Merge pull request #174 from data-douser/data-douser/fix1-cds-extractor-out

Author: data-douser

.github/workflows/run-codeql-unit-tests-javascript.yml

@@ -96,10 +96,26 @@ jobs:
             for cds_file in $(find . -type f \( -iname '*.cds' \) -print)
             do
               echo "I am compiling $cds_file"
+              _out_path="${cds_file}.json"
               cds compile $cds_file \
-                -2 json \
                 --locations \
-                > "$cds_file.json" 2> "$cds_file.err"
+                --to json \
+                --dest "$_out_path" \
+                2> "$cds_file.err"
+              # Check if the output is a regular file or a (sub)directory, where
+              # files generated in an output directory will need to have the file
+              # extension changed from '.json' to '.cds.json', but we don't need
+              # to rename anything if the cds compiler just generated a single
+              # '.cds.json' file.
+              if [ -d "$_out_path" ]
+              then
+                for json_file in $(find "$_out_path" -type f \( -iname '*.json' \) -print)
+                do
+                  _new_path="${json_file%.json}.cds.json"
+                  echo "Renaming CDS compiler generated JSON file $json_file to $_new_path"
+                  mv "$json_file" "$_new_path"
+                done
+              fi
             done
             popd
           done

extractors/cds/tools/index-files.sh

@@ -48,16 +48,62 @@ fi
 
 echo "Processing CDS files to JSON"
 
-# Run the cds compile command on each file in the response file, outputting the compiled JSON to a file with
-# the same name
-while IFS= read -r cds_file; do
-    echo "Processing CDS file $cds_file to:"
-    if ! $cds_command compile "$cds_file" -2 json --locations > "$cds_file.json" 2> "$cds_file.err"; then
-        stderr_truncated=`grep "^\[ERROR\]" "$cds_file.err" | tail -n 4`
-        error_message=$'Could not compile the file '"$cds_file"$'.\nReported error(s):\n```\n'"$stderr_truncated"$'\n```'
-        echo "$error_message"
-        # Log an error diagnostic which appears on the status page
-        "$CODEQL_DIST/codeql" database add-diagnostic --extractor-name cds --ready-for-status-page --source-id cds/compilation-failure --source-name "Failure to compile one or more SAP CAP CDS files" --severity error --markdown-message "$error_message" --file-path "$cds_file" "$CODEQL_EXTRACTOR_CDS_WIP_DATABASE"
+# Run the cds compile command on each file in the response file in order to generate
+# the JSON data we need. Move the generated ".json" files from the temporary output
+# to the original directory and rename them to ".cds.json" so that the JS extractor
+# can distinguish them from regular JSON files / extract them as CDS files.
+while IFS= read -r _cds_file_path; do
+    # The cds compile command chooses how it outputs the JSON. If it creates output
+    # files in a directory, then it will create the directory when it runs. If it
+    # creates a single output file, then it will create the file when it runs. We
+    # create the output path by simply appending ".json" to the input file path,
+    # such that there is nothing further to do if the output is a single file.
+    _cds_compile_json_out="${_cds_file_path}.json"
+    # Remove any existing output directory to avoid conflicts.
+    if [ -d "$_cds_compile_json_out" ]; then
+        echo "WARNING:  overwriting existing temporary output path ${_cds_compile_json_out}."
+        rm -rf "$_cds_compile_json_out"
+    fi
+    _cds_file_err_path="${_cds_file_path}.err"
+    echo "Compiling JSON for source CDS file $_cds_file_path to output path: $_cds_compile_json_out"
+    if ! $cds_command compile "$_cds_file_path" --to json --dest "$_cds_compile_json_out" --locations 2> "$_cds_file_err_path"; then
+        stderr_truncated=`grep "^\[ERROR\]" "$_cds_file_err_path" | tail -n 4`
+        _error_msg=$'ERROR: Could not compile the CDS file '"$_cds_file_path"$'.\nReported error(s):\n```\n'"$stderr_truncated"$'\n```'
+        echo "$_error_msg"
+        # Log the error message as a diagnostic which appears on the Tools status page.
+        "$CODEQL_DIST/codeql" database add-diagnostic --extractor-name cds --ready-for-status-page --source-id cds/compilation-failure --source-name "Failure to compile one or more SAP CAP CDS files" --severity error --markdown-message "$_error_msg" --file-path "$_cds_file_path" "$CODEQL_EXTRACTOR_CDS_WIP_DATABASE"
+        # Continue to the next file.
+        continue
+    fi
+    # Cleanup the error file if it exists and is empty.
+    if [[ -f "$_cds_file_err_path" && -s "$_cds_file_err_path" ]]; then
+        rm -f "$_cds_file_err_path"
+    fi
+    # Replace the ".json" extension with a ".cds.json" extension in the files
+    # generated by the cds compile command. Due to inconsistencies between
+    # different versions of the cds compiler, and contrary to the command-line
+    # documentation for the "cds compile" command, the output may be generated
+    # as a single file in the current directory, or as one or more files in an
+    # output directory.
+    if [ -f "$_cds_compile_json_out" ]; then
+        echo "Compiled CDS source file from $_cds_file_path to $_cds_compile_json_out"
+    elif [ -d "$_cds_compile_json_out" ]; then
+        echo "CDS compiler generated JSON to output directory: $_cds_compile_json_out"
+        if ls "$_cds_compile_json_out"/*.json 1> /dev/null 2>&1; then
+            for _output_json_file in "$_cds_compile_json_out"/*.json; do
+                _cds_json_file_path="${_output_json_file%.json}.cds.json"
+                echo "Renaming CDS compiler generated output '.json' file to $_cds_json_file_path"
+                mv "$_output_json_file" "$_cds_json_file_path"
+            done
+        else
+            _error_msg="ERROR: Detected no '.json' files in CDS compiler output directory: $_cds_compile_json_out"
+            # Log the error message as a diagnostic which appears on the Tools status page.
+            "$CODEQL_DIST/codeql" database add-diagnostic --extractor-name cds --ready-for-status-page --source-id cds/compilation-failure --source-name "Failure to find one or more compiled SAP CAP CDS JSON files" --severity error --markdown-message "$_error_msg" --file-path "$_cds_file_path" "$CODEQL_EXTRACTOR_CDS_WIP_DATABASE"
+        fi
+    else
+        _error_msg="ERROR: Detected no output directory or file for CDS compiler output: $_cds_compile_json_out"
+        # Log the error message as a diagnostic which appears on the Tools status page.
+        "$CODEQL_DIST/codeql" database add-diagnostic --extractor-name cds --ready-for-status-page --source-id cds/compilation-failure --source-name "Failure to find one or more compiled SAP CAP CDS JSON files" --severity error --markdown-message "$_error_msg" --file-path "$_cds_file_path" "$CODEQL_EXTRACTOR_CDS_WIP_DATABASE"
     fi
 done < "$response_file"