Initial commit to fix log output from AuthzWorker.cpp

.github/workflows/javascript-format.yml

@@ -0,0 +1,34 @@
+name: JavaScript Formatter
+on: push
+jobs:
+  format-javascript:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Step 1: Checkout the repository
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      # Step 2: Set up Node.js environment
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18' # Specify your Node.js version
+
+      # Step 3: Install Prettier and ESLint globally
+      - name: Install Prettier
+        run: |
+          npm install -g prettier
+
+      # Step 4: Run Prettier to format code
+      - name: Run prettier
+        run: |
+          prettier "**/*.js" --write
+          git diff
+          git reset --hard
+          prettier --check "**/*.js" 
+
+      # Step 5: Report status
+      - name: Complete
+        run: echo "Formatting completed successfully!"
+

.github/workflows/javascript-lint.yml

@@ -0,0 +1,28 @@
+name: JavaScript Linter
+on: push
+jobs:
+  lint-javascript:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Step 1: Checkout the repository
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      # Step 2: Set up Node.js environment
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18' # MOre up to date version needed for linter
+
+      # Step 3: Instal ESLint and run
+      - name: Install ESLint
+        run: |
+          npm init -y
+          npm install eslint@latest @babel/eslint-parser@latest eslint-define-config globals eslint-plugin-jsdoc --save
+          npx eslint "**/*.js"
+
+      # Step 4: Report status
+      - name: Complete
+        run: echo "Linting completed successfully!"
+

.github/workflows/unit-tests.yml

@@ -2,7 +2,7 @@ name: Unit-Testing
 on: push
 jobs:
   unit-test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     if: ${{ always() }}
     steps:
       - uses: actions/checkout@v2
@@ -15,7 +15,7 @@ jobs:
           ./scripts/generate_datafed.sh
       - name: Build
         run: |
-          cmake -S. -B build -DCMAKE_BUILD_TYPE=Debug
+          cmake -S. -B build -DCMAKE_BUILD_TYPE=Debug -DBUILD_WEB_SERVER=OFF
           cmake --build build -j4
       - name: Run tests
         run: |

.gitignore

@@ -42,6 +42,7 @@ python/datafed_pkg/datafed/Version_pb2.py
 python/datafed_pkg/datafed/__pycache__/
 python/datafed_pkg/dist/
 web/package.json
+repository/gridftp/globus5/authz/mock_core/Version.hpp
 repository/gridftp/globus5/authz/source/Version.hpp
 repository/server/Version.hpp
 scripts/globus/datafed-home-repo.sh

.gitlab-ci.yml

@@ -22,6 +22,8 @@ include:
   - local: .gitlab/stage_provision_client.yml
   - local: .gitlab/stage_image_check.yml
   - local: .gitlab/stage_build.yml
+  - local: .gitlab/stage_test.yml
+  - local: .gitlab/stage_unit.yml
   - local: .gitlab/end_to_end.yml
 
 stages:
@@ -34,9 +36,12 @@ stages:
   - provision-client
   - image-check
   - build
+  - test # Required for gitlab sast templates
+  - unit
   - end-to-end-setup-arango
   - end-to-end-setup
   - end-to-end-test
+  - end-to-end-test-finish-signal
   - log
   - deploy-pypi-package
 
@@ -50,3 +55,4 @@ variables:
   HARBOR_USER: 'robot$$datafed+harbor_datafed_gitlab_ci_registry_2'
   REGISTRY: 'camden.ornl.gov'
   DATAFED_DEPENDENCIES_INSTALL_PATH: "/shared/install"
+  DOCKER_TLS_CERTDIR: "" # Required for running docker in docker

.gitlab/build/build_core_image.yml

@@ -9,10 +9,12 @@ build-core:
   extends: .docker_build_script
   stage: build
   variables:
-    IMAGE_TAG: "datafed/core"
+    PROJECT: "datafed"
+    COMPONENT: "core"
     GIT_STRATEGY: clone
     DOCKER_FILE_PATH: "core/docker/Dockerfile"
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+    BUILD_INTERMEDIATE: "FALSE"
   tags:
     - ci-datafed-core
     - docker
@@ -31,8 +33,21 @@ retag-image:
   extends: .docker_retag_image
   stage: build
   variables:
-    IMAGE_TAG: "datafed/core"
+    PROJECT: "datafed"
+    COMPONENT: "core"
     GIT_STRATEGY: clone
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+    BUILD_INTERMEDIATE: "FALSE"
   tags:
     - docker
+  rules:
+    - changes:
+        - docker/**/*
+        - scripts/**/*
+        - core/**/*
+        - common/**/*
+        - CMakeLists.txt
+        - cmake/**/*
+        - .gitlab-ci.yml
+      when: never
+    - when: on_success

.gitlab/build/build_foxx_image.yml

@@ -9,27 +9,44 @@ build-foxx:
   extends: .docker_build_script
   stage: build
   variables:
-    IMAGE_TAG: "datafed/foxx"
+    PROJECT: "datafed"
+    COMPONENT: "foxx"
     GIT_STRATEGY: clone
     DOCKER_FILE_PATH: "docker/Dockerfile.foxx"
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+    BUILD_INTERMEDIATE: "FALSE"
   tags:
     - docker
   rules:
     - changes:
         - docker/**/*
         - scripts/**/*
-        - web/**/*
+        - core/database/**/*
+        - core/CMakeLists.txt
         - common/proto/**/*
         - .gitlab-ci.yml
+        - CMakeLists.txt
       when: on_success
 
 retag-image:
   extends: .docker_retag_image
   stage: build
   variables:
-    IMAGE_TAG: "datafed/foxx"
+    PROJECT: "datafed"
+    COMPONENT: "foxx"
     GIT_STRATEGY: clone
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+    BUILD_INTERMEDIATE: "FALSE"
   tags:
     - docker
+  rules:
+    - changes:
+        - docker/**/*
+        - scripts/**/*
+        - core/database/**/*
+        - core/CMakeLists.txt
+        - common/proto/**/*
+        - .gitlab-ci.yml
+        - CMakeLists.txt
+      when: never
+    - when: on_success

.gitlab/build/build_gcs_base_image.yml

@@ -8,10 +8,12 @@ include:
 build-gcs-base:
   stage: build
   variables:
-    IMAGE_TAG: "datafed/gcs-base"
+    PROJECT: "datafed"
+    COMPONENT: "gcs-base"
     GIT_SUBMODULE_STRATEGY: recursive
     GIT_STRATEGY: clone
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+    BUILD_INTERMEDIATE: "FALSE"
   tags:
     - ci-datafed-globus
     - docker
@@ -31,28 +33,41 @@ build-gcs-base:
     - cd "external/globus-connect-server-deploy/docker"
     - git checkout "$DATAFED_GCS_SUBMODULE_VERSION"
     - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}"
-    - docker build --progress plain -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" - < "./docker-files/Dockerfile.ubuntu-20.04"
-    - docker tag "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:$CI_COMMIT_SHA"
-    - export DATAFED_HARBOR_REPOSITORY="${IMAGE_TAG}-${BRANCH_LOWER}"
+    - docker build --progress plain -t "${REGISTRY}/${PROJECT}/${COMPONENT}-${BRANCH_LOWER}:latest" - < "./docker-files/Dockerfile.ubuntu-20.04"
+    - docker tag "${REGISTRY}/${PROJECT}/${COMPONENT}-${BRANCH_LOWER}:latest" "${REGISTRY}/${PROJECT}/${COMPONENT}-${BRANCH_LOWER}:$CI_COMMIT_SHA"
+    - export DATAFED_HARBOR_REPOSITORY="${COMPONENT}-${BRANCH_LOWER}"
     - export DATAFED_HARBOR_USERNAME="${HARBOR_USER}"
     - export DATAFED_HARBOR_PASSWORD="${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}"
-    - docker push "${REGISTRY}/${DATAFED_HARBOR_REPOSITORY}:latest"
-    - docker push "${REGISTRY}/${DATAFED_HARBOR_REPOSITORY}:$CI_COMMIT_SHA"
+    - docker push "${REGISTRY}/${PROJECT}/${DATAFED_HARBOR_REPOSITORY}:latest"
+    - docker push "${REGISTRY}/${PROJECT}/${DATAFED_HARBOR_REPOSITORY}:$CI_COMMIT_SHA"
+    - cd "${CI_PROJECT_DIR}"
     - |
-      while [ "$(${CI_PROJECT_DIR}/scripts/ci_harbor_artifact_count.sh)" == "0" ]; do
+      while [ "$(${CI_PROJECT_DIR}/scripts/ci_harbor_artifact_count.sh -r ${DATAFED_HARBOR_REPOSITORY})" == "0" ]; do
         echo "Artifact missing from harbor..."
-        docker push "${REGISTRY}/${DATAFED_HARBOR_REPOSITORY}:latest"
-        docker push "${REGISTRY}/${DATAFED_HARBOR_REPOSITORY}:$CI_COMMIT_SHA"
+        docker push "${REGISTRY}/${PROJECT}/${DATAFED_HARBOR_REPOSITORY}:latest"
+        docker push "${REGISTRY}/${PROJECT}/${DATAFED_HARBOR_REPOSITORY}:$CI_COMMIT_SHA"
         sleep 5  # Optional: Add a sleep to avoid busy waiting 
       done
-    - cat "./harbor_check.log"
+    - cat "${CI_PROJECT_DIR}/harbor_check.log"
 
 retag-image:
   extends: .docker_retag_image
   stage: build
   variables:
-    IMAGE_TAG: "datafed/gcs-base"
+    PROJECT: "datafed"
+    COMPONENT: "gcs-base"
     GIT_STRATEGY: clone
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+    BUILD_INTERMEDIATE: "FALSE"
   tags:
     - docker
+  rules:
+    - changes:
+        - docker/**/*
+        - scripts/**/*
+        - common/**/*
+        - .gitlab-ci.yml
+        - CMakeLists.txt
+        - cmake/**/*
+      when: never
+    - when: on_success

.gitlab/build/build_gcs_image.yml

@@ -8,9 +8,11 @@ include:
 build-gcs:
   stage: build
   variables:
-    IMAGE_TAG: "datafed/gcs"
+    PROJECT: "datafed"
+    COMPONENT: "gcs"
     GIT_STRATEGY: clone
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+    BUILD_INTERMEDIATE: "FALSE"
   tags:
     - ci-datafed-globus
     - docker
@@ -28,31 +30,45 @@ build-gcs:
   script:
     - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
     - echo "$BRANCH_LOWER"
-    - ./scripts/generate_datafed.sh
+    - "${CI_PROJECT_DIR}/scripts/generate_datafed.sh"
     - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}"
-    - docker build --build-arg DEPENDENCIES="${REGISTRY}/datafed/dependencies-${BRANCH_LOWER}:latest" --build-arg RUNTIME="${REGISTRY}/datafed/runtime-${BRANCH_LOWER}:latest" --build-arg GCS_IMAGE="${REGISTRY}/datafed/gcs-base-${BRANCH_LOWER}:latest" -f repository/docker/Dockerfile.gcs -t "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" .
-    - docker tag "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:latest" "${REGISTRY}/${IMAGE_TAG}-${BRANCH_LOWER}:$CI_COMMIT_SHA"
-    - export DATAFED_HARBOR_REPOSITORY="${IMAGE_TAG}-${BRANCH_LOWER}"
+    - docker build --build-arg DEPENDENCIES="${REGISTRY}/datafed/dependencies-${BRANCH_LOWER}:latest" --build-arg RUNTIME="${REGISTRY}/datafed/runtime-${BRANCH_LOWER}:latest" --build-arg GCS_IMAGE="${REGISTRY}/datafed/gcs-base-${BRANCH_LOWER}:latest" -f repository/docker/Dockerfile.gcs -t "${REGISTRY}/${PROJECT}/${COMPONENT}-${BRANCH_LOWER}:latest" .
+    - docker tag "${REGISTRY}/${PROJECT}/${COMPONENT}-${BRANCH_LOWER}:latest" "${REGISTRY}/${PROJECT}/${COMPONENT}-${BRANCH_LOWER}:$CI_COMMIT_SHA"
+    - export DATAFED_HARBOR_REPOSITORY="${COMPONENT}-${BRANCH_LOWER}"
     - export DATAFED_HARBOR_USERNAME="${HARBOR_USER}"
     - export DATAFED_HARBOR_PASSWORD="${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}"
-    - docker push "${REGISTRY}/${DATAFED_HARBOR_REPOSITORY}:latest"
-    - docker push "${REGISTRY}/${DATAFED_HARBOR_REPOSITORY}:$CI_COMMIT_SHA"
+    - docker push "${REGISTRY}/${PROJECT}/${DATAFED_HARBOR_REPOSITORY}:latest"
+    - docker push "${REGISTRY}/${PROJECT}/${DATAFED_HARBOR_REPOSITORY}:$CI_COMMIT_SHA"
     - |
-      while [ "$(./scripts/ci_harbor_artifact_count.sh)" == "0" ]; do
+      while [ "$(${CI_PROJECT_DIR}/scripts/ci_harbor_artifact_count.sh -r ${DATAFED_HARBOR_REPOSITORY})" == "0" ]; do
         echo "Artifact missing from harbor..."
-        docker push "${REGISTRY}/${DATAFED_HARBOR_REPOSITORY}:latest"
-        docker push "${REGISTRY}/${DATAFED_HARBOR_REPOSITORY}:$CI_COMMIT_SHA"
+        docker push "${REGISTRY}/${PROJECT}/${DATAFED_HARBOR_REPOSITORY}:latest"
+        docker push "${REGISTRY}/${PROJECT}/${DATAFED_HARBOR_REPOSITORY}:$CI_COMMIT_SHA"
         sleep 5  # Optional: Add a sleep to avoid busy waiting 
       done
-    - cat "./harbor_check.log"
+    - cat "${CI_PROJECT_DIR}/harbor_check.log"
 
 retag-image:
   extends: .docker_retag_image
   stage: build
   variables:
-    IMAGE_TAG: "datafed/gcs"
+    PROJECT: "datafed"
+    COMPONENT: "gcs"
     GIT_STRATEGY: clone
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+    BUILD_INTERMEDIATE: "FALSE"
   tags:
     - ci-datafed-globus
     - docker
+  rules:
+    - changes:
+        - docker/**/*
+        - scripts/**/*
+        - common/**/*
+        - .gitlab-ci.yml
+        - CMakeLists.txt
+        - cmake/**/*
+        - repository/docker/entrypoint_authz.sh
+        - repository/docker/Dockerfile.gcs
+      when: never
+    - when: on_success

.gitlab/build/build_repo_image.yml

@@ -9,10 +9,12 @@ build-repo:
   extends: .docker_build_script
   stage: build
   variables:
-    IMAGE_TAG: "datafed/repo"
+    PROJECT: "datafed"
+    COMPONENT: "repo"
     GIT_STRATEGY: clone
     DOCKER_FILE_PATH: "repository/docker/Dockerfile"
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+    BUILD_INTERMEDIATE: "FALSE"
   tags:
     - ci-datafed-repo
     - docker
@@ -29,8 +31,20 @@ retag-image:
   extends: .docker_retag_image
   stage: build
   variables:
-    IMAGE_TAG: "datafed/repo"
+    PROJECT: "datafed"
+    COMPONENT: "repo"
     GIT_STRATEGY: clone
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
+    BUILD_INTERMEDIATE: "FALSE"
   tags:
     - docker
+  rules:
+    - changes:
+        - docker/**/*
+        - scripts/**/*
+        - web/**/*
+        - common/proto/**/*
+        - .gitlab-ci.yml
+      when: never
+    - when: on_success
+