ORNL · JoshuaSBrown · Oct 2, 2025 · Sep 25, 2025 · Sep 25, 2025 · Sep 25, 2025
diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml
@@ -40,7 +40,7 @@ jobs:
                   ./scripts/generate_datafed.sh
             - name: Install documentation build dependencies
               run: |
-                  sudo ./scripts/install_docs_dependencies.sh
+                  sudo ./external/DataFedDependencies/scripts/install_docs_dependencies.sh
             - name: Build documentation
               run: |
                   cmake -S. -B build -DBUILD_AUTHZ=OFF -DBUILD_CORE_SERVER=OFF -DBUILD_COMMON=OFF -DBUILD_DOCS=ON -DBUILD_FOXX=OFF -DBUILD_REPO_SERVER=OFF -DBUILD_PYTHON_CLIENT=ON -DBUILD_TESTS=OFF -DBUILD_WEB_SERVER=OFF -DENABLE_UNIT_TESTS=OFF

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -10,9 +10,12 @@ jobs:
             - name: Update debian
               run: apt update
             - name: Install dependencies
+              with:
+                  submodules: recursive
+                  fetch-depth: 0
               run: |
                   ./scripts/generate_datafed.sh
-                  ./scripts/install_core_dependencies.sh
+                  ./external/DataFedDependencies/scripts/install_core_dependencies.sh
             - name: Build
               run: |
                   /opt/datafed/dependencies/bin/cmake -S. -B build -DCMAKE_BUILD_TYPE=Debug -DBUILD_WEB_SERVER=OFF

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -58,3 +58,4 @@ variables:
     REGISTRY: "camden.ornl.gov"
     DATAFED_DEPENDENCIES_INSTALL_PATH: "/shared/install"
     DOCKER_TLS_CERTDIR: "" # Required for running docker in docker
+    GIT_SUBMODULE_STRATEGY: recursive
diff --git a/.gitlab/build/build_gcs_base_image.yml b/.gitlab/build/build_gcs_base_image.yml
@@ -10,7 +10,6 @@ build-gcs-base:
   variables:
     PROJECT: "datafed"
     COMPONENT: "gcs-base"
-    GIT_SUBMODULE_STRATEGY: recursive
     GIT_STRATEGY: clone
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
     BUILD_INTERMEDIATE: "FALSE"

diff --git a/.gitlab/build/force_build_gcs_base_image.yml b/.gitlab/build/force_build_gcs_base_image.yml
@@ -7,7 +7,6 @@ build-gcs-base:
   variables:
     PROJECT: "datafed"
     COMPONENT: "gcs-base"
-    GIT_SUBMODULE_STRATEGY: recursive
     GIT_STRATEGY: clone
     DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count
     GCS_BASE_IMAGE_DISTRO: "debian-12"

diff --git a/.gitlab/stage_provision_client.yml b/.gitlab/stage_provision_client.yml
@@ -7,13 +7,15 @@ provision-client:
     variables:
         GIT_STRATEGY: clone
         DATAFED_PYTHON_DEPENDENCIES_DIR: "${CI_PROJECT_DIR}/dependencies/python"
+        DATAFED_DEPENDENCIES_INSTALL_PATH: "/opt/datafed/dependencies/"
     stage: provision-client
     tags:
         - ci-datafed-client
     before_script:
-        - export PATH=/opt/datafed/dependencies/bin:$PATH
-        - rm -rf $DATAFED_PYTHON_DEPENDENCIES_DIR
+        - sudo chown -R gitlab-runner:gitlab-runner  "$DATAFED_DEPENDENCIES_INSTALL_PATH"
+        - rm -rf "$DATAFED_PYTHON_DEPENDENCIES_DIR"
     script:
-        - ./scripts/generate_datafed.sh
-        - ./scripts/install_client_dependencies.sh
-        - ./scripts/install_end_to_end_test_dependencies.sh
+        - export PATH="/opt/datafed/dependencies/bin:$DATAFED_PYTHON_DEPENDENCIES_DIR/bin:$PATH"
+        - ./external/DataFedDependencies/scripts/generate_dependencies_config.sh
+        - ./external/DataFedDependencies/scripts/install_client_dependencies.sh
+        - ./external/DataFedDependencies/scripts/install_end_to_end_test_dependencies.sh
diff --git a/.gitmodules b/.gitmodules
@@ -1,9 +1,6 @@
 [submodule "external/globus-connect-server-deploy"]
 	path = external/globus-connect-server-deploy
 	url = https://github.com/globus/globus-connect-server-deploy.git
-[submodule "external/protobuf"]
-	path = external/protobuf
-	url = https://github.com/protocolbuffers/protobuf.git
 [submodule "external/DataFedDependencies"]
 	path = external/DataFedDependencies
 	url = https://github.com/ORNL/DataFedDependencies.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -52,7 +52,14 @@ if(NOT EXISTS ${DATAFED_CONFIG_SH})
     "${PROJECT_SOURCE_DIR}/scripts/generate_datafed.sh")
 endif()
 
-file(READ "${PROJECT_SOURCE_DIR}/scripts/dependency_versions.sh" DEPENDENCY_VERSIONS)
+set(DATAFED_DEPENDENCIES_SH "${DataFed_SOURCE_DIR}/external/DataFedDependencies/config/dependencies.sh")
+if(NOT EXISTS ${DATAFED_DEPENDENCIES_SH})
+  message(FATAL_ERROR "Error: File '${DATAFED_DEPENDENCIES_SH}' does not exist. "
+    "Please run generate_dependencies.sh first to populate defaults."
+    "${PROJECT_SOURCE_DIR}/external/DataFedDependencies/scripts/generate_datafed.sh")
+endif()
+
+file(READ "${PROJECT_SOURCE_DIR}/external/DataFedDependencies/scripts/dependency_versions.sh" DEPENDENCY_VERSIONS)
 
 get_version_from_script(${DEPENDENCY_VERSIONS} "DATAFED_DYNAMIC_LIBRARY_PROTOBUF_VERSION" PROTOBUF_LIBRARY_VERSION)
 get_version_from_script(${DEPENDENCY_VERSIONS} "DATAFED_PROTOBUF_VERSION" PROTOBUF_COMPILER_VERSION)
@@ -76,7 +83,7 @@ if(NOT DEFINED DATAFED_DOMAIN)
 endif()
 
 if(NOT DEFINED DATAFED_DEPENDENCIES_INSTALL_PATH)
-  get_value_from_datafed_sh("DATAFED_DEPENDENCIES_INSTALL_PATH" DEPENDENCY_INSTALL_PATH)
+  get_value_from_dependencies_sh("DATAFED_DEPENDENCIES_INSTALL_PATH" DEPENDENCY_INSTALL_PATH)
 endif()
 
 set(CMAKE_PREFIX_PATH "${DEPENDENCY_INSTALL_PATH}")
@@ -187,6 +194,7 @@ endif()
 
 if( BUILD_PYTHON_CLIENT )
   # make target = pydatafed
+  file(COPY ${PROJECT_SOURCE_DIR}/external/DataFedDependencies/python/datafed_pkg/requirements.txt DESTINATION ${PROJECT_SOURCE_DIR}/python/datafed_pkg/requirements.txt)
   add_subdirectory( python EXCLUDE_FROM_ALL )
 endif()
 

diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake
@@ -45,3 +45,24 @@ function(get_value_from_datafed_sh INPUT_KEY OUTPUT_VALUE)
   set(${OUTPUT_VALUE} "${OUTPUT_VAR}" PARENT_SCOPE)
 endfunction()
 
+# Function will get exported value from a shell script
+#
+# i.e. if dependencies.sh has
+#
+# dependencies.sh
+# export MY_NAME="Barry"
+#
+# set(DATAFED_CONFIG_SH "external/DataFedDependencies/config/dependencies.sh")
+# get_value_from_dependencies_sh "MY_NAME" name)
+# message("$name")
+#
+# Will output "Barry"
+function(get_value_from_dependencies_sh INPUT_KEY OUTPUT_VALUE)
+  execute_process(
+    COMMAND bash "-c" "source ${DATAFED_CONFIG_SH} && echo \$${INPUT_KEY}"
+    OUTPUT_VARIABLE OUTPUT_VAR
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+  set(${OUTPUT_VALUE} "${OUTPUT_VAR}" PARENT_SCOPE)
+endfunction()
+
diff --git a/core/database/tests/test_fixture_setup.sh b/core/database/tests/test_fixture_setup.sh
@@ -4,10 +4,10 @@ set -uef -o pipefail
 
 SCRIPT=$(realpath "$BASH_SOURCE[0]")
 SOURCE=$(dirname "$SCRIPT")
-PROJECT_ROOT=$(realpath "${SOURCE}/../../../")
-source "${PROJECT_ROOT}/config/datafed.sh"
-source "${PROJECT_ROOT}/scripts/dependency_versions.sh"
-source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh"
+DATAFED_PROJECT_ROOT=$(realpath "${SOURCE}/../../../")
+source "${DATAFED_PROJECT_ROOT}/config/datafed.sh"
+source "${DATAFED_PROJECT_ROOT}/external/DataFedDependencies/scripts/dependency_versions.sh"
+source "${DATAFED_PROJECT_ROOT}/external/DataFedDependencies/scripts/dependency_install_functions.sh"
 
 Help() {
   echo "$(basename $0) Will initialize fixtures for Foxx tests"
@@ -42,7 +42,7 @@ else
 fi
 
 if [ -z "${FOXX_MAJOR_API_VERSION:-}" ]; then
-  local_FOXX_MAJOR_API_VERSION=$(cat ${PROJECT_ROOT}/cmake/Version.cmake | grep -o -P "(?<=FOXX_API_MAJOR).*(?=\))" | xargs)
+  local_FOXX_MAJOR_API_VERSION=$(cat ${DATAFED_PROJECT_ROOT}/cmake/Version.cmake | grep -o -P "(?<=FOXX_API_MAJOR).*(?=\))" | xargs)
 else
   local_FOXX_MAJOR_API_VERSION=$(printenv FOXX_MAJOR_API_VERSION)
 fi

diff --git a/core/database/tests/test_foxx.sh b/core/database/tests/test_foxx.sh
@@ -12,10 +12,10 @@ set -euf -o pipefail
 
 SCRIPT=$(realpath "$BASH_SOURCE[0]")
 SOURCE=$(dirname "$SCRIPT")
-PROJECT_ROOT=$(realpath ${SOURCE}/../../../)
-source ${PROJECT_ROOT}/config/datafed.sh
-source "${PROJECT_ROOT}/scripts/dependency_versions.sh"
-source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh"
+DATAFED_PROJECT_ROOT=$(realpath ${SOURCE}/../../../)
+source "${DATAFED_PROJECT_ROOT}/config/datafed.sh"
+source "${DATAFED_PROJECT_ROOT}/external/DataFedDependencies/scripts/dependency_versions.sh"
+source "${DATAFED_PROJECT_ROOT}/external/DataFedDependencies/scripts/dependency_install_functions.sh"
 
 Help() {
   echo "$(basename $0) Will run a Foxx unit test"
@@ -46,7 +46,7 @@ else
 fi
 
 if [ -z "${FOXX_MAJOR_API_VERSION:-}" ]; then
-  local_FOXX_MAJOR_API_VERSION=$(cat ${PROJECT_ROOT}/cmake/Version.cmake | grep -o -P "(?<=FOXX_API_MAJOR).*(?=\))" | xargs)
+  local_FOXX_MAJOR_API_VERSION=$(cat ${DATAFED_PROJECT_ROOT}/cmake/Version.cmake | grep -o -P "(?<=FOXX_API_MAJOR).*(?=\))" | xargs)
 else
   local_FOXX_MAJOR_API_VERSION=$(printenv FOXX_MAJOR_API_VERSION)
 fi

diff --git a/core/database/tests/test_setup.sh b/core/database/tests/test_setup.sh
@@ -13,10 +13,10 @@ set -uef -o pipefail
 
 SCRIPT=$(realpath "$BASH_SOURCE[0]")
 SOURCE=$(dirname "$SCRIPT")
-PROJECT_ROOT=$(realpath "${SOURCE}/../../../")
-source "${PROJECT_ROOT}/config/datafed.sh"
-source "${PROJECT_ROOT}/scripts/dependency_versions.sh"
-source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh"
+DATAFED_PROJECT_ROOT=$(realpath "${SOURCE}/../../../")
+source "${DATAFED_PROJECT_ROOT}/config/datafed.sh"
+source "${DATAFED_PROJECT_ROOT}/external/DataFedDependencies/scripts/dependency_versions.sh"
+source "${DATAFED_PROJECT_ROOT}/external/DataFedDependencies/scripts/dependency_install_functions.sh"
 
 Help() {
   echo "$(basename $0) Will set up a configuration file for the core server"
@@ -52,7 +52,7 @@ else
 fi
 
 if [ -z "${FOXX_MAJOR_API_VERSION:-}" ]; then
-  local_FOXX_MAJOR_API_VERSION=$(cat ${PROJECT_ROOT}/cmake/Version.cmake | grep -o -P "(?<=FOXX_API_MAJOR).*(?=\))" | xargs)
+  local_FOXX_MAJOR_API_VERSION=$(cat ${DATAFED_PROJECT_ROOT}/cmake/Version.cmake | grep -o -P "(?<=FOXX_API_MAJOR).*(?=\))" | xargs)
 else
   local_FOXX_MAJOR_API_VERSION=$(printenv FOXX_MAJOR_API_VERSION)
 fi
@@ -115,7 +115,7 @@ if [[ "$output" =~ .*"sdms".* ]]; then
   echo "SDMS already exists do nothing"
 else
   echo "Creating SDMS"
-  arangosh --server.endpoint "tcp://${local_DATAFED_DATABASE_HOST}:8529" --server.password "${local_DATAFED_DATABASE_PASSWORD}" --server.username "${local_DATABASE_USER}" --javascript.execute "${PROJECT_ROOT}/core/database/foxx/db_create.js"
+  arangosh --server.endpoint "tcp://${local_DATAFED_DATABASE_HOST}:8529" --server.password "${local_DATAFED_DATABASE_PASSWORD}" --server.username "${local_DATABASE_USER}" --javascript.execute "${DATAFED_PROJECT_ROOT}/core/database/foxx/db_create.js"
   # Give time for the database to be created
   sleep 2
   arangosh --server.endpoint "tcp://${local_DATAFED_DATABASE_HOST}:8529" --server.password "${local_DATAFED_DATABASE_PASSWORD}" --server.username "${local_DATABASE_USER}" --javascript.execute-string 'db._useDatabase("sdms"); db.config.insert({"_key": "msg_daily", "msg" : "DataFed servers will be off-line for regular maintenance every Sunday night from 11:45 pm until 12:15 am EST Monday morning."}, {overwrite: true});'

diff --git a/core/database/tests/test_teardown.sh b/core/database/tests/test_teardown.sh
@@ -12,8 +12,8 @@ set -euf -o pipefail
 
 SCRIPT=$(realpath "$0")
 SOURCE=$(dirname "$SCRIPT")
-PROJECT_ROOT=$(realpath "${SOURCE}/../../../")
-source "${PROJECT_ROOT}/config/datafed.sh"
+DATAFED_PROJECT_ROOT=$(realpath "${SOURCE}/../../../")
+source "${DATAFED_PROJECT_ROOT}/config/datafed.sh"
 
 PATH_TO_PASSWD_FILE="${SOURCE}/database_temp.password"
 rm "${PATH_TO_PASSWD_FILE}"
diff --git a/core/docker/Dockerfile b/core/docker/Dockerfile
@@ -2,15 +2,16 @@
 # cd ${PROJECT_ROOT} or cd DataFed
 # docker build -f core/docker/Dockerfile .
 
-ARG           BUILD_BASE="debian:bookworm-slim"
-ARG         DEPENDENCIES="dependencies"
-ARG              RUNTIME="runtime"
-ARG          DATAFED_DIR="/datafed"
-ARG DATAFED_INSTALL_PATH="/opt/datafed"
+ARG                        BUILD_BASE="debian:bookworm-slim"
+ARG                      DEPENDENCIES="dependencies"
+ARG                           RUNTIME="runtime"
+ARG                       DATAFED_DIR="/datafed"
+ARG              DATAFED_INSTALL_PATH="/opt/datafed"
 ARG DATAFED_DEPENDENCIES_INSTALL_PATH="/opt/datafed/dependencies"
-ARG            GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal"
-ARG            BUILD_DIR="$DATAFED_DIR/source"
-ARG              LIB_DIR="/usr/local/lib"
+ARG                         BUILD_DIR="$DATAFED_DIR/source"
+ARG         DATAFED_DEPENDENCIES_ROOT="$BUILD_DIR/external/DataFedDependencies"
+ARG                         GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal"
+ARG                           LIB_DIR="/usr/local/lib"
 
 FROM ${DEPENDENCIES} AS core-build
 
@@ -20,33 +21,38 @@ ARG DATAFED_DIR
 ARG BUILD_DIR
 ARG DATAFED_INSTALL_PATH
 ARG DATAFED_DEPENDENCIES_INSTALL_PATH
+ARG DATAFED_DEPENDENCIES_ROOT
 
 ENV DATAFED_INSTALL_PATH="${DATAFED_INSTALL_PATH}"
 # For communicating with repo server
 EXPOSE 7512
 # For listening to web server
 EXPOSE 7513
 
+RUN mkdir -p ${DATAFED_DEPENDENCIES_ROOT}/scripts/ && \
+  mv ./scripts/dependency_versions.sh          ${DATAFED_DEPENDENCIES_ROOT}/scripts/ && \
+  mv ./scripts/generate_dependencies_config.sh ${DATAFED_DEPENDENCIES_ROOT}/scripts/
+
 COPY ./common                          ${BUILD_DIR}/common
 COPY ./core/CMakeLists.txt             ${BUILD_DIR}/core/CMakeLists.txt
 COPY ./CMakeLists.txt                  ${BUILD_DIR}
-COPY ./scripts/dependency_versions.sh  ${BUILD_DIR}/scripts/
 COPY ./scripts/generate_datafed.sh     ${BUILD_DIR}/scripts/
 COPY ./scripts/generate_core_config.sh ${BUILD_DIR}/scripts/
 COPY ./scripts/install_core.sh         ${BUILD_DIR}/scripts/
 COPY ./cmake                           ${BUILD_DIR}/cmake
 COPY ./core/docker/entrypoint.sh       ${BUILD_DIR}/core/docker/
 COPY ./core/server                     ${BUILD_DIR}/core/server
 
-RUN ${BUILD_DIR}/scripts/generate_datafed.sh && \
-	${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake -S. -B build						\
-		-DBUILD_REPO_SERVER=False		\
-		-DBUILD_AUTHZ=False					\
-		-DBUILD_CORE_SERVER=True		\
-		-DBUILD_WEB_SERVER=False		\
-		-DBUILD_DOCS=False					\
-		-DBUILD_PYTHON_CLIENT=False	\
-		-DBUILD_FOXX=False
+RUN ${DATAFED_DEPENDENCIES_ROOT}/scripts/generate_dependencies_config.sh && \
+    ${BUILD_DIR}/scripts/generate_datafed.sh && \
+    ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake -S. -B build \
+      -DBUILD_REPO_SERVER=False \
+      -DBUILD_AUTHZ=False \
+      -DBUILD_CORE_SERVER=True \
+      -DBUILD_WEB_SERVER=False \
+      -DBUILD_DOCS=False \
+      -DBUILD_PYTHON_CLIENT=False \
+      -DBUILD_FOXX=False
 RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build -j 8
 RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build --target install
 
@@ -56,13 +62,15 @@ SHELL ["/bin/bash", "-c"]
 
 ARG DATAFED_DIR
 ARG DATAFED_INSTALL_PATH
+ARG DATAFED_DEPENDENCIES_ROOT
 ARG DATAFED_DEPENDENCIES_INSTALL_PATH
 ARG BUILD_DIR
 ARG LIB_DIR
 
 # The above should also be available at runtime
 ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH"
 ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DEPENDENCIES_INSTALL_PATH}"
+ENV DATAFED_DEPENDENCIES_ROOT="${DATAFED_DEPENDENCIES_ROOT}"
 ENV          DATAFED_DIR="$DATAFED_DIR"
 ENV            BUILD_DIR="$BUILD_DIR"
 ENV              LIB_DIR="$LIB_DIR"
@@ -77,18 +85,21 @@ COPY --chown=datafed:root ./scripts/generate_core_config.sh ${BUILD_DIR}/scripts
 COPY --chown=datafed:root ./scripts/install_core.sh         ${BUILD_DIR}/scripts/install_core.sh
 COPY --chown=datafed:root ./cmake/Version.cmake             ${BUILD_DIR}/cmake/Version.cmake
 COPY --from=core-build --chown=datafed:root ${BUILD_DIR}/core/docker/entrypoint.sh    ${BUILD_DIR}/core/docker/entrypoint.sh
+COPY --from=core-build --chown=datafed:root ${DATAFED_DEPENDENCIES_ROOT}/scripts      ${DATAFED_DEPENDENCIES_ROOT}/scripts
 COPY --from=core-build --chown=datafed:root ${DATAFED_INSTALL_PATH}/core/datafed-core ${DATAFED_INSTALL_PATH}/core/datafed-core
 
 USER root
 
 
-RUN chown -R datafed:root ${DATAFED_DIR} /opt /home/datafed && \
+RUN chown -R datafed:root ${DATAFED_DIR} /opt /home/datafed \
+                 ${DATAFED_DEPENDENCIES_ROOT} && \
     chmod 774 ${DATAFED_DIR} ${BUILD_DIR} ${BUILD_DIR}/core/docker/entrypoint.sh \
                  ${DATAFED_INSTALL_PATH}/core/datafed-core \
                  ${BUILD_DIR}/scripts/generate_datafed.sh \
                  ${BUILD_DIR}/scripts/generate_core_config.sh \
                  ${BUILD_DIR}/scripts/install_core.sh \
                  ${DATAFED_INSTALL_PATH}/core && \
+    find ${DATAFED_DEPENDENCIES_ROOT} -type d -exec chmod 0774 {} + && \
     chmod 664 ${BUILD_DIR}/cmake/Version.cmake && \
     chmod +t ${DATAFED_DIR} ${DATAFED_INSTALL_PATH}
 

diff --git a/doc_source/source/admin/install_bare_metal.rst b/doc_source/source/admin/install_bare_metal.rst
@@ -71,9 +71,9 @@ The npm packages needed primarily by the web server are:
 
 This can be done with a helper scripts these scripts are for ubuntu::
 
-    ./DataFed/scripts/install_core_dependencies.sh
-    ./DataFed/scripts/install_repo_dependencies.sh
-    ./DataFed/scripts/install_ws_dependencies.sh
+    ./DataFed/external/DataFedDependencies/scripts/install_core_dependencies.sh
+    ./DataFed/external/DataFedDependencies/scripts/install_repo_dependencies.sh
+    ./DataFed/external/DataFedDependencies/scripts/install_ws_dependencies.sh
 
 The next step is to enter configuration options that are listed in ./config/datafed.sh. To
 generate a template for this file you will first need to run::