PYTHON-4721 - Create individualized scripts for all shell.exec commands (mongodb#1997)

Co-authored-by: Jib <[email protected]>

Author: NoahStapp

.evergreen/config.yml

@@ -29,7 +29,7 @@ else # Set up virtualenv before installing hatch
     # Ensure hatch does not write to user or global locations.
     touch hatch_config.toml
     HATCH_CONFIG=$(pwd)/hatch_config.toml
-    if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
+    if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
         HATCH_CONFIG=$(cygpath -m "$HATCH_CONFIG")
     fi
     export HATCH_CONFIG

.evergreen/hatch.sh

@@ -53,18 +53,18 @@ if [ -z "${NO_EXT:-}" ]; then
 fi
 
 if [ "$AUTH" != "noauth" ]; then
-    if [ ! -z "$TEST_DATA_LAKE" ]; then
+    if [ -n "$TEST_DATA_LAKE" ]; then
         export DB_USER="mhuser"
         export DB_PASSWORD="pencil"
-    elif [ ! -z "$TEST_SERVERLESS" ]; then
-        source ${DRIVERS_TOOLS}/.evergreen/serverless/secrets-export.sh
+    elif [ -n "$TEST_SERVERLESS" ]; then
+        source "${DRIVERS_TOOLS}"/.evergreen/serverless/secrets-export.sh
         export DB_USER=$SERVERLESS_ATLAS_USER
         export DB_PASSWORD=$SERVERLESS_ATLAS_PASSWORD
         export MONGODB_URI="$SERVERLESS_URI"
         echo "MONGODB_URI=$MONGODB_URI"
         export SINGLE_MONGOS_LB_URI=$MONGODB_URI
         export MULTI_MONGOS_LB_URI=$MONGODB_URI
-    elif [ ! -z "$TEST_AUTH_OIDC" ]; then
+    elif [ -n "$TEST_AUTH_OIDC" ]; then
         export DB_USER=$OIDC_ADMIN_USER
         export DB_PASSWORD=$OIDC_ADMIN_PWD
         export DB_IP="$MONGODB_URI"

.evergreen/run-tests.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -o xtrace
+mkdir out_dir
+# shellcheck disable=SC2156
+find "$MONGO_ORCHESTRATION_HOME" -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \;
+tar zcvf mongodb-logs.tar.gz -C out_dir/ .
+rm -rf out_dir

.evergreen/scripts/archive-mongodb-logs.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -o xtrace
+
+# Enable core dumps if enabled on the machine
+# Copied from https://github.com/mongodb/mongo/blob/master/etc/evergreen.yml
+if [ -f /proc/self/coredump_filter ]; then
+    # Set the shell process (and its children processes) to dump ELF headers (bit 4),
+    # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0).
+    echo 0x13 >/proc/self/coredump_filter
+
+    if [ -f /sbin/sysctl ]; then
+        # Check that the core pattern is set explicitly on our distro image instead
+        # of being the OS's default value. This ensures that coredump names are consistent
+        # across distros and can be picked up by Evergreen.
+        core_pattern=$(/sbin/sysctl -n "kernel.core_pattern")
+        if [ "$core_pattern" = "dump_%e.%p.core" ]; then
+            echo "Enabling coredumps"
+            ulimit -c unlimited
+        fi
+    fi
+fi
+
+if [ "$(uname -s)" = "Darwin" ]; then
+    core_pattern_mac=$(/usr/sbin/sysctl -n "kern.corefile")
+    if [ "$core_pattern_mac" = "dump_%N.%P.core" ]; then
+        echo "Enabling coredumps"
+        ulimit -c unlimited
+    fi
+fi
+
+if [ -n "${skip_crypt_shared}" ]; then
+    export SKIP_CRYPT_SHARED=1
+fi
+
+MONGODB_VERSION=${VERSION} \
+    TOPOLOGY=${TOPOLOGY} \
+    AUTH=${AUTH:-noauth} \
+    SSL=${SSL:-nossl} \
+    STORAGE_ENGINE=${STORAGE_ENGINE:-} \
+    DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS:-} \
+    ORCHESTRATION_FILE=${ORCHESTRATION_FILE:-} \
+    REQUIRE_API_VERSION=${REQUIRE_API_VERSION:-} \
+    LOAD_BALANCER=${LOAD_BALANCER:-} \
+    bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
+# run-orchestration generates expansion file with the MONGODB_URI for the cluster

.evergreen/scripts/bootstrap-mongo-orchestration.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. .evergreen/scripts/env.sh
+set -x
+export BASE_SHA="$1"
+export HEAD_SHA="$2"
+bash .evergreen/run-import-time-test.sh

.evergreen/scripts/check-import-time.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [ -f "$DRIVERS_TOOLS"/.evergreen/csfle/secrets-export.sh ]; then
+  . .evergreen/hatch.sh encryption:teardown
+fi
+rm -rf "${DRIVERS_TOOLS}" || true
+rm -f ./secrets-export.sh || true

.evergreen/scripts/cleanup.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -ex
+#!/bin/bash -eux
 
 # Get the current unique version of this checkout
 # shellcheck disable=SC2154
@@ -29,7 +29,7 @@ fi
 export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
 export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
 
-cat <<EOT > $SCRIPT_DIR/env.sh
+cat <<EOT > "$SCRIPT_DIR"/env.sh
 set -o errexit
 export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
 export CURRENT_VERSION="$CURRENT_VERSION"
@@ -38,6 +38,21 @@ export DRIVERS_TOOLS="$DRIVERS_TOOLS"
 export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
 export MONGODB_BINARIES="$MONGODB_BINARIES"
 export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
+export SETDEFAULTENCODING="${SETDEFAULTENCODING:-}"
+export SKIP_CSOT_TESTS="${SKIP_CSOT_TESTS:-}"
+export MONGODB_STARTED="${MONGODB_STARTED:-}"
+export DISABLE_TEST_COMMANDS="${DISABLE_TEST_COMMANDS:-}"
+export GREEN_FRAMEWORK="${GREEN_FRAMEWORK:-}"
+export NO_EXT="${NO_EXT:-}"
+export COVERAGE="${COVERAGE:-}"
+export COMPRESSORS="${COMPRESSORS:-}"
+export MONGODB_API_VERSION="${MONGODB_API_VERSION:-}"
+export SKIP_HATCH="${SKIP_HATCH:-}"
+export skip_crypt_shared="${skip_crypt_shared:-}"
+export STORAGE_ENGINE="${STORAGE_ENGINE:-}"
+export REQUIRE_API_VERSION="${REQUIRE_API_VERSION:-}"
+export skip_web_identity_auth_test="${skip_web_identity_auth_test:-}"
+export skip_ECS_auth_test="${skip_ECS_auth_test:-}"
 
 export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
 export PATH="$MONGODB_BINARIES:$PATH"

.evergreen/scripts/configure-env.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# Download all the task coverage files.
+aws s3 cp --recursive s3://"$1"/coverage/"$2"/"$3"/coverage/ coverage/

.evergreen/scripts/download-and-merge-coverage.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set +x
+. src/.evergreen/scripts/env.sh
+# shellcheck disable=SC2044
+for filename in $(find $DRIVERS_TOOLS -name \*.json); do
+    perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|$DRIVERS_TOOLS|g" $filename
+done