PYTHON-5398 Support for AWS EKS Pod Identity

.evergreen/generated_configs/tasks.yml

@@ -87,32 +87,40 @@ tasks:
           SUB_TEST_NAME: web-identity
           PYTHON_VERSION: "3.9"
     tags: [auth-aws, auth-aws-web-identity]
-  - name: test-auth-aws-rapid-web-identity-session-name-python3.9
+  - name: test-auth-aws-latest-web-identity-session-name-python3.10
     commands:
       - func: run server
         vars:
           AUTH_AWS: "1"
-          VERSION: rapid
+          VERSION: latest
       - func: assume ec2 role
       - func: run tests
         vars:
           TEST_NAME: auth_aws
           SUB_TEST_NAME: web-identity
+          PYTHON_VERSION: "3.10"
           AWS_ROLE_SESSION_NAME: test
-          PYTHON_VERSION: "3.9"
     tags: [auth-aws, auth-aws-web-identity]
-  - name: test-auth-aws-latest-ecs-python3.10
+  - name: test-auth-aws-latest-eks-python3.11
+    commands:
+      - func: assume ec2 role
+      - func: run tests
+        vars:
+          TEST_NAME: auth_aws
+          SUB_TEST_NAME: eks
+          PYTHON_VERSION: "3.11"
+    tags: [auth-aws, auth-aws-eks]
+  - name: test-auth-aws-latest-ecs-python3.11
     commands:
+      - func: assume ec2 role
       - func: run server
         vars:
           AUTH_AWS: "1"
-          VERSION: latest
-      - func: assume ec2 role
       - func: run tests
         vars:
           TEST_NAME: auth_aws
           SUB_TEST_NAME: ecs
-          PYTHON_VERSION: "3.10"
+          PYTHON_VERSION: "3.11"
     tags: [auth-aws, auth-aws-ecs]
 
   # Backport pr tests

.evergreen/generated_configs/variants.yml

@@ -98,14 +98,14 @@ buildvariants:
     tags: []
   - name: auth-aws-win64
     tasks:
-      - name: .auth-aws !.auth-aws-ecs
+      - name: .auth-aws !.auth-aws-ecs !.auth-aws-eks
     display_name: Auth AWS Win64
     run_on:
       - windows-64-vsMulti-small
     tags: []
   - name: auth-aws-macos
     tasks:
-      - name: .auth-aws !.auth-aws-web-identity !.auth-aws-ecs !.auth-aws-ec2
+      - name: .auth-aws !.auth-aws-web-identity !.auth-aws-ecs !.auth-aws-ec2 !.auth-aws-eks
     display_name: Auth AWS macOS
     run_on:
       - macos-14

.evergreen/scripts/configure-env.sh

@@ -76,6 +76,8 @@ EOT
 rm -rf $DRIVERS_TOOLS
 BRANCH=master
 ORG=mongodb-labs
+BRANCH=DRIVERS-2945
+ORG=blink1073
 git clone --branch $BRANCH https://github.com/$ORG/drivers-evergreen-tools.git $DRIVERS_TOOLS
 
 cat <<EOT > ${DRIVERS_TOOLS}/.env

.evergreen/scripts/generate_config.py

@@ -488,10 +488,12 @@ def create_aws_auth_variants():
         tasks = [".auth-aws"]
         tags = []
         if host_name == "macos":
-            tasks = [".auth-aws !.auth-aws-web-identity !.auth-aws-ecs !.auth-aws-ec2"]
+            tasks = [
+                ".auth-aws !.auth-aws-web-identity !.auth-aws-ecs !.auth-aws-ec2 !.auth-aws-eks"
+            ]
             tags = ["pr"]
         elif host_name == "win64":
-            tasks = [".auth-aws !.auth-aws-ecs"]
+            tasks = [".auth-aws !.auth-aws-ecs !.auth-aws-eks"]
         host = HOSTS[host_name]
         variant = create_variant(
             tasks,
@@ -741,33 +743,38 @@ def create_aws_tasks():
         "env-creds",
         "session-creds",
         "web-identity",
-        "ecs",
+        "web-identity-session-name",
     ]
+    assume_func = FunctionCall(func="assume ec2 role")
     for version, test_type, python in zip_cycle(get_versions_from("4.4"), aws_test_types, CPYTHONS):
         base_name = f"test-auth-aws-{version}"
         base_tags = ["auth-aws"]
         server_vars = dict(AUTH_AWS="1", VERSION=version)
         server_func = FunctionCall(func="run server", vars=server_vars)
-        assume_func = FunctionCall(func="assume ec2 role")
-        tags = [*base_tags, f"auth-aws-{test_type}"]
         name = get_task_name(f"{base_name}-{test_type}", python=python)
         test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, PYTHON_VERSION=python)
+        if test_type == "web-identity-session-name":
+            test_type = test_vars["SUB_TEST_NAME"] = "web-identity"  # noqa:PLW2901
+            test_vars["AWS_ROLE_SESSION_NAME"] = "test"
+        tags = [*base_tags, f"auth-aws-{test_type}"]
         test_func = FunctionCall(func="run tests", vars=test_vars)
         funcs = [server_func, assume_func, test_func]
         tasks.append(EvgTask(name=name, tags=tags, commands=funcs))
 
-        if test_type == "web-identity":
-            tags = [*base_tags, "auth-aws-web-identity"]
-            name = get_task_name(f"{base_name}-web-identity-session-name", python=python)
-            test_vars = dict(
-                TEST_NAME="auth_aws",
-                SUB_TEST_NAME="web-identity",
-                AWS_ROLE_SESSION_NAME="test",
-                PYTHON_VERSION=python,
-            )
-            test_func = FunctionCall(func="run tests", vars=test_vars)
-            funcs = [server_func, assume_func, test_func]
-            tasks.append(EvgTask(name=name, tags=tags, commands=funcs))
+    # These test types use a fixed Python version and the latest server.
+    for test_type in ["eks", "ecs"]:
+        tags = ["auth-aws", f"auth-aws-{test_type}"]
+        base_name = f"test-auth-aws-latest-{test_type}"
+        python = "3.11"
+        name = get_task_name(base_name, python=python)
+        test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, PYTHON_VERSION=python)
+        test_func = FunctionCall(func="run tests", vars=test_vars)
+        funcs = [assume_func, test_func]
+        if test_type == "ecs":
+            server_vars = dict(AUTH_AWS="1")
+            server_func = FunctionCall(func="run server", vars=server_vars)
+            funcs = [assume_func, server_func, test_func]
+        tasks.append(EvgTask(name=name, tags=tags, commands=funcs))
 
     return tasks
 

.evergreen/scripts/generate_config_utils.py

@@ -197,7 +197,7 @@ def get_common_name(base: str, sep: str, **kwargs) -> str:
         display_name = f"{display_name}{sep}{version}"
     for key, value in kwargs.items():
         name = value
-        if key.lower() == "python":
+        if key.lower() == "python" and value is not None:
             if not value.startswith("pypy"):
                 name = f"Python{value}"
             else:

.evergreen/run-mongodb-aws-ecs-test.sh → .evergreen/scripts/run-aws-container-test.sh

@@ -13,14 +13,17 @@ fi
 export MONGODB_URI="$1"
 
 if echo "$MONGODB_URI" | grep -q "@"; then
-  echo "MONGODB_URI unexpectedly contains user credentials in ECS test!";
+  echo "MONGODB_URI unexpectedly contains user credentials in container test!";
   exit 1
 fi
 # Now we can safely enable xtrace
 set -o xtrace
 
 # Install python with pip.
-PYTHON_VER="python3.9"
+PYTHON_VER="python3.xx"
+apt-get -qq update  < /dev/null > /dev/null
+apt-get -qq install software-properties-common -y < /dev/null > /dev/null  # needed for apt-add-repository
+add-apt-repository ppa:deadsnakes/ppa -y || true  # this will fail on debian
 apt-get -qq update  < /dev/null > /dev/null
 apt-get -qq install $PYTHON_VER $PYTHON_VER-venv build-essential $PYTHON_VER-dev -y  < /dev/null > /dev/null
 

.evergreen/scripts/run_tests.py

@@ -11,7 +11,7 @@
 from shutil import which
 
 import pytest
-from utils import DRIVERS_TOOLS, LOGGER, ROOT, run_command
+from utils import DRIVERS_TOOLS, HERE, LOGGER, ROOT, run_command
 
 AUTH = os.environ.get("AUTH", "noauth")
 SSL = os.environ.get("SSL", "nossl")
@@ -159,9 +159,15 @@ def run() -> None:
         result = main("-E -b doctest doc ./doc/_build/doctest".split())
         sys.exit(result)
 
-    # Send ecs tests to run remotely.
-    if TEST_NAME == "auth_aws" and SUB_TEST_NAME == "ecs":
-        run_command(f"{DRIVERS_TOOLS}/.evergreen/auth_aws/aws_setup.sh ecs")
+    # Send ecs and eks tests to run remotely.
+    if TEST_NAME == "auth_aws" and SUB_TEST_NAME in ["ecs", "eks"]:
+        target = f"run-mongodb-aws-{SUB_TEST_NAME}-test.sh"
+        text = (HERE / "run-aws-container-test.sh").read_text()
+        text = text.replace("python3.xx", os.environ["PYTHON_VERSION"])
+        if SUB_TEST_NAME == "eks":
+            text = text.replace("ecs", "eks")
+        (HERE.parent / target).write_text(text)
+        run_command(f"{DRIVERS_TOOLS}/.evergreen/auth_aws/aws_setup.sh {SUB_TEST_NAME}")
         return
 
     # Send OIDC tests to run remotely.

.evergreen/scripts/setup_tests.py

@@ -415,18 +415,23 @@ def handle_test_env() -> None:
 
         setup_kms(sub_test_name)
 
-    if test_name == "auth_aws" and sub_test_name != "ecs-remote":
+    if test_name == "auth_aws" and sub_test_name not in ["ecs-remote", "eks-remote"]:
         auth_aws_dir = f"{DRIVERS_TOOLS}/.evergreen/auth_aws"
         if "AWS_ROLE_SESSION_NAME" in os.environ:
             write_env("AWS_ROLE_SESSION_NAME")
-        if sub_test_name != "ecs":
+        if sub_test_name not in ["ecs", "eks"]:
             aws_setup = f"{auth_aws_dir}/aws_setup.sh"
             run_command(f"bash {aws_setup} {sub_test_name}")
             creds = read_env(f"{auth_aws_dir}/test-env.sh")
             for name, value in creds.items():
                 write_env(name, value)
         else:
             run_command(f"bash {auth_aws_dir}/setup-secrets.sh")
+        if sub_test_name == "eks":
+            # Store AWS creds if they were given.
+            for key in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]:
+                if key in os.environ:
+                    write_env(key, os.environ[key])
 
     if test_name == "atlas_connect":
         get_secrets("drivers/atlas_connect")

.evergreen/scripts/utils.py

@@ -143,8 +143,8 @@ def get_test_options(
         raise ValueError(f"Test '{test_name}' requires a sub_test_name")
     if "auth" in test_name or os.environ.get("AUTH") == "auth":
         opts.auth = True
-        # 'auth_aws ecs' shouldn't have extra auth set.
-        if test_name == "auth_aws" and sub_test_name == "ecs":
+        # auth_aws ecs or eks shouldn't have extra auth set.
+        if test_name == "auth_aws" and sub_test_name in ["ecs", "eks"]:
             opts.auth = False
     if os.environ.get("SSL") == "ssl":
         opts.ssl = True

.gitignore

@@ -30,6 +30,8 @@ expansion.yml
 .evergreen/scripts/test-env.sh
 specifications/
 results.json
+.evergreen/run-mongodb-aws-eks-test.sh
+.evergreen/run-mongodb-aws-ecs-test.sh
 
 # Lambda temp files
 test/lambda/.aws-sam