K8SPSMDB-1192: Introduce e2e-test pytest wrapper

.github/workflows/e2e-py-check.yml

@@ -0,0 +1,31 @@
+name: e2e-tests Python Quality Check
+
+on:
+  pull_request:
+    paths:
+      - 'e2e-tests/**/*.py'
+
+jobs:
+  quality-check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: "pyproject.toml"
+
+      - name: Install dependencies
+        run: uv sync --locked
+
+      - name: Run ruff check
+        run: uv run ruff check e2e-tests/
+
+      - name: Run mypy
+        run: uv run mypy e2e-tests/

.gitignore

@@ -189,3 +189,7 @@ bin/
 projects/
 installers/olm/operator_*.yaml
 installers/olm/bundles
+
+# Test Reports
+e2e-tests/reports/
+e2e-tests/**/__pycache__/

Jenkinsfile

@@ -5,15 +5,29 @@ tests=[]
 void createCluster(String CLUSTER_SUFFIX) {
     withCredentials([string(credentialsId: 'GCP_PROJECT_ID', variable: 'GCP_PROJECT'), file(credentialsId: 'gcloud-key-file', variable: 'CLIENT_SECRET_FILE')]) {
         sh """
-            NODES_NUM=3
             export KUBECONFIG=/tmp/$CLUSTER_NAME-${CLUSTER_SUFFIX}
+            gcloud auth activate-service-account --key-file $CLIENT_SECRET_FILE
+            gcloud config set project $GCP_PROJECT
             ret_num=0
             while [ \${ret_num} -lt 15 ]; do
                 ret_val=0
-                gcloud auth activate-service-account --key-file $CLIENT_SECRET_FILE
-                gcloud config set project $GCP_PROJECT
                 gcloud container clusters list --filter $CLUSTER_NAME-${CLUSTER_SUFFIX} --zone $region --format='csv[no-heading](name)' | xargs gcloud container clusters delete --zone $region --quiet || true
-                gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} --cluster-version=1.30 --machine-type=n1-standard-4 --preemptible --disk-size 30 --num-nodes=\$NODES_NUM --network=jenkins-vpc --subnetwork=jenkins-${CLUSTER_SUFFIX} --no-enable-autoupgrade --cluster-ipv4-cidr=/21 --labels delete-cluster-after-hours=6 --enable-ip-alias --workload-pool=cloud-dev-112233.svc.id.goog && \
+                gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} \
+                    --cluster-version=1.32 \
+                    --machine-type=n1-standard-4 \
+                    --preemptible \
+                    --disk-size 30 \
+                    --num-nodes=3 \
+                    --network=jenkins-vpc \
+                    --subnetwork=jenkins-${CLUSTER_SUFFIX} \
+                    --no-enable-autoupgrade \
+                    --cluster-ipv4-cidr=/21 \
+                    --labels delete-cluster-after-hours=6 \
+                    --enable-ip-alias \
+                    --monitoring=NONE \
+                    --logging=NONE \
+                    --no-enable-managed-prometheus \
+                    --workload-pool=cloud-dev-112233.svc.id.goog && \
                 kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user jenkins@"$GCP_PROJECT".iam.gserviceaccount.com || ret_val=\$?
                 if [ \${ret_val} -eq 0 ]; then break; fi
                 ret_num=\$((ret_num + 1))
@@ -98,6 +112,17 @@ void pushArtifactFile(String FILE_NAME) {
     }
 }
 
+void pushReportFile() {
+    echo "Push logfile final_report.html file to S3!"
+    withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
+        sh """
+            S3_PATH=s3://percona-jenkins-artifactory-public/\$JOB_NAME/\$(git rev-parse --short HEAD)
+            aws s3 ls \$S3_PATH/final_report.html || :
+            aws s3 cp --content-type text/html --quiet final_report.html \$S3_PATH/final_report.html || :
+        """
+    }
+}
+
 void initTests() {
     echo "Populating tests into the tests array!"
 
@@ -130,56 +155,52 @@ void markPassedTests() {
     }
 }
 
-void printKubernetesStatus(String LOCATION, String CLUSTER_SUFFIX) {
-    sh """
-        export KUBECONFIG=/tmp/$CLUSTER_NAME-$CLUSTER_SUFFIX
-        echo "========== KUBERNETES STATUS $LOCATION TEST =========="
-        gcloud container clusters list|grep -E "NAME|$CLUSTER_NAME-$CLUSTER_SUFFIX "
-        echo
-        kubectl get nodes
-        echo
-        kubectl top nodes
-        echo
-        kubectl get pods --all-namespaces
-        echo
-        kubectl top pod --all-namespaces
-        echo
-        kubectl get events --field-selector type!=Normal --all-namespaces --sort-by=".lastTimestamp"
-        echo "======================================================"
-    """
+String formatTime(def time) {
+    if (!time || time == "N/A") return "N/A"
+
+    try {
+        def totalSeconds = time as Double
+        def hours = (totalSeconds / 3600) as Integer
+        def minutes = ((totalSeconds % 3600) / 60) as Integer
+        def seconds = (totalSeconds % 60) as Integer
+
+        return String.format("%02d:%02d:%02d", hours, minutes, seconds)
+
+    } catch (Exception e) {
+        println("Error converting time: ${e.message}")
+        return time.toString()
+    }
 }
 
-TestsReport = '| Test name | Status |\r\n| ------------- | ------------- |'
-TestsReportXML = '<testsuite name=\\"PSMDB\\">\n'
+TestsReport = '| Test Name | Result | Time |\r\n| ----------- | -------- | ------ |'
 
 void makeReport() {
-    def wholeTestAmount=tests.size()
+    def wholeTestAmount = tests.size()
     def startedTestAmount = 0
+    def totalTestTime = 0
 
     for (int i=0; i<tests.size(); i++) {
         def testName = tests[i]["name"]
         def testResult = tests[i]["result"]
         def testTime = tests[i]["time"]
         def testUrl = "${testUrlPrefix}/${env.GIT_BRANCH}/${env.GIT_SHORT_COMMIT}/${testName}.log"
 
+        if (testTime instanceof Number) {
+            totalTestTime += testTime
+        }
+
         if (tests[i]["result"] != "skipped") {
             startedTestAmount++
         }
-        TestsReport = TestsReport + "\r\n| "+ testName +" | ["+ testResult +"]("+ testUrl +") |"
-        TestsReportXML = TestsReportXML + '<testcase name=\\"' + testName + '\\" time=\\"' + testTime + '\\"><'+ testResult +'/></testcase>\n'
+        TestsReport = TestsReport + "\r\n| " + testName + " | [" + testResult + "](" + testUrl + ") | " + formatTime(testTime) + " |"
     }
-    TestsReport = TestsReport + "\r\n| We run $startedTestAmount out of $wholeTestAmount|"
-    TestsReportXML = TestsReportXML + '</testsuite>\n'
-
-    sh """
-        echo "${TestsReportXML}" > TestsReport.xml
-    """
+    TestsReport = TestsReport + "\r\n| We run $startedTestAmount out of $wholeTestAmount | | " + formatTime(totalTestTime) + " |"
 }
 
 void clusterRunner(String cluster) {
     def clusterCreated=0
 
-    for (int i=0; i<tests.size(); i++) {
+    for (int i = 0; i < tests.size(); i++) {
         if (tests[i]["result"] == "skipped" && currentBuild.nextBuild == null) {
             tests[i]["result"] = "failure"
             tests[i]["cluster"] = cluster
@@ -193,6 +214,8 @@ void clusterRunner(String cluster) {
 
     if (clusterCreated >= 1) {
         shutdownCluster(cluster)
+        // Re-check for passed tests after execution
+        markPassedTests()
     }
 }
 
@@ -215,15 +238,18 @@ void runTest(Integer TEST_ID) {
                         export DEBUG_TESTS=1
                     fi
                     export KUBECONFIG=/tmp/$CLUSTER_NAME-$clusterSuffix
-                    time ./e2e-tests/$testName/run
+
+                    source \$HOME/.local/bin/env
+                    uv run pytest e2e-tests/test_pytest_wrapper.py -v -s --test-regex "^${testName}\$" \
+                        --html=e2e-tests/reports/$CLUSTER_NAME-$testName-report.html \
+                        --junitxml=e2e-tests/reports/$CLUSTER_NAME-$testName-report.xml
                 """
             }
             pushArtifactFile("${env.GIT_BRANCH}-${env.GIT_SHORT_COMMIT}-$testName")
             tests[TEST_ID]["result"] = "passed"
             return true
         }
         catch (exc) {
-            printKubernetesStatus("AFTER","$clusterSuffix")
             echo "Test $testName has failed!"
             if (retryCount >= 1 || currentBuild.nextBuild != null) {
                 currentBuild.result = 'FAILURE'
@@ -234,7 +260,7 @@ void runTest(Integer TEST_ID) {
         }
         finally {
             def timeStop = new Date().getTime()
-            def durationSec = (timeStop - timeStart) / 1000
+            def durationSec = (timeStop - timeStart) / 1000.0
             tests[TEST_ID]["time"] = durationSec
             pushLogFile("$testName")
             echo "The $testName test was finished!"
@@ -264,6 +290,11 @@ EOF
         sudo yum install -y google-cloud-cli google-cloud-cli-gke-gcloud-auth-plugin
 
         curl -sL https://github.com/mitchellh/golicense/releases/latest/download/golicense_0.2.0_linux_x86_64.tar.gz | sudo tar -C /usr/local/bin -xzf - golicense
+
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        source \$HOME/.local/bin/env
+        uv python install 3.13
+        uv sync --locked
     """
 }
 
@@ -351,7 +382,7 @@ pipeline {
         CLOUDSDK_CORE_DISABLE_PROMPTS = 1
         CLEAN_NAMESPACE = 1
         OPERATOR_NS = 'psmdb-operator'
-        GIT_SHORT_COMMIT = sh(script: 'git rev-parse --short HEAD', , returnStdout: true).trim()
+        GIT_SHORT_COMMIT = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim()
         VERSION = "${env.GIT_BRANCH}-${env.GIT_SHORT_COMMIT}"
         CLUSTER_NAME = sh(script: "echo jen-psmdb-${env.CHANGE_ID}-${GIT_SHORT_COMMIT}-${env.BUILD_NUMBER} | tr '[:upper:]' '[:lower:]'", , returnStdout: true).trim()
         AUTHOR_NAME = sh(script: "echo ${CHANGE_AUTHOR_EMAIL} | awk -F'@' '{print \$1}'", , returnStdout: true).trim()
@@ -386,7 +417,7 @@ pipeline {
                 prepareNode()
                 script {
                     if (AUTHOR_NAME == 'null') {
-                        AUTHOR_NAME = sh(script: "git show -s --pretty=%ae | awk -F'@' '{print \$1}'", , returnStdout: true).trim()
+                        AUTHOR_NAME = sh(script: "git show -s --pretty=%ae | awk -F'@' '{print \$1}'", returnStdout: true).trim()
                     }
                     for (comment in pullRequest.comments) {
                         println("Author: ${comment.user}, Comment: ${comment.body}")
@@ -499,7 +530,7 @@ pipeline {
                 }
             }
             options {
-                timeout(time: 3, unit: 'HOURS')
+                timeout(time: 4, unit: 'HOURS')
             }
             parallel {
                 stage('cluster1') {
@@ -578,12 +609,21 @@ pipeline {
                             }
                         }
                         makeReport()
-                        step([$class: 'JUnitResultArchiver', testResults: '*.xml', healthScaleFactor: 1.0])
-                        archiveArtifacts '*.xml'
-
+                        if (fileExists('e2e-tests/reports')){
+                            sh """
+                            source \$HOME/.local/bin/env
+                            uv run pytest_html_merger -i e2e-tests/reports -o final_report.html
+                            uv run junitparser merge --glob 'e2e-tests/reports/*.xml' final_report.xml
+                            """
+                            step([$class: 'JUnitResultArchiver', testResults: 'final_report.xml', healthScaleFactor: 1.0])
+                            archiveArtifacts 'final_report.xml, final_report.html'
+                            pushReportFile()
+                        } else {
+                            echo "No report files found in e2e-tests/reports, skipping report generation"
+                        }
                         unstash 'IMAGE'
                         def IMAGE = sh(returnStdout: true, script: "cat results/docker/TAG").trim()
-                        TestsReport = TestsReport + "\r\n\r\ncommit: ${env.CHANGE_URL}/commits/${env.GIT_COMMIT}\r\nimage: `${IMAGE}`\r\n"
+                        TestsReport = TestsReport + "\r\n\r\nCommit: ${env.CHANGE_URL}/commits/${env.GIT_COMMIT}\r\nImage: `${IMAGE}`\r\nTest report: [report](${testUrlPrefix}/${env.GIT_BRANCH}/${env.GIT_SHORT_COMMIT}/final_report.html)\r\n"
                         pullRequest.comment(TestsReport)
                     }
                     deleteOldClusters("$CLUSTER_NAME")

e2e-tests/conftest.py

@@ -0,0 +1,110 @@
+import re
+import pytest
+
+from pathlib import Path
+from typing import Tuple, List
+
+from tools.report_generator import generate_report
+from tools.k8s_resources_collector import collect_k8s_resources, get_namespace
+
+
+def pytest_addoption(parser):
+    """Add custom command line option for test suite file"""
+    parser.addoption(
+        "--test-suite",
+        action="store",
+        default=None,
+        help="Name of the test suite file (will look for run-{name}.csv)",
+    )
+    parser.addoption(
+        "--test-regex",
+        action="store",
+        default=None,
+        help="Run tests matching the given regex pattern",
+    )
+    parser.addoption(
+        "--collect-k8s-resources",
+        action="store_true",
+        default=False,
+        help="Enable collection of K8s resources on test failure",
+    )
+
+
+def get_bash_tests(test_suite: str = "") -> List[Tuple[str, Path]]:
+    """Get bash test scripts from file or all directories"""
+    current_dir = Path(__file__).parent
+    bash_tests: List[Tuple[str, Path]] = []
+
+    if test_suite:
+        file_path = current_dir / f"run-{test_suite}.csv"
+        if not file_path.exists():
+            raise FileNotFoundError(f"Test suite file not found: {file_path}")
+
+        with open(file_path, "r", encoding="utf-8") as f:
+            test_names = [line.strip() for line in f if line.strip()]
+    else:
+        test_names = [d.name for d in current_dir.iterdir() if d.is_dir()]
+
+    for test_name in test_names:
+        test_dir = current_dir / test_name
+        run_script = test_dir / "run"
+        if run_script.exists():
+            bash_tests.append((test_name, run_script))
+
+    return bash_tests
+
+
+def pytest_generate_tests(metafunc):
+    """Generate tests dynamically with regex filtering"""
+    if "test_name" in metafunc.fixturenames and "script_path" in metafunc.fixturenames:
+        test_suite = metafunc.config.getoption("--test-suite")
+        test_regex = metafunc.config.getoption("--test-regex")
+
+        bash_tests = get_bash_tests(test_suite)
+        if test_regex:
+            try:
+                pattern = re.compile(test_regex)
+                filtered_tests = [
+                    (name, path) for name, path in bash_tests if pattern.search(name)
+                ]
+                bash_tests = filtered_tests
+
+                print(f"\nFiltered to {len(bash_tests)} test(s) matching regex '{test_regex}':")
+                for name, _ in bash_tests:
+                    print(f"  - {name}")
+
+            except re.error as e:
+                pytest.exit(f"Invalid regex pattern '{test_regex}': {e}")
+
+        metafunc.parametrize(
+            "test_name,script_path", bash_tests, ids=[name for name, _ in bash_tests]
+        )
+
+
+def pytest_html_report_title(report):
+    report.title = "PSMDB E2E Test Report"
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    outcome = yield
+    report = outcome.get_result()
+
+    if report.when == "call" and report.failed:
+        try:
+            namespace = get_namespace(str(report.longrepr))
+            html_report = generate_report(namespace)
+
+            if not hasattr(report, "extras"):
+                report.extras = []
+            report.extras.extend(html_report)
+
+            collect_resources = item.config.getoption("--collect-k8s-resources")
+            if collect_resources:
+                collect_k8s_resources(
+                    namespace=namespace,
+                    custom_resources=["psmdb", "psmdb-backup", "psmdb-restore"],
+                    output_dir=f"e2e-tests/reports/{namespace}",
+                )
+        except Exception as e:
+            print(f"Error adding K8s info: {e}")