From fa397c990636e2d2f91624e2fce32ab007ef048c Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 23 Apr 2025 14:10:17 -0300 Subject: [PATCH 01/19] Draft --- .github/workflows/checks.yml | 2 +- exasol/slc_ci/api/__init__.py | 5 + exasol/slc_ci/api/check_if_build_needed.py | 18 ++ .../api/export_and_scan_vulnerabilities.py | 40 ++++ exasol/slc_ci/api/get_flavor_ci_model.py | 7 + exasol/slc_ci/api/get_flavors.py | 10 + exasol/slc_ci/api/run_tests.py | 27 +++ exasol/slc_ci/lib/__init__.py | 0 exasol/slc_ci/lib/branch_config.py | 60 ++++++ exasol/slc_ci/lib/check_if_build_needed.py | 46 +++++ exasol/slc_ci/lib/ci_build.py | 78 ++++++++ exasol/slc_ci/lib/ci_export.py | 42 +++++ exasol/slc_ci/lib/ci_prepare.py | 15 ++ exasol/slc_ci/lib/ci_push.py | 44 +++++ exasol/slc_ci/lib/ci_security_scan.py | 36 ++++ exasol/slc_ci/lib/ci_step_output_printer.py | 55 ++++++ exasol/slc_ci/lib/ci_test.py | 110 +++++++++++ .../lib/export_and_scan_vulnerabilities.py | 44 +++++ exasol/slc_ci/lib/git_access.py | 43 +++++ exasol/slc_ci/lib/run_tests.py | 32 ++++ exasol/slc_ci/model/build_config.py | 19 ++ exasol/slc_ci/model/flavor_ci_model.py | 18 ++ exasol/slc_ci/nox/tasks.py | 171 ++++++++++++++++++ .../slc_ci}/version.py | 0 noxconfig.py | 4 +- pyproject.toml | 1 + 26 files changed, 923 insertions(+), 4 deletions(-) create mode 100644 exasol/slc_ci/api/__init__.py create mode 100644 exasol/slc_ci/api/check_if_build_needed.py create mode 100644 exasol/slc_ci/api/export_and_scan_vulnerabilities.py create mode 100644 exasol/slc_ci/api/get_flavor_ci_model.py create mode 100644 exasol/slc_ci/api/get_flavors.py create mode 100644 exasol/slc_ci/api/run_tests.py create mode 100644 exasol/slc_ci/lib/__init__.py create mode 100644 exasol/slc_ci/lib/branch_config.py create mode 100644 exasol/slc_ci/lib/check_if_build_needed.py create mode 100644 exasol/slc_ci/lib/ci_build.py create mode 100644 exasol/slc_ci/lib/ci_export.py create mode 100644 exasol/slc_ci/lib/ci_prepare.py create mode 100644 exasol/slc_ci/lib/ci_push.py create mode 100644 exasol/slc_ci/lib/ci_security_scan.py create mode 100644 exasol/slc_ci/lib/ci_step_output_printer.py create mode 100644 exasol/slc_ci/lib/ci_test.py create mode 100644 exasol/slc_ci/lib/export_and_scan_vulnerabilities.py create mode 100644 exasol/slc_ci/lib/git_access.py create mode 100644 exasol/slc_ci/lib/run_tests.py create mode 100644 exasol/slc_ci/model/build_config.py create mode 100644 exasol/slc_ci/model/flavor_ci_model.py create mode 100644 exasol/slc_ci/nox/tasks.py rename {exasol_script_languages_container_ci => exasol/slc_ci}/version.py (100%) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 1ff74b6..ba892f5 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -20,7 +20,7 @@ jobs: - name: Check Version(s) run: | - poetry run -- version-check exasol_script_languages_container_ci/version.py + poetry run -- version-check exasol/slc_ci/version.py build-matrix: diff --git a/exasol/slc_ci/api/__init__.py b/exasol/slc_ci/api/__init__.py new file mode 100644 index 0000000..fb5c18a --- /dev/null +++ b/exasol/slc_ci/api/__init__.py @@ -0,0 +1,5 @@ +from .check_if_build_needed import check_if_build_needed +from .export_and_scan_vulnerabilities import export_and_scan_vulnerabilities +from .get_flavor_ci_model import get_flavor_ci_model +from .get_flavors import get_flavors +from .run_tests import run_tests diff --git a/exasol/slc_ci/api/check_if_build_needed.py b/exasol/slc_ci/api/check_if_build_needed.py new file mode 100644 index 0000000..4b4ed1c --- /dev/null +++ b/exasol/slc_ci/api/check_if_build_needed.py @@ -0,0 +1,18 @@ +from exasol.slc_ci.lib.check_if_build_needed import ( + check_if_need_to_build as lib_check_if_need_to_build, +) +from exasol.slc_ci.lib.git_access import GitAccess +from exasol.slc_ci.model.build_config import BuildConfig + + +def check_if_build_needed( + flavor: str, build_config: BuildConfig, branch_name: str +) -> bool: + + git_access: GitAccess = GitAccess() + return lib_check_if_need_to_build( + branch_name=branch_name, + build_config=build_config, + flavor=flavor, + git_access=git_access, + ) diff --git a/exasol/slc_ci/api/export_and_scan_vulnerabilities.py b/exasol/slc_ci/api/export_and_scan_vulnerabilities.py new file mode 100644 index 0000000..3e1fd1a --- /dev/null +++ b/exasol/slc_ci/api/export_and_scan_vulnerabilities.py @@ -0,0 +1,40 @@ +from pathlib import Path + +from exasol.slc_ci.lib.ci_build import CIBuild +from exasol.slc_ci.lib.ci_export import CIExport +from exasol.slc_ci.lib.ci_prepare import CIPrepare +from exasol.slc_ci.lib.ci_security_scan import CISecurityScan +from exasol.slc_ci.lib.export_and_scan_vulnerabilities import ( + export_and_scan_vulnerabilities as lib_export_and_scan_vulnerabilities, +) +from exasol.slc_ci.lib.git_access import GitAccess +from exasol.slc_ci.model.build_config import BuildConfig + + +def export_and_scan_vulnerabilities( + flavor: str, + build_config: BuildConfig, + branch_name: str, + docker_user: str, + docker_password: str, + commit_sha: str, +) -> Path: + git_access: GitAccess = GitAccess() + ci_build: CIBuild = CIBuild() + ci_security_scan: CISecurityScan = CISecurityScan() + ci_prepare: CIPrepare = CIPrepare() + ci_export: CIExport = CIExport() + + return lib_export_and_scan_vulnerabilities( + flavor=flavor, + branch_name=branch_name, + docker_user=docker_user, + docker_password=docker_password, + commit_sha=commit_sha, + build_config=build_config, + git_access=git_access, + ci_build=ci_build, + ci_security_scan=ci_security_scan, + ci_prepare=ci_prepare, + ci_export=ci_export, + ) diff --git a/exasol/slc_ci/api/get_flavor_ci_model.py b/exasol/slc_ci/api/get_flavor_ci_model.py new file mode 100644 index 0000000..000757a --- /dev/null +++ b/exasol/slc_ci/api/get_flavor_ci_model.py @@ -0,0 +1,7 @@ +from exasol.slc_ci.model.build_config import BuildConfig +from exasol.slc_ci.model.flavor_ci_model import FlavorCiConfig + + +def get_flavor_ci_model(build_config: BuildConfig, flavor: str) -> FlavorCiConfig: + flavor_ci_config_path = build_config.flavors_path / flavor / "ci.json" + return FlavorCiConfig.model_validate_json(flavor_ci_config_path.read_text()) diff --git a/exasol/slc_ci/api/get_flavors.py b/exasol/slc_ci/api/get_flavors.py new file mode 100644 index 0000000..6ed46da --- /dev/null +++ b/exasol/slc_ci/api/get_flavors.py @@ -0,0 +1,10 @@ +from pathlib import Path +from typing import Iterable + + +def get_flavors(flavors_path: Path) -> Iterable[str]: + if not flavors_path.exists(): + raise ValueError(f"{flavors_path} does not exist") + for p in flavors_path.iterdir(): + if p.is_dir(): + yield p.name diff --git a/exasol/slc_ci/api/run_tests.py b/exasol/slc_ci/api/run_tests.py new file mode 100644 index 0000000..a01095a --- /dev/null +++ b/exasol/slc_ci/api/run_tests.py @@ -0,0 +1,27 @@ +from pathlib import Path +from typing import List + +from exasol.slc_ci.lib.ci_prepare import CIPrepare +from exasol.slc_ci.lib.ci_test import CIExecuteTest +from exasol.slc_ci.lib.run_tests import run_tests as lib_run_tests + + +def run_tests( + flavor: str, + slc_path: Path, + test_folders: List[str], + docker_user: str, + docker_password: str, +) -> None: + ci_test = CIExecuteTest() + ci_prepare = CIPrepare() + + return lib_run_tests( + flavor=flavor, + slc_path=slc_path, + test_folders=test_folders, + docker_user=docker_user, + docker_password=docker_password, + ci_prepare=ci_prepare, + ci_test=ci_test, + ) diff --git a/exasol/slc_ci/lib/__init__.py b/exasol/slc_ci/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/exasol/slc_ci/lib/branch_config.py b/exasol/slc_ci/lib/branch_config.py new file mode 100644 index 0000000..7b276f4 --- /dev/null +++ b/exasol/slc_ci/lib/branch_config.py @@ -0,0 +1,60 @@ +import re +from enum import Enum, auto + + +class BuildSteps(Enum): + BUILD_ALL_ALWAYS = auto() + REBUILD = auto() + PUSH_TO_DOCKER_RELEASE_REPO = auto() + + +class BranchConfig(Enum): + DEVELOP = { + BuildSteps.BUILD_ALL_ALWAYS: True, + BuildSteps.REBUILD: True, + BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: False, + } + MAIN = { + BuildSteps.BUILD_ALL_ALWAYS: True, + BuildSteps.REBUILD: True, + BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: True, + } + REBUILD = { + BuildSteps.BUILD_ALL_ALWAYS: True, + BuildSteps.REBUILD: True, + BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: False, + } + OTHER = { + BuildSteps.BUILD_ALL_ALWAYS: False, + BuildSteps.REBUILD: False, + BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: False, + } + + @staticmethod + def build_always(branch_name: str) -> bool: + return get_branch_config(branch_name).value[BuildSteps.BUILD_ALL_ALWAYS] + + @staticmethod + def rebuild(branch_name) -> bool: + return get_branch_config(branch_name).value[BuildSteps.REBUILD] + + @staticmethod + def push_to_docker_release_repo(branch_name: str) -> bool: + return get_branch_config(branch_name).value[ + BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO + ] + + +def get_branch_config(branch_name: str) -> BranchConfig: + matches = ( + (re.compile(r"refs/heads/(master|main)"), BranchConfig.MAIN), + (re.compile(r"refs/heads/develop"), BranchConfig.DEVELOP), + (re.compile(r"refs/heads/rebuild/.*"), BranchConfig.REBUILD), + ) + + branch_cfg = BranchConfig.OTHER + for branch_regex, branch_config in matches: + if branch_regex.match(branch_name): + branch_cfg = branch_config + break + return branch_cfg diff --git a/exasol/slc_ci/lib/check_if_build_needed.py b/exasol/slc_ci/lib/check_if_build_needed.py new file mode 100644 index 0000000..11fb740 --- /dev/null +++ b/exasol/slc_ci/lib/check_if_build_needed.py @@ -0,0 +1,46 @@ +import logging +from typing import Set + +from exasol.slc_ci.lib.branch_config import BranchConfig +from exasol.slc_ci.lib.git_access import GitAccess +from exasol.slc_ci.model.build_config import BuildConfig + + +def get_all_affected_files(git_access: GitAccess, base_branch: str) -> Set[str]: + base_last_commit_sha = git_access.get_head_commit_sha_of_branch(base_branch) + changed_files = set() # type: ignore + for commit in git_access.get_last_commits(): + if commit == base_last_commit_sha: + break + changed_files.update(git_access.get_files_of_commit(commit)) + return changed_files + + +def check_if_need_to_build( + branch_name: str, build_config: BuildConfig, flavor: str, git_access: GitAccess +) -> bool: + if BranchConfig.build_always(branch_name): + return True + if "[rebuild]" in git_access.get_last_commit_message(): + return True + affected_files = list(get_all_affected_files(git_access, build_config.base_branch)) + logging.debug( + f"check_if_need_to_build: Found files of last commits: {affected_files}" + ) + for ignore_path in build_config.ignore_paths: + affected_files = list( + filter(lambda file: not file.startswith(ignore_path), affected_files) + ) + + if len(affected_files) > 0: + # Now filter out also other flavor folders + this_flavor_path = f"flavors/{flavor}" + affected_files = list( + filter( + lambda file: not file.startswith("flavors") + or file.startswith(this_flavor_path), + affected_files, + ) + ) + logging.debug(f"check_if_need_to_build: filtered files: {affected_files}") + return len(affected_files) > 0 diff --git a/exasol/slc_ci/lib/ci_build.py b/exasol/slc_ci/lib/ci_build.py new file mode 100644 index 0000000..74f2fa3 --- /dev/null +++ b/exasol/slc_ci/lib/ci_build.py @@ -0,0 +1,78 @@ +import logging +from typing import Optional, Tuple + +from exasol.slc.api import build +from exasol.slc.internal.tasks.test.test_container_content import ( + build_test_container_content, +) +from exasol_integration_test_docker_environment.lib.api.build_test_container import ( + build_test_container, +) + +from exasol.slc_ci.lib.ci_step_output_printer import ( + CIStepOutputPrinter, + CIStepOutputPrinterProtocol, +) + + +class CIBuild: + + def __init__( + self, printer: CIStepOutputPrinterProtocol = CIStepOutputPrinter(logging.info) + ): + self._printer = printer + + def build( + self, + flavor_path: Tuple[str, ...], + rebuild: bool, + build_docker_repository: Optional[str], + commit_sha: str, + docker_user: str, + docker_password: str, + test_container_folder: str, + ): + """ + Build the script-language container for given flavor. And also build the test container + """ + + logging.info(f"Running command 'build' with parameters: {locals()}") + if build_docker_repository is None: + slc_image_infos = build( + flavor_path=flavor_path, + force_rebuild=rebuild, + source_docker_tag_prefix=commit_sha, + source_docker_username=docker_user, + source_docker_password=docker_password, + shortcut_build=False, + workers=7, + log_level="WARNING", + use_job_specific_log_file=True, + ) + else: + slc_image_infos = build( + flavor_path=flavor_path, + force_rebuild=rebuild, + source_docker_repository_name=build_docker_repository, + source_docker_tag_prefix=commit_sha, + source_docker_username=docker_user, + source_docker_password=docker_password, + shortcut_build=False, + workers=7, + log_level="WARNING", + use_job_specific_log_file=True, + ) + logging.info( + f"Running command 'build_test_container' with parameters: {locals()}" + ) + content = build_test_container_content(test_container_folder) + test_container_image_infos = build_test_container( + force_rebuild=rebuild, + workers=7, + test_container_content=content, + source_docker_repository_name=build_docker_repository, + source_docker_tag_prefix=commit_sha, + log_level="WARNING", + use_job_specific_log_file=True, + ) + self._printer.print_exasol_docker_images() diff --git a/exasol/slc_ci/lib/ci_export.py b/exasol/slc_ci/lib/ci_export.py new file mode 100644 index 0000000..6bb764c --- /dev/null +++ b/exasol/slc_ci/lib/ci_export.py @@ -0,0 +1,42 @@ +import logging +from pathlib import Path +from typing import Tuple + +from exasol.slc.api import export + +from exasol.slc_ci.lib.ci_step_output_printer import ( + CIStepOutputPrinter, + CIStepOutputPrinterProtocol, +) + + +class CIExport: + + def __init__( + self, printer: CIStepOutputPrinterProtocol = CIStepOutputPrinter(logging.info) + ): + self._printer = printer + + def export(self, flavor_path: Tuple[str, ...]) -> Path: + """ + Export the flavor as tar.gz file + """ + + logging.info(f"Running command 'export' with parameters: {locals()}") + export_result = export( + flavor_path=flavor_path, + workers=7, + log_level="WARNING", + use_job_specific_log_file=True, + ) + self._printer.print_exasol_docker_images() + export_infos = list(export_result.export_infos.values()) + if len(export_infos) != 1: + raise RuntimeError(f"Unexpected number of export infos") + export_info = export_infos[0] + export_flavor_infos = list(export_info.values()) + if len(export_flavor_infos) != 1: + raise RuntimeError(f"Unexpected number of export flavor infos") + + export_flavor_info = export_flavor_infos[0] + return Path(export_flavor_info.cache_file) diff --git a/exasol/slc_ci/lib/ci_prepare.py b/exasol/slc_ci/lib/ci_prepare.py new file mode 100644 index 0000000..8932eed --- /dev/null +++ b/exasol/slc_ci/lib/ci_prepare.py @@ -0,0 +1,15 @@ +import os +from pathlib import Path + +from exasol_integration_test_docker_environment.cli.options.system_options import ( + DEFAULT_OUTPUT_DIRECTORY, +) +from exasol_integration_test_docker_environment.lib.logging import luigi_log_config + + +class CIPrepare: + + def prepare(self): + log_path = Path(DEFAULT_OUTPUT_DIRECTORY) / "jobs" / "logs" / "main.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + os.environ[luigi_log_config.LOG_ENV_VARIABLE_NAME] = f"{log_path.absolute()}" diff --git a/exasol/slc_ci/lib/ci_push.py b/exasol/slc_ci/lib/ci_push.py new file mode 100644 index 0000000..c191d5a --- /dev/null +++ b/exasol/slc_ci/lib/ci_push.py @@ -0,0 +1,44 @@ +import logging +from typing import Tuple + +from exasol.slc.api.push import push + +from exasol.slc_ci.lib.ci_step_output_printer import ( + CIStepOutputPrinter, + CIStepOutputPrinterProtocol, +) + + +class CIPush: + + def __init__( + self, printer: CIStepOutputPrinterProtocol = CIStepOutputPrinter(logging.info) + ): + self._printer = printer + + def push( + self, + flavor_path: Tuple[str, ...], + target_docker_repository: str, + target_docker_tag_prefix: str, + docker_user: str, + docker_password: str, + ): + """ + Push the docker image to Dockerhub + """ + + logging.info(f"Running command 'push' with parameters: {locals()}") + push( + flavor_path=flavor_path, + push_all=True, + force_push=True, + workers=7, + target_docker_repository_name=target_docker_repository, + target_docker_tag_prefix=target_docker_tag_prefix, + target_docker_username=docker_user, + target_docker_password=docker_password, + log_level="WARNING", + use_job_specific_log_file=True, + ) + self._printer.print_exasol_docker_images() diff --git a/exasol/slc_ci/lib/ci_security_scan.py b/exasol/slc_ci/lib/ci_security_scan.py new file mode 100644 index 0000000..ebf26fa --- /dev/null +++ b/exasol/slc_ci/lib/ci_security_scan.py @@ -0,0 +1,36 @@ +import logging +from pathlib import Path +from typing import Tuple + +from exasol.slc.api import security_scan + +from exasol.slc_ci.lib.ci_step_output_printer import ( + CIStepOutputPrinter, + CIStepOutputPrinterProtocol, +) + + +class CISecurityScan: + + def __init__( + self, printer: CIStepOutputPrinterProtocol = CIStepOutputPrinter(logging.info) + ): + self._printer = printer + + def run_security_scan(self, flavor_path: Tuple[str, ...]): + """ + Run security scan and print result + """ + + logging.info(f"Running command 'security_scan' with parameters {locals()}") + security_scan_result = security_scan( + flavor_path=flavor_path, + workers=7, + log_level="WARNING", + use_job_specific_log_file=True, + ) + logging.info("============= SECURITY REPORT ===========") + self._printer.print_file(Path(security_scan_result.report_path)) + self._printer.print_exasol_docker_images() + if not security_scan_result.scans_are_ok: + raise AssertionError("Some security scans not successful.") diff --git a/exasol/slc_ci/lib/ci_step_output_printer.py b/exasol/slc_ci/lib/ci_step_output_printer.py new file mode 100644 index 0000000..b660a0f --- /dev/null +++ b/exasol/slc_ci/lib/ci_step_output_printer.py @@ -0,0 +1,55 @@ +from inspect import cleandoc +from pathlib import Path +from typing import Callable, Protocol + +import docker + + +class CIStepOutputPrinterProtocol(Protocol): + + def print_exasol_docker_images(self): + raise NotImplementedError() + + def print_file(self, filename: Path): + raise NotImplementedError() + + +def _get_exasol_docker_images(): + docker_client = docker.from_env() + try: + exa_images = [ + str(img) for img in docker_client.images.list() if "exasol" in str(img) + ] + return exa_images + finally: + docker_client.close() + return [] + + +class CIStepOutputPrinter(CIStepOutputPrinterProtocol): + + def __init__(self, writer: Callable[[str], None]): + self._writer = writer + + def print_exasol_docker_images(self): + """ + Prints all docker images with "exasol" in it's name to stdout. + :return: None + """ + + self._writer( + cleandoc( + """ + {seperator} + Printing docker images + {seperator} + {images}""" + ).format(seperator=20 * "=", images="\n".join(_get_exasol_docker_images())) + ) + + def print_file(self, filename: Path): + """ + Print the file's content to the writer. + """ + with open(filename) as f: + self._writer(f.read()) diff --git a/exasol/slc_ci/lib/ci_test.py b/exasol/slc_ci/lib/ci_test.py new file mode 100644 index 0000000..5954d2f --- /dev/null +++ b/exasol/slc_ci/lib/ci_test.py @@ -0,0 +1,110 @@ +import logging +from pathlib import Path +from typing import Optional, Protocol, Tuple + +from exasol.slc.api.run_db_tests import run_db_test +from exasol.slc.models.test_result import AllTestsResult + +from exasol.slc_ci.lib.ci_step_output_printer import ( + CIStepOutputPrinter, + CIStepOutputPrinterProtocol, +) + + +class DBTestRunnerProtocol(Protocol): + def run( + self, + flavor_path: Tuple[str, ...], + release_goal: Tuple[str, ...], + test_folder: Tuple[str, ...], + test_container_folder: str, + workers: int, + docker_username: str, + docker_password: str, + use_existing_container: Optional[str], + ) -> AllTestsResult: + raise NotImplementedError() + + +class DBTestRunner(DBTestRunnerProtocol): + def run( + self, + flavor_path: Tuple[str, ...], + release_goal: Tuple[str, ...], + test_folder: Tuple[str, ...], + test_container_folder: str, + workers: int, + docker_username: str, + docker_password: str, + use_existing_container: Optional[str], + ) -> AllTestsResult: + return run_db_test( + flavor_path=flavor_path, + release_goal=release_goal, + test_folder=test_folder, + test_container_folder=test_container_folder, + workers=workers, + source_docker_username=docker_username, + source_docker_password=docker_password, + log_level="WARNING", + use_job_specific_log_file=True, + use_existing_container=use_existing_container, + ) + + +class CIExecuteTest: + + def __init__( + self, + db_test_runner: DBTestRunnerProtocol = DBTestRunner(), + printer: CIStepOutputPrinterProtocol = CIStepOutputPrinter(logging.info), + ): + self._db_test_runner = db_test_runner + self._printer = printer + + def execute_tests( + self, + flavor_path: Tuple[str, ...], + slc_path: Path, + test_folder: str, + docker_user: str, + docker_password: str, + test_container_folder: str, + ): + """ + Run db tests + """ + db_tests_are_ok = self._run_db_tests( + flavor_path=flavor_path, + slc_path=slc_path, + test_folder=test_folder, + docker_user=docker_user, + docker_password=docker_password, + test_container_folder=test_container_folder, + ) + self._printer.print_exasol_docker_images() + if not db_tests_are_ok: + raise AssertionError("Not all tests are ok!") + + def _run_db_tests( + self, + flavor_path: Tuple[str, ...], + slc_path: Path, + test_folder: str, + docker_user: str, + docker_password: str, + test_container_folder: str, + ) -> bool: + logging.info(f"Running command 'run_db_test' for flavor-path {flavor_path}") + db_test_result = self._db_test_runner.run( + flavor_path=flavor_path, + test_folder=(test_folder,), + release_goal=("release",), + workers=7, + docker_username=docker_user, + docker_password=docker_password, + test_container_folder=test_container_folder, + use_existing_container=str(slc_path), + ) + self._printer.print_file(db_test_result.command_line_output_path) + return db_test_result.tests_are_ok diff --git a/exasol/slc_ci/lib/export_and_scan_vulnerabilities.py b/exasol/slc_ci/lib/export_and_scan_vulnerabilities.py new file mode 100644 index 0000000..508937f --- /dev/null +++ b/exasol/slc_ci/lib/export_and_scan_vulnerabilities.py @@ -0,0 +1,44 @@ +import logging +from pathlib import Path + +from exasol.slc_ci.lib.branch_config import BranchConfig +from exasol.slc_ci.lib.ci_build import CIBuild +from exasol.slc_ci.lib.ci_export import CIExport +from exasol.slc_ci.lib.ci_prepare import CIPrepare +from exasol.slc_ci.lib.ci_security_scan import CISecurityScan +from exasol.slc_ci.lib.git_access import GitAccess +from exasol.slc_ci.model.build_config import BuildConfig + + +def export_and_scan_vulnerabilities( + flavor: str, + branch_name: str, + docker_user: str, + docker_password: str, + commit_sha: str, + build_config: BuildConfig, + git_access: GitAccess, + ci_build: CIBuild = CIBuild(), + ci_security_scan: CISecurityScan = CISecurityScan(), + ci_prepare: CIPrepare = CIPrepare(), + ci_export: CIExport = CIExport(), +) -> Path: + logging.info( + f"Running build image and scan vulnerabilities for parameters: {locals()}" + ) + + flavor_path = (f"{build_config.flavors_path.name}/{flavor}",) + test_container_folder = build_config.test_container_folder + rebuild = BranchConfig.rebuild(branch_name) + ci_prepare.prepare() + ci_build.build( + flavor_path=flavor_path, + rebuild=rebuild, + build_docker_repository=build_config.docker_build_repository, + commit_sha=commit_sha, + docker_user=docker_user, + docker_password=docker_password, + test_container_folder=test_container_folder, + ) + ci_security_scan.run_security_scan(flavor_path=flavor_path) + return ci_export.export(flavor_path=flavor_path) diff --git a/exasol/slc_ci/lib/git_access.py b/exasol/slc_ci/lib/git_access.py new file mode 100644 index 0000000..6b58d62 --- /dev/null +++ b/exasol/slc_ci/lib/git_access.py @@ -0,0 +1,43 @@ +from typing import Iterable + +from git import Repo + + +class GitAccess: + + def get_last_commit_message(self): + """ + Assumes that PWD belongs to a GIT repository. Get's the last commit message of this repo and returns it as string + :return: Last commit message of current working directory GIT repository. + """ + return Repo().head.commit.message + + def get_head_commit_sha_of_branch(self, branch_name) -> str: + """ + Returns the last commit sha of given branch. + :raise: ValueError: if the refs with label 'branch_name' does not exists or is not unique. + """ + repo = Repo() + branch = [b for b in repo.refs if b.name == branch_name] # type: ignore + if len(branch) == 0: + ex_msg = f"Branch '{branch_name}' does not exist." + raise ValueError(ex_msg) + elif len(branch) > 1: + ex_msg = f"Branch '{branch_name}' found more than once." + raise ValueError(ex_msg) + return str(branch[0].commit) + + def get_last_commits(self) -> Iterable[str]: + """ + Returns all commit-sha's of the current branch of the repo in the cwd. + """ + repo = Repo() + for c in repo.iter_commits(repo.head): + yield str(c) + + def get_files_of_commit(self, commit_sha) -> Iterable[str]: + """ + Returns the files of the specific commits of the repo in the cwd. + """ + repo = Repo() + return repo.commit(commit_sha).stats.files.keys() # type: ignore diff --git a/exasol/slc_ci/lib/run_tests.py b/exasol/slc_ci/lib/run_tests.py new file mode 100644 index 0000000..d019ff5 --- /dev/null +++ b/exasol/slc_ci/lib/run_tests.py @@ -0,0 +1,32 @@ +import logging +from pathlib import Path +from typing import List + +from exasol.slc_ci.lib.ci_prepare import CIPrepare +from exasol.slc_ci.lib.ci_test import CIExecuteTest + + +def run_tests( + flavor: str, + slc_path: Path, + test_folders: List[str], + docker_user: str, + docker_password: str, + ci_prepare: CIPrepare = CIPrepare(), + ci_test: CIExecuteTest = CIExecuteTest(), +) -> None: + logging.info(f"Run tests for parameters: {locals()}") + + flavor_path = (f"flavors/{flavor}",) + test_container_folder = "test_container" + + ci_prepare.prepare() + for test_folder in test_folders: + ci_test.execute_tests( + flavor_path=flavor_path, + slc_path=slc_path, + test_folder=test_folder, + test_container_folder=test_container_folder, + docker_user=docker_user, + docker_password=docker_password, + ) diff --git a/exasol/slc_ci/model/build_config.py b/exasol/slc_ci/model/build_config.py new file mode 100644 index 0000000..d485a72 --- /dev/null +++ b/exasol/slc_ci/model/build_config.py @@ -0,0 +1,19 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import List + + +@dataclass(frozen=True) +class BuildConfig: + root: Path + + @property + def flavors_path(self): + return self.root / "flavors" + + base_branch: str + ignore_paths: List[str] + + docker_build_repository: str + docker_release_repository: str + test_container_folder: str diff --git a/exasol/slc_ci/model/flavor_ci_model.py b/exasol/slc_ci/model/flavor_ci_model.py new file mode 100644 index 0000000..de35cad --- /dev/null +++ b/exasol/slc_ci/model/flavor_ci_model.py @@ -0,0 +1,18 @@ +from typing import List + +from pydantic import BaseModel + + +class TestSet(BaseModel): + name: str + folders: List[str] + + +class TestConfig(BaseModel): + test_runner: str + test_sets: List[TestSet] + + +class FlavorCiConfig(BaseModel): + build_runner: str + test_config: TestConfig diff --git a/exasol/slc_ci/nox/tasks.py b/exasol/slc_ci/nox/tasks.py new file mode 100644 index 0000000..7f93ea6 --- /dev/null +++ b/exasol/slc_ci/nox/tasks.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +__all__ = [ + "run_find_available_flavors", + "run_get_build_runner_for_flavor", + "run_get_test_runner_for_flavor", + "run_get_test_set_names_for_flavor", + "run_db_tests", + "run_check_if_build_needed", +] + +import argparse +import json +from argparse import ArgumentParser +from pathlib import Path + +import nox +from nox import Session +from slc_build_config import SLC_BUILD_CONFIG + +import exasol.slc_ci.api as api + + +def _parse_flavor(session: Session) -> str: + def parser() -> ArgumentParser: + p = ArgumentParser( + usage=f"nox -s {session.name} -- --flavor ", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + p.add_argument("--flavor") + return p + + args = parser().parse_args(session.posargs) + return args.flavor + + +@nox.session(name="ci:find-available-flavors", python=False) +def run_find_available_flavors(session: Session) -> None: + flavor_list = list(api.get_flavors(SLC_BUILD_CONFIG.flavors_path)) + print(json.dumps(flavor_list, indent=2)) + + +@nox.session(name="ci:get-build-runner-for-flavor", python=False) +def run_get_build_runner_for_flavor(session: nox.Session): + """ + Returns the runner for a flavor + """ + flavor = _parse_flavor(session) + flavor_config = api.get_flavor_ci_model(SLC_BUILD_CONFIG, flavor) + print(flavor_config.build_runner) + + +@nox.session(name="ci:get-test-runner-for-flavor", python=False) +def run_get_test_runner_for_flavor(session: nox.Session): + """ + Returns the test-runner for a flavor + """ + flavor = _parse_flavor(session) + flavor_config = api.get_flavor_ci_model(SLC_BUILD_CONFIG, flavor) + print(flavor_config.test_config.test_runner) + + +@nox.session(name="ci:get-test-set-names-for-flavor", python=False) +def run_get_test_set_names_for_flavor(session: nox.Session): + """ + Returns the test-set names for a flavor as JSON list + """ + flavor = _parse_flavor(session) + flavor_config = api.get_flavor_ci_model(SLC_BUILD_CONFIG, flavor) + test_sets_names = [test_set.name for test_set in flavor_config.test_sets] + print(json.dumps(test_sets_names)) + + +@nox.session(name="ci:run-db-tests", python=False) +def run_db_tests(session: nox.Session): + """ + Runs integration tests + """ + + def parser() -> ArgumentParser: + p = ArgumentParser( + usage="nox -s run-db-tests -- --flavor --test-set ", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + p.add_argument("--flavor") + p.add_argument("--test-set-name") + p.add_argument("--slc-directory") + return p + + args = parser().parse_args(session.posargs) + flavor_config = api.get_flavor_ci_model(SLC_BUILD_CONFIG, args.flavor) + matched_test_set = [ + test_set + for test_set in flavor_config.test_config.test_sets + if test_set.name == args.test_set_name + ] + if len(matched_test_set) != 1: + raise ValueError(f"Invalid test set name: {args.test_set_name}") + test_set_folders = [folder for folder in matched_test_set[0].test_folders] + slc_directory = Path(args.slc_directory) + if not slc_directory.exists(): + raise ValueError(f"{args.slc_directory} does not exist") + slc_files = list(slc_directory.glob(f"{args.flavor}*.tar.gz")) + if len(slc_files) != 1: + raise ValueError( + f"{args.flavor} does not contain expected tar.gz file, but \n {slc_files}" + ) + + api.run_tests( + flavor=args.flavor, + slc_path=args.slc_directory, + test_folders=test_set_folders, + docker_user=args.docker_user, + docker_password=args.docker_password, + ) + + +@nox.session(name="ci:check-if-build-need", python=False) +def run_check_if_build_needed(session: nox.Session): + """ + Returns "True" if rebuild and test is needed, "False" otherwise + """ + + def parser() -> ArgumentParser: + p = ArgumentParser( + usage="nox -s check-if-build-need -- --flavor --branch-name ", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + p.add_argument("--flavor") + p.add_argument("--branch-name") + return p + + args = parser().parse_args(session.posargs) + need_to_build = api.check_if_build_needed( + branch_name=args.branch_name, flavor=args.flavor, build_config=SLC_BUILD_CONFIG + ) + print("True") if need_to_build else print("False") + + +@nox.session(name="ci:check-if-build-need", python=False) +def run_export_and_scan_vulnerabilities(session: nox.Session): + """ + Exports the SLC and runs the vulnerabilities checks. Returns the path to the cached SLC. + """ + + def parser() -> ArgumentParser: + p = ArgumentParser( + usage="nox -s check-if-build-need -- --flavor --branch-name " + "--docker-user --docker-password " + "--docker-build-repository --commit-sha ", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + p.add_argument("--flavor") + p.add_argument("--branch-name") + p.add_argument("--docker-user") + p.add_argument("--docker-password") + p.add_argument("--docker-build-repository") + p.add_argument("--commit-sha") + return p + + args = parser().parse_args(session.posargs) + slc_cache_file = api.export_and_scan_vulnerabilities( + flavor=args.flavor, + build_config=SLC_BUILD_CONFIG, + branch_name=args.branch_name, + docker_user=args.docker_user, + docker_password=args.docker_password, + docker_build_repository=args.docker_build_repository, + commit_sha=args.commit_sha, + ) + print(slc_cache_file) diff --git a/exasol_script_languages_container_ci/version.py b/exasol/slc_ci/version.py similarity index 100% rename from exasol_script_languages_container_ci/version.py rename to exasol/slc_ci/version.py diff --git a/noxconfig.py b/noxconfig.py index 44385cc..133b83f 100644 --- a/noxconfig.py +++ b/noxconfig.py @@ -11,9 +11,7 @@ class Config: root: Path = Path(__file__).parent doc: Path = Path(__file__).parent / "doc" - version_file: Path = ( - Path(__file__).parent / "exasol_script_languages_container_ci" / "version.py" - ) + version_file: Path = Path(__file__).parent / "exasol" / "slc_ci" / "version.py" path_filters: Iterable[str] = ( "dist", ".eggs", diff --git a/pyproject.toml b/pyproject.toml index 2deff0a..d6cae74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ packages = [ {include = "README.md"}, {include = "LICENSE"}, {include = "exasol_script_languages_container_ci"}, + {include = "exasol"}, ] [tool.poetry.dependencies] From 1ae987b90e3e1912448e6b513f295b63d8548619 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 23 Apr 2025 14:16:15 -0300 Subject: [PATCH 02/19] Fixed parser in nox tasks --- exasol/slc_ci/nox/tasks.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/exasol/slc_ci/nox/tasks.py b/exasol/slc_ci/nox/tasks.py index 7f93ea6..7b256fb 100644 --- a/exasol/slc_ci/nox/tasks.py +++ b/exasol/slc_ci/nox/tasks.py @@ -27,7 +27,7 @@ def parser() -> ArgumentParser: usage=f"nox -s {session.name} -- --flavor ", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - p.add_argument("--flavor") + p.add_argument("--flavor", required=True) return p args = parser().parse_args(session.posargs) @@ -82,9 +82,9 @@ def parser() -> ArgumentParser: usage="nox -s run-db-tests -- --flavor --test-set ", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - p.add_argument("--flavor") - p.add_argument("--test-set-name") - p.add_argument("--slc-directory") + p.add_argument("--flavor", required=True) + p.add_argument("--test-set-name", required=True) + p.add_argument("--slc-directory", required=True) return p args = parser().parse_args(session.posargs) @@ -126,8 +126,8 @@ def parser() -> ArgumentParser: usage="nox -s check-if-build-need -- --flavor --branch-name ", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - p.add_argument("--flavor") - p.add_argument("--branch-name") + p.add_argument("--flavor", required=True) + p.add_argument("--branch-name", required=True) return p args = parser().parse_args(session.posargs) @@ -147,15 +147,14 @@ def parser() -> ArgumentParser: p = ArgumentParser( usage="nox -s check-if-build-need -- --flavor --branch-name " "--docker-user --docker-password " - "--docker-build-repository --commit-sha ", + "--commit-sha ", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - p.add_argument("--flavor") - p.add_argument("--branch-name") - p.add_argument("--docker-user") - p.add_argument("--docker-password") - p.add_argument("--docker-build-repository") - p.add_argument("--commit-sha") + p.add_argument("--flavor", required=True) + p.add_argument("--branch-name", required=True) + p.add_argument("--docker-user", required=True) + p.add_argument("--docker-password", required=True) + p.add_argument("--commit-sha", required=True) return p args = parser().parse_args(session.posargs) @@ -165,7 +164,6 @@ def parser() -> ArgumentParser: branch_name=args.branch_name, docker_user=args.docker_user, docker_password=args.docker_password, - docker_build_repository=args.docker_build_repository, commit_sha=args.commit_sha, ) print(slc_cache_file) From bf7bef6cdf8260ae996f0f427d28470cf5b6c17d Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 23 Apr 2025 15:02:52 -0300 Subject: [PATCH 03/19] 1. Removed building of test container 2. fixed nox task name --- exasol/slc_ci/lib/ci_build.py | 19 ------------------- exasol/slc_ci/nox/tasks.py | 2 +- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/exasol/slc_ci/lib/ci_build.py b/exasol/slc_ci/lib/ci_build.py index 74f2fa3..82ea9be 100644 --- a/exasol/slc_ci/lib/ci_build.py +++ b/exasol/slc_ci/lib/ci_build.py @@ -2,12 +2,6 @@ from typing import Optional, Tuple from exasol.slc.api import build -from exasol.slc.internal.tasks.test.test_container_content import ( - build_test_container_content, -) -from exasol_integration_test_docker_environment.lib.api.build_test_container import ( - build_test_container, -) from exasol.slc_ci.lib.ci_step_output_printer import ( CIStepOutputPrinter, @@ -62,17 +56,4 @@ def build( log_level="WARNING", use_job_specific_log_file=True, ) - logging.info( - f"Running command 'build_test_container' with parameters: {locals()}" - ) - content = build_test_container_content(test_container_folder) - test_container_image_infos = build_test_container( - force_rebuild=rebuild, - workers=7, - test_container_content=content, - source_docker_repository_name=build_docker_repository, - source_docker_tag_prefix=commit_sha, - log_level="WARNING", - use_job_specific_log_file=True, - ) self._printer.print_exasol_docker_images() diff --git a/exasol/slc_ci/nox/tasks.py b/exasol/slc_ci/nox/tasks.py index 7b256fb..6a6d940 100644 --- a/exasol/slc_ci/nox/tasks.py +++ b/exasol/slc_ci/nox/tasks.py @@ -137,7 +137,7 @@ def parser() -> ArgumentParser: print("True") if need_to_build else print("False") -@nox.session(name="ci:check-if-build-need", python=False) +@nox.session(name="ci:export-and-scan-vulnerabilities", python=False) def run_export_and_scan_vulnerabilities(session: nox.Session): """ Exports the SLC and runs the vulnerabilities checks. Returns the path to the cached SLC. From 0cbd5a427e29a7e5ed5f6050672f27f1b581b544 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 24 Apr 2025 09:58:36 -0300 Subject: [PATCH 04/19] Removed indention for JSON --- exasol/slc_ci/nox/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/slc_ci/nox/tasks.py b/exasol/slc_ci/nox/tasks.py index 6a6d940..03ba22a 100644 --- a/exasol/slc_ci/nox/tasks.py +++ b/exasol/slc_ci/nox/tasks.py @@ -37,7 +37,7 @@ def parser() -> ArgumentParser: @nox.session(name="ci:find-available-flavors", python=False) def run_find_available_flavors(session: Session) -> None: flavor_list = list(api.get_flavors(SLC_BUILD_CONFIG.flavors_path)) - print(json.dumps(flavor_list, indent=2)) + print(json.dumps(flavor_list)) @nox.session(name="ci:get-build-runner-for-flavor", python=False) From 453404ff866cecbd3703d280e92b9c6b2a01c7e9 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 24 Apr 2025 12:25:33 -0300 Subject: [PATCH 05/19] Fixed printing of docker images --- exasol/slc_ci/lib/ci_step_output_printer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/exasol/slc_ci/lib/ci_step_output_printer.py b/exasol/slc_ci/lib/ci_step_output_printer.py index b660a0f..74d8be1 100644 --- a/exasol/slc_ci/lib/ci_step_output_printer.py +++ b/exasol/slc_ci/lib/ci_step_output_printer.py @@ -23,7 +23,6 @@ def _get_exasol_docker_images(): return exa_images finally: docker_client.close() - return [] class CIStepOutputPrinter(CIStepOutputPrinterProtocol): From 624de152fa79d1c2719caad53b4ce9bddf031057 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 24 Apr 2025 12:48:49 -0300 Subject: [PATCH 06/19] Push build images and fixed nox task --- .../api/export_and_scan_vulnerabilities.py | 3 +++ exasol/slc_ci/api/get_flavor_ci_model.py | 2 +- .../lib/export_and_scan_vulnerabilities.py | 16 ++++++++++++++++ exasol/slc_ci/nox/tasks.py | 2 +- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/exasol/slc_ci/api/export_and_scan_vulnerabilities.py b/exasol/slc_ci/api/export_and_scan_vulnerabilities.py index 3e1fd1a..2c9e9a5 100644 --- a/exasol/slc_ci/api/export_and_scan_vulnerabilities.py +++ b/exasol/slc_ci/api/export_and_scan_vulnerabilities.py @@ -3,6 +3,7 @@ from exasol.slc_ci.lib.ci_build import CIBuild from exasol.slc_ci.lib.ci_export import CIExport from exasol.slc_ci.lib.ci_prepare import CIPrepare +from exasol.slc_ci.lib.ci_push import CIPush from exasol.slc_ci.lib.ci_security_scan import CISecurityScan from exasol.slc_ci.lib.export_and_scan_vulnerabilities import ( export_and_scan_vulnerabilities as lib_export_and_scan_vulnerabilities, @@ -24,6 +25,7 @@ def export_and_scan_vulnerabilities( ci_security_scan: CISecurityScan = CISecurityScan() ci_prepare: CIPrepare = CIPrepare() ci_export: CIExport = CIExport() + ci_push: CIPush = CIPush() return lib_export_and_scan_vulnerabilities( flavor=flavor, @@ -37,4 +39,5 @@ def export_and_scan_vulnerabilities( ci_security_scan=ci_security_scan, ci_prepare=ci_prepare, ci_export=ci_export, + ci_push=ci_push, ) diff --git a/exasol/slc_ci/api/get_flavor_ci_model.py b/exasol/slc_ci/api/get_flavor_ci_model.py index 000757a..5951b29 100644 --- a/exasol/slc_ci/api/get_flavor_ci_model.py +++ b/exasol/slc_ci/api/get_flavor_ci_model.py @@ -4,4 +4,4 @@ def get_flavor_ci_model(build_config: BuildConfig, flavor: str) -> FlavorCiConfig: flavor_ci_config_path = build_config.flavors_path / flavor / "ci.json" - return FlavorCiConfig.model_validate_json(flavor_ci_config_path.read_text()) + return FlavorCiConfig.model_validate_json(flavor_ci_config_path.read_text(), strict=True) diff --git a/exasol/slc_ci/lib/export_and_scan_vulnerabilities.py b/exasol/slc_ci/lib/export_and_scan_vulnerabilities.py index 508937f..95103c9 100644 --- a/exasol/slc_ci/lib/export_and_scan_vulnerabilities.py +++ b/exasol/slc_ci/lib/export_and_scan_vulnerabilities.py @@ -5,6 +5,7 @@ from exasol.slc_ci.lib.ci_build import CIBuild from exasol.slc_ci.lib.ci_export import CIExport from exasol.slc_ci.lib.ci_prepare import CIPrepare +from exasol.slc_ci.lib.ci_push import CIPush from exasol.slc_ci.lib.ci_security_scan import CISecurityScan from exasol.slc_ci.lib.git_access import GitAccess from exasol.slc_ci.model.build_config import BuildConfig @@ -22,6 +23,7 @@ def export_and_scan_vulnerabilities( ci_security_scan: CISecurityScan = CISecurityScan(), ci_prepare: CIPrepare = CIPrepare(), ci_export: CIExport = CIExport(), + ci_push: CIPush = CIPush(), ) -> Path: logging.info( f"Running build image and scan vulnerabilities for parameters: {locals()}" @@ -41,4 +43,18 @@ def export_and_scan_vulnerabilities( test_container_folder=test_container_folder, ) ci_security_scan.run_security_scan(flavor_path=flavor_path) + ci_push.push( + flavor_path=flavor_path, + target_docker_repository=build_config.docker_build_repository, + target_docker_tag_prefix=commit_sha, + docker_user=docker_user, + docker_password=docker_password, + ) + ci_push.push( + flavor_path=flavor_path, + target_docker_repository=build_config.docker_build_repository, + target_docker_tag_prefix="", + docker_user=docker_user, + docker_password=docker_password, + ) return ci_export.export(flavor_path=flavor_path) diff --git a/exasol/slc_ci/nox/tasks.py b/exasol/slc_ci/nox/tasks.py index 03ba22a..963b6f7 100644 --- a/exasol/slc_ci/nox/tasks.py +++ b/exasol/slc_ci/nox/tasks.py @@ -67,7 +67,7 @@ def run_get_test_set_names_for_flavor(session: nox.Session): """ flavor = _parse_flavor(session) flavor_config = api.get_flavor_ci_model(SLC_BUILD_CONFIG, flavor) - test_sets_names = [test_set.name for test_set in flavor_config.test_sets] + test_sets_names = [test_set.name for test_set in flavor_config.test_config.test_sets] print(json.dumps(test_sets_names)) From 50af9655a33f3356c80018467cd6be35840bea79 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 24 Apr 2025 14:07:16 -0300 Subject: [PATCH 07/19] Let nox task print to github output file --- exasol/slc_ci/nox/tasks.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/exasol/slc_ci/nox/tasks.py b/exasol/slc_ci/nox/tasks.py index 963b6f7..b7ea8c6 100644 --- a/exasol/slc_ci/nox/tasks.py +++ b/exasol/slc_ci/nox/tasks.py @@ -147,7 +147,7 @@ def parser() -> ArgumentParser: p = ArgumentParser( usage="nox -s check-if-build-need -- --flavor --branch-name " "--docker-user --docker-password " - "--commit-sha ", + "--commit-sha --github-output $GITHUB_OUTPUT", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p.add_argument("--flavor", required=True) @@ -155,6 +155,7 @@ def parser() -> ArgumentParser: p.add_argument("--docker-user", required=True) p.add_argument("--docker-password", required=True) p.add_argument("--commit-sha", required=True) + p.add_argument("--github-output", required=True) return p args = parser().parse_args(session.posargs) @@ -166,4 +167,6 @@ def parser() -> ArgumentParser: docker_password=args.docker_password, commit_sha=args.commit_sha, ) - print(slc_cache_file) + with open(args.github_output, "a") as f: + print(f"slc_path={slc_cache_file}", file=f) + From a3f000322a57643483c839b217478cf75a15bdb1 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 24 Apr 2025 14:24:15 -0300 Subject: [PATCH 08/19] Fixed `run_db_tests` nox task --- exasol/slc_ci/nox/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exasol/slc_ci/nox/tasks.py b/exasol/slc_ci/nox/tasks.py index b7ea8c6..7d29abf 100644 --- a/exasol/slc_ci/nox/tasks.py +++ b/exasol/slc_ci/nox/tasks.py @@ -96,7 +96,7 @@ def parser() -> ArgumentParser: ] if len(matched_test_set) != 1: raise ValueError(f"Invalid test set name: {args.test_set_name}") - test_set_folders = [folder for folder in matched_test_set[0].test_folders] + test_set_folders = [folder for folder in matched_test_set[0].folders] slc_directory = Path(args.slc_directory) if not slc_directory.exists(): raise ValueError(f"{args.slc_directory} does not exist") @@ -108,7 +108,7 @@ def parser() -> ArgumentParser: api.run_tests( flavor=args.flavor, - slc_path=args.slc_directory, + slc_path=slc_files[0], test_folders=test_set_folders, docker_user=args.docker_user, docker_password=args.docker_password, From 725e2fc4002610f2f22d2247dd25a7f1c765a8e1 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 24 Apr 2025 18:23:00 -0300 Subject: [PATCH 09/19] Extracted parser for nox tasks in separate class --- exasol/slc_ci/api/get_flavor_ci_model.py | 4 +- exasol/slc_ci/api/run_tests.py | 7 +- exasol/slc_ci/lib/run_tests.py | 22 +++- exasol/slc_ci/nox/arg_parser_builder.py | 50 +++++++++ exasol/slc_ci/nox/tasks.py | 130 ++++++++--------------- 5 files changed, 123 insertions(+), 90 deletions(-) create mode 100644 exasol/slc_ci/nox/arg_parser_builder.py diff --git a/exasol/slc_ci/api/get_flavor_ci_model.py b/exasol/slc_ci/api/get_flavor_ci_model.py index 5951b29..f7b6413 100644 --- a/exasol/slc_ci/api/get_flavor_ci_model.py +++ b/exasol/slc_ci/api/get_flavor_ci_model.py @@ -4,4 +4,6 @@ def get_flavor_ci_model(build_config: BuildConfig, flavor: str) -> FlavorCiConfig: flavor_ci_config_path = build_config.flavors_path / flavor / "ci.json" - return FlavorCiConfig.model_validate_json(flavor_ci_config_path.read_text(), strict=True) + return FlavorCiConfig.model_validate_json( + flavor_ci_config_path.read_text(), strict=True + ) diff --git a/exasol/slc_ci/api/run_tests.py b/exasol/slc_ci/api/run_tests.py index a01095a..1c6238a 100644 --- a/exasol/slc_ci/api/run_tests.py +++ b/exasol/slc_ci/api/run_tests.py @@ -4,12 +4,14 @@ from exasol.slc_ci.lib.ci_prepare import CIPrepare from exasol.slc_ci.lib.ci_test import CIExecuteTest from exasol.slc_ci.lib.run_tests import run_tests as lib_run_tests +from exasol.slc_ci.model.flavor_ci_model import FlavorCiConfig def run_tests( flavor: str, slc_path: Path, - test_folders: List[str], + flavor_config: FlavorCiConfig, + test_set: str, docker_user: str, docker_password: str, ) -> None: @@ -19,7 +21,8 @@ def run_tests( return lib_run_tests( flavor=flavor, slc_path=slc_path, - test_folders=test_folders, + flavor_config=flavor_config, + test_set=test_set, docker_user=docker_user, docker_password=docker_password, ci_prepare=ci_prepare, diff --git a/exasol/slc_ci/lib/run_tests.py b/exasol/slc_ci/lib/run_tests.py index d019ff5..1eca6bc 100644 --- a/exasol/slc_ci/lib/run_tests.py +++ b/exasol/slc_ci/lib/run_tests.py @@ -1,27 +1,43 @@ import logging from pathlib import Path -from typing import List from exasol.slc_ci.lib.ci_prepare import CIPrepare from exasol.slc_ci.lib.ci_test import CIExecuteTest +from exasol.slc_ci.model.flavor_ci_model import FlavorCiConfig def run_tests( flavor: str, slc_path: Path, - test_folders: List[str], + flavor_config: FlavorCiConfig, + test_set: str, docker_user: str, docker_password: str, ci_prepare: CIPrepare = CIPrepare(), ci_test: CIExecuteTest = CIExecuteTest(), ) -> None: logging.info(f"Run tests for parameters: {locals()}") + matched_test_set = [ + test_set + for test_set in flavor_config.test_config.test_sets + if test_set.name == test_set + ] + if len(matched_test_set) != 1: + raise ValueError(f"Invalid test set name: {test_set}") + test_set_folders = [folder for folder in matched_test_set[0].folders] + if not slc_path.exists(): + raise ValueError(f"{slc_path} does not exist") + slc_files = list(slc_path.glob(f"{flavor}*.tar.gz")) + if len(slc_files) != 1: + raise ValueError( + f"{flavor} does not contain expected tar.gz file, but \n {slc_files}" + ) flavor_path = (f"flavors/{flavor}",) test_container_folder = "test_container" ci_prepare.prepare() - for test_folder in test_folders: + for test_folder in test_set_folders: ci_test.execute_tests( flavor_path=flavor_path, slc_path=slc_path, diff --git a/exasol/slc_ci/nox/arg_parser_builder.py b/exasol/slc_ci/nox/arg_parser_builder.py new file mode 100644 index 0000000..196fc1a --- /dev/null +++ b/exasol/slc_ci/nox/arg_parser_builder.py @@ -0,0 +1,50 @@ +import argparse + +import nox + + +class ArgumentParserBuilder: + def __init__(self, session: nox.Session) -> None: + self._session = session + self._usages = list() + self._parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + def _add_arg(self, name: str): + self._parser.add_argument(f"--{name}", required=True) + self._usages.append(f"--{name} <{name}>") + + def with_flavor(self) -> "ArgumentParserBuilder": + self._add_arg("flavor") + return self + + def with_docker(self) -> "ArgumentParserBuilder": + self._add_arg("docker-user") + self._add_arg("docker-password") + return self + + def with_testset(self) -> "ArgumentParserBuilder": + self._add_arg("test-set-name") + return self + + def with_slc_directory(self) -> "ArgumentParserBuilder": + self._add_arg("slc-directory") + return self + + def with_github_output(self) -> "ArgumentParserBuilder": + self._add_arg("github-output") + self._add_arg("github-var") + return self + + def with_branch_name(self) -> "ArgumentParserBuilder": + self._add_arg("branch-name") + return self + + def with_commit_sha(self) -> "ArgumentParserBuilder": + self._add_arg("commit-sha") + return self + + def parse(self) -> argparse.Namespace: + self._parser.usage = f"nox -s {self._session.name} {' '.join(self._usages)}" + return self._parser.parse_args(self._session.posargs) diff --git a/exasol/slc_ci/nox/tasks.py b/exasol/slc_ci/nox/tasks.py index 7d29abf..7437704 100644 --- a/exasol/slc_ci/nox/tasks.py +++ b/exasol/slc_ci/nox/tasks.py @@ -7,31 +7,23 @@ "run_get_test_set_names_for_flavor", "run_db_tests", "run_check_if_build_needed", + "run_export_and_scan_vulnerabilities", ] import argparse import json -from argparse import ArgumentParser -from pathlib import Path import nox from nox import Session from slc_build_config import SLC_BUILD_CONFIG import exasol.slc_ci.api as api +from exasol.slc_ci.nox.arg_parser_builder import ArgumentParserBuilder -def _parse_flavor(session: Session) -> str: - def parser() -> ArgumentParser: - p = ArgumentParser( - usage=f"nox -s {session.name} -- --flavor ", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p.add_argument("--flavor", required=True) - return p - - args = parser().parse_args(session.posargs) - return args.flavor +def _write_github_output(args: argparse.Namespace, value: str) -> None: + with open(args.github_output, "a") as github_output: + print(f"{args.github_var}={value}", file=github_output) @nox.session(name="ci:find-available-flavors", python=False) @@ -45,9 +37,9 @@ def run_get_build_runner_for_flavor(session: nox.Session): """ Returns the runner for a flavor """ - flavor = _parse_flavor(session) - flavor_config = api.get_flavor_ci_model(SLC_BUILD_CONFIG, flavor) - print(flavor_config.build_runner) + args = ArgumentParserBuilder(session).with_flavor().with_github_output().parse() + flavor_config = api.get_flavor_ci_model(SLC_BUILD_CONFIG, args.flavor) + _write_github_output(args, flavor_config.build_runner) @nox.session(name="ci:get-test-runner-for-flavor", python=False) @@ -55,9 +47,9 @@ def run_get_test_runner_for_flavor(session: nox.Session): """ Returns the test-runner for a flavor """ - flavor = _parse_flavor(session) - flavor_config = api.get_flavor_ci_model(SLC_BUILD_CONFIG, flavor) - print(flavor_config.test_config.test_runner) + args = ArgumentParserBuilder(session).with_flavor().with_github_output().parse() + flavor_config = api.get_flavor_ci_model(SLC_BUILD_CONFIG, args.flavor) + _write_github_output(args, flavor_config.test_config.test_runner) @nox.session(name="ci:get-test-set-names-for-flavor", python=False) @@ -65,10 +57,12 @@ def run_get_test_set_names_for_flavor(session: nox.Session): """ Returns the test-set names for a flavor as JSON list """ - flavor = _parse_flavor(session) - flavor_config = api.get_flavor_ci_model(SLC_BUILD_CONFIG, flavor) - test_sets_names = [test_set.name for test_set in flavor_config.test_config.test_sets] - print(json.dumps(test_sets_names)) + args = ArgumentParserBuilder(session).with_flavor().with_github_output().parse() + flavor_config = api.get_flavor_ci_model(SLC_BUILD_CONFIG, args.flavor) + test_sets_names = [ + test_set.name for test_set in flavor_config.test_config.test_sets + ] + _write_github_output(args, json.dumps(test_sets_names)) @nox.session(name="ci:run-db-tests", python=False) @@ -76,40 +70,21 @@ def run_db_tests(session: nox.Session): """ Runs integration tests """ - - def parser() -> ArgumentParser: - p = ArgumentParser( - usage="nox -s run-db-tests -- --flavor --test-set ", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p.add_argument("--flavor", required=True) - p.add_argument("--test-set-name", required=True) - p.add_argument("--slc-directory", required=True) - return p - - args = parser().parse_args(session.posargs) + args = ( + ArgumentParserBuilder(session) + .with_flavor() + .with_docker() + .with_testset() + .with_slc_directory() + .parse() + ) flavor_config = api.get_flavor_ci_model(SLC_BUILD_CONFIG, args.flavor) - matched_test_set = [ - test_set - for test_set in flavor_config.test_config.test_sets - if test_set.name == args.test_set_name - ] - if len(matched_test_set) != 1: - raise ValueError(f"Invalid test set name: {args.test_set_name}") - test_set_folders = [folder for folder in matched_test_set[0].folders] - slc_directory = Path(args.slc_directory) - if not slc_directory.exists(): - raise ValueError(f"{args.slc_directory} does not exist") - slc_files = list(slc_directory.glob(f"{args.flavor}*.tar.gz")) - if len(slc_files) != 1: - raise ValueError( - f"{args.flavor} does not contain expected tar.gz file, but \n {slc_files}" - ) api.run_tests( flavor=args.flavor, - slc_path=slc_files[0], - test_folders=test_set_folders, + slc_path=args.slc_directory, + flavor_config=flavor_config, + test_set=args.test_set_name, docker_user=args.docker_user, docker_password=args.docker_password, ) @@ -120,21 +95,17 @@ def run_check_if_build_needed(session: nox.Session): """ Returns "True" if rebuild and test is needed, "False" otherwise """ - - def parser() -> ArgumentParser: - p = ArgumentParser( - usage="nox -s check-if-build-need -- --flavor --branch-name ", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p.add_argument("--flavor", required=True) - p.add_argument("--branch-name", required=True) - return p - - args = parser().parse_args(session.posargs) + args = ( + ArgumentParserBuilder(session) + .with_flavor() + .with_branch_name() + .with_github_output() + .parse() + ) need_to_build = api.check_if_build_needed( branch_name=args.branch_name, flavor=args.flavor, build_config=SLC_BUILD_CONFIG ) - print("True") if need_to_build else print("False") + _write_github_output(args, "True" if need_to_build else "False") @nox.session(name="ci:export-and-scan-vulnerabilities", python=False) @@ -142,23 +113,16 @@ def run_export_and_scan_vulnerabilities(session: nox.Session): """ Exports the SLC and runs the vulnerabilities checks. Returns the path to the cached SLC. """ + args = ( + ArgumentParserBuilder(session) + .with_flavor() + .with_branch_name() + .with_docker() + .with_github_output() + .with_commit_sha() + .parse() + ) - def parser() -> ArgumentParser: - p = ArgumentParser( - usage="nox -s check-if-build-need -- --flavor --branch-name " - "--docker-user --docker-password " - "--commit-sha --github-output $GITHUB_OUTPUT", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p.add_argument("--flavor", required=True) - p.add_argument("--branch-name", required=True) - p.add_argument("--docker-user", required=True) - p.add_argument("--docker-password", required=True) - p.add_argument("--commit-sha", required=True) - p.add_argument("--github-output", required=True) - return p - - args = parser().parse_args(session.posargs) slc_cache_file = api.export_and_scan_vulnerabilities( flavor=args.flavor, build_config=SLC_BUILD_CONFIG, @@ -167,6 +131,4 @@ def parser() -> ArgumentParser: docker_password=args.docker_password, commit_sha=args.commit_sha, ) - with open(args.github_output, "a") as f: - print(f"slc_path={slc_cache_file}", file=f) - + _write_github_output(args, str(slc_cache_file)) From d19f2973663ed9968933d791bb41431e4f8a9a54 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 24 Apr 2025 18:26:51 -0300 Subject: [PATCH 10/19] Fixed nox task ci:find-available-flavors --- exasol/slc_ci/nox/arg_parser_builder.py | 2 +- exasol/slc_ci/nox/tasks.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/exasol/slc_ci/nox/arg_parser_builder.py b/exasol/slc_ci/nox/arg_parser_builder.py index 196fc1a..5aa26e3 100644 --- a/exasol/slc_ci/nox/arg_parser_builder.py +++ b/exasol/slc_ci/nox/arg_parser_builder.py @@ -46,5 +46,5 @@ def with_commit_sha(self) -> "ArgumentParserBuilder": return self def parse(self) -> argparse.Namespace: - self._parser.usage = f"nox -s {self._session.name} {' '.join(self._usages)}" + self._parser.usage = f"nox -s {self._session.name} -- {' '.join(self._usages)}" return self._parser.parse_args(self._session.posargs) diff --git a/exasol/slc_ci/nox/tasks.py b/exasol/slc_ci/nox/tasks.py index 7437704..84b280b 100644 --- a/exasol/slc_ci/nox/tasks.py +++ b/exasol/slc_ci/nox/tasks.py @@ -28,8 +28,9 @@ def _write_github_output(args: argparse.Namespace, value: str) -> None: @nox.session(name="ci:find-available-flavors", python=False) def run_find_available_flavors(session: Session) -> None: + args = ArgumentParserBuilder(session).with_github_output().parse() flavor_list = list(api.get_flavors(SLC_BUILD_CONFIG.flavors_path)) - print(json.dumps(flavor_list)) + _write_github_output(args, json.dumps(flavor_list)) @nox.session(name="ci:get-build-runner-for-flavor", python=False) From b1450a8d279557dfce9a6ea442dfc66cd9acf1cf Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 25 Apr 2025 06:46:29 -0300 Subject: [PATCH 11/19] Use $GITHUB_OUTPUT from env variable --- exasol/slc_ci/nox/arg_parser_builder.py | 3 +-- exasol/slc_ci/nox/tasks.py | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/exasol/slc_ci/nox/arg_parser_builder.py b/exasol/slc_ci/nox/arg_parser_builder.py index 5aa26e3..5c5d9c0 100644 --- a/exasol/slc_ci/nox/arg_parser_builder.py +++ b/exasol/slc_ci/nox/arg_parser_builder.py @@ -32,8 +32,7 @@ def with_slc_directory(self) -> "ArgumentParserBuilder": self._add_arg("slc-directory") return self - def with_github_output(self) -> "ArgumentParserBuilder": - self._add_arg("github-output") + def with_github_var(self) -> "ArgumentParserBuilder": self._add_arg("github-var") return self diff --git a/exasol/slc_ci/nox/tasks.py b/exasol/slc_ci/nox/tasks.py index 84b280b..d48a301 100644 --- a/exasol/slc_ci/nox/tasks.py +++ b/exasol/slc_ci/nox/tasks.py @@ -12,6 +12,7 @@ import argparse import json +import os import nox from nox import Session @@ -22,13 +23,14 @@ def _write_github_output(args: argparse.Namespace, value: str) -> None: - with open(args.github_output, "a") as github_output: - print(f"{args.github_var}={value}", file=github_output) + github_output = os.environ["GITHUB_OUTPUT"] + with open(github_output, "a") as github_output_file: + print(f"{args.github_var}={value}", file=github_output_file) @nox.session(name="ci:find-available-flavors", python=False) def run_find_available_flavors(session: Session) -> None: - args = ArgumentParserBuilder(session).with_github_output().parse() + args = ArgumentParserBuilder(session).with_github_var().parse() flavor_list = list(api.get_flavors(SLC_BUILD_CONFIG.flavors_path)) _write_github_output(args, json.dumps(flavor_list)) @@ -38,7 +40,7 @@ def run_get_build_runner_for_flavor(session: nox.Session): """ Returns the runner for a flavor """ - args = ArgumentParserBuilder(session).with_flavor().with_github_output().parse() + args = ArgumentParserBuilder(session).with_flavor().with_github_var().parse() flavor_config = api.get_flavor_ci_model(SLC_BUILD_CONFIG, args.flavor) _write_github_output(args, flavor_config.build_runner) @@ -48,7 +50,7 @@ def run_get_test_runner_for_flavor(session: nox.Session): """ Returns the test-runner for a flavor """ - args = ArgumentParserBuilder(session).with_flavor().with_github_output().parse() + args = ArgumentParserBuilder(session).with_flavor().with_github_var().parse() flavor_config = api.get_flavor_ci_model(SLC_BUILD_CONFIG, args.flavor) _write_github_output(args, flavor_config.test_config.test_runner) @@ -58,7 +60,7 @@ def run_get_test_set_names_for_flavor(session: nox.Session): """ Returns the test-set names for a flavor as JSON list """ - args = ArgumentParserBuilder(session).with_flavor().with_github_output().parse() + args = ArgumentParserBuilder(session).with_flavor().with_github_var().parse() flavor_config = api.get_flavor_ci_model(SLC_BUILD_CONFIG, args.flavor) test_sets_names = [ test_set.name for test_set in flavor_config.test_config.test_sets @@ -100,7 +102,7 @@ def run_check_if_build_needed(session: nox.Session): ArgumentParserBuilder(session) .with_flavor() .with_branch_name() - .with_github_output() + .with_github_var() .parse() ) need_to_build = api.check_if_build_needed( @@ -119,7 +121,7 @@ def run_export_and_scan_vulnerabilities(session: nox.Session): .with_flavor() .with_branch_name() .with_docker() - .with_github_output() + .with_github_var() .with_commit_sha() .parse() ) From db676114e28e25265dc0ae1c17fae91ff5104cfd Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 25 Apr 2025 13:07:16 -0300 Subject: [PATCH 12/19] Fixed run-db-test --- exasol/slc_ci/api/run_tests.py | 5 ++--- exasol/slc_ci/lib/run_tests.py | 6 +++--- exasol/slc_ci/nox/arg_parser_builder.py | 2 +- exasol/slc_ci/nox/tasks.py | 4 ++-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/exasol/slc_ci/api/run_tests.py b/exasol/slc_ci/api/run_tests.py index 1c6238a..1aaefca 100644 --- a/exasol/slc_ci/api/run_tests.py +++ b/exasol/slc_ci/api/run_tests.py @@ -1,5 +1,4 @@ from pathlib import Path -from typing import List from exasol.slc_ci.lib.ci_prepare import CIPrepare from exasol.slc_ci.lib.ci_test import CIExecuteTest @@ -11,7 +10,7 @@ def run_tests( flavor: str, slc_path: Path, flavor_config: FlavorCiConfig, - test_set: str, + test_set_name: str, docker_user: str, docker_password: str, ) -> None: @@ -22,7 +21,7 @@ def run_tests( flavor=flavor, slc_path=slc_path, flavor_config=flavor_config, - test_set=test_set, + test_set_name=test_set_name, docker_user=docker_user, docker_password=docker_password, ci_prepare=ci_prepare, diff --git a/exasol/slc_ci/lib/run_tests.py b/exasol/slc_ci/lib/run_tests.py index 1eca6bc..e7bccd4 100644 --- a/exasol/slc_ci/lib/run_tests.py +++ b/exasol/slc_ci/lib/run_tests.py @@ -10,7 +10,7 @@ def run_tests( flavor: str, slc_path: Path, flavor_config: FlavorCiConfig, - test_set: str, + test_set_name: str, docker_user: str, docker_password: str, ci_prepare: CIPrepare = CIPrepare(), @@ -20,10 +20,10 @@ def run_tests( matched_test_set = [ test_set for test_set in flavor_config.test_config.test_sets - if test_set.name == test_set + if test_set.name == test_set_name ] if len(matched_test_set) != 1: - raise ValueError(f"Invalid test set name: {test_set}") + raise ValueError(f"Invalid test set name: {test_set_name}") test_set_folders = [folder for folder in matched_test_set[0].folders] if not slc_path.exists(): raise ValueError(f"{slc_path} does not exist") diff --git a/exasol/slc_ci/nox/arg_parser_builder.py b/exasol/slc_ci/nox/arg_parser_builder.py index 5c5d9c0..69f0b7f 100644 --- a/exasol/slc_ci/nox/arg_parser_builder.py +++ b/exasol/slc_ci/nox/arg_parser_builder.py @@ -24,7 +24,7 @@ def with_docker(self) -> "ArgumentParserBuilder": self._add_arg("docker-password") return self - def with_testset(self) -> "ArgumentParserBuilder": + def with_testset_name(self) -> "ArgumentParserBuilder": self._add_arg("test-set-name") return self diff --git a/exasol/slc_ci/nox/tasks.py b/exasol/slc_ci/nox/tasks.py index d48a301..c92c653 100644 --- a/exasol/slc_ci/nox/tasks.py +++ b/exasol/slc_ci/nox/tasks.py @@ -77,7 +77,7 @@ def run_db_tests(session: nox.Session): ArgumentParserBuilder(session) .with_flavor() .with_docker() - .with_testset() + .with_testset_name() .with_slc_directory() .parse() ) @@ -87,7 +87,7 @@ def run_db_tests(session: nox.Session): flavor=args.flavor, slc_path=args.slc_directory, flavor_config=flavor_config, - test_set=args.test_set_name, + test_set_name=args.test_set_name, docker_user=args.docker_user, docker_password=args.docker_password, ) From 6bb19e67e625a506412692bae779fca22357a34a Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 25 Apr 2025 13:22:02 -0300 Subject: [PATCH 13/19] Fixed run-db-test --- exasol/slc_ci/api/run_tests.py | 4 ++-- exasol/slc_ci/lib/run_tests.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/exasol/slc_ci/api/run_tests.py b/exasol/slc_ci/api/run_tests.py index 1aaefca..f311968 100644 --- a/exasol/slc_ci/api/run_tests.py +++ b/exasol/slc_ci/api/run_tests.py @@ -8,7 +8,7 @@ def run_tests( flavor: str, - slc_path: Path, + slc_directory: str, flavor_config: FlavorCiConfig, test_set_name: str, docker_user: str, @@ -19,7 +19,7 @@ def run_tests( return lib_run_tests( flavor=flavor, - slc_path=slc_path, + slc_directory=slc_directory, flavor_config=flavor_config, test_set_name=test_set_name, docker_user=docker_user, diff --git a/exasol/slc_ci/lib/run_tests.py b/exasol/slc_ci/lib/run_tests.py index e7bccd4..b4ff67d 100644 --- a/exasol/slc_ci/lib/run_tests.py +++ b/exasol/slc_ci/lib/run_tests.py @@ -8,7 +8,7 @@ def run_tests( flavor: str, - slc_path: Path, + slc_directory: str, flavor_config: FlavorCiConfig, test_set_name: str, docker_user: str, @@ -25,6 +25,7 @@ def run_tests( if len(matched_test_set) != 1: raise ValueError(f"Invalid test set name: {test_set_name}") test_set_folders = [folder for folder in matched_test_set[0].folders] + slc_path = Path(slc_directory) if not slc_path.exists(): raise ValueError(f"{slc_path} does not exist") slc_files = list(slc_path.glob(f"{flavor}*.tar.gz")) From f2da1de255c6e02b86ee5b34239c05bf4ce1fe60 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 25 Apr 2025 13:36:39 -0300 Subject: [PATCH 14/19] Fixed run-db-test --- exasol/slc_ci/nox/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/slc_ci/nox/tasks.py b/exasol/slc_ci/nox/tasks.py index c92c653..d5256fd 100644 --- a/exasol/slc_ci/nox/tasks.py +++ b/exasol/slc_ci/nox/tasks.py @@ -85,7 +85,7 @@ def run_db_tests(session: nox.Session): api.run_tests( flavor=args.flavor, - slc_path=args.slc_directory, + slc_directory=args.slc_directory, flavor_config=flavor_config, test_set_name=args.test_set_name, docker_user=args.docker_user, From bc5fb730f35536daff2ef66a191d1c94c8ddb82e Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 28 Apr 2025 14:50:39 -0300 Subject: [PATCH 15/19] Added unit tests --- exasol/slc_ci/lib/run_tests.py | 5 +- test/conftest.py | 59 +---- test/integration/fixtures.py | 20 ++ test/integration/v1/__init__.py | 0 test/integration/{ => v1}/test_ci_build.py | 0 test/integration/{ => v1}/test_ci_export.py | 0 test/integration/{ => v1}/test_ci_prepare.py | 0 test/integration/{ => v1}/test_ci_push.py | 0 .../{ => v1}/test_ci_security_scan.py | 0 test/integration/{ => v1}/test_ci_test.py | 0 test/unit/v1/__init__.py | 0 test/unit/{ => v1}/ci_calls.py | 2 +- test/unit/v1/fixtures.py | 35 +++ test/unit/{ => v1}/test_asset_uploader.py | 0 test/unit/{ => v1}/test_ci.py | 4 +- test/unit/{ => v1}/test_ci_prepare.py | 0 test/unit/{ => v1}/test_ci_test.py | 0 test/unit/{ => v1}/test_config_data_model.py | 0 .../test_config_data_model_generator.py | 0 test/unit/{ => v1}/test_env.py | 0 test/unit/{ => v1}/test_ignore_folders.py | 0 test/unit/{ => v1}/test_release.py | 7 +- test/unit/{ => v1}/test_release_uploader.py | 0 test/unit/v2/__init__.py | 0 test/unit/v2/resources/__init__.py | 0 test/unit/v2/resources/slc_build_config.py | 9 + test/unit/v2/test_ci_prepare.py | 60 +++++ test/unit/v2/test_env.py | 22 ++ .../test_export_and_scan_vulnerabilities.py | 66 +++++ test/unit/v2/test_ignore_folders.py | 157 ++++++++++++ test/unit/v2/test_nox_tasks.py | 240 ++++++++++++++++++ test/unit/v2/test_run_tests.py | 50 ++++ 32 files changed, 671 insertions(+), 65 deletions(-) create mode 100644 test/integration/fixtures.py create mode 100644 test/integration/v1/__init__.py rename test/integration/{ => v1}/test_ci_build.py (100%) rename test/integration/{ => v1}/test_ci_export.py (100%) rename test/integration/{ => v1}/test_ci_prepare.py (100%) rename test/integration/{ => v1}/test_ci_push.py (100%) rename test/integration/{ => v1}/test_ci_security_scan.py (100%) rename test/integration/{ => v1}/test_ci_test.py (100%) create mode 100644 test/unit/v1/__init__.py rename test/unit/{ => v1}/ci_calls.py (98%) create mode 100644 test/unit/v1/fixtures.py rename test/unit/{ => v1}/test_asset_uploader.py (100%) rename test/unit/{ => v1}/test_ci.py (97%) rename test/unit/{ => v1}/test_ci_prepare.py (100%) rename test/unit/{ => v1}/test_ci_test.py (100%) rename test/unit/{ => v1}/test_config_data_model.py (100%) rename test/unit/{ => v1}/test_config_data_model_generator.py (100%) rename test/unit/{ => v1}/test_env.py (100%) rename test/unit/{ => v1}/test_ignore_folders.py (100%) rename test/unit/{ => v1}/test_release.py (92%) rename test/unit/{ => v1}/test_release_uploader.py (100%) create mode 100644 test/unit/v2/__init__.py create mode 100644 test/unit/v2/resources/__init__.py create mode 100644 test/unit/v2/resources/slc_build_config.py create mode 100644 test/unit/v2/test_ci_prepare.py create mode 100644 test/unit/v2/test_env.py create mode 100644 test/unit/v2/test_export_and_scan_vulnerabilities.py create mode 100644 test/unit/v2/test_ignore_folders.py create mode 100644 test/unit/v2/test_nox_tasks.py create mode 100644 test/unit/v2/test_run_tests.py diff --git a/exasol/slc_ci/lib/run_tests.py b/exasol/slc_ci/lib/run_tests.py index b4ff67d..a0dafe5 100644 --- a/exasol/slc_ci/lib/run_tests.py +++ b/exasol/slc_ci/lib/run_tests.py @@ -31,8 +31,9 @@ def run_tests( slc_files = list(slc_path.glob(f"{flavor}*.tar.gz")) if len(slc_files) != 1: raise ValueError( - f"{flavor} does not contain expected tar.gz file, but \n {slc_files}" + f"{slc_directory} does not contain expected tar.gz file, but \n {slc_files}" ) + slc_file_path = slc_files[0] flavor_path = (f"flavors/{flavor}",) test_container_folder = "test_container" @@ -41,7 +42,7 @@ def run_tests( for test_folder in test_set_folders: ci_test.execute_tests( flavor_path=flavor_path, - slc_path=slc_path, + slc_path=slc_file_path, test_folder=test_folder, test_container_folder=test_container_folder, docker_user=docker_user, diff --git a/test/conftest.py b/test/conftest.py index b383e37..0464cad 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,26 +1,17 @@ -import json import logging import os -from inspect import cleandoc -from pathlib import Path from tempfile import TemporaryDirectory from unittest import mock from unittest.mock import MagicMock, patch -import pytest -from exasol_script_languages_container_ci.lib.config.config_data_model import ( - Build, - Config, - Ignore, - Release, -) + from exasol_script_languages_container_ci.lib.config.data_model_generator import ( config_data_model_default_output_file, regenerate_config_data_model, ) - -script_path = Path(__file__).absolute().parent +from test.unit.v1.fixtures import * +from test.integration.fixtures import * DISABLE_PYDANTIC_MODEL_GENERATION = "--disable-pydantic-model-generation" @@ -50,21 +41,6 @@ def pytest_configure(config): logger.warning("Generation of pydantic models from json schema disabled") -@pytest.fixture -def resources_path() -> Path: - return script_path / "integration/resources" - - -@pytest.fixture -def flavors_path(resources_path: Path) -> Path: - return resources_path / "flavors" - - -@pytest.fixture -def test_containers_folder(resources_path: Path) -> Path: - return resources_path / "test_containers" - - @pytest.fixture() def mock_settings_env_vars(): with mock.patch.dict(os.environ, {}): @@ -84,13 +60,6 @@ def tmp_test_dir(): os.chdir(old_dir) -@pytest.fixture -def build_config() -> Config: - return Config( - build=Build(ignore=Ignore(paths=["doc"]), base_branch="master"), - release=Release(timeout_in_minutes=1), - ) - @pytest.fixture() def git_access_mock(): @@ -103,25 +72,3 @@ def git_access_mock(): git_access_mock.get_files_of_commit.return_value = ["src/udfclient.cpp"] git_access_mock.get_last_commit_message.return_value = "last commit" return git_access_mock - - -@pytest.fixture -def expected_json_config() -> str: - json = cleandoc( - """ - { - "build": { - "ignore": { - "paths": [ - "a/b/c", - "e/f/g" - ] - }, - "base_branch": "" - }, - "release": { - "timeout_in_minutes": 1 - } - }""" - ) - return json diff --git a/test/integration/fixtures.py b/test/integration/fixtures.py new file mode 100644 index 0000000..e349e23 --- /dev/null +++ b/test/integration/fixtures.py @@ -0,0 +1,20 @@ +from pathlib import Path + +import pytest + +script_path = Path(__file__).absolute().parent + +@pytest.fixture +def resources_path() -> Path: + return script_path / "resources" + + +@pytest.fixture +def flavors_path(resources_path: Path) -> Path: + return resources_path / "flavors" + + +@pytest.fixture +def test_containers_folder(resources_path: Path) -> Path: + return resources_path / "test_containers" + diff --git a/test/integration/v1/__init__.py b/test/integration/v1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/integration/test_ci_build.py b/test/integration/v1/test_ci_build.py similarity index 100% rename from test/integration/test_ci_build.py rename to test/integration/v1/test_ci_build.py diff --git a/test/integration/test_ci_export.py b/test/integration/v1/test_ci_export.py similarity index 100% rename from test/integration/test_ci_export.py rename to test/integration/v1/test_ci_export.py diff --git a/test/integration/test_ci_prepare.py b/test/integration/v1/test_ci_prepare.py similarity index 100% rename from test/integration/test_ci_prepare.py rename to test/integration/v1/test_ci_prepare.py diff --git a/test/integration/test_ci_push.py b/test/integration/v1/test_ci_push.py similarity index 100% rename from test/integration/test_ci_push.py rename to test/integration/v1/test_ci_push.py diff --git a/test/integration/test_ci_security_scan.py b/test/integration/v1/test_ci_security_scan.py similarity index 100% rename from test/integration/test_ci_security_scan.py rename to test/integration/v1/test_ci_security_scan.py diff --git a/test/integration/test_ci_test.py b/test/integration/v1/test_ci_test.py similarity index 100% rename from test/integration/test_ci_test.py rename to test/integration/v1/test_ci_test.py diff --git a/test/unit/v1/__init__.py b/test/unit/v1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/unit/ci_calls.py b/test/unit/v1/ci_calls.py similarity index 98% rename from test/unit/ci_calls.py rename to test/unit/v1/ci_calls.py index 7fa438f..06e5274 100644 --- a/test/unit/ci_calls.py +++ b/test/unit/v1/ci_calls.py @@ -1,4 +1,4 @@ -from test.unit.test_env import test_env +from test.unit.v1.test_env import test_env from unittest.mock import call diff --git a/test/unit/v1/fixtures.py b/test/unit/v1/fixtures.py new file mode 100644 index 0000000..1807456 --- /dev/null +++ b/test/unit/v1/fixtures.py @@ -0,0 +1,35 @@ +from inspect import cleandoc + +import pytest + +from exasol_script_languages_container_ci.lib.config.config_data_model import Config, Build, Ignore, Release + + +@pytest.fixture +def build_config() -> Config: + return Config( + build=Build(ignore=Ignore(paths=["doc"]), base_branch="master"), + release=Release(timeout_in_minutes=1), + ) + + +@pytest.fixture +def expected_json_config() -> str: + json = cleandoc( + """ + { + "build": { + "ignore": { + "paths": [ + "a/b/c", + "e/f/g" + ] + }, + "base_branch": "" + }, + "release": { + "timeout_in_minutes": 1 + } + }""" + ) + return json diff --git a/test/unit/test_asset_uploader.py b/test/unit/v1/test_asset_uploader.py similarity index 100% rename from test/unit/test_asset_uploader.py rename to test/unit/v1/test_asset_uploader.py diff --git a/test/unit/test_ci.py b/test/unit/v1/test_ci.py similarity index 97% rename from test/unit/test_ci.py rename to test/unit/v1/test_ci.py index e1c1a0e..df6beb1 100644 --- a/test/unit/test_ci.py +++ b/test/unit/v1/test_ci.py @@ -1,5 +1,5 @@ -from test.unit import ci_calls -from test.unit.test_env import test_env +from test.unit.v1 import ci_calls +from test.unit.v1.test_env import test_env from typing import Union from unittest.mock import Mock diff --git a/test/unit/test_ci_prepare.py b/test/unit/v1/test_ci_prepare.py similarity index 100% rename from test/unit/test_ci_prepare.py rename to test/unit/v1/test_ci_prepare.py diff --git a/test/unit/test_ci_test.py b/test/unit/v1/test_ci_test.py similarity index 100% rename from test/unit/test_ci_test.py rename to test/unit/v1/test_ci_test.py diff --git a/test/unit/test_config_data_model.py b/test/unit/v1/test_config_data_model.py similarity index 100% rename from test/unit/test_config_data_model.py rename to test/unit/v1/test_config_data_model.py diff --git a/test/unit/test_config_data_model_generator.py b/test/unit/v1/test_config_data_model_generator.py similarity index 100% rename from test/unit/test_config_data_model_generator.py rename to test/unit/v1/test_config_data_model_generator.py diff --git a/test/unit/test_env.py b/test/unit/v1/test_env.py similarity index 100% rename from test/unit/test_env.py rename to test/unit/v1/test_env.py diff --git a/test/unit/test_ignore_folders.py b/test/unit/v1/test_ignore_folders.py similarity index 100% rename from test/unit/test_ignore_folders.py rename to test/unit/v1/test_ignore_folders.py diff --git a/test/unit/test_release.py b/test/unit/v1/test_release.py similarity index 92% rename from test/unit/test_release.py rename to test/unit/v1/test_release.py index 0b6c37b..097a417 100644 --- a/test/unit/test_release.py +++ b/test/unit/v1/test_release.py @@ -1,11 +1,10 @@ -from test.unit import ci_calls -from test.unit.test_env import test_env +from test.unit.v1 import ci_calls +from test.unit.v1.test_env import test_env from typing import Union -from unittest.mock import MagicMock, Mock, create_autospec +from unittest.mock import Mock import pytest -from exasol_script_languages_container_ci.lib.ci import ci from exasol_script_languages_container_ci.lib.ci_build import CIBuild from exasol_script_languages_container_ci.lib.ci_push import CIPush from exasol_script_languages_container_ci.lib.ci_security_scan import CISecurityScan diff --git a/test/unit/test_release_uploader.py b/test/unit/v1/test_release_uploader.py similarity index 100% rename from test/unit/test_release_uploader.py rename to test/unit/v1/test_release_uploader.py diff --git a/test/unit/v2/__init__.py b/test/unit/v2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/unit/v2/resources/__init__.py b/test/unit/v2/resources/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/unit/v2/resources/slc_build_config.py b/test/unit/v2/resources/slc_build_config.py new file mode 100644 index 0000000..3e037bb --- /dev/null +++ b/test/unit/v2/resources/slc_build_config.py @@ -0,0 +1,9 @@ +from pathlib import Path + +from exasol.slc_ci.model.build_config import BuildConfig + +SLC_BUILD_CONFIG = BuildConfig(root=Path(__file__).parent, base_branch="master", + ignore_paths=["doc", "githooks"], + docker_build_repository="test/script-languages-build-cache", + docker_release_repository="test/script-language-container", + test_container_folder="test_container", ) \ No newline at end of file diff --git a/test/unit/v2/test_ci_prepare.py b/test/unit/v2/test_ci_prepare.py new file mode 100644 index 0000000..736afa9 --- /dev/null +++ b/test/unit/v2/test_ci_prepare.py @@ -0,0 +1,60 @@ +import os +from pathlib import Path +from unittest import mock + +import pytest +from exasol_integration_test_docker_environment.cli.options.system_options import ( + DEFAULT_OUTPUT_DIRECTORY, +) +from exasol_integration_test_docker_environment.lib.logging import luigi_log_config + +from exasol.slc_ci.lib.ci_prepare import CIPrepare + +EXPECTED_LOG_PARENT_DIRECTORY = Path(DEFAULT_OUTPUT_DIRECTORY) / "jobs" / "logs" +EXPECTED_LOG_FILE = EXPECTED_LOG_PARENT_DIRECTORY / "main.log" + + +def test_ci_prepare_log_environment_variable_is_set(mock_settings_env_vars): + CIPrepare().prepare() + assert luigi_log_config.LOG_ENV_VARIABLE_NAME in os.environ + + +def test_ci_prepare_log_environment_variable_is_set_to_the_correct_path( + mock_settings_env_vars, +): + CIPrepare().prepare() + expected_path = str(EXPECTED_LOG_FILE.absolute()) + assert os.environ[luigi_log_config.LOG_ENV_VARIABLE_NAME] == expected_path + + +def test_ci_prepare_log_path_parent_directory_doesnt_exists(mock_settings_env_vars): + CIPrepare().prepare() + assert Path(os.environ[luigi_log_config.LOG_ENV_VARIABLE_NAME]).parent.is_dir() + + +def test_ci_prepare_log_path_file_doesnt_exist(mock_settings_env_vars): + CIPrepare().prepare() + actual_path = Path(os.environ[luigi_log_config.LOG_ENV_VARIABLE_NAME]) + assert not actual_path.exists() + + +def test_ci_prepare_log_path_parent_directory_exist(mock_settings_env_vars): + EXPECTED_LOG_PARENT_DIRECTORY.mkdir(parents=True) + CIPrepare().prepare() + assert ( + Path(os.environ[luigi_log_config.LOG_ENV_VARIABLE_NAME]).parent + == EXPECTED_LOG_PARENT_DIRECTORY.absolute() + ) + + +def test_ci_prepare_log_path_file_exists(mock_settings_env_vars): + expected_value = "Test" + EXPECTED_LOG_PARENT_DIRECTORY.mkdir(parents=True) + with EXPECTED_LOG_FILE.open("wt") as f: + f.write(expected_value) + CIPrepare().prepare() + actual_path = Path(os.environ[luigi_log_config.LOG_ENV_VARIABLE_NAME]) + actual_value = actual_path.read_text() + assert ( + actual_value == expected_value and actual_path == EXPECTED_LOG_FILE.absolute() + ) diff --git a/test/unit/v2/test_env.py b/test/unit/v2/test_env.py new file mode 100644 index 0000000..103ac04 --- /dev/null +++ b/test/unit/v2/test_env.py @@ -0,0 +1,22 @@ +from exasol.slc_ci.model.flavor_ci_model import FlavorCiConfig, TestConfig, TestSet +from test.unit.v2.resources.slc_build_config import SLC_BUILD_CONFIG + + +class TestEnv: + docker_user = "test_docker_user" + docker_pwd = "test_docker_pwd" + commit_sha = "test_commit_sha" + branch_name = "test_branch_name" + build_config = SLC_BUILD_CONFIG + flavor_config = FlavorCiConfig( + build_runner="some_build_runner", + test_config=TestConfig( + test_runner="some_test_runner", + test_sets=[TestSet(name="some_test_name", folders=["all", "specific"])], + ), +) + + + + +test_env = TestEnv() diff --git a/test/unit/v2/test_export_and_scan_vulnerabilities.py b/test/unit/v2/test_export_and_scan_vulnerabilities.py new file mode 100644 index 0000000..6a1002e --- /dev/null +++ b/test/unit/v2/test_export_and_scan_vulnerabilities.py @@ -0,0 +1,66 @@ +from pathlib import Path + +from exasol.slc_ci.lib.ci_prepare import CIPrepare +from exasol.slc_ci.lib.export_and_scan_vulnerabilities import export_and_scan_vulnerabilities +from exasol.slc_ci.model.flavor_ci_model import FlavorCiConfig, TestConfig, TestSet +from typing import Union +from unittest.mock import Mock, call, MagicMock + +import pytest + +from exasol_script_languages_container_ci.lib.ci_test import CIExecuteTest +from test.unit.v2.test_env import test_env + +TEST_FLAVOR = "flavor_xyz" + +FLAVOR_CONFIG = FlavorCiConfig( + build_runner="some_build_runner", + test_config=TestConfig( + test_runner="some_test_runner", + test_sets=[TestSet(name="some_test_name", folders=["all", "specific"])], + ), +) + +@pytest.fixture +def slc_directory(tmp_path: Path) -> Path: + with open(str(tmp_path / f'{TEST_FLAVOR}-dummy_slc.tar.gz'), 'w') as f: + f.write("nothing") + return tmp_path + + + +def test_export_and_scan_vulnerabilities(slc_directory, git_access_mock): + res_slc_path = Path("/some_path/slc.tar.gz") + ci_export_mock = MagicMock() + ci_export_mock.export = MagicMock(return_value=res_slc_path) + ci_commands_mock: Union[CIExecuteTest,CIPrepare,Mock] = Mock() + + result = export_and_scan_vulnerabilities( + flavor=TEST_FLAVOR, + branch_name=test_env.branch_name, + docker_user=test_env.docker_user, + docker_password=test_env.docker_pwd, + commit_sha=test_env.commit_sha, + build_config=test_env.build_config, + git_access=git_access_mock, + ci_build=ci_commands_mock, + ci_security_scan=ci_commands_mock, + ci_prepare=ci_commands_mock, + ci_export=ci_export_mock, + ci_push=ci_commands_mock, + ) + assert ci_commands_mock.mock_calls == [ + call.prepare(), + call.build(flavor_path=(f'flavors/{TEST_FLAVOR}',), rebuild=False, + build_docker_repository=test_env.build_config.docker_build_repository, commit_sha=test_env.commit_sha, + docker_user=test_env.docker_user, docker_password=test_env.docker_pwd, + test_container_folder='test_container'), + call.run_security_scan(flavor_path=(f'flavors/{TEST_FLAVOR}',)), + call.push(flavor_path=(f'flavors/{TEST_FLAVOR}',), target_docker_repository=test_env.build_config.docker_build_repository, + target_docker_tag_prefix=test_env.commit_sha, docker_user=test_env.docker_user, + docker_password=test_env.docker_pwd), + call.push(flavor_path=(f'flavors/{TEST_FLAVOR}',), target_docker_repository=test_env.build_config.docker_build_repository, + target_docker_tag_prefix='', docker_user=test_env.docker_user, docker_password=test_env.docker_pwd), + ] + assert ci_export_mock.mock_calls == [call.export(flavor_path=(f'flavors/{TEST_FLAVOR}',))] + assert result == res_slc_path diff --git a/test/unit/v2/test_ignore_folders.py b/test/unit/v2/test_ignore_folders.py new file mode 100644 index 0000000..2542508 --- /dev/null +++ b/test/unit/v2/test_ignore_folders.py @@ -0,0 +1,157 @@ +from pathlib import Path +from typing import List + +import git +import pytest + +from exasol.slc_ci.lib.check_if_build_needed import check_if_need_to_build + +from exasol_script_languages_container_ci.lib.git_access import GitAccess +from test.unit.v2.test_env import test_env + + +def commit_base(repo: git.Repo, repo_path: Path) -> None: + """ + Create dummy commit on base branch with "something" + """ + (repo_path / "something").parent.mkdir(parents=True, exist_ok=True) + open(repo_path / "something", "w").close() + repo.index.add([str(repo_path / "something")]) + repo.index.commit("Base commit") + assert repo.active_branch.name == "master" or repo.active_branch.name == "main" + if repo.active_branch.name == "main": + repo.active_branch.rename("master") + + +def commit_files( + branch_name: str, + repo: git.Repo, + repo_path: Path, + files_to_commit: List[List[str]], + commit_message: str, +) -> None: + """ + Create empty given files (param files_to_commit) and commit them to a "Dummy commit" + """ + commit_base(repo, repo_path) + current = repo.create_head(branch_name) + current.checkout() + for file_list in files_to_commit: + for file in file_list: + (repo_path / file).parent.mkdir(parents=True, exist_ok=True) + open(repo_path / file, "w").close() + repo.index.add([str(repo_path / file)]) + repo.index.commit(commit_message) + + + +TEST_FLAVOR = "flavor_xyz" + +TEST_DATA = [ + # If the last commit contains files not included in the ignore-path list, the build must run + ( + "last_commit_not_ignore_path_build_must_run", + "refs/heads/feature_branch", + [["flavors/flavor_abc/build_steps.py", "doc/something", "src/udfclient.cpp"]], + "message", + True, + ), + # If there are 2 commits, and the last only contains files in the ignore-list, but the first contains + # files not included in the ignore-path list, the build must run + ( + "commit_before_last_commit_not_ignore_path_build_must_run", + "refs/heads/feature_branch", + [ + ["flavors/flavor_abc/build_steps.py", "doc/something", "src/udfclient.cpp"], + ["doc/something"], + ], + "message", + True, + ), + # If last commit(s) contain only files included in the ignore-path-list or another flavor the build must not run + ( + "last_commit_ignore_path_or_another_flavor_build_must_not_run", + "refs/heads/feature_branch", + [["flavors/flavor_abc/build_steps.py", "doc/something"]], + "message", + False, + ), + # If last commit message contains "[rebuild]" the build should always trigger + ( + "rebuild_in_last_commit_msg_build_must_run", + "refs/heads/feature_branch", + [["flavors/flavor_abc/build_steps.py", "doc/something"]], + "message [rebuild]", + True, + ), + # Affected files on current flavor should trigger a build + ( + "changes_in_current_flavor_build_must_run", + "refs/heads/feature_branch", + [[f"flavors/{TEST_FLAVOR}/build_steps.py", "doc/something"]], + "message", + True, + ), + # If there are 2 commits, and the last only contains files in the ignore-list, but the first contains + # files of the current flavor, the build must run + ( + "changes_in_current_flavor_before_last_commit_build_must_run", + "refs/heads/feature_branch", + [ + [f"flavors/{TEST_FLAVOR}/build_steps.py"], + ["flavors/flavor_abc/build_steps.py"], + ], + "message", + True, + ), + ( + "develop_must_always_run", + "refs/heads/develop", + [["doc/something"]], + "message", + True, + ), + # Even if folder should be ignored, in case of develop branch we always expect to run + ( + "master_must_always_run", + "refs/heads/master", + [["doc/something"]], + "message", + True, + ), + # Even if folder should be ignored, in case of master branch we always expect to run + ("main_must_always_run", "refs/heads/main", [["doc/something"]], "message", True), + # Even if folder should be ignored, in case of main branch we always expect to run + ( + "rebuild_must_always_run", + "refs/heads/rebuild/feature_branch", + [["doc/something"]], + "message", + True, + ), + # Even if folder should be ignored, in case of rebuild/* branch we always expect to run +] + + +@pytest.mark.parametrize( + "test_name, branch_name, files_to_commit,commit_message, expected_result", TEST_DATA +) +def test_ignore_folder_should_run_ci( + test_name: str, + branch_name: str, + tmp_test_dir, + files_to_commit, + commit_message: str, + expected_result: bool, +): + """ + This test creates a temporary git repository, commits the given file list (files_for_commit), then runs + ci.check_if_need_to_build() and checks if it returned the expected result + """ + repo_path = Path(tmp_test_dir) + tmp_repo = git.Repo.init(repo_path) + commit_files(branch_name, tmp_repo, repo_path, files_to_commit, commit_message) + assert ( + check_if_need_to_build(branch_name, test_env.build_config, TEST_FLAVOR, GitAccess()) + == expected_result + ) diff --git a/test/unit/v2/test_nox_tasks.py b/test/unit/v2/test_nox_tasks.py new file mode 100644 index 0000000..b710138 --- /dev/null +++ b/test/unit/v2/test_nox_tasks.py @@ -0,0 +1,240 @@ +import argparse +import sys +from pathlib import Path +from unittest import mock +from unittest.mock import MagicMock + +import nox +import pytest +from _pytest.monkeypatch import MonkeyPatch + +import exasol.slc_ci.api as api +from test.unit.v2.test_env import test_env + +FLAVORS = ["flavor_a", "flavor_b"] + + +@pytest.fixture(scope="module") +def nox_tasks(): + resources_path = Path(__file__).parent / "resources" + #exasol.slc_ci.nox.tasks imports 'slc_build_config.py`. We must ensure that this modul can be found by adding it to the sys path. + sys.path.append(str(resources_path)) + import exasol.slc_ci.nox.tasks as nox_tasks + yield nox_tasks + #now remove 'slc_build_config.py` again from sys path. + sys.path.pop() + + +@pytest.fixture +def mock_get_flavors(monkeypatch: MonkeyPatch) -> MagicMock: + mock_function_to_mock = MagicMock(return_value=FLAVORS) + monkeypatch.setattr(api, "get_flavors", mock_function_to_mock) + return mock_function_to_mock + + +@pytest.fixture +def mock_get_flavor_ci_model(monkeypatch: MonkeyPatch) -> MagicMock: + mock_function_to_mock = MagicMock(return_value=test_env.flavor_config) + monkeypatch.setattr(api, "get_flavor_ci_model", mock_function_to_mock) + return mock_function_to_mock + + +@pytest.fixture +def mock_run_tests(monkeypatch: MonkeyPatch) -> MagicMock: + mock_function_to_mock = MagicMock(return_value=None) + monkeypatch.setattr(api, "run_tests", mock_function_to_mock) + return mock_function_to_mock + + +@pytest.fixture +def mock_check_if_build_needed(monkeypatch: MonkeyPatch) -> MagicMock: + mock_function_to_mock = MagicMock(return_value=True) + monkeypatch.setattr(api, "check_if_build_needed", mock_function_to_mock) + return mock_function_to_mock + + +@pytest.fixture +def mock_export_and_scan_vulnerabilities(monkeypatch: MonkeyPatch) -> MagicMock: + mock_function_to_mock = MagicMock(return_value="/tmp/slc.tar.gz") + monkeypatch.setattr(api, "export_and_scan_vulnerabilities", mock_function_to_mock) + return mock_function_to_mock + + +@pytest.fixture +def fake_session_builder(): + def fake_session(session_name: str, *args): + global_config = argparse.Namespace(posargs=args) + session_runner = nox.sessions.SessionRunner( + name=session_name, + signatures=None, + func=None, + global_config=global_config, + manifest=None, + ) + return nox.sessions.Session(session_runner) + + return fake_session + + +@pytest.fixture +def github_output(monkeypatch: MonkeyPatch, tmp_path): + out_file = tmp_path / "out.txt" + monkeypatch.setenv("GITHUB_OUTPUT", str(out_file)) + return out_file + + +def test_run_find_available_flavors( + fake_session_builder, nox_tasks, mock_get_flavors, github_output +) -> None: + fake_session = fake_session_builder( + nox_tasks.run_find_available_flavors.name, "--github-var", "test" + ) + nox_tasks.run_find_available_flavors(fake_session) + assert mock_get_flavors.call_count == 1 + assert mock_get_flavors.call_args == mock.call( + Path(__file__).parent / "resources" / "flavors" + ) + out_content = github_output.read_text() + assert out_content == """test=["flavor_a", "flavor_b"]\n""" + + +def test_run_get_build_runner_for_flavor( + fake_session_builder, nox_tasks, mock_get_flavor_ci_model, github_output +) -> None: + fake_session = fake_session_builder( + nox_tasks.run_get_build_runner_for_flavor.name, + "--flavor", + "flavor_a", + "--github-var", + "test", + ) + nox_tasks.run_get_build_runner_for_flavor(fake_session) + assert mock_get_flavor_ci_model.call_count == 1 + assert mock_get_flavor_ci_model.call_args == mock.call( + nox_tasks.SLC_BUILD_CONFIG, "flavor_a" + ) + out_content = github_output.read_text() + assert out_content == """test=some_build_runner\n""" + + +def test_run_get_test_runner_for_flavor( + fake_session_builder, nox_tasks, mock_get_flavor_ci_model, github_output +) -> None: + fake_session = fake_session_builder( + nox_tasks.run_get_test_runner_for_flavor.name, + "--flavor", + "flavor_a", + "--github-var", + "test", + ) + nox_tasks.run_get_test_runner_for_flavor(fake_session) + assert mock_get_flavor_ci_model.call_count == 1 + assert mock_get_flavor_ci_model.call_args == mock.call( + nox_tasks.SLC_BUILD_CONFIG, "flavor_a" + ) + out_content = github_output.read_text() + assert out_content == """test=some_test_runner\n""" + + +def test_run_get_test_set_names_for_flavor( + fake_session_builder, nox_tasks, mock_get_flavor_ci_model, github_output +) -> None: + fake_session = fake_session_builder( + nox_tasks.run_get_test_set_names_for_flavor.name, + "--flavor", + "flavor_a", + "--github-var", + "test", + ) + nox_tasks.run_get_test_set_names_for_flavor(fake_session) + assert mock_get_flavor_ci_model.call_count == 1 + assert mock_get_flavor_ci_model.call_args == mock.call( + nox_tasks.SLC_BUILD_CONFIG, "flavor_a" + ) + out_content = github_output.read_text() + assert out_content == """test=["some_test_name"]\n""" + + +def test_run_db_tests( + fake_session_builder, nox_tasks, mock_get_flavor_ci_model, mock_run_tests +) -> None: + fake_session = fake_session_builder( + nox_tasks.run_db_tests.name, + "--flavor", + "flavor_a", + "--docker-user", + "some_user", + "--docker-password", + "super_secret", + "--test-set-name", + "all", + "--slc-directory", + "some_directory", + ) + nox_tasks.run_db_tests(fake_session) + assert mock_get_flavor_ci_model.call_count == 1 + assert mock_get_flavor_ci_model.call_args == mock.call( + nox_tasks.SLC_BUILD_CONFIG, "flavor_a" + ) + assert mock_run_tests.call_count == 1 + assert mock_run_tests.call_args == mock.call( + flavor="flavor_a", + slc_directory="some_directory", + flavor_config=test_env.flavor_config, + test_set_name="all", + docker_user="some_user", + docker_password="super_secret", + ) + + +def test_run_check_if_build_needed( + fake_session_builder, nox_tasks, mock_check_if_build_needed, github_output +) -> None: + fake_session = fake_session_builder( + nox_tasks.run_db_tests.name, + "--flavor", + "flavor_a", + "--branch-name", + "feature/abc", + "--github-var", + "test", + ) + nox_tasks.run_check_if_build_needed(fake_session) + assert mock_check_if_build_needed.call_count == 1 + assert mock_check_if_build_needed.call_args == mock.call( + branch_name="feature/abc", flavor="flavor_a", build_config=nox_tasks.SLC_BUILD_CONFIG + ) + out_content = github_output.read_text() + assert out_content == """test=True\n""" + + +def test_run_export_and_scan_vulnerabilities( + fake_session_builder, nox_tasks, mock_export_and_scan_vulnerabilities, github_output +) -> None: + fake_session = fake_session_builder( + nox_tasks.run_db_tests.name, + "--flavor", + "flavor_a", + "--branch-name", + "feature/abc", + "--docker-user", + "some_user", + "--docker-password", + "super_secret", + "--commit-sha", + "ABCDE", + "--github-var", + "test", + ) + nox_tasks.run_export_and_scan_vulnerabilities(fake_session) + assert mock_export_and_scan_vulnerabilities.call_count == 1 + assert mock_export_and_scan_vulnerabilities.call_args == mock.call( + flavor="flavor_a", + build_config=nox_tasks.SLC_BUILD_CONFIG, + branch_name="feature/abc", + docker_user="some_user", + docker_password="super_secret", + commit_sha="ABCDE", + ) + out_content = github_output.read_text() + assert out_content == """test=/tmp/slc.tar.gz\n""" diff --git a/test/unit/v2/test_run_tests.py b/test/unit/v2/test_run_tests.py new file mode 100644 index 0000000..9284583 --- /dev/null +++ b/test/unit/v2/test_run_tests.py @@ -0,0 +1,50 @@ +from pathlib import Path + +from exasol.slc_ci.lib.ci_prepare import CIPrepare +from exasol.slc_ci.lib.run_tests import run_tests +from typing import Union +from unittest.mock import Mock, call + +import pytest + +from exasol_script_languages_container_ci.lib.ci_test import CIExecuteTest +from test.unit.v2.test_env import test_env + + +TEST_FLAVOR = "flavor_xyz" + +@pytest.fixture +def slc_directory(tmp_path: Path) -> Path: + with open(str(tmp_path / f'{TEST_FLAVOR}-dummy_slc.tar.gz'), 'w') as f: + f.write("nothing") + return tmp_path + + +def run_db_test_call(slc_path: Path, test_folder: str): + return call.execute_tests( + flavor_path=(f"flavors/{TEST_FLAVOR}",), + docker_user=test_env.docker_user, + docker_password=test_env.docker_pwd, + test_container_folder="test_container", + slc_path=slc_path, + test_folder=test_folder, + ) + +def test_run_tests(slc_directory): + ci_commands_mock: Union[CIExecuteTest,CIPrepare,Mock] = Mock() + + run_tests( + flavor=TEST_FLAVOR, + slc_directory=str(slc_directory), + flavor_config=test_env.flavor_config, + test_set_name="some_test_name", + docker_user=test_env.docker_user, + docker_password=test_env.docker_pwd, + ci_prepare=ci_commands_mock, + ci_test=ci_commands_mock + ) + assert ci_commands_mock.mock_calls == [ + call.prepare(), + run_db_test_call(slc_path=slc_directory / f"{TEST_FLAVOR}-dummy_slc.tar.gz", test_folder='all'), + run_db_test_call(slc_path=slc_directory / f"{TEST_FLAVOR}-dummy_slc.tar.gz", test_folder='specific'), + ] From f108de901ab25c1dc0019ade5b52556fbf6364b5 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 28 Apr 2025 15:04:00 -0300 Subject: [PATCH 16/19] USe test_container path from build config --- exasol/slc_ci/api/run_tests.py | 3 +++ .../slc_ci/lib/export_and_scan_vulnerabilities.py | 2 +- exasol/slc_ci/lib/run_tests.py | 6 ++++-- exasol/slc_ci/nox/tasks.py | 1 + .../unit/v2/test_export_and_scan_vulnerabilities.py | 13 +++++++------ test/unit/v2/test_nox_tasks.py | 1 + test/unit/v2/test_run_tests.py | 5 +++-- 7 files changed, 20 insertions(+), 11 deletions(-) diff --git a/exasol/slc_ci/api/run_tests.py b/exasol/slc_ci/api/run_tests.py index f311968..dbde026 100644 --- a/exasol/slc_ci/api/run_tests.py +++ b/exasol/slc_ci/api/run_tests.py @@ -3,6 +3,7 @@ from exasol.slc_ci.lib.ci_prepare import CIPrepare from exasol.slc_ci.lib.ci_test import CIExecuteTest from exasol.slc_ci.lib.run_tests import run_tests as lib_run_tests +from exasol.slc_ci.model.build_config import BuildConfig from exasol.slc_ci.model.flavor_ci_model import FlavorCiConfig @@ -10,6 +11,7 @@ def run_tests( flavor: str, slc_directory: str, flavor_config: FlavorCiConfig, + build_config: BuildConfig, test_set_name: str, docker_user: str, docker_password: str, @@ -21,6 +23,7 @@ def run_tests( flavor=flavor, slc_directory=slc_directory, flavor_config=flavor_config, + build_config=build_config, test_set_name=test_set_name, docker_user=docker_user, docker_password=docker_password, diff --git a/exasol/slc_ci/lib/export_and_scan_vulnerabilities.py b/exasol/slc_ci/lib/export_and_scan_vulnerabilities.py index 95103c9..7bcce3c 100644 --- a/exasol/slc_ci/lib/export_and_scan_vulnerabilities.py +++ b/exasol/slc_ci/lib/export_and_scan_vulnerabilities.py @@ -29,7 +29,7 @@ def export_and_scan_vulnerabilities( f"Running build image and scan vulnerabilities for parameters: {locals()}" ) - flavor_path = (f"{build_config.flavors_path.name}/{flavor}",) + flavor_path = (f"{build_config.flavors_path}/{flavor}",) test_container_folder = build_config.test_container_folder rebuild = BranchConfig.rebuild(branch_name) ci_prepare.prepare() diff --git a/exasol/slc_ci/lib/run_tests.py b/exasol/slc_ci/lib/run_tests.py index a0dafe5..13aabd9 100644 --- a/exasol/slc_ci/lib/run_tests.py +++ b/exasol/slc_ci/lib/run_tests.py @@ -3,6 +3,7 @@ from exasol.slc_ci.lib.ci_prepare import CIPrepare from exasol.slc_ci.lib.ci_test import CIExecuteTest +from exasol.slc_ci.model.build_config import BuildConfig from exasol.slc_ci.model.flavor_ci_model import FlavorCiConfig @@ -10,6 +11,7 @@ def run_tests( flavor: str, slc_directory: str, flavor_config: FlavorCiConfig, + build_config: BuildConfig, test_set_name: str, docker_user: str, docker_password: str, @@ -35,8 +37,8 @@ def run_tests( ) slc_file_path = slc_files[0] - flavor_path = (f"flavors/{flavor}",) - test_container_folder = "test_container" + flavor_path = (str( build_config.flavors_path / flavor),) + test_container_folder = build_config.test_container_folder ci_prepare.prepare() for test_folder in test_set_folders: diff --git a/exasol/slc_ci/nox/tasks.py b/exasol/slc_ci/nox/tasks.py index d5256fd..e755540 100644 --- a/exasol/slc_ci/nox/tasks.py +++ b/exasol/slc_ci/nox/tasks.py @@ -87,6 +87,7 @@ def run_db_tests(session: nox.Session): flavor=args.flavor, slc_directory=args.slc_directory, flavor_config=flavor_config, + build_config=SLC_BUILD_CONFIG, test_set_name=args.test_set_name, docker_user=args.docker_user, docker_password=args.docker_password, diff --git a/test/unit/v2/test_export_and_scan_vulnerabilities.py b/test/unit/v2/test_export_and_scan_vulnerabilities.py index 6a1002e..d9620ba 100644 --- a/test/unit/v2/test_export_and_scan_vulnerabilities.py +++ b/test/unit/v2/test_export_and_scan_vulnerabilities.py @@ -49,18 +49,19 @@ def test_export_and_scan_vulnerabilities(slc_directory, git_access_mock): ci_export=ci_export_mock, ci_push=ci_commands_mock, ) + expected_flavor_path = str(test_env.build_config.flavors_path / TEST_FLAVOR) assert ci_commands_mock.mock_calls == [ call.prepare(), - call.build(flavor_path=(f'flavors/{TEST_FLAVOR}',), rebuild=False, + call.build(flavor_path=(expected_flavor_path,), rebuild=False, build_docker_repository=test_env.build_config.docker_build_repository, commit_sha=test_env.commit_sha, docker_user=test_env.docker_user, docker_password=test_env.docker_pwd, - test_container_folder='test_container'), - call.run_security_scan(flavor_path=(f'flavors/{TEST_FLAVOR}',)), - call.push(flavor_path=(f'flavors/{TEST_FLAVOR}',), target_docker_repository=test_env.build_config.docker_build_repository, + test_container_folder=test_env.build_config.test_container_folder,), + call.run_security_scan(flavor_path=(expected_flavor_path,)), + call.push(flavor_path=(expected_flavor_path,), target_docker_repository=test_env.build_config.docker_build_repository, target_docker_tag_prefix=test_env.commit_sha, docker_user=test_env.docker_user, docker_password=test_env.docker_pwd), - call.push(flavor_path=(f'flavors/{TEST_FLAVOR}',), target_docker_repository=test_env.build_config.docker_build_repository, + call.push(flavor_path=(expected_flavor_path,), target_docker_repository=test_env.build_config.docker_build_repository, target_docker_tag_prefix='', docker_user=test_env.docker_user, docker_password=test_env.docker_pwd), ] - assert ci_export_mock.mock_calls == [call.export(flavor_path=(f'flavors/{TEST_FLAVOR}',))] + assert ci_export_mock.mock_calls == [call.export(flavor_path=(expected_flavor_path,))] assert result == res_slc_path diff --git a/test/unit/v2/test_nox_tasks.py b/test/unit/v2/test_nox_tasks.py index b710138..77bc306 100644 --- a/test/unit/v2/test_nox_tasks.py +++ b/test/unit/v2/test_nox_tasks.py @@ -181,6 +181,7 @@ def test_run_db_tests( flavor="flavor_a", slc_directory="some_directory", flavor_config=test_env.flavor_config, + build_config=test_env.build_config, test_set_name="all", docker_user="some_user", docker_password="super_secret", diff --git a/test/unit/v2/test_run_tests.py b/test/unit/v2/test_run_tests.py index 9284583..7a99bf3 100644 --- a/test/unit/v2/test_run_tests.py +++ b/test/unit/v2/test_run_tests.py @@ -22,10 +22,10 @@ def slc_directory(tmp_path: Path) -> Path: def run_db_test_call(slc_path: Path, test_folder: str): return call.execute_tests( - flavor_path=(f"flavors/{TEST_FLAVOR}",), + flavor_path=( str(test_env.build_config.flavors_path / TEST_FLAVOR),), docker_user=test_env.docker_user, docker_password=test_env.docker_pwd, - test_container_folder="test_container", + test_container_folder=test_env.build_config.test_container_folder, slc_path=slc_path, test_folder=test_folder, ) @@ -37,6 +37,7 @@ def test_run_tests(slc_directory): flavor=TEST_FLAVOR, slc_directory=str(slc_directory), flavor_config=test_env.flavor_config, + build_config=test_env.build_config, test_set_name="some_test_name", docker_user=test_env.docker_user, docker_password=test_env.docker_pwd, From 91a1bb3d93bac510608101fc10de60bc2ecb4ae1 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 29 Apr 2025 08:40:44 -0300 Subject: [PATCH 17/19] Added integration tests and unit tests for ci_test --- exasol/slc_ci/lib/ci_export.py | 6 +- .../v1}/__init__.py | 0 test/contract/{ => v1}/test_ci_test.py | 0 .../v2}/__init__.py | 0 test/contract/v2/test_ci_test.py | 83 ++++++++++++ test/integration/v1/test_ci_test.py | 2 +- test/integration/v2/__init__.py | 0 test/integration/v2/test_ci_build.py | 42 ++++++ test/integration/v2/test_ci_export.py | 14 ++ test/integration/v2/test_ci_prepare.py | 26 ++++ test/integration/v2/test_ci_push.py | 29 +++++ test/integration/v2/test_ci_security_scan.py | 34 +++++ test/integration/v2/test_ci_test.py | 27 ++++ test/unit/v2/test_ci_test.py | 122 ++++++++++++++++++ 14 files changed, 382 insertions(+), 3 deletions(-) rename test/{integration/resources/test_containers/failing/tests/test/linker_namespace_sanity => contract/v1}/__init__.py (100%) rename test/contract/{ => v1}/test_ci_test.py (100%) rename test/{integration/resources/test_containers/successful/tests/test/linker_namespace_sanity => contract/v2}/__init__.py (100%) create mode 100644 test/contract/v2/test_ci_test.py create mode 100644 test/integration/v2/__init__.py create mode 100644 test/integration/v2/test_ci_build.py create mode 100644 test/integration/v2/test_ci_export.py create mode 100644 test/integration/v2/test_ci_prepare.py create mode 100644 test/integration/v2/test_ci_push.py create mode 100644 test/integration/v2/test_ci_security_scan.py create mode 100644 test/integration/v2/test_ci_test.py create mode 100644 test/unit/v2/test_ci_test.py diff --git a/exasol/slc_ci/lib/ci_export.py b/exasol/slc_ci/lib/ci_export.py index 6bb764c..60d9255 100644 --- a/exasol/slc_ci/lib/ci_export.py +++ b/exasol/slc_ci/lib/ci_export.py @@ -17,9 +17,10 @@ def __init__( ): self._printer = printer - def export(self, flavor_path: Tuple[str, ...]) -> Path: + def export(self, flavor_path: Tuple[str, ...], output_directory: str = ".build_output") -> Path: """ - Export the flavor as tar.gz file + Export the flavor as tar.gz file. + The returned path is the path of the tar.gz file. """ logging.info(f"Running command 'export' with parameters: {locals()}") @@ -28,6 +29,7 @@ def export(self, flavor_path: Tuple[str, ...]) -> Path: workers=7, log_level="WARNING", use_job_specific_log_file=True, + output_directory=output_directory ) self._printer.print_exasol_docker_images() export_infos = list(export_result.export_infos.values()) diff --git a/test/integration/resources/test_containers/failing/tests/test/linker_namespace_sanity/__init__.py b/test/contract/v1/__init__.py similarity index 100% rename from test/integration/resources/test_containers/failing/tests/test/linker_namespace_sanity/__init__.py rename to test/contract/v1/__init__.py diff --git a/test/contract/test_ci_test.py b/test/contract/v1/test_ci_test.py similarity index 100% rename from test/contract/test_ci_test.py rename to test/contract/v1/test_ci_test.py diff --git a/test/integration/resources/test_containers/successful/tests/test/linker_namespace_sanity/__init__.py b/test/contract/v2/__init__.py similarity index 100% rename from test/integration/resources/test_containers/successful/tests/test/linker_namespace_sanity/__init__.py rename to test/contract/v2/__init__.py diff --git a/test/contract/v2/test_ci_test.py b/test/contract/v2/test_ci_test.py new file mode 100644 index 0000000..20cfb75 --- /dev/null +++ b/test/contract/v2/test_ci_test.py @@ -0,0 +1,83 @@ +from pathlib import Path + +import pytest + +from exasol.slc_ci.lib.ci_export import CIExport +from exasol.slc_ci.lib.ci_test import DBTestRunnerProtocol + +class CreateExistingContainer: + + @pytest.fixture + def use_existing_container(self, tmp_path, flavor_path) -> Path: + ci_export = CIExport() + container_path = ci_export.export(flavor_path=(flavor_path,), output_directory=str(tmp_path)) + return container_path + + +class SuccessfulFlavorContract(CreateExistingContainer): + + @pytest.fixture() + def flavor_path(self, flavors_path): + return str(flavors_path / "successful") + + @pytest.fixture() + def test_container(self, test_containers_folder): + return str(test_containers_folder / "successful") + + @pytest.fixture() + def test_folder(self): + return "successful_test" + + +class SuccessfulFlavorDBTestsContract(SuccessfulFlavorContract): + + @pytest.fixture + def db_test_runner(self) -> DBTestRunnerProtocol: + raise NotImplementedError() + + def test(self, db_test_runner: DBTestRunnerProtocol, test_container: str, flavor_path: str, use_existing_container: Path, test_folder: str): + result = db_test_runner.run( + flavor_path=(flavor_path,), + test_folder=(test_folder,), + release_goal=("release",), + workers=7, + docker_username=None, + docker_password=None, + test_container_folder=test_container, + use_existing_container=str(use_existing_container), + ) + assert result.tests_are_ok and result.command_line_output_path.exists() + + +class FailingRunDBTestFlavorContract(CreateExistingContainer): + + @pytest.fixture() + def flavor_path(self, flavors_path): + return str(flavors_path / "failing_run_db_test") + + @pytest.fixture() + def test_container(self, test_containers_folder): + return str(test_containers_folder / "failing") + + @pytest.fixture() + def test_folder(self): + return "failing_test" + +class FailingRunDBTestFlavorDBTestsContract(FailingRunDBTestFlavorContract): + + @pytest.fixture + def db_test_runner(self) -> DBTestRunnerProtocol: + raise NotImplementedError() + + def test(self, db_test_runner: DBTestRunnerProtocol, test_container: str, flavor_path: str, use_existing_container: Path, test_folder: str): + result = db_test_runner.run( + flavor_path=(flavor_path,), + test_folder=(test_folder,), + release_goal=("release",), + workers=7, + docker_username=None, + docker_password=None, + test_container_folder=test_container, + use_existing_container=str(use_existing_container), + ) + assert not result.tests_are_ok and result.command_line_output_path.exists() diff --git a/test/integration/v1/test_ci_test.py b/test/integration/v1/test_ci_test.py index ab2f828..2d2f07d 100644 --- a/test/integration/v1/test_ci_test.py +++ b/test/integration/v1/test_ci_test.py @@ -1,4 +1,4 @@ -from test.contract.test_ci_test import ( +from test.contract.v1.test_ci_test import ( FailingRunDBTestFlavorDBTestsContract, FailingRunDBTestFlavorLinkerNamespaceTestsContract, SuccessfulFlavorDBTestsContract, diff --git a/test/integration/v2/__init__.py b/test/integration/v2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/integration/v2/test_ci_build.py b/test/integration/v2/test_ci_build.py new file mode 100644 index 0000000..ed7b3fd --- /dev/null +++ b/test/integration/v2/test_ci_build.py @@ -0,0 +1,42 @@ +from test.asserts import not_raises + +import pytest + +from exasol.slc_ci.lib.ci_build import CIBuild + +testdata = [ + ( + "test_docker_build_repository", + "test_commit_sha", + "test_docker_build_repository", + "test_commit_sha_", + ), + (None, "", "exasol/script-language-container", ""), +] + + +@pytest.mark.parametrize( + "input_docker_build_repository,input_commit_sha,expected_docker_build_repository,expected_source_tag_prefix", + testdata, +) +def test( + input_docker_build_repository, + input_commit_sha, + expected_docker_build_repository, + expected_source_tag_prefix, + flavors_path, + test_containers_folder, +): + test_type = "successful" + flavor_path = str(flavors_path / test_type) + test_container_folder = str(test_containers_folder / test_type) + with not_raises(Exception): + CIBuild().build( + flavor_path=(flavor_path,), + rebuild=True, + commit_sha=input_commit_sha, + build_docker_repository=input_docker_build_repository, + docker_user=None, + docker_password=None, + test_container_folder=test_container_folder, + ) diff --git a/test/integration/v2/test_ci_export.py b/test/integration/v2/test_ci_export.py new file mode 100644 index 0000000..556c2d6 --- /dev/null +++ b/test/integration/v2/test_ci_export.py @@ -0,0 +1,14 @@ +from pathlib import Path +from tempfile import TemporaryDirectory + +from exasol.slc_ci.lib.ci_export import CIExport + + +def test(flavors_path): + flavor_name = "successful" + flavor_path = str(flavors_path / flavor_name) + with TemporaryDirectory() as temp_dir: + res = CIExport().export(flavor_path=(flavor_path,), output_directory=temp_dir) + assert res.exists() + assert res.name.endswith("tar.gz") + assert res.name.startswith(flavor_name) diff --git a/test/integration/v2/test_ci_prepare.py b/test/integration/v2/test_ci_prepare.py new file mode 100644 index 0000000..e46d5f9 --- /dev/null +++ b/test/integration/v2/test_ci_prepare.py @@ -0,0 +1,26 @@ +import os +from pathlib import Path + + +from exasol_integration_test_docker_environment.lib.logging import luigi_log_config + +from exasol.slc_ci.lib.ci_build import CIBuild +from exasol.slc_ci.lib.ci_prepare import CIPrepare + + +def test(flavors_path, test_containers_folder, mock_settings_env_vars): + test_type = "successful" + flavor_path = str(flavors_path / test_type) + test_container_folder = str(test_containers_folder / test_type) + CIPrepare().prepare() + CIBuild().build( + flavor_path=(flavor_path,), + rebuild=False, + commit_sha="COMMIT_SHA", + build_docker_repository="input_docker_build_repository", + docker_user=None, + docker_password=None, + test_container_folder=test_container_folder, + ) + log_path = Path(os.environ[luigi_log_config.LOG_ENV_VARIABLE_NAME]) + assert log_path.is_file() diff --git a/test/integration/v2/test_ci_push.py b/test/integration/v2/test_ci_push.py new file mode 100644 index 0000000..3d14421 --- /dev/null +++ b/test/integration/v2/test_ci_push.py @@ -0,0 +1,29 @@ +from exasol_integration_test_docker_environment.testing.docker_registry import ( + LocalDockerRegistryContextManager, +) + +from exasol.slc_ci.lib.ci_push import CIPush + + +def test(flavors_path): + flavor_name = "successful" + flavor_path = str(flavors_path / flavor_name) + with LocalDockerRegistryContextManager("test_ci_push") as registry: + CIPush().push( + flavor_path=(flavor_path,), + target_docker_repository=registry.name, + target_docker_tag_prefix="tag", + docker_user=None, + docker_password=None, + ) + expected_images = { + "name": "test_ci_push", + "tags": [ + f"tag_{flavor_name}-base_test_build_run_GUA7R5J3UM27WOHJSQPX2OJNSIEKWCM5YF5GJXKKXZI53LZPV75Q", + f"tag_{flavor_name}-flavor_test_build_run_G2OIMXJ2S3VS2EUAQNW4KWQLX3B2C27XYZ2SDMF7TQRS3UMAUWJQ", + f"tag_{flavor_name}-release_MNWZZGSSFQ6VCLBDH7CZBEZC4K35QQBSLOW5DSYHF3DFFDX2OOZQ", + ], + } + assert expected_images["name"] == registry.images["name"] and set( + expected_images["tags"] + ) == set(registry.images["tags"]) diff --git a/test/integration/v2/test_ci_security_scan.py b/test/integration/v2/test_ci_security_scan.py new file mode 100644 index 0000000..e25def0 --- /dev/null +++ b/test/integration/v2/test_ci_security_scan.py @@ -0,0 +1,34 @@ +from test.matchers import file_exists_matcher +from unittest.mock import call, create_autospec + +import pytest + +from exasol.slc_ci.lib.ci_security_scan import CISecurityScan +from exasol.slc_ci.lib.ci_step_output_printer import ( + CIStepOutputPrinterProtocol, +) + + +def test_successful_flavor(flavors_path, test_containers_folder): + flavor_path = str(flavors_path / "successful") + printer_mock = create_autospec(CIStepOutputPrinterProtocol) + CISecurityScan(printer=printer_mock).run_security_scan( + flavor_path=(flavor_path,), + ) + assert printer_mock.mock_calls == [ + call.print_file(file_exists_matcher()), + call.print_exasol_docker_images(), + ] + + +def test_failing_security_scan(flavors_path): + flavor_path = str(flavors_path / "failing_security_scan") + printer_mock = create_autospec(CIStepOutputPrinterProtocol) + with pytest.raises(AssertionError, match="Some security scans not successful."): + CISecurityScan(printer=printer_mock).run_security_scan( + flavor_path=(flavor_path,), + ) + assert printer_mock.mock_calls == [ + call.print_file(file_exists_matcher()), + call.print_exasol_docker_images(), + ] diff --git a/test/integration/v2/test_ci_test.py b/test/integration/v2/test_ci_test.py new file mode 100644 index 0000000..8df289b --- /dev/null +++ b/test/integration/v2/test_ci_test.py @@ -0,0 +1,27 @@ +from test.contract.v2.test_ci_test import ( + FailingRunDBTestFlavorDBTestsContract, + SuccessfulFlavorDBTestsContract, +) + +import pytest + +from exasol.slc_ci.lib.ci_test import ( + DBTestRunner, + DBTestRunnerProtocol, +) + + +class TestSuccessfulFlavorDBTestsContract(SuccessfulFlavorDBTestsContract): + + @pytest.fixture + def db_test_runner(self) -> DBTestRunnerProtocol: + return DBTestRunner() + + + +class TestFailingRunDBTestFlavorDBTestsContract(FailingRunDBTestFlavorDBTestsContract): + + @pytest.fixture + def db_test_runner(self) -> DBTestRunnerProtocol: + return DBTestRunner() + diff --git a/test/unit/v2/test_ci_test.py b/test/unit/v2/test_ci_test.py new file mode 100644 index 0000000..01fd4b5 --- /dev/null +++ b/test/unit/v2/test_ci_test.py @@ -0,0 +1,122 @@ +from contextlib import suppress +from pathlib import Path +from test.mock_cast import mock_cast +from typing import Union +from unittest.mock import MagicMock, call, create_autospec + +import pytest +from exasol.slc.models.test_result import AllTestsResult + +from exasol.slc_ci.lib.ci_step_output_printer import ( + CIStepOutputPrinterProtocol, +) +from exasol.slc_ci.lib.ci_test import ( + CIExecuteTest, + DBTestRunnerProtocol, +) + + +class BaseCIExecuteTest: + + @pytest.fixture + def db_test_runner(self) -> DBTestRunnerProtocol: + self.db_test_runner_mock = create_autospec(DBTestRunnerProtocol) + return self.db_test_runner_mock + + @pytest.fixture + def base_setup(self, db_test_runner): + self.printer_mock = create_autospec(CIStepOutputPrinterProtocol) + self.flavor_path = "test_flavor" + self.test_container_folder = "test_container_folder" + self.slc_path = Path.cwd() + self.test_folder = "test_folder" + self.ci_execute_test = CIExecuteTest( + printer=self.printer_mock, db_test_runner=db_test_runner + ) + + @staticmethod + def create_all_tests_result_mock(tests_are_ok: bool): + all_tests_result: Union[MagicMock, AllTestsResult] = create_autospec( + AllTestsResult + ) + all_tests_result.tests_are_ok = tests_are_ok + all_tests_result.command_line_output_path = create_autospec(Path) + return all_tests_result + + def execute_tests(self): + self.ci_execute_test.execute_tests( + flavor_path=(self.flavor_path,), + docker_user=None, + docker_password=None, + test_container_folder=self.test_container_folder, + slc_path=self.slc_path, + test_folder=self.test_folder, + ) + + @pytest.fixture() + def run_db_tests_calls(self): + return [ + call.run( + flavor_path=(self.flavor_path,), + test_folder=(self.test_folder,), + release_goal=("release",), + workers=7, + docker_username=None, + docker_password=None, + test_container_folder=self.test_container_folder, + use_existing_container=str(self.slc_path), + ), + ] + + +class TestSuccessfulFlavor(BaseCIExecuteTest): + + @pytest.fixture + def complete_setup(self, base_setup): + self.db_tests_all_tests_result = self.create_all_tests_result_mock( + tests_are_ok=True + ) + mock_cast(self.db_test_runner_mock.run).side_effect = [ + self.db_tests_all_tests_result, + ] + + def test_ci_step_output_printer_call(self, complete_setup): + self.execute_tests() + assert self.printer_mock.mock_calls == [ + call.print_file(self.db_tests_all_tests_result.command_line_output_path), + call.print_exasol_docker_images(), + ] + + def test_db_test_runner_calls(self, complete_setup, run_db_tests_calls): + self.execute_tests() + assert self.db_test_runner_mock.mock_calls == run_db_tests_calls + + +class TestFailingRunDBTestFlavor(BaseCIExecuteTest): + + @pytest.fixture() + def complete_setup(self, base_setup): + self.db_tests_all_tests_result = self.create_all_tests_result_mock( + tests_are_ok=False + ) + mock_cast(self.db_test_runner_mock.run).side_effect = [ + self.db_tests_all_tests_result, + ] + + @pytest.fixture + def run_suppress_exception(self, complete_setup): + with suppress(Exception): + self.execute_tests() + + def test_raises(self, complete_setup): + with pytest.raises(AssertionError, match="Not all tests are ok!"): + self.execute_tests() + + def test_ci_step_output_printer_call(self, run_suppress_exception): + assert self.printer_mock.mock_calls == [ + call.print_file(self.db_tests_all_tests_result.command_line_output_path), + call.print_exasol_docker_images(), + ] + + def test_db_test_runner_calls(self, run_suppress_exception, run_db_tests_calls): + assert self.db_test_runner_mock.mock_calls == run_db_tests_calls From 08b4f28e7064b1cac81b5f58592bf503d2847a94 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 29 Apr 2025 09:32:52 -0300 Subject: [PATCH 18/19] Fixed formatting, linter errors and type errors --- exasol/slc_ci/lib/ci_export.py | 6 +- exasol/slc_ci/lib/ci_test.py | 8 +-- exasol/slc_ci/lib/run_tests.py | 2 +- exasol/slc_ci/nox/arg_parser_builder.py | 3 +- exasol/slc_ci/nox/tasks.py | 2 +- test/conftest.py | 7 +-- test/contract/v2/test_ci_test.py | 24 +++++++- test/integration/fixtures.py | 2 +- test/integration/v2/test_ci_prepare.py | 1 - test/integration/v2/test_ci_security_scan.py | 4 +- test/integration/v2/test_ci_test.py | 7 +-- test/unit/v1/fixtures.py | 7 ++- test/unit/v2/test_ci_test.py | 9 +-- test/unit/v2/test_env.py | 17 +++--- .../test_export_and_scan_vulnerabilities.py | 55 ++++++++++++------- test/unit/v2/test_ignore_folders.py | 10 ++-- test/unit/v2/test_nox_tasks.py | 17 +++--- test/unit/v2/test_run_tests.py | 28 ++++++---- 18 files changed, 122 insertions(+), 87 deletions(-) diff --git a/exasol/slc_ci/lib/ci_export.py b/exasol/slc_ci/lib/ci_export.py index 60d9255..5ed7687 100644 --- a/exasol/slc_ci/lib/ci_export.py +++ b/exasol/slc_ci/lib/ci_export.py @@ -17,7 +17,9 @@ def __init__( ): self._printer = printer - def export(self, flavor_path: Tuple[str, ...], output_directory: str = ".build_output") -> Path: + def export( + self, flavor_path: Tuple[str, ...], output_directory: str = ".build_output" + ) -> Path: """ Export the flavor as tar.gz file. The returned path is the path of the tar.gz file. @@ -29,7 +31,7 @@ def export(self, flavor_path: Tuple[str, ...], output_directory: str = ".build_o workers=7, log_level="WARNING", use_job_specific_log_file=True, - output_directory=output_directory + output_directory=output_directory, ) self._printer.print_exasol_docker_images() export_infos = list(export_result.export_infos.values()) diff --git a/exasol/slc_ci/lib/ci_test.py b/exasol/slc_ci/lib/ci_test.py index 5954d2f..3ee9ceb 100644 --- a/exasol/slc_ci/lib/ci_test.py +++ b/exasol/slc_ci/lib/ci_test.py @@ -19,8 +19,8 @@ def run( test_folder: Tuple[str, ...], test_container_folder: str, workers: int, - docker_username: str, - docker_password: str, + docker_username: Optional[str], + docker_password: Optional[str], use_existing_container: Optional[str], ) -> AllTestsResult: raise NotImplementedError() @@ -34,8 +34,8 @@ def run( test_folder: Tuple[str, ...], test_container_folder: str, workers: int, - docker_username: str, - docker_password: str, + docker_username: Optional[str], + docker_password: Optional[str], use_existing_container: Optional[str], ) -> AllTestsResult: return run_db_test( diff --git a/exasol/slc_ci/lib/run_tests.py b/exasol/slc_ci/lib/run_tests.py index 13aabd9..fa8b71e 100644 --- a/exasol/slc_ci/lib/run_tests.py +++ b/exasol/slc_ci/lib/run_tests.py @@ -37,7 +37,7 @@ def run_tests( ) slc_file_path = slc_files[0] - flavor_path = (str( build_config.flavors_path / flavor),) + flavor_path = (str(build_config.flavors_path / flavor),) test_container_folder = build_config.test_container_folder ci_prepare.prepare() diff --git a/exasol/slc_ci/nox/arg_parser_builder.py b/exasol/slc_ci/nox/arg_parser_builder.py index 69f0b7f..d3c0cab 100644 --- a/exasol/slc_ci/nox/arg_parser_builder.py +++ b/exasol/slc_ci/nox/arg_parser_builder.py @@ -1,4 +1,5 @@ import argparse +from typing import List import nox @@ -6,7 +7,7 @@ class ArgumentParserBuilder: def __init__(self, session: nox.Session) -> None: self._session = session - self._usages = list() + self._usages: List[str] = list() self._parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) diff --git a/exasol/slc_ci/nox/tasks.py b/exasol/slc_ci/nox/tasks.py index e755540..dd05504 100644 --- a/exasol/slc_ci/nox/tasks.py +++ b/exasol/slc_ci/nox/tasks.py @@ -16,7 +16,7 @@ import nox from nox import Session -from slc_build_config import SLC_BUILD_CONFIG +from slc_build_config import SLC_BUILD_CONFIG #type: ignore # pylint: disable=(import-error), import exasol.slc_ci.api as api from exasol.slc_ci.nox.arg_parser_builder import ArgumentParserBuilder diff --git a/test/conftest.py b/test/conftest.py index 0464cad..db79644 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,17 +1,15 @@ import logging import os from tempfile import TemporaryDirectory +from test.integration.fixtures import * +from test.unit.v1.fixtures import * from unittest import mock from unittest.mock import MagicMock, patch - - from exasol_script_languages_container_ci.lib.config.data_model_generator import ( config_data_model_default_output_file, regenerate_config_data_model, ) -from test.unit.v1.fixtures import * -from test.integration.fixtures import * DISABLE_PYDANTIC_MODEL_GENERATION = "--disable-pydantic-model-generation" @@ -60,7 +58,6 @@ def tmp_test_dir(): os.chdir(old_dir) - @pytest.fixture() def git_access_mock(): """ diff --git a/test/contract/v2/test_ci_test.py b/test/contract/v2/test_ci_test.py index 20cfb75..5deb52b 100644 --- a/test/contract/v2/test_ci_test.py +++ b/test/contract/v2/test_ci_test.py @@ -5,12 +5,15 @@ from exasol.slc_ci.lib.ci_export import CIExport from exasol.slc_ci.lib.ci_test import DBTestRunnerProtocol + class CreateExistingContainer: @pytest.fixture def use_existing_container(self, tmp_path, flavor_path) -> Path: ci_export = CIExport() - container_path = ci_export.export(flavor_path=(flavor_path,), output_directory=str(tmp_path)) + container_path = ci_export.export( + flavor_path=(flavor_path,), output_directory=str(tmp_path) + ) return container_path @@ -35,7 +38,14 @@ class SuccessfulFlavorDBTestsContract(SuccessfulFlavorContract): def db_test_runner(self) -> DBTestRunnerProtocol: raise NotImplementedError() - def test(self, db_test_runner: DBTestRunnerProtocol, test_container: str, flavor_path: str, use_existing_container: Path, test_folder: str): + def test( + self, + db_test_runner: DBTestRunnerProtocol, + test_container: str, + flavor_path: str, + use_existing_container: Path, + test_folder: str, + ): result = db_test_runner.run( flavor_path=(flavor_path,), test_folder=(test_folder,), @@ -63,13 +73,21 @@ def test_container(self, test_containers_folder): def test_folder(self): return "failing_test" + class FailingRunDBTestFlavorDBTestsContract(FailingRunDBTestFlavorContract): @pytest.fixture def db_test_runner(self) -> DBTestRunnerProtocol: raise NotImplementedError() - def test(self, db_test_runner: DBTestRunnerProtocol, test_container: str, flavor_path: str, use_existing_container: Path, test_folder: str): + def test( + self, + db_test_runner: DBTestRunnerProtocol, + test_container: str, + flavor_path: str, + use_existing_container: Path, + test_folder: str, + ): result = db_test_runner.run( flavor_path=(flavor_path,), test_folder=(test_folder,), diff --git a/test/integration/fixtures.py b/test/integration/fixtures.py index e349e23..6b0c404 100644 --- a/test/integration/fixtures.py +++ b/test/integration/fixtures.py @@ -4,6 +4,7 @@ script_path = Path(__file__).absolute().parent + @pytest.fixture def resources_path() -> Path: return script_path / "resources" @@ -17,4 +18,3 @@ def flavors_path(resources_path: Path) -> Path: @pytest.fixture def test_containers_folder(resources_path: Path) -> Path: return resources_path / "test_containers" - diff --git a/test/integration/v2/test_ci_prepare.py b/test/integration/v2/test_ci_prepare.py index e46d5f9..e02c120 100644 --- a/test/integration/v2/test_ci_prepare.py +++ b/test/integration/v2/test_ci_prepare.py @@ -1,7 +1,6 @@ import os from pathlib import Path - from exasol_integration_test_docker_environment.lib.logging import luigi_log_config from exasol.slc_ci.lib.ci_build import CIBuild diff --git a/test/integration/v2/test_ci_security_scan.py b/test/integration/v2/test_ci_security_scan.py index e25def0..79fab69 100644 --- a/test/integration/v2/test_ci_security_scan.py +++ b/test/integration/v2/test_ci_security_scan.py @@ -4,9 +4,7 @@ import pytest from exasol.slc_ci.lib.ci_security_scan import CISecurityScan -from exasol.slc_ci.lib.ci_step_output_printer import ( - CIStepOutputPrinterProtocol, -) +from exasol.slc_ci.lib.ci_step_output_printer import CIStepOutputPrinterProtocol def test_successful_flavor(flavors_path, test_containers_folder): diff --git a/test/integration/v2/test_ci_test.py b/test/integration/v2/test_ci_test.py index 8df289b..3b56975 100644 --- a/test/integration/v2/test_ci_test.py +++ b/test/integration/v2/test_ci_test.py @@ -5,10 +5,7 @@ import pytest -from exasol.slc_ci.lib.ci_test import ( - DBTestRunner, - DBTestRunnerProtocol, -) +from exasol.slc_ci.lib.ci_test import DBTestRunner, DBTestRunnerProtocol class TestSuccessfulFlavorDBTestsContract(SuccessfulFlavorDBTestsContract): @@ -18,10 +15,8 @@ def db_test_runner(self) -> DBTestRunnerProtocol: return DBTestRunner() - class TestFailingRunDBTestFlavorDBTestsContract(FailingRunDBTestFlavorDBTestsContract): @pytest.fixture def db_test_runner(self) -> DBTestRunnerProtocol: return DBTestRunner() - diff --git a/test/unit/v1/fixtures.py b/test/unit/v1/fixtures.py index 1807456..f4c5824 100644 --- a/test/unit/v1/fixtures.py +++ b/test/unit/v1/fixtures.py @@ -2,7 +2,12 @@ import pytest -from exasol_script_languages_container_ci.lib.config.config_data_model import Config, Build, Ignore, Release +from exasol_script_languages_container_ci.lib.config.config_data_model import ( + Build, + Config, + Ignore, + Release, +) @pytest.fixture diff --git a/test/unit/v2/test_ci_test.py b/test/unit/v2/test_ci_test.py index 01fd4b5..e44511f 100644 --- a/test/unit/v2/test_ci_test.py +++ b/test/unit/v2/test_ci_test.py @@ -7,13 +7,8 @@ import pytest from exasol.slc.models.test_result import AllTestsResult -from exasol.slc_ci.lib.ci_step_output_printer import ( - CIStepOutputPrinterProtocol, -) -from exasol.slc_ci.lib.ci_test import ( - CIExecuteTest, - DBTestRunnerProtocol, -) +from exasol.slc_ci.lib.ci_step_output_printer import CIStepOutputPrinterProtocol +from exasol.slc_ci.lib.ci_test import CIExecuteTest, DBTestRunnerProtocol class BaseCIExecuteTest: diff --git a/test/unit/v2/test_env.py b/test/unit/v2/test_env.py index 103ac04..c2c730e 100644 --- a/test/unit/v2/test_env.py +++ b/test/unit/v2/test_env.py @@ -1,6 +1,7 @@ -from exasol.slc_ci.model.flavor_ci_model import FlavorCiConfig, TestConfig, TestSet from test.unit.v2.resources.slc_build_config import SLC_BUILD_CONFIG +from exasol.slc_ci.model.flavor_ci_model import FlavorCiConfig, TestConfig, TestSet + class TestEnv: docker_user = "test_docker_user" @@ -9,14 +10,12 @@ class TestEnv: branch_name = "test_branch_name" build_config = SLC_BUILD_CONFIG flavor_config = FlavorCiConfig( - build_runner="some_build_runner", - test_config=TestConfig( - test_runner="some_test_runner", - test_sets=[TestSet(name="some_test_name", folders=["all", "specific"])], - ), -) - - + build_runner="some_build_runner", + test_config=TestConfig( + test_runner="some_test_runner", + test_sets=[TestSet(name="some_test_name", folders=["all", "specific"])], + ), + ) test_env = TestEnv() diff --git a/test/unit/v2/test_export_and_scan_vulnerabilities.py b/test/unit/v2/test_export_and_scan_vulnerabilities.py index d9620ba..9129cc9 100644 --- a/test/unit/v2/test_export_and_scan_vulnerabilities.py +++ b/test/unit/v2/test_export_and_scan_vulnerabilities.py @@ -1,15 +1,16 @@ from pathlib import Path - -from exasol.slc_ci.lib.ci_prepare import CIPrepare -from exasol.slc_ci.lib.export_and_scan_vulnerabilities import export_and_scan_vulnerabilities -from exasol.slc_ci.model.flavor_ci_model import FlavorCiConfig, TestConfig, TestSet +from test.unit.v2.test_env import test_env from typing import Union -from unittest.mock import Mock, call, MagicMock +from unittest.mock import MagicMock, Mock, call import pytest +from exasol.slc_ci.lib.ci_prepare import CIPrepare +from exasol.slc_ci.lib.export_and_scan_vulnerabilities import ( + export_and_scan_vulnerabilities, +) +from exasol.slc_ci.model.flavor_ci_model import FlavorCiConfig, TestConfig, TestSet from exasol_script_languages_container_ci.lib.ci_test import CIExecuteTest -from test.unit.v2.test_env import test_env TEST_FLAVOR = "flavor_xyz" @@ -21,19 +22,19 @@ ), ) + @pytest.fixture def slc_directory(tmp_path: Path) -> Path: - with open(str(tmp_path / f'{TEST_FLAVOR}-dummy_slc.tar.gz'), 'w') as f: + with open(str(tmp_path / f"{TEST_FLAVOR}-dummy_slc.tar.gz"), "w") as f: f.write("nothing") return tmp_path - def test_export_and_scan_vulnerabilities(slc_directory, git_access_mock): res_slc_path = Path("/some_path/slc.tar.gz") ci_export_mock = MagicMock() ci_export_mock.export = MagicMock(return_value=res_slc_path) - ci_commands_mock: Union[CIExecuteTest,CIPrepare,Mock] = Mock() + ci_commands_mock: Union[CIExecuteTest, CIPrepare, Mock] = Mock() result = export_and_scan_vulnerabilities( flavor=TEST_FLAVOR, @@ -52,16 +53,32 @@ def test_export_and_scan_vulnerabilities(slc_directory, git_access_mock): expected_flavor_path = str(test_env.build_config.flavors_path / TEST_FLAVOR) assert ci_commands_mock.mock_calls == [ call.prepare(), - call.build(flavor_path=(expected_flavor_path,), rebuild=False, - build_docker_repository=test_env.build_config.docker_build_repository, commit_sha=test_env.commit_sha, - docker_user=test_env.docker_user, docker_password=test_env.docker_pwd, - test_container_folder=test_env.build_config.test_container_folder,), + call.build( + flavor_path=(expected_flavor_path,), + rebuild=False, + build_docker_repository=test_env.build_config.docker_build_repository, + commit_sha=test_env.commit_sha, + docker_user=test_env.docker_user, + docker_password=test_env.docker_pwd, + test_container_folder=test_env.build_config.test_container_folder, + ), call.run_security_scan(flavor_path=(expected_flavor_path,)), - call.push(flavor_path=(expected_flavor_path,), target_docker_repository=test_env.build_config.docker_build_repository, - target_docker_tag_prefix=test_env.commit_sha, docker_user=test_env.docker_user, - docker_password=test_env.docker_pwd), - call.push(flavor_path=(expected_flavor_path,), target_docker_repository=test_env.build_config.docker_build_repository, - target_docker_tag_prefix='', docker_user=test_env.docker_user, docker_password=test_env.docker_pwd), + call.push( + flavor_path=(expected_flavor_path,), + target_docker_repository=test_env.build_config.docker_build_repository, + target_docker_tag_prefix=test_env.commit_sha, + docker_user=test_env.docker_user, + docker_password=test_env.docker_pwd, + ), + call.push( + flavor_path=(expected_flavor_path,), + target_docker_repository=test_env.build_config.docker_build_repository, + target_docker_tag_prefix="", + docker_user=test_env.docker_user, + docker_password=test_env.docker_pwd, + ), + ] + assert ci_export_mock.mock_calls == [ + call.export(flavor_path=(expected_flavor_path,)) ] - assert ci_export_mock.mock_calls == [call.export(flavor_path=(expected_flavor_path,))] assert result == res_slc_path diff --git a/test/unit/v2/test_ignore_folders.py b/test/unit/v2/test_ignore_folders.py index 2542508..0397dfd 100644 --- a/test/unit/v2/test_ignore_folders.py +++ b/test/unit/v2/test_ignore_folders.py @@ -1,13 +1,12 @@ from pathlib import Path +from test.unit.v2.test_env import test_env from typing import List import git import pytest from exasol.slc_ci.lib.check_if_build_needed import check_if_need_to_build - -from exasol_script_languages_container_ci.lib.git_access import GitAccess -from test.unit.v2.test_env import test_env +from exasol.slc_ci.lib.git_access import GitAccess def commit_base(repo: git.Repo, repo_path: Path) -> None: @@ -44,7 +43,6 @@ def commit_files( repo.index.commit(commit_message) - TEST_FLAVOR = "flavor_xyz" TEST_DATA = [ @@ -152,6 +150,8 @@ def test_ignore_folder_should_run_ci( tmp_repo = git.Repo.init(repo_path) commit_files(branch_name, tmp_repo, repo_path, files_to_commit, commit_message) assert ( - check_if_need_to_build(branch_name, test_env.build_config, TEST_FLAVOR, GitAccess()) + check_if_need_to_build( + branch_name, test_env.build_config, TEST_FLAVOR, GitAccess() + ) == expected_result ) diff --git a/test/unit/v2/test_nox_tasks.py b/test/unit/v2/test_nox_tasks.py index 77bc306..38773b4 100644 --- a/test/unit/v2/test_nox_tasks.py +++ b/test/unit/v2/test_nox_tasks.py @@ -1,6 +1,7 @@ import argparse import sys from pathlib import Path +from test.unit.v2.test_env import test_env from unittest import mock from unittest.mock import MagicMock @@ -9,7 +10,6 @@ from _pytest.monkeypatch import MonkeyPatch import exasol.slc_ci.api as api -from test.unit.v2.test_env import test_env FLAVORS = ["flavor_a", "flavor_b"] @@ -17,11 +17,12 @@ @pytest.fixture(scope="module") def nox_tasks(): resources_path = Path(__file__).parent / "resources" - #exasol.slc_ci.nox.tasks imports 'slc_build_config.py`. We must ensure that this modul can be found by adding it to the sys path. + # exasol.slc_ci.nox.tasks imports 'slc_build_config.py`. We must ensure that this modul can be found by adding it to the sys path. sys.path.append(str(resources_path)) import exasol.slc_ci.nox.tasks as nox_tasks + yield nox_tasks - #now remove 'slc_build_config.py` again from sys path. + # now remove 'slc_build_config.py` again from sys path. sys.path.pop() @@ -66,10 +67,10 @@ def fake_session(session_name: str, *args): global_config = argparse.Namespace(posargs=args) session_runner = nox.sessions.SessionRunner( name=session_name, - signatures=None, - func=None, + signatures=[], + func=None, #type: ignore global_config=global_config, - manifest=None, + manifest=None, #type: ignore ) return nox.sessions.Session(session_runner) @@ -203,7 +204,9 @@ def test_run_check_if_build_needed( nox_tasks.run_check_if_build_needed(fake_session) assert mock_check_if_build_needed.call_count == 1 assert mock_check_if_build_needed.call_args == mock.call( - branch_name="feature/abc", flavor="flavor_a", build_config=nox_tasks.SLC_BUILD_CONFIG + branch_name="feature/abc", + flavor="flavor_a", + build_config=nox_tasks.SLC_BUILD_CONFIG, ) out_content = github_output.read_text() assert out_content == """test=True\n""" diff --git a/test/unit/v2/test_run_tests.py b/test/unit/v2/test_run_tests.py index 7a99bf3..1dd94c7 100644 --- a/test/unit/v2/test_run_tests.py +++ b/test/unit/v2/test_run_tests.py @@ -1,28 +1,27 @@ from pathlib import Path - -from exasol.slc_ci.lib.ci_prepare import CIPrepare -from exasol.slc_ci.lib.run_tests import run_tests +from test.unit.v2.test_env import test_env from typing import Union from unittest.mock import Mock, call import pytest +from exasol.slc_ci.lib.ci_prepare import CIPrepare +from exasol.slc_ci.lib.run_tests import run_tests from exasol_script_languages_container_ci.lib.ci_test import CIExecuteTest -from test.unit.v2.test_env import test_env - TEST_FLAVOR = "flavor_xyz" + @pytest.fixture def slc_directory(tmp_path: Path) -> Path: - with open(str(tmp_path / f'{TEST_FLAVOR}-dummy_slc.tar.gz'), 'w') as f: + with open(str(tmp_path / f"{TEST_FLAVOR}-dummy_slc.tar.gz"), "w") as f: f.write("nothing") return tmp_path def run_db_test_call(slc_path: Path, test_folder: str): return call.execute_tests( - flavor_path=( str(test_env.build_config.flavors_path / TEST_FLAVOR),), + flavor_path=(str(test_env.build_config.flavors_path / TEST_FLAVOR),), docker_user=test_env.docker_user, docker_password=test_env.docker_pwd, test_container_folder=test_env.build_config.test_container_folder, @@ -30,8 +29,9 @@ def run_db_test_call(slc_path: Path, test_folder: str): test_folder=test_folder, ) + def test_run_tests(slc_directory): - ci_commands_mock: Union[CIExecuteTest,CIPrepare,Mock] = Mock() + ci_commands_mock: Union[CIExecuteTest, CIPrepare, Mock] = Mock() run_tests( flavor=TEST_FLAVOR, @@ -42,10 +42,16 @@ def test_run_tests(slc_directory): docker_user=test_env.docker_user, docker_password=test_env.docker_pwd, ci_prepare=ci_commands_mock, - ci_test=ci_commands_mock + ci_test=ci_commands_mock, ) assert ci_commands_mock.mock_calls == [ call.prepare(), - run_db_test_call(slc_path=slc_directory / f"{TEST_FLAVOR}-dummy_slc.tar.gz", test_folder='all'), - run_db_test_call(slc_path=slc_directory / f"{TEST_FLAVOR}-dummy_slc.tar.gz", test_folder='specific'), + run_db_test_call( + slc_path=slc_directory / f"{TEST_FLAVOR}-dummy_slc.tar.gz", + test_folder="all", + ), + run_db_test_call( + slc_path=slc_directory / f"{TEST_FLAVOR}-dummy_slc.tar.gz", + test_folder="specific", + ), ] From 57e264b9ea23806e22c5548452b41e2a04551e17 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 29 Apr 2025 09:35:42 -0300 Subject: [PATCH 19/19] Fixed formatting --- exasol/slc_ci/nox/tasks.py | 4 +++- test/unit/v2/test_nox_tasks.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/exasol/slc_ci/nox/tasks.py b/exasol/slc_ci/nox/tasks.py index dd05504..f31302e 100644 --- a/exasol/slc_ci/nox/tasks.py +++ b/exasol/slc_ci/nox/tasks.py @@ -16,7 +16,9 @@ import nox from nox import Session -from slc_build_config import SLC_BUILD_CONFIG #type: ignore # pylint: disable=(import-error), +from slc_build_config import ( # type: ignore # pylint: disable=(import-error), + SLC_BUILD_CONFIG, +) import exasol.slc_ci.api as api from exasol.slc_ci.nox.arg_parser_builder import ArgumentParserBuilder diff --git a/test/unit/v2/test_nox_tasks.py b/test/unit/v2/test_nox_tasks.py index 38773b4..d0b4b99 100644 --- a/test/unit/v2/test_nox_tasks.py +++ b/test/unit/v2/test_nox_tasks.py @@ -68,9 +68,9 @@ def fake_session(session_name: str, *args): session_runner = nox.sessions.SessionRunner( name=session_name, signatures=[], - func=None, #type: ignore + func=None, # type: ignore global_config=global_config, - manifest=None, #type: ignore + manifest=None, # type: ignore ) return nox.sessions.Session(session_runner)