Feature/75 implement ci for gh workflows

.github/workflows/checks.yml

@@ -20,7 +20,7 @@ jobs:
 
       - name: Check Version(s)
         run: |
-          poetry run -- version-check exasol_script_languages_container_ci/version.py
+          poetry run -- version-check exasol/slc_ci/version.py
 
 
   build-matrix:

exasol/slc_ci/api/__init__.py

@@ -0,0 +1,5 @@
+from .check_if_build_needed import check_if_build_needed
+from .export_and_scan_vulnerabilities import export_and_scan_vulnerabilities
+from .get_flavor_ci_model import get_flavor_ci_model
+from .get_flavors import get_flavors
+from .run_tests import run_tests

exasol/slc_ci/api/check_if_build_needed.py

@@ -0,0 +1,18 @@
+from exasol.slc_ci.lib.check_if_build_needed import (
+    check_if_need_to_build as lib_check_if_need_to_build,
+)
+from exasol.slc_ci.lib.git_access import GitAccess
+from exasol.slc_ci.model.build_config import BuildConfig
+
+
+def check_if_build_needed(
+    flavor: str, build_config: BuildConfig, branch_name: str
+) -> bool:
+
+    git_access: GitAccess = GitAccess()
+    return lib_check_if_need_to_build(
+        branch_name=branch_name,
+        build_config=build_config,
+        flavor=flavor,
+        git_access=git_access,
+    )

exasol/slc_ci/api/export_and_scan_vulnerabilities.py

@@ -0,0 +1,43 @@
+from pathlib import Path
+
+from exasol.slc_ci.lib.ci_build import CIBuild
+from exasol.slc_ci.lib.ci_export import CIExport
+from exasol.slc_ci.lib.ci_prepare import CIPrepare
+from exasol.slc_ci.lib.ci_push import CIPush
+from exasol.slc_ci.lib.ci_security_scan import CISecurityScan
+from exasol.slc_ci.lib.export_and_scan_vulnerabilities import (
+    export_and_scan_vulnerabilities as lib_export_and_scan_vulnerabilities,
+)
+from exasol.slc_ci.lib.git_access import GitAccess
+from exasol.slc_ci.model.build_config import BuildConfig
+
+
+def export_and_scan_vulnerabilities(
+    flavor: str,
+    build_config: BuildConfig,
+    branch_name: str,
+    docker_user: str,
+    docker_password: str,
+    commit_sha: str,
+) -> Path:
+    git_access: GitAccess = GitAccess()
+    ci_build: CIBuild = CIBuild()
+    ci_security_scan: CISecurityScan = CISecurityScan()
+    ci_prepare: CIPrepare = CIPrepare()
+    ci_export: CIExport = CIExport()
+    ci_push: CIPush = CIPush()
+
+    return lib_export_and_scan_vulnerabilities(
+        flavor=flavor,
+        branch_name=branch_name,
+        docker_user=docker_user,
+        docker_password=docker_password,
+        commit_sha=commit_sha,
+        build_config=build_config,
+        git_access=git_access,
+        ci_build=ci_build,
+        ci_security_scan=ci_security_scan,
+        ci_prepare=ci_prepare,
+        ci_export=ci_export,
+        ci_push=ci_push,
+    )

exasol/slc_ci/api/get_flavor_ci_model.py

@@ -0,0 +1,9 @@
+from exasol.slc_ci.model.build_config import BuildConfig
+from exasol.slc_ci.model.flavor_ci_model import FlavorCiConfig
+
+
+def get_flavor_ci_model(build_config: BuildConfig, flavor: str) -> FlavorCiConfig:
+    flavor_ci_config_path = build_config.flavors_path / flavor / "ci.json"
+    return FlavorCiConfig.model_validate_json(
+        flavor_ci_config_path.read_text(), strict=True
+    )

exasol/slc_ci/api/get_flavors.py

@@ -0,0 +1,10 @@
+from pathlib import Path
+from typing import Iterable
+
+
+def get_flavors(flavors_path: Path) -> Iterable[str]:
+    if not flavors_path.exists():
+        raise ValueError(f"{flavors_path} does not exist")
+    for p in flavors_path.iterdir():
+        if p.is_dir():
+            yield p.name

exasol/slc_ci/api/run_tests.py

@@ -0,0 +1,32 @@
+from pathlib import Path
+
+from exasol.slc_ci.lib.ci_prepare import CIPrepare
+from exasol.slc_ci.lib.ci_test import CIExecuteTest
+from exasol.slc_ci.lib.run_tests import run_tests as lib_run_tests
+from exasol.slc_ci.model.build_config import BuildConfig
+from exasol.slc_ci.model.flavor_ci_model import FlavorCiConfig
+
+
+def run_tests(
+    flavor: str,
+    slc_directory: str,
+    flavor_config: FlavorCiConfig,
+    build_config: BuildConfig,
+    test_set_name: str,
+    docker_user: str,
+    docker_password: str,
+) -> None:
+    ci_test = CIExecuteTest()
+    ci_prepare = CIPrepare()
+
+    return lib_run_tests(
+        flavor=flavor,
+        slc_directory=slc_directory,
+        flavor_config=flavor_config,
+        build_config=build_config,
+        test_set_name=test_set_name,
+        docker_user=docker_user,
+        docker_password=docker_password,
+        ci_prepare=ci_prepare,
+        ci_test=ci_test,
+    )

exasol/slc_ci/lib/branch_config.py

@@ -0,0 +1,60 @@
+import re
+from enum import Enum, auto
+
+
+class BuildSteps(Enum):
+    BUILD_ALL_ALWAYS = auto()
+    REBUILD = auto()
+    PUSH_TO_DOCKER_RELEASE_REPO = auto()
+
+
+class BranchConfig(Enum):
+    DEVELOP = {
+        BuildSteps.BUILD_ALL_ALWAYS: True,
+        BuildSteps.REBUILD: True,
+        BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: False,
+    }
+    MAIN = {
+        BuildSteps.BUILD_ALL_ALWAYS: True,
+        BuildSteps.REBUILD: True,
+        BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: True,
+    }
+    REBUILD = {
+        BuildSteps.BUILD_ALL_ALWAYS: True,
+        BuildSteps.REBUILD: True,
+        BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: False,
+    }
+    OTHER = {
+        BuildSteps.BUILD_ALL_ALWAYS: False,
+        BuildSteps.REBUILD: False,
+        BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO: False,
+    }
+
+    @staticmethod
+    def build_always(branch_name: str) -> bool:
+        return get_branch_config(branch_name).value[BuildSteps.BUILD_ALL_ALWAYS]
+
+    @staticmethod
+    def rebuild(branch_name) -> bool:
+        return get_branch_config(branch_name).value[BuildSteps.REBUILD]
+
+    @staticmethod
+    def push_to_docker_release_repo(branch_name: str) -> bool:
+        return get_branch_config(branch_name).value[
+            BuildSteps.PUSH_TO_DOCKER_RELEASE_REPO
+        ]
+
+
+def get_branch_config(branch_name: str) -> BranchConfig:
+    matches = (
+        (re.compile(r"refs/heads/(master|main)"), BranchConfig.MAIN),
+        (re.compile(r"refs/heads/develop"), BranchConfig.DEVELOP),
+        (re.compile(r"refs/heads/rebuild/.*"), BranchConfig.REBUILD),
+    )
+
+    branch_cfg = BranchConfig.OTHER
+    for branch_regex, branch_config in matches:
+        if branch_regex.match(branch_name):
+            branch_cfg = branch_config
+            break
+    return branch_cfg

exasol/slc_ci/lib/check_if_build_needed.py

@@ -0,0 +1,46 @@
+import logging
+from typing import Set
+
+from exasol.slc_ci.lib.branch_config import BranchConfig
+from exasol.slc_ci.lib.git_access import GitAccess
+from exasol.slc_ci.model.build_config import BuildConfig
+
+
+def get_all_affected_files(git_access: GitAccess, base_branch: str) -> Set[str]:
+    base_last_commit_sha = git_access.get_head_commit_sha_of_branch(base_branch)
+    changed_files = set()  # type: ignore
+    for commit in git_access.get_last_commits():
+        if commit == base_last_commit_sha:
+            break
+        changed_files.update(git_access.get_files_of_commit(commit))
+    return changed_files
+
+
+def check_if_need_to_build(
+    branch_name: str, build_config: BuildConfig, flavor: str, git_access: GitAccess
+) -> bool:
+    if BranchConfig.build_always(branch_name):
+        return True
+    if "[rebuild]" in git_access.get_last_commit_message():
+        return True
+    affected_files = list(get_all_affected_files(git_access, build_config.base_branch))
+    logging.debug(
+        f"check_if_need_to_build: Found files of last commits: {affected_files}"
+    )
+    for ignore_path in build_config.ignore_paths:
+        affected_files = list(
+            filter(lambda file: not file.startswith(ignore_path), affected_files)
+        )
+
+    if len(affected_files) > 0:
+        # Now filter out also other flavor folders
+        this_flavor_path = f"flavors/{flavor}"
+        affected_files = list(
+            filter(
+                lambda file: not file.startswith("flavors")
+                or file.startswith(this_flavor_path),
+                affected_files,
+            )
+        )
+    logging.debug(f"check_if_need_to_build: filtered files: {affected_files}")
+    return len(affected_files) > 0

exasol/slc_ci/lib/ci_build.py

@@ -0,0 +1,59 @@
+import logging
+from typing import Optional, Tuple
+
+from exasol.slc.api import build
+
+from exasol.slc_ci.lib.ci_step_output_printer import (
+    CIStepOutputPrinter,
+    CIStepOutputPrinterProtocol,
+)
+
+
+class CIBuild:
+
+    def __init__(
+        self, printer: CIStepOutputPrinterProtocol = CIStepOutputPrinter(logging.info)
+    ):
+        self._printer = printer
+
+    def build(
+        self,
+        flavor_path: Tuple[str, ...],
+        rebuild: bool,
+        build_docker_repository: Optional[str],
+        commit_sha: str,
+        docker_user: str,
+        docker_password: str,
+        test_container_folder: str,
+    ):
+        """
+        Build the script-language container for given flavor. And also build the test container
+        """
+
+        logging.info(f"Running command 'build' with parameters: {locals()}")
+        if build_docker_repository is None:
+            slc_image_infos = build(
+                flavor_path=flavor_path,
+                force_rebuild=rebuild,
+                source_docker_tag_prefix=commit_sha,
+                source_docker_username=docker_user,
+                source_docker_password=docker_password,
+                shortcut_build=False,
+                workers=7,
+                log_level="WARNING",
+                use_job_specific_log_file=True,
+            )
+        else:
+            slc_image_infos = build(
+                flavor_path=flavor_path,
+                force_rebuild=rebuild,
+                source_docker_repository_name=build_docker_repository,
+                source_docker_tag_prefix=commit_sha,
+                source_docker_username=docker_user,
+                source_docker_password=docker_password,
+                shortcut_build=False,
+                workers=7,
+                log_level="WARNING",
+                use_job_specific_log_file=True,
+            )
+        self._printer.print_exasol_docker_images()

exasol/slc_ci/lib/ci_export.py

@@ -0,0 +1,46 @@
+import logging
+from pathlib import Path
+from typing import Tuple
+
+from exasol.slc.api import export
+
+from exasol.slc_ci.lib.ci_step_output_printer import (
+    CIStepOutputPrinter,
+    CIStepOutputPrinterProtocol,
+)
+
+
+class CIExport:
+
+    def __init__(
+        self, printer: CIStepOutputPrinterProtocol = CIStepOutputPrinter(logging.info)
+    ):
+        self._printer = printer
+
+    def export(
+        self, flavor_path: Tuple[str, ...], output_directory: str = ".build_output"
+    ) -> Path:
+        """
+        Export the flavor as tar.gz file.
+        The returned path is the path of the tar.gz file.
+        """
+
+        logging.info(f"Running command 'export' with parameters: {locals()}")
+        export_result = export(
+            flavor_path=flavor_path,
+            workers=7,
+            log_level="WARNING",
+            use_job_specific_log_file=True,
+            output_directory=output_directory,
+        )
+        self._printer.print_exasol_docker_images()
+        export_infos = list(export_result.export_infos.values())
+        if len(export_infos) != 1:
+            raise RuntimeError(f"Unexpected number of export infos")
+        export_info = export_infos[0]
+        export_flavor_infos = list(export_info.values())
+        if len(export_flavor_infos) != 1:
+            raise RuntimeError(f"Unexpected number of export flavor infos")
+
+        export_flavor_info = export_flavor_infos[0]
+        return Path(export_flavor_info.cache_file)

exasol/slc_ci/lib/ci_prepare.py

@@ -0,0 +1,15 @@
+import os
+from pathlib import Path
+
+from exasol_integration_test_docker_environment.cli.options.system_options import (
+    DEFAULT_OUTPUT_DIRECTORY,
+)
+from exasol_integration_test_docker_environment.lib.logging import luigi_log_config
+
+
+class CIPrepare:
+
+    def prepare(self):
+        log_path = Path(DEFAULT_OUTPUT_DIRECTORY) / "jobs" / "logs" / "main.log"
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        os.environ[luigi_log_config.LOG_ENV_VARIABLE_NAME] = f"{log_path.absolute()}"