From 6bee2483d490e515449396e42e55424496cb83aa Mon Sep 17 00:00:00 2001 From: Jonas Scharpf Date: Thu, 13 Feb 2025 14:30:06 +0100 Subject: [PATCH] Use builders jobs in build-cmk-image ... this was the last job using artifacts from the tstbuilds server and now uses the latest package built for the given commit CMK-21170 Change-Id: I97d4426bd1cce644c0c0ce59bc15e074e7c19015 --- buildscripts/scripts/build-cmk-image.groovy | 146 ++++++++++++++++---- 1 file changed, 119 insertions(+), 27 deletions(-) diff --git a/buildscripts/scripts/build-cmk-image.groovy b/buildscripts/scripts/build-cmk-image.groovy index 4701bc74606..75942d9a2d2 100644 --- a/buildscripts/scripts/build-cmk-image.groovy +++ b/buildscripts/scripts/build-cmk-image.groovy @@ -19,6 +19,8 @@ def main() { "PUSH_TO_REGISTRY_ONLY", "BUILD_IMAGE_WITHOUT_CACHE", "CUSTOM_CMK_BASE_IMAGE", + "DISABLE_CACHE", + "FAKE_WINDOWS_ARTIFACTS", ]); check_environment_variables([ @@ -29,23 +31,30 @@ def main() { "NODE_NAME", ]); - shout("load libaries"); - def versioning = load("${checkout_dir}/buildscripts/scripts/utils/versioning.groovy"); def artifacts_helper = load("${checkout_dir}/buildscripts/scripts/utils/upload_artifacts.groovy"); def test_helper = load("${checkout_dir}/buildscripts/scripts/utils/test_helper.groovy"); + def package_helper = load("${checkout_dir}/buildscripts/scripts/utils/package_helper.groovy"); - def package_dir = "${checkout_dir}/download"; + /// This will get us the location to e.g. "checkmk/master" or "Testing//checkmk/master" + def branch_base_folder = package_helper.branch_base_folder(with_testing_prefix: true); + + def safe_branch_name = versioning.safe_branch_name(scm); def branch_version = versioning.get_branch_version(checkout_dir); // When building from a git tag (VERSION != "daily"), we cannot get the branch name from the scm so used defines.make instead. // this is save on master as there are no tags/versions built other than daily - def branch_name = (VERSION == "daily") ? versioning.safe_branch_name(scm) : branch_version; + def branch_name = (VERSION == "daily") ? safe_branch_name : branch_version; def cmk_version_rc_aware = versioning.get_cmk_version(branch_name, branch_version, VERSION); def cmk_version = versioning.strip_rc_number_from_version(cmk_version_rc_aware); + + def package_dir = "${checkout_dir}/download"; def source_dir = package_dir + "/" + cmk_version_rc_aware; def push_to_registry = PUSH_TO_REGISTRY=='true'; def build_image = PUSH_TO_REGISTRY_ONLY!='true'; + def fake_windows_artifacts = params.FAKE_WINDOWS_ARTIFACTS; + + def relative_job_name = "${branch_base_folder}/builders/build-cmk-distro-package"; print( """ @@ -58,15 +67,116 @@ def main() { |push_to_registry:.... │${push_to_registry}│ |build_image:......... │${build_image}│ |package_dir:......... │${package_dir}│ + |branch_base_folder:.. │${branch_base_folder}│ |=================================================== """.stripMargin()); - currentBuild.description += ( - """ - |Building the CMK docker image - """.stripMargin()); + smart_stage(name: 'Prepare package directory', condition: build_image) { + inside_container() { + dir("${checkout_dir}") { + cleanup_directory("${package_dir}"); + } + } + } + + /// In order to ensure a fixed order for stages executed in parallel, + /// we wait an increasing amount of time (N * 100ms). + /// Without this we end up with a capped build overview matrix in the job view (Jenkins doesn't + /// like changing order or amount of stages, which will happen with stages started `via parallel()` + def timeOffsetForOrder = 0; - shout("build image"); + def stages = [ + "Build source package": { + sleep(0.1 * timeOffsetForOrder++); + def build_instance = null; + + smart_stage( + name: "Build source package", + condition: build_image, + raiseOnError: false, + ) { + build_instance = smart_build( + // see global-defaults.yml, needs to run in minimal container + use_upstream_build: true, + relative_job_name: "${branch_base_folder}/builders/build-cmk-source_tgz", + build_params: [ + CUSTOM_GIT_REF: effective_git_ref, + VERSION: params.VERSION, + EDITION: params.EDITION, + DISABLE_CACHE: params.DISABLE_CACHE, + FAKE_WINDOWS_ARTIFACTS: params.FAKE_WINDOWS_ARTIFACTS, + ], + + build_params_no_check: [ + CIPARAM_OVERRIDE_BUILD_NODE: params.CIPARAM_OVERRIDE_BUILD_NODE, + CIPARAM_CLEANUP_WORKSPACE: params.CIPARAM_CLEANUP_WORKSPACE, + CIPARAM_BISECT_COMMENT: params.CIPARAM_BISECT_COMMENT, + ], + no_remove_others: true, // do not delete other files in the dest dir + download: false, // use copyArtifacts to avoid nested directories + ); + } + smart_stage( + name: "Copy artifacts", + condition: build_instance && build_image, + raiseOnError: false, + ) { + copyArtifacts( + projectName: "${branch_base_folder}/builders/build-cmk-source_tgz", + selector: specific(build_instance.getId()), + target: source_dir, + fingerprintArtifacts: true, + ) + } + }, + "Build Package": { + sleep(0.1 * timeOffsetForOrder++); + def build_instance = null; + + smart_stage( + name: "Build Package", + condition: build_image, + raiseOnError: false, + ) { + build_instance = smart_build( + // see global-defaults.yml, needs to run in minimal container + use_upstream_build: true, + relative_job_name: "${branch_base_folder}/builders/build-cmk-distro-package", + build_params: [ + CUSTOM_GIT_REF: effective_git_ref, + VERSION: params.VERSION, + EDITION: params.EDITION, + DISTRO: "ubuntu-22.04", + DISABLE_CACHE: params.DISABLE_CACHE, + FAKE_WINDOWS_ARTIFACTS: params.FAKE_WINDOWS_ARTIFACTS, + ], + build_params_no_check: [ + CIPARAM_OVERRIDE_BUILD_NODE: params.CIPARAM_OVERRIDE_BUILD_NODE, + CIPARAM_CLEANUP_WORKSPACE: params.CIPARAM_CLEANUP_WORKSPACE, + CIPARAM_BISECT_COMMENT: params.CIPARAM_BISECT_COMMENT, + ], + no_remove_others: true, // do not delete other files in the dest dir + download: false, // use copyArtifacts to avoid nested directories + ); + } + smart_stage( + name: "Copy artifacts", + condition: build_instance && build_image, + raiseOnError: false, + ) { + copyArtifacts( + projectName: "${branch_base_folder}/builders/build-cmk-distro-package", + selector: specific(build_instance.getId()), + target: source_dir, + fingerprintArtifacts: true, + ) + } + } + ]; + + inside_container_minimal(safe_branch_name: safe_branch_name) { + currentBuild.result = parallel(stages).values().every { it } ? "SUCCESS" : "FAILURE"; + } inside_container( args: [ @@ -87,25 +197,7 @@ def main() { usernameVariable: 'NEXUS_USERNAME'), ]) { dir("${checkout_dir}") { - smart_stage(name: 'Prepare package directory', condition: build_image) { - cleanup_directory("${package_dir}"); - } - smart_stage(name: 'Build Image', condition: build_image) { - artifacts_helper.download_deb( - "${INTERNAL_DEPLOY_DEST}", - "${INTERNAL_DEPLOY_PORT}", - "${cmk_version_rc_aware}", - "${source_dir}", - "${EDITION}", - "jammy"); - artifacts_helper.download_source_tar( - "${INTERNAL_DEPLOY_DEST}", - "${INTERNAL_DEPLOY_PORT}", - "${cmk_version_rc_aware}", - "${source_dir}", - "${EDITION}"); - /// TODO: fix this: /// build-cmk-container does not support the downloads dir /// to have an arbitrary location, so we have to provide