diff --git a/toolkit/Makefile b/toolkit/Makefile index cd8c4063132..bca63079332 100644 --- a/toolkit/Makefile +++ b/toolkit/Makefile @@ -143,30 +143,8 @@ include $(SCRIPTS_DIR)/utils.mk ######## REMAINING BUILD FLAGS ######## -ifneq ($(DAILY_BUILD_ID),) - $(warning Using Daily Build $(DAILY_BUILD_ID)) - # Ensure build_arch is set - ifeq ($(build_arch),) - $(error build_arch must be set when using DAILY_BUILD_ID) - endif - # build_arch cannot be directly used because Azure storage do not support container names with '_' char - ifeq ($(build_arch),x86_64) - # The actual repo is found at /, while a duplicate copy of all the rpms can be found at /built_rpms_all/ - # Include both so that the tools that expect a valid repo work, while the tools that expect a basic URL also work. - override PACKAGE_URL_LIST := https://mariner3dailydevrepo.blob.core.windows.net/daily-repo-$(DAILY_BUILD_ID)-x86-64/built_rpms_all \ - https://mariner3dailydevrepo.blob.core.windows.net/daily-repo-$(DAILY_BUILD_ID)-x86-64 - override SRPM_URL_LIST := https://mariner3dailydevrepo.blob.core.windows.net/daily-repo-$(DAILY_BUILD_ID)-x86-64/SRPMS - else - override PACKAGE_URL_LIST := https://mariner3dailydevrepo.blob.core.windows.net/daily-repo-$(DAILY_BUILD_ID)-aarch64/built_rpms_all \ - https://mariner3dailydevrepo.blob.core.windows.net/daily-repo-$(DAILY_BUILD_ID)-aarch64 - override SRPM_URL_LIST := https://mariner3dailydevrepo.blob.core.windows.net/daily-repo-$(DAILY_BUILD_ID)-aarch64/SRPMS - endif -else - ifeq ($(USE_PREVIEW_REPO),y) - # Configure the preview repo - include $(SCRIPTS_DIR)/preview.mk - endif -endif +# Daily build ID and repo configuration +include $(SCRIPTS_DIR)/daily_build.mk CA_CERT ?= TLS_CERT ?= @@ -207,6 +185,12 @@ include $(SCRIPTS_DIR)/build_tag_imagecustomizer.mk # go-tools, clean-go-tools, go-tidy-all (tidy go utilities before committing) go-test-coverage include $(SCRIPTS_DIR)/tools.mk +# Based on daily build configuration, configure testing repos +include $(SCRIPTS_DIR)/preview.mk + +# Dynamically update toolchain and chroot manifests for some flows with: +include $(SCRIPTS_DIR)/manifests.mk + # Bootstrap the toolchain's compilers and other tools with: # toolchain, raw-toolchain, clean-toolchain, check-manifests, check-x86_64-manifests, check-aarch64-manifests include $(SCRIPTS_DIR)/toolchain.mk diff --git a/toolkit/docs/building/building.md b/toolkit/docs/building/building.md index 77c4ccf8dd9..bf3b8aab6ab 100644 --- a/toolkit/docs/building/building.md +++ b/toolkit/docs/building/building.md @@ -353,6 +353,8 @@ The `REPO_LIST` variable supports multiple repo files, and they are prioritized The Azure Linux base repo is implicitly provided and an optional preview repo is available by setting `USE_PREVIEW_REPO=y`. If `DISABLE_UPSTREAM_REPOS=y` is set, any repo that is accessed through the network is disabled. +Daily build packages are available via `DAILY_BUILD_ID`. Use `DAILY_BUILD_ID=lkg`, or `DAILY_BUILD_ID=V-v-YYYYMMDD`. + ### Authentication If supplying custom endpoints for source/SRPM/package servers, accessing these resources may require keys and certificates. The keys and certificates can be set using: @@ -464,7 +466,8 @@ If that is not desired all remote sources can be disabled by clearing the follow #### `PACKAGE_URL_LIST=...` -> Space separated list of URLs to download toolchain RPM packages from, used to populate the toolchain packages if `$(REBUILD_TOOLCHAIN)` is set to `n`. Defaults to the standard distro repos. Overriding this will clear all the default values. May be augmented by passing `USE_PREVIEW_REPO=y` which will uncondinally append the distro's preview repos to what ever set of URLs is being used. +> Space separated list of URLs to download toolchain RPM packages from, used to populate the toolchain packages if `$(REBUILD_TOOLCHAIN)` is set to `n`. Defaults to the standard distro repos. Overriding this will clear all the default values. May be augmented by passing `USE_PREVIEW_REPO=y` which will unconditionally append the distro's preview repos to what ever set of URLs is being used. `DAILY_BUILD_ID=...` will +also augment the URL list. #### `SRPM_URL_LIST=...` @@ -479,6 +482,8 @@ If that is not desired all remote sources can be disabled by clearing the follow > - `mariner-ms-non-oss.repo` and `mariner-ms-non-oss-preview.repo` - Azure Linux repository containing Microsoft non open Source RPMs with sources not viewable to the public. The preview version serves the same purpose as the official preview repo. > +`DAILY_BUILD_ID` and `DAILY_BUILD_REPO` will automatically augment the `REPO_LIST`. + #### Build Enable/Disable Flags #### `REBUILD_TOOLCHAIN=...` @@ -561,6 +566,16 @@ If that is not desired all remote sources can be disabled by clearing the follow > Pull missing packages from the upstream preview repository in addition to the base repository. This will uncondinally append the preview repo sources to `PACKAGE_URL_LIST`, `SRPM_URL_LIST`, and `REPO_LIST`. +#### `DAILY_BUILD_ID=...` + +##### `DAILY_BUILD_ID=`**`""`** *(default)* + +> Disable daily build sources. + +##### `DAILY_BUILD_ID=`**`lkg` | `V-v-YYYYMMDD`** + +> Use the daily build sources for the given date. `lkg` will use the last known good build. + #### `DISABLE_UPSTREAM_REPOS=...` ##### `DISABLE_UPSTREAM_REPOS=`**`n`** *(default)* @@ -665,6 +680,8 @@ sudo make hydrate-rpms PACKAGE_ARCHIVE=./rpms.tar.gz ## All Build Targets +**Help is available via `make help`** + These are the useful build targets: | Target | Description |:---------------------------------|:--- @@ -784,6 +801,7 @@ To reproduce an ISO build, run the same make invocation as before, but set: | PACKAGE_ARCHIVE | | Use with `make hydrate-rpms` to populate a set of rpms from an archive. | DOWNLOAD_SRPMS | n | Pack SRPMs from local SPECs or download published ones? | USE_PREVIEW_REPO | n | Pull missing packages from the upstream preview repository in addition to the base repository? +| DAILY_BUILD_ID | | `lkg` or `V-v-YYYYMMDD` to use the daily build resources for the given date. | DISABLE_UPSTREAM_REPOS | n | Only pull missing packages from local repositories? This does not affect hydrating the toolchain from `$(PACKAGE_URL_LIST)`. | DISABLE_DEFAULT_REPOS | n | Disable pulling packages from PMC. Use this option with `REPO_LIST` if you want to use your own repository exclusively. | CACHED_PACKAGES_ARCHIVE | | Use with `make hydrate-cached-rpms` to populate the external RPMs cache from an archive. diff --git a/toolkit/resources/manifests/package/daily_build_repo.repo.template b/toolkit/resources/manifests/package/daily_build_repo.repo.template new file mode 100644 index 00000000000..78bae9ed399 --- /dev/null +++ b/toolkit/resources/manifests/package/daily_build_repo.repo.template @@ -0,0 +1,10 @@ +[daily-build-repo] +name=Azure Linux {{.DAILY_REPO_NAME}} +baseurl=https://mariner3dailydevrepo.blob.core.windows.net/{{.DAILY_REPO_NAME}} +gpgkey=file:///etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY +gpgcheck=0 +repo_gpgcheck=0 +enabled=1 +skip_if_unavailable=True +sslverify=0 +priority=10 diff --git a/toolkit/scripts/chroot.mk b/toolkit/scripts/chroot.mk index 08af968dcad..ec461909ee4 100644 --- a/toolkit/scripts/chroot.mk +++ b/toolkit/scripts/chroot.mk @@ -21,22 +21,18 @@ clean-chroot-tools: rm -rf $(BUILD_DIR)/worker && \ rm -rf $(BUILD_DIR)/validatechroot -# Worker chroot manifest is a file corresponding to the TOOLCHAIN_MANIFEST name. -toolchain_config_name=$(notdir $(TOOLCHAIN_MANIFEST)) -worker_manifest_name=$(shell echo "$(toolchain_config_name)" | sed -E 's:^toolchain:pkggen_core:' ) -worker_chroot_manifest = $(TOOLCHAIN_MANIFESTS_DIR)/$(worker_manifest_name) #$(TOOLCHAIN_MANIFESTS_DIR)/pkggen_core_$(build_arch).txt # Find the *.rpm corresponding to each of the entries in the manifest # regex operation: (.*\.([^\.]+)\.rpm) extracts *.().rpm" to determine # the exact path of the required rpm # Outputs: $(TOOLCHAIN_RPMS_DIR)//..rpm sed_regex_full_path = 's`(.*\.([^\.]+)\.rpm)`$(TOOLCHAIN_RPMS_DIR)/\2/\1`p' -worker_chroot_rpm_paths := $(shell sed -nr $(sed_regex_full_path) < $(worker_chroot_manifest)) +worker_chroot_rpm_paths := $(shell sed -nr $(sed_regex_full_path) < $(WORKER_CHROOT_MANIFEST)) # The worker chroot depends on specific toolchain RPMs, the $(toolchain_rpms): target in toolchain.mk knows how # to update these RPMs if required. worker_chroot_deps := \ - $(worker_chroot_manifest) \ + $(WORKER_CHROOT_MANIFEST) \ $(worker_chroot_rpm_paths) \ $(PKGGEN_DIR)/worker/create_worker_chroot.sh @@ -45,14 +41,14 @@ $(chroot_worker): $(worker_chroot_deps) $(depend_REBUILD_TOOLCHAIN) $(depend_TOO else $(chroot_worker): endif - $(PKGGEN_DIR)/worker/create_worker_chroot.sh $(BUILD_DIR)/worker $(worker_chroot_manifest) $(TOOLCHAIN_RPMS_DIR) $(LOGS_DIR) + $(PKGGEN_DIR)/worker/create_worker_chroot.sh $(BUILD_DIR)/worker $(WORKER_CHROOT_MANIFEST) $(TOOLCHAIN_RPMS_DIR) $(LOGS_DIR) validate-chroot: $(go-validatechroot) $(chroot_worker) $(go-validatechroot) \ --rpm-dir="$(TOOLCHAIN_RPMS_DIR)" \ --tmp-dir="$(BUILD_DIR)/validatechroot" \ --worker-chroot="$(chroot_worker)" \ - --worker-manifest="$(worker_chroot_manifest)" \ + --worker-manifest="$(WORKER_CHROOT_MANIFEST)" \ --log-file="$(LOGS_DIR)/worker/validate.log" \ --log-level="$(LOG_LEVEL)" \ --log-color="$(LOG_COLOR)" diff --git a/toolkit/scripts/daily_build.mk b/toolkit/scripts/daily_build.mk new file mode 100644 index 00000000000..d990a6ca09c --- /dev/null +++ b/toolkit/scripts/daily_build.mk @@ -0,0 +1,89 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Contains: +# - Configuration for getting daily build packages + +# DAILY_BUILD_ID and DAILY_BUILD_REPO while similar are mutually exclusive. +# +# DAILY_BUILD_ID is points the build tools to the latest daily built +# packages as part of a tight local workflow. The DAILY_BUILD_ID argument +# takes the form: V-v-YYYYMMDD where V-v is the Major-Minor branch or +# may be 'lkg' which will auto-detect the latest valid build. +# +# DAILY_BUILD_REPO also points to the latest daily built packages. However +# this is intended for internal teams to access the packages at build time and +# runtime. The argument takes the form of a repo file supplied by the +# Azure Linux Team's daily artifact feed. The repo file may be provisioned +# in a downstream image so that it has access to the repo at runtime. The +# DAILY_BUILD_REPO argument takes a path to the daily-mariner.repo file. + +##help:var:DAILY_BUILD_ID:{'lkg',}='lkg' will auto select latest daily build repo. An explicit ID of the form 'V-v-YYYYMMDD' where V-v is the Major-Minor branch is also supported. +DAILY_BUILD_ID ?= +##help:var:DAILY_BUILD_ID_UPDATE_MANIFESTS={y,n}=Update the toolchain manifests when using DAILY_BUILD_ID to match the daily build repo. +DAILY_BUILD_ID_UPDATE_MANIFESTS ?= y +##help:var:DAILY_BUILD_REPO={path to daily.repo}=Path to the daily build repo file to use. +DAILY_BUILD_REPO ?= + +daily_lkg_workdir = $(BUILD_DIR)/daily_build_id + +ifneq ($(DAILY_BUILD_ID),) + ifneq ($(DAILY_BUILD_REPO),) + $(error DAILY_BUILD_ID and DAILY_BUILD_REPO are mutually exclusive.) + endif +endif + +ifneq ($(DAILY_BUILD_ID),) + ifeq ($(DAILY_BUILD_ID),lkg) + $(call create_folder,$(daily_lkg_workdir)) + + override DAILY_BUILD_ID := $(shell $(SCRIPTS_DIR)/get_lkg_id.sh $(daily_lkg_workdir)) + ifneq ($(.SHELLSTATUS),0) + $(error Failed to auto detect DAILY_BUILD_ID) + endif + endif + + $(warning Using Daily Build $(DAILY_BUILD_ID)) + # Ensure build_arch is set + ifeq ($(build_arch),) + $(error build_arch must be set when using DAILY_BUILD_ID) + endif + # build_arch cannot be directly used because Azure storage do not support container names with '_' char + daily_build_repo_name := $(subst _,-,daily-repo-$(DAILY_BUILD_ID)-$(build_arch)) + # The actual repo is found at /, while a duplicate copy of all the rpms can be found at /built_rpms_all/ + # Include both so that the tools that expect a valid repo work, while the tools that expect a basic URL also work. + # The ordering is important, we want to always take the daily build versions of packages first since they are NOT + # the same files as the official packages and will fail checksum validation. + override PACKAGE_URL_LIST := https://mariner3dailydevrepo.blob.core.windows.net/$(daily_build_repo_name)/built_rpms_all \ + https://mariner3dailydevrepo.blob.core.windows.net/$(daily_build_repo_name) \ + $(PACKAGE_URL_LIST) + override SRPM_URL_LIST := https://mariner3dailydevrepo.blob.core.windows.net/$(daily_build_repo_name)/SRPMS \ + $(SRPM_URL_LIST) +endif + +ifneq ($(DAILY_BUILD_REPO),) + PACKAGE_ROOT := $(shell grep -m 1 "baseurl" $(DAILY_BUILD_REPO) | sed 's|baseurl=||g') + $(warning ) + $(warning ######################### WARNING #########################) + $(warning Using a Daily Build Repo at following location:) + $(warning $(PACKAGE_ROOT)) + $(warning ######################### WARNING #########################) + $(warning ) + override PACKAGE_URL_LIST += $(PACKAGE_ROOT)/built_rpms_all + override SRPM_URL_LIST += $(PACKAGE_ROOT)/SRPMS + override REPO_LIST += $(DAILY_BUILD_REPO) +endif + +# This does not use $(depend_DAILY_BUILD_ID) because that mechanism will not detect the conversion of "lkg" to a +# specific daily build ID since utils.mk runs before daily_build.mk. +.PHONY: daily_build_id_always_run_phony +$(STATUS_FLAGS_DIR)/daily_build_id.flag: daily_build_id_always_run_phony + @if [ ! -f $@ ]; then \ + echo "Initializing daily build ID sanitization"; \ + touch $@; \ + fi && \ + if [ "$$(cat $@)" = "$(DAILY_BUILD_ID)" ]; then \ + exit 0; \ + fi && \ + echo "#### Daily build ID changed ('$$(cat $@)' -> '$(DAILY_BUILD_ID)') ####" && \ + echo $(DAILY_BUILD_ID) > $@ diff --git a/toolkit/scripts/get_lkg_id.sh b/toolkit/scripts/get_lkg_id.sh new file mode 100755 index 00000000000..12a6732bff6 --- /dev/null +++ b/toolkit/scripts/get_lkg_id.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Prints to stdout the daily build id from the LKG file + +set -e + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +temp_dir="$1" + +LKG_FILENAME="lkg-3.0-dev.json" +LKG_TEMP_FILENAME="$(mktemp -p "$temp_dir" "${LKG_FILENAME}.XXXXXX" )" + +# shellcheck disable=SC2317 +cleanup() { + rm -f "$LKG_TEMP_FILENAME" +} +trap cleanup EXIT + +get_lkg() { + wget -O "$LKG_TEMP_FILENAME" -nv "https://mariner3dailydevrepo.blob.core.windows.net/lkg/$LKG_FILENAME" + DAILY_BUILD_ID=$(jq -r .dailybuildid "$LKG_TEMP_FILENAME" | tr . -) +} + +get_lkg + +if [ -z "$DAILY_BUILD_ID" ]; then + echo "Failed to get DAILY_BUILD_ID from https://mariner3dailydevrepo.blob.core.windows.net/lkg/$LKG_FILENAME" >&2 + exit 1 +fi + +echo "$DAILY_BUILD_ID" + +exit 0 diff --git a/toolkit/scripts/manifests.mk b/toolkit/scripts/manifests.mk new file mode 100644 index 00000000000..e2b92d2bab6 --- /dev/null +++ b/toolkit/scripts/manifests.mk @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Contains: +# - Optional mechanisms for regenerating the toolchain manifests + +TOOLCHAIN_MANIFEST ?= $(TOOLCHAIN_MANIFESTS_DIR)/toolchain_$(build_arch).txt +WORKER_CHROOT_MANIFEST ?= $(TOOLCHAIN_MANIFESTS_DIR)/pkggen_core_$(build_arch).txt + +# We use the contents of the manifests to derive some makefile recipes so we need to ensure they are up to date. It +# is not sufficient to add a recipe to update the manifests, they must be updated prior to the makefile being parsed +# sicne we use them as input to create recipes. A shell command is used to update the manifests if they are out of date. +ifeq ($(DAILY_BUILD_ID_UPDATE_MANIFESTS),y) + ifneq ($(DAILY_BUILD_ID),) + $(call create_folder,$(daily_lkg_workdir)) + + manifest_exit_status := $(call shell_real_build_only, $(SCRIPTS_DIR)/update_manifest.sh $(TOOLCHAIN_MANIFEST) $(build_arch) $(daily_lkg_workdir) $(DAILY_BUILD_ID) 1>&2 ; echo $$?) + ifneq ($(manifest_exit_status),0) + $(error Failed to auto update manifest '$(TOOLCHAIN_MANIFEST)' with DAILY_BUILD_ID '$(DAILY_BUILD_ID)') + endif + + manifest_exit_status := $(call shell_real_build_only, $(SCRIPTS_DIR)/update_manifest.sh $(WORKER_CHROOT_MANIFEST) $(build_arch) $(daily_lkg_workdir) $(DAILY_BUILD_ID) 1>&2 ; echo $$?) + ifneq ($(manifest_exit_status),0) + $(error Failed to auto update manifest '$(WORKER_CHROOT_MANIFEST) with DAILY_BUILD_ID '$(DAILY_BUILD_ID)') + endif + endif +endif diff --git a/toolkit/scripts/pkggen.mk b/toolkit/scripts/pkggen.mk index b7d986835ad..00843eb09ef 100644 --- a/toolkit/scripts/pkggen.mk +++ b/toolkit/scripts/pkggen.mk @@ -65,6 +65,7 @@ clean-cache-worker: clean-cache: clean-cache-worker rm -rf $(CACHED_RPMS_DIR) rm -f $(validate-pkggen-config) + rm -f $(cached_file) @echo Verifying no mountpoints present in $(cache_working_dir) clean-spec-parse: @echo Verifying no mountpoints present in $(parse_working_dir) diff --git a/toolkit/scripts/preview.mk b/toolkit/scripts/preview.mk index e2e5f8081c4..20242caef2d 100644 --- a/toolkit/scripts/preview.mk +++ b/toolkit/scripts/preview.mk @@ -4,43 +4,67 @@ # Contains: # - Create a preview repo file # - Configuration for getting preview packages - -# Ensure USE_PREVIEW_REPO is set to 'y', else this file is not needed. -ifneq ($(USE_PREVIEW_REPO),y) -$(error USE_PREVIEW_REPO is not set to 'y' but preview.mk is included) -endif +# - Create a daily build repo file ######## REPO FILE MANAGEMENT/CREATION ######## -preview_repo := $(toolkit_root)/repos/azurelinux-official-preview.repo -# Intentionally not using $(SPECS) here to avoid issues with custom SPECS directories -preview_repo_source := $(PROJECT_ROOT)/SPECS/azurelinux-repos/azurelinux-official-preview.repo - -# Stop immediately if the preview repo file is missing and cannot be created. -ifeq ($(wildcard $(preview_repo_source))$(wildcard $(preview_repo)),) - $(warning ) - $(warning ######################### ERROR #########################) - $(warning 'USE_PREVIEW_REPO=y' is set but all of the following failed:) - $(warning - packaged default '$(preview_repo)' is missing) - $(warning - backup source '$(preview_repo_source)' is missing) - $(warning ######################### ERROR #########################) - $(error ) -endif +# Handle creating the preview repo file if it is missing + +ifeq ($(USE_PREVIEW_REPO),y) + preview_repo := $(toolkit_root)/repos/azurelinux-official-preview.repo + # Intentionally not using $(SPECS) here to avoid issues with custom SPECS directories + preview_repo_source := $(PROJECT_ROOT)/SPECS/azurelinux-repos/azurelinux-official-preview.repo + + # Stop immediately if the preview repo file is missing and cannot be created. + ifeq ($(wildcard $(preview_repo_source))$(wildcard $(preview_repo)),) + $(warning ) + $(warning ######################### ERROR #########################) + $(warning 'USE_PREVIEW_REPO=y' is set but all of the following failed:) + $(warning - packaged default '$(preview_repo)' is missing) + $(warning - backup source '$(preview_repo_source)' is missing) + $(warning ######################### ERROR #########################) + $(error ) + endif + + # Copy the preview repo file to the expected repo location, but only if we have a target to copy from. + ifeq ($(wildcard $(preview_repo_source)),) -# Copy the preview repo file to the expected repo location, but only if we have a target to copy from. -ifeq ($(wildcard $(preview_repo_source)),) $(preview_repo): ; -else + + else + $(preview_repo): $(preview_repo_source) @echo "Regenerating preview repo file from '$<' to '$@'" mkdir -p $(dir $@) cp $< $@ + + endif + + ######## VARIABLE CONFIGURATION ######## + + override PACKAGE_URL_LIST += https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/preview/base/$(build_arch) + override PACKAGE_URL_LIST += https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/preview/base/debuginfo/$(build_arch) + override PACKAGE_URL_LIST += https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/preview/ms-oss/$(build_arch) + override SRPM_URL_LIST += https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/preview/base/srpms + override REPO_LIST += $(preview_repo) endif -######## VARIABLE CONFIGURATION ######## +# Generate the daily build repo file if DAILY_BUILD_ID is set + +ifneq ($(DAILY_BUILD_ID),) +# The repo file for daily builds is generated from a template file as needed +daily_build_repo_source := $(MANIFESTS_DIR)/package/daily_build_repo.repo.template +daily_build_repo := $(daily_lkg_workdir)/daily_build.repo +override REPO_LIST += $(daily_build_repo) -override PACKAGE_URL_LIST += https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/preview/base/$(build_arch) -override PACKAGE_URL_LIST += https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/preview/base/debuginfo/$(build_arch) -override PACKAGE_URL_LIST += https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/preview/ms-oss/$(build_arch) -override SRPM_URL_LIST += https://packages.microsoft.com/azurelinux/$(RELEASE_MAJOR_ID)/preview/base/srpms -override REPO_LIST += $(preview_repo) +# Ensure daily_build_repo_name was generated correctly +ifeq ($(daily_build_repo_name),) + $(error daily_build_repo_name must be non-empty when using DAILY_BUILD_ID) +endif + +# Copy the daily build repo file and replace the placeholder with the actual daily build ID +$(daily_build_repo): $(daily_build_repo_source) $(STATUS_FLAGS_DIR)/daily_build_id.flag + @echo "Regenerating daily build repo file from '$<' to '$@'" + mkdir -p $(dir $@) + sed 's|{{.DAILY_REPO_NAME}}|$(daily_build_repo_name)|g' $< > $@ +endif diff --git a/toolkit/scripts/toolchain.mk b/toolkit/scripts/toolchain.mk index e85fb75d365..8d2f562abb3 100644 --- a/toolkit/scripts/toolchain.mk +++ b/toolkit/scripts/toolchain.mk @@ -28,7 +28,6 @@ toolchain_files = \ $(call shell_real_build_only, find $(SCRIPTS_DIR)/toolchain -name '*.sh') \ $(SCRIPTS_DIR)/toolchain/container/Dockerfile toolchain_spec_list := $(call shell_real_build_only, $(SCRIPTS_DIR)/toolchain/list_toolchain_specs.sh $(SCRIPTS_DIR)/toolchain/build_official_toolchain_rpms.sh) -TOOLCHAIN_MANIFEST ?= $(TOOLCHAIN_MANIFESTS_DIR)/toolchain_$(build_arch).txt # Find the *.rpm corresponding to each of the entries in the manifest # regex operation: (.*\.([^\.]+)\.rpm) extracts *.().rpm" to determine # the exact path of the required rpm @@ -66,6 +65,7 @@ clean-toolchain: clean-toolchain-rpms rm -rf $(toolchain_logs_dir) rm -rf $(toolchain_from_repos) rm -rf $(STATUS_FLAGS_DIR)/toolchain_local_temp.flag + rm -rf $(STATUS_FLAGS_DIR)/daily_build_auto_cleanup.flag rm -f $(SCRIPTS_DIR)/toolchain/container/toolchain-local-wget-list rm -f $(SCRIPTS_DIR)/toolchain/container/texinfo-perl-fix.patch rm -f $(SCRIPTS_DIR)/toolchain/container/Awt_build_headless_only.patch @@ -86,6 +86,13 @@ clean-toolchain-rpms: @for f in $(toolchain_out_rpms); do rm -vf $$f; done rm -rvf $(TOOLCHAIN_RPMS_DIR) +# We need to clear the toolchain if we are using a daily build. The filenames will all be the same, but the actual +# .rpm files may be fundamentally different. +$(STATUS_FLAGS_DIR)/daily_build_auto_cleanup.flag: $(STATUS_FLAGS_DIR)/daily_build_id.flag + @echo "Daily build ID changed, sanitizing toolchain" + rm -rf $(TOOLCHAIN_RPMS_DIR) + touch $@ + copy-toolchain-rpms: for f in $(toolchain_rpms_buildarch); do cp -vf $(TOOLCHAIN_RPMS_DIR)/$(build_arch)/$$f $(RPMS_DIR)/$(build_arch); done for f in $(toolchain_rpms_noarch); do cp -vf $(TOOLCHAIN_RPMS_DIR)/noarch/$$f $(RPMS_DIR)/noarch; done @@ -274,7 +281,7 @@ $(STATUS_FLAGS_DIR)/toolchain_local_temp.flag: $(selected_toolchain_archive) $(t # The .rpm doesn't exist # The .rpm is older than the archive we are extracting it from # The toolchain configuration has been changed (depend_TOOLCHAIN_ARCHIVE and depend_REBUILD_TOOLCHAIN) -$(toolchain_rpms): $(TOOLCHAIN_MANIFEST) $(STATUS_FLAGS_DIR)/toolchain_local_temp.flag $(depend_TOOLCHAIN_ARCHIVE) $(depend_REBUILD_TOOLCHAIN) +$(toolchain_rpms): $(TOOLCHAIN_MANIFEST) $(STATUS_FLAGS_DIR)/toolchain_local_temp.flag $(STATUS_FLAGS_DIR)/daily_build_auto_cleanup.flag $(depend_TOOLCHAIN_ARCHIVE) $(depend_REBUILD_TOOLCHAIN) tempFile=$(toolchain_local_temp)/$(notdir $@) && \ if [ ! -f $@ \ -o $(selected_toolchain_archive) -nt $@ \ @@ -289,7 +296,7 @@ $(toolchain_rpms): $(TOOLCHAIN_MANIFEST) $(STATUS_FLAGS_DIR)/toolchain_local_tem # No archive was selected, so download from online package server instead. All packages must be available for this step to succeed. else -$(toolchain_rpms): $(TOOLCHAIN_MANIFEST) $(depend_REBUILD_TOOLCHAIN) $(go-downloader) +$(toolchain_rpms): $(TOOLCHAIN_MANIFEST) $(STATUS_FLAGS_DIR)/daily_build_auto_cleanup.flag $(depend_REBUILD_TOOLCHAIN) $(go-downloader) @rpm_filename="$(notdir $@)" && \ rpm_dir="$(dir $@)" && \ log_file="$(toolchain_downloads_logs_dir)/$$rpm_filename.log" && \ diff --git a/toolkit/scripts/update_manifest.sh b/toolkit/scripts/update_manifest.sh new file mode 100755 index 00000000000..5eb98baae80 --- /dev/null +++ b/toolkit/scripts/update_manifest.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Updates the requested manifest file based on the contents of the LKG json. + +set -e + +if [ "$#" -ne 4 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +manifest_file="$1" +arch="$2" +temp_dir="$3" +build_id="$4" + +# Get SUDO_USER if set, USER otherwise +CALLING_USER=${SUDO_USER:-$USER} +echo "Running update_manifest.sh as $CALLING_USER" + +# Convert build_id to the associated filename on the LKG blob +# "3-0-YYYYMMDD" -> "YYYY-MM-DD-3.0-dev" +convert_id() { + branch=$(echo "$1" | cut -d- -f1-2) + date=$(echo "$1" | cut -d- -f3) + + # Ensure the parts are in the correct format + if [[ ! "$branch" =~ ^[0-9]+-[0-9]+$ ]]; then + echo "Invalid branch format: $branch" >&2 + echo "Expected ID format: lkg | --" >&2 + exit 1 + fi + if [[ ! "$date" =~ ^[0-9]{8}$ ]]; then + echo "Invalid date format: $date" >&2 + echo "Expected DAILY_BUILD_ID format: lkg | --" >&2 + exit 1 + fi + # Check we can parse the date + date -d "$date" >/dev/null 2>&1 || { + echo "Invalid date: $date" >&2 + echo "Expected DAILY_BUILD_ID format: lkg | --" >&2 + exit 1 + } + + branch=$(echo "$branch" | tr - .)-dev + date=$(date -d "$date" +%Y-%m-%d) + + echo "${date}-${branch}" +} + +BUILD_FILENAME="$(convert_id "$build_id").json" +BUILD_TEMP_FILENAME="$(mktemp -p "$temp_dir" "${BUILD_FILENAME}.XXXXXX" )" +BUILD_TEMP_MANIFEST_FILENAME="$(mktemp -p "$temp_dir" "$(basename "$manifest_file").XXXXXX" )" + +# shellcheck disable=SC2317 +cleanup() { + rm -f "$BUILD_TEMP_FILENAME" + rm -f "$BUILD_TEMP_MANIFEST_FILENAME" +} +trap cleanup EXIT + +get_commit_info() { + wget -O "$BUILD_TEMP_FILENAME" -nv "https://mariner3dailydevrepo.blob.core.windows.net/lkg/$BUILD_FILENAME" + GIT_COMMIT=$(jq -r .commit "$BUILD_TEMP_FILENAME") +} + +# Check if the current manifest have been changed by the user +check_for_modified_manifests() { + if git status --porcelain | grep -qP "$(basename "$manifest_file")"; then + echo " #### Local modifications to '$manifest_file' detected." + echo " Revert changes to '$manifest_file' before proceeding. ####" >&2 + exit 1 + fi +} + +# We don't want to update the timestamps unless we actually change the file, so download to a temp file and then move it over +# if the files are different. Refuse to overwrite possible user changes to the manifest files. +# $1 - filename (e.g. toolchain_aarch64.txt) +update_manifest() { + local filepath + local arch + local url + filepath="$1" + arch="$2" + basename=$(basename "$filepath") + + url="https://raw.githubusercontent.com/microsoft/CBL-Mariner/$GIT_COMMIT/toolkit/resources/manifests/package/${basename}" + + echo "Checking for updates to $basename manifest from $url" + + wget -nv "${url}" -O "${BUILD_TEMP_MANIFEST_FILENAME}" + # only update if the files are different + if ! cmp -s "${BUILD_TEMP_MANIFEST_FILENAME}" "${filepath}"; then + + # Refuse to overwrite possible user changes to the manifest files. This will fail if LKG changes the manifests + # day-to-day but is better than silently overwriting user changes. + check_for_modified_manifests + + echo "Manifests are different, updating ${basename}" + mv "${BUILD_TEMP_MANIFEST_FILENAME}" "${filepath}" + chown "$CALLING_USER:$CALLING_USER" "${filepath}" + else + echo "Manifests are the same, not updating ${basename}" + fi +} + +get_commit_info + +update_manifest "${manifest_file}" "${arch}" + +exit 0