-
Notifications
You must be signed in to change notification settings - Fork 568
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add automatic mode for DAILY_BUILD_ID (#9899)
Co-authored-by: Jon Slobodzian <[email protected]>
- Loading branch information
1 parent
6c05340
commit 887f440
Showing
11 changed files
with
372 additions
and
64 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
10 changes: 10 additions & 0 deletions
10
toolkit/resources/manifests/package/daily_build_repo.repo.template
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| [daily-build-repo] | ||
| name=Azure Linux {{.DAILY_REPO_NAME}} | ||
| baseurl=https://mariner3dailydevrepo.blob.core.windows.net/{{.DAILY_REPO_NAME}} | ||
| gpgkey=file:///etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY | ||
| gpgcheck=0 | ||
| repo_gpgcheck=0 | ||
| enabled=1 | ||
| skip_if_unavailable=True | ||
| sslverify=0 | ||
| priority=10 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| # Copyright (c) Microsoft Corporation. | ||
| # Licensed under the MIT License. | ||
|
|
||
| # Contains: | ||
| # - Configuration for getting daily build packages | ||
|
|
||
| # DAILY_BUILD_ID and DAILY_BUILD_REPO while similar are mutually exclusive. | ||
| # | ||
| # DAILY_BUILD_ID is points the build tools to the latest daily built | ||
| # packages as part of a tight local workflow. The DAILY_BUILD_ID argument | ||
| # takes the form: V-v-YYYYMMDD where V-v is the Major-Minor branch or | ||
| # may be 'lkg' which will auto-detect the latest valid build. | ||
| # | ||
| # DAILY_BUILD_REPO also points to the latest daily built packages. However | ||
| # this is intended for internal teams to access the packages at build time and | ||
| # runtime. The argument takes the form of a repo file supplied by the | ||
| # Azure Linux Team's daily artifact feed. The repo file may be provisioned | ||
| # in a downstream image so that it has access to the repo at runtime. The | ||
| # DAILY_BUILD_REPO argument takes a path to the daily-mariner.repo file. | ||
|
|
||
| ##help:var:DAILY_BUILD_ID:{'lkg',<build_id>}='lkg' will auto select latest daily build repo. An explicit ID of the form 'V-v-YYYYMMDD' where V-v is the Major-Minor branch is also supported. | ||
| DAILY_BUILD_ID ?= | ||
| ##help:var:DAILY_BUILD_ID_UPDATE_MANIFESTS={y,n}=Update the toolchain manifests when using DAILY_BUILD_ID to match the daily build repo. | ||
| DAILY_BUILD_ID_UPDATE_MANIFESTS ?= y | ||
| ##help:var:DAILY_BUILD_REPO={path to daily.repo}=Path to the daily build repo file to use. | ||
| DAILY_BUILD_REPO ?= | ||
|
|
||
| daily_lkg_workdir = $(BUILD_DIR)/daily_build_id | ||
|
|
||
| ifneq ($(DAILY_BUILD_ID),) | ||
| ifneq ($(DAILY_BUILD_REPO),) | ||
| $(error DAILY_BUILD_ID and DAILY_BUILD_REPO are mutually exclusive.) | ||
| endif | ||
| endif | ||
|
|
||
| ifneq ($(DAILY_BUILD_ID),) | ||
| ifeq ($(DAILY_BUILD_ID),lkg) | ||
| $(call create_folder,$(daily_lkg_workdir)) | ||
|
|
||
| override DAILY_BUILD_ID := $(shell $(SCRIPTS_DIR)/get_lkg_id.sh $(daily_lkg_workdir)) | ||
| ifneq ($(.SHELLSTATUS),0) | ||
| $(error Failed to auto detect DAILY_BUILD_ID) | ||
| endif | ||
| endif | ||
|
|
||
| $(warning Using Daily Build $(DAILY_BUILD_ID)) | ||
| # Ensure build_arch is set | ||
| ifeq ($(build_arch),) | ||
| $(error build_arch must be set when using DAILY_BUILD_ID) | ||
| endif | ||
| # build_arch cannot be directly used because Azure storage do not support container names with '_' char | ||
| daily_build_repo_name := $(subst _,-,daily-repo-$(DAILY_BUILD_ID)-$(build_arch)) | ||
| # The actual repo is found at <URL>/, while a duplicate copy of all the rpms can be found at <URL>/built_rpms_all/ | ||
| # Include both so that the tools that expect a valid repo work, while the tools that expect a basic URL also work. | ||
| # The ordering is important, we want to always take the daily build versions of packages first since they are NOT | ||
| # the same files as the official packages and will fail checksum validation. | ||
| override PACKAGE_URL_LIST := https://mariner3dailydevrepo.blob.core.windows.net/$(daily_build_repo_name)/built_rpms_all \ | ||
| https://mariner3dailydevrepo.blob.core.windows.net/$(daily_build_repo_name) \ | ||
| $(PACKAGE_URL_LIST) | ||
| override SRPM_URL_LIST := https://mariner3dailydevrepo.blob.core.windows.net/$(daily_build_repo_name)/SRPMS \ | ||
| $(SRPM_URL_LIST) | ||
| endif | ||
|
|
||
| ifneq ($(DAILY_BUILD_REPO),) | ||
| PACKAGE_ROOT := $(shell grep -m 1 "baseurl" $(DAILY_BUILD_REPO) | sed 's|baseurl=||g') | ||
| $(warning ) | ||
| $(warning ######################### WARNING #########################) | ||
| $(warning Using a Daily Build Repo at following location:) | ||
| $(warning $(PACKAGE_ROOT)) | ||
| $(warning ######################### WARNING #########################) | ||
| $(warning ) | ||
| override PACKAGE_URL_LIST += $(PACKAGE_ROOT)/built_rpms_all | ||
| override SRPM_URL_LIST += $(PACKAGE_ROOT)/SRPMS | ||
| override REPO_LIST += $(DAILY_BUILD_REPO) | ||
| endif | ||
|
|
||
| # This does not use $(depend_DAILY_BUILD_ID) because that mechanism will not detect the conversion of "lkg" to a | ||
| # specific daily build ID since utils.mk runs before daily_build.mk. | ||
| .PHONY: daily_build_id_always_run_phony | ||
| $(STATUS_FLAGS_DIR)/daily_build_id.flag: daily_build_id_always_run_phony | ||
| @if [ ! -f $@ ]; then \ | ||
| echo "Initializing daily build ID sanitization"; \ | ||
| touch $@; \ | ||
| fi && \ | ||
| if [ "$$(cat $@)" = "$(DAILY_BUILD_ID)" ]; then \ | ||
| exit 0; \ | ||
| fi && \ | ||
| echo "#### Daily build ID changed ('$$(cat $@)' -> '$(DAILY_BUILD_ID)') ####" && \ | ||
| echo $(DAILY_BUILD_ID) > $@ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| #!/bin/bash | ||
| # Copyright (c) Microsoft Corporation. | ||
| # Licensed under the MIT License. | ||
|
|
||
| # Prints to stdout the daily build id from the LKG file | ||
|
|
||
| set -e | ||
|
|
||
| if [ "$#" -ne 1 ]; then | ||
| echo "Usage: $0 <temp_dir>" >&2 | ||
| exit 1 | ||
| fi | ||
|
|
||
| temp_dir="$1" | ||
|
|
||
| LKG_FILENAME="lkg-3.0-dev.json" | ||
| LKG_TEMP_FILENAME="$(mktemp -p "$temp_dir" "${LKG_FILENAME}.XXXXXX" )" | ||
|
|
||
| # shellcheck disable=SC2317 | ||
| cleanup() { | ||
| rm -f "$LKG_TEMP_FILENAME" | ||
| } | ||
| trap cleanup EXIT | ||
|
|
||
| get_lkg() { | ||
| wget -O "$LKG_TEMP_FILENAME" -nv "https://mariner3dailydevrepo.blob.core.windows.net/lkg/$LKG_FILENAME" | ||
| DAILY_BUILD_ID=$(jq -r .dailybuildid "$LKG_TEMP_FILENAME" | tr . -) | ||
| } | ||
|
|
||
| get_lkg | ||
|
|
||
| if [ -z "$DAILY_BUILD_ID" ]; then | ||
| echo "Failed to get DAILY_BUILD_ID from https://mariner3dailydevrepo.blob.core.windows.net/lkg/$LKG_FILENAME" >&2 | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo "$DAILY_BUILD_ID" | ||
|
|
||
| exit 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| # Copyright (c) Microsoft Corporation. | ||
| # Licensed under the MIT License. | ||
|
|
||
| # Contains: | ||
| # - Optional mechanisms for regenerating the toolchain manifests | ||
|
|
||
| TOOLCHAIN_MANIFEST ?= $(TOOLCHAIN_MANIFESTS_DIR)/toolchain_$(build_arch).txt | ||
| WORKER_CHROOT_MANIFEST ?= $(TOOLCHAIN_MANIFESTS_DIR)/pkggen_core_$(build_arch).txt | ||
|
|
||
| # We use the contents of the manifests to derive some makefile recipes so we need to ensure they are up to date. It | ||
| # is not sufficient to add a recipe to update the manifests, they must be updated prior to the makefile being parsed | ||
| # sicne we use them as input to create recipes. A shell command is used to update the manifests if they are out of date. | ||
| ifeq ($(DAILY_BUILD_ID_UPDATE_MANIFESTS),y) | ||
| ifneq ($(DAILY_BUILD_ID),) | ||
| $(call create_folder,$(daily_lkg_workdir)) | ||
|
|
||
| manifest_exit_status := $(call shell_real_build_only, $(SCRIPTS_DIR)/update_manifest.sh $(TOOLCHAIN_MANIFEST) $(build_arch) $(daily_lkg_workdir) $(DAILY_BUILD_ID) 1>&2 ; echo $$?) | ||
| ifneq ($(manifest_exit_status),0) | ||
| $(error Failed to auto update manifest '$(TOOLCHAIN_MANIFEST)' with DAILY_BUILD_ID '$(DAILY_BUILD_ID)') | ||
| endif | ||
|
|
||
| manifest_exit_status := $(call shell_real_build_only, $(SCRIPTS_DIR)/update_manifest.sh $(WORKER_CHROOT_MANIFEST) $(build_arch) $(daily_lkg_workdir) $(DAILY_BUILD_ID) 1>&2 ; echo $$?) | ||
| ifneq ($(manifest_exit_status),0) | ||
| $(error Failed to auto update manifest '$(WORKER_CHROOT_MANIFEST) with DAILY_BUILD_ID '$(DAILY_BUILD_ID)') | ||
| endif | ||
| endif | ||
| endif |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.