support build docker images for arm64 arch (thanos-io#1906)

* Support build docker images for arm64 arch

Signed-off-by: Xiang Dai <[email protected]>

* Use Dockerfile.multi-stage

Signed-off-by: Long Dai <[email protected]>

* CHANGELOG: fix entry

Signed-off-by: Giedrius Statkevičius <[email protected]>

* Makefile: fix SHA values

Signed-off-by: Giedrius Statkevičius <[email protected]>

* .circleci: create new manifest after pushing image

Only create a manifest & push it after pushing images with the new
names.

Signed-off-by: Giedrius Statkevičius <[email protected]>

* fix revert issue

Signed-off-by: Loong <[email protected]>

* update

Signed-off-by: Loong <[email protected]>

* add crossbuild

Signed-off-by: Loong <[email protected]>

* Dockerfile.multi-arch: update SHA

Signed-off-by: Giedrius Statkevičius <[email protected]>

* .dockerignore: remove .promu

Signed-off-by: Giedrius Statkevičius <[email protected]>

* add logic for main

Signed-off-by: Loong <[email protected]>

* Makefile: use $(shell) for getting value

Signed-off-by: Giedrius Statkevičius <[email protected]>

Co-authored-by: Giedrius Statkevičius <[email protected]>
Co-authored-by: Loong <[email protected]>

Author: daixiang0

.circleci/config.yml

@@ -22,7 +22,7 @@ jobs:
       - checkout
       - go/mod-download-cached
       - setup_remote_docker:
-          version: 17.07.0-ce
+          version: 18.06.0-ce
       - run:
           name: Create Secret if PR is not forked
           # GCS integration tests are run only for author's PR that have write access, because these tests
@@ -74,39 +74,46 @@ jobs:
       - checkout
       - go/mod-download-cached
       - setup_remote_docker:
-          version: 17.07.0-ce
+          version: 18.06.0-ce
       - attach_workspace:
           at: .
-      - run: ln -s .build/linux-amd64/thanos thanos
-      - run: make docker
-      - run: docker run thanos --help
+      # Register qemu to support multi-arch.
+      - run: docker run --privileged linuxkit/binfmt:v0.7
+      - run: make crossbuild
+      - run: make docker-build
+      - run: make docker-test
       # Upload to both dockerhub and quay.io.
       - run: echo "${DOCKERHUB_PASSWORD}" | docker login -u="${DOCKERHUB_USERNAME}" --password-stdin
       - run: make docker-push DOCKER_IMAGE_REPO=thanosio/thanos
+      - run: make docker-manifest DOCKER_IMAGE_REPO=thanosio/thanos
       - run: echo "${QUAY_PASSWORD}" | docker login -u="${QUAY_USERNAME}" quay.io --password-stdin
       - run: make docker-push
+      - run: make docker-manifest
 
   publish_release:
     executor: golang
     steps:
       - checkout
       - go/mod-download-cached
       - setup_remote_docker:
-          version: 17.07.0-ce
+          version: 18.06.0-ce
       - attach_workspace:
           at: .
       - run: make tarballs-release
       - store_artifacts:
           path: .tarballs
           destination: releases
-      - run: ln -s .build/linux-amd64/thanos thanos
-      - run: make docker
-      - run: docker run thanos --help
+      # Register qemu to support multi-arch.
+      - run: docker run --privileged linuxkit/binfmt:v0.7
+      - run: make docker-build
+      - run: make docker-test
       # Upload to both dockerhub and quay.io.
       - run: echo "${DOCKERHUB_PASSWORD}" | docker login -u="${DOCKERHUB_USERNAME}" --password-stdin
       - run: make docker-push DOCKER_IMAGE_REPO=thanosio/thanos DOCKER_IMAGE_TAG=$CIRCLE_TAG
+      - run: make docker-manifest DOCKER_IMAGE_REPO=thanosio/thanos DOCKER_IMAGE_TAG=$CIRCLE_TAG
       - run: echo "${QUAY_PASSWORD}" | docker login -u="${QUAY_USERNAME}" quay.io --password-stdin
       - run: make docker-push DOCKER_IMAGE_TAG=$CIRCLE_TAG
+      - run: make docker-manifest DOCKER_IMAGE_TAG=$CIRCLE_TAG
 
 workflows:
   version: 2

.dockerignore

@@ -14,7 +14,6 @@ data/
 test/e2e/e2e_integration_test*
 
 # Ignore promu artifacts.
-/.build
 /.release
 /.tarballs
 

Dockerfile.multi-arch

@@ -0,0 +1,12 @@
+# By default we pin to amd64 sha. Use make docker to automatically adjust for arm64 versions.
+ARG BASE_DOCKER_SHA="97a9aacc097e5dbdec33b0d671adea0785e76d26ff2b979ee28570baf6a9155d"
+
+FROM quay.io/prometheus/busybox@sha256:${BASE_DOCKER_SHA}
+LABEL maintainer="The Thanos Authors"
+
+ARG ARCH="amd64"
+ARG OS="linux"
+
+COPY .build/${OS}-${ARCH}/thanos /bin/thanos
+
+ENTRYPOINT [ "/bin/thanos" ]

Makefile

@@ -12,15 +12,24 @@ arch = $(shell uname -m)
 # just visit https://quay.io/repository/prometheus/busybox?tag=latest&tab=tags.
 # TODO(bwplotka): Pinning is important but somehow quay kills the old images, so make sure to update regularly.
 # Update at 2021.12.08
+AMD64_SHA="97a9aacc097e5dbdec33b0d671adea0785e76d26ff2b979ee28570baf6a9155d"
+ARM64_SHA="5feb736d32e5b57f4944691d00b581f1f9192b3732cab03e3b6034cf0d1c8f2c"
+ 
 ifeq ($(arch), x86_64)
     # amd64
-    BASE_DOCKER_SHA="97a9aacc097e5dbdec33b0d671adea0785e76d26ff2b979ee28570baf6a9155d"
+    BASE_DOCKER_SHA=$(AMD64_SHA)
 else ifeq ($(arch), armv8)
     # arm64
-    BASE_DOCKER_SHA="5feb736d32e5b57f4944691d00b581f1f9192b3732cab03e3b6034cf0d1c8f2c"
+    BASE_DOCKER_SHA=$(ARM64_SHA)
 else
     echo >&2 "only support amd64 or arm64 arch" && exit 1
 endif
+DOCKER_ARCHS       ?= amd64 arm64
+# Generate two target: docker-xxx-amd64, docker-xxx-arm64.
+# Run make docker-xxx -n to see the result with dry run.
+BUILD_DOCKER_ARCHS = $(addprefix docker-build-,$(DOCKER_ARCHS))
+TEST_DOCKER_ARCHS  = $(addprefix docker-test-,$(DOCKER_ARCHS))
+PUSH_DOCKER_ARCHS  = $(addprefix docker-push-,$(DOCKER_ARCHS))
 
 # Ensure everything works even if GOPATH is not set, which is often the case.
 # The `go env GOPATH` will work for all cases for Go 1.8+.
@@ -134,11 +143,20 @@ build: check-git deps $(PROMU)
 	@echo ">> building Thanos binary in $(PREFIX)"
 	@$(PROMU) build --prefix $(PREFIX)
 
+GIT_BRANCH=$(shell $(GIT) rev-parse --abbrev-ref HEAD)
 .PHONY: crossbuild
 crossbuild: ## Builds all binaries for all platforms.
+ifeq ($(GIT_BRANCH), main)
+crossbuild: | $(PROMU)
+	@echo ">> crossbuilding all binaries"
+	# we only care about below two for the main branch
+	$(PROMU) crossbuild -v -p linux/amd64 -p linux/arm64
+else
 crossbuild: | $(PROMU)
 	@echo ">> crossbuilding all binaries"
 	$(PROMU) crossbuild -v
+endif
+
 
 .PHONY: deps
 deps: ## Ensures fresh go.mod and go.sum.
@@ -164,9 +182,33 @@ docker-multi-stage:
 	@echo ">> building docker image 'thanos' with Dockerfile.multi-stage"
 	@docker build -f Dockerfile.multi-stage -t "thanos" --build-arg BASE_DOCKER_SHA=$(BASE_DOCKER_SHA) .
 
-.PHONY: docker-push
+GET_SHA = $(shell echo '$1'_SHA | tr '[:lower:]' '[:upper:]')
+# docker-build builds docker images with multiple architectures.
+.PHONY: docker-build $(BUILD_DOCKER_ARCHS)
+docker-build: $(BUILD_DOCKER_ARCHS)
+$(BUILD_DOCKER_ARCHS): docker-build-%:
+	@docker build -t "$(DOCKER_IMAGE_REPO)-linux-$*:$(DOCKER_IMAGE_TAG)" \
+  --build-arg BASE_DOCKER_SHA=$($(call GET_SHA,$*)) \
+  --build-arg ARCH="$*" \
+  -f Dockerfile.multi-arch .
+
+.PHONY: docker-test $(TEST_DOCKER_ARCHS)
+docker-test: $(TEST_DOCKER_ARCHS)
+$(TEST_DOCKER_ARCHS): docker-test-%:
+	@echo ">> testing image"
+	@docker run "$(DOCKER_IMAGE_REPO)-linux-$*:$(DOCKER_IMAGE_TAG)" --help
+
+# docker-manifest push docker manifest to support multiple architectures.
+.PHONY: docker-manifest
+docker-manifest:
+	@echo ">> creating and pushing manifest"
+	@DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_IMAGE_REPO):$(DOCKER_IMAGE_TAG)" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_IMAGE_REPO)-linux-$(ARCH):$(DOCKER_IMAGE_TAG))
+	@DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_IMAGE_REPO):$(DOCKER_IMAGE_TAG)"
+
+.PHONY: docker-push $(PUSH_DOCKER_ARCHS)
 docker-push: ## Pushes 'thanos' docker image build to "$(DOCKER_IMAGE_REPO):$(DOCKER_IMAGE_TAG)".
-docker-push:
+docker-push: $(PUSH_DOCKER_ARCHS)
+$(PUSH_DOCKER_ARCHS): docker-push-%:
 	@echo ">> pushing image"
 	@docker tag "thanos" "$(DOCKER_IMAGE_REPO):$(DOCKER_IMAGE_TAG)"
 	@docker push "$(DOCKER_IMAGE_REPO):$(DOCKER_IMAGE_TAG)"