Dockerfree build and ecs-init make integration (#3149)

* Resolve go modules/vendor conflicts and add dockerfree-all make target

* add build-time configuration and update scripts/comments

Author: fierlion

.github/workflows/linux.yml

@@ -28,3 +28,5 @@ jobs:
           cd $GITHUB_WORKSPACE/src/github.com/aws/amazon-ecs-agent
           make test-silent
           make analyze-cover-profile
+          make test-init
+          make analyze-cover-profile-init

.github/workflows/static.yml

@@ -29,6 +29,32 @@ jobs:
           make get-deps
           make static-check
 
+  init-check:
+    name: Static Analysis Init
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: src/github.com/aws/amazon-ecs-agent
+      - name: get GO_VERSION
+        run:  |
+          cd $GITHUB_WORKSPACE/src/github.com/aws/amazon-ecs-agent
+          echo "GO_VERSION=$(cat GO_VERSION)" >> $GITHUB_ENV
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/checkout@v2
+        with:
+          path: src/github.com/aws/amazon-ecs-agent
+      - name: run static checks
+        run: |
+          export GOPATH=$GITHUB_WORKSPACE
+          export PATH=$PATH:$(go env GOPATH)/bin
+          export GO111MODULE=auto
+          cd $GITHUB_WORKSPACE/src/github.com/aws/amazon-ecs-agent
+          make get-deps-init
+          make static-check-init
+
   x-platform-build:
     name: Cross platform build
     runs-on: ubuntu-latest

.gitignore

@@ -20,3 +20,22 @@ _bin/
 *.iml
 cover.out
 coverprofile.out
+/amazon-ecs-init*
+/BUILDROOT/
+/x86_64/
+/sources.tar
+/ecs-init-*
+/ecs.conf
+/.deb-done
+/.rpm-done
+/.srpm-done
+/BUILD
+/RPMS
+/SOURCES
+/SRPMS
+/ecs-init.spec
+/sources.tgz
+ecs-agent-*.tar
+/ecs.service
+*.log
+*.DS_Store

Makefile

@@ -39,7 +39,6 @@ all: docker
 gobuild:
 	./scripts/build false
 
-
 # create output directories
 .out-stamp:
 	mkdir -p ./out/test-artifacts ./out/cni-plugins ./out/amazon-ecs-cni-plugins ./out/amazon-vpc-cni-plugins
@@ -49,6 +48,9 @@ gobuild:
 static:
 	./scripts/build
 
+static-with-pause:
+	./scripts/build true "" false true
+
 # Cross-platform build target for static checks
 xplatform-build:
 	GOOS=linux GOARCH=arm64 ./scripts/build true "" false
@@ -119,6 +121,10 @@ gogenerate:
 	go generate -x ./agent/...
 	$(MAKE) goimports
 
+gogenerate-init:
+	PATH=$(PATH):$(shell pwd)/scripts go generate -x ./ecs-init/...
+	$(MAKE) goimports
+
 # 'go' may not be on the $PATH for sudo tests
 GO_EXECUTABLE=$(shell command -v go 2> /dev/null)
 
@@ -140,6 +146,10 @@ test:
 	${GOTEST} -tags unit -coverprofile cover.out -timeout=60s ./agent/...
 	go tool cover -func cover.out > coverprofile.out
 
+test-init:
+	go test -count=1 -short -v -coverprofile cover.out ./ecs-init/...
+	go tool cover -func cover.out > coverprofile-init.out
+
 test-silent:
 	$(eval VERBOSE=)
 	${GOTEST} -tags unit -coverprofile cover.out -timeout=60s ./agent/...
@@ -149,6 +159,10 @@ test-silent:
 analyze-cover-profile: coverprofile.out
 	./scripts/analyze-cover-profile
 
+.PHONY: analyze-cover-profile-init
+analyze-cover-profile-init: coverprofile-init.out
+	./scripts/analyze-cover-profile-init
+
 run-integ-tests: test-registry gremlin container-health-check-image run-sudo-tests
 	ECS_LOGLEVEL=debug ${GOTEST} -tags integration -timeout=30m ./agent/...
 
@@ -158,7 +172,6 @@ run-sudo-tests:
 benchmark-test:
 	go test -run=XX -bench=. ./agent/...
 
-
 .PHONY: build-image-for-ecr upload-images replicate-images
 
 build-image-for-ecr: netkitten volumes-test image-cleanup-test-images fluentd exec-command-agent-test
@@ -223,6 +236,21 @@ cni-plugins: get-cni-sources .out-stamp build-ecs-cni-plugins build-vpc-cni-plug
 	mv $(PWD)/out/amazon-vpc-cni-plugins/* $(PWD)/out/cni-plugins
 	@echo "Built all cni plugins successfully."
 
+# dockerfree build process will build the agent container image from scratch
+# and with minimal dependencies
+# requires glibc-static
+
+dockerfree-pause:
+	GOOS=linux GOARCH=amd64 ./scripts/build-pause
+
+dockerfree-certs:
+	GOOS=linux GOARCH=amd64 ./scripts/get-host-certs
+
+dockerfree-cni-plugins: get-cni-sources
+	GOOS=linux GOARCH=amd64 ./scripts/build-cni-plugins
+
+dockerfree-agent-image: dockerfree-pause dockerfree-certs dockerfree-cni-plugins static-with-pause
+	GOOS=linux GOARCH=amd64 ./scripts/build-agent-image
 
 .PHONY: codebuild
 codebuild: .out-stamp
@@ -259,7 +287,6 @@ image-cleanup-test-images:
 container-health-check-image:
 	$(MAKE) -C misc/container-health $(MFLAGS)
 
-
 # all .go files in the agent, excluding vendor/, model/ and testutils/ directories, and all *_test.go and *_mocks.go files
 GOFILES:=$(shell go list -f '{{$$p := .}}{{range $$f := .GoFiles}}{{$$p.Dir}}/{{$$f}} {{end}}' ./agent/... \
 		| grep -v /testutils/ | grep -v _test\.go$ | grep -v _mocks\.go$ | grep -v /model)
@@ -286,12 +313,23 @@ gogenerate-check: gogenerate
 	# check that gogenerate does not generate a diff.
 	git diff --exit-code
 
+.PHONY: gogenerate-check-init
+gogenerate-check-init: gogenerate-init
+	# check that gogenerate does not generate a diff.
+	git diff --exit-code
+
 .PHONY: static-check
 static-check: gocyclo govet importcheck gogenerate-check
 	# use default checks of staticcheck tool, except style checks (-ST*) and depracation checks (-SA1019)
 	# depracation checks have been left out for now; removing their warnings requires error handling for newer suggested APIs, changes in function signatures and their usages.
 	# https://github.com/dominikh/go-tools/tree/master/cmd/staticcheck
-	staticcheck -tests=false -checks "inherit,-ST*,-SA1019,-SA9002" ./agent/...
+	staticcheck -tests=false -checks "inherit,-ST*,-SA1019,-SA9002,-SA4006" ./agent/...
+
+.PHONY: static-check-init
+static-check-init: gocyclo govet importcheck gogenerate-check-init
+	# use default checks of staticcheck tool, except style checks (-ST*)
+	# https://github.com/dominikh/go-tools/tree/master/cmd/staticcheck
+	staticcheck -tests=false -checks "inherit,-ST*" ./ecs-init/...
 
 .PHONY: goimports
 goimports:
@@ -309,13 +347,52 @@ GOPATH=$(shell go env GOPATH)
 
 get-deps: .get-deps-stamp
 
+get-deps-init:
+	go get golang.org/x/tools/cover
+	go get golang.org/x/tools/cmd/cover
+	go get github.com/golang/mock/mockgen
+	cd "${GOPATH}/src/github.com/golang/mock/mockgen" && git checkout 1.3.1 && go get ./... && go install ./... && cd -
+	GO111MODULE=on go get github.com/fzipp/gocyclo/cmd/[email protected]
+	go get golang.org/x/tools/cmd/goimports
+	go get honnef.co/go/tools/cmd/staticcheck
+
+.generic-rpm-done:
+	./scripts/update-version.sh
+	cp packaging/generic-rpm/amazon-ecs-init.spec amazon-ecs-init.spec
+	cp packaging/generic-rpm/ecs.service ecs.service
+	cp packaging/generic-rpm/amazon-ecs-volume-plugin.service amazon-ecs-volume-plugin.service
+	cp packaging/generic-rpm/amazon-ecs-volume-plugin.socket amazon-ecs-volume-plugin.socket
+	tar -czf ./sources.tgz ecs-init scripts
+	test -e SOURCES || ln -s . SOURCES
+	rpmbuild --define "%_topdir $(PWD)" -bb amazon-ecs-init.spec
+	find RPMS/ -type f -exec cp {} . \;
+	touch .rpm-done
+
+generic-rpm: .generic-rpm-done
+
+dockerfree-all: dockerfree-agent-image generic-rpm
+
+.deb-done: BUILDROOT/ecs-agent.tar
+	./scripts/update-version.sh
+	tar -czf ./amazon-ecs-init_${VERSION}.orig.tar.gz ecs-init scripts README.md
+	cp -r packaging/generic-deb/debian ecs-init scripts README.md BUILDROOT
+	cd BUILDROOT && debuild -uc -us --lintian-opts --suppress-tags bad-distribution-in-changes-file,file-in-unusual-dir
+	touch .deb-done
+
+deb: .deb-done
+
 clean:
 	# ensure docker is running and we can talk to it, abort if not:
 	docker ps > /dev/null
 	-docker rmi $(BUILDER_IMAGE) "amazon/amazon-ecs-agent-cleanbuild:make"
 	-docker rmi $(BUILDER_IMAGE) "amazon/amazon-ecs-agent-cleanbuild-windows:make"
-	rm -f misc/certs/ca-certificates.crt &> /dev/null
+	rm -f misc/certs/host-certs.crt &> /dev/null
+	rm -rf misc/pause-container/image/
+	rm -rf misc/pause-container/rootfs/
+	rm -rf misc/plugins/
+	rm -f misc/pause-container/amazon-ecs-pause.tar
 	rm -rf out/
+	rm -rf rootfs/
 	-$(MAKE) -C $(ECS_CNI_REPOSITORY_SRC_DIR) clean
 	-$(MAKE) -C misc/netkitten $(MFLAGS) clean
 	-$(MAKE) -C misc/volumes-test $(MFLAGS) clean
@@ -329,4 +406,26 @@ clean:
 	-rm -rf $(PWD)/bin
 	-rm -rf cover.out
 	-rm -rf coverprofile.out
-
+	-rm -rf coverprofile-init.out
+	# ecs-init & rpm cleanup
+	-rm -f ecs-init.spec
+	-rm -f amazon-ecs-init.spec
+	-rm -f ecs.conf
+	-rm -f ecs.service
+	-rm -f amazon-ecs-volume-plugin.conf
+	-rm -f amazon-ecs-volume-plugin.service
+	-rm -f amazon-ecs-volume-plugin.socket
+	-rm -rf ./bin
+	-rm -f ./sources.tgz
+	-rm -f ./amazon-ecs-init
+	-rm -f ./ecs-init/ecs-init
+	-rm -f ./amazon-ecs-init-*.rpm
+	-rm -f ./ecs-agent-*.tar
+	-rm -f ./ecs-init-*.src.rpm
+	-rm -rf ./ecs-init-*
+	-rm -rf ./BUILDROOT BUILD RPMS SRPMS SOURCES SPECS
+	-rm -rf ./x86_64
+	-rm -f ./amazon-ecs-init_${VERSION}*
+	-rm -f .srpm-done .rpm-done .generic-rpm-done
+	-rm -f .deb-done
+	-rm -f amazon-ecs-volume-plugin

agent-container/agent-config.json

@@ -0,0 +1 @@
+{"author":"Amazon Web Services, Inc.","config":{"Cmd":["/agent"],"ArgsEscaped":true},"created":"~~timestamp~~","history":[{"created":"~~timestamp~~","author":"Amazon Web Services, Inc.","created_by":"[] + [] === \"\"","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:~~digest~~"]}}

agent-container/agent-image-VERSION

@@ -0,0 +1 @@
+1.0

agent-container/agent-manifest.json

@@ -0,0 +1,2 @@
+[{"Config":"config.json","RepoTags":["amazon/amazon-ecs-agent:~~agentversion~~"],"Layers":["rootfs/layer.tar"]}]
+

agent-container/agent-repositories

@@ -0,0 +1 @@
+{"amazon/amazon-ecs-agent":{"amazon-ecs":"rootfs"}}

dependencies_mocks.go

@@ -0,0 +1,15 @@
+// Copyright 2015-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+//
+// Source: dependencies.go in package cache
+

ecs-init/cache/cache_test.go

@@ -1,3 +1,4 @@
+// +build test
 // Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License"). You may

ecs-init/cache/dependencies.go

@@ -18,7 +18,7 @@ package cache
 // package-level functions.  These interfaces are then used to create mocks
 // for the unit tests.
 
-//go:generate mockgen.sh $GOPACKAGE $GOFILE
+//go:generate mockgen.sh cache $GOFILE  
 
 import (
 	"io"

ecs-init/cache/dependencies_mocks.go

@@ -1,3 +1,4 @@
+//go:build al2
 // +build al2
 
 // Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.

ecs-init/config/config_al2.go

@@ -1,3 +1,4 @@
+//go:build generic_rpm
 // +build generic_rpm
 
 // Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.

ecs-init/config/config_generic_rpm.go

@@ -1,3 +1,4 @@
+//go:build suse || ubuntu || debian
 // +build suse ubuntu debian
 
 // Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.

ecs-init/config/config_suse_ubuntu_debian.go

@@ -1,3 +1,4 @@
+//go:build !suse && !ubuntu && !al2 && !debian && !generic_rpm
 // +build !suse,!ubuntu,!al2,!debian,!generic_rpm
 
 // Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.

ecs-init/config/config_unspecified.go

@@ -1,3 +1,4 @@
+//go:build development
 // +build development
 
 // Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.

ecs-init/config/development.go

@@ -34,7 +34,7 @@ and limitations under the License.
 <seelog type="asyncloop">
 	<outputs formatid="main">
 		<console formatid="console" />
-		<rollingfile filename="` + initLogFile() + `" type="date"
+		<rollingfile filename="`+initLogFile()+`" type="date"
 			 datepattern="2006-01-02-15" archivetype="zip" maxrolls="5" />
 	</outputs>
 	<formats>

ecs-init/config/logger.go

@@ -1,3 +1,4 @@
+//go:build !development
 // +build !development
 
 // Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.

ecs-init/config/release.go

@@ -1,3 +1,4 @@
+// +build test
 // Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License"). You may

ecs-init/docker/backoff_mocks.go

@@ -1,3 +1,4 @@
+// +build test
 // Copyright 2015-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License"). You may

ecs-init/docker/dependencies_mocks.go

@@ -1,3 +1,4 @@
+// +build test
 // Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License"). You may