diff --git a/.gitignore b/.gitignore index 62dc64889..f1e89d046 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ /.classpath /.factorypath /.idea/ +*.iml /.project /.settings /bazel.iml diff --git a/base/BUILD b/base/BUILD index 05eca8269..cf6166717 100644 --- a/base/BUILD +++ b/base/BUILD @@ -1,6 +1,7 @@ package(default_visibility = ["//visibility:public"]) load(":base.bzl", "NONROOT", "distro_components") +load(":distro.bzl", "DISTRO_SUFFIXES") load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") load("@io_bazel_rules_docker//contrib:group.bzl", "group_entry", "group_file") load("@io_bazel_rules_docker//contrib:passwd.bzl", "passwd_entry", "passwd_tar") @@ -113,10 +114,7 @@ go_binary( pure = "on", ) -# Replicate the containers and tests for debian9 and debian10 -distro_components("_debian9") - -distro_components("_debian10") +[distro_components(suffix) for suffix in DISTRO_SUFFIXES] # alias debian9 as the default images alias( diff --git a/base/base.bzl b/base/base.bzl index 52dd859eb..b635c73b2 100644 --- a/base/base.bzl +++ b/base/base.bzl @@ -1,22 +1,11 @@ # defines a function to replicate the container images for different distributions load("@io_bazel_rules_docker//container:container.bzl", "container_image") load("@io_bazel_rules_docker//contrib:test.bzl", "container_test") -load("@package_bundle//file:packages.bzl", "packages") -load("@package_bundle_debian10//file:packages.bzl", packages_debian10 = "packages") +load(":distro.bzl", "DISTRO_PACKAGES", "DISTRO_REPOSITORY") load("//cacerts:cacerts.bzl", "cacerts") NONROOT = 65532 -DISTRO_PACKAGES = { - "_debian9": packages, - "_debian10": packages_debian10, -} - -DISTRO_REPOSITORY = { - "_debian9": "@debian_stretch", - "_debian10": "@debian10", -} - # Replicate everything for debian9 and debian10 def distro_components(distro_suffix): cacerts( diff --git a/base/distro.bzl b/base/distro.bzl new file mode 100644 index 000000000..e4414d16e --- /dev/null +++ b/base/distro.bzl @@ -0,0 +1,14 @@ +load("@package_bundle//file:packages.bzl", "packages") +load("@package_bundle_debian10//file:packages.bzl", packages_debian10 = "packages") + +DISTRO_PACKAGES = { + "_debian9": packages, + "_debian10": packages_debian10, +} + +DISTRO_SUFFIXES = ("_debian9", "_debian10") + +DISTRO_REPOSITORY = { + "_debian9": "@debian_stretch", + "_debian10": "@debian10", +} diff --git a/cacerts/BUILD b/cacerts/BUILD index ce713eee4..805d05be3 100644 --- a/cacerts/BUILD +++ b/cacerts/BUILD @@ -1,6 +1,15 @@ -package(default_visibility = ["//visibility:public"]) +package(default_visibility = ["//:__subpackages__"]) -sh_binary( - name = "extract_certs", - srcs = ["extract.sh"], -) +load("//base:distro.bzl", "DISTRO_PACKAGES", "DISTRO_SUFFIXES") +load(":cacerts.bzl", "cacerts") +load(":java.bzl", "cacerts_java") + +[cacerts( + name = "cacerts" + distro_suffix, + deb = DISTRO_PACKAGES[distro_suffix]["ca-certificates"], +) for distro_suffix in DISTRO_SUFFIXES] + +[cacerts_java( + name = "cacerts_java" + distro_suffix, + cacerts_tar = ":cacerts" + distro_suffix, +) for distro_suffix in DISTRO_SUFFIXES] diff --git a/cacerts/cacerts.bzl b/cacerts/cacerts.bzl index 9b0887720..325d503b0 100644 --- a/cacerts/cacerts.bzl +++ b/cacerts/cacerts.bzl @@ -1,15 +1,36 @@ """A rule to unpack ca certificates from the debian package.""" def _impl(ctx): - ctx.actions.run( - executable = ctx.executable._extract, + ctx.actions.run_shell( + inputs = [ctx.file.deb], + outputs = [ctx.outputs.tar], + tools = [] + ctx.files._build_tar + ctx.files._dpkg_extract, arguments = [ ctx.file.deb.path, ctx.outputs.tar.path, - ctx.outputs.deb.path, ], - inputs = [ctx.file.deb], - outputs = [ctx.outputs.tar, ctx.outputs.deb], + env = { + "EXTRACT_DEB": ctx.executable._dpkg_extract.path, + "BUILD_TAR": ctx.executable._build_tar.path, + }, + command = """ + set -o errexit + set -o xtrace + + $EXTRACT_DEB "$1" ./usr/share/ca-certificates ./usr/share/doc/ca-certificates/copyright + + CERT_FILE=./etc/ssl/certs/ca-certificates.crt + mkdir -p $(dirname $CERT_FILE) + + CERTS=$(find usr/share/ca-certificates -type f | sort) + for cert in $CERTS; do + cat $cert >> $CERT_FILE + done + + $BUILD_TAR --output "$2" \ + --file $CERT_FILE=$CERT_FILE \ + --file ./usr/share/doc/ca-certificates/copyright=./usr/share/doc/ca-certificates/copyright + """, ) cacerts = rule( @@ -19,17 +40,20 @@ cacerts = rule( mandatory = True, ), # Implicit dependencies. - "_extract": attr.label( - default = Label("//cacerts:extract_certs"), + "_build_tar": attr.label( + default = Label("@rules_pkg//:build_tar"), + cfg = "host", + executable = True, + ), + "_dpkg_extract": attr.label( + default = Label("//package_manager:dpkg_extract"), cfg = "host", executable = True, - allow_files = True, ), }, executable = False, outputs = { "tar": "%{name}.tar", - "deb": "%{name}.deb", }, implementation = _impl, ) diff --git a/cacerts/extract.sh b/cacerts/extract.sh deleted file mode 100755 index 4f56b7047..000000000 --- a/cacerts/extract.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# Copyright 2017 Google Inc. All rights reserved. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script extracts the CA certs from the typical debian ca-certificates debian package. -# It would be nicer to do this in Python or Go, but neither of these languages have packages -# that can extract .xz files in their stdlibs. - -set -o errexit - -DEB=$1 -OUT_TAR=$2 -OUT_DEB=$3 - -cp "$DEB" "$OUT_DEB" - -ar -x "$DEB" data.tar.xz -ar -d "$OUT_DEB" data.tar.xz - -tar -xf data.tar.xz ./usr/share/doc/ca-certificates/copyright -tar -xf data.tar.xz ./usr/share/ca-certificates - -# We'll create a new data.tar.xz with flattened certs and -# the copyright -rm data.tar.xz - -# Concat all the certs. -CERT_FILE=./etc/ssl/certs/ca-certificates.crt -mkdir -p $(dirname $CERT_FILE) - -CERTS=$(find usr/share/ca-certificates -type f | sort) -for cert in $CERTS; do - cat $cert >> $CERT_FILE -done - -tar -cf data.tar \ - etc/ \ - usr/share/doc/ca-certificates/copyright - -ar -r "$OUT_DEB" data.tar - -mv data.tar "$OUT_TAR" - -rm -rf usr/share/ca-certificates -rm -rf etc/ssl/certs/ca-certificate.crt - diff --git a/cacerts/java.bzl b/cacerts/java.bzl index 1b522ca3d..054b6e412 100644 --- a/cacerts/java.bzl +++ b/cacerts/java.bzl @@ -16,14 +16,22 @@ def _impl(ctx): ctx.actions.run_shell( outputs = [ctx.outputs.out], inputs = [ctx.file.cacerts_tar], - tools = [ctx.file._jksutil], + tools = [ctx.file._jksutil] + ctx.files._build_tar, arguments = [ctx.file.cacerts_tar.path, ctx.outputs.out.path], + env = { + "CREATE_JKS": ctx.executable._jksutil.path, + "BUILD_TAR": ctx.executable._build_tar.path, + }, command = """ -mkdir -p etc/ssl/certs/java -tar -xOf "$1" etc/ssl/certs/ca-certificates.crt | - """ + ctx.file._jksutil.path + """ > etc/ssl/certs/java/cacerts -tar -cvf "$2" etc/ssl -""", + set -o errexit + set -o xtrace + + mkdir -p etc/ssl/certs/java + tar -xOf "$1" ./etc/ssl/certs/ca-certificates.crt | $CREATE_JKS > etc/ssl/certs/java/cacerts + + $BUILD_TAR --output "$2" \ + --file ./etc/ssl/certs/java/cacerts=./etc/ssl/certs/java/cacerts + """, ) cacerts_java = rule( @@ -42,6 +50,11 @@ file with the JKS file at etc/ssl/certs/java/cacerts. executable = True, allow_single_file = True, ), + "_build_tar": attr.label( + default = Label("@rules_pkg//:build_tar"), + cfg = "host", + executable = True, + ), }, executable = False, outputs = { diff --git a/java/BUILD b/java/BUILD index aa2a1ecf5..05292f67e 100644 --- a/java/BUILD +++ b/java/BUILD @@ -6,7 +6,6 @@ load("@io_bazel_rules_docker//java:image.bzl", "java_image") load("@package_bundle//file:packages.bzl", "packages", "versions") load("@package_bundle_debian10//file:packages.bzl", packages_debian10 = "packages", versions_debian10 = "versions") load("//cacerts:java.bzl", "cacerts_java") -load("//locale:locale.bzl", "locale") load("//java:jre_ver.bzl", "jre_ver") DISTRO_SUFFIXES = ("_debian9", "_debian10") @@ -21,16 +20,6 @@ DISTRO_VERSIONS = { "_debian10": versions_debian10, } -[locale( - name = "locale_java" + distro_suffix, - deb = DISTRO_PACKAGES[distro_suffix]["libc-bin"], -) for distro_suffix in DISTRO_SUFFIXES] - -[cacerts_java( - name = "cacerts_java" + distro_suffix, - cacerts_tar = "//base:cacerts" + distro_suffix + ".tar", -) for distro_suffix in DISTRO_SUFFIXES] - [[container_image( name = "java_base" + mode + distro_suffix, base = ("//cc:cc" if (not ("debug" in mode)) else "//cc:debug") + distro_suffix, @@ -50,8 +39,8 @@ DISTRO_VERSIONS = { "LANG": "C.UTF-8", }, tars = [ - ":cacerts_java" + distro_suffix, - ":locale_java" + distro_suffix, + "//cacerts:cacerts_java" + distro_suffix, + "//locale:locale" + distro_suffix, ], ) for mode in [ "", diff --git a/locale/BUILD b/locale/BUILD index d416861f4..7dcb2afde 100644 --- a/locale/BUILD +++ b/locale/BUILD @@ -1,6 +1,9 @@ -package(default_visibility = ["//visibility:public"]) +package(default_visibility = ["//:__subpackages__"]) -sh_binary( - name = "extract_locale", - srcs = ["extract.sh"], -) +load("//base:distro.bzl", "DISTRO_PACKAGES", "DISTRO_SUFFIXES") +load(":locale.bzl", "locale") + +[locale( + name = "locale" + distro_suffix, + deb = DISTRO_PACKAGES[distro_suffix]["libc-bin"], +) for distro_suffix in DISTRO_SUFFIXES] diff --git a/locale/extract.sh b/locale/extract.sh deleted file mode 100755 index b7a261c34..000000000 --- a/locale/extract.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -set -o errexit - -DEB=$1 -OUT_TAR=$2 - -ar -x "$DEB" data.tar.xz - -tar -xf data.tar.xz ./usr/lib/locale/C.UTF-8 ./usr/share/doc/libc-bin/copyright - -rm data.tar.xz - -# On Mac OS, you need to make sure that GNU Tar is used instead of the BSD tar that is shipped with MacOS. -# To resolve this you can install gnu-tar via Homebrew and symlink it as /usr/local/bin/tar. -tar -c --owner=0 --group=0 -f locale.tar ./usr - -mv locale.tar "$OUT_TAR" - -rm -rf usr diff --git a/locale/locale.bzl b/locale/locale.bzl index f4d1b5268..3136ef72a 100644 --- a/locale/locale.bzl +++ b/locale/locale.bzl @@ -1,14 +1,26 @@ -"""A rule to unpack c locale from the debian package.""" +"""A rule to unpack minimal locales from the debian package.""" def _impl(ctx): - ctx.actions.run( - executable = ctx.executable._extract, + ctx.actions.run_shell( + inputs = [ctx.file.deb], + outputs = [ctx.outputs.tar], + tools = [] + ctx.files._build_tar + ctx.files._dpkg_extract, arguments = [ ctx.file.deb.path, ctx.outputs.tar.path, ], - inputs = [ctx.file.deb], - outputs = [ctx.outputs.tar], + env = { + "EXTRACT_DEB": ctx.executable._dpkg_extract.path, + "BUILD_TAR": ctx.executable._build_tar.path, + }, + command = """ + $EXTRACT_DEB "$1" ./usr/lib/locale/C.UTF-8 ./usr/share/doc/libc-bin/copyright + + $BUILD_TAR --output "$2" \ + --mode 0644 \ + --file ./usr/share/doc/libc-bin/copyright=./usr/share/doc/libc-bin/copyright \ + --file ./usr/lib/locale/C.UTF-8=./usr/lib/locale/C.UTF-8 + """, ) locale = rule( @@ -18,11 +30,15 @@ locale = rule( mandatory = True, ), # Implicit dependencies. - "_extract": attr.label( - default = Label("//locale:extract_locale"), + "_build_tar": attr.label( + default = Label("@rules_pkg//:build_tar"), + cfg = "host", + executable = True, + ), + "_dpkg_extract": attr.label( + default = Label("//package_manager:dpkg_extract"), cfg = "host", executable = True, - allow_files = True, ), }, executable = False, diff --git a/package_manager/BUILD b/package_manager/BUILD index 674bd3d73..ff1ea87dd 100644 --- a/package_manager/BUILD +++ b/package_manager/BUILD @@ -2,7 +2,10 @@ load("@subpar//:subpar.bzl", "par_binary") par_binary( name = "dpkg_parser", - srcs = glob(["**/*.py"]), + srcs = [ + "__init__.py", + "dpkg_parser.py", + ], compiler_args = [ "--interpreter", "/usr/bin/env python", @@ -16,6 +19,12 @@ par_binary( ], ) +sh_binary( + name = "dpkg_extract", + srcs = ["dpkg_extract.sh"], + visibility = ["//:__subpackages__"], +) + py_library( name = "parse_metadata", srcs = ["parse_metadata.py"], @@ -24,7 +33,10 @@ py_library( py_library( name = "util", - srcs = ["util.py"], + srcs = [ + "util.py", + "@rules_pkg//:build_tar", + ], ) py_library( diff --git a/package_manager/dpkg_extract.sh b/package_manager/dpkg_extract.sh new file mode 100755 index 000000000..7de2bef55 --- /dev/null +++ b/package_manager/dpkg_extract.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +set -o errexit + +DEB=$1 +if [ -z "$DEB" ]; then + echo "Usage: dpkg_extract [files to extract]" + exit 1; +fi + +if ! [ -f "$DEB" ]; then + echo "$DEB package was not found" + exit 1; +fi + +shift + +ar -x "$DEB" data.tar.xz + +tar -xf data.tar.xz "$@" + +rm data.tar.xz diff --git a/package_manager/util.py b/package_manager/util.py index 71ac148d4..b64d7772a 100644 --- a/package_manager/util.py +++ b/package_manager/util.py @@ -1,9 +1,12 @@ import hashlib import base64 import collections -import tarfile import os +# Distroless's test.sh runs pylint on all python files, but this module will not exist +# pylint: disable=import-error +from build_tar import TarFile + def sha256_checksum(filename, block_size=65536): sha256 = hashlib.sha256() with open(filename, 'rb') as f: @@ -57,5 +60,5 @@ def build_os_release_tar(distro, os_release_file, os_release_path, tar_file_name os.makedirs(os_release_path) with open(os_release_file, 'w') as os_release: generate_os_release(distro, os_release) - with tarfile.open(tar_file_name, "w") as tar: - tar.add(os_release_file) + with TarFile(output=tar_file_name, directory=None, compression='', root_directory='./', default_mtime=0) as tar: + tar.add_file(os_release_path, os_release_path, mode=0o644)