Cleanup and improve reproducibility of builds (GoogleContainerTools#553)

* Clean up package manager files * Added dpkg_extract and tar utility for reproducible tars * Add test case for build release tar * Migrate java cacert extraction * Updated to archive.py from pkg_tar * Fix tar mode to GNU_TAR for consistency * Use build_tar utility * Fix /etc/os-release file to 0644 file permissions. * Added copyright for libc-bin back to the files. Set default mode to 0644.
flatt-security · Aug 10, 2020 · 4a2e305 · 4a2e305
1 parent f8590f2
commit 4a2e305
Show file tree

Hide file tree

Showing 15 changed files with 160 additions and 145 deletions.
diff --git a/.gitignore b/.gitignore
@@ -6,6 +6,7 @@
 /.classpath
 /.factorypath
 /.idea/
+*.iml
 /.project
 /.settings
 /bazel.iml

diff --git a/base/BUILD b/base/BUILD
@@ -1,6 +1,7 @@
 package(default_visibility = ["//visibility:public"])
 
 load(":base.bzl", "NONROOT", "distro_components")
+load(":distro.bzl", "DISTRO_SUFFIXES")
 load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
 load("@io_bazel_rules_docker//contrib:group.bzl", "group_entry", "group_file")
 load("@io_bazel_rules_docker//contrib:passwd.bzl", "passwd_entry", "passwd_tar")
@@ -113,10 +114,7 @@ go_binary(
     pure = "on",
 )
 
-# Replicate the containers and tests for debian9 and debian10
-distro_components("_debian9")
-
-distro_components("_debian10")
+[distro_components(suffix) for suffix in DISTRO_SUFFIXES]
 
 # alias debian9 as the default images
 alias(

diff --git a/base/base.bzl b/base/base.bzl
@@ -1,22 +1,11 @@
 # defines a function to replicate the container images for different distributions
 load("@io_bazel_rules_docker//container:container.bzl", "container_image")
 load("@io_bazel_rules_docker//contrib:test.bzl", "container_test")
-load("@package_bundle//file:packages.bzl", "packages")
-load("@package_bundle_debian10//file:packages.bzl", packages_debian10 = "packages")
+load(":distro.bzl", "DISTRO_PACKAGES", "DISTRO_REPOSITORY")
 load("//cacerts:cacerts.bzl", "cacerts")
 
 NONROOT = 65532
 
-DISTRO_PACKAGES = {
-    "_debian9": packages,
-    "_debian10": packages_debian10,
-}
-
-DISTRO_REPOSITORY = {
-    "_debian9": "@debian_stretch",
-    "_debian10": "@debian10",
-}
-
 # Replicate everything for debian9 and debian10
 def distro_components(distro_suffix):
     cacerts(

diff --git a/base/distro.bzl b/base/distro.bzl
@@ -0,0 +1,14 @@
+load("@package_bundle//file:packages.bzl", "packages")
+load("@package_bundle_debian10//file:packages.bzl", packages_debian10 = "packages")
+
+DISTRO_PACKAGES = {
+    "_debian9": packages,
+    "_debian10": packages_debian10,
+}
+
+DISTRO_SUFFIXES = ("_debian9", "_debian10")
+
+DISTRO_REPOSITORY = {
+    "_debian9": "@debian_stretch",
+    "_debian10": "@debian10",
+}
diff --git a/cacerts/BUILD b/cacerts/BUILD
@@ -1,6 +1,15 @@
-package(default_visibility = ["//visibility:public"])
+package(default_visibility = ["//:__subpackages__"])
 
-sh_binary(
-    name = "extract_certs",
-    srcs = ["extract.sh"],
-)
+load("//base:distro.bzl", "DISTRO_PACKAGES", "DISTRO_SUFFIXES")
+load(":cacerts.bzl", "cacerts")
+load(":java.bzl", "cacerts_java")
+
+[cacerts(
+    name = "cacerts" + distro_suffix,
+    deb = DISTRO_PACKAGES[distro_suffix]["ca-certificates"],
+) for distro_suffix in DISTRO_SUFFIXES]
+
+[cacerts_java(
+    name = "cacerts_java" + distro_suffix,
+    cacerts_tar = ":cacerts" + distro_suffix,
+) for distro_suffix in DISTRO_SUFFIXES]
diff --git a/cacerts/cacerts.bzl b/cacerts/cacerts.bzl
@@ -1,15 +1,36 @@
 """A rule to unpack ca certificates from the debian package."""
 
 def _impl(ctx):
-    ctx.actions.run(
-        executable = ctx.executable._extract,
+    ctx.actions.run_shell(
+        inputs = [ctx.file.deb],
+        outputs = [ctx.outputs.tar],
+        tools = [] + ctx.files._build_tar + ctx.files._dpkg_extract,
         arguments = [
             ctx.file.deb.path,
             ctx.outputs.tar.path,
-            ctx.outputs.deb.path,
         ],
-        inputs = [ctx.file.deb],
-        outputs = [ctx.outputs.tar, ctx.outputs.deb],
+        env = {
+            "EXTRACT_DEB": ctx.executable._dpkg_extract.path,
+            "BUILD_TAR": ctx.executable._build_tar.path,
+        },
+        command = """
+            set -o errexit
+            set -o xtrace
+
+            $EXTRACT_DEB "$1" ./usr/share/ca-certificates ./usr/share/doc/ca-certificates/copyright
+
+            CERT_FILE=./etc/ssl/certs/ca-certificates.crt
+            mkdir -p $(dirname $CERT_FILE)
+
+            CERTS=$(find usr/share/ca-certificates -type f | sort)
+            for cert in $CERTS; do
+              cat $cert >> $CERT_FILE
+            done
+
+            $BUILD_TAR  --output "$2" \
+                        --file $CERT_FILE=$CERT_FILE \
+                        --file ./usr/share/doc/ca-certificates/copyright=./usr/share/doc/ca-certificates/copyright
+        """,
     )
 
 cacerts = rule(
@@ -19,17 +40,20 @@ cacerts = rule(
             mandatory = True,
         ),
         # Implicit dependencies.
-        "_extract": attr.label(
-            default = Label("//cacerts:extract_certs"),
+        "_build_tar": attr.label(
+            default = Label("@rules_pkg//:build_tar"),
+            cfg = "host",
+            executable = True,
+        ),
+        "_dpkg_extract": attr.label(
+            default = Label("//package_manager:dpkg_extract"),
             cfg = "host",
             executable = True,
-            allow_files = True,
         ),
     },
     executable = False,
     outputs = {
         "tar": "%{name}.tar",
-        "deb": "%{name}.deb",
     },
     implementation = _impl,
 )
diff --git a/cacerts/extract.sh b/cacerts/extract.sh
diff --git a/cacerts/java.bzl b/cacerts/java.bzl
@@ -16,14 +16,22 @@ def _impl(ctx):
     ctx.actions.run_shell(
         outputs = [ctx.outputs.out],
         inputs = [ctx.file.cacerts_tar],
-        tools = [ctx.file._jksutil],
+        tools = [ctx.file._jksutil] + ctx.files._build_tar,
         arguments = [ctx.file.cacerts_tar.path, ctx.outputs.out.path],
+        env = {
+            "CREATE_JKS": ctx.executable._jksutil.path,
+            "BUILD_TAR": ctx.executable._build_tar.path,
+        },
         command = """
-mkdir -p etc/ssl/certs/java
-tar -xOf "$1" etc/ssl/certs/ca-certificates.crt |
-  """ + ctx.file._jksutil.path + """ > etc/ssl/certs/java/cacerts
-tar -cvf "$2" etc/ssl
-""",
+            set -o errexit
+            set -o xtrace
+
+            mkdir -p etc/ssl/certs/java
+            tar -xOf "$1" ./etc/ssl/certs/ca-certificates.crt | $CREATE_JKS > etc/ssl/certs/java/cacerts
+
+            $BUILD_TAR  --output "$2" \
+                        --file ./etc/ssl/certs/java/cacerts=./etc/ssl/certs/java/cacerts
+        """,
     )
 
 cacerts_java = rule(
@@ -42,6 +50,11 @@ file with the JKS file at etc/ssl/certs/java/cacerts.
             executable = True,
             allow_single_file = True,
         ),
+        "_build_tar": attr.label(
+            default = Label("@rules_pkg//:build_tar"),
+            cfg = "host",
+            executable = True,
+        ),
     },
     executable = False,
     outputs = {

diff --git a/java/BUILD b/java/BUILD
@@ -6,7 +6,6 @@ load("@io_bazel_rules_docker//java:image.bzl", "java_image")
 load("@package_bundle//file:packages.bzl", "packages", "versions")
 load("@package_bundle_debian10//file:packages.bzl", packages_debian10 = "packages", versions_debian10 = "versions")
 load("//cacerts:java.bzl", "cacerts_java")
-load("//locale:locale.bzl", "locale")
 load("//java:jre_ver.bzl", "jre_ver")
 
 DISTRO_SUFFIXES = ("_debian9", "_debian10")
@@ -21,16 +20,6 @@ DISTRO_VERSIONS = {
     "_debian10": versions_debian10,
 }
 
-[locale(
-    name = "locale_java" + distro_suffix,
-    deb = DISTRO_PACKAGES[distro_suffix]["libc-bin"],
-) for distro_suffix in DISTRO_SUFFIXES]
-
-[cacerts_java(
-    name = "cacerts_java" + distro_suffix,
-    cacerts_tar = "//base:cacerts" + distro_suffix + ".tar",
-) for distro_suffix in DISTRO_SUFFIXES]
-
 [[container_image(
     name = "java_base" + mode + distro_suffix,
     base = ("//cc:cc" if (not ("debug" in mode)) else "//cc:debug") + distro_suffix,
@@ -50,8 +39,8 @@ DISTRO_VERSIONS = {
         "LANG": "C.UTF-8",
     },
     tars = [
-        ":cacerts_java" + distro_suffix,
-        ":locale_java" + distro_suffix,
+        "//cacerts:cacerts_java" + distro_suffix,
+        "//locale:locale" + distro_suffix,
     ],
 ) for mode in [
     "",

diff --git a/locale/BUILD b/locale/BUILD
@@ -1,6 +1,9 @@
-package(default_visibility = ["//visibility:public"])
+package(default_visibility = ["//:__subpackages__"])
 
-sh_binary(
-    name = "extract_locale",
-    srcs = ["extract.sh"],
-)
+load("//base:distro.bzl", "DISTRO_PACKAGES", "DISTRO_SUFFIXES")
+load(":locale.bzl", "locale")
+
+[locale(
+    name = "locale" + distro_suffix,
+    deb = DISTRO_PACKAGES[distro_suffix]["libc-bin"],
+) for distro_suffix in DISTRO_SUFFIXES]
diff --git a/locale/extract.sh b/locale/extract.sh
diff --git a/locale/locale.bzl b/locale/locale.bzl
@@ -1,14 +1,26 @@
-"""A rule to unpack c locale from the debian package."""
+"""A rule to unpack minimal locales from the debian package."""
 
 def _impl(ctx):
-    ctx.actions.run(
-        executable = ctx.executable._extract,
+    ctx.actions.run_shell(
+        inputs = [ctx.file.deb],
+        outputs = [ctx.outputs.tar],
+        tools = [] + ctx.files._build_tar + ctx.files._dpkg_extract,
         arguments = [
             ctx.file.deb.path,
             ctx.outputs.tar.path,
         ],
-        inputs = [ctx.file.deb],
-        outputs = [ctx.outputs.tar],
+        env = {
+            "EXTRACT_DEB": ctx.executable._dpkg_extract.path,
+            "BUILD_TAR": ctx.executable._build_tar.path,
+        },
+        command = """
+            $EXTRACT_DEB "$1" ./usr/lib/locale/C.UTF-8 ./usr/share/doc/libc-bin/copyright
+
+            $BUILD_TAR  --output "$2" \
+                        --mode 0644 \
+                        --file ./usr/share/doc/libc-bin/copyright=./usr/share/doc/libc-bin/copyright \
+                        --file ./usr/lib/locale/C.UTF-8=./usr/lib/locale/C.UTF-8
+        """,
     )
 
 locale = rule(
@@ -18,11 +30,15 @@ locale = rule(
             mandatory = True,
         ),
         # Implicit dependencies.
-        "_extract": attr.label(
-            default = Label("//locale:extract_locale"),
+        "_build_tar": attr.label(
+            default = Label("@rules_pkg//:build_tar"),
+            cfg = "host",
+            executable = True,
+        ),
+        "_dpkg_extract": attr.label(
+            default = Label("//package_manager:dpkg_extract"),
             cfg = "host",
             executable = True,
-            allow_files = True,
         ),
     },
     executable = False,