Refactor dpkg rules

* Allows multiple sources on the dpkg() rule * Automatically renames packages that are not valid bazel rule names. Prints out a warning with instructions on how to reference the resulting deb. * Allows multiple packages to be part of the same dpkg() rule * Bumps version of dpkg_parser.par to v0.2 Address feedback More feedback, pylint
flatt-security · Jul 24, 2017 · 2c24ffb · 2c24ffb
1 parent e4c72f8
commit 2c24ffb
Show file tree

Hide file tree

Showing 15 changed files with 175 additions and 117 deletions.
diff --git a/WORKSPACE b/WORKSPACE
@@ -14,7 +14,7 @@ load(
     "//package_manager:package_manager.bzl",
     "package_manager_repositories",
     "dpkg_src",
-    "dpkg",
+    "dpkg_list",
 )
 
 package_manager_repositories()
@@ -23,73 +23,43 @@ dpkg_src(
     name = "debian_jessie",
     arch = "amd64",
     distro = "jessie",
-    url = "http://deb.debian.org",
+    sha256 = "8ff5e7a54d4e75bbbcd2f43ebc7cb4a082fbc5493bc9fb2dcdaaeacba6e76dee",
+    snapshot = "20170701T034145Z",
+    url = "http://snapshot.debian.org/archive",
 )
 
 dpkg_src(
     name = "debian_jessie_backports",
     arch = "amd64",
     distro = "jessie-backports",
-    url = "http://deb.debian.org",
-)
-
-# For the glibc base image.
-dpkg(
-    name = "libc6",
-    source = "@debian_jessie//file:Packages.json",
-)
-
-dpkg(
-    name = "ca-certificates",
-    source = "@debian_jessie//file:Packages.json",
-)
-
-dpkg(
-    name = "openssl",
-    source = "@debian_jessie//file:Packages.json",
-)
-
-dpkg(
-    name = "libssl1.0.0",
-    source = "@debian_jessie//file:Packages.json",
-)
-
-# For Java
-dpkg(
-    name = "zlib1g",
-    source = "@debian_jessie//file:Packages.json",
-)
-
-dpkg(
-    name = "openjdk-8-jre-headless",
-    source = "@debian_jessie_backports//file:Packages.json",
-)
-
-dpkg(
-    name = "libgcc1",
-    source = "@debian_jessie//file:Packages.json",
-)
-
-http_file(
-    name = "libstdcpp6",
-    sha256 = "f1509bbabd78e89c861de16931aec5988e1215649688fd4f8dfe1af875a7fbef",
-    url = "http://deb.debian.org/debian/pool/main/g/gcc-4.9/libstdc++6_4.9.2-10_amd64.deb",
-)
-
-# For Python
-dpkg(
-    name = "libpython2.7-minimal",
-    source = "@debian_jessie//file:Packages.json",
-)
-
-dpkg(
-    name = "python2.7-minimal",
-    source = "@debian_jessie//file:Packages.json",
-)
-
-dpkg(
-    name = "libpython2.7-stdlib",
-    source = "@debian_jessie//file:Packages.json",
+    sha256 = "2a493443581bdb4be071359f7fb62122741f233d3596545d88239a4e4ec445e8",
+    snapshot = "20170701T034145Z",
+    url = "http://snapshot.debian.org/archive",
+)
+
+dpkg_list(
+    name = "package_bundle",
+    packages = [
+        "libc6",
+        "ca-certificates",
+        "openssl",
+        "libssl1.0.0",
+
+        #java
+        "zlib1g",
+        "libgcc1",
+        "libstdc++6",
+        "openjdk-8-jre-headless",
+
+        #python
+        "libpython2.7-minimal",
+        "python2.7-minimal",
+        "libpython2.7-stdlib",
+    ],
+    sources = [
+        "@debian_jessie//file:Packages.json",
+        "@debian_jessie_backports//file:Packages.json",
+    ],
 )
 
 # For Jetty

diff --git a/base/BUILD b/base/BUILD
@@ -28,13 +28,15 @@ cacerts(
     name = "cacerts",
 )
 
+load("@package_bundle//file:packages.bzl", "packages")
+
 docker_build(
     name = "base",
     base = ":with_tmp",
     debs = [
-        "@libc6//file:pkg.deb",
-        "@libssl1.0.0//file:pkg.deb",
-        "@openssl//file:pkg.deb",
+        packages["libc6"],
+        packages["libssl1.0.0"],
+        packages["openssl"],
     ],
     tars = [
         ":base_passwd.passwd.tar",

diff --git a/cacerts/cacerts.bzl b/cacerts/cacerts.bzl
@@ -6,10 +6,12 @@ def _impl(ctx):
             inputs = [ctx.executable._extract, ctx.file.deb],
             outputs = [ctx.outputs.out])
 
+load("@package_bundle//file:packages.bzl", "packages")
+
 cacerts = rule(
     attrs = {
         "deb": attr.label(
-            default = Label("@ca-certificates//file:pkg.deb"),
+            default = Label(packages["ca-certificates"]),
             allow_files = [".deb"],
             single_file = True,
         ),

diff --git a/cc/BUILD b/cc/BUILD
@@ -1,13 +1,14 @@
 package(default_visibility = ["//visibility:public"])
 
 load("@io_bazel_rules_docker//docker:docker.bzl", "docker_build")
+load("@package_bundle//file:packages.bzl", "packages")
 
 # An intermediate image for Java and other "mostly statically" compiled languages
 docker_build(
     name = "cc",
     base = "//base:base",
     debs = [
-        "@libgcc1//file:pkg.deb",
-        "@libstdcpp6//file",
+        packages["libgcc1"],
+        packages["libstdc++6"],
     ],
 )
diff --git a/java/BUILD b/java/BUILD
@@ -1,13 +1,14 @@
 package(default_visibility = ["//visibility:public"])
 
 load("@io_bazel_rules_docker//docker:docker.bzl", "docker_build")
+load("@package_bundle//file:packages.bzl", "packages")
 
 docker_build(
     name = "java8",
     base = "//cc:cc",
     debs = [
-        "@zlib1g//file:pkg.deb",
-        "@openjdk-8-jre-headless//file:pkg.deb",
+        packages["zlib1g"],
+        packages["openjdk-8-jre-headless"],
     ],
     entrypoint = [
         "/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java",

diff --git a/package_manager/BUILD b/package_manager/BUILD
@@ -5,18 +5,34 @@ par_binary(
     srcs = glob(["**/*.py"]),
     main = "dpkg_parser.py",
     visibility = ["//visibility:public"],
-    deps = [":parse_metadata"],
+    deps = [
+        ":parse_metadata",
+        ":util",
+    ],
 )
 
 py_library(
     name = "parse_metadata",
     srcs = ["parse_metadata.py"],
 )
 
+py_library(
+    name = "util",
+    srcs = ["util.py"],
+)
+
 py_test(
     name = "parse_metadata_test",
     size = "small",
     srcs = ["parse_metadata_test.py"],
     data = ["testdata/Packages.txt"],
     deps = [":parse_metadata"],
 )
+
+py_test(
+    name = "util_test",
+    size = "small",
+    srcs = ["util_test.py"],
+    data = ["testdata/checksum.txt"],
+    deps = [":util"],
+)
diff --git a/package_manager/dpkg.bzl b/package_manager/dpkg.bzl
@@ -1,25 +1,28 @@
-def _dpkg_impl(repository_ctx):
+def _dpkg_list_impl(repository_ctx):
   repository_ctx.file("file/BUILD", """
 package(default_visibility = ["//visibility:public"])
-exports_files(["pkg.deb"])
+deb_files = glob(["*.deb"])
+exports_files(deb_files + ["packages.bzl"])
 """)
 
   args = [
       repository_ctx.path(repository_ctx.attr._dpkg_parser),
-      "--packages-file", repository_ctx.path(repository_ctx.attr.source),
-      "--package-name", repository_ctx.name
+      "--package-files", ",".join([repository_ctx.path(src_path) for src_path in repository_ctx.attr.sources]),
+      "--packages", ",".join(repository_ctx.attr.packages),
+      "--workspace-name", repository_ctx.name,
   ]
 
   result = repository_ctx.execute(args)
   if result.return_code:
     fail("dpkg_parser command failed: %s (%s)" % (result.stderr, " ".join(args)))
 
-_dpkg = repository_rule(
-    _dpkg_impl,
+_dpkg_list = repository_rule(
+    _dpkg_list_impl,
     attrs = {
-        "source": attr.label(
-            allow_single_file = True,
+        "sources": attr.label_list(
+            allow_files = True,
         ),
+        "packages": attr.string_list(),
         "_dpkg_parser": attr.label(
             executable = True,
             default = Label("@dpkg_parser//file:dpkg_parser.par"),
@@ -38,8 +41,11 @@ exports_files(["Packages.json"])
       "--download-and-extract-only=True",
       "--mirror-url=" + repository_ctx.attr.url,
       "--arch=" + repository_ctx.attr.arch, 
-      "--distro=" + repository_ctx.attr.distro
+      "--distro=" + repository_ctx.attr.distro,
+      "--snapshot=" + repository_ctx.attr.snapshot,
+      "--sha256=" + repository_ctx.attr.sha256,
   ]
+
   result = repository_ctx.execute(args)
   if result.return_code:
     fail("dpkg_parser command failed: %s (%s)" % (result.stderr, " ".join(args)))
@@ -50,6 +56,8 @@ _dpkg_src = repository_rule(
         "url": attr.string(),
         "arch": attr.string(),
         "distro": attr.string(),
+        "snapshot": attr.string(),
+        "sha256": attr.string(),
         "_dpkg_parser": attr.label(
             executable = True,
             default = Label("@dpkg_parser//file:dpkg_parser.par"),
@@ -58,8 +66,8 @@ _dpkg_src = repository_rule(
     },
 )
 
-def dpkg(**kwargs):
-  _dpkg(**kwargs)
+def dpkg_list(**kwargs):
+  _dpkg_list(**kwargs)
 
 def dpkg_src(**kwargs):
   _dpkg_src(**kwargs)