Provide a more concise package "manifest" specification

pkg/experimental/manifest/BUILD

@@ -0,0 +1,25 @@
+# Copyright 2020 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@rules_python//python:defs.bzl", "py_test")
+
+exports_files(glob(["*.bzl"]))
+
+py_test(
+    name = "manifest_test",
+    srcs = ["manifest_test.py"],
+    data = ["manifest.bzl"],
+    python_version = "PY3",
+    srcs_version = "PY3",
+)

pkg/experimental/manifest/examples/BUILD

@@ -0,0 +1,38 @@
+# Copyright 2020 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("//experimental/manifest:manifest.bzl", "pkg_list_manifest")
+load("//experimental:rpm.bzl", "pkg_rpm")
+load(":dest-first.bzl", "manifest_info")
+
+# This is an example to demonstrate usage.  It currently does not function.  All
+# rules are tagged as manual to prevent CI from becoming angry.
+
+manifest = pkg_list_manifest(
+    name = "manifest",
+    default_attrs = manifest_info["default_attrs"],
+    manifest = manifest_info["manifest"],
+    tags = ["manual"],
+)
+
+pkg_rpm(
+    name = "nonbuilding-rpm",
+    data = manifest + [
+        # Other pkg_filegroups and friendes here
+    ],
+    tags = ["manual"],
+    license = "N/A",
+    summary = "don't build me!",
+    # ...
+)

pkg/experimental/manifest/examples/README.md

@@ -0,0 +1,65 @@
+# Why the examples?
+
+We are not sure how to consturct the manifests at this time to a few reasons.
+What comes to mind is:
+
+- `buildifier` -- There is currently no way to tell it [to not
+  reformat](https://github.com/bazelbuild/buildtools/issues/890) some regions of
+  code, a la `//clang format off/on`.  The main result of this is that all
+  whitespace within each of the manifest rows (tuples) is collapsed when
+  buildifier is used in any "fixing" mode.
+
+- Readability: column formatting, file formats?  Starlark is a great starting
+  point here.
+
+- Maintainability: implementations that don't take direct advantage of
+  BUILD-loadable Starlark files appear to require much more code than otherwise.
+
+There may be something we are not aware of, or alternative ideas that we
+overlooked. Feedback is greatly appreciated.
+
+# Options explored
+
+1. `dest-first.bzl` where the manifest is a list of
+   `(destination, action, attributes, source)` tuples.  This is what is currently implemented.
+
+2. `action-first.bzl` where the manifest is a list of
+   `(action, destination, attributes, source)` tuples.
+
+   This is the same as the above, except first two columns are swapped.
+
+3. `action-first-string.bzl` where the manifest is defined as a space-delimited
+    string, ordered like in `action-first.bzl`.
+
+Notes on these:
+
+- 1) and 2) require no custom parsing other than the `attributes` column, which
+  is a simple delimited string.
+
+- 1) and 2) are subject to `buildifier`, 3) is not.
+
+- 1), however is implemented in the code, because it is highly important that
+  the destinations are aligned regardless of overall formatting.
+
+- 3) gives us the most control, but requires writing out a parser.  Depending on
+  the complexity of the file format, this could be impractical to maintain.
+
+Overall, if we could teach `buildifier` not to reformat a region, our preferred
+option is 2), since it has simple code and is easy to read.
+
+# Options not explored
+
+- Moving the transformation from manifest to `pkg_filegroup` list to a
+  repository rule.
+
+  Not explored due to perceived inconvenience and scalability concerns in large
+  monorepos.
+
+- Moving the transformation to some other external utility.
+
+  Not explored due to potential implementation costs.  Also prevents direct
+  reuse of the `pkg_filegroup` rule.
+
+  The main concern with this one is that Bazel is not aware of the contents of
+  the manifest, and will have to be provided additional information that need
+  not be provided when the manifest is available in Starlark.

pkg/experimental/manifest/examples/action-first-string.bzl

@@ -0,0 +1,30 @@
+_default_attrs = "user=root;group=root"
+_manifest = """
+#action     dest                                            attributes...                       source
+"copy",    "/etc/syslog.conf.d/mycomponent.conf",          "unix=0644;section=confignoreplace", "//mycomponent/etc:conf",
+"copy",    "/opt/mycomponent/lib/",                        "unix=0755",                         ":libs_collected",
+"copy",    "/opt/mycomponent/etc/",                        "unix=0600;section=confignoreplace", ":secret_properties",
+"copy",    "/opt/mycomponent/etc/",                        "unix=0644;section=confignoreplace", "//mycomponent/etc:foo.properties",
+"copy",    "/opt/mycomponent/etc/",                        "unix=0644;section=confignoreplace", "//mycomponent/etc:bar.properties",
+"copy",    "/opt/mycomponent/etc/",                        "unix=0644;section=confignoreplace", "//mycomponent/etc:foobar.properties",
+"copy",    "/opt/mycomponent/etc/",                        "unix=0644;section=confignoreplace", "//mycomponent/etc:idkmybffjill.properties",
+"copy",    "/opt/mycomponent/etc/",                        "unix=0644;section=confignoreplace", "//mycomponent/etc/some/subpackage:subpackage.properties"
+"copy",    "/opt/mycomponent/resources/i18n/en_us",        "unix=0644",                         "@i18n//mycomponent/en_us"
+"copy",    "/opt/mycomponent/resources/i18n/en_gb",        "unix=0644",                         "@i18n//mycomponent/en_gb"
+"copy",    "/opt/mycomponent/resources/i18n/es_es",        "unix=0644",                         "@i18n//mycomponent/es_es"
+"copy",    "/opt/mycomponent/resources/i18n/es_mx",        "unix=0644",                         "@i18n//mycomponent/es_mx"
+"copy",    "/opt/mycomponent/bin/mycomponent-service.bin", "unix=0755",                         "//mycomponent/src:service.bin"
+"copy",    "/opt/mycomponent/bin/mycomponent-runner",      "unix=0755",                         "//mycomponent/src:runner"
+"symlink", "/usr/bin/mycomponentd",                        "",                                  "/opt/mycomponent/bin/mycomponent-service-runner"
+"mkdir",   "/opt/mycomponent",                             "unix=0755",                         "",
+"mkdir",   "/opt/mycomponent/lib",                         "unix=0755",                         "",
+"mkdir",   "/opt/mycomponent/resources",                   "unix=0755",                         "",
+"mkdir",   "/opt/mycomponent/state",                       "unix=0755",                         "",
+"mkdir",   "/opt/mycomponent/resources/i18n",              "unix=0755",                         "",
+"""
+
+
+manifest_info = {
+    "default_attrs": _default_attrs,
+    "manifest": _manifest,
+}

pkg/experimental/manifest/examples/action-first.bzl

@@ -0,0 +1,31 @@
+_default_attrs = "user=root;group=root"
+_manifest = [
+    #action     dest                                            attributes...                       source
+    ("copy",    "/etc/syslog.conf.d/mycomponent.conf",          "unix=0644;section=confignoreplace", "//mycomponent/etc:conf"),
+    ("copy",    "/opt/mycomponent/lib/",                        "unix=0755",                         ":libs_collected"),
+    ("copy",    "/opt/mycomponent/etc/",                        "unix=0600;section=confignoreplace", ":secret_properties"),
+    ("copy",    "/opt/mycomponent/etc/",                        "unix=0644;section=confignoreplace", "//mycomponent/etc:foo.properties"),
+    ("copy",    "/opt/mycomponent/etc/",                        "unix=0644;section=confignoreplace", "//mycomponent/etc:bar.properties"),
+    ("copy",    "/opt/mycomponent/etc/",                        "unix=0644;section=confignoreplace", "//mycomponent/etc:foobar.properties"),
+    ("copy",    "/opt/mycomponent/etc/",                        "unix=0644;section=confignoreplace", "//mycomponent/etc:idkmybffjill.properties"),
+    ("copy",    "/opt/mycomponent/etc/",                        "unix=0644;section=confignoreplace", "//mycomponent/etc/some/subpackage:subpackage.properties"),
+    ("copy",    "/opt/mycomponent/resources/i18n/en_us",        "unix=0644",                         "@i18n//mycomponent/en_us"),
+    ("copy",    "/opt/mycomponent/resources/i18n/en_gb",        "unix=0644",                         "@i18n//mycomponent/en_gb"),
+    ("copy",    "/opt/mycomponent/resources/i18n/es_es",        "unix=0644",                         "@i18n//mycomponent/es_es"),
+    ("copy",    "/opt/mycomponent/resources/i18n/es_mx",        "unix=0644",                         "@i18n//mycomponent/es_mx"),
+    ("copy",    "/opt/mycomponent/bin/mycomponent-service.bin", "unix=0755",                         "//mycomponent/src:service.bin"),
+    ("copy",    "/opt/mycomponent/bin/mycomponent-runner",      "unix=0755",                         "//mycomponent/src:runner"),
+    ("symlink", "/usr/bin/mycomponentd",                        "",                                  "/opt/mycomponent/bin/mycomponent-service-runner"),
+
+    ("mkdir",   "/opt/mycomponent",                             "unix=0755",                         ""),
+    ("mkdir",   "/opt/mycomponent/lib",                         "unix=0755",                         ""),
+    ("mkdir",   "/opt/mycomponent/resources",                   "unix=0755",                         ""),
+    ("mkdir",   "/opt/mycomponent/state",                       "unix=0755",                         ""),
+    ("mkdir",   "/opt/mycomponent/resources/i18n",              "unix=0755",                         ""),
+]
+
+
+manifest_info = {
+    "default_attrs": _default_attrs,
+    "manifest": _manifest,
+}

pkg/experimental/manifest/examples/dest-first.bzl

@@ -0,0 +1,32 @@
+_default_attrs = "user=root;group=root"
+_manifest = [
+    #dest                                            action     attributes...                          source
+    ("/etc/syslog.conf.d/mycomponent.conf",          "copy",    "unix=0644;section=config(noreplace)", "//mycomponent/etc:conf"),
+    ("/opt/mycomponent/lib/",                        "copy",    "unix=0755",                           ":libs_collected"),
+    ("/opt/mycomponent/etc/",                        "copy",    "unix=0600;section=config(noreplace)", ":secret_properties"),
+    ("/opt/mycomponent/etc/",                        "copy",    "unix=0644;section=config(noreplace)", "//mycomponent/etc:foo.properties"),
+    ("/opt/mycomponent/etc/",                        "copy",    "unix=0644;section=config(noreplace)", "//mycomponent/etc:bar.properties"),
+    ("/opt/mycomponent/etc/",                        "copy",    "unix=0644;section=config(noreplace)", "//mycomponent/etc:foobar.properties"),
+    ("/opt/mycomponent/etc/",                        "copy",    "unix=0644;section=config(noreplace)", "//mycomponent/etc:idkmybffjill.properties"),
+    ("/opt/mycomponent/etc/",                        "copy",    "unix=0644;section=config(noreplace),", "//mycomponent/etc/some/subpackage:subpackage.properties"),
+    ("/opt/mycomponent/resources/i18n/en_us",        "copy",    "unix=0644",                           "@i18n//mycomponent/en_us"),
+    ("/opt/mycomponent/resources/i18n/en_gb",        "copy",    "unix=0644",                           "@i18n//mycomponent/en_gb"),
+    ("/opt/mycomponent/resources/i18n/es_es",        "copy",    "unix=0644",                           "@i18n//mycomponent/es_es"),
+    ("/opt/mycomponent/resources/i18n/es_mx",        "copy",    "unix=0644",                           "@i18n//mycomponent/es_mx"),
+    ("/opt/mycomponent/bin/mycomponent-service.bin", "copy",    "unix=0755",                           "//mycomponent/src:service.bin"),
+    ("/opt/mycomponent/bin/mycomponent-runner",      "copy",    "unix=0755",                           "//mycomponent/src:runner"),
+    # FIXME: attributes should be allowed to be empty
+    ("/usr/bin/mycomponentd",                        "symlink", "unix=0777",                                    "/opt/mycomponent/bin/mycomponent-service-runner"),
+
+    ("/opt/mycomponent",                             "mkdir",   "unix=0755",                           ""),
+    ("/opt/mycomponent/lib",                         "mkdir",   "unix=0755",                           ""),
+    ("/opt/mycomponent/resources",                   "mkdir",   "unix=0755",                           ""),
+    ("/opt/mycomponent/state",                       "mkdir",   "unix=0755",                           ""),
+    ("/opt/mycomponent/resources/i18n",              "mkdir",   "unix=0755",                           ""),
+
+]
+
+manifest_info = {
+    "default_attrs": _default_attrs,
+    "manifest": _manifest,
+}