Merge remote-tracking branch 'upstream/master'

.bazelci/tests.yml

@@ -24,3 +24,11 @@ tasks:
     # rpmbuild(8) is not available by default on macOS
     - "-//tests:make_rpm_test"
     - "-//experimental/tests:pkg_rpm_basic_test"
+  windows:
+    working_directory: ../pkg
+    # We can not build any RPM files or do the .deb tests.
+    # Also, many other tests are simply broken on windows. So we start with the
+    # one that works and can add to the list as we fix the broken behavior or
+    # broken test.
+    test_targets:
+    - "//tests:zip_test"

CONTRIBUTING.md

@@ -1,7 +1,5 @@
 Want to contribute? Great! First, read this page (including the small print at
-the end). Also note that we do not accept contributions to the published
-toolchain artifacts (contents under https://github.com/bazelbuild/bazel-toolchains/configs/
-folder).
+the end).
 
 ### Before you contribute
 
@@ -25,8 +23,7 @@ guide you.
 ### Code reviews and other contributions.
 
 **All submissions, including submissions by project members, require review.**
-Please follow the instructions in [the contributors
-documentation](http://bazel.build/contributing.html).
+Please follow the instructions in [the patching guide](/patching.md).
 
 ### The small print
 

patching.md

@@ -0,0 +1,30 @@
+# Patch Acceptance Process
+
+1. PRs that change or add behavior are not accepted without being tied to an
+   issue.  Most fixes, even if you think they are obvious, require an issue
+   too. Almost every change breaks someone who has depended on the behavior,
+   even broken behavior.
+1. Significant changes need a design document. Please create an issue describing
+   the proposed change, and post a link to it to [email protected].
+   Wait for discussion to come to agreement before proceeding.
+1. Prepare a git commit that implements the feature. Don't forget to add tests.
+1. Ensure you've signed a [Contributor License
+   Agreement](https://cla.developers.google.com).
+1. Send us a pull request on
+   [GitHub](https://github.com/bazelbuild/rules_pkg/pulls). If you're new to GitHub,
+   read [about pull
+   requests](https://help.github.com/articles/about-pull-requests/). Note that
+   we restrict permissions to create branches on the main repository, so
+   you will need to push your commit to [your own fork of the
+   repository](https://help.github.com/articles/working-with-forks/).
+1. Wait for a repository owner to assign you a reviewer. We strive to do that
+   within 4 business days, but it may take longer. If your review gets lost
+   you can escalate by sending a mail to
+   [[email protected]](mailto:[email protected]).
+1. Work with the reviewer to complete a code review. For each change, create a
+   new commit and push it to make changes to your pull request.
+1. A maintainer will approve the PR and merge it.
+
+For further information about working with Bazel and rules in general:
+1. Read the [Bazel governance plan](https://www.bazel.build/governance.html).
+1. Read the [contributing to Bazel](https://www.bazel.build/contributing.html) guide.

pkg/BUILD

@@ -31,13 +31,25 @@ py_library(
     srcs = [
         "__init__.py",
         "archive.py",
+    ],
+    srcs_version = "PY3",
+    visibility = [
+        "//experimental:__pkg__",
+        "//tests:__pkg__",
+    ],
+)
+
+py_library(
+    name = "helpers",
+    srcs = [
+        "__init__.py",
         "helpers.py",
     ],
     srcs_version = "PY2AND3",
-    # This build rule is not intended for public use, but needs to
-    # be publicly visible because existing projects rely on it.
-    # Use at your own risk.
-    visibility = ["//visibility:public"],
+    visibility = [
+        "//experimental:__pkg__",
+        "//tests:__pkg__",
+    ],
 )
 
 py_binary(
@@ -48,6 +60,7 @@ py_binary(
     visibility = ["//visibility:public"],
     deps = [
         ":archive",
+        ":helpers",
     ],
 )
 
@@ -57,7 +70,10 @@ py_binary(
     python_version = "PY3",
     srcs_version = "PY3",
     visibility = ["//visibility:public"],
-    deps = [":archive"],
+    deps = [
+        ":archive",
+        ":helpers",
+    ],
 )
 
 py_binary(
@@ -66,7 +82,7 @@ py_binary(
     python_version = "PY3",
     visibility = ["//visibility:public"],
     deps = [
-        ":archive",
+        ":helpers",
     ],
 )
 
@@ -78,7 +94,7 @@ py_binary(
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
     deps = [
-        ":archive",
+        ":helpers",
     ],
 )
 
@@ -99,5 +115,12 @@ py_library(
     name = "make_rpm_lib",
     srcs = ["make_rpm.py"],
     srcs_version = "PY2AND3",
-    visibility = ["//visibility:public"],
+    visibility = [
+        "//experimental:__pkg__",
+        "//tests:__pkg__",
+    ],
+    deps = [
+        ":archive",
+        ":helpers",
+    ],
 )

pkg/archive.py

@@ -344,41 +344,7 @@ def add_tar(self,
     if root and root[0] not in ['/', '.']:
       # Root prefix should start with a '/', adds it if missing
       root = '/' + root
-    compression = os.path.splitext(tar)[-1][1:]
-    if compression == 'tgz':
-      compression = 'gz'
-    elif compression == 'bzip2':
-      compression = 'bz2'
-    elif compression == 'lzma':
-      compression = 'xz'
-    elif compression not in ['gz', 'bz2', 'xz']:
-      compression = ''
-    if compression == 'xz':
-      # Python 2 does not support lzma, our py3 support is terrible so let's
-      # just hack around.
-      # Note that we buffer the file in memory and it can have an important
-      # memory footprint but it's probably fine as we don't use them for really
-      # large files.
-      # TODO(dmarting): once our py3 support gets better, compile this tools
-      # with py3 for proper lzma support.
-      if subprocess.call('which xzcat', shell=True, stdout=subprocess.PIPE):
-        raise self.Error('Cannot handle .xz and .lzma compression: '
-                         'xzcat not found.')
-      p = subprocess.Popen('cat %s | xzcat' % tar,
-                           shell=True,
-                           stdout=subprocess.PIPE)
-      f = io.BytesIO(p.stdout.read())
-      p.wait()
-      intar = tarfile.open(fileobj=f, mode='r:')
-    else:
-      if compression in ['gz', 'bz2']:
-        # prevent performance issues due to accidentally-introduced seeks
-        # during intar traversal by opening in "streaming" mode. gz, bz2
-        # are supported natively by python 2.7 and 3.x
-        inmode = 'r|' + compression
-      else:
-        inmode = 'r:' + compression
-      intar = tarfile.open(name=tar, mode=inmode)
+    intar = tarfile.open(name=tar, mode='r:*')
     for tarinfo in intar:
       if name_filter is None or name_filter(tarinfo.name):
         if not self.preserve_mtime:

pkg/experimental/BUILD

@@ -30,7 +30,7 @@ py_binary(
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
     deps = [
-        "//:archive",
+        "//:helpers",
         "//:make_rpm_lib",
     ],
 )

pkg/tests/BUILD

@@ -162,7 +162,6 @@ py_test(
         "no_windows",
     ],
     deps = [
-        "//:archive",
         "//:make_rpm_lib",
     ],
 )
@@ -173,7 +172,7 @@ py_test(
     python_version = "PY3",
     srcs_version = "PY2AND3",
     deps = [
-        "//:archive",
+        "//:helpers",
     ],
 )
 
@@ -192,7 +191,7 @@ pkg_deb(
         "/etc/other",
     ],
     config = ":testdata/config",
-    data = ":test-tar-gz.tar.gz",
+    data = ":test-tar-basic-gz.tar.gz",
     depends = [
         "dep1",
         "dep2",
@@ -227,7 +226,7 @@ pkg_deb(
         "/etc/other",
     ],
     config = ":testdata/config",
-    data = ":test-tar-gz.tar.gz",
+    data = ":test-tar-basic-gz.tar.gz",
     depends = [
         "dep1",
         "dep2",
@@ -245,12 +244,11 @@ pkg_deb(
 )
 
 [pkg_tar(
-    name = "test-tar-%s" % ext[1:],
+    name = "test-tar-basic-%s" % ext[1:],
     srcs = [
         ":etc/nsswitch.conf",
         ":usr/titi",
     ],
-    build_tar = "//:build_tar",
     extension = "tar%s" % ext,
     mode = "0644",
     modes = {"usr/titi": "0755"},
@@ -271,7 +269,7 @@ pkg_deb(
 [pkg_tar(
     name = "test-tar-inclusion-%s" % ext,
     build_tar = "//:build_tar",
-    deps = [":test-tar-%s" % ext],
+    deps = [":test-tar-basic-%s" % ext],
 ) for ext in [
     "",
     "gz",
@@ -345,43 +343,20 @@ pkg_tar(
     portable_mtime = False,
 )
 
-sh_test(
-    name = "build_test_deb",
-    size = "medium",
-    srcs = [
-        "build_test_deb.sh",
-    ],
-    data = [
-        "testenv.sh",
-        ":test-deb.deb",
-        ":test-deb-py2.deb",
-        ":titi_test_all.changes",
-    ],
-    tags = [
-        # dpkg tools are only available on debian
-        "no_macos",
-        "no_windows",
-    ],
-    deps = [
-        "@rules_pkg//third_party/bazel/src/test/shell:bashunit",
-    ],
-)
-
-sh_test(
-    name = "build_test_tar",
+py_test(
+    name = "pkg_tar_test",
     size = "medium",
     srcs = [
-        "build_test_tar.sh",
+        "pkg_tar_test.py",
     ],
     data = [
-        "testenv.sh",
-        ":archive_testdata",
-        ":test-tar-.tar",
-        ":test-tar-bz2.tar.bz2",
+        ":test-tar-basic-.tar",
+        ":test-tar-basic-bz2.tar.bz2",
+        ":test-tar-basic-gz.tar.gz",
+        ":test-tar-basic-xz.tar.xz",
         ":test-tar-empty_dirs.tar",
         ":test-tar-empty_files.tar",
         ":test-tar-files_dict.tar",
-        ":test-tar-gz.tar.gz",
         ":test-tar-inclusion-.tar",
         ":test-tar-inclusion-bz2.tar",
         ":test-tar-inclusion-gz.tar",
@@ -391,17 +366,29 @@ sh_test(
         ":test-tar-strip_prefix-empty.tar",
         ":test-tar-strip_prefix-etc.tar",
         ":test-tar-strip_prefix-none.tar",
-        ":test-tar-xz.tar.xz",
+    ],
+    python_version = "PY3",
+)
+
+sh_test(
+    name = "build_test_deb",
+    size = "medium",
+    srcs = [
+        "build_test_deb.sh",
+    ],
+    data = [
+        "testenv.sh",
+        ":test-deb.deb",
+        ":test-deb-py2.deb",
         ":titi_test_all.changes",
     ],
     tags = [
-        # archive.py requires xzcat, which is not available by default on Mac
+        # dpkg tools are only available on debian
         "no_macos",
-        # TODO(laszlocsomor): fix on Windows or describe why it cannot pass.
         "no_windows",
     ],
     deps = [
-        "//third_party/bazel/src/test/shell:bashunit",
+        "@rules_pkg//third_party/bazel/src/test/shell:bashunit",
     ],
 )