From 29af985d92b54b38246e81b3f709e06ba02f694b Mon Sep 17 00:00:00 2001 From: Cristian Falcas Date: Mon, 20 Jun 2022 22:03:34 +0300 Subject: [PATCH] fix lint --- build_tar/archive.py | 807 +++++++++++++++++++++------------------- build_tar/build_tar.py | 662 ++++++++++++++++---------------- build_tar/helpers.py | 128 +++---- package_manager/util.py | 4 +- 4 files changed, 825 insertions(+), 776 deletions(-) diff --git a/build_tar/archive.py b/build_tar/archive.py index c379dc15d..3addc4b6a 100644 --- a/build_tar/archive.py +++ b/build_tar/archive.py @@ -13,7 +13,6 @@ # limitations under the License. """Archive manipulation library for the Docker rules.""" -# pylint: disable=g-import-not-at-top import gzip import io import os @@ -21,14 +20,15 @@ import tarfile try: - import lzma # pylint: disable=g-import-not-at-top, unused-import - HAS_LZMA = True + import lzma # pylint: disable=unused-import + + HAS_LZMA = True except ImportError: - HAS_LZMA = False + HAS_LZMA = False # This is slightly a lie. We do support xz fallback through the xz tool, but # that is fragile. Users should stick to the expectations provided here. -COMPRESSIONS = ('', 'gz', 'bz2', 'xz') if HAS_LZMA else ('', 'gz', 'bz2') +COMPRESSIONS = ("", "gz", "bz2", "xz") if HAS_LZMA else ("", "gz", "bz2") # Use a deterministic mtime that doesn't confuse other programs. @@ -37,397 +37,428 @@ class SimpleArFile(object): - """A simple AR file reader. + """A simple AR file reader. - This enable to read AR file (System V variant) as described - in https://en.wikipedia.org/wiki/Ar_(Unix). + This enable to read AR file (System V variant) as described + in https://en.wikipedia.org/wiki/Ar_(Unix). - The standard usage of this class is: + The standard usage of this class is: - with SimpleArFile(filename) as ar: - nextFile = ar.next() - while nextFile: - print(nextFile.filename) + with SimpleArFile(filename) as ar: nextFile = ar.next() + while nextFile: + print(nextFile.filename) + nextFile = ar.next() - Upon error, this class will raise a ArError exception. - """ - - # TODO(dmarting): We should use a standard library instead but python 2.7 - # does not have AR reading library. - - class ArError(Exception): - pass - - class SimpleArFileEntry(object): - """Represent one entry in a AR archive. - - Attributes: - filename: the filename of the entry, as described in the archive. - timestamp: the timestamp of the file entry. - owner_id: numeric id of the user and group owning the file. - group_id: numeric id of the user and group owning the file. - mode: unix permission mode of the file - size: size of the file - data: the content of the file. + Upon error, this class will raise a ArError exception. """ - def __init__(self, f): - self.filename = f.read(16).decode('utf-8').strip() - if self.filename.endswith('/'): # SysV variant - self.filename = self.filename[:-1] - self.timestamp = int(f.read(12).strip()) - self.owner_id = int(f.read(6).strip()) - self.group_id = int(f.read(6).strip()) - self.mode = int(f.read(8).strip(), 8) - self.size = int(f.read(10).strip()) - pad = f.read(2) - if pad != b'\x60\x0a': - raise SimpleArFile.ArError('Invalid AR file header') - self.data = f.read(self.size) - - MAGIC_STRING = b'!\n' - - def __init__(self, filename): - self.filename = filename - - def __enter__(self): - self.f = open(self.filename, 'rb') - if self.f.read(len(self.MAGIC_STRING)) != self.MAGIC_STRING: - raise self.ArError('Not a ar file: ' + self.filename) - return self - - def __exit__(self, t, v, traceback): - self.f.close() - - def next(self): - """Read the next file. Returns None when reaching the end of file.""" - # AR sections are two bit aligned using new lines. - if self.f.tell() % 2 != 0: - self.f.read(1) - # An AR sections is at least 60 bytes. Some file might contains garbage - # bytes at the end of the archive, ignore them. - if self.f.tell() > os.fstat(self.f.fileno()).st_size - 60: - return None - return self.SimpleArFileEntry(self.f) + # TODO(dmarting): We should use a standard library instead but python 2.7 + # does not have AR reading library. + + class ArError(Exception): + pass + + class SimpleArFileEntry(object): + """Represent one entry in a AR archive. + + Attributes: + filename: the filename of the entry, as described in the archive. + timestamp: the timestamp of the file entry. + owner_id: numeric id of the user and group owning the file. + group_id: numeric id of the user and group owning the file. + mode: unix permission mode of the file + size: size of the file + data: the content of the file. + """ + + def __init__(self, f): + self.filename = f.read(16).decode("utf-8").strip() + if self.filename.endswith("/"): # SysV variant + self.filename = self.filename[:-1] + self.timestamp = int(f.read(12).strip()) + self.owner_id = int(f.read(6).strip()) + self.group_id = int(f.read(6).strip()) + self.mode = int(f.read(8).strip(), 8) + self.size = int(f.read(10).strip()) + pad = f.read(2) + if pad != b"\x60\x0a": + raise SimpleArFile.ArError("Invalid AR file header") + self.data = f.read(self.size) + + MAGIC_STRING = b"!\n" + + def __init__(self, filename): + self.f=None + self.filename = filename + + def __enter__(self): + self.f = open(self.filename, "rb") + if self.f.read(len(self.MAGIC_STRING)) != self.MAGIC_STRING: + raise self.ArError("Not a ar file: " + self.filename) + return self + + def __exit__(self, t, v, traceback): + self.f.close() + + def next(self): + """Read the next file. Returns None when reaching the end of file.""" + # AR sections are two bit aligned using new lines. + if self.f.tell() % 2 != 0: + self.f.read(1) + # An AR sections is at least 60 bytes. Some file might contains garbage + # bytes at the end of the archive, ignore them. + if self.f.tell() > os.fstat(self.f.fileno()).st_size - 60: + return None + return self.SimpleArFileEntry(self.f) class TarFileWriter(object): - """A wrapper to write tar files.""" - - class Error(Exception): - pass - - def __init__(self, - name, - compression='', - root_directory='.', - default_mtime=None, - preserve_tar_mtimes=True): - """TarFileWriter wraps tarfile.open(). - - Args: - name: the tar file name. - compression: compression type: bzip2, bz2, gz, tgz, xz, lzma. - root_directory: virtual root to prepend to elements in the archive. - default_mtime: default mtime to use for elements in the archive. - May be an integer or the value 'portable' to use the date - 2000-01-01, which is compatible with non *nix OSes'. - preserve_tar_mtimes: if true, keep file mtimes from input tar file. - """ - if compression in ['bzip2', 'bz2']: - mode = 'w:bz2' - else: - mode = 'w:' - self.gz = compression in ['tgz', 'gz'] - # Fallback to xz compression through xz. - self.use_xz_tool = False - if compression in ['xz', 'lzma']: - if HAS_LZMA: - mode = 'w:xz' - else: - self.use_xz_tool = True - self.name = name - self.root_directory = root_directory.rstrip('/').rstrip('\\') - self.root_directory = self.root_directory.replace('\\', '/') - self.preserve_mtime = preserve_tar_mtimes - if default_mtime is None: - self.default_mtime = 0 - elif default_mtime == 'portable': - self.default_mtime = PORTABLE_MTIME - else: - self.default_mtime = int(default_mtime) - - self.fileobj = None - if self.gz: - # The Tarfile class doesn't allow us to specify gzip's mtime attribute. - # Instead, we manually re-implement gzopen from tarfile.py and set mtime. - self.fileobj = gzip.GzipFile( - filename=name, mode='w', compresslevel=9, mtime=self.default_mtime) - self.tar = tarfile.open(name=name, mode=mode, fileobj=self.fileobj) - self.members = set([]) - self.directories = set([]) - - def __enter__(self): - return self - - def __exit__(self, t, v, traceback): - self.close() - - def add_dir(self, - name, - path, - uid=0, - gid=0, - uname='', - gname='', - mtime=None, - mode=None, - depth=100): - """Recursively add a directory. - - Args: - name: the ('/' delimited) path of the directory to add. - path: the (os.path.sep delimited) path of the directory to add. - uid: owner user identifier. - gid: owner group identifier. - uname: owner user names. - gname: owner group names. - mtime: modification time to put in the archive. - mode: unix permission mode of the file, default 0644 (0755). - depth: maximum depth to recurse in to avoid infinite loops - with cyclic mounts. - - Raises: - TarFileWriter.Error: when the recursion depth has exceeded the - `depth` argument. - """ - if not (name == self.root_directory or name.startswith('/') or - name.startswith(self.root_directory + '/')): - name = self.root_directory + '/' + name - if mtime is None: - mtime = self.default_mtime - path = path.rstrip('/').rstrip('\\') - if os.path.isdir(path): - # Remove trailing '/' (index -1 => last character) - if name[-1] in ('/', '\\'): - name = name[:-1] - # Add the x bit to directories to prevent non-traversable directories. - # The x bit is set to 1 only if the read bit is also set. - dirmode = (mode | ((0o444 & mode) >> 2)) if mode else mode - self.add_file(name + '/', - tarfile.DIRTYPE, - uid=uid, - gid=gid, - uname=uname, - gname=gname, - mtime=mtime, - mode=dirmode) - if depth <= 0: - raise self.Error('Recursion depth exceeded, probably in ' - 'an infinite directory loop.') - # Iterate over the sorted list of file so we get a deterministic result. - filelist = os.listdir(path) - filelist.sort() - for f in filelist: - new_name = name + '/' + f - new_path = os.path.join(path, f) - self.add_dir(new_name, new_path, uid, gid, uname, gname, mtime, mode, - depth - 1) - else: - self.add_file(name, - tarfile.REGTYPE, - file_content=path, - uid=uid, - gid=gid, - uname=uname, - gname=gname, - mtime=mtime, - mode=mode) - - def _addfile(self, info, fileobj=None): - """Add a file in the tar file if there is no conflict.""" - if not info.name.endswith('/') and info.type == tarfile.DIRTYPE: - # Enforce the ending / for directories so we correctly deduplicate. - info.name += '/' - if info.name not in self.members: - self.tar.addfile(info, fileobj) - self.members.add(info.name) - elif info.type != tarfile.DIRTYPE: - print('Duplicate file in archive: %s, ' - 'picking first occurrence' % info.name) - - def add_file(self, - name, - kind=tarfile.REGTYPE, - content=None, - link=None, - file_content=None, - uid=0, - gid=0, - uname='', - gname='', - mtime=None, - mode=None): - """Add a file to the current tar. - - Args: - name: the ('/' delimited) path of the file to add. - kind: the type of the file to add, see tarfile.*TYPE. - content: a textual content to put in the file. - link: if the file is a link, the destination of the link. - file_content: file to read the content from. Provide either this - one or `content` to specifies a content for the file. - uid: owner user identifier. - gid: owner group identifier. - uname: owner user names. - gname: owner group names. - mtime: modification time to put in the archive. - mode: unix permission mode of the file, default 0644 (0755). - """ - name = name.replace('\\', '/') - if file_content and os.path.isdir(file_content): - # Recurse into directory - self.add_dir(name, file_content, uid, gid, uname, gname, mtime, mode) - return - if not (name == self.root_directory or name.startswith('/') or - name.startswith(self.root_directory + '/')): - name = self.root_directory + '/' + name - if kind == tarfile.DIRTYPE: - name = name.rstrip('/').rstrip('\\') - if name in self.directories: - return - if mtime is None: - mtime = self.default_mtime - - components = name.rsplit('/', 1) - if len(components) > 1: - d = components[0] - self.add_file(d, - tarfile.DIRTYPE, - uid=uid, - gid=gid, - uname=uname, - gname=gname, - mtime=mtime, - mode=0o755) - tarinfo = tarfile.TarInfo(name) - tarinfo.mtime = mtime - tarinfo.uid = uid - tarinfo.gid = gid - tarinfo.uname = uname - tarinfo.gname = gname - tarinfo.type = kind - if mode is None: - tarinfo.mode = 0o644 if kind == tarfile.REGTYPE else 0o755 - else: - tarinfo.mode = mode - if link: - tarinfo.linkname = link - if content: - content_bytes = content.encode('utf-8') - tarinfo.size = len(content_bytes) - self._addfile(tarinfo, io.BytesIO(content_bytes)) - elif file_content: - with open(file_content, 'rb') as f: - tarinfo.size = os.fstat(f.fileno()).st_size - self._addfile(tarinfo, f) - else: - if kind == tarfile.DIRTYPE: - self.directories.add(name) - self._addfile(tarinfo) - - def add_tar(self, - tar, - rootuid=None, - rootgid=None, - numeric=False, - name_filter=None, - root=None): - """Merge a tar content into the current tar, stripping timestamp. - - Args: - tar: the name of tar to extract and put content into the current tar. - rootuid: user id that we will pretend is root (replaced by uid 0). - rootgid: group id that we will pretend is root (replaced by gid 0). - numeric: set to true to strip out name of owners (and just use the - numeric values). - name_filter: filter out file by names. If not none, this method will be - called for each file to add, given the name and should return true if - the file is to be added to the final tar and false otherwise. - root: place all non-absolute content under given root directory, if not - None. - - Raises: - TarFileWriter.Error: if an error happens when uncompressing the tar file. - """ - if root and root[0] not in ['/', '.']: - # Root prefix should start with a '/', adds it if missing - root = '/' + root - intar = tarfile.open(name=tar, mode='r:*') - for tarinfo in intar: - if name_filter is None or name_filter(tarinfo.name): - if not self.preserve_mtime: - tarinfo.mtime = self.default_mtime - if rootuid is not None and tarinfo.uid == rootuid: - tarinfo.uid = 0 - tarinfo.uname = 'root' - if rootgid is not None and tarinfo.gid == rootgid: - tarinfo.gid = 0 - tarinfo.gname = 'root' - if numeric: - tarinfo.uname = '' - tarinfo.gname = '' - - name = tarinfo.name - if (not name.startswith('/') and - not name.startswith(self.root_directory)): - name = self.root_directory + '/' + name - if root is not None: - if name.startswith('.'): - name = '.' + root + name.lstrip('.') - # Add root dir with same permissions if missing. Note that - # add_file deduplicates directories and is safe to call here. - self.add_file('.' + root, - tarfile.DIRTYPE, - uid=tarinfo.uid, - gid=tarinfo.gid, - uname=tarinfo.uname, - gname=tarinfo.gname, - mtime=tarinfo.mtime, - mode=0o755) - # Relocate internal hardlinks as well to avoid breaking them. - link = tarinfo.linkname - if link.startswith('.') and tarinfo.type == tarfile.LNKTYPE: - tarinfo.linkname = '.' + root + link.lstrip('.') - tarinfo.name = name - - if tarinfo.isfile(): - # use extractfile(tarinfo) instead of tarinfo.name to preserve - # seek position in intar - self._addfile(tarinfo, intar.extractfile(tarinfo)) + """A wrapper to write tar files.""" + + class Error(Exception): + pass + + def __init__( + self, + name, + compression="", + root_directory=".", + default_mtime=None, + preserve_tar_mtimes=True, + ): + """TarFileWriter wraps tarfile.open(). + + Args: + name: the tar file name. + compression: compression type: bzip2, bz2, gz, tgz, xz, lzma. + root_directory: virtual root to prepend to elements in the archive. + default_mtime: default mtime to use for elements in the archive. + May be an integer or the value 'portable' to use the date + 2000-01-01, which is compatible with non *nix OSes'. + preserve_tar_mtimes: if true, keep file mtimes from input tar file. + """ + if compression in ["bzip2", "bz2"]: + mode = "w:bz2" else: - self._addfile(tarinfo) - intar.close() - - def close(self): - """Close the output tar file. - - This class should not be used anymore after calling that method. - - Raises: - TarFileWriter.Error: if an error happens when compressing the output file. - """ - self.tar.close() - # Close the gzip file object if necessary. - if self.fileobj: - self.fileobj.close() - if self.use_xz_tool: - # Support xz compression through xz... until we can use Py3 - if subprocess.call('which xz', shell=True, stdout=subprocess.PIPE): - raise self.Error('Cannot handle .xz and .lzma compression: ' - 'xz not found.') - subprocess.call( - 'mv {0} {0}.d && xz -z {0}.d && mv {0}.d.xz {0}'.format(self.name), - shell=True, - stdout=subprocess.PIPE) + mode = "w:" + self.gz = compression in ["tgz", "gz"] + # Fallback to xz compression through xz. + self.use_xz_tool = False + if compression in ["xz", "lzma"]: + if HAS_LZMA: + mode = "w:xz" + else: + self.use_xz_tool = True + self.name = name + self.root_directory = root_directory.rstrip("/").rstrip("\\") + self.root_directory = self.root_directory.replace("\\", "/") + self.preserve_mtime = preserve_tar_mtimes + if default_mtime is None: + self.default_mtime = 0 + elif default_mtime == "portable": + self.default_mtime = PORTABLE_MTIME + else: + self.default_mtime = int(default_mtime) + + self.fileobj = None + if self.gz: + # The Tarfile class doesn't allow us to specify gzip's mtime attribute. + # Instead, we manually re-implement gzopen from tarfile.py and set mtime. + self.fileobj = gzip.GzipFile( + filename=name, mode="w", compresslevel=9, mtime=self.default_mtime + ) + self.tar = tarfile.open(name=name, mode=mode, fileobj=self.fileobj) + self.members = set([]) + self.directories = set([]) + + def __enter__(self): + return self + + def __exit__(self, t, v, traceback): + self.close() + + def add_dir( + self, + name, + path, + uid=0, + gid=0, + uname="", + gname="", + mtime=None, + mode=None, + depth=100, + ): + """Recursively add a directory. + + Args: + name: the ('/' delimited) path of the directory to add. + path: the (os.path.sep delimited) path of the directory to add. + uid: owner user identifier. + gid: owner group identifier. + uname: owner user names. + gname: owner group names. + mtime: modification time to put in the archive. + mode: unix permission mode of the file, default 0644 (0755). + depth: maximum depth to recurse in to avoid infinite loops + with cyclic mounts. + + Raises: + TarFileWriter.Error: when the recursion depth has exceeded the + `depth` argument. + """ + if not ( + name == self.root_directory + or name.startswith("/") + or name.startswith(self.root_directory + "/") + ): + name = self.root_directory + "/" + name + if mtime is None: + mtime = self.default_mtime + path = path.rstrip("/").rstrip("\\") + if os.path.isdir(path): + # Remove trailing '/' (index -1 => last character) + if name[-1] in ("/", "\\"): + name = name[:-1] + # Add the x bit to directories to prevent non-traversable directories. + # The x bit is set to 1 only if the read bit is also set. + dirmode = (mode | ((0o444 & mode) >> 2)) if mode else mode + self.add_file( + name + "/", + tarfile.DIRTYPE, + uid=uid, + gid=gid, + uname=uname, + gname=gname, + mtime=mtime, + mode=dirmode, + ) + if depth <= 0: + raise self.Error( + "Recursion depth exceeded, probably in " + "an infinite directory loop." + ) + # Iterate over the sorted list of file so we get a deterministic result. + filelist = os.listdir(path) + filelist.sort() + for f in filelist: + new_name = name + "/" + f + new_path = os.path.join(path, f) + self.add_dir( + new_name, new_path, uid, gid, uname, gname, mtime, mode, depth - 1 + ) + else: + self.add_file( + name, + tarfile.REGTYPE, + file_content=path, + uid=uid, + gid=gid, + uname=uname, + gname=gname, + mtime=mtime, + mode=mode, + ) + + def _addfile(self, info, fileobj=None): + """Add a file in the tar file if there is no conflict.""" + if not info.name.endswith("/") and info.type == tarfile.DIRTYPE: + # Enforce the ending / for directories so we correctly deduplicate. + info.name += "/" + if info.name not in self.members: + self.tar.addfile(info, fileobj) + self.members.add(info.name) + elif info.type != tarfile.DIRTYPE: + print( + "Duplicate file in archive: %s, " "picking first occurrence" % info.name + ) + + def add_file( + self, + name, + kind=tarfile.REGTYPE, + content=None, + link=None, + file_content=None, + uid=0, + gid=0, + uname="", + gname="", + mtime=None, + mode=None, + ): + """Add a file to the current tar. + + Args: + name: the ('/' delimited) path of the file to add. + kind: the type of the file to add, see tarfile.*TYPE. + content: a textual content to put in the file. + link: if the file is a link, the destination of the link. + file_content: file to read the content from. Provide either this + one or `content` to specifies a content for the file. + uid: owner user identifier. + gid: owner group identifier. + uname: owner user names. + gname: owner group names. + mtime: modification time to put in the archive. + mode: unix permission mode of the file, default 0644 (0755). + """ + name = name.replace("\\", "/") + if file_content and os.path.isdir(file_content): + # Recurse into directory + self.add_dir(name, file_content, uid, gid, uname, gname, mtime, mode) + return + if not ( + name == self.root_directory + or name.startswith("/") + or name.startswith(self.root_directory + "/") + ): + name = self.root_directory + "/" + name + if kind == tarfile.DIRTYPE: + name = name.rstrip("/").rstrip("\\") + if name in self.directories: + return + if mtime is None: + mtime = self.default_mtime + + components = name.rsplit("/", 1) + if len(components) > 1: + d = components[0] + self.add_file( + d, + tarfile.DIRTYPE, + uid=uid, + gid=gid, + uname=uname, + gname=gname, + mtime=mtime, + mode=0o755, + ) + tarinfo = tarfile.TarInfo(name) + tarinfo.mtime = mtime + tarinfo.uid = uid + tarinfo.gid = gid + tarinfo.uname = uname + tarinfo.gname = gname + tarinfo.type = kind + if mode is None: + tarinfo.mode = 0o644 if kind == tarfile.REGTYPE else 0o755 + else: + tarinfo.mode = mode + if link: + tarinfo.linkname = link + if content: + content_bytes = content.encode("utf-8") + tarinfo.size = len(content_bytes) + self._addfile(tarinfo, io.BytesIO(content_bytes)) + elif file_content: + with open(file_content, "rb") as f: + tarinfo.size = os.fstat(f.fileno()).st_size + self._addfile(tarinfo, f) + else: + if kind == tarfile.DIRTYPE: + self.directories.add(name) + self._addfile(tarinfo) + + def add_tar( + self, + tar, + rootuid=None, + rootgid=None, + numeric=False, + name_filter=None, + root=None, + ): + """Merge a tar content into the current tar, stripping timestamp. + + Args: + tar: the name of tar to extract and put content into the current tar. + rootuid: user id that we will pretend is root (replaced by uid 0). + rootgid: group id that we will pretend is root (replaced by gid 0). + numeric: set to true to strip out name of owners (and just use the + numeric values). + name_filter: filter out file by names. If not none, this method will be + called for each file to add, given the name and should return true if + the file is to be added to the final tar and false otherwise. + root: place all non-absolute content under given root directory, if not + None. + + Raises: + TarFileWriter.Error: if an error happens when uncompressing the tar file. + """ + if root and root[0] not in ["/", "."]: + # Root prefix should start with a '/', adds it if missing + root = "/" + root + intar = tarfile.open(name=tar, mode="r:*") + for tarinfo in intar: + if name_filter is None or name_filter(tarinfo.name): + if not self.preserve_mtime: + tarinfo.mtime = self.default_mtime + if rootuid is not None and tarinfo.uid == rootuid: + tarinfo.uid = 0 + tarinfo.uname = "root" + if rootgid is not None and tarinfo.gid == rootgid: + tarinfo.gid = 0 + tarinfo.gname = "root" + if numeric: + tarinfo.uname = "" + tarinfo.gname = "" + + name = tarinfo.name + if not name.startswith("/") and not name.startswith( + self.root_directory + ): + name = self.root_directory + "/" + name + if root is not None: + if name.startswith("."): + name = "." + root + name.lstrip(".") + # Add root dir with same permissions if missing. Note that + # add_file deduplicates directories and is safe to call here. + self.add_file( + "." + root, + tarfile.DIRTYPE, + uid=tarinfo.uid, + gid=tarinfo.gid, + uname=tarinfo.uname, + gname=tarinfo.gname, + mtime=tarinfo.mtime, + mode=0o755, + ) + # Relocate internal hardlinks as well to avoid breaking them. + link = tarinfo.linkname + if link.startswith(".") and tarinfo.type == tarfile.LNKTYPE: + tarinfo.linkname = "." + root + link.lstrip(".") + tarinfo.name = name + + if tarinfo.isfile(): + # use extractfile(tarinfo) instead of tarinfo.name to preserve + # seek position in intar + self._addfile(tarinfo, intar.extractfile(tarinfo)) + else: + self._addfile(tarinfo) + intar.close() + + def close(self): + """Close the output tar file. + + This class should not be used anymore after calling that method. + + Raises: + TarFileWriter.Error: if an error happens when compressing the output file. + """ + self.tar.close() + # Close the gzip file object if necessary. + if self.fileobj: + self.fileobj.close() + if self.use_xz_tool: + # Support xz compression through xz... until we can use Py3 + if subprocess.call("which xz", shell=True, stdout=subprocess.PIPE): + raise self.Error( + "Cannot handle .xz and .lzma compression: " "xz not found." + ) + subprocess.call( + "mv {0} {0}.d && xz -z {0}.d && mv {0}.d.xz {0}".format(self.name), + shell=True, + stdout=subprocess.PIPE, + ) diff --git a/build_tar/build_tar.py b/build_tar/build_tar.py index f861ea3da..08cf93783 100644 --- a/build_tar/build_tar.py +++ b/build_tar/build_tar.py @@ -19,331 +19,349 @@ import tarfile import tempfile -import archive -import helpers +import archive # pylint: disable=import-error +import helpers # pylint: disable=import-error class TarFile(object): - """A class to generates a TAR file.""" - - class DebError(Exception): - pass - - def __init__(self, output, directory, compression, root_directory, - default_mtime): - self.directory = directory - self.output = output - self.compression = compression - self.root_directory = root_directory - self.default_mtime = default_mtime - - def __enter__(self): - self.tarfile = archive.TarFileWriter( - self.output, - self.compression, - self.root_directory, - default_mtime=self.default_mtime) - return self - - def __exit__(self, t, v, traceback): - self.tarfile.close() - - def add_file(self, f, destfile, mode=None, ids=None, names=None): - """Add a file to the tar file. - - Args: - f: the file to add to the layer - destfile: the name of the file in the layer - mode: force to set the specified mode, by default the value from the - source is taken. - ids: (uid, gid) for the file to set ownership - names: (username, groupname) for the file to set ownership. `f` will be - copied to `self.directory/destfile` in the layer. - """ - dest = destfile.lstrip('/') # Remove leading slashes - if self.directory and self.directory != '/': - dest = self.directory.lstrip('/') + '/' + dest - # If mode is unspecified, derive the mode from the file's mode. - if mode is None: - mode = 0o755 if os.access(f, os.X_OK) else 0o644 - if ids is None: - ids = (0, 0) - if names is None: - names = ('', '') - # Note: normpath changes / to \ on windows, but we expect / style paths. - dest = os.path.normpath(dest).replace(os.path.sep, '/') - self.tarfile.add_file( - dest, - file_content=f, - mode=mode, - uid=ids[0], - gid=ids[1], - uname=names[0], - gname=names[1]) - - def add_empty_file(self, - destfile, - mode=None, - ids=None, - names=None, - kind=tarfile.REGTYPE): - """Add a file to the tar file. - - Args: - destfile: the name of the file in the layer - mode: force to set the specified mode, defaults to 644 - ids: (uid, gid) for the file to set ownership - names: (username, groupname) for the file to set ownership. - kind: type of the file. tarfile.DIRTYPE for directory. An empty file - will be created as `destfile` in the layer. - """ - dest = destfile.lstrip('/') # Remove leading slashes - # If mode is unspecified, assume read only - if mode is None: - mode = 0o644 - if ids is None: - ids = (0, 0) - if names is None: - names = ('', '') - dest = os.path.normpath(dest).replace(os.path.sep, '/') - self.tarfile.add_file( - dest, - content='' if kind == tarfile.REGTYPE else None, - kind=kind, - mode=mode, - uid=ids[0], - gid=ids[1], - uname=names[0], - gname=names[1]) - - def add_empty_dir(self, destpath, mode=None, ids=None, names=None): - """Add a directory to the tar file. - - Args: - destpath: the name of the directory in the layer - mode: force to set the specified mode, defaults to 644 - ids: (uid, gid) for the file to set ownership - names: (username, groupname) for the file to set ownership. An empty - file will be created as `destfile` in the layer. - """ - self.add_empty_file( - destpath, mode=mode, ids=ids, names=names, kind=tarfile.DIRTYPE) - - def add_empty_root_dir(self, destpath, mode=None, ids=None, names=None): - """Add a directory to the root of the tar file. - - Args: - destpath: the name of the directory in the layer - mode: force to set the specified mode, defaults to 644 - ids: (uid, gid) for the file to set ownership - names: (username, groupname) for the file to set ownership. An empty - directory will be created as `destfile` in the root layer. - """ - original_root_directory = self.tarfile.root_directory - self.tarfile.root_directory = destpath - self.add_empty_dir(destpath, mode=mode, ids=ids, names=names) - self.tarfile.root_directory = original_root_directory - - def add_tar(self, tar): - """Merge a tar file into the destination tar file. - - All files presents in that tar will be added to the output file - under self.directory/path. No user name nor group name will be - added to the output. - - Args: - tar: the tar file to add - """ - root = None - if self.directory and self.directory != '/': - root = self.directory - self.tarfile.add_tar(tar, numeric=True, root=root) - - def add_link(self, symlink, destination): - """Add a symbolic link pointing to `destination`. - - Args: - symlink: the name of the symbolic link to add. - destination: where the symbolic link point to. - """ - symlink = os.path.normpath(symlink).replace(os.path.sep, '/') - self.tarfile.add_file(symlink, tarfile.SYMTYPE, link=destination) - - def add_deb(self, deb): - """Extract a debian package in the output tar. - - All files presents in that debian package will be added to the - output tar under the same paths. No user name nor group names will - be added to the output. - - Args: - deb: the tar file to add - - Raises: - DebError: if the format of the deb archive is incorrect. - """ - with archive.SimpleArFile(deb) as arfile: - current = next(arfile) - while current and not current.filename.startswith('data.'): - current = next(arfile) - if not current: - raise self.DebError(deb + ' does not contains a data file!') - tmpfile = tempfile.mkstemp(suffix=os.path.splitext(current.filename)[-1]) - with open(tmpfile[1], 'wb') as f: - f.write(current.data) - self.add_tar(tmpfile[1]) - os.remove(tmpfile[1]) + """A class to generates a TAR file.""" + + class DebError(Exception): + pass + + def __init__(self, output, directory, compression, root_directory, default_mtime): + self.directory = directory + self.output = output + self.compression = compression + self.root_directory = root_directory + self.default_mtime = default_mtime + self.tarfile = None + + def __enter__(self): + self.tarfile = archive.TarFileWriter( + self.output, + self.compression, + self.root_directory, + default_mtime=self.default_mtime, + ) + return self + + def __exit__(self, t, v, traceback): + self.tarfile.close() + + def add_file(self, f, destfile, mode=None, ids=None, names=None): + """Add a file to the tar file. + + Args: + f: the file to add to the layer + destfile: the name of the file in the layer + mode: force to set the specified mode, by default the value from the + source is taken. + ids: (uid, gid) for the file to set ownership + names: (username, groupname) for the file to set ownership. `f` will be + copied to `self.directory/destfile` in the layer. + """ + dest = destfile.lstrip("/") # Remove leading slashes + if self.directory and self.directory != "/": + dest = self.directory.lstrip("/") + "/" + dest + # If mode is unspecified, derive the mode from the file's mode. + if mode is None: + mode = 0o755 if os.access(f, os.X_OK) else 0o644 + if ids is None: + ids = (0, 0) + if names is None: + names = ("", "") + # Note: normpath changes / to \ on windows, but we expect / style paths. + dest = os.path.normpath(dest).replace(os.path.sep, "/") + self.tarfile.add_file( + dest, + file_content=f, + mode=mode, + uid=ids[0], + gid=ids[1], + uname=names[0], + gname=names[1], + ) + + def add_empty_file( + self, destfile, mode=None, ids=None, names=None, kind=tarfile.REGTYPE + ): + """Add a file to the tar file. + + Args: + destfile: the name of the file in the layer + mode: force to set the specified mode, defaults to 644 + ids: (uid, gid) for the file to set ownership + names: (username, groupname) for the file to set ownership. + kind: type of the file. tarfile.DIRTYPE for directory. An empty file + will be created as `destfile` in the layer. + """ + dest = destfile.lstrip("/") # Remove leading slashes + # If mode is unspecified, assume read only + if mode is None: + mode = 0o644 + if ids is None: + ids = (0, 0) + if names is None: + names = ("", "") + dest = os.path.normpath(dest).replace(os.path.sep, "/") + self.tarfile.add_file( + dest, + content="" if kind == tarfile.REGTYPE else None, + kind=kind, + mode=mode, + uid=ids[0], + gid=ids[1], + uname=names[0], + gname=names[1], + ) + + def add_empty_dir(self, destpath, mode=None, ids=None, names=None): + """Add a directory to the tar file. + + Args: + destpath: the name of the directory in the layer + mode: force to set the specified mode, defaults to 644 + ids: (uid, gid) for the file to set ownership + names: (username, groupname) for the file to set ownership. An empty + file will be created as `destfile` in the layer. + """ + self.add_empty_file( + destpath, mode=mode, ids=ids, names=names, kind=tarfile.DIRTYPE + ) + + def add_empty_root_dir(self, destpath, mode=None, ids=None, names=None): + """Add a directory to the root of the tar file. + + Args: + destpath: the name of the directory in the layer + mode: force to set the specified mode, defaults to 644 + ids: (uid, gid) for the file to set ownership + names: (username, groupname) for the file to set ownership. An empty + directory will be created as `destfile` in the root layer. + """ + original_root_directory = self.tarfile.root_directory + self.tarfile.root_directory = destpath + self.add_empty_dir(destpath, mode=mode, ids=ids, names=names) + self.tarfile.root_directory = original_root_directory + + def add_tar(self, tar): + """Merge a tar file into the destination tar file. + + All files presents in that tar will be added to the output file + under self.directory/path. No user name nor group name will be + added to the output. + + Args: + tar: the tar file to add + """ + root = None + if self.directory and self.directory != "/": + root = self.directory + self.tarfile.add_tar(tar, numeric=True, root=root) + + def add_link(self, symlink, destination): + """Add a symbolic link pointing to `destination`. + + Args: + symlink: the name of the symbolic link to add. + destination: where the symbolic link point to. + """ + symlink = os.path.normpath(symlink).replace(os.path.sep, "/") + self.tarfile.add_file(symlink, tarfile.SYMTYPE, link=destination) + + def add_deb(self, deb): + """Extract a debian package in the output tar. + + All files presents in that debian package will be added to the + output tar under the same paths. No user name nor group names will + be added to the output. + + Args: + deb: the tar file to add + + Raises: + DebError: if the format of the deb archive is incorrect. + """ + with archive.SimpleArFile(deb) as arfile: + current = next(arfile) + while current and not current.filename.startswith("data."): + current = next(arfile) + if not current: + raise self.DebError(deb + " does not contains a data file!") + tmpfile = tempfile.mkstemp(suffix=os.path.splitext(current.filename)[-1]) + with open(tmpfile[1], "wb") as f: + f.write(current.data) + self.add_tar(tmpfile[1]) + os.remove(tmpfile[1]) def main(): - parser = argparse.ArgumentParser( - description='Helper for building tar packages', - fromfile_prefix_chars='@') - parser.add_argument('--output', required=True, - help='The output file, mandatory.') - parser.add_argument('--file', action='append', - help='A file to add to the layer.') - parser.add_argument('--manifest', - help='JSON manifest of contents to add to the layer.') - parser.add_argument('--mode', - help='Force the mode on the added files (in octal).') - parser.add_argument( - '--mtime', - help='Set mtime on tar file entries. May be an integer or the' - ' value "portable", to get the value 2000-01-01, which is' - ' is usable with non *nix OSes.') - parser.add_argument('--empty_file', action='append', - help='An empty file to add to the layer.') - parser.add_argument('--empty_dir', action='append', - help='An empty dir to add to the layer.') - parser.add_argument('--empty_root_dir', action='append', - help='An empty dir to add to the layer.') - parser.add_argument('--tar', action='append', - help='A tar file to add to the layer') - parser.add_argument('--deb', action='append', - help='A debian package to add to the layer') - parser.add_argument( - '--link', action='append', - help='Add a symlink a inside the layer pointing to b if a:b is specified') - # TODO(aiuto): Add back in the validation - # flags.register_validator( - # 'link', - # lambda l: all(value.find(':') > 0 for value in l), - # message='--link value should contains a : separator') - parser.add_argument( - '--directory', - help='Directory in which to store the file inside the layer') - parser.add_argument('--compression', - help='Compression (`gz` or `bz2`), default is none.') - parser.add_argument( - '--modes', action='append', - help='Specific mode to apply to specific file (from the file argument),' - ' e.g., path/to/file=0455.') - parser.add_argument( - '--owners', action='append', - help='Specify the numeric owners of individual files, ' - 'e.g. path/to/file=0.0.') - parser.add_argument( - '--owner', default='0.0', - help='Specify the numeric default owner of all files,' - ' e.g., 0.0') - parser.add_argument( - '--owner_name', - help='Specify the owner name of all files, e.g. root.root.') - parser.add_argument( - '--owner_names', action='append', - help='Specify the owner names of individual files, e.g. ' - 'path/to/file=root.root.') - parser.add_argument('--root_directory', default='./', - help='Default root directory is named "."') - options = parser.parse_args() - - # Parse modes arguments - default_mode = None - if options.mode: - # Convert from octal - default_mode = int(options.mode, 8) - - mode_map = {} - if options.modes: - for filemode in options.modes: - (f, mode) = helpers.SplitNameValuePairAtSeparator(filemode, '=') - if f[0] == '/': - f = f[1:] - mode_map[f] = int(mode, 8) - - default_ownername = ('', '') - if options.owner_name: - default_ownername = options.owner_name.split('.', 1) - names_map = {} - if options.owner_names: - for file_owner in options.owner_names: - (f, owner) = helpers.SplitNameValuePairAtSeparator(file_owner, '=') - (user, group) = owner.split('.', 1) - if f[0] == '/': - f = f[1:] - names_map[f] = (user, group) - - default_ids = options.owner.split('.', 1) - default_ids = (int(default_ids[0]), int(default_ids[1])) - ids_map = {} - if options.owners: - for file_owner in options.owners: - (f, owner) = helpers.SplitNameValuePairAtSeparator(file_owner, '=') - (user, group) = owner.split('.', 1) - if f[0] == '/': - f = f[1:] - ids_map[f] = (int(user), int(group)) - - # Add objects to the tar file - with TarFile( - options.output, helpers.GetFlagValue(options.directory), - options.compression, options.root_directory, options.mtime) as output: - - def file_attributes(filename): - if filename.startswith('/'): - filename = filename[1:] - return { - 'mode': mode_map.get(filename, default_mode), - 'ids': ids_map.get(filename, default_ids), - 'names': names_map.get(filename, default_ownername), - } - - if options.manifest: - with open(options.manifest, 'r') as manifest_fp: - manifest = json.load(manifest_fp) - for f in manifest.get('files', []): - output.add_file(f['src'], f['dst'], **file_attributes(f['dst'])) - for f in manifest.get('empty_files', []): - output.add_empty_file(f, **file_attributes(f)) - for d in manifest.get('empty_dirs', []): - output.add_empty_dir(d, **file_attributes(d)) - for d in manifest.get('empty_root_dirs', []): - output.add_empty_root_dir(d, **file_attributes(d)) - for f in manifest.get('symlinks', []): - output.add_link(f['linkname'], f['target']) - for tar in manifest.get('tars', []): - output.add_tar(tar) - for deb in manifest.get('debs', []): - output.add_deb(deb) - - for f in options.file or []: - (inf, tof) = helpers.SplitNameValuePairAtSeparator(f, '=') - output.add_file(inf, tof, **file_attributes(tof)) - for f in options.empty_file or []: - output.add_empty_file(f, **file_attributes(f)) - for f in options.empty_dir or []: - output.add_empty_dir(f, **file_attributes(f)) - for f in options.empty_root_dir or []: - output.add_empty_root_dir(f, **file_attributes(f)) - for tar in options.tar or []: - output.add_tar(tar) - for deb in options.deb or []: - output.add_deb(deb) - for link in options.link or []: - l = helpers.SplitNameValuePairAtSeparator(link, ':') - output.add_link(l[0], l[1]) - - -if __name__ == '__main__': - main() + parser = argparse.ArgumentParser( + description="Helper for building tar packages", fromfile_prefix_chars="@" + ) + parser.add_argument("--output", required=True, help="The output file, mandatory.") + parser.add_argument("--file", action="append", help="A file to add to the layer.") + parser.add_argument( + "--manifest", help="JSON manifest of contents to add to the layer." + ) + parser.add_argument("--mode", help="Force the mode on the added files (in octal).") + parser.add_argument( + "--mtime", + help="Set mtime on tar file entries. May be an integer or the" + ' value "portable", to get the value 2000-01-01, which is' + " is usable with non *nix OSes.", + ) + parser.add_argument( + "--empty_file", action="append", help="An empty file to add to the layer." + ) + parser.add_argument( + "--empty_dir", action="append", help="An empty dir to add to the layer." + ) + parser.add_argument( + "--empty_root_dir", action="append", help="An empty dir to add to the layer." + ) + parser.add_argument("--tar", action="append", help="A tar file to add to the layer") + parser.add_argument( + "--deb", action="append", help="A debian package to add to the layer" + ) + parser.add_argument( + "--link", + action="append", + help="Add a symlink a inside the layer pointing to b if a:b is specified", + ) + # TODO(aiuto): Add back in the validation + # flags.register_validator( + # 'link', + # lambda l: all(value.find(':') > 0 for value in l), + # message='--link value should contains a : separator') + parser.add_argument( + "--directory", help="Directory in which to store the file inside the layer" + ) + parser.add_argument( + "--compression", help="Compression (`gz` or `bz2`), default is none." + ) + parser.add_argument( + "--modes", + action="append", + help="Specific mode to apply to specific file (from the file argument)," + " e.g., path/to/file=0455.", + ) + parser.add_argument( + "--owners", + action="append", + help="Specify the numeric owners of individual files, " + "e.g. path/to/file=0.0.", + ) + parser.add_argument( + "--owner", + default="0.0", + help="Specify the numeric default owner of all files," " e.g., 0.0", + ) + parser.add_argument( + "--owner_name", help="Specify the owner name of all files, e.g. root.root." + ) + parser.add_argument( + "--owner_names", + action="append", + help="Specify the owner names of individual files, e.g. " + "path/to/file=root.root.", + ) + parser.add_argument( + "--root_directory", default="./", help='Default root directory is named "."' + ) + options = parser.parse_args() + + # Parse modes arguments + default_mode = None + if options.mode: + # Convert from octal + default_mode = int(options.mode, 8) + + mode_map = {} + if options.modes: + for filemode in options.modes: + (f, mode) = helpers.SplitNameValuePairAtSeparator(filemode, "=") + if f[0] == "/": + f = f[1:] + mode_map[f] = int(mode, 8) + + default_ownername = ("", "") + if options.owner_name: + default_ownername = options.owner_name.split(".", 1) + names_map = {} + if options.owner_names: + for file_owner in options.owner_names: + (f, owner) = helpers.SplitNameValuePairAtSeparator(file_owner, "=") + (user, group) = owner.split(".", 1) + if f[0] == "/": + f = f[1:] + names_map[f] = (user, group) + + default_ids = options.owner.split(".", 1) + default_ids = (int(default_ids[0]), int(default_ids[1])) + ids_map = {} + if options.owners: + for file_owner in options.owners: + (f, owner) = helpers.SplitNameValuePairAtSeparator(file_owner, "=") + (user, group) = owner.split(".", 1) + if f[0] == "/": + f = f[1:] + ids_map[f] = (int(user), int(group)) + + # Add objects to the tar file + with TarFile( + options.output, + helpers.GetFlagValue(options.directory), + options.compression, + options.root_directory, + options.mtime, + ) as output: + + def file_attributes(filename): + if filename.startswith("/"): + filename = filename[1:] + return { + "mode": mode_map.get(filename, default_mode), + "ids": ids_map.get(filename, default_ids), + "names": names_map.get(filename, default_ownername), + } + + if options.manifest: + with open(options.manifest, "r", encoding="utf-8") as manifest_fp: + manifest = json.load(manifest_fp) + for f in manifest.get("files", []): + output.add_file(f["src"], f["dst"], **file_attributes(f["dst"])) + for f in manifest.get("empty_files", []): + output.add_empty_file(f, **file_attributes(f)) + for d in manifest.get("empty_dirs", []): + output.add_empty_dir(d, **file_attributes(d)) + for d in manifest.get("empty_root_dirs", []): + output.add_empty_root_dir(d, **file_attributes(d)) + for f in manifest.get("symlinks", []): + output.add_link(f["linkname"], f["target"]) + for tar in manifest.get("tars", []): + output.add_tar(tar) + for deb in manifest.get("debs", []): + output.add_deb(deb) + + for f in options.file or []: + (inf, tof) = helpers.SplitNameValuePairAtSeparator(f, "=") + output.add_file(inf, tof, **file_attributes(tof)) + for f in options.empty_file or []: + output.add_empty_file(f, **file_attributes(f)) + for f in options.empty_dir or []: + output.add_empty_dir(f, **file_attributes(f)) + for f in options.empty_root_dir or []: + output.add_empty_root_dir(f, **file_attributes(f)) + for tar in options.tar or []: + output.add_tar(tar) + for deb in options.deb or []: + output.add_deb(deb) + for link in options.link or []: + l = helpers.SplitNameValuePairAtSeparator(link, ":") + output.add_link(l[0], l[1]) + + +if __name__ == "__main__": + main() diff --git a/build_tar/helpers.py b/build_tar/helpers.py index 0ad8e9347..75635dcdd 100644 --- a/build_tar/helpers.py +++ b/build_tar/helpers.py @@ -14,76 +14,78 @@ import os import sys + def SplitNameValuePairAtSeparator(arg, sep): - """Split a string at the first unquoted occurrence of a character. + """Split a string at the first unquoted occurrence of a character. + + Split the string arg at the first unquoted occurrence of the character c. + Here, in the first part of arg, the backslash is considered the + quoting character indicating that the next character is to be + added literally to the first part, even if it is the split character. - Split the string arg at the first unquoted occurrence of the character c. - Here, in the first part of arg, the backslash is considered the - quoting character indicating that the next character is to be - added literally to the first part, even if it is the split character. + Args: + arg: the string to be split + sep: the character at which to split - Args: - arg: the string to be split - sep: the character at which to split + Returns: + The unquoted string before the separator and the string after the + separator. + """ + head = "" + i = 0 + while i < len(arg): + if arg[i] == sep: + return (head, arg[i + 1 :]) + elif arg[i] == "\\": + i += 1 + if i == len(arg): + # dangling quotation symbol + return (head, "") + else: + head += arg[i] + else: + head += arg[i] + i += 1 + # if we leave the loop, the character sep was not found unquoted + return (head, "") - Returns: - The unquoted string before the separator and the string after the - separator. - """ - head = '' - i = 0 - while i < len(arg): - if arg[i] == sep: - return (head, arg[i + 1:]) - elif arg[i] == '\\': - i += 1 - if i == len(arg): - # dangling quotation symbol - return (head, '') - else: - head += arg[i] - else: - head += arg[i] - i += 1 - # if we leave the loop, the character sep was not found unquoted - return (head, '') def GetFlagValue(flagvalue, strip=True): - """Converts a raw flag string to a useable value. + """Converts a raw flag string to a useable value. - 1. Expand @filename style flags to the content of filename. - 2. Cope with Python3 strangness of sys.argv. - sys.argv is not actually proper str types on Unix with Python3 - The bytes of the arg are each directly transcribed to the characters of - the str. It is actually more complex than that, as described in the docs. - https://docs.python.org/3/library/sys.html#sys.argv - https://docs.python.org/3/library/os.html#os.fsencode - https://www.python.org/dev/peps/pep-0383/ + 1. Expand @filename style flags to the content of filename. + 2. Cope with Python3 strangness of sys.argv. + sys.argv is not actually proper str types on Unix with Python3 + The bytes of the arg are each directly transcribed to the characters of + the str. It is actually more complex than that, as described in the docs. + https://docs.python.org/3/library/sys.html#sys.argv + https://docs.python.org/3/library/os.html#os.fsencode + https://www.python.org/dev/peps/pep-0383/ - Args: - flagvalue: (str) raw flag value - strip: (bool) Strip white space. + Args: + flagvalue: (str) raw flag value + strip: (bool) Strip white space. - Returns: - Python2: unicode - Python3: str - """ - if flagvalue: - if sys.version_info[0] < 3: - # python2 gives us raw bytes in argv. - flagvalue = flagvalue.decode('utf-8') - # assertion: py2: flagvalue is unicode - # assertion: py3: flagvalue is str, but in weird format - if flagvalue[0] == '@': - # Subtle: We do not want to re-encode the value here, because it - # is encoded in the right format for file open operations. - with open(flagvalue[1:], 'rb') as f: - flagvalue = f.read().decode('utf-8') - else: - # convert fs specific encoding back to proper unicode. - if sys.version_info[0] > 2: - flagvalue = os.fsencode(flagvalue).decode('utf-8') + Returns: + Python2: unicode + Python3: str + """ + if flagvalue: + if sys.version_info[0] < 3: + # python2 gives us raw bytes in argv. + flagvalue = flagvalue.decode("utf-8") + # assertion: py2: flagvalue is unicode + # assertion: py3: flagvalue is str, but in weird format + if flagvalue[0] == "@": + # Subtle: We do not want to re-encode the value here, because it + # is encoded in the right format for file open operations. + with open(flagvalue[1:], "rb") as f: + flagvalue = f.read().decode("utf-8") + else: + # convert fs specific encoding back to proper unicode. + if sys.version_info[0] > 2: + flagvalue = os.fsencode(flagvalue).decode("utf-8") - if strip: - return flagvalue.strip() - return flagvalue + if strip: + return flagvalue.strip() + return flagvalue diff --git a/package_manager/util.py b/package_manager/util.py index 8c30d7409..d77185932 100644 --- a/package_manager/util.py +++ b/package_manager/util.py @@ -3,9 +3,7 @@ import collections import os -# Distroless's test.sh runs pylint on all python files, but this module will not exist -# pylint: disable=import-error -from build_tar import TarFile +from build_tar.build_tar import TarFile def sha256_checksum(filename, block_size=65536): sha256 = hashlib.sha256()