diff --git a/.gitignore b/.gitignore index 24f09ff3..927781fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .env +.toolset.yml .idea .PKGINFO .SIGN.RSA.alpine-* diff --git a/README.md b/README.md index 123aaec3..10d26c53 100644 --- a/README.md +++ b/README.md @@ -322,7 +322,7 @@ Restricts Pieman to only preparing or upgrading the toolset which is located in Specifies the time zone of the system. -##### TOOLSET_CODENAME="v4-amy" +##### TOOLSET_CODENAME="v5-farnsworth" Specifies the toolset codename. The parameter allows users and developers to switch between different toolsets. Each codename is connected to its directory in `${TOOLSET_DIR}` which, in turn, contains the target toolset. When a codename is passed via `${TOOLSET_CODENAME}` but there is no such directory in `${TOOLSET_DIR}`, the process of creating of the directory and installing the toolset into it will be initiated. diff --git a/essentials.sh b/essentials.sh index 068c12ca..d61ed922 100644 --- a/essentials.sh +++ b/essentials.sh @@ -27,7 +27,7 @@ MENDER_CLIENT_REVISION="1.7.x" PIEMAN_MAJOR_VER=0 -PIEMAN_MINOR_VER=18 +PIEMAN_MINOR_VER=19 PYTHON_MAJOR_VER=3 diff --git a/helpers/apk.sh b/helpers/apk.sh index 95e233a8..ffe210c2 100644 --- a/helpers/apk.sh +++ b/helpers/apk.sh @@ -13,20 +13,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# Gets the Alpine Package Keeper (APK) version for the specified version of -# Alpine Linux. -# Globals: -# None -# Arguments: -# Version of Alpine Linux -# Returns: -# Alpine Package Keeper version -get_apk_tools_version() { - local alpine_version=$1 - - apk_tools_version.py --alpine-version="${alpine_version}" -} - # Runs apk.static to build a chroot environment. # Globals: # BASE_PACKAGES diff --git a/helpers/toolset.sh b/helpers/toolset.sh index 2379bfca..a0fca679 100644 --- a/helpers/toolset.sh +++ b/helpers/toolset.sh @@ -13,90 +13,21 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# Removes -# * the specified files and directories; -# * the .partial file in the current directory. -# Globals: -# None -# Arguments: -# Target directory -# Returns: -# None -finalise_installation() { - for i in "$@"; do - rm -rf "${i}" - done - - rm -f .partial +build_toolset() { + build_toolset.py ${PIEMAN_DIR}/.toolset.yml } -# Gets qemu-user-static 3.1 from Ubuntu 19.04 "Disco Dingo". +# Runs the preprocessor against toolset.yml, located in the root directory of +# Pieman. # Globals: -# None +# PIEMAN_DIR +# PIEMAN_UTILS_DIR +# PYTHON # Arguments: # None # Returns: # None -get_qemu_emulation_binary() { - local package - - do_wget http://mirrors.kernel.org/ubuntu/dists/focal/universe/binary-amd64/Packages.xz - - xz -d Packages.xz - - package="$(grep "Filename: pool/universe/q/qemu/qemu-user-static" Packages | awk '{print $2}')" - - do_wget http://security.ubuntu.com/ubuntu/"${package}" - - ar x "$(basename "${package}")" - - tar xJf data.tar.xz - - cp usr/bin/qemu-aarch64-static . - cp usr/bin/qemu-arm-static . - - # cleanup - rm control.tar.xz - rm data.tar.xz - rm debian-binary - rm Packages - rm "$(basename "${package}")" - rm -r usr -} - -# Checks if the specified Toolset component is partially installed, and if so, -# cleans up its directory and initializes it for the installation, creating the -# .partial file there. -# Globals: -# None -# Arguments: -# Target directory -# Returns: -# 0 if the specified component directory was initialized -# 1 if there was no need to initialize the specified component directory -init_installation_if_needed() { - local dir=$1 - - create_dir "${dir}" - if [ -z "$(ls -A "${dir}")" ] || [ -f "${dir}"/.partial ]; then - rm -rf "${dir:?}"/* - - touch "${dir}"/.partial - - return 0 - fi - - return 1 +run_preprocessor_against_toolset_yml() { + preprocessor.py ${PIEMAN_DIR}/toolset.yml ${PIEMAN_DIR}/.toolset.yml } -# Figures out the number of CPU cores which are available on the current -# machine. -# Globals: -# None -# Arguments: -# None -# Returns: -# Number of available cores -number_of_cores() { - grep -c ^processor /proc/cpuinfo -} diff --git a/pieman.sh b/pieman.sh index adbade25..d8b09a5c 100755 --- a/pieman.sh +++ b/pieman.sh @@ -118,7 +118,7 @@ def_bool_var SUDO_REQUIRE_PASSWORD true def_var TIME_ZONE "Etc/UTC" -def_var TOOLSET_CODENAME "v4-amy" +def_var TOOLSET_CODENAME "v5-farnsworth" def_var TOOLSET_DIR "${PIEMAN_DIR}/toolset" @@ -214,6 +214,9 @@ info "checking toolset ${TOOLSET_CODENAME}" if [ ! -d "${TOOLSET_FULL_PATH}" ]; then info "building toolset ${TOOLSET_CODENAME} since it does not exist" fi + +run_preprocessor_against_toolset_yml + . toolset.sh # shellcheck source=./pieman/pieman/build_status_codes diff --git a/pieman/bin/apk_tools_version.py b/pieman/bin/apk_tools_version.py deleted file mode 100755 index 314d0831..00000000 --- a/pieman/bin/apk_tools_version.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/python3 -# Copyright (C) 2018 Evgeny Golyshev -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""Utility intended to fetch the latest version of the apk-tools-static package -in the specified version of Alpine Linux. The point is that the -apk-tools-static version is not frozen in the stable release of Alpine Linux -and may vary. -""" - -import os -import sys -import time -from argparse import ArgumentParser -from html.parser import HTMLParser -from urllib.error import URLError, HTTPError -from urllib.parse import urljoin -from urllib.request import urlopen - - -ARCH = 'armhf' - -ALPINE_VERSION = '3.12' - -MIRROR = 'http://dl-cdn.alpinelinux.org' - -NAP = 1 - -RETRIES_NUMBER = 5 - - -class CustomHTMLParser(HTMLParser): # pylint: disable=abstract-method - """Simplified HTML parser to find the apk-tools-static version on the - specified page. - """ - - def __init__(self, content): - HTMLParser.__init__(self, convert_charrefs=True) - - self._apk_tools_version = None - self._content = content - - def get_apk_tools_version(self): - """Returns the apk-tools-static version. """ - self.feed(self._content) - return self._apk_tools_version - - def handle_starttag(self, tag, attrs): - if tag == 'a': - for name, value in attrs: - package_name = 'apk-tools-static' - if name == 'href' and value.startswith(package_name): - prefix = len(package_name) + 1 - suffix = len('.apk') - self._apk_tools_version = value[prefix:-suffix] - break - - -def main(): - """The main entry point. """ - - parser = ArgumentParser() - parser.add_argument('--alpine-version', default=ALPINE_VERSION, - help='alpine version', metavar='ALPINE_VERSION') - parser.add_argument('--arch', default=ARCH, - help='target architecture', metavar='ARCH') - parser.add_argument('--mirror', default=MIRROR, - help='mirror', metavar='MIRROR') - args = parser.parse_args() - - address = urljoin(args.mirror, - os.path.join('alpine', 'v' + args.alpine_version, - 'main', args.arch)) - - content = b'' - for attempt in range(1, RETRIES_NUMBER + 1): - try: - content = urlopen(address).read() - break - except HTTPError as exc: - sys.stderr.write('{}: request failed (error code {})\n'. - format(sys.argv[0], exc.code)) - except URLError as exc: - sys.stderr.write('{}: {}\n'.format(sys.argv[0], exc.reason)) - - if attempt != RETRIES_NUMBER: - sys.stderr.write('Retrying in {} seconds...\n'.format(NAP)) - time.sleep(NAP) - - if content == b'' and attempt == RETRIES_NUMBER: - sys.stderr.write('Could not request {} after {} attempts\n'. - format(address, RETRIES_NUMBER)) - sys.exit(1) - - parser = CustomHTMLParser(content.decode('utf8')) - apk_tools_version = parser.get_apk_tools_version() - if not apk_tools_version: - sys.stderr.write('Could not get apk tools version\n') - sys.exit(1) - - print(apk_tools_version) - - -if __name__ == '__main__': - main() diff --git a/pieman/bin/build_toolset.py b/pieman/bin/build_toolset.py new file mode 100755 index 00000000..5292148e --- /dev/null +++ b/pieman/bin/build_toolset.py @@ -0,0 +1,62 @@ +#!/usr/bin/python3 +# Copyright (C) 2020 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import sys +from argparse import ArgumentParser + +from yaml.scanner import ScannerError + +from pieman import toolset, util + + +def main(): + """The main entry point. """ + + parser = ArgumentParser() + parser.add_argument('yml_file', help='path to the Toolset YAML file') + args = parser.parse_args() + + try: + os.environ['TOOLSET_FULL_PATH'] + except KeyError: + util.fatal('The TOOLSET_FULL_PATH environment variable is undefined.') + sys.exit(1) + + try: + toolset_tree = toolset.ToolsetProcessor(args.yml_file) + except ScannerError as exp: + util.fatal('{}'.format(exp)) + sys.exit(1) + except AttributeError as exp: + util.fatal('{}'.format(exp)) + sys.exit(1) + except ModuleNotFoundError as exp: + util.fatal('{}'.format(exp)) + sys.exit(1) + except toolset.MissingRequiredFields as exp: + util.fatal('{}'.format(exp)) + sys.exit(1) + + for name, module in toolset_tree: + for flavour in module['flavours']: + flavour_name = next(iter(flavour)) + mod = module['imported'] + mod.run(**flavour[flavour_name]) + + +if __name__ == '__main__': + main() diff --git a/pieman/bin/preprocessor.py b/pieman/bin/preprocessor.py new file mode 100755 index 00000000..976cc953 --- /dev/null +++ b/pieman/bin/preprocessor.py @@ -0,0 +1,44 @@ +#!/usr/bin/python3 +# Copyright (C) 2020 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys +from argparse import ArgumentParser + +from yaml.scanner import ScannerError + +from pieman import toolset + + +def main(): + """The main entry point. """ + + parser = ArgumentParser() + parser.add_argument('infile', help='path to the file to be processed') + parser.add_argument('outfile', help='path to the result file') + args = parser.parse_args() + + try: + toolset.PreProcessor(args.infile, args.outfile) + except ScannerError as exp: + sys.stderr.write('{}\n'.format(exp)) + sys.exit(1) + except toolset.UndefinedVariable as exp: + sys.stderr.write('{}\n'.format(exp)) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/pieman/pieman/__init__.py b/pieman/pieman/__init__.py index aebe621b..4a862d0d 100644 --- a/pieman/pieman/__init__.py +++ b/pieman/pieman/__init__.py @@ -1,3 +1,3 @@ """The Pieman tools. """ -__version__ = '0.18.0' +__version__ = '0.19.0' diff --git a/pieman/pieman/toolset.py b/pieman/pieman/toolset.py new file mode 100644 index 00000000..edeec470 --- /dev/null +++ b/pieman/pieman/toolset.py @@ -0,0 +1,154 @@ +# Copyright (C) 2020 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import re +import yaml +from importlib import import_module + + +class RootNameIsNotValid(Exception): + pass + + +class MissingRequiredFields(Exception): + def __init__(self, module_name, flavour_name, missing_fields): + super().__init__('The {} flavour of the {} module misses some of the required fields: ' + '{}'.format(flavour_name, module_name, ', '.join(missing_fields))) + + +class UndefinedVariable(Exception): + def __init__(self, var_name): + super().__init__("The variable '{}' is undefined".format(var_name)) + + +class PreProcessor: + def __init__(self, file_name, new_file_name, root_name='toolset'): + with open(file_name, 'r') as infile: + self._toolset = yaml.load(infile, Loader=yaml.FullLoader) + + self._root_name = root_name + self._tree = None + + self._get_tree() + + self._var_re = re.compile(r'\${([\w\d]+)}') + + self._go_through_all_yml(self._tree, self._toolset) + + new_tree = {self._root_name: self._tree} + with open(new_file_name, 'w') as outfile: + yaml.dump(new_tree, outfile, default_flow_style=False) + + # + # Private methods + # + + def _get_tree(self): + try: + self._tree = self._toolset[self._root_name] + except KeyError: + raise RootNameIsNotValid + + def _go_through_all_yml(self, parent_node, node, table_names=None, parent_node_name=''): + table_names = table_names if table_names else {} + + if isinstance(node, dict): + for key, val in node.items(): + table_names['parent_node_name'] = parent_node_name + if not isinstance(val, (dict, list, )): + table_names[key] = val + + self._go_through_all_yml(node, val, table_names, key) + elif isinstance(node, list): + for i in node: + self._go_through_all_yml(node, i) + else: + node_value = parent_node[parent_node_name] + while True: + if not isinstance(node_value, str): + break + + match = self._var_re.search(node_value) + if match is None: + break + + var_name = match[1] + try: + value = table_names[var_name] + except KeyError: + try: + value = os.environ[var_name] + except KeyError: + raise UndefinedVariable(var_name) + + node_value = node_value.replace(match[0], value) + + parent_node[parent_node_name] = node_value + + +class ToolsetProcessor: + def __init__(self, file_name): + with open(file_name, 'r') as infile: + self._toolset = yaml.load(infile, Loader=yaml.FullLoader) + + self._modules = {} + + self._get_root() + + self._process_modules() + + self._validate_modules() + + # + # Private methods + # + + def _get_root(self): + try: + self._root = self._toolset['toolset'] + except KeyError: + raise RootNameIsNotValid + + def _process_modules(self): + """Raises ModuleNotFoundError if one of the specified modules doesn't exist. """ + + for module in self._root: + module_name = next(iter(module)) + mod = self._modules[module_name] = {} + mod['imported'] = imported = import_module('.' + module_name, + package='pieman.toolset_modules') + + if imported.FLAVOURS_ENABLED: + mod['flavours'] = module[module_name] + else: + mod['flavours'] = [{'default': module[module_name]}] + + def _validate_modules(self): + """Raises AttributeError if the REQUIRED_FIELDS attribute is absent in one of the + modules. + """ + + for module_name, module in self._modules.items(): + for flavour in module['flavours']: + flavour_name = next(iter(flavour)) + got_fields = set(flavour[flavour_name].keys()) + required_fields = set(module['imported'].REQUIRED_FIELDS) + missing_fields = required_fields - got_fields + if missing_fields: + raise MissingRequiredFields(module_name, flavour_name, missing_fields) + + def __iter__(self): + return iter(self._modules.items()) diff --git a/pieman/pieman/toolset_modules/apk.py b/pieman/pieman/toolset_modules/apk.py new file mode 100755 index 00000000..77648355 --- /dev/null +++ b/pieman/pieman/toolset_modules/apk.py @@ -0,0 +1,103 @@ +# Copyright (C) 2018-2020 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os.path +import sys +import time +from html.parser import HTMLParser +from urllib.error import URLError, HTTPError +from urllib.parse import urljoin +from urllib.request import urlopen + +from pieman import util + + +ALPINE_VERSION = '3.12' + +FLAVOURS_ENABLED = True + +MIRROR = 'http://dl-cdn.alpinelinux.org' + +REQUIRED_FIELDS = ('arch', 'version', 'dst', ) + +RETRIES_NUMBER = 5 + + +class CustomHTMLParser(HTMLParser): # pylint: disable=abstract-method + """Simplified HTML parser to find the apk-tools-static version on the + specified page. + """ + + def __init__(self, content): + HTMLParser.__init__(self, convert_charrefs=True) + + self._apk_tools_version = None + self._content = content + + def get_apk_tools_version(self): + """Returns the apk-tools-static version. """ + self.feed(self._content) + return self._apk_tools_version + + def handle_starttag(self, tag, attrs): + if tag == 'a': + for name, value in attrs: + package_name = 'apk-tools-static' + if name == 'href' and value.startswith(package_name): + prefix = len(package_name) + 1 + suffix = len('.apk') + self._apk_tools_version = value[prefix:-suffix] + break + + +def run(**kwargs): + pass + # arch = kwargs['arch'] + # version = kwargs['version'] + # dst = os.path.join(os.environ['TOOLSET_FULL_PATH'], kwargs['dst']) + # + # util.mkdir(os.path.dirname(dst)) + # + # address = urljoin(MIRROR, os.path.join('alpine', 'v' + version, 'main', arch)) + # + # content = b'' + # for attempt in range(1, RETRIES_NUMBER + 1): + # try: + # content = urlopen(address).read() + # break + # except HTTPError as exc: + # util.fatal('{}: request failed ' + # '(error code {})'.format(sys.argv[0], exc.code)) + # except URLError as exc: + # util.fatal('{}: {}'.format(sys.argv[0], exc.reason)) + # + # if attempt != RETRIES_NUMBER: + # util.info('{}: retrying in 1 second...'.format(sys.argv[0])) + # time.sleep(1) + # + # if content == b'' and attempt == RETRIES_NUMBER: + # util.fatal('{}: could not request {} after {} ' + # 'attempts'.format(sys.argv[0], address, RETRIES_NUMBER)) + # sys.exit(1) + # + # parser = CustomHTMLParser(content.decode('utf8')) + # apk_tools_version = parser.get_apk_tools_version() + # if not apk_tools_version: + # util.fatal('{}: could not get apk tools version'.format(sys.argv[0])) + # sys.exit(1) + # + # download_link = urljoin(address + '/', 'apk-tools-static-{}.apk'.format(apk_tools_version)) + # util.info('Downloading {}'.format(download_link)) + # util.download(download_link, dst) diff --git a/pieman/pieman/toolset_modules/debootstrap.py b/pieman/pieman/toolset_modules/debootstrap.py new file mode 100644 index 00000000..28aaf3b2 --- /dev/null +++ b/pieman/pieman/toolset_modules/debootstrap.py @@ -0,0 +1,22 @@ +# Copyright (C) 2020 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +FLAVOURS_ENABLED = True + +REQUIRED_FIELDS = ('version', 'dst', ) + + +def run(*args, **kwargs): + pass diff --git a/pieman/pieman/toolset_modules/download.py b/pieman/pieman/toolset_modules/download.py new file mode 100644 index 00000000..463f414a --- /dev/null +++ b/pieman/pieman/toolset_modules/download.py @@ -0,0 +1,14 @@ +# Copyright (C) 2020 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . diff --git a/pieman/pieman/toolset_modules/mender_artifact.py b/pieman/pieman/toolset_modules/mender_artifact.py new file mode 100644 index 00000000..d79a6ef0 --- /dev/null +++ b/pieman/pieman/toolset_modules/mender_artifact.py @@ -0,0 +1,22 @@ +# Copyright (C) 2020 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +FLAVOURS_ENABLED = False + +REQUIRED_FIELDS = ('version', 'git', 'src', 'dst', ) + + +def run(*args, **kwargs): + pass diff --git a/pieman/pieman/toolset_modules/mender_client.py b/pieman/pieman/toolset_modules/mender_client.py new file mode 100644 index 00000000..d79a6ef0 --- /dev/null +++ b/pieman/pieman/toolset_modules/mender_client.py @@ -0,0 +1,22 @@ +# Copyright (C) 2020 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +FLAVOURS_ENABLED = False + +REQUIRED_FIELDS = ('version', 'git', 'src', 'dst', ) + + +def run(*args, **kwargs): + pass diff --git a/pieman/pieman/toolset_modules/qemu_user_static.py b/pieman/pieman/toolset_modules/qemu_user_static.py new file mode 100644 index 00000000..726adba6 --- /dev/null +++ b/pieman/pieman/toolset_modules/qemu_user_static.py @@ -0,0 +1,30 @@ +# Copyright (C) 2020 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Toolset module intended for fetching qemu-user-static the Ubuntu archive. """ + +from pieman import util + + +FLAVOURS_ENABLED = True + +REQUIRED_FIELDS = ('arch', 'codename', 'dst', ) + +UBUNTU_CODENAME = 'focal' + + +def run(*args, **kwargs): + exit_code = util.run_program(['get-qemu-user-static.sh', UBUNTU_CODENAME]) + print(f'----- {exit_code}') diff --git a/pieman/pieman/toolset_modules/uboot.py b/pieman/pieman/toolset_modules/uboot.py new file mode 100644 index 00000000..053fd6c4 --- /dev/null +++ b/pieman/pieman/toolset_modules/uboot.py @@ -0,0 +1,22 @@ +# Copyright (C) 2020 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +FLAVOURS_ENABLED = True + +REQUIRED_FIELDS = ('config', 'toolchain', 'src', 'dst', ) + + +def run(*args, **kwargs): + pass diff --git a/pieman/pieman/util.py b/pieman/pieman/util.py index 3ac45ea3..9f61eed3 100644 --- a/pieman/pieman/util.py +++ b/pieman/pieman/util.py @@ -18,12 +18,23 @@ import logging import os import sys +from curses import tparm, tigetstr, setupterm from urllib.request import urlretrieve -import redis +#import redis + + +setupterm() + LOGGING_FORMATTER = '%(asctime)s %(levelname)-5.5s %(message)s' +RED = tparm(tigetstr('setaf'), 1).decode('utf8') + +YELLOW = tparm(tigetstr('setaf'), 3).decode('utf8') + +RESET = tparm(tigetstr('sgr0')).decode('utf8') + def _reporthook(chunk_number, buffer_size, total_size): """It must accept three numeric parameters: @@ -71,6 +82,14 @@ def download(url, dst, quiet=False): .format(filename)) +def fatal(text): + sys.stderr.write('{}fatal{}: {}\n'.format(RED, RESET, text)) + + +def info(text): + sys.stderr.write('{}info{}: {}\n'.format(YELLOW, RESET, text)) + + def init_logger(logger, log_level, log_file_prefix='', logging_formatter=LOGGING_FORMATTER): """Initializes the logger. """ @@ -88,3 +107,23 @@ def init_logger(logger, log_level, log_file_prefix='', file_handler.setFormatter(formatter) file_handler.setFormatter(formatter) logger.addHandler(file_handler) + + +def mkdir(dir_name): + """Creates the specified directory, making parent directories + as needed. + """ + + if not os.path.exists(dir_name): + os.makedirs(dir_name) + + +def run_program(args): + """Runs the executable file (which is searched for along $PATH) with argument list args. """ + + pid = os.fork() + if pid == 0: # child + os.execvp(args[0], args) + else: # parent + _, status = os.wait() + return status >> 8 diff --git a/pieman/setup.py b/pieman/setup.py index 002394bf..f7042f7f 100644 --- a/pieman/setup.py +++ b/pieman/setup.py @@ -17,7 +17,7 @@ setup(name='pieman', - version='0.18.0', + version='0.19.0', description='Pieman package', long_description=LONG_DESCRIPTION, url='https://github.com/tolstoyevsky/pieman', @@ -26,15 +26,16 @@ maintainer_email='eugulixes@gmail.com', license='https://gnu.org/licenses/gpl-3.0.txt', scripts=[ - 'bin/apk_tools_version.py', 'bin/bsc.py', 'bin/bscd.py', + 'bin/build_toolset.py', 'bin/check_mutually_exclusive_params.py', 'bin/check_redis.py', 'bin/check_wpa_passphrase.py', 'bin/depend_on.py', 'bin/du.py', 'bin/image_attrs.py', + 'bin/preprocessor.py', 'bin/render.py', 'bin/wget.py', ], diff --git a/toolset.sh b/toolset.sh index 9d8f07ed..ac9ed65b 100755 --- a/toolset.sh +++ b/toolset.sh @@ -1,4 +1,4 @@ -# Copyright (C) 2018 Evgeny Golyshev +# Copyright (C) 2018-2020 Evgeny Golyshev # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -13,206 +13,17 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -toolchain_dir="gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabihf" -cross_compiler="${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}/${toolchain_dir}/bin/arm-linux-gnueabihf-" - -toolchain_for_mender_dir="gcc-linaro-6.3.1-2017.05-x86_64_arm-linux-gnueabihf" -cross_compiler_for_mender="${TOOLSET_FULL_PATH}/mender/${toolchain_for_mender_dir}/bin/arm-linux-gnueabihf-" -uboot_tools="${TOOLSET_FULL_PATH}/mender/uboot-mender/tools" -mendersoftware_dir="${TOOLSET_FULL_PATH}"/mender/client/src/mender/vendor/github.com/mendersoftware - -info "checking Mender dependencies" - -if $(are_mender_dependencies_satisfied); then - info "Mender dependencies are satisfied" - mender_dependencies_are_satisfied=true -else - info "Mender dependencies are not satisfied" - mender_dependencies_are_satisfied=false -fi - -info "checking Das U-Boot dependencies" - -if ! $(are_uboot_dependencies_satisfied) && [[ ! -d "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}" ]]; then - fatal "Das U-Boot dependencies are not satisfied" - exit 1 -else - info "Das U-Boot dependencies are satisfied" -fi - -if $(init_installation_if_needed "${TOOLSET_FULL_PATH}/qemu-user-static"); then - info "fetching qemu-user-static" - pushd "${TOOLSET_FULL_PATH}/qemu-user-static" - get_qemu_emulation_binary - - finalise_installation - popd -fi - -if $(init_installation_if_needed "${TOOLSET_FULL_PATH}/apk"); then - info "fetching apk.static for Alpine Linux ${ALPINE_VER}" - pushd "${TOOLSET_FULL_PATH}"/apk - create_dir "${ALPINE_VER}" - - addr=http://dl-cdn.alpinelinux.org/alpine/ - apk_tools_version="$(get_apk_tools_version "${ALPINE_VER}")" - apk_tools_static="apk-tools-static-${apk_tools_version}.apk" - apk_tools_static_path="${TOOLSET_FULL_PATH}/apk/${ALPINE_VER}" - - do_wget "${addr}/v${ALPINE_VER}/main/armhf/${apk_tools_static}" -O "${apk_tools_static_path}/${apk_tools_static}" - - tar -xzf "${apk_tools_static_path}/${apk_tools_static}" -C "${apk_tools_static_path}" - - mv "${apk_tools_static_path}/sbin/apk.static" "${apk_tools_static_path}" - - finalise_installation \ - "${apk_tools_static_path}/${apk_tools_static}" \ - "${apk_tools_static_path}/sbin" - popd -fi - -if [ ! -d "${TOOLSET_FULL_PATH}/debootstrap" ]; then - info "fetching debootstrap ${DEBOOTSTRAP_VER}" - pushd "${TOOLSET_FULL_PATH}" - git clone https://salsa.debian.org/installer-team/debootstrap.git - - git -C debootstrap checkout "${DEBOOTSTRAP_VER}" - popd -else - info "checking if the debootstrap version is equal to or higher ${DEBOOTSTRAP_VER}" - - if ! is_debootstrap_uptodate; then - pushd "${TOOLSET_FULL_PATH}"/debootstrap - info "upgrading debootstrap to ${DEBOOTSTRAP_VER}" - - git checkout master - - git pull - - git checkout ${DEBOOTSTRAP_VER} - popd - fi -fi - -if ${mender_dependencies_are_satisfied} && $(init_installation_if_needed "${TOOLSET_FULL_PATH}/mender"); then - pushd "${TOOLSET_FULL_PATH}/mender" - info "downloading inventory & identity scripts" - do_wget -q -O mender-device-identity "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-device-identity - do_wget -q -O mender-inventory-bootloader-integration "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-inventory-bootloader-integration - do_wget -q -O mender-inventory-hostinfo "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-inventory-hostinfo - do_wget -q -O mender-inventory-network "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-inventory-network - do_wget -q -O mender-inventory-os "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-inventory-os - do_wget -q -O mender-inventory-rootfs-type "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-inventory-rootfs-type - - info "fetching cross-toolchain for building Das U-Boot (Mender flavour) and Mender client" - do_wget "https://releases.linaro.org/components/toolchain/binaries/6.3-2017.05/arm-linux-gnueabihf/${toolchain_for_mender_dir}.tar.xz" -O "${toolchain_for_mender_dir}.tar.xz" - - info "unpacking archive with toolchain for building Das U-Boot (Mender flavour)" - tar xJf "${toolchain_for_mender_dir}.tar.xz" - rm "${toolchain_for_mender_dir}.tar.xz" - - info "fetching Das U-Boot (Mender flavour) from https://github.com/mendersoftware/uboot-mender.git" - git clone https://github.com/mendersoftware/uboot-mender.git -b "${UBOOT_MENDER_BRANCH}" - git -C "uboot-mender" checkout "${UBOOT_MENDER_COMMIT}" - - mkdir -p "${mendersoftware_dir}" - - info "fetching Mender client from https://github.com/mendersoftware/mender.git" - git clone https://github.com/mendersoftware/mender.git "${mendersoftware_dir}"/mender - git -C "${mendersoftware_dir}"/mender checkout "${MENDER_CLIENT_VER}" - - info "fetching Mender Artifacts Library from https://github.com/mendersoftware/mender-artifact.git" - git clone https://github.com/mendersoftware/mender-artifact.git "${mendersoftware_dir}"/mender-artifact - git -C "${mendersoftware_dir}"/mender-artifact checkout "${MENDER_ARTIFACT_VER}" - popd - - pushd "${TOOLSET_FULL_PATH}/mender/uboot-mender" - info "building Das U-Boot (Mender flavour)" - - ARCH=arm CROSS_COMPILE="${cross_compiler_for_mender}" make --quiet distclean - ARCH=arm CROSS_COMPILE="${cross_compiler_for_mender}" make rpi_3_32b_defconfig - ARCH=arm CROSS_COMPILE="${cross_compiler_for_mender}" make PYTHON=python2 -j $(number_of_cores) - ARCH=arm CROSS_COMPILE="${cross_compiler_for_mender}" make envtools -j $(number_of_cores) - - cp "u-boot.bin" "${TOOLSET_FULL_PATH}/mender" - cp tools/env/fw_printenv "${TOOLSET_FULL_PATH}/mender" - - info "generating image for Das U-Boot (Mender flavour)" - "${uboot_tools}"/mkimage -A arm -T script -C none -n "Boot script" -d "${PIEMAN_DIR}"/files/mender/boot.cmd "${TOOLSET_FULL_PATH}"/mender/boot.scr - popd - - pushd "${mendersoftware_dir}"/mender - info "building Mender client" - - env CGO_ENABLED=1 \ - CC="${cross_compiler_for_mender}"gcc \ - GOARCH=arm \ - GOOS=linux \ - GOPATH="${TOOLSET_FULL_PATH}"/mender/client make build - - cp mender "${TOOLSET_FULL_PATH}"/mender - popd - - pushd "${mendersoftware_dir}"/mender-artifact - info "building Mender Artifacts Library" - - env GOPATH="${TOOLSET_FULL_PATH}"/mender/client make build - - cp mender-artifact "${TOOLSET_FULL_PATH}"/mender - popd - - pushd "${TOOLSET_FULL_PATH}/mender" - finalise_installation "${toolchain_for_mender_dir}" client uboot-mender - popd -fi - -if $(init_installation_if_needed "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}"); then - pushd "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}" - info "fetching cross-toolchain for building Das U-Boot" - do_wget "https://releases.linaro.org/components/toolchain/binaries/7.4-2019.02/arm-linux-gnueabihf/${toolchain_dir}.tar.xz" -O "${toolchain_dir}.tar.xz" - - info "unpacking archive with toolchain for building Das U-Boot" - tar xJf "${toolchain_dir}.tar.xz" - rm "${toolchain_dir}.tar.xz" - - info "fetching Das U-Boot ${UBOOT_VER} from ${UBOOT_URL}" - git clone --depth=1 -b "v${UBOOT_VER}" https://github.com/u-boot/u-boot.git "u-boot-${UBOOT_VER}" - popd - - pushd "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}/u-boot-${UBOOT_VER}" - info "building Das U-Boot" - - ARCH=arm CROSS_COMPILE="${cross_compiler}" make orangepi_pc_plus_defconfig - - # The host system may have both Python 2 and 3 installed. U-Boot - # depends on Python 2, so it's necessary to specify it explicitly via - # the PYTHON variable. - ARCH=arm CROSS_COMPILE="${cross_compiler}" PYTHON=python2 make -j $(number_of_cores) - - cp u-boot-sunxi-with-spl.bin "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}"/u-boot-sunxi-with-spl-for-opi-pc-plus.bin - - ARCH=arm CROSS_COMPILE="${cross_compiler}" make orangepi_zero_defconfig - ARCH=arm CROSS_COMPILE="${cross_compiler}" PYTHON=python2 make -j $(number_of_cores) - - cp u-boot-sunxi-with-spl.bin "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}"/u-boot-sunxi-with-spl-for-opi-zero.bin - - cp tools/mkimage "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}" - popd - - pushd "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}" - finalise_installation "${toolchain_dir}" "u-boot-${UBOOT_VER}" uboot-env - popd -fi +build_toolset # Correct ownership if needed -pieman_dir_ownership="$(get_ownership "${PIEMAN_DIR}")" -if [ "$(get_ownership "${TOOLSET_FULL_PATH}")" != "${pieman_dir_ownership}" ]; then - info "correcting ownership for ${TOOLSET_FULL_PATH}" - chown -R "${pieman_dir_ownership}" "${TOOLSET_FULL_PATH}" -fi - -if ${PREPARE_ONLY_TOOLSET}; then - success "exiting since PREPARE_ONLY_TOOLSET is set to true" - - exit 0 -fi +#pieman_dir_ownership="$(get_ownership "${PIEMAN_DIR}")" +#if [ "$(get_ownership "${TOOLSET_FULL_PATH}")" != "${pieman_dir_ownership}" ]; then +# info "correcting ownership for ${TOOLSET_FULL_PATH}" +# chown -R "${pieman_dir_ownership}" "${TOOLSET_FULL_PATH}" +#fi +# +#if ${PREPARE_ONLY_TOOLSET}; then +# success "exiting since PREPARE_ONLY_TOOLSET is set to true" +# +# exit 0 +#fi diff --git a/toolset.yml b/toolset.yml new file mode 100644 index 00000000..1d27eb66 --- /dev/null +++ b/toolset.yml @@ -0,0 +1,53 @@ +toolset: + - apk: + - armhf: + arch: ${parent_node_name} + version: '3.12' + dst: apk/${version}/apk-${parent_node_name}.static + - debootstrap: + - debian: + version: 1.0.118 + git: https://salsa.debian.org/installer-team/debootstrap.git + dst: debootstrap/${parent_node_name} + - uboot: + - mender: + version: 988e0ec54 + config: rpi_3_32b_defconfig + git: https://github.com/mendersoftware/uboot-mender.git + toolchain: https://releases.linaro.org/components/toolchain/binaries/6.3-2017.05/arm-linux-gnueabihf/gcc-linaro-6.3.1-2017.05-x86_64_arm-linux-gnueabihf.tar.xz + src: uboot/uboot-mender/u-boot.bin + dst: uboot/mender-u-boot.bin + - upstream-opi-pc-plus: + version: v2019.01 + config: orangepi_pc_plus_defconfig + git: https://github.com/u-boot/u-boot.git + depth: 1 + toolchain: https://releases.linaro.org/components/toolchain/binaries/7.4-2019.02/arm-linux-gnueabihf/gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabihf.tar.xz + src: uboot/u-boot/u-boot-sunxi-with-spl.bin + dst: uboot/u-boot-sunxi-with-spl-for-opi-pc-plus.bin + - upstream-opi-zero: + config: orangepi_zero_defconfig + dir: uboot/u-boot + toolchain: uboot/gcc-linaro-6.3.1-2017.05-x86_64_arm-linux-gnueabihf + src: uboot/u-boot/u-boot-sunxi-with-spl.bin + dst: uboot/u-boot-sunxi-with-spl-for-opi-zero.bin + - mender_artifact: + version: 2.3.0 + git: https://github.com/mendersoftware/mender-artifact.git + src: mender-artifact/mender-artifact + dst: mender/mender-artifact + - mender_client: + version: 1.7.0 + git: https://raw.githubusercontent.com/mendersoftware/mender + clone_into: ${parent_node_name} + src: ${parent_node_name}/mender + dst: mender/mender + - qemu_user_static: + - arm: + arch: ${parent_node_name} + codename: focal + dst: qemu-user-static/qemu-${parent_node_name}-static + - aarch64: + arch: ${parent_node_name} + codename: focal + dst: qemu-user-static/qemu-${parent_node_name}-static