photon-os-installer: fix tdnf installroot commands

Change-Id: I38b792b77c13a99b7d8bdc174919dd6c48227dc2
Signed-off-by: Shreenidhi Shedi <[email protected]>
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/14961
Reviewed-by: Ankit Jain <[email protected]>
Tested-by: gerrit-photon <[email protected]>

Author: sshedi

SPECS/photon-os-installer/fix-installroot-commands.patch

@@ -0,0 +1,382 @@
+From 2c0c50e89d4f172bb2abf723aef8474204c83afc Mon Sep 17 00:00:00 2001
+From: Shreenidhi Shedi <[email protected]>
+Date: Tue, 21 Dec 2021 18:53:18 +0530
+Subject: [PATCH] installer.py: fix installroot commands
+
+After this fix in tdnf:
+https://github.com/vmware/tdnf/commit/9d2ab6090c1b0dcba4f28763b042b2fd8da3c551
+
+installroot option creates repodir if it doesn't exist inside given root
+path.
+
+This patch fixes it by providing repodir path using
+```
+--setopt=reposdir=... option
+```
+
+Other changes:
+Fixed a bunch of flake8 warnings in installer.py
+Change cachedir path to `.../var/cache/tdnf`
+
+Change-Id: Ibea9fdd555a03c366456c7efffa449defc1b5da7
+Signed-off-by: Shreenidhi Shedi <[email protected]>
+---
+ photon_installer/installer.py | 107 +++++++++++++++++-----------------
+ 1 file changed, 55 insertions(+), 52 deletions(-)
+
+diff --git a/photon_installer/installer.py b/photon_installer/installer.py
+index e09941f..5d019e7 100755
+--- a/photon_installer/installer.py
++++ b/photon_installer/installer.py
+@@ -1,12 +1,11 @@
+ """
+ Photon installer
+ """
+-#/*
+-# * Copyright © 2020 VMware, Inc.
+-# * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-only
+-# */
++# /*
++#  * Copyright © 2020 VMware, Inc.
++#  * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-only
++#  */
+ #
+-#    Author: Mahmoud Bassiouny <[email protected]>
+ 
+ import subprocess
+ import os
+@@ -25,13 +24,13 @@
+ from jsonwrapper import JsonWrapper
+ from progressbar import ProgressBar
+ from window import Window
+-from actionresult import ActionResult
+ from networkmanager import NetworkManager
+ from enum import Enum
+ 
+ BIOSSIZE = 4
+ ESPSIZE = 10
+ 
++
+ class PartitionType(Enum):
+     SWAP = 1
+     LINUX = 2
+@@ -39,6 +38,7 @@ class PartitionType(Enum):
+     ESP = 4
+     BIOS = 5
+ 
++
+ class Installer(object):
+     """
+     Photon installer
+@@ -104,9 +104,9 @@ def __init__(self, working_directory="/mnt/photon-root",
+         self.installer_path = os.path.dirname(os.path.abspath(__file__))
+         self.tdnf_conf_path = self.working_directory + "/tdnf.conf"
+         self.tdnf_repo_path = self.working_directory + "/photon-local.repo"
+-        self.rpm_cache_dir = self.photon_root + '/cache/tdnf/photon-local/rpms'
++        self.rpm_cache_dir = self.photon_root + '/var/cache/tdnf/photon-local/rpms'
+         # used by tdnf.conf as cachedir=, tdnf will append the rest
+-        self.rpm_cache_dir_short = self.photon_root + '/cache/tdnf'
++        self.rpm_cache_dir_short = self.photon_root + '/var/cache/tdnf'
+ 
+         self.setup_grub_command = os.path.dirname(__file__)+"/mk-setup-grub.sh"
+ 
+@@ -116,7 +116,7 @@ def __init__(self, working_directory="/mnt/photon-root",
+     """
+     create, append and validate configuration date - install_config
+     """
+-    def configure(self, install_config, ui_config = None):
++    def configure(self, install_config, ui_config=None):
+         if install_config and 'insecure_installation' in install_config:
+             insecure_installation = install_config.pop('insecure_installation')
+ 
+@@ -155,7 +155,6 @@ def configure(self, install_config, ui_config = None):
+ 
+         self.install_config = install_config
+ 
+-
+     def execute(self):
+         if 'setup_grub_script' in self.install_config:
+             self.setup_grub_command = self.install_config['setup_grub_script']
+@@ -247,17 +246,16 @@ def _add_defaults(self, install_config):
+                 install_config['search_path'].append(dirname)
+ 
+         if 'linux_flavor' not in install_config:
+-            if install_config.get('install_linux_esx', False) == True:
++            if install_config.get('install_linux_esx', False):
+                 install_config['linux_flavor'] = "linux-esx"
+             else:
+-                available_flavors=[]
++                available_flavors = []
+                 for flavor in self.all_linux_flavors:
+                     if flavor in install_config['packages']:
+                         available_flavors.append(flavor)
+                 if len(available_flavors) == 1:
+                     install_config['linux_flavor'] = available_flavors[0]
+ 
+-
+         install_config['install_linux_esx'] = False
+ 
+         # Default Photon docker image
+@@ -277,7 +275,7 @@ def _check_install_config(self, install_config):
+         if len(unknown_keys) > 0:
+             return "Unknown install_config keys: " + ", ".join(unknown_keys)
+ 
+-        if not 'disk' in install_config:
++        if 'disk' not in install_config:
+             return "No disk configured"
+ 
+         # For Ostree install_config['packages'] will be empty list, because ostree
+@@ -507,9 +505,9 @@ def _bind_installer(self):
+         """
+         # Make the photon_root/installer directory if not exits
+         if(self.cmd.run(['mkdir', '-p',
+-                            os.path.join(self.photon_root, "installer")]) != 0 or
++                         os.path.join(self.photon_root, "installer")]) != 0 or
+            self.cmd.run(['mount', '--bind', self.installer_path,
+-                            os.path.join(self.photon_root, "installer")]) != 0):
++                         os.path.join(self.photon_root, "installer")]) != 0):
+             self.logger.error("Fail to bind installer")
+             self.exit_gracefully()
+ 
+@@ -548,9 +546,9 @@ def _unbind_repo_dir(self):
+ 
+     def _get_partuuid(self, path):
+         partuuid = subprocess.check_output(['blkid', '-s', 'PARTUUID', '-o', 'value', path],
+-                                       universal_newlines=True).rstrip('\n')
++                                           universal_newlines=True).rstrip('\n')
+         # Backup way to get uuid/partuuid. Leave it here for later use.
+-        #if partuuidval == '':
++        # if partuuidval == '':
+         #    sgdiskout = Utils.runshellcommand(
+         #        "sgdisk -i 2 {} ".format(disk_device))
+         #    partuuidval = (re.findall(r'Partition unique GUID.*',
+@@ -561,7 +559,7 @@ def _get_uuid(self, path):
+         return subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', path],
+                                        universal_newlines=True).rstrip('\n')
+ 
+-    def _create_fstab(self, fstab_path = None):
++    def _create_fstab(self, fstab_path=None):
+         """
+         update fstab
+         """
+@@ -661,9 +659,12 @@ def _initialize_system(self):
+             self.exit_gracefully()
+ 
+         # Install filesystem rpm
+-        tdnf_cmd = "tdnf install  filesystem --releasever {0} --installroot {1} --assumeyes -c {2}".format(
+-                    self.install_config['photon_release_version'], self.photon_root,
+-                    self.tdnf_conf_path)
++        tdnf_cmd = ("tdnf install -y filesystem --releasever {0} "
++                    "--installroot {1} -c {2} "
++                    "--setopt=reposdir={3}").format(self.install_config['photon_release_version'],
++                                                    self.photon_root,
++                                                    self.tdnf_conf_path,
++                                                    self.working_directory)
+         retval = self.cmd.run(tdnf_cmd)
+         if retval != 0:
+             retval = self.cmd.run(['docker', 'run',
+@@ -683,8 +684,7 @@ def _initialize_system(self):
+         }
+         for device, (mode, dev_type, major, minor) in devices.items():
+             os.mknod(os.path.join(self.photon_root, "dev", device),
+-                    mode | dev_type, os.makedev(major, minor))
+-
++                     mode | dev_type, os.makedev(major, minor))
+ 
+     def _mount_special_folders(self):
+         for d in ["/proc", "/dev", "/dev/pts", "/sys"]:
+@@ -751,8 +751,8 @@ def _setup_grub(self):
+             retval = self.cmd.run('grub2-install --target=i386-pc --force --boot-directory={} {}'.format(self.photon_root + "/boot", self.install_config['disk']))
+             if retval != 0:
+                 retval = self.cmd.run(['grub-install', '--target=i386-pc', '--force',
+-                                   '--boot-directory={}'.format(self.photon_root + "/boot"),
+-                                   self.install_config['disk']])
++                                      '--boot-directory={}'.format(self.photon_root + "/boot"),
++                                      self.install_config['disk']])
+                 if retval != 0:
+                     raise Exception("Unable to setup grub")
+ 
+@@ -823,6 +823,7 @@ def _adjust_packages_based_on_selected_flavor(self):
+         Install slected linux flavor only
+         """
+         redundant_linux_flavors = []
++
+         def filter_packages(package):
+             package = package.split('-')
+             if len(package) > 1:
+@@ -846,7 +847,7 @@ def filter_packages(package):
+                 else:
+                     flavor = ""
+                 redundant_linux_flavors.append(flavor)
+-        self.install_config['packages'] = list(filter(filter_packages,self.install_config['packages']))
++        self.install_config['packages'] = list(filter(filter_packages, self.install_config['packages']))
+ 
+     def _add_packages_to_install(self, package):
+         """
+@@ -880,8 +881,6 @@ def _setup_install_repo(self):
+             # want input RPMS to be removed after installation.
+             if keepcache:
+                 conf_file.write("keepcache=1\n")
+-            conf_file.write("repodir={}\n".format(self.working_directory))
+-            conf_file.write("cachedir={}\n".format(self.rpm_cache_dir_short))
+ 
+     def _install_additional_rpms(self):
+         rpms_path = self.install_config.get('additional_rpms_path', None)
+@@ -889,7 +888,7 @@ def _install_additional_rpms(self):
+         if not rpms_path or not os.path.exists(rpms_path):
+             return
+ 
+-        if self.cmd.run([ 'rpm', '--root', self.photon_root, '-U', rpms_path + '/*.rpm' ]) != 0:
++        if self.cmd.run(['rpm', '--root', self.photon_root, '-U', rpms_path + '/*.rpm']) != 0:
+             self.logger.info('Failed to install additional_rpms from ' + rpms_path)
+             self.exit_gracefully()
+ 
+@@ -903,9 +902,13 @@ def _install_packages(self):
+         packages_to_install = {}
+         total_size = 0
+         stderr = None
+-        tdnf_cmd = "tdnf install --releasever {0} --installroot {1} --assumeyes -c {2} {3}".format(
+-                    self.install_config['photon_release_version'], self.photon_root,
+-                    self.tdnf_conf_path, " ".join(selected_packages))
++        tdnf_cmd = ("tdnf install -y --releasever {0} --installroot {1} "
++                    "-c {2} --setopt=reposdir={3} "
++                    "{4}").format(self.install_config['photon_release_version'],
++                                  self.photon_root,
++                                  self.tdnf_conf_path,
++                                  self.working_directory,
++                                  " ".join(selected_packages))
+         self.logger.debug(tdnf_cmd)
+ 
+         # run in shell to do not throw exception if tdnf not found
+@@ -922,7 +925,7 @@ def _install_packages(self):
+                 if state == 0:
+                     if output == 'Installing:\n':
+                         state = 1
+-                elif state == 1: #N A EVR Size(readable) Size(in bytes)
++                elif state == 1:  # N A EVR Size(readable) Size(in bytes)
+                     if output == '\n':
+                         state = 2
+                         self.progress_bar.update_num_items(total_size)
+@@ -948,7 +951,7 @@ def _install_packages(self):
+ 
+                     self.progress_bar.update_message(output)
+         else:
+-            stdout,stderr = process.communicate()
++            stdout, stderr = process.communicate()
+             self.logger.info(stdout.decode())
+             retval = process.returncode
+             # image creation. host's tdnf might not be available or can be outdated (Photon 1.0)
+@@ -958,9 +961,9 @@ def _install_packages(self):
+                 stderr = None
+                 self.logger.info("Retry 'tdnf install' using docker image")
+                 retval = self.cmd.run(['docker', 'run',
+-                                   '-v', self.rpm_cache_dir+':'+self.rpm_cache_dir,
+-                                   '-v', self.working_directory+':'+self.working_directory,
+-                                   self.install_config['photon_docker_image'], '/bin/sh', '-c', tdnf_cmd])
++                                       '-v', self.rpm_cache_dir+':'+self.rpm_cache_dir,
++                                       '-v', self.working_directory+':'+self.working_directory,
++                                       self.install_config['photon_docker_image'], '/bin/sh', '-c', tdnf_cmd])
+ 
+         # 0 : succeed; 137 : package already installed; 65 : package not found in repo.
+         if retval != 0 and retval != 137:
+@@ -1036,12 +1039,12 @@ def _create_logical_volumes(self, physical_partition, vg_name, lv_partitions, ex
+         """
+         Create logical volumes
+         """
+-        #Remove LVM logical volumes and volume groups if already exists
+-        #Existing lvs & vg should be removed to continue re-installation
+-        #else pvcreate command fails to create physical volumes even if executes forcefully
++        # Remove LVM logical volumes and volume groups if already exists
++        # Existing lvs & vg should be removed to continue re-installation
++        # else pvcreate command fails to create physical volumes even if executes forcefully
+         retval = self.cmd.run(['bash', '-c', 'pvs | grep {}'. format(vg_name)])
+         if retval == 0:
+-            #Remove LV's associated to VG and VG
++            # Remove LV's associated to VG and VG
+             retval = self.cmd.run(["vgremove", "-f", vg_name])
+             if retval != 0:
+                 self.logger.error("Error: Failed to remove existing vg before installation {}". format(vg_name))
+@@ -1071,12 +1074,12 @@ def _create_logical_volumes(self, physical_partition, vg_name, lv_partitions, ex
+             lv_cmd = ['lvcreate', '-y']
+             lv_name = partition['lvm']['lv_name']
+             size = partition['size']
+-            if partition['size'] == 0:
++            if size == 0:
+                 # Each volume group can have only one extensible logical volume
+                 if not extensible_logical_volume:
+                     extensible_logical_volume = partition
+             else:
+-                lv_cmd.extend(['-L', '{}M'.format(partition['size']), '-n', lv_name, vg_name ])
++                lv_cmd.extend(['-L', '{}M'.format(size), '-n', lv_name, vg_name])
+                 retval = self.cmd.run(lv_cmd)
+                 if retval != 0:
+                     raise Exception("Error: Failed to create logical volumes , command: {}".format(lv_cmd))
+@@ -1088,7 +1091,7 @@ def _create_logical_volumes(self, physical_partition, vg_name, lv_partitions, ex
+ 
+         lv_name = extensible_logical_volume['lvm']['lv_name']
+         lv_cmd = ['lvcreate', '-y']
+-        lv_cmd.extend(['-l', '100%FREE', '-n', lv_name, vg_name ])
++        lv_cmd.extend(['-l', '100%FREE', '-n', lv_name, vg_name])
+ 
+         retval = self.cmd.run(lv_cmd)
+         if retval != 0:
+@@ -1152,7 +1155,7 @@ def _get_partition_tree_view(self):
+ 
+         # Add accumulated VG partitions
+         for disk, vg_list in vg_partitions.items():
+-                ptv[disk].extend(vg_list.values())
++            ptv[disk].extend(vg_list.values())
+         return ptv
+ 
+     def _insert_boot_partitions(self):
+@@ -1165,7 +1168,7 @@ def _insert_boot_partitions(self):
+             if ptype == PartitionType.ESP:
+                 esp_found = True
+ 
+-       # Adding boot partition required for ostree if already not present in partitions table
++        # Adding boot partition required for ostree if already not present in partitions table
+         if 'ostree' in self.install_config:
+             mount_points = [partition['mountpoint'] for partition in self.install_config['partitions'] if 'mountpoint' in partition]
+             if '/boot' not in mount_points:
+@@ -1176,12 +1179,12 @@ def _insert_boot_partitions(self):
+ 
+         # Insert efi special partition
+         if not esp_found and (bootmode == 'dualboot' or bootmode == 'efi'):
+-            efi_partition = { 'size': ESPSIZE, 'filesystem': 'vfat', 'mountpoint': '/boot/efi' }
++            efi_partition = {'size': ESPSIZE, 'filesystem': 'vfat', 'mountpoint': '/boot/efi'}
+             self.install_config['partitions'].insert(0, efi_partition)
+ 
+         # Insert bios partition last to be very first
+         if not bios_found and (bootmode == 'dualboot' or bootmode == 'bios'):
+-            bios_partition = { 'size': BIOSSIZE, 'filesystem': 'bios' }
++            bios_partition = {'size': BIOSSIZE, 'filesystem': 'bios'}
+             self.install_config['partitions'].insert(0, bios_partition)
+ 
+     def _partition_disk(self):
+@@ -1242,7 +1245,7 @@ def _partition_disk(self):
+             # Try to use 'sgdisk -m' to convert GPT to MBR and see whether it works.
+             if self.install_config.get('partition_type', 'gpt') == 'msdos':
+                 # m - colon separated partitions list
+-                m = ":".join([str(i) for i in range(1,part_idx)])
++                m = ":".join([str(i) for i in range(1, part_idx)])
+                 retval = self.cmd.run(['sgdisk', '-m', m, disk])
+                 if retval != 0:
+                     raise Exception("Failed to setup efi partition")
+@@ -1300,7 +1303,7 @@ def _format_partitions(self):
+                 mkfs_cmd = ['mkfs', '-t', partition['filesystem']]
+ 
+             if 'fs_options' in partition:
+-                options = re.sub("[^\S]", " ", partition['fs_options']).split()
++                options = re.sub(r"[^\S]", " ", partition['fs_options']).split()
+                 mkfs_cmd.extend(options)
+ 
+             mkfs_cmd.extend([partition['path']])
+@@ -1309,7 +1312,7 @@ def _format_partitions(self):
+             if retval != 0:
+                 raise Exception(
+                     "Failed to format {} partition @ {}".format(partition['filesystem'],
+-                                                         partition['path']))
++                                                                partition['path']))
+ 
+     def getfile(self, filename):
+         """