diff --git a/bin/zopen b/bin/zopen index bbeb141f6..432b38c55 100755 --- a/bin/zopen +++ b/bin/zopen @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -x # # zopen wrapper script to allow single point of entry to zopen # tooling. @@ -35,13 +35,15 @@ Command: config change zopen runtime environment settings diagnostics collects system info for zopen troubleshooting generate generates a new zopen project + info displays detailed information about a package init initializes a zopen environment at the specified location - refresh refreshes your zopen environment and zopen-config file install installs one or more zopen community packages - info displays detailed information about a package list lists information about zopen community packages + mirror performs a synchronization of the GitHub repository + promote promote a zopen environment to another location publish publish zopen package release to github query list local or remote info about zopen community packages + refresh refreshes your zopen environment and zopen-config file remove removes installed zopen community packages update-cacert update the cacert.pem file used by zopen community upgrade upgrades existing zopen community packages @@ -51,6 +53,7 @@ Command: Options: -h, --help, -? display this help and exit -v, --verbose run in verbose mode + --version display the zopen version Examples: zopen --help displays zopen help @@ -59,7 +62,7 @@ Examples: zopen upgrade -y upgrade all installed packages to the latest release, without prompting zopen alt bash list installed alternative bash packages - zopen info vim displays details information about the installed vim package + zopen info vim displays detailed information about the installed vim package zopen usage --pie displays an ASCII-art chart showing biggest space hogs SEE ALSO: @@ -69,13 +72,19 @@ SEE ALSO: zopen-clean(1) zopen-config-helper(1) zopen-generate(1) + zopen-info(1) zopen-init(1) zopen-install(1) - zopen-info(1) + zopen-list(1) + zopen-mirror(1) + zopen-promote(1) zopen-publish(1) zopen-query(1) + zopen-promote(1) + zopen-refresh(1) zopen-remove(1) zopen-update-cacert(1) + zopen-upgrade(1) zopen-usage(1) zopen-whichproject(1) zopen-version(1) @@ -92,41 +101,48 @@ subopts="" subcmd="" help=false version=false +verbFound=false -for arg in $*; do - case "${arg}" in - "alt"|"audit"|"build"|"clean"|"generate"|"init"|"install"|\ - "query"|"remove"|"update-cacert"|"info"|"usage"|"diagnostics"|\ - "whichproject"|"publish") - subcmd="zopen-${arg}" - ;; - "list") - subcmd='zopen-query' - subopts="${subopts} --list" - ;; - "refresh") - subcmd='zopen-init' - subopts="${subopts} --refresh" - ;; - "upgrade") - subcmd='zopen-install' - subopts="${subopts} -u" - ;; - "config") - subcmd="zopen-${arg}-helper" - ;; - "--version") - subcmd='zopen-version' - version=true - ;; - "help" | "--help" | "-?") - help=true - ;; - *) - # let unknown stuff through - subopts="${subopts} ${arg}" - ;; - esac +while [ "$#" -gt 0 ]; do + arg=$1 || shift + if $verbFound; then + subopts="${subopts} ${arg}" + else + case "${arg}" in + "alt"|"audit"|"build"|"clean"|"generate"|"init"|"install"|\ + "query"|"remove"|"update-cacert"|"info"|"usage"|"diagnostics"|\ + "whichproject"|"publish"|"list"|"upgrade"|"mirror"|"promote") + subcmd="zopen-${arg}" + verbFound=true + ;; + "refresh") + subcmd='zopen-init' + subopts="${subopts} --refresh" + verbFound=true + ;; + "config") + subcmd="zopen-${arg}-helper" + verbFound=true + ;; + "--version") + if [ -z "${subcmd}" ]; then + subcmd='zopen-version' + version=true + else + subopts="${subopts} ${arg}" + fi + ;; + "help" | "--help" | "-?") + help=true + verbFound=true + ;; + *) + # let unknown stuff through + subopts="${subopts} ${arg}" + ;; + esac + fi + shift done if [ -z "${subcmd}" ]; then @@ -152,4 +168,5 @@ if ${version}; then fi fi +# shellcheck disable=SC2086 # We want globbing in this case ${subcmd} ${subopts} diff --git a/bin/zopen-alt b/bin/zopen-alt index f170d186d..99418764f 100755 --- a/bin/zopen-alt +++ b/bin/zopen-alt @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -x # # "Alternatives" utility for zopen community - https://github.com/zopencommunity # @@ -55,14 +55,29 @@ mergeNewVersion() { package="$1" newver="$2" - oldver="$2" - [ -z "${package}" ] && printError "Internal error; no packagename provided to merge." - [ -d "${ZOPEN_PKGINSTALL}/${package}/${newver}" ] || printError "Version '${newver}' was not available to set as current" + + [ -z "${package}" ] && printError "Internal error; no package name provided to merge." + newVerDir="${ZOPEN_PKGINSTALL}/${package}/${newver}" + [ -d "${newVerDir}" ] || printError "Version '${newver}' was not available to set as current" + + # We are effectively re-installing so run some scriptlets, noting that for the + # alternative to be available on the system, it must have passed various the + # checks in installPre (GPG/prereq) + if ! processActionScripts "transactionPre"; then + exit 1 + fi if [ -e "${ZOPEN_PKGINSTALL}/${package}/${package}" ]; then printVerbose "Removing main link" - rm -rf "${ZOPEN_PKGINSTALL}/${package}/${package}" + if ! rm -rf "${ZOPEN_PKGINSTALL}/${package}/${package}:?"; then + printSoftError "Unable to remove previously set package link: ${ZOPEN_PKGINSTALL}/${package}/${package}" + printError "Check permissions and retry command." + fi + fi + if ! runLogProgress "mergeIntoSystem \"${package}\" \"${ZOPEN_PKGINSTALL}/${package}//${newver}\" \"${ZOPEN_ROOTFS}\"" \ + "Merging ${package} into symlink mesh" "Merged ${package} into symlink mesh"; then + printSoftError "Unexpected errors merging symlinks into mesh tree" + printError "Use zopen alt to select previous version to ensure known state" fi - mergeIntoSystem "${package}" "${ZOPEN_PKGINSTALL}/${package}/${newver}" "${ZOPEN_ROOTFS}" printVerbose "New version merged; checking for orphaned files from previous version" # This will remove any old symlinks or dirs that might have changed in an up/downgrade # as the merge process overwrites existing files to point to different version. If there was @@ -84,7 +99,12 @@ mergeNewVersion() if [ -e "${ZOPEN_PKGINSTALL}/${package}/${package}/.releaseinfo" ]; then version=$(cat "${ZOPEN_PKGINSTALL}/${package}/${package}/.releaseinfo") fi + printVerbose "Updating package DB" + updatePackageDB + syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE}" "ALT" "setAlt" "Set '${package}' to version:${version};" + processActionScripts "installPost" "${package}" + processActionScripts "transactionPost" } setAlt() @@ -93,7 +113,7 @@ setAlt() newver="$2" if [ -e "${ZOPEN_PKGINSTALL}/${package}/" ]; then printVerbose "${package} is either installed or has been previously" - found=$(zosfind "${ZOPEN_PKGINSTALL}/${package}/" -type l -prune -o -type d -print | sed -e "s#${ZOPEN_PKGINSTALL}/${package}/\([^/]*\).*#\1#" | uniq) + found=$(find "${ZOPEN_PKGINSTALL}/${package}/" -type l -prune -o -type d -print | sed -e "s#${ZOPEN_PKGINSTALL}/${package}/\([^/]*\).*#\1#" | uniq) else printVerbose "${package} has never been installed on the system" fi @@ -145,7 +165,7 @@ listAlts() package=$(echo "${package}" | awk '{$1=$1};1') if [ -e "${ZOPEN_PKGINSTALL}/${package}/" ]; then printVerbose "${package} is either installed or has been previously" - found=$(zosfind "${ZOPEN_PKGINSTALL}/${package}/" -type l -prune -o -type d -print | sed -e "s#${ZOPEN_PKGINSTALL}/${package}/\([^/]*\).*#\1#" | uniq) + found=$(find "${ZOPEN_PKGINSTALL}/${package}/" -type l -prune -o -type d -print | sed -e "s#${ZOPEN_PKGINSTALL}/${package}/\([^/]*\).*#\1#" | uniq) else printVerbose "${package} has never been installed on the system" fi @@ -164,11 +184,11 @@ listAlts() # echo "${found}" | xargs | tr ' ' '\n' | while read repo; do TMP_FIFO_PIPE="${HOME}/altselect.pipe" [ ! -p "${TMP_FIFO_PIPE}" ] || rm -f "${TMP_FIFO_PIPE}" - mkfifo ${TMP_FIFO_PIPE} + mkfifo "${TMP_FIFO_PIPE}" echo "${found}" | xargs | tr ' ' '\n' >> "${TMP_FIFO_PIPE}" & while read repo; do printVerbose "Parsing repo: '${repo}' as '${repo#"${ZOPEN_PKGINSTALL}"/}'" - i=$(expr ${i} + 1) + i=$((i + 1)) if [ "${deref#"${ZOPEN_PKGINSTALL}"/}" = "${repo#"${ZOPEN_PKGINSTALL}"/}" ]; then current=${i} printInfo "${NC}${GREEN}${i}: ${repo#"${ZOPEN_PKGINSTALL}"/} <- current${NC}" @@ -176,7 +196,7 @@ listAlts() printInfo "${i}: ${repo#"${ZOPEN_PKGINSTALL}"/}" fi done < "${TMP_FIFO_PIPE}" - [ ! -p ${TMP_FIFO_PIPE} ] || rm -rf "${TMP_FIFO_PIPE}" + [ ! -p "${TMP_FIFO_PIPE}" ] || rm -rf "${TMP_FIFO_PIPE}" if ${select}; then mutexReq "zopen" "zopen" @@ -202,19 +222,14 @@ while [ $# -gt 0 ]; do case "$1" in "-s" | "--set") shift - [ $# -lt 2 ] && printError "Missing argument(s) for set option. Check program arguments" + [ $# -lt 1 ] && printError "Missing argument for 'set' option. Check program arguments" sett=true select=false - package="$1" - newver="$2" - shift + newver="$1" ;; "--select") select=true sett=false - shift - [ $# -lt 1 ] && printError "Missing argument for select option." - package="$1" ;; "-h" | "--help" | "-?") printHelp "${args}" @@ -228,7 +243,9 @@ while [ $# -gt 0 ]; do verbose=true ;; "--debug") + # shellcheck disable=SC2034 verbose=true + # shellcheck disable=SC2034 debug=true ;; *) @@ -238,6 +255,10 @@ while [ $# -gt 0 ]; do shift done +if ${select} && [ -z "${package}" ]; then + printError "Missing argument for select option." +fi + if ${sett}; then setAlt "${package}" "${newver}" elif [ -n "${package}" ]; then diff --git a/bin/zopen-audit b/bin/zopen-audit index ec4aa5ccc..b14a2a5ec 100755 --- a/bin/zopen-audit +++ b/bin/zopen-audit @@ -84,31 +84,7 @@ if [ $upgrade = true ] && [ $remove = true ]; then exit 1 fi -JSON_VULNERABILITIES_URL="https://raw.githubusercontent.com/zopencommunity/meta/main/docs/api/zopen_vulnerability.json" -LATEST_RELEASES_URL="https://raw.githubusercontent.com/zopencommunity/meta/main/docs/api/zopen_releases_latest.json" - -downloadJsonCaches() -{ - cachedir="${ZOPEN_ROOTFS}/var/cache/zopen" - [ ! -e "${cachedir}" ] && mkdir -p "${cachedir}" - JSON_CVE_CACHE="${cachedir}/zopen_vulnerability.json" - - if ! curlout=$(curlCmd -L --fail --no-progress-meter -o "${JSON_CVE_CACHE}" "${JSON_VULNERABILITIES_URL}"); then - printError "Failed to obtain vulnerability json from ${JSON_VULNERABILITIES_URL}; ${curlout}" - fi - chtag -tc 819 "${JSON_CVE_CACHE}" - - if $upgrade; then - LATEST_RELEASES_CACHE="${cachedir}/zopen_releases_latest.json" - - if ! curlout=$(curlCmd -L --fail --no-progress-meter -o "${LATEST_RELEASES_CACHE}" "${LATEST_RELEASES_URL}"); then - printError "Failed to obtain latest releases json from ${LATEST_RELEASES_URL}; ${curlout}" - fi - chtag -tc 819 "${LATEST_RELEASES_CACHE}" - fi -} - -downloadJsonCaches +updateCaches if [ ! -f "${JSON_CVE_CACHE}" ]; then printError "Vulnerability json cache file not found." @@ -116,7 +92,7 @@ if [ ! -f "${JSON_CVE_CACHE}" ]; then fi printVerbose "Obtained vulnerability json cache." -if [ $upgrade = true ] && [ ! -f "${LATEST_RELEASES_CACHE}" ]; then +if [ $upgrade = true ] && [ ! -f "${JSON_LATEST_CACHE}" ]; then printError "Latest releases json cache file not found." fi printVerbose "Obtained latest releases json cache." @@ -129,7 +105,7 @@ high_vulns=0 critical_vulns=0 # Check for CVEs in all installed projects -installed_packages=$(cd "${ZOPEN_PKGINSTALL}" && zosfind ./*/. ! -name . -prune -type l) +installed_packages=$(cd "${ZOPEN_PKGINSTALL}" && find ./*/. ! -name . -prune -type l) printVerbose "Found all installed packages." # Variables for --upgrade flag @@ -190,7 +166,7 @@ while IFS= read -r repo; do latest_release_vulns="" is_latest_release=true if $upgrade; then - latest_release=$(jq -cr '.release_data | .["'$repo'"] | .[0] | .assets | .[0] | .release' $LATEST_RELEASES_CACHE) + latest_release=$(jqw -cr '.release_data | .["'$repo'"] | .[0] | .assets | .[0] | .release' $JSON_LATEST_CACHE) if [ "$release" != "$latest_release" ]; then is_latest_release=false diff --git a/bin/zopen-build b/bin/zopen-build index f6319de4f..8d33daf3a 100755 --- a/bin/zopen-build +++ b/bin/zopen-build @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -x # # General purpose build script for zopen community ports - https://github.com/zopencommunity # @@ -760,6 +760,7 @@ setDepsEnv() deps="${ZOPEN_DEPS}" orig="${PWD}" + printVerbose "Saving original work directory '${orig}'" # Filter out duplicate deps deps=$(echo "${deps}" | xargs | tr ' ' '\n' | sort -u) for dep in ${deps}; do @@ -787,10 +788,13 @@ setDepsEnv() fi printVerbose "Setting up ${depdir} dependency environment" - cd "${depdir}" && . ./.env + curDir="$(pwd -P)" + cd "${depdir}" || printError "Failed to change to '${depdir}'" + . ./.env if [ $? -gt 0 ]; then printError "Failed to source ${depdir} .env" fi + cd "${curDir}" || printError "Failed to return to '${curDir}'" fi foundDep=true break @@ -812,10 +816,13 @@ setDepsEnv() fi depdir="${path}/${dep}/${dep}" printVerbose "Setting up upgraded ${depdir} dependency environment" - cd "${depdir}" && . ./.env + curDir="$(pwd -P)" + cd "${depdir}" || printError "Failed to change to '${depdir}'" + ./.env if [ $? -gt 0 ]; then printError "Failed to source ${depdir} .env" fi + cd "${curDir}" || printError "Failed to change back to '${curDir}'" versionPath="${depdir}/.version" if [ -r "${versionPath}" ]; then version=$(cat "${versionPath}") @@ -825,7 +832,8 @@ setDepsEnv() fi fi done - cd "${orig}" || exit 99 + cd "${orig}" || printError "Unable to change back to '${orig}'" + printVerbose "Changed back to directory '${orig}'" } setEnv() @@ -1043,6 +1051,7 @@ extractTarBall() fi rm -f "${tarballz}" tagTree "${sourcedir}" + curDir="$(pwd -P)" # TODO: Do we need to return here?? cd "${sourcedir}" || printError "Cannot cd to ${sourcedir}" # Clean up .git* files since we will be creating our own local git repo for applying patches diff --git a/bin/zopen-clean b/bin/zopen-clean index 77b571b09..78b157344 100755 --- a/bin/zopen-clean +++ b/bin/zopen-clean @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -x # # Cleanup utility for zopen community - https://github.com/zopencommunity # @@ -15,6 +15,7 @@ setupMyself() echo "Internal Error. Unable to find common.sh file to source." >&2 exit 8 fi + # shellcheck disable=SC1091 . "${INCDIR}/common.sh" } setupMyself @@ -23,13 +24,13 @@ checkWritable printHelp() { cat << HELPDOC -${ME} is a utility for zopen community to remove uneeded resources +${ME} is a utility for zopen community to remove unneeded resources from the system to save space and prevent clutter. Usage: ${ME} [OPTION] [PACKAGE] Options: - --deep deep clean - run all cleanup operations + --deep deep clean - run all cleanup operations --all apply cleanup command to all applicable packages. -c, --cache [PACKAGE ...] cleans the downloaded package cache; packages will be @@ -50,10 +51,10 @@ Examples: zopen clean -d analyse the zopen file system and remove dangling symlinks zopen clean -u [PACKAGE] remove unused versions for PACKAGE - zopen clean -u --all + zopen clean -u --all remove all unused packages within the zopen environment - zopen clean --deep - + zopen clean --deep + Report bugs at https://github.com/zopencommunity/meta/issues. @@ -74,10 +75,11 @@ removeUnusedPackageVersions() else printInfo "No currently active version of '${needle}'; removing all versions" fi - + counterfile=$(mktempfile "clean" ".rupv") addCleanupTrapCmd "rm -f ${counterfile}" - cd "${ZOPEN_PKGINSTALL}/${needle}" && zosfind . -name "./*" -prune -type d > "${counterfile}" + (cd "${ZOPEN_PKGINSTALL}/${needle}" && find . -name "./*" -prune -type d > "${counterfile}") + unusedCount=0 while read repo; do printVerbose "Parsing repo: '${repo}' as '${repo#./}'" repo="${repo#./}" @@ -96,15 +98,15 @@ removeUnusedPackageVersions() fi fi done < "${counterfile}" + $stats && printInfo "Removed ${unusedCount} unused package version$([ ${unusedCount} -eq 1 ] || echo 's')" return 0 } cleanUnused() { ${deep} && printInfo "Removing any unused package versions" - unusedCount=0 # Updated within the removeUnusedPackageVersions function - if ${all}; then - zopen list --installed --no-header --no-version | while read -r needle; do + if ${all}; then + zopen list --installed --no-header --no-version | while read -r needle; do removeUnusedPackageVersions "${needle}" done elif [ -z "$1" ]; then @@ -114,7 +116,6 @@ cleanUnused() removeUnusedPackageVersions "${needle}" done fi - $stats && printInfo "Removed ${unusedCount} unused package version"$([ ${unusedCount} -eq 1 ] || echo 's') } cleanDangling() @@ -148,7 +149,7 @@ cleanDangling() counterfile=$(mktempfile "clean" ".sl") addCleanupTrapCmd "rm -f ${counterfile}" - zosfind "${ZOPEN_ROOTFS}" -type l -exec test ! -e {} \; -print > "${counterfile}" + find "${ZOPEN_ROOTFS}" -type l -exec test ! -e {} \; -print > "${counterfile}" counter=0 while IFS= read -r sl; do printVerbose "Removing dangling symlink '${sl}'" @@ -177,7 +178,7 @@ cleanPackageCache() else for needle in $1; do printVerbose "Cleaning ${ZOPEN_ROOTFS}/var/cache/zopen entries for '${needle}" - zosfind "${ZOPEN_ROOTFS}"/var/cache/zopen -name "${needle}-*" -exec rm {} \; + find "${ZOPEN_ROOTFS}"/var/cache/zopen -name "${needle}-*" -exec rm {} \; syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_FILE}" "CLEAN" "cleanPackageCache" "Cache for '${needle}' in ${ZOPEN_ROOTFS}/var/cache/zopen cleaned." printVerbose "- Cleaned cached files for '${needle}' in cache at '${ZOPEN_ROOTFS}/var/cache/zopen'." done @@ -238,7 +239,7 @@ while [ $# -gt 0 ]; do unused=true dangling=true cache=true - meta=true + meta=true ;; "--nostats") stats=false @@ -294,7 +295,7 @@ fi ${unused} && cleanUnused "${packagelist}" ${cache} && cleanPackageCache "${packagelist}" -if ${stats}; then +if ${stats}; then afterUsage=$(diskusage "${ZOPEN_ROOTFS}") printInfo "- Disk usage: Before: $(formattedFileSize "${beforeUsage}"); After: $(formattedFileSize "${afterUsage}"); Reclaimed: $(formattedFileSize $((beforeUsage - afterUsage)))" fi diff --git a/bin/zopen-config-helper b/bin/zopen-config-helper index c7296f371..ce61e20d1 100755 --- a/bin/zopen-config-helper +++ b/bin/zopen-config-helper @@ -45,8 +45,46 @@ Examples: zopen config --set is_collecting_stats false disable the is_collecting_stats functionality +Variables + The possible configuration values for the zopen environment are listed + below. The type of value expected is given, one of: + integer|boolean|string. + Where the value must be one of an enumerated set, the default value is + listed first. Validation of those values will be performed at runtime. + + autocacheclean [boolean:true|false] + Tells zopen install to remove downloaded package files from the cache + to preserve space. + + autopkgpurge [boolean:true|false] + Tells zopen install to remove any older version[s] of a package after + a successful install or upgrade + + is_collecting_stats [boolean:true|false] + Configures the zopen environment to collect anonymous statistcs about + package installations. + + override_zos_tools [boolean:false|true] + Tells zopen-config whether to prioritise native z/OS tools (such as in + /bin) on the \$PATH or to use zopen community ports in preference. + zopen community ports can still be referenced using an appropriate + command prefix; for example: gawk, zotfile, zotman + + release_line [string:STABLE|DEV] + Tells zopen install whether to install packages from the STABLE line + or from the DEV line. DEV packages will be newer but might exhibit + unexpected behaviour. The default is to use STABLE packages. + + skip_broken [boolean:true|false] + Tells zopen that if during a transation an action fails, report the + error and continue to the next action when possible. + + skip_size_check [boolean:false|true] + Tells zopen to skip the validation of file space to allow for dynamic + file-systems such as ZFS mounts with AGGRGROW. + Notes: - Configuration options are not validated such that any key/value pairs can + Configuration options are not validated in that any key/value pairs can be added into the global configuration. 3rd-party utilities can store their global configuration into the zopen runtime environment store and use the zopen config tooling to set/retrieve values. Key names for stored properties diff --git a/bin/zopen-diagnostics b/bin/zopen-diagnostics index 95a9af537..c42ae5be1 100755 --- a/bin/zopen-diagnostics +++ b/bin/zopen-diagnostics @@ -33,7 +33,7 @@ printSyntax() print_Diagnostics() { - PLATFORM=$(/bin/uname -s) + PLATFORM=$(uname -s) # Checking if the platform is z/OS if [ ! ${PLATFORM} = "OS/390" ]; then @@ -42,15 +42,15 @@ print_Diagnostics() fi # Check version - VERSION=$(/bin/uname -rsvI 2>/dev/null) + VERSION=$(uname -rsvI 2>/dev/null) if [ -z "$VERSION" ]; then echo "ERROR: This z/OS system does not have a valid version." exit 1 fi - MAJOR=$(echo "$VERSION" | /bin/awk '{print $3}' | /bin/sed 's/^0*//') - MINOR=$(echo "$VERSION" | /bin/awk '{print $2}' | /bin/cut -d'.' -f1 | /bin/sed 's/^0*//') + MAJOR=$(echo "$VERSION" | awk '{print $3}' | sed 's/^0*//') + MINOR=$(echo "$VERSION" | awk '{print $2}' | cut -d'.' -f1 | sed 's/^0*//') if [ -z "$MINOR" ]; then MINOR=0 fi @@ -103,11 +103,11 @@ print_Diagnostics() echo "Architecture: $cpu_arch" echo "Zopen Version: $(zopen --version | head -1 | cut -d ' ' -f4)" echo "Disk Usage for ZOPEN_ROOTFS ($ZOPEN_ROOTFS):" - num1=$(/bin/du -kt $ZOPEN_ROOTFS | tail -1 | tr -s ' ' | cut -d ' ' -f2) - num2=1024 - echo "$(echo "scale=2; $num1 / $num2" | bc) mb" + num1=$(du -kt $ZOPEN_ROOTFS | tail -1 | tr -s ' ' | cut -d ' ' -f2) + num2=1024 + echo "$(echo "scale=2; $num1 / $num2" | bc) mb" echo -e "\nFilesystem Usage (df) for \$ZOPEN_ROOTFS:" - /bin/df -Pm "$ZOPEN_ROOTFS" + df -Pm "$ZOPEN_ROOTFS" } while [ $# -gt 0 ]; do @@ -137,7 +137,7 @@ verbose=false remote_lookup=false if [ -z "$args" ]; then - echo "To open an issue go to https://github.com/zopencommunity/meta/issues and paste the above information" + echo "To open an issue go to https://github.com/zopencommunity/meta/issues and paste the above information" else args_spaceless=$(echo "$args" | cut -d' ' -f2| tr -d ' ') url="To open an issue go to https://github.com/zopencommunity/"$args_spaceless"port/issues and paste the above information" diff --git a/bin/zopen-generate b/bin/zopen-generate index 0715dd245..ff33cac0d 100755 --- a/bin/zopen-generate +++ b/bin/zopen-generate @@ -331,7 +331,7 @@ zopen_get_version() " -/bin/echo "${buildenvContents}" > "${buildenv}" +echo "${buildenvContents}" > "${buildenv}" printInfo "${buildenv} created" diff --git a/bin/zopen-info b/bin/zopen-info index 6bea73a76..5bc96f0f8 100755 --- a/bin/zopen-info +++ b/bin/zopen-info @@ -15,6 +15,7 @@ setupMyself() echo "Internal Error. Unable to find common.sh file to source." >&2 exit 8 fi + # shellcheck disable=SC1091 . "${INCDIR}/common.sh" } setupMyself @@ -38,7 +39,7 @@ printPackageInfo() { remote_lookup=${2:-false} pkghome="${ZOPEN_PKGINSTALL}/${package}/${package}" metadata="${pkghome}/metadata.json" - + if ${remote_lookup} || [ ! -d "${pkghome}" ]; then metadataFile="$zopen_tmp_dir/$LOGNAME.$RANDOM.metadata.json" printVerbose "Performing remote lookup for package '${package}'" @@ -67,7 +68,7 @@ printPackageInfo() { if [ -z "${remote_data}" ] || [ "${remote_data}" = "null" ]; then printError "Package '${package}' not found in remote repository." exit 1 - fi + fi name=$(echo "${remote_data}" | jq -r '.name') url=$(echo "${remote_data}" | jq -r '.assets[0].url') @@ -76,7 +77,6 @@ printPackageInfo() { if ! runAndLog "curlCmd -s -L '${metadataJSONURL}' -o '${metadataFile}'"; then printError "Could not download from ${metadataJSONURL}. Correct any errors and potentially retry." - continue fi repo=$(echo "${repo}" | sed 's#/[^/]*$##') # Strip /download repo=$(echo "${repo}" | sed 's#^[^/]*//[^/]*##') # Strip protocol and host @@ -91,7 +91,7 @@ printPackageInfo() { size=$(echo "${remote_data}" | jq -r '.assets[0].size') expanded_size=$(echo "${remote_data}" | jq -r '.assets[0].expanded_size') community_commitsha=$(echo "${remote_data}" | jq -r '.assets[0].community_commitsha') - runtime_dependencies=$(echo "${remote_data}" | jq -r '.assets[0].runtime_dependencies') + runtime_dependencies=$(echo "${remote_data}" | jq -r '.assets[0].runtime_dependencies') build_dependencies=$(jq -r '.product.build_dependencies | unique_by(.name) | map(.name) | join(" ")' ${metadataFile}) printHeader "==> ${name} (Not Installed)" @@ -130,13 +130,13 @@ printPackageInfo() { if [ -n "${community_commitsha}" ]; then printf "%-20s %s\n" "Community SHA:" "${community_commitsha}" fi - + if [ -n "${runtime_dependencies}" ]; then printHeader "==> Dependencies" printf "%-20s %s\n" "Runtime:" "${runtime_dependencies}" printf "%-20s %s\n" "Build:" "${build_dependencies}" fi - + rm ${metadataFile} return 0 @@ -207,7 +207,7 @@ printPackageInfo() { printHeader "==> Installation Details" printf "%-20s %s\n" "Installed:" "Yes" printf "%-20s %s\n" "Installation Path:" "${installed_path}" - printf "%-20s %s\n" "Installation Size:" "$(/bin/du -s ${installed_path} | awk '{print $1 / 1024 " MB"}')" + printf "%-20s %s\n" "Installation Size:" "$(du -s ${installed_path} | awk '{print $1 / 1024 " MB"}')" if [ -n "${total_tests}" ]; then test_percentage="N/A" @@ -286,6 +286,6 @@ while [ $# -gt 0 ]; do done checkIfConfigLoaded -getReposFromGithub true - +updateCaches +validatePackageList "${package}" printPackageInfo "${package}" "${remote_lookup}" diff --git a/bin/zopen-init b/bin/zopen-init index 6b75d5541..c7484351e 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -1,12 +1,14 @@ -#!/bin/sh +#!/bin/sh -x # Initialize zopen environment - https://github.com/zopencommunity # ME=$(basename "$0") if [ -z "${utildir}" ]; then + # shellcheck disable=SC2155 export utildir="$( cd "$(dirname "$0")" >/dev/null 2>&1 && pwd -P )" fi +# shellcheck disable=SC2034 ZOPEN_DONT_PROCESS_CONFIG=1 # # All zopen-* scripts MUST start with this code to maintain consistency. @@ -20,6 +22,7 @@ setupMyself() echo "Internal Error. Unable to find common.sh file to source." >&2 exit 8 fi + # shellcheck source=/dev/null . "${INCDIR}/common.sh" } setupMyself @@ -48,23 +51,17 @@ Options: --bypass-prereq-checks Bypasses pre-requisite checks - -f, --fs-layout - The filesystem structure to use for installed - packages on disk; packages will be installed to - this location under . - - should be one of: - - usrlclz: /usr/local/zopen (default), - zopen: /usr/zopen, - prod: legacy zopen standard location, - ibm: /usr/lpp, - fhs: File Hierarchical Standard (/opt), - usrlcl: usr/local - -h, -?, --help display this help and exit + --offline + Perform the base installation of the zopen environment without + attempting to update any of the bootstrapping packages. For + use in air-gapped environments or where a custom package + repository will be configed + Note: Post-configuration of alternate package sources might be + required. + --re-init Re-initializes a previous zopen environment or create a new environment using current tooling. @@ -73,6 +70,11 @@ Options: configuration and regenerate configuration files. select the active version for PACKAGE from a list + --rebuild-repo-config + Recreate the default repository configuration file. Will also + refresh the zopen environment configuration, equivalent to + setting the --refresh parameter + --refresh Refreshes the zopen-config file @@ -92,58 +94,56 @@ Examples: interactively bootstrap a zopen environment zopen init --releaseline-dev - interactively bootstrap a zopen environment that - will use Development Releaseline packages - - zopen init --yes --append-to-profile --fs-layout fhs /zopen - non-interactively create a zopen environment at - location '/zopen' on disk, with packages installed - to '/zopen/opt'. The user's .profile will be - updated to source the configuration file at - '/zopen/etc/zopen-config' when new terminal - sessions start + interactively bootstrap a zopen environment that will use + Development Releaseline packages + zopen init --yes --append-to-profile /zopen + non-interactively create a zopen environment at location + '/zopen' on disk, with packages installed to + '/usr/local/zopen'. The user's .profile will be updated to + source the configuration file at '/zopen/etc/zopen-config' + when new terminal sessions start Report bugs at https://github.com/zopencommunity/meta/issues HELPDOC } -args=$* - - -# Constants -# Number of rm processes to run in parallel -RM_FILEPROCS=5 #TODO: adjust based on number of online cpus on system - # Boostrap pax files -CURL_PAX_LOCATION="packages/curl-8.10.1.20241001_214340.zos.pax.Z" -JQ_PAX_LOCATION="packages/jq-1.6.20241001_204116.zos.pax.Z" -GPG_PAX_LOCATION="packages/gnupg-2.5.5.20250407_125451.zos.pax.Z" -PINENTRY_PAX_LOCATION="packages/pinentry-1.3.1.20250131_041514.zos.pax.Z" +CURL_PAX="curl-8.10.1.20241001_214340.zos.pax.Z" +JQ_PAX="jq-1.6.20241001_204116.zos.pax.Z" +GPG_PAX="gnupg-2.5.5.20250407_125451.zos.pax.Z" +PINENTRY_PAX="pinentry-1.3.1.20250131_041514.zos.pax.Z" +PKG_DIR="packages" +CURL_PAX_LOCATION="${PKG_DIR}/${CURL_PAX}" +JQ_PAX_LOCATION="${PKG_DIR}/${JQ_PAX}" +GPG_PAX_LOCATION="${PKG_DIR}/${GPG_PAX}" +PINENTRY_PAX_LOCATION="${PKG_DIR}/${PINENTRY_PAX}" verbose=false isHostIBM=false debug=false +xdebug=false yesToPrompts=false reinitExisting=false appendToProfile=false +offlineInstall=false releaselineDev=false refresh=false -fslayout="usrlclz" +rebuildRepoConfig=false isCollectingStats=false isBot=false # used by jenkins, CI/CD bypassPrereqs=false while [ $# -gt 0 ]; do case "$1" in - "-f" | "--fs-layout" ) - fslayout="$2" - shift - ;; "--re-init" ) reinitExisting=true ;; "--bypass-prereq-checks") - bypassPrereqs=true + bypassPrereqs=true + ;; + "--rebuild-repo-config") + rebuildRepoConfig=true + refresh=true ;; "--refresh" ) refresh=true @@ -167,20 +167,29 @@ while [ $# -gt 0 ]; do "--noenable-stats" ) enableStats=false ;; + "--offline" ) + offlineInstall=true + ;; "-h" | "--help" | "-?") printHelp exit 0 ;; "-v" | "--verbose") + # shellcheck disable=SC2034 verbose=true ;; "--version") - zopen-version $ME + zopen-version "$ME" exit 0 ;; "--debug") + # shellcheck disable=SC2034 debug=true ;; + "--xdebug") + xdebug=true + set -x + ;; "--yes" | "-y") yesToPrompts=true # Automatically answer 'yes' to any questions ;; @@ -193,7 +202,7 @@ while [ $# -gt 0 ]; do esac shift done - +# Wrapper around jq to provide extra help for certain scenarios determineRootFileSystem() { if [ -n "${ZOPEN_ROOT_PATH}" ]; then @@ -247,7 +256,9 @@ determineStatsCollection() elif [ -n "$enableStats" ] && ! $enableStats; then isCollectingStats=false elif [ $analyticsRC -lt 2 ]; then - isCollectingStats=$(jq -re '.is_collecting_stats' "${jsonConfig}") + if ! isCollectingStats=$(jqw -re '.is_collecting_stats' "${jsonConfig}"); then + return 4 + fi return elif $isHostIBM; then printAttention "Attention: Turning on usage statistics as your host is an IBM host. If this is incorrect, rerun with --noenable-stats" @@ -266,7 +277,7 @@ determineStatsCollection() fi if $isCollectingStats; then - if ! /bin/ping -c 1 -t 2 "${ZOPEN_STATS_IP}" >/dev/null 2>/dev/null; then + if ! ping -c 1 -t 2 "${ZOPEN_STATS_IP}" >/dev/null 2>/dev/null; then printAttention "The statistics server url is blocked: ${ZOPEN_STATS_URL}. Turning usage statistics off." isCollectingStats=false fi @@ -288,37 +299,26 @@ init() printHeader "Initializing zopen framework" fi - printDebug "Validating input parameters" - case "${fslayout}" in - "usrlclz") zopen_pkginstall="usr/local/zopen" ;; - "usrlcl") zopen_pkginstall="usr/local" ;; - "fhs") zopen_pkginstall="opt" ;; - "ibm") zopen_pkginstall="usr/lpp" ;; - "prod") zopen_pkginstall="prod" ;; - "zopen") zopen_pkginstall="usr/zopen" ;; - *) printError "${NC}${RED}The filesystem layout ${fslayout} is unrecognised" ;; - esac - determineRootFileSystem determineStatsCollection if ! ${refresh} && ! ${reinitExisting}; then printInfo "- Binaries will be symlinked under \"${rootfs}/usr/local/bin\". Libraries will be symlinked under \"${rootfs}/usr/lib\"" - printInfo "- Packages will be installed and maintained under the directory structure ${fslayout} (${rootfs}/${zopen_pkginstall}). To change, re-run with the -f option." + printInfo "- Packages will be installed and maintained under the directory '${rootfs}/${zopen_pkginstall}'." printInfo "- Collecting usage statistics: ${isCollectingStats}" fi - if ! promptYesOrNo "Do you want to continue?" ${yesToPrompts}; then - printInfo "Exiting..." - exit 0 - fi - if ! ${yesToPrompts} && [ -z "${OVERRIDE_ZOS_TOOLS}" ]; then if promptYesOrNo "Should zopen tools default to overriding the z/OS /bin tools (e.g. man, find, grep)?" ${yesToPrompts}; then OVERRIDE_ZOS_TOOLS=true fi fi + if ! promptYesOrNo "Do you want to continue?" ${yesToPrompts}; then + printInfo "Exiting..." + exit 0 + fi + configFile="${rootfs}/etc/zopen-config" if [ -f "${configFile}" ]; then if [ -e "${configFile}" ]; then @@ -334,8 +334,12 @@ init() [ -n "${reinit}" ] && [ "y" = "${reinit}" ] }; then printInfo "- Re-initializing; zopen-config will be re-created" - if ! rm -rf "${configFile}"; then - printError "Unable to remove existing file. Check permissions and retry command." + reinitExisting=true + backup="zopen-config.$(date +%Y%m%d%H%M%S)" + bupdir=$(dirname "${configFile}")/zopen + [ -e "${bupdir}" ] || { mkdir -p "${bupdir}" || printError "Unable to create directory '${bupdir}'"; } + if ! cp "${configFile}" "${bupdir}/${backup}"; then + printError "Unable to backup existing file. Check permissions and retry command." fi else @@ -347,7 +351,6 @@ init() export ZOPEN_ROOTFS="${rootfs}" ZOPEN_CA_DIR="etc/pki/tls/certs" # Mimic location on some Linux distributions - ZOPEN_JSON_CONFIG="${ZOPEN_ROOTFS}/etc/zopen/config.json" certFileName="cacert.pem" } @@ -381,13 +384,20 @@ generateFileSystem() printVerbose "- Creating symbolic path for prod redirect files" [ -e "${rootfs}/usr/share/zopen/boot" ] || ln -s "${rootfs}/boot" "${rootfs}/usr/share/zopen/boot" [ -e "${rootfs}/etc/zopen" ] || mkdir -p "${rootfs}/etc/zopen" - echo "${zopen_pkginstall}" > "${rootfs}/etc/zopen/fstype" + printVerbose "- Creating product installation location" + zopen_pkginstall="usr/local/zopen" [ -e "${rootfs}/${zopen_pkginstall}" ] || mkdir -p "${rootfs}/${zopen_pkginstall}" [ -e "${rootfs}/usr/share/zopen/prod" ] || ln -s "${rootfs}/${zopen_pkginstall}" "${rootfs}/usr/share/zopen/prod" + printVerbose "- Creating repository directory" + repodir="${rootfs}/etc/zopen/repos.d" + [ -e "${repodir}" ] || mkdir -p "${repodir}" + + printVerbose "- Setting global root envvar" + ZOPEN_PKGINSTALL="${rootfs}/${zopen_pkginstall}" if [ "${ZOPEN_PKGINSTALL}" = "${rootfs}" ]; then - printError "Package install location is the zopen root location; this is not allowed. Exiting". + printError "Package install location is the zopen root location; this is not supported. Exiting". fi printVerbose "- Creating path for certificate lookups: ${rootfs}/${ZOPEN_CA_DIR}" [ -e "${rootfs}/${ZOPEN_CA_DIR}" ] || mkdir -p "${rootfs}/${ZOPEN_CA_DIR}" @@ -399,7 +409,7 @@ generateFileSystem() generateZopenConfig() { printHeader "Generating zopen-config" - writeConfigFile "${configFile}" "${rootfs}" "${zopen_pkginstall}" "${ZOPEN_CA_DIR}/${certFileName}" $OVERRIDE_ZOS_TOOLS + writeConfigFile "${configFile}" "${rootfs}" "${zopen_pkginstall}" "${ZOPEN_CA_DIR}/${certFileName}" "$OVERRIDE_ZOS_TOOLS" printInfo "- Created config in ${configFile}" if ${appendToProfile}; then if ! grep -q ". ${rootfs}/etc/zopen-config" "${HOME}/.profile"; then @@ -411,16 +421,21 @@ EOF fi # Source the config file + # shellcheck disable=SC1090 . "${configFile}" + printInfo "- zopen-config setup complete" +} +updateCACert() +{ printHeader "Updating CA Certificate" export ZOPEN_CA="${ZOPEN_ROOTFS}/${ZOPEN_CA_DIR}/${certFileName}" caCertDir="${ZOPEN_ROOTFS}/${ZOPEN_CA_DIR}" mkdir -p "${caCertDir}" - runAndLog "${MYDIR}/zopen-update-cacert -f "${caCertDir}" " + runAndLog "${MYDIR}/zopen-update-cacert -f ${caCertDir} " [ $? -ne 0 ] && exit $? - printInfo "- zopen bootstrapping complete" + printInfo "- CA Certificate updated" } generateJsonConfiguration() @@ -429,51 +444,123 @@ generateJsonConfiguration() jsonConfigWorking="${jsonConfig}.working" if [ -e "${jsonConfig}" ]; then - releaseLine=$(jq -r '.release_line' "${jsonConfig}") - RM_FILEPROCS=$(jq -r '.num_rm_procs' "${jsonConfig}") - # note that is_collecting_stats is set earlier - if [ -z "${OVERRIDE_ZOS_TOOLS}" ] && jq -er ".override_zos_tools" $jsonConfig >/dev/null; then - OVERRIDE_ZOS_TOOLS=$(jq -r '.override_zos_tools' $jsonConfig) + # isCollectingStats is set elsewhere + if ! releaseLine=$(jqw -r '.release_line' "${jsonConfig}"); then + return 4 fi + if [ -z "${OVERRIDE_ZOS_TOOLS}" ] && jqw -er ".override_zos_tools" "$jsonConfig" >/dev/null; then + OVERRIDE_ZOS_TOOLS=$(jqw -r '.override_zos_tools' "$jsonConfig") + fi + autocacheclean=$(jqw -r '.autocacheclean' "${jsonConfig}") + autopkgpurge=$(jqw -r '.autopkgpurge' "${jsonConfig}") + skip_broken=$(jqw -r '.skip_broken' "${jsonConfig}") + skip_size_check=$(jqw -r '.skip_size_check' "${jsonConfig}") + offline=$(jqw -r '.offline' "${jsonConfig}") fi - if [ -z "${OVERRIDE_ZOS_TOOLS}" ]; then - OVERRIDE_ZOS_TOOLS=false - fi - - if [ -z "$releaseLine" ]; then - if ${releaselineDev}; then - releaseLine="DEV" - printInfo "- Configured zopen to use development (DEV) releaseline packages where available" - else - releaseLine="STABLE" - printInfo "- Configured zopen to use stable releaseline packages" - fi + if ${releaselineDev}; then + # Explicitly added on the command-line so use it + releaseLine="DEV" + printInfo "- Configured zopen to use development (DEV) releaseline packages where available" fi + validateConfigValue "enum" "releaseLine" "$releaseLine" "STABLE" "DEV|STABLE" + validateConfigValue "boolean" "autocacheclean" "$autocacheclean" "true" + validateConfigValue "boolean" "autopkgpurge" "$autopkgpurge" "false" + validateConfigValue "boolean" "skip_broken" "$skip_broken" "false" + validateConfigValue "boolean" "skip_size_check" "$skip_size_check" "false" + validateConfigValue "boolean" "OVERRIDE_ZOS_TOOLS" "$OVERRIDE_ZOS_TOOLS" "false" + validateConfigValue "boolean" "offline" "$offlineInstall" "false" if [ ! -e "${jsonConfig}" ]; then printInfo "Generating global configuration" cat < "${jsonConfig}" -{ - "release_line": "${releaseLine}", - "num_rm_procs": ${RM_FILEPROCS}, - "is_collecting_stats": ${isCollectingStats}, - "override_zos_tools": ${OVERRIDE_ZOS_TOOLS} -} + { + "release_line": "${releaseLine:-STABLE}", + "is_collecting_stats": ${isCollectingStats}, + "override_zos_tools": ${OVERRIDE_ZOS_TOOLS}, + "autocacheclean": ${autocacheclean}, + "autopkgpurge": ${autopkgpurge}, + "skip_broken": ${skip_broken}, + "skip_size_check": ${skip_size_check}, + "offline": ${offline} + } zz else printInfo "Refreshing global configuration" # In case 3rd-party edits have been made (adding custom keys for # example), just update the values that zopen itself knows about inline - if ! jq --arg releaseLine "${releaseLine}" \ - --arg isoverrideZOSTools ${OVERRIDE_ZOS_TOOLS} \ - --arg isCollectingStats ${isCollectingStats} \ - ".release_line = \$releaseLine | .override_zos_tools = \$isoverrideZOSTools | .is_collecting_stats = \$isCollectingStats" \ - "${ZOPEN_JSON_CONFIG}" > "${ZOPEN_JSON_CONFIG}.working"; then + if ! jqw --arg releaseLine "${releaseLine}" \ + --arg isoverrideZOSTools "${OVERRIDE_ZOS_TOOLS}" \ + --arg isCollectingStats "${isCollectingStats}" \ + --arg autocacheclean "${autocacheclean}" \ + --arg autopkgpurge "${autopkgpurge}" \ + --arg skip_broken "${skip_broken}" \ + --arg skip_size_check "${skip_size_check}" \ + --arg offline "${offline}" \ + ".release_line = \$releaseLine | + .override_zos_tools = \$isoverrideZOSTools | + .is_collecting_stats = \$isCollectingStats | + .autocacheclean = \$autocacheclean | + .autopkgpurge = \$autopkgpurge | + .skip_broken = \$skip_broken | + .skip_size_check = \$skip_size_check | + .offline = \$offline + " "${jsonConfig}" > "${jsonConfigWorking}"; then printError "Errors updating existing configuration file. See previous errors for more information and retry command." fi - mv -f "${ZOPEN_JSON_CONFIG}.working" "${ZOPEN_JSON_CONFIG}" + mv -f "${jsonConfigWorking}" "${jsonConfig}" fi + +} + +generateDefaultRepository() +{ + # Check for existing repository definition - if present, use unless + # user wants to rebuild the default repository file + [ -e "${rootfs}/etc/zopen/repos.d/active" ] && + ! ${rebuildRepoConfig} && return 0 + + # If there is no repository definition, generate a default to point to Github + [ ! -e "${rootfs}/etc/zopen/repos.d" ] && mkdir -p "${rootfs}/etc/zopen/repos.d" + + if [ -z "${paxrepourl}" ]; then + paxrepourl="http://raw.githubusercontent.com/${ZOPEN_ORGNAME}/meta/main/docs/api/zopen_releases.json" + fi + + repotype="${paxrepourl%://*}" + [ "${repotype}" = "${paxrepourl}" ] && printError "Unable to parse repository type from '${paxrepourl}'. Ensure valid format: ://://://:// "${repoFile}" +{ + "type": "${repotype}", + "metadata_baseurl": "${baserepourl}", + "metadata_file": "${repometafile}", + "latest_file": "${latestmetafile}" +} +EOS + # Create active link if not already present; if it is present, it'll either point to the default already or + # it'll point to a custom repo - leave as-is. + [ -L "${rootfs}/etc/zopen/repos.d/active" ] || (cd "${rootfs}/etc/zopen/repos.d" && ln -s "${repoFile}" "active") } generateAnalyticsConfiguration() @@ -485,7 +572,7 @@ generateAnalyticsConfiguration() jsonConfig="${rootfs}/var/lib/zopen/analytics.json" # if it's an existing valid json, skip - if [ -s "${jsonConfig}" ] && jq "." -e "${jsonConfig}" 2>/dev/null >/dev/null; then + if [ -s "${jsonConfig}" ] && jqw "." -e "${jsonConfig}" 2>/dev/null >/dev/null; then return; fi jsonConfigWorking="${jsonConfig}.working" @@ -500,19 +587,98 @@ generateAnalyticsConfiguration() "removes": [] } zz - jq '.' $jsonConfigWorking > $jsonConfig + if jqrc=$(jqw '.' "${jsonConfigWorking}" > "${jsonConfig}"); then + rm "${jsonConfigWorking}" + else + printSoftError "${jqrc}" + printError "Unable to update analystics configuration. See previous errors for more details." + fi + registerFileSystem "${UUID}" "${isHostIBM}" "${isBot}" - if [ -n "$ZOPEN_ROOTFS" ] && [ -n "$ZOPEN_LOG_PATH" ] && [ -f "$ZOPEN_LOG_PATH/audit.log" ]; then - processAnalyticsFromLogFile + if [ -n "$ZOPEN_ROOTFS" ]; then + if [ -z "$ZOPEN_LOG_PATH" ]; then + ZOPEN_LOG_PATH="${ZOPEN_ROOTFS}/var/log/zopen" + fi + if [ -f "$ZOPEN_LOG_PATH/audit.log" ]; then + processAnalyticsFromLogFile + fi fi } +installLocalMeta() +{ + metabindir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P) + ulzm="${rootfs}/usr/local/zopen/meta" + target="${ulzm}/meta-bootstrap" + mkdir -p "${target}" + printVerbose "Copying bare meta scripts from '${metabindir}' to '${target}" + cp -R "${metabindir}/../bin" "${target}" + cp -R "${metabindir}/../include" "${target}" + + printVerbose "Creating main symlink for meta" + ln -s "${ulzm}/meta-bootstrap" "${ulzm}/meta" + if ! runLogProgress "mergeIntoSystem \"meta\" \"${ulzm}/meta-bootstrap\" \"${rootfs}\"" \ + "Merging meta into symlink mesh" "Merged meta into symlink mesh"; then + printSoftError "Unexpected errors merging symlinks into mesh" + printError "Use zopen alt to select previous version to ensure known state" + fi + printVerbose "Adding bootstrapped meta to package db" + printVerbose "Creating basic metadata.json" + cat << EOF > "${target}/metadata.json" +{ + "version_scheme": "0.2", + "product": { + "name": "meta", + "version": "0.0.0.0-bootstrap", + "release": "$(date +%Y%m%d_%H%M%S)", + "summary": "meta on z/OS", + "repo": "https://github.com/zopencommunity/metaport", + "license": "https://github.com/zopencommunity/metaport/blob/main/patches/LICENSE", + "zopen_license": "https://github.com/zopencommunity/metaport/blob/main/LICENSE", + "categories": "utilities", + "size": "$(du -ks "${target}" | awk '{print $1 * 1024}')", + "build_dependencies": [], + "runtime_dependencies": [], + "system_prereqs": [] + } +} +EOF + addToInstallTracker "meta" +} + +installRemoteMeta() +{ + printHeader "Installing meta package" + printVerbose "Determining if installing using a DEV meta" + metabindir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P) + metaReleaseLine="STABLE" + ##TODO + metadataFile="${metabindir}/../metadata.json" + if [ ! -r "${metadataFile}" ]; then + printVerbose "No metadata.json file; using stable release" + fi + #Note that the releaseline is knwon as buildline in the metadata file! + if ! bootstrapMetaReleaseline=$(jqw -r '.product.buildline' "${metadataFile}"); then + printVerbose "Could not get releaseline/buildline from package metadata; using stable release" + metaReleaseLine="STABLE" + else + case "${bootstrapMetaReleaseline}" in + "STABLE"|"DEV") printVerbose "Valid releaseline '${bootstrapMetaReleaseline}' from metadata file" + metaReleaseLine="${bootstrapMetaReleaseline}" + ;; + *) printVerbose "Invalid releaseline '${bootstrapMetaReleaseline}'; defaulting to STABLE" + metaReleaseLine="STABLE" + ;; + esac + fi + "${MYDIR}"/zopen-install "${installOptions}" -y meta --release-line "${metaReleaseLine}" +} installPrereqs() { printHeader "Installing zopen pre-requisites" # Comma-separated list of pax locations - PAX_LOCATIONS="$CURL_PAX_LOCATION,$JQ_PAX_LOCATION,${GPG_PAX_LOCATION},${PINENTRY_PAX_LOCATION}" + PAX_LOCATIONS="$CURL_PAX_LOCATION,$JQ_PAX_LOCATION,${PINENTRY_PAX_LOCATION},${GPG_PAX_LOCATION}" echo "$PAX_LOCATIONS" | tr ',' '\n' | while read -r pax; do printVerbose "- Check for shipped ${pax}" @@ -531,42 +697,86 @@ installPrereqs() cachedir="${rootfs}/var/cache/zopen" basepax=$(basename ${pax}) [ -e "${cachedir}/${basepax}" ] || printError "Could not locate bootstrap ${pax}." - paxrc=$(pax -rf "${cachedir}/${basepax}" -s##${rootfs}/boot/#) - if [ $? -gt 0 ]; then - printError "Failed to extract pax file ${basepax}" + if ! paxrc=$(pax -rf "${cachedir}/${basepax}" "-s##${rootfs}/boot/#"); then + printError "Failed to extract pax file ${basepax}: ${paxrc}" fi done # Sourcing the environment from within the actual directories - for tool in "curl" "jq" "gnupg" "pinentry"; do + curwd="$(pwd -P)" + for tool in "curl" "jq" "pinentry" "gnupg"; do printVerbose "Sourcing environment to trigger any setup required for ${tool}" tooldir=$(ls "${rootfs}/boot" | grep ${tool} | head -1) [ -e "${rootfs}/boot/${tooldir}" ] || printError "Could not locate ${tool} directory '${rootfs}/boot/${tooldir}/' for bootstrap. Re-run 'zopen init' to retry" chmod -R 755 "${rootfs}/boot/${tooldir}" - curwd="${PWD}" cd "${rootfs}/boot/${tooldir}/" || printError "Could not access ${tool} bootstrap in directory '${rootfs}/boot/${tooldir}/'. Re-run 'zopen init' to retry" + # shellcheck disable=SC1091 . ./.env cd "${curwd}" || printError "Could not change to ${curwd}. Re-run 'zopen init' to retry" done + cd "${curwd}" || printError "Could not return to ${curwd}." printInfo "- Sourcing environment" } -updateBootstrappedTools() { +# Used to ensure that there are initial versions of the core products available for use. +# - install/update the packages from the repo +# - in an offline/airgapped environment, this involves installing the bootstrapped paxes +# and copying over the meta that was used in the initial install +updateBootstrappedTools() +{ printInfo "- Installing/updating bootstrapped curl, jq, gpg, pinentry and meta to install updated versions (if available)" installOptions="--force" + toolInstall=0 if $bypassPrereqs; then installOptions="${installOptions} --bypass-prereq-checks"; fi if ${refresh}; then - ${MYDIR}/zopen-install ${installOptions} -y curl jq gpg pinentry + "${MYDIR}"/zopen-install "${installOptions}" --yes curl jq pinentry gpg toolInstall=$? + elif ${offlineInstall}; then + printVerbose "Installing bootstrap tools" + PAX_LOCATIONS="$CURL_PAX_LOCATION,$JQ_PAX_LOCATION,${PINENTRY_PAX_LOCATION},${GPG_PAX_LOCATION}" + + workingPaxList="$PAX_LOCATIONS" + printVerbose " Parsing without subshell to ensure exit" + while [ -n "${workingPaxList}" ]; do + case "${workingPaxList}" in + *,*) + pax="${workingPaxList%%,*}" + workingPaxList="${workingPaxList#*,}" + ;; + *) + # Last item [hence no comma separator] + pax="${workingPaxList}" + workingPaxList="" + ;; + esac + printVerbose "Validating pax '${pax}'" + paxLocation=$(findrev "${MYDIR}" "${pax}") + printVerbose "Pax installation from '${paxLocation}/${pax}'" + if [ -e "${paxLocation}/${pax}" ]; then + installOptions="-y --bypass-prereq-checks --skip-verify" + ${verbose} && installOptions="${installOptions} --verbose" + ${debug} && installOptions="${installOptions} --debug" + ${xdebug} && installOptions="${installOptions} --xdebug" + # shellcheck disable=SC2086 # We want split options for command + zopen install "${paxLocation}/${pax}" ${installOptions} + toolInstall=$? + [ "${toolInstall}" -ne 0 ] && printError "Unable to install '${pax}' locally; see previous errors and retry installation using the '--re-init' parameter" + else + printError "Could not locate ${pax} to install as bootstrap" + fi + done else - ${MYDIR}/zopen-install ${installOptions} -y curl jq gpg pinentry meta + installRemoteMeta # Need to ensure meta is available + # shellcheck disable=SC2086 # We want installOptions to be split + "${MYDIR}/zopen-install" ${installOptions} --yes curl jq pinentry gpg toolInstall=$? fi - [ ${toolInstall} -ne 0 ] && printError "Unable to install curl, jq, gpg, pinentry and/or meta; see previous errors and retry the initilisation using the '--re-init' parameter" + [ "${toolInstall}" -ne 0 ] && printError "Unable to install curl, jq, gpg, pinentry and/or meta; see previous errors and retry the initilisation using the '--re-init' parameter" + # shellcheck disable=SC1090 . "${configFile}" } @@ -626,12 +836,17 @@ processCaCerts() fi } +createInitialPackageDB() +{ + updatePackageDB +} + deinit() { + syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE},${CAT_FILE}" "INIT" "" "zopen environment initialised as ${ZOPEN_ROOTFS}" printHeader "Initialization successfully completed" printInfo "${NC}${YELLOW}IMPORTANT: Run '. ${rootfs}/etc/zopen-config' to enable zopen environment for current session or add it to your .profile.${NC}" printInfo "${NC}${YELLOW}IMPORTANT: If you prefer GNU tools over z/OS /bin tools, run '. ${rootfs}/etc/zopen-config --override-zos-tools' in the current session or add it to your .profile.${NC}" - syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE},${CAT_FILE}" "INIT" "" "zopen environment initialised as ${ZOPEN_ROOTFS}" if ! ${refresh}; then myparent="$(cd "$(dirname "${MYDIR}")" >/dev/null 2>&1 && pwd -P )" myparentpaxz="${myparent}.pax.Z" @@ -641,11 +856,29 @@ deinit() fi } +# Prompt/pre-fill config questions init +# Check and create if missing required file system layout generateFileSystem +# During install, install the pre-packaged required packages ! ${refresh} && installPrereqs +# On an install, ensure there is a bootstrapped meta to fallback to +# Note this stage +! (${refresh} || ${reinitExisting}) && installLocalMeta +# (Re)Create the config.json control file generateJsonConfiguration +# Create the default repository - even if offline to use as a template +generateDefaultRepository +# Create the zopen-config configuration script generateZopenConfig +# Ensure the latest CA certificates are available +! ${offlineInstall} && updateCACert +# Create the analytics collector configuration generateAnalyticsConfiguration -updateBootstrappedTools +# Create the installation tracking file; if not existing, recreate +[ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || createInitialPackageDB +# Update the bootstrapped tools as needed - if offline, just install the +# shipped paxes +! ${refresh} && updateBootstrappedTools + deinit diff --git a/bin/zopen-install b/bin/zopen-install index fa6fdf7e0..90c821621 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -x # # Install utility for zopen community - https://github.com/zopencommunity # @@ -6,674 +6,94 @@ # # All zopen-* scripts MUST start with this code to maintain consistency. # -#set -x setupMyself() { - ME=$(basename $0) + ME=$(basename "$0") MYDIR="$(cd "$(dirname "$0")" > /dev/null 2>&1 && pwd -P)" INCDIR="${MYDIR}/../include" if ! [ -d "${INCDIR}" ] && ! [ -f "${INCDIR}/common.sh" ]; then echo "Internal Error. Unable to find common.sh file to source." >&2 exit 8 fi + # shellcheck source=/dev/null . "${INCDIR}/common.sh" } setupMyself checkWritable -startGPGAgent() { - printInfo "- Starting gpg-agent..." - - SOCKET_PATH=$(gpgconf --list-dirs agent-socket) - if [ -r "$SOCKET_PATH" ]; then - printVerbose "gpg-agent is already running (socket found at $SOCKET_PATH)." - return 0 - fi - - if eval "$(gpg-agent --daemon --disable-scdaemon)" >/dev/null 2>&1; then - if [ -r "$SOCKET_PATH" ]; then - printVerbose "gpg-agent started successfully (socket created at $SOCKET_PATH)." - else - printWarning "gpg-agent started, but socket was not created at $SOCKET_PATH. Please verify your GPG installation." - fi - else - if [ -r "$SOCKET_PATH" ]; then - printWarning "gpg-agent started successfully (socket created at $SOCKET_PATH), but gpg-agent returned a non-zero return code." - else - printError "Failed to start gpg-agent. Reinstall or upgrade GPG using \"zopen install --reinstall gpg -y\" or \"zopen upgrade gpg -y\"." - fi - fi -} - -gpgCleanup() { - printVerbose "Cleaning up $SIGNATURE_FILE, $PUBLIC_KEY_FILE and $TMP_GPG_DIR" - [ -e "$SIGNATURE_FILE" ] && rm -f "$SIGNATURE_FILE" - [ -e "$PUBLIC_KEY_FILE" ] && rm -f "$PUBLIC_KEY_FILE" - [ -d "$TMP_GPG_DIR" ] && rm -rf "$TMP_GPG_DIR" -} - -verifySignatureOfPax() -{ - printInfo "- Performing GPG signature verification of pax file..." - # Extracting values and checking for errors - if ! FILE_TO_VERIFY=$(jq -e -r '.product.pax' "${metadataFile}"); then - printError "Failed to extract 'pax' from ${metadataFile}" >&2 - fi - - if ! SIGNATURE=$(jq -e -r '.product.signature' "${metadataFile}"); then - printVerbose "Failed to extract 'signature' from ${metadataFile}" >&2 - return - fi - - if ! PUBLIC_KEY=$(jq -e -r '.product.public_key' "${metadataFile}"); then - printError "Failed to extract 'public_key' from ${metadataFile}" >&2 - fi - - - # Create a temporary directory for GPG keyring - TMP_GPG_DIR="$zopen_tmp_dir/zopen_gpg_verify_$LOGNAME_$$" - mkdir -p "$TMP_GPG_DIR" - - SIGNATURE_FILE="$zopen_tmp_dir/zopen_signedfile.$LOGNAME.$$.asc" - PUBLIC_KEY_FILE="$zopen_tmp_dir/zopen_scriptpubkey.$LOGNAME.$$.asc" - /bin/printf "%b" "$SIGNATURE" | tr -d '"' > "$SIGNATURE_FILE" - /bin/printf "%b" "$PUBLIC_KEY" | tr -d '"' > "$PUBLIC_KEY_FILE" - - startGPGAgent - - printVerbose "Importing public key..." - gpg_output=$(gpg --no-default-keyring --keyring "$TMP_GPG_DIR/pubring.kbx" --batch --yes --import "$PUBLIC_KEY_FILE" 2>&1) - if [ $? -ne 0 ]; then - gpgCleanup - printError "Importing public key failed. See output:\n$gpg_output.\n Verification aborted." - fi - printVerbose "$gpg_output" - - # Verify that the key was imported successfully - printVerbose "Checking if public key is imported..." - gpg_output=$(gpg --no-default-keyring --keyring "$TMP_GPG_DIR/pubring.kbx" --check-sigs 2>&1) - if [ $? -ne 0 ]; then - gpgCleanup - printError "Public key was not imported. See output:\n$gpg_output.\nVerification aborted." - fi - printVerbose "$gpg_output" - - # Verify the signature - printInfo "- Verifying the gpg signature..." - if [ ! -f "$SIGNATURE_FILE" ]; then - gpgCleanup - printError "Signature file does not exist. Please raise an issue." - fi - - gpg_output=$(gpg --no-default-keyring --keyring "$TMP_GPG_DIR/pubring.kbx" --verify "$SIGNATURE_FILE" "$FILE_TO_VERIFY" 2>&1) - printVerbose "$gpg_output" - if echo "$gpg_output" | grep -q "Good signature from"; then - gpgCleanup - printInfo "- Signature successfully verified." - else - gpgCleanup - printError "Verification failed. See the output:\n$gpg_output" - fi -} - -printSyntax() +printHelp() { - args=$* cat << HELPDOC ${ME} is a utility to download/install a zopen community package. -Usage: ${ME} [OPION] [PACKAGE] - [PACKAGE] is a package to install. Multiple packages can be specified. +Usage: zopen install [OPTION] [PARAMETERS] [PACKAGES] Options: - --all download/install all zopen community packages. - --cache-only do not install dependencies. - --download-only download package to current directory. - --help print this help. - --install-or-upgrade installs the package if not installed, - or upgrades the package if installed. - --bypass-prereq-checks Ignores pre-req checks - --local-install download and unpackage to current directory. - --no-deps do not install dependencies. - --no-set-active do not change the pinned version. - --nosymlink do not integrate into filesystem through - symlink redirection. - -r, --reinstall reinstall already installed zopen community packages. - --release-line [stable, dev] the release line to build off of. - --select select a version to install. - --skip-upgrade do not upgrade. - --force force install, bypassing locks. - -u, --update, --upgrade updates installed zopen community packages. - -v, --verbose print verbose messages. - --version print version. - -y, --yes automatically answer yes to prompts. + --all download all available packages and install + --bypass-prereq-checks ignores pre-req checks + --download-only download installable package with no install + --install-or-upgrade installs the package if not installed, or upgrades + the package if installed (deprecated as default behaviour) + --nodeps do not install dependencies + --noset-active unpackage onto the system but do not change the currently + active version + --reinstall reinstall currently installed package(s) + --reinstall-with-deps reinstall package(s) and their dependencies + --release-line [stable|dev] + the release line to install from + --select select a version to install from available versions + --skip-broken continue installing other packages as part of a + transaction where a package failed to install + --skip-verify skips the package verification allowing un-trusted + packages. Caveat usor! + -v, --verbose run in verbose mode + -y, --yes automatically answer yes to prompts + -h,-?, --help display this help and exit + + +Examples: + zopen install foo + install package foo if not already installed + zopen install --release-line DEV foo + install package foo from the DEV releaseline if + available + zopen install -y foo bar --nodeps + install packages foo and bar without asking for + user confirmation and without installing any + dependencies + zopen install /tmp/foo-1.2.3-4.zos.pax.Z + install package foo from the specified location + zopen install /tmp/foo-1.2.3-4.zos.pax.Z bar-1.2.3.4.zos.pax.Z + install packages foo and bar from the specified + locations + +Report bugs at https://github.com/zopencommunity/meta/issues HELPDOC } -installDependencies() -( - name=$1 - printVerbose "List of dependencies to install: ${dependencies}" - skipupgrade_lcl=${skipupgrade} - skipupgrade=true - skipverify=false; - echo "${dependencies}" | xargs | tr ' ' '\n' | sort | while read dep; do - printVerbose "Removing '${dep}' from dependency queue '${dependencies}'" - dependencies=$(echo "${dependencies}" | sed -e "s/${dep}//" | tr -s ' ') - handlePackageInstall "${dep}" true - done - skipupgrade=${skipupgrade_lcl} -) - -handlePackageInstall() -{ - - fullname="$1" - isRuntimeDependency=$2 - if [ -z "$isRuntimeDependency" ]; then - isRuntimeDependency=false - fi - printVerbose "Name to install: ${fullname}, parsing any version ('=') or tag ('%') has been specified" - name=$(echo "${fullname}" | sed -e 's#[=%].*##') - repo="${name}" - versioned=$(echo "${fullname}" | cut -s -d '=' -f 2) - tagged=$(echo "${fullname}" | cut -s -d '%' -f 2) - printDebug "Name:${name};version:${versioned};tag:${tagged};repo:${repo}" - printInfo "${NC}${HEADERCOLOR}${BOLD}Processing package: ${name}${NC}" - - nameSansPort=$(echo "${name}" | sed -e 's#\(.*\)port$#\1#') - # findutilsport -> findutils - # findutils -> findutils - if [[ "${nameSansPort}" != "${name}" ]] ; then - printError "Please install using base project name without port suffix.\nTry: zopen install ${nameSansPort}" - fi - - getAllReleasesFromGithub "${repo}" - - if ${localInstall}; then - printVerbose "Local install to current directory" - rootInstallDir="${PWD}" - else - printVerbose "Setting install root to: ${ZOPEN_PKGINSTALL}" - rootInstallDir="${ZOPEN_PKGINSTALL}" - fi - - originalFileVersion="" - printVerbose "Checking for meta files in '${rootInstallDir}/${name}/${name}'" - printVerbose "Finding version/release information." - if [ -e "${rootInstallDir}/${name}/${name}/.releaseinfo" ]; then - originalFileVersion=$(cat "${rootInstallDir}/${name}/${name}/.releaseinfo") - printVerbose "Found originalFileVersion=${originalFileVersion} (port is already installed)." - elif [ -e "${rootInstallDir}/${name}/${name}/.version" ]; then - originalFileVersion=$(cat "${rootInstallDir}/${name}/${name}/.version") - printVerbose "Found originalFileVersion=${originalFileVersion} (port is already installed)." - else - printVerbose "Could not detect existing installation at ${rootInstallDir}/${name}/${name}" - fi - - printVerbose "Finding releaseline information." - installedReleaseLine="" - if [ -e "${rootInstallDir}/${name}/${name}/.releaseline" ]; then - installedReleaseLine=$(cat "${rootInstallDir}/${name}/${name}/.releaseline") - printVerbose "Installed product from releaseline: ${installedReleaseLine}" - else - printVerbose "No current releaseline for package." - fi - - releaseMetadata="" - downloadURL="" - # Options where the user explicitly sets a version/tag/releaseline currently ignore any configured release-line, - # either for a previous package install or system default - if [ -n "${versioned}" ]; then - printVerbose "Specific version ${versioned} requested - checking existence and URL." - requestedMajor=$(echo "${versioned}" | awk -F'.' '{print $1}') - requestedMinor=$(echo "${versioned}" | awk -F'.' '{print $2}') - requestedPatch=$(echo "${versioned}" | awk -F'.' '{print $3}') - requestedSubrelease=$(echo "${versioned}" | awk -F'.' '{print $4}') - requestedVersion="${requestedMajor}\\\.${requestedMinor}\\\.${requestedPatch}\\\.${requestedSubrelease}" - printVerbose "Finding URL for latest release matching version prefix: requestedVersion: ${requestedVersion}" - releaseMetadata=$(/bin/printf "%s" "${releases}" | jq -e -r '. | map(select(.assets[].name | test("'${requestedVersion}'")))[0]') - elif [ -n "${tagged}" ]; then - printVerbose "Explicit tagged version '${tagged}' specified. Checking for match." - releaseMetadata=$(/bin/printf "%s" "${releases}" | jq -e -r '.[] | select(.tag_name == "'${tagged}'")') - printVerbose "Use quick check for asset to check for existence of metadata for specific messages." - asset=$(/bin/printf "%s" "${releaseMetadata}" | jq -e -r '.assets[0]') - if [ $? -ne 0 ]; then - printError "Could not find release tagged '${tagged}' in repo '${repo}'" - fi - elif ${selectVersion}; then - # Explicitly allow the user to select a release to install; useful if there are broken installs - # as a known good release can be found, selected and pinned! - printVerbose "List individual releases and allow selection." - i=$(/bin/printf "%s" "${releases}" | jq -r 'length - 1') - printInfo "Versions available for install:" - /bin/printf "%s" "${releases}" | jq -e -r 'to_entries | map("\(.key): \(.value.tag_name) - \(.value.assets[0].name) - size: \(.value.assets[0].expanded_size | tonumber/ (1024 * 1024))mb")[]' - - printVerbose "Getting user selection." - valid=false - while ! ${valid}; do - echo "Enter version to install (0-${i}): " - read selection < /dev/tty - if [[ ! -z $(echo "${selection}" | sed -e 's/[0-9]*//') ]]; then - echo "Invalid input, must be a number between 0 and ${i}." - elif [ "${selection}" -ge 0 ] && [ "${selection}" -le "${i}" ]; then - valid=true - fi - done - printVerbose "Selecting item ${selection} from array" - releaseMetadata="$(/bin/printf "%s" "${releases}" | jq -e -r ".[${selection}]")" - - elif [ ! -z "${releaseLine}" ]; then - printVerbose "Install from release line '${releaseLine}' specified" - validatedReleaseLine=$(validateReleaseLine "${releaseLine}") - if [ -z "${validatedReleaseLine}" ]; then - printError "Invalid releaseline specified: '${releaseLine}'; Valid values: DEV or STABLE." - fi - printVerbose "Finding latest asset on the release line" - releaseMetadata="$(/bin/printf "%s" "${releases}" | jq -e -r '. | map(select(.tag_name | startswith("'${releaseLine}'")))[0]')" - printVerbose "Use quick check for asset to check for existence of metadata." - asset="$(/bin/printf "%s" "${releaseMetadata}" | jq -e -r '.assets[0]')" - if [ $? -ne 0 ]; then - printError "Could not find release-line ${releaseLine} for repo: ${repo}." - fi - - else - printVerbose "No explicit version/tag/releaseline, checking for pre-existing package&releaseline." - if [ -n "${installedReleaseLine}" ]; then - printVerbose "Found existing releaseline '${installedReleaseLine}', restricting to only that releaseline." - validatedReleaseLine="${installedReleaseLine}" # Already validated when stored - else - printVerbose "Checking for system-configured releaseline." - sysrelline=$(getReleaseLine) - printVerbose "Validating value: ${sysrelline}" - validatedReleaseLine=$(validateReleaseLine "${sysrelline}") - if [ -n "${validatedReleaseLine}" ]; then - printVerbose "zopen system configured to use releaseline '${sysrelline}'; restricting to that releaseline." - else - printWarning "zopen misconfigured to use an unknown releaseline of '${sysrelline}'; defaulting to STABLE packages." - printWarning "Set the contents of '${ZOPEN_ROOTFS}/etc/zopen/releaseline' to a valid value to remove this message." - printWarning "Valid values are: DEV | STABLE." - validatedReleaseLine="STABLE" - fi - fi - # We have some situations that could arise - # 1. the port being installed has no releaseline tagging yet (ie. no releases tagged STABLE_* or DEV_*) - # 2. system is configured for STABLE but only has DEV stream available - # 3. system is configured for DEV but only has DEV stream available - # 4. the port being installed has got full releaseline tagging - # The issue could arise that the user has switched the system from DEV->STABLE or vice-versa so package - # stream mismatches could arise but in normal case, once a package is installed [that has releaseline tagging] - # then that specific releaseline will be used - printVerbose "Finding any releases tagged with ${validatedReleaseLine} and getting the first (newest/latest)" - releaseMetadata="$(/bin/printf "%s" "${releases}" | jq -e -r '. | map(select(.tag_name | startswith("'${validatedReleaseLine}'")))[0]')" - printVerbose "Use quick check for asset to check for existence of metadata" - asset="$(/bin/printf "%s" "${releaseMetadata}" | jq -e -r '.assets[0]')" - if [ $? -eq 0 ]; then - # Case 4... - printVerbose "Found a specific '${validatedReleaseLine}' release-line tagged version; installing..." - else - # Case 2 & 3 - printVerbose "No releases on releaseline '${validatedReleaseLine}'; checking alternative releaseline." - alt=$(echo "${validatedReleaseLine}" | awk ' /DEV/ { print "STABLE" } /STABLE/ { print "DEV" }') - releaseMetadata="$(/bin/printf "%s" "${releases}" | jq -e -r '. | map(select(.tag_name | startswith("'${alt}'")))[0]')" - printVerbose "Use quick check for asset to check for existence of metadata" - asset="$(/bin/printf "%s" "${releaseMetadata}" | jq -e -r '.assets[0]')" - if [ $? -eq 0 ]; then - printVerbose "Found a release on the '${alt}' release line so release tagging is active." - if [ "DEV" = "${validatedReleaseLine}" ]; then - # The system will be configured to use DEV packages where available but if none, use latest - printInfo "No specific DEV releaseline package, using latest available" - releaseMetadata="$(/bin/printf "%s" "${releases}" | jq -e -r ".[0]")" - else - printVerbose "The system is configured to only use STABLE releaseline packages but there are none." - printInfo "No release available on the '${validatedReleaseLine}' releaseline." - fi - else - # Case 1 - old package that has no release tagging yet (no DEV or STABLE), just install latest - printVerbose "Installing latest release" - releaseMetadata="$(/bin/printf "%s" "${releases}" | jq -e -r ".[0]")" - fi - fi - fi - - printVerbose "Getting specific asset details from metadata: ${releaseMetadata}" - if [ -z "${asset}" ] || [ "null" = "${asset}" ]; then - printVerbose "Asset not found during previous logic; setting now." - asset=$(/bin/printf "%s" "${releaseMetadata}" | jq -e -r '.assets[0]') - fi - if [ -z "${asset}" ]; then - printError "Unable to determine download asset for ${name}" - fi - - tagname=$(/bin/printf "%s" "${releaseMetadata}" | jq -e -r ".tag_name" | sed "s/\([^_]*\)_.*/\1/") - installedReleaseLine=$(validateReleaseLine "${tagname}") - - downloadURL=$(/bin/printf "%s" "${asset}" | jq -e -r '.url') - metadataJSONURL="$(dirname "${downloadURL}")/metadata.json" - expanded_size=$(/bin/printf "%s" "${asset}" | jq -e -r '.expanded_size') - size=$(/bin/printf "%s" "${asset}" | jq -e -r '.size') - - if [ -z "${downloadURL}" ]; then - printError "Unable to determine download location for ${name}" - fi - downloadFile=$(basename "${downloadURL}") - metadataFile="${downloadFile}.json" # use the same basename as the pax + .json to avoid collision - downloadFileVer=$(echo "${downloadFile}" | sed -E 's/.*-(.*)\.zos\.pax\.Z/\1/') - printVerbose "Downloading port from URL: ${downloadURL} to file: ${downloadFile} (ver=${downloadFileVer})" - - if ${downloadOnly}; then - printVerbose "Skipping installation, downloading only." - else - printVerbose "Install=${downloadFileVer};Original=${originalFileVersion};${upgradeInstalled};${installOrUpgrade};${reinstall}" - if [ "${downloadFileVer}" = "${originalFileVersion}" ]; then - if ! ${reinstall}; then - printInfo "${NC}${GREEN}Package ${name} is already installed at the requested version: ${downloadFileVer} and due to the absence of the 'reinstall' flag, will not be reinstalled.${NC}" - return - fi - printInfo "- Reinstalling version '${downloadFileVer}' of ${name}..." - fi - - printVerbose "Checking if package is not installed but scheduled for upgrade." - if [ -z "${originalFileVersion}" ]; then - printVerbose "No previous version found." - if ${installOrUpgrade}; then - printVerbose "Package ${name} was not installed, so installing instead of upgrading." - elif ${upgradeInstalled}; then - printError "Package ${name} can not be upgraded as it is not installed." - continue - fi - unInstallOldVersion=false - printInfo "- Installing ${name}..." - elif ${skipupgrade}; then - printInfo "Package ${name} has a newer release '${downloadFileVer}' but was not specified for an upgrade." - continue - elif ! ${setactive}; then - printVerbose "Current version '${originalFileVersion}' will remain active." - unInstallOldVersion=false - else - printVerbose "Previous version '${originalFileVersion}' installed." - if [ -e "${rootInstallDir}/${name}/${name}/.pinned" ]; then - printWarning "- Version '${originalFileVersion}' has been pinned; upgrade to '${downloadFileVer}' skipped." - syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE},${CAT_INSTALL}" "DOWNLOAD" "handlePackageInstall" "Attempt to change pinned package '${name}' skipped." - continue - else - printInfo "- Replacing ${name} version '${originalFileVersion}' with '${downloadFileVer}'" - unInstallOldVersion=true - currentversiondir=$(cd "${rootInstallDir}/${name}/${name}" && pwd -P) - currentlinkfile="${currentversiondir}/.links" - fi - fi - fi - - printVerbose "Ensuring we are in the correct working download location '${downloadDir}'" - cd "${downloadDir}" || exit - if [ ! -n "${downloadOnly}" ] || [ ! -n "${localInstall}" ]; then - printVerbose "Checking current directory for already downloaded package [file name comparison]." - location="current directory" - else - printVerbose "Checking cache for already downloaded package [file name comparison]." - location="zopen package cache" - fi - - # Download the metadata json file - if ! runAndLog "curlCmd -L '${metadataJSONURL}' -o '${metadataFile}'" ${redirectToDevNull}; then - printError "Could not download from ${metadataJSONURL}. Correct any errors and potentially retry." - continue - fi - pax=${downloadFile} - if [ -f "${pax}" ]; then - printInfo "- Found existing file '${pax}' in ${location}" - else - printInfo "- Downloading ${pax} file from remote to ${location}..." - if ! ${verbose}; then - redirectToDevNull="2>/dev/null" - fi - - # Check partition size before download package spinner starts - checkAvailableSize "${name}" "${size}" - progressHandler "network" "- Download complete." & - ph=$! - killph="kill -HUP ${ph}" - addCleanupTrapCmd "${killph}" - - if ! runAndLog "curlCmd -L '${downloadURL}' -O ${redirectToDevNull}"; then - printError "Could not download from ${downloadURL}. Correct any errors and potentially retry." - continue - fi - ${killph} 2> /dev/null # if the timer is not running, the kill will fail - syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_NETWORK},${CAT_PACKAGE},${CAT_FILE}" "DOWNLOAD" "handlePackageInstall" "Downloaded remote file '${pax}'" - fi - if [ ! -f "${pax}" ]; then - printError "${pax} was not found after download!?" - fi - - if ${downloadOnly}; then - printVerbose "Pax was downloaded to local dir '${downloadDir}'" - elif ${cacheOnly}; then - printVerbose "Pax was downloaded to zopen cache '${downloadDir}'" - else - printVerbose "Installing ${pax}" - installdirname="${name}/${pax%.pax.Z}" # Use full pax name as default - - printInfo "- Processing ${pax}..." - baseinstalldir="." - paxredirect="" - if ! ${localInstall}; then - baseinstalldir="${rootInstallDir}" - paxredirect="-s %[^/]*/%${rootInstallDir}/${installdirname}/%" - printVerbose "Non-local install, extracting with '${paxredirect}'" - else - printInfo "- Local install specified." - paxredirect="-s %[^/]*/%${installdirname}/%" - printVerbose "Non-local install, extracting with '${paxredirect}'" - fi - - megabytes=$(echo "scale=2; ${expanded_size} / (1024 * 1024)" | bc) - printInfo "After this operation, ${NC}${HEADERCOLOR}${BOLD}${megabytes} MB${NC} of additional disk space will be used." - if ! promptYesOrNo "Do you want to continue?" ${yesToPrompts}; then - if ! ${force}; then - mutexFree "zopen" - fi - printInfo "Exiting..." - exit 0 - fi - - printVerbose "Check for existing directory for version '${installdirname}'" - if [ -d "${baseinstalldir}/${installdirname}" ]; then - printInfo "- Clearing existing directory and contents" - rm -rf "${baseinstalldir}/${installdirname}" - fi - - metadataVersion=$(jq -r '.version_scheme' "${metadataFile}" 2>/dev/null) - is_greater=$(echo "$metadataVersion > 0.1" | bc -l) - - if [ "$is_greater" -eq 1 ] && ! $skipverify; then - if ! command -v gpg> /dev/null; then - skipverify=false; - printWarning "GPG is not installed" - else - verifySignatureOfPax - fi - fi - if ! runLogProgress "pax -rf ${pax} -p p ${paxredirect} ${redirectToDevNull}" "Expanding ${pax}" "Expanded"; then - printWarning "Errors unpaxing, package directory state unknown." - printWarning "Use zopen alt to select previous version." - continue - fi - - if ${localInstall}; then - rm -f "${pax}" - fi - - # Some installation have installation caveats - installCaveat=$(jq -r '.product.install_caveats // empty' "${metadataFile}" 2>/dev/null) - if [ -n "$installCaveat" ]; then - /bin/printf "${NC}${HEADERCOLOR}${BOLD}${name}${NC}:\n ${installCaveat}\n" >> ${caveatsFile} - fi - - - systemPrereqs=$(jq -r '.product.system_prereqs // empty | map(.name) | join(" ")' "${metadataFile}" 2>/dev/null) - if ! $bypassPrereqs; then - if [ -z "${systemPrereqs}" ]; then - systemPrereqs="${systemPrereqs} zos24" # set the min requirement as z/OS 2.4 - fi - if [ -n "$systemPrereqs" ]; then - if [ ! -r "$ZOPEN_SYSTEM_PREREQ_SCRIPT" ]; then - printError "$ZOPEN_SYSTEM_PREREQ_SCRIPT does not exist. Check file permissions and reinstall the meta package or reinitialize the zopen environment. If the error persists, open an issue." - else - . $ZOPEN_SYSTEM_PREREQ_SCRIPT - for prereq in $(echo "${systemPrereqs}" | xargs | tr ' ' '\n' | sort -u); do - printInfo "- Checking system pre-req requirement $prereq" - if command -V "${prereq}" >/dev/null 2>&1; then - if ! ( ${prereq} ); then - printError "Failed system pre-req check \"$prereq\". If you wish to bypass this, install with --bypass-prereq-checks" - fi - else - printError "Prereq \"$prereq\" does not exist in $ZOPEN_SYSTEM_PREREQ_SCRIPT. Consider upgrading meta or open an issue if it persists." - fi - done - fi - fi - else - syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE},${CAT_INSTALL}" "BYPASS" "handlePackageInstall" "Bypassing prereq checks ${systemPrereqs} for '${name}'." - fi - # Let the individual tools emit their own caveats - #if [ -d "${baseinstalldir}/${installdirname}/altbin" ]; then - # /bin/printf "${NC}${HEADERCOLOR}${BOLD}${name}${NC}:\n${name} has tools provided under altbin/ that conflict with tools under /bin.\nTo use them set ZOPEN_TOOLSET_OVERRIDE and then re-source the zopen-config.\n" >> ${caveatsFile} - #fi - - - if ${setactive}; then - if [ -L "${baseinstalldir}/${name}/${name}" ]; then - printVerbose "Removing old symlink '${baseinstalldir}/${name}/${name}'" - rm -f "${baseinstalldir}/${name}/${name}" - fi - if ! ln -s "${baseinstalldir}/${installdirname}" "${baseinstalldir}/${name}/${name}"; then - printError "Could not create symbolic link name." - fi - fi - - printVerbose "Adding version '${downloadFileVer}' to info file." - # Add file version information as a .releaseinfo file - echo "${downloadFileVer}" > "${baseinstalldir}/${installdirname}/.releaseinfo" - - # Check for a .version file from the pax - if present good, if not - # generate one from the file name as the tag isn't granular enough to really - # be used in dependency checks - if [ ! -f "${baseinstalldir}/${installdirname}/.version" ]; then - echo "${downloadFileVer}" > "${baseinstalldir}/${installdirname}/.version" - fi - - printVerbose "Adding releaseline '${installedReleaseLine}' metadata to ${baseinstalldir}/${installdirname}/.releaseline" - echo "${installedReleaseLine}" > "${baseinstalldir}/${installdirname}/.releaseline" - - if ${setactive}; then - if ! ${nosymlink}; then - mergeIntoSystem "${name}" "${baseinstalldir}/${installdirname}" "${ZOPEN_ROOTFS}" - misrc=$? - printVerbose "The merge completed with: ${misrc}" - fi - - printInfo "- Checking for env file." - if [ -f "${baseinstalldir}/${name}/${name}/.env" ] || [ -f "${baseinstalldir}/${name}/${name}/.appenv" ]; then - printInfo "- .env file found, adding to profiled processing." - mkdir -p "${ZOPEN_ROOTFS}/etc/profiled/${name}" - cat << EOF > "${ZOPEN_ROOTFS}/etc/profiled/${name}/dotenv" -curdir=\$(pwd) -cd "\${ZOPEN_ROOTFS}${ZOPEN_PKGINSTALL##"${ZOPEN_ROOTFS}"}/${name}/${name}" >/dev/null 2>&1 -# If .appenv exists, source it as it's quicker -if [ -f ".appenv" ]; then - . ./.appenv -elif [ -f ".env" ]; then - . ./.env -fi -cd \${curdir} >/dev/null 2>&1 -EOF - printInfo "- Sourcing environment to run any setup." - cd "${baseinstalldir}/${name}/${name}" && ./setup.sh - fi - fi - if ${unInstallOldVersion}; then - printVerbose "New version merged; checking for orphaned files from previous version." - # This will remove any old symlinks or dirs that might have changed in an upgrade - # as the merge process overwrites existing files to point to different version - unsymlinkFromSystem "${name}" "${ZOPEN_ROOTFS}" "${currentlinkfile}" "${baseinstalldir}/${name}/${name}/.links" - fi - - if ${setactive}; then - printVerbose "Marking this version as installed." - touch "${baseinstalldir}/${name}/${name}/.active" - installedList="${name} ${installedList}" - syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_INSTALL},${CAT_PACKAGE}" "DOWNLOAD" "handlePackageInstall" "Installed package:'${name}';version:${downloadFileVer};install_dir='${baseinstalldir}/${installdirname}';" - fi - - registerInstall "$name" "${downloadFileVer}" "${upgradeInstalled}" ${isRuntimeDependency} - - if ${doNotInstallDeps}; then - printInfo "- Skipping dependency installation." - elif ${reinstall}; then - printVerbose "- Reinstalling so no dependency reinstall (unless explicitly listed)." - else - printInfo "- Checking for runtime dependencies." - printVerbose "Checking for .runtimedeps file." - if [ -e "${baseinstalldir}/${name}/${name}/.runtimedeps" ]; then - dependencies=$(cat "${baseinstalldir}/${name}/${name}/.runtimedeps") - fi - printVerbose "Checking for runtime dependencies from the git metadata." - if echo "${statusline}" | grep "Runtime Dependencies:" > /dev/null; then - gitmetadependencies="$(echo "${statusline}" | sed -e "s#.*Runtime Dependencies:<\/b> ##" -e "s#
.*##")" - if [ ! "${gitmetadependencies}" = "No dependencies" ]; then - dependencies="${dependencies} ${gitmetadependencies}" - fi - fi - dependencies=$(deleteDuplicateEntries "${dependencies}" " ") - if [ -n "${dependencies}" ]; then - printInfo "- ${name} depends on: ${dependencies}" - printInfo "- Installing dependencies." - installDependencies "${name}" "${dependencies}" - else - printInfo "- No runtime dependencies found." - fi - fi - printInfo "${NC}${GREEN}Successfully installed: ${name}${NC}" - fi # (download only) -} - -installPorts() -( - ports="$1" - printVerbose "Ports to install: ${ports}" - if ! ${force}; then - mutexReq "zopen" "zopen" - fi - caveatsFile=$(mktempfile "caveats") - echo "${ports}" | xargs | tr ' ' '\n' | while read port; do - handlePackageInstall "${port}" - done - if [ -s "${caveatsFile}" ]; then - printHeader "Installation Caveats" - cat "${caveatsFile}" - fi - rm -f "${caveatsFile}" - if ! ${force}; then - mutexFree "zopen" - fi -) - # Main code start here +# Need to set a number of variables for use in the install function +# which is common between install & upgrade args=$* -upgradeInstalled=false verbose=false debug=false +xdebug=false +quiet=false selectVersion=false +# shellcheck disable=SC2034 +bypassPrereqs=false +# shellcheck disable=SC2034 setActive=true -cacheOnly=false downloadOnly=false -localInstall=false reinstall=false +reinstallDeps=false installOrUpgrade=false +# shellcheck disable=SC2034 nosymlink=false +skip_broken=false +skip_size_check=false +# shellcheck disable=SC2034 skipupgrade=false skipverify=false doNotInstallDeps=false @@ -682,191 +102,221 @@ yesToPrompts=false bypassPrereqs=false force=false chosenRepos="" +fileinstall=false + while [ $# -gt 0 ]; do case "$1" in - "-u" | "--update" | "--upgrade") - upgradeInstalled=true # Upgrade packages - ;; - "-r" | "-reinstall" | "--reinstall") - reinstall=true # If package already installed, reinstall - ;; - "--install-or-upgrade") - installOrUpgrade=true # Upgrade package or install if not present - ;; - "--bypass-prereq-checks") - bypassPrereqs=true - ;; - "--local-install") - localInstall=true # Install the package into current directory - ;; - "--no-symlink") - nosymlink=true # Do not mesh the package into the file system; leave as stand-alone - ;; - "--no-deps") - doNotInstallDeps=true - ;; - "--cache-only") - cacheOnly=true # Download remote pax file to cache only (no install) - ;; - "--release-line") - shift - releaseLine=$(echo "$1" | awk '{print toupper($0)}') - ;; - "--yes" | "-y") - yesToPrompts=true # Automatically answer 'yes' to any questions - ;; - "--download-only") - downloadOnly=true # Download remote pax file to current directory only - ;; - "--no-set-active") - setactive=false # Install package as normal but keep existing installation as active - ;; - "--skip-upgrade") - skipupgrade=true # Do not upgrade any packages - ;; - "--skip-verify" | "-sv") - skipverify=true # Verify signature of packages - ;; - "--force") - force=true # Bypasses locks - ;; - "--all") - all=true # Install all packages - ;; - "--select") - selectVersion=true # Display a selction table to allow version picking - ;; - "-h" | "--help" | "-?" ) - printSyntax "${args}" - exit 0 - ;; - "--debug") - verbose=true - debug=true + "-r" | "-reinstall" | "--reinstall") + # shellcheck disable=SC2034 + reinstall=true # If package already installed, reinstall + ;; + "--reinstall-with-deps") + # shellcheck disable=SC2034 + reinstallDeps=true # If reinstalling, also reinstall dependencies + # shellcheck disable=SC2034 + reinstall=true + ;; + "--install-or-upgrade") + # shellcheck disable=SC2034 + installOrUpgrade=true # Upgrade package or install if not present + ;; + "--bypass-prereq-checks") + # shellcheck disable=SC2034 + bypassPrereqs=true + ;; + "--no-symlink") + # shellcheck disable=SC2034 + setactive=true # Do not mesh the package into the file system; leave as stand-alone + ;; + "--nodeps" | "--no-deps") # Deprecate --no-deps for consistency + doNotInstallDeps=true + ;; + "--release-line") + shift + # shellcheck disable=SC2034 + releaseLine=$(echo "$1" | awk '{print toupper($0)}') + ;; + "--yes" | "-y") + # shellcheck disable=SC2034 + yesToPrompts=true # Automatically answer 'yes' to any questions + ;; + "--download-only") + downloadOnly=true # Download remote package files to current directory only + ;; + "--noset-active" | "--no-set-active") # Deprecate --no-set-active for consistency + # shellcheck disable=SC2034 + setactive=false # Install package as normal but keep existing installation as active + ;; + "--skip-broken" | "-sb") + # shellcheck disable=SC2034 + skip_broken=true + ;; + "--skip-verify" | "-sv") + # shellcheck disable=SC2034 + skipverify=true # Verify signature of packages + ;; + "--skip_size_check" | "-st") + # shellcheck disable=SC2034 + skip_size_check=true + ;; + "--force") + # shellcheck disable=SC2034 + force=true # Bypasses locks + ;; + "--all") + all=true # Install all packages + ;; + "--select") + # shellcheck disable=SC2034 + selectVersion=true # Display a selction table to allow version picking + ;; + "-h" | "--help" | "-?") + printHelp "${args}" + exit 0 + ;; + "--debug") + verbose=true + # shellcheck disable=SC2034 + debug=true + ;; + "-v" | "--verbose") + # shellcheck disable=SC2034 + verbose=true + ;; + "--xdebug") + xdebug=true ;; - "-v" | "--verbose") - verbose=true + "--quiet") + # shellcheck disable=SC2034 + quiet=true ;; "--version") - zopen-version ${ME} + zopen-version "${ME}" exit 0 ;; + + -*) printError "Unsupported parameter '$1'";; *) - chosenRepos="${chosenRepos} $1" + # Generate a long @@ separated string to allow for embedded + # spaces in hardcoded pax filenames + chosenRepos="${chosenRepos}@@$1" ;; esac shift done -if [ -z "${chosenRepos}" ]; then - if ! ${all} && ! ${upgradeInstalled}; then - printInfo "No ports selected for installation." - exit 4 - fi - if ${upgradeInstalled}; then - printVerbose "No specific port to upgrade, upgrade all installed packages." - printInfo "- Querying for installed packages." - progressHandler "spinner" "- Query complete." & - ph=$! - killph="kill -HUP ${ph}" - addCleanupTrapCmd "${killph}" - chosenRepos="$(${MYDIR}/zopen-query --list --installed --no-header --no-version 2>&1)" - zqrc=$? - ${killph} 2> /dev/null # if the timer is not running, the kill will fail - sleep 1 # give the above process time to clear - if [ ${zqrc} -ne 0 ]; then - printError "Query for installed packages unexpectedly failed; zopen-query returned message: '${chosenRepos}'" - fi - fi +${xdebug} && set -x && printVerbose "Enabled command execution trace" + +if ! ${all} && [ -z "${chosenRepos}" ]; then + printInfo "No packages selected for installation." + exit 4 fi -checkIfConfigLoaded +# If any of the parameters passed in point to an existing file, then +# the user is attempting to install a port directly from the file system +# rather than a repo +printDebug "Checking input parameters for actual files" +potentials=$(echo "${chosenRepos}" | sed 's/@@/ /g') +for installRepo in ${potentials}; do + [ -f "${installRepo}" ] || continue + # The parameter is a file, check it is actually a supported package or ignore + case "${installRepo}" in + (*.pax.Z) + printVerbose "Input parameter '$installRepo' has a supported input suffix" + # Treat all input parameters as file parameters + fileinstall=true + break + ;; + *) printVerbose "Handling input parameter '$installRepo' as a package name" ;; + esac +done + +# Sanity check for argument clashes +if ${selectVersion}; then + if ${all}; then + printError "Conflicting program arguments, --select cannot be used with --all" + elif ${fileinstall}; then + printError "Conflicting program argument, --select cannot be used with local file installations" + fi +fi +if ${fileinstall} && ${all}; then + printError "Conflicting program argument, --all cannot be used with local file installations" +fi -export SSL_CERT_FILE="${ZOPEN_CA}" -export GIT_SSL_CAINFO="${ZOPEN_CA}" -export CURL_CA_BUNDLE="${ZOPEN_CA}" +if ! ${fileinstall}; then + printVerbose "Using potentially remote files; ensuring configured for remote access" + checkIfConfigLoaded +else + printVerbose "At least one local file was specified for installation" +fi +mutexReq "zopen" "zopen" if ${downloadOnly}; then downloadDir="${PWD}" - printVerbose "Downloading pax to current directory '${downloadDir}'" -elif ${localInstall}; then - downloadDir="${PWD}" - printVerbose "Installing to current directory '${downloadDir}'" + printDebug "Downloading pax to current directory '${downloadDir}'" else - printVerbose "Installing to zopen file system: ${ZOPEN_ROOTFS}" + printDebug "Installing to zopen file system: ${ZOPEN_ROOTFS}" if [ -z "${ZOPEN_ROOTFS}" ]; then - printError "Unable to locate zopen file system, \$ZOPEN_ROOTFS is undefined." + printError "Unable to locate zopen file system, \${ZOPEN_ROOTFS} is undefined. Re-source zopen-config and retry command." fi downloadDir="${ZOPEN_ROOTFS}/var/cache/zopen" fi if [ ! -d "${downloadDir}" ]; then - mkdir -p "${downloadDir}" - if [ $? -gt 0 ]; then - printError "Could not create download directory: ${downloadDir}" + if ! mkdir -p "${downloadDir}"; then + printError "Could not create download directory: ${downloadDir}. Check permissions and retry command." fi fi -if [ -n "${downloadDir}" ] && [ -d "${downloadDir}" ]; then - cd "${downloadDir}" || exit +printDebug "Checking if installing from pax files: ${fileinstall}" +if ! ${fileinstall}; then + printVerbose "Querying metadata for latest package information" + updateCaches + grfgRc=$? + [ 0 -ne ${grfgRc} ] && exit ${grfgRc}; fi -printVerbose "Working directory: ${downloadDir}" -# Parse passed in repositories and check if valid zopen framework repos -printInfo "- Querying repo for latest package information." -getReposFromGithub -grfgRc=$? -${killph} 2> /dev/null # if the timer is not running, the kill will fail -[ 0 -ne "${grfgRc}" ] && exit "${grfgRc}" - -foundPort=false -installArray="" - -if ${all}; then - if ! ${yesToPrompts}; then - # Sum up the pax size + expanded size of the latest releases of each port - spaceRequiredBytes=$(jq '.release_data | to_entries | map(select(.value | length > 0)) | map(.value[0].assets[0]) | reduce .[] as $item ({}; . + {($item.name): (($item.size | tonumber) + ($item.expanded_size | tonumber))}) | [.[]] | add' ${JSON_CACHE}) - echo "Space: $spaceRequiredBytes" - spaceRequiredMB=$(echo "scale=0; ${spaceRequiredBytes} / (1024 * 1024)" | bc) - availableSpaceMB=$(/bin/df -m ${ZOPEN_ROOTFS} | sed "1d" | awk '{ print $3 }' | awk -F'/' '{ print $1 }') - - printInfo "You have chosen to install all tools. An estimated ${spaceRequiredMB} MB of additional disk space will be used." - if [ $availableSpaceMB -lt $spaceRequiredMB ]; then - printWarning "Your zopen file-system ($ZOPEN_ROOTFS) only has ${availableSpaceMB} MB of available space." - fi - printInfo "Enter 'all' to confirm full installation. (This can take a VERY long time!):" - confirmall=$(getInput) - if [ ! "xall" = "x${confirmall}" ]; then - printError "Cancelling full installation." - fi - fi - for repo in $(echo ${repo_results}); do - installArray="${installArray} ${repo}" +if ${fileinstall}; then + printDebug "Installing from files as listed in arguments: '${chosenRepos}'" + printDebug "Fully-qualifying files" + absoluteFiles="" + + for fn in $(echo "${chosenRepos}" | sed 's/@@/ /g'); do + absoluteFiles="${absoluteFiles} $(toAbsolutePath "${fn}")" done - installArray=$(strtrim "${installArray}") + # generate the install list JSON from the @@-delimited inputs + installList=$(echo "${absoluteFiles}" \ + | jq --raw-input \ + 'def make_object($url): {asset:{url: ( "file://" + $url )}}; . | split("@@") | map(select(.!="")|make_object(.)) | {"installqueue" :.} ') else - chosenRepos=$(strtrim "${chosenRepos}") - invalidlist="" - for chosenRepo in $(echo "${chosenRepos}" | tr ',' ' ' | tr -s ' '); do - printVerbose "Processing repo: ${chosenRepo}" - printVerbose "Stripping any version (%), tag (#) or port suffixes" - toolrepo=$(echo "${chosenRepo}" | sed -e 's#%.*##' -e 's#=.*##') - toolfound=$(echo "${repo_results}" | awk -vtoolrepo="${toolrepo}" '$0 == toolrepo {print}') - if [ "${toolfound}" = "${toolrepo}" ]; then - printVerbose "Adding '${chosenRepo}' to the install queue." - installArray="${installArray} ${chosenRepo}" - printVerbose "Removing valid port from input list." - chosenRepos=$(echo "${chosenRepos}" | sed -e "s#^${chosenRepo}\$##") - else - invalidlist=$(/bin/printf "%s %s" "${invalidlist}" "${chosenRepo}") - fi - done + if ${all}; then + # shellcheck disable=SC2034 + doNotInstallDeps=true + installList=$(jq --raw-output '.release_data| keys[]' "${JSON_CACHE}") + installListCount=$(jq --raw-output '.release_data| keys | length' "${JSON_CACHE}") + printInfo "- Installing all currently-uninstalled packages" + printInfo "- Checking installation status for ${installListCount} package$([ "${installListCount}" -ne 1 ] && echo "s")" + else + installList=$(echo "$chosenRepos" | sed "s/@@/ /g") + validatePackageList "${installList}" + fi + if ! generateInstallGraph "${installList}"; then + printError "Unable to generate install graph" + fi fi -printVerbose "Checking whether any invalid ports were specified." -if [ -n "${invalidlist}" ]; then - printSoftError "The following requested port(s) do not exist:\n\t$(echo "${invalidlist}" | tr -s '[:space:]')" - printError "Check port name(s), remove any port suffixes and retry command." +if [ 0 -eq "$(echo "${installList}" | jq --raw-output '.installqueue| length')" ]; then + printInfo "- No packages require installation" +else + if ${verbose}; then + printInfo " - The following package(s) will be installed:" + echo "${installList}" | jq --raw-output '.installqueue | sort| .[] | .portname ' + fi + if ! processRepoInstallFile; then + exitrc=1 + else + printInfo "Installation complete." + fi fi - -installPorts "${installArray}" +mutexFree "zopen" +exit "${exitrc:-0}" diff --git a/bin/zopen-list b/bin/zopen-list new file mode 100755 index 000000000..47be6a62f --- /dev/null +++ b/bin/zopen-list @@ -0,0 +1,344 @@ +#!/bin/sh +# +# List utility for zopen community- https://github.com/zopencommunity +# + +# +# All zopen-* scripts MUST start with this code to maintain consistency. +# +setupMyself() +{ + ME=$(basename $0) + MYDIR="$(cd "$(dirname "$0")" > /dev/null 2>&1 && pwd -P)" + INCDIR="${MYDIR}/../include" + if ! [ -d "${INCDIR}" ] && ! [ -f "${INCDIR}/common.sh" ]; then + echo "Internal Error. Unable to find common.sh file to source." >&2 + exit 8 + fi + # shellcheck source=/dev/null + . "${INCDIR}/common.sh" +} +setupMyself + +printHelp(){ + cat << HELPDOC +zopen list - list information about local packages + +Usage: zopen list [OPTION] [VERB] [PACKAGE] + +Options: + --available lists all available packages + --installed list only installed packages (default output) + --details provide version information on packages + --full provides more information about packages + --[no]header output explanation header for columns + --verbose run in verbose mode + --version print version + --version-info lists version information for installed packages when + combined with the --installed option + -h,-?, --help display this help and exit + +Examples: + zopen list --installed + list packages installed on the system + +Report bugs at https://github.com/zopencommunity/meta/issues + +HELPDOC +} + +listUpdatesForInstalled() +{ + updateCaches + printInfo "Packages available for update:" + + pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" + installed=$(zopen list --installed --details | \ + awk '{gsub(/[[:space:]]+/, "@", $0); lines=(NR==1) ? "\""$0"\"" : lines",""\""$0"\""} END {print "["lines"]"}') + #installed="[${installed%,}]" # wrap in a JSON array and strip any trailing ',' char + jq --raw-output --arg installed "${installed}" \ + "$(jqfunctions)"' def parse_installed: ($installed | fromjson) | map(split("@") | {name: .[0], version: .[1], release: .[2]}); +.release_data | to_entries | +map(.key as $pkg_name | .value[0].assets | + map(. as $asset | + (parse_installed | map(select(.name == $pkg_name)) | .[0]) as $installed_pkg | + if $installed_pkg then + $asset.version as $asset_version | + $asset.release as $asset_release | + # Compare versions + if ($installed_pkg.version != $asset_version) or ($installed_pkg.release != $asset_release) then + if ($installed_pkg.version < $asset_version) or + ($installed_pkg.version == $asset_version and $installed_pkg.release < $asset_release) then + { + "package": $pkg_name, + "installed_version": $installed_pkg.version, + "installed_release": $installed_pkg.release, + "available_version": $asset_version, + "available_release": $asset_release + } + else null end + else null end + else null end + ) +) | flatten | . as $upgrades | +($upgrades | map(.package | length) | max) as $max_packagename_length | +($upgrades | map((.installed_version + .installed_release ) | length ) | max)as $max_installedstring_length | +$upgrades | map(select(.)) | .[] +| pr(.package;" ";(($max_packagename_length | tonumber) + 4)) + + pl((.installed_version + "-" + .installed_release);" ";($max_installedstring_length + 3)) + + c("==>";" ";5) + + pr((.available_version + "-" + .available_release);" ";($max_installedstring_length + 3)) + ' "${JSON_CACHE}" | sed 's/@/ /g' +} + +listAvailablePackages() +{ + updateCaches + wpackage=20 + wversion=23 + winstalled=10 + wtag=36 + wrellne=12 + wdownload=12 + wexpsz=12 + wqual=10 + if ${urlinclude}; then + wurl=6 + urltext="URL" + else + wurl=0 + urltext="" + fi + if ${header}; then + if ${details} || ${full}; then + printf "${NC}${UNDERLINE}%-${wpackage}s%-${winstalled}s%-${wtag}s%-${wdownload}s%-${wexpsz}s%-${wqual}s%-${wurl}s${NC}\n" \ + "$(text_center "Package")" "$(text_center "Installed")" "$(text_center "Latest Tag")"\ + "$(text_center "Download size")" "$(text_center "Disk usage")" "$(text_center "Quality")" \ + "$(text_center "${urltext}")" + + else + printf "${NC}${UNDERLINE}%-${wpackage}s%-${winstalled}s%-${wversion}s%-${wtag}s${NC}\n" \ + "$(text_center "Package")" "$(text_center "Installed")" "$(text_center "Version")" "$(text_center "Latest Tag")" + fi + fi + installed=$(zopen list --installed | awk -v ORS=',' '{print "\""$0"\""}') + installed="[${installed%,}]" # wrap in a JSON array and strip any trailing ',' char + + if ${details} || ${full}; then + jq --raw-output --argjson installed "${installed}" \ + --arg wpackage "${wpackage}" \ + --arg winstalled "${winstalled}" \ + --arg wtag "${wtag}" \ + --arg wdownload "${wdownload}" \ + --arg wexpsz "${wexpsz}" \ + --arg wqual "${wqual}" \ + --arg urltext "${urltext}" \ + "$(jqfunctions)"' .release_data| + to_entries |sort_by(.key) | .[] | + .key as $key | + (.value[0].name | split(" ")[0]) as $actualname | + .value[0].assets[0] as $va | + $va.url as $url | + ($url | match(".*-([^-]*).[0-9]{8}_[0-9]{6}.zos.pax.Z").captures[0].string) as $version | + .value[0].tag_name as $tn | + (if ($installed | index($key)) then "installed" else "available" end) as $ins | + $va.expanded_size as $es | + $va.size as $ds | + ($va.passed_tests | if (.=="") then "0" else . end | tonumber) as $pt | + ($va.total_tests | if (.=="") then "99999" else . end |tonumber) as $tt | + ($pt/$tt*100) as $pts | + ($pts | if (.<0) then "No tests" else ($pts|r(1) |tostring + "%" ) end) as $q| + pr($actualname;"@";($wpackage | tonumber)) + + c($ins;"@";($winstalled | tonumber)) + + c($tn;"@";($wtag | tonumber)) + + c($ds;"@";($wdownload | tonumber)) + + c($es;"@";($wexpsz | tonumber)) + + c(($q);"@";($wqual | tonumber)) + + $urltext ' \ + "${JSON_CACHE}" | sed 's/@/ /g' + else + jq --raw-output --argjson installed "${installed}" \ + --arg wpackage "${wpackage}" \ + --arg winstalled "${winstalled}" \ + --arg wversion "${wversion}" \ + --arg wtag "${wtag}" \ + "$(jqfunctions)"' .release_data| + to_entries |sort_by(.key) | .[] | + .key as $key | + (.value[0].name | split(" ")[0]) as $actualname | + .value[0].assets[0].url as $url | + ($url | match(".*-([^-]*).[0-9]{8}_[0-9]{6}.zos.pax.Z").captures[0].string) as $version | + .value[0].tag_name as $tn | + (if ($installed | index($key)) then "installed" else "available" end) as $ins | + .value[0].assets[0] as $va | + pr($actualname;"@";($wpackage | tonumber)) + + c($ins;"@";($winstalled | tonumber)) + + c($version;"@";($wversion | tonumber)) + + c($tn;"@";($wtag | tonumber)) ' \ + "${JSON_CACHE}" | sed 's/@/ /g' + fi +} + +listInstalledEntries() +{ + wpackage=20 + wversion=26 + wrelease=14 + wrellne=12 + wexpsz=12 + wqual=8 + if ${urlinclude}; then + wurl=6 + urltext="URL" + else + wurl=0 + urltext="" + fi + pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" + if ! [ -e "${pdb}" ]; then + printWarning "No package database found. Regenerating (subsequent calls will be faster)" + updatePackageDB + fi + if ${header} && ${details}; then + if ${full}; then + printf "${NC}${UNDERLINE}%-${wpackage}s%-${wversion}s%-${wrelease}s%-${wrellne}s%-${wexpsz}s%-${wqual}s%-${wurl}s${NC}\n" \ + "$(text_center "Package" ${wpackage})" "$(text_center "Version" ${wversion})" "$(text_center "Release" ${wrelease})" \ + "$(text_center "Releaseline" ${wrellne})" "$(text_center "Disk usage" ${wexpsz})" \ + "$(text_center "Quality" ${wqual})" "$(text_center "${urltext}" ${wurl})" + else + printf "${NC}${UNDERLINE}%-${wpackage}s%-${wversion}s%-${wrelease}s${NC}\n" \ + "$(text_center "Package" ${wpackage})" "$(text_center "Version" ${wversion})" "$(text_center "Release" ${wrelease})" + fi + fi + if ${details}; then + printVerbose "Details requested" + if ${full}; then + printVerbose "Full details requested" + jq --raw-output \ + --arg wpackage "${wpackage}" \ + --arg wversion "${wversion}" \ + --arg wrelease "${wrelease}" \ + --arg wrellne "${wrellne}" \ + --arg wexpsz "${wexpsz}" \ + --arg wqual "${wqual}" \ + --arg urlinclude "${urlinclude}" \ + --arg wurl "${wurl}" \ + "$(jqfunctions)"' .[] + | (keys|sort)[] as $key + |.[$key].product.test_status as $ts + |($ts.total_success | tonumber? // null) as $succ + |($ts.total_tests|tonumber? // null) as $tests + |( + if ($succ == -1) then "-1" + elif ($tests == null or $succ == null) then "n/a" + elif ($tests == 0) then "0" + else (((100 * $succ / $tests) * 10 | round) / 10) |r(2)| tostring + "%" + end + ) as $ts_str + | pr($key;"@";($wpackage | tonumber)) + + c((.[$key].product?.version? // "unknown");"@";($wversion | tonumber)) + + c((.[$key].product?.release? // "unknown");"@";($wrelease | tonumber)) + + c((.[$key].product.buildline? // "unknown");"@";($wrellne | tonumber)) + + c((.[$key].product.size? // "unknown");"@";($wexpsz | tonumber)) + + c($ts_str;"@";($wqual | tonumber)) + + ( + if ($urlinclude == "true") then + c((.[$key].product.repo? // "unknown");"@";($wurl | tonumber)) + else + "" + end + )' \ + "${pdb}" | sed 's/@/ /g' + else + jq --raw-output \ + --arg wpackage "${wpackage}" \ + --arg wversion "${wversion}" \ + --arg wrelease "${wrelease}" \ + "$(jqfunctions)"' .[] | + (keys|sort)[] as $key | + pr($key;"@";($wpackage | tonumber)) + + c(.[$key].product.version;"@";($wversion | tonumber)) + + c(.[$key].product.release;"@";($wrelease | tonumber))' \ + "${pdb}" | sed 's/@/ /g' + fi + elif ${versioninfo}; then + jq --raw-output \ + --arg wpackage "${wpackage}" \ + --arg wversion "${wversion}" \ + "$(jqfunctions)"' .[] | + (keys|sort)[] as $key | + .[$key].product.test_status as $ts | + pr($key;"@";($wpackage | tonumber)) + + c((.[$key].product?.version? // "unknown");"@";($wversion | tonumber))' \ + "${pdb}" | sed 's/@/ /g' + else + jq --raw-output \ + '[.[] | keys[] as $key | $key] | unique[] ' \ + "${pdb}" + fi +} + + +# Main code start here +# Note that functions use padding chars of '@' to work round a quirk of jq in that +# it will collapse repeated spaces into tab characters - which proves irksome when tryinh +# to pad/format column layouts; '@' chars get post-processed into spaces + +verbose=false +header=false +details=false +full=false +commandVerb="installed" # Default list to display +urlinclude=false # whether to include download url in output information +versioninfo=false + +while [ $# -gt 0 ]; do + printVerbose "Parsing option: $1" + case "$1" in + "--available") commandVerb="available";; + "--installed") commandVerb="installed";; + "--updates") commandVerb="updates";; + "--details") details=true ;; + "--full") full=true; details=true ;; + "--header") header=true ;; + "--noheader") header=false ;; + "--no-header") header=false ;; # legacy, hidden in preference to the above + "--no-version") versioninfo=false ;; # legacy/deprecated hence hidden + "--url") urlinclude=true ;; + "--version-info") versioninfo=true ;; + "-h" | "--help" | "-?") + printHelp + exit 0 + ;; + "--version") + zopen-version "${ME}" + exit 0 + ;; + "--verbose") + # shellcheck disable=SC2034 + verbose=true + ;; + "--xdebug") + set -x + ;; + -*) + printError "Unknown option '$1'" + ;; + *) + chosenRepos="${chosenRepos} $1" + ;; + esac + shift +done + +checkIfConfigLoaded + +case "${commandVerb}" in + "available") listAvailablePackages;; + "updates") listUpdatesForInstalled;; + "installed") listInstalledEntries;; + *) assertFailed "Unrecognised command verb '${commandVerb}'";; +esac + +exit 0 + diff --git a/bin/zopen-mirror b/bin/zopen-mirror new file mode 100755 index 000000000..f299e5adb --- /dev/null +++ b/bin/zopen-mirror @@ -0,0 +1,861 @@ +#!/bin/sh +# Repo mirror utility - stands alone from other scripts so has no +# direct dependencies on existing functions. As such, it has some +# cut-down versions of common features and an independent version +VERSION=0.4.0 + + +# Global constants +ZOPEN_PROJECT_NAME="zopen community" +ZOPEN_GITHUB="https://github.com/zopencommunity" +RELEASE_JSON_URL="https://api.github.com/repos/zopencommunity/meta/contents/docs/api/zopen_releases.json?ref=main" +LATEST_JSON_URL="https://api.github.com/repos/zopencommunity/meta/contents/docs/api/zopen_releases_latest.json?ref=main" + +# Help text +printHelp() { + cat <
--target /zotrepo + + FTP - creates a mirror in the /parent/ftp.repo on the remote machine, using + credentials stored in a .netrc file for both the ftp server and + GitHub. + ${0} --type ftp --netrc --target /%2Fparent/ftp.repo --genRepoFile + +Description: + ${0} is a small utility that will mirror the packages made available by the + ${ZOPEN_PROJECT_NAME} for instalation using the zopen package manager. + The mirror will download - and upload if required - the packages from the + various GitHub repositories to the location specified and optionally + provide configuration files that can be used to configure the zopen package + manager to use the alternative repository. + +Details: + The currently supported mirror types are: + FLAT - mirrors the GH contents into a single directory on the local + system. For example: /zopen/repo.mirror + ALPHA - mirrors the GH contents into a local directory structure where the + heirarchy is the initial package letter as a directory name containing + packages starting with that letter. For example: //a/... + NAME - mirrors the GH contents into a unique local directory per package. + Each directory would contain only releases for that package. For + example: //automake + ALPHANAME - A combination of ALPHA and NAME. For example: /a/automake + FTP - Mirrors the GH repo to a remote ftp server. Specify the --target + in the following format: "/". Note that + if contains a leading '/' this will need to be + replaced with '%2F'; if there is no leading '/', the mirror will be + to a directory within the users home directory [the directory where + the user will start in after login]. Requires either the --netrc or + --dstuser and --dstpassword parameters for credentials where anonymous + login is disabled. Note that ftp is inherently insecure! + SFTP - As 'FTP'' but utilises the SFTP protocol + SAMBA - As 'FTP' but uses the SAMBA protocol, allowing Windows or Linux + network shares. --target should be the URL with share but no leading + protocol definition. For example, samba.example.com/share_path - + smb://samba.example.com/share_path will fail + + NEXUS - Utilises a Sonatype Nexus Repository. Requires either the --netrc or + --dstuser and --dstpassword parameters for credentials + +Parameter details: + --genRepoFile + The --genRepoFile parameter will create a file that is configured for the + specified mirror type. For example, running the mirror for an FTP-type + mirror will produce a file with the contents: + { + "type": "ftp", + "metadata_baseurl": "ftpserver.name/%2Fzopen/repo.mirror")", + "metadata_file": "metadata.json" + } + This file can be placed into the /etc/zopen/repos.d directory and + then symlinked to be the active version. + --activate [ZOPEN_ROOTFS] + The --activate parameter will move a generated repository description file + [as generated with the --genRepoFile parameter] to be the active zopen + repository for the given zopen environment. Note this does require running + in a z/OS environment with authority to write to the + /etc/zopen/repos.d directory. + --netrc + The underlying curl process will use a .netrc to locate credentials for the + user, both for the remote operation and if the --ghsrcpat is not specified, + the credentials to use to access Github. The ZOPEN_CURL_PARAMS environment + variable can be used to specify the location ot the netrc file, the default + is a .netrc from the users \$HOME directory. A sample entry for .netrc to + access an ftp server would be: + machine SERVER_HOSTNAME_OR_IP + login SERVER_USERNAME + password SERVER_PASSWORD + + --ghsrcpat, --noghcreds + To prevent rate-limiting, --ghsrcpat should be specified however this does + expose the Personal Access Token to shell history, process listings etc. + Specifying --noghcreds prevents the check for the existence of the ghsrcpat + option, allowing a Github configuration in the user's .netrc file to be used + to authenticate. A sample entry into .netrc for a Github PAT looks like: + machine api.github.com + login YOUR_GITHUB_USERNAME + password YOUR_PERSONAL_ACCESS_TOKEN + +FULLHELP + fi +cat << FOOTER +Report bugs at ${ZOPEN_GITHUB}/meta/issues . +FOOTER + +} + +# Utility functions +displayVersion() { + log ${VERSION} +} + +log() { + ! $quiet && /bin/printf "$*\n" +} +verbose() { + $verbose && /bin/printf "${NC}${MAGENTA}$*${NC}\n" +} + +warning() { + /bin/printf " ${NC}${YELLOW}!! $*${NC}\n" +} + +err() { + /bin/printf " ${NC}${RED}!! $*${NC}\n" + exit 1 +} + +defineANSI() +{ + # Standard tty codes + ESC="" # Start of Escape Sequence; EBCDIC=\047, ASCII=\033 + platform=$(uname) + case ${platform} in + "OS/390") ESC=$(printf "\047");; + *) ESC=$(printf "\033");; + esac + CSI="[" # Control Sequence Introducer + CNL="E" # Cursor Next Line + CPL="F" # Cursor Previous Line + CHA="G" # Cursor Horizontal Absolute - column selector + EL="K" # Erase In Line + SGR="m" # Select Graphic Rendition + + # shellcheck disable=SC2034 + ERASELINE="${ESC}${CSI}2${EL}" + # shellcheck disable=SC2034 + CRSRHIDE="${ESC}${CSI}?25l" + # shellcheck disable=SC2034 + CRSRSHOW="${ESC}${CSI}?25h" + # shellcheck disable=SC2034 + CRSRSOL="${ESC}${CSI}0${CPL}" + # shellcheck disable=SC2034 + CRSRPL="${ESC}${CSI}1${CPL}" # Move to start of previous line + # shellcheck disable=SC2034 + CRSRUP="A" # CUU + # shellcheck disable=SC2034 + CRSRDOWN="B" # CUF + # shellcheck disable=SC2034 + CRSRRIGHT="C" # CUB + # shellcheck disable=SC2034 + CRSRLEFT="D" # CUD + # Color-type codes, needs explicit terminal settings + if [ ! "${_BPX_TERMPATH-x}" = "OMVS" ] && [ -z "${NO_COLOR}" ] && [ ! "${FORCE_COLOR-x}" = "0" ] && [ -t 1 ] && [ -t 2 ]; then + ANSION=true + BLACK="${ESC}${CSI}30${SGR}" + RED="${ESC}${CSI}31${SGR}" + GREEN="${ESC}${CSI}32${SGR}" + YELLOW="${ESC}${CSI}33${SGR}" + BLUE="${ESC}${CSI}34${SGR}" + MAGENTA=$(printf "${ESC}${CSI}35${SGR}") + CYAN="${ESC}${CSI}36${SGR}" + GRAY="${ESC}${CSI}37${SGR}" + BOLD="${ESC}${CSI}1${SGR}" + UNDERLINE="${ESC}${CSI}4${SGR}" + HEADERCOLOR="${MAGENTA}" + WARNINGCOLOR="${MAGENTA}" + # Keep the following last in the list - command trace can interpret the codes + # above and make the output display interesting depending on what the last + # ANSI command was! + NC=$(printf "${ESC}${CSI}0${SGR}") + else + ANSION=false + unset esc RED GREEN YELLOW BOLD UNDERLINE NC + unset esc BLACK RED GREEN YELLOW BLUE MAGENTA CYAN GRAY + unset UNDERLINE NC HEADERCOLOR WARNINGCOLOR + fi +} + +curlCmd() { + if ! type curl >/dev/null; then + err "Could not find curl on the path." + fi + curl "$@" 2>&1 +} + +initial() { + echo "$1" | cut -c1 +} + +downloadRepoMetadata() { + JSON_CACHE="${tmpdir:=/tmp}/zopen_releases.json.$$" + JSON_CACHE_LATEST="${tmpdir:=/tmp}/zopen_releases_latest.json.$$" + + if ! curlout=$(curlCmd -L --no-progress-meter -H 'Accept: application/vnd.github.v3.raw' -H "${authHeader}"\ + -o "${JSON_CACHE}" \ + "${RELEASE_JSON_URL}"); then + err "Failed to obtain json cache from '${RELEASE_JSON_URL}'; ${curlout}" + fi + if type chtag > /dev/null 2>&1; then + # z/OS only - run chtag to ensure the file is processed as ASCII + chtag -tc 819 "${JSON_CACHE}" + fi + if [ ! -f "${JSON_CACHE}" ]; then + err "Could not find json cache locally after download from '${RELEASE_JSON_URL}'" + fi + + if ! curlout=$(curlCmd -L --no-progress-meter -H 'Accept: application/vnd.github.v3.raw' -H "${authHeader}"\ + -o "${JSON_CACHE_LATEST}" \ + "${LATEST_JSON_URL}"); then + err "Failed to obtain json cache from '${LATEST_JSON_URL}'; ${curlout}" + fi + if type chtag > /dev/null 2>&1; then + # z/OS only - run chtag to ensure the file is processed as ASCII + chtag -tc 819 "${JSON_CACHE_LATEST}" + fi + if [ ! -f "${JSON_CACHE_LATEST}" ]; then + err "Could not find json cache locally after download from '${LATEST_JSON_URL}'" + fi + +} + +getPkgFile() { + mirror="$1" + portNumber=$2 + repourl=$(jq -r --arg portNumber "${portNumber}" ".release_data.\"${mirror}\"[$portNumber].assets[0].url" "${JSON_CACHE}") + filename=$(basename "${repourl}") + [ -z "${filename}" ] && err "Unable to parse filename for download at '${repourl}'." + if eval "${type}Repo checkForFile ${filename} ${mirror}"; then + verbose "${filename} already in repository" + return 1 + fi + outfile="${tmpdir:=/tmp}/${filename}" + [ -e "${outfile}" ] && rm "${outfile}" + if ! curlout=$(curlCmd --no-progress-meter -L -H "${authHeader}" -o "${outfile}" "${repourl}"); then + err "Curl command to retrive package file failed '${curlout}'. Resolve issue and retry request." + fi +} + +getPkgMetadata(){ + # Use previously set variables (from getPkgFile) with a ".json" suffix! + outfile="${tmpdir:=/tmp}/${filename}.json" + [ -e "${outfile}" ] && rm "${outfile}" + if ! curlout=$(curlCmd --no-progress-meter -L -H "${authHeader}" -o "${outfile}" "$(dirname "${repourl}")/metadata.json"); then + err "Curl command to retrive package metadata failed '${curlout}'. Resolve issue and retry request." + fi + if type chtag > /dev/null 2>&1; then + # z/OS only - run chtag to ensure the file is processed as ASCII + chtag -tc 819 "${outfile}" + fi +} + +## Repository handlers +_AbstractFileRepo(){ + case "$1" in + initialise) + [ -e "${target}" ] || mkdir -p "${target}" + [ -d "${target}" ] || err "Repo mirror target '${target}' is not a directory" + [ -w "${target}" ] || err "Repo mirror target '${target}' is not writable" + return 0 + ;; + checkForFile) err "Unimplemented action verb for abstract type" ;; + mirrorFile) err "Unimplemented action verb for abstract type";; + generateRepoD) + repoDFile="${target}/repod.json" + [ -e "${repoDFile}" ] && + mv "${repoDFile}" "${repoDFile}.$(date "+%C%m%d%H%M%S")" + mkdir -p "$(dirname "${repoDFile}")" + cat <"${repoDFile}" +{ + "type": "file", + "metadata_baseurl": "$(dirname "${rewrittenRepoFile}")", + "metadata_file": "$(basename "${rewrittenRepoFile}")", + "latest_file": "$(basename "${rewrittenLatestFile}")" +} +EOS + echo "${repoDFile}" + ;; + rewriteMetadata) + jqQuery=$2 + rewrittenRepoFile="${target%/}/metadata.json" + rewrittenLatestFile="${target%/}/latest_metadata.json" + if ! rewrite=$(jq --arg zopenrepo "${ZOPEN_GITHUB}" --arg t "${target%/}" \ + "${jqQuery}" "${JSON_CACHE}" >"${rewrittenRepoFile}"); then + err "Unable to generate metadata JSON file '${rewrittenRepoFile}' for mirrored repo. ${rewrite}" + fi + if ! rewrite=$(jq --arg zopenrepo "${ZOPEN_GITHUB}" --arg t "${target%/}" \ + "${jqQuery}" "${JSON_CACHE_LATEST}" >"${rewrittenLatestFile}"); then + err "Unable to generate metadata JSON file '${rewrittenLatestFile}' for mirrored repo. ${rewrite}" + fi + ;; + *) err "Unsupported action verb for repo type";; + esac +} +FLATRepo(){ + case "$1" in + initialise) _AbstractFileRepo "initialise" ;; + checkForFile) + filename=$(basename "$2") + [ -e "${target}/${filename}" ] && return 0 + return 1 + ;; + mirrorFile) + targetFile="${target}/$(basename "$2")" + [ -e "${targetFile}" ] && verbose "$2 already in repo" && return 0 + # A mv would generally work, however on z/OS, if there is an issue + # setting the new gid there'll be a message; mimic the mv with a + # copy-n-delete as still want to get any error messages - redirecting & + # analyzing the output from the mv complicates the code! + cp -f "${outfile}" "${targetFile}" + rm "${outfile}" + ;; + generateRepoD)_AbstractFileRepo "generateRepoD" ;; + rewriteMetadata) + _AbstractFileRepo "rewriteMetadata" \ + 'walk(if type == "string" then gsub($zopenrepo + "/.*/(?.*)$"; "file://" + $t + "/" + .f) else . end)' + ;; + *) err "Unsupported action verb for repo type";; + esac +} +ALPHARepo(){ + case "$1" in + initialise) _AbstractFileRepo "initialise" ;; + checkForFile) + filename=$(basename "$2") + [ -e "${target}/$(initial "${filename}")/${filename}" ] && return 0 + return 1 + ;; + mirrorFile) + filename=$(basename "$2") + targetFile="${target}/$(initial "${filename}")/${filename}" + [ -e "${targetFile}" ] && verbose "${filename} already downloaded" && return 0 + [ ! -d "$(dirname "${targetFile}")" ] && mkdir -p "$(dirname "${targetFile}")" + [ -e "${outfile}" ] || err "Could not find downloaded file '${outfile}'" + cp -f "${outfile}" "${targetFile}" + rm "${outfile}" + ;; + generateRepoD) _AbstractFileRepo "generateRepoD" ;; + rewriteMetadata) + _AbstractFileRepo "rewriteMetadata" \ + 'walk(if type == "string" then gsub($zopenrepo + "/(?.).*/(?.*)$"; "file://" + $t + "/" + .i + "/" + .f) else . end)' + ;; + *) err "Unsupported action verb for repo type";; + esac +} + +NAMERepo(){ + case "$1" in + initialise) _AbstractFileRepo "initialise" ;; + checkForFile) + filename=$(basename "$2") + pkgname=$3 + [ -e "${target}/${pkgname}/${filename}" ] && return 0 + return 1 + ;; + mirrorFile) + filename=$(basename "$2") + pkgname=$3 + targetFile="${target}/${pkgname}/${filename}" + [ -e "${targetFile}" ] && verbose "${filename} already downloaded" && return 0 + [ ! -d "$(dirname "${targetFile}")" ] && mkdir -p "$(dirname "${targetFile}")" + cp -f "${outfile}" "${targetFile}" + rm "${outfile}" + ;; + generateRepoD) _AbstractFileRepo "generateRepoD" ;; + rewriteMetadata) + _AbstractFileRepo "rewriteMetadata" \ + 'walk(if type == "string" then gsub($zopenrepo + "/(?.*)port/.*/(?.*)$"; "file://" + $t + "/" + .n + "/" + .f) else . end)' + ;; + *) err "Unsupported action verb for repo type";; + esac +} + +ALPHANAMERepo(){ + case "$1" in + initialise) _AbstractFileRepo "initialise" ;; + checkForFile) + filename=$(basename "$2") + pkgname=$3 + [ -e "${target}/$(initial "${filename}")/${pkgname}/${filename}" ] && return 0 + return 1 + ;; + mirrorFile) + filename=$(basename "$2") + pkgname=$3 + targetFile="${target}/$(initial "${filename}")/${pkgname}/${filename}" + [ -e "${targetFile}" ] && verbose "${filename} already downloaded" && return 0 + [ -d "$(dirname "${targetFile}")" ] || mkdir -p "$(dirname "${targetFile}")" + cp -f "${outfile}" "${targetFile}" + rm "${outfile}" + ;; + generateRepoD) _AbstractFileRepo "generateRepoD" ;; + rewriteMetadata) + _AbstractFileRepo "rewriteMetadata" \ + 'walk(if type == "string" then gsub($zopenrepo + "/(?.)(?.*)port/.*/(?.*)$"; "file://" + $t + "/" + .i + "/" + .i + .n + "/" + .f) else . end)' + + ;; + *) err "Unsupported action verb for repo type";; + esac +} + +_AbstractRemoteRepo(){ + case "$1" in + initialise) + # Get a directory listing of the remote and grep for the filename - crude + # but works. Cache the result in a file; a variable would be potentially quicker + # but if the package count gets too large, the variable will overflow. + if [ ! -f "${DIR_LIST_CACHE}" ]; then + curlparams=$(printf "%s --silent --show-error " "${curlparams}") + if ${netrc}; then + curlparams=$(printf " %s %s" "${curlparams}" "--netrc") + else + curlparams=$(printf " %s %s" "${curlparams}" "--user ${dstuser}:${dstpassword}") + fi + # shellcheck disable=SC2086 + if ! curlCmd ${curlparams} "$2://${target}/" > "${DIR_LIST_CACHE}"; then + err "Failed to otain directory listing at '$2://${target}/': $(cat ${DIR_LIST_CACHE})" + fi + fi + ;; + checkForFile) grep -q "$(basename "$2")" "${DIR_LIST_CACHE}" ;; + mirrorFile) + if ${netrc}; then + curlparams=$(printf " %s %s" "${curlparams}" "--netrc") + else + curlparams=$(printf " %s %s" "${curlparams}" "--user ${dstuser}:${dstpassword}") + fi + curlparams=$(printf "%s --silent --show-error" "${curlparams}") + curlCmd ${curlparams} "--upload-file" "$2" "$3://${target}/" + ;; + generateRepoD) + repoDFile=$(echo "${target}" | sed "s/[\\/:']//g") + repoDFile="${repoDFile}/repod.json" + mkdir -p "$(dirname "${repoDFile}")" + [ -e "${repoDFile}" ] && + mv "${repoDFile}" "${repoDFile}.$(date "+%C%m%d%H%M%S")" + cat <"${repoDFile}" +{ + "type": "$2", + "metadata_baseurl": "${target}")", + "metadata_file": "$(basename "${rewrittenRepoFile}")", + "latest_file": "$(basename "${rewrittenLatestFile}")" +} +EOS + echo "${repoDFile}" + ;; + rewriteMetadata) + jqQuery='walk(if type == "string" then gsub($zopenrepo + "/(?.)(?.*)port/.*/(?.*)$"; "$protocol://$target/" + .f) else . end)' + rewrittenRepoFile="${target%/}/metadata.json" + rewrittenLatestFile="${target%/}/latest_metadata.json" + if ! rewrite=$(jq --arg zopenrepo "${ZOPEN_GITHUB}" --arg t "${target%/}" \ + "${jqQuery}" "${JSON_CACHE}" >"${rewrittenRepoFile}"); then + err "Unable to generate metadata JSON file '${rewrittenRepoFile}' for mirrored repo. ${rewrite}" + fi + if ! rewrite=$(jq --arg zopenrepo "${ZOPEN_GITHUB}" --arg t "${target%/}" \ + "${jqQuery}" "${JSON_CACHE_LATEST}" >"${rewrittenLatestFile}"); then + err "Unable to generate metadata JSON file '${rewrittenLatestFile}' for mirrored repo. ${rewrite}" + fi + # Note there is no upload of the files; this is the abstract so no transport + ;; + + *) err "Unsupported action verb for repo type";; + esac +} + +FTPRepo(){ + this="ftp" + case "$1" in + initialise) _AbstractRemoteRepo "initialise" "${this}";; + checkForFile) + curlparams="--ssl --list-only" + _AbstractRemoteRepo "checkForFile" "$2" "${this}" + ;; + mirrorFile) + curlparams="--ssl" + _AbstractRemoteRepo "mirrorFile" "$2" "${this}" + ;; + generateRepoD) _AbstractRemoteRepo "generateRepoD" "${this}" ;; + rewriteMetadata) + _AbstractRemoteRepo "rewriteMetadata" "${this}" + FTPRepo "mirrorFile" "${rewrittenRepoFile}" "${this}" + FTPRepo "mirrorFile" "${rewrittenLatestFile}" "${this}" + ;; + *) err "Unsupported action verb for repo type";; + esac +} +SFTPRepo(){ + this="sftp" + case "$1" in + initialise) _AbstractRemoteRepo "initialise" "${this}" ;; + checkForFile) + curlparams="--list-only" + _AbstractRemoteRepo "checkForFile" "$2" "${this}" + ;; + mirrorFile) _AbstractRemoteRepo "mirrorFile" "$2" "${this}" ;; + generateRepoD) _AbstractRemoteRepo "generateRepoD" "${this}" ;; + rewriteMetadata) + _AbstractRemoteRepo "rewriteMetadata" "${this}" + SFTPRepo "mirrorFile" "${rewrittenRepoFile}" "${this}" + SFTPRepo "mirrorFile" "${rewrittenLatestFile}" "${this}" + ;; + *) err "Unsupported action verb for repo type";; + esac +} +SAMBARepo(){ + this="sftp" + case "$1" in + initialise) _AbstractRemoteRepo "initialise" "${this}" ;; + checkForFile) + curlparams="--list-only" + _AbstractRemoteRepo "checkForFile" "$2" "${this}" + ;; + mirrorFile) _AbstractRemoteRepo "mirrorFile" "$2" "${this}" ;; + generateRepoD) _AbstractRemoteRepo "generateRepoD" "${this}" ;; + rewriteMetadata) + _AbstractRemoteRepo "rewriteMetadata" "${this}" + SAMBARepo "mirrorFile" "${rewrittenRepoFile}" "${this}" + SAMBARepo "mirrorFile" "${rewrittenLatestFile}" "${this}" + ;; + *) err "Unsupported action verb for repo type";; + esac +} + +NEXUSRepo(){ + this="nexus" + case "$1" in + initialise) _AbstractRemoteRepo "initialise" "${this}" ;; + checkForFile) + curlparams="--fail -w \"\\n%{http_code}\" -s -I" + if ${netrc}; then + curlparams=$(printf "%s %s" "${curlparams}" "--netrc") + else + curlparams=$(printf "%s %s" "${curlparams}" "--user ${dstuser}:${dstpassword}") + fi + curlCmd ${curlparams} $target/$2 | grep "HTTP/1.1 200 OK" >/dev/null + ;; + mirrorFile) + file="$2" + #TODO read Nexus config, use other auth? + nexushost=$(echo "${target}" | sed -e "s#^[^/]*//##" -e "s#\([^/]*\).*#\1#") + # Check if Nexus credentials are provided + if ${netrc}; then + verbose "Using netrc credentials so let curl handle this" + else + verbose "Using Basic Auth" + if [ -z "$dstuser" ] || [ -z "$dstpassword" ]; then + error "Basic authentication credentials missing" + fi + fi + file="${tmpdir:=/tmp}/${file}" + if [ ! -e "${file}" ]; then + warning "File '${file}' not found; ignoring upload request" + return + fi + if [ ! -r "${file}" ]; then + warning "No permissions to upload file '${file}'; ignoring upload request" + return + fi + filename=$(basename "${file}") + verbose "Attempting to upload '${file}' to '$target/$filename" + if ! httprc=$(curlCmd -fs -u "${dstuser}:${dstpassword}" --upload-file "$file" "$target/$filename"); then + case ${httprc} in + "403") warning "403: Authentication failure uploading file '${file}'";; + "200") verbose "200: File '$/tmp/avro-c-packaging-master.20240130_092928.zos.pax.Z{file}' uploaded successfully";; + *) warning "${httprc}: Unxpected response code during upload of file '${file}'"; + esac + fi + ;; + generateRepoD) + repoDFile=$(echo "${target}" | sed "s/[\\/:']//g") + repoDFile="${repoDFile}/repod.json" + mkdir -p "$(dirname "${repoDFile}")" + [ -e "${repoDFile}" ] && + mv "${repoDFile}" "${repoDFile}.$(date "+%C%m%d%H%M%S")" + cat <"${repoDFile}" +{ + "type": "nexus", + "metadata_baseurl": "${target}")", + "metadata_file": "$(basename "${rewrittenRepoFile}")", + "latest +} +EOS + echo "${repoDFile}" + ;; + rewriteMetadata) + _AbstractRemoteRepo "rewriteMetadata" "${this}" + NEXUSRepo "mirrorFile" "${rewrittenRepoFile}" "${this}" + NEXUSRepo "mirrorFile" "${rewrittenLatestFile}" "${this}" + ;; + *) err "Unsupported action verb for repo type";; + esac + +} + + +runMirror() { + log "Mirroring packages" + terminal=false + if [ -t 0 ] && [ -t 1 ]; then + terminal=true + fi + + cnt=$(jq --raw-output '.release_data| keys | length' "${JSON_CACHE}") + log "${cnt} " + + if ! initRepo=$(eval '${type}Repo "initialise"'); then + err "Unable to initialise repository mirror: ${initRepo}" + fi + if $terminal; then + # Get terminal width - try tput first, fall back to stty, default to 80. + cols=$(tput cols 2>/dev/null || stty size 2>/dev/null | cut -d' ' -f2 || echo 80) + effectiveCols=$((cols - 6)) + pct=$((cnt / effectiveCols)) + [ "$pct" -eq 0 ] && pct=1 # Ensure pct is at least 1 to avoid division by zero + i=0 + printf "0%%" + fi + + for mirror in $(jq --raw-output '.release_data| keys[]' "${JSON_CACHE}"); do + if "${verbose}"; then + verbose "Mirroring: ${mirror}..." + elif ! "${terminal}"; then + : + else + i=$((i + 1)) + if [ $(( i % pct)) -eq 0 ]; then + printf "." + fi + fi + # Just grab the latest version [offset 0 in array]; + # TODO: determine if mirror providers want to provide a full sync - + # trying to install package versions that haven't been mirrored will result + # in can't be downloaded errors + if getPkgFile "${mirror}" 0; then # Remember to add the port suffix! + if ! getOutput=$(eval '${type}Repo "mirrorFile" "${outfile}" "${mirror}"'); then + err "Errors mirroring '${outfile}': ${getOutput}" + fi + fi + # Need to also get the metadata.json for the file - outfile will get updated + if getPkgMetadata; then + if ! getOutput=$(eval '${type}Repo "mirrorFile" "${outfile}" "${mirror}"'); then + err "Errors mirroring '${outfile}': ${getOutput}" + fi + fi + done + + if $terminal; then + printf " 100%%\n" + else + printf "\n" + fi + + log "Mirroring complete, generating metadata JSON" + eval '${type}Repo "rewriteMetadata"' +} + +generateRepositoryMetadata() { + log "Generate repository information file" + if ! repoFile=$(eval '${type}Repo "generateRepoD"'); then + err "Could not generate repository file. Details: ${repoFile}" + fi +} + +activateRepo() { + repodDir="${zopenrootfs}/etc/zopen/repos.d" + [ ! -d "${repodDir}" ] && mkdir "${repodDir}" + activeSymlink="${repodDir}/active" + [ -L "${activeSymlink}" ] && rm "${activeSymlink}" + if [ ! -e "${repoFile}" ]; then + log "$LOG_ERROR" "Could not find file '${repoFile}' to use as active repository" + exit 8 + fi + repoDName=$(basename "${repoFile}") + if [ -e "${repodDir}/${repoDName}" ]; then + mv "${repodDir}/${repoDName}" "${repodDir}/${repoDName}.$(date "+%C%m%d%H%M%S")" + fi + cp "${repoFile}" "${repodDir}/${repoDName}" + if ! symlink=$(ln -s "${repodDir}/${repoDName}" "${activeSymlink}"); then + log "$LOG_ERROR" "Could not set new repository '${repoFile}' as active." + [ -n "${symlink}" ] && log "$LOG_ERROR" "Details: ${symlink}" + exit 8 + fi +} + +parseArgs() { + while [ $# -gt 0 ]; do + case $1 in + --netrc) + netrc=true + ;; + --ghsrcpat) + [ $# -lt 2 ] && err "Missing value for option '$1'" + ghsrcpat="$2" + shift + ;; + --type) + [ $# -lt 2 ] && err "Missing value for option '$1'" + type="$2" + shift + ;; + --tmpdir) + [ $# -lt 2 ] && err "Missing value for option '$1'" + tmpdir="$2" + shift + ;; + --genRepoFile) + genRepoFile=true + ;; + --activate) + [ $# -lt 2 ] && err "Missing zopen rootfs for option '$1'" + zopenrootfs="$2" + activate=true + shift + ;; + --dstuser) + [ $# -lt 2 ] && err "Missing value for option '$1'" + dstuser="$2" + shift + ;; + --dstpassword) + [ $# -lt 2 ] && err "Missing value for option '$1'" + dstpassword="$2" + shift + ;; + --noghcreds) + noghcreds=true + ;; + --target) + [ $# -lt 2 ] && err "Missing value for option '$1'" + target="$2" + shift + ;; + -? | --?) printHelp false && exit 0 ;; + --help) printHelp true && exit 0 ;; + --verbose) verbose=true ;; + --quiet) quiet=true ;; + --version) displayVersion && exit 0 ;; + *) err "Bad parameter '$1'. " ;; + esac + shift + done +} + +verbose=false +quiet=false +type="" +genRepoFile=true # ?false? +activate=false +zopenrootfs="" +noghcreds=false +ghsrcpat="" +netrc=false +authHeader="" + +export PATH=/bin:$PATH # Always prefer native tools; makes a difference on z/OS +defineANSI +parseArgs "$@" + +log "zopen repository mirror" +if $activate; then + [ -z "${zopenrootfs}" ] && err "Missing zopenrootfs definition for the --activate option." + [ ! -d "${zopenrootfs}/etc/zopen" ] && err "No zopen file system detected at '${zopenrootfs}'. The --activate option updates a zopen file system so must run on z/OS. Correct parameters and retry request." + zopenrootfs=$(cd "${zopenrootfs}" && pwd -P) # use absolute name if possible +fi + +if "${netrc}"; then + verbose "Using .netrc for download authentication" +elif "${noghcreds}"; then + verbose "Skipping GitHub authorization - using anonymous" +else + # TODO: use more secure than cli param for PAT? + [ -z "${ghsrcpat}" ] && err "Github personal access token (PAT) required to prevent rate-limiting" + [ -n "${ghsrcpat}" ] && authHeader="Authorization: token ${ghsrcpat}" +fi + +[ -z "${type}" ] && err "Missing mirror type" +[ -z "${target}" ] && err "Parameter --target required for '${type}' mirror type" + +type=$(echo "${type}" | /bin/awk '{print(toupper($1))}') +verbose "Checking repository type '${type}' for validity" +case "${type}" in +FLAT | ALPHA | NAME | ALPHANAME) + [ ! -d "${target}" ] && mkdir -p "${target}" + target=$(cd "${target}" && pwd -P) # Use absolute name if possible + ;; +NEXUS|FTP|SFTP|SAMBA) + if ! "${netrc}"; then + [ -z "${dstuser}" ] && err "No remote repo username given" + [ -z "${dstpassword}" ] && err "No remote repo password given" + else + verbose "Using .netrc for credentials for upload to remote server" + fi + case "${type}" in + FTP|SFTP|SAMBA) + target="${target#*://}" # Strip any protocol definitions - add back later + DIR_LIST_CACHE="${tmpdir:=/tmp}/zopen_mirror_dir_cache.$$" + [ -f "${DIR_LIST_CACHE}" ] && rm "${DIR_LIST_CACHE}" ;; + *) :;; + esac +;; +*) err "Unsupported mirror type: '${type}'. Check parameter and retry command." ;; +esac + +downloadRepoMetadata +runMirror +if ${genRepoFile}; then + generateRepositoryMetadata +fi +if ${activate}; then + activateRepo +fi +[ -e "${DIR_LIST_CACHE}" ] && echo "rm \"${DIR_LIST_CACHE}\"" + +log "Mirror operation complete" + diff --git a/bin/zopen-promote b/bin/zopen-promote index 0a9f643e4..4ee0720dc 100755 --- a/bin/zopen-promote +++ b/bin/zopen-promote @@ -242,54 +242,64 @@ else mkdir -p "${promotefs}" fi -printInfo "- Promoting from '${rootfs}' to '${promotefs}'..." - -printDebug "Generating temporary pipe" -FIFO_PIPE_STDOUT=$(mktempfile "" ".pipe") - -progressHandler "spinner" "- promote complete" & -ph=$! -killph="kill -HUP ${ph}" -addCleanupTrapCmd "${killph}" -#cp -fRT "${rootfs}/" Fails to keep some relative symlinks correct - -printDebug "Need to pax old dir and unpax to new dest to keep symlink structure" -# Some files are set with permission -r--r--r-- which means they cannot be copied -# over without some work. Rather than search for these files, attempt to copy -# and catch the errors, forcibly re-copying the file over afterwards; saves -# iterating the entire file system twice - -printDebug "Generating temporary pipe" -FIFO_PIPE_STDOUT=$(mktempfile "promotestdout" ".pipe") -[ ! -p "${FIFO_PIPE_STDOUT}" ] || rm -f "${FIFO_PIPE_STDOUT}" -mkfifo "${FIFO_PIPE_STDOUT}" && chtag -tc 819 "${FIFO_PIPE_STDOUT}" && addCleanupTrapCmd "rm -rf ${FIFO_PIPE_STDOUT}" -cd "${rootfs}" && pax -rw -p p "." "${promotefs}" 2>>"${FIFO_PIPE_STDOUT}" & -while read OUTMSG; do - printDebug "Parsing output: '${OUTMSG}'" - destFile=$(echo "${OUTMSG}" | sed 's#.*FSUM7148.*"\(.*\)".*EDC5111I.*#\1#') - if [ "${destFile}" = "${OUTMSG}" ]; then - printSoftError "${OUTMSG}" - printError "Error cloning '${srcFile}' to '${destFile}'. Correct the reported error and retry command" - else - printDebug "Permission fail; trying to force copy of '${srcFile}' to '${destFile}'" - srcFile="${rootfs}${destFile#"${promotefs}"}" - printDebug "Copying sourcefile '${srcFile}' to '${destFile}'" - cp -f "${srcFile}" "${destFile}" +promoteFS(){ + #cp -fRT "${rootfs}/" Fails to keep some relative symlinks correct + + printDebug "Need to pax old dir and unpax to new dest to keep symlink structure" + # Some files are set with permission -r--r--r-- which means they cannot be copied + # over without some work. Rather than search for these files, attempt to copy + # and catch the errors, forcibly re-copying the file over afterwards; saves + # iterating the entire file system twice + + printDebug "Generating temporary pipe" + FIFO_PIPE_STDOUT=$(mktempfile "promotestdout" ".pipe") + [ ! -p "${FIFO_PIPE_STDOUT}" ] || rm -f "${FIFO_PIPE_STDOUT}" + mkfifo "${FIFO_PIPE_STDOUT}" && chtag -tc 819 "${FIFO_PIPE_STDOUT}" + addCleanupTrapCmd "rm -rf ${FIFO_PIPE_STDOUT}" + cd "${rootfs}" && pax -rw -p p "." "${promotefs}" 2>>"${FIFO_PIPE_STDOUT}" & + while read OUTMSG; do + printDebug "Parsing output: '${OUTMSG}'" + destFile=$(echo "${OUTMSG}" | sed 's#.*FSUM7148.*"\(.*\)".*EDC5111I.*#\1#') + if [ "${destFile}" = "${OUTMSG}" ]; then + printSoftError "${OUTMSG}" + printError "Error cloning '${srcFile}' to '${destFile}'. Correct the reported error and retry command" + else + printDebug "Permission fail; trying to force copy of '${srcFile}' to '${destFile}'" + srcFile="${rootfs}${destFile#"${promotefs}"}" + printDebug "Copying sourcefile '${srcFile}' to '${destFile}'" + cp -f "${srcFile}" "${destFile}" + + [ "$?" -ne 0 ] && printError "Could not promote environment; permission issue cloning '${srcFile}' to '${destFile}'. Check permissions and retry command." + fi - [ "$?" -ne 0 ] && printError "Could not promote environment; permission issue cloning '${srcFile}' to '${destFile}'. Check permissions and retry command." - fi + done < "${FIFO_PIPE_STDOUT}" + [ -n "${FIFO_PIPE_STDOUT}" ] && [ -e "${FIFO_PIPE_STDOUT}" ] && rm -f "${FIFO_PIPE_STDOUT}" +} -done < "${FIFO_PIPE_STDOUT}" -[ -n "${FIFO_PIPE_STDOUT}" ] && [ -e "${FIFO_PIPE_STDOUT}" ] && rm -f "${FIFO_PIPE_STDOUT}" -${killph} 2>/dev/null # if the timer is not running, the kill will fail +if ! runLogProgress "promoteFS" "Promoting from '${rootfs}' to '${promotefs}'" "Promoted from '${rootfs}' to '${promotefs}'" "mirror"; then + printError "Unable to promote file system from '${rootfs}' to '${promotefs}'" +fi printVerbose "Grabbing pkginstall location from within promoted env" if [ -e "${promotefs}/etc/zopen/fstype" ]; then zopen_pkginstall=$(cat "${promotefs}/etc/zopen/fstype") else - printError "Unable to locate '${promotefs}/etc/zopen/fstype'; unrecognisable file system. Check permissions" + printInfo "Unable to locate '${promotefs}/etc/zopen/fstype'; unrecognisable file system. Using default." fi +rmDanglingLinks(){ + flecnt=0 + + tmpfile=$(mktempfile "zopen_promote" ".pipe") + addCleanupTrapCmd "rm -rf ${tmpfile}" + + find "${promotefs}" -type l -exec test ! -e {} \; -print > "${tmpfile}" + while IFS= read -r sl; do + printVerbose "Removing symlink '${sl}'" + # rm -f "${sl}" + flecnt=$(expr ${flecnt} + 1) + done < "${tmpfile}" +} if ! ${keepzopentooling}; then printDebug "Stripping zopen tools from cloned environment; removing meta packages" @@ -300,17 +310,9 @@ if ! ${keepzopentooling}; then # As packages can install to any subfolder of the zopen filesystem, need to traverse # along every path under that filesystem - even if '/''. As this is a cloned # environment, can just check for links related to "meta/meta*" - progressHandler "spinner" "- Dangling link removal complete" & - ph=$! - killph="kill -HUP ${ph}" - addCleanupTrapCmd "${killph}" - flecnt=0 - zosfind "${promotefs}" -type l -exec test ! -e {} \; -print | while read sl; do - printVerbose "Removing symlink '${sl}'" - rm -f ${sl} - flecnt=$(expr ${flecnt} + 1) - done - ${killph} 2>/dev/null # if the timer is not running, the kill will fail + if ! runLogProgress "rmDanglingLinks" "Removing dangling links" "Removed dangling links"; then + printError "Unable to remove dangling links. Review any errors. Manual cleanup might be required" + fi syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_ZOPEN}" "PROMOTE" "mainline" "meta tooling checked for and removed from promoted environment; ${flecnt} link(s) removed" fi diff --git a/bin/zopen-query b/bin/zopen-query index 02d595a93..ed7f7c90d 100755 --- a/bin/zopen-query +++ b/bin/zopen-query @@ -15,6 +15,7 @@ setupMyself() echo "Internal Error. Unable to find common.sh file to source." >&2 exit 8 fi + # shellcheck source=/dev/null . "${INCDIR}/common.sh" } setupMyself @@ -23,20 +24,28 @@ filter_implemented=false upgradeable_implemented=false whatprovides_implemented=false -printSyntax() -{ - args=$* - echo "${ME} - a utility for zopen community to query packages and repos." - echo "" - echo "Usage: ${ME} [OPTION] [VERB] [PACKAGE]" - echo " VERB is the action to take, which is one of" - echo " --list, --remote-search, --installed" - echo " PACKAGE is a package, specified for --remote-search" - echo "" - echo "Verbs:" - echo " -i, --installed list installed zopen community." - echo " --list list all available zopen community." - echo " --remote-search regex match package against available zopen community" +printHelp(){ + cat << HELPDOC +zopen query - a utility for zopen community to query packages and repos. + +Usage: zopen query [OPTION] [VERB] [PACKAGE] + +Options: + -d, --details include full details for listings + -i, --installed list only installed pacakages + --list list all available packages + --no-header suppress the header for the output + --no-version suppress version information, return package names + --remote-search + regex match package against available packages + --verbose run in verbose mode + --version print version + -h,-?, --help display this help and exit +HELPDOC +if ${filter_implemented}; then + echo " --filter COLOR apply COLOR (quality) filter." + echo " green: all pass, blue: most pass, yellow: some pass, red: no pass, gray: skipped" +fi if ${upgradeable_implemented}; then echo " --upgradeable list packages where an upgrade is available." fi @@ -91,7 +100,8 @@ printDetailListEntries() if [ ! -z "$1" ] && ! ${noheader}; then printf "${NC}${UNDERLINE}%-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s${NC}\n" "Package" "Installed" "Available" "Latest Tag" "Categories" fi - echo "${repoArray}" | xargs | tr ' ' '\n' | sort | while read repo; do + jqw -r '.release_data | keys[]' "${JSON_CACHE}" |\ + xargs | tr ' ' '\n' | sort | while read repo; do listport=false if [ -z "${needle}" ]; then listport=true @@ -107,8 +117,8 @@ printDetailListEntries() if [ $? -ne 0 ]; then printError "Unable to retrieve remote information" fi - latestVersion=$(/bin/printf "%s" "${latest}" | jq -e -r '.assets[0].name' | sed -E 's/.*-(.*)\.zos\.pax\.Z/\1/') - categories=$(/bin/printf "%s" "${latest}" | jq -e -r '.assets[0].categories') + latestVersion=$(printf "%s" "${latest}" | jq -e -r '.assets[0].name' | sed -E 's/.*-(.*)\.zos\.pax\.Z/\1/') + categories=$(printf "%s" "${latest}" | jq -e -r '.assets[0].categories') if [ -n "${category_filter}" ] && ! echo "${categories}" | grep -q "${category_filter}"; then continue fi @@ -136,7 +146,6 @@ printDetailListEntries() fi done else - printVerbose "Checking repoArray: ${repoArray}" scrcols=$(getScreenCols) numcols=7 colwidth=$((scrcols / numcols - 1)) @@ -144,7 +153,8 @@ printDetailListEntries() if ! ${noheader}; then printf "${NC}${UNDERLINE}%-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s${NC}\n" "Package" "Installed" "Latest Tag" "Download Size" "Expanded Size" "Quality" "Categories" fi - echo "${repoArray}" | xargs | tr ' ' '\n' | sort | while read repo; do + jqw -r '.release_data | keys[]' "${JSON_CACHE}" | \ + xargs | tr ' ' '\n' | sort | while read repo; do listport=false if [ -z "${needle}" ]; then listport=true @@ -170,12 +180,12 @@ printDetailListEntries() ;; 0) printVerbose "Latest release request successful" - latestTag="$(/bin/printf "%s" "${latest}" | jq -e -r '.tag_name')" - passed="$(/bin/printf "%s" "${latest}" | jq -e -r '.assets[0].passed_tests')" - total="$(/bin/printf "%s" "${latest}" | jq -e -r '.assets[0].total_tests')" - expandedsize="$(/bin/printf "%s" "${latest}" | jq -e -r '.assets[0].expanded_size')" - downloadsize="$(/bin/printf "%s" "${latest}" | jq -e -r '.assets[0].size')" - categories=$(/bin/printf "%s" "${latest}" | jq -e -r '.assets[0].categories') + latestTag="$(printf "%s" "${latest}" | jq -e -r '.tag_name')" + passed="$(printf "%s" "${latest}" | jq -e -r '.assets[0].passed_tests')" + total="$(printf "%s" "${latest}" | jq -e -r '.assets[0].total_tests')" + expandedsize="$(printf "%s" "${latest}" | jq -e -r '.assets[0].expanded_size')" + downloadsize="$(printf "%s" "${latest}" | jq -e -r '.assets[0].size')" + categories=$(printf "%s" "${latest}" | jq -e -r '.assets[0].categories') if [ "${onlyUpgradesAvailable}" -gt 0 ]; then if [ "${originalTag}" = "Not installed" -o "${originalTag}" = "${latestTag}" ]; then continue @@ -184,17 +194,17 @@ printDetailListEntries() if [ -n "$total" ] && [ ${total} -gt 0 ]; then percentage=$(echo "scale=0; 100 * (${passed}) / ${total}" | bc) fi - /bin/printf "%-${colwidth}s %-${colwidth}s %-${colwidth}s" "${repo}" "${originalTag}" "${latestTag}" - /bin/printf " %-${colwidth}s" "${downloadsize}" - /bin/printf " %-${colwidth}s" "${expandedsize}" + printf "%-${colwidth}s %-${colwidth}s %-${colwidth}s" "${repo}" "${originalTag}" "${latestTag}" + printf " %-${colwidth}s" "${downloadsize}" + printf " %-${colwidth}s" "${expandedsize}" if [ -z "${percentage}" ]; then - /bin/printf "${NC}${RED}%-${colwidth}s${NC}" "No tests" + printf "${NC}${RED}%-${colwidth}s${NC}" "No tests" else - /bin/printf "${NC}$(colorizepct "${percentage}")%-${colwidth}s${NC}" "${percentage}" + printf "${NC}$(colorizepct "${percentage}")%-${colwidth}s${NC}" "${percentage}" fi - /bin/printf " %-${colwidth}s" "${categories}" - /bin/printf "\n" + printf " %-${colwidth}s" "${categories}" + printf "\n" ;; *) printError "Error while trying to retrieve latest repo release" ;; esac @@ -206,96 +216,12 @@ printDetailListEntries() exit 0 } -printInstalledEntries() -{ - needle="${2}" - - scrcols=$(getScreenCols) - [ "${details}" -eq 0 ] && numcols=4 || numcols=6 - colwidth=$((scrcols / numcols - 1)) - printVerbose "Screen width: ${scrcols}; colwidth:${colwidth}" - if [ -n "$1" ] && ! ${noheader}; then - if [ "${details}" -eq 0 ]; then - printf "${NC}${UNDERLINE}%-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s${NC}\n" "Package" "Version" "File" "Releaseline" - else - printf "${NC}${UNDERLINE}%-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s${NC}\n" "Package" "Version" "File" "Releaseline" "Expanded Size" "Quality" - fi - fi - printVerbose "Getting list of symlinks in the package install directory (that point to specific versions)" - installedPackages=$(cd "${ZOPEN_PKGINSTALL}" && zosfind ./*/. ! -name . -prune -type l) - printVerbose "Packages: ${installedPackages}" - echo "${installedPackages}" | xargs | tr ' ' '\n' | sort | while read repo; do - repo="${repo##*/}" - pkghome="${ZOPEN_PKGINSTALL}/${repo}/${repo}" - if [ ! -e "${pkghome}/.active" ]; then - printVerbose "Symlink '${repo}' in '${ZOPEN_PKGINSTALL}' is not active; skipping" - continue - fi - - # Check if the current package matches the needle (if provided) - if [ -n "${needle}" ] && ! echo "${repo}" | grep -q "${needle}"; then - printVerbose "Skipping '${repo}' as it does not match the needle '${needle}'" - continue - fi - - if ${noversion}; then - printf "%s\n" "${repo}" - continue - fi - - if [ -e "${pkghome}/.releaseinfo" ]; then - originalTag=$(cat "${pkghome}/.releaseinfo") - else - originalTag="N/A" - fi - if [ -e "${pkghome}/.version" ]; then - dotversion=$(cat "${pkghome}/.version") - else - dotversion="N/A" - fi - - releaseline="" - if [ -e "${pkghome}/.releaseline" ]; then - releaseline=$(cat "${pkghome}/.releaseline") - fi - if [ -z "${releaseline}" ]; then - releaseline="n/a" - fi - - printVerbose "Original tag: ${originalTag} for repo: ${repo}" - if [ -z "$1" ]; then - printInfo "${originalTag}" - else - fileversion="$(cd "${ZOPEN_PKGINSTALL}/${repo}/${repo}" > /dev/null 2>&1 && pwd -P | xargs basename)" - printf "%-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s" "${repo}" "${dotversion}" "${fileversion}" "${releaseline}" - if [ "${details}" -eq 1 ]; then - # Extra headers: disk size and quality - disksizestr=$(du "${ZOPEN_PKGINSTALL}/${repo}" | tail -n 1) - disksizestr=$(echo "${disksizestr}" | sed 's#\([0-9]*\).*#\1#') - disksize=$((disksizestr * 512)) - printf "%-${colwidth}d" "${disksize}" - - if [ -e "${pkghome}/test.status" ]; then - teststatus=$(cat "${pkghome}/test.status") - percentage=$(echo "${teststatus}" | sed 's/[^-]*-\([^%\.]*\).*/\1/') - printf "${NC}$(colorizepct "${percentage}")%-${colwidth}s${NC}" "${percentage}" - else - printf "${NC}${RED}%-${colwidth}s${NC}" "No tests" - fi - fi - - printf "\n" - fi - done - exit 0 -} - whatProvides() { needle="$1" printVerbose "Finding matches outside of ZOPEN_PKGINSTALL (${ZOPEN_PKGINSTALL})" # Find any symlinks that match the needle and can then be dereferenced - found=$(zosfind "${ZOPEN_ROOTFS}" -name "${ZOPEN_PKGINSTALL}/\*" -prune -o -type l -print | grep "${needle}") + found=$(find "${ZOPEN_ROOTFS}" -name "${ZOPEN_PKGINSTALL}/\*" -prune -o -type l -print | grep "${needle}") printVerbose "Found list: '${found}'" if [ -z "${found}" ]; then printInfo "No package provides '${needle}'" @@ -305,7 +231,7 @@ whatProvides() echo "${found}" | xargs | tr ' ' '\n' | while read foundmatch; do printVerbose "Parsing '${foundmatch}'" if [ ! -d "${foundmatch}" ]; then - dereferenced=$(deref "${foundmatch}") + dereferenced=$(deref_symlink "${foundmatch}") fullpackage=$(echo "${dereferenced}" | sed "s#${ZOPEN_PKGINSTALL}/\([^/]*\).*#\1#") printInfo "Package '${fullpackage}' provides: '${foundmatch}'" @@ -322,6 +248,7 @@ noheader=false noversion=false localoption=true upgradeable=false +list=false unset category_filter details=0 needle= @@ -333,28 +260,24 @@ while [ $# -gt 0 ]; do printVerbose "Parsing option: $1" case "$1" in "--list") - list=1 + list=true localoption=false ;; "-i" | "--installed") localoption=true - installed=1 - list= - if [ -n "$2" ] && [[ "$2" != -* ]]; then - needle="$2" - shift - fi + installed=true + list=true ;; "-wp" | "--whatprovides") localoption=true - whatprovides=1 + whatprovides=true shift [ -n "$1" ] || printError "Missing file argument" needle=$1 ;; "--remote-search") localoption=false - remotesearch=1 + remotesearch=true shift [ -n "$1" ] || printError "Missing package argument" needle=$1 @@ -380,18 +303,21 @@ while [ $# -gt 0 ]; do "-d" | "--details") details=1 ;; - "-h" | "--h" | "-help" | "--help" | "-?" | "-syntax") - printSyntax "${args}" + "-h" | "--help" | "-?") + printHelp exit 0 ;; "--version") zopen-version "${ME}" exit 0 ;; - "-v" | "--v" | "-verbose" | "--verbose") + "--verbose") # shellcheck disable=SC2034 verbose=true ;; + "--xdebug") + set -x + ;; -*) printError "Unknown option '$1'" ;; @@ -421,14 +347,25 @@ fi if ! ${localoption}; then # Retrieve all repositories - getReposFromGithub true # zopen query is a read only operation + updateCaches grfgRc=$? [ 0 -ne "${grfgRc}" ] && exit "${grfgRc}" - repoArray="${repo_results}" fi -! ${upgradeable} || printDetailListEntries "${details}" "" 1 -[ -z "${remotesearch}" ] || printDetailListEntries "${details}" "${needle}" 0 -[ -z "${list}" ] || printDetailListEntries "${details}" "" 0 -[ -z "${installed}" ] || printInstalledEntries "${details}" "${needle}" 0 -[ -z "${whatprovides}" ] || whatProvides "${needle}" +# Use dedicated zopen-list script for listing system packages, passing +# appropriate parameters to mimic the old query behaviour +if ${list}; then + if ${installed}; then + if [ "${details}" -eq 0 ]; then + (zopen list --installed) + else + (zopen list --installed --details --header) + fi + else + (zopen list ) + fi +else + ! ${upgradeable} || printDetailListEntries "${details}" "" 1 + ! ${remotesearch} || printDetailListEntries "${details}" "${needle}" 0 + ! ${whatprovides} || whatProvides "${needle}" +fi \ No newline at end of file diff --git a/bin/zopen-remove b/bin/zopen-remove index d61143200..c42a076d4 100755 --- a/bin/zopen-remove +++ b/bin/zopen-remove @@ -15,6 +15,7 @@ setupMyself() echo "Internal Error. Unable to find common.sh file to source." >&2 exit 8 fi + # shellcheck disable=SC1091 . "${INCDIR}/common.sh" } setupMyself @@ -51,27 +52,36 @@ HELPDOC removePackages() { pkglist=$* - + processActionScripts "transactionPre" + # Create a temp file name to trigger the post-transaction action script + # triggering - run only if an action was taken (which creates the file) + runpostTxnActionScripts=$(mktempfile "txnpost" "run") echo "${pkglist}" | xargs | tr ' ' '\n' | sort | while read pkg; do printHeader "Removing package: ${pkg}" - printInfo "- Checking status of package '${pkg}'" - if [ ! -f ${ZOPEN_PKGINSTALL}/${pkg}/${pkg}/.active ]; then - printInfo "${NC}${YELLOW}Package '${pkg}' is not installed${NC}" + processActionScripts "removePre" + printInfo "- Querying install status of package '${pkg}'" + # Check for the actual file rather than the database; if the db is + # corrupt, the file is the tell-tale + if [ ! -f "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}/.active" ]; then + printInfo "${NC}${YELLOW}- Package '${pkg}' is not installed${NC}" else - printInfo "- Away to remove '${pkg}'," - if ! ${yesToPrompts}; then - while true; do - printInfo "- Do you want to continue? [y/n/a]" - read continueInstall < /dev/tty - case "${continueInstall}" in - "y") break;; - "n") mutexFree "zopen"; printInfo "Exiting..."; exit 0 ;; - "a") yesToPrompts=true; break;; - *) echo "?";; - esac - done + + if preReqList=$(checkIfPrereq "${pkg}"); then + printWarning "${pkg} is a runtime prerequiste of the following:\n$(echo "${preReqList}" | sed '$d')" + printWarning "Removing ${pkg} could result in broken packages" + if ! promptYesNoAlways "Continue removal of '${pkg}' ?" false; then + mutexFree "zopen"; + printInfo "Exiting..."; + exit 0 + fi + else + if ! promptYesNoAlways "Confirm removal of '${pkg}'?" ${yesToPrompts}; then + mutexFree "zopen"; + printInfo "Exiting..."; + exit 0 + fi fi - printInfo "- Unmeshing from system" + printInfo "- Unmeshing '${pkg}' from system" version="unknown" if [ -e "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}/.releaseinfo" ]; then version=$(cat "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}/.releaseinfo") @@ -83,27 +93,34 @@ removePackages() if ${purge}; then printInfo "- Purging package" printVerbose "Checking if we are currently in a directory that is to be purged" - [ "${PWD##"${ZOPEN_PKGINSTALL}"/"${pkg}"/"${pkg}"}" != "${PWD}" ] && cd ${ZOPEN_PKGINSTALL} - rm -rf $(cd "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}" && pwd -P) - syslog ${ZOPEN_LOG_PATH}/audit.log ${LOG_A} "${CAT_PACKAGE},${CAT_REMOVE}" "REMOVE" "removePackage" "Purging package:'${needle};version:${version};" + [ "${PWD##"${ZOPEN_PKGINSTALL}"/"${pkg}"/"${pkg}"}" != "${PWD}" ] && cd "${ZOPEN_PKGINSTALL}" + rm -rf "$(cd "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}" && pwd -P)" + syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE},${CAT_REMOVE}" "REMOVE" "removePackage" "Purging package:'${needle};version:${version};" registerRemove "${pkg}" "${version}" else - printInfo "- Removing metadata file to mark uninstall" + printVerbose "- Removing metadata file to mark uninstall" rm -f "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}/.active" - printInfo "- Breaking link from current to versioned" + printVerbose "- Breaking link from current to versioned" rm -f "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}" fi printVerbose "Main symlink removed, removing dangling symlinks" unsymlinkFromSystem "${pkg}" "${ZOPEN_ROOTFS}" "${installedlinksfile}" fi - printInfo "- Removing profiled entry" + printVerbose "- Removing profiled entry" [ -d "${ZOPEN_ROOTFS}/etc/profiled/${pkg}" ] && rm -rf "${ZOPEN_ROOTFS}/etc/profiled/${pkg}" + removeFromInstallTracker "${pkg}" + processActionScripts "removePost" syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE},${CAT_REMOVE}" "REMOVE" "removePackage" "Removed package:'${needle};version:${version};" + touch "${runpostTxnActionScripts}" printInfo "${NC}${GREEN}Successfully removed: ${pkg}${NC}" fi done + if [ -e "${runpostTxnActionScripts}" ]; then + processActionScripts "transactionPost" "mandb"; + rm "${runpostTxnActionScripts}" + fi } # Main code start here diff --git a/bin/zopen-upgrade b/bin/zopen-upgrade new file mode 100755 index 000000000..89536f9ed --- /dev/null +++ b/bin/zopen-upgrade @@ -0,0 +1,175 @@ +#!/bin/sh +# Upgrade utility for zopen community- https://github.com/zopencommunity +# +# All zopen-* scripts MUST start with this code to maintain consistency +# +setupMyself() +{ + ME=$(basename $0) + MYDIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 && pwd -P )" + INCDIR="${MYDIR}/../include" + if ! [ -d "${INCDIR}" ] && ! [ -f "${INCDIR}/common.sh" ]; then + echo "Internal Error. Unable to find common.sh file to source" >&2 + exit 8 + fi + # shellcheck disable=SC1091 + . "${INCDIR}/common.sh" +} +setupMyself +checkWritable + +printHelp(){ + cat << HELPDOC +zopen upgrade is a utility to upgrade installed packages to a later release + +Usage: zopen upgrade [OPTION] [PARAMETERS] [PACKAGES] + +Options: + -y, --yes automatically answer yes to prompts + -v, --verbose run in verbose mode + -h,-?, --help display this help and exit + +Examples: + zopen upgrade foo + upgrade package foo if installed + zopen upgrade -y upgrade all packages to latest version on their + releaseline + +Report bugs at https://github.com/zopencommunity/meta/issues + +HELPDOC +} + + + +# Main code start here +# Need to set a number of variables for use in the install function +# which is common between install & upgrade +args=$* +verbose=false +debug=false +xdebug=false +# shellcheck disable=SC2034 +selectVersion=false +# shellcheck disable=SC2034 +setActive=true +# shellcheck disable=SC2034 +downloadOnly=false +# shellcheck disable=SC2034 +reinstall=false +# shellcheck disable=SC2034 +reinstallDeps=false +# shellcheck disable=SC2034 +nosymlink=false +# shellcheck disable=SC2034 +doNotInstallDeps=true +# shellcheck disable=SC2034 +bypassPrereqs=false + +yesToPrompts=false +chosenRepos="" +while [ $# -gt 0 ]; do + case "$1" in + "--bypass-prereq-checks") + # shellcheck disable=SC2034 + bypassPrereqs=true + ;; + "--yes" | "-y") + # shellcheck disable=SC2034 + yesToPrompts=true # Automatically answer 'yes' to any questions + ;; + "-h" | "--help" | "-?") + printHelp "${args}" + exit 0 + ;; + "-v" | "--verbose") + verbose=true + ;; + "--debug") + verbose=true + debug=true + ;; + "--xdebug") + # shellcheck disable=SC2034 + verbose=true + # shellcheck disable=SC2034 + debug=true + xdebug=true + ;; + "--version") + zopen-version "${ME}" + exit 0 + ;; + -*) printError "Unsupported parameter '$1'" ;; + *) + chosenRepos=" ${chosenRepos} $1 "; + ;; + esac + shift; +done + +${xdebug} && set -x && printVerbose "Enabled command execution trace" + +checkIfConfigLoaded + +export SSL_CERT_FILE="${ZOPEN_CA}" +export GIT_SSL_CAINFO="${ZOPEN_CA}" +export CURL_CA_BUNDLE="${ZOPEN_CA}" + +printDebug "Installing to zopen file system: ${ZOPEN_ROOTFS}" +if [ -z "${ZOPEN_ROOTFS}" ]; then + printError "Unable to locate zopen file system, \${ZOPEN_ROOTFS} is undefined" +fi +downloadDir="${ZOPEN_ROOTFS}/var/cache/zopen" + + +if [ ! -d "${downloadDir}" ]; then + if ! mkdir -p "${downloadDir}"; then + printError "Could not create download directory: ${downloadDir}" + fi +fi + +# Parse passed in repositories and check if valid zopen framework repos +printVerbose "Querying remote repo for latest package information" +updateCaches +grfgRc=$? +[ 0 -ne ${grfgRc} ] && exit ${grfgRc}; +# shellcheck disable=SC2034 +installArray="" + +mutexReq "zopen" "zopen" +printDebug "Parsing list of packages to install and verifying validity" + +if [ -z "${chosenRepos}" ]; then + if ! chosenRepos=$(zopen list --installed); then + printSoftError "Unable to retrieve list of currently installed packages" + printError "Details: ${chosenRepos}" + fi +else + validatePackageList "${chosenRepos}" +fi + +if ! generateInstallGraph "${chosenRepos}"; then + printError "Unable to generate upgrade graph" +fi + +# shellcheck disable=SC2154 +upgrades=$(echo "${installList}" | jq --raw-output '.installqueue| length') +if [ 0 -eq "${upgrades}" ]; then + printInfo "- No available updates" +else + pkgcount=$(echo "${installList}" | jq --raw-output '.installqueue | sort| length ') + if [ "$pkgcount" -gt 1 ]; then + pkgInsert="${pkgcount} packages" + else + pkgInsert="package" + fi + printInfo "- The following ${pkgInsert} will be upgraded:" + echo "${installList}" | jq --raw-output '.installqueue | sort| .[] | .portname ' + if promptYesNoAlways "Continue upgrading ${pkgInsert}?" ${yesToPrompts}; then + processRepoInstallFile + fi + printInfo "Upgrade operation complete" +fi +mutexFree "zopen" + diff --git a/bin/zopen-usage b/bin/zopen-usage index 73263e07b..a03faa72a 100755 --- a/bin/zopen-usage +++ b/bin/zopen-usage @@ -80,8 +80,8 @@ render_pie_chart_awk(){ center_x=$((radius + 1)) center_y=$radius aspect="0.6" # Reasonable guess? - - if ! pie=$(/bin/awk -v FS="|" \ + + if ! pie=$(awk -v FS="|" \ -v title="${title}" \ -v legend_title="${legend_title}" \ -v radius="${radius}" \ @@ -102,10 +102,10 @@ render_pie_chart_awk(){ for (theta = current_angle; theta <= end_angle; theta++) { rad = theta * a2r; # Calculate Cartesian coordinates - sohcahtoa... - x = round(cx + r * cos(rad)); + x = round(cx + r * cos(rad)); y = round(cy + r * sin(rad)*aspect); # Bounds check - + if (x >= 0 && x <= (width+1) && y >= 0 && y < height) { if (use_ansi == "true") { chart[x,y] = ansi_csi "[" color "m" color_symbol ansi_csi "[0m"; @@ -165,7 +165,7 @@ render_pie_chart_awk(){ end_angle = int(current_angle + angle_size); if (use_ansi == "true") { modidx=idx % colors_size + 1 # Note % to wrap?? - color = colors_array[modidx] ; + color = colors_array[modidx] ; if (NR > colors_size) { # We cannot chart this, so add it to the "other" category color="31" @@ -200,7 +200,7 @@ render_pie_chart_awk(){ legend[31] = sprintf("%7s: %11sk (%4.1f%%): %25s", ansi_csi "[31m" color_symbol ansi_csi "[0m", (datasize - pie_size)/1024, ((datasize - pie_size) / datasize*100), "Other assorted") } else { legend["?"] = sprintf("%7s: %11sk (%4.1f%%): %25s", "?", (datasize - pie_size)/1024, ((datasize - pie_size) / datasize*100), "Other assorted") - } + } } # Set the centre character to a generic, unused value chart[cx,cy] = "O"; @@ -224,12 +224,12 @@ render_pie_chart_awk(){ for (i = 1; i <= n; i++) { print legendrev[i] } - + }' "${datafile}" ); then printError "Could not generate pie chart: ${pie}" fi # shellcheck disable=SC2059 - /bin/printf "${pie}\n" + printf "${pie}\n" return 0 } @@ -251,7 +251,7 @@ list_usage(){ percentage=$(echo "scale=2; 100 * ${other_size} / ${datasize}" | bc) printf "%10sK %-10s: Other\n" "${other_size}" "(${percentage}%)" fi - zopen_env_size=$(zosdu -kts "${zopen_rootfs}" 2>/dev/null | awk '{printf "%.2f", ($1 / 1024)}') + zopen_env_size=$(du -kts "${zopen_rootfs}" 2>/dev/null | awk '{printf "%.2f", ($1 / 1024)}') available=$(df -k "${zopen_rootfs}" | tail -n 1 | tr -s " " | cut -d' ' -f3) printf " The zopen environment is using %sMb of total storage\n" "${zopen_env_size}" echo "${available}" | awk -F'/' '{ printf " There is %.2f Mb available for the zopen environment\n" , ($1 / 1024) }' @@ -260,7 +260,7 @@ list_usage(){ add_dir_size(){ dir=$1 [ -e "${dir}" ] || return 1 - size=$(zosdu -kts "${dir}" 2>/dev/null | awk '{print $1}') + size=$(du -kts "${dir}" 2>/dev/null | awk '{print $1}') # Do size first as it gets sorted on echo "${size}|${dir}" >> "${datafile}" echo "${size}" @@ -276,7 +276,7 @@ create_datafile() { printError "Specified zopen environment '${zopen_rootfs}' does not appear to be valid" datafile=$(mktempfile "pie" ".data") - zopen_env_size=$(zosdu -kts "${zopen_rootfs}" 2>/dev/null | awk '{print $1}') + zopen_env_size=$(du -kts "${zopen_rootfs}" 2>/dev/null | awk '{print $1}') # Get explicit directories to include for dir in "${zopen_rootfs}"/usr/local/zopen/*; do # Get space usage of the directory (in format "size dir") diff --git a/bin/zopen-whichproject b/bin/zopen-whichproject index 6745a409b..acb353bbb 100755 --- a/bin/zopen-whichproject +++ b/bin/zopen-whichproject @@ -67,32 +67,18 @@ if ! command -v jq > /dev/null 2>&1; then exit 1 fi -JSON_FILES_URL="https://raw.githubusercontent.com/zopencommunity/meta/main/docs/api/zopen_files.json" -JSON_FILES_URL="https://raw.githubusercontent.com/zopencommunity/meta/e859e89e75fab31c1d5b4764b9ee9f511c52b682/docs/api/zopen_files.json" - -downloadJsonCaches() -{ - cachedir="${ZOPEN_ROOTFS}/var/cache/zopen" - [ ! -e "${cachedir}" ] && mkdir -p "${cachedir}" - JSON_FILES_CACHE="${cachedir}/zopen_files.json" - - if ! curlout=$(curlCmd -L --fail --no-progress-meter -o "${JSON_FILES_CACHE}" "${JSON_FILES_URL}"); then - printError "Failed to obtain vulnerability json from ${JSON_FILES_URL}; ${curlout}" - fi - chtag -tc 819 "${JSON_FILES_CACHE}" -} - -downloadJsonCaches +updateCaches "JSON_FILES_CACHE" # Perform the search using jq printVerbose "Searching for '${query_item}' in ${JSON_FILES_CACHE}..." # The jq query to find the project key(s) containing the item +# shellcheck disable=SC2016 jq_query='.project_files | to_entries[] | select(.value.binaries[]? == $item or .value.libs[]? == $item) | .key' # Execute jq, passing the query item as an argument # Handle potential errors from jq (e.g., file not found, invalid JSON) -matching_projects=$(jq -r --arg item "$query_item" "${jq_query}" "${JSON_FILES_CACHE}" 2> /dev/null | /bin/sort -u) +matching_projects=$(jq -r --arg item "$query_item" "${jq_query}" "${JSON_FILES_CACHE}" 2> /dev/null | sort -u) jq_exit_status=$? if [ ${jq_exit_status} -ne 0 ] && [ ${jq_exit_status} -ne 4 ]; then # jq exits 4 if no results found, others are errors diff --git a/include/common.sh b/include/common.sh index 1697170dc..e5b1b6b9d 100755 --- a/include/common.sh +++ b/include/common.sh @@ -5,19 +5,27 @@ zopenInitialize() { + # Hardcode the z/OS-supplied tooling to ensure zopen only uses standard + # tools + export PATH=/bin:"$PATH" + # Capture start time before setting trap fullProcessStartTime=${SECONDS} - + + # Monitor the intial parent process; as subshells inherit + # variables, this will not be overridden due to param expansion + parentPid=${parentPid:-$$} + # Create the cleanup pipeline and exit handler trap "cleanupFunction" EXIT INT TERM QUIT HUP - + # Temporary files for zopen_tmp_dir in "${TMPDIR}" "${TMP}" /tmp; do - if [ ! -z ${zopen_tmp_dir} ] && [ -d ${zopen_tmp_dir} ]; then + if [ ! -z "${zopen_tmp_dir}" ] && [ -d "${zopen_tmp_dir}" ]; then break fi done - + if [ ! -d "${zopen_tmp_dir}" ]; then printError "Temporary directory not found. Please specify \$TMPDIR, \$TMP or have a valid /tmp directory." fi @@ -27,19 +35,71 @@ zopenInitialize() if [ -z "${ZOPEN_DONT_PROCESS_CONFIG}" ]; then processConfig fi + + ZOPEN_ORGNAME="zopencommunity" + # shellcheck disable=SC2034 + ZOPEN_GITHUB="https://github.com/${ZOPEN_ORGNAME}" + # shellcheck disable=SC2034 ZOPEN_ANALYTICS_JSON="${ZOPEN_ROOTFS}/var/lib/zopen/analytics.json" - ZOPEN_JSON_CACHE_URL="https://raw.githubusercontent.com/zopencommunity/meta/main/docs/api/zopen_releases.json" + # shellcheck disable=SC2034 + ZOPEN_JSON_CACHE_URL="https://raw.githubusercontent.com/${ZOPEN_ORGNAME}/meta/main/docs/api/zopen_releases.json" + # shellcheck disable=SC2034 + ZOPEN_LATEST_RELEASE_JSON="https://raw.githubusercontent.com/${ZOPEN_ORGNAME}/meta/main/docs/api/zopen_releases_latest.json" + # shellcheck disable=SC2034 + JSON_FILES_CACHE_URL="https://raw.githubusercontent.com/zopencommunity/meta/main/docs/api/zopen_files.json" ZOPEN_JSON_CONFIG="${ZOPEN_ROOTFS}/etc/zopen/config.json" + ZOPEN_CACHEDURL_DIR="${ZOPEN_ROOTFS}/var/cache/zopen" + ZOPEN_REPOS_DIR="${ZOPEN_ROOTFS}/etc/zopen/repos.d" + + if [ -n "${INCDIR}" ]; then - ZOPEN_SYSTEM_PREREQ_SCRIPT="${INCDIR}/prereq.sh" + ZOPEN_SCRIPTLET_DIR="${INCDIR}/scriptlets" else - ZOPEN_SYSTEM_PREREQ_SCRIPT="${ZOPEN_ROOTFS}/usr/local/zopen/meta/meta/include/prereq.sh" + ZOPEN_SCRIPTLET_DIR="${ZOPEN_ROOTFS}/usr/local/zopen/meta/meta/include/scriptlets" fi -} +} addCleanupTrapCmd(){ - newcmd=$1 - [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" + # Attempt to remove any redirects; rather than test, simpler to remove + # and re-add if present. + newcmd="$1" + # shellcheck disable=SC2009 # no pgrep. + #mypid=$(ps -ef |grep -v grep |grep $0 | tr -s ' ' | cut -f3 -d ' ') + mypid=$(exec sh -c 'echo ${PPID}') + tmpscriptfile="${zopen_tmp_dir}/zopen_trap.${mypid}.scr" + echo "${newcmd}" >> "${tmpscriptfile}" + + if [ $$ -eq "${parentPid}" ]; then + # Re-register handlers if already done; quicker than testing presensce + for trappedSignal in "EXIT" "INT" "TERM" "QUIT" "HUP"; do + trap cleanup "${trappedSignal}" + done + fi +} + +cleanup() { + ogExitCode=$? # Save the original exit code + # shellcheck disable=SC2009 # no pgrep. + #mypid=$(ps -ef |grep -v grep |grep ps | tr -s ' ' | cut -f3 -d ' ') + mypid=$(exec sh -c 'echo ${PPID}') + tmpscriptfile="${zopen_tmp_dir}/zopen_trap.${mypid}.scr" + if [ -f "${tmpscriptfile}" ] && [ -s "${tmpscriptfile}" ]; then + # Execute the commands in the cleanup file by sourcing it + # shellcheck disable=SC1090 + . "${tmpscriptfile}" > /dev/null 2>&1 + rm "${tmpscriptfile}" + fi + return ${ogExitCode} +} + +addCleanupTrapCmd2(){ + # Attempt to remove any redirects; rather than test, simpler to remove + # and re-add if present. + newcmd=$(echo "$cmd" | sed -E "s/[ \t]*[1-9]*[<>][>|&]?[ \t]*[^ \t]*//g") + newcmd="${newcmd} >/dev/null 2>&1" + # Command Trace MUST be disabled as the output from this can become + # interleaved with output when calling zopen sub-processes. + # Small timing window if the script is killed between the creation # and removal of the temporary file; would be easier if zos sh # didn't have a bug -trap can't be piped/redirected anywhere except @@ -47,20 +107,44 @@ addCleanupTrapCmd(){ # and run in the subshell before returning trap handler(s)!?! tmpscriptfile="${zopen_tmp_dir}/clean.tmp" trap > "${tmpscriptfile}" 2>&1 && script=$(cat "${tmpscriptfile}") - rm "${tmpscriptfile}" if [ -n "${script}" ]; then - for trappedSignal in "EXIT" "INT" "TERM" "QUIT" "HUP"; do - newtrapcmd=$(echo "${script}" | while read trapcmd; do - sigcmd=$(echo "${trapcmd}" | zossed "s/trap -- \"\(.*\)\" ${trappedSignal}.*/\1/") - [ "${sigcmd}" = "${trapcmd}" ] && continue - printf "%s;%s 2>/dev/null" "${sigcmd}" "${newcmd}" | tr -s ';' - break + for trappedSignal in "EXIT" "INT" "TERM" "QUIT" "HUP"; do + newtrapcmd=$( + echo "${script}" | while read trapcmd; do + sigcmd=$(echo "${trapcmd}" | + sed "s/trap -- \"\(.*\)\" ${trappedSignal}.*/\1/") + # No match/replace in sed, then sigcmd remains unchanged + [ "${sigcmd}" = "${trapcmd}" ] && continue + # There was a match, so sigcmd contains the string of commands + # to run for the trap. Need to remove the exit command (which + # returns the exit code) and the initial set of the exit code + # and ensure it is last. Note that eval is used to run commands + # with potentially embedded variables (like the return code + # capture/return) + suffix="eval exit \$exitrc" + prfx="exitrc=\$?" + sigcmd=$(echo "${sigcmd}" |awk -v prfx="$prfx" -v suffix="$suffix" ' + BEGIN{ FS=";"; OFS=";" sep=""} + { + for (i=1; i<=NF; i++){ + if (substr($i, 1, length(prfx)) != prfx && + substr($i, 1, length(suffix)) != suffix) { + printf "%s%s >/dev/null 2>&1", sep, $i + sep = OFS + } + } + } + END{ printf "\n" }' + ) + printf "%s;%s;%s;eval exit \${exitrc}" "${prfx}" "${newcmd}" "${sigcmd}" | tr -s ';' + break done - ) - if [ -n "${newtrapcmd}" ]; then - trap -- "${newtrapcmd}" "${trappedSignal}" - fi - done + ) + if [ -n "${newtrapcmd}" ]; then + trap -- "${newtrapcmd}" "${trappedSignal}" + fi + done + rm "${tmpscriptfile}" 2>/dev/null fi [ -n "${xtrc}" ] && set -x } @@ -69,11 +153,11 @@ addCleanupTrapCmd(){ cleanupOnExit() { rv=$? - [ -f ${ZOPEN_TEMP_C_FILE} ] && rm -rf ${ZOPEN_TEMP_C_FILE} - [ -p ${TMP_FIFO_PIPE} ] && rm -rf ${TMP_FIFO_PIPE} + [ -f "${ZOPEN_TEMP_C_FILE}" ] && rm -rf "${ZOPEN_TEMP_C_FILE}" + [ -p "${TMP_FIFO_PIPE}" ] && rm -rf "${TMP_FIFO_PIPE}" if [ ! -z "${TEE_PID}" ]; then - if kill -0 ${TEE_PID} 2> /dev/null; then - kill -9 ${TEE_PID} + if kill -0 "${TEE_PID}" 2> /dev/null; then + kill -9 "${TEE_PID}" fi fi [ -e "${TMP_GPG_DIR}" ] && rm -rf "${TMP_GPG_DIR}" @@ -89,17 +173,29 @@ cleanupFunction() : } + +# showConfigParmWarning +# writes warning messages when a bad config parameter is found +# input: $1 - name of the paramter +# $2 - current value of the parameter +# $3 - valid value +# $4 - default that will be uesd +showConfigParmWarning(){ + printWarning "Found invalid value '$2' for $1 configuration parameter [should be $3]. Defaulting to '$4'" + if type zopen-config-helper >/dev/null 2>&1; then + printWarning "Run 'zopen config --set $1 [$3]' to update configuration if required." + else + printWarning "Update zopen configuration file with a valid value [$3] for parameter '$1' if required." + fi +} + # getParentProcess # returns the parent process for the specified process -# input: $1 - the pid to get the parent of -# return: 0 for error or parent process id getParentProcess() { - parent=$(ps -o ppid= -p "$1") - if parent=$(ps -o ppid= -p "$1"); then - return 0 - fi - return "${parent}" + [ -z "$1" ] && assertFailed "No process name given" + # Get Parent Pid (ppid), with no heading (='') and strip blanks (awk) + ps -o ppid= -p "$1" | awk '{$1=$1; print}' } # getCurrentVersionDir @@ -114,8 +210,10 @@ getCurrentVersionDir(){ [ ! -L "${ZOPEN_PKGINSTALL}/${needle}/${needle}" ] \ && echo ""\ && return 4 - cd "${ZOPEN_PKGINSTALL}/${needle}/${needle}" 2> /dev/null \ + ( # Do not modify current environment! + cd "${ZOPEN_PKGINSTALL}/${needle}/${needle}" 2> /dev/null \ && pwd -P 2> /dev/null + ) } # isPackageActive @@ -129,19 +227,19 @@ isPackageActive(){ getCurrentVersionDir "$needle" } -# Given two input files, return those lines in haystack file that are +# Given two input files, return those lines in haystack file that are # not in needles file diffFile() { haystackfile="$1" needlesfile="$2" [ -n "${needlesfile}" ] || printError "Internal error; needle file was empty/non-existent." - diff=$(awk 'NR==FNR{needles[$0];next} + diff=$(awk 'NR==FNR{needles[$0];next} !($0 in needles) {print}' "${needlesfile}" "${haystackfile}") echo "${diff}" } -# Given two input lists (with \n delimiters), return those lines in +# Given two input lists (with \n delimiters), return those lines in # haystack that are not in needles diffList() { @@ -149,11 +247,12 @@ diffList() needles="$2" haystackfile=$(mktempfile "haystack") echo "${haystack}" >"${haystackfile}" - [ -e "${haystackfile}" ] && addCleanupTrapCmd "rm -rf ${tempdir}" + [ -e "${haystackfile}" ] && addCleanupTrapCmd "rm -rf ${haystackfile}" needlesfile=$(mktempfile "needles") echo "${needles}" >"${needlesfile}" [ -e "${needlesfile}" ] && addCleanupTrapCmd "rm -rf ${needlesfile}" diffFile "${needlesfile}" "${haystackfile}" + rm -rf "${needlesfile}" "${haystackfile}" } # Generate a file name that has a high probability of being unique for @@ -183,13 +282,13 @@ mktempfile() # Create a temporary directory mktempdir() { - tempdir=$(mktempfile "$1") + tempdir=$(mktempfile "$1" "$2") [ ! -e "${tempdir}" ] && mkdir "${tempdir}" && addCleanupTrapCmd "rm -rf ${tempdir}" && echo "${tempdir}" } isPermString() { - test=$(echo "$1" | zossed "s/[-+rwxugo,=]//g") + test=$(echo "$1" | sed "s/[-+rwxugo,=]//g") if [ -n "${test}" ]; then printDebug "Permission string '$1' was invalid" false; @@ -209,14 +308,12 @@ writeConfigFile(){ cat << EOF > "${configFile}" #!/bin/false # Script currently intended to be sourced, not run # zopen community Configuration file -# Main root location for the zopen installation; can be changed if the -# underlying root location is copied/moved elsewhere as locations are -# relative to this envvar value -displayHelp() { + +zot_displayHelp() { echo "usage: . zopen-config [--eknv] [--knv] [--quiet] [-?|--help]" echo " --override-zos-tools Adds altbin/ dir to the PATH and altman/ dir to MANPATH, overriding the native z/OS tooling." echo " --nooverride-zos-tools Does not add altbin/ and altman/ dir to PATH and MANPATH." -echo " --override-zos-tools-subset=" +echo " --override-zos-tools-subset " echo " Override a subset of zos tools. Containing a subset of packages to override, delimited by newlines." echo " --knv Display zopen environment variables " echo " --eknv Display zopen environment variables, prefixed with an" @@ -237,26 +334,46 @@ knv=false exportknv="" displayText=true unset overrideFile -while [ \$# -gt 0 ]; do - case "\$1" in - --eknv) exportknv="export "; knv=true;; - --knv) knv=true;; - --override-zos-tools) export ZOPEN_TOOLSET_OVERRIDE=1;; - --nooverride-zos-tools) unset ZOPEN_TOOLSET_OVERRIDE;; - --override-zos-tools-subset) shift; export ZOPEN_TOOLSET_OVERRIDE=1; overrideFile="\$1";; - --quiet) displayText=false;; - -?|--help) displayHelp; return 0;; - esac - shift + +zot_parse() { + while [ "\$#" -gt 0 ]; do + case \$1 in + --eknv) exportknv="export "; knv=true ;; + --knv) knv=true ;; + --override-zos-tools) export ZOPEN_TOOLSET_OVERRIDE=1;; + --nooverride-zos-tools) unset ZOPEN_TOOLSET_OVERRIDE ;; + --override-zos-tools-subset) + [ "\$#" -ge 2 ] || { printf '%s\n' "error: --override-zos-tools-subset needs a file" >&2; return 2; } + export ZOPEN_TOOLSET_OVERRIDE=1 + overrideFile=\$2; + [ -f "\${overrideFile}" ] || { echo "Error. Override file '\${overrideFile}' is not a file." >&2; return 4; } + [ -r "\${overrideFile}" ] || { echo "Error. Override file '\${overrideFile}' is not readable." >&2; return 4; } + shift 2; continue ;; + --quiet) displayText=false ;; + --) shift; break ;; + -\?|--help) zot_displayHelp; return 0 ;; + --*) printf 'error: unknown option: %s\n' "\$1" >&2; return 2 ;; + *) # positional argument if you support any; collect or ignore + ;; + esac + shift + done + return 0 +} +zot_parse "\$@" + +for local_tmp in "\${TMPDIR}" "\${TMP}" /tmp; do + [ -n "\${local_tmp}" ] && [ -d "\${local_tmp}" ] && break done -if \${knv}; then - /bin/env | /bin/sort > /tmp/zopen-config-env-orig.\$\$ -fi -if [ -n "\${overrideFile}" ] && [ ! -f "\${overrideFile}" ]; then - echo "Override file '\${overrideFile}' is not a file. Skipping..." +if \${knv}; then + original_env="\${local_tmp}/zopen-config-env-orig."\$\$ + /bin/env | /bin/sort > \${original_env} fi +# Main root location for the zopen installation; can be changed if the +# underlying root location is copied/moved elsewhere as locations are +# relative to this envvar value ZOPEN_ROOTFS="${rootfs}" export ZOPEN_ROOTFS @@ -274,20 +391,32 @@ fi zot="zopen community" -sanitizeEnvVar() +zot_sanitizeEnvVar() { # remove any envvar entries that match the specified regex value="\$1" delim="\$2" prefix="\$3" - echo "\${value}" | awk -v RS="\${delim}" -v DLIM="\${delim}" -v PRFX="\${prefix}" '{ if (match(\$1, PRFX)==0) {printf("%s%s",\$1,DLIM)}}' + echo "\${value}" | /bin/awk -v RS="\${delim}" -v DLIM="\${delim}" -v PRFX="\${prefix}" '{ if (match(\$1, PRFX)==0) {printf("%s%s",\$1,DLIM)}}' + } -deleteDuplicateEntries() +zot_deleteDuplicateEntries() { value="\$1" delim="\$2" - echo "\${value}\${delim}" | awk -v RS="\${delim}" '!(\$0 in a) {a[\$0]; printf("%s%s", col, \$0); col=RS; }' | /bin/sed "s/\${delim}$//" + if [ -z "\${value}" ]; then + /bin/echo "" + return + fi + + echo "\${value}\${delim}" | /bin/awk -v RS="\${delim}" -v ORS="\${delim}" ' + BEGIN {col=""} + !(\$0 in a) { + a[\$0]++; + printf("%s%s", col, \$0); + col=ORS; + }' | /bin/sed "s/\${delim}\$//" } # zopen community environment variables @@ -310,39 +439,49 @@ if [ -n "\$SSH_CONNECTION" ] && [ -z "\$PS1" ] || [ ! -t 1 ]; then fi if [ -z "\${ZOPEN_QUICK_LOAD}" ]; then - if [ -e "\${ZOPEN_ROOTFS}/etc/profiled" ]; then - dotenvs=\$(find "\${ZOPEN_ROOTFS}/etc/profiled" -type f -name 'dotenv' -print) + if [ -d "\${ZOPEN_ROOTFS}/etc/profiled" ]; then + if \$displayText; then - /bin/printf "Processing \$zot configuration..." + /bin/printf "Processing %s configuration..." "\$zot" fi - for dotenv in \$dotenvs; do - . \$dotenv - done + tmpfile="\${local_tmp}/zopen_config.tmp" + + if /bin/find "\${ZOPEN_ROOTFS}/etc/profiled" -type f -name 'dotenv' -print > "\${tmpfile}"; then + while IFS= read -r dotenv; do + [ -r "\${dotenv}" ] && . "\${dotenv}" + done < "\${tmpfile}" + else + echo "Error: find command failed" >&2 + [ -e "\${tmpfile}" ] && rm "\${tmpfile}" + return 1 + fi + [ -e "\${tmpfile}" ] && rm "\${tmpfile}" + if \$displayText; then /bin/echo "DONE" if [ -n "\${ZOPEN_TOOLSET_OVERRIDE}" ]; then /bin/echo "NOTE: Conflicting tools (eg. man, cat, grep, make) will take precedence over z/OS /bin tools. Pass the option --nooverride-zos-tools to avoid this." else - /bin/echo "NOTE: Conflicting tools (eg. man, cat, grep, make) will NOT take precedence over z/OS /bin tools; Use the prefixed executables instead (eg. zotman, gcat, ggrep, gmake). Pass the option --override-zos-tools if you prefer zopen tools or --help for further options." + /bin/echo "NOTE: Conflicting tools (eg. man, cat, grep, make) will NOT take precedence over z/OS /bin tools; use the prefixed executables instead (eg. zotman, gcat, ggrep, gmake). Pass the option --override-zos-tools if you prefer zopen tools or --help for further options." fi fi unset dotenvs fi fi unset displayText -PATH=\${ZOPEN_ROOTFS}/usr/local/bin:\${ZOPEN_ROOTFS}/usr/bin:\${ZOPEN_ROOTFS}/bin:\${ZOPEN_ROOTFS}/boot:\$(sanitizeEnvVar "\${PATH}" ":" "^\${ZOPEN_PKGINSTALL}/.*\$") -MANPATH=\${ZOPEN_ROOTFS}/usr/local/share/man:\${ZOPEN_ROOTFS}/usr/local/share/man/\%L:\${ZOPEN_ROOTFS}/usr/share/man:\${ZOPEN_ROOTFS}/usr/share/man/\%L:\$(sanitizeEnvVar "\${MANPATH}" ":" "^\${ZOPEN_PKGINSTALL}/.*\$") +PATH=\${ZOPEN_ROOTFS}/usr/local/bin:\${ZOPEN_ROOTFS}/usr/bin:\${ZOPEN_ROOTFS}/bin:\${ZOPEN_ROOTFS}/boot:\$(zot_sanitizeEnvVar "\${PATH}" ":" "^\${ZOPEN_PKGINSTALL}/.*\$") +MANPATH=\${ZOPEN_ROOTFS}/usr/local/share/man:\${ZOPEN_ROOTFS}/usr/local/share/man/\%L:\${ZOPEN_ROOTFS}/usr/share/man:\${ZOPEN_ROOTFS}/usr/share/man/\%L:\$(zot_sanitizeEnvVar "\${MANPATH}" ":" "^\${ZOPEN_PKGINSTALL}/.*\$") -if [ -n "\$ZOPEN_TOOLSET_OVERRIDE" ]; then +if [ -n "\${ZOPEN_TOOLSET_OVERRIDE}" ]; then if [ -n "\${overrideFile}" ] && [ -f "\${overrideFile}" ]; then - PATH=\$(sanitizeEnvVar "\${PATH}" ":" "^\${ZOPEN_ROOTFS}/usr/local/altbin.*\$") + PATH=\$(zot_sanitizeEnvVar "\${PATH}" ":" "^\${ZOPEN_ROOTFS}/usr/local/altbin.*\$") while IFS= read -r project; do - if [ -d "\$ZOPEN_PKGINSTALL/\$project/\$project/altbin" ]; then - PATH="\$ZOPEN_PKGINSTALL/\$project/\$project/altbin:\$PATH" + if [ -d "\${ZOPEN_PKGINSTALL}/\${project}/\${project}/altbin" ]; then + PATH="\${ZOPEN_PKGINSTALL}/\${project}/\${project}/altbin:\$PATH" fi - if [ -d "\$ZOPEN_PKGINSTALL/\$project/\$project/share/altman" ]; then - MANPATH="\$ZOPEN_PKGINSTALL/\$project/\$project/share/altman:\$MANPATH" + if [ -d "\${ZOPEN_PKGINSTALL}/\${project}/\${project}/share/altman" ]; then + MANPATH="\${ZOPEN_PKGINSTALL}/\${project}/\${project}/share/altman:\$MANPATH" fi done < "\${overrideFile}" else @@ -350,19 +489,20 @@ if [ -n "\$ZOPEN_TOOLSET_OVERRIDE" ]; then MANPATH="\${ZOPEN_ROOTFS}/usr/local/share/altman:\$MANPATH" fi else - PATH=\$(sanitizeEnvVar "\${PATH}" ":" "^\${ZOPEN_ROOTFS}/usr/local/altbin.*\$") - MANPATH=\$(sanitizeEnvVar "\${MANPATH}" ":" "^\${ZOPEN_ROOTFS}/usr/local/share/altman.*\$") + PATH=\$(zot_sanitizeEnvVar "\${PATH}" ":" "^\${ZOPEN_ROOTFS}/usr/local/altbin.*\$") + MANPATH=\$(zot_sanitizeEnvVar "\${MANPATH}" ":" "^\${ZOPEN_ROOTFS}/usr/local/share/altman.*\$") fi -export PATH=\$(deleteDuplicateEntries "\${PATH}" ":") -LIBPATH=\${ZOPEN_ROOTFS}/usr/local/lib:\${ZOPEN_ROOTFS}/usr/lib:\$(sanitizeEnvVar "\${LIBPATH}" ":" "^\${ZOPEN_PKGINSTALL}/.*\$") -export LIBPATH=\$(deleteDuplicateEntries "\${LIBPATH}" ":") -export MANPATH=\$(deleteDuplicateEntries "\${MANPATH}" ":") +export PATH=\$(zot_deleteDuplicateEntries "\${PATH}" ":") +LIBPATH=\${ZOPEN_ROOTFS}/usr/local/lib:\${ZOPEN_ROOTFS}/usr/lib:\$(zot_sanitizeEnvVar "\${LIBPATH}" ":" "^\${ZOPEN_PKGINSTALL}/.*\$") +export LIBPATH=\$(zot_deleteDuplicateEntries "\${LIBPATH}" ":") +export MANPATH=\$(zot_deleteDuplicateEntries "\${MANPATH}" ":") if \${knv}; then - /bin/env | /bin/sort > /tmp/zopen-config-env-modded.\$\$ - diffout=\$(/bin/diff /tmp/zopen-config-env-orig.\$\$ /tmp/zopen-config-env-modded.\$\$ | /bin/grep -E '^[>]' | /bin/cut -c3- ) + modified_env="\${local_tmp}/zopen-config-env-modded."\$\$ + /bin/env | /bin/sort > \${modified_env} + diffout=\$(/bin/diff \${original_env} \${modified_env} | /bin/grep -E '^>' | /bin/cut -c3- ) echo "\${diffout}" | while IFS= read -r knvp; do newval="" envvar="\${knvp%%=*}" @@ -370,29 +510,45 @@ if \${knv}; then IFS=":" for token in \${knvp##*=}; do tok=\$(echo "\${token}" | /bin/sed -e 's#/usr/local/zopen/\([^/]*\)/[^/]*/#/usr/local/zopen/\1/\1/#') - newval=\$(printf "%s:%s" "\${newval}" "\${tok}") + newval=\$(/bin/printf "%s:%s" "\${newval}" "\${tok}") done echo "\${exportknv}\${envvar}=\${newval#*:}" IFS="\${cIFS}" done - rm /tmp/zopen-config-env-orig.\$\$ /tmp/zopen-config-env-modded.\$\$ 2>/dev/null + rm \${original_env} \${modified_env} 2>/dev/null fi - +# Cleanup +unset -f zot_deleteDuplicateEntries 2>/dev/null +unset -f zot_displayHelp 2>/dev/null +unset -f zot_sanitizeEnvVar 2>/dev/null +unset -f zot_parse 2>/dev/null EOF } +# Wrap the curl command with additional standard parameters curlCmd() { # Take the list of parameters and concat them with # any custom parameters the user requires in ZOPEN_CURL_PARAMS - if [ ! -t 1 ] || [ ! -t 2 ]; then - extra_curl_options="--no-progress-meter" - fi + extra_curl_options="--no-progress-meter" curl ${ZOPEN_CURL_PARAMS} ${extra_curl_options} $* } +# Wrap the gpg command with additional standard parameters +gpgCmd() +{ + if ! gpgout=$(gpg --no-secmem-warning $*); then + if echo "${gpgout}" | grep -q "EDC5128I No such device"; then + printVerbose "Ignoring mmap issue not applicable on z/OS currently" + return 0 + fi + echo "${gpgout}" + return 1 + fi +} + validateReleaseLine() { echo "$1" | awk ' @@ -403,22 +559,32 @@ validateReleaseLine() } # Attempt to fully dereference a symlink without any bashisms or arcane set logic -# using some simplistic (recursive!) logic -deref() -{ - testpath="$1" - if [ -L "${testpath}" ]; then - child=$(basename "${testpath}") - symlink=$(ls -l "${testpath}" | zossed 's/.*-> \(.*\)/\1/') - parent=$(dirname "${testpath}") - relpath="${parent}/${symlink}" - relparent=$(dirname "${relpath}") - abspath=$(cd "${relparent}" && pwd -P) - testpath="${abspath}/${child}" - deref "${testpath}" - else - [ -n "${testpath}" ] && echo "${testpath}" && unset testpath - fi +deref_symlink() { + symlink="$1" + if [ -L "${symlink}" ]; then + target=$(ls -l "$symlink" | awk '{print $NF}') + else + target="${symlink}" + fi + targetName=$(basename "${target}") + targetDir=$(dirname "${target}") + # Check if dir starts with a '/'; if not need to calculate absolute + # fully-qualified name + case "$target" in + /*) targetDir=$(cd "${targetDir}" && pwd -P);; + *) + symlink_dir=$(dirname "$symlink") + targetDir=$(cd "$(pwd -P)/${symlink_dir}" && pwd -P) + esac + absolute_dir=$(cd "$targetDir" && pwd -P) # Resolves symlinked dirs + echo "$absolute_dir/$targetName" +} + +toAbsolutePath() { + case "$1" in + /*) echo "$1" ;; # Already absolute + * | . | .. | ./* | ../*) echo "$(cd "$(dirname "$1")" && pwd -P)/$(basename "$1")" ;; + esac } #return 1 if brightness is dark, 0 if light, and 255 if unknown (considered to be dark as default) @@ -438,28 +604,48 @@ darkbackground() { defineANSI() { # Standard tty codes - ESC="\047" + ESC=$(printf "\047") # Start of Escape Sequence; EBCDIC=\047, ASCII=\033 + CSI="[" # Control Sequence Introducer + CNL="E" # Cursor Next Line + CPL="F" # Cursor Previous Line + CHA="G" # Cursor Horizontal Absolute - column selector + EL="K" # Erase In Line + SGR="m" # Select Graphic Rendition + + # shellcheck disable=SC2034 + ERASELINE=$(printf "${ESC}${CSI}2${EL}") # shellcheck disable=SC2034 - ERASELINE="${ESC}[2K" + CRSRHIDE=$(printf "${ESC}${CSI}?25l") # shellcheck disable=SC2034 - CRSRHIDE="${ESC}[?25l" + CRSRSHOW=$(printf "${ESC}${CSI}?25h") + CRSRSOL=$(printf "${ESC}${CSI}0${CHA}") # shellcheck disable=SC2034 - CRSRSHOW="${ESC}[?25h" + CRSRPL=$(printf "${ESC}${CSI}1${CPL}") # Move to start of previous line + # shellcheck disable=SC2034 + CRSRUP="A" # CUU + # shellcheck disable=SC2034 + CRSRDOWN="B" # CUF + # shellcheck disable=SC2034 + CRSRRIGHT="C" # CUB + # shellcheck disable=SC2034 + CRSRLEFT="D" # CUD + + # Color-type codes, needs explicit terminal settings if [ ! "${_BPX_TERMPATH-x}" = "OMVS" ] && [ -z "${NO_COLOR}" ] && [ ! "${FORCE_COLOR-x}" = "0" ] && [ -t 1 ] && [ -t 2 ]; then - esc="\047" - BLACK="${esc}[30m" - RED="${esc}[31m" - GREEN="${esc}[32m" - YELLOW="${esc}[33m" - BLUE="${esc}[34m" - MAGENTA="${esc}[35m" - CYAN="${esc}[36m" - GRAY="${esc}[37m" - BOLD="${esc}[1m" - UNDERLINE="${esc}[4m" - NC="${esc}[0m" + ANSION=true + #ESC="\047" + BLACK=$(printf "${ESC}${CSI}30${SGR}") + RED="${ESC}${CSI}31${SGR}" + GREEN="${ESC}${CSI}32${SGR}" + YELLOW="${ESC}${CSI}33${SGR}" + BLUE="${ESC}${CSI}34${SGR}" + MAGENTA="${ESC}${CSI}35${SGR}" + CYAN="${ESC}${CSI}36${SGR}" + GRAY="${ESC}${CSI}37${SGR}" + BOLD="${ESC}${CSI}1${SGR}" + UNDERLINE="${ESC}${CSI}4${SGR}" darkbackground bg=$? if [ $bg -ne 0 ]; then @@ -471,17 +657,24 @@ defineANSI() HEADERCOLOR="${MAGENTA}" WARNINGCOLOR="${MAGENTA}" fi + # The following should be the last ANSI declaration. With -x trace active, the ANSI + # codes might be interpreted by the terminal when outputing the command trace. Having + # NC as the last value ensures that the text is returned to normal + NC="${ESC}${CSI}0${SGR}" else # unset esc RED GREEN YELLOW BOLD UNDERLINE NC - + ANSION=false esc='' + # shellcheck disable=SC2034 BLACK='' RED='' GREEN='' YELLOW='' BLUE='' MAGENTA='' + # shellcheck disable=SC2034 CYAN='' + # shellcheck disable=SC2034 GRAY='' BOLD='' UNDERLINE='' @@ -496,19 +689,32 @@ ansiline() { deltax=$1 deltay=$2 - echostr=$3 - if [ ${deltax} -gt 0 ]; then - echostr="${ESC}[${deltax}A${echostr}" - elif [ ${deltax} -lt 0 ]; then - echostr="${ESC}[$(expr ${deltax} \* -1)A${echostr}" + echostr="$3" + ansimove "$1" "$2" + echo "${echostr}\c" +} + +ansimove() +{ + deltax=$1 + deltay=$2 + + movestr="" + if [ -n "${deltax}" ]; then + if [ "${deltax}" -gt 0 ]; then + movestr="${ESC}${CSI}${deltax}${CRSRRIGHT}" + elif [ "${deltax}" -lt 0 ]; then + movestr="${ESC}${CSI}$((deltax * -1))${CRSRLEFT}" + fi fi - if [ ${deltay} -gt 0 ]; then - echostr="${ESC}[${deltax}C${echostr}" - elif [ ${deltay} -lt 0 ]; then - echostr="${ESC}[$(expr ${deltax} \* -1)D${echostr}" + if [ -n "${deltay}" ]; then + if [ "${deltay}" -gt 0 ]; then + movestr="${movestr}${ESC}${CSI}${deltay}${CRSRDOWN}" + elif [ "${deltay}" -lt 0 ]; then + movestr="${movestr}${ESC}${CSI}$((deltay * -1))${CRSRUP}" + fi fi - /bin/echo "${echostr}" - + echo "${movestr}\c" } getScreenCols() @@ -516,46 +722,21 @@ getScreenCols() # If stdout/stderr are associated with a tty terminal if [ -t 1 ] && [ -t 2 ]; then # Note tput does not handle ssh sessions too well... - stty | awk -F'[/=;]' '/columns/ { print $4}' | tr -d " " + lclcols=$(stty | awk -F'[/=;]' '/columns/ { print $4}' | tr -d " ") elif [ ! -z "${COLUMNS}" ]; then - echo "${COLUMNS}" - else - echo "$(tput cols)" + lclcols="${COLUMNS}" + elif ! lclcols=$(tput cols 2>/dev/null); then + # tput can fail if the terminal type is unrecognised; use fallback + lclcols=80 fi -} - -zossed() -{ - # Use the standard z/OS sed utility; If the sed package is installed - # GNU sed becomes the dominant version which might change how - # matching is performed - /bin/sed "$@" -} - -zosfind() -{ - # Use the standard z/OS find utility; If the findutils package is installed, - # the installed find command takes precedence but is not compatible with the - # standard zos find [regex searches for "-name" are not allowed, but - # "-wholename" is not available on standard zosfind. For the tooling to be - # consistent across platforms (where findutils is/is not installed) use the - # standard zos version - /bin/find "$@" -} - -zosdu() -{ - # Use the standard z/OS du utility; if the coreutils package is installed, this - # changes the behaviour of the du command so we need to ensure we support those - # who do not have that installed - /bin/du "$@" + echo "${lclcols}" } findrev() { haystack="$1" needle="$2" - while [[ "${haystack}" != "" && "${haystack}" != "/" && "${haystack}" != "./" && ! -e "${haystack}/${needle}" ]]; do + while [ "${haystack}" != "" ] && [ "${haystack}" != "/" ] && [ "${haystack}" != "./" ] && [ ! -e "${haystack}/${needle}" ]; do haystack=${haystack%/*} done echo "${haystack}" @@ -563,7 +744,28 @@ findrev() strtrim() { - echo "$1" | zossed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' + echo "$1" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' +} + +text_center() +{ + if [ $# -lt 2 ]; then + echo "$1" + return + fi + echo "$1" | awk -v strlen="$2" \ + '{spc = strlen - length;padding = int(spc / 2);pad = spc - padding;printf "%*s%s%*s\n", pad, "", $0, padding, ""}' +} + +text_padrightr() { + padchar="${3:- }" # Default to space char + echo "" | awk -v str="$1" -v n="$2" -v pc="$padchar" \ + 'function padr(n,acc, c){ + if (++acc>=n) return c; + return c padr(n, acc, c); + } + BEGIN{ print str padr(n,0,pc);} + ' } defineEnvironment() @@ -586,46 +788,109 @@ defineEnvironment() # Required for proper operation of (USS shipped) sed export _UNIX03=YES - # Use /bin/cat as the pager in case xlclang help is displayed, we don't want to wait for input - export PAGER=/bin/cat + # Use cat as the pager in case xlclang help is displayed, we don't want to wait for input + export PAGER=cat # Set a default umask of read and execute for user and group umask 0022 } # -# For now, explicitly specify /bin/echo to ensure we get the EBCDIC echo since the escape +# For now, explicitly specify echo to ensure we get the EBCDIC echo since the escape # sequences are EBCDIC escape sequences # printColors() { - /bin/echo "$@" + echo "$@" } -mutexReq() -{ - mutex=$1 +# ancestorPid +# Determines if the given pid belongs to the ancestory lineage of current process +# (including self) +# inputs: $1 pid to validate +# return: 0 pid is an ancestor or self +# 1 pid is not in ancestor lineage +ancestorPid() { + target="$1" + + case "$target" in + ''|*[!0-9]*) assertFailed "Bad pid '${target}' given to ancestorPid()" ;; + esac + mypid=$(exec sh -c 'echo ${PPID}') # $$ does not seem reliable!?! + [ "${target}" -eq "${mypid}" ] && return 0 + ap="${PPID:-0}" # Get ancestor pid (parent at current level) + while [ -n "${ap}" ] && [ "${ap}" -gt 1 ] 2>/dev/null; do + [ "$ap" -eq "${target}" ] && return 0 + # Get ancestor pid (parent of $ap) and trim spaces (awk) + if ! ap="$(/bin/ps -o ppid= -p "${ap}" 2>/dev/null | /bin/awk '{print $1}')"; then + # awk should just work, so the failure came from ps, likely from traversing the + # lineage too far so no permission to go further. + return 1 + fi + [ -z "${ap}" ] && break + done + return 1 +} + +# mutexReq +# Finds appropriate metadata for the specified port(s) and +# includes that in the installation file +# inputs: $1 name of mutex to being requested +# return: 0 mutex sucessfully locked +# 1 mutex lock unsuccessful (fast exit) +mutexReq() { + mutex="$1" lockdir="${ZOPEN_ROOTFS}/var/lock" - [ -e lockdir ] || mkdir -p ${lockdir} - mutex="${lockdir}/${mutex}" - mypid=$(exec sh -c 'echo ${PPID}') - mygrandparent=$(/bin/ps -o ppid= -p "$mypid" | awk '{print $1}') - if [ -e "${mutex}" ]; then - lockedpid=$(cat ${mutex}) - { - [ ! "${lockedpid}" = "${mypid}" ] && [ ! "${lockedpid}" = "${PPID}" ] && [ ! "${lockedpid}" = "${mygrandparent}" ] - } && kill -0 "${lockedpid}" 2> /dev/null && echo "Aborting, Active process '${lockedpid}' holds the '$2' lock: '${mutex}'" && exit -1 + [ -d "$lockdir" ] || /bin/mkdir -p "${lockdir}" || printError "Cannot create ${lockdir}" + mutexdir="${lockdir}/${mutex}.lock" + mypid=$(exec sh -c 'echo ${PPID}') # $$ does not seem reliable!?! + + printVerbose "Using mkdir's atomicity as mutex request" + # Network Filesystems might not be so atomic, but smaller window than using a + # locak file + if /bin/mkdir "${mutexdir}" 2>/dev/null; then + printVerbose "Lockdir created - mutex acquired; Adding tracking pid" + umask 077 + /bin/printf '%s\n' "${mypid}" > "${mutexdir}/pid" + addCleanupTrapCmd "rmdir '${mutexdir}'" + return 0 + fi + + printVerbose "mkdir('${mutexdir}') failed; lock already held. Checking details" + lockedpid="$(/bin/cat "${mutexdir}/pid" 2>/dev/null)" + if ! kill -0 "${lockedpid}" 2>/dev/null; then + printVerbose "Detected stale lock '${lockedpid}' [pid not active]. Removing mutex and retrying" + rm "${mutexdir}/pid" || printError "Cannot remove lockpid file'${mutexdir}/pid'" + /bin/rmdir "${mutexdir}" 2>/dev/null || printError "Cannot remove lockfile dir '${mutexdir}'" + /bin/sleep 1 + mutexReq "$mutex" + return $? + fi + + if ancestorPid "${lockedpid}"; then + printVerbose "Ancestor '${lockedpid}' owns mutex; re-entering." + return 0 fi - addCleanupTrapCmd "rm -rf ${mutex}" - echo "${mypid}" > ${mutex} + printError "Aborting, active process '${lockedpid}' holds the '${name}' lock: '${lockpath}'" } -mutexFree() -{ - mutex=$1 +# mutexFree +# Releases the specified mutex; mutexReq checks for stale locks but best to +# clean up! +# inputs: $1 name of mutex to release +# return: 0 +mutexFree() { + name="$1" + mypid=$(exec sh -c 'echo ${PPID}') # $$ does not seem reliable!?! lockdir="${ZOPEN_ROOTFS}/var/lock" - mutex="${lockdir}/${mutex}" - [ -e "${mutex}" ] && rm -f ${mutex} + lockpath="${lockdir}/${name}.lock" + if [ -d "${lockpath}" ]; then + lockedpid="$(/bin/cat "${lockpath}/pid" 2>/dev/null)" + if [ "${lockedpid}" -eq "${mypid}" ] 2>/dev/null || ! kill -0 "${lockedpid}" 2>/dev/null; then + rmdir "${lockpath}" 2>/dev/null || true + fi + fi + return 0 } relativePath2() @@ -658,7 +923,7 @@ relativePath2() # if the target is longer than the source, there might be some additional # elements in the shifted $0 to append if [ $# -gt 0 ]; then - relativePath=${relativePath}/$(echo $* | zossed "s/ /\//g") + relativePath=${relativePath}/$(echo $* | sed "s/ /\//g") fi IFS="${currentIFS}" echo "${relativePath}" @@ -675,11 +940,10 @@ mergeIntoSystem() [ -z "${rebaseusr}" ] && rebaseusr="usr/local" currentDir="${PWD}" - targetdir="${rootfs}/${rebaseusr}" # The main rootfs/usr location printDebug "Calculating the offset path to store from root" offset=$(dirname "${versioneddir#"${rootfs}"/}") - version=$(basename ${versioneddir}) + version=$(basename "${versioneddir}") tmptime=$(date +%Y%m%d%H%M%S) processingDir="${rootfs}/tmp/zopen.${tmptime}" printDebug "Temporary processing dir evaluated to: ${processingDir}" @@ -692,7 +956,7 @@ mergeIntoSystem() mv "${versioneddir}" "${virtualStore}" printDebug "Creating main linked directory in store" - $(cd "${virtualStore}" && ln -s "${version}" "${name}") + cd "${virtualStore}" && ln -s "${version}" "${name}" printDebug "Creating virtual root directory structure" mkdir -p "${processingDir}/${rebaseusr}" @@ -704,19 +968,18 @@ mergeIntoSystem() printDebug "Generating symlink tree" printDebug "Creating directory structure" - curdir="${PWD}" - cd "${virtualStore}/${name}" || exit + cd "${virtualStore}/${name}" || printError "Unable to change to virtual store at '${virtualStore}/${name}'" # since 'ln *' doesn't invoke globbing to allow multiple files at once, # abuse the Recurse option; this results in "already exists" errors but # ignore them as the first call should generate the correct link but # subsequent calls would generate a symlink that has incorrect dereferencing # and ignoring them is actually faster than individually creating the links! - zosfind . -type d | sort -r | while read dir; do - dir=$(echo "${dir}" | zossed "s#^./##") + find . -type d | sort -r | while read dir; do + dir=$(echo "${dir}" | sed "s#^./##") printDebug "Processing dir: ${dir}" - [ ${dir} = "." ] && continue + [ "${dir}" = "." ] && continue mkdir -p "${processingDir}/${rebaseusr}/${dir}" - cd "${processingDir}/${rebaseusr}/${dir}" || exit + cd "${processingDir}/${rebaseusr}/${dir}" || printError "Unable to change to processing directory '${processingDir}/${rebaseusr}/${dir}'" dirrelpath=$(relativePath2 "${virtualStore}/${name}/${dir}" "${processingDir}/${rebaseusr}/${dir}") ln -Rs "${dirrelpath}/" "." 2> /dev/null done @@ -732,26 +995,80 @@ mergeIntoSystem() printDebug "Generating intermediary tar file" # Need '-S' to allow long symlinks - $(cd "${processingDir}" && tar -S -cf "${tarfile}" "usr") + cd "${processingDir}" && tar -S -cf "${tarfile}" "usr" - printDebug "Generating listing for remove processing (including main symlink)." - listing=$(tar tf "${processingDir}/${tarfile}" 2> /dev/null | sort -r) + printDebug "Generating listing for remove processing (including main symlink)" echo "Installed files:" > "${versioneddir}/.links" - echo "${listing}" >> "${versioneddir}/.links" + tar tf "${processingDir}/${tarfile}" 2> /dev/null| sort -r >> "${versioneddir}/.links" - printDebug "Extracting tar to rootfs." + printDebug "Extracting tar to rootfs" cd "${processingDir}" && tar xf "${tarfile}" -C "${rootfs}" 2> /dev/null printDebug "Cleaning temp resources." rm -rf "${processingDir}" 2> /dev/null - printDebug "Switching to previous cwd - current work dir was purged." - cd "${currentDir}" || exit + printDebug "Switching to previous cwd - current work dir was purged" + cd "${currentDir}" || printError "Unable to change to '${currentDir}'" - printInfo "- Integration complete." return 0 } +rmSymlinksFSCheck(){ + # Slower method needed to analyse each link to see if it has + # become orphaned. Only relevent when removing a package as + # upgrades/alt-switching can supply a list of files + # Use sed to skip header line in .links file + # Note that the contents of the links file are ordered such that + # processing occurs depth-first; if, after removing orphaned symlinks, + # a directory is empty, then it can be removed. + printDebug "Creating Temporary dirname file" + tempDirFile=$(mktempfile "unsymlink") + [ -e "${tempDirFile}" ] && rm -f "${tempDirFile}" >/dev/null 2>&1 + touch "${tempDirFile}" + tempTrash=$(mktempfile "unsymlink" "trash") + [ -e "${tempTrash}" ] && rm -f "${tempTrash}" >/dev/null 2>&1 + addCleanupTrapCmd "rm -rf ${tempDirFile}" + addCleanupTrapCmd "rm -rf ${tempTrash}" + printDebug "Using temporary file ${tempDirFile}" + while read filetounlink; do + filetounlink=$(echo "${filetounlink}" | sed 's/\(.*\).symbolic.*/\1/') + filename="$filetounlink" + [ -z "${filetounlink}" ] && continue + filetounlink="${ZOPEN_ROOTFS}/${filetounlink}" + [ ! -e "${filetounlink}" ] && continue # If not there, can'e be removed! + if [ -d "${filetounlink}" ]; then + # Add to the directory queue for checking once files are gone if unique + ispresent=$(grep "^${filetounlink}[ ]*$" "${tempDirFile}") + if [ -z "${ispresent}" ]; then + echo " ${filetounlink} " >> "${tempDirFile}" + fi + elif [ -L "${filetounlink}" ]; then + if [ ! -f "${filetounlink}" ]; then + # the linked-to file no longer exists (ie. the symlink is dangling) + rm -f "${filetounlink}" > /dev/null 2>&1 + fi + else + echo "Unprocessable file: '${filetounlink}'" >> "${tempTrash}" + fi + done < /dev/null 2>&1 + fi + done +} + # The following function will remove any orphaned symlinks left after either: # - a different version has been installed (where the old symlinks are not reused for # that different version version ie. the file has been removed from updated version @@ -763,84 +1080,27 @@ unsymlinkFromSystem() rootfs=$2 dotlinks=$3 newfilelist=$4 - if [ -e "${dotlinks}" ]; then - printInfo "- Checking for obsoleted files in ${rootfs}/usr/ tree from ${pkg}" + if [ -e "${dotlinks}" ]; then if [ -e "${newfilelist}" ]; then - printDebug "Release change, so the list of changes to physically remove should be smaller" - printDebug "Starting spinner..." - progressHandler "spinner" "- Check complete" & - ph=$! - killph="kill -HUP ${ph}" - addCleanupTrapCmd "${killph}" - obsoleteList=$(diffFile "${dotlinks}" "${newfilelist}") - echo "${obsoleteList}" | while read obsoleteFile; do - [ -z "${obsoleteFile}" ] && return 0 - obsoleteFile="${ZOPEN_ROOTFS}/${obsoleteFile}" - obsoleteFile="${obsoleteFile%% symbolic*}" - printDebug "Checking obsoletefile '${obsoleteFile}'" - if [ -L "${obsoleteFile}" ] && [ ! -e "${obsoleteFile}" ]; then - # the linked-to file no longer exists (ie. the symlink is dangling) - rm -f "${obsoleteFile}" > /dev/null 2>&1 - fi - done - ${killph} 2>/dev/null # if the timer is not running, the kill will fail - sleep 1 # give spinner time to exit if running + if ! runLogProgress "rmSymlinksFileDiff" \ + "Checking for file differences in mesh" \ + "Checked for file differences in mesh" "linkcheck"; then + printError "Unable to remove symlinks links. Review any errors. Manual cleanup using zopen-alt might be required" + fi else - # Slower method needed to analyse each link to see if it has - # become orphaned. Only relevent when removing a package as - # upgrades/alt-switching can supply a list of files - # Use sed to skip header line in .links file - # Note that the contents of the links file are ordered such that - # processing occurs depth-first; if, after removing orphaned symlinks, - # a directory is empty, then it can be removed. - nfiles=$(zossed '1d;$d' "${dotlinks}" | wc -l | tr -d ' ') - printDebug "Creating Temporary dirname file" - tempDirFile=$(mktempfile "unsymlink") - [ -e "${tempDirFile}" ] && rm -f "${tempDirFile}" >/dev/null 2>&1 - touch "${tempDirFile}" - tempTrash=$(mktempfile "unsymlink" "trash") - [ -e "${tempTrash}" ] && rm -f "${tempTrash}" >/dev/null 2>&1 - addCleanupTrapCmd "rm -rf ${tempDirFile}" - printDebug "Using temporary file ${tempDirFile}" - printInfo "- Checking ${nfiles} potential links" - printDebug "Starting spinner..." - progressHandler "spinner" "- Complete" & - ph=$! - killph="kill -HUP ${ph}" - addCleanupTrapCmd "${killph}" - - while read filetounlink; do - filetounlink=$(echo "${filetounlink}" | zossed 's/\(.*\).symbolic.*/\1/') - filename="$filetounlink" - [ -z "${filetounlink}" ] && continue - filetounlink="${ZOPEN_ROOTFS}/${filetounlink}" - [ ! -e "${filetounlink}" ] && continue # If not there, can'e be removed! - if [ -d "${filetounlink}" ]; then - # Add to the directory queue for checking once files are gone if unique - ispresent=$(grep "^${filetounlink}[ ]*$" "${tempDirFile}") - if [ -z "${ispresent}" ]; then - echo " ${filetounlink} " >> "${tempDirFile}" - fi - elif [ -L "${filetounlink}" ]; then - if [ ! -f "${filetounlink}" ]; then - # the linked-to file no longer exists (ie. the symlink is dangling) - rm -f "${filetounlink}" > /dev/null 2>&1 - fi - else - echo "Unprocessable file: '${filetounlink}'" >> "${tempTrash}" - fi - done </dev/null # if the timer is not running, the kill will fail - sleep 1 # ensure the spinner has stopped if running + if ! runLogProgress "rmSymlinksFSCheck" \ + "Checking for orphaned symlinks in mesh" \ + "Checked for orphaned symlinks in mesh" "linkcheck"; then + printError "Unable to remove symlinks links. Review any errors. Manual cleanup using zopen-alt might be required" + fi if [ -e "${tempDirFile}" ]; then ndirs=$(uniq < "${tempDirFile}" | wc -l | tr -d ' ') printVerbose "- Checking ${ndirs} dir links" - for d in $(uniq < "${tempDirFile}" | sort -r) ; do + for d in $(uniq < "${tempDirFile}" | sort -r) ; do [ -d "${d}" ] && rmdir "${d}" >/dev/null 2>&1 done + rm "${tempDirFile}" fi if [ -e "${tempTrash}" ]; then printSoftError "Issues found while trying to remove the following files:" @@ -858,20 +1118,22 @@ EOF printDebug() { [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" + # shellcheck disable=SC2154 if ${debug}; then - printColors "${NC}${BLUE}${BOLD}:DEBUG:${NC}: '${1}'" + printColors "${NC}${BLUE}${BOLD}:DEBUG:${NC}: '${1}'" >&2 fi - [ ! -z "${xtrc}" ] && set -x + [ -n "${xtrc}" ] && set -x return 0 } printVerbose() { [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" + # shellcheck disable=SC2154 if ${verbose}; then - printColors "${NC}${GREEN}${BOLD}VERBOSE${NC}: ${1}" + printColors "${NC}${GREEN}${BOLD}VERBOSE${NC}: ${1}" >&2 fi - [ ! -z "${xtrc}" ] && set -x + [ -n "${xtrc}" ] && set -x return 0 } @@ -882,7 +1144,7 @@ printHeader() fi [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" printColors "${NC}${HEADERCOLOR}${BOLD}${UNDERLINE}${1}${NC}" - [ ! -z "${xtrc}" ] && set -x + [ -n "${xtrc}" ] && set -x return 0 } @@ -890,7 +1152,7 @@ printAttention() { [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" printColors "${NC}${MAGENTA}${BOLD}${UNDERLINE}${1}${NC}" - [ ! -z "${xtrc}" ] && set -x + [ -n "${xtrc}" ] && set -x return 0 } @@ -908,26 +1170,28 @@ runAndLog() runLogProgress() { printVerbose "$1" - if [ -n "$2" ]; then - printInfo "- $2" + printInfo "- ${2:-Running}" + completeText="${3:-Complete}" + animation="${4:-spinner}" + + if ! ${verbose}; then + progressHandler "${animation}" "${completeText}" & + PROGRESS_HANDLER=$! + killph="kill -HUP ${PROGRESS_HANDLER}" + addCleanupTrapCmd "${killph}" + sleep 1 # Give the handler time to wakeup! + eval "$1" + rc=$? + ${killph} >/dev/null 2>&1 # if the timer is not running, the kill will fail + waitforpid ${PROGRESS_HANDLER} # Make sure it's finished writing to screen else - printInfo "- Running" + eval "$1" + rc=$? fi - if [ -n "$3" ]; then - completeText="$3" - else - completeText="Complete" - fi - progressHandler "spinner" "- ${completeText}" & - ph=$! - killph="kill -HUP ${ph} 2>/dev/null" - addCleanupTrapCmd "${killph}" - eval "$1" - rc=$? - if [ ! -z "${SSH_TTY}" ]; then - chtag -r ${SSH_TTY} + if [ -n "${SSH_TTY}" ]; then + chtag -r "${SSH_TTY}" fi - ${killph} 2> /dev/null # if the timer is not running, the kill will fail + return "${rc}" } @@ -936,8 +1200,7 @@ spinloop() # in the absence of generic ms/ns reporting, spin-loop instead - not ideal # but without pre-reqing packages... i=$1 - while [ ${i} -ge 0 ]; do - : + while [ "${i}" -ge 0 ]; do i=$((i - 1)) done } @@ -947,45 +1210,77 @@ progressAnimation() [ $# -eq 0 ] && printError "Internal error: no animation strings." animcnt=$# anim=1 - ansiline 0 0 "$1" - while :; do - spinloop 1000 + firstFrame=true + while true; do + spinloop 3000 # Check for daemonization of this process (ie. orphaned and PPID=1) # Cannot actually use "$PPID" as it is set at script initialization # and not updated when the parent changes so need to query. - getParentProcess "$$" >/dev/null 2>&1 - ppid=$? - [ "${ppid}" -eq 1 ] && kill INT "${ppid}" >/dev/null 2>&1 + if ! ppid=$(getParentProcess "$$"); then + printVerbose "Cannot determine parent process, disable animation" + exit 1 + fi + if [ "${ppid}" = "1" ]; then + # We have been daemonized and owned by PPID=1 + kill HUP "$$" >/dev/null 2>&1 + sleep 1 > /dev/null 2>&1 + fi anim=$((anim + 1)) [ ${anim} -gt ${animcnt} ] && anim=1 - ansiline 1 -1 $(getNthArrayArg "${anim}" "$@") + if ${firstFrame}; then + printf "${ERASELINE}%s\n" "$(getNthArrayArg "${anim}" "$@")" + firstFrame=false + else + printf "${CRSRPL}${ERASELINE}%s\n" "$(getNthArrayArg "${anim}" "$@")" + fi done } -getNthArrayArg () { - shift "$1" - echo "$1" +getNthArrayArg () +{ + shift "$1" + echo "$1\c" +} + +waitforpid() +{ + while kill -0 "$1" >/dev/null 2>&1; do + sleep 1 + done } progressHandler() { - if [ ! "${_BPX_TERMPATH-x}" = "OMVS" ] && [ -z "${NO_COLOR}" ] && [ ! "${FORCE_COLOR-x}" = "0" ] && [ -t 1 ] && [ -t 2 ]; then - [ -z "${-%%*x*}" ] && set +x # Disable -x debug if set for this process - type=$1 - completiontext=$2 # Custom end text (when the process is complete) - trapcmd="exit;" - if [ -n "${completiontext}" ]; then - trapcmd="/bin/echo \"\047[1A\047[30D\047[2K${completiontext}\"; ${trapcmd}" - fi + + if [ -z "${-%%*x*}" ]; then + # Command trace is active so any progress animation + # writing to screen will interleave, making things cluttered. + # Sleep for 1s (to allow the caller to setup signal handling) and exit + sleep 1 + exit 0 + fi + type=$1 + completiontext=$2 # Custom end text (when the process is complete) + trap "exit" EXIT # If there is an animation error, it will just exit + if ${ANSION}; then + trapcmd="/bin/printf \"${CRSRSHOW}${ERASELINE}${CRSRPL}${ERASELINE}${CRSRPL}${ERASELINE}- ${completiontext:-Done}\n\";exit" # shellcheck disable=SC2064 trap "${trapcmd}" HUP + # shellcheck disable=SC2059 + printf "${CRSRHIDE}" case "${type}" in - "spinner") progressAnimation '-' '\' '|' '/' ;; - "network") progressAnimation '-----' '>----' '->---' '-->--' '--->-' '---->' '-----' '----<' '---<-' '--<--' '-<---' '<----' ;; + "spinner") progressAnimation '-' '/' '|' '\\' ;; + "network") progressAnimation '-----' '>----' '->---' '-->--' '--->-' '---->' '-----' '----<' '---<-' '--<--' '-<---' '<----' ;; + "mirror") progressAnimation '#______' '##_____' '#=#____' '#==#___' '#===#__' '#====#_' '#=====#' '#_====#' '#__===#' '#___==#' '#____=#' '#_____#' ;; + "trash") progressAnimation 'O________' '_O_______' '__O______' '___o_____' '____o____' '_____o___' '______.__' '_______._' '________.' ;; "linkcheck") progressAnimation '======|' '?=====|' '-?====|' '--?===|' '---?==|' '----?=|' '-----?|' '------|' '?-----|' '=?----|' '==?---|' '===?--|' '====?-|' '=====?|';; - *) progressAnimation '.' 'o' 'O' 'O' 'o' '.' - ;; + "pkgcheck") progressAnimation '?###?###' '#?###?##' '##?###?#' '###?###?';; + *) progressAnimation '.' 'o' 'O' 'O' 'o' '.' ;; esac + else + trapcmd="/bin/printf \"${completiontext}\n\";exit" + # shellcheck disable=SC2064 + trap "${trapcmd}" HUP fi } @@ -997,21 +1292,24 @@ runInBackgroundWithTimeoutAndLog() printVerbose "${command} with timeout of ${timeout}s." eval "${command} &; TEEPID=$!" PID=$! + [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" # Disable cmd trace if active n=0 while [ ${n} -le ${timeout} ]; do kill -0 "${PID}" 2> /dev/null if [ $? != 0 ]; then wait "${PID}" - if [ ! -z "${SSH_TTY}" ]; then - chtag -r ${SSH_TTY} + if [ -n "${SSH_TTY}" ]; then + chtag -r "${SSH_TTY}" fi rc=$? - return "${rc}" + [ -n "${xtrc}" ] && set -x # Re-enable cmd trace + return ${rc} else sleep 1 - n=$(expr ${n} + 1) + n=$(( n + 1)) fi done + [ -n "${xtrc}" ] && set -x # Re0enable cmd trace kill -9 "${PID}" 2>/dev/null kill -9 "${TEEPID}" 2>/dev/null printError "TIMEOUT: (PID: ${PID}): ${command}" @@ -1024,13 +1322,23 @@ printSoftError() [ -n "${xtrc}" ] && set -x } +assertFailed() +{ + # Used to indicate that something that should have been set in an internal + # call was missing/borken - a program error rather than a user error + [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" + printColors "${NC}${RED}${BOLD}***INTERNAL ASSERTION ERROR: ${NC}${RED}${1}${NC}" >&2 + [ -n "${xtrc}" ] && set -x + mutexFree "zopen" # prevent lock from lingering around after an error + exit 12 +} + printError() { [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" printColors "${NC}${RED}${BOLD}***ERROR: ${NC}${RED}${1}${NC}" >&2 [ -n "${xtrc}" ] && set -x mutexFree "zopen" # prevent lock from lingering around after an error - cleanupFunction exit 4 } @@ -1062,14 +1370,14 @@ getInputHidden() addCleanupTrapCmd "stty echo" stty -echo read zopen_input - echo ${zopen_input} + echo "${zopen_input}" stty echo } getInput() { read zopen_input - echo ${zopen_input} + echo "${zopen_input}" } printElapsedTime() @@ -1077,7 +1385,7 @@ printElapsedTime() printType=$1 functionName=$2 startTime=$3 - elapsedTime=$((${SECONDS} - ${startTime})) + elapsedTime=$((SECONDS - startTime)) elapsedTimeOutput="${functionName} completed in ${elapsedTime} seconds." @@ -1097,6 +1405,7 @@ processConfig() if [ -z "${ZOPEN_ROOTFS}" ]; then relativeRootDir="$(cd "$(dirname "$0")/../.." > /dev/null 2>&1 && pwd -P)" if [ -f "${relativeRootDir}/etc/zopen-config" ]; then + # shellcheck source=/dev/null . "${relativeRootDir}/etc/zopen-config" else printError "Source the zopen-config prior to running $0." @@ -1113,7 +1422,7 @@ checkIfConfigLoaded() errorMessage="Certificate at ${ZOPEN_CA} could not be accessed. Ensure zopen init has run and zopen-config has been sourced." fi - if [ ! -z "${errorMessage}" ]; then + if [ -n "${errorMessage}" ]; then if [ -r "${mydir}/../../../etc/zopen-config" ]; then relativeConfigDir="$(cd "$(dirname "${mydir}")/../../etc/" > /dev/null 2>&1 && pwd -P)" errorMessage="${errorMessage} Run '. ${relativeConfigDir}/zopen-config' or add it to your .profile." @@ -1125,28 +1434,28 @@ checkIfConfigLoaded() parseDeps() { dep="$1" - version=$(echo ${dep} | awk -F '[>=<]+' '{print $2}') + version=$(echo "${dep}" | awk -F '[>=<]+' '{print $2}') if [ -z "${version}" ]; then operator="" - dep=$(echo ${dep} | awk -F '[>=<]+' '{print $1}') + dep=$(echo "${dep}" | awk -F '[>=<]+' '{print $1}') else - operator=$(echo ${dep} | awk -F '[0-9.]+' '{print $1}' | awk -F '^[a-zA-Z]+' '{print $2}') - dep=$(echo ${dep} | awk -F '[>=<]+' '{print $1}') + operator=$(echo "${dep}" | awk -F '[0-9.]+' '{print $1}' | awk -F '^[a-zA-Z]+' '{print $2}') + dep=$(echo "${dep}" | awk -F '[>=<]+' '{print $1}') case ${operator} in ">=") ;; "=") ;; *) printError "${operator} is not supported." ;; esac - major=$(echo ${version} | awk -F. '{print $1}') - minor=$(echo ${version} | awk -F. '{print $2}') + major=$(echo "${version}" | awk -F. '{print $1}') + minor=$(echo "${version}" | awk -F. '{print $2}') if [ -z "${minor}" ]; then minor=0 fi - patch=$(echo ${version} | awk -F. '{print $3}') + patch=$(echo "${version}" | awk -F. '{print $3}') if [ -z "${patch}" ]; then patch=0 fi - prerelease=$(echo ${version} | awk -F. '{print $4}') + prerelease=$(echo "${version}" | awk -F. '{print $4}') if [ -z "${prerelease}" ]; then prerelease=0 fi @@ -1204,7 +1513,7 @@ deleteDuplicateEntries() { value=$1 delim=$2 - echo "${value}${delim}" | awk -v RS="${delim}" '!($0 in a) {a[$0]; printf("%s%s", col, $0); col=RS; }' | zossed "s/${delim}$//" + echo "${value}${delim}" | awk -v RS="${delim}" '!($0 in a) {a[$0]; printf("%s%s", col, $0); col=RS; }' | sed "s/${delim}$//" } # Logging Types @@ -1233,78 +1542,182 @@ syslog() module=$4 # zopen- location=$5 # function msg=$6 # Message text - if [ ! -e "${fd}" ]; then - mkdir -p "$(dirname "${fd}")" - touch "${fd}" + parent=$(dirname "${fd}") + + output="$(date +"%F %T") $(id | cut -d' ' -f1)::${module}:${type}:${categories}:${location}:${msg}" + if [ ! -d "${parent}" ]; then + mkdir -p "${parent}" 2>/dev/null || { + printWarning "Cannot create log directory '${parent}'; writing syslog to stderr" + echo "${output}" >&2 + } + elif [ ! -e "${fd}" ]; then + touch "${fd}" 2>/dev/null || { + printWarning "Cannot create log file '${fd}'; writing syslog to stderr" + echo "${output}" >&2 + } + elif [ ! -w "${fd}" ]; then + printWarning "No write permission to log file '${fd}'; writing syslog to stderr" + echo "${output}" >&2 + else + echo "${output}" >> "${fd}" fi - echo "$(date +"%F %T") $(id | cut -d' ' -f1)::${module}:${type}:${categories}:${location}:${msg}" >> "${fd}" -} -downloadJSONCache() -{ - from_readonly=$1 +} - if [ -z "${JSON_CACHE}" ]; then - cachedir="${ZOPEN_ROOTFS}/var/cache/zopen" - [ ! -e "${cachedir}" ] && mkdir -p "${cachedir}" - JSON_CACHE="${cachedir}/zopen_releases.json" - JSON_TIMESTAMP="${cachedir}/zopen_releases.timestamp" - JSON_TIMESTAMP_CURRENT="${cachedir}/zopen_releases.timestamp.current" +jqGetKey(){ + # If the key is not present, the error causes jq to exit with a + # non-zero return code + jq -er --arg key "$1" '.[$key] // error("Missing key: " + $key)' "$2" +} - if [ -n "$from_readonly" ]; then - if [ -r "$JSON_CACHE" ] && [ ! -w "$JSON_CACHE" ]; then - return; # Skip the download for read only operations when you know you can't write to it - fi +validateReposDEntry(){ + activeRepo=$1 + requiredKeys="type metadata_baseurl metadata_file latest_file" + printVerbose "Ensuring repo file is valid; required keys: ${requiredKeys}" + for reposDEntry in ${requiredKeys}; do + if ! jqk=$(jqGetKey "${reposDEntry}" "${activeRepo}"); then + printSoftError "Repository definition at '${activeRepo}' incomplete; missing required key: ${reposDEntry}." + printError "Running zopen init --refresh might resolve this or recreating file from a backup" + elif [ -z "${jqk}" ]; then + printSoftError "Repository definition at '${activeRepo}' incomplete; required key '${reposDEntry}' has no value" + printError "Check the value in '${activeRepo}' is correct and retry command" + else + printVerbose "Key '${reposDEntry}' found with value '${jqk}'" fi + done +} + +# buildCacheURL +# Given a cache filename, generate the appropriate URL to retrieve +# the latest cache data from, taking into account the different repo types +# inputs: cacheFileName - the name of the cache to update +# stdout: the url to check for the latest cache +# return: 0 success +# 1 failure +buildCacheURL(){ + cacheFileName=$1 + activeRepo="${ZOPEN_REPOS_DIR}/active" + dereffedLink=$(deref_symlink "${activeRepo}") + [ ! -e "${dereffedLink}" ] && printError "Could not access linked repository configuration at '${dereffedLink}'. Check file to ensure valid repository configuration or refresh default configuration with zopen init --refresh -y." + if ! validateReposDEntry "${dereffedLink}"; then + return 1 + fi + # Failures in the following indicate the configuration file isn't setup correctly + type=$(jqGetKey "type" "${dereffedLink}") || printError "Unrecognised key 'type' in configuration file." + base=$(jqGetKey "metadata_baseurl" "${dereffedLink}") || printError "Unrecognised key 'metadata_baseurl' in configuration file." + case "${type}" in + http|https) printf "%s://%s/%s\n" "${type}" "${base}" "${cacheFileName}" + ;; + file) printf "%s:%s/%s\n" "${type}" "${base}" "${cacheFileName}" + ;; + *) printError "Unsupported repository type '${type}'.";; + esac +} + +# shellcheck disable=SC2120 +updateCaches() +{ + if [ $# -gt 0 ]; then + caches="$1" + else + caches="JSON_CACHE JSON_LATEST_CACHE JSON_CVE_CACHE JSON_FILES_CACHE" + fi + for cache in $caches; do + printVerbose "Checking if the $cache already downloaded in this session" + eval "value=\${$cache}" + if [ -z "${value}" ]; then + case "$cache" in + JSON_CACHE) cacheFile="zopen_releases.json" ;; + JSON_LATEST_CACHE) cacheFile="zopen_releases_latest.json" ;; + JSON_CVE_CACHE) cacheFile="zopen_vulnerability.json" ;; + JSON_FILES_CACHE) cacheFile="zopen_files.json" ;; + *) assertFailed "Invalid cache variable specified: $cache" + esac + fqCacheFile="${ZOPEN_CACHEDURL_DIR}/${cacheFile}" + eval "${cache}=${fqCacheFile}" + url=$(buildCacheURL "${cacheFile}") || printError "Unable to build url to update release data cache" + downloadJSONCacheIfExpired "${fqCacheFile}" "${url}" + fi + done +} + +downloadJSONCacheIfExpired() +{ + fileToCache="$1" + cacheUrl="$2" + cacheTimestamp="${fileToCache}.timestamp" + cacheTimestampCurrent="${fileToCache}.timestamp.current" + + printVerbose "Ensuring cache directory exists" + if [ ! -e "${ZOPEN_CACHEDURL_DIR}" ]; then + mkdir -p "${ZOPEN_CACHEDURL_DIR}" || printError "Cannot create cache directory for metadata at '${cachedir}'. Check permissions and retry." + fi + + if [ -r "${fileToCache}" ] && [ ! -w "${fileToCache}" ]; then + # Skip the download for read only operations when you know you can't write to it + printWarning "Unable to write to cache at '${fileToCache}'; using stale cache data" + return; + fi + # Need to check that we can read & write to the JSON timestamp cache files - if [ -e "${JSON_TIMESTAMP_CURRENT}" ]; then - [ ! -w "${JSON_TIMESTAMP_CURRENT}" ] || [ ! -r "${JSON_TIMESTAMP_CURRENT}" ] && printError "Cannot access cache at '${JSON_TIMESTAMP_CURRENT}'. Check permissions and retry request." + if [ -e "${cacheTimestampCurrent}" ]; then + [ ! -w "${cacheTimestampCurrent}" ] || [ ! -r "${cacheTimestampCurrent}" ] && printError "Cannot access cache at '${cacheTimestampCurrent}'. Check permissions and retry request." fi - if [ -e "${JSON_TIMESTAMP}" ]; then - [ ! -w "${JSON_TIMESTAMP}" ] || [ ! -r "${JSON_TIMESTAMP}" ] && printError "Cannot access cache at '${JSON_TIMESTAMP}'. Check permissions and retry request." + if [ -e "${cacheTimestamp}" ]; then + [ ! -w "${cacheTimestamp}" ] || [ ! -r "${cacheTimestamp}" ] && printError "Cannot access cache at '${cacheTimestamp}'. Check permissions and retry request." fi - if [ -e "${JSON_CACHE}" ]; then - [ ! -w "${JSON_CACHE}" ] || [ ! -r "${JSON_CACHE}" ] && printError "Cannot access cache at '${JSON_CACHE}'. Check permissions and retry request." + if [ -e "${fileToCache}" ]; then + [ ! -w "${fileToCache}" ] || [ ! -r "${fileToCache}" ] && printError "Cannot access cache at '${JSON_CACHE}'. Check permissions and retry request." fi - if ! curlout=$(curlCmd -L --no-progress-meter -I "${ZOPEN_JSON_CACHE_URL}" -o "${JSON_TIMESTAMP_CURRENT}"); then - printError "Failed to obtain json cache timestamp from ${ZOPEN_JSON_CACHE_URL}; ${curlout}" + if ! curlCmd --fail --location --silent --head "${cacheUrl}" -o "${cacheTimestampCurrent}"; then + printError "Failed to obtain json cache timestamp from ${cacheUrl}." + fi + if command -v chtag >/dev/null 2>&1; then + printVerbose "Issuing chtag command" + chtag -tc 819 "${cacheTimestampCurrent}" fi - chtag -tc 819 "${JSON_TIMESTAMP_CURRENT}" - if [ -f "${JSON_CACHE}" ] && [ -f "${JSON_TIMESTAMP}" ] && grep -q 'ETag' "${JSON_TIMESTAMP_CURRENT}" && [ "$(grep 'ETag' "${JSON_TIMESTAMP_CURRENT}")" = "$(grep 'ETag' "${JSON_TIMESTAMP}")" ]; then - return + if [ -f "${fileToCache}" ] && [ -f "${cacheTimestamp}" ]; then + # Extract eyecatchers - either ETag or Last-Modified depending on repo type + prevETag=$(grep -i '^ETag' "${cacheTimestamp}") + currETag=$(grep -i '^ETag' "${cacheTimestampCurrent}") + prevLastMod=$(grep -i '^Last-Modified' "${cacheTimestamp}") + currLastMod=$(grep -i '^Last-Modified' "${cacheTimestampCurrent}") + # Compare values if they exist and match - if they do, then the cache is current + if [ -n "$prevETag" ] && [ -n "$currETag" ]; then + printVerbose "Comparing previous ETag '${prevETag}' with current '${currETag}" + [ "$prevETag" = "$currETag" ] && return + elif [ -n "$prevLastMod" ] && [ -n "$currLastMod" ]; then + printVerbose "Comparing previous ETag '${prevLastMod}' with current '${currLastMod}" + [ "$prevLastMod" = "$currLastMod" ] && return + fi fi printVerbose "Replacing old timestamp with latest." - mv -f "${JSON_TIMESTAMP_CURRENT}" "${JSON_TIMESTAMP}" + mv -f "${cacheTimestampCurrent}" "${cacheTimestamp}" - if ! curlout=$(curlCmd -L --no-progress-meter -o "${JSON_CACHE}" "${ZOPEN_JSON_CACHE_URL}"); then - printError "Failed to obtain json cache from ${ZOPEN_JSON_CACHE_URL}; ${curlout}" + if ! curlCmd --fail --location --silent -o "${fileToCache}" "${cacheUrl}"; then + printError "Failed to obtain json cache from '${cacheUrl}'" fi - chtag -tc 819 "${JSON_CACHE}" - fi - - if [ ! -f "${JSON_CACHE}" ]; then - printError "Could not download json cache from ${ZOPEN_JSON_CACHE_URL}" + chtag -tc 819 "${fileToCache}" + if [ ! -f "${fileToCache}" ]; then + printError "Could not download json cache from '${cacheUrl}" fi } -getReposFromGithub() -{ - downloadJSONCache $1 - repo_results="$(cat "${JSON_CACHE}" | jq -r '.release_data | keys[]')" -} +# isValidRepo +# Queries the main repository list to determine if the input is valid, This +# uses jq itself to return 0 or 1 with no output +# inputs: $1 the port name. -getAllReleasesFromGithub() +# return: 0 valid port name +# 1 invalid port name +isValidRepo() { - downloadJSONCache $1 - repo="$1" - releases="$(jq -e -r '.release_data."'${repo}'"' "${JSON_CACHE}")" - if [ $? -ne 0 ]; then - printError "Could not get all releases for ${repo}" - fi + updateCaches + jq -r --arg needle "$1" 'if .release_data | has($needle) then empty else error("") end' "${JSON_CACHE}" > /dev/null 2>&1 } # Initializes a default environment for consistency in zopen builds @@ -1338,57 +1751,25 @@ checkWritable() fi } -getReleaseLine() -{ - jsonConfig="${ZOPEN_ROOTFS}/etc/zopen/config.json" - if [ ! -f "${jsonConfig}" ]; then - jq -r '.release_line' $jsonConfig - else - echo "STABLE" - fi -} - -getRMProcs() +generateUUID() { - jsonConfig="${ZOPEN_ROOTFS}/etc/zopen/config.json" - if [ ! -f "${jsonConfig}" ]; then - jq -r '.num_rm_procs' $jsonConfig - else - echo "5" # default - fi + date_part=$(date +%s) + random_part=$((RANDOM)) + uuid="${date_part}-${random_part}" + echo "${uuid}" } isURLReachable() { url="$1" timeout=5 - if curl -s --fail --max-time $timeout "$url" > /dev/null; then + if curlCmd -s --fail --max-time $timeout "$url" > /dev/null; then return 0 else return 1 fi } -checkAvailableSize() -{ - - package="$1" - packageSize="$2" - printInfo "- Checking available size to install ${package}." - - printDebug "Package Size: ${packageSize} bytes" - packageSize=$(echo "scale=2; ${packageSize} / 1024" | bc) - printDebug "Package Size: ${packageSize} k" - partitionSize=$(/bin/df -k . | tail -1 | awk '{print $3}' | cut -f1 -d '/') - printDebug "Partition Size: ${partitionSize}k [free on '$(pwd -P)']" - - if [ 1 -eq "$(echo "${packageSize} > ${partitionSize}" | bc)" ]; then - printError "Not enough space in partition." - fi - printInfo "- Enough space to install ${package}. Proceeding with installation." - return 0 -} - promptYesOrNo() { message="$1" skip=$2 @@ -1422,14 +1803,14 @@ asciiecho() return 2 fi if [ "$(chtag -p "${file}" | cut -f2 -d' ')" = "IBM-1047" ]; then - if ! /bin/iconv -f IBM-1047 -t ISO8859-1 < "${file}" > "${file}_ascii" || ! chtag -tc ISO8859-1 "${file}_ascii" || ! mv "${file}_ascii" "${file}"; then + if ! iconv -f IBM-1047 -t ISO8859-1 < "${file}" > "${file}_ascii" || ! chtag -tc ISO8859-1 "${file}_ascii" || ! mv "${file}_ascii" "${file}"; then printError "Unable to convert EBCDIC text to ASCII for ${file}" >&2 fi fi return 0 } -a2e() +a2e() { source="$1" @@ -1439,18 +1820,1195 @@ a2e() fi if [ "$(chtag -p "${source}" | cut -f2 -d' ')" = "ISO8859-1" ]; then - /bin/iconv -f ISO8859-1 -t IBM-1047 "$source" > "$source.bk" + iconv -f ISO8859-1 -t IBM-1047 "$source" > "$source.bk" chtag -tc 1047 "$source.bk" mv "$source.bk" "$source" fi } -diskusage() -{ - path=$1 - # awk to "trim" output" - if ! size=$(zosdu -kts "${path}" | /bin/awk '{print ($1)}'); then - printError "Unable to generate disk usage (du) report for '${path}'" +testGPGAgentSocket() { + # Test whether the gpg socket that has been found is actually valid - + # orphaned sockets without a daemon can lead to gpg errors when trying + # to restart + if ! gpg-connect-agent --socket="$1" /bye >/dev/null 2>&1; then + return 1 + fi + return 0 +} + +startGPGAgent() { + printInfo "- Starting gpg-agent..." + + SOCKET_PATH=$(gpgconf --list-dirs agent-socket) + printVerbose "gpg socket path: ${SOCKET_PATH}" + if [ -r "$SOCKET_PATH" ]; then + if testGPGAgentSocket "${SOCKET_PATH}"; then + printVerbose "gpg-agent is already running (socket found at ${SOCKET_PATH})." + return 0 + else + printWarning "Found orphaned GPG agent socket at ${SOCKET_PATH}; removing and attempting restart of gpg-agent." + rm -f "${SOCKET_PATH}" + fi + else + printWarning "gpg socket at '$SOCKET_PATH' cannot be read" + fi + + if eval "$(gpg-agent --daemon --disable-scdaemon)" >/dev/null 2>&1; then + if [ -r "$SOCKET_PATH" ]; then + printVerbose "gpg-agent started successfully (socket created at $SOCKET_PATH)." + else + printWarning "gpg-agent start initiated, but socket was not created at $SOCKET_PATH. Check file permissions for directory '$(dirname "$SOCKET_PATH")' and files contained" + printWarning "gpg-agent daemon might not complete initialisation" + fi + else + if [ -r "$SOCKET_PATH" ]; then + printWarning "gpg-agent started successfully (socket created at $SOCKET_PATH), but gpg-agent returned a non-zero return code." + else + printError "Failed to start gpg-agent. Check file permissions for directory '$(dirname "$SOCKET_PATH")' and files contained." + fi + fi +} + +promptYesNoAlways() { + message="$1" + skip=$2 + if ! ${skip}; then + while true; do + printf "${message} [y/n/a] " + read answer < /dev/tty + answer=$(echo "${answer}" | tr '[A-Z]' '[a-z]') + case "${answer}" in + y|yes) return 0;; + n|no) return 1;; + a|always) yesToPrompts=true; return 0;; + esac + done + fi + [ "$yesToPrompts" = "true" ] && return 0 || return 1 +} + +getVersionedMetadata() +{ + repo=$1 + invalidPortAssetFile=$2 + printDebug "Specific version ${versioned} requested - checking existence and URL" + requestedVersion=$(echo "${versioned}" | awk -F'.' '{print $4}') + printDebug "Finding metadata for latest release matching version prefix: requestedVersion: ${versioned}" + if ! releasemetadata=$(jq --arg repo "${repo}" --arg requestedVersion "${versioned}" \ + '.release_data[$repo] | map(select(.assets[].name | test($requestedVersion)))[0]' "${JSON_CACHE}"); then + printSoftError "Could not find metadata for ${repo} in repo metadata '${JSON_CACHE}'" 2> "${invalidPortAssetFile}" + return 1 + elif [ "${releasemetadata}" = "null" ]; then + printSoftError "Could not find specified version '${versioned}' for package '${repo}'" 2> "${invalidPortAssetFile}" + return 1 + fi + printDebug "Check for asset" + asset=$(printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]') + if ! asset=$(printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]'); then + printSoftError "Could not find asset for release version '${versioned}' in repo '${repo}'" 2> "${invalidPortAssetFile}" + return 1 + fi + return 0 +} + +getTaggedMetadata() +{ + repo=$1 + invalidPortAssetFile=$2 + printDebug "Explicit tagged version '${tagged}' specified. Checking for match" + if ! releasemetadata=$(jq -e --arg repo "${repo}" --arg tagged "${tagged}" \ + '.release_data[$repo][] | select(.tag_name == $tagged)' "${JSON_CACHE}"); then + printSoftError "Tagged release '${tagged}' was not found for ${repo}" 2> "${invalidPortAssetFile}" + return 1 + fi + printDebug "Check for asset" + asset=$(printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]') + if ! asset=$(printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]'); then + printSoftError "Could not find asset for release tagged '${tagged}' in repo '${repo}'" 2> "${invalidPortAssetFile}" + return 1 + fi + return 0 +} + +getSelectMetadata() +{ + # As this is running within the generate... logic, a progress handler will have been started. + # This needs to be terminated before trying to write to screen. This does mean that messages + # can be written to the screen rather than directed to a temporary error file + # shellcheck disable=SC2154 + kill -HUP "${PROGRESS_HANDLER}" >/dev/null 2>&1 # if the timer is not running, the kill will fail + waitforpid "${PROGRESS_HANDLER}" # Make sure it's finished writing to screen + + repo="$1" + # Explicitly allow the user to select a release to install; useful if there are broken installs + # as a known good release can be found, selected and pinned! + printDebug "List individual releases and allow selection" + i=$(jq --arg repo "${repo}" '.release_data[$repo] | length - 1' "${JSON_CACHE}") + printInfo "Versions available for install:" + if ! jq --raw-output --arg repo "${repo}" \ + '.release_data[$repo] | to_entries | map("\(.key): \(.value.tag_name) - \(.value.assets[0].name) [\( ( .value.assets[0].expanded_size|tonumber)*1000 / (1024 * 1024) | ceil | . / 1000)Mb]")[]' "${JSON_CACHE}"; then + printError "Unable to enumerate asset version strings" + fi + printDebug "Getting user selection" + valid=false + while ! ${valid}; do + echo "Enter version to install (0-${i}): " + read selection < /dev/tty + if [ -n "$(echo "${selection}" | sed -e 's/[0-9]*//')" ]; then + echo "Invalid input, must be a number between 0 and ${i}" + elif [ "${selection}" -ge 0 ] && [ "${selection}" -le "${i}" ]; then + valid=true + fi + done + printVerbose "Selecting item ${selection} from array" + releasemetadata=$(jq --arg repo "${repo}" --arg selection "${selection}" \ + '.release_data[$repo][$selection | tonumber]' "${JSON_CACHE}") +} + +getReleaseLineMetadata() +{ + repo=$1 + invalidPortAssetFile=$2 + printDebug "Install from release line '${releaseLine}' specified" + validatedReleaseLine=$(validateReleaseLine "${releaseLine}") + if [ -z "${validatedReleaseLine}" ]; then + printSoftError "Invalid releaseline specified: '${releaseLine}'; Valid values: DEV or STABLE" 2> "${invalidPortAssetFile}" + return 1 + fi + printDebug "Finding latest asset on the release line" + if ! releasemetadata=$(jq --arg repo "${repo}" --arg releaseLine "${validatedReleaseLine}" \ + '.release_data[$repo] | map(select(.tag_name | startswith($releaseLine)))[0]' "${JSON_CACHE}"); then + printSoftError "Could not find metadata for ${repo} in repository metadata at '${JSON_CACHE}'" 2> "${invalidPortAssetFile}" + return 1 + fi + printDebug "Use quick check for asset to check for existence of metadata" + if ! asset="$(printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')"; then + printSoftError "Could not find asset for releaseline '${validatedReleaseLine}' for repo '${repo}' in ${JSON_CACHE}" 2> "${invalidPortAssetFile}" + return 1 + fi + unset releaseLine +} + +getReleaseLineFromInstalledPkg(){ + repo="$1" + metadataFile="${ZOPEN_PKGINSTALL}/${repo}//${repo}/metadata.json" + if [ ! -r "${metadataFile}" ]; then + printVerbose "No metadata.json file for '${repo}'; default to stable" + echo "STABLE" && return 0 + fi + #Note that the releaseline is knwon as buildline in the metadata file! + if ! installedReleaseline=$(jqw -r '.product.buildline' "${metadataFile}"); then + printError "Could not get releaseline/buildline from package metadata for '${repo}'" + fi + case "${installedReleaseline}" in + "STABLE"|"DEV") printVerbose "Valid releaseline '${installedReleaseline}' from metadata file";; + "null") printVerbose "Explicit release line for package '${repo}' unknown; git clone perhaps." + installedReleaseline="UNKNOWN" + ;; + *) printError "Invalid releaseline '${installedReleaseline}' for package '${repo}'" ;; + esac + echo "${installedReleaseline}" + return 0 +} + +calculateReleaseLineMetadata() +{ + repo="$1" + validatedReleaseLine="UNKNOWN" + printDebug "No explicit version/tag/releaseline, checking for pre-existing package&releaseline" + if isPackageActive "${repo}"; then + printVerbose "Package '${repo}' already installed; attempting to determine releaseline from metadata" + if ! validatedReleaseLine=$(getReleaseLineFromInstalledPkg "${repo}"); then + return 1 + fi + fi + case "${validatedReleaseLine}" in + "UNKNOWN") + printVerbose "No specific releaseline from package itself. Checking for system-configured releaseline" + if [ -e "${ZOPEN_JSON_CONFIG}" ]; then + printDebug "Using v2 configuration: '${ZOPEN_JSON_CONFIG}'" + sysrelline=$(jq -re '.release_line' "${ZOPEN_JSON_CONFIG}") + elif [ -e "${ZOPEN_ROOTFS}/etc/zopen/releaseline" ] ; then + printDebug "Using legacy file-based config" + sysrelline=$(awk ' {print toupper($1)}') < "${ZOPEN_ROOTFS}/etc/zopen/releaseline" + else + printWarning "Cannot determine system or package release line (STABLE|DEV) for package '{repo}'" + printWarning "Using default of 'STABLE'" + validatedReleaseLine="STABLE" + fi + if [ -n "${sysrelline}" ]; then + printDebug "Validating value: ${sysrelline}" + validatedReleaseLine=$(validateReleaseLine "${sysrelline}") + if [ -n "${validatedReleaseLine}" ]; then + printVerbose "zopen system configured to use releaseline '${sysrelline}'; restricting to that releaseline" + else + printWarning "zopen misconfigured to use an unknown releaseline of '${sysrelline}'; defaulting to STABLE packages" + printWarning "Set the contents of '${ZOPEN_ROOTFS}/etc/zopen/releaseline' to a valid value to remove this message" + printWarning "Valid values are: DEV | STABLE" + validatedReleaseLine="STABLE" + fi + fi + ;; + STABLE|DEV) printVerbose "Using releaseline '${validatedReleaseLine}' for package '${repo}'" ;; + *) assertFailed "Invalid value for relaseline '${validatedReleaseLine}' calculated!" ;; + esac + + # We have some situations that could arise + # 1. the port being installed has no releaseline tagging yet (ie. no releases tagged STABLE_* or DEV_*) + # 2. system is configured for STABLE but only has DEV stream available + # 3. system is configured for DEV but only has STABLE stream available + # 4. the port being installed has got full releaseline tagging + # The issue could arise that the user has switched the system from DEV->STABLE or vice-versa so package + # stream mismatches could arise but in normal case, once a package is installed [that has releaseline tagging] + # then that specific releaseline will be used + printDebug "Finding any releases tagged with ${validatedReleaseLine} and getting the first (newest/latest)" + releasemetadata=$(jq --arg repo "${repo}" --arg releaseLine "${validatedReleaseLine}" \ + '.release_data[$repo] | map(select(.tag_name | startswith($releaseLine)))[0]' "${JSON_CACHE}") + + printDebug "Use quick check for asset to check for existence of metadata" + asset="$(printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')" + [ "${asset}" = "null" ] && asset="" # jq uses null, translate to sh's empty + + if [ -n "${asset}" ]; then + # Case 4... + printVerbose "Found a specific '${validatedReleaseLine}' release-line tagged version; installing..." + else + # Case 2 & 3 + printDebug "No releases on releaseline '${validatedReleaseLine}'; checking alternative releaseline" + alt=$(echo "${validatedReleaseLine}" | awk ' /DEV/ { print "STABLE" } /STABLE/ { print "DEV" }') + releasemetadata=$(jq --arg repo "${repo}" --arg releaseLine "${alt}" \ + '.release_data[$repo] | map(select(.tag_name | startswith($releaseLine)))[0]' "${JSON_CACHE}") + printDebug "Use quick check for asset to check for existence of metadata" + asset="$(printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')" + [ "${asset}" = "null" ] && asset="" # jq uses null, translate to sh's empty + if [ $? -eq 0 ]; then + printDebug "Found a release on the '${alt}' release line so release tagging is active" + if [ "DEV" = "${validatedReleaseLine}" ]; then + # The system will be configured to use DEV packages where available but if none, use latest + printInfo "No specific DEV releaseline package, using latest available release" + releasemetadata=$(jq --arg repo "${repo}" '.release_data[$repo][0]' "${JSON_CACHE}") + else + printVerbose "The system is configured to only use STABLE releaseline packages but there are none" + printInfo "No release available on the '${validatedReleaseLine}' releaseline." + fi + else + # Case 1 - old package that has no release tagging yet (no DEV or STABLE), just install latest + printVerbose "Installing latest release" + releasemetadata=$(jq --arg repo "${repo}" '.release_data[$repo][0]' "${JSON_CACHE}") + fi + fi +} + +parseRepoName() +{ + fullname="$1" + printDebug "Name to install: ${fullname}, parsing any version ('=') or tag ('%') that has been specified" + name=$(echo "${fullname}" | sed -e 's#[=%].*##') + repo="${name}" + versioned=$(echo "${fullname}" | cut -s -d '=' -f 2) + tagged=$(echo "${fullname}" | cut -s -d '%' -f 2) + printDebug "Name:${name};version:${versioned};tag:${tagged};repo:${repo}" +} + +getPortMetaData(){ + # This is running inside a progresshandler - route error messages to + # $2 + portRequested="$1" + invalidPortAssetFile="$2" + printDebug "Removing any version (=) or tag (%) suffixes fron '${portRequested}" + portName=$(echo "${portRequested}" | sed -e 's#%.*##' -e 's#=.*##') + + if ! isValidRepo "${portName}"; then + echo "${portName}: no matching port found" >> "${invalidPortAssetFile}" + return 1 + fi + + parseRepoName "${portRequested}" # To set the various status flags below + updateCaches + if [ -n "${versioned}" ]; then + if ! getVersionedMetadata "${portName}" "${invalidPortAssetFile}"; then + return 1 + fi + elif [ -n "${tagged}" ]; then + if ! getTaggedMetadata "${portName}" "${invalidPortAssetFile}"; then + return 1 + fi + elif # shellcheck disable=SC2154 + ${selectVersion}; then + selectVersion=false # Need to set this to prevent selection of dependencies + if ! getSelectMetadata "${portName}" "${invalidPortAssetFile}"; then + return 1 + fi + elif [ -n "${releaseLine}" ]; then + # The release line to use was explicitly mentioned on the cli + if ! getReleaseLineMetadata "${portName}" "${invalidPortAssetFile}"; then + return 1 + fi + else + # No explict release line given, calculate + if ! calculateReleaseLineMetadata "${portName}" "${invalidPortAssetFile}"; then + return 1 + fi + fi + if [ -z "${asset}" ] || [ "null" = "${asset}" ]; then + printDebug "Asset not found during previous logic; setting now from metadata" + if [ -z "${releasemetadata}" ]; then + echo "${portName}: metadata could not be found" >> "${invalidPortAssetFile}" + return 1 + fi + printDebug "Getting specific asset details using metadata: ${releasemetadata}" + asset=$(printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]') + fi + if [ -z "${asset}" ]; then + echo "${portName} asset metadata could not be found" 2> "${invalidPortAssetFile}" + return 1 + fi + return 0 +} + +# createDependancyGraph +# analyzes the input file to create the list of all packages that are to +# be pulled in during install - and recurses if any added packages themselves +# pull in dependancies, dependencies being added to the front of the install queue +# inputs: $1 the file to use for install ports +# $2 an error file for outputing failures +# return: 0 for success (output of pwd -P command) +# 8 if error +createDependancyGraph() +{ + invalidPortAssetFile=$1 && shift + printDebug "Getting list of dependencies" + dependencies=$(echo "${installList}" | jq --raw-output '.installqueue[] | select(.asset.runtime_dependencies | test("No dependencies") | not )| map(try(.runtime_dependencies |= split(" ")))| .[] | .runtime_dependencies[] ') + printDebug "Removing any dependencies already on install queue" + installing=$(echo "${installList}" | jq --raw-output '.installqueue[] | .portname') + # TODO: Use JQ to diff? + missing=$(diffList "${installing}" "${dependencies}" ) + if [ -z "${missing}" ]; then + printDebug "All dependencies are in the install graph" + return 0 + fi + printDebug "Adding dependencies to install graph" + if ! addToInstallGraph "dependancy" "${invalidPortAssetFile}" "${missing}"; then + return 1 + fi + # Recurse in case the now-installing dependencies themselves have dependencies + # Recursive dependencies should not break as the initial package will have been + # marked for installation + if ! createDependancyGraph "${invalidPortAssetFile}"; then + return 1 + fi + return 0 +} + +# addToInstallGraph +# Finds appropriate metadata for the specified port(s) and +# includes that in the installation file +# inputs: $1 if the install list comes from dependency analysis +# $2 an error file for outputing failures +# $* requested list of packages to install +# return: 0 for success (output of pwd -P command) +# 8 if error +addToInstallGraph(){ + installtype=$1 && shift + invalidPortAssetFile=$1 && shift + pkgList="$1" + printDebug "Adding ${pkgList} to install graph" + for portRequested in ${pkgList}; do + if ! getPortMetaData "${portRequested}" "${invalidPortAssetFile}"; then + break + fi + ## Merge asset into JSON install list + installList=$(echo "${installList}" | \ + jq ".installqueue += [{\"portname\":\"${portName}\", \"asset\":${asset}, \"installtype\":\"${installtype}\"}]") + done + [ -e "${invalidPortAssetFile}" ] && return 1 + return 0 +} + +# This uses the release json file to find whether any of the inputs are invalid +# package names. Note that the Github API returns repository names in lowercase +# which potentially breaks packages whose key does not match their repo name +# (NATSport for example) - we need to ensure we use the reponame in lowercase. +validatePackageList(){ + installees="$1" + # shellcheck disable=SC2086 # Using set -f disables globbing + printVerbose "Stripping any version/tagging" + installees=$(set -f; echo ${installees} |awk -v ORS=, -v RS=' ' '{$1=$1; sub(/[=%].*/,x); print "\""$1"\""}') + # Check the metadata cache file to see if the needle is either the name of a + # port (so a key) or is defined in a tag + invalidPortList=$(jq -r --argjson needles "[${installees%%,}]" \ + '. as $data | $needles| map( select( . as $needle| ( + ($data.release_data | keys_unsorted | any(. == $needle)) + ) | not ) ) | unique | join(" ")' "${JSON_CACHE}" + ) + + if [ -n "${invalidPortList}" ]; then + # Calculate word lengths for nicer formatting + set -- ${invalidPortList} + minlen=1 + for toolrepo in "$@"; do + wordlen=${#toolrepo} + minlen=$(( wordlen > minlen ? wordlen : minlen )) + done + + # Respin loop actually checking for suggestions + set -- ${invalidPortList} + printSoftError "The following port name$( [ $# -eq 1 ] && echo " was" || echo "s were") not recognised:" + for toolrepo in "$@"; do + printVerbose "Finding suggestions for '${toolrepo}'[${#toolrepo}]" + suggestion=$(toolSuggestion "${toolrepo}") + # Use printf to print without the printSoftError prefix for cleaner output + if [ -n "${suggestion}" ]; then + printf " %-${minlen}s [Did you mean '%s'?]\n" "${toolrepo}" "${suggestion}" >&2 + else + printf " %${minlen}s\n" "${toolrepo}" >&2 + fi + done + printError "Check port name(s) and retry command." + fi + +} + + + +dedupStringList() +{ delim="$1" && shift + str="$1" + echo "${str}"| awk -v delim="${delim}" ' + { dlm=""; for (i=1; i<=NF; i++) {if (!seen[$i]++) {printf "%s%s", dlm, $i};dlm=delim};print ""}' +} + +# generateInstallGraph +# generates a file with details for packages that are to be installed from +# the in-use repository, reporting errors if ports were invalid and +# triggering dependency graph population +# inputs: $1 the file to use for validated ports +# $* requested list of packages to install +# return: 0 for success (output of pwd -P command) +# 8 if error +generateInstallGraph(){ + installList="{}" + printDebug "Parsing list of packages to install and verifying validity" + portsToInstall="$1" # start with the initial list + portsToInstall=$(dedupStringList ' ' "${portsToInstall}") + + # Create the following file here to trigger cleanup - otherwise, multiple + # tempfiles could be created depending on dependency graph depth + invalidPortAssetFile=$(mktempfile "invalid" "port") + addCleanupTrapCmd "rm -rf ${invalidPortAssetFile}" + if ! runLogProgress " addToInstallGraph \"install\" \"${invalidPortAssetFile}\" \"${portsToInstall}\"" \ + "Creating install graph" "Created install graph" "linkcheck"; then + if [ -e "${invalidPortAssetFile}" ]; then + printSoftError "Cannot complete install request" + while read invalidPort; do + printf "%s\n" "${invalidPort}" + done < "${invalidPortAssetFile}" + exit 1 + else + printError "Unexpected error while creating install graph. Correct errors and retry command." + fi + fi + + # shellcheck disable=SC2154 + if ! ${doNotInstallDeps} && { (! ${reinstall} && ! ${reinstallDeps}) || (${reinstall} && ${reinstallDeps}); }; then + # doNotInstallDeps was not explicitly set and we are either not reinstalling(vanilla install) or we + # are reinstalling AND the reinstallDependencies flag is set; we do not want + # to reinstall a package and all it's dependencies by default, just the package itself + printVerbose "Calculating dependancy graph" + if ! runLogProgress "createDependancyGraph \"${invalidPortAssetFile}\"" \ + "Creating dependancy graph" "Created dependancy graph" "linkcheck"; then + if [ -e "${invalidPortAssetFile}" ]; then + printSoftError "The following ports cannot be installed: " + while read invalidPort; do + printf "${WARNING} %s\n" "${invalidPort}" + done < "${invalidPortAssetFile}" + printError "Confirm port names, remove any 'port' suffixes and retry command." + else + printError "Unexpected error while creating dependancy graph. Correct errors and retry command." + fi + fi + else + printVerbose "- Skipping dependency analysis" + fi + + # shellcheck disable=SC2154 + if "${reinstall}"; then + printVerbose "Not pruning already installed packages as reinstalling" + else + parseGraph + fi +} + +parseGraph() +{ + printDebug "Parsing graph for valid entries" + # shellcheck disable=SC2154 + if "${downloadOnly}"; then + # Download the pax files, even if already installed or if they would + # fail validation - the user wants the files downloaded for whatever reason + return 0 + fi + if ! processActionScripts "parseGraphPre" "${installList}"; then + exit 1 + fi + + # Prune already installed packages at the requested level; compare the + # incoming file name against the port name, version and release already on the system + # - seems to be the easiest comparison since some data is not in zopen_release vs metadata.json + # and a local pax won't have a remote repo but should have a file name! + +installList=$(jq --argjson install_list "${installList}" ' +. as $input | +{ + "installqueue": ( + $install_list.installqueue | map(select( .portname as $pnn | .asset.release as $r | +[$input[] | to_entries[] | select(.key == $pnn) | .value]| all(.product.release != $r) + ))) +} ' "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" +) + if ! processActionScripts "parseGraphPost"; then + exit 1 + fi +} + +checkIfPrereq(){ + # This jq query analyses the package database, looking for objects where the + # runtime dependency contains $1 (the removee). It extracts the keys from those + # objects into $dependents then outputs the flattened dependents array along with + # either true or false depending on if there was a prereq found (true) or if not + # false; the exit-status parameter then sets the exit code of jq dependning on that + # true/false value! + jq -r --exit-status --arg removee "$1" \ + '[.[] | to_entries | map(select(.value.product.runtime_dependencies[]?.name == $removee)) | .[].key] | . as $dependents | $dependents[], ($dependents | length > 0)' \ + "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" +} + +# spaceValidate +# Ensures there is sufficient available disk space for an operation to continue. +# Note that auto-expanding Filesystems might display the warning, even if an +# expansion is possible; as such, allow the user to continue. Caveat consumptor... +# Input: $1 - cacheBytes: number of bytes required for caching packages +# $2 - packageBytes: number of bytes require for uncompressing package +# stdout: Formatted file size to 3 decimal places with TGMk suffix +spaceValidate(){ + cacheBytes=$1 + packageBytes=$2 + + skip_size_check=$(jqw -re '.skip_size_check' "${ZOPEN_JSON_CONFIG}") + if "${skip_size_check}"; then + printVerbose "Skipping size check due to skip_size_check=${skip_size_check}" + return + fi + + cacheBytesKb=$(echo "scale=0; ${cacheBytes} / 1024" | bc) + packageBytesKb=$(echo "scale=0; ${packageBytes} / 1024" | bc) + spaceRequiredKb=$(echo "scale=0; ${cacheBytesKb} + ${packageBytesKb}" | bc) + + if ${reinstall}; then + # During a reinstall, the existing package size should remain constant as the + # package should overwrite the existing with the same files. However + # there might be a need to download the package into the cache - if + # autocacheclean is active, then this should be temporary; if not, then the + # cache will grow - inform the user either way + printVerbose "Reinstall of package, so package file size delta should be 0!" + spaceRequiredKb=$(echo "scale=0; (${cacheBytes}) / 1024" | bc) + if ! isCacheClean=$(zopen config --get autocacheclean); then + printError "Could not determine autocacheclean status" + fi + if ${isCacheClean}; then + printInfo "During this operation, $(formattedFileSize "${spaceRequiredKb}") of disk space will be used." + else + printInfo "After this operation, $(formattedFileSize "${spaceRequiredKb}") of additional cache will be used." + fi + elif ${downloadOnly}; then + # To download to current dir, then only need to make sure there is enough disk space + # for the pax, the "cached" size + spaceRequiredKb=$(echo "scale=0; (${cacheBytes}) / 1024" | bc) + printInfo "After this operation, $(formattedFileSize "${spaceRequiredKb}") of additional disk space will be used." + else + # If not a reinstall, there is a space requirement for both the + # package and the expanded package during install; if autocache clean is on, + # some of that will be released + if ${isCacheClean}; then + printInfo "During this operation, up to $(formattedFileSize "${spaceRequiredKb}") of disk space will be required." + fi + printInfo "After this operation, $(formattedFileSize "${packageBytesKb}") of disk space will be used." + fi + availableSpaceKb=$(df -k "${ZOPEN_ROOTFS}" | sed "1d" | awk '{ print $3 }' | awk -F'/' '{ print $1 }') + if [ "${availableSpaceKb}" -lt "${spaceRequiredKb}" ]; then + printWarning "Your zopen file-system (${ZOPEN_ROOTFS}) only has $(formattedFileSize "${availableSpaceKb}") of available space." + fi + if ! ${yesToPrompts} || [ "${availableSpaceKb}" -lt "${spaceRequiredKb}" ]; then + while true; do + printf "Do you want to continue [y/n/a]? " + read continueInstall < /dev/tty + case "${continueInstall}" in + "y") break;; + "n") mutexFree "zopen"; printInfo "Exiting..."; exit 0 ;; + "a") yesToPrompts=true; break;; + *) echo "?";; + esac + done + fi +} + +processRepoInstallFile(){ + printVerbose "Beginning port installation" + mutexReq "zopen" + + processActionScripts "transactionPre" + if [ 0 -eq "$(echo "${installList}" | jq --raw-output '.installqueue| length')" ]; then + printInfo "- No packages to install" + return 0 + fi + + # shellcheck disable=SC2154 + if ${fileinstall}; then + : + else + if ! ${quiet} >/dev/null 2>&1; then + xIFS=$IFS + IFS=' ' + hdr=false + for installee in $(echo "${installList}" | jq --raw-output '.installqueue | map( (select(.installtype=="install") | .portname| sub(" ";"") ))| @sh'); do + installee=$(echo "${installee}" |tr -d "\'" ) + [ -z "${installee}" ] && continue + if ! ${hdr}; then + printHeader "Installing the following packages:" + hdr=true + fi + printInfo "${installee}" + + done + hdr=false + for dependee in $(echo "${installList}" | jq --raw-output '.installqueue | map( (select(.installtype=="dependancy") | .portname| sub(" ";"") ))| @sh'); do + dependee=$(echo "${dependee}" |tr -d "\'" ) + [ -z "${dependee}" ] && continue + if ! ${hdr}; then + printHeader "Dependent packages to install:" + hdr=true + fi + printInfo "${dependee}" + done + IFS=${xIFS} + fi + cacheSpaceRequired=$(echo "${installList}" | jq --raw-output '.installqueue| map(.asset.size)| reduce .[] as $total (0; .+($total|tonumber))') + actualRequiredBytes=$(echo "${installList}" | jq --raw-output '.installqueue| map(.asset.expanded_size)| reduce .[] as $total (0; .+($total|tonumber))') + spaceValidate "${cacheSpaceRequired}" "${actualRequiredBytes}" + fi + + processActionScripts "transactionPre" + for installurl in $(echo "${installList}" | jq --raw-output '.installqueue |map( (.asset.url| sub(" ";"") ))| @sh'); do + printVerbose "Analysing :'${installurl}'" + installurl=$(echo "${installurl}" | tr -d "' ") + getInstallFile "${installurl}" + if $downloadOnly; then + continue + fi + installFile="${installurl##*/}" + if [ ! "${installFile%.zos.pax.Z}" = "${installFile}" ]; then + # Found zos.pax.Z format + if ! installFromPax "${installFile}"; then + printError "Package installation terminated" + fi + else + printError "Unrecognised install file format" + fi + done + processActionScripts "transactionPost" + mutexFree "zopen" + printVerbose "Port installation complete" +} + +getInstallFile() +{ + installurl="$1" + + downloadToDir="${ZOPEN_ROOTFS}/var/cache/zopen" + if $downloadOnly; then + downloadToDir="." + else + downloadToDir="${ZOPEN_ROOTFS}/var/cache/zopen" + fi + + if [ -e "${downloadToDir}/${installurl##*/}" ]; then + printVerbose "Install file '${installurl##*/}' already in local cache at '${downloadToDir}'" + else + [ -e "${downloadToDir}" ] || mkdir -p "${downloadToDir}" + [ -w "${downloadToDir}" ] || printError "No permission to save install file to '${downloadToDir}'. Check permissions and retry command." + printVerbose "Downloading installable file" + if ! runAndLog "cd ${downloadToDir} && curlCmd -L ${installurl} -O >/dev/null 2>&1"; then + printError "Could not download from ${installurl}. Correct any errors and potentially retry" + fi + fi + + # Check if this is a pax-flie install; in which case there is no "remote" JSON + case "${installurl}" in + file://*) return 0 ;; + *) :;; + esac + + metadataFile="$(basename "${installurl}").json" + if [ -e "${downloadToDir}/${metadataFile}" ]; then + printVerbose "Corresponding metadata '${metadataFile}' already in local cache" + else + printVerbose "Downloading corresponding metadata" + # check if it is in the same location just with a different suffix (as in a mirror) + # if not, likely is the original Github repo which uses a subdirectory for the metadata + # Try again with this URL + metadataJSONURL="${installurl}.json" + + printVerbose "Checking for existence of remote metadata file" + if curlOut=$(curlCmd -s -o /dev/null -I -w "%{http_code}" "${metadataJSONURL}"); then + # test for 404 - if it is, then this is likely the Main GitHub repo which hosts + # the metadata as a release so use alternative URL + if [ "${curlOut}" = "404" ]; then + metadataJSONURL_ext="$(dirname "${installurl}")/metadata.json" + printVerbose " Metadata not found at '${metadataJSONURL}'. Trying '${metadataJSONURL_ext}'" + metadataJSONURL="${metadataJSONURL_ext}" + fi + else + printSoftError "Curl issue trying to download from '${metadataJSONURL}'" + [ -n "${curlOut}" ] && printSoftError + exit 1 + fi + + if ! curlOut=$(cd "${downloadToDir}" && curlCmd -L "${metadataJSONURL}" -o "${metadataFile}"); then + printSoftError "Curl issue trying to download from '${metadataJSONURL}'" + [ -n "${curlOut}" ] && printSoftError + exit 1 + else + printVerbose "Metadata downloaded from '${metadataJSONURL}' to '${metadataFile}'" + fi + + if command -v chtag >/dev/null 2>&1; then + # Curl currently does not know on z/OS to set the text flag for text files + printVerbose "Metadata file downloaded, ensuring 'text' flag set for z/OS" + chtag -t "${metadataFile}" + fi + fi +} + +extractMetadataFromPax() +{ + # If there is an issue with the pax or the target, it is possible for pax + # itself to report an error and sit waiting for user input; + if ! pax -rf "$1" -s "%[^/]*/%/tmp/%" '*/metadata.json' ; then + if ! details=$(pax -rf "$1" -s "%[^/]*/%/tmp/%" '*/package.json'); then + printSoftError "Could not extract package metadata from file '$1'." + [ -n "${details}" ] && printSoftError "Details: ${details}" + return 1 + else + echo "/tmp/package.json" + fi + else + echo "/tmp/metadata.json" + fi +} + +installFromPax() +{ + pax="${downloadToDir}/$1" + printDebug "Installing from '${pax}'" + + if ! metadatafile=$(extractMetadataFromPax "${pax}"); then + return 1 + fi + + # Ideally we would use the following, + # name=$(jq --raw-output '.product.name' "${metadatafile}") + # but name does not always map to the actual repo package name at present! + # The Github API also lowercases repository names when getting all + # repositories which causes issues for NATSport for example. As such, we + # can grab the name from .product.repo, lowercase and it should match the + # metadata + printVerbose "Extracting product name from .product.repo" + + name=$(jq --raw-output '.product.repo | capture(".+/(?[^/].+)(port)")| .name' \ + "${metadatafile}") + if [ -z "${name}" ] || [ "${name##*[^ ]*}" = "" ]; then + printError "Unable to determine name from .product.repo in '${metadatafile}'. Check metadata is correct." + fi + name=$(awk -v n="${name}" 'BEGIN{print (tolower(n))}') + + + if ! processActionScripts "installPre" "${name}" "${metadatafile}" "${pax}"; then + skip_broken=$(jq -re '.skip_broken' "${ZOPEN_JSON_CONFIG}") + if "${skip_broken}"; then + printError "Failed installation pre-requisite check(s) for '${name}'. Correct previous errors and retry command" + fi + printSoftError "Skipping package '${name}' due to failed pre-requisite check(s)" + return 0 # 0 as we have handled the issue + fi + + # Store current installation directory (if exists) + currentderef=$(cd "${ZOPEN_PKGINSTALL}/${name}/${name}" > /dev/null 2>&1 && pwd -P) + + paxname="${installurl##*/}" + installdirname="${name}/${paxname%.pax.Z}" # Use full pax name as default + + baseinstalldir="${ZOPEN_PKGINSTALL}" + paxredirect="-s %[^/]*/%${ZOPEN_PKGINSTALL}/${installdirname}/%" + + printDebug "Check for existing directory for version '${installdirname}'" + if [ -d "${ZOPEN_PKGINSTALL}/${installdirname}" ]; then + printVerbose "- Clearing existing directory and contents at '${ZOPEN_PKGINSTALL}/${installdirname}'" + rm -rf "${ZOPEN_PKGINSTALL:?}/${installdirname}" || printError "Installation directory resolved to '/'; cannot recursively clear contents!" + fi + + if [ ! -e "${pax}" ]; then + printError "Pax file to install not available at '${pax}'" + elif [ ! -r "${pax}" ]; then + printError "Pax file to install not readable at '${pax}'" + fi + + # shellcheck disable=SC2154 + if ! runLogProgress "pax -rf ${pax} -p p ${paxredirect} >/dev/null 2>&1" \ + "Expanding file: ${pax}" "Expanded file: ${pax}"; then + printSoftError "Unexpected errors during unpaxing, package directory state unknown" + printError "Use zopen alt to select previous version to ensure known state" + fi + + setactiveLcl=true + if [ -e "${ZOPEN_PKGINSTALL}/${name}/${name}/.pinned" ]; then + printWarning "Current version of ${name} is pinned; not setting updated version as active" + printWarning "Remove .pinned file and run 'zopen alt meta' to use new version" + setactiveLcl=false + unInstallOldVersion=false + fi + # shellcheck disable=SC2154 + if $setactive && $setactiveLcl; then + if [ -L "${ZOPEN_PKGINSTALL}/${name}/${name}" ]; then + printDebug "Removing old symlink '${ZOPEN_PKGINSTALL}/${name}/${name}'" + rm -f "${ZOPEN_PKGINSTALL}/${name}/${name}" + fi + if ! ln -s "${ZOPEN_PKGINSTALL}/${installdirname}" "${ZOPEN_PKGINSTALL}/${name}/${name}"; then + printError "Could not create symbolic link name" + fi + if ! ${nosymlink}; then + if ! runLogProgress "mergeIntoSystem \"${name}\" \"${ZOPEN_PKGINSTALL}/${installdirname}\" \"${ZOPEN_ROOTFS}\"" \ + "Merging ${name} into symlink mesh" "Merged ${name} into symlink mesh"; then + printSoftError "Unexpected errors merging symlinks into mesh" + printError "Use zopen alt to select previous version to ensure known state" + fi + fi + + printVerbose "- Checking for env file" + if [ -f "${ZOPEN_PKGINSTALL}/${name}/${name}/.env" ] || [ -f "${ZOPEN_PKGINSTALL}/${name}/${name}/.appenv" ]; then + printVerbose "- .env file found, adding to profiled processing" + mkdir -p "${ZOPEN_ROOTFS}/etc/profiled/${name}" + cat << EOF > "${ZOPEN_ROOTFS}/etc/profiled/${name}/dotenv" +curdir=\$(pwd) +cd "${ZOPEN_PKGINSTALL}/${name}/${name}" >/dev/null 2>&1 +# If .appenv exists, source it as it's quicker +if [ -f ".appenv" ]; then + . ./.appenv +elif [ -f ".env" ]; then + . ./.env +fi +cd \${curdir} >/dev/null 2>&1 +EOF + if [ "${name}" = "meta" ]; then + printVerbose "Meta is handled by the zopen-meta-init-refresh script" + # running it's setup now can affect the currently running meta + else + printVerbose "- Running any setup scripts in sub-shell to preserve environment" + ( # Prevent cd from modifying current environment + cd "${ZOPEN_PKGINSTALL}/${name}/${name}" && [ -r "./setup.sh" ] && ./setup.sh >/dev/null + ) + fi + fi + fi + if ${unInstallOldVersion}; then + printDebug "New version merged; checking for orphaned files from previous version" + # This will remove any old symlinks or dirs that might have changed in an upgrade + # as the merge process overwrites existing files to point to different version + unsymlinkFromSystem "${name}" "${ZOPEN_ROOTFS}" "${currentderef}/.links" "${baseinstalldir}/${name}/${name}/.links" + fi + + if $setactive && $setactiveLcl; then + printDebug "Marking this version as installed" + touch "${ZOPEN_PKGINSTALL}/${name}/${name}/.active" + installedList="${name} ${installedList}" + syslog "${ZOPEN_LOG_PATH:-${ZOPEN_ROOTFS}/var/log}/audit.log" "${LOG_A}" "${CAT_INSTALL},${CAT_PACKAGE}" "DOWNLOAD" "handlePackageInstall" "Installed package:'${name}';version:${downloadFileVer};install_dir='${baseinstalldir}/${installdirname}';" + addToInstallTracker "${name}" + # Some installation have installation caveats + installCaveat=$(jq -r '.product.install_caveats // empty' "${metadatafile}" 2>/dev/null) + if [ -n "$installCaveat" ]; then + printf "${name}:\n%s\n" "${installCaveat}">> "${ZOPEN_ROOTFS}/var/cache/install_caveats.tmp" + fi + + processActionScripts "installPost" "${name}" + fi + printInfo "${NC}${GREEN}Successfully installed ${name}${NC}" +} + + +getActivePackageDirs() +{ + (unset CD_PATH; cd "${ZOPEN_PKGINSTALL}" && find ./*/. ! -name . -prune -type l) +} + + +# processActionScripts +# runs any scriptlets that are applicable to the current phase of an administration +# command (install/update/remove/alternative) +# inputs: $1 the phase of the transaction that is currently executing +# $2 the name of the package being administered +# return: 0 for success (nb. Warnings may haeve been printed to screen) +# 8 on error +processActionScripts() +{ + printVerbose "Processing phase '${1}' scriptlets" + [ $# -lt 1 ] && printError "Internal error; missing action phase" + phase=$1 + shift # Drop the initial parameter + + case "${phase}" in + "installPre") scriptletDir="${ZOPEN_SCRIPTLET_DIR}/installPre";; + "installPost") scriptletDir="${ZOPEN_SCRIPTLET_DIR}/installPost";; + "removePre") scriptletDir="${ZOPEN_SCRIPTLET_DIR}/removePre";; + "removePost") scriptletDir="${ZOPEN_SCRIPTLET_DIR}/removePost";; + "transactionPre") scriptletDir="${ZOPEN_SCRIPTLET_DIR}/transactionPre";; + "transactionPost") scriptletDir="${ZOPEN_SCRIPTLET_DIR}/transactionPost";; + "parseGraphPre") scriptletDir="${ZOPEN_SCRIPTLET_DIR}/parseGraphPre";; + "parseGraphPost") scriptletDir="${ZOPEN_SCRIPTLET_DIR}/parseGraphPost";; + *) assertFailed "Invalid process action phase '${phase}'" + esac + printVerbose "Running script[s] from '${scriptletDir}'" + + if [ ! -d "${scriptletDir}" ]; then + printDebug "No script directory for phase: ${phase}" + return 0 + fi + curDir=$(pwd -P) + unset CDPATH; + cd "${scriptletDir}" || return 1 + + scriptletRcFile=$(mktempfile "zopen_actionscripts" ".err") + find . -type l | while IFS= read -r scriptletFile; do + if [ ! -r "${scriptletFile}" ]; then + printWarning "Script '${scriptletDir}/${scriptletFile}' is not readable. Check permissions" >> "${scriptletRcFile}" 2>&1 + continue + fi + printVerbose "Attempting to run script '${scriptletFile}'" + # Run script in a subshell to prevent environment modification + printf "${CRSRSOL}${ERASELINE}Running ${scriptletFile}" + scriptletOutput=$({ + # shellcheck disable=SC1090 + # shellcheck disable=SC2240 + . "$scriptletFile" "$@" + echo $? # Append exit status to output + } 2>&1 + ) + printf "${CRSRSOL}${ERASELINE}${CRSRSOL}" + scriptletRc=$(echo "${scriptletOutput}" | tail -n 1) # Extract exit status + scriptletBody=$(echo "${scriptletOutput}" | sed '$d') # Extract script output + if [ "${scriptletRc}" -ne 0 ]; then { + printWarning "Scriptlet '${scriptletFile}' failed with exit code ${scriptletRc}" + printWarning "Details:" + printWarning "${scriptletBody}" + } >> "${scriptletRcFile}" 2>&1 + elif ${verbose}; then + if [ -n "${scriptletBody}" ]; then + # Show the scriptlet output + echo "${scriptletBody}" + fi + fi + done + + if [ -s "${scriptletRcFile}" ]; then # Check if errorLog is non-empty + printWarning "One or more scripts failed:" + cat "${scriptletRcFile}" + rm -f "${scriptletRcFile}" + cd "${curDir}" || printSoftError "Could not return to previous directory" + return 1 + fi + rm -f "${scriptletRcFile}" + cd "${curDir}" || printSoftError "Could not return to previous directory" + return 0 +} + +# updatePackageDB +# Updates/generates the installed package database +# return: 0 for success +# 8 in error +updatePackageDB() +{ + printVerbose "Updating the installed package tracker db" + + pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" + if [ -e "${pdb}" ]; then + backup=$(mktempfile "updatepdb" "bkk") + addCleanupTrapCmd "rm -rf ${backup}" + cp "${pdb}" "${backup}" + rm "${pdb}" + else + printVerbose "No current package database [new install?]. Creating empty array[]" + printf "[\n]\n" > "${pdb}" + fi + + if [ "$(unset CD_PATH; cd "${ZOPEN_PKGINSTALL}"; ls -A | wc -l)" -eq 0 ]; then + printVerbose "No packages found to add to database [new install?]" + return + fi + + + if ! pkgdirs=$(getActivePackageDirs); then + printError "Unable to update the package db" + fi + for pkgdir in ${pkgdirs}; do + metadataFile="${ZOPEN_PKGINSTALL}/${pkgdir}/metadata.json" + if [ ! -e "${metadataFile}" ]; then + # TODO: Fallback to filesystem analysis [depending on backward compatability] + printWarning "No metadata.json found in '${ZOPEN_PKGINSTALL}/${pkgdir}' [recent install/old package?]" + continue + fi + escapedJSONFile=$(mktempfile "escaped" "json") + addCleanupTrapCmd "rm -rf ${escapedJSONFile}" + stripControlCharacters "${metadataFile}" "${escapedJSONFile}" + if [ ! -e "${pdb}" ]; then + echo "[]" > "${pdb}" + fi + + # Grab the repo name from the metadata.json and lowercase it to match + # what the Github API returns when enumerating repsitories! + mdj=$(jq '(.product.repo | capture(".+/(?[^/].+)(port)")| .name | ascii_downcase) as $name + | [{($name):.}]' \ + "${escapedJSONFile}") + + if [ -z "${mdj}" ]; then + pkg=$(basename "${pkgdir}") + printWarning "Cannot locate metadata for installed package '${pkg}' at location '${metadataFile}'. Check file existence and permissions" + else + printVerbose "Valid metadata found for package '${pkg}' - add to install tracker" + if ! jq --argjson mdj "${mdj}" '. += $mdj' \ + "${pdb}" > \ + "${pdb}.working"; then + [ -e "${pdb}.working" ] && "${pdb}.working" + [ -e "${pdb}" ] && mv -f "${pdb}" "${pdb}.broken" # Save for potential diagnostics + printSoftError "Could not add metadata for '$(basename "${pkgdir}")' to install tracker." + printError "Run 'zopen init --refresh' to attempt database regeneration and re-run command." + fi + mv "${pdb}.working" "${pdb}" + fi + done +} + +stripControlCharacters(){ + [ ! -f "$1" ] && assertFailed "No input file specified for parsing!" + [ -e "$2" ] && assertFailed "Output file exists so cannot be used for output!" + tr -d '[:cntrl:]' > "$2" < "$1" + +} +# JSONcontrolChar2Unicode +# ensures escaping of characters in JSON. For example, if a caveat has an +# unescaped '\n'/0x0A jq will fail to process it. Escape control characters +# 0x00->0x1F and reverse solidus. Note this should also only process unescaped +# sequences by checking whether the character prior to the sequence is not a +# reverse-solidus (so start-of-line [^] or any character [^\] ). If there was +# a preceding character, then capture/use that in the regex (\1) so it does not +# get discarded +# +# inputs: $1 the input JSON file +# $2 the output file, with sanitised JSON +# return: 0 for success (nb. Warnings may haeve been printed to screen) +# 8 on error +JSONcontrolChar2Unicode() { + [ ! -f "$1" ] && assertFailed "No input file specified for parsing!" + [ -e "$2" ] && assertFailed "Output file exists so cannot be used for output!" + sed -E ' # Note this is a long string! + s/\\/\\\\/g; # Escape reverse-solidus; the following are the control chars + s/(^|[^\\])\x00/\1\\u0000/g; s/(^|[^\\])\x01/\1\\u0001/g; + s/(^|[^\\])\x02/\1\\u0002/g; s/(^|[^\\])\x03/\1\\u0003/g; + s/(^|[^\\])\x04/\1\\u0004/g; s/(^|[^\\])\x05/\1\\u0005/g; + s/(^|[^\\])\x06/\1\\u0006/g; s/(^|[^\\])\x07/\1\\u0007/g; + s/(^|[^\\])\x08/\1\\u0008/g; s/(^|[^\\])\x09/\1\\u0009/g; + s/(^|[^\\])\x0A/\1\\u000A/g; s/(^|[^\\])\x0B/\1\\u000B/g; + s/(^|[^\\])\x0C/\1\\u000C/g; s/(^|[^\\])\x0D/\1\\u000D/g; + s/(^|[^\\])\x0E/\1\\u000E/g; s/(^|[^\\])\x0F/\1\\u000F/g; + s/(^|[^\\])\x10/\1\\u0010/g; s/(^|[^\\])\x11/\1\\u0011/g; + s/(^|[^\\])\x12/\1\\u0012/g; s/(^|[^\\])\x13/\1\\u0013/g; + s/(^|[^\\])\x14/\1\\u0014/g; s/(^|[^\\])\x15/\1\\u0015/g; + s/(^|[^\\])\x16/\1\\u0016/g; s/(^|[^\\])\x17/\1\\u0017/g; + s/(^|[^\\])\x18/\1\\u0018/g; s/(^|[^\\])\x19/\1\\u0019/g; + s/(^|[^\\])\x1A/\1\\u001A/g; s/(^|[^\\])\x1B/\1\\u001B/g; + s/(^|[^\\])\x1C/\1\\u001C/g; s/(^|[^\\])\x1D/\1\\u001D/g; + s/(^|[^\\])\x1E/\1\\u001E/g; s/(^|[^\\])\x1F/\1\\u001F/g; + ' "$1" > "$2" +} + +# addToInstallTracker +# Records the installation into the database that tracks which packages have +# been installed +# inputs: $1 the name of the package +# return: 0 for success (nb. Warnings may haeve been printed to screen) +# 8 on error +addToInstallTracker() +{ + pkg=$1 + pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" + if [ ! -e "${pdb}" ]; then + # Generate the packageDB + printWarning "No package tracker found, regenerating [subsequent runs will be faster]" + updatePackageDB + fi + metadataJson=$(cat "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}/metadata.json") + if ! jq --argjson mdj "[{\"${pkg}\":${metadataJson}}]" \ + "if any(.[]; has(\"${pkg}\")) then . |= map( if has(\"${pkg}\") then \$mdj[] else . end ) else . + \$mdj end" \ + "${pdb}" > \ + "${pdb}.working"; then + printError "Could not update metadata for '${pkg}' in package tracker. Run zopen -init -re-init to attempt regeneration." + fi + mv "${pdb}.working" "${pdb}" +} + +# removeFromInstallTracker +# Removes the installation of the package from the database, making it +# uninstalled +# inputs: $1 the name of the package +# return: 0 for success (nb. Warnings may haeve been printed to screen) +# 8 on error +removeFromInstallTracker() +{ + pkg=$1 + pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" + if [ ! -e "${pdb}" ]; then + # Generate the packageDB + printWarning "No package tracker found, regenerating [subsequent runs will be faster]" + updatePackageDB + fi + if ! jq \ + "map(select(has(\"${pkg}\") | not))" \ + "${pdb}" > \ + "${pdb}.working"; then + printError "Could not add metadata for '${pkg}' to install tracker. Run zopen --re-init to attempt regeneration." + fi + mv "${pdb}.working" "${pdb}" +} + +jqfunctions() +{ + # Return a set of helper functions for jq that can be prepended to + # any jq query + # pl(s;c;n) - padLeft with character 'c' to length 'n' + # pr(s;c;n) - padRight with chacater 'c' to length 'n' + # c(s;l) - center the string 's' in a string of length 'l' + # r(dp) - round decimal to 'dp' decimal places (needs '*' & '/' 10^dp hack) + # shellcheck disable=SC2016 + printf "%s;%s;%s;%s;" \ + 'def pl(s;c;n):c*(n-(s|length))+s' \ + 'def pr(s;c;n):s+c*(n-(s|length))' \ + 'def c(s;c;l):(((l - (s|length))/2 | floor ) // 0) as $lp|((l - (s|length) - $lp) // 0 )as $rp|pl("";c; $lp) + s + pr("";c; $rp)' \ + 'def r(dp):.*pow(10;dp)|round/pow(10;dp)' +} + +diskusage() +{ + path=$1 + # awk to "trim" output" + if ! size=$(du -kts "${path}" | awk '{print ($1)}'); then + printError "Unable to generate disk usage (du) report for '${path}'" fi echo "${size}" } @@ -1458,7 +3016,7 @@ diskusage() formattedFileSize() { filesize=$1 # in kb - # Use awk rather than $((..)) to get floating points, using the + # Use awk rather than $((..)) to get floating points, using the # "repeated divisions and count" method to generate an offset echo "${filesize}" | awk '{ num = $1; @@ -1473,9 +3031,366 @@ formattedFileSize() num = num / 1000; unit = "M"; } printf "%.3f%s\n", num, unit; - }' + }' } +# Used to perform validation of config values. Legacy +# values might use 0:1 or true:false so handle both, +# ignoring case +isFalse() +{ + case "$1" in + 0|[Ff][Aa][Ll][Ss][Ee] ) return 0;; + *) return 1;; + esac +} +isTrue() +{ + case "$1" in + 1|[Tt][Rr][Uu][Ee] ) return 0;; + *) return 1;; + esac +} + +jqw() +{ + if ! type jq >/dev/null 2>&1; then + printSoftError "Cannot locate 'jq'." + printSoftError "If recovering a system, run . ./.env from the following location:" + printError " /usr/local/zopen/jq/ and retry command" + fi + jq "$@" +} + +# Lookup the list of ports and find the closest suggestion. This uses +# awk to parse the dictionary directly from the cache file [piped via jq] +toolSuggestion(){ + word="$1" + threshold=4 # If the word is too far from alternatives, do not suggest + jq -r '.release_data | keys[]' "${JSON_CACHE}" | \ + awk -v s1="${word}" -v th="${threshold}" ' + BEGIN { best = ""; current=th;} + END { if (current <= th) print best } + { + s2=$1 + len1 = length(s1) + len2 = length(s2) + for (i = 0; i <= len1; i++) d[i,0] = i + for (j = 0; j <= len2; j++) d[0,j] = j + for (i = 1; i <= len1; i++) { + for (j = 1; j <= len2; j++) { + cost = (substr(s1, i, 1) == substr(s2, j, 1)) ? 0 : 1 + + del = d[i-1,j] + 1 + ins = d[i,j-1] + 1 + # Renamed variable to avoid conflict with the built-in "sub" function in awk. + substitution = d[i-1,j-1] + cost + + min = del + if (ins < min) min = ins + if (substitution < min) min = substitution + d[i,j] = min + } + } + distance=d[len1,len2] + if (distance == 0) { print s2; exit } # Found a match - should not happen + if (distance < current) { + best=s2 + current = distance + } + } + ' + +} + + +levenshtein() { + word1="$1" + word2="$2" + + # Calculates the distance and prints it to stdout. + awk -v s1="$word1" -v s2="$word2" ' + BEGIN { + len1 = length(s1) + len2 = length(s2) + for (i = 0; i <= len1; i++) d[i,0] = i + for (j = 0; j <= len2; j++) d[0,j] = j + for (i = 1; i <= len1; i++) { + for (j = 1; j <= len2; j++) { + cost = (substr(s1, i, 1) == substr(s2, j, 1)) ? 0 : 1 + + del = d[i-1,j] + 1 + ins = d[i,j-1] + 1 + # Renamed variable to avoid conflict with the built-in "sub" function in awk. + substitution = d[i-1,j-1] + cost + + min = del + if (ins < min) min = ins + if (substitution < min) min = substitution + d[i,j] = min + } + } + print d[len1,len2] + }' +} + +# Finds the closest match for a misspelled word from a list of valid words +findSuggestion() { + misspelled="$1" + dictionary="$2" + best_match="" + min_distance=4 + + for word in ${dictionary}; do + distance=$(levenshtein "${misspelled}" "${word}") + if [ "${distance}" -lt "${min_distance}" ]; then + min_distance=${distance} + best_match="${word}" + fi + done + echo "${best_match}" +} + +validateConfigValue() +{ + type=$1 + key=$2 + value=$3 + default=$4 + enums=$5 + case "${type}" in + boolean) + case "${value:=$default}" in + "0"|"1"|[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee]) :;; # Valid value + "null") value="$default" # jq found no entry (new config?) + ;; + *) showConfigParmWarning "${key}" "${value}" "true|false" "${default}" + value="$default" + ;; + esac + ;; + enum) + if [ "${value:=$default}" = "null" ]; then + value="$default" # jq found no entry (new config?) + elif echo "${enums}" | awk -v RS='|' -v val="${value}" ' + BEGIN {notfound=1} + { if (toupper(val) == toupper($0)) {notfound=0; exit 0;} } + END { exit notfound }'; then + : # Value value + else + showConfigParmWarning "${key}" "${value}" "${enums}" "${default}" + value="$default" + fi + ;; + *) assertFailed "Unknown config value type '${type}'" + esac + eval "$key=\"${value}\"" +} + +# Map functions - no arrays or maps in POSIX.1 shell so improvise +# Default separators +DEFAULT_KVSEP="=" +DEFAULT_ENTRYSEP="|" + +# Create a new map with specified separators +createMap() { + kvsep="$1" + entrysep="$2" + echo "${kvsep} ${entrysep} " +} + +# Get key-value separator from map +getKVSeparator() { + map=$1 + echo "${map}" | awk '{print $1}' +} + +# Get entry separator from map +getEntrySeparator() { + map=$1 + echo "${map}" | awk '{print $2}' +} + +# Get entries from map +getEntries() { + map=$1 + echo "${map}" | cut -d' ' -f3- +} + +# Add entry to map (initializes if blank) +addEntry() { + map="$1" + key="$2" + value="$3" + + if [ -z "${map}" ]; then + kvsep="$DEFAULT_KVSEP" + entrysep="$DEFAULT_ENTRYSEP" + entries="" + else + kvsep=$(getKVSeparator "${map}") + entrysep=$(getEntrySeparator "${map}") + entries=$(getEntries "${map}") + fi + newEntry="${key}${kvsep}${value}" + echo "${kvsep} ${entrysep} ${entries}${newEntry}${entrysep}" +} + +# Immutable add (fails if key exists) +immutableAddEntry() { + map="$1" + key="$2" + value="$3" + + if hasKey "${map}" "$key"; then + echo "${map}" + return 1 + else + addEntry "${map}" "$key" "$value" + return 0 + fi +} + +# Get value for a key +getValue() { + map="$1" + needle="$2" + kvsep=$(getKVSeparator "${map}") + entrysep=$(getEntrySeparator "${map}") + entries=$(getEntries "${map}") + + IFS="$entrysep" set -- $entries + for knv; do + key=$(echo "$knv" | awk -F"$kvsep" '{print $1}') + if [ "$key" = "$needle" ]; then + echo "$knv" | awk -F"$kvsep" '{print $2}' + return 0 + fi + done + return 1 +} + +# Check if key exists +hasKey() { + map="$1" + needle="$2" + getValue "${map}" "$needle" > /dev/null +} + +# Update value for a key +updateValue() { + map="$1" + needle="$2" + newValue="$3" + kvsep=$(getKVSeparator "${map}") + entrysep=$(getEntrySeparator "${map}") + entries=$(getEntries "${map}") + newMap="${kvsep} ${entrysep} " + + IFS="$entrysep" set -- $entries + for knv; do + key=$(echo "$knv" | awk -F"$kvsep" '{print $1}') + if [ "$key" = "$needle" ]; then + knv="${key}${kvsep}${newValue}" + fi + newMap="${newMap}${knv}${entrysep}" + done + echo "$newMap" +} + +# Delete a key +deleteKey() { + map="$1" + needle="$2" + kvsep=$(getKVSeparator "${map}") + entrysep=$(getEntrySeparator "${map}") + entries=$(getEntries "${map}") + newMap="${kvsep} ${entrysep} " + + IFS="$entrysep" set -- $entries + for knv; do + key=$(echo "$knv" | awk -F"$kvsep" '{print $1}') + if [ "$key" != "$needle" ]; then + newMap="${newMap}${knv}${entrysep}" + fi + done + echo "$newMap" +} + +# List all keys +listKeys() { + map="$1" + kvsep=$(getKVSeparator "${map}") + entrysep=$(getEntrySeparator "${map}") + entries=$(getEntries "${map}") + + IFS="$entrysep" set -- $entries + for knv; do + echo "$knv" | awk -F"$kvsep" '{print $1}' + done +} + +# Print map entries +prettyPrint() { + map="$1" + wrap_width="${2:-80}" + indent="${3:-4}" + + kvsep=$(getKVSeparator "${map}") + entrysep=$(getEntrySeparator "${map}") + entries=$(getEntries "${map}") + + printf "%s" "${entries}" | awk -v kvsep="${kvsep}" -v RS="${entrysep}" -v wrap="${wrap_width}" -v lhindent="${indent}"\ + ' + { + split($0, kv, kvsep) # Split into key/value pair on the key separator + keys[NR] = kv[1] # Store the key in the keys array [no maps in awk] + values[NR] = kv[2] # Store associated value in value array [at same offset!] + # Determine what the longest key name is as we use that to calculate the + # offset to write values at. + if (length(kv[1]) > maxlen) maxlen = length(kv[1]) + } + END { + indent = maxlen + 3 # Space between biggest key and start of value text + for (i = 1; i <= NR; i++) { # Loop all records + printf "%*s%-*s ", lhindent, "", maxlen, keys[i] # Output key + line = "" + # Need to format the value such that it breaks into sections + # then indented to where the initial line started - pretty! + split(values[i], words, /[ \t]+/) + for (j = 1; j <= length(words); j++) { + word = words[j] + printf "Parsed word:>>%s\n", word + if (length(line) + length(word) + 1 > wrap - indent) { + # There is no space in current line buffer for word - output and rebuild + print line + line = sprintf("%*s%*s%s", lhindent, "", indent, "", word) + } else { + # Space in buffer for word + if (length(line) == 0) { + # First word so just use as-is in buffer + line = sprintf("%*s%s", lhindent, "", word) + printf "Started new line:>>%s\n", line + } else { + # ! first word so insert space char and append to buffer + line = line " " word + printf "Line is now :>>%s\n", line + } + } + } + # Anything left in the line buffer can be written + if (length(line) > 0) print line + } + }' +} + + + +# Main code + + +# shellcheck disable=SC1091 +# shellcheck disable=SC2086 . ${INCDIR}/analytics.sh zopenInitialize diff --git a/include/scriptlets/installPost/autopkgpurge b/include/scriptlets/installPost/autopkgpurge new file mode 100755 index 000000000..ffba304be --- /dev/null +++ b/include/scriptlets/installPost/autopkgpurge @@ -0,0 +1,25 @@ +#!/bin/sh +this="autopkgpurge" +printVerbose "Running '${this}' script" +pkg=$1 +[ -z "${pkg}" ] && return +if type zopen-config-helper >/dev/null 2>&1; then + isactive=$(zopen-config-helper --get autopkgpurge) +else + isactive=false +fi + +if isTrue "${isactive}"; then + (zopen clean --unused "${pkg}" >/dev/null 2>&1) +elif isFalse "${isactive}"; then + printVerbose "autopkgpurge is disabled; no clean will occur" +else + printWarning "Invalid value '${isactive}' for autopkgpurge config parameter [should be true or false]." + if type zopen-config-helper >/dev/null 2>&1; then + printWarning "Run zopen config --set autopkgpurge [true|false] to update configuration" + else + printWarning "Update zopen configuration file to contain a valid value for parameter 'autopkgpurge'" + fi + printWarning "Unused package clean will not be performed" +fi + diff --git a/include/scriptlets/installPost/man-db b/include/scriptlets/installPost/man-db new file mode 100755 index 000000000..b94fdb6ed --- /dev/null +++ b/include/scriptlets/installPost/man-db @@ -0,0 +1,22 @@ +#!/bin/sh +this="man-db" +printVerbose "Running '${this}' script" +if ! zopen list --installed | grep "^man-db\$"; then + return 0 # No man-db installed +else + echo "- Updating man database for '$1'" + # Check if package shipped any man pages + manDir="${ZOPEN_PKGINSTALL}/${1}/${1}/usr/share/man" + [ -d "${manDir}" ] || return 0 + # Man pages were shipped; scan from the directory - the man + # directories are symlinked from ZOPEN_ROOTFS/usr/share/man/manX + # so they all point to the same location anyway. + if ! mandbOut=$(mandb "${manDir}" >/dev/null 2>&1); then + printSoftError "man-db update completed with non-zero return code." + printSoftError "Details: ${mandbOut}" + printSoftError "Re-run 'mandb' manually for additional information." + return 1 + fi + echo "- Update of man-db complete." +fi +return 0 diff --git a/include/scriptlets/installPost/zopen-meta-init-refresh b/include/scriptlets/installPost/zopen-meta-init-refresh new file mode 100755 index 000000000..b0ec0d730 --- /dev/null +++ b/include/scriptlets/installPost/zopen-meta-init-refresh @@ -0,0 +1,9 @@ +#!/bin/sh +this="meta-init-refresh" +printVerbose "Running '${this}' script" +pkg=$1 +[ -z "${pkg}" ] && return +case "${pkg}" in + meta ) (zopen init --refresh -y) ;; + * ) :;; # No action +esac diff --git a/include/scriptlets/installPre/systemPreReqCheck b/include/scriptlets/installPre/systemPreReqCheck new file mode 100755 index 000000000..82669bf92 --- /dev/null +++ b/include/scriptlets/installPre/systemPreReqCheck @@ -0,0 +1,39 @@ +#!/bin/sh +this="systemPreReqCheck" +printVerbose "Running '${this}' script" +pkg=$1 +metadataFile=$2 + +[ -z "${pkg}" ] && return +# shellcheck disable=SC2154 +if ! ${bypassPrereqs}; then + systemPrereqs=$(jq -r '.product.system_prereqs // empty | map(.name) | join(" ")' "${metadataFile}" 2>/dev/null) + if [ -z "${systemPrereqs}" ]; then + systemPrereqs="${systemPrereqs} zos24" # set the min requirement as z/OS 2.4 + fi + if [ -n "$systemPrereqs" ]; then + ZOPEN_SYSTEM_PREREQ_SCRIPT="../../prereq.sh" + if [ ! -r "${ZOPEN_SYSTEM_PREREQ_SCRIPT}" ]; then + printSoftError "${ZOPEN_SYSTEM_PREREQ_SCRIPT} does not exist. Check file permissions and reinstall the meta package or reinitialize the zopen environment. If the error persists, open an issue." + return 1 + else + # shellcheck disable=SC1090 + . ${ZOPEN_SYSTEM_PREREQ_SCRIPT} + for prereq in $(echo "${systemPrereqs}" | xargs | tr ' ' '\n' | sort -u); do + printInfo "- Checking system pre-req requirement ${prereq}" + if command -V "${prereq}" >/dev/null 2>&1; then + if ! ( ${prereq} ); then + printSoftError "Failed system pre-req check \"${prereq}\". If you wish to bypass this, install with --bypass-prereq-checks" + return 1 + fi + else + printSoftError "Prereq \"${prereq}\" does not exist in ${ZOPEN_SYSTEM_PREREQ_SCRIPT}. Consider upgrading meta or open an issue if it persists." + return 1 + fi + done + fi + fi +else + syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE},${CAT_INSTALL}" "BYPASS" "handlePackageInstall" "Bypassing prereq checks ${systemPrereqs} for '${name}'." +fi + diff --git a/include/scriptlets/installPre/verifySignatureOfPax b/include/scriptlets/installPre/verifySignatureOfPax new file mode 100644 index 000000000..9420fb3ee --- /dev/null +++ b/include/scriptlets/installPre/verifySignatureOfPax @@ -0,0 +1,143 @@ +#!/bin/sh +this="verifySignatureOfPax" +printVerbose "Running '${this}' script" +pkg=$1 +metadataFile=$2 +paxFile=$3 + +if ${skipverify}; then + printWarning "Package validation with GPG explicitly skipped" + return 0 +fi +printInfo "- Performing GPG signature verification for '${pkg}' pax file '${paxFile}'" +if ! [ -e "${metadataFile}" ]; then + printError "Metadata not found for '${pkg}'" + return 1 +fi +metadataVersion=$(jq -r '.version_scheme' "${metadataFile}" 2>/dev/null) +[ -n "${metadataVersion}" ] || printError "Metadata version could not be read from metadata file '${metadataFile}'" +is_greater=$(echo "$metadataVersion > 0.1" | bc -l) + +if [ "$is_greater" -eq 1 ] && ! $skipverify; then + if ! command -v gpg> /dev/null; then + skipverify=false; + printWarning "GPG is not installed - package validation cannot be performed" + printWarning "Run 'zopen install gpg' or use the --skip-verify parameter" + return 1 + fi +else + printVerbose "Skipping verification" +fi + +# The following function will attempt to retrieve a value from the metadata for this +# package. It will attempt to find it in the metadata included inside the pax file +# and previously extracted and failing that, will fallback to checking the existence +# of the package in the cache [the metadata downloaded [might be]/is different] +extractFromMetaData(){ + key=$1 + mdf=$2 + if ! sRc=$(jq -e -r "${key}" "${mdf}"); then + mdf="${ZOPEN_ROOTFS}/var/cache/zopen/$(basename "$3").json" + [ -e "${mdf}" ] || return 1 + if ! sRc=$(jq -e -r "${key}" "${mdf}"); then + return 1 + fi + fi + echo "${sRc}" && return 0 +} + +# Extracting values and checking for errors +# If this is a local pax file, then there should not be a .product.pax in the +# metadata. Local paxes need to be installed with --skipverify since do not +# contain the .product.pax and .signature and .public_key entries could be invalid +# and unreliable/untrustworthy +if ! FILE_TO_VERIFY=$(extractFromMetaData ".product.pax" "${metadataFile}" "${paxFile}"); then + printWarning "Metadata not found for '${pkg}' - package validation cannot be performed" + printWarning "Use the --skip-verify parameter or configure the zopen environment" + printWarning "to enable local/offline installations without package verification." + return 1 +fi + +if [ ! "${FILE_TO_VERIFY}" = "$(basename "${paxFile}")" ]; then + printSoftError "Mismatch between pax file '${paxFile}' and metadata pax name '${FILE_TO_VERIFY}'" + return 1 +fi + +if ! SIGNATURE=$(extractFromMetaData ".product.signature" "${metadataFile}" "${paxFile}"); then + # Lack of a .signature indicates the package has not yet been signed so allow installation + printVerbose "No .signature found in the metadata for '${pkg}; continuing installation" + return 0 +fi + +if ! PUBLIC_KEY=$(extractFromMetaData ".product.public_key" "${metadataFile}" "${paxFile}"); then + # Since we found the .signature above, not finding the public key is not good; there is an + # error with the metadata, either corrupted in download, in the pax or initial generation + printSoftError "The metadata for '${pkg}' is inconsistent. Clear caches and retry command." + return 1 +fi + +# Create a temporary directory for GPG keyring +SIGNATURE_FILE=$(mktempfile "signedfile" ".asc") +PUBLIC_KEY_FILE=$(mktempfile "scriptpubkey" ".asc") +addCleanupTrapCmd "rm -rf ${SIGNATURE_FILE}" +addCleanupTrapCmd "rm -rf ${PUBLIC_KEY_FILE}" + +#TMP_GPG_DIR=$(mktempdir "gpg" "verify") +#SIGNATURE_FILE="${TMP_GPG_DIR}/signedfile.asc" +#PUBLIC_KEY_FILE="${TMP_GPG_DIR}/scriptpubkey.asc" +printf "%b" "${SIGNATURE}" | tr -d '"' > "${SIGNATURE_FILE}" +printf "%b" "$PUBLIC_KEY" | tr -d '"' > "${PUBLIC_KEY_FILE}" + + +if [ ! -f "${PUBLIC_KEY_FILE}" ]; then + printSoftError "Unable to locate created public key file" + return 1 +fi + +if ! startGPGAgent; then + printSoftError "Unable to start GPG agent" + return 1 +fi + +printVerbose "Importing public key to keyring file..." +KEYRING_FILE=$(mktempfile "pubring" ".kbx") +addCleanupTrapCmd "rm -rf ${KEYRING_FILE}" +addCleanupTrapCmd "rm -rf ${KEYRING_FILE}~" # Also remove a potential lock file? + + if ! gpg_output=$(gpgCmd --no-default-keyring --keyring "${KEYRING_FILE}" --batch --yes --import "$PUBLIC_KEY_FILE" 2>&1); then + printSoftError "Importing public key failed. Details:" + printSoftError "${gpg_output}" + printSoftError "Verification aborted." + return 1 + fi + printVerbose "${gpg_output}" + + # Verify that the key was imported successfully + printVerbose "Checking if public key is imported..." + + if ! gpg_output=$(gpgCmd --no-default-keyring --keyring "${KEYRING_FILE}" --check-sigs 2>&1); then + printSoftError "Public key was not imported. See output:\n${gpg_output}.\nVerification aborted." + return 1 + fi + + printVerbose "${gpg_output}" + + # Verify the signature + printInfo "- Verifying the gpg signature..." + if [ ! -f "${SIGNATURE_FILE}" ]; then + printSoftError "Signature file does not exist. Please raise an issue." + return 1 + fi + + gpg_output=$(gpgCmd --no-default-keyring --keyring "${KEYRING_FILE}" --verify "${SIGNATURE_FILE}" "${paxFile}" 2>&1) + printVerbose "${gpg_output}" + if ! echo "${gpg_output}" | grep -q "Good signature from"; then + printSoftError "Verification failed. Details:" + printSoftError "${gpg_output}" + return 1 + fi + + printInfo "- Signature successfully verified." + return 0 + + diff --git a/include/scriptlets/transactionPost/list_caveats b/include/scriptlets/transactionPost/list_caveats new file mode 100755 index 000000000..6e5baf9f7 --- /dev/null +++ b/include/scriptlets/transactionPost/list_caveats @@ -0,0 +1,7 @@ +#!/bin/sh +this="list_caveats" +printVerbose "Running '${this}' script" +caveats_file="${ZOPEN_ROOTFS}/var/cache/install_caveats.tmp" +[ -e "${caveats_file}" ] || return 0 +cat "${caveats_file}" +rm "${caveats_file}" diff --git a/include/scriptlets/transactionPost/man-db b/include/scriptlets/transactionPost/man-db new file mode 100755 index 000000000..285194538 --- /dev/null +++ b/include/scriptlets/transactionPost/man-db @@ -0,0 +1,29 @@ +#!/bin/sh +this="man-db" +printVerbose "Running '${this}' script" +fullScan=false +while [ $# -gt 0 ]; do + case "$1" in + "mandb") fullScan=true ;; + *):;; + esac + shift +done +if ! $fullScan; then + # The option to run a full mandb scan was not passed; do not run scan + return 0 +fi + +if ! zopen list --installed | grep "^man-db\$"; then + return 0 # No man-db installed +else + echo "- Updating man-db database after package changes..." + if ! mandbOut=$(mandb >/dev/null 2>&1); then + printSoftError "man-db update completed with non-zero return code." + printSoftError "Details: ${mandbOut}" + printSoftError "Re-run 'mandb' manually for additional information." + return 1 + fi + echo "- Update of man-db complete." +fi +return 0 diff --git a/include/scriptlets/transactionPost/zopen-clean-autocache b/include/scriptlets/transactionPost/zopen-clean-autocache new file mode 100755 index 000000000..a0d789c1b --- /dev/null +++ b/include/scriptlets/transactionPost/zopen-clean-autocache @@ -0,0 +1,23 @@ +#!/bin/sh +this="zopen-clean-autocache" +printVerbose "Running '${this}' script" +if type zopen-config-helper >/dev/null 2>&1; then + isactive=$(zopen-config-helper --get autocacheclean) +else + isactive=true # Default to true +fi + +if isTrue "${isactive}"; then + (zopen clean --cache >/dev/null 2>&1) + echo "- Cache cleaned." +elif isFalse "${isactive}"; then + printVerbose "autoclean cache is disable; no clean will occur" +else + printWarning "Invalid value '${isactive}' for autocacheclean config parameter [should be 'true' or 'false']." + if type zopen-config-helper >/dev/null 2>&1; then + printWarning "Run zopen config --set autocacheclean [true|false] to update configuration" + else + printWarning "Update zopen configuration file to contain a valid value for parameter 'autocacheclean'" + fi + printWarning "Autocache clean will not be performed" +fi diff --git a/include/scriptlets/transactionPost/zopen-config-list b/include/scriptlets/transactionPost/zopen-config-list new file mode 100755 index 000000000..6cbf4498f --- /dev/null +++ b/include/scriptlets/transactionPost/zopen-config-list @@ -0,0 +1,30 @@ +#!/bin/sh +this="zopen-config-list" +printVerbose "Running '${this}' script" +cfgFile="${ZOPEN_ROOTFS}/etc/zopen-config" +cfgFileWorking="${ZOPEN_ROOTFS}/tmp/zopen-config.tmp" +envFile="${ZOPEN_ROOTFS}/etc/zopen/zopen.env" +envFileWorking="${envFile}.working" +# Running in a "clean" environment - this requires an executable version of the +# configuration file + +cp -f "${cfgFile}" "${cfgFileWorking}" +chmod +x "${cfgFileWorking}" + +if ! (env -i "${cfgFileWorking}" --knv | sort > "${envFileWorking}" ); then + printSoftError "Unable to generate zopen environment key/value pair" + return 1 +fi +if [ ! -e "${envFileWorking}" ]; then + printSoftError "Could not locate working file" + return 1 +fi +if ! [ -e "${envFile}" ]; then + mv "${envFileWorking}" "${envFile}" + return 0 +fi +if ! diff=$(diff "${envFileWorking}" "${envFile}"); then + printSoftError "${diff}" + printSoftError "Could not compute difference in new key-value pairs. Correct errors and run and redirect '${ZOPEN_ROOTFS}/etc/zopen-config knv | sort' manually." + return 1 +fi diff --git a/include/zopen_version b/include/zopen_version index b60d71966..ac39a106c 100644 --- a/include/zopen_version +++ b/include/zopen_version @@ -1 +1 @@ -0.8.4 +0.9.0