From d49ad10031f9527c8c23ed016b1482c9d5ca5ff5 Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 07:44:44 +0000 Subject: [PATCH 001/179] Support upgrade and list as standalone scripts --- bin/zopen | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/bin/zopen b/bin/zopen index 13e0959bf..d9fac9c9d 100755 --- a/bin/zopen +++ b/bin/zopen @@ -91,22 +91,15 @@ version=false for arg in $*; do case "${arg}" in - "alt"|"audit"|"build"|"clean"|"generate"|"init"|"install"|\ - "query"|"remove"|"update-cacert"|"info"|"usage"|"diagnostics") + "alt"|"build"|"clean"|"generate"|"init"|"install"|\ + "query"|"remove"|"update-cacert"|"info"|"usage"|"diagnostics"|\ + "list"|"upgrade") subcmd="zopen-${arg}" ;; - "list") - subcmd='zopen-query' - subopts="${subopts} --list" - ;; "refresh") subcmd='zopen-init' subopts="${subopts} --refresh" ;; - "upgrade") - subcmd='zopen-install' - subopts="${subopts} -u" - ;; "config") subcmd="zopen-${arg}-helper" ;; From 3fbd950fd018110fee4e2fe39427152ae069adc2 Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 07:49:55 +0000 Subject: [PATCH 002/179] Enable alternative repo support - generate default GH repo if not setup Generate package install database if not present Enable autocacheclean capability (to clear downloaded paxes by default) Enable scriptlets for pre/post actions (man + cleanup) Remove custom filesystem layouts and RM_FILEPROCs Resolve some shellcheck issues --- bin/zopen-init | 168 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 119 insertions(+), 49 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index 3c82eaa82..30f8e1c26 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -4,9 +4,11 @@ ME=$(basename "$0") if [ -z "${utildir}" ]; then + # shellcheck disable=SC2155 export utildir="$( cd "$(dirname "$0")" >/dev/null 2>&1 && pwd -P )" fi +# shellcheck disable=SC2034 ZOPEN_DONT_PROCESS_CONFIG=1 # # All zopen-* scripts MUST start with this code to maintain consistency. @@ -20,6 +22,7 @@ setupMyself() echo "Internal Error. Unable to find common.sh file to source." >&2 exit 8 fi + # shellcheck source=/dev/null . "${INCDIR}/common.sh" } setupMyself @@ -48,20 +51,6 @@ Options: --bypass-prereq-checks Bypasses pre-requisite checks - -f, --fs-layout - The filesystem structure to use for installed - packages on disk; packages will be installed to - this location under . - - should be one of: - - usrlclz: /usr/local/zopen (default), - zopen: /usr/zopen, - prod: legacy zopen standard location, - ibm: /usr/lpp, - fhs: File Hierarchical Standard (/opt), - usrlcl: usr/local - -h, -?, --help display this help and exit @@ -92,24 +81,20 @@ Examples: interactively bootstrap a zopen environment zopen init --releaseline-dev - interactively bootstrap a zopen environment that - will use Development Releaseline packages - - zopen init --yes --append-to-profile --fs-layout fhs /zopen - non-interactively create a zopen environment at - location '/zopen' on disk, with packages installed - to '/zopen/opt'. The user's .profile will be - updated to source the configuration file at - '/zopen/etc/zopen-config' when new terminal - sessions start + interactively bootstrap a zopen environment that will use + Development Releaseline packages + zopen init --yes --append-to-profile /zopen + non-interactively create a zopen environment at location + '/zopen' on disk, with packages installed to + '/usr/local/zopen'. The user's .profile will be updated to + source the configuration file at '/zopen/etc/zopen-config' + when new terminal sessions start Report bugs at https://github.com/zopencommunity/meta/issues HELPDOC } -args=$* - # Constants # Number of rm processes to run in parallel @@ -129,16 +114,11 @@ reinitExisting=false appendToProfile=false releaselineDev=false refresh=false -fslayout="usrlclz" isCollectingStats=false isBot=false # used by jenkins, CI/CD bypassPrereqs=false while [ $# -gt 0 ]; do case "$1" in - "-f" | "--fs-layout" ) - fslayout="$2" - shift - ;; "--re-init" ) reinitExisting=true ;; @@ -172,6 +152,7 @@ while [ $# -gt 0 ]; do exit 0 ;; "-v" | "--verbose") + # shellcheck disable=SC2034 verbose=true ;; "--version") @@ -179,6 +160,7 @@ while [ $# -gt 0 ]; do exit 0 ;; "--debug") + # shellcheck disable=SC2034 debug=true ;; "--yes" | "-y") @@ -288,24 +270,13 @@ init() printHeader "Initializing zopen framework" fi - printDebug "Validating input parameters" - case "${fslayout}" in - "usrlclz") zopen_pkginstall="usr/local/zopen" ;; - "usrlcl") zopen_pkginstall="usr/local" ;; - "fhs") zopen_pkginstall="opt" ;; - "ibm") zopen_pkginstall="usr/lpp" ;; - "prod") zopen_pkginstall="prod" ;; - "zopen") zopen_pkginstall="usr/zopen" ;; - *) printError "${NC}${RED}The filesystem layout ${fslayout} is unrecognised" ;; - esac - determineRootFileSystem determineStatsCollection if ! ${refresh} && ! ${reinitExisting}; then printInfo "- Binaries will be symlinked under \"${rootfs}/usr/local/bin\". Libraries will be symlinked under \"${rootfs}/usr/lib\"" - printInfo "- Packages will be installed and maintained under the directory structure ${fslayout} (${rootfs}/${zopen_pkginstall}). To change, re-run with the -f option." - printInfo "- Collecting usage statistics: ${isCollectingStats}" + printInfo "- Packages will be installed and maintained under the directory '${rootfs}/${zopen_pkginstall}'." + printInfo "- Collecting usage statistics: $isCollectingStats" fi if ! promptYesOrNo "Do you want to continue?" ${yesToPrompts}; then @@ -381,13 +352,34 @@ generateFileSystem() printVerbose "- Creating symbolic path for prod redirect files" [ -e "${rootfs}/usr/share/zopen/boot" ] || ln -s "${rootfs}/boot" "${rootfs}/usr/share/zopen/boot" [ -e "${rootfs}/etc/zopen" ] || mkdir -p "${rootfs}/etc/zopen" - echo "${zopen_pkginstall}" > "${rootfs}/etc/zopen/fstype" - + + printVerbose "- Creating product installation location" + zopen_pkginstall="usr/local/zopen" [ -e "${rootfs}/${zopen_pkginstall}" ] || mkdir -p "${rootfs}/${zopen_pkginstall}" [ -e "${rootfs}/usr/share/zopen/prod" ] || ln -s "${rootfs}/${zopen_pkginstall}" "${rootfs}/usr/share/zopen/prod" + printVerbose "- Creating repository directory" + repodir="${rootfs}/etc/zopen/repos.d" + [ -e "${repodir}" ] || mkdir -p "${repodir}" + + printVerbose "- Creating action scriptlet directories" + piaDir="${rootfs}/etc/zopen/scriptlets/preInstall" + [ -e "${piaDir}" ] || mkdir -p "${piaDir}" + piaDir="${rootfs}/etc/zopen/scriptlets/postInstall" + [ -e "${piaDir}" ] || mkdir -p "${piaDir}" + praDir="${rootfs}/etc/zopen/scriptlets/preRemove" + [ -e "${praDir}" ] || mkdir -p "${praDir}" + praDir="${rootfs}/etc/zopen/scriptlets/postRemove" + [ -e "${praDir}" ] || mkdir -p "${praDir}" + ptaDir="${rootfs}/etc/zopen/scriptlets/preTransaction" + [ -e "${ptaDir}" ] || mkdir -p "${ptaDir}" + ptaDir="${rootfs}/etc/zopen/scriptlets/postTransaction" + [ -e "${ptaDir}" ] || mkdir -p "${ptaDir}" + + printVerbose "- Setting global root envvar" + ZOPEN_PKGINSTALL="${rootfs}/${zopen_pkginstall}" if [ "${ZOPEN_PKGINSTALL}" = "${rootfs}" ]; then - printError "Package install location is the zopen root location; this is not allowed. Exiting". + printError "Package install location is the zopen root location; this is not supported. Exiting". fi printVerbose "- Creating path for certificate lookups: ${rootfs}/${ZOPEN_CA_DIR}" [ -e "${rootfs}/${ZOPEN_CA_DIR}" ] || mkdir -p "${rootfs}/${ZOPEN_CA_DIR}" @@ -441,7 +433,7 @@ generateJsonConfiguration() OVERRIDE_ZOS_TOOLS=false fi - if [ -z "$releaseLine" ]; then + if [ -z "${releaseLine}" ]; then if ${releaselineDev}; then releaseLine="DEV" printInfo "- Configured zopen to use development (DEV) releaseline packages where available" @@ -474,6 +466,82 @@ zz fi mv -f "${ZOPEN_JSON_CONFIG}.working" "${ZOPEN_JSON_CONFIG}" fi + jq '.' "${jsonConfigWorking}" > "${jsonConfig}" + echo 1 > "${rootfs}/etc/zopen/autocacheclean" +} + +generateDefaultRepository() +{ + # Check for existing repository definition - if present, use. + [ -e "${rootfs}/etc/zopen/repos.d/active" ] && return 0 + + # If there is no repository definition, generate a default to point to Github + [ ! -e "${rootfs}/etc/zopen/repos.d" ] && mkdir -p "${rootfs}/etc/zopen/repos.d" + + if [ -z "${paxrepourl}" ]; then + paxrepourl="http://raw.githubusercontent.com/ZOSOpenTools/meta/main/docs/api/zopen_releases.json" + fi + + repotype="${paxrepourl%://*}" + [ "${repotype}" = "${paxrepourl}" ] && printError "Unable to parse repository type from '${paxrepourl}'. Ensure valid format: ://://://:// "${repoFile}" +{ + "type": "${repotype}", + "metadata_baseurl": "${baserepourl}", + "metadata_file": "${repometafile}" +} +EOS + (cd "${rootfs}/etc/zopen/repos.d" && ln -s "${repoFile}" "active") +} + +generateDefaultScriptlets() +{ + printVerbose "- Generating man-db scriptlet" + cat << EOF > "${rootfs}/etc/zopen/scriptlets/postTransaction/man-db" +#!/bin/false # Expects to be sourced in zopen-* scripts, not run directly! +# Do not modify this file; changes will be overwritten automatically! +if ! installed=\$(zopen list --installed | grep "^man-db\$"); then + : # No man-db installed +else + echo "- Updating man-db database..." + if ! mandb >/dev/null 2>&1; then + printWarning "man-db update completed with non-zero return code." + printWarning "Re-run 'mandb' manually for additional information." + else + echo "- Update of man-db complete." + fi +fi +EOF + printVerbose "- Generating zopen-clean scriptlet" + cat << EOF > "${rootfs}/etc/zopen/scriptlets/postTransaction/zopen-clean" +#!/bin/false # Expects to be sourced in zopen-* scripts, not run directly! +# Do not modify this file; changes will be overwritten automatically! +[ -e "\${ZOPEN_ROOTFS}/etc/zopen/autocacheclean" ] || exit 0 +isactive=\$(cat "\${ZOPEN_ROOTFS}/etc/zopen/autocacheclean") +if [ "\${isactive}" -eq 1 ]; then + echo "- Cleaning install cache..." + (zopen clean --cache >/dev/null 2>&1) + echo "- Cache cleaned." +fi +EOF } generateAnalyticsConfiguration() @@ -645,7 +713,9 @@ init generateFileSystem ! ${refresh} && installPrereqs generateJsonConfiguration -generateZopenConfig +generateDefaultRepository generateAnalyticsConfiguration +generateDefaultScriptlets +[ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || updatePackageDB updateBootstrappedTools deinit From e846ebeaa94094abb07f54b5846e20c9736048e9 Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 07:53:35 +0000 Subject: [PATCH 003/179] Enable use of pre/post scriptlets (for man-db processing) --- bin/zopen-alt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/zopen-alt b/bin/zopen-alt index f170d186d..e104b124f 100755 --- a/bin/zopen-alt +++ b/bin/zopen-alt @@ -55,7 +55,7 @@ mergeNewVersion() { package="$1" newver="$2" - oldver="$2" + processActionScripts "txn-pre" [ -z "${package}" ] && printError "Internal error; no packagename provided to merge." [ -d "${ZOPEN_PKGINSTALL}/${package}/${newver}" ] || printError "Version '${newver}' was not available to set as current" if [ -e "${ZOPEN_PKGINSTALL}/${package}/${package}" ]; then @@ -85,6 +85,7 @@ mergeNewVersion() version=$(cat "${ZOPEN_PKGINSTALL}/${package}/${package}/.releaseinfo") fi syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE}" "ALT" "setAlt" "Set '${package}' to version:${version};" + processActionScripts "txn-post" } setAlt() @@ -228,7 +229,9 @@ while [ $# -gt 0 ]; do verbose=true ;; "--debug") + # shellcheck disable=SC2034 verbose=true + # shellcheck disable=SC2034 debug=true ;; *) From cb967682c8c5eed322ecedd99eac94f2c749fd93 Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 07:54:32 +0000 Subject: [PATCH 004/179] Use pre/post scriptlets (for man-db); Update installation tracker db --- bin/zopen-remove | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/bin/zopen-remove b/bin/zopen-remove index d61143200..289221594 100755 --- a/bin/zopen-remove +++ b/bin/zopen-remove @@ -15,6 +15,7 @@ setupMyself() echo "Internal Error. Unable to find common.sh file to source." >&2 exit 8 fi + # shellcheck disable=SC1091 . "${INCDIR}/common.sh" } setupMyself @@ -51,11 +52,15 @@ HELPDOC removePackages() { pkglist=$* - + processActionScripts "txn-pre" + # Create a temp file name to trigger the post-transaction action script + # triggering - run only if an action was taken (which creates the file) + runpostTxnActionScripts=$(mktempfile "txnpost" "run") echo "${pkglist}" | xargs | tr ' ' '\n' | sort | while read pkg; do printHeader "Removing package: ${pkg}" + processActionScripts "remove-pre" printInfo "- Checking status of package '${pkg}'" - if [ ! -f ${ZOPEN_PKGINSTALL}/${pkg}/${pkg}/.active ]; then + if [ ! -f "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}/.active" ]; then printInfo "${NC}${YELLOW}Package '${pkg}' is not installed${NC}" else printInfo "- Away to remove '${pkg}'," @@ -83,27 +88,34 @@ removePackages() if ${purge}; then printInfo "- Purging package" printVerbose "Checking if we are currently in a directory that is to be purged" - [ "${PWD##"${ZOPEN_PKGINSTALL}"/"${pkg}"/"${pkg}"}" != "${PWD}" ] && cd ${ZOPEN_PKGINSTALL} - rm -rf $(cd "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}" && pwd -P) - syslog ${ZOPEN_LOG_PATH}/audit.log ${LOG_A} "${CAT_PACKAGE},${CAT_REMOVE}" "REMOVE" "removePackage" "Purging package:'${needle};version:${version};" + [ "${PWD##"${ZOPEN_PKGINSTALL}"/"${pkg}"/"${pkg}"}" != "${PWD}" ] && cd "${ZOPEN_PKGINSTALL}" + rm -rf "$(cd "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}" && pwd -P)" + syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE},${CAT_REMOVE}" "REMOVE" "removePackage" "Purging package:'${needle};version:${version};" registerRemove "${pkg}" "${version}" else - printInfo "- Removing metadata file to mark uninstall" + printVerbose "- Removing metadata file to mark uninstall" rm -f "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}/.active" - printInfo "- Breaking link from current to versioned" + printVerbose "- Breaking link from current to versioned" rm -f "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}" fi printVerbose "Main symlink removed, removing dangling symlinks" unsymlinkFromSystem "${pkg}" "${ZOPEN_ROOTFS}" "${installedlinksfile}" fi - printInfo "- Removing profiled entry" + printVerbose "- Removing profiled entry" [ -d "${ZOPEN_ROOTFS}/etc/profiled/${pkg}" ] && rm -rf "${ZOPEN_ROOTFS}/etc/profiled/${pkg}" + removeFromInstallTracker "${pkg}" + processActionScripts "remove-post" syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE},${CAT_REMOVE}" "REMOVE" "removePackage" "Removed package:'${needle};version:${version};" + touch "${runpostTxnActionScripts}" printInfo "${NC}${GREEN}Successfully removed: ${pkg}${NC}" fi done + if [ -e "${runpostTxnActionScripts}" ]; then + processActionScripts "txn-post"; + rm "${runpostTxnActionScripts}" + fi } # Main code start here From 1988ee66b45c12733f83ed506d4ca66f6ff3778b Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 07:56:00 +0000 Subject: [PATCH 005/179] Use renamed function to get repository information; Add delay after progress handler to stop screen overlays --- bin/zopen-clean | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/zopen-clean b/bin/zopen-clean index 2d1e147f0..27252faba 100755 --- a/bin/zopen-clean +++ b/bin/zopen-clean @@ -15,6 +15,7 @@ setupMyself() echo "Internal Error. Unable to find common.sh file to source." >&2 exit 8 fi + # shellcheck disable=SC1091 . "${INCDIR}/common.sh" } setupMyself @@ -116,6 +117,7 @@ cleanDangling() rm -f "${sl}" done ${killph} 2> /dev/null # if the timer is not running, the kill will fail + waitforpid ${ph} # Make sure it's finished writing to screen syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_FILE}" "CLEAN" "cleanDangling" "zopen system at '${ZOPEN_ROOTFS}' scanned for dangling symlinks" } @@ -210,7 +212,7 @@ done ${dangling} && cleanDangling && exit ${meta} && cleanMetadata -getReposFromGithub +getRepos if $all; then printVerbose "Generating complete list of all packages" # shellcheck disable=SC2154 From 96e908fa9737ce4bb7300f956d7db176db1e9c1d Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 07:58:44 +0000 Subject: [PATCH 006/179] Add new script to provide "list" capabilities; Utilize package installation db and jq to massively speed up processing Uses new jqfunctions() function to "import" additional standard functions into jq queries --- bin/zopen-list | 311 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 311 insertions(+) create mode 100755 bin/zopen-list diff --git a/bin/zopen-list b/bin/zopen-list new file mode 100755 index 000000000..326006a2a --- /dev/null +++ b/bin/zopen-list @@ -0,0 +1,311 @@ +#!/bin/sh +# +# List utility for z/OS Open Tools - https://github.com/ZOSOpenTools +# + +# +# All zopen-* scripts MUST start with this code to maintain consistency. +# +setupMyself() +{ + ME=$(basename $0) + MYDIR="$(cd "$(dirname "$0")" > /dev/null 2>&1 && pwd -P)" + INCDIR="${MYDIR}/../include" + if ! [ -d "${INCDIR}" ] && ! [ -f "${INCDIR}/common.sh" ]; then + echo "Internal Error. Unable to find common.sh file to source." >&2 + exit 8 + fi + # shellcheck source=/dev/null + . "${INCDIR}/common.sh" +} +setupMyself + +printHelp(){ + cat << HELPDOC +zopen list - list information about local packages + +Usage: zopen list [OPTION] [VERB] [PACKAGE] + +Options: + --available lists all available z/OS Open Tools + --details provide version information on packages + --full provides more information about packages + --installed list only installed z/OS Open Tools + --[no]header output explanation header for columns + --verbose run in verbose mode + --version print version + --version-info lists version information for installed packages when + combined with the --installed option + -h,-?, --help display this help and exit + +Examples: + zopen list --installed + list packages installed on the system + +Report bugs at https://github.com/ZOSOpenTools/meta/issues . + +HELPDOC +} +text_center() +{ + if [ $# -lt 2 ]; then + echo "$1" + return + fi + echo "$1" | awk -v strlen="$2" \ + '{spc = strlen - length;padding = int(spc / 2);pad = spc - padding;printf "%*s%s%*s\n", pad, "", $0, padding, ""}' +} + +text_padrightr() { + padchar="${3:- }" # Default to space char + echo "" | awk -v str="$1" -v n="$2" -v pc="$padchar" \ + 'function padr(n,acc, c){ + if (++acc>=n) return c; + return c padr(n, acc, c); + } + BEGIN{ print str padr(n,0,pc);} + ' +} + + +listLatestForInstalled() +{ + wpackage=20 + wversion=12 + wrelease=14 + wtag=36 + # wurl=11 Note required unless additional columns added + if ${header}; then + printf "${NC}${UNDERLINE}%-${wpackage}s%-${winstalled}s%-${wtag}s%-${wdownload}s%-${wexpsz}s%-${wqual}s${NC}\n" \ + "$(text_center "Package")" "$(text_center "Version")" "$(text_center "Release")"\ + "$(text_center "Tag Name")" "$(text_center "Download URL")" + fi + pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" + installed=$(zopen list --installed | awk -v ORS=',' '{print "\""$0"\""}') + installed="[${installed%,}]" # wrap in a JSON array and strip any trailing ',' char + jq --raw-output --argjson installed "${installed}" \ + "$(jqfunctions) .release_data| + to_entries |sort_by(.key) | .[] | + .key as \$key | + select( \$installed | index(\$key)) | + .value[0].assets[0] as \$va | + \$va.url as \$url | + (\$url | match(\".*-([^-]*).([0-9]{8}_[0-9]{6}).zos.pax.Z\").captures) as \$caps | + \$caps[0].string as \$version | + \$caps[1].string as \$release | + .value[0].tag_name as \$tn | + pr(\$key;\"@\";${wpackage}) + + pr(\$version;\"@\"; ${wversion}) + + pr(\$release;\"@\"; ${wrelease}) + + pr(\$tn;\"@\";${wtag}) + + \$url " + "${JSON_CACHE}" | sed 's/@/ /g' +} + +listAvailablePackages() +{ + downloadJSONCache + wpackage=20 + wversion=23 + winstalled=10 + wtag=36 + wrellne=12 + wdownload=10 + wexpsz=10 + wqual=7 + if ${urlinclude}; then + wurl=6 + urltext="URL" + else + wurl=0 + urltext="" + fi + if ${header}; then + if ${details} || ${full}; then + printf "${NC}${UNDERLINE}%-${wpackage}s%-${winstalled}s%-${wtag}s%-${wdownload}s%-${wexpsz}s%-${wqual}s%-${wurl}s${NC}\n" \ + "$(text_center "Package")" "$(text_center "Installed")" "$(text_center "Latest Tag")"\ + "$(text_center "Download size")" "$(text_center "Disk usage")" "$(text_center "Quality")" \ + "$(text_center "${urltext}")" + + else + printf "${NC}${UNDERLINE}%-${wpackage}s%-${winstalled}s%-${wversion}s%-${wtag}s${NC}\n" \ + "$(text_center "Package")" "$(text_center "Installed")" "$(text_center "Version")" "$(text_center "Latest Tag")" + fi + fi + installed=$(zopen list --installed | awk -v ORS=',' '{print "\""$0"\""}') + installed="[${installed%,}]" # wrap in a JSON array and strip any trailing ',' char + + if ${details} || ${full}; then + if ${urlinclude}; then + urltext='$url' + else + urltext="" + fi + jq --raw-output --argjson installed "${installed}" \ + "$(jqfunctions) .release_data| + to_entries |sort_by(.key) | .[] | + .key as \$key | + .value[0].assets[0] as \$va | + \$va.url as \$url | + (\$url | match(\".*-([^-]*).[0-9]{8}_[0-9]{6}.zos.pax.Z\").captures[0].string) as \$version | + .value[0].tag_name as \$tn | + (if (\$installed | index(\$key)) then \"true\" else \"false\" end) as \$ins | + \$va.expanded_size as \$es | + \$va.size as \$ds | + (\$va.passed_tests | if (.==\"\") then \"0\" else . end | tonumber) as \$pt | + (\$va.total_tests | if (.==\"\") then \"99999\" else . end |tonumber) as \$tt | + ((\$pt/\$tt*100|r(2))|tostring + \"%\") as \$q | + pr(\$key;\"@\";${wpackage}) + + c(\$ins;\"@\";${winstalled}) + + c(\$tn;\"@\";${wtag}) + + c(\$ds;\"@\";${wdownload}) + + c(\$es;\"@\";${wexpsz}) + + c((\$q);\"@\";${wqual}) + + ${urltext} " \ + "${JSON_CACHE}" | sed 's/@/ /g' + else + jq --raw-output --argjson installed "${installed}" \ + "$(jqfunctions) .release_data| + to_entries |sort_by(.key) | .[] | + .key as \$key | + .value[0].assets[0].url as \$url | + (\$url | match(\".*-([^-]*).[0-9]{8}_[0-9]{6}.zos.pax.Z\").captures[0].string) as \$version | + .value[0].tag_name as \$tn | + (if (\$installed | index(\$key)) then \"true\" else \"false\" end) as \$ins | + .value[0].assets[0] as \$va | + pr(\$key;\"@\";${wpackage}) + + c(\$ins;\"@\";${winstalled}) + + c(\$version;\"@\";${wversion}) + + c(\$tn;\"@\";${wtag}) " \ + "${JSON_CACHE}" | sed 's/@/ /g' + fi +} + +listInstalledEntries() +{ + wpackage=20 + wversion=26 + wrelease=14 + wrellne=12 + wexpsz=12 + wqual=8 + if ${urlinclude}; then + wurl=6 + urltext="URL" + else + wurl=0 + urltext="" + fi + if ${header} && ${details}; then + if ${full}; then + printf "${NC}${UNDERLINE}%-${wpackage}s%-${wversion}s%-${wrelease}s%-${wrellne}s%-${wexpsz}s%-${wqual}s%-${wurl}s${NC}\n" \ + "$(text_center "Package")" "$(text_center "Version")" "$(text_center "Release")" \ + "$(text_center "Releaseline")" "$(text_center "Disk usage")" \ + "$(text_center "Quality")" "$(text_center "${urltext}")" + else + printf "${NC}${UNDERLINE}%-${wpackage}s%-${wversion}s%-${wrelease}s${NC}\n" \ + "$(text_center "Package")" "$(text_center "Version")" "$(text_center "Release")" + fi + fi + pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" + if ! [ -e "${pdb}" ]; then + printWarning "No package database found. Regenerating (subsequent calls will be faster)" + updatePackageDB + fi + if ${details}; then + if ${full}; then + jq --raw-output \ + "$(jqfunctions) .[] | + keys[] as \$key | + .[\$key].product.test_status as \$ts | + pr(\$key;\"@\";${wpackage}) + + c((.[\$key].product?.version? // \"unknown\");\"@\";${wversion}) + + c((.[\$key].product?.release? // \"unknown\");\"@\";${wrelease}) + + c((.[\$key].product.buildline? // \"unknown\");\"@\";${wrellne}) + + c((.[\$key].product.size? // \"unknown\");\"@\";${wexpsz}) + + c(((if (\$ts? and (\$ts.total_tests | tonumber?) and (\$ts.total_success | tonumber?)) then (\$ts.total_success | tonumber) / (\$ts.total_tests | tonumber) else 0 end | .*100)| r(2) |tostring);\"@\";${wqual})" \ + "${pdb}" | sed 's/@/ /g' + else + jq --raw-output \ + "$(jqfunctions) .[] | + keys[] as \$key | + pr(\$key;\"@\";${wpackage}) + + c(.[\$key].product.version;\"@\";${wversion}) + + c(.[\$key].product.release;\"@\";${wrelease})" \ + "${pdb}" | sed 's/@/ /g' + fi + else + jq --raw-output \ + '[.[] | keys[] as $key | $key] | unique[] ' \ + "${pdb}" + fi +} + +# Main code start here +# Note that functions use padding chars of '@' to work round a quirk of jq in that +# it will collapse repeated spaces into tab characters - which proves irksome when tryinh +# to pad/format column layouts; '@' chars get post-processed into spaces + +verbose=false +header=false +details=false +full=false +installed=false +urlinclude=false # whether to include download url in output information +versioninfo=false + +if [ $# -eq 0 ]; then + printError "No option provided" +fi + +while [ $# -gt 0 ]; do + printVerbose "Parsing option: $1" + case "$1" in + "--available") available=true ;; + "--installed") installed=true ;; + "--details") details=true ;; + "--full") full=true; details=true ;; + "--header") header=true ;; + "--noheader") header=false ;; + "--url") urlinclude=true ;; + "--version-info") versioninfo=true ;; + "-h" | "--help" | "-?") + printHelp + exit 0 + ;; + "--version") + zopen-version "${ME}" + exit 0 + ;; + "--verbose") + # shellcheck disable=SC2034 + verbose=true + ;; + "--xdebug") + set -x + ;; + -*) + printError "Unknown option '$1'" + ;; + *) + chosenRepos="${chosenRepos} $1" + ;; + esac + shift +done + +checkIfConfigLoaded +if ${installed}; then + if ${versioninfo}; then + listInstalledVerInfo + else + listInstalledEntries + fi + exit 0 +fi +if ${available}; then + listAvailablePackages + exit 0 +fi + + From 410b0e2101cfe1f3b5e4a5fa0465ac0823c16d5c Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 07:59:07 +0000 Subject: [PATCH 007/179] Add delay to prevent screen overlays --- bin/zopen-promote | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/zopen-promote b/bin/zopen-promote index 0a9f643e4..ee41a3f33 100755 --- a/bin/zopen-promote +++ b/bin/zopen-promote @@ -282,6 +282,7 @@ while read OUTMSG; do done < "${FIFO_PIPE_STDOUT}" [ -n "${FIFO_PIPE_STDOUT}" ] && [ -e "${FIFO_PIPE_STDOUT}" ] && rm -f "${FIFO_PIPE_STDOUT}" ${killph} 2>/dev/null # if the timer is not running, the kill will fail +waitforpid ${ph} # Make sure it's finished writing to screen printVerbose "Grabbing pkginstall location from within promoted env" if [ -e "${promotefs}/etc/zopen/fstype" ]; then @@ -311,6 +312,7 @@ if ! ${keepzopentooling}; then flecnt=$(expr ${flecnt} + 1) done ${killph} 2>/dev/null # if the timer is not running, the kill will fail + waitforpid ${ph} # Make sure it's finished writing to screen syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_ZOPEN}" "PROMOTE" "mainline" "meta tooling checked for and removed from promoted environment; ${flecnt} link(s) removed" fi From 02a3ae6a5bb9f2d4cb913f55871ef2061a60f3df Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 08:02:15 +0000 Subject: [PATCH 008/179] Utilize dedicated zopen-list script to provide list capabilities --- bin/zopen-query | 81 +++++++++++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 30 deletions(-) diff --git a/bin/zopen-query b/bin/zopen-query index 02d595a93..c25e20654 100755 --- a/bin/zopen-query +++ b/bin/zopen-query @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh # # Query utility for zopen community - https://github.com/zopencommunity # @@ -15,6 +15,7 @@ setupMyself() echo "Internal Error. Unable to find common.sh file to source." >&2 exit 8 fi + # shellcheck source=/dev/null . "${INCDIR}/common.sh" } setupMyself @@ -23,20 +24,28 @@ filter_implemented=false upgradeable_implemented=false whatprovides_implemented=false -printSyntax() -{ - args=$* - echo "${ME} - a utility for zopen community to query packages and repos." - echo "" - echo "Usage: ${ME} [OPTION] [VERB] [PACKAGE]" - echo " VERB is the action to take, which is one of" - echo " --list, --remote-search, --installed" - echo " PACKAGE is a package, specified for --remote-search" - echo "" - echo "Verbs:" - echo " -i, --installed list installed zopen community." - echo " --list list all available zopen community." - echo " --remote-search regex match package against available zopen community" +printHelp(){ + cat << HELPDOC +zopen query - a utility for zopen community to query packages and repos. + +Usage: zopen query [OPTION] [VERB] [PACKAGE] + +Options: + -d, --details include full details for listings + -i, --installed list only installed z/OS Open Tools + --list list all available z/OS Open Tools + --no-header suppress the header for the output + --no-version suppress version information, return package names + --remote-search + regex match package against available packages + --verbose run in verbose mode + --version print version + -h,-?, --help display this help and exit +HELPDOC +if ${filter_implemented}; then + echo " --filter COLOR apply COLOR (quality) filter." + echo " green: all pass, blue: most pass, yellow: some pass, red: no pass, gray: skipped" +fi if ${upgradeable_implemented}; then echo " --upgradeable list packages where an upgrade is available." fi @@ -322,6 +331,7 @@ noheader=false noversion=false localoption=true upgradeable=false +list=false unset category_filter details=0 needle= @@ -333,17 +343,13 @@ while [ $# -gt 0 ]; do printVerbose "Parsing option: $1" case "$1" in "--list") - list=1 + list=true localoption=false ;; "-i" | "--installed") localoption=true installed=1 - list= - if [ -n "$2" ] && [[ "$2" != -* ]]; then - needle="$2" - shift - fi + list=true ;; "-wp" | "--whatprovides") localoption=true @@ -380,18 +386,21 @@ while [ $# -gt 0 ]; do "-d" | "--details") details=1 ;; - "-h" | "--h" | "-help" | "--help" | "-?" | "-syntax") - printSyntax "${args}" + "-h" | "--help" | "-?") + printHelp exit 0 ;; "--version") zopen-version "${ME}" exit 0 ;; - "-v" | "--v" | "-verbose" | "--verbose") + "--verbose") # shellcheck disable=SC2034 verbose=true ;; + "--xdebug") + set -x + ;; -*) printError "Unknown option '$1'" ;; @@ -421,14 +430,26 @@ fi if ! ${localoption}; then # Retrieve all repositories - getReposFromGithub true # zopen query is a read only operation + getRepos grfgRc=$? [ 0 -ne "${grfgRc}" ] && exit "${grfgRc}" repoArray="${repo_results}" fi -! ${upgradeable} || printDetailListEntries "${details}" "" 1 -[ -z "${remotesearch}" ] || printDetailListEntries "${details}" "${needle}" 0 -[ -z "${list}" ] || printDetailListEntries "${details}" "" 0 -[ -z "${installed}" ] || printInstalledEntries "${details}" "${needle}" 0 -[ -z "${whatprovides}" ] || whatProvides "${needle}" +# Use dedicated zopen-list script for listing system packages, passing +# appropriate parameters to mimic the old query behaviour +if ${list}; then + if ${installed}; then + if [ "${details}" -eq 0 ]; then + (zopen list --installed) + else + (zopen list --installed --details --header) + fi + else + (zopen list ) + fi +else + ! ${upgradeable} || printDetailListEntries "${details}" "" 1 + [ -z "${remotesearch}" ] || printDetailListEntries "${details}" "${needle}" 0 + [ -z "${whatprovides}" ] || whatProvides "${needle}" +fi \ No newline at end of file From 679a7c058bae0fa33cac3259afd616d6e81b3652 Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 08:07:07 +0000 Subject: [PATCH 009/179] Split into install and upgrade scripts. Move bulk of pax handling logic into common.sh Add detection of "local" pax file to install from file rather than repo --- bin/zopen-install | 825 +++++++++------------------------------------- 1 file changed, 147 insertions(+), 678 deletions(-) diff --git a/bin/zopen-install b/bin/zopen-install index 0792ee8a9..0585f442b 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -6,16 +6,16 @@ # # All zopen-* scripts MUST start with this code to maintain consistency. # -#set -x setupMyself() { - ME=$(basename $0) + ME=$(basename "$0") MYDIR="$(cd "$(dirname "$0")" > /dev/null 2>&1 && pwd -P)" INCDIR="${MYDIR}/../include" if ! [ -d "${INCDIR}" ] && ! [ -f "${INCDIR}/common.sh" ]; then echo "Internal Error. Unable to find common.sh file to source." >&2 exit 8 fi + # shellcheck source=/dev/null . "${INCDIR}/common.sh" } setupMyself @@ -118,32 +118,27 @@ verifySignatureOfPax() printSyntax() { - args=$* cat << HELPDOC ${ME} is a utility to download/install a zopen community package. -Usage: ${ME} [OPION] [PACKAGE] - [PACKAGE] is a package to install. Multiple packages can be specified. +Usage: zopen install [OPTION] [PARAMETERS] [PACKAGES] Options: --all download/install all zopen community packages. - --cache-only do not install dependencies. + --bypass-prereq-checks Ignores pre-req checks --download-only download package to current directory. + --force force install, bypassing locks. --help print this help. --install-or-upgrade installs the package if not installed, or upgrades the package if installed. - --bypass-prereq-checks Ignores pre-req checks - --local-install download and unpackage to current directory. --no-deps do not install dependencies. --no-set-active do not change the pinned version. --nosymlink do not integrate into filesystem through symlink redirection. - -r, --reinstall reinstall already installed zopen community packages. + --reinstall reinstall already installed zopen community packages. --release-line [stable, dev] the release line to build off of. --select select a version to install. - --skip-upgrade do not upgrade. - --force force install, bypassing locks. - -u, --update, --upgrade updates installed zopen community packages. + -u, --upgrade check for package updates and apply -v, --verbose print verbose messages. --version print version. -y, --yes automatically answer yes to prompts. @@ -151,526 +146,17 @@ Options: HELPDOC } -installDependencies() -( - name=$1 - printVerbose "List of dependencies to install: ${dependencies}" - skipupgrade_lcl=${skipupgrade} - skipupgrade=true - skipverify=false; - echo "${dependencies}" | xargs | tr ' ' '\n' | sort | while read dep; do - printVerbose "Removing '${dep}' from dependency queue '${dependencies}'" - dependencies=$(echo "${dependencies}" | sed -e "s/${dep}//" | tr -s ' ') - handlePackageInstall "${dep}" true - done - skipupgrade=${skipupgrade_lcl} -) - -handlePackageInstall() -{ - - fullname="$1" - isRuntimeDependency=$2 - if [ -z "$isRuntimeDependency" ]; then - isRuntimeDependency=false - fi - printVerbose "Name to install: ${fullname}, parsing any version ('=') or tag ('%') has been specified" - name=$(echo "${fullname}" | sed -e 's#[=%].*##') - repo="${name}" - versioned=$(echo "${fullname}" | cut -s -d '=' -f 2) - tagged=$(echo "${fullname}" | cut -s -d '%' -f 2) - printDebug "Name:${name};version:${versioned};tag:${tagged};repo:${repo}" - printInfo "${NC}${HEADERCOLOR}${BOLD}Processing package: ${name}${NC}" - - nameSansPort=$(echo "${name}" | sed -e 's#\(.*\)port$#\1#') - # findutilsport -> findutils - # findutils -> findutils - if [[ "${nameSansPort}" != "${name}" ]] ; then - printError "Please install using base project name without port suffix.\nTry: zopen install ${nameSansPort}" - fi - - getAllReleasesFromGithub "${repo}" - - if ${localInstall}; then - printVerbose "Local install to current directory" - rootInstallDir="${PWD}" - else - printVerbose "Setting install root to: ${ZOPEN_PKGINSTALL}" - rootInstallDir="${ZOPEN_PKGINSTALL}" - fi - - originalFileVersion="" - printVerbose "Checking for meta files in '${rootInstallDir}/${name}/${name}'" - printVerbose "Finding version/release information." - if [ -e "${rootInstallDir}/${name}/${name}/.releaseinfo" ]; then - originalFileVersion=$(cat "${rootInstallDir}/${name}/${name}/.releaseinfo") - printVerbose "Found originalFileVersion=${originalFileVersion} (port is already installed)." - elif [ -e "${rootInstallDir}/${name}/${name}/.version" ]; then - originalFileVersion=$(cat "${rootInstallDir}/${name}/${name}/.version") - printVerbose "Found originalFileVersion=${originalFileVersion} (port is already installed)." - else - printVerbose "Could not detect existing installation at ${rootInstallDir}/${name}/${name}" - fi - - printVerbose "Finding releaseline information." - installedReleaseLine="" - if [ -e "${rootInstallDir}/${name}/${name}/.releaseline" ]; then - installedReleaseLine=$(cat "${rootInstallDir}/${name}/${name}/.releaseline") - printVerbose "Installed product from releaseline: ${installedReleaseLine}" - else - printVerbose "No current releaseline for package." - fi - - releaseMetadata="" - downloadURL="" - # Options where the user explicitly sets a version/tag/releaseline currently ignore any configured release-line, - # either for a previous package install or system default - if [ -n "${versioned}" ]; then - printVerbose "Specific version ${versioned} requested - checking existence and URL." - requestedMajor=$(echo "${versioned}" | awk -F'.' '{print $1}') - requestedMinor=$(echo "${versioned}" | awk -F'.' '{print $2}') - requestedPatch=$(echo "${versioned}" | awk -F'.' '{print $3}') - requestedSubrelease=$(echo "${versioned}" | awk -F'.' '{print $4}') - requestedVersion="${requestedMajor}\\\.${requestedMinor}\\\.${requestedPatch}\\\.${requestedSubrelease}" - printVerbose "Finding URL for latest release matching version prefix: requestedVersion: ${requestedVersion}" - releaseMetadata=$(/bin/printf "%s" "${releases}" | jq -e -r '. | map(select(.assets[].name | test("'${requestedVersion}'")))[0]') - elif [ -n "${tagged}" ]; then - printVerbose "Explicit tagged version '${tagged}' specified. Checking for match." - releaseMetadata=$(/bin/printf "%s" "${releases}" | jq -e -r '.[] | select(.tag_name == "'${tagged}'")') - printVerbose "Use quick check for asset to check for existence of metadata for specific messages." - asset=$(/bin/printf "%s" "${releaseMetadata}" | jq -e -r '.assets[0]') - if [ $? -ne 0 ]; then - printError "Could not find release tagged '${tagged}' in repo '${repo}'" - fi - elif ${selectVersion}; then - # Explicitly allow the user to select a release to install; useful if there are broken installs - # as a known good release can be found, selected and pinned! - printVerbose "List individual releases and allow selection." - i=$(/bin/printf "%s" "${releases}" | jq -r 'length - 1') - printInfo "Versions available for install:" - /bin/printf "%s" "${releases}" | jq -e -r 'to_entries | map("\(.key): \(.value.tag_name) - \(.value.assets[0].name) - size: \(.value.assets[0].expanded_size | tonumber/ (1024 * 1024))mb")[]' - - printVerbose "Getting user selection." - valid=false - while ! ${valid}; do - echo "Enter version to install (0-${i}): " - read selection < /dev/tty - if [[ ! -z $(echo "${selection}" | sed -e 's/[0-9]*//') ]]; then - echo "Invalid input, must be a number between 0 and ${i}." - elif [ "${selection}" -ge 0 ] && [ "${selection}" -le "${i}" ]; then - valid=true - fi - done - printVerbose "Selecting item ${selection} from array" - releaseMetadata="$(/bin/printf "%s" "${releases}" | jq -e -r ".[${selection}]")" - - elif [ ! -z "${releaseLine}" ]; then - printVerbose "Install from release line '${releaseLine}' specified" - validatedReleaseLine=$(validateReleaseLine "${releaseLine}") - if [ -z "${validatedReleaseLine}" ]; then - printError "Invalid releaseline specified: '${releaseLine}'; Valid values: DEV or STABLE." - fi - printVerbose "Finding latest asset on the release line" - releaseMetadata="$(/bin/printf "%s" "${releases}" | jq -e -r '. | map(select(.tag_name | startswith("'${releaseLine}'")))[0]')" - printVerbose "Use quick check for asset to check for existence of metadata." - asset="$(/bin/printf "%s" "${releaseMetadata}" | jq -e -r '.assets[0]')" - if [ $? -ne 0 ]; then - printError "Could not find release-line ${releaseLine} for repo: ${repo}." - fi - - else - printVerbose "No explicit version/tag/releaseline, checking for pre-existing package&releaseline." - if [ -n "${installedReleaseLine}" ]; then - printVerbose "Found existing releaseline '${installedReleaseLine}', restricting to only that releaseline." - validatedReleaseLine="${installedReleaseLine}" # Already validated when stored - else - printVerbose "Checking for system-configured releaseline." - sysrelline=$(getReleaseLine) - printVerbose "Validating value: ${sysrelline}" - validatedReleaseLine=$(validateReleaseLine "${sysrelline}") - if [ -n "${validatedReleaseLine}" ]; then - printVerbose "zopen system configured to use releaseline '${sysrelline}'; restricting to that releaseline." - else - printWarning "zopen misconfigured to use an unknown releaseline of '${sysrelline}'; defaulting to STABLE packages." - printWarning "Set the contents of '${ZOPEN_ROOTFS}/etc/zopen/releaseline' to a valid value to remove this message." - printWarning "Valid values are: DEV | STABLE." - validatedReleaseLine="STABLE" - fi - fi - # We have some situations that could arise - # 1. the port being installed has no releaseline tagging yet (ie. no releases tagged STABLE_* or DEV_*) - # 2. system is configured for STABLE but only has DEV stream available - # 3. system is configured for DEV but only has DEV stream available - # 4. the port being installed has got full releaseline tagging - # The issue could arise that the user has switched the system from DEV->STABLE or vice-versa so package - # stream mismatches could arise but in normal case, once a package is installed [that has releaseline tagging] - # then that specific releaseline will be used - printVerbose "Finding any releases tagged with ${validatedReleaseLine} and getting the first (newest/latest)" - releaseMetadata="$(/bin/printf "%s" "${releases}" | jq -e -r '. | map(select(.tag_name | startswith("'${validatedReleaseLine}'")))[0]')" - printVerbose "Use quick check for asset to check for existence of metadata" - asset="$(/bin/printf "%s" "${releaseMetadata}" | jq -e -r '.assets[0]')" - if [ $? -eq 0 ]; then - # Case 4... - printVerbose "Found a specific '${validatedReleaseLine}' release-line tagged version; installing..." - else - # Case 2 & 3 - printVerbose "No releases on releaseline '${validatedReleaseLine}'; checking alternative releaseline." - alt=$(echo "${validatedReleaseLine}" | awk ' /DEV/ { print "STABLE" } /STABLE/ { print "DEV" }') - releaseMetadata="$(/bin/printf "%s" "${releases}" | jq -e -r '. | map(select(.tag_name | startswith("'${alt}'")))[0]')" - printVerbose "Use quick check for asset to check for existence of metadata" - asset="$(/bin/printf "%s" "${releaseMetadata}" | jq -e -r '.assets[0]')" - if [ $? -eq 0 ]; then - printVerbose "Found a release on the '${alt}' release line so release tagging is active." - if [ "DEV" = "${validatedReleaseLine}" ]; then - # The system will be configured to use DEV packages where available but if none, use latest - printInfo "No specific DEV releaseline package, using latest available" - releaseMetadata="$(/bin/printf "%s" "${releases}" | jq -e -r ".[0]")" - else - printVerbose "The system is configured to only use STABLE releaseline packages but there are none." - printInfo "No release available on the '${validatedReleaseLine}' releaseline." - fi - else - # Case 1 - old package that has no release tagging yet (no DEV or STABLE), just install latest - printVerbose "Installing latest release" - releaseMetadata="$(/bin/printf "%s" "${releases}" | jq -e -r ".[0]")" - fi - fi - fi - - printVerbose "Getting specific asset details from metadata: ${releaseMetadata}" - if [ -z "${asset}" ] || [ "null" = "${asset}" ]; then - printVerbose "Asset not found during previous logic; setting now." - asset=$(/bin/printf "%s" "${releaseMetadata}" | jq -e -r '.assets[0]') - fi - if [ -z "${asset}" ]; then - printError "Unable to determine download asset for ${name}" - fi - - tagname=$(/bin/printf "%s" "${releaseMetadata}" | jq -e -r ".tag_name" | sed "s/\([^_]*\)_.*/\1/") - installedReleaseLine=$(validateReleaseLine "${tagname}") - - downloadURL=$(/bin/printf "%s" "${asset}" | jq -e -r '.url') - metadataJSONURL="$(dirname "${downloadURL}")/metadata.json" - expanded_size=$(/bin/printf "%s" "${asset}" | jq -e -r '.expanded_size') - size=$(/bin/printf "%s" "${asset}" | jq -e -r '.size') - - if [ -z "${downloadURL}" ]; then - printError "Unable to determine download location for ${name}" - fi - downloadFile=$(basename "${downloadURL}") - metadataFile="${downloadFile}.json" # use the same basename as the pax + .json to avoid collision - downloadFileVer=$(echo "${downloadFile}" | sed -E 's/.*-(.*)\.zos\.pax\.Z/\1/') - printVerbose "Downloading port from URL: ${downloadURL} to file: ${downloadFile} (ver=${downloadFileVer})" - - if ${downloadOnly}; then - printVerbose "Skipping installation, downloading only." - else - printVerbose "Install=${downloadFileVer};Original=${originalFileVersion};${upgradeInstalled};${installOrUpgrade};${reinstall}" - if [ "${downloadFileVer}" = "${originalFileVersion}" ]; then - if ! ${reinstall}; then - printInfo "${NC}${GREEN}Package ${name} is already installed at the requested version: ${downloadFileVer} and due to the absence of the 'reinstall' flag, will not be reinstalled.${NC}" - return - fi - printInfo "- Reinstalling version '${downloadFileVer}' of ${name}..." - fi - - printVerbose "Checking if package is not installed but scheduled for upgrade." - if [ -z "${originalFileVersion}" ]; then - printVerbose "No previous version found." - if ${installOrUpgrade}; then - printVerbose "Package ${name} was not installed, so installing instead of upgrading." - elif ${upgradeInstalled}; then - printError "Package ${name} can not be upgraded as it is not installed." - continue - fi - unInstallOldVersion=false - printInfo "- Installing ${name}..." - elif ${skipupgrade}; then - printInfo "Package ${name} has a newer release '${downloadFileVer}' but was not specified for an upgrade." - continue - elif ! ${setactive}; then - printVerbose "Current version '${originalFileVersion}' will remain active." - unInstallOldVersion=false - else - printVerbose "Previous version '${originalFileVersion}' installed." - if [ -e "${rootInstallDir}/${name}/${name}/.pinned" ]; then - printWarning "- Version '${originalFileVersion}' has been pinned; upgrade to '${downloadFileVer}' skipped." - syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE},${CAT_INSTALL}" "DOWNLOAD" "handlePackageInstall" "Attempt to change pinned package '${name}' skipped." - continue - else - printInfo "- Replacing ${name} version '${originalFileVersion}' with '${downloadFileVer}'" - unInstallOldVersion=true - currentversiondir=$(cd "${rootInstallDir}/${name}/${name}" && pwd -P) - currentlinkfile="${currentversiondir}/.links" - fi - fi - fi - - printVerbose "Ensuring we are in the correct working download location '${downloadDir}'" - cd "${downloadDir}" || exit - if [ ! -n "${downloadOnly}" ] || [ ! -n "${localInstall}" ]; then - printVerbose "Checking current directory for already downloaded package [file name comparison]." - location="current directory" - else - printVerbose "Checking cache for already downloaded package [file name comparison]." - location="zopen package cache" - fi - - # Download the metadata json file - if ! runAndLog "curlCmd -L '${metadataJSONURL}' -o '${metadataFile}'" ${redirectToDevNull}; then - printError "Could not download from ${metadataJSONURL}. Correct any errors and potentially retry." - continue - fi - pax=${downloadFile} - if [ -f "${pax}" ]; then - printInfo "- Found existing file '${pax}' in ${location}" - else - printInfo "- Downloading ${pax} file from remote to ${location}..." - if ! ${verbose}; then - redirectToDevNull="2>/dev/null" - fi - - # Check partition size before download package spinner starts - checkAvailableSize "${name}" "${size}" - progressHandler "network" "- Download complete." & - ph=$! - killph="kill -HUP ${ph}" - addCleanupTrapCmd "${killph}" - - if ! runAndLog "curlCmd -L '${downloadURL}' -O ${redirectToDevNull}"; then - printError "Could not download from ${downloadURL}. Correct any errors and potentially retry." - continue - fi - ${killph} 2> /dev/null # if the timer is not running, the kill will fail - syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_NETWORK},${CAT_PACKAGE},${CAT_FILE}" "DOWNLOAD" "handlePackageInstall" "Downloaded remote file '${pax}'" - fi - if [ ! -f "${pax}" ]; then - printError "${pax} was not found after download!?" - fi - - if ${downloadOnly}; then - printVerbose "Pax was downloaded to local dir '${downloadDir}'" - elif ${cacheOnly}; then - printVerbose "Pax was downloaded to zopen cache '${downloadDir}'" - else - printVerbose "Installing ${pax}" - installdirname="${name}/${pax%.pax.Z}" # Use full pax name as default - - printInfo "- Processing ${pax}..." - baseinstalldir="." - paxredirect="" - if ! ${localInstall}; then - baseinstalldir="${rootInstallDir}" - paxredirect="-s %[^/]*/%${rootInstallDir}/${installdirname}/%" - printVerbose "Non-local install, extracting with '${paxredirect}'" - else - printInfo "- Local install specified." - paxredirect="-s %[^/]*/%${installdirname}/%" - printVerbose "Non-local install, extracting with '${paxredirect}'" - fi - - megabytes=$(echo "scale=2; ${expanded_size} / (1024 * 1024)" | bc) - printInfo "After this operation, ${NC}${HEADERCOLOR}${BOLD}${megabytes} MB${NC} of additional disk space will be used." - if ! promptYesOrNo "Do you want to continue?" ${yesToPrompts}; then - if ! ${force}; then - mutexFree "zopen" - fi - printInfo "Exiting..." - exit 0 - fi - - printVerbose "Check for existing directory for version '${installdirname}'" - if [ -d "${baseinstalldir}/${installdirname}" ]; then - printInfo "- Clearing existing directory and contents" - rm -rf "${baseinstalldir}/${installdirname}" - fi - - metadataVersion=$(jq -r '.version_scheme' "${metadataFile}" 2>/dev/null) - is_greater=$(echo "$metadataVersion > 0.1" | bc -l) - - if [ "$is_greater" -eq 1 ] && ! $skipverify; then - if ! command -v gpg> /dev/null; then - skipverify=false; - printWarning "GPG is not installed" - else - verifySignatureOfPax - fi - fi - if ! runLogProgress "pax -rf ${pax} -p p ${paxredirect} ${redirectToDevNull}" "Expanding ${pax}" "Expanded"; then - printWarning "Errors unpaxing, package directory state unknown." - printWarning "Use zopen alt to select previous version." - continue - fi - - if ${localInstall}; then - rm -f "${pax}" - fi - - # Some installation have installation caveats - installCaveat=$(jq -r '.product.install_caveats // empty' "${metadataFile}" 2>/dev/null) - if [ -n "$installCaveat" ]; then - printf "${NC}${HEADERCOLOR}${BOLD}${name}${NC}:\n ${installCaveat}\n" >> ${caveatsFile} - fi - - - systemPrereqs=$(jq -r '.product.system_prereqs // empty | map(.name) | join(" ")' "${metadataFile}" 2>/dev/null) - if ! $bypassPrereqs; then - if [ -z "${systemPrereqs}" ]; then - systemPrereqs="${systemPrereqs} zos24" # set the min requirement as z/OS 2.4 - fi - if [ -n "$systemPrereqs" ]; then - if [ ! -r "$ZOPEN_SYSTEM_PREREQ_SCRIPT" ]; then - printError "$ZOPEN_SYSTEM_PREREQ_SCRIPT does not exist. Check file permissions and reinstall the meta package or reinitialize the zopen environment. If the error persists, open an issue." - else - . $ZOPEN_SYSTEM_PREREQ_SCRIPT - for prereq in $(echo "${systemPrereqs}" | xargs | tr ' ' '\n' | sort -u); do - printInfo "- Checking system pre-req requirement $prereq" - if command -V "${prereq}" >/dev/null 2>&1; then - if ! ( ${prereq} ); then - printError "Failed system pre-req check \"$prereq\". If you wish to bypass this, install with --bypass-prereq-checks" - fi - else - printError "Prereq \"$prereq\" does not exist in $ZOPEN_SYSTEM_PREREQ_SCRIPT. Consider upgrading meta or open an issue if it persists." - fi - done - fi - fi - else - syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE},${CAT_INSTALL}" "BYPASS" "handlePackageInstall" "Bypassing prereq checks ${systemPrereqs} for '${name}'." - fi - # Let the individual tools emit their own caveats - #if [ -d "${baseinstalldir}/${installdirname}/altbin" ]; then - # printf "${NC}${HEADERCOLOR}${BOLD}${name}${NC}:\n${name} has tools provided under altbin/ that conflict with tools under /bin.\nTo use them set ZOPEN_TOOLSET_OVERRIDE and then re-source the zopen-config.\n" >> ${caveatsFile} - #fi - - - if ${setactive}; then - if [ -L "${baseinstalldir}/${name}/${name}" ]; then - printVerbose "Removing old symlink '${baseinstalldir}/${name}/${name}'" - rm -f "${baseinstalldir}/${name}/${name}" - fi - if ! ln -s "${baseinstalldir}/${installdirname}" "${baseinstalldir}/${name}/${name}"; then - printError "Could not create symbolic link name." - fi - fi - - printVerbose "Adding version '${downloadFileVer}' to info file." - # Add file version information as a .releaseinfo file - echo "${downloadFileVer}" > "${baseinstalldir}/${installdirname}/.releaseinfo" - - # Check for a .version file from the pax - if present good, if not - # generate one from the file name as the tag isn't granular enough to really - # be used in dependency checks - if [ ! -f "${baseinstalldir}/${installdirname}/.version" ]; then - echo "${downloadFileVer}" > "${baseinstalldir}/${installdirname}/.version" - fi - - printVerbose "Adding releaseline '${installedReleaseLine}' metadata to ${baseinstalldir}/${installdirname}/.releaseline" - echo "${installedReleaseLine}" > "${baseinstalldir}/${installdirname}/.releaseline" - - if ${setactive}; then - if ! ${nosymlink}; then - mergeIntoSystem "${name}" "${baseinstalldir}/${installdirname}" "${ZOPEN_ROOTFS}" - misrc=$? - printVerbose "The merge completed with: ${misrc}" - fi - - printInfo "- Checking for env file." - if [ -f "${baseinstalldir}/${name}/${name}/.env" ] || [ -f "${baseinstalldir}/${name}/${name}/.appenv" ]; then - printInfo "- .env file found, adding to profiled processing." - mkdir -p "${ZOPEN_ROOTFS}/etc/profiled/${name}" - cat << EOF > "${ZOPEN_ROOTFS}/etc/profiled/${name}/dotenv" -curdir=\$(pwd) -cd "\${ZOPEN_ROOTFS}${ZOPEN_PKGINSTALL##"${ZOPEN_ROOTFS}"}/${name}/${name}" >/dev/null 2>&1 -# If .appenv exists, source it as it's quicker -if [ -f ".appenv" ]; then - . ./.appenv -elif [ -f ".env" ]; then - . ./.env -fi -cd \${curdir} >/dev/null 2>&1 -EOF - printInfo "- Sourcing environment to run any setup." - cd "${baseinstalldir}/${name}/${name}" && ./setup.sh - fi - fi - if ${unInstallOldVersion}; then - printVerbose "New version merged; checking for orphaned files from previous version." - # This will remove any old symlinks or dirs that might have changed in an upgrade - # as the merge process overwrites existing files to point to different version - unsymlinkFromSystem "${name}" "${ZOPEN_ROOTFS}" "${currentlinkfile}" "${baseinstalldir}/${name}/${name}/.links" - fi - - if ${setactive}; then - printVerbose "Marking this version as installed." - touch "${baseinstalldir}/${name}/${name}/.active" - installedList="${name} ${installedList}" - syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_INSTALL},${CAT_PACKAGE}" "DOWNLOAD" "handlePackageInstall" "Installed package:'${name}';version:${downloadFileVer};install_dir='${baseinstalldir}/${installdirname}';" - fi - - registerInstall "$name" "${downloadFileVer}" "${upgradeInstalled}" ${isRuntimeDependency} - - if ${doNotInstallDeps}; then - printInfo "- Skipping dependency installation." - elif ${reinstall}; then - printVerbose "- Reinstalling so no dependency reinstall (unless explicitly listed)." - else - printInfo "- Checking for runtime dependencies." - printVerbose "Checking for .runtimedeps file." - if [ -e "${baseinstalldir}/${name}/${name}/.runtimedeps" ]; then - dependencies=$(cat "${baseinstalldir}/${name}/${name}/.runtimedeps") - fi - printVerbose "Checking for runtime dependencies from the git metadata." - if echo "${statusline}" | grep "Runtime Dependencies:" > /dev/null; then - gitmetadependencies="$(echo "${statusline}" | sed -e "s#.*Runtime Dependencies:<\/b> ##" -e "s#
.*##")" - if [ ! "${gitmetadependencies}" = "No dependencies" ]; then - dependencies="${dependencies} ${gitmetadependencies}" - fi - fi - dependencies=$(deleteDuplicateEntries "${dependencies}" " ") - if [ -n "${dependencies}" ]; then - printInfo "- ${name} depends on: ${dependencies}" - printInfo "- Installing dependencies." - installDependencies "${name}" "${dependencies}" - else - printInfo "- No runtime dependencies found." - fi - fi - printInfo "${NC}${GREEN}Successfully installed: ${name}${NC}" - fi # (download only) -} - -installPorts() -( - ports="$1" - printVerbose "Ports to install: ${ports}" - if ! ${force}; then - mutexReq "zopen" "zopen" - fi - caveatsFile=$(mktempfile "caveats") - echo "${ports}" | xargs | tr ' ' '\n' | while read port; do - handlePackageInstall "${port}" - done - if [ -s "${caveatsFile}" ]; then - printHeader "Installation Caveats" - cat "${caveatsFile}" - fi - rm -f "${caveatsFile}" - if ! ${force}; then - mutexFree "zopen" - fi -) - # Main code start here +# Need to set a number of variables for use in the install function +# which is common between install & upgrade args=$* -upgradeInstalled=false verbose=false debug=false +xdebug=false selectVersion=false +# shellcheck disable=SC2034 setActive=true -cacheOnly=false downloadOnly=false -localInstall=false reinstall=false installOrUpgrade=false nosymlink=false @@ -678,195 +164,178 @@ skipupgrade=false skipverify=false doNotInstallDeps=false all=false -yesToPrompts=false +yesToPrompts=falseskipupgrade bypassPrereqs=false force=false chosenRepos="" +fileinstall=false + while [ $# -gt 0 ]; do case "$1" in - "-u" | "--update" | "--upgrade") - upgradeInstalled=true # Upgrade packages - ;; - "-r" | "-reinstall" | "--reinstall") - reinstall=true # If package already installed, reinstall - ;; - "--install-or-upgrade") - installOrUpgrade=true # Upgrade package or install if not present - ;; - "--bypass-prereq-checks") - bypassPrereqs=true - ;; - "--local-install") - localInstall=true # Install the package into current directory - ;; - "--no-symlink") - nosymlink=true # Do not mesh the package into the file system; leave as stand-alone - ;; - "--no-deps") - doNotInstallDeps=true - ;; - "--cache-only") - cacheOnly=true # Download remote pax file to cache only (no install) - ;; - "--release-line") - shift - releaseLine=$(echo "$1" | awk '{print toupper($0)}') - ;; - "--yes" | "-y") - yesToPrompts=true # Automatically answer 'yes' to any questions - ;; - "--download-only") - downloadOnly=true # Download remote pax file to current directory only - ;; - "--no-set-active") - setactive=false # Install package as normal but keep existing installation as active - ;; - "--skip-upgrade") - skipupgrade=true # Do not upgrade any packages - ;; - "--skip-verify" | "-sv") - skipverify=true # Verify signature of packages - ;; - "--force") - force=true # Bypasses locks - ;; - "--all") - all=true # Install all packages - ;; - "--select") - selectVersion=true # Display a selction table to allow version picking - ;; - "-h" | "--help" | "-?" ) - printSyntax "${args}" - exit 0 - ;; - "--debug") - verbose=true - debug=true - ;; - "-v" | "--verbose") - verbose=true + "-r" | "-reinstall" | "--reinstall") + # shellcheck disable=SC2034 + reinstall=true # If package already installed, reinstall + ;; + "--install-or-upgrade") + # shellcheck disable=SC2034 + installOrUpgrade=true # Upgrade package or install if not present + ;; + "--bypass-prereq-checks") + bypassPrereqs=true + ;; + "--no-symlink") + # shellcheck disable=SC2034 + nosymlink=true # Do not mesh the package into the file system; leave as stand-alone + ;; + "--no-deps") + doNotInstallDeps=true + ;; + "--release-line") + shift + # shellcheck disable=SC2034 + releaseLine=$(echo "$1" | awk '{print toupper($0)}') + ;; + "--yes" | "-y") + # shellcheck disable=SC2034 + yesToPrompts=true # Automatically answer 'yes' to any questions + ;; + "--download-only") + downloadOnly=true # Download remote package files to current directory only + ;; + "--no-set-active") + # shellcheck disable=SC2034 + setactive=false # Install package as normal but keep existing installation as active + ;; + "--skip-verify" | "-sv") + # shellcheck disable=SC2034 + skipverify=true # Verify signature of packages + ;; + "--force") + # shellcheck disable=SC2034 + force=true # Bypasses locks + ;; + "--all") + all=true # Install all packages + ;; + "--select") + # shellcheck disable=SC2034 + selectVersion=true # Display a selction table to allow version picking + ;; + "-h" | "--help" | "-?") + printHelp "${args}" + exit 0 + ;; + "--debug") + verbose=true + # shellcheck disable=SC2034 + debug=true + ;; + "-v" | "--verbose") + # shellcheck disable=SC2034 + verbose=true + ;; + "--xdebug") + xdebug=true ;; "--version") - zopen-version ${ME} + zopen-version "${ME}" exit 0 ;; + + -*) printError "Unsupported parameter '$1'";; *) - chosenRepos="${chosenRepos} $1" + # Generate a long @@ separated string to allow for embedded + # spaces in hardcoded pax filenames + chosenRepos="${chosenRepos}@@$1" ;; esac shift done -if [ -z "${chosenRepos}" ]; then - if ! ${all} && ! ${upgradeInstalled}; then - printInfo "No ports selected for installation." - exit 4 - fi - if ${upgradeInstalled}; then - printVerbose "No specific port to upgrade, upgrade all installed packages." - printInfo "- Querying for installed packages." - progressHandler "spinner" "- Query complete." & - ph=$! - killph="kill -HUP ${ph}" - addCleanupTrapCmd "${killph}" - chosenRepos="$(${MYDIR}/zopen-query --list --installed --no-header --no-version 2>&1)" - zqrc=$? - ${killph} 2> /dev/null # if the timer is not running, the kill will fail - sleep 1 # give the above process time to clear - if [ ${zqrc} -ne 0 ]; then - printError "Query for installed packages unexpectedly failed; zopen-query returned message: '${chosenRepos}'" - fi - fi +${xdebug} && set -x && printVerbose "Enabled command execution trace" + +if ! ${all} && [ -z "${chosenRepos}" ]; then + printInfo "No packages selected for installation." + exit 4 fi checkIfConfigLoaded -export SSL_CERT_FILE="${ZOPEN_CA}" -export GIT_SSL_CAINFO="${ZOPEN_CA}" -export CURL_CA_BUNDLE="${ZOPEN_CA}" +#export SSL_CERT_FILE="${ZOPEN_CA}" +#export GIT_SSL_CAINFO="${ZOPEN_CA}" +#export CURL_CA_BUNDLE="${ZOPEN_CA}" + +# If any of the parameters passed in point to an existing file, then +# the user is attempting to install a port directly from the file system +# rather than a repo +printDebug "Checking input parameters for actual files" +potentials=$(echo "${chosenRepos}" | sed 's/@@/ /g') +for installRepo in ${potentials}; do + [ -e "${installRepo}" ] && fileinstall=true && break +done if ${downloadOnly}; then downloadDir="${PWD}" - printVerbose "Downloading pax to current directory '${downloadDir}'" -elif ${localInstall}; then - downloadDir="${PWD}" - printVerbose "Installing to current directory '${downloadDir}'" + printDebug "Downloading pax to current directory '${downloadDir}'" else - printVerbose "Installing to zopen file system: ${ZOPEN_ROOTFS}" + printDebug "Installing to zopen file system: ${ZOPEN_ROOTFS}" if [ -z "${ZOPEN_ROOTFS}" ]; then - printError "Unable to locate zopen file system, \$ZOPEN_ROOTFS is undefined." + printError "Unable to locate zopen file system, \${ZOPEN_ROOTFS} is undefined. Re-source zopen-config and retry command." fi downloadDir="${ZOPEN_ROOTFS}/var/cache/zopen" fi if [ ! -d "${downloadDir}" ]; then - mkdir -p "${downloadDir}" - if [ $? -gt 0 ]; then - printError "Could not create download directory: ${downloadDir}" + if ! mkdir -p "${downloadDir}"; then + printError "Could not create download directory: ${downloadDir}. Check permissions and retry command." fi fi -if [ -n "${downloadDir}" ] && [ -d "${downloadDir}" ]; then - cd "${downloadDir}" || exit +printDebug "Checking if installing from pax files: ${fileinstall}" +if ! ${fileinstall}; then + printVerbose "Querying metadata for latest package information" + getRepos + grfgRc=$? + [ 0 -ne ${grfgRc} ] && exit ${grfgRc}; fi -printVerbose "Working directory: ${downloadDir}" -# Parse passed in repositories and check if valid zopen framework repos -printInfo "- Querying repo for latest package information." -getReposFromGithub -grfgRc=$? -${killph} 2> /dev/null # if the timer is not running, the kill will fail -[ 0 -ne "${grfgRc}" ] && exit "${grfgRc}" - -foundPort=false -installArray="" - -if ${all}; then - if ! ${yesToPrompts}; then - # Sum up the pax size + expanded size of the latest releases of each port - spaceRequiredBytes=$(jq '.release_data | to_entries | map(select(.value | length > 0)) | map(.value[0].assets[0]) | reduce .[] as $item ({}; . + {($item.name): (($item.size | tonumber) + ($item.expanded_size | tonumber))}) | [.[]] | add' ${JSON_CACHE}) - echo "Space: $spaceRequiredBytes" - spaceRequiredMB=$(echo "scale=0; ${spaceRequiredBytes} / (1024 * 1024)" | bc) - availableSpaceMB=$(/bin/df -m ${ZOPEN_ROOTFS} | sed "1d" | awk '{ print $3 }' | awk -F'/' '{ print $1 }') - - printInfo "You have chosen to install all tools. An estimated ${spaceRequiredMB} MB of additional disk space will be used." - if [ $availableSpaceMB -lt $spaceRequiredMB ]; then - printWarning "Your zopen file-system ($ZOPEN_ROOTFS) only has ${availableSpaceMB} MB of available space." - fi - printInfo "Enter 'all' to confirm full installation. (This can take a VERY long time!):" - confirmall=$(getInput) - if [ ! "xall" = "x${confirmall}" ]; then - printError "Cancelling full installation." - fi - fi - for repo in $(echo ${repo_results}); do - installArray="${installArray} ${repo}" - done - installArray=$(strtrim "${installArray}") +if ${fileinstall}; then + printDebug "Installing from files as listed in arguments: '${chosenRepos}'" + # generate the install list JSON from the @@-delimited inputs + installList=$(echo "${chosenRepos}" \ + | jq --raw-input --arg d "$(pwd -P)" \ + 'def make_object($url): {asset:{url: ( "file://" + $d + "/" + $url )}}; . | split("@@") | map(select(.!="")|make_object(.)) | {"installqueue" :.} ') else - chosenRepos=$(strtrim "${chosenRepos}") - invalidlist="" - for chosenRepo in $(echo "${chosenRepos}" | tr ',' ' ' | tr -s ' '); do - printVerbose "Processing repo: ${chosenRepo}" - printVerbose "Stripping any version (%), tag (#) or port suffixes" - toolrepo=$(echo "${chosenRepo}" | sed -e 's#%.*##' -e 's#=.*##') - toolfound=$(echo "${repo_results}" | awk -vtoolrepo="${toolrepo}" '$0 == toolrepo {print}') - if [ "${toolfound}" = "${toolrepo}" ]; then - printVerbose "Adding '${chosenRepo}' to the install queue." - installArray="${installArray} ${chosenRepo}" - printVerbose "Removing valid port from input list." - chosenRepos=$(echo "${chosenRepos}" | sed -e "s#^${chosenRepo}\$##") - else - invalidlist=$(printf "%s %s" "${invalidlist}" "${chosenRepo}") - fi - done + if ${all}; then + # shellcheck disable=SC2034 + doNotInstallDeps=true + installList=$(jq --raw-output '.release_data| keys[]' "${JSON_CACHE}") + installListCount=$(jq --raw-output '.release_data| keys | length' "${JSON_CACHE}") + printInfo "- Installing all currently-uninstalled packages" + printInfo "- Checking installation status for '${installListCount}' packages" + else + installList=$(echo "$chosenRepos" | sed "s/@@/ /g") + validateInstallList "${installList}" + fi + printInfo "- Generating install graph" + progressHandler "spinner" & + gigph=$! + killph="kill -HUP ${gigph}" + addCleanupTrapCmd "${killph}" + generateInstallGraph "${installList}" + ${killph} 2>/dev/null # if the timer is not running, the kill will fail + waitforpid ${gigph} # Make sure it's finished writing to screen fi -printVerbose "Checking whether any invalid ports were specified." -if [ -n "${invalidlist}" ]; then - printSoftError "The following requested port(s) do not exist:\n\t$(echo "${invalidlist}" | tr -s '[:space:]')" - printError "Check port name(s), remove any port suffixes and retry command." +if [ 0 -eq "$(echo "${installList}" | jq --raw-output '.installqueue| length')" ]; then + printInfo "- No packages for install" +else + if ${verbose}; then + printInfo " - The following package(s) will be installed:" + echo "${installList}" | jq --raw-output '.installqueue | sort| .[] | .portname ' + fi + processRepoInstallFile + fi -installPorts "${installArray}" +exit From 2fdcbf7bb901d25bc0c322556f76c6831b1afe0f Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 08:09:46 +0000 Subject: [PATCH 010/179] New script to allow better zopen --help display and separation of concerns --- bin/zopen-upgrade | 183 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100755 bin/zopen-upgrade diff --git a/bin/zopen-upgrade b/bin/zopen-upgrade new file mode 100755 index 000000000..0c9166c1d --- /dev/null +++ b/bin/zopen-upgrade @@ -0,0 +1,183 @@ +#!/bin/sh +# Upgrade utility for z/OS Open Tools - https://github.com/ZOSOpenTools +# +# All zopen-* scripts MUST start with this code to maintain consistency +# +setupMyself() +{ + ME=$(basename $0) + MYDIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 && pwd -P )" + INCDIR="${MYDIR}/../include" + if ! [ -d "${INCDIR}" ] && ! [ -f "${INCDIR}/common.sh" ]; then + echo "Internal Error. Unable to find common.sh file to source" >&2 + exit 8 + fi + # shellcheck disable=SC1091 + . "${INCDIR}/common.sh" +} +setupMyself +checkWritable + +printHelp(){ + cat << HELPDOC +zopen upgrade is a utility for z/OS Open Tools to upgrade packages to +a later release + +Usage: zopen upgrade [OPTION] [PARAMETERS] [PACKAGES] + +Options: + -y, --yes automatically answer yes to prompts + -v, --verbose run in verbose mode + -h,-?, --help display this help and exit + +Examples: + zopen upgrade foo + upgrade package foo if installed + zopen upgrade -y upgrade all packages to latest version on their + releaseline + +Report bugs at https://github.com/ZOSOpenTools/meta/issues . + +HELPDOC +} + + + +# Main code start here +# Need to set a number of variables for use in the install function +# which is common between install & upgrade +args=$* +verbose=false +debug=false +xdebug=false +# shellcheck disable=SC2034 +selectVersion=false +# shellcheck disable=SC2034 +setActive=true +# shellcheck disable=SC2034 +downloadOnly=false +# shellcheck disable=SC2034 +reinstall=false +nosymlink=false +doNotInstallDeps=true +yesToPrompts=false +chosenRepos="" +while [ $# -gt 0 ]; do + case "$1" in + "--yes" | "-y") + yesToPrompts=true # Automatically answer 'yes' to any questions + ;; + "--no-set-active") + setactive=false # Install package as normal but keep existing installation as active + ;; + "-h" | "--help" | "-?") + printHelp "${args}" + exit 0 + ;; + "-v" | "--verbose") + verbose=true + ;; + "--debug") + verbose=true + debug=true + ;; + "--xdebug") + verbose=true + debug=true + xdebug=true + ;; + "--version") + zopen-version "${ME}" + exit 0 + ;; + -*) printError "Unsupported parameter '$1'" ;; + *) + chosenRepos=" ${chosenRepos} $1 "; + ;; + esac + shift; +done + +${xdebug} && set -x && printVerbose "Enabled command execution trace" + +if [ -z "${chosenRepos}" ]; then + printVerbose "No specific port to upgrade, upgrade all installed packages" + printInfo "- Querying installed packages" + progressHandler "mirror" & + ph=$! + killph="kill -HUP ${ph}" + addCleanupTrapCmd "${killph}" + chosenRepos=$(zopen list --installed) + ${killph} 2>/dev/null # if the timer is not running, the kill will fail + waitforpid ${ph} # Make sure it's finished writing to screen +fi + +checkIfConfigLoaded + +export SSL_CERT_FILE="${ZOPEN_CA}" +export GIT_SSL_CAINFO="${ZOPEN_CA}" +export CURL_CA_BUNDLE="${ZOPEN_CA}" + +printDebug "Installing to zopen file system: ${ZOPEN_ROOTFS}" +if [ -z "${ZOPEN_ROOTFS}" ]; then + printError "Unable to locate zopen file system, \${ZOPEN_ROOTFS} is undefined" +fi +downloadDir="${ZOPEN_ROOTFS}/var/cache/zopen" + + +if [ ! -d "${downloadDir}" ]; then + mkdir -p "${downloadDir}" + if [ $? -gt 0 ]; then + printError "Could not create download directory: ${downloadDir}" + fi +fi + +# Parse passed in repositories and check if valid zopen framework repos +printVerbose "Querying remote repo for latest package information" +getRepos +grfgRc=$? +[ 0 -ne ${grfgRc} ] && exit ${grfgRc}; +installArray="" + +mutexReq "zopen" "zopen" +printDebug "Parsing list of packages to install and verifying validity" + +if [ -z "${chosenRepos}" ]; then + badportlist="" + for chosenRepo in $(echo "${chosenRepos}" | tr ',' ' '); do + printVerbose "Processing repo: ${chosenRepo}" + printDebug "Stripping any version (%), tag (#) or port suffixes and trim" + toolrepo=$(echo "${chosenRepo}" | sed -e 's#%.*##' -e 's#=.*##' -e 's#.*port##') + toolfound=$(echo "${repo_results}" | awk -vtoolrepo="${toolrepo}" '$0 == toolrepo {print}') + if [ "${toolfound}" = "${toolrepo}" ]; then + printVerbose "Adding '${chosenRepo}' to the install queue" + installArray=$(printf "%s\n%s" "${installArray}" "${chosenRepo}") + else + badportlist=$(printf "%s %s" "${badportlist}" "${toolrepo}") + fi + done + + if [ -z "${badportlist}" ]; then + printSoftError "The following requested port(s) do not exist:\n\t${badportlist}" + fi + upgradeList=$(jq --raw-output '.release_data| keys[]' "${JSON_CACHE}") + generateInstallGraph "${upgradeList}" +else + validateInstallList "${chosenRepos}" + printInfo "- Generating install graph" + progressHandler "spinner" & + ph=$! + killph="kill -HUP ${ph}" + addCleanupTrapCmd "${killph}" + generateInstallGraph "${chosenRepos}" + ${killph} 2>/dev/null # if the timer is not running, the kill will fail + waitforpid ${ph} # Make sure it's finished writing to screen +fi + +if [ 0 -eq "$(echo "${installList}" | jq --raw-output '.installqueue| length')" ]; then + printInfo "- No packages to upgrade" +else + processRepoInstallFile +fi + +mutexFree "zopen" From 82761c9b7dcc250c602f49d2d30ab94fbd0c36a2 Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 08:59:23 +0000 Subject: [PATCH 011/179] Prevent screen overlay issues with ansi progress displays add animations --- include/common.sh | 89 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 28 deletions(-) diff --git a/include/common.sh b/include/common.sh index 492966c9c..a6f2bbcff 100755 --- a/include/common.sh +++ b/include/common.sh @@ -447,6 +447,7 @@ defineANSI() # Color-type codes, needs explicit terminal settings if [ ! "${_BPX_TERMPATH-x}" = "OMVS" ] && [ -z "${NO_COLOR}" ] && [ ! "${FORCE_COLOR-x}" = "0" ] && [ -t 1 ] && [ -t 2 ]; then + ANSION=true esc="\047" BLACK="${esc}[30m" RED="${esc}[31m" @@ -472,15 +473,18 @@ defineANSI() fi else # unset esc RED GREEN YELLOW BOLD UNDERLINE NC - + ANSION=false esc='' + # shellcheck disable=SC2034 BLACK='' RED='' GREEN='' YELLOW='' BLUE='' MAGENTA='' + # shellcheck disable=SC2034 CYAN='' + # shellcheck disable=SC2034 GRAY='' BOLD='' UNDERLINE='' @@ -495,19 +499,31 @@ ansiline() { deltax=$1 deltay=$2 - echostr=$3 - if [ ${deltax} -gt 0 ]; then - echostr="${ESC}[${deltax}A${echostr}" - elif [ ${deltax} -lt 0 ]; then - echostr="${ESC}[$(expr ${deltax} \* -1)A${echostr}" + echostr="$3" + ansimove $1 $2 + /bin/echo "${echostr}\c" +} + +ansimove() +{ + deltax=$1 + deltay=$2 + movestr="" + if [ -n "${deltax}" ]; then + if [ "${deltax}" -gt 0 ]; then + movestr="${ESC}[${deltax}C" + elif [ "${deltax}" -lt 0 ]; then + movestr="${ESC}[$(expr "${deltax}" \* -1)D" + fi fi - if [ ${deltay} -gt 0 ]; then - echostr="${ESC}[${deltax}C${echostr}" - elif [ ${deltay} -lt 0 ]; then - echostr="${ESC}[$(expr ${deltax} \* -1)D${echostr}" + if [ -n "${deltay}" ]; then + if [ "${deltay}" -gt 0 ]; then + movestr="${movestr}${ESC}[${deltay}B" + elif [ "${deltay}" -lt 0 ]; then + movestr="${movestr}${ESC}[$(expr "${deltay}" \* -1)A" + fi fi - /bin/echo "${echostr}" - + /bin/echo "${movestr}\c" } getScreenCols() @@ -782,6 +798,7 @@ unsymlinkFromSystem() fi done ${killph} 2>/dev/null # if the timer is not running, the kill will fail + waitforpid ${ph} # Make sure it's finished writing to screen sleep 1 # give spinner time to exit if running else # Slower method needed to analyse each link to see if it has @@ -800,9 +817,9 @@ unsymlinkFromSystem() [ -e "${tempTrash}" ] && rm -f "${tempTrash}" >/dev/null 2>&1 addCleanupTrapCmd "rm -rf ${tempDirFile}" printDebug "Using temporary file ${tempDirFile}" - printInfo "- Checking ${nfiles} potential links" + printInfo "- Checking ${nfiles} potentially obsolete file links" printDebug "Starting spinner..." - progressHandler "spinner" "- Complete" & + progressHandler "linkcheck" "- Complete" & ph=$! killph="kill -HUP ${ph}" addCleanupTrapCmd "${killph}" @@ -831,7 +848,7 @@ unsymlinkFromSystem() $(zossed '1d;$d' "${dotlinks}") EOF ${killph} 2>/dev/null # if the timer is not running, the kill will fail - sleep 1 # ensure the spinner has stopped if running + waitforpid ${ph} # Make sure it's finished writing to screen if [ -e "${tempDirFile}" ]; then ndirs=$(uniq < "${tempDirFile}" | wc -l | tr -d ' ') printVerbose "- Checking ${ndirs} dir links" @@ -914,7 +931,7 @@ runLogProgress() fi progressHandler "spinner" "- ${completeText}" & ph=$! - killph="kill -HUP ${ph} 2>/dev/null" + killph="kill -HUP ${ph}" addCleanupTrapCmd "${killph}" eval "$1" rc=$? @@ -922,6 +939,7 @@ runLogProgress() chtag -r ${SSH_TTY} fi ${killph} 2> /dev/null # if the timer is not running, the kill will fail + waitforpid ${ph} # Make sure it's finished writing to screen return "${rc}" } @@ -941,7 +959,7 @@ progressAnimation() [ $# -eq 0 ] && printError "Internal error: no animation strings." animcnt=$# anim=1 - ansiline 0 0 "$1" +# ansiline 0 0 "$1\n\c" while :; do spinloop 1000 # Check for daemonization of this process (ie. orphaned and PPID=1) @@ -952,34 +970,49 @@ progressAnimation() [ "${ppid}" -eq 1 ] && kill INT "${ppid}" >/dev/null 2>&1 anim=$((anim + 1)) [ ${anim} -gt ${animcnt} ] && anim=1 - ansiline 1 -1 $(getNthArrayArg "${anim}" "$@") + ansiline -10 0 "$(getNthArrayArg "${anim}" "$@")" + ansimove -10 0 done } -getNthArrayArg () { +getNthArrayArg () +{ shift "$1" - echo "$1" + echo "$1\c" +} + +waitforpid() +{ +while kill -0 "$1" >/dev/null 2>&1; do + sleep 1 +done } progressHandler() { - if [ ! "${_BPX_TERMPATH-x}" = "OMVS" ] && [ -z "${NO_COLOR}" ] && [ ! "${FORCE_COLOR-x}" = "0" ] && [ -t 1 ] && [ -t 2 ]; then + # if [ ! "${_BPX_TERMPATH-x}" = "OMVS" ] && [ -z "${NO_COLOR}" ] && [ ! "${FORCE_COLOR-x}" = "0" ] && [ -t 1 ] && [ -t 2 ]; then + if ${ANSION}; then [ -z "${-%%*x*}" ] && set +x # Disable -x debug if set for this process type=$1 completiontext=$2 # Custom end text (when the process is complete) trapcmd="exit;" if [ -n "${completiontext}" ]; then - trapcmd="/bin/echo \"\047[1A\047[30D\047[2K${completiontext}\"; ${trapcmd}" + ansiline + #trapcmd="/bin/echo \"\047[0A\047[10D\047[2K${completiontext}\n\c\"; ${trapcmd}" + trapcmd="/bin/echo \"\047[10D\047[K${completiontext}\n\c\"; ${trapcmd}" + else + #trapcmd="/bin/echo \"\047[0A\047[10D\c\"; ${trapcmd}" + trapcmd="/bin/echo \"\047[10D\047[K\c\"; ${trapcmd}" fi # shellcheck disable=SC2064 trap "${trapcmd}" HUP case "${type}" in - "spinner") progressAnimation '-' '\' '|' '/' - ;; - "network") progressAnimation '-----' '>----' '->---' '-->--' '--->-' '---->' '-----' '----<' '---<-' '--<--' '-<---' '<----' - ;; - *) progressAnimation '.' 'o' 'O' 'O' 'o' '.' - ;; + "spinner") progressAnimation '-' '>' '|' '>' ;; + "network") progressAnimation '-----' '>----' '->---' '-->--' '--->-' '---->' '-----' '----<' '---<-' '--<--' '-<---' '<----' ;; + "mirror") progressAnimation '#______' '##_____' '#=#____' '#==#___' '#===#__' '#====#_' '#=====#' '#_====#' '#__===#' '#___==#' '#____=#' '#_____#' ;; + "trash") progressAnimation 'O________' '_O_______' '__O______' '___o_____' '____o____' '_____o___' '______.__' '_______._' '________.' ;; + "linkcheck")progressAnimation '------>' '?----->' '-?---->' '--?--->' '---?-->' '----?->' '-----?>';; + *) progressAnimation '.' 'o' 'O' 'O' 'o' '.' ;; esac fi } From 0fc35f9c4782529da36014951242d95bd0425d0d Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 11:00:33 +0000 Subject: [PATCH 012/179] Code tidy - remove unused, correct shellcheck issues --- include/common.sh | 93 +++++++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 52 deletions(-) diff --git a/include/common.sh b/include/common.sh index a6f2bbcff..dbd6fe3d3 100755 --- a/include/common.sh +++ b/include/common.sh @@ -570,7 +570,7 @@ findrev() { haystack="$1" needle="$2" - while [[ "${haystack}" != "" && "${haystack}" != "/" && "${haystack}" != "./" && ! -e "${haystack}/${needle}" ]]; do + while [ "${haystack}" != "" ] && [ "${haystack}" != "/" ] && [ "${haystack}" != "./" ] && [ ! -e "${haystack}/${needle}" ]; do haystack=${haystack%/*} done echo "${haystack}" @@ -688,7 +688,6 @@ mergeIntoSystem() [ -z "${rebaseusr}" ] && rebaseusr="usr/local" currentDir="${PWD}" - targetdir="${rootfs}/${rebaseusr}" # The main rootfs/usr location printDebug "Calculating the offset path to store from root" offset=$(dirname "${versioneddir#"${rootfs}"/}") @@ -705,7 +704,7 @@ mergeIntoSystem() mv "${versioneddir}" "${virtualStore}" printDebug "Creating main linked directory in store" - $(cd "${virtualStore}" && ln -s "${version}" "${name}") + cd "${virtualStore}" && ln -s "${version}" "${name}" printDebug "Creating virtual root directory structure" mkdir -p "${processingDir}/${rebaseusr}" @@ -717,8 +716,7 @@ mergeIntoSystem() printDebug "Generating symlink tree" printDebug "Creating directory structure" - curdir="${PWD}" - cd "${virtualStore}/${name}" || exit + cd "${virtualStore}/${name}" || printError "Unable to change to virtual store at '${virtualStore}/${name}'" # since 'ln *' doesn't invoke globbing to allow multiple files at once, # abuse the Recurse option; this results in "already exists" errors but # ignore them as the first call should generate the correct link but @@ -727,9 +725,9 @@ mergeIntoSystem() zosfind . -type d | sort -r | while read dir; do dir=$(echo "${dir}" | zossed "s#^./##") printDebug "Processing dir: ${dir}" - [ ${dir} = "." ] && continue + [ "${dir}" = "." ] && continue mkdir -p "${processingDir}/${rebaseusr}/${dir}" - cd "${processingDir}/${rebaseusr}/${dir}" || exit + cd "${processingDir}/${rebaseusr}/${dir}" || printError "Unable to change to processing directory '${processingDir}/${rebaseusr}/${dir}'" dirrelpath=$(relativePath2 "${virtualStore}/${name}/${dir}" "${processingDir}/${rebaseusr}/${dir}") ln -Rs "${dirrelpath}/" "." 2> /dev/null done @@ -745,7 +743,7 @@ mergeIntoSystem() printDebug "Generating intermediary tar file" # Need '-S' to allow long symlinks - $(cd "${processingDir}" && tar -S -cf "${tarfile}" "usr") + cd "${processingDir}" && tar -S -cf "${tarfile}" "usr" printDebug "Generating listing for remove processing (including main symlink)." listing=$(tar tf "${processingDir}/${tarfile}" 2> /dev/null | sort -r) @@ -758,8 +756,8 @@ mergeIntoSystem() printDebug "Cleaning temp resources." rm -rf "${processingDir}" 2> /dev/null - printDebug "Switching to previous cwd - current work dir was purged." - cd "${currentDir}" || exit + printDebug "Switching to previous cwd - current work dir was purged" + cd "${currentDir}" || printError "Unable to change to '${currentDir}'" printInfo "- Integration complete." return 0 @@ -872,20 +870,22 @@ EOF printDebug() { [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" + # shellcheck disable=SC2154 if ${debug}; then printColors "${NC}${BLUE}${BOLD}:DEBUG:${NC}: '${1}'" fi - [ ! -z "${xtrc}" ] && set -x + [ -n "${xtrc}" ] && set -x return 0 } printVerbose() { [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" + # shellcheck disable=SC2154 if ${verbose}; then printColors "${NC}${GREEN}${BOLD}VERBOSE${NC}: ${1}" fi - [ ! -z "${xtrc}" ] && set -x + [ -n "${xtrc}" ] && set -x return 0 } @@ -893,7 +893,7 @@ printHeader() { [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" printColors "${NC}${HEADERCOLOR}${BOLD}${UNDERLINE}${1}${NC}" - [ ! -z "${xtrc}" ] && set -x + [ -n "${xtrc}" ] && set -x return 0 } @@ -901,7 +901,7 @@ printAttention() { [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" printColors "${NC}${MAGENTA}${BOLD}${UNDERLINE}${1}${NC}" - [ ! -z "${xtrc}" ] && set -x + [ -n "${xtrc}" ] && set -x return 0 } @@ -935,8 +935,8 @@ runLogProgress() addCleanupTrapCmd "${killph}" eval "$1" rc=$? - if [ ! -z "${SSH_TTY}" ]; then - chtag -r ${SSH_TTY} + if [ -n "${SSH_TTY}" ]; then + chtag -r "${SSH_TTY}" fi ${killph} 2> /dev/null # if the timer is not running, the kill will fail waitforpid ${ph} # Make sure it's finished writing to screen @@ -1030,14 +1030,14 @@ runInBackgroundWithTimeoutAndLog() kill -0 "${PID}" 2> /dev/null if [ $? != 0 ]; then wait "${PID}" - if [ ! -z "${SSH_TTY}" ]; then - chtag -r ${SSH_TTY} + if [ -n "${SSH_TTY}" ]; then + chtag -r "${SSH_TTY}" fi rc=$? - return "${rc}" + return ${rc} else sleep 1 - n=$(expr ${n} + 1) + n=$(( n + 1)) fi done kill -9 "${PID}" 2>/dev/null @@ -1067,7 +1067,7 @@ printWarning() [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" printColors "${NC}${WARNINGCOLOR}${BOLD}***WARNING: ${NC}${YELLOW}${1}${NC}" >&2 [ -n "${xtrc}" ] && set -x - return 0 + return 0; } printInfo() @@ -1075,7 +1075,7 @@ printInfo() [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" printColors "$1" [ -n "${xtrc}" ] && set -x - return 0 + return 0; } # Used to input sensitive data - turns off echo to the screen for the input @@ -1087,14 +1087,14 @@ getInputHidden() addCleanupTrapCmd "stty echo" stty -echo read zopen_input - echo ${zopen_input} + echo "${zopen_input}" stty echo } getInput() { read zopen_input - echo ${zopen_input} + echo "${zopen_input}" } printElapsedTime() @@ -1102,7 +1102,7 @@ printElapsedTime() printType=$1 functionName=$2 startTime=$3 - elapsedTime=$((${SECONDS} - ${startTime})) + elapsedTime=$((SECONDS - startTime)) elapsedTimeOutput="${functionName} completed in ${elapsedTime} seconds." @@ -1122,6 +1122,7 @@ processConfig() if [ -z "${ZOPEN_ROOTFS}" ]; then relativeRootDir="$(cd "$(dirname "$0")/../.." > /dev/null 2>&1 && pwd -P)" if [ -f "${relativeRootDir}/etc/zopen-config" ]; then + # shellcheck source=/dev/null . "${relativeRootDir}/etc/zopen-config" else printError "Source the zopen-config prior to running $0." @@ -1138,7 +1139,7 @@ checkIfConfigLoaded() errorMessage="Certificate at ${ZOPEN_CA} could not be accessed. Ensure zopen init has run and zopen-config has been sourced." fi - if [ ! -z "${errorMessage}" ]; then + if [ -n "${errorMessage}" ]; then if [ -r "${mydir}/../../../etc/zopen-config" ]; then relativeConfigDir="$(cd "$(dirname "${mydir}")/../../etc/" > /dev/null 2>&1 && pwd -P)" errorMessage="${errorMessage} Run '. ${relativeConfigDir}/zopen-config' or add it to your .profile." @@ -1150,28 +1151,28 @@ checkIfConfigLoaded() parseDeps() { dep="$1" - version=$(echo ${dep} | awk -F '[>=<]+' '{print $2}') + version=$(echo "${dep}" | awk -F '[>=<]+' '{print $2}') if [ -z "${version}" ]; then operator="" - dep=$(echo ${dep} | awk -F '[>=<]+' '{print $1}') + dep=$(echo "${dep}" | awk -F '[>=<]+' '{print $1}') else - operator=$(echo ${dep} | awk -F '[0-9.]+' '{print $1}' | awk -F '^[a-zA-Z]+' '{print $2}') - dep=$(echo ${dep} | awk -F '[>=<]+' '{print $1}') + operator=$(echo "${dep}" | awk -F '[0-9.]+' '{print $1}' | awk -F '^[a-zA-Z]+' '{print $2}') + dep=$(echo "${dep}" | awk -F '[>=<]+' '{print $1}') case ${operator} in ">=") ;; "=") ;; *) printError "${operator} is not supported." ;; esac - major=$(echo ${version} | awk -F. '{print $1}') - minor=$(echo ${version} | awk -F. '{print $2}') + major=$(echo "${version}" | awk -F. '{print $1}') + minor=$(echo "${version}" | awk -F. '{print $2}') if [ -z "${minor}" ]; then minor=0 fi - patch=$(echo ${version} | awk -F. '{print $3}') + patch=$(echo "${version}" | awk -F. '{print $3}') if [ -z "${patch}" ]; then patch=0 fi - prerelease=$(echo ${version} | awk -F. '{print $4}') + prerelease=$(echo "${version}" | awk -F. '{print $4}') if [ -z "${prerelease}" ]; then prerelease=0 fi @@ -1361,31 +1362,19 @@ checkWritable() fi } -getReleaseLine() +generateUUID() { - jsonConfig="${ZOPEN_ROOTFS}/etc/zopen/config.json" - if [ ! -f "${jsonConfig}" ]; then - jq -r '.release_line' $jsonConfig - else - echo "STABLE" - fi -} - -getRMProcs() -{ - jsonConfig="${ZOPEN_ROOTFS}/etc/zopen/config.json" - if [ ! -f "${jsonConfig}" ]; then - jq -r '.num_rm_procs' $jsonConfig - else - echo "5" # default - fi + date_part=$(date +%s) + random_part=$((RANDOM)) + uuid="${date_part}-${random_part}" + echo "${uuid}" } isURLReachable() { url="$1" timeout=5 - if curl -s --fail --max-time $timeout "$url" > /dev/null; then + if curlCmd -s --fail --max-time $timeout "$url" > /dev/null; then return 0 else return 1 From 73ed6b5e5df695a0bd21692c76f2d8357e688408 Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 11:07:29 +0000 Subject: [PATCH 013/179] Remove intermediary variable - output directly --- include/common.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/include/common.sh b/include/common.sh index dbd6fe3d3..b02ccf0e4 100755 --- a/include/common.sh +++ b/include/common.sh @@ -745,12 +745,11 @@ mergeIntoSystem() # Need '-S' to allow long symlinks cd "${processingDir}" && tar -S -cf "${tarfile}" "usr" - printDebug "Generating listing for remove processing (including main symlink)." - listing=$(tar tf "${processingDir}/${tarfile}" 2> /dev/null | sort -r) + printDebug "Generating listing for remove processing (including main symlink)" echo "Installed files:" > "${versioneddir}/.links" - echo "${listing}" >> "${versioneddir}/.links" - - printDebug "Extracting tar to rootfs." + tar tf "${processingDir}/${tarfile}" 2> /dev/null| sort -r >> "${versioneddir}/.links" + + printDebug "Extracting tar to rootfs" cd "${processingDir}" && tar xf "${tarfile}" -C "${rootfs}" 2> /dev/null printDebug "Cleaning temp resources." @@ -774,10 +773,10 @@ unsymlinkFromSystem() rootfs=$2 dotlinks=$3 newfilelist=$4 - if [ -e "${dotlinks}" ]; then - printInfo "- Checking for obsoleted files in ${rootfs}/usr/ tree from ${pkg}" + if [ -e "${dotlinks}" ]; then if [ -e "${newfilelist}" ]; then + printInfo "- Checking for file differences switching versions of '${pkg}'" printDebug "Release change, so the list of changes to physically remove should be smaller" printDebug "Starting spinner..." progressHandler "spinner" "- Check complete" & From 72bce1c3e315dc4ebcf2f235e50c46cfeab67d35 Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 11:11:51 +0000 Subject: [PATCH 014/179] Use dynamic location for JSON cache file --- include/common.sh | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/include/common.sh b/include/common.sh index b02ccf0e4..68fd14b38 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1265,16 +1265,28 @@ syslog() echo "$(date +"%F %T") $(id | cut -d' ' -f1)::${module}:${type}:${categories}:${location}:${msg}" >> "${fd}" } +getJSONCacheURL(){ + activeRepo="${ZOPEN_ROOTFS}/etc/zopen/repos.d/active" + [ ! -e "${activeRepo}" ] && printError "Could not access repository configuration at '${activeRepo}'. Check file to ensure valid repository configuration or refresh default configuration with zopen init --refresh -y." + + type=$(jq -r ".type" "${activeRepo}") + base=$(jq -r ".metadata_baseurl" "${activeRepo}") + filename=$(jq -r ".metadata_file" "${activeRepo}") + case "${type}" in + http|https) printf "%s://%s/%s" "${type}" "${base}" "${filename}";; + file) printf "%s:%s/%s" "${type}" "${base}" "${filename}";; + *) printError "Unsupported repository type '${type}'.";; + esac +} + downloadJSONCache() { from_readonly=$1 if [ -z "${JSON_CACHE}" ]; then - cachedir="${ZOPEN_ROOTFS}/var/cache/zopen" - [ ! -e "${cachedir}" ] && mkdir -p "${cachedir}" - JSON_CACHE="${cachedir}/zopen_releases.json" - JSON_TIMESTAMP="${cachedir}/zopen_releases.timestamp" - JSON_TIMESTAMP_CURRENT="${cachedir}/zopen_releases.timestamp.current" + JSON_CACHE="${ZOPEN_ROOTFS}/var/cache/zopen/zopen_releases.json" + JSON_TIMESTAMP="${ZOPEN_ROOTFS}/var/cache/zopen/zopen_releases.timestamp" + JSON_TIMESTAMP_CURRENT="${ZOPEN_ROOTFS}/var/cache/zopen/zopen_releases.timestamp.current" if [ -n "$from_readonly" ]; then if [ -r "$JSON_CACHE" ] && [ ! -w "$JSON_CACHE" ]; then @@ -1293,26 +1305,30 @@ downloadJSONCache() [ ! -w "${JSON_CACHE}" ] || [ ! -r "${JSON_CACHE}" ] && printError "Cannot access cache at '${JSON_CACHE}'. Check permissions and retry request." fi - if ! curlout=$(curlCmd -L --no-progress-meter -I "${ZOPEN_JSON_CACHE_URL}" -o "${JSON_TIMESTAMP_CURRENT}"); then - printError "Failed to obtain json cache timestamp from ${ZOPEN_JSON_CACHE_URL}; ${curlout}" + jsonCacheURL=$(getJSONCacheURL) + if ! curlCmd -f -L -s -I "${jsonCacheURL}" -o "${JSON_TIMESTAMP_CURRENT}"; then + printError "Failed to obtain json cache timestamp from ${jsonCacheURL}." fi chtag -tc 819 "${JSON_TIMESTAMP_CURRENT}" - if [ -f "${JSON_CACHE}" ] && [ -f "${JSON_TIMESTAMP}" ] && grep -q 'ETag' "${JSON_TIMESTAMP_CURRENT}" && [ "$(grep 'ETag' "${JSON_TIMESTAMP_CURRENT}")" = "$(grep 'ETag' "${JSON_TIMESTAMP}")" ]; then + if [ -f "${JSON_CACHE}" ] \ + && [ -f "${JSON_TIMESTAMP}" ] \ + && [ "$(grep 'Last-Modified' "${JSON_TIMESTAMP_CURRENT}")" = "$(grep 'Last-Modified' "${JSON_TIMESTAMP}")" ]; then + # Metadata cache unchanged return fi printVerbose "Replacing old timestamp with latest." mv -f "${JSON_TIMESTAMP_CURRENT}" "${JSON_TIMESTAMP}" - if ! curlout=$(curlCmd -L --no-progress-meter -o "${JSON_CACHE}" "${ZOPEN_JSON_CACHE_URL}"); then - printError "Failed to obtain json cache from ${ZOPEN_JSON_CACHE_URL}; ${curlout}" + if ! curlCmd -f -L -s -o "${JSON_CACHE}" "${jsonCacheURL}"; then + printError "Failed to obtain json cache from '${jsonCacheURL}'" fi chtag -tc 819 "${JSON_CACHE}" fi if [ ! -f "${JSON_CACHE}" ]; then - printError "Could not download json cache from ${ZOPEN_JSON_CACHE_URL}" + printError "Could not download json cache from '${jsonCacheURL}" fi } From 67eb8933427200d865fcb6a5dae97e36b674b4ec Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 11:12:32 +0000 Subject: [PATCH 015/179] Rename functions to reflect not just GH --- include/common.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/common.sh b/include/common.sh index 68fd14b38..6d1094caa 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1332,13 +1332,13 @@ downloadJSONCache() fi } -getReposFromGithub() +getRepos() { - downloadJSONCache $1 - repo_results="$(cat "${JSON_CACHE}" | jq -r '.release_data | keys[]')" + downloadJSONCache + repo_results="$(jq -r '.release_data | keys[]' "${JSON_CACHE}")" } -getAllReleasesFromGithub() +getRepoReleases() { downloadJSONCache $1 repo="$1" From bc931821b7f9d7d027d420c8015958c8b9d9953b Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 11:16:55 +0000 Subject: [PATCH 016/179] Add common functions that can be included into a jq query for formatting padleft, padright, center, round --- include/common.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/include/common.sh b/include/common.sh index 6d1094caa..6219a69d2 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1474,4 +1474,20 @@ a2e() . ${INCDIR}/analytics.sh +jqfunctions() +{ + # Return a set of helper functions for jq that can be prepended to + # any jq query + # pl(s;c;n) - padLeft with character 'c' to length 'n' + # pr(s;c;n) - padRight with chacater 'c' to length 'n' + # c(s;l) - center the string 's' in a string of length 'l' + # r(dp) - round decimal to 'dp' decimal places (needs '*' & '/' 10^dp hack) + # shellcheck disable=SC2016 + printf "%s;%s;%s;%s;" \ + 'def pl(s;c;n):c*(n-(s|length))+s' \ + 'def pr(s;c;n):s+c*(n-(s|length))' \ + 'def c(s;c;l):(((l - (s|length))/2 | floor ) // 0) as $lp|((l - (s|length) - $lp) // 0 )as $rp|pl("";c; $lp) + s + pr("";c; $rp)' \ + 'def r(dp):.*pow(10;dp)|round/pow(10;dp)' +} + zopenInitialize From 826d86280ab01dfee2f3f318e55469f3022afc6e Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 11:44:33 +0000 Subject: [PATCH 017/179] Add utility functions New prompt that allows an 'A' option to answer yes to all future questions Reponame formatter to extract a repo name regardless of any version/tag suffixes Given a list of strings, generate a de-deuplicated list separated by the specified delimiter string/char Space validate function that will check whether there is sufficient disk space for the required operation to continue, given the required number of bytes passed in. Return all "active" package directories, as in those within the usr/local/zopen structure where there is an active link, indicating that a package is linked into the system (ie. active) --- include/common.sh | 76 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/include/common.sh b/include/common.sh index 6219a69d2..78cb8179b 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1472,7 +1472,79 @@ a2e() fi } -. ${INCDIR}/analytics.sh +promptYesNoAlways() { + message="$1" + skip=$2 + if ! ${skip}; then + while true; do + printInfo "${message} [y/n/a]" + read answer < /dev/tty + answer=$(echo "${answer}" | tr '[A-Z]' '[a-z]') + case "${answer}" in + y|Y) return 0;; + n|N) return 1;; + a|A) ye + esac + + if [ "y" = "${answer}" ] || [ "yes" = "${answer}" ]; then + return 0 + fi + if [ "n" = "${answer}" ] || [ "no" = "${answer}" ]; then + return 1 + fi + + done + fi + return 0 +} + + +parseRepoName() +{ + fullname="$1" + printDebug "Name to install: ${fullname}, parsing any version ('=') or tag ('%') that has been specified" + name=$(echo "${fullname}" | sed -e 's#[=%].*##') + repo="${name}" + versioned=$(echo "${fullname}" | cut -s -d '=' -f 2) + tagged=$(echo "${fullname}" | cut -s -d '%' -f 2) + printDebug "Name:${name};version:${versioned};tag:${tagged};repo:${repo}" +} + +dedupStringList() +{ delim="$1" && shift + str="$1" + echo "${str}"| awk -v delim="${delim}" ' + { dlm=""; for (i=1; i<=NF; i++) {if (!seen[$i]++) {printf "%s%s", dlm, $i};dlm=delim};print ""}' +} + +spaceValidate(){ + spaceRequiredBytes=$1 + spaceRequiredMB=$(echo "scale=0; ${spaceRequiredBytes} / (1024 * 1024)" | bc) + availableSpaceMB=$(/bin/df -m "${ZOPEN_ROOTFS}" | sed "1d" | awk '{ print $3 }' | awk -F'/' '{ print $1 }') + + printInfo "After this operation, ${spaceRequiredMB} MB of additional disk space will be used." + if [ "${availableSpaceMB}" -lt "${spaceRequiredMB}" ]; then + printWarning "Your zopen file-system (${ZOPEN_ROOTFS}) only has ${availableSpaceMB} MB of available space." + fi + if ! ${yesToPrompts} || [ "${availableSpaceMB}" -lt "${spaceRequiredMB}" ]; then + while true; do + printInfo "Do you want to continue? [y/n/a]" + read continueInstall < /dev/tty + case "${continueInstall}" in + "y") break;; + "n") mutexFree "zopen"; printInfo "Exiting..."; exit 0 ;; + "a") yesToPrompts=true; break;; + *) echo "?";; + esac + done + fi +} + +getActivePackageDirs() +{ + (unset CD_PATH; cd "${ZOPEN_PKGINSTALL}" && zosfind ./*/. ! -name . -prune -type l) +} + jqfunctions() { @@ -1490,4 +1562,6 @@ jqfunctions() 'def r(dp):.*pow(10;dp)|round/pow(10;dp)' } +# shellcheck disable=SC1091 +. "${INCDIR}/analytics.sh" zopenInitialize From 679bb221050b48b95362c2fd3fc38137db6953c4 Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 11:49:03 +0000 Subject: [PATCH 018/179] Run pre/post action scriplets Called with a specific phase, scriptlets in the appropriate location for that phase get sourced/executed. Default scripts are provided for the post-transaction phase (ie. after all packages are installed/removed/alternated) to run man-db and to execute cleanup of pax files --- include/common.sh | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/include/common.sh b/include/common.sh index 78cb8179b..15af2c11a 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1545,6 +1545,34 @@ getActivePackageDirs() (unset CD_PATH; cd "${ZOPEN_PKGINSTALL}" && zosfind ./*/. ! -name . -prune -type l) } +processActionScripts() +{ + phase=$1 + case $phase in + "install-post") scriptDir="${ZOPEN_ROOTFS}/etc/zopen/scriptlets/postInstall";; + "remove-post") scriptDir="${ZOPEN_ROOTFS}/etc/zopen/scriptlets/postRemove";; + "txn-post") scriptDir="${ZOPEN_ROOTFS}/etc/zopen/scriptlets/postTransaction";; + "install-pre") scriptDir="${ZOPEN_ROOTFS}/etc/zopen/scriptlets/preInstall";; + "remove-pre") scriptDir="${ZOPEN_ROOTFS}/etc/zopen/scriptlets/preRemove";; + "txn-pre") scriptDir="${ZOPEN_ROOTFS}/etc/zopen/scriptlets/preTransaction";; + *) printError "Internal error; invalid post action phase '${phase}'" + esac + ( + [ -d "${scriptDir}" ] || return 0 # No script directory + unset CDPATH; + cd "${scriptDir}" || exit # the subshell + find . | while read scriptFile; + do + if [ ! -e "${scriptFile}" ]; then + printWarning "Script '${scriptDir}/${scriptFile}' is not executable. Check permissions" + continue + fi + # shellcheck disable=SC1090 + . "${scriptFile}" + done + ) +} + jqfunctions() { From 3de15e6ed1722d2761eca7bbcf93837d38dd51d1 Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 11:52:09 +0000 Subject: [PATCH 019/179] Logic to determine which JSON asset to use --- include/common.sh | 179 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) diff --git a/include/common.sh b/include/common.sh index 15af2c11a..03a666a15 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1498,6 +1498,145 @@ promptYesNoAlways() { return 0 } +getVersionedMetadata() +{ + printDebug "Specific version ${versioned} requested - checking existence and URL" + requestedMajor=$(echo "${versioned}" | awk -F'.' '{print $1}') + requestedMinor=$(echo "${versioned}" | awk -F'.' '{print $2}') + requestedPatch=$(echo "${versioned}" | awk -F'.' '{print $3}') + requestedSubrelease=$(echo "${versioned}" | awk -F'.' '{print $4}') + requestedVersion="${requestedMajor}\\\.${requestedMinor}\\\.${requestedPatch}\\\.${requestedSubrelease}" + printDebug "Finding URL for latest release matching version prefix: requestedVersion: ${requestedVersion}" + releasemetadata=$(/bin/printf "%s" "${releases}" | jq -e -r '. | map(select(.assets[].name | test("'${requestedVersion}'")))[0]') +} + +getTaggedMetadata() +{ + printDebug "Explicit tagged version '${tagged}' specified. Checking for match" + releasemetadata=$(/bin/printf "%s" "${releases}" | jq -e -r '.[] | select(.tag_name == "'"${tagged}"'")') + printDebug "Use quick check for asset to check for existence of metadata for specific messages" + asset=$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]') + if [ $? -ne 0 ]; then + printError "Could not find release tagged '${tagged}' in repo '${repo}'" + fi +} + +getSelectMetadata() +{ + # As this is running within the generate... logic, a progress handler will have been started. + # This needs to be terminated before trying to write to screen + # shellcheck disable=SC2154 + kill -HUP "${gigph}" 2>/dev/null # if the timer is not running, the kill will fail + waitforpid "${gigph}" # Make sure it's finished writing to screen + + # Explicitly allow the user to select a release to install; useful if there are broken installs + # as a known good release can be found, selected and pinned! + printDebug "List individual releases and allow selection" + i=$(/bin/printf "%s" "${releases}" | jq -r 'length - 1') + printInfo "Versions available for install:" + /bin/printf "%s" "${releases}" | jq --raw-output 'to_entries | map("\(.key): \(.value.tag_name) - \(.value.assets[0].name) [\( ( .value.assets[0].expanded_size|tonumber)*1000 / (1024 * 1024) | ceil | . / 1000)Mb]")[]' + printDebug "Getting user selection" + valid=false + while ! ${valid}; do + echo "Enter version to install (0-${i}): " + read selection < /dev/tty + if [ ! -z $(echo "${selection}" | sed -e 's/[0-9]*//') ]; then + echo "Invalid input, must be a number between 0 and ${i}" + elif [ "${selection}" -ge 0 ] && [ "${selection}" -le "${i}" ]; then + valid=true + fi + done + printVerbose "Selecting item ${selection} from array" + releasemetadata="$(/bin/printf "%s" "${releases}" | jq -e -r ".[${selection}]")" +} + +getReleaseLineMetadata() +{ + printDebug "Install from release line '${releaseLine}' specified" + validatedReleaseLine=$(validateReleaseLine "${releaseLine}") + if [ -z "${validatedReleaseLine}" ]; then + printError "Invalid releaseline specified: '${releaseLine}'; Valid values: DEV or STABLE" + fi + printDebug "Finding latest asset on the release line" + releasemetadata="$(/bin/printf "%s" "${releases}" | jq -e -r '. | map(select(.tag_name | startswith("'${releaseLine}'")))[0]')" + printDebug "Use quick check for asset to check for existence of metadata" + asset="$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')" + if [ $? -ne 0 ]; then + printError "Could not find release-line ${releaseLine} for repo: ${repo}" + fi +} + +calculateReleaseLineMetadata() +{ + printDebug "No explicit version/tag/releaseline, checking for pre-existing package&releaseline" + if [ -n "${installedReleaseLine}" ]; then + printDebug "Found existing releaseline '${installedReleaseLine}', restricting to only that releaseline" + validatedReleaseLine="${installedReleaseLine}" # Already validated when stored + else + printDebug "Checking for system-configured releaseline" + if [ -e "${ZOPEN_ROOTFS}/etc/zopen/config.json" ]; then + printDebug "Using v2 configuration: '${ZOPEN_ROOTFS}/etc/zopen/config.json}'" + sysrelline=$(jq -re '.release_line' "${ZOPEN_ROOTFS}/etc/zopen/config.json") + elif [ -e "${ZOPEN_ROOTFS}/etc/zopen/releaseline" ] ; then + printDebug "Using legacy file-based config" + sysrelline=$(awk ' {print toupper($1)}') < "${ZOPEN_ROOTFS}/etc/zopen/releaseline" + fi + printDebug "Validating value: ${sysrelline}" + validatedReleaseLine=$(validateReleaseLine "${sysrelline}") + if [ -n "${validatedReleaseLine}" ]; then + printDebug "zopen system configured to use releaseline '${sysrelline}'; restricting to that releaseline" + else + printWarning "zopen misconfigured to use an unknown releaseline of '${sysrelline}'; defaulting to STABLE packages" + printWarning "Set the contents of '${ZOPEN_ROOTFS}/etc/zopen/releaseline' to a valid value to remove this message" + printWarning "Valid values are: DEV | STABLE" + validatedReleaseLine="STABLE" + fi + fi + + printDebug "Parsing releases: ${releases}" + # We have some situations that could arise + # 1. the port being installed has no releaseline tagging yet (ie. no releases tagged STABLE_* or DEV_*) + # 2. system is configured for STABLE but only has DEV stream available + # 3. system is configured for DEV but only has DEV stream available + # 4. the port being installed has got full releaseline tagging + # The issue could arise that the user has switched the system from DEV->STABLE or vice-versa so package + # stream mismatches could arise but in normal case, once a package is installed [that has releaseline tagging] + # then that specific releaseline will be used + printDebug "Finding any releases tagged with ${validatedReleaseLine} and getting the first (newest/latest)" + releasemetadata="$(/bin/printf "%s" "${releases}" | jq -e -r '. | map(select(.tag_name | startswith("'${validatedReleaseLine}'")))[0]')" + + printDebug "Use quick check for asset to check for existence of metadata" + asset="$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')" + [ "${asset}" = "null" ] && asset="" # jq uses null, translate to sh's empty + + if [ -n "${asset}" ]; then + # Case 4... + printVerbose "Found a specific '${validatedReleaseLine}' release-line tagged version; installing..." + else + # Case 2 & 3 + printDebug "No releases on releaseline '${validatedReleaseLine}'; checking alternative releaseline" + alt=$(echo "${validatedReleaseLine}" | awk ' /DEV/ { print "STABLE" } /STABLE/ { print "DEV" }') + releasemetadata="$(/bin/printf "%s" "${releases}" | jq -e -r '. | map(select(.tag_name | startswith("'${alt}'")))[0]')" + printDebug "Use quick check for asset to check for existence of metadata" + asset="$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')" + [ "${asset}" = "null" ] && asset="" # jq uses null, translate to sh's empty + if [ $? -eq 0 ]; then + printDebug "Found a release on the '${alt}' release line so release tagging is active" + if [ "DEV" = "${validatedReleaseLine}" ]; then + # The system will be configured to use DEV packages where available but if none, use latest + printInfo "No specific DEV releaseline package, using latest available" + releasemetadata="$(/bin/printf "%s" "${releases}" | jq -e -r ".[0]")" + else + printVerbose "The system is configured to only use STABLE releaseline packages but there are none" + printInfo "No release available on the '${validatedReleaseLine}' releaseline." + fi + else + # Case 1 - old package that has no release tagging yet (no DEV or STABLE), just install latest + printVerbose "Installing latest release" + releasemetadata="$(/bin/printf "%s" "${releases}" | jq -e -r ".[0]")" + fi + fi +} parseRepoName() { @@ -1510,6 +1649,46 @@ parseRepoName() printDebug "Name:${name};version:${versioned};tag:${tagged};repo:${repo}" } +getPortMetaData(){ + portRequested="$1" + invalidPortAssetFile="$2" + printDebug "Removing any version (%) or tag (#) suffixes fron '${portRequested}" + portName=$(echo "${portRequested}" | sed -e 's#%.*##' -e 's#=.*##') + validatedPort=$(echo "${repo_metadata}" | awk -vportName="${portName}" '$0 == portName {print}') + if [ -z "${validatedPort}" ]; then + echo "${portName}: no matching port found" >> "${invalidPortAssetFile}" + return 1 + fi + parseRepoName "${portRequested}" # To set the various status flags below + getRepoReleases "${validatedPort}" + if [ -n "${versioned}" ]; then + getVersionedMetadata + elif [ -n "${tagged}" ]; then + getTaggedMetadata + elif # shellcheck disable=SC2154 + ${selectVersion}; then + getSelectMetadata + elif [ -n "${releaseLine}" ]; then + getReleaseLineMetadata + else + calculateReleaseLineMetadata + fi + if [ -z "${releasemetadata}" ]; then + echo "${portName}: metadata could not be found" >> "${invalidPortAssetFile}" + return 1 + fi + printDebug "Getting specific asset details using metadata: ${releasemetadata}" + if [ -z "${asset}" ] || [ "null" = "${asset}" ]; then + printDebug "Asset not found during previous logic; setting now" + asset=$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]') + fi + if [ -z "${asset}" ]; then + echo "${portName} asset metadata could not be found" >> "${invalidPortAssetFile}" + return 1 + fi + return 0 +} + dedupStringList() { delim="$1" && shift str="$1" From 375788508c742073427b3ad978a72118f42c6528 Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 11:53:51 +0000 Subject: [PATCH 020/179] Package installation database Functions to generate a complete database if none exists, add a package installation to the database and to remove from database --- include/common.sh | 70 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/include/common.sh b/include/common.sh index 03a666a15..c202a23f1 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1752,6 +1752,76 @@ processActionScripts() ) } +updatePackageDB() +{ + printVerbose "Updating the installed package tracker db" + if ! pkgdirs=$(getActivePackageDirs); then + printError "Unable to update the package db" + fi + + pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" + if [ -e "${pdb}" ]; then + backup=$(mktempfile "updatepdb" "bkk") + cp "${pdb}" "${backup}" + rm "${pdb}" + fi + pkgdirs=$(getActivePackageDirs) + for pkgdir in ${pkgdirs}; do + if [ ! -e "${ZOPEN_PKGINSTALL}/${pkgdir}/metadata.json" ]; then + printWarning "No metadata.json found in '${ZOPEN_PKGINSTALL}/${pkgdir}'. Bad package install?" + continue + fi + metadata=$(cat "${ZOPEN_PKGINSTALL}/${pkgdir}/metadata.json") + if [ ! -e "${pdb}" ]; then + echo "[]" > "${pdb}" + fi + + mdj=$(echo "${metadata}" | jq '. as $metadata | .product.repo | match(".*/ZOSOpenTools/(.*)port").captures[0].string | [{(.):$metadata}]') + if ! jq --argjson mdj "${mdj}" '. += $mdj' \ + "${pdb}" > \ + "${pdb}.working"; then + printError "Could not add metadata for '${pkg}' to install tracker. Run zopen --re-init to attempt regeneration." + fi + mv "${pdb}.working" "${pdb}" + done +} + +addToInstallTracker() +{ + pkg=$1 + pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" + if [ ! -e "${pdb}" ]; then + # Generate the packageDB + printWarning "No package tracker found, regenerating [subsequent runs will be faster]" + updatePackageDB + fi + metadataJson=$(cat "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}/metadata.json") + if ! jq --argjson mdj "[{\"${pkg}\":${metadataJson}}]" \ + "if any(.[]; has(\"${pkg}\")) then . |= map( if has(\"${pkg}\") then \$mdj[] else . end ) else . + \$mdj end" \ + "${pdb}" > \ + "${pdb}.working"; then + printError "Could not update metadata for '${pkg}' in package tracker. Run zopen -init -re-init to attempt regeneration." + fi + mv "${pdb}.working" "${pdb}" +} + +removeFromInstallTracker() +{ + pkg=$1 + pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" + if [ ! -e "${pdb}" ]; then + # Generate the packageDB + printWarning "No package tracker found, regenerating [subsequent runs will be faster]" + updatePackageDB + fi + if ! jq \ + "map(select(has(\"${pkg}\") | not))" \ + "${pdb}" > \ + "${pdb}.working"; then + printError "Could not add metadata for '${pkg}' to install tracker. Run zopen --re-init to attempt regeneration." + fi + mv "${pdb}.working" "${pdb}" +} jqfunctions() { From c907f945d979f7ec8afd51f092f29b2872d96fe5 Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 11:55:58 +0000 Subject: [PATCH 021/179] Calculate any inputted port names that are invalid Uses jq to query against main JSON release data as the reference --- include/common.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/include/common.sh b/include/common.sh index c202a23f1..5dcaca9f7 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1689,6 +1689,19 @@ getPortMetaData(){ return 0 } +validateInstallList(){ + installees="$1" + # shellcheck disable=SC2086 # Using set -f disables globbing + installees=$(set -f; echo ${installees} |awk -v ORS=, -v RS=' ' '{$1=$1; sub(/[=%].*/,x); print "\""$1"\""}') + invalidPortList=$(jq -r --argjson needles "[${installees%%,}]" \ + '.release_data| keys as $haystack | $needles | map(select(. as $needle | $haystack | index($needle)|not)) | .[]' "${JSON_CACHE}") + if [ -n "${invalidPortList}" ]; then + printSoftError "The following ports could not be installed:" + printSoftError " $(echo "${invalidPortList}" | awk -v OFS=' ' -v ORS=' ' '{$1=$1};1' )" + printError "Check port name(s), remove any extra 'port' suffixes and retry command." + fi +} + dedupStringList() { delim="$1" && shift str="$1" From 51693c8f42135e13bc676617e752a5269c380dbe Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 12:01:36 +0000 Subject: [PATCH 022/179] Implement file installation graph Gather dependencies, recurse, then prune to product an output of packages to install --- include/common.sh | 122 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/include/common.sh b/include/common.sh index 5dcaca9f7..3f19cae1f 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1689,6 +1689,61 @@ getPortMetaData(){ return 0 } +# createDependancyGraph +# analyzes the input file to create the list of all packages that are to +# be pulled in during install - and recurses if any added packages themselves +# pull in dependancies, dependencies being added to the front of the install queue +# inputs: $1 the file to use for install ports +# $2 an error file for outputing failures +# return: 0 for success (output of pwd -P command) +# 8 if error +createDependancyGraph(){ + invalidPortAssetFile=$1 && shift + printDebug "Getting list of dependencies" + dependencies=$(echo "${installList}" | jq --raw-output '.installqueue[] | select(.asset.runtime_dependencies | test("No dependencies") | not )| map(try(.runtime_dependencies |= split(" ")))| .[] | .runtime_dependencies[] ') + printDebug "Removing any dependencies already on install queue" + installing=$(echo "${installList}" | jq --raw-output '.installqueue[] | .portname') + missing=$(diffList "${installing}" "${dependencies}" ) + if [ -z "${missing}" ]; then + printDebug "All dependencies are in the install graph" + return 0 + fi + printDebug "Adding dependencies to install graph" + addToInstallGraph "${invalidPortAssetFile}" "${missing}" + # Recurse in case the now-installing dependencies themselves have dependencies + # Recursive dependencies should not break as the initial package will have been + # marked for installation + createDependancyGraph "${invalidPortAssetFile}" +} + +# addToInstallGraph +# Finds appropriate metadata for the specified port(s) and +# includes that in the installation file +# inputs: $1 the file to use for validated ports +# $2 an error file for outputing failures +# $* requested list of packages to install +# return: 0 for success (output of pwd -P command) +# 8 if error +addToInstallGraph(){ + invalidPortAssetFile=$1 && shift + pkgList="$1" + printDebug "Adding pkgList to install graph" + for portRequested in ${pkgList}; do + if ! getPortMetaData "${portRequested}" "${invalidPortAssetFile}"; then + continue + fi + ## Merge asset into output file - note the lack of inline file edit hence the mv + installList=$(echo "${installList}" | jq ".installqueue += [{\"portname\":\"${validatedPort}\", \"asset\":${asset}}]") + done + if [ -e "${invalidPortAssetFile}" ]; then + printSoftError "The following ports cannot be installed: " + while read invalidPort; do + printf "${WARNING} %s\n" "${invalidPort}" + done < "${invalidPortAssetFile}" + printError "Confirm port names, remove any 'port' suffixes and retry command." + fi +} + validateInstallList(){ installees="$1" # shellcheck disable=SC2086 # Using set -f disables globbing @@ -1709,6 +1764,73 @@ dedupStringList() { dlm=""; for (i=1; i<=NF; i++) {if (!seen[$i]++) {printf "%s%s", dlm, $i};dlm=delim};print ""}' } +# generateInstallGraph +# generates a file with details for packages that are to be installed from +# the in-use repository, reporting errors if ports were invalid and +# triggering dependency graph population +# inputs: $1 the file to use for validated ports +# $* requested list of packages to install +# return: 0 for success (output of pwd -P command) +# 8 if error +generateInstallGraph(){ + installList="{}" + printDebug "Parsing list of packages to install and verifying validity" + portsToInstall="$1" # start with the initial list + portsToInstall=$(dedupStringList ' ' "${portsToInstall}") + repo_metadata="${repo_results}" + # Create the following file here to trigger cleanup - otherwise, multiple + # tempfiles could be created depending on dependency graph depth + invalidPortAssetFile=$(mktempfile "invalid" "port") + addCleanupTrapCmd "rm -rf ${invalidPortAssetFile}" + addToInstallGraph "${invalidPortAssetFile}" "${portsToInstall}" + if + # shellcheck disable=SC2154 + ${doNotInstallDeps}; then + printVerbose "- Skipping dependency analysis" + else + # calculate dependancy graph + createDependancyGraph "${invalidPortAssetFile}" + fi + pruneGraph +} + +pruneGraph() +{ + printDebug "Pruning entries in graph if already installed" + if "${reinstall}"; then + # Reinstall packages if they are already installed - no prune needed + return 0 + fi + if "${downloadOnly}"; then + # Download the pax files, even if already installed as they are not + # being reinstalled so no prune required + return 0 + fi + # Prune already installed packages at the requested level; compare the + # incoming file name against the port name, version and release already on the system + # - seems to be the easiest comparison since some data is not in zopen_release vs metadata.json + # and a local pax won't have a remote repo but should have a file name! + installed=$(zopen list --installed --details) + # Ignore the version string - it varies across ports so use name and build time as that + # should be unique enough + installed=$(echo "${installed}"| awk 'BEGIN{ORS = "," } {print "\"" $1 "@=@" $3 "\""}') + installed="[${installed%,}]" + + installList=$(echo "${installList}" | \ + jq --argjson installees "${installed}" \ + '.installqueue |= + map( + select(.asset.url | + capture(".*/ZOSOpenTools/(?[^/]*)port.*-(?[^-]*)\\.(?\\d{8}_\\d{6}?)\\.zos\\.pax\\.Z$") |.rel as $rel | .name as $name | + $installees | map( + .|capture("(?[^@]*)@=@(?\\d{8}_\\d{6}?)$")|.iname as $iname | .irel as $irel | + ($iname+"-"+$irel) == ($name+"-"+$rel) + ) | any == false + ) + )'\ + ) +} + spaceValidate(){ spaceRequiredBytes=$1 spaceRequiredMB=$(echo "scale=0; ${spaceRequiredBytes} / (1024 * 1024)" | bc) From 43c626d1e84dd792d712fc3f355a07f58f32c779 Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 12:04:09 +0000 Subject: [PATCH 023/179] Process an installation file, running pre/post actions --- include/common.sh | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/include/common.sh b/include/common.sh index 3f19cae1f..3fb7cec90 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1854,6 +1854,45 @@ spaceValidate(){ fi } +processRepoInstallFile(){ + printVerbose "Beginning port installation" + mutexReq "zopen" + + processActionScripts "txn-pre" + if [ 0 -eq "$(echo "${installList}" | jq --raw-output '.installqueue| length')" ]; then + printInfo "- No packages to install" + return 0 + fi + + # shellcheck disable=SC2154 + if ${fileinstall}; then + : + else + spaceRequiredBytes=$(echo "${installList}" | jq --raw-output '.installqueue| map(.asset.size, .asset.expanded_size)| reduce .[] as $total (0; .+($total|tonumber))') + spaceValidate "${spaceRequiredBytes}" + fi + + processActionScripts "txn-pre" + for installurl in $(echo "${installList}" | jq --raw-output '.installqueue |map( (.asset.url| sub(" ";"") ))| @sh'); do + printVerbose "Analysing :'${installurl}'" + installurl=$(echo "${installurl}" | tr -d "' ") + getInstallFile "${installurl}" + if $downloadOnly; then + continue + fi + installFile="${installurl##*/}" + if [ ! "${installFile%.zos.pax.Z}" = "${installFile}" ]; then + # Found zos.pax.Z format + installFromPax "${installFile}" + else + printError "Unrecognised install file format" + fi + done + processActionScripts "txn-post" + mutexFree "zopen" + printVerbose "Port installation complete" +} + getActivePackageDirs() { (unset CD_PATH; cd "${ZOPEN_PKGINSTALL}" && zosfind ./*/. ! -name . -prune -type l) From 7addf6965f5b230419b8f9c4735bcb3ff312a91b Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 12:05:56 +0000 Subject: [PATCH 024/179] Function to download pax file from repo Location is either the zopen cache or to current directory, dependent on parameters --- include/common.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/include/common.sh b/include/common.sh index 3fb7cec90..a9789b8ac 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1893,6 +1893,26 @@ processRepoInstallFile(){ printVerbose "Port installation complete" } +getInstallFile() +{ + installurl="$1" + downloadToDir="${ZOPEN_ROOTFS}/var/cache/zopen" + if $downloadOnly; then + downloadToDir="." + else + downloadToDir="${ZOPEN_ROOTFS}/var/cache/zopen" + fi + if [ -e "${downloadToDir}/${installurl##*/}" ]; then + : + else + [ -e "${downloadToDir}" ] || mkdir -p "${downloadToDir}" + [ -w "${downloadToDir}" ] || printError "No permission to save install file to '${downloadToDir}'. Check permissions and retry command." + if ! runAndLog "cd ${downloadToDir} && curlCmd --no-progress-meter -L ${installurl} -O ${redirectToDevNull}"; then + printError "Could not download from ${installurl}. Correct any errors and potentially retry" + fi + fi +} + getActivePackageDirs() { (unset CD_PATH; cd "${ZOPEN_PKGINSTALL}" && zosfind ./*/. ! -name . -prune -type l) From ff963c5967de16272e3a777dd750ca13f2e92dbe Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 12:06:49 +0000 Subject: [PATCH 025/179] Implement function to extract the metadata.json from pax file --- include/common.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/include/common.sh b/include/common.sh index a9789b8ac..7ff6cefba 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1913,6 +1913,21 @@ getInstallFile() fi } +extractMetadataFromPax() +{ + if ! pax -rf "$1" -s "%[^/]*/%/tmp/%" '*/metadata.json' ; then + if ! details=$(pax -rf "$1" -s "%[^/]*/%/tmp/%" '*/package.json'); then + printSoftError "Could not extract package metadata from file '$1'." + [ -n "${details}" ] && printSoftError "Details: ${details}" + exit 8 + else + echo "/tmp/package.json" + fi + else + echo "/tmp/metadata.json" + fi +} + getActivePackageDirs() { (unset CD_PATH; cd "${ZOPEN_PKGINSTALL}" && zosfind ./*/. ! -name . -prune -type l) From 44c8b41d1a61dd3b74e5d258d1ca9f78da3de61b Mon Sep 17 00:00:00 2001 From: DevonianTeuchter Date: Thu, 14 Mar 2024 12:08:40 +0000 Subject: [PATCH 026/179] Function to handle pax installation All paxes, whether remotely-gathered or a local-install are treated the same once the pax is on disk for installation; this function uses that on-disk pax to install. Pre/post installation scriptlets are triggered (if available). --- include/common.sh | 87 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/include/common.sh b/include/common.sh index 7ff6cefba..5096a1ff8 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1928,6 +1928,93 @@ extractMetadataFromPax() fi } +installFromPax() +{ + pax="${downloadToDir}/$1" + printDebug "Installing from '${pax}'" + processActionScripts "install-pre" + metadatafile=$(extractMetadataFromPax "${pax}") + # Ideally we would use the following, but name does not always map + # to the actual repo package name at present. The repo name is in the + # repo field so can extract from there instead + #name=$(jq --raw-output '.product.name' "${metadatafile}") + name=$(jq --raw-output '.product.repo | match(".*/ZOSOpenTools/(.*)port").captures[0].string' "${metadatafile}") + paxname="${installurl##*/}" + installdirname="${name}/${paxname%.pax.Z}" # Use full pax name as default + + baseinstalldir="${ZOPEN_PKGINSTALL}" + paxredirect="-s %[^/]*/%${ZOPEN_PKGINSTALL}/${installdirname}/%" + + printDebug "Check for existing directory for version '${installdirname}'" + if [ -d "${ZOPEN_PKGINSTALL}/${installdirname}" ]; then + printVerbose "- Clearing existing directory and contents" + rm -rf "${ZOPEN_PKGINSTALL}/${installdirname}" + fi + + # shellcheck disable=SC2154 + if ! runLogProgress "pax -rf ${pax} -p p ${paxredirect} ${redirectToDevNull}" "Expanding ${pax}" "Expanded"; then + printSoftError "Unexpected errors during unpaxing, package directory state unknown" + printError "Use zopen alt to select previous version to ensure known state" + fi + + if [ -e "${ZOPEN_PKGINSTALL}/${name}/${name}/.pinned" ]; then + printWarning "Current version of ${name} is pinned; not setting updated version as active" + setactive=false + unInstallOldVersion=false + fi + # shellcheck disable=SC2154 + if ${setactive}; then + if [ -L "${ZOPEN_PKGINSTALL}/${name}/${name}" ]; then + printDebug "Removing old symlink '${ZOPEN_PKGINSTALL}/${name}/${name}'" + rm -f "${ZOPEN_PKGINSTALL}/${name}/${name}" + fi + if ! ln -s "${ZOPEN_PKGINSTALL}/${installdirname}" "${ZOPEN_PKGINSTALL}/${name}/${name}"; then + printError "Could not create symbolic link name" + fi + if ! ${nosymlink}; then + mergeIntoSystem "${name}" "${ZOPEN_PKGINSTALL}/${installdirname}" "${ZOPEN_ROOTFS}" + misrc=$? + printDebug "The merge complete with: ${misrc}" + fi + + printVerbose "- Checking for env file" + if [ -f "${ZOPEN_PKGINSTALL}/${name}/${name}/.env" ] || [ -f "${ZOPEN_PKGINSTALL}/${name}/${name}/.appenv" ]; then + printVerbose "- .env file found, adding to profiled processing" + mkdir -p "${ZOPEN_ROOTFS}/etc/profiled/${name}" + cat << EOF > "${ZOPEN_ROOTFS}/etc/profiled/${name}/dotenv" +curdir=\$(pwd) +cd "${ZOPEN_PKGINSTALL}/${name}/${name}" >/dev/null 2>&1 +# If .appenv exists, source it as it's quicker +if [ -f ".appenv" ]; then + . ./.appenv +elif [ -f ".env" ]; then + . ./.env +fi +cd \${curdir} >/dev/null 2>&1 +EOF + printVerbose "- Running any setup scripts" + cd "${ZOPEN_PKGINSTALL}/${name}/${name}" && [ -r "./setup.sh" ] && ./setup.sh >/dev/null + fi + fi + if ${unInstallOldVersion}; then + printDebug "New version merged; checking for orphaned files from previous version" + # This will remove any old symlinks or dirs that might have changed in an upgrade + # as the merge process overwrites existing files to point to different version + unsymlinkFromSystem "${name}" "${ZOPEN_ROOTFS}" "${currentlinkfile}" "${baseinstalldir}/${name}/${name}/.links" + fi + + if ${setactive}; then + printDebug "Marking this version as installed" + touch "${ZOPEN_PKGINSTALL}/${name}/${name}/.active" + installedList="${name} ${installedList}" + syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_INSTALL},${CAT_PACKAGE}" "DOWNLOAD" "handlePackageInstall" "Installed package:'${name}';version:${downloadFileVer};install_dir='${baseinstalldir}/${installdirname}';" + addToInstallTracker "${name}" + processActionScripts "install-post" + fi + printInfo "${NC}${GREEN}Successfully installed ${name}${NC}" +} + + getActivePackageDirs() { (unset CD_PATH; cd "${ZOPEN_PKGINSTALL}" && zosfind ./*/. ! -name . -prune -type l) From ded466c0800bf2df19ce1eb91d4454d0022adf38 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 18 Mar 2024 16:15:10 +0000 Subject: [PATCH 027/179] Re-implement caveat support using scriptlet Fix merge issue that removed link optimization --- bin/zopen-init | 9 +++++++++ include/common.sh | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index 30f8e1c26..9a85c2090 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -541,6 +541,15 @@ if [ "\${isactive}" -eq 1 ]; then (zopen clean --cache >/dev/null 2>&1) echo "- Cache cleaned." fi +EOF + printVerbose "- Generating caveats scriptlet" + cat << EOF > "${rootfs}/etc/zopen/scriptlets/postTransaction/list_caveats" +#!/bin/false # Expects to be sourced in zopen-* scripts, not run directly! +# Do not modify this file; changes will be overwritten automatically! +caveats_file="\${ZOPEN_ROOTFS}/var/cache/install_caveats.tmp" +[ -e "\${caveats_file}" ] || exit 0 +cat "\${caveats_file}" +rm "\${caveats_file}" EOF } diff --git a/include/common.sh b/include/common.sh index 5096a1ff8..953174373 100755 --- a/include/common.sh +++ b/include/common.sh @@ -958,7 +958,6 @@ progressAnimation() [ $# -eq 0 ] && printError "Internal error: no animation strings." animcnt=$# anim=1 -# ansiline 0 0 "$1\n\c" while :; do spinloop 1000 # Check for daemonization of this process (ie. orphaned and PPID=1) @@ -1933,12 +1932,17 @@ installFromPax() pax="${downloadToDir}/$1" printDebug "Installing from '${pax}'" processActionScripts "install-pre" + metadatafile=$(extractMetadataFromPax "${pax}") # Ideally we would use the following, but name does not always map # to the actual repo package name at present. The repo name is in the # repo field so can extract from there instead #name=$(jq --raw-output '.product.name' "${metadatafile}") name=$(jq --raw-output '.product.repo | match(".*/ZOSOpenTools/(.*)port").captures[0].string' "${metadatafile}") + + # Store current installation directory (if exists) + currentderef=$(cd "${ZOPEN_PKGINSTALL}/${name}/${name}" > /dev/null 2>&1 && pwd -P) + paxname="${installurl##*/}" installdirname="${name}/${paxname%.pax.Z}" # Use full pax name as default @@ -2000,7 +2004,7 @@ EOF printDebug "New version merged; checking for orphaned files from previous version" # This will remove any old symlinks or dirs that might have changed in an upgrade # as the merge process overwrites existing files to point to different version - unsymlinkFromSystem "${name}" "${ZOPEN_ROOTFS}" "${currentlinkfile}" "${baseinstalldir}/${name}/${name}/.links" + unsymlinkFromSystem "${name}" "${ZOPEN_ROOTFS}" "${currentderef}/.links" "${baseinstalldir}/${name}/${name}/.links" fi if ${setactive}; then @@ -2009,6 +2013,12 @@ EOF installedList="${name} ${installedList}" syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_INSTALL},${CAT_PACKAGE}" "DOWNLOAD" "handlePackageInstall" "Installed package:'${name}';version:${downloadFileVer};install_dir='${baseinstalldir}/${installdirname}';" addToInstallTracker "${name}" + # Some installation have installation caveats + installCaveat=$(jq -r '.product.install_caveats // empty' "${metadatafile}" 2>/dev/null) + if [ -n "$installCaveat" ]; then + printf "${name}:\n%s\n" "${installCaveat}">> "${ZOPEN_ROOTFS}/var/cache/install_caveats.tmp" + fi + processActionScripts "install-post" fi printInfo "${NC}${GREEN}Successfully installed ${name}${NC}" From f71835ced7a17de607108714def4155dda15a704 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 19 Mar 2024 18:53:17 +0000 Subject: [PATCH 028/179] Add mutex during install Resolve json.config writing issue New animation for package check. --- bin/zopen-init | 3 ++- bin/zopen-install | 3 ++- bin/zopen-upgrade | 2 +- include/common.sh | 5 ++--- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index 9a85c2090..02d2f53e8 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -450,7 +450,8 @@ generateJsonConfiguration() "release_line": "${releaseLine}", "num_rm_procs": ${RM_FILEPROCS}, "is_collecting_stats": ${isCollectingStats}, - "override_zos_tools": ${OVERRIDE_ZOS_TOOLS} + "override_zos_tools": ${OVERRIDE_ZOS_TOOLS}, + "autocacheclean": 1 } zz else diff --git a/bin/zopen-install b/bin/zopen-install index 0585f442b..c0ced5139 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -274,6 +274,7 @@ for installRepo in ${potentials}; do [ -e "${installRepo}" ] && fileinstall=true && break done +mutexReq "zopen" "zopen" if ${downloadOnly}; then downloadDir="${PWD}" printDebug "Downloading pax to current directory '${downloadDir}'" @@ -337,5 +338,5 @@ else processRepoInstallFile fi - +mutexFree "zopen" exit diff --git a/bin/zopen-upgrade b/bin/zopen-upgrade index 0c9166c1d..3948b308c 100755 --- a/bin/zopen-upgrade +++ b/bin/zopen-upgrade @@ -103,7 +103,7 @@ ${xdebug} && set -x && printVerbose "Enabled command execution trace" if [ -z "${chosenRepos}" ]; then printVerbose "No specific port to upgrade, upgrade all installed packages" printInfo "- Querying installed packages" - progressHandler "mirror" & + progressHandler "pkgcheck" & ph=$! killph="kill -HUP ${ph}" addCleanupTrapCmd "${killph}" diff --git a/include/common.sh b/include/common.sh index 953174373..6fc8aa2eb 100755 --- a/include/common.sh +++ b/include/common.sh @@ -996,20 +996,19 @@ progressHandler() trapcmd="exit;" if [ -n "${completiontext}" ]; then ansiline - #trapcmd="/bin/echo \"\047[0A\047[10D\047[2K${completiontext}\n\c\"; ${trapcmd}" trapcmd="/bin/echo \"\047[10D\047[K${completiontext}\n\c\"; ${trapcmd}" else - #trapcmd="/bin/echo \"\047[0A\047[10D\c\"; ${trapcmd}" trapcmd="/bin/echo \"\047[10D\047[K\c\"; ${trapcmd}" fi # shellcheck disable=SC2064 trap "${trapcmd}" HUP case "${type}" in - "spinner") progressAnimation '-' '>' '|' '>' ;; + "spinner") progressAnimation '-' '>' '|' '<' ;; "network") progressAnimation '-----' '>----' '->---' '-->--' '--->-' '---->' '-----' '----<' '---<-' '--<--' '-<---' '<----' ;; "mirror") progressAnimation '#______' '##_____' '#=#____' '#==#___' '#===#__' '#====#_' '#=====#' '#_====#' '#__===#' '#___==#' '#____=#' '#_____#' ;; "trash") progressAnimation 'O________' '_O_______' '__O______' '___o_____' '____o____' '_____o___' '______.__' '_______._' '________.' ;; "linkcheck")progressAnimation '------>' '?----->' '-?---->' '--?--->' '---?-->' '----?->' '-----?>';; + "pkgcheck") progressAnimation '?###?###' '#?###?##' '##?###?#' '###?###?';; *) progressAnimation '.' 'o' 'O' 'O' 'o' '.' ;; esac fi From b1fde6f3143274e3d761d558bc6b75c00eea7773 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 29 Apr 2024 21:58:11 +0100 Subject: [PATCH 029/179] Fix issue during zopen init and packagedb Rework progress handler/trap handler Improve list processing. --- bin/zopen-init | 14 ++-- bin/zopen-install | 20 +++--- bin/zopen-list | 102 +++++++++++++++++++-------- bin/zopen-promote | 2 +- bin/zopen-upgrade | 6 +- include/common.sh | 176 +++++++++++++++++++++++++++++++++++----------- 6 files changed, 234 insertions(+), 86 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index 02d2f53e8..94d7adc43 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -101,10 +101,15 @@ HELPDOC RM_FILEPROCS=5 #TODO: adjust based on number of online cpus on system # Boostrap pax files -CURL_PAX_LOCATION="packages/curl-8.10.1.20241001_214340.zos.pax.Z" -JQ_PAX_LOCATION="packages/jq-1.6.20241001_204116.zos.pax.Z" -GPG_PAX_LOCATION="packages/gnupg-2.4.4.20241001_233258.zos.pax.Z" -PINENTRY_PAX_LOCATION="packages/pinentry-1.2.1.20241002_001628.zos.pax.Z" +CURL_PAX="curl-8.10.1.20241001_214340.zos.pax.Z" +JQ_PAX="jq-1.6.20241001_204116.zos.pax.Z" +GPG_PAX="gnupg-2.4.4.20241001_233258.zos.pax.Z" +PINENTRY_PAX="pinentry-1.2.1.20241002_001628.zos.pax.Z" +PKG_DIR="packages" +CURL_PAX_LOCATION="${PKG_DIR}/${CURL_PAX}" +JQ_PAX_LOCATION="${PKG_DIR}/${JQ_PAX}" +GPG_PAX_LOCATION="${PKG_DIR}/${GPG_PAX}" +PINENTRY_PAX_LOCATION="${PKG_DIR}/${JQ_PAX}" verbose=false isHostIBM=false @@ -726,6 +731,7 @@ generateJsonConfiguration generateDefaultRepository generateAnalyticsConfiguration generateDefaultScriptlets +! ${refresh} && updateBootstrappedTools [ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || updatePackageDB updateBootstrappedTools deinit diff --git a/bin/zopen-install b/bin/zopen-install index c0ced5139..8491cbeb1 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -153,6 +153,7 @@ args=$* verbose=false debug=false xdebug=false +quiet=false selectVersion=false # shellcheck disable=SC2034 setActive=true @@ -237,6 +238,10 @@ while [ $# -gt 0 ]; do "--xdebug") xdebug=true ;; + "--quiet") + # shellcheck disable=SC2034 + quiet=true + ;; "--version") zopen-version "${ME}" exit 0 @@ -259,21 +264,20 @@ if ! ${all} && [ -z "${chosenRepos}" ]; then exit 4 fi -checkIfConfigLoaded - -#export SSL_CERT_FILE="${ZOPEN_CA}" -#export GIT_SSL_CAINFO="${ZOPEN_CA}" -#export CURL_CA_BUNDLE="${ZOPEN_CA}" - # If any of the parameters passed in point to an existing file, then # the user is attempting to install a port directly from the file system # rather than a repo printDebug "Checking input parameters for actual files" potentials=$(echo "${chosenRepos}" | sed 's/@@/ /g') for installRepo in ${potentials}; do - [ -e "${installRepo}" ] && fileinstall=true && break + [ -f "${installRepo}" ] && fileinstall=true && break done +if [ ! ${fileinstall} ]; then + printVerbose "Using potentially remote files; ensuring configured for remte access" + checkIfConfigLoaded +fi + mutexReq "zopen" "zopen" if ${downloadOnly}; then downloadDir="${PWD}" @@ -319,7 +323,7 @@ else validateInstallList "${installList}" fi printInfo "- Generating install graph" - progressHandler "spinner" & + progressHandler "spinner" "- Generated install graph" & gigph=$! killph="kill -HUP ${gigph}" addCleanupTrapCmd "${killph}" diff --git a/bin/zopen-list b/bin/zopen-list index 326006a2a..05696281b 100755 --- a/bin/zopen-list +++ b/bin/zopen-list @@ -46,27 +46,6 @@ Report bugs at https://github.com/ZOSOpenTools/meta/issues . HELPDOC } -text_center() -{ - if [ $# -lt 2 ]; then - echo "$1" - return - fi - echo "$1" | awk -v strlen="$2" \ - '{spc = strlen - length;padding = int(spc / 2);pad = spc - padding;printf "%*s%s%*s\n", pad, "", $0, padding, ""}' -} - -text_padrightr() { - padchar="${3:- }" # Default to space char - echo "" | awk -v str="$1" -v n="$2" -v pc="$padchar" \ - 'function padr(n,acc, c){ - if (++acc>=n) return c; - return c padr(n, acc, c); - } - BEGIN{ print str padr(n,0,pc);} - ' -} - listLatestForInstalled() { @@ -149,7 +128,7 @@ listAvailablePackages() \$va.url as \$url | (\$url | match(\".*-([^-]*).[0-9]{8}_[0-9]{6}.zos.pax.Z\").captures[0].string) as \$version | .value[0].tag_name as \$tn | - (if (\$installed | index(\$key)) then \"true\" else \"false\" end) as \$ins | + (if (\$installed | index(\$key)) then \"installed\" else \"available\" end) as \$ins | \$va.expanded_size as \$es | \$va.size as \$ds | (\$va.passed_tests | if (.==\"\") then \"0\" else . end | tonumber) as \$pt | @@ -171,7 +150,7 @@ listAvailablePackages() .value[0].assets[0].url as \$url | (\$url | match(\".*-([^-]*).[0-9]{8}_[0-9]{6}.zos.pax.Z\").captures[0].string) as \$version | .value[0].tag_name as \$tn | - (if (\$installed | index(\$key)) then \"true\" else \"false\" end) as \$ins | + (if (\$installed | index(\$key)) then \"installed\" else \"available\" end) as \$ins | .value[0].assets[0] as \$va | pr(\$key;\"@\";${wpackage}) + c(\$ins;\"@\";${winstalled}) + @@ -182,6 +161,77 @@ listAvailablePackages() } listInstalledEntries() +{ + wpackage=20 + wversion=26 + wrelease=14 + wrellne=12 + wexpsz=12 + wqual=8 + if ${urlinclude}; then + wurl=6 + urltext="URL" + else + wurl=0 + urltext="" + fi + if ${header} && ${details}; then + if ${full}; then + printf "${NC}${UNDERLINE}%-${wpackage}s%-${wversion}s%-${wrelease}s%-${wrellne}s%-${wexpsz}s%-${wqual}s%-${wurl}s${NC}\n" \ + "$(text_center "Package")" "$(text_center "Version")" "$(text_center "Release")" \ + "$(text_center "Releaseline")" "$(text_center "Disk usage")" \ + "$(text_center "Quality")" "$(text_center "${urltext}")" + else + printf "${NC}${UNDERLINE}%-${wpackage}s%-${wversion}s%-${wrelease}s${NC}\n" \ + "$(text_center "Package")" "$(text_center "Version")" "$(text_center "Release")" + fi + fi + pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" + if ! [ -e "${pdb}" ]; then + printWarning "No package database found. Regenerating (subsequent calls will be faster)" + updatePackageDB + fi + if ${details}; then + if ${full}; then + jq --raw-output \ + "$(jqfunctions) .[] | + keys[] as \$key | + .[\$key].product.test_status as \$ts | + pr(\$key;\"@\";${wpackage}) + + c((.[\$key].product?.version? // \"unknown\");\"@\";${wversion}) + + c((.[\$key].product?.release? // \"unknown\");\"@\";${wrelease}) + + c((.[\$key].product.buildline? // \"unknown\");\"@\";${wrellne}) + + c((.[\$key].product.size? // \"unknown\");\"@\";${wexpsz}) + + c((( if (\$ts? and (\$ts.total_tests | tonumber?) and (\$ts.total_success | tonumber?)) \ + then (\$ts.total_success | tonumber) / (\$ts.total_tests | tonumber) \ + else 0 \ + end | .*100)| r(2) |tostring);\"@\";${wqual})" \ + "${pdb}" | sed 's/@/ /g' + else + jq --raw-output \ + "$(jqfunctions) .[] | + keys[] as \$key | + pr(\$key;\"@\";${wpackage}) + + c(.[\$key].product.version;\"@\";${wversion}) + + c(.[\$key].product.release;\"@\";${wrelease})" \ + "${pdb}" | sed 's/@/ /g' + fi + elif ${versioninfo}; then + jq --raw-output \ + "$(jqfunctions) .[] | + keys[] as \$key | + .[\$key].product.test_status as \$ts | + pr(\$key;\"@\";${wpackage}) + + c((.[\$key].product?.version? // \"unknown\");\"@\";${wversion})" \ + "${pdb}" | sed 's/@/ /g' + else + jq --raw-output \ + '[.[] | keys[] as $key | $key] | unique[] ' \ + "${pdb}" + fi +} + +istInstalledEntries() { wpackage=20 wversion=26 @@ -296,11 +346,7 @@ done checkIfConfigLoaded if ${installed}; then - if ${versioninfo}; then - listInstalledVerInfo - else - listInstalledEntries - fi + listInstalledEntries exit 0 fi if ${available}; then diff --git a/bin/zopen-promote b/bin/zopen-promote index ee41a3f33..4541cffce 100755 --- a/bin/zopen-promote +++ b/bin/zopen-promote @@ -247,7 +247,7 @@ printInfo "- Promoting from '${rootfs}' to '${promotefs}'..." printDebug "Generating temporary pipe" FIFO_PIPE_STDOUT=$(mktempfile "" ".pipe") -progressHandler "spinner" "- promote complete" & +progressHandler "spinner" "- Promoted from '${rootfs}' to '${promotefs}'." & ph=$! killph="kill -HUP ${ph}" addCleanupTrapCmd "${killph}" diff --git a/bin/zopen-upgrade b/bin/zopen-upgrade index 3948b308c..0680f6353 100755 --- a/bin/zopen-upgrade +++ b/bin/zopen-upgrade @@ -103,7 +103,7 @@ ${xdebug} && set -x && printVerbose "Enabled command execution trace" if [ -z "${chosenRepos}" ]; then printVerbose "No specific port to upgrade, upgrade all installed packages" printInfo "- Querying installed packages" - progressHandler "pkgcheck" & + progressHandler "pkgcheck" "- Query of installed packages complete" & ph=$! killph="kill -HUP ${ph}" addCleanupTrapCmd "${killph}" @@ -164,8 +164,8 @@ if [ -z "${chosenRepos}" ]; then generateInstallGraph "${upgradeList}" else validateInstallList "${chosenRepos}" - printInfo "- Generating install graph" - progressHandler "spinner" & + printInfo "- Generating upgrade graph" + progressHandler "spinner" "- Generated upgrade graph" & ph=$! killph="kill -HUP ${ph}" addCleanupTrapCmd "${killph}" diff --git a/include/common.sh b/include/common.sh index 6fc8aa2eb..dfc6b413f 100755 --- a/include/common.sh +++ b/include/common.sh @@ -5,13 +5,17 @@ zopenInitialize() { - # Create the cleanup pipeline and exit handler - trap "cleanupFunction" EXIT INT TERM QUIT HUP + # Create the cleanup pipeline and exit handler - the export grabs the + # program exit code (for EXIT) or the signal otherwise to return from + # the program (rather than the return code from the functions in the + # trap handler) + trap "eval export exitrc=\$?" EXIT INT TERM QUIT HUP defineEnvironment defineANSI if [ -z "${ZOPEN_DONT_PROCESS_CONFIG}" ]; then processConfig fi + # shellcheck disable=SC2034 ZOPEN_ANALYTICS_JSON="${ZOPEN_ROOTFS}/var/lib/zopen/analytics.json" ZOPEN_JSON_CACHE_URL="https://raw.githubusercontent.com/zopencommunity/meta/main/docs/api/zopen_releases.json" ZOPEN_JSON_CONFIG="${ZOPEN_ROOTFS}/etc/zopen/config.json" @@ -24,6 +28,8 @@ zopenInitialize() addCleanupTrapCmd(){ newcmd=$1 + # Command Trace MUST be disabled as the output from this can become + # interleaved with output when calling zopen sub-processes. [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" # Small timing window if the script is killed between the creation # and removal of the temporary file; would be easier if zos sh @@ -32,20 +38,40 @@ addCleanupTrapCmd(){ # and run in the subshell before returning trap handler(s)!?! tmpscriptfile="clean.tmp" trap > "${tmpscriptfile}" 2>&1 && script=$(cat "${tmpscriptfile}") - rm "${tmpscriptfile}" if [ -n "${script}" ]; then for trappedSignal in "EXIT" "INT" "TERM" "QUIT" "HUP"; do - newtrapcmd=$(echo "${script}" | while read trapcmd; do - sigcmd=$(echo "${trapcmd}" | zossed "s/trap -- \"\(.*\)\" ${trappedSignal}.*/\1/") - [ "${sigcmd}" = "${trapcmd}" ] && continue - printf "%s;%s 2>/dev/null" "${sigcmd}" "${newcmd}" | tr -s ';' - break - done + newtrapcmd=$( + echo "${script}" | while read trapcmd; do + sigcmd=$(echo "${trapcmd}" | + zossed "s/trap -- \"\(.*\)\" ${trappedSignal}.*/\1/") + # No match/replace in sed, then sigcmd remains unchanged + [ "${sigcmd}" = "${trapcmd}" ] && continue + # There was a match, so sigcmd contains the string of commands + # to run for the trap. Need to remove the exit command (which + # returns the exit code) and ensure it is last + # Note that eval is used to run commands with potentially + # embedded variables (like the return code capture/return) + prfx="eval exit " + sigcmd=$(echo "${sigcmd}" |awk -v prfx="$prfx" ' + BEGIN{ FS=";"; OFS=";" } + { + for (i=1; i<=NF; i++){ + if (substr($i, 1, length(prfx)) != prfx) { + printf "%s%s", sep, $i; sep="$OFS" + } + } + } + END{ printf "\n" }' + ) + printf "%s;%s 2>/dev/null;eval exit \${exitrc}" "${sigcmd}" "${newcmd}" | tr -s ';' + break + done ) if [ -n "${newtrapcmd}" ]; then trap -- "${newtrapcmd}" "${trappedSignal}" fi done + rm "${tmpscriptfile}" 2>/dev/null fi [ -n "${xtrc}" ] && set -x } @@ -176,6 +202,7 @@ mktempfile() mktempfile "$1" "${suffix}" # recurse and try again else echo "${tempfile}" + addCleanupTrapCmd "rm -rf ${tempfile}" fi } @@ -444,6 +471,16 @@ defineANSI() CRSRHIDE="${ESC}[?25l" # shellcheck disable=SC2034 CRSRSHOW="${ESC}[?25h" + # shellcheck disable=SC2034 + CRSRPL="${ESC}[1F" # Move to start of previous line + # shellcheck disable=SC2034 + CRSRUP="A" + # shellcheck disable=SC2034 + CRSRDOWN="B" + # shellcheck disable=SC2034 + CRSRRIGHT="C" + # shellcheck disable=SC2034 + CRSRLEFT="D" # Color-type codes, needs explicit terminal settings if [ ! "${_BPX_TERMPATH-x}" = "OMVS" ] && [ -z "${NO_COLOR}" ] && [ ! "${FORCE_COLOR-x}" = "0" ] && [ -t 1 ] && [ -t 2 ]; then @@ -508,19 +545,20 @@ ansimove() { deltax=$1 deltay=$2 + movestr="" if [ -n "${deltax}" ]; then if [ "${deltax}" -gt 0 ]; then - movestr="${ESC}[${deltax}C" + movestr="${ESC}[${deltax}${CRSRRIGHT}" elif [ "${deltax}" -lt 0 ]; then - movestr="${ESC}[$(expr "${deltax}" \* -1)D" + movestr="${ESC}[$(expr "${deltax}" \* -1)${CRSRLEFT}" fi fi if [ -n "${deltay}" ]; then if [ "${deltay}" -gt 0 ]; then - movestr="${movestr}${ESC}[${deltay}B" + movestr="${movestr}${ESC}[${deltay}${CRSRDOWN}" elif [ "${deltay}" -lt 0 ]; then - movestr="${movestr}${ESC}[$(expr "${deltay}" \* -1)A" + movestr="${movestr}${ESC}[$(expr "${deltay}" \* -1)${CRSRUP}" fi fi /bin/echo "${movestr}\c" @@ -535,7 +573,7 @@ getScreenCols() elif [ ! -z "${COLUMNS}" ]; then echo "${COLUMNS}" else - echo "$(tput cols)" + tput cols fi } @@ -581,6 +619,27 @@ strtrim() echo "$1" | zossed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' } +text_center() +{ + if [ $# -lt 2 ]; then + echo "$1" + return + fi + echo "$1" | awk -v strlen="$2" \ + '{spc = strlen - length;padding = int(spc / 2);pad = spc - padding;printf "%*s%s%*s\n", pad, "", $0, padding, ""}' +} + +text_padrightr() { + padchar="${3:- }" # Default to space char + echo "" | awk -v str="$1" -v n="$2" -v pc="$padchar" \ + 'function padr(n,acc, c){ + if (++acc>=n) return c; + return c padr(n, acc, c); + } + BEGIN{ print str padr(n,0,pc);} + ' +} + defineEnvironment() { # Required for proper operation of z/OS auto-conversion support @@ -778,8 +837,7 @@ unsymlinkFromSystem() if [ -e "${newfilelist}" ]; then printInfo "- Checking for file differences switching versions of '${pkg}'" printDebug "Release change, so the list of changes to physically remove should be smaller" - printDebug "Starting spinner..." - progressHandler "spinner" "- Check complete" & + progressHandler "spinner" "- Checked for file differences switching versions of '${pkg}'" & ph=$! killph="kill -HUP ${ph}" addCleanupTrapCmd "${killph}" @@ -815,8 +873,7 @@ unsymlinkFromSystem() addCleanupTrapCmd "rm -rf ${tempDirFile}" printDebug "Using temporary file ${tempDirFile}" printInfo "- Checking ${nfiles} potentially obsolete file links" - printDebug "Starting spinner..." - progressHandler "linkcheck" "- Complete" & + progressHandler "linkcheck" "- Checked ${nfiles} potentially obsolete file links" & ph=$! killph="kill -HUP ${ph}" addCleanupTrapCmd "${killph}" @@ -924,11 +981,11 @@ runLogProgress() printInfo "- Running" fi if [ -n "$3" ]; then - completeText="$3" + completeText="- $3" else - completeText="Complete" + completeText="- Complete" fi - progressHandler "spinner" "- ${completeText}" & + progressHandler "spinner" "${completeText}" & ph=$! killph="kill -HUP ${ph}" addCleanupTrapCmd "${killph}" @@ -968,8 +1025,7 @@ progressAnimation() [ "${ppid}" -eq 1 ] && kill INT "${ppid}" >/dev/null 2>&1 anim=$((anim + 1)) [ ${anim} -gt ${animcnt} ] && anim=1 - ansiline -10 0 "$(getNthArrayArg "${anim}" "$@")" - ansimove -10 0 + /bin/echo "${ESC}[1F${ESC}[0k$(getNthArrayArg "${anim}" "$@")" done } @@ -981,29 +1037,35 @@ getNthArrayArg () waitforpid() { -while kill -0 "$1" >/dev/null 2>&1; do - sleep 1 -done + while kill -0 "$1" >/dev/null 2>&1; do + sleep 1 + done } progressHandler() { - # if [ ! "${_BPX_TERMPATH-x}" = "OMVS" ] && [ -z "${NO_COLOR}" ] && [ ! "${FORCE_COLOR-x}" = "0" ] && [ -t 1 ] && [ -t 2 ]; then + + if [ -z "${-%%*x*}" ]; then + # Command trace is active so any progress animation + # writing to screen will interleave, making things cluttered. + # Sleep for 1s (to allow the caller to setup signal handling) and exit + sleep 1 + exit 0 + fi if ${ANSION}; then - [ -z "${-%%*x*}" ] && set +x # Disable -x debug if set for this process type=$1 completiontext=$2 # Custom end text (when the process is complete) trapcmd="exit;" if [ -n "${completiontext}" ]; then - ansiline - trapcmd="/bin/echo \"\047[10D\047[K${completiontext}\n\c\"; ${trapcmd}" + trapcmd="/bin/echo \"${CRSRSHOW}${CRSRPL}${ERASELINE}${CRSRPL}${ERASELINE}${completiontext}\n\c\"; ${trapcmd}" else - trapcmd="/bin/echo \"\047[10D\047[K\c\"; ${trapcmd}" + trapcmd="/bin/echo \"${CRSRSHOW}${CRSRPL}${ERASELINE}\c\"; ${trapcmd}" fi # shellcheck disable=SC2064 trap "${trapcmd}" HUP + /bin/echo "${CRSRHIDE}" case "${type}" in - "spinner") progressAnimation '-' '>' '|' '<' ;; + "spinner") progressAnimation '-' '/' '|' '\\' ;; "network") progressAnimation '-----' '>----' '->---' '-->--' '--->-' '---->' '-----' '----<' '---<-' '--<--' '-<---' '<----' ;; "mirror") progressAnimation '#______' '##_____' '#=#____' '#==#___' '#===#__' '#====#_' '#=====#' '#_====#' '#__===#' '#___==#' '#____=#' '#_____#' ;; "trash") progressAnimation 'O________' '_O_______' '__O______' '___o_____' '____o____' '_____o___' '______.__' '_______._' '________.' ;; @@ -1055,7 +1117,6 @@ printError() printColors "${NC}${RED}${BOLD}***ERROR: ${NC}${RED}${1}${NC}" >&2 [ -n "${xtrc}" ] && set -x mutexFree "zopen" # prevent lock from lingering around after an error - cleanupFunction exit 4 } @@ -1707,7 +1768,7 @@ createDependancyGraph(){ return 0 fi printDebug "Adding dependencies to install graph" - addToInstallGraph "${invalidPortAssetFile}" "${missing}" + addToInstallGraph "dependancy" "${invalidPortAssetFile}" "${missing}" # Recurse in case the now-installing dependencies themselves have dependencies # Recursive dependencies should not break as the initial package will have been # marked for installation @@ -1717,12 +1778,13 @@ createDependancyGraph(){ # addToInstallGraph # Finds appropriate metadata for the specified port(s) and # includes that in the installation file -# inputs: $1 the file to use for validated ports +# inputs: $1 if the install list comes from dependency analysis # $2 an error file for outputing failures # $* requested list of packages to install # return: 0 for success (output of pwd -P command) # 8 if error addToInstallGraph(){ + installtype=$1 && shift invalidPortAssetFile=$1 && shift pkgList="$1" printDebug "Adding pkgList to install graph" @@ -1731,7 +1793,7 @@ addToInstallGraph(){ continue fi ## Merge asset into output file - note the lack of inline file edit hence the mv - installList=$(echo "${installList}" | jq ".installqueue += [{\"portname\":\"${validatedPort}\", \"asset\":${asset}}]") + installList=$(echo "${installList}" | jq ".installqueue += [{\"portname\":\"${validatedPort}\", \"asset\":${asset}, \"installtype\":\"${installtype}\"}]") done if [ -e "${invalidPortAssetFile}" ]; then printSoftError "The following ports cannot be installed: " @@ -1780,7 +1842,7 @@ generateInstallGraph(){ # tempfiles could be created depending on dependency graph depth invalidPortAssetFile=$(mktempfile "invalid" "port") addCleanupTrapCmd "rm -rf ${invalidPortAssetFile}" - addToInstallGraph "${invalidPortAssetFile}" "${portsToInstall}" + addToInstallGraph "install" "${invalidPortAssetFile}" "${portsToInstall}" if # shellcheck disable=SC2154 ${doNotInstallDeps}; then @@ -1795,10 +1857,12 @@ generateInstallGraph(){ pruneGraph() { printDebug "Pruning entries in graph if already installed" + # shellcheck disable=SC2154 if "${reinstall}"; then # Reinstall packages if they are already installed - no prune needed return 0 fi + # shellcheck disable=SC2154 if "${downloadOnly}"; then # Download the pax files, even if already installed as they are not # being reinstalled so no prune required @@ -1819,7 +1883,7 @@ pruneGraph() '.installqueue |= map( select(.asset.url | - capture(".*/ZOSOpenTools/(?[^/]*)port.*-(?[^-]*)\\.(?\\d{8}_\\d{6}?)\\.zos\\.pax\\.Z$") |.rel as $rel | .name as $name | + capture(".*/(?.*)-(?[^-]*)\\.(?\\d{8}_\\d{6}?)\\.zos\\.pax\\.Z$") |.rel as $rel | .name as $name | $installees | map( .|capture("(?[^@]*)@=@(?\\d{8}_\\d{6}?)$")|.iname as $iname | .irel as $irel | ($iname+"-"+$irel) == ($name+"-"+$rel) @@ -1866,6 +1930,32 @@ processRepoInstallFile(){ if ${fileinstall}; then : else + if ! ${quiet} >/dev/null 2>&1; then + xIFS=$IFS + IFS=' ' + hdr=false + for installee in $(echo "${installList}" | jq --raw-output '.installqueue | map( (select(.installtype=="install") | .portname| sub(" ";"") ))| @sh'); do + installee=$(echo "${installee}" |tr -d "\'" ) + [ -z "${installee}" ] && continue + if ! ${hdr}; then + printHeader "Installing the following packages:" + hdr=true + fi + printInfo "${installee}" + + done + hdr=false + for dependee in $(echo "${installList}" | jq --raw-output '.installqueue | map( (select(.installtype=="dependancy") | .portname| sub(" ";"") ))| @sh'); do + dependee=$(echo "${dependee}" |tr -d "\'" ) + [ -z "${dependee}" ] && continue + if ! ${hdr}; then + printHeader "Dependent packages to install:" + hdr=true + fi + printInfo "${dependee}" + done + IFS=${xIFS} + fi spaceRequiredBytes=$(echo "${installList}" | jq --raw-output '.installqueue| map(.asset.size, .asset.expanded_size)| reduce .[] as $total (0; .+($total|tonumber))') spaceValidate "${spaceRequiredBytes}" fi @@ -1955,7 +2045,7 @@ installFromPax() fi # shellcheck disable=SC2154 - if ! runLogProgress "pax -rf ${pax} -p p ${paxredirect} ${redirectToDevNull}" "Expanding ${pax}" "Expanded"; then + if ! runLogProgress "pax -rf ${pax} -p p ${paxredirect} ${redirectToDevNull}" "Expanding ${pax}" "Expanded ${pax}"; then printSoftError "Unexpected errors during unpaxing, package directory state unknown" printError "Use zopen alt to select previous version to ensure known state" fi @@ -2072,11 +2162,13 @@ updatePackageDB() fi pkgdirs=$(getActivePackageDirs) for pkgdir in ${pkgdirs}; do - if [ ! -e "${ZOPEN_PKGINSTALL}/${pkgdir}/metadata.json" ]; then - printWarning "No metadata.json found in '${ZOPEN_PKGINSTALL}/${pkgdir}'. Bad package install?" + metadataFile="${ZOPEN_PKGINSTALL}/${pkgdir}/metadata.json" + if [ ! -e "${metadataFile}" ]; then + printVerbose "Falling back to filesystem analysis" + printWarning "No metadata.json found in '${ZOPEN_PKGINSTALL}/${pkgdir}'. Old package install?" continue fi - metadata=$(cat "${ZOPEN_PKGINSTALL}/${pkgdir}/metadata.json") + metadata=$(cat "${metadataFile}") if [ ! -e "${pdb}" ]; then echo "[]" > "${pdb}" fi From 0b5c43faee8fb5153bb07837c3b3a6c277ca0fd5 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 12 Sep 2024 15:25:43 +0100 Subject: [PATCH 030/179] Move scriptlets into meta/include dir for consistency Update ANSI handling Improve temp file handling. --- bin/zopen | 2 +- bin/zopen-alt | 6 +- bin/zopen-init | 147 +++---- bin/zopen-install | 60 ++- bin/zopen-remove | 11 +- include/common.sh | 391 ++++++++++++------ include/scriptlets/installPost/autopkgpurge | 21 + .../installPost/zopen-meta-init-refresh | 7 + .../scriptlets/installPre/systemPreReqCheck | 24 ++ include/scriptlets/removePost/man-db | 12 + .../scriptlets/transactionPost/list_caveats | 5 + include/scriptlets/transactionPost/man-db | 12 + .../transactionPost/zopen-clean-autocache | 24 ++ .../transactionPost/zopen-config-list | 16 + 14 files changed, 498 insertions(+), 240 deletions(-) create mode 100755 include/scriptlets/installPost/autopkgpurge create mode 100755 include/scriptlets/installPost/zopen-meta-init-refresh create mode 100755 include/scriptlets/installPre/systemPreReqCheck create mode 100755 include/scriptlets/removePost/man-db create mode 100755 include/scriptlets/transactionPost/list_caveats create mode 100755 include/scriptlets/transactionPost/man-db create mode 100755 include/scriptlets/transactionPost/zopen-clean-autocache create mode 100755 include/scriptlets/transactionPost/zopen-config-list diff --git a/bin/zopen b/bin/zopen index d9fac9c9d..b5e2c9be1 100755 --- a/bin/zopen +++ b/bin/zopen @@ -91,7 +91,7 @@ version=false for arg in $*; do case "${arg}" in - "alt"|"build"|"clean"|"generate"|"init"|"install"|\ + "alt"|"audit"|"build"|"clean"|"generate"|"init"|"install"|\ "query"|"remove"|"update-cacert"|"info"|"usage"|"diagnostics"|\ "list"|"upgrade") subcmd="zopen-${arg}" diff --git a/bin/zopen-alt b/bin/zopen-alt index e104b124f..651fe6b51 100755 --- a/bin/zopen-alt +++ b/bin/zopen-alt @@ -55,8 +55,8 @@ mergeNewVersion() { package="$1" newver="$2" - processActionScripts "txn-pre" - [ -z "${package}" ] && printError "Internal error; no packagename provided to merge." + processActionScripts "transactionpre" + [ -z "${package}" ] && printError "Internal error; no package name provided to merge." [ -d "${ZOPEN_PKGINSTALL}/${package}/${newver}" ] || printError "Version '${newver}' was not available to set as current" if [ -e "${ZOPEN_PKGINSTALL}/${package}/${package}" ]; then printVerbose "Removing main link" @@ -85,7 +85,7 @@ mergeNewVersion() version=$(cat "${ZOPEN_PKGINSTALL}/${package}/${package}/.releaseinfo") fi syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE}" "ALT" "setAlt" "Set '${package}' to version:${version};" - processActionScripts "txn-post" + processActionScripts "transactionpost" } setAlt() diff --git a/bin/zopen-init b/bin/zopen-init index 94d7adc43..d3843c434 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -98,7 +98,7 @@ HELPDOC # Constants # Number of rm processes to run in parallel -RM_FILEPROCS=5 #TODO: adjust based on number of online cpus on system +rm_fileprocs=5 #TODO: adjust based on number of online cpus on system # Boostrap pax files CURL_PAX="curl-8.10.1.20241001_214340.zos.pax.Z" @@ -281,7 +281,7 @@ init() if ! ${refresh} && ! ${reinitExisting}; then printInfo "- Binaries will be symlinked under \"${rootfs}/usr/local/bin\". Libraries will be symlinked under \"${rootfs}/usr/lib\"" printInfo "- Packages will be installed and maintained under the directory '${rootfs}/${zopen_pkginstall}'." - printInfo "- Collecting usage statistics: $isCollectingStats" + printInfo "- Collecting usage statistics: ${isCollectingStats}" fi if ! promptYesOrNo "Do you want to continue?" ${yesToPrompts}; then @@ -367,20 +367,6 @@ generateFileSystem() repodir="${rootfs}/etc/zopen/repos.d" [ -e "${repodir}" ] || mkdir -p "${repodir}" - printVerbose "- Creating action scriptlet directories" - piaDir="${rootfs}/etc/zopen/scriptlets/preInstall" - [ -e "${piaDir}" ] || mkdir -p "${piaDir}" - piaDir="${rootfs}/etc/zopen/scriptlets/postInstall" - [ -e "${piaDir}" ] || mkdir -p "${piaDir}" - praDir="${rootfs}/etc/zopen/scriptlets/preRemove" - [ -e "${praDir}" ] || mkdir -p "${praDir}" - praDir="${rootfs}/etc/zopen/scriptlets/postRemove" - [ -e "${praDir}" ] || mkdir -p "${praDir}" - ptaDir="${rootfs}/etc/zopen/scriptlets/preTransaction" - [ -e "${ptaDir}" ] || mkdir -p "${ptaDir}" - ptaDir="${rootfs}/etc/zopen/scriptlets/postTransaction" - [ -e "${ptaDir}" ] || mkdir -p "${ptaDir}" - printVerbose "- Setting global root envvar" ZOPEN_PKGINSTALL="${rootfs}/${zopen_pkginstall}" if [ "${ZOPEN_PKGINSTALL}" = "${rootfs}" ]; then @@ -426,38 +412,64 @@ generateJsonConfiguration() jsonConfigWorking="${jsonConfig}.working" if [ -e "${jsonConfig}" ]; then + # isCollectingStats is set elsewhere releaseLine=$(jq -r '.release_line' "${jsonConfig}") - RM_FILEPROCS=$(jq -r '.num_rm_procs' "${jsonConfig}") - # note that is_collecting_stats is set earlier + rm_fileprocs=$(jq -r '.num_rm_procs' "${jsonConfig}") if [ -z "${OVERRIDE_ZOS_TOOLS}" ] && jq -er ".override_zos_tools" $jsonConfig >/dev/null; then OVERRIDE_ZOS_TOOLS=$(jq -r '.override_zos_tools' $jsonConfig) fi - fi - - if [ -z "${OVERRIDE_ZOS_TOOLS}" ]; then - OVERRIDE_ZOS_TOOLS=false - fi - - if [ -z "${releaseLine}" ]; then - if ${releaselineDev}; then - releaseLine="DEV" - printInfo "- Configured zopen to use development (DEV) releaseline packages where available" - else - releaseLine="STABLE" - printInfo "- Configured zopen to use stable releaseline packages" - fi - fi + autocacheclean=$(jq -r '.autocacheclean' "${jsonConfig}") + autopkgpurge=$(jq -r '.autopkgpurge' "${jsonConfig}") + fi + + if ${releaselineDev}; then + # Explicitly added on the command-line so use it + releaseLine="DEV" + printInfo "- Configured zopen to use development (DEV) releaseline packages where available" + fi + case "${releaseLine}" in + "") releaseLine="STABLE" + printInfo "- Configured zopen to use stable releaseline packages" + ;; + "STABLE"|"DEV") :;; # Use as-is + *) printWarning "Current value for release line '${releaseLine}' invalid. Defaulting to 'STABLE'." + releaseLine="STABLE" + ;; + esac + case "${autocacheclean:=1}" in + "0"|"1") :;; # Valid value + "null") autocacheclean=1 # Default to true; jq found no entry (new config) + ;; + *) showConfigParmWarning "autocacheclean" "${autocacheclean}" "0|1" "1" + autocacheclean=1 + + ;; + esac + case "${autopkgpurge:=0}" in + "0"|"1") :;; # Valid value + "null")autopkgpurge=0 # Default to false; jq found no entry (new config) + ;; + *) showConfigParmWarning "autopkgpurge" "${autopkgpurge}" "0|1" "0" + autopkgpurge=0 + ;; + esac + case "${OVERRIDE_ZOS_TOOLS}" in + "true"|"false") :;; # Valid value + "null") OVERRIDE_ZOS_TOOLS=false ;; # No entry in config file + *) OVERRIDE_ZOS_TOOLS=false ;; # Bad envvar / empty value in config + esac if [ ! -e "${jsonConfig}" ]; then printInfo "Generating global configuration" - cat < "${jsonConfig}" -{ - "release_line": "${releaseLine}", - "num_rm_procs": ${RM_FILEPROCS}, - "is_collecting_stats": ${isCollectingStats}, - "override_zos_tools": ${OVERRIDE_ZOS_TOOLS}, - "autocacheclean": 1 -} + cat < "${jsonConfigWorking}" + { + "release_line": "${releaseLine:-STABLE}", + "num_rm_procs": ${rm_fileprocs:-5}, + "is_collecting_stats": ${isCollectingStats}, + "override_zos_tools": ${OVERRIDE_ZOS_TOOLS}, + "autocacheclean": ${autocacheclean}, + "autopkgpurge": ${autopkgpurge} + } zz else printInfo "Refreshing global configuration" @@ -472,8 +484,7 @@ zz fi mv -f "${ZOPEN_JSON_CONFIG}.working" "${ZOPEN_JSON_CONFIG}" fi - jq '.' "${jsonConfigWorking}" > "${jsonConfig}" - echo 1 > "${rootfs}/etc/zopen/autocacheclean" + } generateDefaultRepository() @@ -518,47 +529,6 @@ EOS (cd "${rootfs}/etc/zopen/repos.d" && ln -s "${repoFile}" "active") } -generateDefaultScriptlets() -{ - printVerbose "- Generating man-db scriptlet" - cat << EOF > "${rootfs}/etc/zopen/scriptlets/postTransaction/man-db" -#!/bin/false # Expects to be sourced in zopen-* scripts, not run directly! -# Do not modify this file; changes will be overwritten automatically! -if ! installed=\$(zopen list --installed | grep "^man-db\$"); then - : # No man-db installed -else - echo "- Updating man-db database..." - if ! mandb >/dev/null 2>&1; then - printWarning "man-db update completed with non-zero return code." - printWarning "Re-run 'mandb' manually for additional information." - else - echo "- Update of man-db complete." - fi -fi -EOF - printVerbose "- Generating zopen-clean scriptlet" - cat << EOF > "${rootfs}/etc/zopen/scriptlets/postTransaction/zopen-clean" -#!/bin/false # Expects to be sourced in zopen-* scripts, not run directly! -# Do not modify this file; changes will be overwritten automatically! -[ -e "\${ZOPEN_ROOTFS}/etc/zopen/autocacheclean" ] || exit 0 -isactive=\$(cat "\${ZOPEN_ROOTFS}/etc/zopen/autocacheclean") -if [ "\${isactive}" -eq 1 ]; then - echo "- Cleaning install cache..." - (zopen clean --cache >/dev/null 2>&1) - echo "- Cache cleaned." -fi -EOF - printVerbose "- Generating caveats scriptlet" - cat << EOF > "${rootfs}/etc/zopen/scriptlets/postTransaction/list_caveats" -#!/bin/false # Expects to be sourced in zopen-* scripts, not run directly! -# Do not modify this file; changes will be overwritten automatically! -caveats_file="\${ZOPEN_ROOTFS}/var/cache/install_caveats.tmp" -[ -e "\${caveats_file}" ] || exit 0 -cat "\${caveats_file}" -rm "\${caveats_file}" -EOF -} - generateAnalyticsConfiguration() { if ! ${isCollectingStats}; then @@ -583,7 +553,13 @@ generateAnalyticsConfiguration() "removes": [] } zz - jq '.' $jsonConfigWorking > $jsonConfig + if jqrc=$(jq '.' $jsonConfigWorking > $jsonConfig); then + rm "${jsonConfigWorking}" + else + printSoftError "${jqrc}" + printError "Unable to update analystics configuration. See previous errors for more details." + fi + registerFileSystem "${UUID}" "${isHostIBM}" "${isBot}" if [ -n "$ZOPEN_ROOTFS" ] && [ -n "$ZOPEN_LOG_PATH" ] && [ -f "$ZOPEN_LOG_PATH/audit.log" ]; then @@ -730,7 +706,6 @@ generateFileSystem generateJsonConfiguration generateDefaultRepository generateAnalyticsConfiguration -generateDefaultScriptlets ! ${refresh} && updateBootstrappedTools [ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || updatePackageDB updateBootstrappedTools diff --git a/bin/zopen-install b/bin/zopen-install index 8491cbeb1..94e4fd61f 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -124,24 +124,40 @@ ${ME} is a utility to download/install a zopen community package. Usage: zopen install [OPTION] [PARAMETERS] [PACKAGES] Options: - --all download/install all zopen community packages. - --bypass-prereq-checks Ignores pre-req checks - --download-only download package to current directory. - --force force install, bypassing locks. - --help print this help. - --install-or-upgrade installs the package if not installed, - or upgrades the package if installed. - --no-deps do not install dependencies. - --no-set-active do not change the pinned version. - --nosymlink do not integrate into filesystem through - symlink redirection. - --reinstall reinstall already installed zopen community packages. - --release-line [stable, dev] the release line to build off of. - --select select a version to install. - -u, --upgrade check for package updates and apply - -v, --verbose print verbose messages. - --version print version. - -y, --yes automatically answer yes to prompts. + --all download all available packages and install + --bypass-prereq-checks ignores pre-req checks + --download-only download installable package with no install + --install-or-upgrade installs the package if not installed, or upgrades + the package if installed (deprecated as default behaviour) + --nodeps do not install dependencies + --noset-active unpackage onto the system but do not change the currently + active version + --reinstall reinstall currently installed package(s) + --release-line [stable|dev] + the release line to install from + --select select a version to install from available versions + -v, --verbose run in verbose mode + -y, --yes automatically answer yes to prompts + -h,-?, --help display this help and exit + + +Examples: + zopen install foo + install package foo if not already installed + zopen install --release-line DEV foo + install package foo from the DEV releaseline if + available + zopen install -y foo bar --nodeps + install packages foo and bar without asking for + user confirmation and without installing any + dependencies + zopen install /tmp/foo-1.2.3-4.zos.pax.Z + install package foo from the specified location + zopen install /tmp/foo-1.2.3-4.zos.pax.Z bar-1.2.3.4.zos.pax.Z + install packages foo and bar from the specified + locations + +Report bugs at https://github.com/ZOSOpenTools/meta/issues . HELPDOC } @@ -186,9 +202,9 @@ while [ $# -gt 0 ]; do ;; "--no-symlink") # shellcheck disable=SC2034 - nosymlink=true # Do not mesh the package into the file system; leave as stand-alone + setactive=true # Do not mesh the package into the file system; leave as stand-alone ;; - "--no-deps") + "--nodeps" | "--no-deps") # Deprecate --no-deps for consistency doNotInstallDeps=true ;; "--release-line") @@ -203,7 +219,7 @@ while [ $# -gt 0 ]; do "--download-only") downloadOnly=true # Download remote package files to current directory only ;; - "--no-set-active") + "--noset-active" | "--no-set-active") # Deprecate --no-set-active for consistency # shellcheck disable=SC2034 setactive=false # Install package as normal but keep existing installation as active ;; @@ -274,7 +290,7 @@ for installRepo in ${potentials}; do done if [ ! ${fileinstall} ]; then - printVerbose "Using potentially remote files; ensuring configured for remte access" + printVerbose "Using potentially remote files; ensuring configured for remote access" checkIfConfigLoaded fi diff --git a/bin/zopen-remove b/bin/zopen-remove index 289221594..7f6190051 100755 --- a/bin/zopen-remove +++ b/bin/zopen-remove @@ -52,14 +52,15 @@ HELPDOC removePackages() { pkglist=$* - processActionScripts "txn-pre" + processActionScripts "transactionpre" # Create a temp file name to trigger the post-transaction action script # triggering - run only if an action was taken (which creates the file) runpostTxnActionScripts=$(mktempfile "txnpost" "run") + addCleanupTrapCmd "rm -rf ${runpostTxnActionScripts}" echo "${pkglist}" | xargs | tr ' ' '\n' | sort | while read pkg; do printHeader "Removing package: ${pkg}" - processActionScripts "remove-pre" - printInfo "- Checking status of package '${pkg}'" + processActionScripts "removepre" + printInfo "- Querying install status of package '${pkg}'" if [ ! -f "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}/.active" ]; then printInfo "${NC}${YELLOW}Package '${pkg}' is not installed${NC}" else @@ -106,14 +107,14 @@ removePackages() [ -d "${ZOPEN_ROOTFS}/etc/profiled/${pkg}" ] && rm -rf "${ZOPEN_ROOTFS}/etc/profiled/${pkg}" removeFromInstallTracker "${pkg}" - processActionScripts "remove-post" + processActionScripts "removepost" syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE},${CAT_REMOVE}" "REMOVE" "removePackage" "Removed package:'${needle};version:${version};" touch "${runpostTxnActionScripts}" printInfo "${NC}${GREEN}Successfully removed: ${pkg}${NC}" fi done if [ -e "${runpostTxnActionScripts}" ]; then - processActionScripts "txn-post"; + processActionScripts "transactionpost"; rm "${runpostTxnActionScripts}" fi } diff --git a/include/common.sh b/include/common.sh index dfc6b413f..6f3e3ab75 100755 --- a/include/common.sh +++ b/include/common.sh @@ -20,14 +20,14 @@ zopenInitialize() ZOPEN_JSON_CACHE_URL="https://raw.githubusercontent.com/zopencommunity/meta/main/docs/api/zopen_releases.json" ZOPEN_JSON_CONFIG="${ZOPEN_ROOTFS}/etc/zopen/config.json" if [ -n "${INCDIR}" ]; then - ZOPEN_SYSTEM_PREREQ_SCRIPT="${INCDIR}/prereq.sh" + ZOPEN_SCRIPTLET_DIR="${INCDIR}/scriptlets" else - ZOPEN_SYSTEM_PREREQ_SCRIPT="${ZOPEN_ROOTFS}/usr/local/zopen/meta/meta/include/prereq.sh" + ZOPEN_SCRIPTLET_DIR="${ZOPEN_ROOTFS}/usr/local/zopen/meta/meta/include/scriptlets" fi } addCleanupTrapCmd(){ - newcmd=$1 + newcmd="$1 >/dev/null 2>&1" # Command Trace MUST be disabled as the output from this can become # interleaved with output when calling zopen sub-processes. [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" @@ -36,41 +36,45 @@ addCleanupTrapCmd(){ # didn't have a bug -trap can't be piped/redirected anywhere except # a file like it can in bash or non-zos sh as it seems to create # and run in the subshell before returning trap handler(s)!?! - tmpscriptfile="clean.tmp" + tmpscriptfile="/tmp/clean.tmp" trap > "${tmpscriptfile}" 2>&1 && script=$(cat "${tmpscriptfile}") if [ -n "${script}" ]; then - for trappedSignal in "EXIT" "INT" "TERM" "QUIT" "HUP"; do - newtrapcmd=$( + for trappedSignal in "EXIT" "INT" "TERM" "QUIT" "HUP"; do + newtrapcmd=$( echo "${script}" | while read trapcmd; do - sigcmd=$(echo "${trapcmd}" | - zossed "s/trap -- \"\(.*\)\" ${trappedSignal}.*/\1/") - # No match/replace in sed, then sigcmd remains unchanged - [ "${sigcmd}" = "${trapcmd}" ] && continue - # There was a match, so sigcmd contains the string of commands - # to run for the trap. Need to remove the exit command (which - # returns the exit code) and ensure it is last - # Note that eval is used to run commands with potentially - # embedded variables (like the return code capture/return) - prfx="eval exit " - sigcmd=$(echo "${sigcmd}" |awk -v prfx="$prfx" ' - BEGIN{ FS=";"; OFS=";" } - { - for (i=1; i<=NF; i++){ - if (substr($i, 1, length(prfx)) != prfx) { - printf "%s%s", sep, $i; sep="$OFS" - } + sigcmd=$(echo "${trapcmd}" | + zossed "s/trap -- \"\(.*\)\" ${trappedSignal}.*/\1/") + # No match/replace in sed, then sigcmd remains unchanged + [ "${sigcmd}" = "${trapcmd}" ] && continue + # There was a match, so sigcmd contains the string of commands + # to run for the trap. Need to remove the exit command (which + # returns the exit code) and the initial set of the exit code + # and ensure it is last. Note that eval is used to run commands + # with potentially embedded variables (like the return code + # capture/return) + suffix="eval exit \$exitrc" + prfx="exitrc=\$?" + sigcmd=$(echo "${sigcmd}" |awk -v prfx="$prfx" -v suffix="$suffix" ' + BEGIN{ FS=";"; OFS=";" sep=""} + { + for (i=1; i<=NF; i++){ + if (substr($i, 1, length(prfx)) != prfx && + substr($i, 1, length(suffix)) != suffix) { + printf "%s%s >/dev/null 2>&1", sep, $i + sep = OFS } - } - END{ printf "\n" }' - ) - printf "%s;%s 2>/dev/null;eval exit \${exitrc}" "${sigcmd}" "${newcmd}" | tr -s ';' - break - done - ) - if [ -n "${newtrapcmd}" ]; then - trap -- "${newtrapcmd}" "${trappedSignal}" - fi - done + } + } + END{ printf "\n" }' + ) + printf "%s;%s;%s;eval exit \${exitrc}" "${prfx}" "${newcmd}" "${sigcmd}" | tr -s ';' + break + done + ) + if [ -n "${newtrapcmd}" ]; then + trap -- "${newtrapcmd}" "${trappedSignal}" + fi + done rm "${tmpscriptfile}" 2>/dev/null fi [ -n "${xtrc}" ] && set -x @@ -114,6 +118,22 @@ cleanupFunction() : } + +# showConfigParmWarning +# writes warning messages when a bad config parameter is found +# input: $1 - name of the paramter +# $2 - current value of the parameter +# $3 - valid value +# $4 - default that will be uesd +showConfigParmWarning(){ + printWarning "Found invalid value '$2' for $1 configuration parameter [should be $3]. Defaulting to '$4'" + if type zopen-config-helper >/dev/null 2>&1; then + printWarning "Run 'zopen config --set $1 [$3]' to update configuration if required." + else + printWarning "Update zopen configuration file with a valid value [$3] for parameter '$1' if required." + fi +} + # getParentProcess # returns the parent process for the specified process # input: $1 - the pid to get the parent of @@ -174,11 +194,12 @@ diffList() needles="$2" haystackfile=$(mktempfile "haystack") echo "${haystack}" >"${haystackfile}" - [ -e "${haystackfile}" ] && addCleanupTrapCmd "rm -rf ${tempdir}" + [ -e "${haystackfile}" ] && addCleanupTrapCmd "rm -rf ${haystackfile}" needlesfile=$(mktempfile "needles") echo "${needles}" >"${needlesfile}" [ -e "${needlesfile}" ] && addCleanupTrapCmd "rm -rf ${needlesfile}" diffFile "${needlesfile}" "${haystackfile}" + rm -rf "${needlesfile}" "${haystackfile}" } # Generate a file name that has a high probability of being unique for @@ -202,7 +223,6 @@ mktempfile() mktempfile "$1" "${suffix}" # recurse and try again else echo "${tempfile}" - addCleanupTrapCmd "rm -rf ${tempfile}" fi } @@ -464,39 +484,49 @@ darkbackground() { defineANSI() { # Standard tty codes - ESC="\047" + ESC=$(printf "\047") # Start of Escape Sequence; EBCDIC=\047, ASCII=\033 + CSI="[" # Control Sequence Introducer + CNL="E" # Cursor Next Line + CPL="F" # Cursor Previous Line + CHA="G" # Cursor Horizontal Absolute - column selector + EL="K" # Erase In Line + SGR="m" # Select Graphic Rendition + # shellcheck disable=SC2034 - ERASELINE="${ESC}[2K" + ERASELINE=$(printf "${ESC}${CSI}2${EL}") # shellcheck disable=SC2034 - CRSRHIDE="${ESC}[?25l" + CRSRHIDE=$(printf "${ESC}${CSI}?25l") # shellcheck disable=SC2034 - CRSRSHOW="${ESC}[?25h" + CRSRSHOW=$(printf "${ESC}${CSI}?25h") + CRSRSOL=$(printf "${ESC}${CSI}0${CHA}") # shellcheck disable=SC2034 - CRSRPL="${ESC}[1F" # Move to start of previous line + CRSRPL=$(printf "${ESC}${CSI}1${CPL}") # Move to start of previous line # shellcheck disable=SC2034 - CRSRUP="A" + CRSRUP="A" # CUU # shellcheck disable=SC2034 - CRSRDOWN="B" + CRSRDOWN="B" # CUF # shellcheck disable=SC2034 - CRSRRIGHT="C" + CRSRRIGHT="C" # CUB # shellcheck disable=SC2034 - CRSRLEFT="D" + CRSRLEFT="D" # CUD + + # Color-type codes, needs explicit terminal settings if [ ! "${_BPX_TERMPATH-x}" = "OMVS" ] && [ -z "${NO_COLOR}" ] && [ ! "${FORCE_COLOR-x}" = "0" ] && [ -t 1 ] && [ -t 2 ]; then ANSION=true - esc="\047" - BLACK="${esc}[30m" - RED="${esc}[31m" - GREEN="${esc}[32m" - YELLOW="${esc}[33m" - BLUE="${esc}[34m" - MAGENTA="${esc}[35m" - CYAN="${esc}[36m" - GRAY="${esc}[37m" - BOLD="${esc}[1m" - UNDERLINE="${esc}[4m" - NC="${esc}[0m" + #ESC="\047" + BLACK=$(printf "${ESC}${CSI}30${SGR}") + RED="${ESC}${CSI}31${SGR}" + GREEN="${ESC}${CSI}32${SGR}" + YELLOW="${ESC}${CSI}33${SGR}" + BLUE="${ESC}${CSI}34${SGR}" + MAGENTA="${ESC}${CSI}35${SGR}" + CYAN="${ESC}${CSI}36${SGR}" + GRAY="${ESC}${CSI}37${SGR}" + BOLD="${ESC}${CSI}1${SGR}" + UNDERLINE="${ESC}${CSI}4${SGR}" + NC="${ESC}${CSI}0${SGR}" darkbackground bg=$? if [ $bg -ne 0 ]; then @@ -537,8 +567,8 @@ ansiline() deltax=$1 deltay=$2 echostr="$3" - ansimove $1 $2 - /bin/echo "${echostr}\c" + ansimove "$1" "$2" + zosecho "${echostr}\c" } ansimove() @@ -549,19 +579,19 @@ ansimove() movestr="" if [ -n "${deltax}" ]; then if [ "${deltax}" -gt 0 ]; then - movestr="${ESC}[${deltax}${CRSRRIGHT}" + movestr="${ESC}${CSI}${deltax}${CRSRRIGHT}" elif [ "${deltax}" -lt 0 ]; then - movestr="${ESC}[$(expr "${deltax}" \* -1)${CRSRLEFT}" + movestr="${ESC}${CSI}$((deltax * -1))${CRSRLEFT}" fi fi if [ -n "${deltay}" ]; then if [ "${deltay}" -gt 0 ]; then - movestr="${movestr}${ESC}[${deltay}${CRSRDOWN}" + movestr="${movestr}${ESC}${CSI}${deltay}${CRSRDOWN}" elif [ "${deltay}" -lt 0 ]; then - movestr="${movestr}${ESC}[$(expr "${deltay}" \* -1)${CRSRUP}" + movestr="${movestr}${ESC}${CSI}$((deltay * -1))${CRSRUP}" fi fi - /bin/echo "${movestr}\c" + zosecho "${movestr}\c" } getScreenCols() @@ -569,12 +599,27 @@ getScreenCols() # If stdout/stderr are associated with a tty terminal if [ -t 1 ] && [ -t 2 ]; then # Note tput does not handle ssh sessions too well... - stty | awk -F'[/=;]' '/columns/ { print $4}' | tr -d " " + lclcols=$(stty | awk -F'[/=;]' '/columns/ { print $4}' | tr -d " ") elif [ ! -z "${COLUMNS}" ]; then - echo "${COLUMNS}" - else - tput cols + lclcols="${COLUMNS}" + elif ! lclcols=$(tput cols 2>/dev/null); then + # tput can fail if the terminal type is unrecognised; use fallback + lclcols=80 fi + echo "${lclcols}" +} + +zostr() +{ + # Use the standard z/OS 'tr' command + /bin/tr "$@" +} +zosecho() +{ + # Use the standard z/OS echo utility; supports use of ANSI colour when + # required and is consistent across shells as it uses the EBCDIC ANSI + # control codes + /bin/echo "$@" } zossed() @@ -666,12 +711,12 @@ defineEnvironment() } # -# For now, explicitly specify /bin/echo to ensure we get the EBCDIC echo since the escape +# For now, explicitly specify zosecho to ensure we get the EBCDIC echo since the escape # sequences are EBCDIC escape sequences # printColors() { - /bin/echo "$@" + zosecho "$@" } mutexReq() @@ -854,7 +899,6 @@ unsymlinkFromSystem() done ${killph} 2>/dev/null # if the timer is not running, the kill will fail waitforpid ${ph} # Make sure it's finished writing to screen - sleep 1 # give spinner time to exit if running else # Slower method needed to analyse each link to see if it has # become orphaned. Only relevent when removing a package as @@ -871,6 +915,7 @@ unsymlinkFromSystem() tempTrash=$(mktempfile "unsymlink" "trash") [ -e "${tempTrash}" ] && rm -f "${tempTrash}" >/dev/null 2>&1 addCleanupTrapCmd "rm -rf ${tempDirFile}" + addCleanupTrapCmd "rm -rf ${tempTrash}" printDebug "Using temporary file ${tempDirFile}" printInfo "- Checking ${nfiles} potentially obsolete file links" progressHandler "linkcheck" "- Checked ${nfiles} potentially obsolete file links" & @@ -909,6 +954,7 @@ EOF for d in $(uniq < "${tempDirFile}" | sort -r) ; do [ -d "${d}" ] && rmdir "${d}" >/dev/null 2>&1 done + rm "${tempDirFile}" fi if [ -e "${tempTrash}" ]; then printSoftError "Issues found while trying to remove the following files:" @@ -994,7 +1040,7 @@ runLogProgress() if [ -n "${SSH_TTY}" ]; then chtag -r "${SSH_TTY}" fi - ${killph} 2> /dev/null # if the timer is not running, the kill will fail + ${killph} 2>/dev/null # if the timer is not running, the kill will fail waitforpid ${ph} # Make sure it's finished writing to screen return "${rc}" } @@ -1025,7 +1071,7 @@ progressAnimation() [ "${ppid}" -eq 1 ] && kill INT "${ppid}" >/dev/null 2>&1 anim=$((anim + 1)) [ ${anim} -gt ${animcnt} ] && anim=1 - /bin/echo "${ESC}[1F${ESC}[0k$(getNthArrayArg "${anim}" "$@")" + printf "${CRSRSOL}${ERASELINE}%s" "$(getNthArrayArg "${anim}" "$@")" done } @@ -1037,9 +1083,11 @@ getNthArrayArg () waitforpid() { + sleep 1 while kill -0 "$1" >/dev/null 2>&1; do sleep 1 done + sleep 1 } progressHandler() @@ -1057,13 +1105,13 @@ progressHandler() completiontext=$2 # Custom end text (when the process is complete) trapcmd="exit;" if [ -n "${completiontext}" ]; then - trapcmd="/bin/echo \"${CRSRSHOW}${CRSRPL}${ERASELINE}${CRSRPL}${ERASELINE}${completiontext}\n\c\"; ${trapcmd}" + trapcmd="/bin/printf \"${CRSRSHOW}${ERASELINE}${CRSRPL}${ERASELINE}${completiontext}\n\"; ${trapcmd}" else - trapcmd="/bin/echo \"${CRSRSHOW}${CRSRPL}${ERASELINE}\c\"; ${trapcmd}" + trapcmd="/bin/printf \"${CRSRSHOW}${ERASELINE}${CRSRPL}${ERASELINE}\"; ${trapcmd}" fi # shellcheck disable=SC2064 trap "${trapcmd}" HUP - /bin/echo "${CRSRHIDE}" + printf "${CRSRHIDE}" case "${type}" in "spinner") progressAnimation '-' '/' '|' '\\' ;; "network") progressAnimation '-----' '>----' '->---' '-->--' '--->-' '---->' '-----' '----<' '---<-' '--<--' '-<---' '<----' ;; @@ -1111,6 +1159,17 @@ printSoftError() [ -n "${xtrc}" ] && set -x } +assertFailed() +{ + # Used to indicate that something that should have been set in an internal + # call was missing/borken - a program error rather than a user error + [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" + printColors "${NC}${RED}${BOLD}***INTERNAL ASSERTION ERROR: ${NC}${RED}${1}${NC}" >&2 + [ -n "${xtrc}" ] && set -x + mutexFree "zopen" # prevent lock from lingering around after an error + exit 12 +} + printError() { [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" @@ -1125,7 +1184,7 @@ printWarning() [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" printColors "${NC}${WARNINGCOLOR}${BOLD}***WARNING: ${NC}${YELLOW}${1}${NC}" >&2 [ -n "${xtrc}" ] && set -x - return 0; + return 0 } printInfo() @@ -1133,7 +1192,7 @@ printInfo() [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" printColors "$1" [ -n "${xtrc}" ] && set -x - return 0; + return 0 } # Used to input sensitive data - turns off echo to the screen for the input @@ -1343,9 +1402,11 @@ downloadJSONCache() from_readonly=$1 if [ -z "${JSON_CACHE}" ]; then - JSON_CACHE="${ZOPEN_ROOTFS}/var/cache/zopen/zopen_releases.json" - JSON_TIMESTAMP="${ZOPEN_ROOTFS}/var/cache/zopen/zopen_releases.timestamp" - JSON_TIMESTAMP_CURRENT="${ZOPEN_ROOTFS}/var/cache/zopen/zopen_releases.timestamp.current" + cachedir="${ZOPEN_ROOTFS}/var/cache/zopen" + [ ! -e "${cachedir}" ] && mkdir -p "${cachedir}" + JSON_CACHE="${cachedir}/zopen_releases.json" + JSON_TIMESTAMP="${cachedir}/zopen_releases.timestamp" + JSON_TIMESTAMP_CURRENT="${cachedir}/zopen_releases.timestamp.current" if [ -n "$from_readonly" ]; then if [ -r "$JSON_CACHE" ] && [ ! -w "$JSON_CACHE" ]; then @@ -1599,7 +1660,7 @@ getSelectMetadata() while ! ${valid}; do echo "Enter version to install (0-${i}): " read selection < /dev/tty - if [ ! -z $(echo "${selection}" | sed -e 's/[0-9]*//') ]; then + if [ -n "$(echo "${selection}" | sed -e 's/[0-9]*//')" ]; then echo "Invalid input, must be a number between 0 and ${i}" elif [ "${selection}" -ge 0 ] && [ "${selection}" -le "${i}" ]; then valid=true @@ -1633,9 +1694,9 @@ calculateReleaseLineMetadata() validatedReleaseLine="${installedReleaseLine}" # Already validated when stored else printDebug "Checking for system-configured releaseline" - if [ -e "${ZOPEN_ROOTFS}/etc/zopen/config.json" ]; then - printDebug "Using v2 configuration: '${ZOPEN_ROOTFS}/etc/zopen/config.json}'" - sysrelline=$(jq -re '.release_line' "${ZOPEN_ROOTFS}/etc/zopen/config.json") + if [ -e "${ZOPEN_JSON_CONFIG}" ]; then + printDebug "Using v2 configuration: '${ZOPEN_JSON_CONFIG}'" + sysrelline=$(jq -re '.release_line' "${ZOPEN_JSON_CONFIG}") elif [ -e "${ZOPEN_ROOTFS}/etc/zopen/releaseline" ] ; then printDebug "Using legacy file-based config" sysrelline=$(awk ' {print toupper($1)}') < "${ZOPEN_ROOTFS}/etc/zopen/releaseline" @@ -1843,26 +1904,26 @@ generateInstallGraph(){ invalidPortAssetFile=$(mktempfile "invalid" "port") addCleanupTrapCmd "rm -rf ${invalidPortAssetFile}" addToInstallGraph "install" "${invalidPortAssetFile}" "${portsToInstall}" - if - # shellcheck disable=SC2154 - ${doNotInstallDeps}; then + # shellcheck disable=SC2154 + if ${doNotInstallDeps}; then printVerbose "- Skipping dependency analysis" else # calculate dependancy graph createDependancyGraph "${invalidPortAssetFile}" fi - pruneGraph + + # shellcheck disable=SC2154 + if "${reinstall}"; then + printVerbose "Not pruning already installed packages as reinstalling" + else + pruneGraph + fi } pruneGraph() { printDebug "Pruning entries in graph if already installed" # shellcheck disable=SC2154 - if "${reinstall}"; then - # Reinstall packages if they are already installed - no prune needed - return 0 - fi - # shellcheck disable=SC2154 if "${downloadOnly}"; then # Download the pax files, even if already installed as they are not # being reinstalled so no prune required @@ -1920,7 +1981,7 @@ processRepoInstallFile(){ printVerbose "Beginning port installation" mutexReq "zopen" - processActionScripts "txn-pre" + processActionScripts "transactionpre" if [ 0 -eq "$(echo "${installList}" | jq --raw-output '.installqueue| length')" ]; then printInfo "- No packages to install" return 0 @@ -1960,7 +2021,7 @@ processRepoInstallFile(){ spaceValidate "${spaceRequiredBytes}" fi - processActionScripts "txn-pre" + processActionScripts "transactionpre" for installurl in $(echo "${installList}" | jq --raw-output '.installqueue |map( (.asset.url| sub(" ";"") ))| @sh'); do printVerbose "Analysing :'${installurl}'" installurl=$(echo "${installurl}" | tr -d "' ") @@ -1976,7 +2037,7 @@ processRepoInstallFile(){ printError "Unrecognised install file format" fi done - processActionScripts "txn-post" + processActionScripts "transactionpost" mutexFree "zopen" printVerbose "Port installation complete" } @@ -2020,7 +2081,6 @@ installFromPax() { pax="${downloadToDir}/$1" printDebug "Installing from '${pax}'" - processActionScripts "install-pre" metadatafile=$(extractMetadataFromPax "${pax}") # Ideally we would use the following, but name does not always map @@ -2028,6 +2088,9 @@ installFromPax() # repo field so can extract from there instead #name=$(jq --raw-output '.product.name' "${metadatafile}") name=$(jq --raw-output '.product.repo | match(".*/ZOSOpenTools/(.*)port").captures[0].string' "${metadatafile}") + if ! processActionScripts "installpre" "${name}" "${metadatafile}"; then + printError "Failed installation pre-requisite check(s) for '${name}'. Correct previous errors and retry command" + fi # Store current installation directory (if exists) currentderef=$(cd "${ZOPEN_PKGINSTALL}/${name}/${name}" > /dev/null 2>&1 && pwd -P) @@ -2045,7 +2108,7 @@ installFromPax() fi # shellcheck disable=SC2154 - if ! runLogProgress "pax -rf ${pax} -p p ${paxredirect} ${redirectToDevNull}" "Expanding ${pax}" "Expanded ${pax}"; then + if ! runLogProgress "pax -rf ${pax} -p p ${paxredirect} ${redirectToDevNull}" "Expanding: ${pax}" "Expanded: ${pax}"; then printSoftError "Unexpected errors during unpaxing, package directory state unknown" printError "Use zopen alt to select previous version to ensure known state" fi @@ -2108,7 +2171,7 @@ EOF printf "${name}:\n%s\n" "${installCaveat}">> "${ZOPEN_ROOTFS}/var/cache/install_caveats.tmp" fi - processActionScripts "install-post" + processActionScripts "installpost" "${name}" fi printInfo "${NC}${GREEN}Successfully installed ${name}${NC}" } @@ -2119,70 +2182,146 @@ getActivePackageDirs() (unset CD_PATH; cd "${ZOPEN_PKGINSTALL}" && zosfind ./*/. ! -name . -prune -type l) } + +# processActionScripts +# runs any scriptlets that are applicable to the current phase of an administration +# command (install/update/remove/alternative) +# inputs: $1 the phase of the transaction that is currently executing +# $2 the name of the package being administered +# return: 0 for success (nb. Warnings may haeve been printed to screen) +# 8 on error processActionScripts() { + set -x + printVerbose "Processing phase '${1}' scriptlets" + [ $# -lt 1 ] && printError "Internal error; missing action phase" phase=$1 - case $phase in - "install-post") scriptDir="${ZOPEN_ROOTFS}/etc/zopen/scriptlets/postInstall";; - "remove-post") scriptDir="${ZOPEN_ROOTFS}/etc/zopen/scriptlets/postRemove";; - "txn-post") scriptDir="${ZOPEN_ROOTFS}/etc/zopen/scriptlets/postTransaction";; - "install-pre") scriptDir="${ZOPEN_ROOTFS}/etc/zopen/scriptlets/preInstall";; - "remove-pre") scriptDir="${ZOPEN_ROOTFS}/etc/zopen/scriptlets/preRemove";; - "txn-pre") scriptDir="${ZOPEN_ROOTFS}/etc/zopen/scriptlets/preTransaction";; - *) printError "Internal error; invalid post action phase '${phase}'" + shift # Drop the initial parameter + + case "${phase}" in + "installpost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/installpost";; + "removepost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/removepost";; + "transactionpost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/transactionpost";; + "installpre") scriptDir="${ZOPEN_SCRIPTLET_DIR}/installpre";; + "removepre") scriptDir="${ZOPEN_SCRIPTLET_DIR}/removepre";; + "transactionpre") scriptDir="${ZOPEN_SCRIPTLET_DIR}/transactionpre";; + *) assertFailed "Invalid process action phase '${phase}'" esac - ( + printVerbose "Running script[s] from '${scriptDir}'" + scriptRc=$( + # Running in a sub-shell so the scripts do not directly affect the current + # environment - status is returned to scriptRc [ -d "${scriptDir}" ] || return 0 # No script directory unset CDPATH; cd "${scriptDir}" || exit # the subshell - find . | while read scriptFile; - do - if [ ! -e "${scriptFile}" ]; then + find . -type f | while read scriptFile; do + if [ ! -x "${scriptFile}" ]; then printWarning "Script '${scriptDir}/${scriptFile}' is not executable. Check permissions" continue fi - # shellcheck disable=SC1090 - . "${scriptFile}" + printVerbose "Running script '${scriptFile}'" + # Call each scriptlet with the remaining parameters passed to this function + if ! src=$("${scriptFile}" "$@"); then + exit "${src:=-1}" + fi done ) + set +x + return "${scriptRc:-0}" } +# updatePackageDB +# Updates/generates the installed package database +# return: 0 for success +# 8 in error updatePackageDB() { printVerbose "Updating the installed package tracker db" - if ! pkgdirs=$(getActivePackageDirs); then - printError "Unable to update the package db" - fi pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" if [ -e "${pdb}" ]; then backup=$(mktempfile "updatepdb" "bkk") + addCleanupTrapCmd "rm -rf ${backup}" cp "${pdb}" "${backup}" rm "${pdb}" fi - pkgdirs=$(getActivePackageDirs) + if ! pkgdirs=$(getActivePackageDirs); then + printError "Unable to update the package db" + fi for pkgdir in ${pkgdirs}; do metadataFile="${ZOPEN_PKGINSTALL}/${pkgdir}/metadata.json" if [ ! -e "${metadataFile}" ]; then - printVerbose "Falling back to filesystem analysis" + # TODO: Fallback to filesystem analysis [depending on backward compatability] printWarning "No metadata.json found in '${ZOPEN_PKGINSTALL}/${pkgdir}'. Old package install?" continue fi - metadata=$(cat "${metadataFile}") + escapedJSONFile=$(mktempfile "escaped" "json") + addCleanupTrapCmd "rm -rf ${escapedJSONFile}" + stripControlCharacters "${metadataFile}" "${escapedJSONFile}" if [ ! -e "${pdb}" ]; then echo "[]" > "${pdb}" fi - - mdj=$(echo "${metadata}" | jq '. as $metadata | .product.repo | match(".*/ZOSOpenTools/(.*)port").captures[0].string | [{(.):$metadata}]') + mdj=$(jq '. as $metadata | .product.repo | match(".*/ZOSOpenTools/(.*)port").captures[0].string | [{(.):$metadata}]' \ + "${escapedJSONFile}") if ! jq --argjson mdj "${mdj}" '. += $mdj' \ "${pdb}" > \ "${pdb}.working"; then - printError "Could not add metadata for '${pkg}' to install tracker. Run zopen --re-init to attempt regeneration." + printSoftError "Could not add metadata for '$(basename "${pkgdir}")' to install tracker." + printError "Run zopen --re-init to attempt database regeneration and zopen install --reinstall $(basename "${pkgdir}")" fi mv "${pdb}.working" "${pdb}" done } +stripControlCharacters(){ + [ ! -f "$1" ] && assertFailed "No input file specified for parsing!" + [ -e "$2" ] && assertFailed "Output file exists so cannot be used for output!" + tr -d '[:cntrl:]' > "$2" < "$1" + +} +# JSONcontrolChar2Unicode +# ensures escaping of characters in JSON. For example, if a caveat has an +# unescaped '\n'/0x0A jq will fail to process it. Escape control characters +# 0x00->0x1F and reverse solidus. Note this should also only process unescaped +# sequences by checking whether the character prior to the sequence is not a +# reverse-solidus (so start-of-line [^] or any character [^\] ). If there was +# a preceding character, then capture/use that in the regex (\1) so it does not +# get discarded +# +# inputs: $1 the input JSON file +# $2 the output file, with sanitised JSON +# return: 0 for success (nb. Warnings may haeve been printed to screen) +# 8 on error +JSONcontrolChar2Unicode() { + [ ! -f "$1" ] && assertFailed "No input file specified for parsing!" + [ -e "$2" ] && assertFailed "Output file exists so cannot be used for output!" + zossed -E ' # Note this is a long string! + s/\\/\\\\/g; # Escape reverse-solidus; the following are the control chars + s/(^|[^\\])\x00/\1\\u0000/g; s/(^|[^\\])\x01/\1\\u0001/g; + s/(^|[^\\])\x02/\1\\u0002/g; s/(^|[^\\])\x03/\1\\u0003/g; + s/(^|[^\\])\x04/\1\\u0004/g; s/(^|[^\\])\x05/\1\\u0005/g; + s/(^|[^\\])\x06/\1\\u0006/g; s/(^|[^\\])\x07/\1\\u0007/g; + s/(^|[^\\])\x08/\1\\u0008/g; s/(^|[^\\])\x09/\1\\u0009/g; + s/(^|[^\\])\x0A/\1\\u000A/g; s/(^|[^\\])\x0B/\1\\u000B/g; + s/(^|[^\\])\x0C/\1\\u000C/g; s/(^|[^\\])\x0D/\1\\u000D/g; + s/(^|[^\\])\x0E/\1\\u000E/g; s/(^|[^\\])\x0F/\1\\u000F/g; + s/(^|[^\\])\x10/\1\\u0010/g; s/(^|[^\\])\x11/\1\\u0011/g; + s/(^|[^\\])\x12/\1\\u0012/g; s/(^|[^\\])\x13/\1\\u0013/g; + s/(^|[^\\])\x14/\1\\u0014/g; s/(^|[^\\])\x15/\1\\u0015/g; + s/(^|[^\\])\x16/\1\\u0016/g; s/(^|[^\\])\x17/\1\\u0017/g; + s/(^|[^\\])\x18/\1\\u0018/g; s/(^|[^\\])\x19/\1\\u0019/g; + s/(^|[^\\])\x1A/\1\\u001A/g; s/(^|[^\\])\x1B/\1\\u001B/g; + s/(^|[^\\])\x1C/\1\\u001C/g; s/(^|[^\\])\x1D/\1\\u001D/g; + s/(^|[^\\])\x1E/\1\\u001E/g; s/(^|[^\\])\x1F/\1\\u001F/g; + ' "$1" > "$2" +} + +# addToInstallTracker +# Records the installation into the database that tracks which packages have +# been installed +# inputs: $1 the name of the package +# return: 0 for success (nb. Warnings may haeve been printed to screen) +# 8 on error addToInstallTracker() { pkg=$1 @@ -2202,6 +2341,12 @@ addToInstallTracker() mv "${pdb}.working" "${pdb}" } +# removeFromInstallTracker +# Removes the installation of the package from the database, making it +# uninstalled +# inputs: $1 the name of the package +# return: 0 for success (nb. Warnings may haeve been printed to screen) +# 8 on error removeFromInstallTracker() { pkg=$1 diff --git a/include/scriptlets/installPost/autopkgpurge b/include/scriptlets/installPost/autopkgpurge new file mode 100755 index 000000000..bef3aa532 --- /dev/null +++ b/include/scriptlets/installPost/autopkgpurge @@ -0,0 +1,21 @@ +#!/bin/sh +pkg=$1 +[ -z "${pkg}" ] && return +if type zopen-config-helper >/dev/null 2>&1; then + isactive=$(zopen-config-helper --get autopkgpurge) +else + isactive=0 +fi +case "${isactive}" in + 0 ) :;; # Take no action + 1 ) (zopen clean --unused "${pkg}" >/dev/null 2>&1) + ;; + *) printWarning "Invalid value '${isactive}' for autopkgpurge config parameter [should be 0 or 1]." + if type zopen-config-helper >/dev/null 2>&1; then + printWarning "Run zopen config --set autopkgpurge [0|1] to update configuration" + else + printWarning "Update zopen configuration file to contain a valid value for parameter 'autopkgpurge'" + fi + printWarning "Unused package clean will not be performed" + ;; +esac diff --git a/include/scriptlets/installPost/zopen-meta-init-refresh b/include/scriptlets/installPost/zopen-meta-init-refresh new file mode 100755 index 000000000..af254b1e4 --- /dev/null +++ b/include/scriptlets/installPost/zopen-meta-init-refresh @@ -0,0 +1,7 @@ +#!/bin/sh +pkg=$1 +[ -z "${pkg}" ] && return +case "${pkg}" in + meta ) (zopen init --refresh -y) ;; + * ) :;; # No action +esac diff --git a/include/scriptlets/installPre/systemPreReqCheck b/include/scriptlets/installPre/systemPreReqCheck new file mode 100755 index 000000000..179bf902e --- /dev/null +++ b/include/scriptlets/installPre/systemPreReqCheck @@ -0,0 +1,24 @@ +#!/bin/sh +pkg=$1 +metadataFile=$2 +[ -z "${pkg}" ] && return +# shellcheck disable=SC2154 +if ! "${bypassPrereqs}"; then + systemPrereqs=$(jq -r '.product.system_prereqs // empty' "${metadataFile}" 2>/dev/null) + systemPrereqs="${systemPrereqs} zos24" # zos24 should be min requirement - always add it + if [ -n "${systemPrereqs}" ]; then + if [ ! -d "${ZOPEN_SYSTEM_PREREQS_DIR}" ]; then + printWarning "${ZOPEN_SYSTEM_PREREQS_DIR} does not exist. You should upgrade meta. Bypassing prereq check." + fi + for prereq in $(echo "${systemPrereqs}" | xargs | tr ' ' '\n' | sort -u); do + printHeader "Checking system pre-req requirement '${prereq}'" + if [ -e "${ZOPEN_SYSTEM_PREREQS_DIR}/${prereq}" ]; then + if ! /bin/sh -c "${ZOPEN_SYSTEM_PREREQS_DIR}/${prereq}"; then + printError "Failed system pre-req check '${prereq}'. If you wish to bypass this, install with --bypass-prereq-checks" + fi + else + printWarning "${ZOPEN_SYSTEM_PREREQS_DIR}/${prereq} does not exist. You should upgrade meta. Bypassing prereq check." + fi + done + fi +fi diff --git a/include/scriptlets/removePost/man-db b/include/scriptlets/removePost/man-db new file mode 100755 index 000000000..b871a8fa9 --- /dev/null +++ b/include/scriptlets/removePost/man-db @@ -0,0 +1,12 @@ +#!/bin/sh +if ! zopen list --installed | grep "^man-db\$"; then + : # No man-db installed +else + echo "- Updating man-db database after package removal..." + if ! mandb >/dev/null 2>&1; then + printWarning "man-db update completed with non-zero return code." + printWarning "Re-run 'mandb' manually for additional information." + else + echo "- Update of man-db complete." + fi +fi diff --git a/include/scriptlets/transactionPost/list_caveats b/include/scriptlets/transactionPost/list_caveats new file mode 100755 index 000000000..a23349496 --- /dev/null +++ b/include/scriptlets/transactionPost/list_caveats @@ -0,0 +1,5 @@ +#!/bin/sh +caveats_file="${ZOPEN_ROOTFS}/var/cache/install_caveats.tmp" +[ -e "${caveats_file}" ] || exit 0 +cat "${caveats_file}" +rm "${caveats_file}" diff --git a/include/scriptlets/transactionPost/man-db b/include/scriptlets/transactionPost/man-db new file mode 100755 index 000000000..febb5e00e --- /dev/null +++ b/include/scriptlets/transactionPost/man-db @@ -0,0 +1,12 @@ +#!/bin/sh +if ! zopen list --installed | grep "^man-db\$"; then + : # No man-db installed +else + echo "- Updating man-db database after package changes..." + if ! mandb >/dev/null 2>&1; then + printWarning "man-db update completed with non-zero return code." + printWarning "Re-run 'mandb' manually for additional information." + else + echo "- Update of man-db complete." + fi +fi diff --git a/include/scriptlets/transactionPost/zopen-clean-autocache b/include/scriptlets/transactionPost/zopen-clean-autocache new file mode 100755 index 000000000..6f7ba1a78 --- /dev/null +++ b/include/scriptlets/transactionPost/zopen-clean-autocache @@ -0,0 +1,24 @@ +#!/bin/sh +printInfo "RUNNING AUTO CLEAN" +if type zopen-config-helper >/dev/null 2>&1; then +printInfo "USING CONFIG-HELPER" + isactive=$(zopen-config-helper --get autocacheclean) +else + isactive=1 # Default to true +fi +case "${isactive}" in + 0 ) :;; # Take no action + 1 ) echo "- Cleaning install cache..." + printInfo "CLEANING !!" + (zopen clean --cache >/dev/null 2>&1) + echo "- Cache cleaned." + ;; + *) printWarning "Invalid value '${isactive}' for autocacheclean config parameter [should be 0 or 1]." + if type zopen-config-helper >/dev/null 2>&1; then + printWarning "Run zopen config --set autocacheclean [0|1] to update configuration" + else + printWarning "Update zopen configuration file to contain a valid value for parameter 'autocacheclean'" + fi + printWarning "Autocache clean will not be performed" + ;; +esac diff --git a/include/scriptlets/transactionPost/zopen-config-list b/include/scriptlets/transactionPost/zopen-config-list new file mode 100755 index 000000000..f61da1637 --- /dev/null +++ b/include/scriptlets/transactionPost/zopen-config-list @@ -0,0 +1,16 @@ +#!/bin/sh +envFile="${ZOPEN_ROOTFS}/etc/zopen/zopen.env" +envFileWorking="${envFile}.working" +# Running in a "clean" environment +if ! (env -i . "${ZOPEN_ROOTFS}"/etc/zopen-config --knv | sort > "${envFileWorking}" ); then + printError "Unable to generate zopen environment key/value pair" +fi +[ -e "${envFileWorking}" ] || printError "Could not locate working file" +if ! [ -e "${envFile}" ]; then + mv "${envFileWorking}" "${envFile}" + exit 0 +fi +if ! diff=$(diff "${envFileWorking}" "${envFile}"); then + printSoftError "${diff}" + printError "Could not compute difference in new key-value pairs. Correct errors and run and redirect '${ZOPEN_ROOTFS}/etc/zopen-config knv | sort' manually." +fi From d2db05ef60917b6ca239950c7dcc9f9bba9c6747 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 12 Sep 2024 16:00:23 +0100 Subject: [PATCH 031/179] Match install script with new location --- bin/zopen-alt | 10 +++++----- bin/zopen-remove | 8 ++++---- include/common.sh | 24 +++++++++++------------- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/bin/zopen-alt b/bin/zopen-alt index 651fe6b51..55895d997 100755 --- a/bin/zopen-alt +++ b/bin/zopen-alt @@ -55,7 +55,7 @@ mergeNewVersion() { package="$1" newver="$2" - processActionScripts "transactionpre" + processActionScripts "transactionPre" [ -z "${package}" ] && printError "Internal error; no package name provided to merge." [ -d "${ZOPEN_PKGINSTALL}/${package}/${newver}" ] || printError "Version '${newver}' was not available to set as current" if [ -e "${ZOPEN_PKGINSTALL}/${package}/${package}" ]; then @@ -85,7 +85,7 @@ mergeNewVersion() version=$(cat "${ZOPEN_PKGINSTALL}/${package}/${package}/.releaseinfo") fi syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE}" "ALT" "setAlt" "Set '${package}' to version:${version};" - processActionScripts "transactionpost" + processActionScripts "transactionPost" } setAlt() @@ -165,11 +165,11 @@ listAlts() # echo "${found}" | xargs | tr ' ' '\n' | while read repo; do TMP_FIFO_PIPE="${HOME}/altselect.pipe" [ ! -p "${TMP_FIFO_PIPE}" ] || rm -f "${TMP_FIFO_PIPE}" - mkfifo ${TMP_FIFO_PIPE} + mkfifo "${TMP_FIFO_PIPE}" echo "${found}" | xargs | tr ' ' '\n' >> "${TMP_FIFO_PIPE}" & while read repo; do printVerbose "Parsing repo: '${repo}' as '${repo#"${ZOPEN_PKGINSTALL}"/}'" - i=$(expr ${i} + 1) + i=$((i + 1)) if [ "${deref#"${ZOPEN_PKGINSTALL}"/}" = "${repo#"${ZOPEN_PKGINSTALL}"/}" ]; then current=${i} printInfo "${NC}${GREEN}${i}: ${repo#"${ZOPEN_PKGINSTALL}"/} <- current${NC}" @@ -177,7 +177,7 @@ listAlts() printInfo "${i}: ${repo#"${ZOPEN_PKGINSTALL}"/}" fi done < "${TMP_FIFO_PIPE}" - [ ! -p ${TMP_FIFO_PIPE} ] || rm -rf "${TMP_FIFO_PIPE}" + [ ! -p "${TMP_FIFO_PIPE}" ] || rm -rf "${TMP_FIFO_PIPE}" if ${select}; then mutexReq "zopen" "zopen" diff --git a/bin/zopen-remove b/bin/zopen-remove index 7f6190051..2a57e5f62 100755 --- a/bin/zopen-remove +++ b/bin/zopen-remove @@ -52,14 +52,14 @@ HELPDOC removePackages() { pkglist=$* - processActionScripts "transactionpre" + processActionScripts "transactionPre" # Create a temp file name to trigger the post-transaction action script # triggering - run only if an action was taken (which creates the file) runpostTxnActionScripts=$(mktempfile "txnpost" "run") addCleanupTrapCmd "rm -rf ${runpostTxnActionScripts}" echo "${pkglist}" | xargs | tr ' ' '\n' | sort | while read pkg; do printHeader "Removing package: ${pkg}" - processActionScripts "removepre" + processActionScripts "removePre" printInfo "- Querying install status of package '${pkg}'" if [ ! -f "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}/.active" ]; then printInfo "${NC}${YELLOW}Package '${pkg}' is not installed${NC}" @@ -107,14 +107,14 @@ removePackages() [ -d "${ZOPEN_ROOTFS}/etc/profiled/${pkg}" ] && rm -rf "${ZOPEN_ROOTFS}/etc/profiled/${pkg}" removeFromInstallTracker "${pkg}" - processActionScripts "removepost" + processActionScripts "removePost" syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE},${CAT_REMOVE}" "REMOVE" "removePackage" "Removed package:'${needle};version:${version};" touch "${runpostTxnActionScripts}" printInfo "${NC}${GREEN}Successfully removed: ${pkg}${NC}" fi done if [ -e "${runpostTxnActionScripts}" ]; then - processActionScripts "transactionpost"; + processActionScripts "transactionPost"; rm "${runpostTxnActionScripts}" fi } diff --git a/include/common.sh b/include/common.sh index 6f3e3ab75..a27ec2660 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1981,7 +1981,7 @@ processRepoInstallFile(){ printVerbose "Beginning port installation" mutexReq "zopen" - processActionScripts "transactionpre" + processActionScripts "transactionPre" if [ 0 -eq "$(echo "${installList}" | jq --raw-output '.installqueue| length')" ]; then printInfo "- No packages to install" return 0 @@ -2021,7 +2021,7 @@ processRepoInstallFile(){ spaceValidate "${spaceRequiredBytes}" fi - processActionScripts "transactionpre" + processActionScripts "transactionPre" for installurl in $(echo "${installList}" | jq --raw-output '.installqueue |map( (.asset.url| sub(" ";"") ))| @sh'); do printVerbose "Analysing :'${installurl}'" installurl=$(echo "${installurl}" | tr -d "' ") @@ -2037,7 +2037,7 @@ processRepoInstallFile(){ printError "Unrecognised install file format" fi done - processActionScripts "transactionpost" + processActionScripts "transactionPost" mutexFree "zopen" printVerbose "Port installation complete" } @@ -2088,7 +2088,7 @@ installFromPax() # repo field so can extract from there instead #name=$(jq --raw-output '.product.name' "${metadatafile}") name=$(jq --raw-output '.product.repo | match(".*/ZOSOpenTools/(.*)port").captures[0].string' "${metadatafile}") - if ! processActionScripts "installpre" "${name}" "${metadatafile}"; then + if ! processActionScripts "installPre" "${name}" "${metadatafile}"; then printError "Failed installation pre-requisite check(s) for '${name}'. Correct previous errors and retry command" fi @@ -2171,7 +2171,7 @@ EOF printf "${name}:\n%s\n" "${installCaveat}">> "${ZOPEN_ROOTFS}/var/cache/install_caveats.tmp" fi - processActionScripts "installpost" "${name}" + processActionScripts "installPost" "${name}" fi printInfo "${NC}${GREEN}Successfully installed ${name}${NC}" } @@ -2192,19 +2192,18 @@ getActivePackageDirs() # 8 on error processActionScripts() { - set -x printVerbose "Processing phase '${1}' scriptlets" [ $# -lt 1 ] && printError "Internal error; missing action phase" phase=$1 shift # Drop the initial parameter case "${phase}" in - "installpost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/installpost";; - "removepost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/removepost";; - "transactionpost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/transactionpost";; - "installpre") scriptDir="${ZOPEN_SCRIPTLET_DIR}/installpre";; - "removepre") scriptDir="${ZOPEN_SCRIPTLET_DIR}/removepre";; - "transactionpre") scriptDir="${ZOPEN_SCRIPTLET_DIR}/transactionpre";; + "installPost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/installPost";; + "removePost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/removePost";; + "transactionPost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/transactionPost";; + "installPre") scriptDir="${ZOPEN_SCRIPTLET_DIR}/installPre";; + "removePre") scriptDir="${ZOPEN_SCRIPTLET_DIR}/removePre";; + "transactionPre") scriptDir="${ZOPEN_SCRIPTLET_DIR}/transactionPre";; *) assertFailed "Invalid process action phase '${phase}'" esac printVerbose "Running script[s] from '${scriptDir}'" @@ -2226,7 +2225,6 @@ processActionScripts() fi done ) - set +x return "${scriptRc:-0}" } From fcd7ce22088f447558761910deaf25d4b7c828c5 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 1 Oct 2024 12:30:53 +0100 Subject: [PATCH 032/179] Pre-req checks --- bin/zopen-alt | 2 + bin/zopen-init | 46 +++++++--- bin/zopen-install | 1 + bin/zopen-promote | 6 +- bin/zopen-query | 84 ------------------- include/common.sh | 29 +++++++ .../systemPreReqCheck | 0 7 files changed, 68 insertions(+), 100 deletions(-) rename include/scriptlets/{installPre => preReqCheck}/systemPreReqCheck (100%) diff --git a/bin/zopen-alt b/bin/zopen-alt index 55895d997..646bb5127 100755 --- a/bin/zopen-alt +++ b/bin/zopen-alt @@ -56,6 +56,7 @@ mergeNewVersion() package="$1" newver="$2" processActionScripts "transactionPre" + processActionScripts "installPre" # We are effectively re-installing [ -z "${package}" ] && printError "Internal error; no package name provided to merge." [ -d "${ZOPEN_PKGINSTALL}/${package}/${newver}" ] || printError "Version '${newver}' was not available to set as current" if [ -e "${ZOPEN_PKGINSTALL}/${package}/${package}" ]; then @@ -85,6 +86,7 @@ mergeNewVersion() version=$(cat "${ZOPEN_PKGINSTALL}/${package}/${package}/.releaseinfo") fi syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE}" "ALT" "setAlt" "Set '${package}' to version:${version};" + processActionScripts "installPost" processActionScripts "transactionPost" } diff --git a/bin/zopen-init b/bin/zopen-init index d3843c434..6d4c3e957 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -180,6 +180,16 @@ while [ $# -gt 0 ]; do esac shift done +# Wrapper around jq to provide extra help for certain scenarios +jqw() +{ + if ! type jq >/dev/null 2>&1; then + printSoftError "Cannot locate 'jq'." + printSoftError "If recovering a system, run . ./.env from the following location:" + printError " /usr/local/zopen/jq/ and retry command" + fi + jq "$@" +} determineRootFileSystem() { @@ -234,7 +244,9 @@ determineStatsCollection() elif [ -n "$enableStats" ] && ! $enableStats; then isCollectingStats=false elif [ $analyticsRC -lt 2 ]; then - isCollectingStats=$(jq -re '.is_collecting_stats' "${jsonConfig}") + if ! isCollectingStats=$(jqw -re '.is_collecting_stats' "${jsonConfig}"); then + return 4 + fi return elif $isHostIBM; then printAttention "Attention: Turning on usage statistics as your host is an IBM host. If this is incorrect, rerun with --noenable-stats" @@ -413,13 +425,15 @@ generateJsonConfiguration() if [ -e "${jsonConfig}" ]; then # isCollectingStats is set elsewhere - releaseLine=$(jq -r '.release_line' "${jsonConfig}") - rm_fileprocs=$(jq -r '.num_rm_procs' "${jsonConfig}") - if [ -z "${OVERRIDE_ZOS_TOOLS}" ] && jq -er ".override_zos_tools" $jsonConfig >/dev/null; then - OVERRIDE_ZOS_TOOLS=$(jq -r '.override_zos_tools' $jsonConfig) + if ! releaseLine=$(jqw -r '.release_line' "${jsonConfig}"); then + return 4 + fi + rm_fileprocs=$(jqw -r '.num_rm_procs' "${jsonConfig}") + if [ -z "${OVERRIDE_ZOS_TOOLS}" ] && jqw -er ".override_zos_tools" $jsonConfig >/dev/null; then + OVERRIDE_ZOS_TOOLS=$(jqw -r '.override_zos_tools' $jsonConfig) fi - autocacheclean=$(jq -r '.autocacheclean' "${jsonConfig}") - autopkgpurge=$(jq -r '.autopkgpurge' "${jsonConfig}") + autocacheclean=$(jqw -r '.autocacheclean' "${jsonConfig}") + autopkgpurge=$(jqw -r '.autopkgpurge' "${jsonConfig}") fi if ${releaselineDev}; then @@ -475,14 +489,14 @@ zz printInfo "Refreshing global configuration" # In case 3rd-party edits have been made (adding custom keys for # example), just update the values that zopen itself knows about inline - if ! jq --arg releaseLine "${releaseLine}" \ + if ! jqw --arg releaseLine "${releaseLine}" \ --arg isoverrideZOSTools ${OVERRIDE_ZOS_TOOLS} \ --arg isCollectingStats ${isCollectingStats} \ ".release_line = \$releaseLine | .override_zos_tools = \$isoverrideZOSTools | .is_collecting_stats = \$isCollectingStats" \ - "${ZOPEN_JSON_CONFIG}" > "${ZOPEN_JSON_CONFIG}.working"; then + "${jsonConfig}" > "${jsonConfigWorking}"; then printError "Errors updating existing configuration file. See previous errors for more information and retry command." fi - mv -f "${ZOPEN_JSON_CONFIG}.working" "${ZOPEN_JSON_CONFIG}" + mv -f "${jsonConfigWorking}" "${jsonConfig}" fi } @@ -538,7 +552,7 @@ generateAnalyticsConfiguration() jsonConfig="${rootfs}/var/lib/zopen/analytics.json" # if it's an existing valid json, skip - if [ -s "${jsonConfig}" ] && jq "." -e "${jsonConfig}" 2>/dev/null >/dev/null; then + if [ -s "${jsonConfig}" ] && jqw "." -e "${jsonConfig}" 2>/dev/null >/dev/null; then return; fi jsonConfigWorking="${jsonConfig}.working" @@ -553,7 +567,7 @@ generateAnalyticsConfiguration() "removes": [] } zz - if jqrc=$(jq '.' $jsonConfigWorking > $jsonConfig); then + if jqrc=$(jqw '.' $jsonConfigWorking > $jsonConfig); then rm "${jsonConfigWorking}" else printSoftError "${jqrc}" @@ -700,13 +714,21 @@ deinit() fi } +echo "DBG!! ZOPEN_ROOTFS=${ZOPEN_ROOTFS}" init +echo "DBG!! ZOPEN_ROOTFS=${ZOPEN_ROOTFS}" generateFileSystem +echo "DBG!! ZOPEN_ROOTFS=${ZOPEN_ROOTFS}" ! ${refresh} && installPrereqs +echo "DBG!! ZOPEN_ROOTFS=${ZOPEN_ROOTFS}" generateJsonConfiguration +echo "DBG!! ZOPEN_ROOTFS=${ZOPEN_ROOTFS}" generateDefaultRepository +echo "DBG!! ZOPEN_ROOTFS=${ZOPEN_ROOTFS}" generateAnalyticsConfiguration +echo "DBG!! ZOPEN_ROOTFS=${ZOPEN_ROOTFS}" ! ${refresh} && updateBootstrappedTools +echo "DBG!! ZOPEN_ROOTFS=${ZOPEN_ROOTFS}" [ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || updatePackageDB updateBootstrappedTools deinit diff --git a/bin/zopen-install b/bin/zopen-install index 94e4fd61f..23b2a5095 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -172,6 +172,7 @@ xdebug=false quiet=false selectVersion=false # shellcheck disable=SC2034 +bypassPrereqs=false setActive=true downloadOnly=false reinstall=false diff --git a/bin/zopen-promote b/bin/zopen-promote index 4541cffce..09d1d8d30 100755 --- a/bin/zopen-promote +++ b/bin/zopen-promote @@ -244,9 +244,6 @@ fi printInfo "- Promoting from '${rootfs}' to '${promotefs}'..." -printDebug "Generating temporary pipe" -FIFO_PIPE_STDOUT=$(mktempfile "" ".pipe") - progressHandler "spinner" "- Promoted from '${rootfs}' to '${promotefs}'." & ph=$! killph="kill -HUP ${ph}" @@ -262,7 +259,8 @@ printDebug "Need to pax old dir and unpax to new dest to keep symlink structure" printDebug "Generating temporary pipe" FIFO_PIPE_STDOUT=$(mktempfile "promotestdout" ".pipe") [ ! -p "${FIFO_PIPE_STDOUT}" ] || rm -f "${FIFO_PIPE_STDOUT}" -mkfifo "${FIFO_PIPE_STDOUT}" && chtag -tc 819 "${FIFO_PIPE_STDOUT}" && addCleanupTrapCmd "rm -rf ${FIFO_PIPE_STDOUT}" +mkfifo "${FIFO_PIPE_STDOUT}" && chtag -tc 819 "${FIFO_PIPE_STDOUT}" +addCleanupTrapCmd "rm -rf ${FIFO_PIPE_STDOUT}" cd "${rootfs}" && pax -rw -p p "." "${promotefs}" 2>>"${FIFO_PIPE_STDOUT}" & while read OUTMSG; do printDebug "Parsing output: '${OUTMSG}'" diff --git a/bin/zopen-query b/bin/zopen-query index c25e20654..4fc9b3e1c 100755 --- a/bin/zopen-query +++ b/bin/zopen-query @@ -215,90 +215,6 @@ printDetailListEntries() exit 0 } -printInstalledEntries() -{ - needle="${2}" - - scrcols=$(getScreenCols) - [ "${details}" -eq 0 ] && numcols=4 || numcols=6 - colwidth=$((scrcols / numcols - 1)) - printVerbose "Screen width: ${scrcols}; colwidth:${colwidth}" - if [ -n "$1" ] && ! ${noheader}; then - if [ "${details}" -eq 0 ]; then - printf "${NC}${UNDERLINE}%-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s${NC}\n" "Package" "Version" "File" "Releaseline" - else - printf "${NC}${UNDERLINE}%-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s${NC}\n" "Package" "Version" "File" "Releaseline" "Expanded Size" "Quality" - fi - fi - printVerbose "Getting list of symlinks in the package install directory (that point to specific versions)" - installedPackages=$(cd "${ZOPEN_PKGINSTALL}" && zosfind ./*/. ! -name . -prune -type l) - printVerbose "Packages: ${installedPackages}" - echo "${installedPackages}" | xargs | tr ' ' '\n' | sort | while read repo; do - repo="${repo##*/}" - pkghome="${ZOPEN_PKGINSTALL}/${repo}/${repo}" - if [ ! -e "${pkghome}/.active" ]; then - printVerbose "Symlink '${repo}' in '${ZOPEN_PKGINSTALL}' is not active; skipping" - continue - fi - - # Check if the current package matches the needle (if provided) - if [ -n "${needle}" ] && ! echo "${repo}" | grep -q "${needle}"; then - printVerbose "Skipping '${repo}' as it does not match the needle '${needle}'" - continue - fi - - if ${noversion}; then - printf "%s\n" "${repo}" - continue - fi - - if [ -e "${pkghome}/.releaseinfo" ]; then - originalTag=$(cat "${pkghome}/.releaseinfo") - else - originalTag="N/A" - fi - if [ -e "${pkghome}/.version" ]; then - dotversion=$(cat "${pkghome}/.version") - else - dotversion="N/A" - fi - - releaseline="" - if [ -e "${pkghome}/.releaseline" ]; then - releaseline=$(cat "${pkghome}/.releaseline") - fi - if [ -z "${releaseline}" ]; then - releaseline="n/a" - fi - - printVerbose "Original tag: ${originalTag} for repo: ${repo}" - if [ -z "$1" ]; then - printInfo "${originalTag}" - else - fileversion="$(cd "${ZOPEN_PKGINSTALL}/${repo}/${repo}" > /dev/null 2>&1 && pwd -P | xargs basename)" - printf "%-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s" "${repo}" "${dotversion}" "${fileversion}" "${releaseline}" - if [ "${details}" -eq 1 ]; then - # Extra headers: disk size and quality - disksizestr=$(du "${ZOPEN_PKGINSTALL}/${repo}" | tail -n 1) - disksizestr=$(echo "${disksizestr}" | sed 's#\([0-9]*\).*#\1#') - disksize=$((disksizestr * 512)) - printf "%-${colwidth}d" "${disksize}" - - if [ -e "${pkghome}/test.status" ]; then - teststatus=$(cat "${pkghome}/test.status") - percentage=$(echo "${teststatus}" | sed 's/[^-]*-\([^%\.]*\).*/\1/') - printf "${NC}$(colorizepct "${percentage}")%-${colwidth}s${NC}" "${percentage}" - else - printf "${NC}${RED}%-${colwidth}s${NC}" "No tests" - fi - fi - - printf "\n" - fi - done - exit 0 -} - whatProvides() { needle="$1" diff --git a/include/common.sh b/include/common.sh index a27ec2660..9bea3000b 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1836,6 +1836,32 @@ createDependancyGraph(){ createDependancyGraph "${invalidPortAssetFile}" } +checkPreReqs(){ + pkg=$1 + metadata=$2 +[ -z "${pkg}" ] && return 12 +# shellcheck disable=SC2154 +if ! "${bypassPrereqs:-false}"; then + systemPrereqs=$(echo "${metadata}" | jq -r '.product.system_prereqs // empty' 2>/dev/null) + systemPrereqs="${systemPrereqs} zos24" # zos24 should be min requirement - always add it + if [ -n "${systemPrereqs}" ]; then + if [ ! -d "${ZOPEN_SYSTEM_PREREQS_DIR}" ]; then + printWarning "${ZOPEN_SYSTEM_PREREQS_DIR} does not exist. You should upgrade meta. Bypassing prereq check." + fi + for prereq in $(echo "${systemPrereqs}" | xargs | tr ' ' '\n' | sort -u); do + printHeader "Checking system pre-req requirement '${prereq}'" + if [ -e "${ZOPEN_SYSTEM_PREREQS_DIR}/${prereq}" ]; then + if ! /bin/sh -c "${ZOPEN_SYSTEM_PREREQS_DIR}/${prereq}"; then + printError "Failed system pre-req check '${prereq}'. If you wish to bypass this, install with --bypass-prereq-checks" + fi + else + printWarning "${ZOPEN_SYSTEM_PREREQS_DIR}/${prereq} does not exist. You should upgrade meta. Bypassing prereq check." + fi + done + fi +fi + +} # addToInstallGraph # Finds appropriate metadata for the specified port(s) and # includes that in the installation file @@ -1853,6 +1879,9 @@ addToInstallGraph(){ if ! getPortMetaData "${portRequested}" "${invalidPortAssetFile}"; then continue fi + if ! preReqFailed=$(checkPreReqs "${asset}"); then + echo "${preReqFailed}" >> "${invalidPortAssetFile}" + fi ## Merge asset into output file - note the lack of inline file edit hence the mv installList=$(echo "${installList}" | jq ".installqueue += [{\"portname\":\"${validatedPort}\", \"asset\":${asset}, \"installtype\":\"${installtype}\"}]") done diff --git a/include/scriptlets/installPre/systemPreReqCheck b/include/scriptlets/preReqCheck/systemPreReqCheck similarity index 100% rename from include/scriptlets/installPre/systemPreReqCheck rename to include/scriptlets/preReqCheck/systemPreReqCheck From 1a9e4d2cb75e91315e809748fae984a476c0ca4f Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 29 Oct 2024 12:41:29 +0000 Subject: [PATCH 033/179] Minor bug fixes --- bin/zopen-init | 13 +++++++++---- bin/zopen-upgrade | 6 +++--- include/common.sh | 35 ++++++++++++++++++++++++++++------- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index 6d4c3e957..0bd3edb5f 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -510,7 +510,7 @@ generateDefaultRepository() [ ! -e "${rootfs}/etc/zopen/repos.d" ] && mkdir -p "${rootfs}/etc/zopen/repos.d" if [ -z "${paxrepourl}" ]; then - paxrepourl="http://raw.githubusercontent.com/ZOSOpenTools/meta/main/docs/api/zopen_releases.json" + paxrepourl="http://raw.githubusercontent.com/${ZOPEN_ORGNAME}/meta/main/docs/api/zopen_releases.json" fi repotype="${paxrepourl%://*}" @@ -567,7 +567,7 @@ generateAnalyticsConfiguration() "removes": [] } zz - if jqrc=$(jqw '.' $jsonConfigWorking > $jsonConfig); then + if jqrc=$(jqw '.' "${jsonConfigWorking}" > "${jsonConfig}"); then rm "${jsonConfigWorking}" else printSoftError "${jqrc}" @@ -576,8 +576,13 @@ zz registerFileSystem "${UUID}" "${isHostIBM}" "${isBot}" - if [ -n "$ZOPEN_ROOTFS" ] && [ -n "$ZOPEN_LOG_PATH" ] && [ -f "$ZOPEN_LOG_PATH/audit.log" ]; then - processAnalyticsFromLogFile + if [ -n "$ZOPEN_ROOTFS" ]; then + if [ -z "$ZOPEN_LOG_PATH" ]; then + ZOPEN_LOG_PATH="${ZOPEN_ROOTFS}/var/log/zopen" + fi + if [ -f "$ZOPEN_LOG_PATH/audit.log" ]; then + processAnalyticsFromLogFile + fi fi } diff --git a/bin/zopen-upgrade b/bin/zopen-upgrade index 0680f6353..3bcdec4e6 100755 --- a/bin/zopen-upgrade +++ b/bin/zopen-upgrade @@ -65,11 +65,9 @@ chosenRepos="" while [ $# -gt 0 ]; do case "$1" in "--yes" | "-y") + # shellcheck disable=SC2034 yesToPrompts=true # Automatically answer 'yes' to any questions ;; - "--no-set-active") - setactive=false # Install package as normal but keep existing installation as active - ;; "-h" | "--help" | "-?") printHelp "${args}" exit 0 @@ -82,7 +80,9 @@ while [ $# -gt 0 ]; do debug=true ;; "--xdebug") + # shellcheck disable=SC2034 verbose=true + # shellcheck disable=SC2034 debug=true xdebug=true ;; diff --git a/include/common.sh b/include/common.sh index 9bea3000b..5cc880abd 100755 --- a/include/common.sh +++ b/include/common.sh @@ -15,15 +15,19 @@ zopenInitialize() if [ -z "${ZOPEN_DONT_PROCESS_CONFIG}" ]; then processConfig fi + + ZOPEN_ORGNAME="zopencommunity" + ZOPEN_GITHUB="https://github.com/${ZOPEN_ORGNAME}" # shellcheck disable=SC2034 ZOPEN_ANALYTICS_JSON="${ZOPEN_ROOTFS}/var/lib/zopen/analytics.json" - ZOPEN_JSON_CACHE_URL="https://raw.githubusercontent.com/zopencommunity/meta/main/docs/api/zopen_releases.json" + ZOPEN_JSON_CACHE_URL="https://raw.githubusercontent.com/${ZOPEN_ORGNAME}/meta/main/docs/api/zopen_releases.json" ZOPEN_JSON_CONFIG="${ZOPEN_ROOTFS}/etc/zopen/config.json" if [ -n "${INCDIR}" ]; then ZOPEN_SCRIPTLET_DIR="${INCDIR}/scriptlets" else ZOPEN_SCRIPTLET_DIR="${ZOPEN_ROOTFS}/usr/local/zopen/meta/meta/include/scriptlets" fi + } addCleanupTrapCmd(){ @@ -526,7 +530,6 @@ defineANSI() GRAY="${ESC}${CSI}37${SGR}" BOLD="${ESC}${CSI}1${SGR}" UNDERLINE="${ESC}${CSI}4${SGR}" - NC="${ESC}${CSI}0${SGR}" darkbackground bg=$? if [ $bg -ne 0 ]; then @@ -538,6 +541,10 @@ defineANSI() HEADERCOLOR="${MAGENTA}" WARNINGCOLOR="${MAGENTA}" fi + # The following should be the last ANSI declaration. With -x trace active, the ANSI + # codes might be interpreted by the terminal when outputing the command trace. Having + # NC as the last value ensures that the text is returned to normal + NC="${ESC}${CSI}0${SGR}" else # unset esc RED GREEN YELLOW BOLD UNDERLINE NC ANSION=false @@ -2116,7 +2123,8 @@ installFromPax() # to the actual repo package name at present. The repo name is in the # repo field so can extract from there instead #name=$(jq --raw-output '.product.name' "${metadatafile}") - name=$(jq --raw-output '.product.repo | match(".*/ZOSOpenTools/(.*)port").captures[0].string' "${metadatafile}") + # Ideally, use $reponame in the match but jq seems to have issues with that! + name=$(jq ---arg reponame "${ZOPEN_ORGNAME}" -raw-output '.product.repo | match(".*/zopencommunity/(.*)port").captures[0].string' "${metadatafile}") if ! processActionScripts "installPre" "${name}" "${metadatafile}"; then printError "Failed installation pre-requisite check(s) for '${name}'. Correct previous errors and retry command" fi @@ -2192,7 +2200,7 @@ EOF printDebug "Marking this version as installed" touch "${ZOPEN_PKGINSTALL}/${name}/${name}/.active" installedList="${name} ${installedList}" - syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_INSTALL},${CAT_PACKAGE}" "DOWNLOAD" "handlePackageInstall" "Installed package:'${name}';version:${downloadFileVer};install_dir='${baseinstalldir}/${installdirname}';" + syslog "${ZOPEN_LOG_PATH:-${ZOPEN_ROOTFS}/var/log}/audit.log" "${LOG_A}" "${CAT_INSTALL},${CAT_PACKAGE}" "DOWNLOAD" "handlePackageInstall" "Installed package:'${name}';version:${downloadFileVer};install_dir='${baseinstalldir}/${installdirname}';" addToInstallTracker "${name}" # Some installation have installation caveats installCaveat=$(jq -r '.product.install_caveats // empty' "${metadatafile}" 2>/dev/null) @@ -2283,18 +2291,31 @@ updatePackageDB() continue fi escapedJSONFile=$(mktempfile "escaped" "json") - addCleanupTrapCmd "rm -rf ${escapedJSONFile}" + # addCleanupTrapCmd "rm -rf ${escapedJSONFile}" stripControlCharacters "${metadataFile}" "${escapedJSONFile}" if [ ! -e "${pdb}" ]; then echo "[]" > "${pdb}" fi - mdj=$(jq '. as $metadata | .product.repo | match(".*/ZOSOpenTools/(.*)port").captures[0].string | [{(.):$metadata}]' \ + # Ideally, use $reponame in the match but jq seems to have issues with that! + mdj=$(jq --arg reponame "${ZOPEN_ORGNAME}" '. as $metadata | .product.repo | match(".*/zopencommunity/(.*)port").captures[0].string | [{(.):$metadata}]' \ "${escapedJSONFile}") + if [ -z "${mdj}" ]; then + # Try legacy repository + mdj=$(jq --arg reponame "${ZOPEN_ORGNAME}" '. as $metadata | .product.repo | match(".*/ZOSOpenTools/(.*)port").captures[0].string | [{(.):$metadata}]' \ + "${escapedJSONFile}") + fi + + if [ -z "${mdj}" ]; then + pkg=$(basename "${pkgdir}") + printWarning "Cannot locate metadata for installed package '${pkg}' at location '${metadataFile}'. Check file existence and permissions" + fi if ! jq --argjson mdj "${mdj}" '. += $mdj' \ "${pdb}" > \ "${pdb}.working"; then + [ -e "${pdb}.working" ] && "${pdb}.working" + [ -e "${pdb}" ] && mv -f "${pdb}" "${pdb}.broken" # Save for potential diagnostics printSoftError "Could not add metadata for '$(basename "${pkgdir}")' to install tracker." - printError "Run zopen --re-init to attempt database regeneration and zopen install --reinstall $(basename "${pkgdir}")" + printError "Run 'zopen init --refresh' to attempt database regeneration and re-run command." fi mv "${pdb}.working" "${pdb}" done From 5b19515ce0be4e71ab01cf6ec0208fbbd61c0711 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 13 Jan 2025 15:49:51 +0000 Subject: [PATCH 034/179] Scriptlet processing updates --- bin/zopen-alt | 9 ++- bin/zopen-init | 8 --- bin/zopen-install | 72 ++----------------- bin/zopen-upgrade | 6 +- include/common.sh | 176 ++++++++++++++++++++++++++-------------------- 5 files changed, 114 insertions(+), 157 deletions(-) diff --git a/bin/zopen-alt b/bin/zopen-alt index 646bb5127..cd093d233 100755 --- a/bin/zopen-alt +++ b/bin/zopen-alt @@ -55,8 +55,13 @@ mergeNewVersion() { package="$1" newver="$2" - processActionScripts "transactionPre" - processActionScripts "installPre" # We are effectively re-installing + # We are effectively re-installing so run scriptlets + if ! processActionScripts "transactionPre"; then + exit 1 + fi + if ! processActionScripts "installPre"; then + exit 1 + fi [ -z "${package}" ] && printError "Internal error; no package name provided to merge." [ -d "${ZOPEN_PKGINSTALL}/${package}/${newver}" ] || printError "Version '${newver}' was not available to set as current" if [ -e "${ZOPEN_PKGINSTALL}/${package}/${package}" ]; then diff --git a/bin/zopen-init b/bin/zopen-init index 0bd3edb5f..2c40034ff 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -719,21 +719,13 @@ deinit() fi } -echo "DBG!! ZOPEN_ROOTFS=${ZOPEN_ROOTFS}" init -echo "DBG!! ZOPEN_ROOTFS=${ZOPEN_ROOTFS}" generateFileSystem -echo "DBG!! ZOPEN_ROOTFS=${ZOPEN_ROOTFS}" ! ${refresh} && installPrereqs -echo "DBG!! ZOPEN_ROOTFS=${ZOPEN_ROOTFS}" generateJsonConfiguration -echo "DBG!! ZOPEN_ROOTFS=${ZOPEN_ROOTFS}" generateDefaultRepository -echo "DBG!! ZOPEN_ROOTFS=${ZOPEN_ROOTFS}" generateAnalyticsConfiguration -echo "DBG!! ZOPEN_ROOTFS=${ZOPEN_ROOTFS}" ! ${refresh} && updateBootstrappedTools -echo "DBG!! ZOPEN_ROOTFS=${ZOPEN_ROOTFS}" [ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || updatePackageDB updateBootstrappedTools deinit diff --git a/bin/zopen-install b/bin/zopen-install index 23b2a5095..1bba3a7a9 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -x # # Install utility for zopen community - https://github.com/zopencommunity # @@ -52,71 +52,7 @@ gpgCleanup() { [ -d "$TMP_GPG_DIR" ] && rm -rf "$TMP_GPG_DIR" } -verifySignatureOfPax() -{ - printInfo "- Performing GPG signature verification of pax file..." - # Extracting values and checking for errors - if ! FILE_TO_VERIFY=$(jq -e -r '.product.pax' "${metadataFile}"); then - printError "Failed to extract 'pax' from ${metadataFile}" >&2 - fi - - if ! SIGNATURE=$(jq -e -r '.product.signature' "${metadataFile}"); then - printVerbose "Failed to extract 'signature' from ${metadataFile}" >&2 - return - fi - - if ! PUBLIC_KEY=$(jq -e -r '.product.public_key' "${metadataFile}"); then - printError "Failed to extract 'public_key' from ${metadataFile}" >&2 - fi - - - # Create a temporary directory for GPG keyring - TMP_GPG_DIR="$zopen_tmp_dir/zopen_gpg_verify_$LOGNAME_$$" - mkdir -p "$TMP_GPG_DIR" - - SIGNATURE_FILE="$zopen_tmp_dir/zopen_signedfile.$LOGNAME.$$.asc" - PUBLIC_KEY_FILE="$zopen_tmp_dir/zopen_scriptpubkey.$LOGNAME.$$.asc" - printf "%b" "$SIGNATURE" | tr -d '"' > "$SIGNATURE_FILE" - printf "%b" "$PUBLIC_KEY" | tr -d '"' > "$PUBLIC_KEY_FILE" - - startGPGAgent - - printVerbose "Importing public key..." - gpg_output=$(gpg --no-default-keyring --keyring "$TMP_GPG_DIR/pubring.kbx" --batch --yes --import "$PUBLIC_KEY_FILE" 2>&1) - if [ $? -ne 0 ]; then - gpgCleanup - printError "Importing public key failed. See output:\n$gpg_output.\n Verification aborted." - fi - printVerbose "$gpg_output" - - # Verify that the key was imported successfully - printVerbose "Checking if public key is imported..." - gpg_output=$(gpg --no-default-keyring --keyring "$TMP_GPG_DIR/pubring.kbx" --check-sigs 2>&1) - if [ $? -ne 0 ]; then - gpgCleanup - printError "Public key was not imported. See output:\n$gpg_output.\nVerification aborted." - fi - printVerbose "$gpg_output" - - # Verify the signature - printInfo "- Verifying the gpg signature..." - if [ ! -f "$SIGNATURE_FILE" ]; then - gpgCleanup - printError "Signature file does not exist. Please raise an issue." - fi - - gpg_output=$(gpg --no-default-keyring --keyring "$TMP_GPG_DIR/pubring.kbx" --verify "$SIGNATURE_FILE" "$FILE_TO_VERIFY" 2>&1) - printVerbose "$gpg_output" - if echo "$gpg_output" | grep -q "Good signature from"; then - gpgCleanup - printInfo "- Signature successfully verified." - else - gpgCleanup - printError "Verification failed. See the output:\n$gpg_output" - fi -} - -printSyntax() +printHelp() { cat << HELPDOC ${ME} is a utility to download/install a zopen community package. @@ -182,7 +118,7 @@ skipupgrade=false skipverify=false doNotInstallDeps=false all=false -yesToPrompts=falseskipupgrade +yesToPrompts=false bypassPrereqs=false force=false chosenRepos="" @@ -357,7 +293,7 @@ else echo "${installList}" | jq --raw-output '.installqueue | sort| .[] | .portname ' fi processRepoInstallFile - + printInfo "Installation complete." fi mutexFree "zopen" exit diff --git a/bin/zopen-upgrade b/bin/zopen-upgrade index 3bcdec4e6..44b74abee 100755 --- a/bin/zopen-upgrade +++ b/bin/zopen-upgrade @@ -174,10 +174,12 @@ else waitforpid ${ph} # Make sure it's finished writing to screen fi +# shellcheck disable=SC2154 if [ 0 -eq "$(echo "${installList}" | jq --raw-output '.installqueue| length')" ]; then - printInfo "- No packages to upgrade" + printInfo "- No available updates" else processRepoInstallFile + printInfo "Upgrade operation complete" fi - mutexFree "zopen" + diff --git a/include/common.sh b/include/common.sh index 5cc880abd..35e33b60d 100755 --- a/include/common.sh +++ b/include/common.sh @@ -233,7 +233,7 @@ mktempfile() # Create a temporary directory mktempdir() { - tempdir=$(mktempfile "$1") + tempdir=$(mktempfile "$1" "$2") [ ! -e "${tempdir}" ] && mkdir "${tempdir}" && addCleanupTrapCmd "rm -rf ${tempdir}" && echo "${tempdir}" } @@ -1843,32 +1843,6 @@ createDependancyGraph(){ createDependancyGraph "${invalidPortAssetFile}" } -checkPreReqs(){ - pkg=$1 - metadata=$2 -[ -z "${pkg}" ] && return 12 -# shellcheck disable=SC2154 -if ! "${bypassPrereqs:-false}"; then - systemPrereqs=$(echo "${metadata}" | jq -r '.product.system_prereqs // empty' 2>/dev/null) - systemPrereqs="${systemPrereqs} zos24" # zos24 should be min requirement - always add it - if [ -n "${systemPrereqs}" ]; then - if [ ! -d "${ZOPEN_SYSTEM_PREREQS_DIR}" ]; then - printWarning "${ZOPEN_SYSTEM_PREREQS_DIR} does not exist. You should upgrade meta. Bypassing prereq check." - fi - for prereq in $(echo "${systemPrereqs}" | xargs | tr ' ' '\n' | sort -u); do - printHeader "Checking system pre-req requirement '${prereq}'" - if [ -e "${ZOPEN_SYSTEM_PREREQS_DIR}/${prereq}" ]; then - if ! /bin/sh -c "${ZOPEN_SYSTEM_PREREQS_DIR}/${prereq}"; then - printError "Failed system pre-req check '${prereq}'. If you wish to bypass this, install with --bypass-prereq-checks" - fi - else - printWarning "${ZOPEN_SYSTEM_PREREQS_DIR}/${prereq} does not exist. You should upgrade meta. Bypassing prereq check." - fi - done - fi -fi - -} # addToInstallGraph # Finds appropriate metadata for the specified port(s) and # includes that in the installation file @@ -1886,9 +1860,6 @@ addToInstallGraph(){ if ! getPortMetaData "${portRequested}" "${invalidPortAssetFile}"; then continue fi - if ! preReqFailed=$(checkPreReqs "${asset}"); then - echo "${preReqFailed}" >> "${invalidPortAssetFile}" - fi ## Merge asset into output file - note the lack of inline file edit hence the mv installList=$(echo "${installList}" | jq ".installqueue += [{\"portname\":\"${validatedPort}\", \"asset\":${asset}, \"installtype\":\"${installtype}\"}]") done @@ -1952,42 +1923,49 @@ generateInstallGraph(){ if "${reinstall}"; then printVerbose "Not pruning already installed packages as reinstalling" else - pruneGraph + parseGraph fi } -pruneGraph() +parseGraph() { - printDebug "Pruning entries in graph if already installed" + printDebug "Parsing graph for valid entries" # shellcheck disable=SC2154 if "${downloadOnly}"; then - # Download the pax files, even if already installed as they are not - # being reinstalled so no prune required + # Download the pax files, even if already installed or if they would + # fail validation - the user wants the files downloaded for whatever reason return 0 fi - # Prune already installed packages at the requested level; compare the - # incoming file name against the port name, version and release already on the system - # - seems to be the easiest comparison since some data is not in zopen_release vs metadata.json - # and a local pax won't have a remote repo but should have a file name! - installed=$(zopen list --installed --details) - # Ignore the version string - it varies across ports so use name and build time as that - # should be unique enough - installed=$(echo "${installed}"| awk 'BEGIN{ORS = "," } {print "\"" $1 "@=@" $3 "\""}') - installed="[${installed%,}]" - - installList=$(echo "${installList}" | \ - jq --argjson installees "${installed}" \ - '.installqueue |= - map( - select(.asset.url | - capture(".*/(?.*)-(?[^-]*)\\.(?\\d{8}_\\d{6}?)\\.zos\\.pax\\.Z$") |.rel as $rel | .name as $name | - $installees | map( - .|capture("(?[^@]*)@=@(?\\d{8}_\\d{6}?)$")|.iname as $iname | .irel as $irel | - ($iname+"-"+$irel) == ($name+"-"+$rel) - ) | any == false - ) - )'\ - ) + if ! processActionScripts "parseGraphPre" "${installList}"; then + exit 1 + fi + + # Prune already installed packages at the requested level; compare the + # incoming file name against the port name, version and release already on the system + # - seems to be the easiest comparison since some data is not in zopen_release vs metadata.json + # and a local pax won't have a remote repo but should have a file name! + installed=$(zopen list --installed --details) + # Ignore the version string - it varies across ports so use name and build time as that + # should be unique enough + installed=$(echo "${installed}"| awk 'BEGIN{ORS = "," } {print "\"" $1 "@=@" $3 "\""}') + installed="[${installed%,}]" + + installList=$(echo "${installList}" | \ + jq --argjson installees "${installed}" \ + '.installqueue |= + map( + select(.asset.url | + capture(".*/(?.*)-(?[^-]*)\\.(?\\d{8}_\\d{6}?)\\.zos\\.pax\\.Z$") |.rel as $rel | .name as $name | + $installees | map( + .|capture("(?[^@]*)@=@(?\\d{8}_\\d{6}?)$")|.iname as $iname | .irel as $irel | + ($iname+"-"+$irel) == ($name+"-"+$rel) + ) | any == false + ) + )'\ + ) + if ! processActionScripts "parseGraphPost"; then + exit 1 + fi } spaceValidate(){ @@ -2081,6 +2059,8 @@ processRepoInstallFile(){ getInstallFile() { installurl="$1" + metadataJSONURL="$(dirname "${installurl}")/metadata.json" + metadataFile="$(basename "${installurl}").json" downloadToDir="${ZOPEN_ROOTFS}/var/cache/zopen" if $downloadOnly; then downloadToDir="." @@ -2092,9 +2072,14 @@ getInstallFile() else [ -e "${downloadToDir}" ] || mkdir -p "${downloadToDir}" [ -w "${downloadToDir}" ] || printError "No permission to save install file to '${downloadToDir}'. Check permissions and retry command." + printVerbose "Downloading installable file" if ! runAndLog "cd ${downloadToDir} && curlCmd --no-progress-meter -L ${installurl} -O ${redirectToDevNull}"; then printError "Could not download from ${installurl}. Correct any errors and potentially retry" fi + printVerbose "Downloading corresponding metadata" + if ! runAndLog "curlCmd -L '${metadataJSONURL}' -o '${metadataFile}'" "${redirectToDevNull}"; then + printError "Could not download from ${metadataJSONURL}. Correct any errors and potentially retry." + fi fi } @@ -2124,8 +2109,8 @@ installFromPax() # repo field so can extract from there instead #name=$(jq --raw-output '.product.name' "${metadatafile}") # Ideally, use $reponame in the match but jq seems to have issues with that! - name=$(jq ---arg reponame "${ZOPEN_ORGNAME}" -raw-output '.product.repo | match(".*/zopencommunity/(.*)port").captures[0].string' "${metadatafile}") - if ! processActionScripts "installPre" "${name}" "${metadatafile}"; then + name=$(jq --arg reponame "${ZOPEN_ORGNAME}" --raw-output '.product.repo | match(".*/zopencommunity/(.*)port").captures[0].string' "${metadatafile}") + if ! processActionScripts "installPre" "${name}" "${metadatafile}" "${pax}"; then printError "Failed installation pre-requisite check(s) for '${name}'. Correct previous errors and retry command" fi @@ -2235,34 +2220,56 @@ processActionScripts() shift # Drop the initial parameter case "${phase}" in - "installPost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/installPost";; - "removePost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/removePost";; - "transactionPost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/transactionPost";; "installPre") scriptDir="${ZOPEN_SCRIPTLET_DIR}/installPre";; + "installPost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/installPost";; "removePre") scriptDir="${ZOPEN_SCRIPTLET_DIR}/removePre";; + "removePost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/removePost";; "transactionPre") scriptDir="${ZOPEN_SCRIPTLET_DIR}/transactionPre";; + "transactionPost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/transactionPost";; + "parseGraphPre") scriptDir="${ZOPEN_SCRIPTLET_DIR}/parseGraphPre";; + "parseGraphPost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/parseGraphPost";; *) assertFailed "Invalid process action phase '${phase}'" esac printVerbose "Running script[s] from '${scriptDir}'" - scriptRc=$( - # Running in a sub-shell so the scripts do not directly affect the current - # environment - status is returned to scriptRc - [ -d "${scriptDir}" ] || return 0 # No script directory + + if [ ! -d "${scriptDir}" ]; then + printDebug "No script directory for phase: ${phase}" + return 0 + fi unset CDPATH; - cd "${scriptDir}" || exit # the subshell - find . -type f | while read scriptFile; do - if [ ! -x "${scriptFile}" ]; then - printWarning "Script '${scriptDir}/${scriptFile}' is not executable. Check permissions" + # shellcheck disable=SC2164 + cd "${scriptDir}" + scriptRcFile=$(mktempfile "actionscripts" ".err") + find . -type l | while read scriptFile; do + # Note: The script only needs to be readable as it is not executed as a + # separate process; only read permission required when sourced with '.' + if [ ! -r "${scriptFile}" ]; then + printWarning "Script '${scriptDir}/${scriptFile}' is not readable. Check permissions" continue fi - printVerbose "Running script '${scriptFile}'" - # Call each scriptlet with the remaining parameters passed to this function - if ! src=$("${scriptFile}" "$@"); then - exit "${src:=-1}" + printVerbose "Attempting to run script '${scriptFile}'" + # Call each scriptlet with the remaining parameters passed to this function as + # subshells so that any modification in the scripts does not affect the runtime + # Any errors, then output the status code as the final output + scriptOutput=$( + # shellcheck disable=SC1090 + # shellcheck disable=SC2240 # This works on z/OS's /bin/sh + . "${scriptFile}" "$@" 2>&1 + echo $? + ) + scriptRc=$(echo "${scriptOutput}" | tail -n 1) + echo "${scriptOutput}" | head -n -1 # Don't print last (status) line! + + if [ "${scriptRc}" -ne 0 ]; then + touch "${scriptRcFile}" # Needed to indicate error outside subshell + break fi done - ) - return "${scriptRc:-0}" + if [ -e "${scriptRcFile}" ]; then + rm -f "${scriptRcFile}" + return 1 + fi + return 0 # If we get to here, then none of the scripts returned a fatal error } # updatePackageDB @@ -2429,6 +2436,21 @@ jqfunctions() 'def r(dp):.*pow(10;dp)|round/pow(10;dp)' } +startGPGAgent() +{ + printInfo "- Starting gpg-agent..." + if ! gpg-agent --daemon --disable-scdaemon; then + printError "Error running gpg-agent command. Review error messages and retry command." + fi + # Wait a moment to ensure the gpg-agent has time to start + sleep 2 + # Check to confirm if gpg-agent started successfully + # shellcheck disable=SC2009 # pgrep not on z/OS currently! + if ! ps -ef | grep -v grep | grep "gpg-agent" > /dev/null; then + printError "Failed to start the gpg-agent. Reinstall or upgrade GPG using \"zopen install --reinstall gpg -y\" or \"zopen upgrade gpg -y\" command." + fi +} + # shellcheck disable=SC1091 . "${INCDIR}/analytics.sh" zopenInitialize From 116e7417dc81ca83f438eb619f8f5e0709f3f07d Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 13 Jan 2025 15:50:18 +0000 Subject: [PATCH 035/179] Scriptlets - including prereq and signature verification --- include/scriptlets/installPost/autopkgpurge | 2 + .../installPost/zopen-meta-init-refresh | 2 + .../scriptlets/installPre/systemPreReqCheck | 39 +++++ .../installPre/verifySignatureOfPax | 134 ++++++++++++++++++ .../scriptlets/preReqCheck/systemPreReqCheck | 24 ---- include/scriptlets/removePost/man-db | 8 +- .../scriptlets/transactionPost/list_caveats | 4 +- include/scriptlets/transactionPost/man-db | 6 +- .../transactionPost/zopen-clean-autocache | 4 +- .../transactionPost/zopen-config-list | 26 +++- 10 files changed, 211 insertions(+), 38 deletions(-) create mode 100755 include/scriptlets/installPre/systemPreReqCheck create mode 100644 include/scriptlets/installPre/verifySignatureOfPax delete mode 100755 include/scriptlets/preReqCheck/systemPreReqCheck mode change 100755 => 100644 include/scriptlets/removePost/man-db diff --git a/include/scriptlets/installPost/autopkgpurge b/include/scriptlets/installPost/autopkgpurge index bef3aa532..039de76bc 100755 --- a/include/scriptlets/installPost/autopkgpurge +++ b/include/scriptlets/installPost/autopkgpurge @@ -1,4 +1,6 @@ #!/bin/sh +this="autopkgpurge" +printVerbose "Running '${this}' script" pkg=$1 [ -z "${pkg}" ] && return if type zopen-config-helper >/dev/null 2>&1; then diff --git a/include/scriptlets/installPost/zopen-meta-init-refresh b/include/scriptlets/installPost/zopen-meta-init-refresh index af254b1e4..b0ec0d730 100755 --- a/include/scriptlets/installPost/zopen-meta-init-refresh +++ b/include/scriptlets/installPost/zopen-meta-init-refresh @@ -1,4 +1,6 @@ #!/bin/sh +this="meta-init-refresh" +printVerbose "Running '${this}' script" pkg=$1 [ -z "${pkg}" ] && return case "${pkg}" in diff --git a/include/scriptlets/installPre/systemPreReqCheck b/include/scriptlets/installPre/systemPreReqCheck new file mode 100755 index 000000000..2d1817dfa --- /dev/null +++ b/include/scriptlets/installPre/systemPreReqCheck @@ -0,0 +1,39 @@ +#!/bin/sh -x +this="systemPreReqCheck" +printVerbose "Running '${this}' script" +pkg=$1 +metadataFile=$2 + +[ -z "${pkg}" ] && return +# shellcheck disable=SC2154 +if ! ${bypassPrereqs}; then + systemPrereqs=$(jq -r '.product.system_prereqs // empty | map(.name) | join(" ")' "${metadataFile}" 2>/dev/null) + if [ -z "${systemPrereqs}" ]; then + systemPrereqs="${systemPrereqs} zos24" # set the min requirement as z/OS 2.4 + fi + if [ -n "$systemPrereqs" ]; then + ZOPEN_SYSTEM_PREREQ_SCRIPT="../../prereq.sh" + if [ ! -r "${ZOPEN_SYSTEM_PREREQ_SCRIPT}" ]; then + printSoftError "${ZOPEN_SYSTEM_PREREQ_SCRIPT} does not exist. Check file permissions and reinstall the meta package or reinitialize the zopen environment. If the error persists, open an issue." + return 1 + else + # shellcheck disable=SC1090 + . ${ZOPEN_SYSTEM_PREREQ_SCRIPT} + for prereq in $(echo "${systemPrereqs}" | xargs | tr ' ' '\n' | sort -u); do + printInfo "- Checking system pre-req requirement ${prereq}" + if command -V "${prereq}" >/dev/null 2>&1; then + if ! ( ${prereq} ); then + printSoftError "Failed system pre-req check \"${prereq}\". If you wish to bypass this, install with --bypass-prereq-checks" + return 1 + fi + else + printSoftError "Prereq \"${prereq}\" does not exist in ${ZOPEN_SYSTEM_PREREQ_SCRIPT}. Consider upgrading meta or open an issue if it persists." + return 1 + fi + done + fi + fi +else + syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE},${CAT_INSTALL}" "BYPASS" "handlePackageInstall" "Bypassing prereq checks ${systemPrereqs} for '${name}'." +fi + diff --git a/include/scriptlets/installPre/verifySignatureOfPax b/include/scriptlets/installPre/verifySignatureOfPax new file mode 100644 index 000000000..70e6470d8 --- /dev/null +++ b/include/scriptlets/installPre/verifySignatureOfPax @@ -0,0 +1,134 @@ +#!/bin/sh +this="verifySignatureOfPax" +printVerbose "Running '${this}' script" +pkg=$1 +metadataFile=$2 +paxFile=$3 + +printInfo "- Performing GPG signature verification for '${pkg}' pax file" +metadataVersion=$(jq -r '.version_scheme' "${metadataFile}" 2>/dev/null) +is_greater=$(echo "$metadataVersion > 0.1" | bc -l) + +if [ "$is_greater" -eq 1 ] && ! $skipverify; then + if ! command -v gpg> /dev/null; then + skipverify=false; + printWarning "GPG is not installed - package validation cannot be performed" + return + fi +fi + +# The following function will attempt to retrieve a value from the metadata for this +# package. It will attempt to find it in the metadata included inside the pax file +# and previously extracted and failing that, will fallback to checking the existence +# of the package in the cache [the metadata downloaded [might be]/is different] +extractFromMetaData(){ + key=$1 + mdf=$2 + if ! sRc=$(jq -e -r "${key}" "${mdf}"); then + printVerbose "Could not locate ${key} in the pax's metadata. Checking cache..." + mdf="${ZOPEN_ROOTFS}/var/cache/zopen/$(basename "$3").json" + if ! sRc=$(jq -e -r "${key}" "${mdf}"); then + printSoftError "Failed to extract '${key}' from package metadata" >&2 + return 1 + fi + fi + echo "${sRc}" && return 0 + +} +# Extracting values and checking for errors +# If this is a local pax file, then there might not be a .product.pax in the +# metadata; check for a corresponding .json file in the cache. +# Local paxes need to be installed with --skipverify until they contain the +# .product.pax (and .signature and .public_key) entry +#if ! FILE_TO_VERIFY=$(jq -e -r '.product.pax' "${metadataFile}"); then +# printVerbose "Could not locate .product.pax in the pax's metadata. Checking cache..." +# metadataFile="${ZOPEN_ROOTFS}/var/cache/zopen/${paxFile}.json" +# if ! FILE_TO_VERIFY=$(jq -e -r '.product.pax' "${metadataFile}"); then +# printSoftError "Failed to extract 'pax' from ${metadataFile}" >&2 +# return 1 +# fi +#fi +if ! FILE_TO_VERIFY=$(extractFromMetaData ".product.pax" "${metadataFile}" "${paxFile}"); then + return 1 +fi + +if [ ! "${FILE_TO_VERIFY}" = "$(basename "${paxFile}")" ]; then + printSoftError "Mismatch between pax file '${paxFile}' and metadata pax name '${FILE_TO_VERIFY}'" + return 1 +fi + +if ! SIGNATURE=$(extractFromMetaData ".product.signature" "${metadataFile}" "${paxFile}"); then + # Lack of a .signature indicates the package has not yet been signed so allow installation + printVerbose "No .signature found in the metadata for '${pkg}'; continuing installation" + return 0 +fi + +if ! PUBLIC_KEY=$(extractFromMetaData ".product.public_key" "${metadataFile}" "${paxFile}"); then + # Since we found the .signature above, not finding the public key is not good; there is an + # error with the metadata, either corrupted in download, in the pax or initial generation + printSoftError "The metadata for '${pkg}' is inconsistent. Clear caches and retry command." + return 1 +fi + +# Create a temporary directory for GPG keyring +SIGNATURE_FILE=$(mktempfile "signedfile" ".asc") +PUBLIC_KEY_FILE=$(mktempfile "scriptpubkey" ".asc") + +#TMP_GPG_DIR=$(mktempdir "gpg" "verify") +#SIGNATURE_FILE="${TMP_GPG_DIR}/signedfile.asc" +#PUBLIC_KEY_FILE="${TMP_GPG_DIR}/scriptpubkey.asc" +printf "%b" "${SIGNATURE}" | tr -d '"' > "${SIGNATURE_FILE}" +printf "%b" "$PUBLIC_KEY" | tr -d '"' > "${PUBLIC_KEY_FILE}" + +[ -e "${SIGNATURE_FILE}" ] && addCleanupTrapCmd "rm -rf ${SIGNATURE_FILE}" +[ -e "${PUBLIC_KEY}" ] && addCleanupTrapCmd "rm -rf ${PUBLIC_KEY}" + +if [ ! -f "${PUBLIC_KEY_FILE}" ]; then + printSoftError "Unable to locate created public key file" + return 1 +fi + +startGPGAgent + +printVerbose "Importing public key to keyring file..." +${KEYRING_FILE} +KEYRING_FILE=$(mktempfile "pubring" ".kbx") + + if ! gpg_output=$(gpg --no-default-keyring --keyring "${KEYRING_FILE}" --batch --yes --import "$PUBLIC_KEY_FILE" 2>&1); then + printSoftError "Importing public key failed. Details:" + printSoftError "${gpg_output}" + printSoftError "Verification aborted." + return 1 + fi + printVerbose "${gpg_output}" + + # Verify that the key was imported successfully + printVerbose "Checking if public key is imported..." + + if ! gpg_output=$(gpg --no-default-keyring --keyring "${KEYRING_FILE}" --check-sigs 2>&1); then + printSoftError "Public key was not imported. See output:\n${gpg_output}.\nVerification aborted." + return 1 + fi + + printVerbose "${gpg_output}" + + # Verify the signature + printInfo "- Verifying the gpg signature..." + if [ ! -f "${SIGNATURE_FILE}" ]; then + printSoftError "Signature file does not exist. Please raise an issue." + return 1 + fi + +# gpg_output=$(gpg --no-default-keyring --keyring "${KEYRING_FILE}" --verify "${SIGNATURE_FILE}" "${FILE_TO_VERIFY}" 2>&1) + gpg_output=$(gpg --no-default-keyring --keyring "${KEYRING_FILE}" --verify "${SIGNATURE_FILE}" "${paxFile}" 2>&1) + printVerbose "${gpg_output}" + if ! echo "${gpg_output}" | grep -q "Good signature from"; then + printSoftError "Verification failed. Details:" + printSoftError "${gpg_output}" + return 1 + fi + + printInfo "- Signature successfully verified." + return 0 + + diff --git a/include/scriptlets/preReqCheck/systemPreReqCheck b/include/scriptlets/preReqCheck/systemPreReqCheck deleted file mode 100755 index 179bf902e..000000000 --- a/include/scriptlets/preReqCheck/systemPreReqCheck +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -pkg=$1 -metadataFile=$2 -[ -z "${pkg}" ] && return -# shellcheck disable=SC2154 -if ! "${bypassPrereqs}"; then - systemPrereqs=$(jq -r '.product.system_prereqs // empty' "${metadataFile}" 2>/dev/null) - systemPrereqs="${systemPrereqs} zos24" # zos24 should be min requirement - always add it - if [ -n "${systemPrereqs}" ]; then - if [ ! -d "${ZOPEN_SYSTEM_PREREQS_DIR}" ]; then - printWarning "${ZOPEN_SYSTEM_PREREQS_DIR} does not exist. You should upgrade meta. Bypassing prereq check." - fi - for prereq in $(echo "${systemPrereqs}" | xargs | tr ' ' '\n' | sort -u); do - printHeader "Checking system pre-req requirement '${prereq}'" - if [ -e "${ZOPEN_SYSTEM_PREREQS_DIR}/${prereq}" ]; then - if ! /bin/sh -c "${ZOPEN_SYSTEM_PREREQS_DIR}/${prereq}"; then - printError "Failed system pre-req check '${prereq}'. If you wish to bypass this, install with --bypass-prereq-checks" - fi - else - printWarning "${ZOPEN_SYSTEM_PREREQS_DIR}/${prereq} does not exist. You should upgrade meta. Bypassing prereq check." - fi - done - fi -fi diff --git a/include/scriptlets/removePost/man-db b/include/scriptlets/removePost/man-db old mode 100755 new mode 100644 index b871a8fa9..f5d940cb7 --- a/include/scriptlets/removePost/man-db +++ b/include/scriptlets/removePost/man-db @@ -1,11 +1,13 @@ #!/bin/sh +this="man-db" +printVerbose "Running '${this}' script" if ! zopen list --installed | grep "^man-db\$"; then : # No man-db installed else - echo "- Updating man-db database after package removal..." + echo "- Updating man-db database after package changes..." if ! mandb >/dev/null 2>&1; then - printWarning "man-db update completed with non-zero return code." - printWarning "Re-run 'mandb' manually for additional information." + printSoftError "man-db update completed with non-zero return code." + printSoftError "Re-run 'mandb' manually for additional information." else echo "- Update of man-db complete." fi diff --git a/include/scriptlets/transactionPost/list_caveats b/include/scriptlets/transactionPost/list_caveats index a23349496..6e5baf9f7 100755 --- a/include/scriptlets/transactionPost/list_caveats +++ b/include/scriptlets/transactionPost/list_caveats @@ -1,5 +1,7 @@ #!/bin/sh +this="list_caveats" +printVerbose "Running '${this}' script" caveats_file="${ZOPEN_ROOTFS}/var/cache/install_caveats.tmp" -[ -e "${caveats_file}" ] || exit 0 +[ -e "${caveats_file}" ] || return 0 cat "${caveats_file}" rm "${caveats_file}" diff --git a/include/scriptlets/transactionPost/man-db b/include/scriptlets/transactionPost/man-db index febb5e00e..f5d940cb7 100755 --- a/include/scriptlets/transactionPost/man-db +++ b/include/scriptlets/transactionPost/man-db @@ -1,11 +1,13 @@ #!/bin/sh +this="man-db" +printVerbose "Running '${this}' script" if ! zopen list --installed | grep "^man-db\$"; then : # No man-db installed else echo "- Updating man-db database after package changes..." if ! mandb >/dev/null 2>&1; then - printWarning "man-db update completed with non-zero return code." - printWarning "Re-run 'mandb' manually for additional information." + printSoftError "man-db update completed with non-zero return code." + printSoftError "Re-run 'mandb' manually for additional information." else echo "- Update of man-db complete." fi diff --git a/include/scriptlets/transactionPost/zopen-clean-autocache b/include/scriptlets/transactionPost/zopen-clean-autocache index 6f7ba1a78..cade3f475 100755 --- a/include/scriptlets/transactionPost/zopen-clean-autocache +++ b/include/scriptlets/transactionPost/zopen-clean-autocache @@ -1,7 +1,7 @@ #!/bin/sh -printInfo "RUNNING AUTO CLEAN" +this="zopen-clean-autocache" +printVerbose "Running '${this}' script" if type zopen-config-helper >/dev/null 2>&1; then -printInfo "USING CONFIG-HELPER" isactive=$(zopen-config-helper --get autocacheclean) else isactive=1 # Default to true diff --git a/include/scriptlets/transactionPost/zopen-config-list b/include/scriptlets/transactionPost/zopen-config-list index f61da1637..6cbf4498f 100755 --- a/include/scriptlets/transactionPost/zopen-config-list +++ b/include/scriptlets/transactionPost/zopen-config-list @@ -1,16 +1,30 @@ #!/bin/sh +this="zopen-config-list" +printVerbose "Running '${this}' script" +cfgFile="${ZOPEN_ROOTFS}/etc/zopen-config" +cfgFileWorking="${ZOPEN_ROOTFS}/tmp/zopen-config.tmp" envFile="${ZOPEN_ROOTFS}/etc/zopen/zopen.env" envFileWorking="${envFile}.working" -# Running in a "clean" environment -if ! (env -i . "${ZOPEN_ROOTFS}"/etc/zopen-config --knv | sort > "${envFileWorking}" ); then - printError "Unable to generate zopen environment key/value pair" +# Running in a "clean" environment - this requires an executable version of the +# configuration file + +cp -f "${cfgFile}" "${cfgFileWorking}" +chmod +x "${cfgFileWorking}" + +if ! (env -i "${cfgFileWorking}" --knv | sort > "${envFileWorking}" ); then + printSoftError "Unable to generate zopen environment key/value pair" + return 1 +fi +if [ ! -e "${envFileWorking}" ]; then + printSoftError "Could not locate working file" + return 1 fi -[ -e "${envFileWorking}" ] || printError "Could not locate working file" if ! [ -e "${envFile}" ]; then mv "${envFileWorking}" "${envFile}" - exit 0 + return 0 fi if ! diff=$(diff "${envFileWorking}" "${envFile}"); then printSoftError "${diff}" - printError "Could not compute difference in new key-value pairs. Correct errors and run and redirect '${ZOPEN_ROOTFS}/etc/zopen-config knv | sort' manually." + printSoftError "Could not compute difference in new key-value pairs. Correct errors and run and redirect '${ZOPEN_ROOTFS}/etc/zopen-config knv | sort' manually." + return 1 fi From f69af4946adb1a35e5a92ddb93c8d131ff2660cf Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Jan 2025 10:16:26 +0000 Subject: [PATCH 036/179] Disable command trace --- bin/zopen-install | 2 +- include/scriptlets/installPre/systemPreReqCheck | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/zopen-install b/bin/zopen-install index 1bba3a7a9..bc8642e15 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -1,4 +1,4 @@ -#!/bin/sh -x +#!/bin/sh # # Install utility for zopen community - https://github.com/zopencommunity # diff --git a/include/scriptlets/installPre/systemPreReqCheck b/include/scriptlets/installPre/systemPreReqCheck index 2d1817dfa..82669bf92 100755 --- a/include/scriptlets/installPre/systemPreReqCheck +++ b/include/scriptlets/installPre/systemPreReqCheck @@ -1,4 +1,4 @@ -#!/bin/sh -x +#!/bin/sh this="systemPreReqCheck" printVerbose "Running '${this}' script" pkg=$1 From 9def9c1541d20fd640fd734f874c599cec87507d Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Jan 2025 15:48:10 +0000 Subject: [PATCH 037/179] Ensure bootstrapped Pinentry comes before gnugp --- bin/zopen-init | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index 2c40034ff..9062cf2fb 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -109,7 +109,7 @@ PKG_DIR="packages" CURL_PAX_LOCATION="${PKG_DIR}/${CURL_PAX}" JQ_PAX_LOCATION="${PKG_DIR}/${JQ_PAX}" GPG_PAX_LOCATION="${PKG_DIR}/${GPG_PAX}" -PINENTRY_PAX_LOCATION="${PKG_DIR}/${JQ_PAX}" +PINENTRY_PAX_LOCATION="${PKG_DIR}/${PINENTRY_PAX}" verbose=false isHostIBM=false @@ -590,7 +590,7 @@ installPrereqs() { printHeader "Installing zopen pre-requisites" # Comma-separated list of pax locations - PAX_LOCATIONS="$CURL_PAX_LOCATION,$JQ_PAX_LOCATION,${GPG_PAX_LOCATION},${PINENTRY_PAX_LOCATION}" + PAX_LOCATIONS="$CURL_PAX_LOCATION,$JQ_PAX_LOCATION,${PINENTRY_PAX_LOCATION},${GPG_PAX_LOCATION}" echo "$PAX_LOCATIONS" | tr ',' '\n' | while read -r pax; do printVerbose "- Check for shipped ${pax}" @@ -617,7 +617,7 @@ installPrereqs() done # Sourcing the environment from within the actual directories - for tool in "curl" "jq" "gnupg" "pinentry"; do + for tool in "curl" "jq" "pinentry "gnupg""; do printVerbose "Sourcing environment to trigger any setup required for ${tool}" tooldir=$(ls "${rootfs}/boot" | grep ${tool} | head -1) @@ -638,10 +638,10 @@ updateBootstrappedTools() { installOptions="${installOptions} --bypass-prereq-checks"; fi if ${refresh}; then - ${MYDIR}/zopen-install ${installOptions} -y curl jq gpg pinentry + ${MYDIR}/zopen-install ${installOptions} -y curl jq pinentry gpg toolInstall=$? else - ${MYDIR}/zopen-install ${installOptions} -y curl jq gpg pinentry meta + ${MYDIR}/zopen-install ${installOptions} -y curl jq pinentry gpg meta toolInstall=$? fi [ ${toolInstall} -ne 0 ] && printError "Unable to install curl, jq, gpg, pinentry and/or meta; see previous errors and retry the initilisation using the '--re-init' parameter" @@ -725,7 +725,6 @@ generateFileSystem generateJsonConfiguration generateDefaultRepository generateAnalyticsConfiguration -! ${refresh} && updateBootstrappedTools [ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || updatePackageDB updateBootstrappedTools deinit From 7280be99e60c863bd2fd52d56e01cc58ccd56052 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Jan 2025 15:58:21 +0000 Subject: [PATCH 038/179] Sourcing issue with gnupg & pinentry --- bin/zopen-init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/zopen-init b/bin/zopen-init index 9062cf2fb..3fa7d77aa 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -617,7 +617,7 @@ installPrereqs() done # Sourcing the environment from within the actual directories - for tool in "curl" "jq" "pinentry "gnupg""; do + for tool in "curl" "jq" "pinentry" "gnupg"; do printVerbose "Sourcing environment to trigger any setup required for ${tool}" tooldir=$(ls "${rootfs}/boot" | grep ${tool} | head -1) From 69e20c1604b66c52e0efcb734c1c2c801e57e4eb Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Jan 2025 19:49:39 +0000 Subject: [PATCH 039/179] Generate database after updating tools --- bin/zopen-init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/zopen-init b/bin/zopen-init index 3fa7d77aa..6198fa536 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -725,6 +725,6 @@ generateFileSystem generateJsonConfiguration generateDefaultRepository generateAnalyticsConfiguration -[ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || updatePackageDB updateBootstrappedTools +[ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || updatePackageDB deinit From 7e6cac1f639055318c2185e535d458349bf7194b Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Jan 2025 23:33:54 +0000 Subject: [PATCH 040/179] Add back in zopen-config generate code --- bin/zopen-init | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/zopen-init b/bin/zopen-init index 6198fa536..fc015447d 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -724,6 +724,7 @@ generateFileSystem ! ${refresh} && installPrereqs generateJsonConfiguration generateDefaultRepository +generateZopenConfig generateAnalyticsConfiguration updateBootstrappedTools [ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || updatePackageDB From cdc938ef576ee81b98a5f58a27d89cddcd4210d6 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Jan 2025 23:36:29 +0000 Subject: [PATCH 041/179] Handle metadata that still references legacy ZOSOpenTools --- include/common.sh | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/include/common.sh b/include/common.sh index 35e33b60d..137c00c05 100755 --- a/include/common.sh +++ b/include/common.sh @@ -20,6 +20,7 @@ zopenInitialize() ZOPEN_GITHUB="https://github.com/${ZOPEN_ORGNAME}" # shellcheck disable=SC2034 ZOPEN_ANALYTICS_JSON="${ZOPEN_ROOTFS}/var/lib/zopen/analytics.json" + # shellcheck disable=SC2034 ZOPEN_JSON_CACHE_URL="https://raw.githubusercontent.com/${ZOPEN_ORGNAME}/meta/main/docs/api/zopen_releases.json" ZOPEN_JSON_CONFIG="${ZOPEN_ROOTFS}/etc/zopen/config.json" if [ -n "${INCDIR}" ]; then @@ -2104,12 +2105,21 @@ installFromPax() printDebug "Installing from '${pax}'" metadatafile=$(extractMetadataFromPax "${pax}") - # Ideally we would use the following, but name does not always map - # to the actual repo package name at present. The repo name is in the - # repo field so can extract from there instead - #name=$(jq --raw-output '.product.name' "${metadatafile}") - # Ideally, use $reponame in the match but jq seems to have issues with that! - name=$(jq --arg reponame "${ZOPEN_ORGNAME}" --raw-output '.product.repo | match(".*/zopencommunity/(.*)port").captures[0].string' "${metadatafile}") + # Ideally we would use the following, + # name=$(jq --raw-output '.product.name' "${metadatafile}") + # but name does not always map to the actual repo package name at present! + # The repo name is in the.product.repo field so can extract from there instead. + # Note that at present some metadata might refer to the legacy repo ZOSOpenTools + # so fall back to that + printVerbose "Extracting product name from repo" + name=$(jq --arg reponame "${ZOPEN_ORGNAME}" --raw-output '.product.repo | match(".*/\($reponame)/(.*)port").captures[0].string' "${metadatafile}") + if [ -z "${name}" ] || [ "${name##*[^ ]*}" = "" ]; then + name=$(jq --arg reponame "ZOSOpenTools" --raw-output '.product.repo | match(".*/\($reponame)/(.*)port").captures[0].string' "${metadatafile}") + fi + if [ -z "${name}" ] || [ "${name##*[^ ]*}" = "" ]; then + printError "Unable to determine name from .product.repo in '${metadatafile}'. Check metadata is correct." + fi + if ! processActionScripts "installPre" "${name}" "${metadatafile}" "${pax}"; then printError "Failed installation pre-requisite check(s) for '${name}'. Correct previous errors and retry command" fi @@ -2298,7 +2308,7 @@ updatePackageDB() continue fi escapedJSONFile=$(mktempfile "escaped" "json") - # addCleanupTrapCmd "rm -rf ${escapedJSONFile}" + addCleanupTrapCmd "rm -rf ${escapedJSONFile}" stripControlCharacters "${metadataFile}" "${escapedJSONFile}" if [ ! -e "${pdb}" ]; then echo "[]" > "${pdb}" From 0c7af7530eca5a3486df2a70473981fd90bda235 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Jan 2025 23:38:33 +0000 Subject: [PATCH 042/179] GPG QoL changes as a scriptlet --- include/common.sh | 24 +++++++++++++++++-- .../installPre/verifySignatureOfPax | 18 +++++++------- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/include/common.sh b/include/common.sh index 137c00c05..5015d3c33 100755 --- a/include/common.sh +++ b/include/common.sh @@ -434,6 +434,7 @@ EOF } +# Wrap the curl command with additional standard parameters curlCmd() { # Take the list of parameters and concat them with @@ -444,6 +445,19 @@ curlCmd() curl ${ZOPEN_CURL_PARAMS} ${extra_curl_options} $* } +# Wrap the gpg command with additional standard parameters +gpgCmd() +{ + if ! gpgout=$(gpg --no-secmem-warning $*); then + if echo "${gpgout}" | grep -q "EDC5128I No such device"; then + printVerbose "Ignoring mmap issue not applicable on z/OS currently" + return 0 + fi + echo "${gpgout}" + return 1 + fi +} + validateReleaseLine() { echo "$1" | awk ' @@ -2448,8 +2462,14 @@ jqfunctions() startGPGAgent() { - printInfo "- Starting gpg-agent..." - if ! gpg-agent --daemon --disable-scdaemon; then + + # shellcheck disable=SC2009 # Not on z/OS currently + if ps -ef | grep [g]pg-agent; then + printInfo "- Re-using gpg-agent" + return + fi + printInfo "- Starting gpg-agent" + if ! gpg-agent --daemon --disable-scdaemon --no-secmem-warning; then printError "Error running gpg-agent command. Review error messages and retry command." fi # Wait a moment to ensure the gpg-agent has time to start diff --git a/include/scriptlets/installPre/verifySignatureOfPax b/include/scriptlets/installPre/verifySignatureOfPax index 70e6470d8..5cc32a848 100644 --- a/include/scriptlets/installPre/verifySignatureOfPax +++ b/include/scriptlets/installPre/verifySignatureOfPax @@ -28,7 +28,7 @@ extractFromMetaData(){ printVerbose "Could not locate ${key} in the pax's metadata. Checking cache..." mdf="${ZOPEN_ROOTFS}/var/cache/zopen/$(basename "$3").json" if ! sRc=$(jq -e -r "${key}" "${mdf}"); then - printSoftError "Failed to extract '${key}' from package metadata" >&2 + printVerbose "Failed to extract '${key}' from package metadata" >&2 return 1 fi fi @@ -49,6 +49,7 @@ extractFromMetaData(){ # fi #fi if ! FILE_TO_VERIFY=$(extractFromMetaData ".product.pax" "${metadataFile}" "${paxFile}"); then + printSoftError "Failed to extract '${key}' from package metadata" >&2 return 1 fi @@ -59,7 +60,7 @@ fi if ! SIGNATURE=$(extractFromMetaData ".product.signature" "${metadataFile}" "${paxFile}"); then # Lack of a .signature indicates the package has not yet been signed so allow installation - printVerbose "No .signature found in the metadata for '${pkg}'; continuing installation" + printVerbose "No .signature found in the metadata for '${pkg}; continuing installation" return 0 fi @@ -73,6 +74,8 @@ fi # Create a temporary directory for GPG keyring SIGNATURE_FILE=$(mktempfile "signedfile" ".asc") PUBLIC_KEY_FILE=$(mktempfile "scriptpubkey" ".asc") +addCleanupTrapCmd "rm -rf ${SIGNATURE_FILE}" +addCleanupTrapCmd "rm -rf ${PUBLIC_KEY}" #TMP_GPG_DIR=$(mktempdir "gpg" "verify") #SIGNATURE_FILE="${TMP_GPG_DIR}/signedfile.asc" @@ -80,8 +83,6 @@ PUBLIC_KEY_FILE=$(mktempfile "scriptpubkey" ".asc") printf "%b" "${SIGNATURE}" | tr -d '"' > "${SIGNATURE_FILE}" printf "%b" "$PUBLIC_KEY" | tr -d '"' > "${PUBLIC_KEY_FILE}" -[ -e "${SIGNATURE_FILE}" ] && addCleanupTrapCmd "rm -rf ${SIGNATURE_FILE}" -[ -e "${PUBLIC_KEY}" ] && addCleanupTrapCmd "rm -rf ${PUBLIC_KEY}" if [ ! -f "${PUBLIC_KEY_FILE}" ]; then printSoftError "Unable to locate created public key file" @@ -93,8 +94,9 @@ startGPGAgent printVerbose "Importing public key to keyring file..." ${KEYRING_FILE} KEYRING_FILE=$(mktempfile "pubring" ".kbx") +addCleanupTrapCmd "rm -rf ${KEYRING_FILE}" - if ! gpg_output=$(gpg --no-default-keyring --keyring "${KEYRING_FILE}" --batch --yes --import "$PUBLIC_KEY_FILE" 2>&1); then + if ! gpg_output=$(gpgCmd --no-default-keyring --keyring "${KEYRING_FILE}" --batch --yes --import "$PUBLIC_KEY_FILE" 2>&1); then printSoftError "Importing public key failed. Details:" printSoftError "${gpg_output}" printSoftError "Verification aborted." @@ -105,7 +107,7 @@ KEYRING_FILE=$(mktempfile "pubring" ".kbx") # Verify that the key was imported successfully printVerbose "Checking if public key is imported..." - if ! gpg_output=$(gpg --no-default-keyring --keyring "${KEYRING_FILE}" --check-sigs 2>&1); then + if ! gpg_output=$(gpgCmd --no-default-keyring --keyring "${KEYRING_FILE}" --check-sigs 2>&1); then printSoftError "Public key was not imported. See output:\n${gpg_output}.\nVerification aborted." return 1 fi @@ -119,8 +121,8 @@ KEYRING_FILE=$(mktempfile "pubring" ".kbx") return 1 fi -# gpg_output=$(gpg --no-default-keyring --keyring "${KEYRING_FILE}" --verify "${SIGNATURE_FILE}" "${FILE_TO_VERIFY}" 2>&1) - gpg_output=$(gpg --no-default-keyring --keyring "${KEYRING_FILE}" --verify "${SIGNATURE_FILE}" "${paxFile}" 2>&1) +# gpg_output=$(gpgCmd --no-default-keyring --keyring "${KEYRING_FILE}" --verify "${SIGNATURE_FILE}" "${FILE_TO_VERIFY}" 2>&1) + gpg_output=$(gpgCmd --no-default-keyring --keyring "${KEYRING_FILE}" --verify "${SIGNATURE_FILE}" "${paxFile}" 2>&1) printVerbose "${gpg_output}" if ! echo "${gpg_output}" | grep -q "Good signature from"; then printSoftError "Verification failed. Details:" From bd6e28c0865552567a54bdd73a5fa44e8a56ff90 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Jan 2025 23:43:30 +0000 Subject: [PATCH 043/179] Simplify and fix cleanup logic --- include/common.sh | 46 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/include/common.sh b/include/common.sh index 5015d3c33..040119799 100755 --- a/include/common.sh +++ b/include/common.sh @@ -5,11 +5,9 @@ zopenInitialize() { - # Create the cleanup pipeline and exit handler - the export grabs the - # program exit code (for EXIT) or the signal otherwise to return from - # the program (rather than the return code from the functions in the - # trap handler) - trap "eval export exitrc=\$?" EXIT INT TERM QUIT HUP + # Monitor the intial parent process; as subshells inherit + # variables, this will not be overridden due to param expansion + parentPid=${parentPid:-$$} defineEnvironment defineANSI if [ -z "${ZOPEN_DONT_PROCESS_CONFIG}" ]; then @@ -30,12 +28,44 @@ zopenInitialize() fi } - addCleanupTrapCmd(){ - newcmd="$1 >/dev/null 2>&1" + # Attempt to remove any redirects; rather than test, simpler to remove + # and re-add if present. + newcmd=$(echo "$cmd" | zossed -E "s/[ \t]*[1-9]*[<>][>|&]?[ \t]*[^ \t]*//g") + newcmd="${newcmd} >/dev/null 2>&1" + + tmpscriptfile="/tmp/zopen_trap.scr" + echo "${newcmd}" >> "${tmpscriptfile}" + + if [ $$ -eq "${parentPid}" ]; then + # Re-register handlers if already done; quicker than testing presensce + for trappedSignal in "EXIT" "INT" "TERM" "QUIT" "HUP"; do + trap cleanup "${trappedSignal}" + done + fi +} + +cleanup() { + printVerbose "Performing cleanup [in the parent process]" + if [ -f "${tmpscriptfile}" ] && [ -s "${tmpscriptfile}" ]; then + # Execute the commands in the cleanup file by sourcing it + # shellcheck disable=SC1090 + . "${tmpscriptfile}" + rm "${tmpscriptfile}" + else + printVerbose "No cleanup script to run" + fi + printElapsedTime info "${0}" ${fullProcessStartTime} +} + +addCleanupTrapCmd2(){ + # Attempt to remove any redirects; rather than test, simpler to remove + # and re-add if present. + newcmd=$(echo "$cmd" | zossed -E "s/[ \t]*[1-9]*[<>][>|&]?[ \t]*[^ \t]*//g") + newcmd="${newcmd} >/dev/null 2>&1" # Command Trace MUST be disabled as the output from this can become # interleaved with output when calling zopen sub-processes. - [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" + # Small timing window if the script is killed between the creation # and removal of the temporary file; would be easier if zos sh # didn't have a bug -trap can't be piped/redirected anywhere except From 80a195a3950b33e3c6bb13aaae0d441e52b00037 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 16 Jan 2025 00:02:35 +0000 Subject: [PATCH 044/179] Assume cleanup commands need redirection! --- include/common.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/common.sh b/include/common.sh index 040119799..e320fe269 100755 --- a/include/common.sh +++ b/include/common.sh @@ -31,8 +31,7 @@ zopenInitialize() addCleanupTrapCmd(){ # Attempt to remove any redirects; rather than test, simpler to remove # and re-add if present. - newcmd=$(echo "$cmd" | zossed -E "s/[ \t]*[1-9]*[<>][>|&]?[ \t]*[^ \t]*//g") - newcmd="${newcmd} >/dev/null 2>&1" + newcmd="$1 >/dev/null 2>&1" tmpscriptfile="/tmp/zopen_trap.scr" echo "${newcmd}" >> "${tmpscriptfile}" From 34ed26f276fc191fdc9b1546d719051eb0c6f95c Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 16 Jan 2025 00:30:02 +0000 Subject: [PATCH 045/179] Cleaner sub-process exits --- bin/zopen-install | 2 +- include/common.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/zopen-install b/bin/zopen-install index bc8642e15..df0b7bb08 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -296,4 +296,4 @@ else printInfo "Installation complete." fi mutexFree "zopen" -exit +exit 0 diff --git a/include/common.sh b/include/common.sh index e320fe269..575088b2a 100755 --- a/include/common.sh +++ b/include/common.sh @@ -54,7 +54,6 @@ cleanup() { else printVerbose "No cleanup script to run" fi - printElapsedTime info "${0}" ${fullProcessStartTime} } addCleanupTrapCmd2(){ From df1cd163a94ffd4091dd93b699d0be6c8340c751 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 16 Jan 2025 09:17:21 +0000 Subject: [PATCH 046/179] Ensure updates to config.json remain --- bin/zopen-init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/zopen-init b/bin/zopen-init index fc015447d..471473f81 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -475,7 +475,7 @@ generateJsonConfiguration() if [ ! -e "${jsonConfig}" ]; then printInfo "Generating global configuration" - cat < "${jsonConfigWorking}" + cat < "${jsonConfig}" { "release_line": "${releaseLine:-STABLE}", "num_rm_procs": ${rm_fileprocs:-5}, From d7ca7c7511ddd4763fa611bc34bef66eaaf5d00f Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 16 Jan 2025 12:52:32 +0000 Subject: [PATCH 047/179] More cleanup of files --- include/common.sh | 4 ++-- include/scriptlets/installPre/verifySignatureOfPax | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/common.sh b/include/common.sh index 575088b2a..0e6109813 100755 --- a/include/common.sh +++ b/include/common.sh @@ -31,7 +31,7 @@ zopenInitialize() addCleanupTrapCmd(){ # Attempt to remove any redirects; rather than test, simpler to remove # and re-add if present. - newcmd="$1 >/dev/null 2>&1" + newcmd="$1" tmpscriptfile="/tmp/zopen_trap.scr" echo "${newcmd}" >> "${tmpscriptfile}" @@ -49,7 +49,7 @@ cleanup() { if [ -f "${tmpscriptfile}" ] && [ -s "${tmpscriptfile}" ]; then # Execute the commands in the cleanup file by sourcing it # shellcheck disable=SC1090 - . "${tmpscriptfile}" + . "${tmpscriptfile}" > /dev/null 2>&1 rm "${tmpscriptfile}" else printVerbose "No cleanup script to run" diff --git a/include/scriptlets/installPre/verifySignatureOfPax b/include/scriptlets/installPre/verifySignatureOfPax index 5cc32a848..df0e83b63 100644 --- a/include/scriptlets/installPre/verifySignatureOfPax +++ b/include/scriptlets/installPre/verifySignatureOfPax @@ -75,7 +75,7 @@ fi SIGNATURE_FILE=$(mktempfile "signedfile" ".asc") PUBLIC_KEY_FILE=$(mktempfile "scriptpubkey" ".asc") addCleanupTrapCmd "rm -rf ${SIGNATURE_FILE}" -addCleanupTrapCmd "rm -rf ${PUBLIC_KEY}" +addCleanupTrapCmd "rm -rf ${PUBLIC_KEY_FILE}" #TMP_GPG_DIR=$(mktempdir "gpg" "verify") #SIGNATURE_FILE="${TMP_GPG_DIR}/signedfile.asc" @@ -92,9 +92,9 @@ fi startGPGAgent printVerbose "Importing public key to keyring file..." -${KEYRING_FILE} KEYRING_FILE=$(mktempfile "pubring" ".kbx") addCleanupTrapCmd "rm -rf ${KEYRING_FILE}" +addCleanupTrapCmd "rm -rf ${KEYRING_FILE}~" # Also remove a potential lock file? if ! gpg_output=$(gpgCmd --no-default-keyring --keyring "${KEYRING_FILE}" --batch --yes --import "$PUBLIC_KEY_FILE" 2>&1); then printSoftError "Importing public key failed. Details:" From 87b44fe82fa29d9f84564d5cf4e6f5ce480a3592 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 16 Jan 2025 13:36:21 +0000 Subject: [PATCH 048/179] Propogate skipping of gpg validation --- include/scriptlets/installPre/verifySignatureOfPax | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/scriptlets/installPre/verifySignatureOfPax b/include/scriptlets/installPre/verifySignatureOfPax index df0e83b63..579113b86 100644 --- a/include/scriptlets/installPre/verifySignatureOfPax +++ b/include/scriptlets/installPre/verifySignatureOfPax @@ -5,6 +5,10 @@ pkg=$1 metadataFile=$2 paxFile=$3 +if ${skipverify}; then + printWarning "Package validation with GPG explicitly skipped" + return 0 +fi printInfo "- Performing GPG signature verification for '${pkg}' pax file" metadataVersion=$(jq -r '.version_scheme' "${metadataFile}" 2>/dev/null) is_greater=$(echo "$metadataVersion > 0.1" | bc -l) From 52bb3e37b509178a280d181f7a90bb60e1adadd9 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 21 Jan 2025 20:14:50 +0000 Subject: [PATCH 049/179] Additional error handling --- bin/zopen-install | 9 ++++++--- include/common.sh | 10 +++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/bin/zopen-install b/bin/zopen-install index df0b7bb08..217729998 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -292,8 +292,11 @@ else printInfo " - The following package(s) will be installed:" echo "${installList}" | jq --raw-output '.installqueue | sort| .[] | .portname ' fi - processRepoInstallFile - printInfo "Installation complete." + if ! processRepoInstallFile; then + exitrc=1 + else + printInfo "Installation complete." + fi fi mutexFree "zopen" -exit 0 +exit "${exitrc:-0}" diff --git a/include/common.sh b/include/common.sh index 0e6109813..eaf9fb00d 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2089,7 +2089,9 @@ processRepoInstallFile(){ installFile="${installurl##*/}" if [ ! "${installFile%.zos.pax.Z}" = "${installFile}" ]; then # Found zos.pax.Z format - installFromPax "${installFile}" + if ! installFromPax "${installFile}"; then + printError "Package installation terminated" + fi else printError "Unrecognised install file format" fi @@ -2132,7 +2134,7 @@ extractMetadataFromPax() if ! details=$(pax -rf "$1" -s "%[^/]*/%/tmp/%" '*/package.json'); then printSoftError "Could not extract package metadata from file '$1'." [ -n "${details}" ] && printSoftError "Details: ${details}" - exit 8 + return 1 else echo "/tmp/package.json" fi @@ -2146,7 +2148,9 @@ installFromPax() pax="${downloadToDir}/$1" printDebug "Installing from '${pax}'" - metadatafile=$(extractMetadataFromPax "${pax}") + if ! metadatafile=$(extractMetadataFromPax "${pax}"); then + return 1 + fi # Ideally we would use the following, # name=$(jq --raw-output '.product.name' "${metadatafile}") # but name does not always map to the actual repo package name at present! From 7c76653af5af11074c4a4c3bd414036df90be030 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 21 Jan 2025 20:18:33 +0000 Subject: [PATCH 050/179] Chtag downloaded metadata --- include/common.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/common.sh b/include/common.sh index eaf9fb00d..25164dbb2 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2124,6 +2124,12 @@ getInstallFile() printVerbose "Downloading corresponding metadata" if ! runAndLog "curlCmd -L '${metadataJSONURL}' -o '${metadataFile}'" "${redirectToDevNull}"; then printError "Could not download from ${metadataJSONURL}. Correct any errors and potentially retry." + else + if command -v chtag >/dev/null 2>&1; then + # Curl currently does not know on z/OS to set the text flag for text files + printVerbose "Metadata file downloaded, ensuring 'text' flag set for z/OS" + chtag -t "${metadataFile}" + fi fi fi } From b97e5928772f4e6637a3cab5a9994a10cdc49860 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 21 Jan 2025 20:34:26 +0000 Subject: [PATCH 051/179] Always hide curl progress meter. --- include/common.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/include/common.sh b/include/common.sh index 25164dbb2..7b5ac834a 100755 --- a/include/common.sh +++ b/include/common.sh @@ -467,9 +467,7 @@ curlCmd() { # Take the list of parameters and concat them with # any custom parameters the user requires in ZOPEN_CURL_PARAMS - if [ ! -t 1 ] || [ ! -t 2 ]; then - extra_curl_options="--no-progress-meter" - fi + extra_curl_options="--no-progress-meter" curl ${ZOPEN_CURL_PARAMS} ${extra_curl_options} $* } From 4fec9737342076a097b946f874a505ded7e01b6d Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 21 Jan 2025 21:11:18 +0000 Subject: [PATCH 052/179] Add help about all configuration variables --- bin/zopen-config-helper | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/bin/zopen-config-helper b/bin/zopen-config-helper index c7296f371..ff3f90915 100755 --- a/bin/zopen-config-helper +++ b/bin/zopen-config-helper @@ -45,8 +45,41 @@ Examples: zopen config --set is_collecting_stats false disable the is_collecting_stats functionality +Variables + The possible configuration values for the zopen environment are listed + below. The type of value expected is given, one of: + integer|boolean|string. + Where the value must be one of an enumerated set, the default value is + listed first. Validation of those values will be performed at runtime. + + autocacheclean [integer:0|1] + Tells zopen install to remove downloaded package files from the cache + to preserve space. + + autopkgpurge [integer:0|1] + Tells zopen install to remove any older version[s] of a package after + a successful install or upgrade + + is_collecting_stats [boolean:true|false] + Configures the zopen environment to collect anonymous statistcs about + package installations. + + num_rm_procs (deprecated) [integer:5] + Tells zopen remove how many subshells to use during removal processing + + override_zos_tools [boolean:false|true] + Tells zopen-config whether to prioritise native z/OS tools (such as in + /bin) on the \$PATH or to use zopen community ports in preference. + zopen community ports can still be referenced using an appropriate + command prefix; for example: gawk, zotfile, zotman + + release_line [string:STABLE|DEV] + Tells zopen install whether to install packages from the STABLE line + or from the DEV line. DEV packages will be newer but might exhibit + unexpected behaviour. The default is to use STABLE packages. + Notes: - Configuration options are not validated such that any key/value pairs can + Configuration options are not validated in that any key/value pairs can be added into the global configuration. 3rd-party utilities can store their global configuration into the zopen runtime environment store and use the zopen config tooling to set/retrieve values. Key names for stored properties From 64c1be7029a849fa500aa73ba4c842a2c9d1bd04 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 21 Jan 2025 22:34:37 +0000 Subject: [PATCH 053/179] For parity, reinstall only named package(s) [or use new option for deps] --- bin/zopen-install | 11 +++++++++++ bin/zopen-upgrade | 4 ++++ include/common.sh | 20 ++++++++++++++++---- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/bin/zopen-install b/bin/zopen-install index 217729998..e9c08b58e 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -69,6 +69,7 @@ Options: --noset-active unpackage onto the system but do not change the currently active version --reinstall reinstall currently installed package(s) + --reinstall-with-deps reinstall package(s) and their dependencies --release-line [stable|dev] the release line to install from --select select a version to install from available versions @@ -109,11 +110,15 @@ quiet=false selectVersion=false # shellcheck disable=SC2034 bypassPrereqs=false +# shellcheck disable=SC2034 setActive=true downloadOnly=false reinstall=false +reinstallDeps=false installOrUpgrade=false +# shellcheck disable=SC2034 nosymlink=false +# shellcheck disable=SC2034 skipupgrade=false skipverify=false doNotInstallDeps=false @@ -130,6 +135,12 @@ while [ $# -gt 0 ]; do # shellcheck disable=SC2034 reinstall=true # If package already installed, reinstall ;; + "--reinstall-with-deps") + # shellcheck disable=SC2034 + reinstallDeps=true # If reinstalling, also reinstall dependencies + # shellcheck disable=SC2034 + reinstall=true + ;; "--install-or-upgrade") # shellcheck disable=SC2034 installOrUpgrade=true # Upgrade package or install if not present diff --git a/bin/zopen-upgrade b/bin/zopen-upgrade index 44b74abee..fb63cfd19 100755 --- a/bin/zopen-upgrade +++ b/bin/zopen-upgrade @@ -58,7 +58,11 @@ setActive=true downloadOnly=false # shellcheck disable=SC2034 reinstall=false +# shellcheck disable=SC2034 +reinstallDeps=false +# shellcheck disable=SC2034 nosymlink=false +# shellcheck disable=SC2034 doNotInstallDeps=true yesToPrompts=false chosenRepos="" diff --git a/include/common.sh b/include/common.sh index 7b5ac834a..6a2d3b0e5 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1952,13 +1952,25 @@ generateInstallGraph(){ invalidPortAssetFile=$(mktempfile "invalid" "port") addCleanupTrapCmd "rm -rf ${invalidPortAssetFile}" addToInstallGraph "install" "${invalidPortAssetFile}" "${portsToInstall}" + # shellcheck disable=SC2154 - if ${doNotInstallDeps}; then - printVerbose "- Skipping dependency analysis" - else - # calculate dependancy graph + if ! ${doNotInstallDeps} && { (! ${reinstall} && ! ${reinstallDeps}) || (${reinstall} && ${reinstallDeps}); }; then + # doNotInstallDeps was not explicitly set and we are either not reinstalling(vanilla install) or we + # are reinstalling AND the reinstallDependencies flag is set; we do not want + # to reinstall a package and all it's dependencies by default, just the package itself + printVerbose "Calculating dependancy graph" createDependancyGraph "${invalidPortAssetFile}" + else + printVerbose "- Skipping dependency analysis" fi + + # shellcheck disable=SC2154 + #if ${doNotInstallDeps}; then + # printVerbose "- Skipping dependency analysis" + #else + # calculate dependancy graph + # createDependancyGraph "${invalidPortAssetFile}" + #fi # shellcheck disable=SC2154 if "${reinstall}"; then From eb0a64240f529e8d29160f4f7431a72efbb6c606 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 4 Feb 2025 19:01:30 +0000 Subject: [PATCH 054/179] QOL: Fix param parsing, more shellcheck issues and unneeded code --- bin/zopen-alt | 27 ++++++++++++++------------- bin/zopen-install | 16 +++++++++++++++- bin/zopen-remove | 6 +++--- bin/zopen-upgrade | 16 ++-------------- 4 files changed, 34 insertions(+), 31 deletions(-) diff --git a/bin/zopen-alt b/bin/zopen-alt index cd093d233..883c2334b 100755 --- a/bin/zopen-alt +++ b/bin/zopen-alt @@ -55,15 +55,17 @@ mergeNewVersion() { package="$1" newver="$2" - # We are effectively re-installing so run scriptlets + + [ -z "${package}" ] && printError "Internal error; no package name provided to merge." + newVerDir="${ZOPEN_PKGINSTALL}/${package}/${newver}" + [ -d "${newVerDir}" ] || printError "Version '${newver}' was not available to set as current" + + # We are effectively re-installing so run some scriptlets, noting that for the + # alternative to be available on the system, it must have passed various the + # checks in installPre (GPG/prereq) if ! processActionScripts "transactionPre"; then exit 1 fi - if ! processActionScripts "installPre"; then - exit 1 - fi - [ -z "${package}" ] && printError "Internal error; no package name provided to merge." - [ -d "${ZOPEN_PKGINSTALL}/${package}/${newver}" ] || printError "Version '${newver}' was not available to set as current" if [ -e "${ZOPEN_PKGINSTALL}/${package}/${package}" ]; then printVerbose "Removing main link" rm -rf "${ZOPEN_PKGINSTALL}/${package}/${package}" @@ -210,19 +212,14 @@ while [ $# -gt 0 ]; do case "$1" in "-s" | "--set") shift - [ $# -lt 2 ] && printError "Missing argument(s) for set option. Check program arguments" + [ $# -lt 1 ] && printError "Missing argument for 'set' option. Check program arguments" sett=true select=false - package="$1" - newver="$2" - shift + newver="$1" ;; "--select") select=true sett=false - shift - [ $# -lt 1 ] && printError "Missing argument for select option." - package="$1" ;; "-h" | "--help" | "-?") printHelp "${args}" @@ -248,6 +245,10 @@ while [ $# -gt 0 ]; do shift done +if ${select} && [ -z "${package}" ]; then + printError "Missing argument for select option." +fi + if ${sett}; then setAlt "${package}" "${newver}" elif [ -n "${package}" ]; then diff --git a/bin/zopen-install b/bin/zopen-install index e9c08b58e..3756288cd 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -237,9 +237,23 @@ for installRepo in ${potentials}; do [ -f "${installRepo}" ] && fileinstall=true && break done -if [ ! ${fileinstall} ]; then +# Sanity check for argument clashes +if ${selectVersion}; then + if ${all}; then + printError "Conflicting program arguments, --select cannot be used with --all" + elif ${fileinstall}; then + printError "Conflicting program argument, --select cannot be used with local file installations" + fi +fi +if ${fileinstall} && ${all}; then + printError "Conflicting program argument, --all cannot be used with local file installations" +fi + +if ! ${fileinstall}; then printVerbose "Using potentially remote files; ensuring configured for remote access" checkIfConfigLoaded +else + printVerbose "At least one local file was specified for installation" fi mutexReq "zopen" "zopen" diff --git a/bin/zopen-remove b/bin/zopen-remove index 2a57e5f62..a2c6dc2c3 100755 --- a/bin/zopen-remove +++ b/bin/zopen-remove @@ -62,12 +62,12 @@ removePackages() processActionScripts "removePre" printInfo "- Querying install status of package '${pkg}'" if [ ! -f "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}/.active" ]; then - printInfo "${NC}${YELLOW}Package '${pkg}' is not installed${NC}" + printInfo "${NC}${YELLOW}- Package '${pkg}' is not installed${NC}" else printInfo "- Away to remove '${pkg}'," if ! ${yesToPrompts}; then while true; do - printInfo "- Do you want to continue? [y/n/a]" + /bin/printf "- Do you want to continue [y/n/a]? " read continueInstall < /dev/tty case "${continueInstall}" in "y") break;; @@ -77,7 +77,7 @@ removePackages() esac done fi - printInfo "- Unmeshing from system" + printInfo "- Unmeshing '${pkg}' from system" version="unknown" if [ -e "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}/.releaseinfo" ]; then version=$(cat "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}/.releaseinfo") diff --git a/bin/zopen-upgrade b/bin/zopen-upgrade index fb63cfd19..f3e7a3683 100755 --- a/bin/zopen-upgrade +++ b/bin/zopen-upgrade @@ -104,18 +104,6 @@ done ${xdebug} && set -x && printVerbose "Enabled command execution trace" -if [ -z "${chosenRepos}" ]; then - printVerbose "No specific port to upgrade, upgrade all installed packages" - printInfo "- Querying installed packages" - progressHandler "pkgcheck" "- Query of installed packages complete" & - ph=$! - killph="kill -HUP ${ph}" - addCleanupTrapCmd "${killph}" - chosenRepos=$(zopen list --installed) - ${killph} 2>/dev/null # if the timer is not running, the kill will fail - waitforpid ${ph} # Make sure it's finished writing to screen -fi - checkIfConfigLoaded export SSL_CERT_FILE="${ZOPEN_CA}" @@ -130,8 +118,7 @@ downloadDir="${ZOPEN_ROOTFS}/var/cache/zopen" if [ ! -d "${downloadDir}" ]; then - mkdir -p "${downloadDir}" - if [ $? -gt 0 ]; then + if ! mkdir -p "${downloadDir}"; then printError "Could not create download directory: ${downloadDir}" fi fi @@ -141,6 +128,7 @@ printVerbose "Querying remote repo for latest package information" getRepos grfgRc=$? [ 0 -ne ${grfgRc} ] && exit ${grfgRc}; +# shellcheck disable=SC2034 installArray="" mutexReq "zopen" "zopen" From c75e77bf5a1376095ff0c8b367253c14ed2f9c94 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 5 Feb 2025 07:45:21 +0000 Subject: [PATCH 055/179] Read file into jq directly for processing, no piping into it --- include/common.sh | 205 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 160 insertions(+), 45 deletions(-) diff --git a/include/common.sh b/include/common.sh index 6a2d3b0e5..1e5d71c86 100755 --- a/include/common.sh +++ b/include/common.sh @@ -20,6 +20,7 @@ zopenInitialize() ZOPEN_ANALYTICS_JSON="${ZOPEN_ROOTFS}/var/lib/zopen/analytics.json" # shellcheck disable=SC2034 ZOPEN_JSON_CACHE_URL="https://raw.githubusercontent.com/${ZOPEN_ORGNAME}/meta/main/docs/api/zopen_releases.json" + ZOPEN_LATEST_RELEASE_JSON="https://raw.githubusercontent.com/${ZOPEN_ORGNAME}/meta/main/docs/api/zopen_releases_latest.json" ZOPEN_JSON_CONFIG="${ZOPEN_ROOTFS}/etc/zopen/config.json" if [ -n "${INCDIR}" ]; then ZOPEN_SCRIPTLET_DIR="${INCDIR}/scriptlets" @@ -843,7 +844,7 @@ mergeIntoSystem() printDebug "Calculating the offset path to store from root" offset=$(dirname "${versioneddir#"${rootfs}"/}") - version=$(basename ${versioneddir}) + version=$(basename "${versioneddir}") tmptime=$(date +%Y%m%d%H%M%S) processingDir="${rootfs}/tmp/zopen.${tmptime}" printDebug "Temporary processing dir evaluated to: ${processingDir}" @@ -1445,10 +1446,75 @@ getJSONCacheURL(){ esac } -downloadJSONCache() +updateJSONCaches() +{ + printVerbose "Ensuring cache directory exists" + cachedir="${ZOPEN_ROOTFS}/var/cache/zopen" + [ ! -e "${cachedir}" ] && mkdir -p "${cachedir}" + jsonCacheURL=$(getJSONCacheURL) + + printVerbose "Checking if the JSON_CACHE already downloaded in this session" + if [ -z "${JSON_CACHE}" ]; then + JSON_CACHE="${cachedir}/zopen_releases.json" + downloadJSONCacheIfExpired "${JSON_CACHE}" "${jsonCacheURL}" + fi + if [ -z "${JSON_LATEST_CACHE}" ]; then + JSON_LATEST_CACHE="${cachedir}/zopen_releases_latest.json" + latestReleaseURL="$(dirname "${jsonCacheURL}")/zopen_releases_latest.json" + downloadJSONCacheIfExpired "${JSON_LATEST_CACHE}" "${latestReleaseURL}" + fi +} + +downloadJSONCacheIfExpired() { - from_readonly=$1 + fileToCache="$1" + cacheUrl="$2" + cacheTimestamp="${fileToCache}.timestamp" + cacheTimestampCurrent="${fileToCache}.timestamp.current" + + # Need to check that we can read & write to the JSON timestamp cache files + if [ -e "${cacheTimestampCurrent}" ]; then + [ ! -w "${cacheTimestampCurrent}" ] || [ ! -r "${cacheTimestampCurrent}" ] && printError "Cannot access cache at '${cacheTimestampCurrent}'. Check permissions and retry request." + fi + if [ -e "${cacheTimestamp}" ]; then + [ ! -w "${cacheTimestamp}" ] || [ ! -r "${cacheTimestamp}" ] && printError "Cannot access cache at '${cacheTimestamp}'. Check permissions and retry request." + fi + if [ -e "${fileToCache}" ]; then + [ ! -w "${fileToCache}" ] || [ ! -r "${fileToCache}" ] && printError "Cannot access cache at '${JSON_CACHE}'. Check permissions and retry request." + fi + if ! curlCmd -f -L -s -I "${cacheUrl}" -o "${cacheTimestampCurrent}"; then + printError "Failed to obtain json cache timestamp from ${cacheUrl}." + fi + chtag -tc 819 "${cacheTimestampCurrent}" + + if [ -f "${fileToCache}" ] \ + && [ -f "${cacheTimestamp}" ] \ + && [ "$(grep 'Last-Modified' "${cacheTimestampCurrent}")" = "$(grep 'Last-Modified' "${cacheTimestamp}")" ]; then + # Metadata cache unchanged + return + fi + + printVerbose "Replacing old timestamp with latest." + mv -f "${cacheTimestampCurrent}" "${cacheTimestamp}" + + if ! curlCmd -f -L -s -o "${fileToCache}" "${cacheUrl}"; then + printError "Failed to obtain json cache from '${cacheUrl}'" + fi + chtag -tc 819 "${fileToCache}" + if [ ! -f "${fileToCache}" ]; then + printError "Could not download json cache from '${cacheUrl}" + fi +} + +downloadJSONCache() +{ + if ! updateJSONCaches; then + return 1 + else + return 0 + fi + ##TODORM>> if [ -z "${JSON_CACHE}" ]; then cachedir="${ZOPEN_ROOTFS}/var/cache/zopen" [ ! -e "${cachedir}" ] && mkdir -p "${cachedir}" @@ -1498,22 +1564,45 @@ downloadJSONCache() if [ ! -f "${JSON_CACHE}" ]; then printError "Could not download json cache from '${jsonCacheURL}" fi + ## < /dev/null 2>&1 +} + +#Deprecated getRepoReleases() { - downloadJSONCache $1 + updateJSONCaches repo="$1" - releases="$(jq -e -r '.release_data."'${repo}'"' "${JSON_CACHE}")" - if [ $? -ne 0 ]; then - printError "Could not get all releases for ${repo}" - fi + ##TODO + ##TDORM releases="$(jq -e -r '.release_data."'${repo}'"' "${JSON_CACHE}")" + ##TDORM if [ $? -ne 0 ]; then + ##TDORM printError "Could not get all releases for ${repo}" + ##TDORM fi } initDefaultEnvironment() @@ -1675,13 +1764,16 @@ getVersionedMetadata() requestedSubrelease=$(echo "${versioned}" | awk -F'.' '{print $4}') requestedVersion="${requestedMajor}\\\.${requestedMinor}\\\.${requestedPatch}\\\.${requestedSubrelease}" printDebug "Finding URL for latest release matching version prefix: requestedVersion: ${requestedVersion}" - releasemetadata=$(/bin/printf "%s" "${releases}" | jq -e -r '. | map(select(.assets[].name | test("'${requestedVersion}'")))[0]') + releasemetadata=$(jq --arg repo "${repo}" --arg requestedVersion "${requestedVersion}" \ + '.release_data.[$repo] | map(select(.assets[].version | test($requestedVersion)))[0]' "${JSON_CACHE}") } getTaggedMetadata() { printDebug "Explicit tagged version '${tagged}' specified. Checking for match" releasemetadata=$(/bin/printf "%s" "${releases}" | jq -e -r '.[] | select(.tag_name == "'"${tagged}"'")') + releasemetadata=$(jq --arg repo "${repo}" --arg requestedVersion "${requestedVersion}" \ + '.release_data.[$repo][] | select(.tag_name == $tag_name)' "${JSON_CACHE}") printDebug "Use quick check for asset to check for existence of metadata for specific messages" asset=$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]') if [ $? -ne 0 ]; then @@ -1696,13 +1788,17 @@ getSelectMetadata() # shellcheck disable=SC2154 kill -HUP "${gigph}" 2>/dev/null # if the timer is not running, the kill will fail waitforpid "${gigph}" # Make sure it's finished writing to screen - + + repo="$1" # Explicitly allow the user to select a release to install; useful if there are broken installs # as a known good release can be found, selected and pinned! printDebug "List individual releases and allow selection" - i=$(/bin/printf "%s" "${releases}" | jq -r 'length - 1') + i=$(jq --arg repo "${repo}" '.release_data.[$repo] | length - 1' "${JSON_CACHE}") printInfo "Versions available for install:" - /bin/printf "%s" "${releases}" | jq --raw-output 'to_entries | map("\(.key): \(.value.tag_name) - \(.value.assets[0].name) [\( ( .value.assets[0].expanded_size|tonumber)*1000 / (1024 * 1024) | ceil | . / 1000)Mb]")[]' + if ! jq --raw-output --arg repo "${repo}" \ + '.release_data.[$repo] | to_entries | map("\(.key): \(.value.tag_name) - \(.value.assets[0].name) [\( ( .value.assets[0].expanded_size|tonumber)*1000 / (1024 * 1024) | ceil | . / 1000)Mb]")[]' "${JSON_CACHE}"; then + printError "Unable to enumerate asset version strings" + fi printDebug "Getting user selection" valid=false while ! ${valid}; do @@ -1715,7 +1811,8 @@ getSelectMetadata() fi done printVerbose "Selecting item ${selection} from array" - releasemetadata="$(/bin/printf "%s" "${releases}" | jq -e -r ".[${selection}]")" + releasemetadata=$(jq --arg repo "${repo}" --arg selection "${selection}" \ + '.release_data.[$repo][$selection | tonumber]' "${JSON_CACHE}") } getReleaseLineMetadata() @@ -1726,16 +1823,19 @@ getReleaseLineMetadata() printError "Invalid releaseline specified: '${releaseLine}'; Valid values: DEV or STABLE" fi printDebug "Finding latest asset on the release line" - releasemetadata="$(/bin/printf "%s" "${releases}" | jq -e -r '. | map(select(.tag_name | startswith("'${releaseLine}'")))[0]')" + releasemetadata=$(jq --arg repo "${repo}" --arg releaseLine "${validatedReleaseLine}" \ + '.release_data.[$repo] | map(select(.tag_name | startswith($releaseLine)))[0]' "${JSON_CACHE}") printDebug "Use quick check for asset to check for existence of metadata" asset="$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')" if [ $? -ne 0 ]; then - printError "Could not find release-line ${releaseLine} for repo: ${repo}" + printError "Could not find release-line ${releaseLine} releases for repo: ${repo}" fi + unset "${releaseLine}" } calculateReleaseLineMetadata() { + repo="$1" printDebug "No explicit version/tag/releaseline, checking for pre-existing package&releaseline" if [ -n "${installedReleaseLine}" ]; then printDebug "Found existing releaseline '${installedReleaseLine}', restricting to only that releaseline" @@ -1761,17 +1861,17 @@ calculateReleaseLineMetadata() fi fi - printDebug "Parsing releases: ${releases}" # We have some situations that could arise # 1. the port being installed has no releaseline tagging yet (ie. no releases tagged STABLE_* or DEV_*) # 2. system is configured for STABLE but only has DEV stream available - # 3. system is configured for DEV but only has DEV stream available + # 3. system is configured for DEV but only has STABLE stream available # 4. the port being installed has got full releaseline tagging # The issue could arise that the user has switched the system from DEV->STABLE or vice-versa so package # stream mismatches could arise but in normal case, once a package is installed [that has releaseline tagging] # then that specific releaseline will be used printDebug "Finding any releases tagged with ${validatedReleaseLine} and getting the first (newest/latest)" - releasemetadata="$(/bin/printf "%s" "${releases}" | jq -e -r '. | map(select(.tag_name | startswith("'${validatedReleaseLine}'")))[0]')" + releasemetadata=$(jq --arg repo "${repo}" --arg releaseLine "${validatedReleaseLine}" \ + '.release_data.[$repo] | map(select(.tag_name | startswith($releaseLine)))[0]' "${JSON_CACHE}") printDebug "Use quick check for asset to check for existence of metadata" asset="$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')" @@ -1784,7 +1884,8 @@ calculateReleaseLineMetadata() # Case 2 & 3 printDebug "No releases on releaseline '${validatedReleaseLine}'; checking alternative releaseline" alt=$(echo "${validatedReleaseLine}" | awk ' /DEV/ { print "STABLE" } /STABLE/ { print "DEV" }') - releasemetadata="$(/bin/printf "%s" "${releases}" | jq -e -r '. | map(select(.tag_name | startswith("'${alt}'")))[0]')" + releasemetadata=$(jq --arg repo "${repo}" --arg releaseLine "${alt}" \ + '.release_data.[$repo] | map(select(.tag_name | startswith($releaseLine)))[0]' "${JSON_CACHE}") printDebug "Use quick check for asset to check for existence of metadata" asset="$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')" [ "${asset}" = "null" ] && asset="" # jq uses null, translate to sh's empty @@ -1792,8 +1893,8 @@ calculateReleaseLineMetadata() printDebug "Found a release on the '${alt}' release line so release tagging is active" if [ "DEV" = "${validatedReleaseLine}" ]; then # The system will be configured to use DEV packages where available but if none, use latest - printInfo "No specific DEV releaseline package, using latest available" - releasemetadata="$(/bin/printf "%s" "${releases}" | jq -e -r ".[0]")" + printInfo "No specific DEV releaseline package, using latest available release" + releasemetadata=$(jq --arg repo "${repo}" '.release_data.[$repo][0]' "${JSON_CACHE}") else printVerbose "The system is configured to only use STABLE releaseline packages but there are none" printInfo "No release available on the '${validatedReleaseLine}' releaseline." @@ -1801,7 +1902,7 @@ calculateReleaseLineMetadata() else # Case 1 - old package that has no release tagging yet (no DEV or STABLE), just install latest printVerbose "Installing latest release" - releasemetadata="$(/bin/printf "%s" "${releases}" | jq -e -r ".[0]")" + releasemetadata=$(jq --arg repo "${repo}" '.release_data.[$repo][0]' "${JSON_CACHE}") fi fi } @@ -1822,24 +1923,26 @@ getPortMetaData(){ invalidPortAssetFile="$2" printDebug "Removing any version (%) or tag (#) suffixes fron '${portRequested}" portName=$(echo "${portRequested}" | sed -e 's#%.*##' -e 's#=.*##') - validatedPort=$(echo "${repo_metadata}" | awk -vportName="${portName}" '$0 == portName {print}') - if [ -z "${validatedPort}" ]; then + + if ! isValidRepo "${portName}"; then echo "${portName}: no matching port found" >> "${invalidPortAssetFile}" return 1 fi + parseRepoName "${portRequested}" # To set the various status flags below - getRepoReleases "${validatedPort}" + getRepoReleases "${portName}" if [ -n "${versioned}" ]; then - getVersionedMetadata + getVersionedMetadata "${portName}" elif [ -n "${tagged}" ]; then - getTaggedMetadata + getTaggedMetadata "${portName}" elif # shellcheck disable=SC2154 ${selectVersion}; then - getSelectMetadata + selectVersion=false # Need to set this to prevent selection of dependencies + getSelectMetadata "${portName}" elif [ -n "${releaseLine}" ]; then - getReleaseLineMetadata + getReleaseLineMetadata "${portName}" else - calculateReleaseLineMetadata + calculateReleaseLineMetadata "${portName}" fi if [ -z "${releasemetadata}" ]; then echo "${portName}: metadata could not be found" >> "${invalidPortAssetFile}" @@ -1865,12 +1968,14 @@ getPortMetaData(){ # $2 an error file for outputing failures # return: 0 for success (output of pwd -P command) # 8 if error -createDependancyGraph(){ +createDependancyGraph() +{ invalidPortAssetFile=$1 && shift printDebug "Getting list of dependencies" dependencies=$(echo "${installList}" | jq --raw-output '.installqueue[] | select(.asset.runtime_dependencies | test("No dependencies") | not )| map(try(.runtime_dependencies |= split(" ")))| .[] | .runtime_dependencies[] ') printDebug "Removing any dependencies already on install queue" installing=$(echo "${installList}" | jq --raw-output '.installqueue[] | .portname') + # TODO: Use JQ to diff? missing=$(diffList "${installing}" "${dependencies}" ) if [ -z "${missing}" ]; then printDebug "All dependencies are in the install graph" @@ -1901,8 +2006,9 @@ addToInstallGraph(){ if ! getPortMetaData "${portRequested}" "${invalidPortAssetFile}"; then continue fi - ## Merge asset into output file - note the lack of inline file edit hence the mv - installList=$(echo "${installList}" | jq ".installqueue += [{\"portname\":\"${validatedPort}\", \"asset\":${asset}, \"installtype\":\"${installtype}\"}]") + ## Merge asset into JSON install list + installList=$(echo "${installList}" | \ + jq ".installqueue += [{\"portname\":\"${portName}\", \"asset\":${asset}, \"installtype\":\"${installtype}\"}]") done if [ -e "${invalidPortAssetFile}" ]; then printSoftError "The following ports cannot be installed: " @@ -1916,6 +2022,7 @@ addToInstallGraph(){ validateInstallList(){ installees="$1" # shellcheck disable=SC2086 # Using set -f disables globbing + printVerbose "Stripping any version/tagging" installees=$(set -f; echo ${installees} |awk -v ORS=, -v RS=' ' '{$1=$1; sub(/[=%].*/,x); print "\""$1"\""}') invalidPortList=$(jq -r --argjson needles "[${installees%%,}]" \ '.release_data| keys as $haystack | $needles | map(select(. as $needle | $haystack | index($needle)|not)) | .[]' "${JSON_CACHE}") @@ -1946,7 +2053,7 @@ generateInstallGraph(){ printDebug "Parsing list of packages to install and verifying validity" portsToInstall="$1" # start with the initial list portsToInstall=$(dedupStringList ' ' "${portsToInstall}") - repo_metadata="${repo_results}" + # Create the following file here to trigger cleanup - otherwise, multiple # tempfiles could be created depending on dependency graph depth invalidPortAssetFile=$(mktempfile "invalid" "port") @@ -2032,7 +2139,7 @@ spaceValidate(){ fi if ! ${yesToPrompts} || [ "${availableSpaceMB}" -lt "${spaceRequiredMB}" ]; then while true; do - printInfo "Do you want to continue? [y/n/a]" + /bin/printf "Do you want to continue [y/n/a]? " read continueInstall < /dev/tty case "${continueInstall}" in "y") break;; @@ -2167,21 +2274,29 @@ installFromPax() if ! metadatafile=$(extractMetadataFromPax "${pax}"); then return 1 fi + # Ideally we would use the following, # name=$(jq --raw-output '.product.name' "${metadatafile}") # but name does not always map to the actual repo package name at present! - # The repo name is in the.product.repo field so can extract from there instead. + # The repo name is in the.product.repo field so can extract from there instead - + # though this also has issues for some packages like NATS/nats ... # Note that at present some metadata might refer to the legacy repo ZOSOpenTools # so fall back to that - printVerbose "Extracting product name from repo" - name=$(jq --arg reponame "${ZOPEN_ORGNAME}" --raw-output '.product.repo | match(".*/\($reponame)/(.*)port").captures[0].string' "${metadatafile}") - if [ -z "${name}" ] || [ "${name##*[^ ]*}" = "" ]; then - name=$(jq --arg reponame "ZOSOpenTools" --raw-output '.product.repo | match(".*/\($reponame)/(.*)port").captures[0].string' "${metadatafile}") - fi - if [ -z "${name}" ] || [ "${name##*[^ ]*}" = "" ]; then - printError "Unable to determine name from .product.repo in '${metadatafile}'. Check metadata is correct." + if [ -n "${USEPRODNAME}" ]; then + printVerbose "Extracting product name from .product.repo" + # name=$(jq --raw-output '.product.name' "${metadatafile}") + else + printVerbose "Extracting product name from .product.repo" + name=$(jq --arg reponame "${ZOPEN_ORGNAME}" --raw-output '.product.repo | match(".*/\($reponame)/(.*)port").captures[0].string' "${metadatafile}") + if [ -z "${name}" ] || [ "${name##*[^ ]*}" = "" ]; then + name=$(jq --arg reponame "ZOSOpenTools" --raw-output '.product.repo | match(".*/\($reponame)/(.*)port").captures[0].string' "${metadatafile}") + fi + if [ -z "${name}" ] || [ "${name##*[^ ]*}" = "" ]; then + printError "Unable to determine name from .product.repo in '${metadatafile}'. Check metadata is correct." + fi fi + if ! processActionScripts "installPre" "${name}" "${metadatafile}" "${pax}"; then printError "Failed installation pre-requisite check(s) for '${name}'. Correct previous errors and retry command" fi @@ -2517,7 +2632,7 @@ startGPGAgent() return fi printInfo "- Starting gpg-agent" - if ! gpg-agent --daemon --disable-scdaemon --no-secmem-warning; then + if ! gpg-agent --daemon --disable-scdaemon; then printError "Error running gpg-agent command. Review error messages and retry command." fi # Wait a moment to ensure the gpg-agent has time to start From 78c6e79684fdaa132f3f5b0fcbfc530ddcd0118b Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 5 Feb 2025 07:46:50 +0000 Subject: [PATCH 056/179] Rework (simplify) progresshandler calls --- bin/zopen-alt | 12 ++- bin/zopen-clean | 17 +-- bin/zopen-install | 11 +- bin/zopen-promote | 100 +++++++++--------- bin/zopen-upgrade | 33 ++---- include/common.sh | 260 ++++++++++++++++++++++++---------------------- 6 files changed, 214 insertions(+), 219 deletions(-) diff --git a/bin/zopen-alt b/bin/zopen-alt index 883c2334b..19226cfa9 100755 --- a/bin/zopen-alt +++ b/bin/zopen-alt @@ -68,9 +68,17 @@ mergeNewVersion() fi if [ -e "${ZOPEN_PKGINSTALL}/${package}/${package}" ]; then printVerbose "Removing main link" - rm -rf "${ZOPEN_PKGINSTALL}/${package}/${package}" + if ! rm -rf "${ZOPEN_PKGINSTALL}/${package}/${package}:?"; then + printSoftError "Unable to remove previously set package link: ${ZOPEN_PKGINSTALL}/${package}/${package}" + printError "Check permissions and retry command." + fi + fi + if ! runLogProgress "mergeIntoSystem \"${package}\" \"${ZOPEN_PKGINSTALL}/${package}//${newver}\" \"${ZOPEN_ROOTFS}\"" \ + "Merging ${package} into symlink mesh" "Merged ${package} into symlink mesh"; then + printSoftError "Unexpected errors merging symlinks into mesh tree" + printError "Use zopen alt to select previous version to ensure known state" fi - mergeIntoSystem "${package}" "${ZOPEN_PKGINSTALL}/${package}/${newver}" "${ZOPEN_ROOTFS}" + ##TODORM mergeIntoSystem "${package}" "${ZOPEN_PKGINSTALL}/${package}/${newver}" "${ZOPEN_ROOTFS}" printVerbose "New version merged; checking for orphaned files from previous version" # This will remove any old symlinks or dirs that might have changed in an up/downgrade # as the merge process overwrites existing files to point to different version. If there was diff --git a/bin/zopen-clean b/bin/zopen-clean index 27252faba..a7cf27aa8 100755 --- a/bin/zopen-clean +++ b/bin/zopen-clean @@ -108,17 +108,20 @@ cleanDangling() exit 4 fi fi - progressHandler "spinner" "- Dangling link removal complete" & - ph=$! - killph="kill -HUP ${ph}" - addCleanupTrapCmd "${killph}" + + if ! runLogProgress "rmSymlinks" "Removing orphaned symlinks" "Removed orphaned symlinks" "trash"; then + printError "Unexpected error while removing symlinks. Correct errors and retry command." + fi + syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_FILE}" "CLEAN" "cleanDangling" "zopen system at '${ZOPEN_ROOTFS}' scanned for dangling symlinks" +} + +rmSymlinks() +{ + # shellcheck disable=SC2317 zosfind "${ZOPEN_ROOTFS}" -type l -exec test ! -e {} \; -print | while read sl; do printVerbose "Removing dangling symlink '${sl}'" rm -f "${sl}" done - ${killph} 2> /dev/null # if the timer is not running, the kill will fail - waitforpid ${ph} # Make sure it's finished writing to screen - syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_FILE}" "CLEAN" "cleanDangling" "zopen system at '${ZOPEN_ROOTFS}' scanned for dangling symlinks" } cleanPackageCache() diff --git a/bin/zopen-install b/bin/zopen-install index 3756288cd..b628c8bff 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -300,14 +300,9 @@ else installList=$(echo "$chosenRepos" | sed "s/@@/ /g") validateInstallList "${installList}" fi - printInfo "- Generating install graph" - progressHandler "spinner" "- Generated install graph" & - gigph=$! - killph="kill -HUP ${gigph}" - addCleanupTrapCmd "${killph}" - generateInstallGraph "${installList}" - ${killph} 2>/dev/null # if the timer is not running, the kill will fail - waitforpid ${gigph} # Make sure it's finished writing to screen + if ! generateInstallGraph "${installList}"; then + printError "Unable to generate install graph" + fi fi if [ 0 -eq "$(echo "${installList}" | jq --raw-output '.installqueue| length')" ]; then diff --git a/bin/zopen-promote b/bin/zopen-promote index 09d1d8d30..9b87d3230 100755 --- a/bin/zopen-promote +++ b/bin/zopen-promote @@ -242,45 +242,43 @@ else mkdir -p "${promotefs}" fi -printInfo "- Promoting from '${rootfs}' to '${promotefs}'..." - -progressHandler "spinner" "- Promoted from '${rootfs}' to '${promotefs}'." & -ph=$! -killph="kill -HUP ${ph}" -addCleanupTrapCmd "${killph}" -#cp -fRT "${rootfs}/" Fails to keep some relative symlinks correct - -printDebug "Need to pax old dir and unpax to new dest to keep symlink structure" -# Some files are set with permission -r--r--r-- which means they cannot be copied -# over without some work. Rather than search for these files, attempt to copy -# and catch the errors, forcibly re-copying the file over afterwards; saves -# iterating the entire file system twice - -printDebug "Generating temporary pipe" -FIFO_PIPE_STDOUT=$(mktempfile "promotestdout" ".pipe") -[ ! -p "${FIFO_PIPE_STDOUT}" ] || rm -f "${FIFO_PIPE_STDOUT}" -mkfifo "${FIFO_PIPE_STDOUT}" && chtag -tc 819 "${FIFO_PIPE_STDOUT}" -addCleanupTrapCmd "rm -rf ${FIFO_PIPE_STDOUT}" -cd "${rootfs}" && pax -rw -p p "." "${promotefs}" 2>>"${FIFO_PIPE_STDOUT}" & -while read OUTMSG; do - printDebug "Parsing output: '${OUTMSG}'" - destFile=$(echo "${OUTMSG}" | sed 's#.*FSUM7148.*"\(.*\)".*EDC5111I.*#\1#') - if [ "${destFile}" = "${OUTMSG}" ]; then - printSoftError "${OUTMSG}" - printError "Error cloning '${srcFile}' to '${destFile}'. Correct the reported error and retry command" - else - printDebug "Permission fail; trying to force copy of '${srcFile}' to '${destFile}'" - srcFile="${rootfs}${destFile#"${promotefs}"}" - printDebug "Copying sourcefile '${srcFile}' to '${destFile}'" - cp -f "${srcFile}" "${destFile}" +promoteFS(){ + #cp -fRT "${rootfs}/" Fails to keep some relative symlinks correct + + printDebug "Need to pax old dir and unpax to new dest to keep symlink structure" + # Some files are set with permission -r--r--r-- which means they cannot be copied + # over without some work. Rather than search for these files, attempt to copy + # and catch the errors, forcibly re-copying the file over afterwards; saves + # iterating the entire file system twice + + printDebug "Generating temporary pipe" + FIFO_PIPE_STDOUT=$(mktempfile "promotestdout" ".pipe") + [ ! -p "${FIFO_PIPE_STDOUT}" ] || rm -f "${FIFO_PIPE_STDOUT}" + mkfifo "${FIFO_PIPE_STDOUT}" && chtag -tc 819 "${FIFO_PIPE_STDOUT}" + addCleanupTrapCmd "rm -rf ${FIFO_PIPE_STDOUT}" + cd "${rootfs}" && pax -rw -p p "." "${promotefs}" 2>>"${FIFO_PIPE_STDOUT}" & + while read OUTMSG; do + printDebug "Parsing output: '${OUTMSG}'" + destFile=$(echo "${OUTMSG}" | sed 's#.*FSUM7148.*"\(.*\)".*EDC5111I.*#\1#') + if [ "${destFile}" = "${OUTMSG}" ]; then + printSoftError "${OUTMSG}" + printError "Error cloning '${srcFile}' to '${destFile}'. Correct the reported error and retry command" + else + printDebug "Permission fail; trying to force copy of '${srcFile}' to '${destFile}'" + srcFile="${rootfs}${destFile#"${promotefs}"}" + printDebug "Copying sourcefile '${srcFile}' to '${destFile}'" + cp -f "${srcFile}" "${destFile}" + + [ "$?" -ne 0 ] && printError "Could not promote environment; permission issue cloning '${srcFile}' to '${destFile}'. Check permissions and retry command." + fi - [ "$?" -ne 0 ] && printError "Could not promote environment; permission issue cloning '${srcFile}' to '${destFile}'. Check permissions and retry command." - fi + done < "${FIFO_PIPE_STDOUT}" + [ -n "${FIFO_PIPE_STDOUT}" ] && [ -e "${FIFO_PIPE_STDOUT}" ] && rm -f "${FIFO_PIPE_STDOUT}" +} -done < "${FIFO_PIPE_STDOUT}" -[ -n "${FIFO_PIPE_STDOUT}" ] && [ -e "${FIFO_PIPE_STDOUT}" ] && rm -f "${FIFO_PIPE_STDOUT}" -${killph} 2>/dev/null # if the timer is not running, the kill will fail -waitforpid ${ph} # Make sure it's finished writing to screen +if ! runLogProgress "promoteFS" "Promoting from '${rootfs}' to '${promotefs}'" "Promoted from '${rootfs}' to '${promotefs}'" "mirror"; then + printError "Unable to promote file system from '${rootfs}' to '${promotefs}'" +fi printVerbose "Grabbing pkginstall location from within promoted env" if [ -e "${promotefs}/etc/zopen/fstype" ]; then @@ -289,6 +287,19 @@ else printError "Unable to locate '${promotefs}/etc/zopen/fstype'; unrecognisable file system. Check permissions" fi +rmDanglingLinks(){ + flecnt=0 + + tmpfile=$(mktempfile "zopen_promote" ".pipe") + addCleanupTrapCmd "rm -rf ${tmpfile}" + + zosfind "${promotefs}" -type l -exec test ! -e {} \; -print > "${tmpfile}" + while IFS= read -r sl; do + printVerbose "Removing symlink '${sl}'" + # rm -f "${sl}" + flecnt=$(expr ${flecnt} + 1) + done < "${tmpfile}" +} if ! ${keepzopentooling}; then printDebug "Stripping zopen tools from cloned environment; removing meta packages" @@ -299,18 +310,9 @@ if ! ${keepzopentooling}; then # As packages can install to any subfolder of the zopen filesystem, need to traverse # along every path under that filesystem - even if '/''. As this is a cloned # environment, can just check for links related to "meta/meta*" - progressHandler "spinner" "- Dangling link removal complete" & - ph=$! - killph="kill -HUP ${ph}" - addCleanupTrapCmd "${killph}" - flecnt=0 - zosfind "${promotefs}" -type l -exec test ! -e {} \; -print | while read sl; do - printVerbose "Removing symlink '${sl}'" - rm -f ${sl} - flecnt=$(expr ${flecnt} + 1) - done - ${killph} 2>/dev/null # if the timer is not running, the kill will fail - waitforpid ${ph} # Make sure it's finished writing to screen + if ! runLogProgress "rmDanglingLinks" "Removing dangling links" "Removed dangling links"; then + printError "Unable to remove dangling links. Review any errors. Manual cleanup might be required" + fi syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_ZOPEN}" "PROMOTE" "mainline" "meta tooling checked for and removed from promoted environment; ${flecnt} link(s) removed" fi diff --git a/bin/zopen-upgrade b/bin/zopen-upgrade index f3e7a3683..fcb3de33d 100755 --- a/bin/zopen-upgrade +++ b/bin/zopen-upgrade @@ -135,35 +135,16 @@ mutexReq "zopen" "zopen" printDebug "Parsing list of packages to install and verifying validity" if [ -z "${chosenRepos}" ]; then - badportlist="" - for chosenRepo in $(echo "${chosenRepos}" | tr ',' ' '); do - printVerbose "Processing repo: ${chosenRepo}" - printDebug "Stripping any version (%), tag (#) or port suffixes and trim" - toolrepo=$(echo "${chosenRepo}" | sed -e 's#%.*##' -e 's#=.*##' -e 's#.*port##') - toolfound=$(echo "${repo_results}" | awk -vtoolrepo="${toolrepo}" '$0 == toolrepo {print}') - if [ "${toolfound}" = "${toolrepo}" ]; then - printVerbose "Adding '${chosenRepo}' to the install queue" - installArray=$(printf "%s\n%s" "${installArray}" "${chosenRepo}") - else - badportlist=$(printf "%s %s" "${badportlist}" "${toolrepo}") - fi - done - - if [ -z "${badportlist}" ]; then - printSoftError "The following requested port(s) do not exist:\n\t${badportlist}" + if ! chosenRepos=$(zopen list --installed); then + printSoftError "Unable to retrieve list of currently installed packages" + printError "Details: ${chosenRepos}" fi - upgradeList=$(jq --raw-output '.release_data| keys[]' "${JSON_CACHE}") - generateInstallGraph "${upgradeList}" else validateInstallList "${chosenRepos}" - printInfo "- Generating upgrade graph" - progressHandler "spinner" "- Generated upgrade graph" & - ph=$! - killph="kill -HUP ${ph}" - addCleanupTrapCmd "${killph}" - generateInstallGraph "${chosenRepos}" - ${killph} 2>/dev/null # if the timer is not running, the kill will fail - waitforpid ${ph} # Make sure it's finished writing to screen +fi + +if ! generateInstallGraph "${chosenRepos}"; then + printError "Unable to generate upgrade graph" fi # shellcheck disable=SC2154 diff --git a/include/common.sh b/include/common.sh index 1e5d71c86..ef447508f 100755 --- a/include/common.sh +++ b/include/common.sh @@ -170,15 +170,11 @@ showConfigParmWarning(){ # getParentProcess # returns the parent process for the specified process -# input: $1 - the pid to get the parent of -# return: 0 for error or parent process id getParentProcess() { - parent=$(ps -o ppid= -p "$1") - if parent=$(ps -o ppid= -p "$1"); then - return 0 - fi - return "${parent}" + [ -z "$1" ] && assertFailed "No process name given" + # Get Parent Pid (ppid), with no heading (='') and strip blanks (awk) + ps -o ppid= -p "$1" | awk '{$1=$1; print}' } # getCurrentVersionDir @@ -911,10 +907,66 @@ mergeIntoSystem() printDebug "Switching to previous cwd - current work dir was purged" cd "${currentDir}" || printError "Unable to change to '${currentDir}'" - printInfo "- Integration complete." return 0 } +rmSymlinksFSCheck(){ + # Slower method needed to analyse each link to see if it has + # become orphaned. Only relevent when removing a package as + # upgrades/alt-switching can supply a list of files + # Use sed to skip header line in .links file + # Note that the contents of the links file are ordered such that + # processing occurs depth-first; if, after removing orphaned symlinks, + # a directory is empty, then it can be removed. + nfiles=$(zossed '1d;$d' "${dotlinks}" | wc -l | tr -d ' ') + printDebug "Creating Temporary dirname file" + tempDirFile=$(mktempfile "unsymlink") + [ -e "${tempDirFile}" ] && rm -f "${tempDirFile}" >/dev/null 2>&1 + touch "${tempDirFile}" + tempTrash=$(mktempfile "unsymlink" "trash") + [ -e "${tempTrash}" ] && rm -f "${tempTrash}" >/dev/null 2>&1 + addCleanupTrapCmd "rm -rf ${tempDirFile}" + addCleanupTrapCmd "rm -rf ${tempTrash}" + printDebug "Using temporary file ${tempDirFile}" + while read filetounlink; do + filetounlink=$(echo "${filetounlink}" | zossed 's/\(.*\).symbolic.*/\1/') + filename="$filetounlink" + [ -z "${filetounlink}" ] && continue + filetounlink="${ZOPEN_ROOTFS}/${filetounlink}" + [ ! -e "${filetounlink}" ] && continue # If not there, can'e be removed! + if [ -d "${filetounlink}" ]; then + # Add to the directory queue for checking once files are gone if unique + ispresent=$(grep "^${filetounlink}[ ]*$" "${tempDirFile}") + if [ -z "${ispresent}" ]; then + echo " ${filetounlink} " >> "${tempDirFile}" + fi + elif [ -L "${filetounlink}" ]; then + if [ ! -f "${filetounlink}" ]; then + # the linked-to file no longer exists (ie. the symlink is dangling) + rm -f "${filetounlink}" > /dev/null 2>&1 + fi + else + echo "Unprocessable file: '${filetounlink}'" >> "${tempTrash}" + fi + done < /dev/null 2>&1 + fi + done +} + # The following function will remove any orphaned symlinks left after either: # - a different version has been installed (where the old symlinks are not reused for # that different version version ie. the file has been removed from updated version @@ -929,74 +981,17 @@ unsymlinkFromSystem() if [ -e "${dotlinks}" ]; then if [ -e "${newfilelist}" ]; then - printInfo "- Checking for file differences switching versions of '${pkg}'" - printDebug "Release change, so the list of changes to physically remove should be smaller" - progressHandler "spinner" "- Checked for file differences switching versions of '${pkg}'" & - ph=$! - killph="kill -HUP ${ph}" - addCleanupTrapCmd "${killph}" - obsoleteList=$(diffFile "${dotlinks}" "${newfilelist}") - echo "${obsoleteList}" | while read obsoleteFile; do - [ -z "${obsoleteFile}" ] && return 0 - obsoleteFile="${ZOPEN_ROOTFS}/${obsoleteFile}" - obsoleteFile="${obsoleteFile%% symbolic*}" - printDebug "Checking obsoletefile '${obsoleteFile}'" - if [ -L "${obsoleteFile}" ] && [ ! -e "${obsoleteFile}" ]; then - # the linked-to file no longer exists (ie. the symlink is dangling) - rm -f "${obsoleteFile}" > /dev/null 2>&1 - fi - done - ${killph} 2>/dev/null # if the timer is not running, the kill will fail - waitforpid ${ph} # Make sure it's finished writing to screen + if ! runLogProgress "rmSymlinksFileDiff" \ + "Checking for file differences in mesh" \ + "Checked for file differences mesh" "linkcheck"; then + printError "Unable to remove symlinks links. Review any errors. Manual cleanup using zopen-alt might be required" + fi else - # Slower method needed to analyse each link to see if it has - # become orphaned. Only relevent when removing a package as - # upgrades/alt-switching can supply a list of files - # Use sed to skip header line in .links file - # Note that the contents of the links file are ordered such that - # processing occurs depth-first; if, after removing orphaned symlinks, - # a directory is empty, then it can be removed. - nfiles=$(zossed '1d;$d' "${dotlinks}" | wc -l | tr -d ' ') - printDebug "Creating Temporary dirname file" - tempDirFile=$(mktempfile "unsymlink") - [ -e "${tempDirFile}" ] && rm -f "${tempDirFile}" >/dev/null 2>&1 - touch "${tempDirFile}" - tempTrash=$(mktempfile "unsymlink" "trash") - [ -e "${tempTrash}" ] && rm -f "${tempTrash}" >/dev/null 2>&1 - addCleanupTrapCmd "rm -rf ${tempDirFile}" - addCleanupTrapCmd "rm -rf ${tempTrash}" - printDebug "Using temporary file ${tempDirFile}" - printInfo "- Checking ${nfiles} potentially obsolete file links" - progressHandler "linkcheck" "- Checked ${nfiles} potentially obsolete file links" & - ph=$! - killph="kill -HUP ${ph}" - addCleanupTrapCmd "${killph}" - - while read filetounlink; do - filetounlink=$(echo "${filetounlink}" | zossed 's/\(.*\).symbolic.*/\1/') - filename="$filetounlink" - [ -z "${filetounlink}" ] && continue - filetounlink="${ZOPEN_ROOTFS}/${filetounlink}" - [ ! -e "${filetounlink}" ] && continue # If not there, can'e be removed! - if [ -d "${filetounlink}" ]; then - # Add to the directory queue for checking once files are gone if unique - ispresent=$(grep "^${filetounlink}[ ]*$" "${tempDirFile}") - if [ -z "${ispresent}" ]; then - echo " ${filetounlink} " >> "${tempDirFile}" - fi - elif [ -L "${filetounlink}" ]; then - if [ ! -f "${filetounlink}" ]; then - # the linked-to file no longer exists (ie. the symlink is dangling) - rm -f "${filetounlink}" > /dev/null 2>&1 - fi - else - echo "Unprocessable file: '${filetounlink}'" >> "${tempTrash}" - fi - done </dev/null # if the timer is not running, the kill will fail - waitforpid ${ph} # Make sure it's finished writing to screen + if ! runLogProgress "rmSymlinksFSCheck" \ + "Checking for orphaned symlinks in mesh" \ + "Checked for orphaned symlinks in mesh" "linkcheck"; then + printError "Unable to remove symlinks links. Review any errors. Manual cleanup using zopen-alt might be required" + fi if [ -e "${tempDirFile}" ]; then ndirs=$(uniq < "${tempDirFile}" | wc -l | tr -d ' ') printVerbose "- Checking ${ndirs} dir links" @@ -1012,7 +1007,7 @@ EOF done < "${tempTrash}" printError "Manual removal of files might be required" fi - fi + fi else printDebug "No list of current links to check - package was not installed/active" fi @@ -1070,17 +1065,11 @@ runAndLog() runLogProgress() { printVerbose "$1" - if [ -n "$2" ]; then - printInfo "- $2" - else - printInfo "- Running" - fi - if [ -n "$3" ]; then - completeText="- $3" - else - completeText="- Complete" - fi - progressHandler "spinner" "${completeText}" & + printInfo "- ${2:-Running}" + completeText="${3:-Complete}" + animation="${4:-spinner}" + + progressHandler "${animation}" "${completeText}" & ph=$! killph="kill -HUP ${ph}" addCleanupTrapCmd "${killph}" @@ -1089,7 +1078,7 @@ runLogProgress() if [ -n "${SSH_TTY}" ]; then chtag -r "${SSH_TTY}" fi - ${killph} 2>/dev/null # if the timer is not running, the kill will fail + ${killph} >/dev/null 2>&1 # if the timer is not running, the kill will fail waitforpid ${ph} # Make sure it's finished writing to screen return "${rc}" } @@ -1099,8 +1088,7 @@ spinloop() # in the absence of generic ms/ns reporting, spin-loop instead - not ideal # but without pre-reqing packages... i=$1 - while [ ${i} -ge 0 ]; do - : + while [ "${i}" -ge 0 ]; do i=$((i - 1)) done } @@ -1110,17 +1098,29 @@ progressAnimation() [ $# -eq 0 ] && printError "Internal error: no animation strings." animcnt=$# anim=1 - while :; do - spinloop 1000 + firstFrame=true + while true; do + spinloop 3000 # Check for daemonization of this process (ie. orphaned and PPID=1) # Cannot actually use "$PPID" as it is set at script initialization # and not updated when the parent changes so need to query. - getParentProcess "$$" >/dev/null 2>&1 - ppid=$? - [ "${ppid}" -eq 1 ] && kill INT "${ppid}" >/dev/null 2>&1 + if ! ppid=$(getParentProcess "$$"); then + printVerbose "Cannot determine parent process, disable animation" + exit 1 + fi + if [ "${ppid}" -eq 1 ]; then + # We have been daemonized and owned by PPID=1 + kill HUP "$$" >/dev/null 2>&1 + sleep 1 > /dev/null 2>&1 + fi anim=$((anim + 1)) [ ${anim} -gt ${animcnt} ] && anim=1 - printf "${CRSRSOL}${ERASELINE}%s" "$(getNthArrayArg "${anim}" "$@")" + if ${firstFrame}; then + printf "${ERASELINE}%s\n" "$(getNthArrayArg "${anim}" "$@")" + firstFrame=false + else + printf "${CRSRPL}${ERASELINE}%s\n" "$(getNthArrayArg "${anim}" "$@")" + fi done } @@ -1132,11 +1132,9 @@ getNthArrayArg () waitforpid() { - sleep 1 while kill -0 "$1" >/dev/null 2>&1; do sleep 1 done - sleep 1 } progressHandler() @@ -1149,17 +1147,14 @@ progressHandler() sleep 1 exit 0 fi + type=$1 + completiontext=$2 # Custom end text (when the process is complete) + trap "exit" EXIT # If there is an animation error, it will just exit if ${ANSION}; then - type=$1 - completiontext=$2 # Custom end text (when the process is complete) - trapcmd="exit;" - if [ -n "${completiontext}" ]; then - trapcmd="/bin/printf \"${CRSRSHOW}${ERASELINE}${CRSRPL}${ERASELINE}${completiontext}\n\"; ${trapcmd}" - else - trapcmd="/bin/printf \"${CRSRSHOW}${ERASELINE}${CRSRPL}${ERASELINE}\"; ${trapcmd}" - fi + trapcmd="/bin/printf \"${CRSRSHOW}${ERASELINE}${CRSRPL}${ERASELINE}${CRSRPL}${ERASELINE}- ${completiontext:-Done}\n\";exit" # shellcheck disable=SC2064 trap "${trapcmd}" HUP + # shellcheck disable=SC2059 printf "${CRSRHIDE}" case "${type}" in "spinner") progressAnimation '-' '/' '|' '\\' ;; @@ -1170,6 +1165,10 @@ progressHandler() "pkgcheck") progressAnimation '?###?###' '#?###?##' '##?###?#' '###?###?';; *) progressAnimation '.' 'o' 'O' 'O' 'o' '.' ;; esac + else + trapcmd="/bin/printf \"${completiontext}\n\";exit" + # shellcheck disable=SC2064 + trap "${trapcmd}" HUP fi } @@ -1734,25 +1733,17 @@ promptYesNoAlways() { skip=$2 if ! ${skip}; then while true; do - printInfo "${message} [y/n/a]" + /bin/printf "${message} [y/n/a]" read answer < /dev/tty answer=$(echo "${answer}" | tr '[A-Z]' '[a-z]') case "${answer}" in - y|Y) return 0;; - n|N) return 1;; - a|A) ye + y|yes) return 0;; + n|no) return 1;; + a|always) return 2;; esac - - if [ "y" = "${answer}" ] || [ "yes" = "${answer}" ]; then - return 0 - fi - if [ "n" = "${answer}" ] || [ "no" = "${answer}" ]; then - return 1 - fi - done fi - return 0 + return 1 } getVersionedMetadata() @@ -1786,7 +1777,7 @@ getSelectMetadata() # As this is running within the generate... logic, a progress handler will have been started. # This needs to be terminated before trying to write to screen # shellcheck disable=SC2154 - kill -HUP "${gigph}" 2>/dev/null # if the timer is not running, the kill will fail + kill -HUP "${gigph}" >/dev/null 2>&1 # if the timer is not running, the kill will fail waitforpid "${gigph}" # Make sure it's finished writing to screen repo="$1" @@ -2058,7 +2049,11 @@ generateInstallGraph(){ # tempfiles could be created depending on dependency graph depth invalidPortAssetFile=$(mktempfile "invalid" "port") addCleanupTrapCmd "rm -rf ${invalidPortAssetFile}" - addToInstallGraph "install" "${invalidPortAssetFile}" "${portsToInstall}" + if ! runLogProgress " addToInstallGraph \"install\" \"\${invalidPortAssetFile}\" \"\${portsToInstall}\"" \ + "Creating install graph" "Created install graph" "linkcheck"; then + printError "Unexpected error whilecreating install graph. Correct errors and retry command." + fi + ##TODORM addToInstallGraph "install" "${invalidPortAssetFile}" "${portsToInstall}" # shellcheck disable=SC2154 if ! ${doNotInstallDeps} && { (! ${reinstall} && ! ${reinstallDeps}) || (${reinstall} && ${reinstallDeps}); }; then @@ -2066,11 +2061,16 @@ generateInstallGraph(){ # are reinstalling AND the reinstallDependencies flag is set; we do not want # to reinstall a package and all it's dependencies by default, just the package itself printVerbose "Calculating dependancy graph" - createDependancyGraph "${invalidPortAssetFile}" + if ! runLogProgress "createDependancyGraph \"\${invalidPortAssetFile}\"" \ + "Creating dependancy graph" "Created dependancy graph" "linkcheck"; then + printError "Unexpected error while creating dependancy graph. Correct errors and retry command." + fi + ##TODORM createDependancyGraph "${invalidPortAssetFile}" else printVerbose "- Skipping dependency analysis" fi + ##TODORM>> # shellcheck disable=SC2154 #if ${doNotInstallDeps}; then # printVerbose "- Skipping dependency analysis" @@ -2078,7 +2078,7 @@ generateInstallGraph(){ # calculate dependancy graph # createDependancyGraph "${invalidPortAssetFile}" #fi - + ##< Date: Thu, 6 Feb 2025 12:27:48 +0000 Subject: [PATCH 057/179] QOL update to signature verification scriptlet --- include/scriptlets/installPre/verifySignatureOfPax | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/scriptlets/installPre/verifySignatureOfPax b/include/scriptlets/installPre/verifySignatureOfPax index 579113b86..254e54e0e 100644 --- a/include/scriptlets/installPre/verifySignatureOfPax +++ b/include/scriptlets/installPre/verifySignatureOfPax @@ -17,8 +17,11 @@ if [ "$is_greater" -eq 1 ] && ! $skipverify; then if ! command -v gpg> /dev/null; then skipverify=false; printWarning "GPG is not installed - package validation cannot be performed" + printWarning "Run 'zopen install gpg' or use the --skip-verify parameter" return fi +else + printVerbose "Skipping verification" fi # The following function will attempt to retrieve a value from the metadata for this From cab76a1a41039ccca4ad9c800a6e3bd7b147915f Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 6 Feb 2025 12:36:50 +0000 Subject: [PATCH 058/179] Better pre-installed pkg processing and local pax file handling QOL message improvements --- bin/zopen-install | 8 +++++++- bin/zopen-upgrade | 15 +++++++++++++-- include/common.sh | 35 ++++++++++++++++++++++++++++------- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/bin/zopen-install b/bin/zopen-install index b628c8bff..6bc7adb3d 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -284,8 +284,14 @@ fi if ${fileinstall}; then printDebug "Installing from files as listed in arguments: '${chosenRepos}'" + printDebug "Fully-qualifying files" + absoluteFiles="" + + for fn in $(echo "${chosenRepos}" | sed 's/@@/ /g'); do + absoluteFiles="${absoluteFiles} $(toAbsolutePath "${fn}")" + done # generate the install list JSON from the @@-delimited inputs - installList=$(echo "${chosenRepos}" \ + installList=$(echo "${absoluteFiles}" \ | jq --raw-input --arg d "$(pwd -P)" \ 'def make_object($url): {asset:{url: ( "file://" + $d + "/" + $url )}}; . | split("@@") | map(select(.!="")|make_object(.)) | {"installqueue" :.} ') else diff --git a/bin/zopen-upgrade b/bin/zopen-upgrade index fcb3de33d..87040dffb 100755 --- a/bin/zopen-upgrade +++ b/bin/zopen-upgrade @@ -148,10 +148,21 @@ if ! generateInstallGraph "${chosenRepos}"; then fi # shellcheck disable=SC2154 -if [ 0 -eq "$(echo "${installList}" | jq --raw-output '.installqueue| length')" ]; then +upgrades=$(echo "${installList}" | jq --raw-output '.installqueue| length') +if [ 0 -eq "${upgrades}" ]; then printInfo "- No available updates" else - processRepoInstallFile + pkgcount=$(echo "${installList}" | jq --raw-output '.installqueue | sort| length ') + if [ "$pkgcount" -gt 1 ]; then + pkgInsert="${pkgcount} packages" + else + pkgInsert="package" + fi + printInfo "- The following ${pkgInsert} will be upgraded:" + echo "${installList}" | jq --raw-output '.installqueue | sort| .[] | .portname ' + if promptYesNoAlways "Continue upgrading ${pkgInsert}?" ${yesToPrompts}; then + processRepoInstallFile + fi printInfo "Upgrade operation complete" fi mutexFree "zopen" diff --git a/include/common.sh b/include/common.sh index ef447508f..15c4f7705 100755 --- a/include/common.sh +++ b/include/common.sh @@ -509,6 +509,12 @@ deref() fi } +toAbsolutePath() { + case "$1" in + /*) echo "$1" ;; # Already absolute + * | . | .. | ./* | ../*) echo "$(cd "$(dirname "$1")" && pwd -P)/$(basename "$1")" ;; + esac +} #return 1 if brightness is dark, 0 if light, and 255 if unknown (considered to be dark as default) darkbackground() { if [ "${#COLORFGBG}" -ge 3 ]; then @@ -983,7 +989,7 @@ unsymlinkFromSystem() if [ -e "${newfilelist}" ]; then if ! runLogProgress "rmSymlinksFileDiff" \ "Checking for file differences in mesh" \ - "Checked for file differences mesh" "linkcheck"; then + "Checked for file differences in mesh" "linkcheck"; then printError "Unable to remove symlinks links. Review any errors. Manual cleanup using zopen-alt might be required" fi else @@ -1070,8 +1076,8 @@ runLogProgress() animation="${4:-spinner}" progressHandler "${animation}" "${completeText}" & - ph=$! - killph="kill -HUP ${ph}" + PROGRESS_HANDLER=$! + killph="kill -HUP ${PROGRESS_HANDLER}" addCleanupTrapCmd "${killph}" eval "$1" rc=$? @@ -1079,7 +1085,7 @@ runLogProgress() chtag -r "${SSH_TTY}" fi ${killph} >/dev/null 2>&1 # if the timer is not running, the kill will fail - waitforpid ${ph} # Make sure it's finished writing to screen + waitforpid ${PROGRESS_HANDLER} # Make sure it's finished writing to screen return "${rc}" } @@ -1108,7 +1114,7 @@ progressAnimation() printVerbose "Cannot determine parent process, disable animation" exit 1 fi - if [ "${ppid}" -eq 1 ]; then + if [ "${ppid}" = "1" ]; then # We have been daemonized and owned by PPID=1 kill HUP "$$" >/dev/null 2>&1 sleep 1 > /dev/null 2>&1 @@ -1777,8 +1783,8 @@ getSelectMetadata() # As this is running within the generate... logic, a progress handler will have been started. # This needs to be terminated before trying to write to screen # shellcheck disable=SC2154 - kill -HUP "${gigph}" >/dev/null 2>&1 # if the timer is not running, the kill will fail - waitforpid "${gigph}" # Make sure it's finished writing to screen + kill -HUP "${PROGRESS_HANDLER}" >/dev/null 2>&1 # if the timer is not running, the kill will fail + waitforpid "${PROGRESS_HANDLER}" # Make sure it's finished writing to screen repo="$1" # Explicitly allow the user to select a release to install; useful if there are broken installs @@ -2104,6 +2110,18 @@ parseGraph() # incoming file name against the port name, version and release already on the system # - seems to be the easiest comparison since some data is not in zopen_release vs metadata.json # and a local pax won't have a remote repo but should have a file name! + +installList=$(jq --argjson install_list "${installList}" ' +. as $input | +{ + "installqueue": ( + $install_list.installqueue | map(select( .portname as $pnn | .asset.release as $r | +[$input[] | to_entries[] | select(.key == $pnn) | .value]| all(.product.release != $r) + ))) +} ' "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" +) +##TODORM>> +hidden(){ installed=$(zopen list --installed --details) # Ignore the version string - it varies across ports so use name and build time as that # should be unique enough @@ -2123,6 +2141,8 @@ parseGraph() ) )'\ ) +} +##< Date: Thu, 6 Feb 2025 12:39:42 +0000 Subject: [PATCH 059/179] Handle scriptlet errors --- include/common.sh | 88 +++++++++++++++++++++++++---------------------- 1 file changed, 46 insertions(+), 42 deletions(-) diff --git a/include/common.sh b/include/common.sh index 15c4f7705..53ded8f03 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2434,56 +2434,60 @@ processActionScripts() shift # Drop the initial parameter case "${phase}" in - "installPre") scriptDir="${ZOPEN_SCRIPTLET_DIR}/installPre";; - "installPost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/installPost";; - "removePre") scriptDir="${ZOPEN_SCRIPTLET_DIR}/removePre";; - "removePost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/removePost";; - "transactionPre") scriptDir="${ZOPEN_SCRIPTLET_DIR}/transactionPre";; - "transactionPost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/transactionPost";; - "parseGraphPre") scriptDir="${ZOPEN_SCRIPTLET_DIR}/parseGraphPre";; - "parseGraphPost") scriptDir="${ZOPEN_SCRIPTLET_DIR}/parseGraphPost";; + "installPre") scriptletDir="${ZOPEN_SCRIPTLET_DIR}/installPre";; + "installPost") scriptletDir="${ZOPEN_SCRIPTLET_DIR}/installPost";; + "removePre") scriptletDir="${ZOPEN_SCRIPTLET_DIR}/removePre";; + "removePost") scriptletDir="${ZOPEN_SCRIPTLET_DIR}/removePost";; + "transactionPre") scriptletDir="${ZOPEN_SCRIPTLET_DIR}/transactionPre";; + "transactionPost") scriptletDir="${ZOPEN_SCRIPTLET_DIR}/transactionPost";; + "parseGraphPre") scriptletDir="${ZOPEN_SCRIPTLET_DIR}/parseGraphPre";; + "parseGraphPost") scriptletDir="${ZOPEN_SCRIPTLET_DIR}/parseGraphPost";; *) assertFailed "Invalid process action phase '${phase}'" esac - printVerbose "Running script[s] from '${scriptDir}'" + printVerbose "Running script[s] from '${scriptletDir}'" - if [ ! -d "${scriptDir}" ]; then + if [ ! -d "${scriptletDir}" ]; then printDebug "No script directory for phase: ${phase}" return 0 fi unset CDPATH; - # shellcheck disable=SC2164 - cd "${scriptDir}" - scriptRcFile=$(mktempfile "actionscripts" ".err") - find . -type l | while read scriptFile; do - # Note: The script only needs to be readable as it is not executed as a - # separate process; only read permission required when sourced with '.' - if [ ! -r "${scriptFile}" ]; then - printWarning "Script '${scriptDir}/${scriptFile}' is not readable. Check permissions" - continue - fi - printVerbose "Attempting to run script '${scriptFile}'" - # Call each scriptlet with the remaining parameters passed to this function as - # subshells so that any modification in the scripts does not affect the runtime - # Any errors, then output the status code as the final output - scriptOutput=$( - # shellcheck disable=SC1090 - # shellcheck disable=SC2240 # This works on z/OS's /bin/sh - . "${scriptFile}" "$@" 2>&1 - echo $? - ) - scriptRc=$(echo "${scriptOutput}" | tail -n 1) - echo "${scriptOutput}" | head -n -1 # Don't print last (status) line! - - if [ "${scriptRc}" -ne 0 ]; then - touch "${scriptRcFile}" # Needed to indicate error outside subshell - break - fi - done - if [ -e "${scriptRcFile}" ]; then - rm -f "${scriptRcFile}" - return 1 + cd "${scriptletDir}" || return 1 + + scriptletRcFile=$(mktempfile "zopen_actionscripts" ".err") + find . -type l | while IFS= read -r scriptletFile; do + if [ ! -r "${scriptletFile}" ]; then + printWarning "Script '${scriptletDir}/${scriptletFile}' is not readable. Check permissions" >> "${scriptletRcFile}" 2>&1 + continue + fi + printVerbose "Attempting to run script '${scriptletFile}'" + # Run script in a subshell to prevent environment modification + /bin/printf "${CRSRSOL}${ERASELINE}Running ${scriptletFile}" + scriptletOutput=$({ + # shellcheck disable=SC1090 + # shellcheck disable=SC2240 + . "$scriptletFile" "$@" + echo $? # Append exit status to output + } 2>&1 + ) + /bin/printf "${CRSRSOL}${ERASELINE}${CRSRSOL}" + scriptletRc=$(echo "${scriptletOutput}" | tail -n 1) # Extract exit status + scriptletBody=$(echo "${scriptletOutput}" | sed '$d') # Extract script output + if [ "${scriptletRc}" -ne 0 ]; then { + printWarning "Scriptlet '${scriptletFile}' failed with exit code ${scriptletRc}" + printWarning "Details:" + printWarning "${scriptletBody}" + } >> "${scriptletRcFile}" fi - return 0 # If we get to here, then none of the scripts returned a fatal error + done + + if [ -s "${scriptletRcFile}" ]; then # Check if errorLog is non-empty + printWarning "One or more scripts failed:" + cat "${scriptletRcFile}" + rm -f "${scriptletRcFile}" + return 1 + fi + rm -f "${scriptletRcFile}" + return 0 } # updatePackageDB From 51107216abc95991e6f8ade353bd93a8d67eda13 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 6 Feb 2025 12:40:25 +0000 Subject: [PATCH 060/179] Warn when uninstalling a package pre-req --- bin/zopen-remove | 29 +++++++++++++++++------------ include/common.sh | 17 +++++++++++++++-- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/bin/zopen-remove b/bin/zopen-remove index a2c6dc2c3..4a236c72c 100755 --- a/bin/zopen-remove +++ b/bin/zopen-remove @@ -61,21 +61,26 @@ removePackages() printHeader "Removing package: ${pkg}" processActionScripts "removePre" printInfo "- Querying install status of package '${pkg}'" + # Check for the actual file rather than the database; if the db is + # corrupt, the file is the tell-tale if [ ! -f "${ZOPEN_PKGINSTALL}/${pkg}/${pkg}/.active" ]; then printInfo "${NC}${YELLOW}- Package '${pkg}' is not installed${NC}" else - printInfo "- Away to remove '${pkg}'," - if ! ${yesToPrompts}; then - while true; do - /bin/printf "- Do you want to continue [y/n/a]? " - read continueInstall < /dev/tty - case "${continueInstall}" in - "y") break;; - "n") mutexFree "zopen"; printInfo "Exiting..."; exit 0 ;; - "a") yesToPrompts=true; break;; - *) echo "?";; - esac - done + + if preReqList=$(checkIfPrereq "${pkg}"); then + printWarning "${pkg} is a runtime prerequiste of the following:\n$(echo "${preReqList}" | sed '$d')" + printWarning "Removing ${pkg} could result in broken packages" + if ! promptYesNoAlways "Continue removal of '${pkg}' ?" false; then + mutexFree "zopen"; + printInfo "Exiting..."; + exit 0 + fi + else + if ! promptYesNoAlways "Confirm removal of '${pkg}'?" ${yesToPrompts}; then + mutexFree "zopen"; + printInfo "Exiting..."; + exit 0 + fi fi printInfo "- Unmeshing '${pkg}' from system" version="unknown" diff --git a/include/common.sh b/include/common.sh index 53ded8f03..814ac487c 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1739,13 +1739,13 @@ promptYesNoAlways() { skip=$2 if ! ${skip}; then while true; do - /bin/printf "${message} [y/n/a]" + /bin/printf "${message} [y/n/a] " read answer < /dev/tty answer=$(echo "${answer}" | tr '[A-Z]' '[a-z]') case "${answer}" in y|yes) return 0;; n|no) return 1;; - a|always) return 2;; + a|always) yesToPrompts=true; return 0;; esac done fi @@ -2148,6 +2148,19 @@ hidden(){ fi } +checkIfPrereq(){ + # This jq query analyses the package database, looking for objects where the + # runtime dependency contains $1 (the removee). It extracts the keys from those + # objects into $dependents then outputs the flattened dependents array along with + # eithr true or false depending on if there was a prereq found (true) or if not + # false; the exit-status parameter then sets the exit code of jq dependning on that + # true/false value! + jq -r --exit-status --arg removee "$1" \ + '[.[] | to_entries | map(select(.value.product.runtime_dependencies[]?.name == $removee)) | .[].key] | . as $dependents | $dependents[], ($dependents | length > 0)' \ + "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" +} + + spaceValidate(){ spaceRequiredBytes=$1 spaceRequiredMB=$(echo "scale=0; ${spaceRequiredBytes} / (1024 * 1024)" | bc) From 8deeaa67e48bae2c217ddd32bf9b47dbea58e913 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Fri, 14 Feb 2025 10:56:07 +0000 Subject: [PATCH 061/179] Older jq issue with variable keys & fix merge issues --- bin/zopen-init | 7 ++++- include/common.sh | 71 ++++++++++++++++++++++++++--------------------- 2 files changed, 45 insertions(+), 33 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index 471473f81..390df312f 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -704,6 +704,11 @@ processCaCerts() fi } +createInitialPackageDB() +{ + updatePackageDB +} + deinit() { printHeader "Initialization successfully completed" @@ -726,6 +731,6 @@ generateJsonConfiguration generateDefaultRepository generateZopenConfig generateAnalyticsConfiguration +[ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || createInitialPackageDB updateBootstrappedTools -[ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || updatePackageDB deinit diff --git a/include/common.sh b/include/common.sh index 814ac487c..f4c4f11ba 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1734,6 +1734,30 @@ a2e() fi } +startGPGAgent() { + printInfo "- Starting gpg-agent..." + + SOCKET_PATH=$(gpgconf --list-dirs agent-socket) + if [ -r "$SOCKET_PATH" ]; then + printVerbose "gpg-agent is already running (socket found at $SOCKET_PATH)." + return 0 + fi + + if eval "$(gpg-agent --daemon --disable-scdaemon)" >/dev/null 2>&1; then + if [ -r "$SOCKET_PATH" ]; then + printVerbose "gpg-agent started successfully (socket created at $SOCKET_PATH)." + else + printWarning "gpg-agent started, but socket was not created at $SOCKET_PATH. Please verify your GPG installation." + fi + else + if [ -r "$SOCKET_PATH" ]; then + printWarning "gpg-agent started successfully (socket created at $SOCKET_PATH), but gpg-agent returned a non-zero return code." + else + printError "Failed to start gpg-agent. Reinstall or upgrade GPG using \"zopen install --reinstall gpg -y\" or \"zopen upgrade gpg -y\"." + fi + fi +} + promptYesNoAlways() { message="$1" skip=$2 @@ -1762,7 +1786,7 @@ getVersionedMetadata() requestedVersion="${requestedMajor}\\\.${requestedMinor}\\\.${requestedPatch}\\\.${requestedSubrelease}" printDebug "Finding URL for latest release matching version prefix: requestedVersion: ${requestedVersion}" releasemetadata=$(jq --arg repo "${repo}" --arg requestedVersion "${requestedVersion}" \ - '.release_data.[$repo] | map(select(.assets[].version | test($requestedVersion)))[0]' "${JSON_CACHE}") + '.release_data[$repo] | map(select(.assets[].version | test($requestedVersion)))[0]' "${JSON_CACHE}") } getTaggedMetadata() @@ -1770,7 +1794,7 @@ getTaggedMetadata() printDebug "Explicit tagged version '${tagged}' specified. Checking for match" releasemetadata=$(/bin/printf "%s" "${releases}" | jq -e -r '.[] | select(.tag_name == "'"${tagged}"'")') releasemetadata=$(jq --arg repo "${repo}" --arg requestedVersion "${requestedVersion}" \ - '.release_data.[$repo][] | select(.tag_name == $tag_name)' "${JSON_CACHE}") + '.release_data[$repo][] | select(.tag_name == $tag_name)' "${JSON_CACHE}") printDebug "Use quick check for asset to check for existence of metadata for specific messages" asset=$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]') if [ $? -ne 0 ]; then @@ -1790,10 +1814,10 @@ getSelectMetadata() # Explicitly allow the user to select a release to install; useful if there are broken installs # as a known good release can be found, selected and pinned! printDebug "List individual releases and allow selection" - i=$(jq --arg repo "${repo}" '.release_data.[$repo] | length - 1' "${JSON_CACHE}") + i=$(jq --arg repo "${repo}" '.release_data[$repo] | length - 1' "${JSON_CACHE}") printInfo "Versions available for install:" if ! jq --raw-output --arg repo "${repo}" \ - '.release_data.[$repo] | to_entries | map("\(.key): \(.value.tag_name) - \(.value.assets[0].name) [\( ( .value.assets[0].expanded_size|tonumber)*1000 / (1024 * 1024) | ceil | . / 1000)Mb]")[]' "${JSON_CACHE}"; then + '.release_data[$repo] | to_entries | map("\(.key): \(.value.tag_name) - \(.value.assets[0].name) [\( ( .value.assets[0].expanded_size|tonumber)*1000 / (1024 * 1024) | ceil | . / 1000)Mb]")[]' "${JSON_CACHE}"; then printError "Unable to enumerate asset version strings" fi printDebug "Getting user selection" @@ -1809,7 +1833,7 @@ getSelectMetadata() done printVerbose "Selecting item ${selection} from array" releasemetadata=$(jq --arg repo "${repo}" --arg selection "${selection}" \ - '.release_data.[$repo][$selection | tonumber]' "${JSON_CACHE}") + '.release_data[$repo][$selection | tonumber]' "${JSON_CACHE}") } getReleaseLineMetadata() @@ -1821,7 +1845,7 @@ getReleaseLineMetadata() fi printDebug "Finding latest asset on the release line" releasemetadata=$(jq --arg repo "${repo}" --arg releaseLine "${validatedReleaseLine}" \ - '.release_data.[$repo] | map(select(.tag_name | startswith($releaseLine)))[0]' "${JSON_CACHE}") + '.release_data[$repo] | map(select(.tag_name | startswith($releaseLine)))[0]' "${JSON_CACHE}") printDebug "Use quick check for asset to check for existence of metadata" asset="$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')" if [ $? -ne 0 ]; then @@ -1868,7 +1892,7 @@ calculateReleaseLineMetadata() # then that specific releaseline will be used printDebug "Finding any releases tagged with ${validatedReleaseLine} and getting the first (newest/latest)" releasemetadata=$(jq --arg repo "${repo}" --arg releaseLine "${validatedReleaseLine}" \ - '.release_data.[$repo] | map(select(.tag_name | startswith($releaseLine)))[0]' "${JSON_CACHE}") + '.release_data[$repo] | map(select(.tag_name | startswith($releaseLine)))[0]' "${JSON_CACHE}") printDebug "Use quick check for asset to check for existence of metadata" asset="$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')" @@ -1882,7 +1906,7 @@ calculateReleaseLineMetadata() printDebug "No releases on releaseline '${validatedReleaseLine}'; checking alternative releaseline" alt=$(echo "${validatedReleaseLine}" | awk ' /DEV/ { print "STABLE" } /STABLE/ { print "DEV" }') releasemetadata=$(jq --arg repo "${repo}" --arg releaseLine "${alt}" \ - '.release_data.[$repo] | map(select(.tag_name | startswith($releaseLine)))[0]' "${JSON_CACHE}") + '.release_data[$repo] | map(select(.tag_name | startswith($releaseLine)))[0]' "${JSON_CACHE}") printDebug "Use quick check for asset to check for existence of metadata" asset="$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')" [ "${asset}" = "null" ] && asset="" # jq uses null, translate to sh's empty @@ -1891,7 +1915,7 @@ calculateReleaseLineMetadata() if [ "DEV" = "${validatedReleaseLine}" ]; then # The system will be configured to use DEV packages where available but if none, use latest printInfo "No specific DEV releaseline package, using latest available release" - releasemetadata=$(jq --arg repo "${repo}" '.release_data.[$repo][0]' "${JSON_CACHE}") + releasemetadata=$(jq --arg repo "${repo}" '.release_data[$repo][0]' "${JSON_CACHE}") else printVerbose "The system is configured to only use STABLE releaseline packages but there are none" printInfo "No release available on the '${validatedReleaseLine}' releaseline." @@ -1899,7 +1923,7 @@ calculateReleaseLineMetadata() else # Case 1 - old package that has no release tagging yet (no DEV or STABLE), just install latest printVerbose "Installing latest release" - releasemetadata=$(jq --arg repo "${repo}" '.release_data.[$repo][0]' "${JSON_CACHE}") + releasemetadata=$(jq --arg repo "${repo}" '.release_data[$repo][0]' "${JSON_CACHE}") fi fi } @@ -2557,6 +2581,10 @@ updatePackageDB() fi mv "${pdb}.working" "${pdb}" done + if [ ! -e "${pdb}" ]; then + printVerbose "No currently installed packages [new install?]. Creating empty array[]" + printf "[\n]\n" > "${pdb}" + fi } stripControlCharacters(){ @@ -2667,27 +2695,6 @@ jqfunctions() 'def r(dp):.*pow(10;dp)|round/pow(10;dp)' } -startGPGAgent() -{ - - # shellcheck disable=SC2009 # Not on z/OS currently - if ps -ef | grep [g]pg-agent; then - printInfo "- Re-using gpg-agent" - return - fi - printInfo "- Starting gpg-agent" - if ! gpg-agent --daemon --disable-scdaemon; then - printError "Error running gpg-agent command. Review error messages and retry command." - fi - # Wait a moment to ensure the gpg-agent has time to start - sleep 2 - # Check to confirm if gpg-agent started successfully - # shellcheck disable=SC2009 # pgrep not on z/OS currently! - if ! ps -ef | grep -v grep | grep "gpg-agent" > /dev/null; then - printError "Failed to start the gpg-agent. Reinstall or upgrade GPG using \"zopen install --reinstall gpg -y\" or \"zopen upgrade gpg -y\" command." - fi -} +. ${INCDIR}/analytics.sh -# shellcheck disable=SC1091 -. "${INCDIR}/analytics.sh" zopenInitialize From 71358e411c1fe2e965e2918bd6de438400275d65 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 4 Mar 2025 10:55:07 +0000 Subject: [PATCH 062/179] Ensure promptYesNoAlways enforces always --- include/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common.sh b/include/common.sh index f4c4f11ba..26982c171 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1773,7 +1773,7 @@ promptYesNoAlways() { esac done fi - return 1 + [ "$yesToPrompts" = "true" ] && return 0 || return 1 } getVersionedMetadata() From cf72c0b7dd7ba8c313cdd02f6695eebc09b7fc9e Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 4 Mar 2025 10:56:00 +0000 Subject: [PATCH 063/179] Handle print* function output change to redirection --- include/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common.sh b/include/common.sh index 26982c171..a1f8f93e4 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2513,7 +2513,7 @@ processActionScripts() printWarning "Scriptlet '${scriptletFile}' failed with exit code ${scriptletRc}" printWarning "Details:" printWarning "${scriptletBody}" - } >> "${scriptletRcFile}" + } >> "${scriptletRcFile}" 2>&1 fi done From 733c9a4f104352438733aca501d42fdb89bb4bcc Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 4 Mar 2025 17:18:26 +0000 Subject: [PATCH 064/179] Ensure skip-verify is listed in help --- bin/zopen-install | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/zopen-install b/bin/zopen-install index 6bc7adb3d..5389363fc 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -73,6 +73,8 @@ Options: --release-line [stable|dev] the release line to install from --select select a version to install from available versions + --skip-verify skips the package verification allowing un-trusted + packages. Caveat usor! -v, --verbose run in verbose mode -y, --yes automatically answer yes to prompts -h,-?, --help display this help and exit From 04497dfe9586c0bc4eb899060bd42c8178e4f996 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 4 Mar 2025 17:25:57 +0000 Subject: [PATCH 065/179] Fix issues running with --verbose and output additional diagnostic information when active --- include/common.sh | 48 +++++++++++-------- .../installPre/verifySignatureOfPax | 6 +-- include/scriptlets/removePost/man-db | 14 ------ include/scriptlets/transactionPost/man-db | 10 ++-- .../transactionPost/zopen-clean-autocache | 1 - 5 files changed, 36 insertions(+), 43 deletions(-) delete mode 100644 include/scriptlets/removePost/man-db diff --git a/include/common.sh b/include/common.sh index a1f8f93e4..4e8c4b119 100755 --- a/include/common.sh +++ b/include/common.sh @@ -46,15 +46,12 @@ addCleanupTrapCmd(){ } cleanup() { - printVerbose "Performing cleanup [in the parent process]" - if [ -f "${tmpscriptfile}" ] && [ -s "${tmpscriptfile}" ]; then - # Execute the commands in the cleanup file by sourcing it - # shellcheck disable=SC1090 - . "${tmpscriptfile}" > /dev/null 2>&1 - rm "${tmpscriptfile}" - else - printVerbose "No cleanup script to run" - fi + if [ -f "${tmpscriptfile}" ] && [ -s "${tmpscriptfile}" ]; then + # Execute the commands in the cleanup file by sourcing it + # shellcheck disable=SC1090 + . "${tmpscriptfile}" > /dev/null 2>&1 + rm "${tmpscriptfile}" + fi } addCleanupTrapCmd2(){ @@ -1075,17 +1072,23 @@ runLogProgress() completeText="${3:-Complete}" animation="${4:-spinner}" - progressHandler "${animation}" "${completeText}" & - PROGRESS_HANDLER=$! - killph="kill -HUP ${PROGRESS_HANDLER}" - addCleanupTrapCmd "${killph}" - eval "$1" - rc=$? + if ! ${verbose}; then + progressHandler "${animation}" "${completeText}" & + PROGRESS_HANDLER=$! + killph="kill -HUP ${PROGRESS_HANDLER}" + addCleanupTrapCmd "${killph}" + eval "$1" + rc=$? + ${killph} >/dev/null 2>&1 # if the timer is not running, the kill will fail + waitforpid ${PROGRESS_HANDLER} # Make sure it's finished writing to screen + else + eval "$1" + rc=$? + fi if [ -n "${SSH_TTY}" ]; then chtag -r "${SSH_TTY}" fi - ${killph} >/dev/null 2>&1 # if the timer is not running, the kill will fail - waitforpid ${PROGRESS_HANDLER} # Make sure it's finished writing to screen + return "${rc}" } @@ -2079,7 +2082,7 @@ generateInstallGraph(){ # tempfiles could be created depending on dependency graph depth invalidPortAssetFile=$(mktempfile "invalid" "port") addCleanupTrapCmd "rm -rf ${invalidPortAssetFile}" - if ! runLogProgress " addToInstallGraph \"install\" \"\${invalidPortAssetFile}\" \"\${portsToInstall}\"" \ + if ! runLogProgress " addToInstallGraph \"install\" \"${invalidPortAssetFile}\" \"${portsToInstall}\"" \ "Creating install graph" "Created install graph" "linkcheck"; then printError "Unexpected error whilecreating install graph. Correct errors and retry command." fi @@ -2091,7 +2094,7 @@ generateInstallGraph(){ # are reinstalling AND the reinstallDependencies flag is set; we do not want # to reinstall a package and all it's dependencies by default, just the package itself printVerbose "Calculating dependancy graph" - if ! runLogProgress "createDependancyGraph \"\${invalidPortAssetFile}\"" \ + if ! runLogProgress "createDependancyGraph \"${invalidPortAssetFile}\"" \ "Creating dependancy graph" "Created dependancy graph" "linkcheck"; then printError "Unexpected error while creating dependancy graph. Correct errors and retry command." fi @@ -2176,7 +2179,7 @@ checkIfPrereq(){ # This jq query analyses the package database, looking for objects where the # runtime dependency contains $1 (the removee). It extracts the keys from those # objects into $dependents then outputs the flattened dependents array along with - # eithr true or false depending on if there was a prereq found (true) or if not + # either true or false depending on if there was a prereq found (true) or if not # false; the exit-status parameter then sets the exit code of jq dependning on that # true/false value! jq -r --exit-status --arg removee "$1" \ @@ -2514,6 +2517,11 @@ processActionScripts() printWarning "Details:" printWarning "${scriptletBody}" } >> "${scriptletRcFile}" 2>&1 + elif ${verbose}; then + if [ -n "${scriptletBody}" ]; then + # Show the scriptlet output + echo "${scriptletBody}" + fi fi done diff --git a/include/scriptlets/installPre/verifySignatureOfPax b/include/scriptlets/installPre/verifySignatureOfPax index 254e54e0e..30d179e29 100644 --- a/include/scriptlets/installPre/verifySignatureOfPax +++ b/include/scriptlets/installPre/verifySignatureOfPax @@ -18,7 +18,7 @@ if [ "$is_greater" -eq 1 ] && ! $skipverify; then skipverify=false; printWarning "GPG is not installed - package validation cannot be performed" printWarning "Run 'zopen install gpg' or use the --skip-verify parameter" - return + return 1 fi else printVerbose "Skipping verification" @@ -32,16 +32,14 @@ extractFromMetaData(){ key=$1 mdf=$2 if ! sRc=$(jq -e -r "${key}" "${mdf}"); then - printVerbose "Could not locate ${key} in the pax's metadata. Checking cache..." mdf="${ZOPEN_ROOTFS}/var/cache/zopen/$(basename "$3").json" if ! sRc=$(jq -e -r "${key}" "${mdf}"); then - printVerbose "Failed to extract '${key}' from package metadata" >&2 return 1 fi fi echo "${sRc}" && return 0 - } + # Extracting values and checking for errors # If this is a local pax file, then there might not be a .product.pax in the # metadata; check for a corresponding .json file in the cache. diff --git a/include/scriptlets/removePost/man-db b/include/scriptlets/removePost/man-db deleted file mode 100644 index f5d940cb7..000000000 --- a/include/scriptlets/removePost/man-db +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -this="man-db" -printVerbose "Running '${this}' script" -if ! zopen list --installed | grep "^man-db\$"; then - : # No man-db installed -else - echo "- Updating man-db database after package changes..." - if ! mandb >/dev/null 2>&1; then - printSoftError "man-db update completed with non-zero return code." - printSoftError "Re-run 'mandb' manually for additional information." - else - echo "- Update of man-db complete." - fi -fi diff --git a/include/scriptlets/transactionPost/man-db b/include/scriptlets/transactionPost/man-db index f5d940cb7..d2c567872 100755 --- a/include/scriptlets/transactionPost/man-db +++ b/include/scriptlets/transactionPost/man-db @@ -2,13 +2,15 @@ this="man-db" printVerbose "Running '${this}' script" if ! zopen list --installed | grep "^man-db\$"; then - : # No man-db installed + return 0 # No man-db installed else echo "- Updating man-db database after package changes..." - if ! mandb >/dev/null 2>&1; then + if ! mandbOut=$(mandb >/dev/null 2>&1); then printSoftError "man-db update completed with non-zero return code." + printSoftError "Details: ${mandbOut}" printSoftError "Re-run 'mandb' manually for additional information." - else - echo "- Update of man-db complete." + return 1 fi + echo "- Update of man-db complete." fi +return 0 diff --git a/include/scriptlets/transactionPost/zopen-clean-autocache b/include/scriptlets/transactionPost/zopen-clean-autocache index cade3f475..4c5c0be1f 100755 --- a/include/scriptlets/transactionPost/zopen-clean-autocache +++ b/include/scriptlets/transactionPost/zopen-clean-autocache @@ -9,7 +9,6 @@ fi case "${isactive}" in 0 ) :;; # Take no action 1 ) echo "- Cleaning install cache..." - printInfo "CLEANING !!" (zopen clean --cache >/dev/null 2>&1) echo "- Cache cleaned." ;; From 67def024e809d87f881c1b026f108cb2e42dc306 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 5 Mar 2025 15:18:06 +0000 Subject: [PATCH 066/179] Prevent error if local file is not actually an install package --- bin/zopen-install | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/bin/zopen-install b/bin/zopen-install index 5389363fc..7fc550ba8 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -236,7 +236,17 @@ fi printDebug "Checking input parameters for actual files" potentials=$(echo "${chosenRepos}" | sed 's/@@/ /g') for installRepo in ${potentials}; do - [ -f "${installRepo}" ] && fileinstall=true && break + [ -f "${installRepo}" ] || continue + # The parameter is a file, check it is actually a supported package or ignore + case "${installRepo}" in + (*.pax.Z) + printVerbose "Input parameter '$installRepo' has a supported input suffix" + # Treat all input parameters as file parameters + fileinstall=true + break + ;; + *) printVerbose "Handling input parameter '$installRepo' as a package name" ;; + esac done # Sanity check for argument clashes From 3171846515f95b9c1aa2cbae6bdb2c51df6bff46 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 5 Mar 2025 21:51:27 +0000 Subject: [PATCH 067/179] Failed gpg start does not propogate --- include/scriptlets/installPre/verifySignatureOfPax | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/scriptlets/installPre/verifySignatureOfPax b/include/scriptlets/installPre/verifySignatureOfPax index 30d179e29..ee133fafe 100644 --- a/include/scriptlets/installPre/verifySignatureOfPax +++ b/include/scriptlets/installPre/verifySignatureOfPax @@ -94,7 +94,10 @@ if [ ! -f "${PUBLIC_KEY_FILE}" ]; then return 1 fi -startGPGAgent +if ! startGPGAgent; then + printSoftError "Unable to start GPG agent" + return 1 +fi printVerbose "Importing public key to keyring file..." KEYRING_FILE=$(mktempfile "pubring" ".kbx") From 875b17a3cfa615efeb844a9580cbe30076f04f57 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 5 Mar 2025 22:04:03 +0000 Subject: [PATCH 068/179] Simplify jq queries usage of shell parameters to aid debug and allow fixing of queries and output displays Remove unused code, tidy up indentation and add additional diagnostics/verbose entries. --- bin/zopen-list | 283 +++++++++++++++++++++---------------------------- 1 file changed, 122 insertions(+), 161 deletions(-) diff --git a/bin/zopen-list b/bin/zopen-list index 05696281b..ef45c84c5 100755 --- a/bin/zopen-list +++ b/bin/zopen-list @@ -28,14 +28,14 @@ Usage: zopen list [OPTION] [VERB] [PACKAGE] Options: --available lists all available z/OS Open Tools + --installed list only installed z/OS Open Tools (default output) --details provide version information on packages --full provides more information about packages - --installed list only installed z/OS Open Tools --[no]header output explanation header for columns --verbose run in verbose mode --version print version --version-info lists version information for installed packages when - combined with the --installed option + combined with the --installed option -h,-?, --help display this help and exit Examples: @@ -109,7 +109,7 @@ listAvailablePackages() else printf "${NC}${UNDERLINE}%-${wpackage}s%-${winstalled}s%-${wversion}s%-${wtag}s${NC}\n" \ "$(text_center "Package")" "$(text_center "Installed")" "$(text_center "Version")" "$(text_center "Latest Tag")" - fi + fi fi installed=$(zopen list --installed | awk -v ORS=',' '{print "\""$0"\""}') installed="[${installed%,}]" # wrap in a JSON array and strip any trailing ',' char @@ -121,43 +121,54 @@ listAvailablePackages() urltext="" fi jq --raw-output --argjson installed "${installed}" \ - "$(jqfunctions) .release_data| + --arg wpackage "${wpackage}" \ + --arg winstalled "${winstalled}" \ + --arg wtag "${wtag}" \ + --arg wdownload "${wdownload}" \ + --arg wexpsz "${wexpsz}" \ + --arg wqual "${wqual}" \ + --arg urltext "${urltext}" \ + "$(jqfunctions)"' .release_data| to_entries |sort_by(.key) | .[] | - .key as \$key | - .value[0].assets[0] as \$va | - \$va.url as \$url | - (\$url | match(\".*-([^-]*).[0-9]{8}_[0-9]{6}.zos.pax.Z\").captures[0].string) as \$version | - .value[0].tag_name as \$tn | - (if (\$installed | index(\$key)) then \"installed\" else \"available\" end) as \$ins | - \$va.expanded_size as \$es | - \$va.size as \$ds | - (\$va.passed_tests | if (.==\"\") then \"0\" else . end | tonumber) as \$pt | - (\$va.total_tests | if (.==\"\") then \"99999\" else . end |tonumber) as \$tt | - ((\$pt/\$tt*100|r(2))|tostring + \"%\") as \$q | - pr(\$key;\"@\";${wpackage}) + - c(\$ins;\"@\";${winstalled}) + - c(\$tn;\"@\";${wtag}) + - c(\$ds;\"@\";${wdownload}) + - c(\$es;\"@\";${wexpsz}) + - c((\$q);\"@\";${wqual}) + - ${urltext} " \ - "${JSON_CACHE}" | sed 's/@/ /g' + .key as $key | + .value[0].assets[0] as $va | + $va.url as $url | + ($url | match(".*-([^-]*).[0-9]{8}_[0-9]{6}.zos.pax.Z").captures[0].string) as $version | + .value[0].tag_name as $tn | + (if ($installed | index($key)) then "installed" else "available" end) as $ins | + $va.expanded_size as $es | + $va.size as $ds | + ($va.passed_tests | if (.=="") then "0" else . end | tonumber) as $pt | + ($va.total_tests | if (.=="") then "99999" else . end |tonumber) as $tt | + (($pt/$tt*100|r(2))|tostring + "%") as $q | + pr($key;"@";($wpackage | tonumber)) + + c($ins;"@";($winstalled | tonumber)) + + c($tn;"@";($wtag | tonumber)) + + c($ds;"@";($wdownload | tonumber)) + + c($es;"@";($wexpsz | tonumber)) + + c(($q);"@";($wqual | tonumber)) + + $urltext ' \ + "${JSON_CACHE}" | sed 's/@/ /g' else jq --raw-output --argjson installed "${installed}" \ - "$(jqfunctions) .release_data| + --arg wpackage "${wpackage}" \ + --arg winstalled "${winstalled}" \ + --arg wversion "${wversion}" \ + --arg wtag "${wtag}" \ + "$(jqfunctions)"' .release_data| to_entries |sort_by(.key) | .[] | - .key as \$key | - .value[0].assets[0].url as \$url | - (\$url | match(\".*-([^-]*).[0-9]{8}_[0-9]{6}.zos.pax.Z\").captures[0].string) as \$version | - .value[0].tag_name as \$tn | - (if (\$installed | index(\$key)) then \"installed\" else \"available\" end) as \$ins | - .value[0].assets[0] as \$va | - pr(\$key;\"@\";${wpackage}) + - c(\$ins;\"@\";${winstalled}) + - c(\$version;\"@\";${wversion}) + - c(\$tn;\"@\";${wtag}) " \ - "${JSON_CACHE}" | sed 's/@/ /g' - fi + .key as $key | + .value[0].assets[0].url as $url | + ($url | match(".*-([^-]*).[0-9]{8}_[0-9]{6}.zos.pax.Z").captures[0].string) as $version | + .value[0].tag_name as $tn | + (if ($installed | index($key)) then "installed" else "available" end) as $ins | + .value[0].assets[0] as $va | + pr($key;"@";($wpackage | tonumber)) + + c($ins;"@";($winstalled | tonumber)) + + c($version;"@";($wversion | tonumber)) + + c($tn;"@";($wtag | tonumber)) ' \ + "${JSON_CACHE}" | sed 's/@/ /g' + fi } listInstalledEntries() @@ -178,13 +189,13 @@ listInstalledEntries() if ${header} && ${details}; then if ${full}; then printf "${NC}${UNDERLINE}%-${wpackage}s%-${wversion}s%-${wrelease}s%-${wrellne}s%-${wexpsz}s%-${wqual}s%-${wurl}s${NC}\n" \ - "$(text_center "Package")" "$(text_center "Version")" "$(text_center "Release")" \ - "$(text_center "Releaseline")" "$(text_center "Disk usage")" \ - "$(text_center "Quality")" "$(text_center "${urltext}")" + "$(text_center "Package" ${wpackage})" "$(text_center "Version" ${wversion})" "$(text_center "Release" ${wrelease})" \ + "$(text_center "Releaseline" ${wrellne})" "$(text_center "Disk usage" ${wexpsz})" \ + "$(text_center "Quality" ${wqual})" "$(text_center "${urltext}" ${wurl})" else printf "${NC}${UNDERLINE}%-${wpackage}s%-${wversion}s%-${wrelease}s${NC}\n" \ - "$(text_center "Package")" "$(text_center "Version")" "$(text_center "Release")" - fi + "$(text_center "Package" ${wpackage})" "$(text_center "Version" ${wversion})" "$(text_center "Release" ${wrelease})" + fi fi pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" if ! [ -e "${pdb}" ]; then @@ -192,105 +203,56 @@ listInstalledEntries() updatePackageDB fi if ${details}; then + printVerbose "Details requested" if ${full}; then + printVerbose "Full details requested" jq --raw-output \ - "$(jqfunctions) .[] | - keys[] as \$key | - .[\$key].product.test_status as \$ts | - pr(\$key;\"@\";${wpackage}) + - c((.[\$key].product?.version? // \"unknown\");\"@\";${wversion}) + - c((.[\$key].product?.release? // \"unknown\");\"@\";${wrelease}) + - c((.[\$key].product.buildline? // \"unknown\");\"@\";${wrellne}) + - c((.[\$key].product.size? // \"unknown\");\"@\";${wexpsz}) + - c((( if (\$ts? and (\$ts.total_tests | tonumber?) and (\$ts.total_success | tonumber?)) \ - then (\$ts.total_success | tonumber) / (\$ts.total_tests | tonumber) \ - else 0 \ - end | .*100)| r(2) |tostring);\"@\";${wqual})" \ + --arg wpackage "${wpackage}" \ + --arg wversion "${wversion}" \ + --arg wrelease "${wrelease}" \ + --arg wrellne "${wrellne}" \ + --arg wexpsz "${wexpsz}" \ + --arg wqual "${wqual}" \ + "$(jqfunctions)"' .[] | + keys[] as $key | + .[$key].product.test_status as $ts | + pr($key;"@";($wpackage | tonumber)) + + c((.[$key].product?.version? // "unknown");"@";($wversion | tonumber)) + + c((.[$key].product?.release? // "unknown");"@";($wrelease | tonumber)) + + c((.[$key].product.buildline? // "unknown");"@";($wrellne | tonumber)) + + c((.[$key].product.size? // "unknown");"@";($wexpsz | tonumber)) + + c(((if ($ts? and ($ts.total_tests | tonumber?) and ($ts.total_success | tonumber?)) then ($ts.total_success | tonumber) / ($ts.total_tests | tonumber) else 0 end | .*100)| r(2) |tostring);"@";($wqual | tonumber))' \ "${pdb}" | sed 's/@/ /g' else jq --raw-output \ - "$(jqfunctions) .[] | - keys[] as \$key | - pr(\$key;\"@\";${wpackage}) + - c(.[\$key].product.version;\"@\";${wversion}) + - c(.[\$key].product.release;\"@\";${wrelease})" \ - "${pdb}" | sed 's/@/ /g' + --arg wpackage "${wpackage}" \ + --arg wversion "${wversion}" \ + --arg wrelease "${wrelease}" \ + "$(jqfunctions)"' .[] | + keys[] as $key | + pr($key;"@";($wpackage | tonumber)) + + c(.[$key].product.version;"@";($wversion | tonumber)) + + c(.[$key].product.release;"@";($wrelease | tonumber))' \ + "${pdb}" | sed 's/@/ /g' fi elif ${versioninfo}; then jq --raw-output \ - "$(jqfunctions) .[] | - keys[] as \$key | - .[\$key].product.test_status as \$ts | - pr(\$key;\"@\";${wpackage}) + - c((.[\$key].product?.version? // \"unknown\");\"@\";${wversion})" \ - "${pdb}" | sed 's/@/ /g' + --arg wpackage "${wpackage}" \ + --arg wversion "${wversion}" \ + "$(jqfunctions)"' .[] | + keys[] as $key | + .[$key].product.test_status as $ts | + pr($key;"@";($wpackage | tonumber)) + + c((.[$key].product?.version? // "unknown");"@";($wversion | tonumber))' \ + "${pdb}" | sed 's/@/ /g' else jq --raw-output \ - '[.[] | keys[] as $key | $key] | unique[] ' \ - "${pdb}" - fi -} - -istInstalledEntries() -{ - wpackage=20 - wversion=26 - wrelease=14 - wrellne=12 - wexpsz=12 - wqual=8 - if ${urlinclude}; then - wurl=6 - urltext="URL" - else - wurl=0 - urltext="" + '[.[] | keys[] as $key | $key] | unique[] ' \ + "${pdb}" fi - if ${header} && ${details}; then - if ${full}; then - printf "${NC}${UNDERLINE}%-${wpackage}s%-${wversion}s%-${wrelease}s%-${wrellne}s%-${wexpsz}s%-${wqual}s%-${wurl}s${NC}\n" \ - "$(text_center "Package")" "$(text_center "Version")" "$(text_center "Release")" \ - "$(text_center "Releaseline")" "$(text_center "Disk usage")" \ - "$(text_center "Quality")" "$(text_center "${urltext}")" - else - printf "${NC}${UNDERLINE}%-${wpackage}s%-${wversion}s%-${wrelease}s${NC}\n" \ - "$(text_center "Package")" "$(text_center "Version")" "$(text_center "Release")" - fi - fi - pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" - if ! [ -e "${pdb}" ]; then - printWarning "No package database found. Regenerating (subsequent calls will be faster)" - updatePackageDB - fi - if ${details}; then - if ${full}; then - jq --raw-output \ - "$(jqfunctions) .[] | - keys[] as \$key | - .[\$key].product.test_status as \$ts | - pr(\$key;\"@\";${wpackage}) + - c((.[\$key].product?.version? // \"unknown\");\"@\";${wversion}) + - c((.[\$key].product?.release? // \"unknown\");\"@\";${wrelease}) + - c((.[\$key].product.buildline? // \"unknown\");\"@\";${wrellne}) + - c((.[\$key].product.size? // \"unknown\");\"@\";${wexpsz}) + - c(((if (\$ts? and (\$ts.total_tests | tonumber?) and (\$ts.total_success | tonumber?)) then (\$ts.total_success | tonumber) / (\$ts.total_tests | tonumber) else 0 end | .*100)| r(2) |tostring);\"@\";${wqual})" \ - "${pdb}" | sed 's/@/ /g' - else - jq --raw-output \ - "$(jqfunctions) .[] | - keys[] as \$key | - pr(\$key;\"@\";${wpackage}) + - c(.[\$key].product.version;\"@\";${wversion}) + - c(.[\$key].product.release;\"@\";${wrelease})" \ - "${pdb}" | sed 's/@/ /g' - fi - else - jq --raw-output \ - '[.[] | keys[] as $key | $key] | unique[] ' \ - "${pdb}" - fi } + # Main code start here # Note that functions use padding chars of '@' to work round a quirk of jq in that # it will collapse repeated spaces into tab characters - which proves irksome when tryinh @@ -300,46 +262,45 @@ verbose=false header=false details=false full=false -installed=false +available=false +installed=true +updates=false urlinclude=false # whether to include download url in output information versioninfo=false -if [ $# -eq 0 ]; then - printError "No option provided" -fi - while [ $# -gt 0 ]; do printVerbose "Parsing option: $1" case "$1" in - "--available") available=true ;; - "--installed") installed=true ;; - "--details") details=true ;; - "--full") full=true; details=true ;; - "--header") header=true ;; - "--noheader") header=false ;; - "--url") urlinclude=true ;; - "--version-info") versioninfo=true ;; - "-h" | "--help" | "-?") - printHelp - exit 0 - ;; - "--version") - zopen-version "${ME}" - exit 0 - ;; - "--verbose") - # shellcheck disable=SC2034 - verbose=true - ;; - "--xdebug") - set -x - ;; - -*) - printError "Unknown option '$1'" - ;; - *) - chosenRepos="${chosenRepos} $1" - ;; + "--available") available=true; installed=false; updates=false;; + "--installed") available=false; installed=true; updates=false;; + "--updates") available=false; installed=true; updates=true ;; + "--details") details=true ;; + "--full") full=true; details=true ;; + "--header") header=true ;; + "--noheader") header=false ;; + "--url") urlinclude=true ;; + "--version-info") versioninfo=true ;; + "-h" | "--help" | "-?") + printHelp + exit 0 + ;; + "--version") + zopen-version "${ME}" + exit 0 + ;; + "--verbose") + # shellcheck disable=SC2034 + verbose=true + ;; + "--xdebug") + set -x + ;; + -*) + printError "Unknown option '$1'" + ;; + *) + chosenRepos="${chosenRepos} $1" + ;; esac shift done From 15d752ff672c2bc9830c09a0a41bfe2ed8cf5e83 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 6 Mar 2025 17:38:36 +0000 Subject: [PATCH 069/179] Implement list of available updates (rework legacy/obsolete code) --- bin/zopen-list | 78 +++++++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 33 deletions(-) diff --git a/bin/zopen-list b/bin/zopen-list index ef45c84c5..345169081 100755 --- a/bin/zopen-list +++ b/bin/zopen-list @@ -47,38 +47,48 @@ Report bugs at https://github.com/ZOSOpenTools/meta/issues . HELPDOC } -listLatestForInstalled() +listUpdatesForInstalled() { - wpackage=20 - wversion=12 - wrelease=14 - wtag=36 - # wurl=11 Note required unless additional columns added - if ${header}; then - printf "${NC}${UNDERLINE}%-${wpackage}s%-${winstalled}s%-${wtag}s%-${wdownload}s%-${wexpsz}s%-${wqual}s${NC}\n" \ - "$(text_center "Package")" "$(text_center "Version")" "$(text_center "Release")"\ - "$(text_center "Tag Name")" "$(text_center "Download URL")" - fi + downloadJSONCache + printInfo "Packages available for update:" + pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" - installed=$(zopen list --installed | awk -v ORS=',' '{print "\""$0"\""}') - installed="[${installed%,}]" # wrap in a JSON array and strip any trailing ',' char - jq --raw-output --argjson installed "${installed}" \ - "$(jqfunctions) .release_data| - to_entries |sort_by(.key) | .[] | - .key as \$key | - select( \$installed | index(\$key)) | - .value[0].assets[0] as \$va | - \$va.url as \$url | - (\$url | match(\".*-([^-]*).([0-9]{8}_[0-9]{6}).zos.pax.Z\").captures) as \$caps | - \$caps[0].string as \$version | - \$caps[1].string as \$release | - .value[0].tag_name as \$tn | - pr(\$key;\"@\";${wpackage}) + - pr(\$version;\"@\"; ${wversion}) + - pr(\$release;\"@\"; ${wrelease}) + - pr(\$tn;\"@\";${wtag}) + - \$url " - "${JSON_CACHE}" | sed 's/@/ /g' + installed=$(zopen list --installed --details | \ + awk '{gsub(/[[:space:]]+/, "@", $0); lines=(NR==1) ? "\""$0"\"" : lines",""\""$0"\""} END {print "["lines"]"}') + #installed="[${installed%,}]" # wrap in a JSON array and strip any trailing ',' char + jq --raw-output --arg installed "${installed}" \ + "$(jqfunctions)"' def parse_installed: ($installed | fromjson) | map(split("@") | {name: .[0], version: .[1], release: .[2]}); +.release_data | to_entries | +map(.key as $pkg_name | .value[0].assets | + map(. as $asset | + (parse_installed | map(select(.name == $pkg_name)) | .[0]) as $installed_pkg | + if $installed_pkg then + $asset.version as $asset_version | + $asset.release as $asset_release | + # Compare versions + if ($installed_pkg.version != $asset_version) or ($installed_pkg.release != $asset_release) then + if ($installed_pkg.version < $asset_version) or + ($installed_pkg.version == $asset_version and $installed_pkg.release < $asset_release) then + { + "package": $pkg_name, + "installed_version": $installed_pkg.version, + "installed_release": $installed_pkg.release, + "available_version": $asset_version, + "available_release": $asset_release + } + else null end + else null end + else null end + ) +) | flatten | . as $upgrades | +($upgrades | map(.package | length) | max) as $max_packagename_length | +($upgrades | map((.installed_version + .installed_release ) | length ) | max)as $max_installedstring_length | +$upgrades | map(select(.)) | .[] +| pr(.package;" ";(($max_packagename_length | tonumber) + 4)) + + pl((.installed_version + "-" + .installed_release);" ";($max_installedstring_length + 3)) + + c("==>";" ";5) + + pr((.available_version + "-" + .available_release);" ";($max_installedstring_length + 3)) + ' "${JSON_CACHE}" | sed 's/@/ /g' } listAvailablePackages() @@ -273,7 +283,7 @@ while [ $# -gt 0 ]; do case "$1" in "--available") available=true; installed=false; updates=false;; "--installed") available=false; installed=true; updates=false;; - "--updates") available=false; installed=true; updates=true ;; + "--updates") available=false; installed=false; updates=true ;; "--details") details=true ;; "--full") full=true; details=true ;; "--header") header=true ;; @@ -314,5 +324,7 @@ if ${available}; then listAvailablePackages exit 0 fi - - +if ${updates}; then + listUpdatesForInstalled + exit 0 +fi From 32c7fbf70ceec41c644c9f54a803de6e1dd4de95 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 11 Mar 2025 14:28:08 +0000 Subject: [PATCH 070/179] Update message for clarity when nothing needs installing --- bin/zopen-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/zopen-install b/bin/zopen-install index 7fc550ba8..e6f7cb6c9 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -324,7 +324,7 @@ else fi if [ 0 -eq "$(echo "${installList}" | jq --raw-output '.installqueue| length')" ]; then - printInfo "- No packages for install" + printInfo "- No packages require installation" else if ${verbose}; then printInfo " - The following package(s) will be installed:" From 52209457351b1ac8241cb804f365cbab267c3f46 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 11 Mar 2025 16:10:49 +0000 Subject: [PATCH 071/179] Update zopen-info with agnostic repo support reinstate readonly cache processing --- bin/zopen-info | 2 +- include/common.sh | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/bin/zopen-info b/bin/zopen-info index b7a938b1d..7b26a78b1 100755 --- a/bin/zopen-info +++ b/bin/zopen-info @@ -273,6 +273,6 @@ while [ $# -gt 0 ]; do done checkIfConfigLoaded -getReposFromGithub true +getRepos true printPackageInfo "${package}" "${remote_lookup}" diff --git a/include/common.sh b/include/common.sh index 4e8c4b119..9b2766ee8 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1456,6 +1456,18 @@ getJSONCacheURL(){ updateJSONCaches() { + if [ -n "${JSON_CACHE}" ]; then + printVerbose "Cache already downloaded/checked during this session" + return 0 + fi + + from_readonly="$1" + if [ -n "${from_readonly}" ]; then + if [ -r "${JSON_CACHE}" ] && [ ! -w "${JSON_CACHE}" ]; then + return; # Skip the download for read only operations when you know you can't write to it + fi + fi + printVerbose "Ensuring cache directory exists" cachedir="${ZOPEN_ROOTFS}/var/cache/zopen" [ ! -e "${cachedir}" ] && mkdir -p "${cachedir}" @@ -1517,7 +1529,7 @@ downloadJSONCacheIfExpired() downloadJSONCache() { - if ! updateJSONCaches; then + if ! updateJSONCaches "$1"; then return 1 else return 0 @@ -1583,7 +1595,7 @@ downloadJSONCache() # 1 failure getRepos() { - updateJSONCaches + updateJSONCaches "$1" # shellcheck disable=SC2034 repo_results="$(jq -r '.release_data | keys[]' "${JSON_CACHE}")" } From 48be860dea3284abf1daa884625e69f305290b65 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 13 Mar 2025 14:55:02 +0000 Subject: [PATCH 072/179] Ensure use of /bin/echo not coreutils/shell in progresshandler --- include/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/common.sh b/include/common.sh index 9b2766ee8..737404d88 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1135,8 +1135,8 @@ progressAnimation() getNthArrayArg () { - shift "$1" - echo "$1\c" + shift "$1" + zosecho "$1\c" } waitforpid() From b279702ed7c7cf2c12f527f150c22390112fc9bf Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Sun, 16 Mar 2025 22:51:01 +0000 Subject: [PATCH 073/179] support legacy options in zopen-list --- bin/zopen-list | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/zopen-list b/bin/zopen-list index 345169081..e3da8e430 100755 --- a/bin/zopen-list +++ b/bin/zopen-list @@ -288,6 +288,8 @@ while [ $# -gt 0 ]; do "--full") full=true; details=true ;; "--header") header=true ;; "--noheader") header=false ;; + "--no-header") header=false ;; # legacy, hidden in preference to the above + "--no-version") versioninfo=false ;; # legacy/deprecated hence hidden "--url") urlinclude=true ;; "--version-info") versioninfo=true ;; "-h" | "--help" | "-?") From 6df8214803b743b4639673de2ff0cb7ad4f2032d Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Sun, 16 Mar 2025 23:00:32 +0000 Subject: [PATCH 074/179] Fix insert to error message --- include/scriptlets/installPre/verifySignatureOfPax | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/scriptlets/installPre/verifySignatureOfPax b/include/scriptlets/installPre/verifySignatureOfPax index ee133fafe..8bebd7b46 100644 --- a/include/scriptlets/installPre/verifySignatureOfPax +++ b/include/scriptlets/installPre/verifySignatureOfPax @@ -54,7 +54,7 @@ extractFromMetaData(){ # fi #fi if ! FILE_TO_VERIFY=$(extractFromMetaData ".product.pax" "${metadataFile}" "${paxFile}"); then - printSoftError "Failed to extract '${key}' from package metadata" >&2 + printSoftError "Failed to extract '.product.pax' from package metadata" >&2 return 1 fi From b62485073256ac5367117cd864e844bae258f032 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Sun, 16 Mar 2025 23:10:05 +0000 Subject: [PATCH 075/179] More granularity & specific messages to space check --- include/common.sh | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/include/common.sh b/include/common.sh index 737404d88..c6b5215ec 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2201,11 +2201,33 @@ checkIfPrereq(){ spaceValidate(){ - spaceRequiredBytes=$1 - spaceRequiredMB=$(echo "scale=0; ${spaceRequiredBytes} / (1024 * 1024)" | bc) - availableSpaceMB=$(/bin/df -m "${ZOPEN_ROOTFS}" | sed "1d" | awk '{ print $3 }' | awk -F'/' '{ print $1 }') + cacheBytes=$1 + packageBytes=$2 + + if ${reinstall}; then + # During a reinstall, the existing package size should remain constant as the + # package should overwrite the existing with the same files. However + # there might be a need to download the package into the cache - if + # autocacheclean is active, then this should be temporary; if not, then the + # cache will grow - infrom the user either way + printVerbose "Reinstall of package, so pacakge file size delta should be 0!" + spaceRequiredMB=$(echo "scale=0; (${cacheBytes}) / (1024 * 1024)" | bc) + if ! isCacheClean=$(zopen config --get autocacheclean); then + printError "Could not determine autocacheclean status" + fi + if [ "${isCacheClean}" -eq 1 ]; then + printInfo "During this operation, ${spaceRequiredMB} MB of disk space will be used." + else + printInfo "After this operation, ${spaceRequiredMB} MB of additional cache will be used." + fi + else + # If not a reinstall, assume there is a need for both the package and the expanded + # package to be required + spaceRequiredMB=$(echo "scale=0; (${cacheBytes} + ${packageBytes}) / (1024 * 1024)" | bc) + printInfo "After this operation, ${spaceRequiredMB} MB of additional disk space will be used." + fi - printInfo "After this operation, ${spaceRequiredMB} MB of additional disk space will be used." + availableSpaceMB=$(/bin/df -m "${ZOPEN_ROOTFS}" | sed "1d" | awk '{ print $3 }' | awk -F'/' '{ print $1 }') if [ "${availableSpaceMB}" -lt "${spaceRequiredMB}" ]; then printWarning "Your zopen file-system (${ZOPEN_ROOTFS}) only has ${availableSpaceMB} MB of available space." fi @@ -2263,8 +2285,9 @@ processRepoInstallFile(){ done IFS=${xIFS} fi - spaceRequiredBytes=$(echo "${installList}" | jq --raw-output '.installqueue| map(.asset.size, .asset.expanded_size)| reduce .[] as $total (0; .+($total|tonumber))') - spaceValidate "${spaceRequiredBytes}" + cacheSpaceRequired=$(echo "${installList}" | jq --raw-output '.installqueue| map(.asset.size)| reduce .[] as $total (0; .+($total|tonumber))') + actualRequiredBytes=$(echo "${installList}" | jq --raw-output '.installqueue| map(.asset.expanded_size)| reduce .[] as $total (0; .+($total|tonumber))') + spaceValidate "${cacheSpaceRequired}" "${actualRequiredBytes}" fi processActionScripts "transactionPre" From 3490324fe0db084cadc4a2778a8b34e3192c9a8f Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 17 Mar 2025 21:12:15 +0000 Subject: [PATCH 076/179] Additional error handling around alt-repos, versioning and tagged releases --- bin/zopen-init | 7 +- include/common.sh | 314 ++++++++++++++++++++++++++++------------------ 2 files changed, 196 insertions(+), 125 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index 390df312f..191faaf93 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -526,7 +526,9 @@ generateDefaultRepository() repometafile="${paxrepourl##*/}" [ "${repometafile}" = "${paxrepourl}" ] && printError "Unable to parse repository file from '${paxrepourl}'. Ensure valid format: ://://][>|&]?[ \t]*[^ \t]*//g") newcmd="${newcmd} >/dev/null 2>&1" # Command Trace MUST be disabled as the output from this can become @@ -95,7 +96,7 @@ addCleanupTrapCmd2(){ sep = OFS } } - } + } END{ printf "\n" }' ) printf "%s;%s;%s;eval exit \${exitrc}" "${prfx}" "${newcmd}" "${sigcmd}" | tr -s ';' @@ -171,7 +172,7 @@ getParentProcess() { [ -z "$1" ] && assertFailed "No process name given" # Get Parent Pid (ppid), with no heading (='') and strip blanks (awk) - ps -o ppid= -p "$1" | awk '{$1=$1; print}' + ps -o ppid= -p "$1" | awk '{$1=$1; print}' } # getCurrentVersionDir @@ -201,19 +202,19 @@ isPackageActive(){ getCurrentVersionDir "$needle" } -# Given two input files, return those lines in haystack file that are +# Given two input files, return those lines in haystack file that are # not in needles file diffFile() { haystackfile="$1" needlesfile="$2" [ -n "${needlesfile}" ] || printError "Internal error; needle file was empty/non-existent." - diff=$(awk 'NR==FNR{needles[$0];next} + diff=$(awk 'NR==FNR{needles[$0];next} !($0 in needles) {print}' "${needlesfile}" "${haystackfile}") echo "${diff}" } -# Given two input lists (with \n delimiters), return those lines in +# Given two input lists (with \n delimiters), return those lines in # haystack that are not in needles diffList() { @@ -900,7 +901,7 @@ mergeIntoSystem() printDebug "Generating listing for remove processing (including main symlink)" echo "Installed files:" > "${versioneddir}/.links" tar tf "${processingDir}/${tarfile}" 2> /dev/null| sort -r >> "${versioneddir}/.links" - + printDebug "Extracting tar to rootfs" cd "${processingDir}" && tar xf "${tarfile}" -C "${rootfs}" 2> /dev/null @@ -915,7 +916,7 @@ mergeIntoSystem() rmSymlinksFSCheck(){ # Slower method needed to analyse each link to see if it has - # become orphaned. Only relevent when removing a package as + # become orphaned. Only relevent when removing a package as # upgrades/alt-switching can supply a list of files # Use sed to skip header line in .links file # Note that the contents of the links file are ordered such that @@ -966,7 +967,7 @@ rmSymlinksFileDiff(){ if [ -L "${obsoleteFile}" ] && [ ! -e "${obsoleteFile}" ]; then # the linked-to file no longer exists (ie. the symlink is dangling) rm -f "${obsoleteFile}" > /dev/null 2>&1 - fi + fi done } @@ -998,7 +999,7 @@ unsymlinkFromSystem() if [ -e "${tempDirFile}" ]; then ndirs=$(uniq < "${tempDirFile}" | wc -l | tr -d ' ') printVerbose "- Checking ${ndirs} dir links" - for d in $(uniq < "${tempDirFile}" | sort -r) ; do + for d in $(uniq < "${tempDirFile}" | sort -r) ; do [ -d "${d}" ] && rmdir "${d}" >/dev/null 2>&1 done rm "${tempDirFile}" @@ -1010,7 +1011,7 @@ unsymlinkFromSystem() done < "${tempTrash}" printError "Manual removal of files might be required" fi - fi + fi else printDebug "No list of current links to check - package was not installed/active" fi @@ -1148,9 +1149,9 @@ waitforpid() progressHandler() { - + if [ -z "${-%%*x*}" ]; then - # Command trace is active so any progress animation + # Command trace is active so any progress animation # writing to screen will interleave, making things cluttered. # Sleep for 1s (to allow the caller to setup signal handling) and exit sleep 1 @@ -1440,16 +1441,21 @@ syslog() echo "$(date +"%F %T") $(id | cut -d' ' -f1)::${module}:${type}:${categories}:${location}:${msg}" >> "${fd}" } -getJSONCacheURL(){ +getJSONCacheURLs(){ activeRepo="${ZOPEN_ROOTFS}/etc/zopen/repos.d/active" [ ! -e "${activeRepo}" ] && printError "Could not access repository configuration at '${activeRepo}'. Check file to ensure valid repository configuration or refresh default configuration with zopen init --refresh -y." type=$(jq -r ".type" "${activeRepo}") base=$(jq -r ".metadata_baseurl" "${activeRepo}") filename=$(jq -r ".metadata_file" "${activeRepo}") + latest_metadata=$(jq -r ".latest_metadata" "${activeRepo}") case "${type}" in - http|https) printf "%s://%s/%s" "${type}" "${base}" "${filename}";; - file) printf "%s:%s/%s" "${type}" "${base}" "${filename}";; + http|https) printf "%s://%s/%s\n%s://%s/%s" \ + "${type}" "${base}" "${filename}" "${type}" "${base}" "${latest_metadata}" + ;; + file) printf "%s:%s/%s\n%s:%s/%s" \ + "${type}" "${base}" "${filename}" "${type}" "${base}" "${latest_metadata}" + ;; *) printError "Unsupported repository type '${type}'.";; esac } @@ -1471,17 +1477,17 @@ updateJSONCaches() printVerbose "Ensuring cache directory exists" cachedir="${ZOPEN_ROOTFS}/var/cache/zopen" [ ! -e "${cachedir}" ] && mkdir -p "${cachedir}" - jsonCacheURL=$(getJSONCacheURL) + jsonCacheURLs=$(getJSONCacheURLs) # Also sets $latest_metadata printVerbose "Checking if the JSON_CACHE already downloaded in this session" if [ -z "${JSON_CACHE}" ]; then JSON_CACHE="${cachedir}/zopen_releases.json" - downloadJSONCacheIfExpired "${JSON_CACHE}" "${jsonCacheURL}" + downloadJSONCacheIfExpired "${JSON_CACHE}" "$(echo "${jsonCacheURLs}" | head -n 1)" fi if [ -z "${JSON_LATEST_CACHE}" ]; then JSON_LATEST_CACHE="${cachedir}/zopen_releases_latest.json" - latestReleaseURL="$(dirname "${jsonCacheURL}")/zopen_releases_latest.json" - downloadJSONCacheIfExpired "${JSON_LATEST_CACHE}" "${latestReleaseURL}" + latestReleaseURL="$(dirname "${jsonCacheURL}")/${latest_metadata}" + downloadJSONCacheIfExpired "${JSON_LATEST_CACHE}" "$(echo "${jsonCacheURLs}" | tail -n 1)" fi } @@ -1503,7 +1509,7 @@ downloadJSONCacheIfExpired() [ ! -w "${fileToCache}" ] || [ ! -r "${fileToCache}" ] && printError "Cannot access cache at '${JSON_CACHE}'. Check permissions and retry request." fi - if ! curlCmd -f -L -s -I "${cacheUrl}" -o "${cacheTimestampCurrent}"; then + if ! curlCmd --fail --location --silent --head "${cacheUrl}" -o "${cacheTimestampCurrent}"; then printError "Failed to obtain json cache timestamp from ${cacheUrl}." fi chtag -tc 819 "${cacheTimestampCurrent}" @@ -1518,7 +1524,7 @@ downloadJSONCacheIfExpired() printVerbose "Replacing old timestamp with latest." mv -f "${cacheTimestampCurrent}" "${cacheTimestamp}" - if ! curlCmd -f -L -s -o "${fileToCache}" "${cacheUrl}"; then + if ! curlCmd --fail --location --silent -o "${fileToCache}" "${cacheUrl}"; then printError "Failed to obtain json cache from '${cacheUrl}'" fi chtag -tc 819 "${fileToCache}" @@ -1547,7 +1553,7 @@ downloadJSONCache() return; # Skip the download for read only operations when you know you can't write to it fi fi - + # Need to check that we can read & write to the JSON timestamp cache files if [ -e "${JSON_TIMESTAMP_CURRENT}" ]; then [ ! -w "${JSON_TIMESTAMP_CURRENT}" ] || [ ! -r "${JSON_TIMESTAMP_CURRENT}" ] && printError "Cannot access cache at '${JSON_TIMESTAMP_CURRENT}'. Check permissions and retry request." @@ -1601,7 +1607,7 @@ getRepos() } # isValidRepo -# Queries the main repository list to determine if the input is valid, This +# Queries the main repository list to determine if the input is valid, This # uses jq itself to return 0 or 1 with no output # inputs: $1 the port name. @@ -1618,7 +1624,7 @@ getRepoReleases() { updateJSONCaches repo="$1" - ##TODO + ##TODO ##TDORM releases="$(jq -e -r '.release_data."'${repo}'"' "${JSON_CACHE}")" ##TDORM if [ $? -ne 0 ]; then ##TDORM printError "Could not get all releases for ${repo}" @@ -1654,7 +1660,7 @@ checkWritable() fi } -generateUUID() +generateUUID() { date_part=$(date +%s) random_part=$((RANDOM)) @@ -1674,8 +1680,8 @@ isURLReachable() { } checkAvailableSize() -{ - +{ + package="$1" packageSize="$2" printInfo "- Checking available size to install ${package}." @@ -1685,7 +1691,7 @@ checkAvailableSize() printDebug "Package Size: ${packageSize} k" partitionSize=$(/bin/df -k . | tail -1 | awk '{print $3}' | cut -f1 -d '/') printDebug "Partition Size: ${partitionSize}k [free on '$(pwd -P)']" - + if [ 1 -eq "$(echo "${packageSize} > ${partitionSize}" | bc)" ]; then printError "Not enough space in partition." fi @@ -1733,7 +1739,7 @@ asciiecho() return 0 } -a2e() +a2e() { source="$1" @@ -1793,38 +1799,56 @@ promptYesNoAlways() { getVersionedMetadata() { + repo=$1 + invalidPortAssetFile=$2 printDebug "Specific version ${versioned} requested - checking existence and URL" - requestedMajor=$(echo "${versioned}" | awk -F'.' '{print $1}') - requestedMinor=$(echo "${versioned}" | awk -F'.' '{print $2}') - requestedPatch=$(echo "${versioned}" | awk -F'.' '{print $3}') - requestedSubrelease=$(echo "${versioned}" | awk -F'.' '{print $4}') - requestedVersion="${requestedMajor}\\\.${requestedMinor}\\\.${requestedPatch}\\\.${requestedSubrelease}" - printDebug "Finding URL for latest release matching version prefix: requestedVersion: ${requestedVersion}" - releasemetadata=$(jq --arg repo "${repo}" --arg requestedVersion "${requestedVersion}" \ - '.release_data[$repo] | map(select(.assets[].version | test($requestedVersion)))[0]' "${JSON_CACHE}") + requestedVersion=$(echo "${versioned}" | awk -F'.' '{print $4}') + printDebug "Finding metadata for latest release matching version prefix: requestedVersion: ${versioned}" + if ! releasemetadata=$(jq --arg repo "${repo}" --arg requestedVersion "${versioned}" \ + '.release_data[$repo] | map(select(.assets[].name | test($requestedVersion)))[0]' "${JSON_CACHE}"); then + printSoftError "Could not find metadata for ${repo} in repo metadata '${JSON_CACHE}'" 2> "${invalidPortAssetFile}" + return 1 + elif [ "${releasemetadata}" = "null" ]; then + printSoftError "Could not find specified version '${versioned}' for package '${repo}'" 2> "${invalidPortAssetFile}" + return 1 + fi + printDebug "Check for asset" + asset=$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]') + if ! asset=$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]'); then + printSoftError "Could not find asset for release version '${versioned}' in repo '${repo}'" 2> "${invalidPortAssetFile}" + return 1 + fi + return 0 } getTaggedMetadata() { + repo=$1 + invalidPortAssetFile=$2 printDebug "Explicit tagged version '${tagged}' specified. Checking for match" - releasemetadata=$(/bin/printf "%s" "${releases}" | jq -e -r '.[] | select(.tag_name == "'"${tagged}"'")') - releasemetadata=$(jq --arg repo "${repo}" --arg requestedVersion "${requestedVersion}" \ - '.release_data[$repo][] | select(.tag_name == $tag_name)' "${JSON_CACHE}") - printDebug "Use quick check for asset to check for existence of metadata for specific messages" + if ! releasemetadata=$(jq -e --arg repo "${repo}" --arg tagged "${tagged}" \ + '.release_data[$repo][] | select(.tag_name == $tagged)' "${JSON_CACHE}"); then + printSoftError "Tagged release '${tagged}' was not found for ${repo}" 2> "${invalidPortAssetFile}" + return 1 + fi + printDebug "Check for asset" asset=$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]') - if [ $? -ne 0 ]; then - printError "Could not find release tagged '${tagged}' in repo '${repo}'" + if ! asset=$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]'); then + printSoftError "Could not find asset for release tagged '${tagged}' in repo '${repo}'" 2> "${invalidPortAssetFile}" + return 1 fi + return 0 } getSelectMetadata() { # As this is running within the generate... logic, a progress handler will have been started. - # This needs to be terminated before trying to write to screen + # This needs to be terminated before trying to write to screen. This does mean that messages + # can be written to the screen rather than directed to a temporary error file # shellcheck disable=SC2154 kill -HUP "${PROGRESS_HANDLER}" >/dev/null 2>&1 # if the timer is not running, the kill will fail waitforpid "${PROGRESS_HANDLER}" # Make sure it's finished writing to screen - + repo="$1" # Explicitly allow the user to select a release to install; useful if there are broken installs # as a known good release can be found, selected and pinned! @@ -1852,19 +1876,25 @@ getSelectMetadata() } getReleaseLineMetadata() -{ +{ + repo=$1 + invalidPortAssetFile=$2 printDebug "Install from release line '${releaseLine}' specified" validatedReleaseLine=$(validateReleaseLine "${releaseLine}") if [ -z "${validatedReleaseLine}" ]; then - printError "Invalid releaseline specified: '${releaseLine}'; Valid values: DEV or STABLE" + printSoftError "Invalid releaseline specified: '${releaseLine}'; Valid values: DEV or STABLE" 2> "${invalidPortAssetFile}" + return 1 fi printDebug "Finding latest asset on the release line" - releasemetadata=$(jq --arg repo "${repo}" --arg releaseLine "${validatedReleaseLine}" \ - '.release_data[$repo] | map(select(.tag_name | startswith($releaseLine)))[0]' "${JSON_CACHE}") + if ! releasemetadata=$(jq --arg repo "${repo}" --arg releaseLine "${validatedReleaseLine}" \ + '.release_data[$repo] | map(select(.tag_name | startswith($releaseLine)))[0]' "${JSON_CACHE}"); then + printSoftError "Could not find metadata for ${repo} in repository metadata at '${JSON_CACHE}'" 2> "${invalidPortAssetFile}" + return 1 + fi printDebug "Use quick check for asset to check for existence of metadata" - asset="$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')" - if [ $? -ne 0 ]; then - printError "Could not find release-line ${releaseLine} releases for repo: ${repo}" + if ! asset="$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')"; then + printSoftError "Could not find asset for releaseline '${validatedReleaseLine}' for repo '${repo}' in ${JSON_CACHE}" 2> "${invalidPortAssetFile}" + return 1 fi unset "${releaseLine}" } @@ -1876,7 +1906,7 @@ calculateReleaseLineMetadata() if [ -n "${installedReleaseLine}" ]; then printDebug "Found existing releaseline '${installedReleaseLine}', restricting to only that releaseline" validatedReleaseLine="${installedReleaseLine}" # Already validated when stored - else + else printDebug "Checking for system-configured releaseline" if [ -e "${ZOPEN_JSON_CONFIG}" ]; then printDebug "Using v2 configuration: '${ZOPEN_JSON_CONFIG}'" @@ -1955,9 +1985,11 @@ parseRepoName() } getPortMetaData(){ + # This is running inside a progresshandler - route error messages to + # $2 portRequested="$1" invalidPortAssetFile="$2" - printDebug "Removing any version (%) or tag (#) suffixes fron '${portRequested}" + printDebug "Removing any version (=) or tag (%) suffixes fron '${portRequested}" portName=$(echo "${portRequested}" | sed -e 's#%.*##' -e 's#=.*##') if ! isValidRepo "${portName}"; then @@ -1968,29 +2000,39 @@ getPortMetaData(){ parseRepoName "${portRequested}" # To set the various status flags below getRepoReleases "${portName}" if [ -n "${versioned}" ]; then - getVersionedMetadata "${portName}" + if ! getVersionedMetadata "${portName}" "${invalidPortAssetFile}"; then + return 1 + fi elif [ -n "${tagged}" ]; then - getTaggedMetadata "${portName}" + if ! getTaggedMetadata "${portName}" "${invalidPortAssetFile}"; then + return 1 + fi elif # shellcheck disable=SC2154 ${selectVersion}; then selectVersion=false # Need to set this to prevent selection of dependencies - getSelectMetadata "${portName}" - elif [ -n "${releaseLine}" ]; then - getReleaseLineMetadata "${portName}" + if ! getSelectMetadata "${portName}" "${invalidPortAssetFile}"; then + return 1 + fi + elif [ -n "${releaseLine}" ]; then + if ! getReleaseLineMetadata "${portName}" "${invalidPortAssetFile}"; then + return 1 + fi else - calculateReleaseLineMetadata "${portName}" - fi - if [ -z "${releasemetadata}" ]; then - echo "${portName}: metadata could not be found" >> "${invalidPortAssetFile}" - return 1 + if ! calculateReleaseLineMetadata "${portName}" "${invalidPortAssetFile}"; then + return 1 + fi fi - printDebug "Getting specific asset details using metadata: ${releasemetadata}" if [ -z "${asset}" ] || [ "null" = "${asset}" ]; then - printDebug "Asset not found during previous logic; setting now" + printDebug "Asset not found during previous logic; setting now from metadata" + if [ -z "${releasemetadata}" ]; then + echo "${portName}: metadata could not be found" >> "${invalidPortAssetFile}" + return 1 + fi + printDebug "Getting specific asset details using metadata: ${releasemetadata}" asset=$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]') fi if [ -z "${asset}" ]; then - echo "${portName} asset metadata could not be found" >> "${invalidPortAssetFile}" + echo "${portName} asset metadata could not be found" 2> "${invalidPortAssetFile}" return 1 fi return 0 @@ -2006,7 +2048,7 @@ getPortMetaData(){ # 8 if error createDependancyGraph() { - invalidPortAssetFile=$1 && shift + invalidPortAssetFile=$1 && shift printDebug "Getting list of dependencies" dependencies=$(echo "${installList}" | jq --raw-output '.installqueue[] | select(.asset.runtime_dependencies | test("No dependencies") | not )| map(try(.runtime_dependencies |= split(" ")))| .[] | .runtime_dependencies[] ') printDebug "Removing any dependencies already on install queue" @@ -2035,7 +2077,7 @@ createDependancyGraph() # 8 if error addToInstallGraph(){ installtype=$1 && shift - invalidPortAssetFile=$1 && shift + invalidPortAssetFile=$1 && shift pkgList="$1" printDebug "Adding pkgList to install graph" for portRequested in ${pkgList}; do @@ -2046,13 +2088,8 @@ addToInstallGraph(){ installList=$(echo "${installList}" | \ jq ".installqueue += [{\"portname\":\"${portName}\", \"asset\":${asset}, \"installtype\":\"${installtype}\"}]") done - if [ -e "${invalidPortAssetFile}" ]; then - printSoftError "The following ports cannot be installed: " - while read invalidPort; do - printf "${WARNING} %s\n" "${invalidPort}" - done < "${invalidPortAssetFile}" - printError "Confirm port names, remove any 'port' suffixes and retry command." - fi + [ -e "${invalidPortAssetFile}" ] && return 1 + return 0 } validateInstallList(){ @@ -2072,7 +2109,7 @@ validateInstallList(){ dedupStringList() { delim="$1" && shift str="$1" - echo "${str}"| awk -v delim="${delim}" ' + echo "${str}"| awk -v delim="${delim}" ' { dlm=""; for (i=1; i<=NF; i++) {if (!seen[$i]++) {printf "%s%s", dlm, $i};dlm=delim};print ""}' } @@ -2093,10 +2130,18 @@ generateInstallGraph(){ # Create the following file here to trigger cleanup - otherwise, multiple # tempfiles could be created depending on dependency graph depth invalidPortAssetFile=$(mktempfile "invalid" "port") - addCleanupTrapCmd "rm -rf ${invalidPortAssetFile}" + #addCleanupTrapCmd "rm -rf ${invalidPortAssetFile}" if ! runLogProgress " addToInstallGraph \"install\" \"${invalidPortAssetFile}\" \"${portsToInstall}\"" \ "Creating install graph" "Created install graph" "linkcheck"; then - printError "Unexpected error whilecreating install graph. Correct errors and retry command." + if [ -e "${invalidPortAssetFile}" ]; then + printSoftError "Cannot complete install request" + while read invalidPort; do + printf "%s\n" "${invalidPort}" + done < "${invalidPortAssetFile}" + exit 1 + else + printError "Unexpected error while creating install graph. Correct errors and retry command." + fi fi ##TODORM addToInstallGraph "install" "${invalidPortAssetFile}" "${portsToInstall}" @@ -2108,7 +2153,15 @@ generateInstallGraph(){ printVerbose "Calculating dependancy graph" if ! runLogProgress "createDependancyGraph \"${invalidPortAssetFile}\"" \ "Creating dependancy graph" "Created dependancy graph" "linkcheck"; then - printError "Unexpected error while creating dependancy graph. Correct errors and retry command." + if [ -e "${invalidPortAssetFile}" ]; then + printSoftError "The following ports cannot be installed: " + while read invalidPort; do + /bin/printf "${WARNING} %s\n" "${invalidPort}" + done < "${invalidPortAssetFile}" + printError "Confirm port names, remove any 'port' suffixes and retry command." + else + printError "Unexpected error while creating dependancy graph. Correct errors and retry command." + fi fi ##TODORM createDependancyGraph "${invalidPortAssetFile}" else @@ -2125,11 +2178,11 @@ generateInstallGraph(){ #fi ##<> @@ -2166,7 +2219,7 @@ hidden(){ # should be unique enough installed=$(echo "${installed}"| awk 'BEGIN{ORS = "," } {print "\"" $1 "@=@" $3 "\""}') installed="[${installed%,}]" - + installList=$(echo "${installList}" | \ jq --argjson installees "${installed}" \ '.installqueue |= @@ -2180,7 +2233,7 @@ hidden(){ ) )'\ ) -} +} ##</dev/null 2>&1; then xIFS=$IFS @@ -2315,9 +2368,8 @@ processRepoInstallFile(){ getInstallFile() { - installurl="$1" - metadataJSONURL="$(dirname "${installurl}")/metadata.json" - metadataFile="$(basename "${installurl}").json" + installurl="$1" + downloadToDir="${ZOPEN_ROOTFS}/var/cache/zopen" if $downloadOnly; then downloadToDir="." @@ -2333,15 +2385,31 @@ getInstallFile() if ! runAndLog "cd ${downloadToDir} && curlCmd --no-progress-meter -L ${installurl} -O ${redirectToDevNull}"; then printError "Could not download from ${installurl}. Correct any errors and potentially retry" fi + printVerbose "Downloading corresponding metadata" + # check if it is in the same location just with a different suffix (as in a mirror) + # if not, likely is the original Github repo which uses a subdirectory for the metadata... + # Try again with this extended URL + metadataFile="$(basename "${installurl}").json" + metadataJSONURL="${installurl}.json" if ! runAndLog "curlCmd -L '${metadataJSONURL}' -o '${metadataFile}'" "${redirectToDevNull}"; then - printError "Could not download from ${metadataJSONURL}. Correct any errors and potentially retry." - else - if command -v chtag >/dev/null 2>&1; then - # Curl currently does not know on z/OS to set the text flag for text files - printVerbose "Metadata file downloaded, ensuring 'text' flag set for z/OS" - chtag -t "${metadataFile}" + printVerbose "Not located explicit file, try extended URL location" + metadataJSONURL_ext="$(dirname "${installurl}")/metadata.json" + if ! runAndLog "curlCmd -L '${metadataJSONURL_ext}' -o '${metadataFile}'" "${redirectToDevNull}"; then + printSoftError "Could not download package metadata. Tried the following locations:" + printSoftErrpr "'${metadataJSONURL}' '${metadataJSONURL_ext}'" + printError "Correct any errors and retry command" + else + printVerbose "Metadata downloaded from extended location '${metadataJSONURL_ext}'" fi + else + printVerbose "Metadata downloaded from '${metadataJSONURL}'" + fi + + if command -v chtag >/dev/null 2>&1; then + # Curl currently does not know on z/OS to set the text flag for text files + printVerbose "Metadata file downloaded, ensuring 'text' flag set for z/OS" + chtag -t "${metadataFile}" fi fi } @@ -2370,8 +2438,8 @@ installFromPax() return 1 fi - # Ideally we would use the following, - # name=$(jq --raw-output '.product.name' "${metadatafile}") + # Ideally we would use the following, + # name=$(jq --raw-output '.product.name' "${metadatafile}") # but name does not always map to the actual repo package name at present! # The repo name is in the.product.repo field so can extract from there instead - # though this also has issues for some packages like NATS/nats ... @@ -2379,14 +2447,14 @@ installFromPax() # so fall back to that if [ -n "${USEPRODNAME}" ]; then printVerbose "Extracting product name from .product.repo" - # name=$(jq --raw-output '.product.name' "${metadatafile}") + # name=$(jq --raw-output '.product.name' "${metadatafile}") else printVerbose "Extracting product name from .product.repo" name=$(jq --arg reponame "${ZOPEN_ORGNAME}" --raw-output '.product.repo | match(".*/\($reponame)/(.*)port").captures[0].string' "${metadatafile}") - if [ -z "${name}" ] || [ "${name##*[^ ]*}" = "" ]; then + if [ -z "${name}" ] || [ "${name##*[^ ]*}" = "" ]; then name=$(jq --arg reponame "ZOSOpenTools" --raw-output '.product.repo | match(".*/\($reponame)/(.*)port").captures[0].string' "${metadatafile}") fi - if [ -z "${name}" ] || [ "${name##*[^ ]*}" = "" ]; then + if [ -z "${name}" ] || [ "${name##*[^ ]*}" = "" ]; then printError "Unable to determine name from .product.repo in '${metadatafile}'. Check metadata is correct." fi fi @@ -2406,7 +2474,7 @@ installFromPax() paxredirect="-s %[^/]*/%${ZOPEN_PKGINSTALL}/${installdirname}/%" printDebug "Check for existing directory for version '${installdirname}'" - if [ -d "${ZOPEN_PKGINSTALL}/${installdirname}" ]; then + if [ -d "${ZOPEN_PKGINSTALL}/${installdirname}" ]; then printVerbose "- Clearing existing directory and contents" rm -rf "${ZOPEN_PKGINSTALL}/${installdirname}" fi @@ -2432,14 +2500,14 @@ installFromPax() fi if ! ln -s "${ZOPEN_PKGINSTALL}/${installdirname}" "${ZOPEN_PKGINSTALL}/${name}/${name}"; then printError "Could not create symbolic link name" - fi + fi if ! ${nosymlink}; then if ! runLogProgress "mergeIntoSystem \"${name}\" \"${ZOPEN_PKGINSTALL}/${installdirname}\" \"${ZOPEN_ROOTFS}\"" \ "Merging ${name} into symlink mesh" "Merged ${name} into symlink mesh"; then printSoftError "Unexpected errors merging symlinks into mesh" printError "Use zopen alt to select previous version to ensure known state" fi - ##TODORM mergeIntoSystem "${name}" "${ZOPEN_PKGINSTALL}/${installdirname}" "${ZOPEN_ROOTFS}" + ##TODORM mergeIntoSystem "${name}" "${ZOPEN_PKGINSTALL}/${installdirname}" "${ZOPEN_ROOTFS}" ##TODORM misrc=$? ##TODORM printDebug "The merge complete with: ${misrc}" fi @@ -2475,7 +2543,7 @@ EOF touch "${ZOPEN_PKGINSTALL}/${name}/${name}/.active" installedList="${name} ${installedList}" syslog "${ZOPEN_LOG_PATH:-${ZOPEN_ROOTFS}/var/log}/audit.log" "${LOG_A}" "${CAT_INSTALL},${CAT_PACKAGE}" "DOWNLOAD" "handlePackageInstall" "Installed package:'${name}';version:${downloadFileVer};install_dir='${baseinstalldir}/${installdirname}';" - addToInstallTracker "${name}" + addToInstallTracker "${name}" # Some installation have installation caveats installCaveat=$(jq -r '.product.install_caveats // empty' "${metadatafile}" 2>/dev/null) if [ -n "$installCaveat" ]; then @@ -2542,7 +2610,7 @@ processActionScripts() # shellcheck disable=SC2240 . "$scriptletFile" "$@" echo $? # Append exit status to output - } 2>&1 + } 2>&1 ) /bin/printf "${CRSRSOL}${ERASELINE}${CRSRSOL}" scriptletRc=$(echo "${scriptletOutput}" | tail -n 1) # Extract exit status @@ -2639,12 +2707,12 @@ stripControlCharacters(){ # JSONcontrolChar2Unicode # ensures escaping of characters in JSON. For example, if a caveat has an # unescaped '\n'/0x0A jq will fail to process it. Escape control characters -# 0x00->0x1F and reverse solidus. Note this should also only process unescaped -# sequences by checking whether the character prior to the sequence is not a +# 0x00->0x1F and reverse solidus. Note this should also only process unescaped +# sequences by checking whether the character prior to the sequence is not a # reverse-solidus (so start-of-line [^] or any character [^\] ). If there was # a preceding character, then capture/use that in the regex (\1) so it does not # get discarded -# +# # inputs: $1 the input JSON file # $2 the output file, with sanitised JSON # return: 0 for success (nb. Warnings may haeve been printed to screen) @@ -2653,22 +2721,22 @@ JSONcontrolChar2Unicode() { [ ! -f "$1" ] && assertFailed "No input file specified for parsing!" [ -e "$2" ] && assertFailed "Output file exists so cannot be used for output!" zossed -E ' # Note this is a long string! - s/\\/\\\\/g; # Escape reverse-solidus; the following are the control chars + s/\\/\\\\/g; # Escape reverse-solidus; the following are the control chars s/(^|[^\\])\x00/\1\\u0000/g; s/(^|[^\\])\x01/\1\\u0001/g; s/(^|[^\\])\x02/\1\\u0002/g; s/(^|[^\\])\x03/\1\\u0003/g; - s/(^|[^\\])\x04/\1\\u0004/g; s/(^|[^\\])\x05/\1\\u0005/g; + s/(^|[^\\])\x04/\1\\u0004/g; s/(^|[^\\])\x05/\1\\u0005/g; s/(^|[^\\])\x06/\1\\u0006/g; s/(^|[^\\])\x07/\1\\u0007/g; s/(^|[^\\])\x08/\1\\u0008/g; s/(^|[^\\])\x09/\1\\u0009/g; - s/(^|[^\\])\x0A/\1\\u000A/g; s/(^|[^\\])\x0B/\1\\u000B/g; + s/(^|[^\\])\x0A/\1\\u000A/g; s/(^|[^\\])\x0B/\1\\u000B/g; s/(^|[^\\])\x0C/\1\\u000C/g; s/(^|[^\\])\x0D/\1\\u000D/g; s/(^|[^\\])\x0E/\1\\u000E/g; s/(^|[^\\])\x0F/\1\\u000F/g; - s/(^|[^\\])\x10/\1\\u0010/g; s/(^|[^\\])\x11/\1\\u0011/g; + s/(^|[^\\])\x10/\1\\u0010/g; s/(^|[^\\])\x11/\1\\u0011/g; s/(^|[^\\])\x12/\1\\u0012/g; s/(^|[^\\])\x13/\1\\u0013/g; s/(^|[^\\])\x14/\1\\u0014/g; s/(^|[^\\])\x15/\1\\u0015/g; - s/(^|[^\\])\x16/\1\\u0016/g; s/(^|[^\\])\x17/\1\\u0017/g; + s/(^|[^\\])\x16/\1\\u0016/g; s/(^|[^\\])\x17/\1\\u0017/g; s/(^|[^\\])\x18/\1\\u0018/g; s/(^|[^\\])\x19/\1\\u0019/g; s/(^|[^\\])\x1A/\1\\u001A/g; s/(^|[^\\])\x1B/\1\\u001B/g; - s/(^|[^\\])\x1C/\1\\u001C/g; s/(^|[^\\])\x1D/\1\\u001D/g; + s/(^|[^\\])\x1C/\1\\u001C/g; s/(^|[^\\])\x1D/\1\\u001D/g; s/(^|[^\\])\x1E/\1\\u001E/g; s/(^|[^\\])\x1F/\1\\u001F/g; ' "$1" > "$2" } @@ -2729,7 +2797,7 @@ jqfunctions() # pl(s;c;n) - padLeft with character 'c' to length 'n' # pr(s;c;n) - padRight with chacater 'c' to length 'n' # c(s;l) - center the string 's' in a string of length 'l' - # r(dp) - round decimal to 'dp' decimal places (needs '*' & '/' 10^dp hack) + # r(dp) - round decimal to 'dp' decimal places (needs '*' & '/' 10^dp hack) # shellcheck disable=SC2016 printf "%s;%s;%s;%s;" \ 'def pl(s;c;n):c*(n-(s|length))+s' \ From 7d5e96cf41f35c150cf59ae4650b0d895f5cac7d Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 18 Mar 2025 20:55:24 +0000 Subject: [PATCH 077/179] Add zopen-mirror capability Also adds missing "SEE ALSO" info for list & upgrade --- bin/zopen | 6 +- bin/zopen-mirror | 853 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 858 insertions(+), 1 deletion(-) create mode 100755 bin/zopen-mirror diff --git a/bin/zopen b/bin/zopen index b5e2c9be1..0f179cda1 100755 --- a/bin/zopen +++ b/bin/zopen @@ -40,6 +40,7 @@ Command: install installs one or more zopen community packages info displays detailed information about a package list lists information about zopen community packages + mirror performs a synchronization of the GitHub repository query list local or remote info about zopen community packages remove removes installed zopen community packages update-cacert update the cacert.pem file used by zopen community @@ -70,9 +71,12 @@ SEE ALSO: zopen-init(1) zopen-install(1) zopen-info(1) + zopen-list(1) + zopen-mirror(1) zopen-query(1) zopen-remove(1) zopen-update-cacert(1) + zopen-upgrade(1) zopen-usage(1) zopen-version(1) @@ -93,7 +97,7 @@ for arg in $*; do case "${arg}" in "alt"|"audit"|"build"|"clean"|"generate"|"init"|"install"|\ "query"|"remove"|"update-cacert"|"info"|"usage"|"diagnostics"|\ - "list"|"upgrade") + "list"|"upgrade"|"mirror") subcmd="zopen-${arg}" ;; "refresh") diff --git a/bin/zopen-mirror b/bin/zopen-mirror new file mode 100755 index 000000000..111635d0f --- /dev/null +++ b/bin/zopen-mirror @@ -0,0 +1,853 @@ +#!/bin/sh -x +# Repo mirror utility - stands alone from other scripts so has no +# direct dependencies on existing functions. As such, it has some +# cut-down versions of common features and an independent version +VERSION=0.4.0 + + +# Global constants +ZOPEN_PROJECT_NAME="zopen community" +ZOPEN_GITHUB="https://github.com/zopencommunity" +RELEASE_JSON_URL="https://api.github.com/repos/zopencommunity/meta/contents/docs/api/zopen_releases.json?ref=main" +LATEST_JSON_URL="https://api.github.com/repos/zopencommunity/meta/contents/docs/api/zopen_releases_latest.json?ref=main" + +# Help text +printHelp() { + cat <
--target /zotrepo + + FTP - creates a mirror in the /parent/ftp.repo on the remote machine, using + credentials stored in a .netrc file for both the ftp server and + GitHub. + ${0} --type ftp --netrc --target /%2Fparent/ftp.repo --genRepoFile + +Description: + ${0} is a small utility that will mirror the packages made available by the + ${ZOPEN_PROJECT_NAME} for instalation using the zopen package manager. + The mirror will download - and upload if required - the packages from the + various GitHub repositories to the location specified and optionally + provide configuration files that can be used to configure the zopen package + manager to use the alternative repository. + +Details: + The currently supported mirror types are: + FLAT - mirrors the GH contents into a single directory on the local + system. For example: /zopen/repo.mirror + ALPHA - mirrors the GH contents into a local directory structure where the + heirarchy is the initial package letter as a directory name containing + packages starting with that letter. For example: //a/... + NAME - mirrors the GH contents into a unique local directory per package. + Each directory would contain only releases for that package. For + example: //automake + ALPHANAME - A combination of ALPHA and NAME. For example: /a/automake + FTP - Mirrors the GH repo to a remote ftp server. Specify the --target + in the following format: "/". Note that + if contains a leading '/' this will need to be + replaced with '%2F'; if there is no leading '/', the mirror will be + to a directory within the users home directory [the directory where + the user will start in after login]. Requires either the --netrc or + --dstuser and --dstpassword parameters for credentials where anonymous + login is disabled. Note that ftp is inherently insecure! + SFTP - As 'FTP'' but utilises the SFTP protocol + SAMBA - As 'FTP' but uses the SAMBA protocol, allowing Windows or Linux + network shares. --target should be the URL with share but no leading + protocol definition. For example, samba.example.com/share_path - + smb://samba.example.com/share_path will fail + + NEXUS - Utilises a Sonatype Nexus Repository. Requires either the --netrc or + --dstuser and --dstpassword parameters for credentials + +Parameter details: + --genRepoFile + The --genRepoFile parameter will create a file that is configured for the + specified mirror type. For example, running the mirror for an FTP-type + mirror will produce a file with the contents: + { + "type": "ftp", + "metadata_baseurl": "ftpserver.name/%2Fzopen/repo.mirror")", + "metadata_file": "metadata.json" + } + This file can be placed into the /etc/zopen/repos.d directory and + then symlinked to be the active version. + --activate [ZOPEN_ROOTFS] + The --activate parameter will move a generated repository description file + [as generated with the --genRepoFile parameter] to be the active zopen + repository for the given zopen environment. Note this does require running + in a z/OS environment with authority to write to the + /etc/zopen/repos.d directory. + --netrc + The underlying curl process will use a .netrc to locate credentials for the + user, both for the remote operation and if the --ghsrcpat is not specified, + the credentials to use to access Github. The ZOPEN_CURL_PARAMS environment + variable can be used to specify the location ot the netrc file, the default + is a .netrc from the users \$HOME directory. A sample entry for .netrc to + access an ftp server would be: + machine SERVER_HOSTNAME_OR_IP + login SERVER_USERNAME + password SERVER_PASSWORD + + --ghsrcpat, --noghcreds + To prevent rate-limiting, --ghsrcpat should be specified however this does + expose the Personal Access Token to shell history, process listings etc. + Specifying --noghcreds prevents the check for the existence of the ghsrcpat + option, allowing a Github configuration in the user's .netrc file to be used + to authenticate. A sample entry into .netrc for a Github PAT looks like: + machine api.github.com + login YOUR_GITHUB_USERNAME + password YOUR_PERSONAL_ACCESS_TOKEN + +FULLHELP + fi +cat << FOOTER +Report bugs at ${ZOPEN_GITHUB}/meta/issues . +FOOTER + +} + +# Utility functions +displayVersion() { + log ${VERSION} +} + +log() { + ! $quiet && /bin/printf "$*\n" +} +verbose() { + $verbose && /bin/printf "${NC}${MAGENTA}$*${NC}\n" +} + +warning() { + /bin/printf " ${NC}${YELLOW}!! $*${NC}\n" +} + +err() { + /bin/printf " ${NC}${RED}!! $*${NC}\n" + exit 1 +} + +defineANSI() +{ + # Standard tty codes + ESC="" # Start of Escape Sequence; EBCDIC=\047, ASCII=\033 + platform=$(uname) + case ${platform} in + "OS/390") ESC=$(printf "\047");; + *) ESC=$(printf "\033");; + esac + CSI="[" # Control Sequence Introducer + CNL="E" # Cursor Next Line + CPL="F" # Cursor Previous Line + CHA="G" # Cursor Horizontal Absolute - column selector + EL="K" # Erase In Line + SGR="m" # Select Graphic Rendition + + # shellcheck disable=SC2034 + ERASELINE="${ESC}${CSI}2${EL}" + # shellcheck disable=SC2034 + CRSRHIDE="${ESC}${CSI}?25l" + # shellcheck disable=SC2034 + CRSRSHOW="${ESC}${CSI}?25h" + # shellcheck disable=SC2034 + CRSRSOL="${ESC}${CSI}0${CPL}" + # shellcheck disable=SC2034 + CRSRPL="${ESC}${CSI}1${CPL}" # Move to start of previous line + # shellcheck disable=SC2034 + CRSRUP="A" # CUU + # shellcheck disable=SC2034 + CRSRDOWN="B" # CUF + # shellcheck disable=SC2034 + CRSRRIGHT="C" # CUB + # shellcheck disable=SC2034 + CRSRLEFT="D" # CUD + # Color-type codes, needs explicit terminal settings + if [ ! "${_BPX_TERMPATH-x}" = "OMVS" ] && [ -z "${NO_COLOR}" ] && [ ! "${FORCE_COLOR-x}" = "0" ] && [ -t 1 ] && [ -t 2 ]; then + ANSION=true + BLACK="${ESC}${CSI}30${SGR}" + RED="${ESC}${CSI}31${SGR}" + GREEN="${ESC}${CSI}32${SGR}" + YELLOW="${ESC}${CSI}33${SGR}" + BLUE="${ESC}${CSI}34${SGR}" + MAGENTA=$(printf "${ESC}${CSI}35${SGR}") + CYAN="${ESC}${CSI}36${SGR}" + GRAY="${ESC}${CSI}37${SGR}" + BOLD="${ESC}${CSI}1${SGR}" + UNDERLINE="${ESC}${CSI}4${SGR}" + HEADERCOLOR="${MAGENTA}" + WARNINGCOLOR="${MAGENTA}" + # Keep the following last in the list - command trace can interpret the codes + # above and make the output display interesting depending on what the last + # ANSI command was! + NC=$(printf "${ESC}${CSI}0${SGR}") + else + ANSION=false + unset esc RED GREEN YELLOW BOLD UNDERLINE NC + unset esc BLACK RED GREEN YELLOW BLUE MAGENTA CYAN GRAY + unset UNDERLINE NC HEADERCOLOR WARNINGCOLOR + fi +} + +curlCmd() { + if ! type curl >/dev/null; then + err "Could not find curl on the path." + fi + curl "$@" 2>&1 +} + +initial() { + echo "$1" | cut -c1 +} + +downloadRepoMetadata() { + JSON_CACHE="${tmpdir:=/tmp}/zopen_releases.json.$$" + JSON_CACHE_LATEST="${tmpdir:=/tmp}/zopen_releases_latest.json.$$" + + if ! curlout=$(curlCmd -L --no-progress-meter -H 'Accept: application/vnd.github.v3.raw' -H "${authHeader}"\ + -o "${JSON_CACHE}" \ + "${RELEASE_JSON_URL}"); then + err "Failed to obtain json cache from '${RELEASE_JSON_URL}'; ${curlout}" + fi + if type chtag > /dev/null 2>&1; then + # z/OS only - run chtag to ensure the file is processed as ASCII + chtag -tc 819 "${JSON_CACHE}" + fi + if [ ! -f "${JSON_CACHE}" ]; then + err "Could not find json cache locally after download from '${RELEASE_JSON_URL}'" + fi + + if ! curlout=$(curlCmd -L --no-progress-meter -H 'Accept: application/vnd.github.v3.raw' -H "${authHeader}"\ + -o "${JSON_CACHE_LATEST}" \ + "${LATEST_JSON_URL}"); then + err "Failed to obtain json cache from '${LATEST_JSON_URL}'; ${curlout}" + fi + if type chtag > /dev/null 2>&1; then + # z/OS only - run chtag to ensure the file is processed as ASCII + chtag -tc 819 "${JSON_CACHE_LATEST}" + fi + if [ ! -f "${JSON_CACHE_LATEST}" ]; then + err "Could not find json cache locally after download from '${LATEST_JSON_URL}'" + fi + +} + +getPkgFile() { + repo="$1" + portNumber=$2 + repourl=$(jq -r --arg portNumber "${portNumber}" ".release_data.\"${repo%%port}\"[$portNumber].assets[0].url" "${JSON_CACHE}") + filename=$(basename "${repourl}") + [ -z "${filename}" ] && err "Unable to parse filename for download at '${repourl}'." + if eval "${type}Repo checkForFile ${filename}"; then + verbose "${filename} already in repository" + return 1 + fi + outfile="${tmpdir:=/tmp}/${filename}" + [ -e "${outfile}" ] && rm "${outfile}" + if ! curlout=$(curlCmd --no-progress-meter -L -H "${authHeader}" -o "${outfile}" "${repourl}"); then + err "Curl command to retrive package file failed '${curlout}'. Resolve issue and retry request." + fi +} + +getPkgMetadata(){ + # Use previously set variables (from getPkgFile) with a ".json" suffix! + outfile="${tmpdir:=/tmp}/${filename}.json" + [ -e "${outfile}" ] && rm "${outfile}" + if ! curlout=$(curlCmd --no-progress-meter -L -H "${authHeader}" -o "${outfile}" "${repourl}/metadata.json"); then + err "Curl command to retrive package metadata failed '${curlout}'. Resolve issue and retry request." + fi +} + +## Repository handlers +_AbstractFileRepo(){ + case "$1" in + initialise) + [ -e "${target}" ] || mkdir -p "${target}" + [ -d "${target}" ] || err "Repo mirror target '${target}' is not a directory" + [ -w "${target}" ] || err "Repo mirror target '${target}' is not writable" + return 0 + ;; + checkForFile) err "Unimplemented action verb for abstract type" ;; + mirrorFile) err "Unimplemented action verb for abstract type";; + generateRepoD) + repoDFile="${target}/repod.json" + [ -e "${repoDFile}" ] && + mv "${repoDFile}" "${repoDFile}.$(date "+%C%m%d%H%M%S")" + mkdir -p "$(dirname "${repoDFile}")" + cat <"${repoDFile}" +{ + "type": "file", + "metadata_baseurl": "$(dirname "${rewrittenRepoFile}")", + "metadata_file": "$(basename "${rewrittenRepoFile}")", + "latest_file": "$(basename "${rewrittenLatestFile}")" +} +EOS + echo "${repoDFile}" + ;; + rewriteMetadata) + jqQuery=$2 + rewrittenRepoFile="${target%/}/metadata.json" + rewrittenLatestFile="${target%/}/latest_metadata.json" + if ! rewrite=$(jq --arg zopenrepo "${ZOPEN_GITHUB}" --arg t "${target%/}" \ + "${jqQuery}" "${JSON_CACHE}" >"${rewrittenRepoFile}"); then + err "Unable to generate metadata JSON file '${rewrittenRepoFile}' for mirrored repo. ${rewrite}" + fi + if ! rewrite=$(jq --arg zopenrepo "${ZOPEN_GITHUB}" --arg t "${target%/}" \ + "${jqQuery}" "${JSON_CACHE_LATEST}" >"${rewrittenLatestFile}"); then + err "Unable to generate metadata JSON file '${rewrittenLatestFile}' for mirrored repo. ${rewrite}" + fi + ;; + *) err "Unsupported action verb for repo type";; + esac +} +FLATRepo(){ + case "$1" in + initialise) _AbstractFileRepo "initialise" ;; + checkForFile) + filename=$(basename "$2") + [ -e "${target}/${filename}" ] && return 0 + return 1 + ;; + mirrorFile) + targetFile="${target}/$(basename "$2")" + [ -e "${targetFile}" ] && verbose "$2 already in repo" && return 0 + # A mv would generally work, however on z/OS, if there is an issue + # setting the new gid there'll be a message; mimic the mv with a + # copy-n-delete as still want to get any error messages - redirecting & + # analyzing the output from the mv complicates the code! + cp -f "${outfile}" "${targetFile}" + rm "${outfile}" + ;; + generateRepoD)_AbstractFileRepo "generateRepoD" ;; + rewriteMetadata) + _AbstractFileRepo "rewriteMetadata" \ + 'walk(if type == "string" then gsub($zopenrepo + "/.*/(?.*)$"; "file://" + $t + "/" + .f) else . end)' + ;; + *) err "Unsupported action verb for repo type";; + esac +} +ALPHARepo(){ + case "$1" in + initialise) _AbstractFileRepo "initialise" ;; + checkForFile) + filename=$(basename "$2") + [ -e "${target}/$(initial "${filename}")/${filename}" ] && return 0 + return 1 + ;; + mirrorFile) + filename=$(basename "$2") + targetFile="${target}/$(initial "${filename}")/${filename}" + [ -e "${targetFile}" ] && verbose "${filename} already downloaded" && return 0 + [ ! -d "$(dirname "${targetFile}")" ] && mkdir -p "$(dirname "${targetFile}")" + [ -e "${outfile}" ] || err "Could not find downloaded file '${outfile}'" + cp -f "${outfile}" "${targetFile}" + rm "${outfile}" + ;; + generateRepoD) _AbstractFileRepo "generateRepoD" ;; + rewriteMetadata) + _AbstractFileRepo "rewriteMetadata" \ + 'walk(if type == "string" then gsub($zopenrepo + "/(?.).*/(?.*)$"; "file://" + $t + "/" + .i + "/" + .f) else . end)' + ;; + *) err "Unsupported action verb for repo type";; + esac +} + +NAMERepo(){ + case "$1" in + initialise) _AbstractFileRepo "initialise" ;; + checkForFile) + filename=$(basename "$2") + pkgname=$(echo "${filename}" | \ + sed -e "s/\(\.*\)-[^-]*[0-9]*_[0-9]*\..*/\1/") + [ -e "${target}/${pkgname}/${filename}" ] && return 0 + return 1 + ;; + mirrorFile) + filename=$(basename "$2") + pkgname=$(echo "${filename}" | \ + sed -e "s/\(\.*\)-[^-]*[0-9]*_[0-9]*\..*/\1/") + targetFile="${target}/${pkgname}/${filename}" + [ -e "${targetFile}" ] && verbose "${filename} already downloaded" && return 0 + [ ! -d "$(dirname "${targetFile}")" ] && mkdir -p "$(dirname "${targetFile}")" + cp -f "${outfile}" "${targetFile}" + rm "${outfile}" + ;; + generateRepoD) _AbstractFileRepo "generateRepoD" ;; + rewriteMetadata) + _AbstractFileRepo "rewriteMetadata" \ + 'walk(if type == "string" then gsub($zopenrepo + "/(?.*)port/.*/(?.*)$"; "file://" + $t + "/" + .n + "/" + .f) else . end)' + ;; + *) err "Unsupported action verb for repo type";; + esac +} + +ALPHANAMERepo(){ + case "$1" in + initialise) _AbstractFileRepo "initialise" ;; + checkForFile) + filename=$(basename "$2") + pkgname=$(echo "${filename}" | \ + sed -e "s/\(\.*\)-[^-]*[0-9]*_[0-9]*\..*/\1/") + [ -e "${target}/$(initial "${filename}")/${pkgname}/${filename}" ] && return 0 + return 1 + ;; + mirrorFile) + filename=$(basename "$2") + pkgname=$(echo "${filename}" | \ + sed -e "s/\(\.*\)-[^-]*[0-9]*_[0-9]*\..*/\1/") + targetFile="${target}/$(initial "${filename}")/${pkgname}/${filename}" + [ -e "${targetFile}" ] && verbose "${filename} already downloaded" && return 0 + [ -d "$(dirname "${targetFile}")" ] || mkdir -p "$(dirname "${targetFile}")" + cp -f "${outfile}" "${targetFile}" + rm "${outfile}" + ;; + generateRepoD) _AbstractFileRepo "generateRepoD" ;; + rewriteMetadata) + _AbstractFileRepo "rewriteMetadata" \ + 'walk(if type == "string" then gsub($zopenrepo + "/(?.)(?.*)port/.*/(?.*)$"; "file://" + $t + "/" + .i + "/" + .i + .n + "/" + .f) else . end)' + + ;; + *) err "Unsupported action verb for repo type";; + esac +} + +_AbstractRemoteRepo(){ + case "$1" in + initialise) + # Get a directory listing of the remote and grep for the filename - crude + # but works. Cache the result in a file; a variable would be potentially quicker + # but if the package count gets too large, the variable will overflow. + if [ ! -f "${DIR_LIST_CACHE}" ]; then + curlparams=$(printf "%s --silent --show-error " "${curlparams}") + if ${netrc}; then + curlparams=$(printf " %s %s" "${curlparams}" "--netrc") + else + curlparams=$(printf " %s %s" "${curlparams}" "--user ${dstuser}:${dstpassword}") + fi + # shellcheck disable=SC2086 + if ! curlCmd ${curlparams} "$2://${target}/" > "${DIR_LIST_CACHE}"; then + err "Failed to otain directory listing at '$2://${target}/': $(cat ${DIR_LIST_CACHE})" + fi + fi + ;; + checkForFile) grep -q "$(basename "$2")" "${DIR_LIST_CACHE}" ;; + mirrorFile) + if ${netrc}; then + curlparams=$(printf " %s %s" "${curlparams}" "--netrc") + else + curlparams=$(printf " %s %s" "${curlparams}" "--user ${dstuser}:${dstpassword}") + fi + curlparams=$(printf "%s --silent --show-error" "${curlparams}") + curlCmd ${curlparams} "--upload-file" "$2" "$3://${target}/" + ;; + generateRepoD) + repoDFile=$(echo "${target}" | sed "s/[\\/:']//g") + repoDFile="${repoDFile}/repod.json" + mkdir -p "$(dirname "${repoDFile}")" + [ -e "${repoDFile}" ] && + mv "${repoDFile}" "${repoDFile}.$(date "+%C%m%d%H%M%S")" + cat <"${repoDFile}" +{ + "type": "$2", + "metadata_baseurl": "${target}")", + "metadata_file": "$(basename "${rewrittenRepoFile}")", + "latest_file": "$(basename "${rewrittenLatestFile}")" +} +EOS + echo "${repoDFile}" + ;; + rewriteMetadata) + jqQuery='walk(if type == "string" then gsub($zopenrepo + "/(?.)(?.*)port/.*/(?.*)$"; "$protocol://$target/" + .f) else . end)' + rewrittenRepoFile="${target%/}/metadata.json" + rewrittenLatestFile="${target%/}/latest_metadata.json" + if ! rewrite=$(jq --arg zopenrepo "${ZOPEN_GITHUB}" --arg t "${target%/}" \ + "${jqQuery}" "${JSON_CACHE}" >"${rewrittenRepoFile}"); then + err "Unable to generate metadata JSON file '${rewrittenRepoFile}' for mirrored repo. ${rewrite}" + fi + if ! rewrite=$(jq --arg zopenrepo "${ZOPEN_GITHUB}" --arg t "${target%/}" \ + "${jqQuery}" "${JSON_CACHE_LATEST}" >"${rewrittenLatestFile}"); then + err "Unable to generate metadata JSON file '${rewrittenLatestFile}' for mirrored repo. ${rewrite}" + fi + # Note there is no upload of the files; this is the abstract so no transport + ;; + + *) err "Unsupported action verb for repo type";; + esac +} + +FTPRepo(){ + this="ftp" + case "$1" in + initialise) _AbstractRemoteRepo "initialise" "${this}";; + checkForFile) + curlparams="--ssl --list-only" + _AbstractRemoteRepo "checkForFile" "$2" "${this}" + ;; + mirrorFile) + curlparams="--ssl" + _AbstractRemoteRepo "mirrorFile" "$2" "${this}" + ;; + generateRepoD) _AbstractRemoteRepo "generateRepoD" "${this}" ;; + rewriteMetadata) + _AbstractRemoteRepo "rewriteMetadata" "${this}" + FTPRepo "mirrorFile" "${rewrittenRepoFile}" "${this}" + FTPRepo "mirrorFile" "${rewrittenLatestFile}" "${this}" + ;; + *) err "Unsupported action verb for repo type";; + esac +} +SFTPRepo(){ + this="sftp" + case "$1" in + initialise) _AbstractRemoteRepo "initialise" "${this}" ;; + checkForFile) + curlparams="--list-only" + _AbstractRemoteRepo "checkForFile" "$2" "${this}" + ;; + mirrorFile) _AbstractRemoteRepo "mirrorFile" "$2" "${this}" ;; + generateRepoD) _AbstractRemoteRepo "generateRepoD" "${this}" ;; + rewriteMetadata) + _AbstractRemoteRepo "rewriteMetadata" "${this}" + SFTPRepo "mirrorFile" "${rewrittenRepoFile}" "${this}" + SFTPRepo "mirrorFile" "${rewrittenLatestFile}" "${this}" + ;; + *) err "Unsupported action verb for repo type";; + esac +} +SAMBARepo(){ + this="sftp" + case "$1" in + initialise) _AbstractRemoteRepo "initialise" "${this}" ;; + checkForFile) + curlparams="--list-only" + _AbstractRemoteRepo "checkForFile" "$2" "${this}" + ;; + mirrorFile) _AbstractRemoteRepo "mirrorFile" "$2" "${this}" ;; + generateRepoD) _AbstractRemoteRepo "generateRepoD" "${this}" ;; + rewriteMetadata) + _AbstractRemoteRepo "rewriteMetadata" "${this}" + SAMBARepo "mirrorFile" "${rewrittenRepoFile}" "${this}" + SAMBARepo "mirrorFile" "${rewrittenLatestFile}" "${this}" + ;; + *) err "Unsupported action verb for repo type";; + esac +} + +NEXUSRepo(){ + this="nexus" + case "$1" in + initialise) _AbstractRemoteRepo "initialise" "${this}" ;; + checkForFile) + curlparams="--fail -w \"\\n%{http_code}\" -s -I" + if ${netrc}; then + curlparams=$(printf "%s %s" "${curlparams}" "--netrc") + else + curlparams=$(printf "%s %s" "${curlparams}" "--user ${dstuser}:${dstpassword}") + fi + curlCmd ${curlparams} $target/$2 | grep "HTTP/1.1 200 OK" >/dev/null + ;; + mirrorFile) + file="$2" + #TODO read Nexus config, use other auth? + nexushost=$(echo "${target}" | sed -e "s#^[^/]*//##" -e "s#\([^/]*\).*#\1#") + # Check if Nexus credentials are provided + if ${netrc}; then + verbose "Using netrc credentials so let curl handle this" + else + verbose "Using Basic Auth" + if [ -z "$dstuser" ] || [ -z "$dstpassword" ]; then + error "Basic authentication credentials missing" + fi + fi + file="${tmpdir:=/tmp}/${file}" + if [ ! -e "${file}" ]; then + warning "File '${file}' not found; ignoring upload request" + return + fi + if [ ! -r "${file}" ]; then + warning "No permissions to upload file '${file}'; ignoring upload request" + return + fi + filename=$(basename "${file}") + verbose "Attempting to upload '${file}' to '$target/$filename" + if ! httprc=$(curlCmd -fs -u "${dstuser}:${dstpassword}" --upload-file "$file" "$target/$filename"); then + case ${httprc} in + "403") warning "403: Authentication failure uploading file '${file}'";; + "200") verbose "200: File '$/tmp/avro-c-packaging-master.20240130_092928.zos.pax.Z{file}' uploaded successfully";; + *) warning "${httprc}: Unxpected response code during upload of file '${file}'"; + esac + fi + ;; + generateRepoD) + repoDFile=$(echo "${target}" | sed "s/[\\/:']//g") + repoDFile="${repoDFile}/repod.json" + mkdir -p "$(dirname "${repoDFile}")" + [ -e "${repoDFile}" ] && + mv "${repoDFile}" "${repoDFile}.$(date "+%C%m%d%H%M%S")" + cat <"${repoDFile}" +{ + "type": "nexus", + "metadata_baseurl": "${target}")", + "metadata_file": "$(basename "${rewrittenRepoFile}")", + "latest +} +EOS + echo "${repoDFile}" + ;; + rewriteMetadata) + _AbstractRemoteRepo "rewriteMetadata" "${this}" + NEXUSRepo "mirrorFile" "${rewrittenRepoFile}" "${this}" + NEXUSRepo "mirrorFile" "${rewrittenLatestFile}" "${this}" + ;; + *) err "Unsupported action verb for repo type";; + esac + +} + + +runMirror() { + log "Mirroring packages" + terminal=false + if [ -t 0 ] && [ -t 1 ]; then + terminal=true + fi + + cnt=$(jq --raw-output '.release_data| keys | length' "${JSON_CACHE}") + log "${cnt} " + + if ! initRepo=$(eval '${type}Repo "initialise"'); then + err "Unable to initialise repository mirror: ${initRepo}" + fi + if $terminal; then + # Get terminal width - try tput first, fall back to stty, default to 80. + cols=$(tput cols 2>/dev/null || stty size 2>/dev/null | cut -d' ' -f2 || echo 80) + effectiveCols=$((cols - 6)) + pct=$((cnt / effectiveCols)) + [ "$pct" -eq 0 ] && pct=1 # Ensure pct is at least 1 to avoid division by zero + i=0 + printf "0%%" + fi + + for mirror in $(jq --raw-output '.release_data| keys[]' "${JSON_CACHE}"); do + if "${verbose}"; then + verbose "Mirroring: ${mirror}..." + elif ! "${terminal}"; then + : + else + i=$((i + 1)) + if [ $(( i % pct)) -eq 0 ]; then + printf "." + fi + fi + # Just grab the latest version [offset 0 in array]; + # TODO: determine if mirror providers want to provide a full sync - + # trying to install package versions that haven't been mirrored will result + # in can't be downloaded errors + if getPkgFile "${mirror}port" 0; then # Remember to add the port suffix! + if ! getOutput=$(eval '${type}Repo "mirrorFile" "${outfile}"'); then + err "Errors mirroring '${outfile}': ${getOutput}" + fi + fi + # Need to also get the metadata.json for the file - outfile will get updated + if getPkgMetadata; then + if ! getOutput=$(eval '${type}Repo "mirrorFile" "${outfile}"'); then + err "Errors mirroring '${outfile}': ${getOutput}" + fi + fi + done + + if $terminal; then + printf " 100%%\n" + else + printf "\n" + fi + + log "Mirroring complete, generating metadata JSON" + eval '${type}Repo "rewriteMetadata"' +} + +generateRepositoryMetadata() { + log "Generate repository information file" + if ! repoFile=$(eval '${type}Repo "generateRepoD"'); then + err "Could not generate repository file. Details: ${repoFile}" + fi +} + +activateRepo() { + set -x + repodDir="${zopenrootfs}/etc/zopen/repos.d" + [ ! -d "${repodDir}" ] && mkdir "${repodDir}" + activeSymlink="${repodDir}/active" + [ -L "${activeSymlink}" ] && rm "${activeSymlink}" + if ! symlink=$(ln -s "${repoFile}" "${activeSymlink}"); then + log "$LOG_ERROR" "Could not set new repository '${repoFile}' as active." + [ -n "${symlink}" ] && log "$LOG_ERROR" "Details: ${symlink}" + exit 8 + fi +} + +parseArgs() { + while [ $# -gt 0 ]; do + case $1 in + --netrc) + netrc=true + ;; + --ghsrcpat) + [ $# -lt 2 ] && err "Missing value for option '$1'" + ghsrcpat="$2" + shift + ;; + --type) + [ $# -lt 2 ] && err "Missing value for option '$1'" + type="$2" + shift + ;; + --tmpdir) + [ $# -lt 2 ] && err "Missing value for option '$1'" + tmpdir="$2" + shift + ;; + --genRepoFile) + genRepoFile=true + ;; + --activate) + [ $# -lt 2 ] && err "Missing zopen rootfs for option '$1'" + zopenrootfs="$2" + activate=true + shift + ;; + --dstuser) + [ $# -lt 2 ] && err "Missing value for option '$1'" + dstuser="$2" + shift + ;; + --dstpassword) + [ $# -lt 2 ] && err "Missing value for option '$1'" + dstpassword="$2" + shift + ;; + --noghcreds) + noghcreds=true + ;; + --target) + [ $# -lt 2 ] && err "Missing value for option '$1'" + target="$2" + shift + ;; + -? | --?) printHelp false && exit 0 ;; + --help) printHelp true && exit 0 ;; + --verbose) verbose=true ;; + --quiet) quiet=true ;; + --version) displayVersion && exit 0 ;; + *) err "Bad parameter '$1'. " ;; + esac + shift + done +} + +verbose=false +quiet=false +type="" +genRepoFile=true # ?false? +activate=false +zopenrootfs="" +noghcreds=false +ghsrcpat="" +netrc=false +authHeader="" + +export PATH=/bin:$PATH # Always prefer native tools; makes a difference on z/OS +defineANSI +parseArgs "$@" + +log "zopen repository mirror" +if $activate; then + [ -z "${zopenrootfs}" ] && err "Missing zopenrootfs definition for the --activate option." + [ ! -d "${zopenrootfs}/etc/zopen" ] && err "No zopen file system detected at '${zopenrootfs}'. The --activate option updates a zopen file system so must run on z/OS. Correct parameters and retry request." + zopenrootfs=$(cd "${zopenrootfs}" && pwd -P) # use absolute name if possible +fi + +if "${netrc}"; then + verbose "Using .netrc for download authentication" +elif "${noghcreds}"; then + verbose "Skipping GitHub authorization - using anonymous" +else + # TODO: use more secure than cli param for PAT?. + [ -z "${ghsrcpat}" ] && err "Github personal access token (PAT) required to prevent rate-limiting" + [ -n "${ghsrcpat}" ] && authHeader="Authorization: token ${ghsrcpat}" +fi + +[ -z "${type}" ] && err "Missing mirror type" +[ -z "${target}" ] && err "Parameter --target required for '${type}' mirror type" + +type=$(echo "${type}" | /bin/awk '{print(toupper($1))}') +verbose "Checking repository type '${type}' for validity" +case "${type}" in +FLAT | ALPHA | NAME | ALPHANAME) + [ ! -d "${target}" ] && mkdir -p "${target}" + target=$(cd "${target}" && pwd -P) # Use absolute name if possible + ;; +NEXUS|FTP|SFTP|SAMBA) + if ! "${netrc}"; then + [ -z "${dstuser}" ] && err "No remote repo username given" + [ -z "${dstpassword}" ] && err "No remote repo password given" + else + verbose "Using .netrc for credentials for upload to remote server" + fi + case "${type}" in + FTP|SFTP|SAMBA) + target="${target#*://}" # Strip any protocol definitions - add back later + DIR_LIST_CACHE="${tmpdir:=/tmp}/zopen_mirror_dir_cache.$$" + [ -f "${DIR_LIST_CACHE}" ] && rm "${DIR_LIST_CACHE}" ;; + *) :;; + esac +;; +*) err "Unsupported mirror type: '${type}'. Check parameter and retry command." ;; +esac + +downloadRepoMetadata +runMirror +if ${genRepoFile}; then + generateRepositoryMetadata +fi +if ${activate}; then + activateRepo +fi +[ -e "${DIR_LIST_CACHE}" ] && echo "rm \"${DIR_LIST_CACHE}\"" + +log "Mirror operation complete" + From 21d22475d49fd3efbc09d521253c7ce410d9641e Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 18 Mar 2025 21:34:51 +0000 Subject: [PATCH 078/179] Ensure mirror tags json files and downloads correctly --- bin/zopen-mirror | 8 ++++++-- include/common.sh | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/bin/zopen-mirror b/bin/zopen-mirror index 111635d0f..50bedd979 100755 --- a/bin/zopen-mirror +++ b/bin/zopen-mirror @@ -287,9 +287,13 @@ getPkgMetadata(){ # Use previously set variables (from getPkgFile) with a ".json" suffix! outfile="${tmpdir:=/tmp}/${filename}.json" [ -e "${outfile}" ] && rm "${outfile}" - if ! curlout=$(curlCmd --no-progress-meter -L -H "${authHeader}" -o "${outfile}" "${repourl}/metadata.json"); then + if ! curlout=$(curlCmd --no-progress-meter -L -H "${authHeader}" -o "${outfile}" "$(dirname "${repourl}")/metadata.json"); then err "Curl command to retrive package metadata failed '${curlout}'. Resolve issue and retry request." fi + if type chtag > /dev/null 2>&1; then + # z/OS only - run chtag to ensure the file is processed as ASCII + chtag -tc 819 "${outfile}" + fi } ## Repository handlers @@ -806,7 +810,7 @@ if "${netrc}"; then elif "${noghcreds}"; then verbose "Skipping GitHub authorization - using anonymous" else - # TODO: use more secure than cli param for PAT?. + # TODO: use more secure than cli param for PAT? [ -z "${ghsrcpat}" ] && err "Github personal access token (PAT) required to prevent rate-limiting" [ -n "${ghsrcpat}" ] && authHeader="Authorization: token ${ghsrcpat}" fi diff --git a/include/common.sh b/include/common.sh index fa4f2f260..e5f2a3636 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1448,7 +1448,7 @@ getJSONCacheURLs(){ type=$(jq -r ".type" "${activeRepo}") base=$(jq -r ".metadata_baseurl" "${activeRepo}") filename=$(jq -r ".metadata_file" "${activeRepo}") - latest_metadata=$(jq -r ".latest_metadata" "${activeRepo}") + latest_metadata=$(jq -r ".latest_file" "${activeRepo}") case "${type}" in http|https) printf "%s://%s/%s\n%s://%s/%s" \ "${type}" "${base}" "${filename}" "${type}" "${base}" "${latest_metadata}" From 483b76c0824bfc48501540713092d693533ae557 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 19 Mar 2025 07:22:23 +0000 Subject: [PATCH 079/179] Actually use the correct json key for latest flie --- bin/zopen-init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/zopen-init b/bin/zopen-init index 191faaf93..2c96ba1ce 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -540,7 +540,7 @@ generateDefaultRepository() "type": "${repotype}", "metadata_baseurl": "${baserepourl}", "metadata_file": "${repometafile}", - "latest_metadata": "${latestmetafile}" + "latest_file": "${latestmetafile}" } EOS (cd "${rootfs}/etc/zopen/repos.d" && ln -s "${repoFile}" "active") From 49cd7b7bd906f75717926d7e183fa5241218761f Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 19 Mar 2025 09:09:30 +0000 Subject: [PATCH 080/179] Rework metadata downloads to work around GH .json response with 404 --- include/common.sh | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/include/common.sh b/include/common.sh index e5f2a3636..121224bfe 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2382,28 +2382,38 @@ getInstallFile() [ -e "${downloadToDir}" ] || mkdir -p "${downloadToDir}" [ -w "${downloadToDir}" ] || printError "No permission to save install file to '${downloadToDir}'. Check permissions and retry command." printVerbose "Downloading installable file" - if ! runAndLog "cd ${downloadToDir} && curlCmd --no-progress-meter -L ${installurl} -O ${redirectToDevNull}"; then + if ! runAndLog "cd ${downloadToDir} && curlCmd -L ${installurl} -O ${redirectToDevNull}"; then printError "Could not download from ${installurl}. Correct any errors and potentially retry" fi printVerbose "Downloading corresponding metadata" # check if it is in the same location just with a different suffix (as in a mirror) - # if not, likely is the original Github repo which uses a subdirectory for the metadata... - # Try again with this extended URL + # if not, likely is the original Github repo which uses a subdirectory for the metadata + # Try again with this URL metadataFile="$(basename "${installurl}").json" metadataJSONURL="${installurl}.json" - if ! runAndLog "curlCmd -L '${metadataJSONURL}' -o '${metadataFile}'" "${redirectToDevNull}"; then - printVerbose "Not located explicit file, try extended URL location" + + printVerbose "Checking for existence of remote metadata file" + if curlOut=$(curlCmd -s -o /dev/null -I -w "%{http_code}" "${metadataJSONURL}"); then + # test for 404 - if it is, then this is likely the Main GitHub repo which hosts + # the metadata as a release so use alternative URL + if [ "${curlOut}" = "404" ]; then metadataJSONURL_ext="$(dirname "${installurl}")/metadata.json" - if ! runAndLog "curlCmd -L '${metadataJSONURL_ext}' -o '${metadataFile}'" "${redirectToDevNull}"; then - printSoftError "Could not download package metadata. Tried the following locations:" - printSoftErrpr "'${metadataJSONURL}' '${metadataJSONURL_ext}'" - printError "Correct any errors and retry command" - else - printVerbose "Metadata downloaded from extended location '${metadataJSONURL_ext}'" + printVerbose " Metadata not found at '${metadataJSONURL}'. Trying '${metadataJSONURL_ext}'" + metadataJSONURL="${metadataJSONURL_ext}" fi else - printVerbose "Metadata downloaded from '${metadataJSONURL}'" + printSoftError "Curl issue trying to download from '${metadataJSONURL}'" + [ -n "${curlOut}" ] && printSoftError + exit 1 + fi + + if ! curlOut=$(cd "${downloadToDir}" && curlCmd -L "${metadataJSONURL}" -o "${metadataFile}"); then + printSoftError "Curl issue trying to download from '${metadataJSONURL}'" + [ -n "${curlOut}" ] && printSoftError + exit 1 + else + printVerbose "Metadata downloaded from '${metadataJSONURL}' to '${metadataFile}'" fi if command -v chtag >/dev/null 2>&1; then From adad93a8fca5cdc4ff2a8a95182d1561c67a1b93 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 19 Mar 2025 11:36:33 +0000 Subject: [PATCH 081/179] Use mirror pkg name rather than trying to calculate --- bin/zopen-mirror | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/bin/zopen-mirror b/bin/zopen-mirror index 50bedd979..d2d02c438 100755 --- a/bin/zopen-mirror +++ b/bin/zopen-mirror @@ -267,12 +267,12 @@ downloadRepoMetadata() { } getPkgFile() { - repo="$1" + mirror="$1" portNumber=$2 - repourl=$(jq -r --arg portNumber "${portNumber}" ".release_data.\"${repo%%port}\"[$portNumber].assets[0].url" "${JSON_CACHE}") + repourl=$(jq -r --arg portNumber "${portNumber}" ".release_data.\"${mirror}\"[$portNumber].assets[0].url" "${JSON_CACHE}") filename=$(basename "${repourl}") [ -z "${filename}" ] && err "Unable to parse filename for download at '${repourl}'." - if eval "${type}Repo checkForFile ${filename}"; then + if eval "${type}Repo checkForFile ${filename} ${mirror}"; then verbose "${filename} already in repository" return 1 fi @@ -395,15 +395,13 @@ NAMERepo(){ initialise) _AbstractFileRepo "initialise" ;; checkForFile) filename=$(basename "$2") - pkgname=$(echo "${filename}" | \ - sed -e "s/\(\.*\)-[^-]*[0-9]*_[0-9]*\..*/\1/") + pkgname=$3 [ -e "${target}/${pkgname}/${filename}" ] && return 0 return 1 ;; mirrorFile) filename=$(basename "$2") - pkgname=$(echo "${filename}" | \ - sed -e "s/\(\.*\)-[^-]*[0-9]*_[0-9]*\..*/\1/") + pkgname=$3 targetFile="${target}/${pkgname}/${filename}" [ -e "${targetFile}" ] && verbose "${filename} already downloaded" && return 0 [ ! -d "$(dirname "${targetFile}")" ] && mkdir -p "$(dirname "${targetFile}")" @@ -424,15 +422,13 @@ ALPHANAMERepo(){ initialise) _AbstractFileRepo "initialise" ;; checkForFile) filename=$(basename "$2") - pkgname=$(echo "${filename}" | \ - sed -e "s/\(\.*\)-[^-]*[0-9]*_[0-9]*\..*/\1/") + pkgname=$3 [ -e "${target}/$(initial "${filename}")/${pkgname}/${filename}" ] && return 0 return 1 ;; mirrorFile) filename=$(basename "$2") - pkgname=$(echo "${filename}" | \ - sed -e "s/\(\.*\)-[^-]*[0-9]*_[0-9]*\..*/\1/") + pkgname=$3 targetFile="${target}/$(initial "${filename}")/${pkgname}/${filename}" [ -e "${targetFile}" ] && verbose "${filename} already downloaded" && return 0 [ -d "$(dirname "${targetFile}")" ] || mkdir -p "$(dirname "${targetFile}")" @@ -471,7 +467,7 @@ _AbstractRemoteRepo(){ checkForFile) grep -q "$(basename "$2")" "${DIR_LIST_CACHE}" ;; mirrorFile) if ${netrc}; then - curlparams=$(printf " %s %s" "${curlparams}" "--netrc") + curlparams=$(printf " %s %s" "${curlparams}" "--netrc") else curlparams=$(printf " %s %s" "${curlparams}" "--user ${dstuser}:${dstpassword}") fi @@ -681,14 +677,14 @@ runMirror() { # TODO: determine if mirror providers want to provide a full sync - # trying to install package versions that haven't been mirrored will result # in can't be downloaded errors - if getPkgFile "${mirror}port" 0; then # Remember to add the port suffix! - if ! getOutput=$(eval '${type}Repo "mirrorFile" "${outfile}"'); then + if getPkgFile "${mirror}" 0; then # Remember to add the port suffix! + if ! getOutput=$(eval '${type}Repo "mirrorFile" "${outfile}" "${mirror}"'); then err "Errors mirroring '${outfile}': ${getOutput}" fi fi # Need to also get the metadata.json for the file - outfile will get updated if getPkgMetadata; then - if ! getOutput=$(eval '${type}Repo "mirrorFile" "${outfile}"'); then + if ! getOutput=$(eval '${type}Repo "mirrorFile" "${outfile}" "${mirror}"'); then err "Errors mirroring '${outfile}': ${getOutput}" fi fi From 5c3199d9fcce24c3353cc173f9b887dab1c948c7 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 19 Mar 2025 14:44:24 +0000 Subject: [PATCH 082/179] Easier activation of repo when on z/OS --- bin/zopen-mirror | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/bin/zopen-mirror b/bin/zopen-mirror index d2d02c438..e26b47ed6 100755 --- a/bin/zopen-mirror +++ b/bin/zopen-mirror @@ -310,7 +310,7 @@ _AbstractFileRepo(){ generateRepoD) repoDFile="${target}/repod.json" [ -e "${repoDFile}" ] && - mv "${repoDFile}" "${repoDFile}.$(date "+%C%m%d%H%M%S")" + mv "${repoDFile}" "${repoDFile}.$(date "+%C%m%d%H%M%S")" mkdir -p "$(dirname "${repoDFile}")" cat <"${repoDFile}" { @@ -708,12 +708,20 @@ generateRepositoryMetadata() { } activateRepo() { - set -x repodDir="${zopenrootfs}/etc/zopen/repos.d" [ ! -d "${repodDir}" ] && mkdir "${repodDir}" activeSymlink="${repodDir}/active" [ -L "${activeSymlink}" ] && rm "${activeSymlink}" - if ! symlink=$(ln -s "${repoFile}" "${activeSymlink}"); then + if [ ! -e "${repoFile}" ]; then + log "$LOG_ERROR" "Could not find file '${repoFile}' to use as active repository" + exit 8 + fi + repoDName=$(basename "${repoFile}") + if [ -e "${repodDir}/${repoDName}" ]; then + mv "${repodDir}/${repoDName}" "${repodDir}/${repoDName}.$(date "+%C%m%d%H%M%S")" + fi + cp "${repoFile}" "${repodDir}/${repoDName}" + if ! symlink=$(ln -s "${repodDir}/${repoDName}" "${activeSymlink}"); then log "$LOG_ERROR" "Could not set new repository '${repoFile}' as active." [ -n "${symlink}" ] && log "$LOG_ERROR" "Details: ${symlink}" exit 8 From 70bd6090e38738694ce97de287ac170177428a99 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 19 Mar 2025 16:03:04 +0000 Subject: [PATCH 083/179] Ensure metadata and pax are both available, downloaded as needed --- include/common.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/include/common.sh b/include/common.sh index 121224bfe..85175df47 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2376,8 +2376,9 @@ getInstallFile() else downloadToDir="${ZOPEN_ROOTFS}/var/cache/zopen" fi + if [ -e "${downloadToDir}/${installurl##*/}" ]; then - : + printVerbose "Install file '${installurl##*/}' already in local cache at '${downloadToDir}'" else [ -e "${downloadToDir}" ] || mkdir -p "${downloadToDir}" [ -w "${downloadToDir}" ] || printError "No permission to save install file to '${downloadToDir}'. Check permissions and retry command." @@ -2385,12 +2386,15 @@ getInstallFile() if ! runAndLog "cd ${downloadToDir} && curlCmd -L ${installurl} -O ${redirectToDevNull}"; then printError "Could not download from ${installurl}. Correct any errors and potentially retry" fi - + fi + metadataFile="$(basename "${installurl}").json" + if [ -e "${downloadToDir}/${metadataFile}" ]; then + printVerbose "Corresponding metadata '${metadataFile}' already in local cache" + else printVerbose "Downloading corresponding metadata" # check if it is in the same location just with a different suffix (as in a mirror) # if not, likely is the original Github repo which uses a subdirectory for the metadata # Try again with this URL - metadataFile="$(basename "${installurl}").json" metadataJSONURL="${installurl}.json" printVerbose "Checking for existence of remote metadata file" From 3f76ff7c273ef62addc007d5694a8d51b20195b0 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Mon, 31 Mar 2025 09:08:34 -0400 Subject: [PATCH 084/179] fix typos in zopen and zopen-clean help output --- bin/zopen | 4 +++- bin/zopen-clean | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/zopen b/bin/zopen index 0f179cda1..919bd61a1 100755 --- a/bin/zopen +++ b/bin/zopen @@ -50,6 +50,7 @@ Command: Options: -h, --help, -? display this help and exit -v, --verbose run in verbose mode + --version display the zopen version Examples: zopen --help displays zopen help @@ -58,7 +59,7 @@ Examples: zopen upgrade -y upgrade all installed packages to the latest release, without prompting zopen alt bash list installed alternative bash packages - zopen info vim displays details information about the installed vim package + zopen info vim displays detailed information about the installed vim package zopen usage --pie displays an ASCII-art chart showing biggest space hogs SEE ALSO: @@ -74,6 +75,7 @@ SEE ALSO: zopen-list(1) zopen-mirror(1) zopen-query(1) + zopen-refresh(1) zopen-remove(1) zopen-update-cacert(1) zopen-upgrade(1) diff --git a/bin/zopen-clean b/bin/zopen-clean index a7cf27aa8..754594e60 100755 --- a/bin/zopen-clean +++ b/bin/zopen-clean @@ -24,7 +24,7 @@ checkWritable printHelp() { cat << HELPDOC -${ME} is a utility for zopen community to remove uneeded resources +${ME} is a utility for zopen community to remove unneeded resources from the system to save space and prevent clutter. Usage: ${ME} [OPTION] [PACKAGE] From 0a76b9c73b287140275ed6ee2b3fc0650d3406e4 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 2 Apr 2025 12:33:23 +0100 Subject: [PATCH 085/179] Correct list output for "No Tests" --- bin/zopen-list | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/bin/zopen-list b/bin/zopen-list index e3da8e430..a3e439dba 100755 --- a/bin/zopen-list +++ b/bin/zopen-list @@ -1,6 +1,6 @@ #!/bin/sh # -# List utility for z/OS Open Tools - https://github.com/ZOSOpenTools +# List utility for zopen community- https://github.com/zopencommunity # # @@ -27,8 +27,8 @@ zopen list - list information about local packages Usage: zopen list [OPTION] [VERB] [PACKAGE] Options: - --available lists all available z/OS Open Tools - --installed list only installed z/OS Open Tools (default output) + --available lists all available packages + --installed list only installed packages (default output) --details provide version information on packages --full provides more information about packages --[no]header output explanation header for columns @@ -42,7 +42,7 @@ Examples: zopen list --installed list packages installed on the system -Report bugs at https://github.com/ZOSOpenTools/meta/issues . +Report bugs at https://github.com/zopencommunity/meta/issues HELPDOC } @@ -99,9 +99,9 @@ listAvailablePackages() winstalled=10 wtag=36 wrellne=12 - wdownload=10 - wexpsz=10 - wqual=7 + wdownload=12 + wexpsz=12 + wqual=10 if ${urlinclude}; then wurl=6 urltext="URL" @@ -150,7 +150,8 @@ listAvailablePackages() $va.size as $ds | ($va.passed_tests | if (.=="") then "0" else . end | tonumber) as $pt | ($va.total_tests | if (.=="") then "99999" else . end |tonumber) as $tt | - (($pt/$tt*100|r(2))|tostring + "%") as $q | + ($pt/$tt*100) as $pts | + ($pts | if (.<0) then "No tests" else ($pts|r(1) |tostring + "%" ) end) as $q| pr($key;"@";($wpackage | tonumber)) + c($ins;"@";($winstalled | tonumber)) + c($tn;"@";($wtag | tonumber)) + From e72331385bb5c411f2fcf9ea81fa0b46891100c9 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 2 Apr 2025 12:37:21 +0100 Subject: [PATCH 086/179] Sanity check and attempt to rebuild the default repository config file --- bin/zopen-init | 25 ++++++++--- include/common.sh | 110 ++++++++++++++++++++++++++++++++-------------- 2 files changed, 97 insertions(+), 38 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index 2c96ba1ce..77e712b50 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -62,6 +62,11 @@ Options: configuration and regenerate configuration files. select the active version for PACKAGE from a list + --rebuild-repo-config + Recreate the default repository configuration file. Will also + refresh the zopen environment configuration, equivalent to + setting the --refresh parameter + --refresh Refreshes the zopen-config file @@ -119,6 +124,7 @@ reinitExisting=false appendToProfile=false releaselineDev=false refresh=false +rebuildRepoConfig=false isCollectingStats=false isBot=false # used by jenkins, CI/CD bypassPrereqs=false @@ -130,6 +136,10 @@ while [ $# -gt 0 ]; do "--bypass-prereq-checks") bypassPrereqs=true ;; + "--rebuild-repo-config") + rebuildRepoConfig=true + refresh=true + ;; "--refresh" ) refresh=true yesToPrompts=true # updating zopen-config is not as scary @@ -503,8 +513,10 @@ zz generateDefaultRepository() { - # Check for existing repository definition - if present, use. - [ -e "${rootfs}/etc/zopen/repos.d/active" ] && return 0 + # Check for existing repository definition - if present, use unless + # user wants to rebuild the default repository file + [ -e "${rootfs}/etc/zopen/repos.d/active" ] && + ! ${rebuildRepoConfig} && return 0 # If there is no repository definition, generate a default to point to Github [ ! -e "${rootfs}/etc/zopen/repos.d" ] && mkdir -p "${rootfs}/etc/zopen/repos.d" @@ -516,7 +528,7 @@ generateDefaultRepository() repotype="${paxrepourl%://*}" [ "${repotype}" = "${paxrepourl}" ] && printError "Unable to parse repository type from '${paxrepourl}'. Ensure valid format: ://://:// "${repoFile}" { @@ -543,7 +556,9 @@ generateDefaultRepository() "latest_file": "${latestmetafile}" } EOS - (cd "${rootfs}/etc/zopen/repos.d" && ln -s "${repoFile}" "active") + # Create active link if not already present; if it is present, it'll either point to the default already or + # it'll point to a custom repo - leave as-is. + [ -L "${rootfs}/etc/zopen/repos.d/active" ] || (cd "${rootfs}/etc/zopen/repos.d" && ln -s "${repoFile}" "active") } generateAnalyticsConfiguration() diff --git a/include/common.sh b/include/common.sh index 85175df47..06e866e59 100755 --- a/include/common.sh +++ b/include/common.sh @@ -489,22 +489,25 @@ validateReleaseLine() } # Attempt to fully dereference a symlink without any bashisms or arcane set logic -# using some simplistic (recursive!) logic -deref() -{ - testpath="$1" - if [ -L "${testpath}" ]; then - child=$(basename "${testpath}") - symlink=$(ls -l "${testpath}" | zossed 's/.*-> \(.*\)/\1/') - parent=$(dirname "${testpath}") - relpath="${parent}/${symlink}" - relparent=$(dirname "${relpath}") - abspath=$(cd "${relparent}" && pwd -P) - testpath="${abspath}/${child}" - deref "${testpath}" - else - [ -n "${testpath}" ] && echo "${testpath}" && unset testpath - fi +deref_symlink() { + symlink="$1" + if [ -L "${symlink}"]; then + target=$(ls -l "$symlink" | awk '{print $NF}') + else + target="${symlink}" + fi + targetName=$(basename "${target}") + targetDir=$(dirname "${target}") + # Check if dir starts with a '/'; if not need to calculate absolute + # fully-qualified name + case "$target" in + /*) targetDir=$(cd "${targetDir}" && pwd -P);; + *) + symlink_dir=$(dirname "$symlink") + targetDir=$(cd "$(pwd -P)/${symlink_dir}" && pwd -P) + esac + absolute_dir=$(cd "$targetDir" && pwd -P) # Resolves symlinked dirs + echo "$absolute_dir/$targetName" } toAbsolutePath() { @@ -513,6 +516,7 @@ toAbsolutePath() { * | . | .. | ./* | ../*) echo "$(cd "$(dirname "$1")" && pwd -P)/$(basename "$1")" ;; esac } + #return 1 if brightness is dark, 0 if light, and 255 if unknown (considered to be dark as default) darkbackground() { if [ "${#COLORFGBG}" -ge 3 ]; then @@ -1441,20 +1445,48 @@ syslog() echo "$(date +"%F %T") $(id | cut -d' ' -f1)::${module}:${type}:${categories}:${location}:${msg}" >> "${fd}" } -getJSONCacheURLs(){ - activeRepo="${ZOPEN_ROOTFS}/etc/zopen/repos.d/active" - [ ! -e "${activeRepo}" ] && printError "Could not access repository configuration at '${activeRepo}'. Check file to ensure valid repository configuration or refresh default configuration with zopen init --refresh -y." +jqGetKey(){ + # If there key is not present, the error causes jq to exit with a + # non-zero return code + jq -er --arg key "$1" '.[$key] // error("Missing key: " + $key)' "$2" +} - type=$(jq -r ".type" "${activeRepo}") - base=$(jq -r ".metadata_baseurl" "${activeRepo}") - filename=$(jq -r ".metadata_file" "${activeRepo}") - latest_metadata=$(jq -r ".latest_file" "${activeRepo}") +validateReposDEntry(){ + activeRepo=$1 + requiredKeys="type metadata_baseurl metadata_file latest_file" + printVerbose "Ensuring repo file is valid; required keys: ${requiredKeys}" + for reposDEntry in ${requiredKeys}; do + if ! jqk=$(jqGetKey "${reposDEntry}" "${activeRepo}"); then + printSoftError "Repository definition at '${activeRepo}' incomplete; missing required key: ${reposDEntry}." + printError "Running zopen init --refresh might resolve this or recreating file from a backup" + elif [ -z "${jqk}" ]; then + printSoftError "Repository definition at '${activeRepo}' incomplete; required key '${reposDEntry}' has no value" + printError "Check the value in '${activeRepo}' is correct and retry command" + else + printVerbose "Key '${reposDEntry}' found with value '${jqk}'" + fi + done +} + +getJSONCacheURLs(){ + reposdDir="${ZOPEN_ROOTFS}/etc/zopen/repos.d" + activeRepo="${reposdDir}/active" + dereffedLink=$(deref_symlink "${activeRepo}") + [ ! -e "${dereffedLink}" ] && printError "Could not access linked repository configuration at '${dereffedLink}'. Check file to ensure valid repository configuration or refresh default configuration with zopen init --refresh -y." + if ! validateReposDEntry "${dereffedLink}"; then + return 1 + fi + type=$(jqGetKey "type" "${dereffedLink}") + base=$(jqGetKey "metadata_baseurl" "${dereffedLink}") + filename=$(jqGetKey "metadata_file" "${dereffedLink}") + latest_metadata=$(jqGetKey "latest_file" "${dereffedLink}") + case "${type}" in - http|https) printf "%s://%s/%s\n%s://%s/%s" \ - "${type}" "${base}" "${filename}" "${type}" "${base}" "${latest_metadata}" + http|https) jsonCacheURLs=$(printf "%s://%s/%s\n%s://%s/%s" \ + "${type}" "${base}" "${filename}" "${type}" "${base}" "${latest_metadata}") ;; - file) printf "%s:%s/%s\n%s:%s/%s" \ - "${type}" "${base}" "${filename}" "${type}" "${base}" "${latest_metadata}" + file) jsonCacheURLs=$(printf "%s:%s/%s\n%s:%s/%s" \ + "${type}" "${base}" "${filename}" "${type}" "${base}" "${latest_metadata}") ;; *) printError "Unsupported repository type '${type}'.";; esac @@ -1477,12 +1509,14 @@ updateJSONCaches() printVerbose "Ensuring cache directory exists" cachedir="${ZOPEN_ROOTFS}/var/cache/zopen" [ ! -e "${cachedir}" ] && mkdir -p "${cachedir}" - jsonCacheURLs=$(getJSONCacheURLs) # Also sets $latest_metadata + if ! getJSONCacheURLs; then + return 1 + fi printVerbose "Checking if the JSON_CACHE already downloaded in this session" if [ -z "${JSON_CACHE}" ]; then JSON_CACHE="${cachedir}/zopen_releases.json" - downloadJSONCacheIfExpired "${JSON_CACHE}" "$(echo "${jsonCacheURLs}" | head -n 1)" + downloadJSONCacheIfExpired "${JSON_CACHE}" "$(echo "${jsonCacheURLs}" | tail -n 2 | head -n 1)" fi if [ -z "${JSON_LATEST_CACHE}" ]; then JSON_LATEST_CACHE="${cachedir}/zopen_releases_latest.json" @@ -1601,7 +1635,9 @@ downloadJSONCache() # 1 failure getRepos() { - updateJSONCaches "$1" + if ! updateJSONCaches "$1"; then + return 1 + fi # shellcheck disable=SC2034 repo_results="$(jq -r '.release_data | keys[]' "${JSON_CACHE}")" } @@ -1615,14 +1651,18 @@ getRepos() # 1 invalid port name isValidRepo() { - updateJSONCaches + if ! updateJSONCaches; then + return 1 + fi jq -r --arg needle "$1" 'if .release_data | has($needle) then empty else error("") end' "${JSON_CACHE}" > /dev/null 2>&1 } #Deprecated getRepoReleases() { - updateJSONCaches + if ! updateJSONCaches; then + return 1 + fi repo="$1" ##TODO ##TDORM releases="$(jq -e -r '.release_data."'${repo}'"' "${JSON_CACHE}")" @@ -1998,7 +2038,9 @@ getPortMetaData(){ fi parseRepoName "${portRequested}" # To set the various status flags below - getRepoReleases "${portName}" + if ! getRepoReleases "${portName}"; then + return 1 + fi if [ -n "${versioned}" ]; then if ! getVersionedMetadata "${portName}" "${invalidPortAssetFile}"; then return 1 @@ -2430,6 +2472,8 @@ getInstallFile() extractMetadataFromPax() { + # If there is an issue with the pax or the target, it is possible for pax + # itself to report an error and sit waiting for user input; if ! pax -rf "$1" -s "%[^/]*/%/tmp/%" '*/metadata.json' ; then if ! details=$(pax -rf "$1" -s "%[^/]*/%/tmp/%" '*/package.json'); then printSoftError "Could not extract package metadata from file '$1'." From 99e7cf7ece434d347588c9ffe674c761cab2fc24 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 2 Apr 2025 14:52:21 +0100 Subject: [PATCH 087/179] Ensure zopen community terminology used --- bin/zopen-install | 2 +- bin/zopen-query | 6 +++--- bin/zopen-upgrade | 7 +++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/bin/zopen-install b/bin/zopen-install index e6f7cb6c9..10654e861 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -96,7 +96,7 @@ Examples: install packages foo and bar from the specified locations -Report bugs at https://github.com/ZOSOpenTools/meta/issues . +Report bugs at https://github.com/zopencommunity/meta/issues HELPDOC } diff --git a/bin/zopen-query b/bin/zopen-query index 4fc9b3e1c..0c762f18a 100755 --- a/bin/zopen-query +++ b/bin/zopen-query @@ -32,8 +32,8 @@ Usage: zopen query [OPTION] [VERB] [PACKAGE] Options: -d, --details include full details for listings - -i, --installed list only installed z/OS Open Tools - --list list all available z/OS Open Tools + -i, --installed list only installed pacakages + --list list all available packages --no-header suppress the header for the output --no-version suppress version information, return package names --remote-search @@ -230,7 +230,7 @@ whatProvides() echo "${found}" | xargs | tr ' ' '\n' | while read foundmatch; do printVerbose "Parsing '${foundmatch}'" if [ ! -d "${foundmatch}" ]; then - dereferenced=$(deref "${foundmatch}") + dereferenced=$(deref_symlink "${foundmatch}") fullpackage=$(echo "${dereferenced}" | sed "s#${ZOPEN_PKGINSTALL}/\([^/]*\).*#\1#") printInfo "Package '${fullpackage}' provides: '${foundmatch}'" diff --git a/bin/zopen-upgrade b/bin/zopen-upgrade index 87040dffb..8a8fc2053 100755 --- a/bin/zopen-upgrade +++ b/bin/zopen-upgrade @@ -1,5 +1,5 @@ #!/bin/sh -# Upgrade utility for z/OS Open Tools - https://github.com/ZOSOpenTools +# Upgrade utility for zopen community- https://github.com/zopencommunity # # All zopen-* scripts MUST start with this code to maintain consistency # @@ -20,8 +20,7 @@ checkWritable printHelp(){ cat << HELPDOC -zopen upgrade is a utility for z/OS Open Tools to upgrade packages to -a later release +zopen upgrade is a utility to upgrade installed packages to a later release Usage: zopen upgrade [OPTION] [PARAMETERS] [PACKAGES] @@ -36,7 +35,7 @@ Examples: zopen upgrade -y upgrade all packages to latest version on their releaseline -Report bugs at https://github.com/ZOSOpenTools/meta/issues . +Report bugs at https://github.com/zopencommunity/meta/issues HELPDOC } From d7635af30496250f1e95393fb09c71740b6b09b1 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 2 Apr 2025 15:07:23 +0100 Subject: [PATCH 088/179] Disable command trace --- bin/zopen-mirror | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/zopen-mirror b/bin/zopen-mirror index e26b47ed6..f299e5adb 100755 --- a/bin/zopen-mirror +++ b/bin/zopen-mirror @@ -1,4 +1,4 @@ -#!/bin/sh -x +#!/bin/sh # Repo mirror utility - stands alone from other scripts so has no # direct dependencies on existing functions. As such, it has some # cut-down versions of common features and an independent version From 6f6c22906b3ec70782dd76e5e1b2c29957e666d4 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 2 Apr 2025 17:19:51 +0100 Subject: [PATCH 089/179] Merge issue with common.sh --- include/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common.sh b/include/common.sh index 7d84dabdc..984a03746 100755 --- a/include/common.sh +++ b/include/common.sh @@ -491,7 +491,7 @@ validateReleaseLine() # Attempt to fully dereference a symlink without any bashisms or arcane set logic deref_symlink() { symlink="$1" - if [ -L "${symlink}"]; then + if [ -L "${symlink}" ]; then target=$(ls -l "$symlink" | awk '{print $NF}') else target="${symlink}" From fd9144e5aba13671bec3057aa2067dc3a1ad0dd4 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 14 May 2025 07:39:02 +0100 Subject: [PATCH 090/179] Allow install without initial update of tools --- bin/zopen-init | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/bin/zopen-init b/bin/zopen-init index 77e712b50..f9f66f3a0 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -53,6 +53,14 @@ Options: -h, -?, --help display this help and exit + + --offline + Perform the base installation of the zopen environment without + attempting to update any of the bootstrapping packages. For + use in air-gapped environments or where a custom package + repository will be configed + Note: Post-configuration of alternate package sources might be + required. --re-init Re-initializes a previous zopen environment or @@ -122,6 +130,7 @@ debug=false yesToPrompts=false reinitExisting=false appendToProfile=false +offlineInstall=false releaselineDev=false refresh=false rebuildRepoConfig=false @@ -162,6 +171,9 @@ while [ $# -gt 0 ]; do "--noenable-stats" ) enableStats=false ;; + "--offline" ) + offlineInstall=true + ;; "-h" | "--help" | "-?") printHelp exit 0 @@ -750,5 +762,5 @@ generateDefaultRepository generateZopenConfig generateAnalyticsConfiguration [ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || createInitialPackageDB -updateBootstrappedTools +! ${offlineInstall} && updateBootstrappedTools deinit From 7513fd90bf053aaf9f442aaa1748ab02ac6211af Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 14 May 2025 07:42:09 +0100 Subject: [PATCH 091/179] Remove commented out code --- bin/zopen-alt | 1 - include/common.sh | 88 ----------------------------------------------- 2 files changed, 89 deletions(-) diff --git a/bin/zopen-alt b/bin/zopen-alt index 19226cfa9..876e88723 100755 --- a/bin/zopen-alt +++ b/bin/zopen-alt @@ -78,7 +78,6 @@ mergeNewVersion() printSoftError "Unexpected errors merging symlinks into mesh tree" printError "Use zopen alt to select previous version to ensure known state" fi - ##TODORM mergeIntoSystem "${package}" "${ZOPEN_PKGINSTALL}/${package}/${newver}" "${ZOPEN_ROOTFS}" printVerbose "New version merged; checking for orphaned files from previous version" # This will remove any old symlinks or dirs that might have changed in an up/downgrade # as the merge process overwrites existing files to point to different version. If there was diff --git a/include/common.sh b/include/common.sh index 984a03746..d55781c99 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1574,57 +1574,6 @@ downloadJSONCache() else return 0 fi - ##TODORM>> - if [ -z "${JSON_CACHE}" ]; then - cachedir="${ZOPEN_ROOTFS}/var/cache/zopen" - [ ! -e "${cachedir}" ] && mkdir -p "${cachedir}" - JSON_CACHE="${cachedir}/zopen_releases.json" - JSON_TIMESTAMP="${cachedir}/zopen_releases.timestamp" - JSON_TIMESTAMP_CURRENT="${cachedir}/zopen_releases.timestamp.current" - - if [ -n "$from_readonly" ]; then - if [ -r "$JSON_CACHE" ] && [ ! -w "$JSON_CACHE" ]; then - return; # Skip the download for read only operations when you know you can't write to it - fi - fi - - # Need to check that we can read & write to the JSON timestamp cache files - if [ -e "${JSON_TIMESTAMP_CURRENT}" ]; then - [ ! -w "${JSON_TIMESTAMP_CURRENT}" ] || [ ! -r "${JSON_TIMESTAMP_CURRENT}" ] && printError "Cannot access cache at '${JSON_TIMESTAMP_CURRENT}'. Check permissions and retry request." - fi - if [ -e "${JSON_TIMESTAMP}" ]; then - [ ! -w "${JSON_TIMESTAMP}" ] || [ ! -r "${JSON_TIMESTAMP}" ] && printError "Cannot access cache at '${JSON_TIMESTAMP}'. Check permissions and retry request." - fi - if [ -e "${JSON_CACHE}" ]; then - [ ! -w "${JSON_CACHE}" ] || [ ! -r "${JSON_CACHE}" ] && printError "Cannot access cache at '${JSON_CACHE}'. Check permissions and retry request." - fi - - jsonCacheURL=$(getJSONCacheURL) - if ! curlCmd -f -L -s -I "${jsonCacheURL}" -o "${JSON_TIMESTAMP_CURRENT}"; then - printError "Failed to obtain json cache timestamp from ${jsonCacheURL}." - fi - chtag -tc 819 "${JSON_TIMESTAMP_CURRENT}" - - if [ -f "${JSON_CACHE}" ] \ - && [ -f "${JSON_TIMESTAMP}" ] \ - && [ "$(grep 'Last-Modified' "${JSON_TIMESTAMP_CURRENT}")" = "$(grep 'Last-Modified' "${JSON_TIMESTAMP}")" ]; then - # Metadata cache unchanged - return - fi - - printVerbose "Replacing old timestamp with latest." - mv -f "${JSON_TIMESTAMP_CURRENT}" "${JSON_TIMESTAMP}" - - if ! curlCmd -f -L -s -o "${JSON_CACHE}" "${jsonCacheURL}"; then - printError "Failed to obtain json cache from '${jsonCacheURL}'" - fi - chtag -tc 819 "${JSON_CACHE}" - fi - - if [ ! -f "${JSON_CACHE}" ]; then - printError "Could not download json cache from '${jsonCacheURL}" - fi - ## <> - # shellcheck disable=SC2154 - #if ${doNotInstallDeps}; then - # printVerbose "- Skipping dependency analysis" - #else - # calculate dependancy graph - # createDependancyGraph "${invalidPortAssetFile}" - #fi - ##<> -hidden(){ - installed=$(zopen list --installed --details) - # Ignore the version string - it varies across ports so use name and build time as that - # should be unique enough - installed=$(echo "${installed}"| awk 'BEGIN{ORS = "," } {print "\"" $1 "@=@" $3 "\""}') - installed="[${installed%,}]" - - installList=$(echo "${installList}" | \ - jq --argjson installees "${installed}" \ - '.installqueue |= - map( - select(.asset.url | - capture(".*/(?.*)-(?[^-]*)\\.(?\\d{8}_\\d{6}?)\\.zos\\.pax\\.Z$") |.rel as $rel | .name as $name | - $installees | map( - .|capture("(?[^@]*)@=@(?\\d{8}_\\d{6}?)$")|.iname as $iname | .irel as $irel | - ($iname+"-"+$irel) == ($name+"-"+$rel) - ) | any == false - ) - )'\ - ) -} -##< Date: Wed, 14 May 2025 07:47:31 +0100 Subject: [PATCH 092/179] Handle "no-package" systems db creation --- include/common.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/include/common.sh b/include/common.sh index d55781c99..8370e5d81 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2622,7 +2622,17 @@ updatePackageDB() addCleanupTrapCmd "rm -rf ${backup}" cp "${pdb}" "${backup}" rm "${pdb}" + else + printVerbose "No current package database [new install?]. Creating empty array[]" + printf "[\n]\n" > "${pdb}" + fi + + if [ $(unset CD_PATH; cd "${ZOPEN_PKGINSTALL}"; ls -A | wc -l) -eq 0 ]; then + printVerbose "No packages found to add to database [new install?]" + return fi + + if ! pkgdirs=$(getActivePackageDirs); then printError "Unable to update the package db" fi @@ -2662,10 +2672,6 @@ updatePackageDB() fi mv "${pdb}.working" "${pdb}" done - if [ ! -e "${pdb}" ]; then - printVerbose "No currently installed packages [new install?]. Creating empty array[]" - printf "[\n]\n" > "${pdb}" - fi } stripControlCharacters(){ From df0dcc3eb64a06246611fe31a3dbe38b6f840dc4 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 14 May 2025 07:48:38 +0100 Subject: [PATCH 093/179] Ensure caches update --- include/common.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/common.sh b/include/common.sh index 8370e5d81..fcbb72fcd 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1550,7 +1550,8 @@ downloadJSONCacheIfExpired() if [ -f "${fileToCache}" ] \ && [ -f "${cacheTimestamp}" ] \ - && [ "$(grep 'Last-Modified' "${cacheTimestampCurrent}")" = "$(grep 'Last-Modified' "${cacheTimestamp}")" ]; then + && grep -q 'ETag' "${JSON_TIMESTAMP_CURRENT}" \ + && [ "$(grep 'ETag' "${cacheTimestampCurrent}")" = "$(grep 'ETag' "${cacheTimestamp}")" ]; then # Metadata cache unchanged return fi From 608022b3047006c8d23809e50149ce6728bc6891 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 14 May 2025 07:51:08 +0100 Subject: [PATCH 094/179] More granular package installs (kb v Mb) --- include/common.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/include/common.sh b/include/common.sh index fcbb72fcd..658bf8165 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2221,8 +2221,9 @@ spaceValidate(){ # there might be a need to download the package into the cache - if # autocacheclean is active, then this should be temporary; if not, then the # cache will grow - infrom the user either way - printVerbose "Reinstall of package, so pacakge file size delta should be 0!" + printVerbose "Reinstall of package, so package file size delta should be 0!" spaceRequiredMB=$(echo "scale=0; (${cacheBytes}) / (1024 * 1024)" | bc) + spaceRequiredKb=$(( cacheBytes / 1024 )) if ! isCacheClean=$(zopen config --get autocacheclean); then printError "Could not determine autocacheclean status" fi @@ -2234,15 +2235,16 @@ spaceValidate(){ else # If not a reinstall, assume there is a need for both the package and the expanded # package to be required + spaceRequiredKb=$(( (cacheBytes + packageBytes) / 1024 )) spaceRequiredMB=$(echo "scale=0; (${cacheBytes} + ${packageBytes}) / (1024 * 1024)" | bc) printInfo "After this operation, ${spaceRequiredMB} MB of additional disk space will be used." + printInfo "After this operation, $(formattedFileSize ${spaceRequiredKb}) of additional disk space will be used." fi - - availableSpaceMB=$(/bin/df -m "${ZOPEN_ROOTFS}" | sed "1d" | awk '{ print $3 }' | awk -F'/' '{ print $1 }') - if [ "${availableSpaceMB}" -lt "${spaceRequiredMB}" ]; then - printWarning "Your zopen file-system (${ZOPEN_ROOTFS}) only has ${availableSpaceMB} MB of available space." + availableSpaceKb=$(/bin/df -k "${ZOPEN_ROOTFS}" | sed "1d" | awk '{ print $3 }' | awk -F'/' '{ print $1 }') + if [ "${availableSpaceKb}" -lt "${spaceRequiredKb}" ]; then + printWarning "Your zopen file-system (${ZOPEN_ROOTFS}) only has $(formattedFileSize "${availableSpaceKb}") of available space." fi - if ! ${yesToPrompts} || [ "${availableSpaceMB}" -lt "${spaceRequiredMB}" ]; then + if ! ${yesToPrompts} || [ "${availableSpaceKb}" -lt "${spaceRequiredKb}" ]; then while true; do /bin/printf "Do you want to continue [y/n/a]? " read continueInstall < /dev/tty From 6131e4175fa669d716b38ba7993b890923af56e9 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 14 May 2025 07:52:08 +0100 Subject: [PATCH 095/179] Local pax install validation --- include/common.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/include/common.sh b/include/common.sh index 658bf8165..eb6c15ae7 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2451,10 +2451,16 @@ installFromPax() printDebug "Check for existing directory for version '${installdirname}'" if [ -d "${ZOPEN_PKGINSTALL}/${installdirname}" ]; then - printVerbose "- Clearing existing directory and contents" - rm -rf "${ZOPEN_PKGINSTALL}/${installdirname}" + printVerbose "- Clearing existing directory and contents at '${ZOPEN_PKGINSTALL}/${installdirname}'" + rm -rf "${ZOPEN_PKGINSTALL:?}/${installdirname}" || printError "Installation directory resolved to '/'; cannot recursively clear contents!" fi + if [ ! -e "${pax}" ]; then + printError "Pax file to install not available at '${pax}'" + elif [ ! -r "${pax}" ]; then + printError "Pax file to install not readable at '${pax}'" + fi + # shellcheck disable=SC2154 if ! runLogProgress "pax -rf ${pax} -p p ${paxredirect} ${redirectToDevNull}" \ "Expanding file: ${pax}" "Expanded file: ${pax}"; then From 0339adf05b97cfd158358b421da836309af1dc80 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 14 May 2025 07:59:55 +0100 Subject: [PATCH 096/179] Use correct variable for cached filename --- include/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common.sh b/include/common.sh index eb6c15ae7..ece6d2a5f 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1550,7 +1550,7 @@ downloadJSONCacheIfExpired() if [ -f "${fileToCache}" ] \ && [ -f "${cacheTimestamp}" ] \ - && grep -q 'ETag' "${JSON_TIMESTAMP_CURRENT}" \ + && grep -q 'ETag' "${cacheTimestamp}" \ && [ "$(grep 'ETag' "${cacheTimestampCurrent}")" = "$(grep 'ETag' "${cacheTimestamp}")" ]; then # Metadata cache unchanged return From 3aed2d98315c746622a5c87bdea8a02aff156459 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 14 May 2025 13:08:58 +0100 Subject: [PATCH 097/179] Remove extra size report. --- include/common.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/include/common.sh b/include/common.sh index ece6d2a5f..1112cc43e 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2237,7 +2237,6 @@ spaceValidate(){ # package to be required spaceRequiredKb=$(( (cacheBytes + packageBytes) / 1024 )) spaceRequiredMB=$(echo "scale=0; (${cacheBytes} + ${packageBytes}) / (1024 * 1024)" | bc) - printInfo "After this operation, ${spaceRequiredMB} MB of additional disk space will be used." printInfo "After this operation, $(formattedFileSize ${spaceRequiredKb}) of additional disk space will be used." fi availableSpaceKb=$(/bin/df -k "${ZOPEN_ROOTFS}" | sed "1d" | awk '{ print $3 }' | awk -F'/' '{ print $1 }') From fa031fc71c23aa76183c4c132bb6e936bdec2af6 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 15 May 2025 12:22:24 +0100 Subject: [PATCH 098/179] Allow skipping of broken packages during install --- bin/zopen-config-helper | 4 ++++ bin/zopen-init | 14 ++++++++++++-- bin/zopen-install | 8 ++++++++ include/common.sh | 7 ++++++- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/bin/zopen-config-helper b/bin/zopen-config-helper index ff3f90915..67309a4da 100755 --- a/bin/zopen-config-helper +++ b/bin/zopen-config-helper @@ -77,6 +77,10 @@ Variables Tells zopen install whether to install packages from the STABLE line or from the DEV line. DEV packages will be newer but might exhibit unexpected behaviour. The default is to use STABLE packages. + + skip_broken [integer:0|1] + Tells zopen that if during a transation an action fails, report the + error and continue to the next action when possible. Notes: Configuration options are not validated in that any key/value pairs can diff --git a/bin/zopen-init b/bin/zopen-init index f9f66f3a0..738c38660 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -455,7 +455,8 @@ generateJsonConfiguration() OVERRIDE_ZOS_TOOLS=$(jqw -r '.override_zos_tools' $jsonConfig) fi autocacheclean=$(jqw -r '.autocacheclean' "${jsonConfig}") - autopkgpurge=$(jqw -r '.autopkgpurge' "${jsonConfig}") + autopkgpurge=$(jqw -r '.autopkgpurge' "${jsonConfig}") + skip_broken=$(jqw -r '.skip_broken' "${jsonConfig}") fi if ${releaselineDev}; then @@ -489,6 +490,14 @@ generateJsonConfiguration() autopkgpurge=0 ;; esac + case "${skip_broken:=0}" in + "0"|"1") :;; # Valid value + "null") skip_broken=0 # Default to false + ;; + *) showConfigParmWarning "skip_broken" "${skip_broken}" "0|1" "0" + skip_broken=0 + ;; + esac case "${OVERRIDE_ZOS_TOOLS}" in "true"|"false") :;; # Valid value "null") OVERRIDE_ZOS_TOOLS=false ;; # No entry in config file @@ -504,7 +513,8 @@ generateJsonConfiguration() "is_collecting_stats": ${isCollectingStats}, "override_zos_tools": ${OVERRIDE_ZOS_TOOLS}, "autocacheclean": ${autocacheclean}, - "autopkgpurge": ${autopkgpurge} + "autopkgpurge": ${autopkgpurge}, + "skip_broken": ${skip_broken} } zz else diff --git a/bin/zopen-install b/bin/zopen-install index 10654e861..caae9a083 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -73,6 +73,8 @@ Options: --release-line [stable|dev] the release line to install from --select select a version to install from available versions + --skip-broken continue installing other packages as part of a + transaction where a package failed to install --skip-verify skips the package verification allowing un-trusted packages. Caveat usor! -v, --verbose run in verbose mode @@ -120,6 +122,7 @@ reinstallDeps=false installOrUpgrade=false # shellcheck disable=SC2034 nosymlink=false +skip_broken=false # shellcheck disable=SC2034 skipupgrade=false skipverify=false @@ -148,6 +151,7 @@ while [ $# -gt 0 ]; do installOrUpgrade=true # Upgrade package or install if not present ;; "--bypass-prereq-checks") + # shellcheck disable=SC2034 bypassPrereqs=true ;; "--no-symlink") @@ -173,6 +177,10 @@ while [ $# -gt 0 ]; do # shellcheck disable=SC2034 setactive=false # Install package as normal but keep existing installation as active ;; + "--skip-broken" | "-sb") + # shellcheck disable=SC2034 + skip_broken=true + ;; "--skip-verify" | "-sv") # shellcheck disable=SC2034 skipverify=true # Verify signature of packages diff --git a/include/common.sh b/include/common.sh index 1112cc43e..94d10d59a 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2436,7 +2436,12 @@ installFromPax() if ! processActionScripts "installPre" "${name}" "${metadatafile}" "${pax}"; then - printError "Failed installation pre-requisite check(s) for '${name}'. Correct previous errors and retry command" + skip_broken=$(jq -re '.skip_broken' "${ZOPEN_JSON_CONFIG}") + if [ "${skip_broken}" -ne 1 ]; then + printError "Failed installation pre-requisite check(s) for '${name}'. Correct previous errors and retry command" + fi + printSoftError "Skipping package '${name}' due to failed pre-requisite check(s)" + return 0 # 0 as we have handled the issue fi # Store current installation directory (if exists) From 34951ae98c69f309810025ce83b0feac60cda9f0 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 15 May 2025 12:23:34 +0100 Subject: [PATCH 099/179] Try to preserve exit codes from failures --- include/common.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/include/common.sh b/include/common.sh index 94d10d59a..01cd0024a 100755 --- a/include/common.sh +++ b/include/common.sh @@ -47,12 +47,14 @@ addCleanupTrapCmd(){ } cleanup() { + ogExitCode=$? # Save the original exit code if [ -f "${tmpscriptfile}" ] && [ -s "${tmpscriptfile}" ]; then # Execute the commands in the cleanup file by sourcing it # shellcheck disable=SC1090 . "${tmpscriptfile}" > /dev/null 2>&1 rm "${tmpscriptfile}" fi + return ${ogExitCode} } addCleanupTrapCmd2(){ @@ -795,7 +797,10 @@ mutexFree() mutex=$1 lockdir="${ZOPEN_ROOTFS}/var/lock" mutex="${lockdir}/${mutex}" - [ -e "${mutex}" ] && rm -f ${mutex} + if [ -e "${mutex}" ]; then + rm -f ${mutex} + fi + return 0 } relativePath2() From 3c5dc54444a139151a42b8d11733f64887cc8f67 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 15 May 2025 12:24:11 +0100 Subject: [PATCH 100/179] Turn off debug trace during process handling --- include/common.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/common.sh b/include/common.sh index 01cd0024a..a5bdaed9c 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1199,6 +1199,7 @@ runInBackgroundWithTimeoutAndLog() printVerbose "${command} with timeout of ${timeout}s." eval "${command} &; TEEPID=$!" PID=$! + [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" # Disable cmd trace if active n=0 while [ ${n} -le ${timeout} ]; do kill -0 "${PID}" 2> /dev/null @@ -1208,12 +1209,14 @@ runInBackgroundWithTimeoutAndLog() chtag -r "${SSH_TTY}" fi rc=$? + [ -n "${xtrc}" ] && set -x # Re-enable cmd trace return ${rc} else sleep 1 n=$(( n + 1)) fi done + [ -n "${xtrc}" ] && set -x # Re0enable cmd trace kill -9 "${PID}" 2>/dev/null kill -9 "${TEEPID}" 2>/dev/null printError "TIMEOUT: (PID: ${PID}): ${command}" From 7d109abd0cd00a3f9d68bc36804962060c3498fc Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 19 May 2025 12:36:26 +0100 Subject: [PATCH 101/179] Use Last-Modified for file-type repos, etag for http --- include/common.sh | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/include/common.sh b/include/common.sh index a5bdaed9c..9814a8367 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1556,12 +1556,20 @@ downloadJSONCacheIfExpired() fi chtag -tc 819 "${cacheTimestampCurrent}" - if [ -f "${fileToCache}" ] \ - && [ -f "${cacheTimestamp}" ] \ - && grep -q 'ETag' "${cacheTimestamp}" \ - && [ "$(grep 'ETag' "${cacheTimestampCurrent}")" = "$(grep 'ETag' "${cacheTimestamp}")" ]; then - # Metadata cache unchanged - return + if [ -f "${fileToCache}" ] && [ -f "${cacheTimestamp}" ]; then + # Extract eyecatchers - either ETag or Last-Modified depending on repo type + prevETag=$(grep -i '^ETag' "${cacheTimestamp}") + currETag=$(grep -i '^ETag' "${cacheTimestampCurrent}") + prevLastMod=$(grep -i '^Last-Modified' "${cacheTimestamp}") + currLastMod=$(grep -i '^Last-Modified' "${cacheTimestampCurrent}") + # Compare values if they exist and match - if they do, then the cache is current + if [ -n "$prevETag" ] && [ -n "$currETag" ]; then + printVerbose "Comparing previous ETag '${prevETag}' with current '${currETag}" + [ "$prevETag" = "$currETag" ] && return + elif [ -n "$prevLastMod" ] && [ -n "$currLastMod" ]; then + printVerbose "Comparing previous ETag '${prevLastMod}' with current '${currLastMod}" + [ "$prevLastMod" = "$currLastMod" ] && return + fi fi printVerbose "Replacing old timestamp with latest." From 1038a80231342c42a8b0d629f15240366f601887 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 4 Jun 2025 13:17:46 +0100 Subject: [PATCH 102/179] Don't update CA cert when offline init --- bin/zopen-init | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index 738c38660..126f7a864 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -428,16 +428,20 @@ EOF fi # Source the config file + # shellcheck disable=SC1090 . "${configFile}" - + printInfo "- zopen-config setup complete" +} +updateCACert() +{ printHeader "Updating CA Certificate" export ZOPEN_CA="${ZOPEN_ROOTFS}/${ZOPEN_CA_DIR}/${certFileName}" caCertDir="${ZOPEN_ROOTFS}/${ZOPEN_CA_DIR}" mkdir -p "${caCertDir}" - runAndLog "${MYDIR}/zopen-update-cacert -f "${caCertDir}" " + runAndLog "${MYDIR}/zopen-update-cacert -f ${caCertDir} " [ $? -ne 0 ] && exit $? - printInfo "- zopen bootstrapping complete" + printInfo "- CA Certificate updated" } generateJsonConfiguration() @@ -770,6 +774,7 @@ generateFileSystem generateJsonConfiguration generateDefaultRepository generateZopenConfig +! ${offlineInstall} && updateCACert generateAnalyticsConfiguration [ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || createInitialPackageDB ! ${offlineInstall} && updateBootstrappedTools From a7cb39b9a03ba7f531ec05eea55d939fdaec5d64 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 4 Jun 2025 13:18:53 +0100 Subject: [PATCH 103/179] Allow bypass for upgrades if a prereq is added --- bin/zopen-upgrade | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/zopen-upgrade b/bin/zopen-upgrade index 8a8fc2053..2d928e4b3 100755 --- a/bin/zopen-upgrade +++ b/bin/zopen-upgrade @@ -63,10 +63,17 @@ reinstallDeps=false nosymlink=false # shellcheck disable=SC2034 doNotInstallDeps=true +# shellcheck disable=SC2034 +bypassPrereqs=false + yesToPrompts=false chosenRepos="" while [ $# -gt 0 ]; do case "$1" in + "--bypass-prereq-checks") + # shellcheck disable=SC2034 + bypassPrereqs=true + ;; "--yes" | "-y") # shellcheck disable=SC2034 yesToPrompts=true # Automatically answer 'yes' to any questions From 1cc98961ddd543a38a3d8050e3ddb2d44527d9d8 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 4 Jun 2025 13:19:37 +0100 Subject: [PATCH 104/179] Pinned package breaks upgrades --- include/common.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/common.sh b/include/common.sh index 9814a8367..ae4c35345 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2488,14 +2488,15 @@ installFromPax() printError "Use zopen alt to select previous version to ensure known state" fi + setactiveLcl=true if [ -e "${ZOPEN_PKGINSTALL}/${name}/${name}/.pinned" ]; then printWarning "Current version of ${name} is pinned; not setting updated version as active" printWarning "Remove .pinned file and run 'zopen alt meta' to use new version" - setactive=false + setactiveLcl=false unInstallOldVersion=false fi # shellcheck disable=SC2154 - if ${setactive}; then + if $setactive && $setactiveLcl; then if [ -L "${ZOPEN_PKGINSTALL}/${name}/${name}" ]; then printDebug "Removing old symlink '${ZOPEN_PKGINSTALL}/${name}/${name}'" rm -f "${ZOPEN_PKGINSTALL}/${name}/${name}" @@ -2537,7 +2538,7 @@ EOF unsymlinkFromSystem "${name}" "${ZOPEN_ROOTFS}" "${currentderef}/.links" "${baseinstalldir}/${name}/${name}/.links" fi - if ${setactive}; then + if $setactive && $setactiveLcl; then printDebug "Marking this version as installed" touch "${ZOPEN_PKGINSTALL}/${name}/${name}/.active" installedList="${name} ${installedList}" From e8fd601be2d8bdbe9d6bc9597c8e7905881f8537 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 4 Jun 2025 14:53:49 +0100 Subject: [PATCH 105/179] Offline mode installer --- bin/zopen-init | 36 +++++++++++++++++++++++++++++++++++- include/common.sh | 7 +++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/bin/zopen-init b/bin/zopen-init index 126f7a864..47a09aaf4 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -432,6 +432,7 @@ EOF . "${configFile}" printInfo "- zopen-config setup complete" } + updateCACert() { printHeader "Updating CA Certificate" @@ -684,6 +685,39 @@ updateBootstrappedTools() { if ${refresh}; then ${MYDIR}/zopen-install ${installOptions} -y curl jq pinentry gpg toolInstall=$? + elif ${offlineInstall}; then + printVerbose "As offline, no access to meta.pax so copy currently running" + metabindir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P) + ulzm="${rootfs}/usr/local/zopen/meta" + target="${ulzm}/meta-bootstrap" + mkdir -p "${target}" + printVerbose "Copying bare meta scrtips from '${metabindir}' to '${target}" + cp -R "${metabindir}/../bin" "${target}" + cp -R "${metabindir}/../include" "${target}" + printVerbose "Creating main symlink for meta" + ln -s "${ulzm}/meta-bootstrap" "${ulzm}/meta" + if ! runLogProgress "mergeIntoSystem \"meta\" \"${ulzm}/meta-bootstrap\" \"${rootfs}\"" \ + "Merging meta into symlink mesh" "Merged meta into symlink mesh"; then + printSoftError "Unexpected errors merging symlinks into mesh" + printError "Use zopen alt to select previous version to ensure known state" + fi + + printVerbose "Installing bootstrap tools" + PAX_LOCATIONS="$CURL_PAX_LOCATION,$JQ_PAX_LOCATION,${PINENTRY_PAX_LOCATION},${GPG_PAX_LOCATION}" + + echo "$PAX_LOCATIONS" | tr ',' '\n' | while read -r pax; do + printVerbose "- Check for shipped ${pax}" + paxLocation=$(findrev "${MYDIR}" "${pax}") + echo "${paxLocation}/${pax}" + if [ -e "${paxLocation}/${pax}" ]; then + # Use local file install to make available; note that this should use + # the bootstrapped tools until the install completes and then gets + # sourced in + zopen install "${paxLocation}/${pax}" -y --bypass-prereq-checks --skip-verify + else + printError "Could not locate ${pax} in the zopen cache" + fi + done else ${MYDIR}/zopen-install ${installOptions} -y curl jq pinentry gpg meta toolInstall=$? @@ -777,5 +811,5 @@ generateZopenConfig ! ${offlineInstall} && updateCACert generateAnalyticsConfiguration [ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || createInitialPackageDB -! ${offlineInstall} && updateBootstrappedTools +updateBootstrappedTools deinit diff --git a/include/common.sh b/include/common.sh index ae4c35345..8f75e429d 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2362,6 +2362,13 @@ getInstallFile() printError "Could not download from ${installurl}. Correct any errors and potentially retry" fi fi + + # Check if this is a pax-flie install; in which case there is no "remote" JSON + case "${installurl}" in + file://*) return 0 ;; + *) :;; + esac + metadataFile="$(basename "${installurl}").json" if [ -e "${downloadToDir}/${metadataFile}" ]; then printVerbose "Corresponding metadata '${metadataFile}' already in local cache" From 735d4a8cdddb9d85ba83ffc861155a8cec9367b8 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 4 Jun 2025 15:22:54 +0100 Subject: [PATCH 106/179] Correctly generate local file location --- bin/zopen-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/zopen-install b/bin/zopen-install index caae9a083..e99e41fc0 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -312,8 +312,8 @@ if ${fileinstall}; then done # generate the install list JSON from the @@-delimited inputs installList=$(echo "${absoluteFiles}" \ - | jq --raw-input --arg d "$(pwd -P)" \ - 'def make_object($url): {asset:{url: ( "file://" + $d + "/" + $url )}}; . | split("@@") | map(select(.!="")|make_object(.)) | {"installqueue" :.} ') + | jq --raw-input \ + 'def make_object($url): {asset:{url: ( "file://" + $url )}}; . | split("@@") | map(select(.!="")|make_object(.)) | {"installqueue" :.} ') else if ${all}; then # shellcheck disable=SC2034 From a1d62e8dfd85022f02e76ebc85c5298a22f2ebc3 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Fri, 6 Jun 2025 12:36:09 +0100 Subject: [PATCH 107/179] Fix unused counter for all shells --- bin/zopen-clean | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/zopen-clean b/bin/zopen-clean index 2f3fce793..b43f91eeb 100755 --- a/bin/zopen-clean +++ b/bin/zopen-clean @@ -79,6 +79,7 @@ removeUnusedPackageVersions() counterfile=$(mktempfile "clean" ".rupv") addCleanupTrapCmd "rm -f ${counterfile}" cd "${ZOPEN_PKGINSTALL}/${needle}" && zosfind . -name "./*" -prune -type d > "${counterfile}" + unusedCount=0 while read repo; do printVerbose "Parsing repo: '${repo}' as '${repo#./}'" repo="${repo#./}" @@ -97,13 +98,13 @@ removeUnusedPackageVersions() fi fi done < "${counterfile}" + $stats && printInfo "Removed ${unusedCount} unused package version$([ ${unusedCount} -eq 1 ] || echo 's')" return 0 } cleanUnused() { ${deep} && printInfo "Removing any unused package versions" - unusedCount=0 # Updated within the removeUnusedPackageVersions function if ${all}; then zopen list --installed --no-header --no-version | while read -r needle; do removeUnusedPackageVersions "${needle}" @@ -115,7 +116,6 @@ cleanUnused() removeUnusedPackageVersions "${needle}" done fi - $stats && printInfo "Removed ${unusedCount} unused package version"$([ ${unusedCount} -eq 1 ] || echo 's') } cleanDangling() From 0f1e34cd119da9493613b1dd14cc7c8ac57cd9b1 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 17 Jun 2025 13:39:22 +0100 Subject: [PATCH 108/179] Handle removal of fstype during promote --- bin/zopen-promote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/zopen-promote b/bin/zopen-promote index 9b87d3230..92dbe9707 100755 --- a/bin/zopen-promote +++ b/bin/zopen-promote @@ -284,7 +284,7 @@ printVerbose "Grabbing pkginstall location from within promoted env" if [ -e "${promotefs}/etc/zopen/fstype" ]; then zopen_pkginstall=$(cat "${promotefs}/etc/zopen/fstype") else - printError "Unable to locate '${promotefs}/etc/zopen/fstype'; unrecognisable file system. Check permissions" + printInfo "Unable to locate '${promotefs}/etc/zopen/fstype'; unrecognisable file system. Using default." fi rmDanglingLinks(){ From 24169823d7fa27e1998ebc7f8832feac60c167ba Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 17 Jun 2025 13:44:12 +0100 Subject: [PATCH 109/179] Only parse package man pages during install, not full scan --- bin/zopen-alt | 2 +- bin/zopen-remove | 3 +-- include/scriptlets/installPost/man-db | 22 ++++++++++++++++++++++ include/scriptlets/transactionPost/man-db | 13 +++++++++++++ 4 files changed, 37 insertions(+), 3 deletions(-) create mode 100755 include/scriptlets/installPost/man-db diff --git a/bin/zopen-alt b/bin/zopen-alt index 876e88723..9395557ef 100755 --- a/bin/zopen-alt +++ b/bin/zopen-alt @@ -100,7 +100,7 @@ mergeNewVersion() version=$(cat "${ZOPEN_PKGINSTALL}/${package}/${package}/.releaseinfo") fi syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE}" "ALT" "setAlt" "Set '${package}' to version:${version};" - processActionScripts "installPost" + processActionScripts "installPost" "${package}" processActionScripts "transactionPost" } diff --git a/bin/zopen-remove b/bin/zopen-remove index 4a236c72c..c42a076d4 100755 --- a/bin/zopen-remove +++ b/bin/zopen-remove @@ -56,7 +56,6 @@ removePackages() # Create a temp file name to trigger the post-transaction action script # triggering - run only if an action was taken (which creates the file) runpostTxnActionScripts=$(mktempfile "txnpost" "run") - addCleanupTrapCmd "rm -rf ${runpostTxnActionScripts}" echo "${pkglist}" | xargs | tr ' ' '\n' | sort | while read pkg; do printHeader "Removing package: ${pkg}" processActionScripts "removePre" @@ -119,7 +118,7 @@ removePackages() fi done if [ -e "${runpostTxnActionScripts}" ]; then - processActionScripts "transactionPost"; + processActionScripts "transactionPost" "mandb"; rm "${runpostTxnActionScripts}" fi } diff --git a/include/scriptlets/installPost/man-db b/include/scriptlets/installPost/man-db new file mode 100755 index 000000000..b94fdb6ed --- /dev/null +++ b/include/scriptlets/installPost/man-db @@ -0,0 +1,22 @@ +#!/bin/sh +this="man-db" +printVerbose "Running '${this}' script" +if ! zopen list --installed | grep "^man-db\$"; then + return 0 # No man-db installed +else + echo "- Updating man database for '$1'" + # Check if package shipped any man pages + manDir="${ZOPEN_PKGINSTALL}/${1}/${1}/usr/share/man" + [ -d "${manDir}" ] || return 0 + # Man pages were shipped; scan from the directory - the man + # directories are symlinked from ZOPEN_ROOTFS/usr/share/man/manX + # so they all point to the same location anyway. + if ! mandbOut=$(mandb "${manDir}" >/dev/null 2>&1); then + printSoftError "man-db update completed with non-zero return code." + printSoftError "Details: ${mandbOut}" + printSoftError "Re-run 'mandb' manually for additional information." + return 1 + fi + echo "- Update of man-db complete." +fi +return 0 diff --git a/include/scriptlets/transactionPost/man-db b/include/scriptlets/transactionPost/man-db index d2c567872..285194538 100755 --- a/include/scriptlets/transactionPost/man-db +++ b/include/scriptlets/transactionPost/man-db @@ -1,6 +1,19 @@ #!/bin/sh this="man-db" printVerbose "Running '${this}' script" +fullScan=false +while [ $# -gt 0 ]; do + case "$1" in + "mandb") fullScan=true ;; + *):;; + esac + shift +done +if ! $fullScan; then + # The option to run a full mandb scan was not passed; do not run scan + return 0 +fi + if ! zopen list --installed | grep "^man-db\$"; then return 0 # No man-db installed else From 0e2d920d1b17aaf8ea46579bed8b841996e56630 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 17 Jun 2025 13:44:58 +0100 Subject: [PATCH 110/179] Better handling of error during init --- bin/zopen-init | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index 47a09aaf4..a07410dbe 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -679,6 +679,7 @@ installPrereqs() updateBootstrappedTools() { printInfo "- Installing/updating bootstrapped curl, jq, gpg, pinentry and meta to install updated versions (if available)" installOptions="--force" + toolInstall=0 if $bypassPrereqs; then installOptions="${installOptions} --bypass-prereq-checks"; fi @@ -691,7 +692,7 @@ updateBootstrappedTools() { ulzm="${rootfs}/usr/local/zopen/meta" target="${ulzm}/meta-bootstrap" mkdir -p "${target}" - printVerbose "Copying bare meta scrtips from '${metabindir}' to '${target}" + printVerbose "Copying bare meta scripts from '${metabindir}' to '${target}" cp -R "${metabindir}/../bin" "${target}" cp -R "${metabindir}/../include" "${target}" printVerbose "Creating main symlink for meta" @@ -705,24 +706,34 @@ updateBootstrappedTools() { printVerbose "Installing bootstrap tools" PAX_LOCATIONS="$CURL_PAX_LOCATION,$JQ_PAX_LOCATION,${PINENTRY_PAX_LOCATION},${GPG_PAX_LOCATION}" - echo "$PAX_LOCATIONS" | tr ',' '\n' | while read -r pax; do - printVerbose "- Check for shipped ${pax}" + workingPaxList="$PAX_LOCATIONS" + printVerbose " Parsing without subshell to ensure exit" + while [ -n "${workingPaxList}" ]; do + case "${workingPaxList}" in + *,*) + pax="${workingPaxList%%,*}" + workingPaxList="${workingPaxList#*,}" + ;; + *) + # Last item [hence no comma separator] + pax="${workingPaxList}" + workingPaxList="" + ;; + esac + printVerbose "Validating pax '${pax}'" paxLocation=$(findrev "${MYDIR}" "${pax}") echo "${paxLocation}/${pax}" if [ -e "${paxLocation}/${pax}" ]; then - # Use local file install to make available; note that this should use - # the bootstrapped tools until the install completes and then gets - # sourced in zopen install "${paxLocation}/${pax}" -y --bypass-prereq-checks --skip-verify else - printError "Could not locate ${pax} in the zopen cache" + printError "Could not locate ${pax} to install as bootstrap" fi - done + done else ${MYDIR}/zopen-install ${installOptions} -y curl jq pinentry gpg meta toolInstall=$? fi - [ ${toolInstall} -ne 0 ] && printError "Unable to install curl, jq, gpg, pinentry and/or meta; see previous errors and retry the initilisation using the '--re-init' parameter" + [ "${toolInstall}" -ne 0 ] && printError "Unable to install curl, jq, gpg, pinentry and/or meta; see previous errors and retry the initilisation using the '--re-init' parameter" . "${configFile}" } From c0a837b150f2a1ca2d81f9802e9b116539b3f07b Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 17 Jun 2025 13:45:55 +0100 Subject: [PATCH 111/179] split cleanup logic per pid --- include/common.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/include/common.sh b/include/common.sh index 8f75e429d..05ea52441 100755 --- a/include/common.sh +++ b/include/common.sh @@ -34,8 +34,10 @@ addCleanupTrapCmd(){ # Attempt to remove any redirects; rather than test, simpler to remove # and re-add if present. newcmd="$1" - - tmpscriptfile="/tmp/zopen_trap.scr" + # shellcheck disable=SC2009 # no pgrep. + #mypid=$(ps -ef |grep -v grep |grep $0 | tr -s ' ' | cut -f3 -d ' ') + mypid=$(exec sh -c 'echo ${PPID}') + tmpscriptfile="/tmp/zopen_trap.${mypid}.scr" echo "${newcmd}" >> "${tmpscriptfile}" if [ $$ -eq "${parentPid}" ]; then @@ -48,6 +50,10 @@ addCleanupTrapCmd(){ cleanup() { ogExitCode=$? # Save the original exit code + # shellcheck disable=SC2009 # no pgrep. + #mypid=$(ps -ef |grep -v grep |grep ps | tr -s ' ' | cut -f3 -d ' ') + mypid=$(exec sh -c 'echo ${PPID}') + tmpscriptfile="/tmp/zopen_trap.${mypid}.scr" if [ -f "${tmpscriptfile}" ] && [ -s "${tmpscriptfile}" ]; then # Execute the commands in the cleanup file by sourcing it # shellcheck disable=SC1090 @@ -2138,7 +2144,7 @@ generateInstallGraph(){ # Create the following file here to trigger cleanup - otherwise, multiple # tempfiles could be created depending on dependency graph depth invalidPortAssetFile=$(mktempfile "invalid" "port") - #addCleanupTrapCmd "rm -rf ${invalidPortAssetFile}" + addCleanupTrapCmd "rm -rf ${invalidPortAssetFile}" if ! runLogProgress " addToInstallGraph \"install\" \"${invalidPortAssetFile}\" \"${portsToInstall}\"" \ "Creating install graph" "Created install graph" "linkcheck"; then if [ -e "${invalidPortAssetFile}" ]; then From 827672ee01ea2d52ab31daa8247f82282cf7d463 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 17 Jun 2025 13:47:06 +0100 Subject: [PATCH 112/179] Better calculation during --downloadonly install --- include/common.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/common.sh b/include/common.sh index 05ea52441..9c0d1ec88 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2254,6 +2254,12 @@ spaceValidate(){ else printInfo "After this operation, ${spaceRequiredMB} MB of additional cache will be used." fi + elif ${downloadOnly}; then + # To download to current dir, then only need to make sure there is enough disk space + # for the pax, the "cached" size + spaceRequiredMB=$(echo "scale=0; (${cacheBytes}) / (1024 * 1024)" | bc) + spaceRequiredKb=$(( cacheBytes / 1024 )) + printInfo "After this operation, $(formattedFileSize ${spaceRequiredKb}) of additional disk space will be used." else # If not a reinstall, assume there is a need for both the package and the expanded # package to be required From 15d05913851d2d25df685967324490162a24bd03 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 15 Jul 2025 15:04:35 +0100 Subject: [PATCH 113/179] Don't assume /tmp; allow $TMPDIR or $TMP --- include/common.sh | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/include/common.sh b/include/common.sh index 9c0d1ec88..7a5e9ab70 100755 --- a/include/common.sh +++ b/include/common.sh @@ -291,9 +291,7 @@ writeConfigFile(){ cat << EOF > "${configFile}" #!/bin/false # Script currently intended to be sourced, not run # zopen community Configuration file -# Main root location for the zopen installation; can be changed if the -# underlying root location is copied/moved elsewhere as locations are -# relative to this envvar value + displayHelp() { echo "usage: . zopen-config [--eknv] [--knv] [--quiet] [-?|--help]" echo " --override-zos-tools Adds altbin/ dir to the PATH and altman/ dir to MANPATH, overriding the native z/OS tooling." @@ -331,14 +329,22 @@ while [ \$# -gt 0 ]; do esac shift done + +for local_tmp in "\${TMPDIR}" "\${TMP}" /tmp; do + [ -n "\${local_tmp}" ] && [ -d "\${local_tmp}" ] && break +done + if \${knv}; then - /bin/env | /bin/sort > /tmp/zopen-config-env-orig.\$\$ + /bin/env | /bin/sort > \${local_tmp}/zopen-config-env-orig.\$\$ fi if [ -n "\${overrideFile}" ] && [ ! -f "\${overrideFile}" ]; then echo "Override file '\${overrideFile}' is not a file. Skipping..." fi +# Main root location for the zopen installation; can be changed if the +# underlying root location is copied/moved elsewhere as locations are +# relative to this envvar value ZOPEN_ROOTFS="${rootfs}" export ZOPEN_ROOTFS @@ -443,8 +449,8 @@ export LIBPATH=\$(deleteDuplicateEntries "\${LIBPATH}" ":") export MANPATH=\$(deleteDuplicateEntries "\${MANPATH}" ":") if \${knv}; then - /bin/env | /bin/sort > /tmp/zopen-config-env-modded.\$\$ - diffout=\$(/bin/diff /tmp/zopen-config-env-orig.\$\$ /tmp/zopen-config-env-modded.\$\$ | /bin/grep -E '^[>]' | /bin/cut -c3- ) + /bin/env | /bin/sort > \${local_tmp}/zopen-config-env-modded.\$\$ + diffout=\$(/bin/diff \${local_tmp}/zopen-config-env-orig.\$\$ \${local_tmp}/zopen-config-env-modded.\$\$ | /bin/grep -E '^[>]' | /bin/cut -c3- ) echo "\${diffout}" | while IFS= read -r knvp; do newval="" envvar="\${knvp%%=*}" @@ -457,7 +463,7 @@ if \${knv}; then echo "\${exportknv}\${envvar}=\${newval#*:}" IFS="\${cIFS}" done - rm /tmp/zopen-config-env-orig.\$\$ /tmp/zopen-config-env-modded.\$\$ 2>/dev/null + rm \${local_tmp}/zopen-config-env-orig.\$\$ \${local_tmp}/zopen-config-env-modded.\$\$ 2>/dev/null fi From 7e359504abb3d7ae035a952f2ddf2a4b5b16c217 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 15 Jul 2025 15:06:31 +0100 Subject: [PATCH 114/179] Stop leak of functions into env during sourcing --- include/common.sh | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/include/common.sh b/include/common.sh index 7a5e9ab70..ff55bd3db 100755 --- a/include/common.sh +++ b/include/common.sh @@ -362,7 +362,7 @@ fi zot="zopen community" -sanitizeEnvVar() +zot_sanitizeEnvVar() { # remove any envvar entries that match the specified regex value="\$1" @@ -371,7 +371,7 @@ sanitizeEnvVar() echo "\${value}" | awk -v RS="\${delim}" -v DLIM="\${delim}" -v PRFX="\${prefix}" '{ if (match(\$1, PRFX)==0) {printf("%s%s",\$1,DLIM)}}' } -deleteDuplicateEntries() +zot_deleteDuplicateEntries() { value="\$1" delim="\$2" @@ -419,12 +419,12 @@ if [ -z "\${ZOPEN_QUICK_LOAD}" ]; then fi fi unset displayText -PATH=\${ZOPEN_ROOTFS}/usr/local/bin:\${ZOPEN_ROOTFS}/usr/bin:\${ZOPEN_ROOTFS}/bin:\${ZOPEN_ROOTFS}/boot:\$(sanitizeEnvVar "\${PATH}" ":" "^\${ZOPEN_PKGINSTALL}/.*\$") -MANPATH=\${ZOPEN_ROOTFS}/usr/local/share/man:\${ZOPEN_ROOTFS}/usr/local/share/man/\%L:\${ZOPEN_ROOTFS}/usr/share/man:\${ZOPEN_ROOTFS}/usr/share/man/\%L:\$(sanitizeEnvVar "\${MANPATH}" ":" "^\${ZOPEN_PKGINSTALL}/.*\$") +PATH=\${ZOPEN_ROOTFS}/usr/local/bin:\${ZOPEN_ROOTFS}/usr/bin:\${ZOPEN_ROOTFS}/bin:\${ZOPEN_ROOTFS}/boot:\$(zot_sanitizeEnvVar "\${PATH}" ":" "^\${ZOPEN_PKGINSTALL}/.*\$") +MANPATH=\${ZOPEN_ROOTFS}/usr/local/share/man:\${ZOPEN_ROOTFS}/usr/local/share/man/\%L:\${ZOPEN_ROOTFS}/usr/share/man:\${ZOPEN_ROOTFS}/usr/share/man/\%L:\$(zot_sanitizeEnvVar "\${MANPATH}" ":" "^\${ZOPEN_PKGINSTALL}/.*\$") if [ -n "\$ZOPEN_TOOLSET_OVERRIDE" ]; then if [ -n "\${overrideFile}" ] && [ -f "\${overrideFile}" ]; then - PATH=\$(sanitizeEnvVar "\${PATH}" ":" "^\${ZOPEN_ROOTFS}/usr/local/altbin.*\$") + PATH=\$(zot_sanitizeEnvVar "\${PATH}" ":" "^\${ZOPEN_ROOTFS}/usr/local/altbin.*\$") while IFS= read -r project; do if [ -d "\$ZOPEN_PKGINSTALL/\$project/\$project/altbin" ]; then PATH="\$ZOPEN_PKGINSTALL/\$project/\$project/altbin:\$PATH" @@ -438,15 +438,15 @@ if [ -n "\$ZOPEN_TOOLSET_OVERRIDE" ]; then MANPATH="\${ZOPEN_ROOTFS}/usr/local/share/altman:\$MANPATH" fi else - PATH=\$(sanitizeEnvVar "\${PATH}" ":" "^\${ZOPEN_ROOTFS}/usr/local/altbin.*\$") - MANPATH=\$(sanitizeEnvVar "\${MANPATH}" ":" "^\${ZOPEN_ROOTFS}/usr/local/share/altman.*\$") + PATH=\$(zot_sanitizeEnvVar "\${PATH}" ":" "^\${ZOPEN_ROOTFS}/usr/local/altbin.*\$") + MANPATH=\$(zot_sanitizeEnvVar "\${MANPATH}" ":" "^\${ZOPEN_ROOTFS}/usr/local/share/altman.*\$") fi -export PATH=\$(deleteDuplicateEntries "\${PATH}" ":") -LIBPATH=\${ZOPEN_ROOTFS}/usr/local/lib:\${ZOPEN_ROOTFS}/usr/lib:\$(sanitizeEnvVar "\${LIBPATH}" ":" "^\${ZOPEN_PKGINSTALL}/.*\$") -export LIBPATH=\$(deleteDuplicateEntries "\${LIBPATH}" ":") -export MANPATH=\$(deleteDuplicateEntries "\${MANPATH}" ":") +export PATH=\$(zot_deleteDuplicateEntries "\${PATH}" ":") +LIBPATH=\${ZOPEN_ROOTFS}/usr/local/lib:\${ZOPEN_ROOTFS}/usr/lib:\$(zot_sanitizeEnvVar "\${LIBPATH}" ":" "^\${ZOPEN_PKGINSTALL}/.*\$") +export LIBPATH=\$(zot_deleteDuplicateEntries "\${LIBPATH}" ":") +export MANPATH=\$(zot_deleteDuplicateEntries "\${MANPATH}" ":") if \${knv}; then /bin/env | /bin/sort > \${local_tmp}/zopen-config-env-modded.\$\$ @@ -466,7 +466,10 @@ if \${knv}; then rm \${local_tmp}/zopen-config-env-orig.\$\$ \${local_tmp}/zopen-config-env-modded.\$\$ 2>/dev/null fi - +# Cleanup +unset -f zot_deleteDuplicateEntries 2>/dev/null +unset -f zot_displayHelp 2>/dev/null +unset -f zot_sanitizeEnvVar 2>/dev/null EOF } From 6456378815ae1b0ae2a437f6b41ada840ac58290 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 15 Jul 2025 15:09:56 +0100 Subject: [PATCH 115/179] Issue with zsh passing vars to awk; remove awk call use expansion --- include/common.sh | 44 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/include/common.sh b/include/common.sh index ff55bd3db..28debab33 100755 --- a/include/common.sh +++ b/include/common.sh @@ -364,18 +364,54 @@ zot="zopen community" zot_sanitizeEnvVar() { - # remove any envvar entries that match the specified regex + # remove any envvar entries that match the specified prefix, in a shell-agnostic + # manner value="\$1" delim="\$2" - prefix="\$3" - echo "\${value}" | awk -v RS="\${delim}" -v DLIM="\${delim}" -v PRFX="\${prefix}" '{ if (match(\$1, PRFX)==0) {printf("%s%s",\$1,DLIM)}}' + prefix="\$3" + if [ -z "\${value}" ]; then + return + fi + result="" + oldIFS="\${IFS}" + IFS="\${delim}" + for entry in \${value}; do + case "\${entry}" in + "\${prefix}"/*) + # Skip entries that start with the prefix followed by / + ;; + *) + # Keep entries that don't match + if [ -z "\${result}" ]; then + # First time through + result="\${entry}" + else + # At least one previous result; insert delimiter + result="\${result}\${delim}\${entry}" + fi + ;; + esac + done + IFS="\${oldIFS}" + /bin/echo "\${result}" } zot_deleteDuplicateEntries() { value="\$1" delim="\$2" - echo "\${value}\${delim}" | awk -v RS="\${delim}" '!(\$0 in a) {a[\$0]; printf("%s%s", col, \$0); col=RS; }' | /bin/sed "s/\${delim}$//" + if [ -z "\${value}" ]; then + /bin/echo "" + return + fi + + echo "\${value}\${delim}" | awk -v RS="\${delim}" -v ORS="\${delim}" ' + BEGIN={col=""} + !(\$0 in a) { + a[\$0] = 1; + /bin/printf("%s%s", col, \$0); + col=ORS; + }' | /bin/sed "s/\${delim}$//" } # zopen community environment variables From 309dd48a426754d13ec6e2cbef1397fc2d79ead9 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 15 Jul 2025 15:15:13 +0100 Subject: [PATCH 116/179] Failure during dotenv find now stops --- include/common.sh | 50 +++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/include/common.sh b/include/common.sh index 28debab33..b862d84ff 100755 --- a/include/common.sh +++ b/include/common.sh @@ -368,10 +368,10 @@ zot_sanitizeEnvVar() # manner value="\$1" delim="\$2" - prefix="\$3" + prefix="\$3" if [ -z "\${value}" ]; then return - fi + fi result="" oldIFS="\${IFS}" IFS="\${delim}" @@ -408,9 +408,9 @@ zot_deleteDuplicateEntries() echo "\${value}\${delim}" | awk -v RS="\${delim}" -v ORS="\${delim}" ' BEGIN={col=""} !(\$0 in a) { - a[\$0] = 1; + a[\$0] = 1; /bin/printf("%s%s", col, \$0); - col=ORS; + col=ORS; }' | /bin/sed "s/\${delim}$//" } @@ -434,15 +434,25 @@ if [ -n "\$SSH_CONNECTION" ] && [ -z "\$PS1" ] || [ ! -t 1 ]; then fi if [ -z "\${ZOPEN_QUICK_LOAD}" ]; then - if [ -e "\${ZOPEN_ROOTFS}/etc/profiled" ]; then - dotenvs=\$(find "\${ZOPEN_ROOTFS}/etc/profiled" -type f -name 'dotenv' -print) + if [ -d "\${ZOPEN_ROOTFS}/etc/profiled" ]; then + if \$displayText; then /bin/printf "Processing \$zot configuration..." fi - for dotenv in \$dotenvs; do - . \$dotenv - done + tmpfile="\${local_tmp}/zopen_config.tmp" + + if /bin/find "\${ZOPEN_ROOTFS}/etc/profiled" -type f -name 'dotenv' -print > "\${tmpfile}"; then + while IFS= read -r dotenv; do + [ -r "\${dotenv}" ] && . "\${dotenv}" + done < "\${tmpfile}" + else + echo "Error: find command failed" >&2 + [ -e "\${tmpfile}" ] && rm "\${tmpfile}" + exit 1 + fi + [ -e "\${tmpfile}" ] && rm "\${tmpfile}" + if \$displayText; then /bin/echo "DONE" if [ -n "\${ZOPEN_TOOLSET_OVERRIDE}" ]; then @@ -1505,7 +1515,7 @@ syslog() } jqGetKey(){ - # If there key is not present, the error causes jq to exit with a + # If there key is not present, the error causes jq to exit with a # non-zero return code jq -er --arg key "$1" '.[$key] // error("Missing key: " + $key)' "$2" } @@ -1539,7 +1549,7 @@ getJSONCacheURLs(){ base=$(jqGetKey "metadata_baseurl" "${dereffedLink}") filename=$(jqGetKey "metadata_file" "${dereffedLink}") latest_metadata=$(jqGetKey "latest_file" "${dereffedLink}") - + case "${type}" in http|https) jsonCacheURLs=$(printf "%s://%s/%s\n%s://%s/%s" \ "${type}" "${base}" "${filename}" "${type}" "${base}" "${latest_metadata}") @@ -2073,10 +2083,12 @@ getPortMetaData(){ return 1 fi elif [ -n "${releaseLine}" ]; then + # The release line to use was explicitly mentioned on the cli if ! getReleaseLineMetadata "${portName}" "${invalidPortAssetFile}"; then return 1 fi else + # No explict release line given, calculate if ! calculateReleaseLineMetadata "${portName}" "${invalidPortAssetFile}"; then return 1 fi @@ -2400,7 +2412,7 @@ processRepoInstallFile(){ getInstallFile() { - installurl="$1" + installurl="$1" downloadToDir="${ZOPEN_ROOTFS}/var/cache/zopen" if $downloadOnly; then @@ -2429,7 +2441,7 @@ getInstallFile() metadataFile="$(basename "${installurl}").json" if [ -e "${downloadToDir}/${metadataFile}" ]; then printVerbose "Corresponding metadata '${metadataFile}' already in local cache" - else + else printVerbose "Downloading corresponding metadata" # check if it is in the same location just with a different suffix (as in a mirror) # if not, likely is the original Github repo which uses a subdirectory for the metadata @@ -2469,8 +2481,8 @@ getInstallFile() extractMetadataFromPax() { - # If there is an issue with the pax or the target, it is possible for pax - # itself to report an error and sit waiting for user input; + # If there is an issue with the pax or the target, it is possible for pax + # itself to report an error and sit waiting for user input; if ! pax -rf "$1" -s "%[^/]*/%/tmp/%" '*/metadata.json' ; then if ! details=$(pax -rf "$1" -s "%[^/]*/%/tmp/%" '*/package.json'); then printSoftError "Could not extract package metadata from file '$1'." @@ -2544,7 +2556,7 @@ installFromPax() elif [ ! -r "${pax}" ]; then printError "Pax file to install not readable at '${pax}'" fi - + # shellcheck disable=SC2154 if ! runLogProgress "pax -rf ${pax} -p p ${paxredirect} ${redirectToDevNull}" \ "Expanding file: ${pax}" "Expanded file: ${pax}"; then @@ -2720,7 +2732,7 @@ updatePackageDB() printVerbose "No current package database [new install?]. Creating empty array[]" printf "[\n]\n" > "${pdb}" fi - + if [ $(unset CD_PATH; cd "${ZOPEN_PKGINSTALL}"; ls -A | wc -l) -eq 0 ]; then printVerbose "No packages found to add to database [new install?]" return @@ -2889,7 +2901,7 @@ diskusage() formattedFileSize() { filesize=$1 # in kb - # Use awk rather than $((..)) to get floating points, using the + # Use awk rather than $((..)) to get floating points, using the # "repeated divisions and count" method to generate an offset echo "${filesize}" | awk '{ num = $1; @@ -2904,7 +2916,7 @@ formattedFileSize() num = num / 1000; unit = "M"; } printf "%.3f%s\n", num, unit; - }' + }' } . ${INCDIR}/analytics.sh From eafd68062b317aaed9f93b590fe6799d00bff630 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 15 Jul 2025 15:16:31 +0100 Subject: [PATCH 117/179] Fail early if install issue during zopen-init --- bin/zopen-init | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index a07410dbe..976940bab 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -722,18 +722,21 @@ updateBootstrappedTools() { esac printVerbose "Validating pax '${pax}'" paxLocation=$(findrev "${MYDIR}" "${pax}") - echo "${paxLocation}/${pax}" + printVerbose "Pax installation from '${paxLocation}/${pax}'" if [ -e "${paxLocation}/${pax}" ]; then zopen install "${paxLocation}/${pax}" -y --bypass-prereq-checks --skip-verify + toolInstall=$? + [ "${toolInstall}" -ne 0 ] && printError "Unable to install '${pax}' locally; see previous errors and retry installation using the '--re-init' parameter" else printError "Could not locate ${pax} to install as bootstrap" fi done else - ${MYDIR}/zopen-install ${installOptions} -y curl jq pinentry gpg meta + "${MYDIR}/zopen-install" ${installOptions} -y curl jq pinentry gpg meta toolInstall=$? fi [ "${toolInstall}" -ne 0 ] && printError "Unable to install curl, jq, gpg, pinentry and/or meta; see previous errors and retry the initilisation using the '--re-init' parameter" + # shellcheck disable=SC1090 . "${configFile}" } From bf66c3256ab23be851b2fbf5d780263c475a4240 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 13 Aug 2025 16:23:10 +0100 Subject: [PATCH 118/179] Issues with awk syntax when creating config --- include/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/common.sh b/include/common.sh index b862d84ff..3a1d423e3 100755 --- a/include/common.sh +++ b/include/common.sh @@ -406,10 +406,10 @@ zot_deleteDuplicateEntries() fi echo "\${value}\${delim}" | awk -v RS="\${delim}" -v ORS="\${delim}" ' - BEGIN={col=""} + BEGIN {col=""} !(\$0 in a) { a[\$0] = 1; - /bin/printf("%s%s", col, \$0); + printf("%s%s", col, \$0); col=ORS; }' | /bin/sed "s/\${delim}$//" } From 5b614aed9715ea378910e71afb5387f23d00e0f3 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 14 Aug 2025 13:28:42 +0100 Subject: [PATCH 119/179] Add bootstrapped meta to package tracker during offline init --- bin/zopen-init | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/bin/zopen-init b/bin/zopen-init index 976940bab..fad87ec9b 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -702,6 +702,28 @@ updateBootstrappedTools() { printSoftError "Unexpected errors merging symlinks into mesh" printError "Use zopen alt to select previous version to ensure known state" fi + printVerbose "Adding bootstrapped meta to package db" + printVerbose "Creating basic metadata.json" + cat << EOF > "${target}/metadata.json" +{ + "version_scheme": "0.2", + "product": { + "name": "meta", + "version": "0.0.0.0-bootstrap", + "release": "$(date +%Y%m%d_%H%M%S)", + "summary": "meta on z/OS", + "repo": "https://github.com/zopencommunity/metaport", + "license": "https://github.com/zopencommunity/metaport/blob/main/patches/LICENSE", + "zopen_license": "https://github.com/zopencommunity/metaport/blob/main/LICENSE", + "categories": "utilities", + "size": "$(du -ks "${target}" | awk '{print $1 * 1024}')", + "build_dependencies": [], + "runtime_dependencies": [], + "system_prereqs": [] + } +} +EOF + addToInstallTracker "meta" printVerbose "Installing bootstrap tools" PAX_LOCATIONS="$CURL_PAX_LOCATION,$JQ_PAX_LOCATION,${PINENTRY_PAX_LOCATION},${GPG_PAX_LOCATION}" From 4bacec34018ce1e4e61402926405b0101816ea91 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 14 Aug 2025 13:35:24 +0100 Subject: [PATCH 120/179] Move GPG code to common.sh; add socket sanity check --- bin/zopen-install | 31 ------------------------------- include/common.sh | 20 ++++++++++++++++++-- 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/bin/zopen-install b/bin/zopen-install index e99e41fc0..f6f61d1d8 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -21,37 +21,6 @@ setupMyself() setupMyself checkWritable -startGPGAgent() { - printInfo "- Starting gpg-agent..." - - SOCKET_PATH=$(gpgconf --list-dirs agent-socket) - if [ -r "$SOCKET_PATH" ]; then - printVerbose "gpg-agent is already running (socket found at $SOCKET_PATH)." - return 0 - fi - - if eval "$(gpg-agent --daemon --disable-scdaemon)" >/dev/null 2>&1; then - if [ -r "$SOCKET_PATH" ]; then - printVerbose "gpg-agent started successfully (socket created at $SOCKET_PATH)." - else - printWarning "gpg-agent started, but socket was not created at $SOCKET_PATH. Please verify your GPG installation." - fi - else - if [ -r "$SOCKET_PATH" ]; then - printWarning "gpg-agent started successfully (socket created at $SOCKET_PATH), but gpg-agent returned a non-zero return code." - else - printError "Failed to start gpg-agent. Reinstall or upgrade GPG using \"zopen install --reinstall gpg -y\" or \"zopen upgrade gpg -y\"." - fi - fi -} - -gpgCleanup() { - printVerbose "Cleaning up $SIGNATURE_FILE, $PUBLIC_KEY_FILE and $TMP_GPG_DIR" - [ -e "$SIGNATURE_FILE" ] && rm -f "$SIGNATURE_FILE" - [ -e "$PUBLIC_KEY_FILE" ] && rm -f "$PUBLIC_KEY_FILE" - [ -d "$TMP_GPG_DIR" ] && rm -rf "$TMP_GPG_DIR" -} - printHelp() { cat << HELPDOC diff --git a/include/common.sh b/include/common.sh index 3a1d423e3..cebd90637 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1822,13 +1822,29 @@ a2e() fi } +testGPGAgentSocket() { + # Test whether the gpg socket that has been found is actually valid - + # orphaned sockets without a daemon can lead to gpg errors when trying + # to restart + if ! gpg-connect-agent --socket="$1" /bye >/dev/null 2>&1; then + return 1 + fi + return 0 +} + startGPGAgent() { printInfo "- Starting gpg-agent..." SOCKET_PATH=$(gpgconf --list-dirs agent-socket) + printVerbose "gpg socket path: ${SOCKET_PATH}" if [ -r "$SOCKET_PATH" ]; then - printVerbose "gpg-agent is already running (socket found at $SOCKET_PATH)." - return 0 + if testGPGAgentSocket "${SOCKET_PATH}"; then + printVerbose "gpg-agent is already running (socket found at ${SOCKET_PATH})." + return 0 + else + printWarning "Found orphaned GPG agent socket at ${SOCKET_PATH}; removing and attempting restart of gpg-agent." + rm -f "${SOCKET_PATH}" + fi fi if eval "$(gpg-agent --daemon --disable-scdaemon)" >/dev/null 2>&1; then From 76bcf2d70b48933258ac33a21a007605eb9096e4 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 14 Aug 2025 13:37:51 +0100 Subject: [PATCH 121/179] Fix more Shellcheck issues and remove spurious blank spaces --- bin/zopen-init | 64 ++++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index fad87ec9b..2bef7aa17 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -53,13 +53,13 @@ Options: -h, -?, --help display this help and exit - + --offline Perform the base installation of the zopen environment without attempting to update any of the bootstrapping packages. For use in air-gapped environments or where a custom package repository will be configed - Note: Post-configuration of alternate package sources might be + Note: Post-configuration of alternate package sources might be required. --re-init @@ -94,13 +94,13 @@ Examples: interactively bootstrap a zopen environment zopen init --releaseline-dev - interactively bootstrap a zopen environment that will use + interactively bootstrap a zopen environment that will use Development Releaseline packages zopen init --yes --append-to-profile /zopen - non-interactively create a zopen environment at location - '/zopen' on disk, with packages installed to - '/usr/local/zopen'. The user's .profile will be updated to - source the configuration file at '/zopen/etc/zopen-config' + non-interactively create a zopen environment at location + '/zopen' on disk, with packages installed to + '/usr/local/zopen'. The user's .profile will be updated to + source the configuration file at '/zopen/etc/zopen-config' when new terminal sessions start Report bugs at https://github.com/zopencommunity/meta/issues @@ -143,7 +143,7 @@ while [ $# -gt 0 ]; do reinitExisting=true ;; "--bypass-prereq-checks") - bypassPrereqs=true + bypassPrereqs=true ;; "--rebuild-repo-config") rebuildRepoConfig=true @@ -183,7 +183,7 @@ while [ $# -gt 0 ]; do verbose=true ;; "--version") - zopen-version $ME + zopen-version "$ME" exit 0 ;; "--debug") @@ -391,7 +391,7 @@ generateFileSystem() printVerbose "- Creating symbolic path for prod redirect files" [ -e "${rootfs}/usr/share/zopen/boot" ] || ln -s "${rootfs}/boot" "${rootfs}/usr/share/zopen/boot" [ -e "${rootfs}/etc/zopen" ] || mkdir -p "${rootfs}/etc/zopen" - + printVerbose "- Creating product installation location" zopen_pkginstall="usr/local/zopen" [ -e "${rootfs}/${zopen_pkginstall}" ] || mkdir -p "${rootfs}/${zopen_pkginstall}" @@ -416,7 +416,7 @@ generateFileSystem() generateZopenConfig() { printHeader "Generating zopen-config" - writeConfigFile "${configFile}" "${rootfs}" "${zopen_pkginstall}" "${ZOPEN_CA_DIR}/${certFileName}" $OVERRIDE_ZOS_TOOLS + writeConfigFile "${configFile}" "${rootfs}" "${zopen_pkginstall}" "${ZOPEN_CA_DIR}/${certFileName}" "$OVERRIDE_ZOS_TOOLS" printInfo "- Created config in ${configFile}" if ${appendToProfile}; then if ! grep -q ". ${rootfs}/etc/zopen-config" "${HOME}/.profile"; then @@ -456,12 +456,12 @@ generateJsonConfiguration() return 4 fi rm_fileprocs=$(jqw -r '.num_rm_procs' "${jsonConfig}") - if [ -z "${OVERRIDE_ZOS_TOOLS}" ] && jqw -er ".override_zos_tools" $jsonConfig >/dev/null; then - OVERRIDE_ZOS_TOOLS=$(jqw -r '.override_zos_tools' $jsonConfig) + if [ -z "${OVERRIDE_ZOS_TOOLS}" ] && jqw -er ".override_zos_tools" "$jsonConfig" >/dev/null; then + OVERRIDE_ZOS_TOOLS=$(jqw -r '.override_zos_tools' "$jsonConfig") fi autocacheclean=$(jqw -r '.autocacheclean' "${jsonConfig}") autopkgpurge=$(jqw -r '.autopkgpurge' "${jsonConfig}") - skip_broken=$(jqw -r '.skip_broken' "${jsonConfig}") + skip_broken=$(jqw -r '.skip_broken' "${jsonConfig}") fi if ${releaselineDev}; then @@ -484,7 +484,7 @@ generateJsonConfiguration() ;; *) showConfigParmWarning "autocacheclean" "${autocacheclean}" "0|1" "1" autocacheclean=1 - + ;; esac case "${autopkgpurge:=0}" in @@ -527,8 +527,8 @@ zz # In case 3rd-party edits have been made (adding custom keys for # example), just update the values that zopen itself knows about inline if ! jqw --arg releaseLine "${releaseLine}" \ - --arg isoverrideZOSTools ${OVERRIDE_ZOS_TOOLS} \ - --arg isCollectingStats ${isCollectingStats} \ + --arg isoverrideZOSTools "${OVERRIDE_ZOS_TOOLS}" \ + --arg isCollectingStats "${isCollectingStats}" \ ".release_line = \$releaseLine | .override_zos_tools = \$isoverrideZOSTools | .is_collecting_stats = \$isCollectingStats" \ "${jsonConfig}" > "${jsonConfigWorking}"; then printError "Errors updating existing configuration file. See previous errors for more information and retry command." @@ -542,7 +542,7 @@ generateDefaultRepository() { # Check for existing repository definition - if present, use unless # user wants to rebuild the default repository file - [ -e "${rootfs}/etc/zopen/repos.d/active" ] && + [ -e "${rootfs}/etc/zopen/repos.d/active" ] && ! ${rebuildRepoConfig} && return 0 # If there is no repository definition, generate a default to point to Github @@ -554,12 +554,12 @@ generateDefaultRepository() repotype="${paxrepourl%://*}" [ "${repotype}" = "${paxrepourl}" ] && printError "Unable to parse repository type from '${paxrepourl}'. Ensure valid format: ://://:// "${repoFile}" -{ - "type": "${repotype}", - "metadata_baseurl": "${baserepourl}", +{ + "type": "${repotype}", + "metadata_baseurl": "${baserepourl}", "metadata_file": "${repometafile}", - "latest_file": "${latestmetafile}" + "latest_file": "${latestmetafile}" } EOS # Create active link if not already present; if it is present, it'll either point to the default already or @@ -618,7 +618,7 @@ zz printSoftError "${jqrc}" printError "Unable to update analystics configuration. See previous errors for more details." fi - + registerFileSystem "${UUID}" "${isHostIBM}" "${isBot}" if [ -n "$ZOPEN_ROOTFS" ]; then @@ -654,10 +654,9 @@ installPrereqs() cachedir="${rootfs}/var/cache/zopen" basepax=$(basename ${pax}) [ -e "${cachedir}/${basepax}" ] || printError "Could not locate bootstrap ${pax}." - paxrc=$(pax -rf "${cachedir}/${basepax}" -s##${rootfs}/boot/#) - if [ $? -gt 0 ]; then - printError "Failed to extract pax file ${basepax}" + if ! paxrc=$(pax -rf "${cachedir}/${basepax}" "-s##${rootfs}/boot/#"); then + printError "Failed to extract pax file ${basepax}: ${paxrc}" fi done @@ -670,6 +669,7 @@ installPrereqs() chmod -R 755 "${rootfs}/boot/${tooldir}" curwd="${PWD}" cd "${rootfs}/boot/${tooldir}/" || printError "Could not access ${tool} bootstrap in directory '${rootfs}/boot/${tooldir}/'. Re-run 'zopen init' to retry" + # shellcheck disable=SC1091 . ./.env cd "${curwd}" || printError "Could not change to ${curwd}. Re-run 'zopen init' to retry" done @@ -684,7 +684,7 @@ updateBootstrappedTools() { installOptions="${installOptions} --bypass-prereq-checks"; fi if ${refresh}; then - ${MYDIR}/zopen-install ${installOptions} -y curl jq pinentry gpg + "${MYDIR}"/zopen-install "${installOptions}" -y curl jq pinentry gpg toolInstall=$? elif ${offlineInstall}; then printVerbose "As offline, no access to meta.pax so copy currently running" @@ -695,6 +695,7 @@ updateBootstrappedTools() { printVerbose "Copying bare meta scripts from '${metabindir}' to '${target}" cp -R "${metabindir}/../bin" "${target}" cp -R "${metabindir}/../include" "${target}" + printVerbose "Creating main symlink for meta" ln -s "${ulzm}/meta-bootstrap" "${ulzm}/meta" if ! runLogProgress "mergeIntoSystem \"meta\" \"${ulzm}/meta-bootstrap\" \"${rootfs}\"" \ @@ -736,7 +737,7 @@ EOF pax="${workingPaxList%%,*}" workingPaxList="${workingPaxList#*,}" ;; - *) + *) # Last item [hence no comma separator] pax="${workingPaxList}" workingPaxList="" @@ -754,6 +755,7 @@ EOF fi done else + # shellcheck disable=SC2086 # We want installOptions to be split "${MYDIR}/zopen-install" ${installOptions} -y curl jq pinentry gpg meta toolInstall=$? fi From 668b54e585381e302b033605e38351c2b2b128d3 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 22 Sep 2025 14:57:52 +0100 Subject: [PATCH 122/179] Add ability to skip size checks [on a dynamic fs for example] Use internal is true/false --- bin/zopen-config-helper | 13 +++-- bin/zopen-init | 58 ++++++++++++------- bin/zopen-install | 5 ++ include/common.sh | 58 +++++++++++-------- include/scriptlets/installPost/autopkgpurge | 30 +++++----- .../transactionPost/zopen-clean-autocache | 32 +++++----- 6 files changed, 116 insertions(+), 80 deletions(-) diff --git a/bin/zopen-config-helper b/bin/zopen-config-helper index 67309a4da..ce61e20d1 100755 --- a/bin/zopen-config-helper +++ b/bin/zopen-config-helper @@ -52,11 +52,11 @@ Variables Where the value must be one of an enumerated set, the default value is listed first. Validation of those values will be performed at runtime. - autocacheclean [integer:0|1] + autocacheclean [boolean:true|false] Tells zopen install to remove downloaded package files from the cache to preserve space. - autopkgpurge [integer:0|1] + autopkgpurge [boolean:true|false] Tells zopen install to remove any older version[s] of a package after a successful install or upgrade @@ -64,9 +64,6 @@ Variables Configures the zopen environment to collect anonymous statistcs about package installations. - num_rm_procs (deprecated) [integer:5] - Tells zopen remove how many subshells to use during removal processing - override_zos_tools [boolean:false|true] Tells zopen-config whether to prioritise native z/OS tools (such as in /bin) on the \$PATH or to use zopen community ports in preference. @@ -78,10 +75,14 @@ Variables or from the DEV line. DEV packages will be newer but might exhibit unexpected behaviour. The default is to use STABLE packages. - skip_broken [integer:0|1] + skip_broken [boolean:true|false] Tells zopen that if during a transation an action fails, report the error and continue to the next action when possible. + skip_size_check [boolean:false|true] + Tells zopen to skip the validation of file space to allow for dynamic + file-systems such as ZFS mounts with AGGRGROW. + Notes: Configuration options are not validated in that any key/value pairs can be added into the global configuration. 3rd-party utilities can store their diff --git a/bin/zopen-init b/bin/zopen-init index 9dca5c7ec..6d48f829a 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -455,13 +455,13 @@ generateJsonConfiguration() if ! releaseLine=$(jqw -r '.release_line' "${jsonConfig}"); then return 4 fi - rm_fileprocs=$(jqw -r '.num_rm_procs' "${jsonConfig}") if [ -z "${OVERRIDE_ZOS_TOOLS}" ] && jqw -er ".override_zos_tools" "$jsonConfig" >/dev/null; then OVERRIDE_ZOS_TOOLS=$(jqw -r '.override_zos_tools' "$jsonConfig") fi autocacheclean=$(jqw -r '.autocacheclean' "${jsonConfig}") autopkgpurge=$(jqw -r '.autopkgpurge' "${jsonConfig}") skip_broken=$(jqw -r '.skip_broken' "${jsonConfig}") + skip_size_check=$(jqw -r '.skip_size_check' "${jsonConfig}") fi if ${releaselineDev}; then @@ -478,31 +478,39 @@ generateJsonConfiguration() releaseLine="STABLE" ;; esac - case "${autocacheclean:=1}" in - "0"|"1") :;; # Valid value - "null") autocacheclean=1 # Default to true; jq found no entry (new config) + case "${autocacheclean:=true}" in + "0"|"1"|[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee]) :;; # Valid value + "null") autocacheclean="true" # Default to true; jq found no entry (new config) ;; - *) showConfigParmWarning "autocacheclean" "${autocacheclean}" "0|1" "1" - autocacheclean=1 + *) showConfigParmWarning "autocacheclean" "${autocacheclean}" "true|false" "true" + autocacheclean=true ;; esac - case "${autopkgpurge:=0}" in - "0"|"1") :;; # Valid value - "null")autopkgpurge=0 # Default to false; jq found no entry (new config) + case "${autopkgpurge:=false}" in + "0"|"1"|[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee]) :;; # Valid value + "null")autopkgpurge=false # Default to false; jq found no entry (new config) ;; - *) showConfigParmWarning "autopkgpurge" "${autopkgpurge}" "0|1" "0" - autopkgpurge=0 + *) showConfigParmWarning "autopkgpurge" "${autopkgpurge}" "true|false" "false" + autopkgpurge=false ;; esac - case "${skip_broken:=0}" in - "0"|"1") :;; # Valid value - "null") skip_broken=0 # Default to false + case "${skip_broken:=false}" in + "0"|"1"|[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee]) :;; # Valid value + "null") skip_broken=false # Default to false ;; - *) showConfigParmWarning "skip_broken" "${skip_broken}" "0|1" "0" - skip_broken=0 + *) showConfigParmWarning "skip_broken" "${skip_broken}" "true|false" "false" + skip_broken=false ;; esac + case "${skip_size_check:=false}" in + "0"|"1"|[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee]) :;; # Valid value + "null") skip_size_check=false # Default to false + ;; + *) showConfigParmWarning "skip_size_check" "${skip_size_check}" "true|false" "false" + skip_size_check=false + ;; + esac case "${OVERRIDE_ZOS_TOOLS}" in "true"|"false") :;; # Valid value "null") OVERRIDE_ZOS_TOOLS=false ;; # No entry in config file @@ -514,12 +522,12 @@ generateJsonConfiguration() cat < "${jsonConfig}" { "release_line": "${releaseLine:-STABLE}", - "num_rm_procs": ${rm_fileprocs:-5}, "is_collecting_stats": ${isCollectingStats}, "override_zos_tools": ${OVERRIDE_ZOS_TOOLS}, "autocacheclean": ${autocacheclean}, "autopkgpurge": ${autopkgpurge}, - "skip_broken": ${skip_broken} + "skip_broken": ${skip_broken}, + "skip_size_check": ${skip_size_check} } zz else @@ -529,8 +537,18 @@ zz if ! jqw --arg releaseLine "${releaseLine}" \ --arg isoverrideZOSTools "${OVERRIDE_ZOS_TOOLS}" \ --arg isCollectingStats "${isCollectingStats}" \ - ".release_line = \$releaseLine | .override_zos_tools = \$isoverrideZOSTools | .is_collecting_stats = \$isCollectingStats" \ - "${jsonConfig}" > "${jsonConfigWorking}"; then + --arg autocacheclean "${autocacheclean}" \ + --arg autopkgpurge "${autopkgpurge}" \ + --arg skip_broken "${skip_broken}" \ + --arg skip_size_check "${skip_size_check}" \ + ".release_line = \$releaseLine | + .override_zos_tools = \$isoverrideZOSTools | + .is_collecting_stats = \$isCollectingStats + .autocacheclean = \$autocacheclean | + .autopkgpurge = \$autopkgpurge | + .skip_broken = \$skip_broken | + .skip_size_check = \$skip_size_check + " "${jsonConfig}" > "${jsonConfigWorking}"; then printError "Errors updating existing configuration file. See previous errors for more information and retry command." fi mv -f "${jsonConfigWorking}" "${jsonConfig}" diff --git a/bin/zopen-install b/bin/zopen-install index f6f61d1d8..e3f00af77 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -92,6 +92,7 @@ installOrUpgrade=false # shellcheck disable=SC2034 nosymlink=false skip_broken=false +skip_size_check=false # shellcheck disable=SC2034 skipupgrade=false skipverify=false @@ -154,6 +155,10 @@ while [ $# -gt 0 ]; do # shellcheck disable=SC2034 skipverify=true # Verify signature of packages ;; + "--skip_size_check" | "-st") + # shellcheck disable=SC2034 + skip_size_check=true + ;; "--force") # shellcheck disable=SC2034 force=true # Bypasses locks diff --git a/include/common.sh b/include/common.sh index f78523678..911eea2b8 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1756,26 +1756,6 @@ isURLReachable() { fi } -checkAvailableSize() -{ - - package="$1" - packageSize="$2" - printInfo "- Checking available size to install ${package}." - - printDebug "Package Size: ${packageSize} bytes" - packageSize=$(echo "scale=2; ${packageSize} / 1024" | bc) - printDebug "Package Size: ${packageSize} k" - partitionSize=$(/bin/df -k . | tail -1 | awk '{print $3}' | cut -f1 -d '/') - printDebug "Partition Size: ${partitionSize}k [free on '$(pwd -P)']" - - if [ 1 -eq "$(echo "${packageSize} > ${partitionSize}" | bc)" ]; then - printError "Not enough space in partition." - fi - printInfo "- Enough space to install ${package}. Proceeding with installation." - return 0 -} - promptYesOrNo() { message="$1" skip=$2 @@ -2176,7 +2156,7 @@ addToInstallGraph(){ installtype=$1 && shift invalidPortAssetFile=$1 && shift pkgList="$1" - printDebug "Adding pkgList to install graph" + printDebug "Adding ${pkgList} to install graph" for portRequested in ${pkgList}; do if ! getPortMetaData "${portRequested}" "${invalidPortAssetFile}"; then continue @@ -2320,19 +2300,25 @@ spaceValidate(){ cacheBytes=$1 packageBytes=$2 + skip_size_check=$(jq -re '.skip_size_check' "${ZOPEN_JSON_CONFIG}") + if "${skip_size_check}"; then + printVerbose "Skipping size check due to skip_size_check=${skip_size_check}" + return + fi + if ${reinstall}; then # During a reinstall, the existing package size should remain constant as the # package should overwrite the existing with the same files. However # there might be a need to download the package into the cache - if # autocacheclean is active, then this should be temporary; if not, then the - # cache will grow - infrom the user either way + # cache will grow - inform the user either way printVerbose "Reinstall of package, so package file size delta should be 0!" spaceRequiredMB=$(echo "scale=0; (${cacheBytes}) / (1024 * 1024)" | bc) spaceRequiredKb=$(( cacheBytes / 1024 )) if ! isCacheClean=$(zopen config --get autocacheclean); then printError "Could not determine autocacheclean status" fi - if [ "${isCacheClean}" -eq 1 ]; then + if ${isCacheClean}; then printInfo "During this operation, ${spaceRequiredMB} MB of disk space will be used." else printInfo "After this operation, ${spaceRequiredMB} MB of additional cache will be used." @@ -2555,7 +2541,7 @@ installFromPax() if ! processActionScripts "installPre" "${name}" "${metadatafile}" "${pax}"; then skip_broken=$(jq -re '.skip_broken' "${ZOPEN_JSON_CONFIG}") - if [ "${skip_broken}" -ne 1 ]; then + if "${skip_broken}"; then printError "Failed installation pre-requisite check(s) for '${name}'. Correct previous errors and retry command" fi printSoftError "Skipping package '${name}' due to failed pre-requisite check(s)" @@ -2944,6 +2930,30 @@ formattedFileSize() } printf "%.3f%s\n", num, unit; }' } + +# Used to perform validation of config values. Legacy +# values might use 0:1 or true:false so handle both, +# ignoring case +isFalse() +{ + case "$1" in + 0|[Ff][Aa][Ll][Ss][Ee] ) return 0;; + *) return 1;; + esac +} +isTrue() +{ + case "$1" in + 1|[Tt][Rr][Uu][Ee] ) return 0;; + *) return 1;; + esac +} + +# Main code + + +# shellcheck disable=SC1091 +# shellcheck disable=SC2086 . ${INCDIR}/analytics.sh zopenInitialize diff --git a/include/scriptlets/installPost/autopkgpurge b/include/scriptlets/installPost/autopkgpurge index 039de76bc..832ff4828 100755 --- a/include/scriptlets/installPost/autopkgpurge +++ b/include/scriptlets/installPost/autopkgpurge @@ -6,18 +6,20 @@ pkg=$1 if type zopen-config-helper >/dev/null 2>&1; then isactive=$(zopen-config-helper --get autopkgpurge) else - isactive=0 + isactive=false fi -case "${isactive}" in - 0 ) :;; # Take no action - 1 ) (zopen clean --unused "${pkg}" >/dev/null 2>&1) - ;; - *) printWarning "Invalid value '${isactive}' for autopkgpurge config parameter [should be 0 or 1]." - if type zopen-config-helper >/dev/null 2>&1; then - printWarning "Run zopen config --set autopkgpurge [0|1] to update configuration" - else - printWarning "Update zopen configuration file to contain a valid value for parameter 'autopkgpurge'" - fi - printWarning "Unused package clean will not be performed" - ;; -esac + +if isTrue "${isactive}"; then + (zopen clean --unused "${pkg}" >/dev/null 2>&1) +elif isFalse "${isactive}"; then + printVebose "autopkgpurge is disabled; no clean will occur" +else + printWarning "Invalid value '${isactive}' for autopkgpurge config parameter [should be true or false]." + if type zopen-config-helper >/dev/null 2>&1; then + printWarning "Run zopen config --set autopkgpurge [true|false] to update configuration" + else + printWarning "Update zopen configuration file to contain a valid value for parameter 'autopkgpurge'" + fi + printWarning "Unused package clean will not be performed" +fi + diff --git a/include/scriptlets/transactionPost/zopen-clean-autocache b/include/scriptlets/transactionPost/zopen-clean-autocache index 4c5c0be1f..bdf03abf3 100755 --- a/include/scriptlets/transactionPost/zopen-clean-autocache +++ b/include/scriptlets/transactionPost/zopen-clean-autocache @@ -4,20 +4,20 @@ printVerbose "Running '${this}' script" if type zopen-config-helper >/dev/null 2>&1; then isactive=$(zopen-config-helper --get autocacheclean) else - isactive=1 # Default to true + isactive=true # Default to true +fi + +if isTrue "${isactive}"; then + (zopen clean --cache >/dev/null 2>&1) + echo "- Cache cleaned." +elif isFalse "${isactive}"; then + printVebose "autoclean cache is disable; no clean will occur" +else + printWarning "Invalid value '${isactive}' for autocacheclean config parameter [should be 'true' or 'false']." + if type zopen-config-helper >/dev/null 2>&1; then + printWarning "Run zopen config --set autocacheclean [true|false] to update configuration" + else + printWarning "Update zopen configuration file to contain a valid value for parameter 'autocacheclean'" + fi + printWarning "Autocache clean will not be performed" fi -case "${isactive}" in - 0 ) :;; # Take no action - 1 ) echo "- Cleaning install cache..." - (zopen clean --cache >/dev/null 2>&1) - echo "- Cache cleaned." - ;; - *) printWarning "Invalid value '${isactive}' for autocacheclean config parameter [should be 0 or 1]." - if type zopen-config-helper >/dev/null 2>&1; then - printWarning "Run zopen config --set autocacheclean [0|1] to update configuration" - else - printWarning "Update zopen configuration file to contain a valid value for parameter 'autocacheclean'" - fi - printWarning "Autocache clean will not be performed" - ;; -esac From fcfeccf43dec491f6eac7c7158f07ce682cf8b52 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 22 Sep 2025 14:58:45 +0100 Subject: [PATCH 123/179] Ensure permission to write audit logs or issue as warning --- include/common.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/include/common.sh b/include/common.sh index 911eea2b8..70f876f9b 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1519,7 +1519,14 @@ syslog() mkdir -p "$(dirname "${fd}")" touch "${fd}" fi - echo "$(date +"%F %T") $(id | cut -d' ' -f1)::${module}:${type}:${categories}:${location}:${msg}" >> "${fd}" + output="$(date +"%F %T") $(id | cut -d' ' -f1)::${module}:${type}:${categories}:${location}:${msg}" + if [ -w "${fd}" ]; then + echo "${output}" >> "${fd}" + else + printWarning "No write permission to log file '${fd}'; writing to stderr" + echo "${output}" >&2 + fi + } jqGetKey(){ From faeabb5550cd744c64646e0f810d567e6e880251 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 22 Sep 2025 15:00:51 +0100 Subject: [PATCH 124/179] Fix package capitalisation issues with upgrading --- bin/zopen-install | 2 +- bin/zopen-list | 6 ++++-- bin/zopen-upgrade | 2 +- include/common.sh | 53 +++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 53 insertions(+), 10 deletions(-) diff --git a/bin/zopen-install b/bin/zopen-install index e3f00af77..7cc29ac84 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -298,7 +298,7 @@ else printInfo "- Checking installation status for '${installListCount}' packages" else installList=$(echo "$chosenRepos" | sed "s/@@/ /g") - validateInstallList "${installList}" + validatePackageList "${installList}" fi if ! generateInstallGraph "${installList}"; then printError "Unable to generate install graph" diff --git a/bin/zopen-list b/bin/zopen-list index a3e439dba..460824dcb 100755 --- a/bin/zopen-list +++ b/bin/zopen-list @@ -141,6 +141,7 @@ listAvailablePackages() "$(jqfunctions)"' .release_data| to_entries |sort_by(.key) | .[] | .key as $key | + (.value[0].name | split(" ")[0]) as $actualname | .value[0].assets[0] as $va | $va.url as $url | ($url | match(".*-([^-]*).[0-9]{8}_[0-9]{6}.zos.pax.Z").captures[0].string) as $version | @@ -152,7 +153,7 @@ listAvailablePackages() ($va.total_tests | if (.=="") then "99999" else . end |tonumber) as $tt | ($pt/$tt*100) as $pts | ($pts | if (.<0) then "No tests" else ($pts|r(1) |tostring + "%" ) end) as $q| - pr($key;"@";($wpackage | tonumber)) + + pr($actualname;"@";($wpackage | tonumber)) + c($ins;"@";($winstalled | tonumber)) + c($tn;"@";($wtag | tonumber)) + c($ds;"@";($wdownload | tonumber)) + @@ -169,12 +170,13 @@ listAvailablePackages() "$(jqfunctions)"' .release_data| to_entries |sort_by(.key) | .[] | .key as $key | + (.value[0].name | split(" ")[0]) as $actualname | .value[0].assets[0].url as $url | ($url | match(".*-([^-]*).[0-9]{8}_[0-9]{6}.zos.pax.Z").captures[0].string) as $version | .value[0].tag_name as $tn | (if ($installed | index($key)) then "installed" else "available" end) as $ins | .value[0].assets[0] as $va | - pr($key;"@";($wpackage | tonumber)) + + pr($actualname;"@";($wpackage | tonumber)) + c($ins;"@";($winstalled | tonumber)) + c($version;"@";($wversion | tonumber)) + c($tn;"@";($wtag | tonumber)) ' \ diff --git a/bin/zopen-upgrade b/bin/zopen-upgrade index 2d928e4b3..fc1f4a356 100755 --- a/bin/zopen-upgrade +++ b/bin/zopen-upgrade @@ -146,7 +146,7 @@ if [ -z "${chosenRepos}" ]; then printError "Details: ${chosenRepos}" fi else - validateInstallList "${chosenRepos}" + validatePackageList "${chosenRepos}" fi if ! generateInstallGraph "${chosenRepos}"; then diff --git a/include/common.sh b/include/common.sh index 70f876f9b..23fe663c7 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2182,7 +2182,8 @@ validateInstallList(){ printVerbose "Stripping any version/tagging" installees=$(set -f; echo ${installees} |awk -v ORS=, -v RS=' ' '{$1=$1; sub(/[=%].*/,x); print "\""$1"\""}') invalidPortList=$(jq -r --argjson needles "[${installees%%,}]" \ - '.release_data| keys as $haystack | $needles | map(select(. as $needle | $haystack | index($needle)|not)) | .[]' "${JSON_CACHE}") + '.release_data| keys as $haystack | $needles | map(select(. as $needle | $haystack | index($needle)|not)) | .[]' \ + "${JSON_CACHE}") if [ -n "${invalidPortList}" ]; then printSoftError "The following ports could not be installed:" printSoftError " $(echo "${invalidPortList}" | awk -v OFS=' ' -v ORS=' ' '{$1=$1};1' )" @@ -2190,6 +2191,39 @@ validateInstallList(){ fi } +# This uses the release json file to find whether any of the inputs are invalid +# package names. Ideally, this would use the keys of the release_data however, the +# NATS package breaks this as it is known as 'nats' in the repo but the port name is +# NATS. As such, use .product.name then fallback to the .url as a sanity check +# Note: this is not to make the match "case-insensitive", but to use whichever the +# user uses as the name - if they know it as NATS, accept that! +validatePackageList(){ + installees="$1" + # shellcheck disable=SC2086 # Using set -f disables globbing + printVerbose "Stripping any version/tagging" + installees=$(set -f; echo ${installees} |awk -v ORS=, -v RS=' ' '{$1=$1; sub(/[=%].*/,x); print "\""$1"\""}') + invalidPortList=$(jq -r --argjson needles "[${installees%%,}]" \ +'$needles + | map( + select( + (. as $needle) | + (any( + .release_data | to_entries[] | .value[]; + .tag_name | test("_[^_]*" + $needle + "[^_]*port") + ) | not) + ) + )' \ + "${JSON_CACHE}") + if [ -n "${invalidPortList}" ]; then + printSoftError "The following ports could not be installed:" + printSoftError " $(echo "${invalidPortList}" | awk -v OFS=' ' -v ORS=' ' '{$1=$1};1' )" + printError "Check port name(s), remove any extra 'port' suffixes and retry command." + fi + +} + + + dedupStringList() { delim="$1" && shift str="$1" @@ -2532,8 +2566,8 @@ installFromPax() # Note that at present some metadata might refer to the legacy repo ZOSOpenTools # so fall back to that if [ -n "${USEPRODNAME}" ]; then - printVerbose "Extracting product name from .product.repo" - # name=$(jq --raw-output '.product.name' "${metadatafile}") + printVerbose "Extracting product name from .product.name" + name=$(jq --raw-output '.product.name' "${metadatafile}") else printVerbose "Extracting product name from .product.repo" name=$(jq --arg reponame "${ZOPEN_ORGNAME}" --raw-output '.product.repo | match(".*/\($reponame)/(.*)port").captures[0].string' "${metadatafile}") @@ -2775,11 +2809,18 @@ updatePackageDB() echo "[]" > "${pdb}" fi # Ideally, use $reponame in the match but jq seems to have issues with that! - mdj=$(jq --arg reponame "${ZOPEN_ORGNAME}" '. as $metadata | .product.repo | match(".*/zopencommunity/(.*)port").captures[0].string | [{(.):$metadata}]' \ + + mdj=$(jq '[{(.product.name):.}]' \ "${escapedJSONFile}") + if [ -z "${mdj}" ]; then + # If we couldn't get the repo name from .product.name, use (.*)port - this might result + # in odd cases [NATS...] + mdj=$(jq '. as $metadata | .product.repo | match(".*/zopencommunity/(.*)port").captures[0].string | [{(.):$metadata}]' \ + "${escapedJSONFile}") + fi if [ -z "${mdj}" ]; then - # Try legacy repository - mdj=$(jq --arg reponame "${ZOPEN_ORGNAME}" '. as $metadata | .product.repo | match(".*/ZOSOpenTools/(.*)port").captures[0].string | [{(.):$metadata}]' \ + # Try legacy repository [for really old packages!] + mdj=$(jq '. as $metadata | .product.repo | match(".*/ZOSOpenTools/(.*)port").captures[0].string | [{(.):$metadata}]' \ "${escapedJSONFile}") fi From 7b53890f0bd20b989cdebfdede6eabb0785f1d74 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 30 Sep 2025 13:25:54 +0100 Subject: [PATCH 125/179] Add missing pipe/continuation character --- bin/zopen-init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/zopen-init b/bin/zopen-init index 6d48f829a..1e3557f6d 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -543,7 +543,7 @@ zz --arg skip_size_check "${skip_size_check}" \ ".release_line = \$releaseLine | .override_zos_tools = \$isoverrideZOSTools | - .is_collecting_stats = \$isCollectingStats + .is_collecting_stats = \$isCollectingStats | .autocacheclean = \$autocacheclean | .autopkgpurge = \$autopkgpurge | .skip_broken = \$skip_broken | From 5de49d602e3652e5acfeba9887acbdf943f4017a Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 30 Sep 2025 15:39:32 +0100 Subject: [PATCH 126/179] Typo on printVerbose --- include/scriptlets/installPost/autopkgpurge | 2 +- include/scriptlets/transactionPost/zopen-clean-autocache | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/scriptlets/installPost/autopkgpurge b/include/scriptlets/installPost/autopkgpurge index 832ff4828..ffba304be 100755 --- a/include/scriptlets/installPost/autopkgpurge +++ b/include/scriptlets/installPost/autopkgpurge @@ -12,7 +12,7 @@ fi if isTrue "${isactive}"; then (zopen clean --unused "${pkg}" >/dev/null 2>&1) elif isFalse "${isactive}"; then - printVebose "autopkgpurge is disabled; no clean will occur" + printVerbose "autopkgpurge is disabled; no clean will occur" else printWarning "Invalid value '${isactive}' for autopkgpurge config parameter [should be true or false]." if type zopen-config-helper >/dev/null 2>&1; then diff --git a/include/scriptlets/transactionPost/zopen-clean-autocache b/include/scriptlets/transactionPost/zopen-clean-autocache index bdf03abf3..a0d789c1b 100755 --- a/include/scriptlets/transactionPost/zopen-clean-autocache +++ b/include/scriptlets/transactionPost/zopen-clean-autocache @@ -11,7 +11,7 @@ if isTrue "${isactive}"; then (zopen clean --cache >/dev/null 2>&1) echo "- Cache cleaned." elif isFalse "${isactive}"; then - printVebose "autoclean cache is disable; no clean will occur" + printVerbose "autoclean cache is disable; no clean will occur" else printWarning "Invalid value '${isactive}' for autocacheclean config parameter [should be 'true' or 'false']." if type zopen-config-helper >/dev/null 2>&1; then From b83d2eeaedaace67c566a977cd308206a8cb609d Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 30 Sep 2025 16:32:48 +0100 Subject: [PATCH 127/179] Hardcode /bin into path; replace any zos* wrappers & /bin/ prefixes --- bin/zopen-alt | 4 +- bin/zopen-audit | 2 +- bin/zopen-clean | 24 +++---- bin/zopen-diagnostics | 18 +++--- bin/zopen-generate | 2 +- bin/zopen-info | 12 ++-- bin/zopen-init | 4 +- bin/zopen-promote | 2 +- bin/zopen-query | 34 +++++----- bin/zopen-usage | 22 +++---- bin/zopen-whichproject | 2 +- include/common.sh | 138 +++++++++++++++-------------------------- 12 files changed, 114 insertions(+), 150 deletions(-) diff --git a/bin/zopen-alt b/bin/zopen-alt index 9395557ef..3356ac1f9 100755 --- a/bin/zopen-alt +++ b/bin/zopen-alt @@ -110,7 +110,7 @@ setAlt() newver="$2" if [ -e "${ZOPEN_PKGINSTALL}/${package}/" ]; then printVerbose "${package} is either installed or has been previously" - found=$(zosfind "${ZOPEN_PKGINSTALL}/${package}/" -type l -prune -o -type d -print | sed -e "s#${ZOPEN_PKGINSTALL}/${package}/\([^/]*\).*#\1#" | uniq) + found=$(find "${ZOPEN_PKGINSTALL}/${package}/" -type l -prune -o -type d -print | sed -e "s#${ZOPEN_PKGINSTALL}/${package}/\([^/]*\).*#\1#" | uniq) else printVerbose "${package} has never been installed on the system" fi @@ -162,7 +162,7 @@ listAlts() package=$(echo "${package}" | awk '{$1=$1};1') if [ -e "${ZOPEN_PKGINSTALL}/${package}/" ]; then printVerbose "${package} is either installed or has been previously" - found=$(zosfind "${ZOPEN_PKGINSTALL}/${package}/" -type l -prune -o -type d -print | sed -e "s#${ZOPEN_PKGINSTALL}/${package}/\([^/]*\).*#\1#" | uniq) + found=$(find "${ZOPEN_PKGINSTALL}/${package}/" -type l -prune -o -type d -print | sed -e "s#${ZOPEN_PKGINSTALL}/${package}/\([^/]*\).*#\1#" | uniq) else printVerbose "${package} has never been installed on the system" fi diff --git a/bin/zopen-audit b/bin/zopen-audit index ec4aa5ccc..0aea798f3 100755 --- a/bin/zopen-audit +++ b/bin/zopen-audit @@ -129,7 +129,7 @@ high_vulns=0 critical_vulns=0 # Check for CVEs in all installed projects -installed_packages=$(cd "${ZOPEN_PKGINSTALL}" && zosfind ./*/. ! -name . -prune -type l) +installed_packages=$(cd "${ZOPEN_PKGINSTALL}" && find ./*/. ! -name . -prune -type l) printVerbose "Found all installed packages." # Variables for --upgrade flag diff --git a/bin/zopen-clean b/bin/zopen-clean index b43f91eeb..c52ffe223 100755 --- a/bin/zopen-clean +++ b/bin/zopen-clean @@ -30,7 +30,7 @@ from the system to save space and prevent clutter. Usage: ${ME} [OPTION] [PACKAGE] Options: - --deep deep clean - run all cleanup operations + --deep deep clean - run all cleanup operations --all apply cleanup command to all applicable packages. -c, --cache [PACKAGE ...] cleans the downloaded package cache; packages will be @@ -51,10 +51,10 @@ Examples: zopen clean -d analyse the zopen file system and remove dangling symlinks zopen clean -u [PACKAGE] remove unused versions for PACKAGE - zopen clean -u --all + zopen clean -u --all remove all unused packages within the zopen environment - zopen clean --deep - + zopen clean --deep + Report bugs at https://github.com/zopencommunity/meta/issues. @@ -75,10 +75,10 @@ removeUnusedPackageVersions() else printInfo "No currently active version of '${needle}'; removing all versions" fi - + counterfile=$(mktempfile "clean" ".rupv") addCleanupTrapCmd "rm -f ${counterfile}" - cd "${ZOPEN_PKGINSTALL}/${needle}" && zosfind . -name "./*" -prune -type d > "${counterfile}" + cd "${ZOPEN_PKGINSTALL}/${needle}" && find . -name "./*" -prune -type d > "${counterfile}" unusedCount=0 while read repo; do printVerbose "Parsing repo: '${repo}' as '${repo#./}'" @@ -105,8 +105,8 @@ removeUnusedPackageVersions() cleanUnused() { ${deep} && printInfo "Removing any unused package versions" - if ${all}; then - zopen list --installed --no-header --no-version | while read -r needle; do + if ${all}; then + zopen list --installed --no-header --no-version | while read -r needle; do removeUnusedPackageVersions "${needle}" done elif [ -z "$1" ]; then @@ -149,7 +149,7 @@ cleanDangling() counterfile=$(mktempfile "clean" ".sl") addCleanupTrapCmd "rm -f ${counterfile}" - zosfind "${ZOPEN_ROOTFS}" -type l -exec test ! -e {} \; -print > "${counterfile}" + find "${ZOPEN_ROOTFS}" -type l -exec test ! -e {} \; -print > "${counterfile}" counter=0 while IFS= read -r sl; do printVerbose "Removing dangling symlink '${sl}'" @@ -178,7 +178,7 @@ cleanPackageCache() else for needle in $1; do printVerbose "Cleaning ${ZOPEN_ROOTFS}/var/cache/zopen entries for '${needle}" - zosfind "${ZOPEN_ROOTFS}"/var/cache/zopen -name "${needle}-*" -exec rm {} \; + find "${ZOPEN_ROOTFS}"/var/cache/zopen -name "${needle}-*" -exec rm {} \; syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_FILE}" "CLEAN" "cleanPackageCache" "Cache for '${needle}' in ${ZOPEN_ROOTFS}/var/cache/zopen cleaned." printVerbose "- Cleaned cached files for '${needle}' in cache at '${ZOPEN_ROOTFS}/var/cache/zopen'." done @@ -239,7 +239,7 @@ while [ $# -gt 0 ]; do unused=true dangling=true cache=true - meta=true + meta=true ;; "--nostats") stats=false @@ -295,7 +295,7 @@ fi ${unused} && cleanUnused "${packagelist}" ${cache} && cleanPackageCache "${packagelist}" -if ${stats}; then +if ${stats}; then afterUsage=$(diskusage "${ZOPEN_ROOTFS}") printInfo "- Disk usage: Before: $(formattedFileSize "${beforeUsage}"); After: $(formattedFileSize "${afterUsage}"); Reclaimed: $(formattedFileSize $((beforeUsage - afterUsage)))" fi diff --git a/bin/zopen-diagnostics b/bin/zopen-diagnostics index 95a9af537..c42ae5be1 100755 --- a/bin/zopen-diagnostics +++ b/bin/zopen-diagnostics @@ -33,7 +33,7 @@ printSyntax() print_Diagnostics() { - PLATFORM=$(/bin/uname -s) + PLATFORM=$(uname -s) # Checking if the platform is z/OS if [ ! ${PLATFORM} = "OS/390" ]; then @@ -42,15 +42,15 @@ print_Diagnostics() fi # Check version - VERSION=$(/bin/uname -rsvI 2>/dev/null) + VERSION=$(uname -rsvI 2>/dev/null) if [ -z "$VERSION" ]; then echo "ERROR: This z/OS system does not have a valid version." exit 1 fi - MAJOR=$(echo "$VERSION" | /bin/awk '{print $3}' | /bin/sed 's/^0*//') - MINOR=$(echo "$VERSION" | /bin/awk '{print $2}' | /bin/cut -d'.' -f1 | /bin/sed 's/^0*//') + MAJOR=$(echo "$VERSION" | awk '{print $3}' | sed 's/^0*//') + MINOR=$(echo "$VERSION" | awk '{print $2}' | cut -d'.' -f1 | sed 's/^0*//') if [ -z "$MINOR" ]; then MINOR=0 fi @@ -103,11 +103,11 @@ print_Diagnostics() echo "Architecture: $cpu_arch" echo "Zopen Version: $(zopen --version | head -1 | cut -d ' ' -f4)" echo "Disk Usage for ZOPEN_ROOTFS ($ZOPEN_ROOTFS):" - num1=$(/bin/du -kt $ZOPEN_ROOTFS | tail -1 | tr -s ' ' | cut -d ' ' -f2) - num2=1024 - echo "$(echo "scale=2; $num1 / $num2" | bc) mb" + num1=$(du -kt $ZOPEN_ROOTFS | tail -1 | tr -s ' ' | cut -d ' ' -f2) + num2=1024 + echo "$(echo "scale=2; $num1 / $num2" | bc) mb" echo -e "\nFilesystem Usage (df) for \$ZOPEN_ROOTFS:" - /bin/df -Pm "$ZOPEN_ROOTFS" + df -Pm "$ZOPEN_ROOTFS" } while [ $# -gt 0 ]; do @@ -137,7 +137,7 @@ verbose=false remote_lookup=false if [ -z "$args" ]; then - echo "To open an issue go to https://github.com/zopencommunity/meta/issues and paste the above information" + echo "To open an issue go to https://github.com/zopencommunity/meta/issues and paste the above information" else args_spaceless=$(echo "$args" | cut -d' ' -f2| tr -d ' ') url="To open an issue go to https://github.com/zopencommunity/"$args_spaceless"port/issues and paste the above information" diff --git a/bin/zopen-generate b/bin/zopen-generate index 0715dd245..ff33cac0d 100755 --- a/bin/zopen-generate +++ b/bin/zopen-generate @@ -331,7 +331,7 @@ zopen_get_version() " -/bin/echo "${buildenvContents}" > "${buildenv}" +echo "${buildenvContents}" > "${buildenv}" printInfo "${buildenv} created" diff --git a/bin/zopen-info b/bin/zopen-info index 9629db36c..865374c65 100755 --- a/bin/zopen-info +++ b/bin/zopen-info @@ -38,7 +38,7 @@ printPackageInfo() { remote_lookup=${2:-false} pkghome="${ZOPEN_PKGINSTALL}/${package}/${package}" metadata="${pkghome}/metadata.json" - + if ${remote_lookup} || [ ! -d "${pkghome}" ]; then metadataFile="$zopen_tmp_dir/$LOGNAME.$RANDOM.metadata.json" printVerbose "Performing remote lookup for package '${package}'" @@ -67,7 +67,7 @@ printPackageInfo() { if [ -z "${remote_data}" ] || [ "${remote_data}" = "null" ]; then printError "Package '${package}' not found in remote repository." exit 1 - fi + fi name=$(echo "${remote_data}" | jq -r '.name') url=$(echo "${remote_data}" | jq -r '.assets[0].url') @@ -91,7 +91,7 @@ printPackageInfo() { size=$(echo "${remote_data}" | jq -r '.assets[0].size') expanded_size=$(echo "${remote_data}" | jq -r '.assets[0].expanded_size') community_commitsha=$(echo "${remote_data}" | jq -r '.assets[0].community_commitsha') - runtime_dependencies=$(echo "${remote_data}" | jq -r '.assets[0].runtime_dependencies') + runtime_dependencies=$(echo "${remote_data}" | jq -r '.assets[0].runtime_dependencies') build_dependencies=$(jq -r '.product.build_dependencies | unique_by(.name) | map(.name) | join(" ")' ${metadataFile}) printHeader "==> ${name} (Not Installed)" @@ -130,13 +130,13 @@ printPackageInfo() { if [ -n "${community_commitsha}" ]; then printf "%-20s %s\n" "Community SHA:" "${community_commitsha}" fi - + if [ -n "${runtime_dependencies}" ]; then printHeader "==> Dependencies" printf "%-20s %s\n" "Runtime:" "${runtime_dependencies}" printf "%-20s %s\n" "Build:" "${build_dependencies}" fi - + rm ${metadataFile} return 0 @@ -207,7 +207,7 @@ printPackageInfo() { printHeader "==> Installation Details" printf "%-20s %s\n" "Installed:" "Yes" printf "%-20s %s\n" "Installation Path:" "${installed_path}" - printf "%-20s %s\n" "Installation Size:" "$(/bin/du -s ${installed_path} | awk '{print $1 / 1024 " MB"}')" + printf "%-20s %s\n" "Installation Size:" "$(du -s ${installed_path} | awk '{print $1 / 1024 " MB"}')" if [ -n "${total_tests}" ]; then test_percentage="N/A" diff --git a/bin/zopen-init b/bin/zopen-init index 1e3557f6d..3b8a4313f 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -287,7 +287,7 @@ determineStatsCollection() fi if $isCollectingStats; then - if ! /bin/ping -c 1 -t 2 "${ZOPEN_STATS_IP}" >/dev/null 2>/dev/null; then + if ! ping -c 1 -t 2 "${ZOPEN_STATS_IP}" >/dev/null 2>/dev/null; then printAttention "The statistics server url is blocked: ${ZOPEN_STATS_URL}. Turning usage statistics off." isCollectingStats=false fi @@ -547,7 +547,7 @@ zz .autocacheclean = \$autocacheclean | .autopkgpurge = \$autopkgpurge | .skip_broken = \$skip_broken | - .skip_size_check = \$skip_size_check + .skip_size_check = \$skip_size_check " "${jsonConfig}" > "${jsonConfigWorking}"; then printError "Errors updating existing configuration file. See previous errors for more information and retry command." fi diff --git a/bin/zopen-promote b/bin/zopen-promote index 92dbe9707..4ee0720dc 100755 --- a/bin/zopen-promote +++ b/bin/zopen-promote @@ -293,7 +293,7 @@ rmDanglingLinks(){ tmpfile=$(mktempfile "zopen_promote" ".pipe") addCleanupTrapCmd "rm -rf ${tmpfile}" - zosfind "${promotefs}" -type l -exec test ! -e {} \; -print > "${tmpfile}" + find "${promotefs}" -type l -exec test ! -e {} \; -print > "${tmpfile}" while IFS= read -r sl; do printVerbose "Removing symlink '${sl}'" # rm -f "${sl}" diff --git a/bin/zopen-query b/bin/zopen-query index 0c762f18a..a6b0ab14c 100755 --- a/bin/zopen-query +++ b/bin/zopen-query @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh # # Query utility for zopen community - https://github.com/zopencommunity # @@ -116,8 +116,8 @@ printDetailListEntries() if [ $? -ne 0 ]; then printError "Unable to retrieve remote information" fi - latestVersion=$(/bin/printf "%s" "${latest}" | jq -e -r '.assets[0].name' | sed -E 's/.*-(.*)\.zos\.pax\.Z/\1/') - categories=$(/bin/printf "%s" "${latest}" | jq -e -r '.assets[0].categories') + latestVersion=$(printf "%s" "${latest}" | jq -e -r '.assets[0].name' | sed -E 's/.*-(.*)\.zos\.pax\.Z/\1/') + categories=$(printf "%s" "${latest}" | jq -e -r '.assets[0].categories') if [ -n "${category_filter}" ] && ! echo "${categories}" | grep -q "${category_filter}"; then continue fi @@ -179,12 +179,12 @@ printDetailListEntries() ;; 0) printVerbose "Latest release request successful" - latestTag="$(/bin/printf "%s" "${latest}" | jq -e -r '.tag_name')" - passed="$(/bin/printf "%s" "${latest}" | jq -e -r '.assets[0].passed_tests')" - total="$(/bin/printf "%s" "${latest}" | jq -e -r '.assets[0].total_tests')" - expandedsize="$(/bin/printf "%s" "${latest}" | jq -e -r '.assets[0].expanded_size')" - downloadsize="$(/bin/printf "%s" "${latest}" | jq -e -r '.assets[0].size')" - categories=$(/bin/printf "%s" "${latest}" | jq -e -r '.assets[0].categories') + latestTag="$(printf "%s" "${latest}" | jq -e -r '.tag_name')" + passed="$(printf "%s" "${latest}" | jq -e -r '.assets[0].passed_tests')" + total="$(printf "%s" "${latest}" | jq -e -r '.assets[0].total_tests')" + expandedsize="$(printf "%s" "${latest}" | jq -e -r '.assets[0].expanded_size')" + downloadsize="$(printf "%s" "${latest}" | jq -e -r '.assets[0].size')" + categories=$(printf "%s" "${latest}" | jq -e -r '.assets[0].categories') if [ "${onlyUpgradesAvailable}" -gt 0 ]; then if [ "${originalTag}" = "Not installed" -o "${originalTag}" = "${latestTag}" ]; then continue @@ -193,17 +193,17 @@ printDetailListEntries() if [ -n "$total" ] && [ ${total} -gt 0 ]; then percentage=$(echo "scale=0; 100 * (${passed}) / ${total}" | bc) fi - /bin/printf "%-${colwidth}s %-${colwidth}s %-${colwidth}s" "${repo}" "${originalTag}" "${latestTag}" - /bin/printf " %-${colwidth}s" "${downloadsize}" - /bin/printf " %-${colwidth}s" "${expandedsize}" + printf "%-${colwidth}s %-${colwidth}s %-${colwidth}s" "${repo}" "${originalTag}" "${latestTag}" + printf " %-${colwidth}s" "${downloadsize}" + printf " %-${colwidth}s" "${expandedsize}" if [ -z "${percentage}" ]; then - /bin/printf "${NC}${RED}%-${colwidth}s${NC}" "No tests" + printf "${NC}${RED}%-${colwidth}s${NC}" "No tests" else - /bin/printf "${NC}$(colorizepct "${percentage}")%-${colwidth}s${NC}" "${percentage}" + printf "${NC}$(colorizepct "${percentage}")%-${colwidth}s${NC}" "${percentage}" fi - /bin/printf " %-${colwidth}s" "${categories}" - /bin/printf "\n" + printf " %-${colwidth}s" "${categories}" + printf "\n" ;; *) printError "Error while trying to retrieve latest repo release" ;; esac @@ -220,7 +220,7 @@ whatProvides() needle="$1" printVerbose "Finding matches outside of ZOPEN_PKGINSTALL (${ZOPEN_PKGINSTALL})" # Find any symlinks that match the needle and can then be dereferenced - found=$(zosfind "${ZOPEN_ROOTFS}" -name "${ZOPEN_PKGINSTALL}/\*" -prune -o -type l -print | grep "${needle}") + found=$(find "${ZOPEN_ROOTFS}" -name "${ZOPEN_PKGINSTALL}/\*" -prune -o -type l -print | grep "${needle}") printVerbose "Found list: '${found}'" if [ -z "${found}" ]; then printInfo "No package provides '${needle}'" diff --git a/bin/zopen-usage b/bin/zopen-usage index 73263e07b..a03faa72a 100755 --- a/bin/zopen-usage +++ b/bin/zopen-usage @@ -80,8 +80,8 @@ render_pie_chart_awk(){ center_x=$((radius + 1)) center_y=$radius aspect="0.6" # Reasonable guess? - - if ! pie=$(/bin/awk -v FS="|" \ + + if ! pie=$(awk -v FS="|" \ -v title="${title}" \ -v legend_title="${legend_title}" \ -v radius="${radius}" \ @@ -102,10 +102,10 @@ render_pie_chart_awk(){ for (theta = current_angle; theta <= end_angle; theta++) { rad = theta * a2r; # Calculate Cartesian coordinates - sohcahtoa... - x = round(cx + r * cos(rad)); + x = round(cx + r * cos(rad)); y = round(cy + r * sin(rad)*aspect); # Bounds check - + if (x >= 0 && x <= (width+1) && y >= 0 && y < height) { if (use_ansi == "true") { chart[x,y] = ansi_csi "[" color "m" color_symbol ansi_csi "[0m"; @@ -165,7 +165,7 @@ render_pie_chart_awk(){ end_angle = int(current_angle + angle_size); if (use_ansi == "true") { modidx=idx % colors_size + 1 # Note % to wrap?? - color = colors_array[modidx] ; + color = colors_array[modidx] ; if (NR > colors_size) { # We cannot chart this, so add it to the "other" category color="31" @@ -200,7 +200,7 @@ render_pie_chart_awk(){ legend[31] = sprintf("%7s: %11sk (%4.1f%%): %25s", ansi_csi "[31m" color_symbol ansi_csi "[0m", (datasize - pie_size)/1024, ((datasize - pie_size) / datasize*100), "Other assorted") } else { legend["?"] = sprintf("%7s: %11sk (%4.1f%%): %25s", "?", (datasize - pie_size)/1024, ((datasize - pie_size) / datasize*100), "Other assorted") - } + } } # Set the centre character to a generic, unused value chart[cx,cy] = "O"; @@ -224,12 +224,12 @@ render_pie_chart_awk(){ for (i = 1; i <= n; i++) { print legendrev[i] } - + }' "${datafile}" ); then printError "Could not generate pie chart: ${pie}" fi # shellcheck disable=SC2059 - /bin/printf "${pie}\n" + printf "${pie}\n" return 0 } @@ -251,7 +251,7 @@ list_usage(){ percentage=$(echo "scale=2; 100 * ${other_size} / ${datasize}" | bc) printf "%10sK %-10s: Other\n" "${other_size}" "(${percentage}%)" fi - zopen_env_size=$(zosdu -kts "${zopen_rootfs}" 2>/dev/null | awk '{printf "%.2f", ($1 / 1024)}') + zopen_env_size=$(du -kts "${zopen_rootfs}" 2>/dev/null | awk '{printf "%.2f", ($1 / 1024)}') available=$(df -k "${zopen_rootfs}" | tail -n 1 | tr -s " " | cut -d' ' -f3) printf " The zopen environment is using %sMb of total storage\n" "${zopen_env_size}" echo "${available}" | awk -F'/' '{ printf " There is %.2f Mb available for the zopen environment\n" , ($1 / 1024) }' @@ -260,7 +260,7 @@ list_usage(){ add_dir_size(){ dir=$1 [ -e "${dir}" ] || return 1 - size=$(zosdu -kts "${dir}" 2>/dev/null | awk '{print $1}') + size=$(du -kts "${dir}" 2>/dev/null | awk '{print $1}') # Do size first as it gets sorted on echo "${size}|${dir}" >> "${datafile}" echo "${size}" @@ -276,7 +276,7 @@ create_datafile() { printError "Specified zopen environment '${zopen_rootfs}' does not appear to be valid" datafile=$(mktempfile "pie" ".data") - zopen_env_size=$(zosdu -kts "${zopen_rootfs}" 2>/dev/null | awk '{print $1}') + zopen_env_size=$(du -kts "${zopen_rootfs}" 2>/dev/null | awk '{print $1}') # Get explicit directories to include for dir in "${zopen_rootfs}"/usr/local/zopen/*; do # Get space usage of the directory (in format "size dir") diff --git a/bin/zopen-whichproject b/bin/zopen-whichproject index 6745a409b..53bef4616 100755 --- a/bin/zopen-whichproject +++ b/bin/zopen-whichproject @@ -92,7 +92,7 @@ jq_query='.project_files | to_entries[] | select(.value.binaries[]? == $item or # Execute jq, passing the query item as an argument # Handle potential errors from jq (e.g., file not found, invalid JSON) -matching_projects=$(jq -r --arg item "$query_item" "${jq_query}" "${JSON_FILES_CACHE}" 2> /dev/null | /bin/sort -u) +matching_projects=$(jq -r --arg item "$query_item" "${jq_query}" "${JSON_FILES_CACHE}" 2> /dev/null | sort -u) jq_exit_status=$? if [ ${jq_exit_status} -ne 0 ] && [ ${jq_exit_status} -ne 4 ]; then # jq exits 4 if no results found, others are errors diff --git a/include/common.sh b/include/common.sh index 23fe663c7..a71c3e8d6 100755 --- a/include/common.sh +++ b/include/common.sh @@ -5,19 +5,23 @@ zopenInitialize() { + # Hardcode the z/OS-supplied tooling to ensure zopen only uses standard + # tools + export PATH=/bin:"$PATH" + # Capture start time before setting trap fullProcessStartTime=${SECONDS} - + # Create the cleanup pipeline and exit handler trap "cleanupFunction" EXIT INT TERM QUIT HUP - + # Temporary files for zopen_tmp_dir in "${TMPDIR}" "${TMP}" /tmp; do if [ ! -z ${zopen_tmp_dir} ] && [ -d ${zopen_tmp_dir} ]; then break fi done - + if [ ! -d "${zopen_tmp_dir}" ]; then printError "Temporary directory not found. Please specify \$TMPDIR, \$TMP or have a valid /tmp directory." fi @@ -29,6 +33,7 @@ zopenInitialize() fi ZOPEN_ORGNAME="zopencommunity" + # shellcheck disable=SC2034 ZOPEN_GITHUB="https://github.com/${ZOPEN_ORGNAME}" # shellcheck disable=SC2034 ZOPEN_ANALYTICS_JSON="${ZOPEN_ROOTFS}/var/lib/zopen/analytics.json" @@ -80,7 +85,7 @@ cleanup() { addCleanupTrapCmd2(){ # Attempt to remove any redirects; rather than test, simpler to remove # and re-add if present. - newcmd=$(echo "$cmd" | zossed -E "s/[ \t]*[1-9]*[<>][>|&]?[ \t]*[^ \t]*//g") + newcmd=$(echo "$cmd" | sed -E "s/[ \t]*[1-9]*[<>][>|&]?[ \t]*[^ \t]*//g") newcmd="${newcmd} >/dev/null 2>&1" # Command Trace MUST be disabled as the output from this can become # interleaved with output when calling zopen sub-processes. @@ -97,7 +102,7 @@ addCleanupTrapCmd2(){ newtrapcmd=$( echo "${script}" | while read trapcmd; do sigcmd=$(echo "${trapcmd}" | - zossed "s/trap -- \"\(.*\)\" ${trappedSignal}.*/\1/") + sed "s/trap -- \"\(.*\)\" ${trappedSignal}.*/\1/") # No match/replace in sed, then sigcmd remains unchanged [ "${sigcmd}" = "${trapcmd}" ] && continue # There was a match, so sigcmd contains the string of commands @@ -271,7 +276,7 @@ mktempdir() isPermString() { - test=$(echo "$1" | zossed "s/[-+rwxugo,=]//g") + test=$(echo "$1" | sed "s/[-+rwxugo,=]//g") if [ -n "${test}" ]; then printDebug "Permission string '$1' was invalid" false; @@ -504,7 +509,7 @@ if \${knv}; then IFS=":" for token in \${knvp##*=}; do tok=\$(echo "\${token}" | /bin/sed -e 's#/usr/local/zopen/\([^/]*\)/[^/]*/#/usr/local/zopen/\1/\1/#') - newval=\$(printf "%s:%s" "\${newval}" "\${tok}") + newval=\$(/bin/printf "%s:%s" "\${newval}" "\${tok}") done echo "\${exportknv}\${envvar}=\${newval#*:}" IFS="\${cIFS}" @@ -561,14 +566,14 @@ deref_symlink() { fi targetName=$(basename "${target}") targetDir=$(dirname "${target}") - # Check if dir starts with a '/'; if not need to calculate absolute + # Check if dir starts with a '/'; if not need to calculate absolute # fully-qualified name case "$target" in - /*) targetDir=$(cd "${targetDir}" && pwd -P);; - *) + /*) targetDir=$(cd "${targetDir}" && pwd -P);; + *) symlink_dir=$(dirname "$symlink") targetDir=$(cd "$(pwd -P)/${symlink_dir}" && pwd -P) - esac + esac absolute_dir=$(cd "$targetDir" && pwd -P) # Resolves symlinked dirs echo "$absolute_dir/$targetName" } @@ -684,7 +689,7 @@ ansiline() deltay=$2 echostr="$3" ansimove "$1" "$2" - zosecho "${echostr}\c" + echo "${echostr}\c" } ansimove() @@ -707,7 +712,7 @@ ansimove() movestr="${movestr}${ESC}${CSI}$((deltay * -1))${CRSRUP}" fi fi - zosecho "${movestr}\c" + echo "${movestr}\c" } getScreenCols() @@ -725,46 +730,6 @@ getScreenCols() echo "${lclcols}" } -zostr() -{ - # Use the standard z/OS 'tr' command - /bin/tr "$@" -} -zosecho() -{ - # Use the standard z/OS echo utility; supports use of ANSI colour when - # required and is consistent across shells as it uses the EBCDIC ANSI - # control codes - /bin/echo "$@" -} - -zossed() -{ - # Use the standard z/OS sed utility; If the sed package is installed - # GNU sed becomes the dominant version which might change how - # matching is performed - /bin/sed "$@" -} - -zosfind() -{ - # Use the standard z/OS find utility; If the findutils package is installed, - # the installed find command takes precedence but is not compatible with the - # standard zos find [regex searches for "-name" are not allowed, but - # "-wholename" is not available on standard zosfind. For the tooling to be - # consistent across platforms (where findutils is/is not installed) use the - # standard zos version - /bin/find "$@" -} - -zosdu() -{ - # Use the standard z/OS du utility; if the coreutils package is installed, this - # changes the behaviour of the du command so we need to ensure we support those - # who do not have that installed - /bin/du "$@" -} - findrev() { haystack="$1" @@ -777,7 +742,7 @@ findrev() strtrim() { - echo "$1" | zossed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' + echo "$1" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' } text_center() @@ -821,20 +786,20 @@ defineEnvironment() # Required for proper operation of (USS shipped) sed export _UNIX03=YES - # Use /bin/cat as the pager in case xlclang help is displayed, we don't want to wait for input - export PAGER=/bin/cat + # Use cat as the pager in case xlclang help is displayed, we don't want to wait for input + export PAGER=cat # Set a default umask of read and execute for user and group umask 0022 } # -# For now, explicitly specify zosecho to ensure we get the EBCDIC echo since the escape +# For now, explicitly specify echo to ensure we get the EBCDIC echo since the escape # sequences are EBCDIC escape sequences # printColors() { - zosecho "$@" + echo "$@" } mutexReq() @@ -844,7 +809,7 @@ mutexReq() [ -e lockdir ] || mkdir -p ${lockdir} mutex="${lockdir}/${mutex}" mypid=$(exec sh -c 'echo ${PPID}') - mygrandparent=$(/bin/ps -o ppid= -p "$mypid" | awk '{print $1}') + mygrandparent=$(ps -o ppid= -p "$mypid" | awk '{print $1}') if [ -e "${mutex}" ]; then lockedpid=$(cat ${mutex}) { @@ -896,7 +861,7 @@ relativePath2() # if the target is longer than the source, there might be some additional # elements in the shifted $0 to append if [ $# -gt 0 ]; then - relativePath=${relativePath}/$(echo $* | zossed "s/ /\//g") + relativePath=${relativePath}/$(echo $* | sed "s/ /\//g") fi IFS="${currentIFS}" echo "${relativePath}" @@ -947,8 +912,8 @@ mergeIntoSystem() # ignore them as the first call should generate the correct link but # subsequent calls would generate a symlink that has incorrect dereferencing # and ignoring them is actually faster than individually creating the links! - zosfind . -type d | sort -r | while read dir; do - dir=$(echo "${dir}" | zossed "s#^./##") + find . -type d | sort -r | while read dir; do + dir=$(echo "${dir}" | sed "s#^./##") printDebug "Processing dir: ${dir}" [ "${dir}" = "." ] && continue mkdir -p "${processingDir}/${rebaseusr}/${dir}" @@ -994,7 +959,6 @@ rmSymlinksFSCheck(){ # Note that the contents of the links file are ordered such that # processing occurs depth-first; if, after removing orphaned symlinks, # a directory is empty, then it can be removed. - nfiles=$(zossed '1d;$d' "${dotlinks}" | wc -l | tr -d ' ') printDebug "Creating Temporary dirname file" tempDirFile=$(mktempfile "unsymlink") [ -e "${tempDirFile}" ] && rm -f "${tempDirFile}" >/dev/null 2>&1 @@ -1005,7 +969,7 @@ rmSymlinksFSCheck(){ addCleanupTrapCmd "rm -rf ${tempTrash}" printDebug "Using temporary file ${tempDirFile}" while read filetounlink; do - filetounlink=$(echo "${filetounlink}" | zossed 's/\(.*\).symbolic.*/\1/') + filetounlink=$(echo "${filetounlink}" | sed 's/\(.*\).symbolic.*/\1/') filename="$filetounlink" [ -z "${filetounlink}" ] && continue filetounlink="${ZOPEN_ROOTFS}/${filetounlink}" @@ -1025,7 +989,7 @@ rmSymlinksFSCheck(){ echo "Unprocessable file: '${filetounlink}'" >> "${tempTrash}" fi done < "${file}_ascii" || ! chtag -tc ISO8859-1 "${file}_ascii" || ! mv "${file}_ascii" "${file}"; then + if ! iconv -f IBM-1047 -t ISO8859-1 < "${file}" > "${file}_ascii" || ! chtag -tc ISO8859-1 "${file}_ascii" || ! mv "${file}_ascii" "${file}"; then printError "Unable to convert EBCDIC text to ASCII for ${file}" >&2 fi fi @@ -1813,7 +1777,7 @@ a2e() fi if [ "$(chtag -p "${source}" | cut -f2 -d' ')" = "ISO8859-1" ]; then - /bin/iconv -f ISO8859-1 -t IBM-1047 "$source" > "$source.bk" + iconv -f ISO8859-1 -t IBM-1047 "$source" > "$source.bk" chtag -tc 1047 "$source.bk" mv "$source.bk" "$source" fi @@ -1864,7 +1828,7 @@ promptYesNoAlways() { skip=$2 if ! ${skip}; then while true; do - /bin/printf "${message} [y/n/a] " + printf "${message} [y/n/a] " read answer < /dev/tty answer=$(echo "${answer}" | tr '[A-Z]' '[a-z]') case "${answer}" in @@ -1893,8 +1857,8 @@ getVersionedMetadata() return 1 fi printDebug "Check for asset" - asset=$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]') - if ! asset=$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]'); then + asset=$(printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]') + if ! asset=$(printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]'); then printSoftError "Could not find asset for release version '${versioned}' in repo '${repo}'" 2> "${invalidPortAssetFile}" return 1 fi @@ -1912,8 +1876,8 @@ getTaggedMetadata() return 1 fi printDebug "Check for asset" - asset=$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]') - if ! asset=$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]'); then + asset=$(printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]') + if ! asset=$(printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]'); then printSoftError "Could not find asset for release tagged '${tagged}' in repo '${repo}'" 2> "${invalidPortAssetFile}" return 1 fi @@ -1972,7 +1936,7 @@ getReleaseLineMetadata() return 1 fi printDebug "Use quick check for asset to check for existence of metadata" - if ! asset="$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')"; then + if ! asset="$(printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')"; then printSoftError "Could not find asset for releaseline '${validatedReleaseLine}' for repo '${repo}' in ${JSON_CACHE}" 2> "${invalidPortAssetFile}" return 1 fi @@ -2020,7 +1984,7 @@ calculateReleaseLineMetadata() '.release_data[$repo] | map(select(.tag_name | startswith($releaseLine)))[0]' "${JSON_CACHE}") printDebug "Use quick check for asset to check for existence of metadata" - asset="$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')" + asset="$(printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')" [ "${asset}" = "null" ] && asset="" # jq uses null, translate to sh's empty if [ -n "${asset}" ]; then @@ -2033,7 +1997,7 @@ calculateReleaseLineMetadata() releasemetadata=$(jq --arg repo "${repo}" --arg releaseLine "${alt}" \ '.release_data[$repo] | map(select(.tag_name | startswith($releaseLine)))[0]' "${JSON_CACHE}") printDebug "Use quick check for asset to check for existence of metadata" - asset="$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')" + asset="$(printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]')" [ "${asset}" = "null" ] && asset="" # jq uses null, translate to sh's empty if [ $? -eq 0 ]; then printDebug "Found a release on the '${alt}' release line so release tagging is active" @@ -2113,7 +2077,7 @@ getPortMetaData(){ return 1 fi printDebug "Getting specific asset details using metadata: ${releasemetadata}" - asset=$(/bin/printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]') + asset=$(printf "%s" "${releasemetadata}" | jq -e -r '.assets[0]') fi if [ -z "${asset}" ]; then echo "${portName} asset metadata could not be found" 2> "${invalidPortAssetFile}" @@ -2273,7 +2237,7 @@ generateInstallGraph(){ if [ -e "${invalidPortAssetFile}" ]; then printSoftError "The following ports cannot be installed: " while read invalidPort; do - /bin/printf "${WARNING} %s\n" "${invalidPort}" + printf "${WARNING} %s\n" "${invalidPort}" done < "${invalidPortAssetFile}" printError "Confirm port names, remove any 'port' suffixes and retry command." else @@ -2377,13 +2341,13 @@ spaceValidate(){ spaceRequiredMB=$(echo "scale=0; (${cacheBytes} + ${packageBytes}) / (1024 * 1024)" | bc) printInfo "After this operation, $(formattedFileSize ${spaceRequiredKb}) of additional disk space will be used." fi - availableSpaceKb=$(/bin/df -k "${ZOPEN_ROOTFS}" | sed "1d" | awk '{ print $3 }' | awk -F'/' '{ print $1 }') + availableSpaceKb=$(df -k "${ZOPEN_ROOTFS}" | sed "1d" | awk '{ print $3 }' | awk -F'/' '{ print $1 }') if [ "${availableSpaceKb}" -lt "${spaceRequiredKb}" ]; then printWarning "Your zopen file-system (${ZOPEN_ROOTFS}) only has $(formattedFileSize "${availableSpaceKb}") of available space." fi if ! ${yesToPrompts} || [ "${availableSpaceKb}" -lt "${spaceRequiredKb}" ]; then while true; do - /bin/printf "Do you want to continue [y/n/a]? " + printf "Do you want to continue [y/n/a]? " read continueInstall < /dev/tty case "${continueInstall}" in "y") break;; @@ -2687,7 +2651,7 @@ EOF getActivePackageDirs() { - (unset CD_PATH; cd "${ZOPEN_PKGINSTALL}" && zosfind ./*/. ! -name . -prune -type l) + (unset CD_PATH; cd "${ZOPEN_PKGINSTALL}" && find ./*/. ! -name . -prune -type l) } @@ -2733,7 +2697,7 @@ processActionScripts() fi printVerbose "Attempting to run script '${scriptletFile}'" # Run script in a subshell to prevent environment modification - /bin/printf "${CRSRSOL}${ERASELINE}Running ${scriptletFile}" + printf "${CRSRSOL}${ERASELINE}Running ${scriptletFile}" scriptletOutput=$({ # shellcheck disable=SC1090 # shellcheck disable=SC2240 @@ -2741,7 +2705,7 @@ processActionScripts() echo $? # Append exit status to output } 2>&1 ) - /bin/printf "${CRSRSOL}${ERASELINE}${CRSRSOL}" + printf "${CRSRSOL}${ERASELINE}${CRSRSOL}" scriptletRc=$(echo "${scriptletOutput}" | tail -n 1) # Extract exit status scriptletBody=$(echo "${scriptletOutput}" | sed '$d') # Extract script output if [ "${scriptletRc}" -ne 0 ]; then { @@ -2812,7 +2776,7 @@ updatePackageDB() mdj=$(jq '[{(.product.name):.}]' \ "${escapedJSONFile}") - if [ -z "${mdj}" ]; then + if [ -z "${mdj}" ]; then # If we couldn't get the repo name from .product.name, use (.*)port - this might result # in odd cases [NATS...] mdj=$(jq '. as $metadata | .product.repo | match(".*/zopencommunity/(.*)port").captures[0].string | [{(.):$metadata}]' \ @@ -2862,7 +2826,7 @@ stripControlCharacters(){ JSONcontrolChar2Unicode() { [ ! -f "$1" ] && assertFailed "No input file specified for parsing!" [ -e "$2" ] && assertFailed "Output file exists so cannot be used for output!" - zossed -E ' # Note this is a long string! + sed -E ' # Note this is a long string! s/\\/\\\\/g; # Escape reverse-solidus; the following are the control chars s/(^|[^\\])\x00/\1\\u0000/g; s/(^|[^\\])\x01/\1\\u0001/g; s/(^|[^\\])\x02/\1\\u0002/g; s/(^|[^\\])\x03/\1\\u0003/g; @@ -2952,7 +2916,7 @@ diskusage() { path=$1 # awk to "trim" output" - if ! size=$(zosdu -kts "${path}" | /bin/awk '{print ($1)}'); then + if ! size=$(du -kts "${path}" | awk '{print ($1)}'); then printError "Unable to generate disk usage (du) report for '${path}'" fi echo "${size}" From 5a29ff570c8e6c6e27370152421715299ff5a9eb Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 30 Sep 2025 16:33:51 +0100 Subject: [PATCH 128/179] Replace jq query for package name validation --- include/common.sh | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/include/common.sh b/include/common.sh index a71c3e8d6..da7035c77 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2158,7 +2158,7 @@ validateInstallList(){ # This uses the release json file to find whether any of the inputs are invalid # package names. Ideally, this would use the keys of the release_data however, the # NATS package breaks this as it is known as 'nats' in the repo but the port name is -# NATS. As such, use .product.name then fallback to the .url as a sanity check +# NATS. As such, use .product.name then fallback to the .tag as a sanity check # Note: this is not to make the match "case-insensitive", but to use whichever the # user uses as the name - if they know it as NATS, accept that! validatePackageList(){ @@ -2166,17 +2166,23 @@ validatePackageList(){ # shellcheck disable=SC2086 # Using set -f disables globbing printVerbose "Stripping any version/tagging" installees=$(set -f; echo ${installees} |awk -v ORS=, -v RS=' ' '{$1=$1; sub(/[=%].*/,x); print "\""$1"\""}') + # Check the metadata cache file to see if the needle is either the name of a + # port (so a key) or is defined in a tag invalidPortList=$(jq -r --argjson needles "[${installees%%,}]" \ -'$needles - | map( - select( - (. as $needle) | - (any( - .release_data | to_entries[] | .value[]; - .tag_name | test("_[^_]*" + $needle + "[^_]*port") - ) | not) +'. as $data +| $needles +| map( + select( + . as $needle| ( + ($data.release_data | keys_unsorted | any(. == $needle)) + or + ($data.release_data | to_entries[] | .value[] | .tag_name | (test("_[^_]*" + . + "[^_]*port")) + ) + | not ) - )' \ + ) + ) +| unique | join(" ")' \ "${JSON_CACHE}") if [ -n "${invalidPortList}" ]; then printSoftError "The following ports could not be installed:" From 3195a6df65efd89d9ba6423176ea5120e5ba7825 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 1 Oct 2025 10:29:50 +0100 Subject: [PATCH 129/179] Add typo suggestions for package names from PR 1087 https://github.com/zopencommunity/meta/pull/1087 Add in optimized version that loops within single awk instance rather than looping in shell and invoking awk repeatedly --- include/common.sh | 143 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 114 insertions(+), 29 deletions(-) diff --git a/include/common.sh b/include/common.sh index da7035c77..70e8b2d00 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2140,21 +2140,6 @@ addToInstallGraph(){ return 0 } -validateInstallList(){ - installees="$1" - # shellcheck disable=SC2086 # Using set -f disables globbing - printVerbose "Stripping any version/tagging" - installees=$(set -f; echo ${installees} |awk -v ORS=, -v RS=' ' '{$1=$1; sub(/[=%].*/,x); print "\""$1"\""}') - invalidPortList=$(jq -r --argjson needles "[${installees%%,}]" \ - '.release_data| keys as $haystack | $needles | map(select(. as $needle | $haystack | index($needle)|not)) | .[]' \ - "${JSON_CACHE}") - if [ -n "${invalidPortList}" ]; then - printSoftError "The following ports could not be installed:" - printSoftError " $(echo "${invalidPortList}" | awk -v OFS=' ' -v ORS=' ' '{$1=$1};1' )" - printError "Check port name(s), remove any extra 'port' suffixes and retry command." - fi -} - # This uses the release json file to find whether any of the inputs are invalid # package names. Ideally, this would use the keys of the release_data however, the # NATS package breaks this as it is known as 'nats' in the repo but the port name is @@ -2169,25 +2154,36 @@ validatePackageList(){ # Check the metadata cache file to see if the needle is either the name of a # port (so a key) or is defined in a tag invalidPortList=$(jq -r --argjson needles "[${installees%%,}]" \ -'. as $data -| $needles -| map( - select( - . as $needle| ( + '. as $data | $needles| map( select( . as $needle| ( ($data.release_data | keys_unsorted | any(. == $needle)) or ($data.release_data | to_entries[] | .value[] | .tag_name | (test("_[^_]*" + . + "[^_]*port")) - ) - | not - ) - ) + ) | not ) ) ) | unique | join(" ")' "${JSON_CACHE}" ) -| unique | join(" ")' \ - "${JSON_CACHE}") + if [ -n "${invalidPortList}" ]; then - printSoftError "The following ports could not be installed:" - printSoftError " $(echo "${invalidPortList}" | awk -v OFS=' ' -v ORS=' ' '{$1=$1};1' )" - printError "Check port name(s), remove any extra 'port' suffixes and retry command." + printSoftError "The following port names were not recognised:" + + # Calculate word lengths for nicer formatting + set -- ${invalidPortList} + minlen=1 + for toolrepo in "$@"; do + wordlen=${#toolrepo} + minlen=$(( wordlen > minlen ? wordlen : minlen )) + done + # Respin loop actually checking for suggestions + set -- ${invalidPortList} + for toolrepo in "$@"; do + printDebug "Finding suggestions for '${toolrepo}'[${#toolrepo}]" + suggestion=$(toolSuggestion "${toolrepo}") + # Use printf to print without the printSoftError prefix for cleaner output + if [ -n "${suggestion}" ]; then + printf " %-${minlen}s [Did you mean '%s'?]\n" "${toolrepo}" "${suggestion}" >&2 + else + printf " %${minlen}s\n" "${toolrepo}" >&2 + fi + done + printError "Check port name(s) and retry command." fi } @@ -2967,6 +2963,95 @@ isTrue() esac } +# Lookup the list of ports and find the closest suggestion. This uses +# awk to parse the dictionary directly from the cache file [piped via jq] +toolSuggestion(){ + word="$1" + threshold=4 # If the word is too far from alternatives, do not suggest + jq -r '.release_data | keys[]' "${JSON_CACHE}" | \ + awk -v s1="${word}" -v th="${threshold}" ' + BEGIN { best = ""; current=th;} + END { if (current <= th) print best } + { + s2=$1 + len1 = length(s1) + len2 = length(s2) + for (i = 0; i <= len1; i++) d[i,0] = i + for (j = 0; j <= len2; j++) d[0,j] = j + for (i = 1; i <= len1; i++) { + for (j = 1; j <= len2; j++) { + cost = (substr(s1, i, 1) == substr(s2, j, 1)) ? 0 : 1 + + del = d[i-1,j] + 1 + ins = d[i,j-1] + 1 + # Renamed variable to avoid conflict with the built-in "sub" function in awk. + substitution = d[i-1,j-1] + cost + + min = del + if (ins < min) min = ins + if (substitution < min) min = substitution + d[i,j] = min + } + } + distance=d[len1,len2] + if (distance == 0) { print s2; exit } # Found a match - should not happen + if (distance < current) { + best=s2 + current = distance + } + } + ' + +} + + +levenshtein() { + word1="$1" + word2="$2" + + # Calculates the distance and prints it to stdout. + awk -v s1="$word1" -v s2="$word2" ' + BEGIN { + len1 = length(s1) + len2 = length(s2) + for (i = 0; i <= len1; i++) d[i,0] = i + for (j = 0; j <= len2; j++) d[0,j] = j + for (i = 1; i <= len1; i++) { + for (j = 1; j <= len2; j++) { + cost = (substr(s1, i, 1) == substr(s2, j, 1)) ? 0 : 1 + + del = d[i-1,j] + 1 + ins = d[i,j-1] + 1 + # Renamed variable to avoid conflict with the built-in "sub" function in awk. + substitution = d[i-1,j-1] + cost + + min = del + if (ins < min) min = ins + if (substitution < min) min = substitution + d[i,j] = min + } + } + print d[len1,len2] + }' +} + +# Finds the closest match for a misspelled word from a list of valid words +findSuggestion() { + misspelled="$1" + dictionary="$2" + best_match="" + min_distance=4 + + for word in ${dictionary}; do + distance=$(levenshtein "${misspelled}" "${word}") + if [ "${distance}" -lt "${min_distance}" ]; then + min_distance=${distance} + best_match="${word}" + fi + done + echo "${best_match}" +} + # Main code From a1c8815aff4c66a7d7ecc14c0d947e6249ab371f Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 6 Oct 2025 21:15:59 +0100 Subject: [PATCH 130/179] Update packageDb when alternating versions --- bin/zopen-alt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/zopen-alt b/bin/zopen-alt index 3356ac1f9..08873ae0d 100755 --- a/bin/zopen-alt +++ b/bin/zopen-alt @@ -99,6 +99,9 @@ mergeNewVersion() if [ -e "${ZOPEN_PKGINSTALL}/${package}/${package}/.releaseinfo" ]; then version=$(cat "${ZOPEN_PKGINSTALL}/${package}/${package}/.releaseinfo") fi + printVerbose "Updating package DB" + updatePackageDB + syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE}" "ALT" "setAlt" "Set '${package}' to version:${version};" processActionScripts "installPost" "${package}" processActionScripts "transactionPost" From 6b06117130ff17dc112a92c984a2cecb445bccac Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 6 Oct 2025 21:17:06 +0100 Subject: [PATCH 131/179] Fix zopen X --version --- bin/zopen | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/zopen b/bin/zopen index d65466a3e..6df11ab9b 100755 --- a/bin/zopen +++ b/bin/zopen @@ -114,8 +114,12 @@ for arg in $*; do subcmd="zopen-${arg}-helper" ;; "--version") - subcmd='zopen-version' - version=true + if [ -z "${subcmd}" ]; then + subcmd='zopen-version' + version=true + else + subopts="${subopts} ${arg}" + fi ;; "help" | "--help" | "-?") help=true From 8ad4ddb0404d4818fcc58ee4a3221a917641494e Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 6 Oct 2025 21:20:20 +0100 Subject: [PATCH 132/179] Refix matadata.json parsing to handle GH API --- include/common.sh | 67 ++++++++++++++++------------------------------- 1 file changed, 22 insertions(+), 45 deletions(-) diff --git a/include/common.sh b/include/common.sh index 70e8b2d00..266736bcd 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1494,7 +1494,7 @@ syslog() } jqGetKey(){ - # If there key is not present, the error causes jq to exit with a + # If the key is not present, the error causes jq to exit with a # non-zero return code jq -er --arg key "$1" '.[$key] // error("Missing key: " + $key)' "$2" } @@ -1670,11 +1670,6 @@ getRepoReleases() return 1 fi repo="$1" - ##TODO - ##TDORM releases="$(jq -e -r '.release_data."'${repo}'"' "${JSON_CACHE}")" - ##TDORM if [ $? -ne 0 ]; then - ##TDORM printError "Could not get all releases for ${repo}" - ##TDORM fi } # Initializes a default environment for consistency in zopen builds @@ -2141,11 +2136,9 @@ addToInstallGraph(){ } # This uses the release json file to find whether any of the inputs are invalid -# package names. Ideally, this would use the keys of the release_data however, the -# NATS package breaks this as it is known as 'nats' in the repo but the port name is -# NATS. As such, use .product.name then fallback to the .tag as a sanity check -# Note: this is not to make the match "case-insensitive", but to use whichever the -# user uses as the name - if they know it as NATS, accept that! +# package names. Note that the Github API returns repository names in lowercase +# which potentially breaks packages whose key does not match their repo name +# (NATSport for example) - we need to ensure we use the reponame in lowercase. validatePackageList(){ installees="$1" # shellcheck disable=SC2086 # Using set -f disables globbing @@ -2156,9 +2149,7 @@ validatePackageList(){ invalidPortList=$(jq -r --argjson needles "[${installees%%,}]" \ '. as $data | $needles| map( select( . as $needle| ( ($data.release_data | keys_unsorted | any(. == $needle)) - or - ($data.release_data | to_entries[] | .value[] | .tag_name | (test("_[^_]*" + . + "[^_]*port")) - ) | not ) ) ) | unique | join(" ")' "${JSON_CACHE}" + ) | not ) ) | unique | join(" ")' "${JSON_CACHE}" ) if [ -n "${invalidPortList}" ]; then @@ -2446,7 +2437,7 @@ getInstallFile() [ -e "${downloadToDir}" ] || mkdir -p "${downloadToDir}" [ -w "${downloadToDir}" ] || printError "No permission to save install file to '${downloadToDir}'. Check permissions and retry command." printVerbose "Downloading installable file" - if ! runAndLog "cd ${downloadToDir} && curlCmd -L ${installurl} -O ${redirectToDevNull}"; then + if ! runAndLog "cd ${downloadToDir} && curlCmd -L ${installurl} -O >/dev/null 2>&1"; then printError "Could not download from ${installurl}. Correct any errors and potentially retry" fi fi @@ -2527,23 +2518,18 @@ installFromPax() # Ideally we would use the following, # name=$(jq --raw-output '.product.name' "${metadatafile}") # but name does not always map to the actual repo package name at present! - # The repo name is in the.product.repo field so can extract from there instead - - # though this also has issues for some packages like NATS/nats ... - # Note that at present some metadata might refer to the legacy repo ZOSOpenTools - # so fall back to that - if [ -n "${USEPRODNAME}" ]; then - printVerbose "Extracting product name from .product.name" - name=$(jq --raw-output '.product.name' "${metadatafile}") - else - printVerbose "Extracting product name from .product.repo" - name=$(jq --arg reponame "${ZOPEN_ORGNAME}" --raw-output '.product.repo | match(".*/\($reponame)/(.*)port").captures[0].string' "${metadatafile}") - if [ -z "${name}" ] || [ "${name##*[^ ]*}" = "" ]; then - name=$(jq --arg reponame "ZOSOpenTools" --raw-output '.product.repo | match(".*/\($reponame)/(.*)port").captures[0].string' "${metadatafile}") - fi - if [ -z "${name}" ] || [ "${name##*[^ ]*}" = "" ]; then - printError "Unable to determine name from .product.repo in '${metadatafile}'. Check metadata is correct." - fi + # The Github API also lowercases repository names when getting all + # repositories which causes issues for NATSport for example. As such, we + # can grab the name from .product.repo, lowercase and it should match the + # metadata + printVerbose "Extracting product name from .product.repo" + + name=$(jq --raw-output '.product.repo | capture(".+/(?[^/].+)(port)")| .name' \ + "${metadatafile}") + if [ -z "${name}" ] || [ "${name##*[^ ]*}" = "" ]; then + printError "Unable to determine name from .product.repo in '${metadatafile}'. Check metadata is correct." fi + name=$(awk -v n="${name}" 'BEGIN{print (tolower(n))}') if ! processActionScripts "installPre" "${name}" "${metadatafile}" "${pax}"; then @@ -2577,7 +2563,7 @@ installFromPax() fi # shellcheck disable=SC2154 - if ! runLogProgress "pax -rf ${pax} -p p ${paxredirect} ${redirectToDevNull}" \ + if ! runLogProgress "pax -rf ${pax} -p p ${paxredirect} >/dev/null 2>&1" \ "Expanding file: ${pax}" "Expanded file: ${pax}"; then printSoftError "Unexpected errors during unpaxing, package directory state unknown" printError "Use zopen alt to select previous version to ensure known state" @@ -2774,21 +2760,12 @@ updatePackageDB() if [ ! -e "${pdb}" ]; then echo "[]" > "${pdb}" fi - # Ideally, use $reponame in the match but jq seems to have issues with that! - mdj=$(jq '[{(.product.name):.}]' \ - "${escapedJSONFile}") - if [ -z "${mdj}" ]; then - # If we couldn't get the repo name from .product.name, use (.*)port - this might result - # in odd cases [NATS...] - mdj=$(jq '. as $metadata | .product.repo | match(".*/zopencommunity/(.*)port").captures[0].string | [{(.):$metadata}]' \ + # Grab the repo name from the metadata.json and lowercase it to match + # what the Github API returns when enumerating repsitories! + mdj=$(jq '(.product.repo | capture(".+/(?[^/].+)(port)")| .name | ascii_downcase) as $name + | [{($name):.}]' \ "${escapedJSONFile}") - fi - if [ -z "${mdj}" ]; then - # Try legacy repository [for really old packages!] - mdj=$(jq '. as $metadata | .product.repo | match(".*/ZOSOpenTools/(.*)port").captures[0].string | [{(.):$metadata}]' \ - "${escapedJSONFile}") - fi if [ -z "${mdj}" ]; then pkg=$(basename "${pkgdir}") From 2fd21fb4f2ad1c6610827209f181c1f320e5107b Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 6 Oct 2025 21:21:01 +0100 Subject: [PATCH 133/179] Bump version number --- include/zopen_version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/zopen_version b/include/zopen_version index b60d71966..ac39a106c 100644 --- a/include/zopen_version +++ b/include/zopen_version @@ -1 +1 @@ -0.8.4 +0.9.0 From f8b9fe982974d7506e3ad2c350d711d2010c775a Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 6 Oct 2025 21:23:42 +0100 Subject: [PATCH 134/179] Backup rather than destroy old etc/zopen-config --- bin/zopen-init | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index 3b8a4313f..20767a389 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -344,8 +344,9 @@ init() [ -n "${reinit}" ] && [ "y" = "${reinit}" ] }; then printInfo "- Re-initializing; zopen-config will be re-created" - if ! rm -rf "${configFile}"; then - printError "Unable to remove existing file. Check permissions and retry command." + backup="zopen-config.$(date +%Y%m%d%H%M%S)" + if ! cp "${configFile}" "${backup}"; then + printError "Unable to backup existing file. Check permissions and retry command." fi else From 4aeee75dacf791444877f055b513b2c51891ca0e Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Oct 2025 22:02:14 +0100 Subject: [PATCH 135/179] Refactor cache lookups --- bin/zopen-audit | 28 +------- bin/zopen-info | 8 ++- bin/zopen-init | 10 --- bin/zopen-install | 4 +- bin/zopen-list | 4 +- bin/zopen-query | 20 +++--- bin/zopen-upgrade | 2 +- bin/zopen-whichproject | 18 +---- include/common.sh | 155 +++++++++++++++++++---------------------- 9 files changed, 97 insertions(+), 152 deletions(-) diff --git a/bin/zopen-audit b/bin/zopen-audit index 0aea798f3..6c9bcc16e 100755 --- a/bin/zopen-audit +++ b/bin/zopen-audit @@ -84,31 +84,7 @@ if [ $upgrade = true ] && [ $remove = true ]; then exit 1 fi -JSON_VULNERABILITIES_URL="https://raw.githubusercontent.com/zopencommunity/meta/main/docs/api/zopen_vulnerability.json" -LATEST_RELEASES_URL="https://raw.githubusercontent.com/zopencommunity/meta/main/docs/api/zopen_releases_latest.json" - -downloadJsonCaches() -{ - cachedir="${ZOPEN_ROOTFS}/var/cache/zopen" - [ ! -e "${cachedir}" ] && mkdir -p "${cachedir}" - JSON_CVE_CACHE="${cachedir}/zopen_vulnerability.json" - - if ! curlout=$(curlCmd -L --fail --no-progress-meter -o "${JSON_CVE_CACHE}" "${JSON_VULNERABILITIES_URL}"); then - printError "Failed to obtain vulnerability json from ${JSON_VULNERABILITIES_URL}; ${curlout}" - fi - chtag -tc 819 "${JSON_CVE_CACHE}" - - if $upgrade; then - LATEST_RELEASES_CACHE="${cachedir}/zopen_releases_latest.json" - - if ! curlout=$(curlCmd -L --fail --no-progress-meter -o "${LATEST_RELEASES_CACHE}" "${LATEST_RELEASES_URL}"); then - printError "Failed to obtain latest releases json from ${LATEST_RELEASES_URL}; ${curlout}" - fi - chtag -tc 819 "${LATEST_RELEASES_CACHE}" - fi -} - -downloadJsonCaches +updateCaches if [ ! -f "${JSON_CVE_CACHE}" ]; then printError "Vulnerability json cache file not found." @@ -116,7 +92,7 @@ if [ ! -f "${JSON_CVE_CACHE}" ]; then fi printVerbose "Obtained vulnerability json cache." -if [ $upgrade = true ] && [ ! -f "${LATEST_RELEASES_CACHE}" ]; then +if [ $upgrade = true ] && [ ! -f "${JSON_LATEST_CACHE}" ]; then printError "Latest releases json cache file not found." fi printVerbose "Obtained latest releases json cache." diff --git a/bin/zopen-info b/bin/zopen-info index 865374c65..c2ba3a851 100755 --- a/bin/zopen-info +++ b/bin/zopen-info @@ -15,6 +15,7 @@ setupMyself() echo "Internal Error. Unable to find common.sh file to source." >&2 exit 8 fi + # shellcheck disable=SC1091 . "${INCDIR}/common.sh" } setupMyself @@ -286,6 +287,7 @@ while [ $# -gt 0 ]; do done checkIfConfigLoaded -getRepos true - -printPackageInfo "${package}" "${remote_lookup}" +updateCaches +validatePackageList "${package}" +gatherPackageInfo "${package}" "${remote_lookup}" +outputPackageInfo \ No newline at end of file diff --git a/bin/zopen-init b/bin/zopen-init index 20767a389..1e105cd88 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -203,16 +203,6 @@ while [ $# -gt 0 ]; do shift done # Wrapper around jq to provide extra help for certain scenarios -jqw() -{ - if ! type jq >/dev/null 2>&1; then - printSoftError "Cannot locate 'jq'." - printSoftError "If recovering a system, run . ./.env from the following location:" - printError " /usr/local/zopen/jq/ and retry command" - fi - jq "$@" -} - determineRootFileSystem() { if [ -n "${ZOPEN_ROOT_PATH}" ]; then diff --git a/bin/zopen-install b/bin/zopen-install index 7cc29ac84..7de84f03b 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -271,7 +271,7 @@ fi printDebug "Checking if installing from pax files: ${fileinstall}" if ! ${fileinstall}; then printVerbose "Querying metadata for latest package information" - getRepos + updateCaches grfgRc=$? [ 0 -ne ${grfgRc} ] && exit ${grfgRc}; fi @@ -295,7 +295,7 @@ else installList=$(jq --raw-output '.release_data| keys[]' "${JSON_CACHE}") installListCount=$(jq --raw-output '.release_data| keys | length' "${JSON_CACHE}") printInfo "- Installing all currently-uninstalled packages" - printInfo "- Checking installation status for '${installListCount}' packages" + printInfo "- Checking installation status for ${installListCount} package$([ "${installListCount}" -ne 1 ] && echo "s")" else installList=$(echo "$chosenRepos" | sed "s/@@/ /g") validatePackageList "${installList}" diff --git a/bin/zopen-list b/bin/zopen-list index 460824dcb..ca4668819 100755 --- a/bin/zopen-list +++ b/bin/zopen-list @@ -49,7 +49,7 @@ HELPDOC listUpdatesForInstalled() { - downloadJSONCache + updateCaches printInfo "Packages available for update:" pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" @@ -93,7 +93,7 @@ $upgrades | map(select(.)) | .[] listAvailablePackages() { - downloadJSONCache + updateCaches wpackage=20 wversion=23 winstalled=10 diff --git a/bin/zopen-query b/bin/zopen-query index a6b0ab14c..ed7f7c90d 100755 --- a/bin/zopen-query +++ b/bin/zopen-query @@ -100,7 +100,8 @@ printDetailListEntries() if [ ! -z "$1" ] && ! ${noheader}; then printf "${NC}${UNDERLINE}%-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s${NC}\n" "Package" "Installed" "Available" "Latest Tag" "Categories" fi - echo "${repoArray}" | xargs | tr ' ' '\n' | sort | while read repo; do + jqw -r '.release_data | keys[]' "${JSON_CACHE}" |\ + xargs | tr ' ' '\n' | sort | while read repo; do listport=false if [ -z "${needle}" ]; then listport=true @@ -145,7 +146,6 @@ printDetailListEntries() fi done else - printVerbose "Checking repoArray: ${repoArray}" scrcols=$(getScreenCols) numcols=7 colwidth=$((scrcols / numcols - 1)) @@ -153,7 +153,8 @@ printDetailListEntries() if ! ${noheader}; then printf "${NC}${UNDERLINE}%-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s %-${colwidth}s${NC}\n" "Package" "Installed" "Latest Tag" "Download Size" "Expanded Size" "Quality" "Categories" fi - echo "${repoArray}" | xargs | tr ' ' '\n' | sort | while read repo; do + jqw -r '.release_data | keys[]' "${JSON_CACHE}" | \ + xargs | tr ' ' '\n' | sort | while read repo; do listport=false if [ -z "${needle}" ]; then listport=true @@ -264,19 +265,19 @@ while [ $# -gt 0 ]; do ;; "-i" | "--installed") localoption=true - installed=1 + installed=true list=true ;; "-wp" | "--whatprovides") localoption=true - whatprovides=1 + whatprovides=true shift [ -n "$1" ] || printError "Missing file argument" needle=$1 ;; "--remote-search") localoption=false - remotesearch=1 + remotesearch=true shift [ -n "$1" ] || printError "Missing package argument" needle=$1 @@ -346,10 +347,9 @@ fi if ! ${localoption}; then # Retrieve all repositories - getRepos + updateCaches grfgRc=$? [ 0 -ne "${grfgRc}" ] && exit "${grfgRc}" - repoArray="${repo_results}" fi # Use dedicated zopen-list script for listing system packages, passing @@ -366,6 +366,6 @@ if ${list}; then fi else ! ${upgradeable} || printDetailListEntries "${details}" "" 1 - [ -z "${remotesearch}" ] || printDetailListEntries "${details}" "${needle}" 0 - [ -z "${whatprovides}" ] || whatProvides "${needle}" + ! ${remotesearch} || printDetailListEntries "${details}" "${needle}" 0 + ! ${whatprovides} || whatProvides "${needle}" fi \ No newline at end of file diff --git a/bin/zopen-upgrade b/bin/zopen-upgrade index fc1f4a356..89536f9ed 100755 --- a/bin/zopen-upgrade +++ b/bin/zopen-upgrade @@ -131,7 +131,7 @@ fi # Parse passed in repositories and check if valid zopen framework repos printVerbose "Querying remote repo for latest package information" -getRepos +updateCaches grfgRc=$? [ 0 -ne ${grfgRc} ] && exit ${grfgRc}; # shellcheck disable=SC2034 diff --git a/bin/zopen-whichproject b/bin/zopen-whichproject index 53bef4616..acb353bbb 100755 --- a/bin/zopen-whichproject +++ b/bin/zopen-whichproject @@ -67,27 +67,13 @@ if ! command -v jq > /dev/null 2>&1; then exit 1 fi -JSON_FILES_URL="https://raw.githubusercontent.com/zopencommunity/meta/main/docs/api/zopen_files.json" -JSON_FILES_URL="https://raw.githubusercontent.com/zopencommunity/meta/e859e89e75fab31c1d5b4764b9ee9f511c52b682/docs/api/zopen_files.json" - -downloadJsonCaches() -{ - cachedir="${ZOPEN_ROOTFS}/var/cache/zopen" - [ ! -e "${cachedir}" ] && mkdir -p "${cachedir}" - JSON_FILES_CACHE="${cachedir}/zopen_files.json" - - if ! curlout=$(curlCmd -L --fail --no-progress-meter -o "${JSON_FILES_CACHE}" "${JSON_FILES_URL}"); then - printError "Failed to obtain vulnerability json from ${JSON_FILES_URL}; ${curlout}" - fi - chtag -tc 819 "${JSON_FILES_CACHE}" -} - -downloadJsonCaches +updateCaches "JSON_FILES_CACHE" # Perform the search using jq printVerbose "Searching for '${query_item}' in ${JSON_FILES_CACHE}..." # The jq query to find the project key(s) containing the item +# shellcheck disable=SC2016 jq_query='.project_files | to_entries[] | select(.value.binaries[]? == $item or .value.libs[]? == $item) | .key' # Execute jq, passing the query item as an argument diff --git a/include/common.sh b/include/common.sh index 266736bcd..c3a87bdc9 100755 --- a/include/common.sh +++ b/include/common.sh @@ -41,7 +41,13 @@ zopenInitialize() ZOPEN_JSON_CACHE_URL="https://raw.githubusercontent.com/${ZOPEN_ORGNAME}/meta/main/docs/api/zopen_releases.json" # shellcheck disable=SC2034 ZOPEN_LATEST_RELEASE_JSON="https://raw.githubusercontent.com/${ZOPEN_ORGNAME}/meta/main/docs/api/zopen_releases_latest.json" + # shellcheck disable=SC2034 + JSON_FILES_CACHE_URL="https://raw.githubusercontent.com/zopencommunity/meta/main/docs/api/zopen_files.json" ZOPEN_JSON_CONFIG="${ZOPEN_ROOTFS}/etc/zopen/config.json" + ZOPEN_CACHEDURL_DIR="${ZOPEN_ROOTFS}/var/cache/zopen" + ZOPEN_REPOS_DIR="${ZOPEN_ROOTFS}/etc/zopen/repos.d" + + if [ -n "${INCDIR}" ]; then ZOPEN_SCRIPTLET_DIR="${INCDIR}/scriptlets" else @@ -1516,61 +1522,59 @@ validateReposDEntry(){ done } -getJSONCacheURLs(){ - reposdDir="${ZOPEN_ROOTFS}/etc/zopen/repos.d" - activeRepo="${reposdDir}/active" +# buildCacheURL +# Given a cache filename, generate the appropriate URL to retrieve +# the latest cache data from, taking into account the different repo types +# inputs: cacheFileName - the name of the cache to update +# stdout: the url to check for the latest cache +# return: 0 success +# 1 failure +buildCacheURL(){ + cacheFileName=$1 + activeRepo="${ZOPEN_REPOS_DIR}/active" dereffedLink=$(deref_symlink "${activeRepo}") [ ! -e "${dereffedLink}" ] && printError "Could not access linked repository configuration at '${dereffedLink}'. Check file to ensure valid repository configuration or refresh default configuration with zopen init --refresh -y." if ! validateReposDEntry "${dereffedLink}"; then return 1 fi - type=$(jqGetKey "type" "${dereffedLink}") - base=$(jqGetKey "metadata_baseurl" "${dereffedLink}") - filename=$(jqGetKey "metadata_file" "${dereffedLink}") - latest_metadata=$(jqGetKey "latest_file" "${dereffedLink}") - + # Failures in the following indicate the configuration file isn't setup correctly + type=$(jqGetKey "type" "${dereffedLink}") || printError "Unrecognised key 'type' in configuration file." + base=$(jqGetKey "metadata_baseurl" "${dereffedLink}") || printError "Unrecognised key 'metadata_baseurl' in configuration file." + case "${type}" in - http|https) jsonCacheURLs=$(printf "%s://%s/%s\n%s://%s/%s" \ - "${type}" "${base}" "${filename}" "${type}" "${base}" "${latest_metadata}") + http|https) printf "%s://%s/%s\n" "${type}" "${base}" "${cacheFileName}" ;; - file) jsonCacheURLs=$(printf "%s:%s/%s\n%s:%s/%s" \ - "${type}" "${base}" "${filename}" "${type}" "${base}" "${latest_metadata}") + file) printf "%s:%s/%s\n" "${type}" "${base}" "${cacheFileName}" ;; - *) printError "Unsupported repository type '${type}'.";; + *) printError "Unsupported repository type '${type}'.";; esac } -updateJSONCaches() +# shellcheck disable=SC2120 +updateCaches() { - if [ -n "${JSON_CACHE}" ]; then - printVerbose "Cache already downloaded/checked during this session" - return 0 - fi - - from_readonly="$1" - if [ -n "${from_readonly}" ]; then - if [ -r "${JSON_CACHE}" ] && [ ! -w "${JSON_CACHE}" ]; then - return; # Skip the download for read only operations when you know you can't write to it + if [ $# -gt 0 ]; then + caches="$1" + else + caches="JSON_CACHE JSON_LATEST_CACHE JSON_CVE_CACHE JSON_FILES_CACHE" + fi + for cache in $caches; do + printVerbose "Checking if the $cache already downloaded in this session" + eval "value=\${$cache}" + if [ -z "${value}" ]; then + case "$cache" in + JSON_CACHE) cacheFile="zopen_releases.json" ;; + JSON_LATEST_CACHE) cacheFile="zopen_releases_latest.json" ;; + JSON_CVE_CACHE) cacheFile="zopen_vulnerability.json" ;; + JSON_FILES_CACHE) cacheFile="zopen_files.json" ;; + *) assertFailed "Invalid cache variable specified: $cache" + esac + fqCacheFile="${ZOPEN_CACHEDURL_DIR}/${cacheFile}" + eval "${cache}=${fqCacheFile}" + url=$(buildCacheURL "${cacheFile}") || printError "Unable to build url to update release data cache" + downloadJSONCacheIfExpired "${fqCacheFile}" "${url}" fi - fi - - printVerbose "Ensuring cache directory exists" - cachedir="${ZOPEN_ROOTFS}/var/cache/zopen" - [ ! -e "${cachedir}" ] && mkdir -p "${cachedir}" - if ! getJSONCacheURLs; then - return 1 - fi - - printVerbose "Checking if the JSON_CACHE already downloaded in this session" - if [ -z "${JSON_CACHE}" ]; then - JSON_CACHE="${cachedir}/zopen_releases.json" - downloadJSONCacheIfExpired "${JSON_CACHE}" "$(echo "${jsonCacheURLs}" | tail -n 2 | head -n 1)" - fi - if [ -z "${JSON_LATEST_CACHE}" ]; then - JSON_LATEST_CACHE="${cachedir}/zopen_releases_latest.json" - latestReleaseURL="$(dirname "${jsonCacheURL}")/${latest_metadata}" - downloadJSONCacheIfExpired "${JSON_LATEST_CACHE}" "$(echo "${jsonCacheURLs}" | tail -n 1)" - fi + done } downloadJSONCacheIfExpired() @@ -1580,6 +1584,17 @@ downloadJSONCacheIfExpired() cacheTimestamp="${fileToCache}.timestamp" cacheTimestampCurrent="${fileToCache}.timestamp.current" + printVerbose "Ensuring cache directory exists" + if [ ! -e "${ZOPEN_CACHEDURL_DIR}" ]; then + mkdir -p "${ZOPEN_CACHEDURL_DIR}" || printError "Cannot create cache directory for metadata at '${cachedir}'. Check permissions and retry." + fi + + if [ -r "${fileToCache}" ] && [ ! -w "${fileToCache}" ]; then + # Skip the download for read only operations when you know you can't write to it + printWarning "Unable to write to cache at '${fileToCache}'; using stale cache data" + return; + fi + # Need to check that we can read & write to the JSON timestamp cache files if [ -e "${cacheTimestampCurrent}" ]; then [ ! -w "${cacheTimestampCurrent}" ] || [ ! -r "${cacheTimestampCurrent}" ] && printError "Cannot access cache at '${cacheTimestampCurrent}'. Check permissions and retry request." @@ -1594,7 +1609,10 @@ downloadJSONCacheIfExpired() if ! curlCmd --fail --location --silent --head "${cacheUrl}" -o "${cacheTimestampCurrent}"; then printError "Failed to obtain json cache timestamp from ${cacheUrl}." fi - chtag -tc 819 "${cacheTimestampCurrent}" + if command -v chtag >/dev/null 2>&1; then + printVerbose "Issuing chtag command" + chtag -tc 819 "${cacheTimestampCurrent}" + fi if [ -f "${fileToCache}" ] && [ -f "${cacheTimestamp}" ]; then # Extract eyecatchers - either ETag or Last-Modified depending on repo type @@ -1624,30 +1642,6 @@ downloadJSONCacheIfExpired() fi } -downloadJSONCache() -{ - if ! updateJSONCaches "$1"; then - return 1 - else - return 0 - fi -} - -# getRepos -# Queries the main repository list to obtain a list of all available port -# names, populating the repo_results global var -# inputs: none -# return: 0 success -# 1 failure -getRepos() -{ - if ! updateJSONCaches "$1"; then - return 1 - fi - # shellcheck disable=SC2034 - repo_results="$(jq -r '.release_data | keys[]' "${JSON_CACHE}")" -} - # isValidRepo # Queries the main repository list to determine if the input is valid, This # uses jq itself to return 0 or 1 with no output @@ -1657,21 +1651,10 @@ getRepos() # 1 invalid port name isValidRepo() { - if ! updateJSONCaches; then - return 1 - fi + updateCaches jq -r --arg needle "$1" 'if .release_data | has($needle) then empty else error("") end' "${JSON_CACHE}" > /dev/null 2>&1 } -#Deprecated -getRepoReleases() -{ - if ! updateJSONCaches; then - return 1 - fi - repo="$1" -} - # Initializes a default environment for consistency in zopen builds initDefaultEnvironment() { @@ -2037,9 +2020,7 @@ getPortMetaData(){ fi parseRepoName "${portRequested}" # To set the various status flags below - if ! getRepoReleases "${portName}"; then - return 1 - fi + updateCaches if [ -n "${versioned}" ]; then if ! getVersionedMetadata "${portName}" "${invalidPortAssetFile}"; then return 1 @@ -2940,6 +2921,16 @@ isTrue() esac } +jqw() +{ + if ! type jq >/dev/null 2>&1; then + printSoftError "Cannot locate 'jq'." + printSoftError "If recovering a system, run . ./.env from the following location:" + printError " /usr/local/zopen/jq/ and retry command" + fi + jq "$@" +} + # Lookup the list of ports and find the closest suggestion. This uses # awk to parse the dictionary directly from the cache file [piped via jq] toolSuggestion(){ From 7fb1f8a358a0ff9b2db19773c79c952876b6e587 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Oct 2025 22:02:59 +0100 Subject: [PATCH 136/179] Use jqw wrapper in audit script --- bin/zopen-audit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/zopen-audit b/bin/zopen-audit index 6c9bcc16e..b14a2a5ec 100755 --- a/bin/zopen-audit +++ b/bin/zopen-audit @@ -166,7 +166,7 @@ while IFS= read -r repo; do latest_release_vulns="" is_latest_release=true if $upgrade; then - latest_release=$(jq -cr '.release_data | .["'$repo'"] | .[0] | .assets | .[0] | .release' $LATEST_RELEASES_CACHE) + latest_release=$(jqw -cr '.release_data | .["'$repo'"] | .[0] | .assets | .[0] | .release' $JSON_LATEST_CACHE) if [ "$release" != "$latest_release" ]; then is_latest_release=false From 31d00d9e4950d6ff22ec9f8d2f67f59602bd68a8 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Oct 2025 22:04:10 +0100 Subject: [PATCH 137/179] Write debug/verbose to stderr --- include/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/common.sh b/include/common.sh index c3a87bdc9..1f9f3d398 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1064,7 +1064,7 @@ printDebug() [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" # shellcheck disable=SC2154 if ${debug}; then - printColors "${NC}${BLUE}${BOLD}:DEBUG:${NC}: '${1}'" + printColors "${NC}${BLUE}${BOLD}:DEBUG:${NC}: '${1}'" >&2 fi [ -n "${xtrc}" ] && set -x return 0 @@ -1075,7 +1075,7 @@ printVerbose() [ -z "${-%%*x*}" ] && set +x && xtrc="-x" || xtrc="" # shellcheck disable=SC2154 if ${verbose}; then - printColors "${NC}${GREEN}${BOLD}VERBOSE${NC}: ${1}" + printColors "${NC}${GREEN}${BOLD}VERBOSE${NC}: ${1}" >&2 fi [ -n "${xtrc}" ] && set -x return 0 From e3ad6cdb3f4cb2121e03667728d5d30739d7064a Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Oct 2025 22:04:38 +0100 Subject: [PATCH 138/179] Give progresshandler time to start [prevent line overwrites] --- include/common.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/include/common.sh b/include/common.sh index 1f9f3d398..7e94e8564 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1123,6 +1123,7 @@ runLogProgress() PROGRESS_HANDLER=$! killph="kill -HUP ${PROGRESS_HANDLER}" addCleanupTrapCmd "${killph}" + sleep 1 # Give the handler time to wakeup! eval "$1" rc=$? ${killph} >/dev/null 2>&1 # if the timer is not running, the kill will fail From ee035b13442e1ef280bd349370f3fd38816d614f Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Oct 2025 22:05:45 +0100 Subject: [PATCH 139/179] Writing syslog on read-only install during promote fails --- include/common.sh | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/include/common.sh b/include/common.sh index 7e94e8564..e7cc0d11f 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1486,16 +1486,24 @@ syslog() module=$4 # zopen- location=$5 # function msg=$6 # Message text - if [ ! -e "${fd}" ]; then - mkdir -p "$(dirname "${fd}")" - touch "${fd}" - fi + parent=$(dirname "${fd}") + output="$(date +"%F %T") $(id | cut -d' ' -f1)::${module}:${type}:${categories}:${location}:${msg}" - if [ -w "${fd}" ]; then - echo "${output}" >> "${fd}" - else - printWarning "No write permission to log file '${fd}'; writing to stderr" + if [ ! -d "${parent}" ]; then + mkdir -p "${parent}" 2>/dev/null || { + printWarning "Cannot create log directory '${parent}'; writing syslog to stderr" + echo "${output}" >&2 + } + elif [ ! -e "${fd}" ]; then + touch "${fd}" 2>/dev/null || { + printWarning "Cannot create log file '${fd}'; writing syslog to stderr" + echo "${output}" >&2 + } + elif [ ! -w "${fd}" ]; then + printWarning "No write permission to log file '${fd}'; writing syslog to stderr" echo "${output}" >&2 + else + echo "${output}" >> "${fd}" fi } From 2674ab5245a0123ec2fc371506b5ae673ab0ec3a Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Oct 2025 22:06:57 +0100 Subject: [PATCH 140/179] clearer messages on non-writable ~/.gnupg --- include/common.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/include/common.sh b/include/common.sh index e7cc0d11f..2d1f0dd1c 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1793,19 +1793,22 @@ startGPGAgent() { printWarning "Found orphaned GPG agent socket at ${SOCKET_PATH}; removing and attempting restart of gpg-agent." rm -f "${SOCKET_PATH}" fi + else + printWarning "gpg socket at '$SOCKET_PATH' cannot be read" fi if eval "$(gpg-agent --daemon --disable-scdaemon)" >/dev/null 2>&1; then if [ -r "$SOCKET_PATH" ]; then printVerbose "gpg-agent started successfully (socket created at $SOCKET_PATH)." else - printWarning "gpg-agent started, but socket was not created at $SOCKET_PATH. Please verify your GPG installation." + printWarning "gpg-agent start initiated, but socket was not created at $SOCKET_PATH. Check file permissions for directory '$(dirname "$SOCKET_PATH")' and files contained" + printWarning "gpg-agent daemon might not complete initialisation" fi else if [ -r "$SOCKET_PATH" ]; then printWarning "gpg-agent started successfully (socket created at $SOCKET_PATH), but gpg-agent returned a non-zero return code." else - printError "Failed to start gpg-agent. Reinstall or upgrade GPG using \"zopen install --reinstall gpg -y\" or \"zopen upgrade gpg -y\"." + printError "Failed to start gpg-agent. Check file permissions for directory '$(dirname "$SOCKET_PATH")' and files contained." fi fi } From 51918f9f8e30f9505c632244f3303f8afe3c9945 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Oct 2025 22:07:54 +0100 Subject: [PATCH 141/179] Warning message tweaks for clarity --- include/common.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/common.sh b/include/common.sh index 2d1f0dd1c..259530f4f 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2146,8 +2146,6 @@ validatePackageList(){ ) if [ -n "${invalidPortList}" ]; then - printSoftError "The following port names were not recognised:" - # Calculate word lengths for nicer formatting set -- ${invalidPortList} minlen=1 @@ -2155,10 +2153,12 @@ validatePackageList(){ wordlen=${#toolrepo} minlen=$(( wordlen > minlen ? wordlen : minlen )) done + # Respin loop actually checking for suggestions set -- ${invalidPortList} + printSoftError "The following port name$( [ $# -eq 1 ] && echo " was" || echo "s were") not recognised:" for toolrepo in "$@"; do - printDebug "Finding suggestions for '${toolrepo}'[${#toolrepo}]" + printVerbose "Finding suggestions for '${toolrepo}'[${#toolrepo}]" suggestion=$(toolSuggestion "${toolrepo}") # Use printf to print without the printSoftError prefix for cleaner output if [ -n "${suggestion}" ]; then From c681a2950d901a39bb00bc35764e97150da3df34 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Oct 2025 22:08:44 +0100 Subject: [PATCH 142/179] More robust space validation messages --- include/common.sh | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/include/common.sh b/include/common.sh index 259530f4f..c4b3fc1fa 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2286,7 +2286,13 @@ checkIfPrereq(){ "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" } - +# spaceValidate +# Ensures there is sufficient available disk space for an operation to continue. +# Note that auto-expanding Filesystems might display the warning, even if an +# expansion is possible; as such, allow the user to continue. Caveat consumptor... +# Input: $1 - cacheBytes: number of bytes required for caching packages +# $2 - packageBytes: number of bytes require for uncompressing package +# stdout: Formatted file size to 3 decimal places with TGMk suffix spaceValidate(){ cacheBytes=$1 packageBytes=$2 @@ -2297,6 +2303,10 @@ spaceValidate(){ return fi + cacheBytesKb=$(echo "scale=0; ${cacheBytes} / 1024" | bc) + packageBytesKb=$(echo "scale=0; ${packageBytes} / 1024" | bc) + spaceRequiredKb=$(echo "scale=0; ${cacheBytesKb} + ${packageBytesKb}" | bc) + if ${reinstall}; then # During a reinstall, the existing package size should remain constant as the # package should overwrite the existing with the same files. However @@ -2304,28 +2314,28 @@ spaceValidate(){ # autocacheclean is active, then this should be temporary; if not, then the # cache will grow - inform the user either way printVerbose "Reinstall of package, so package file size delta should be 0!" - spaceRequiredMB=$(echo "scale=0; (${cacheBytes}) / (1024 * 1024)" | bc) - spaceRequiredKb=$(( cacheBytes / 1024 )) + spaceRequiredKb=$(echo "scale=0; (${cacheBytes}) / 1024" | bc) if ! isCacheClean=$(zopen config --get autocacheclean); then printError "Could not determine autocacheclean status" fi if ${isCacheClean}; then - printInfo "During this operation, ${spaceRequiredMB} MB of disk space will be used." + printInfo "During this operation, $(formattedFileSize "${spaceRequiredKb}") of disk space will be used." else - printInfo "After this operation, ${spaceRequiredMB} MB of additional cache will be used." + printInfo "After this operation, $(formattedFileSize "${spaceRequiredKb}") of additional cache will be used." fi elif ${downloadOnly}; then # To download to current dir, then only need to make sure there is enough disk space # for the pax, the "cached" size - spaceRequiredMB=$(echo "scale=0; (${cacheBytes}) / (1024 * 1024)" | bc) - spaceRequiredKb=$(( cacheBytes / 1024 )) - printInfo "After this operation, $(formattedFileSize ${spaceRequiredKb}) of additional disk space will be used." + spaceRequiredKb=$(echo "scale=0; (${cacheBytes}) / 1024" | bc) + printInfo "After this operation, $(formattedFileSize "${spaceRequiredKb}") of additional disk space will be used." else - # If not a reinstall, assume there is a need for both the package and the expanded - # package to be required - spaceRequiredKb=$(( (cacheBytes + packageBytes) / 1024 )) - spaceRequiredMB=$(echo "scale=0; (${cacheBytes} + ${packageBytes}) / (1024 * 1024)" | bc) - printInfo "After this operation, $(formattedFileSize ${spaceRequiredKb}) of additional disk space will be used." + # If not a reinstall, there is a space requirement for both the + # package and the expanded package during install; if autocache clean is on, + # some of that will be released + if ${isCacheClean}; then + printInfo "During this operation, up to $(formattedFileSize "${spaceRequiredKb}") of disk space will be required." + fi + printInfo "After this operation, $(formattedFileSize "${packageBytesKb}") of disk space will be used." fi availableSpaceKb=$(df -k "${ZOPEN_ROOTFS}" | sed "1d" | awk '{ print $3 }' | awk -F'/' '{ print $1 }') if [ "${availableSpaceKb}" -lt "${spaceRequiredKb}" ]; then From 0fbc0f15dced548be4c442da5e583eb545fc9eb7 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Oct 2025 22:09:13 +0100 Subject: [PATCH 143/179] Message tweak for older package (or git installed like meta) --- include/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common.sh b/include/common.sh index c4b3fc1fa..904e53e9e 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2754,7 +2754,7 @@ updatePackageDB() metadataFile="${ZOPEN_PKGINSTALL}/${pkgdir}/metadata.json" if [ ! -e "${metadataFile}" ]; then # TODO: Fallback to filesystem analysis [depending on backward compatability] - printWarning "No metadata.json found in '${ZOPEN_PKGINSTALL}/${pkgdir}'. Old package install?" + printWarning "No metadata.json found in '${ZOPEN_PKGINSTALL}/${pkgdir}' [recent install/old package?]" continue fi escapedJSONFile=$(mktempfile "escaped" "json") From 19b8cf51925f86b320070961cf9b946753c8723d Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Oct 2025 22:12:52 +0100 Subject: [PATCH 144/179] Message tweaks; variable definiing in zopen-config --- include/common.sh | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/include/common.sh b/include/common.sh index 904e53e9e..7f471dace 100755 --- a/include/common.sh +++ b/include/common.sh @@ -307,7 +307,7 @@ displayHelp() { echo "usage: . zopen-config [--eknv] [--knv] [--quiet] [-?|--help]" echo " --override-zos-tools Adds altbin/ dir to the PATH and altman/ dir to MANPATH, overriding the native z/OS tooling." echo " --nooverride-zos-tools Does not add altbin/ and altman/ dir to PATH and MANPATH." -echo " --override-zos-tools-subset=" +echo " --override-zos-tools-subset " echo " Override a subset of zos tools. Containing a subset of packages to override, delimited by newlines." echo " --knv Display zopen environment variables " echo " --eknv Display zopen environment variables, prefixed with an" @@ -346,11 +346,8 @@ for local_tmp in "\${TMPDIR}" "\${TMP}" /tmp; do done if \${knv}; then - /bin/env | /bin/sort > \${local_tmp}/zopen-config-env-orig.\$\$ -fi - -if [ -n "\${overrideFile}" ] && [ ! -f "\${overrideFile}" ]; then - echo "Override file '\${overrideFile}' is not a file. Skipping..." + original_env="\${local_tmp}/zopen-config-env-orig."\$\$ + /bin/env | /bin/sort > \${original_env} fi # Main root location for the zopen installation; can be changed if the @@ -448,7 +445,7 @@ if [ -z "\${ZOPEN_QUICK_LOAD}" ]; then if [ -d "\${ZOPEN_ROOTFS}/etc/profiled" ]; then if \$displayText; then - /bin/printf "Processing \$zot configuration..." + /bin/printf "Processing %s configuration..." "\$zot" fi tmpfile="\${local_tmp}/zopen_config.tmp" @@ -460,7 +457,7 @@ if [ -z "\${ZOPEN_QUICK_LOAD}" ]; then else echo "Error: find command failed" >&2 [ -e "\${tmpfile}" ] && rm "\${tmpfile}" - exit 1 + return 1 fi [ -e "\${tmpfile}" ] && rm "\${tmpfile}" @@ -469,7 +466,7 @@ if [ -z "\${ZOPEN_QUICK_LOAD}" ]; then if [ -n "\${ZOPEN_TOOLSET_OVERRIDE}" ]; then /bin/echo "NOTE: Conflicting tools (eg. man, cat, grep, make) will take precedence over z/OS /bin tools. Pass the option --nooverride-zos-tools to avoid this." else - /bin/echo "NOTE: Conflicting tools (eg. man, cat, grep, make) will NOT take precedence over z/OS /bin tools; Use the prefixed executables instead (eg. zotman, gcat, ggrep, gmake). Pass the option --override-zos-tools if you prefer zopen tools or --help for further options." + /bin/echo "NOTE: Conflicting tools (eg. man, cat, grep, make) will NOT take precedence over z/OS /bin tools; use the prefixed executables instead (eg. zotman, gcat, ggrep, gmake). Pass the option --override-zos-tools if you prefer zopen tools or --help for further options." fi fi unset dotenvs @@ -479,15 +476,15 @@ unset displayText PATH=\${ZOPEN_ROOTFS}/usr/local/bin:\${ZOPEN_ROOTFS}/usr/bin:\${ZOPEN_ROOTFS}/bin:\${ZOPEN_ROOTFS}/boot:\$(zot_sanitizeEnvVar "\${PATH}" ":" "^\${ZOPEN_PKGINSTALL}/.*\$") MANPATH=\${ZOPEN_ROOTFS}/usr/local/share/man:\${ZOPEN_ROOTFS}/usr/local/share/man/\%L:\${ZOPEN_ROOTFS}/usr/share/man:\${ZOPEN_ROOTFS}/usr/share/man/\%L:\$(zot_sanitizeEnvVar "\${MANPATH}" ":" "^\${ZOPEN_PKGINSTALL}/.*\$") -if [ -n "\$ZOPEN_TOOLSET_OVERRIDE" ]; then +if [ -n "\${ZOPEN_TOOLSET_OVERRIDE}" ]; then if [ -n "\${overrideFile}" ] && [ -f "\${overrideFile}" ]; then PATH=\$(zot_sanitizeEnvVar "\${PATH}" ":" "^\${ZOPEN_ROOTFS}/usr/local/altbin.*\$") while IFS= read -r project; do - if [ -d "\$ZOPEN_PKGINSTALL/\$project/\$project/altbin" ]; then - PATH="\$ZOPEN_PKGINSTALL/\$project/\$project/altbin:\$PATH" + if [ -d "\${ZOPEN_PKGINSTALL}/\${project}/\${project}/altbin" ]; then + PATH="\${ZOPEN_PKGINSTALL}/\${project}/\${project}/altbin:\$PATH" fi - if [ -d "\$ZOPEN_PKGINSTALL/\$project/\$project/share/altman" ]; then - MANPATH="\$ZOPEN_PKGINSTALL/\$project/\$project/share/altman:\$MANPATH" + if [ -d "\${ZOPEN_PKGINSTALL}/\${project}/\${project}/share/altman" ]; then + MANPATH="\${ZOPEN_PKGINSTALL}/\${project}/\${project}/share/altman:\$MANPATH" fi done < "\${overrideFile}" else @@ -506,8 +503,9 @@ export LIBPATH=\$(zot_deleteDuplicateEntries "\${LIBPATH}" ":") export MANPATH=\$(zot_deleteDuplicateEntries "\${MANPATH}" ":") if \${knv}; then - /bin/env | /bin/sort > \${local_tmp}/zopen-config-env-modded.\$\$ - diffout=\$(/bin/diff \${local_tmp}/zopen-config-env-orig.\$\$ \${local_tmp}/zopen-config-env-modded.\$\$ | /bin/grep -E '^[>]' | /bin/cut -c3- ) + modified_env="\${local_tmp}/zopen-config-env-modded."\$\$ + /bin/env | /bin/sort > \${modified_env} + diffout=\$(/bin/diff \${original_env} \${modified_env} | /bin/grep -E '^>' | /bin/cut -c3- ) echo "\${diffout}" | while IFS= read -r knvp; do newval="" envvar="\${knvp%%=*}" @@ -520,7 +518,7 @@ if \${knv}; then echo "\${exportknv}\${envvar}=\${newval#*:}" IFS="\${cIFS}" done - rm \${local_tmp}/zopen-config-env-orig.\$\$ \${local_tmp}/zopen-config-env-modded.\$\$ 2>/dev/null + rm \${original_env} \${modified_env} 2>/dev/null fi # Cleanup From 35189ee6503d28e64784c4fea29f1aa843bdcfc9 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Oct 2025 22:14:04 +0100 Subject: [PATCH 145/179] Revert zot_santizeEnvvar; fix zot_deleteDuplicateEntries --- include/common.sh | 36 ++++++------------------------------ 1 file changed, 6 insertions(+), 30 deletions(-) diff --git a/include/common.sh b/include/common.sh index 7f471dace..e59b9e3b9 100755 --- a/include/common.sh +++ b/include/common.sh @@ -372,36 +372,12 @@ zot="zopen community" zot_sanitizeEnvVar() { - # remove any envvar entries that match the specified prefix, in a shell-agnostic - # manner + # remove any envvar entries that match the specified regex value="\$1" delim="\$2" prefix="\$3" - if [ -z "\${value}" ]; then - return - fi - result="" - oldIFS="\${IFS}" - IFS="\${delim}" - for entry in \${value}; do - case "\${entry}" in - "\${prefix}"/*) - # Skip entries that start with the prefix followed by / - ;; - *) - # Keep entries that don't match - if [ -z "\${result}" ]; then - # First time through - result="\${entry}" - else - # At least one previous result; insert delimiter - result="\${result}\${delim}\${entry}" - fi - ;; - esac - done - IFS="\${oldIFS}" - /bin/echo "\${result}" + echo "\${value}" | /bin/awk -v RS="\${delim}" -v DLIM="\${delim}" -v PRFX="\${prefix}" '{ if (match(\$1, PRFX)==0) {printf("%s%s",\$1,DLIM)}}' + } zot_deleteDuplicateEntries() @@ -413,13 +389,13 @@ zot_deleteDuplicateEntries() return fi - echo "\${value}\${delim}" | awk -v RS="\${delim}" -v ORS="\${delim}" ' + echo "\${value}\${delim}" | /bin/awk -v RS="\${delim}" -v ORS="\${delim}" ' BEGIN {col=""} !(\$0 in a) { - a[\$0] = 1; + a[\$0]++; printf("%s%s", col, \$0); col=ORS; - }' | /bin/sed "s/\${delim}$//" + }' | /bin/sed "s/\${delim}//" } # zopen community environment variables From 96a20a48d9211731a1b85a426fae52769dbf10b0 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Oct 2025 22:14:45 +0100 Subject: [PATCH 146/179] Move zopen-config arg parse into function --- include/common.sh | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/include/common.sh b/include/common.sh index e59b9e3b9..76e953949 100755 --- a/include/common.sh +++ b/include/common.sh @@ -303,7 +303,7 @@ writeConfigFile(){ #!/bin/false # Script currently intended to be sourced, not run # zopen community Configuration file -displayHelp() { +zot_displayHelp() { echo "usage: . zopen-config [--eknv] [--knv] [--quiet] [-?|--help]" echo " --override-zos-tools Adds altbin/ dir to the PATH and altman/ dir to MANPATH, overriding the native z/OS tooling." echo " --nooverride-zos-tools Does not add altbin/ and altman/ dir to PATH and MANPATH." @@ -328,18 +328,33 @@ knv=false exportknv="" displayText=true unset overrideFile -while [ \$# -gt 0 ]; do - case "\$1" in - --eknv) exportknv="export "; knv=true;; - --knv) knv=true;; - --override-zos-tools) export ZOPEN_TOOLSET_OVERRIDE=1;; - --nooverride-zos-tools) unset ZOPEN_TOOLSET_OVERRIDE;; - --override-zos-tools-subset) shift; export ZOPEN_TOOLSET_OVERRIDE=1; overrideFile="\$1";; - --quiet) displayText=false;; - -?|--help) displayHelp; return 0;; - esac - shift -done + +zot_parse() { + while [ "\$#" -gt 0 ]; do + case \$1 in + --eknv) exportknv="export "; knv=true ;; + --knv) knv=true ;; + --override-zos-tools) export ZOPEN_TOOLSET_OVERRIDE=1;; + --nooverride-zos-tools) unset ZOPEN_TOOLSET_OVERRIDE ;; + --override-zos-tools-subset) + [ "\$#" -ge 2 ] || { printf '%s\n' "error: --override-zos-tools-subset needs a file" >&2; return 2; } + export ZOPEN_TOOLSET_OVERRIDE=1 + overrideFile=\$2; + [ -f "\${overrideFile}" ] || { echo "Error. Override file '\${overrideFile}' is not a file." >&2; return 4; } + [ -r "\${overrideFile}" ] || { echo "Error. Override file '\${overrideFile}' is not readable." >&2; return 4; } + shift 2; continue ;; + --quiet) displayText=false ;; + --) shift; break ;; + -\?|--help) zot_displayHelp; return 0 ;; + --*) printf 'error: unknown option: %s\n' "\$1" >&2; return 2 ;; + *) # positional argument if you support any; collect or ignore + ;; + esac + shift + done + return 0 +} +zot_parse "\$@" for local_tmp in "\${TMPDIR}" "\${TMP}" /tmp; do [ -n "\${local_tmp}" ] && [ -d "\${local_tmp}" ] && break From 4cf3c3adca3f7611c3b98448ccb2b53ed65082af Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 15 Oct 2025 22:18:47 +0100 Subject: [PATCH 147/179] Backup config file to etc, not CWD! --- bin/zopen-init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/zopen-init b/bin/zopen-init index 1e105cd88..31a2323b7 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -335,7 +335,7 @@ init() }; then printInfo "- Re-initializing; zopen-config will be re-created" backup="zopen-config.$(date +%Y%m%d%H%M%S)" - if ! cp "${configFile}" "${backup}"; then + if ! cp "${configFile}" "$(dirname "${configFile}")/${backup}"; then printError "Unable to backup existing file. Check permissions and retry command." fi From 3499ea83e269e09f597237475a46054705c01c97 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 16 Oct 2025 13:19:32 +0100 Subject: [PATCH 148/179] Fix regex for zot_deleteDuplicateEntries --- include/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common.sh b/include/common.sh index 76e953949..37b031046 100755 --- a/include/common.sh +++ b/include/common.sh @@ -410,7 +410,7 @@ zot_deleteDuplicateEntries() a[\$0]++; printf("%s%s", col, \$0); col=ORS; - }' | /bin/sed "s/\${delim}//" + }' | /bin/sed "s/\${delim}\$//" } # zopen community environment variables From 3a9b92332a866c214d5b68a5550bce47036f91bc Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Fri, 17 Oct 2025 13:19:08 +0100 Subject: [PATCH 149/179] Move zopen-config backups into /etc/zopen --- bin/zopen-init | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/zopen-init b/bin/zopen-init index 31a2323b7..b84f50d74 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -335,7 +335,9 @@ init() }; then printInfo "- Re-initializing; zopen-config will be re-created" backup="zopen-config.$(date +%Y%m%d%H%M%S)" - if ! cp "${configFile}" "$(dirname "${configFile}")/${backup}"; then + bupdir=$(dirname "${configFile}")/zopen + [ -e "${bupdir}" ] || { mkdir -p "${bupdir}" || printError "Unable to create directory '${bupdir}'"; } + if ! cp "${configFile}" "${bupdir}/${backup}"; then printError "Unable to backup existing file. Check permissions and retry command." fi From 8003d98d6cf2a2fca36db46eca2b14e4d86563de Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 20 Oct 2025 14:53:21 +0100 Subject: [PATCH 150/179] Ensure valid metadata added to install tracker --- include/common.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/include/common.sh b/include/common.sh index 37b031046..b5917f2a9 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2762,16 +2762,18 @@ updatePackageDB() if [ -z "${mdj}" ]; then pkg=$(basename "${pkgdir}") printWarning "Cannot locate metadata for installed package '${pkg}' at location '${metadataFile}'. Check file existence and permissions" + else + printVerbose "Valid metadata found for package '${pkg}' - add to install tracker" + if ! jq --argjson mdj "${mdj}" '. += $mdj' \ + "${pdb}" > \ + "${pdb}.working"; then + [ -e "${pdb}.working" ] && "${pdb}.working" + [ -e "${pdb}" ] && mv -f "${pdb}" "${pdb}.broken" # Save for potential diagnostics + printSoftError "Could not add metadata for '$(basename "${pkgdir}")' to install tracker." + printError "Run 'zopen init --refresh' to attempt database regeneration and re-run command." + fi + mv "${pdb}.working" "${pdb}" fi - if ! jq --argjson mdj "${mdj}" '. += $mdj' \ - "${pdb}" > \ - "${pdb}.working"; then - [ -e "${pdb}.working" ] && "${pdb}.working" - [ -e "${pdb}" ] && mv -f "${pdb}" "${pdb}.broken" # Save for potential diagnostics - printSoftError "Could not add metadata for '$(basename "${pkgdir}")' to install tracker." - printError "Run 'zopen init --refresh' to attempt database regeneration and re-run command." - fi - mv "${pdb}.working" "${pdb}" done } From 5634473dbf1fde663a0a0c9729e2f698578a5fcc Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 21 Oct 2025 21:32:14 +0100 Subject: [PATCH 151/179] Add zopen-promote as top-level verb --- bin/zopen | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bin/zopen b/bin/zopen index 6df11ab9b..d7e62b22d 100755 --- a/bin/zopen +++ b/bin/zopen @@ -35,14 +35,15 @@ Command: config change zopen runtime environment settings diagnostics collects system info for zopen troubleshooting generate generates a new zopen project + info displays detailed information about a package init initializes a zopen environment at the specified location - refresh refreshes your zopen environment and zopen-config file install installs one or more zopen community packages - info displays detailed information about a package list lists information about zopen community packages mirror performs a synchronization of the GitHub repository + promote promote a zopen environment to another location publish publish zopen package release to github query list local or remote info about zopen community packages + refresh refreshes your zopen environment and zopen-config file remove removes installed zopen community packages update-cacert update the cacert.pem file used by zopen community upgrade upgrades existing zopen community packages @@ -71,13 +72,15 @@ SEE ALSO: zopen-clean(1) zopen-config-helper(1) zopen-generate(1) + zopen-info(1) zopen-init(1) zopen-install(1) - zopen-info(1) zopen-list(1) zopen-mirror(1) + zopen-promote(1) zopen-publish(1) zopen-query(1) + zopen-promote(1) zopen-refresh(1) zopen-remove(1) zopen-update-cacert(1) @@ -103,7 +106,7 @@ for arg in $*; do case "${arg}" in "alt"|"audit"|"build"|"clean"|"generate"|"init"|"install"|\ "query"|"remove"|"update-cacert"|"info"|"usage"|"diagnostics"|\ - "whichproject"|"publish"|"list"|"upgrade"|"mirror") + "whichproject"|"publish"|"list"|"upgrade"|"mirror"|"promote") subcmd="zopen-${arg}" ;; "refresh") From 6acebffe687c39f15a777e5669c4acb160bf3fce Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 21 Oct 2025 21:32:39 +0100 Subject: [PATCH 152/179] Prevent leak of zot_parse function to shell --- include/common.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/include/common.sh b/include/common.sh index b5917f2a9..67aacb141 100755 --- a/include/common.sh +++ b/include/common.sh @@ -516,6 +516,7 @@ fi unset -f zot_deleteDuplicateEntries 2>/dev/null unset -f zot_displayHelp 2>/dev/null unset -f zot_sanitizeEnvVar 2>/dev/null +unset -f zot_parse 2>/dev/null EOF } From ce28a46a5116b812cc502bb2cffbfd9be55295a1 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 21 Oct 2025 21:36:40 +0100 Subject: [PATCH 153/179] Rework mutex logic; check further in ancestor lineage --- include/common.sh | 100 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 80 insertions(+), 20 deletions(-) diff --git a/include/common.sh b/include/common.sh index 67aacb141..84be0272b 100755 --- a/include/common.sh +++ b/include/common.sh @@ -798,31 +798,91 @@ printColors() echo "$@" } -mutexReq() -{ - mutex=$1 +# ancestorPid +# Determines if the given pid belongs to the ancestory lineage of current process +# (including self) +# inputs: $1 pid to validate +# return: 0 pid is an ancestor or self +# 1 pid is not in ancestor lineage +ancestorPid() { + target="$1" + + case "$target" in + ''|*[!0-9]*) assertFailed "Bad pid '${target}' given to ancestorPid()" ;; + esac + mypid=$(exec sh -c 'echo ${PPID}') # $$ does not seem reliable!?! + [ "${target}" -eq "${mypid}" ] && return 0 + ap="${PPID:-0}" # Get ancestor pid (parent at current level) + while [ -n "${ap}" ] && [ "${ap}" -gt 1 ] 2>/dev/null; do + [ "$ap" -eq "${target}" ] && return 0 + # Get ancestor pid (parent of $ap) and trim spaces (awk) + if ! ap="$(/bin/ps -o ppid= -p "${ap}" 2>/dev/null | /bin/awk '{print $1}')"; then + # awk should just work, so the failure came from ps, likely from traversing the + # lineage too far so no permission to go further. + return 1 + fi + [ -z "${ap}" ] && break + done + return 1 +} + +# mutexReq +# Finds appropriate metadata for the specified port(s) and +# includes that in the installation file +# inputs: $1 name of mutex to being requested +# return: 0 mutex sucessfully locked +# 1 mutex lock unsuccessful (fast exit) +mutexReq() { + mutex="$1" lockdir="${ZOPEN_ROOTFS}/var/lock" - [ -e lockdir ] || mkdir -p ${lockdir} - mutex="${lockdir}/${mutex}" - mypid=$(exec sh -c 'echo ${PPID}') - mygrandparent=$(ps -o ppid= -p "$mypid" | awk '{print $1}') - if [ -e "${mutex}" ]; then - lockedpid=$(cat ${mutex}) - { - [ ! "${lockedpid}" = "${mypid}" ] && [ ! "${lockedpid}" = "${PPID}" ] && [ ! "${lockedpid}" = "${mygrandparent}" ] - } && kill -0 "${lockedpid}" 2> /dev/null && echo "Aborting, Active process '${lockedpid}' holds the '$2' lock: '${mutex}'" && exit -1 + [ -d "$lockdir" ] || /bin/mkdir -p "${lockdir}" || printError "Cannot create ${lockdir}" + mutexdir="${lockdir}/${mutex}.lock" + mypid=$(exec sh -c 'echo ${PPID}') # $$ does not seem reliable!?! + + printVerbose "Using mkdir's atomicity as mutex request" + # Network Filesystems might not be so atomic, but smaller window than using a + # locak file + if /bin/mkdir "${mutexdir}" 2>/dev/null; then + printVerbose "Lockdir created - mutex acquired; Adding tracking pid" + umask 077 + /bin/printf '%s\n' "${mypid}" > "${mutexdir}/pid" + addCleanupTrapCmd "rmdir '${mutexdir}'" + return 0 + fi + + printVerbose "mkdir('${mutexdir}') failed; lock already held. Checking details" + lockedpid="$(/bin/cat "${mutexdir}/pid" 2>/dev/null)" + if ! kill -0 "${lockedpid}" 2>/dev/null; then + printVerbose "Detected stale lock '${lockedpid}' [pid not active]. Removing mutex and retrying" + rm "${mutexdir}/pid" || printError "Cannot remove lockpid file'${mutexdir}/pid'" + /bin/rmdir "${mutexdir}" 2>/dev/null || printError "Cannot remove lockfile dir '${mutexdir}'" + /bin/sleep 1 + mutexReq "$mutex" + return $? fi - addCleanupTrapCmd "rm -rf ${mutex}" - echo "${mypid}" > ${mutex} + + if ancestorPid "${lockedpid}"; then + printVerbose "Ancestor '${lockedpid}' owns mutex; re-entering." + return 0 + fi + printError "Aborting, active process '${lockedpid}' holds the '${name}' lock: '${lockpath}'" } -mutexFree() -{ - mutex=$1 +# mutexFree +# Releases the specified mutex; mutexReq checks for stale locks but best to +# clean up! +# inputs: $1 name of mutex to release +# return: 0 +mutexFree() { + name="$1" + mypid=$(exec sh -c 'echo ${PPID}') # $$ does not seem reliable!?! lockdir="${ZOPEN_ROOTFS}/var/lock" - mutex="${lockdir}/${mutex}" - if [ -e "${mutex}" ]; then - rm -f ${mutex} + lockpath="${lockdir}/${name}.lock" + if [ -d "${lockpath}" ]; then + lockedpid="$(/bin/cat "${lockpath}/pid" 2>/dev/null)" + if [ "${lockedpid}" -eq "${mypid}" ] 2>/dev/null || ! kill -0 "${lockedpid}" 2>/dev/null; then + rmdir "${lockpath}" 2>/dev/null || true + fi fi return 0 } From cc912e87bbf416e655a14b9d38cca03c87a7e5a9 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 30 Oct 2025 15:47:29 +0000 Subject: [PATCH 154/179] Prevent recursive zopen-init --- include/common.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/include/common.sh b/include/common.sh index 84be0272b..9befb162d 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2661,8 +2661,13 @@ elif [ -f ".env" ]; then fi cd \${curdir} >/dev/null 2>&1 EOF - printVerbose "- Running any setup scripts" - cd "${ZOPEN_PKGINSTALL}/${name}/${name}" && [ -r "./setup.sh" ] && ./setup.sh >/dev/null + if [ "${name}" = "meta" ]; then + printVerbose "Meta is handled by the zopen-meta-init-refresh script" + # running it's setup now can affect the currently running meta + else + printVerbose "- Running any setup scripts in sub-shell to preserve environment" + cd "${ZOPEN_PKGINSTALL}/${name}/${name}" && [ -r "./setup.sh" ] && ./setup.sh >/dev/null + fi fi fi if ${unInstallOldVersion}; then From 362904fc0716de0f07ad1497b6d4b803c199ddab Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 30 Oct 2025 15:52:28 +0000 Subject: [PATCH 155/179] Validate metadata.json could be found for gpg --- include/scriptlets/installPre/verifySignatureOfPax | 1 + 1 file changed, 1 insertion(+) diff --git a/include/scriptlets/installPre/verifySignatureOfPax b/include/scriptlets/installPre/verifySignatureOfPax index 8bebd7b46..10e7d1a32 100644 --- a/include/scriptlets/installPre/verifySignatureOfPax +++ b/include/scriptlets/installPre/verifySignatureOfPax @@ -11,6 +11,7 @@ if ${skipverify}; then fi printInfo "- Performing GPG signature verification for '${pkg}' pax file" metadataVersion=$(jq -r '.version_scheme' "${metadataFile}" 2>/dev/null) +[ -n "${metadataVersion}" ] || printError "Metadata version could not be read from metadata file '${metadataFile}'" is_greater=$(echo "$metadataVersion > 0.1" | bc -l) if [ "$is_greater" -eq 1 ] && ! $skipverify; then From 806893da1e03a7cedc170e142494edbf6748718b Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Thu, 30 Oct 2025 15:57:52 +0000 Subject: [PATCH 156/179] Prevent bootstrap update when refreshing/offline --- bin/zopen-init | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index b84f50d74..1b8c25bff 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -127,6 +127,7 @@ PINENTRY_PAX_LOCATION="${PKG_DIR}/${PINENTRY_PAX}" verbose=false isHostIBM=false debug=false +xdebug=false yesToPrompts=false reinitExisting=false appendToProfile=false @@ -190,6 +191,10 @@ while [ $# -gt 0 ]; do # shellcheck disable=SC2034 debug=true ;; + "--xdebug") + xdebug=true + set -x + ;; "--yes" | "-y") yesToPrompts=true # Automatically answer 'yes' to any questions ;; @@ -758,7 +763,12 @@ EOF paxLocation=$(findrev "${MYDIR}" "${pax}") printVerbose "Pax installation from '${paxLocation}/${pax}'" if [ -e "${paxLocation}/${pax}" ]; then - zopen install "${paxLocation}/${pax}" -y --bypass-prereq-checks --skip-verify + installOptions="-y --bypass-prereq-checks --skip-verify" + ${verbose} && installOptions="${installOptions} --verbose" + ${debug} && installOptions="${installOptions} --debug" + ${xdebug} && installOptions="${installOptions} --xdebug" + # shellcheck disable=SC2086 # We want split options for command + zopen install "${paxLocation}/${pax}" ${installOptions} toolInstall=$? [ "${toolInstall}" -ne 0 ] && printError "Unable to install '${pax}' locally; see previous errors and retry installation using the '--re-init' parameter" else @@ -860,5 +870,8 @@ generateZopenConfig ! ${offlineInstall} && updateCACert generateAnalyticsConfiguration [ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || createInitialPackageDB -updateBootstrappedTools +if ! ${refresh} && ! ${offlineInstall}; then + updateBootstrappedTools +fi + deinit From 498b2a2e1e3ae6146edc8dd5bb22c18838d8b80a Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 18 Nov 2025 16:03:37 +0000 Subject: [PATCH 157/179] fix listings --- bin/zopen-list | 79 ++++++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/bin/zopen-list b/bin/zopen-list index ca4668819..47be6a62f 100755 --- a/bin/zopen-list +++ b/bin/zopen-list @@ -125,11 +125,6 @@ listAvailablePackages() installed="[${installed%,}]" # wrap in a JSON array and strip any trailing ',' char if ${details} || ${full}; then - if ${urlinclude}; then - urltext='$url' - else - urltext="" - fi jq --raw-output --argjson installed "${installed}" \ --arg wpackage "${wpackage}" \ --arg winstalled "${winstalled}" \ @@ -199,6 +194,11 @@ listInstalledEntries() wurl=0 urltext="" fi + pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" + if ! [ -e "${pdb}" ]; then + printWarning "No package database found. Regenerating (subsequent calls will be faster)" + updatePackageDB + fi if ${header} && ${details}; then if ${full}; then printf "${NC}${UNDERLINE}%-${wpackage}s%-${wversion}s%-${wrelease}s%-${wrellne}s%-${wexpsz}s%-${wqual}s%-${wurl}s${NC}\n" \ @@ -210,11 +210,6 @@ listInstalledEntries() "$(text_center "Package" ${wpackage})" "$(text_center "Version" ${wversion})" "$(text_center "Release" ${wrelease})" fi fi - pdb="${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" - if ! [ -e "${pdb}" ]; then - printWarning "No package database found. Regenerating (subsequent calls will be faster)" - updatePackageDB - fi if ${details}; then printVerbose "Details requested" if ${full}; then @@ -226,15 +221,33 @@ listInstalledEntries() --arg wrellne "${wrellne}" \ --arg wexpsz "${wexpsz}" \ --arg wqual "${wqual}" \ - "$(jqfunctions)"' .[] | - keys[] as $key | - .[$key].product.test_status as $ts | - pr($key;"@";($wpackage | tonumber)) + + --arg urlinclude "${urlinclude}" \ + --arg wurl "${wurl}" \ + "$(jqfunctions)"' .[] + | (keys|sort)[] as $key + |.[$key].product.test_status as $ts + |($ts.total_success | tonumber? // null) as $succ + |($ts.total_tests|tonumber? // null) as $tests + |( + if ($succ == -1) then "-1" + elif ($tests == null or $succ == null) then "n/a" + elif ($tests == 0) then "0" + else (((100 * $succ / $tests) * 10 | round) / 10) |r(2)| tostring + "%" + end + ) as $ts_str + | pr($key;"@";($wpackage | tonumber)) + c((.[$key].product?.version? // "unknown");"@";($wversion | tonumber)) + c((.[$key].product?.release? // "unknown");"@";($wrelease | tonumber)) + c((.[$key].product.buildline? // "unknown");"@";($wrellne | tonumber)) + c((.[$key].product.size? // "unknown");"@";($wexpsz | tonumber)) + - c(((if ($ts? and ($ts.total_tests | tonumber?) and ($ts.total_success | tonumber?)) then ($ts.total_success | tonumber) / ($ts.total_tests | tonumber) else 0 end | .*100)| r(2) |tostring);"@";($wqual | tonumber))' \ + c($ts_str;"@";($wqual | tonumber)) + + ( + if ($urlinclude == "true") then + c((.[$key].product.repo? // "unknown");"@";($wurl | tonumber)) + else + "" + end + )' \ "${pdb}" | sed 's/@/ /g' else jq --raw-output \ @@ -242,7 +255,7 @@ listInstalledEntries() --arg wversion "${wversion}" \ --arg wrelease "${wrelease}" \ "$(jqfunctions)"' .[] | - keys[] as $key | + (keys|sort)[] as $key | pr($key;"@";($wpackage | tonumber)) + c(.[$key].product.version;"@";($wversion | tonumber)) + c(.[$key].product.release;"@";($wrelease | tonumber))' \ @@ -253,7 +266,7 @@ listInstalledEntries() --arg wpackage "${wpackage}" \ --arg wversion "${wversion}" \ "$(jqfunctions)"' .[] | - keys[] as $key | + (keys|sort)[] as $key | .[$key].product.test_status as $ts | pr($key;"@";($wpackage | tonumber)) + c((.[$key].product?.version? // "unknown");"@";($wversion | tonumber))' \ @@ -275,18 +288,16 @@ verbose=false header=false details=false full=false -available=false -installed=true -updates=false +commandVerb="installed" # Default list to display urlinclude=false # whether to include download url in output information versioninfo=false while [ $# -gt 0 ]; do printVerbose "Parsing option: $1" case "$1" in - "--available") available=true; installed=false; updates=false;; - "--installed") available=false; installed=true; updates=false;; - "--updates") available=false; installed=false; updates=true ;; + "--available") commandVerb="available";; + "--installed") commandVerb="installed";; + "--updates") commandVerb="updates";; "--details") details=true ;; "--full") full=true; details=true ;; "--header") header=true ;; @@ -321,15 +332,13 @@ while [ $# -gt 0 ]; do done checkIfConfigLoaded -if ${installed}; then - listInstalledEntries - exit 0 -fi -if ${available}; then - listAvailablePackages - exit 0 -fi -if ${updates}; then - listUpdatesForInstalled - exit 0 -fi + +case "${commandVerb}" in + "available") listAvailablePackages;; + "updates") listUpdatesForInstalled;; + "installed") listInstalledEntries;; + *) assertFailed "Unrecognised command verb '${commandVerb}'";; +esac + +exit 0 + From 5b9122fe6a634085d7c25e24f32d849bef779c4d Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 18 Nov 2025 16:04:28 +0000 Subject: [PATCH 158/179] Fix zopen-info --- bin/zopen-info | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/zopen-info b/bin/zopen-info index c2ba3a851..35b022c34 100755 --- a/bin/zopen-info +++ b/bin/zopen-info @@ -77,7 +77,6 @@ printPackageInfo() { if ! runAndLog "curlCmd -s -L '${metadataJSONURL}' -o '${metadataFile}'"; then printError "Could not download from ${metadataJSONURL}. Correct any errors and potentially retry." - continue fi repo=$(echo "${repo}" | sed 's#/[^/]*$##') # Strip /download repo=$(echo "${repo}" | sed 's#^[^/]*//[^/]*##') # Strip protocol and host @@ -289,5 +288,5 @@ done checkIfConfigLoaded updateCaches validatePackageList "${package}" -gatherPackageInfo "${package}" "${remote_lookup}" +printPackageInfo "${package}" "${remote_lookup}" outputPackageInfo \ No newline at end of file From 67ae972d026dcb6f0f4f33e69b79cac9cc66d48f Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 18 Nov 2025 16:08:26 +0000 Subject: [PATCH 159/179] Fix meta install during init --- bin/zopen-init | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index 1b8c25bff..35a5f55ca 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -108,11 +108,6 @@ Report bugs at https://github.com/zopencommunity/meta/issues HELPDOC } - -# Constants -# Number of rm processes to run in parallel -rm_fileprocs=5 #TODO: adjust based on number of online cpus on system - # Boostrap pax files CURL_PAX="curl-8.10.1.20241001_214340.zos.pax.Z" JQ_PAX="jq-1.6.20241001_204116.zos.pax.Z" @@ -313,17 +308,17 @@ init() printInfo "- Collecting usage statistics: ${isCollectingStats}" fi - if ! promptYesOrNo "Do you want to continue?" ${yesToPrompts}; then - printInfo "Exiting..." - exit 0 - fi - if ! ${yesToPrompts} && [ -z "${OVERRIDE_ZOS_TOOLS}" ]; then if promptYesOrNo "Should zopen tools default to overriding the z/OS /bin tools (e.g. man, find, grep)?" ${yesToPrompts}; then OVERRIDE_ZOS_TOOLS=true fi fi + if ! promptYesOrNo "Do you want to continue?" ${yesToPrompts}; then + printInfo "Exiting..." + exit 0 + fi + configFile="${rootfs}/etc/zopen-config" if [ -f "${configFile}" ]; then if [ -e "${configFile}" ]; then @@ -647,6 +642,29 @@ zz fi } +installMeta() +{ + printHeader "Installing meta package" + printVerbose "Determining if installing using a DEV meta" + metabindir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P) + metaReleaseLine="STABLE" + ##TODO + metadataFile="${metabindir}/../metadata.json" + if [ ! -r "${metadataFile}" ]; then + printVerbose "No metadata.json file; using stable release" + fi + #Note that the releaseline is knwon as buildline in the metadata file! + if ! bootstrapMetaReleaseline=$(jqw -r '.product.buildline' "${metadataFile}"); then + printVerbose "Could not get releaseline/buildline from package metadata; using stable release" + fi + case "${bootstrapMetaReleaseline}" in + "STABLE"|"DEV") printVerbose "Valid releaseline '${bootstrapMetaReleaseline}' from metadata file" + metaReleaseLine="${bootstrapMetaReleaseline}";; + *) printVerbose "Invalid releaseline '${bootstrapMetaReleaseline}'; defaulting to STABLE" + esac + "${MYDIR}"/zopen-install "${installOptions}" -y meta --release-line "${metaReleaseLine}" +} + installPrereqs() { printHeader "Installing zopen pre-requisites" @@ -871,7 +889,8 @@ generateZopenConfig generateAnalyticsConfiguration [ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || createInitialPackageDB if ! ${refresh} && ! ${offlineInstall}; then - updateBootstrappedTools + installMeta + updateBootstrappedTools fi deinit From deec9449151fff0f6f794a0caf81e254111f7a61 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 18 Nov 2025 16:08:50 +0000 Subject: [PATCH 160/179] Update releaseline detection --- include/common.sh | 228 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 224 insertions(+), 4 deletions(-) diff --git a/include/common.sh b/include/common.sh index 9befb162d..c88369985 100755 --- a/include/common.sh +++ b/include/common.sh @@ -1980,16 +1980,37 @@ getReleaseLineMetadata() printSoftError "Could not find asset for releaseline '${validatedReleaseLine}' for repo '${repo}' in ${JSON_CACHE}" 2> "${invalidPortAssetFile}" return 1 fi - unset "${releaseLine}" + unset releaseLine +} + +getReleaseLineFromInstalledPkg(){ + repo="$1" + metadataFile="${ZOPEN_PKGINSTALL}/${repo}//${repo}/metadata.json" + if [ ! -r "${metadataFile}" ]; then + printVerbose "No metadata.json file for '${repo}'; default to stable" + echo "STABLE" && return 0 + fi + #Note that the releaseline is knwon as buildline in the metadata file! + if ! installedReleaseline=$(jqw -r '.product.buildline' "${metadataFile}"); then + printError "Could not get releaseline/buildline from package metadata for '${repo}'" + fi + case "${installedReleaseline}" in + "STABLE"|"DEV") printVerbose "Valid releaseline '${installedReleaseline}' from metadata file";; + *) printError "Invalid releaseline '${installedReleaseline}' for package '${repo}'" + esac + echo "${installedReleaseline}" + return 0 } calculateReleaseLineMetadata() { repo="$1" printDebug "No explicit version/tag/releaseline, checking for pre-existing package&releaseline" - if [ -n "${installedReleaseLine}" ]; then - printDebug "Found existing releaseline '${installedReleaseLine}', restricting to only that releaseline" - validatedReleaseLine="${installedReleaseLine}" # Already validated when stored + if isPackageActive "${repo}"; then + printVerbose "Package '${repo}' already installed; determining releaseline" + if ! validatedReleaseLine=$(getReleaseLineFromInstalledPkg "${repo}"); then + return 1 + fi else printDebug "Checking for system-configured releaseline" if [ -e "${ZOPEN_JSON_CONFIG}" ]; then @@ -3099,6 +3120,205 @@ findSuggestion() { echo "${best_match}" } +# Map functions - no arrays or maps in POSIX.1 shell so improvise +# Default separators +DEFAULT_KVSEP="=" +DEFAULT_ENTRYSEP="|" + +# Create a new map with specified separators +createMap() { + kvsep="$1" + entrysep="$2" + echo "${kvsep} ${entrysep} " +} + +# Get key-value separator from map +getKVSeparator() { + map=$1 + echo "${map}" | awk '{print $1}' +} + +# Get entry separator from map +getEntrySeparator() { + map=$1 + echo "${map}" | awk '{print $2}' +} + +# Get entries from map +getEntries() { + map=$1 + echo "${map}" | cut -d' ' -f3- +} + +# Add entry to map (initializes if blank) +addEntry() { + map="$1" + key="$2" + value="$3" + + if [ -z "${map}" ]; then + kvsep="$DEFAULT_KVSEP" + entrysep="$DEFAULT_ENTRYSEP" + entries="" + else + kvsep=$(getKVSeparator "${map}") + entrysep=$(getEntrySeparator "${map}") + entries=$(getEntries "${map}") + fi + newEntry="${key}${kvsep}${value}" + echo "${kvsep} ${entrysep} ${entries}${newEntry}${entrysep}" +} + +# Immutable add (fails if key exists) +immutableAddEntry() { + map="$1" + key="$2" + value="$3" + + if hasKey "${map}" "$key"; then + echo "${map}" + return 1 + else + addEntry "${map}" "$key" "$value" + return 0 + fi +} + +# Get value for a key +getValue() { + map="$1" + needle="$2" + kvsep=$(getKVSeparator "${map}") + entrysep=$(getEntrySeparator "${map}") + entries=$(getEntries "${map}") + + IFS="$entrysep" set -- $entries + for knv; do + key=$(echo "$knv" | awk -F"$kvsep" '{print $1}') + if [ "$key" = "$needle" ]; then + echo "$knv" | awk -F"$kvsep" '{print $2}' + return 0 + fi + done + return 1 +} + +# Check if key exists +hasKey() { + map="$1" + needle="$2" + getValue "${map}" "$needle" > /dev/null +} + +# Update value for a key +updateValue() { + map="$1" + needle="$2" + newValue="$3" + kvsep=$(getKVSeparator "${map}") + entrysep=$(getEntrySeparator "${map}") + entries=$(getEntries "${map}") + newMap="${kvsep} ${entrysep} " + + IFS="$entrysep" set -- $entries + for knv; do + key=$(echo "$knv" | awk -F"$kvsep" '{print $1}') + if [ "$key" = "$needle" ]; then + knv="${key}${kvsep}${newValue}" + fi + newMap="${newMap}${knv}${entrysep}" + done + echo "$newMap" +} + +# Delete a key +deleteKey() { + map="$1" + needle="$2" + kvsep=$(getKVSeparator "${map}") + entrysep=$(getEntrySeparator "${map}") + entries=$(getEntries "${map}") + newMap="${kvsep} ${entrysep} " + + IFS="$entrysep" set -- $entries + for knv; do + key=$(echo "$knv" | awk -F"$kvsep" '{print $1}') + if [ "$key" != "$needle" ]; then + newMap="${newMap}${knv}${entrysep}" + fi + done + echo "$newMap" +} + +# List all keys +listKeys() { + map="$1" + kvsep=$(getKVSeparator "${map}") + entrysep=$(getEntrySeparator "${map}") + entries=$(getEntries "${map}") + + IFS="$entrysep" set -- $entries + for knv; do + echo "$knv" | awk -F"$kvsep" '{print $1}' + done +} + +# Print map entries +prettyPrint() { + map="$1" + wrap_width="${2:-80}" + indent="${3:-4}" + + kvsep=$(getKVSeparator "${map}") + entrysep=$(getEntrySeparator "${map}") + entries=$(getEntries "${map}") + + printf "%s" "${entries}" | awk -v kvsep="${kvsep}" -v RS="${entrysep}" -v wrap="${wrap_width}" -v lhindent="${indent}"\ + ' + { + split($0, kv, kvsep) # Split into key/value pair on the key separator + keys[NR] = kv[1] # Store the key in the keys array [no maps in awk] + values[NR] = kv[2] # Store associated value in value array [at same offset!] + # Determine what the longest key name is as we use that to calculate the + # offset to write values at. + if (length(kv[1]) > maxlen) maxlen = length(kv[1]) + } + END { + indent = maxlen + 3 # Space between biggest key and start of value text + for (i = 1; i <= NR; i++) { # Loop all records + printf "%*s%-*s ", lhindent, "", maxlen, keys[i] # Output key + line = "" + # Need to format the value such that it breaks into sections + # then indented to where the initial line started - pretty! + split(values[i], words, /[ \t]+/) + for (j = 1; j <= length(words); j++) { + word = words[j] + printf "Parsed word:>>%s\n", word + if (length(line) + length(word) + 1 > wrap - indent) { + # There is no space in current line buffer for word - output and rebuild + print line + line = sprintf("%*s%*s%s", lhindent, "", indent, "", word) + } else { + # Space in buffer for word + if (length(line) == 0) { + # First word so just use as-is in buffer + line = sprintf("%*s%s", lhindent, "", word) + printf "Started new line:>>%s\n", line + } else { + # ! first word so insert space char and append to buffer + line = line " " word + printf "Line is now :>>%s\n", line + } + } + } + # Anything left in the line buffer can be written + if (length(line) > 0) print line + } + }' +} + + + # Main code From 725163864373e08a0a05a303c9630b66b514cdb7 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Wed, 19 Nov 2025 10:42:34 +0000 Subject: [PATCH 161/179] Add back in parentPid tracker [main merge issue] --- include/common.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/common.sh b/include/common.sh index c88369985..f629e0923 100755 --- a/include/common.sh +++ b/include/common.sh @@ -12,6 +12,10 @@ zopenInitialize() # Capture start time before setting trap fullProcessStartTime=${SECONDS} + # Monitor the intial parent process; as subshells inherit + # variables, this will not be overridden due to param expansion + parentPid=${parentPid:-$$} + # Create the cleanup pipeline and exit handler trap "cleanupFunction" EXIT INT TERM QUIT HUP From 244f0d46021aa9c0bcad43051357b9ac9b180df0 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 25 Nov 2025 20:47:18 +0000 Subject: [PATCH 162/179] Log before final message; error can cause scroll of IMPORTANT! --- bin/zopen-init | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/zopen-init b/bin/zopen-init index 35a5f55ca..ac91143a5 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -866,6 +866,7 @@ createInitialPackageDB() deinit() { + syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE},${CAT_FILE}" "INIT" "" "zopen environment initialised as ${ZOPEN_ROOTFS}" printHeader "Initialization successfully completed" printInfo "${NC}${YELLOW}IMPORTANT: Run '. ${rootfs}/etc/zopen-config' to enable zopen environment for current session or add it to your .profile.${NC}" printInfo "${NC}${YELLOW}IMPORTANT: If you prefer GNU tools over z/OS /bin tools, run '. ${rootfs}/etc/zopen-config --override-zos-tools' in the current session or add it to your .profile.${NC}" From 96ccd17022dfe81486e8973ab9173afd23bfe051 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 25 Nov 2025 21:00:12 +0000 Subject: [PATCH 163/179] Ensure use of tmp dir (and quoted strings) --- include/common.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/include/common.sh b/include/common.sh index f629e0923..abf6c7a77 100755 --- a/include/common.sh +++ b/include/common.sh @@ -21,7 +21,7 @@ zopenInitialize() # Temporary files for zopen_tmp_dir in "${TMPDIR}" "${TMP}" /tmp; do - if [ ! -z ${zopen_tmp_dir} ] && [ -d ${zopen_tmp_dir} ]; then + if [ ! -z "${zopen_tmp_dir}" ] && [ -d "${zopen_tmp_dir}" ]; then break fi done @@ -66,7 +66,7 @@ addCleanupTrapCmd(){ # shellcheck disable=SC2009 # no pgrep. #mypid=$(ps -ef |grep -v grep |grep $0 | tr -s ' ' | cut -f3 -d ' ') mypid=$(exec sh -c 'echo ${PPID}') - tmpscriptfile="/tmp/zopen_trap.${mypid}.scr" + tmpscriptfile="${zopen_tmp_dir}/zopen_trap.${mypid}.scr" echo "${newcmd}" >> "${tmpscriptfile}" if [ $$ -eq "${parentPid}" ]; then @@ -82,7 +82,7 @@ cleanup() { # shellcheck disable=SC2009 # no pgrep. #mypid=$(ps -ef |grep -v grep |grep ps | tr -s ' ' | cut -f3 -d ' ') mypid=$(exec sh -c 'echo ${PPID}') - tmpscriptfile="/tmp/zopen_trap.${mypid}.scr" + tmpscriptfile="${zopen_tmp_dir}/zopen_trap.${mypid}.scr" if [ -f "${tmpscriptfile}" ] && [ -s "${tmpscriptfile}" ]; then # Execute the commands in the cleanup file by sourcing it # shellcheck disable=SC1090 @@ -153,11 +153,11 @@ addCleanupTrapCmd2(){ cleanupOnExit() { rv=$? - [ -f ${ZOPEN_TEMP_C_FILE} ] && rm -rf ${ZOPEN_TEMP_C_FILE} - [ -p ${TMP_FIFO_PIPE} ] && rm -rf ${TMP_FIFO_PIPE} + [ -f "${ZOPEN_TEMP_C_FILE}" ] && rm -rf "${ZOPEN_TEMP_C_FILE}" + [ -p "${TMP_FIFO_PIPE}" ] && rm -rf "${TMP_FIFO_PIPE}" if [ ! -z "${TEE_PID}" ]; then - if kill -0 ${TEE_PID} 2> /dev/null; then - kill -9 ${TEE_PID} + if kill -0 "${TEE_PID}" 2> /dev/null; then + kill -9 "${TEE_PID}" fi fi [ -e "${TMP_GPG_DIR}" ] && rm -rf "${TMP_GPG_DIR}" From afb48586a56518b2242ed98d5bb5237db5480078 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 25 Nov 2025 21:38:05 +0000 Subject: [PATCH 164/179] Remove duplicate syslog --- bin/zopen-init | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/zopen-init b/bin/zopen-init index ac91143a5..c23bb27c9 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -870,7 +870,6 @@ deinit() printHeader "Initialization successfully completed" printInfo "${NC}${YELLOW}IMPORTANT: Run '. ${rootfs}/etc/zopen-config' to enable zopen environment for current session or add it to your .profile.${NC}" printInfo "${NC}${YELLOW}IMPORTANT: If you prefer GNU tools over z/OS /bin tools, run '. ${rootfs}/etc/zopen-config --override-zos-tools' in the current session or add it to your .profile.${NC}" - syslog "${ZOPEN_LOG_PATH}/audit.log" "${LOG_A}" "${CAT_PACKAGE},${CAT_FILE}" "INIT" "" "zopen environment initialised as ${ZOPEN_ROOTFS}" if ! ${refresh}; then myparent="$(cd "$(dirname "${MYDIR}")" >/dev/null 2>&1 && pwd -P )" myparentpaxz="${myparent}.pax.Z" From f387f815db967ae21f1449918aa757cf578689ac Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 25 Nov 2025 21:40:27 +0000 Subject: [PATCH 165/179] Prevent cd from modifying environment; apply shellcheck fixes --- include/common.sh | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/include/common.sh b/include/common.sh index abf6c7a77..cd403066e 100755 --- a/include/common.sh +++ b/include/common.sh @@ -210,8 +210,10 @@ getCurrentVersionDir(){ [ ! -L "${ZOPEN_PKGINSTALL}/${needle}/${needle}" ] \ && echo ""\ && return 4 - cd "${ZOPEN_PKGINSTALL}/${needle}/${needle}" 2> /dev/null \ + ( # Do not modify current environment! + cd "${ZOPEN_PKGINSTALL}/${needle}/${needle}" 2> /dev/null \ && pwd -P 2> /dev/null + ) } # isPackageActive @@ -2691,7 +2693,9 @@ EOF # running it's setup now can affect the currently running meta else printVerbose "- Running any setup scripts in sub-shell to preserve environment" - cd "${ZOPEN_PKGINSTALL}/${name}/${name}" && [ -r "./setup.sh" ] && ./setup.sh >/dev/null + ( # Prevent cd from modifying current environment + cd "${ZOPEN_PKGINSTALL}/${name}/${name}" && [ -r "./setup.sh" ] && ./setup.sh >/dev/null + ) fi fi fi @@ -2751,13 +2755,14 @@ processActionScripts() "parseGraphPost") scriptletDir="${ZOPEN_SCRIPTLET_DIR}/parseGraphPost";; *) assertFailed "Invalid process action phase '${phase}'" esac - printVerbose "Running script[s] from '${scriptletDir}'" + printVerbose "Running script[s] from '${scriptletDir}'" - if [ ! -d "${scriptletDir}" ]; then - printDebug "No script directory for phase: ${phase}" - return 0 - fi - unset CDPATH; + if [ ! -d "${scriptletDir}" ]; then + printDebug "No script directory for phase: ${phase}" + return 0 + fi + curDir=$(pwd -P) + unset CDPATH; cd "${scriptletDir}" || return 1 scriptletRcFile=$(mktempfile "zopen_actionscripts" ".err") @@ -2796,9 +2801,11 @@ processActionScripts() printWarning "One or more scripts failed:" cat "${scriptletRcFile}" rm -f "${scriptletRcFile}" + cd "${curDir}" || printSoftError "Could not return to previous directory" return 1 fi rm -f "${scriptletRcFile}" + cd "${curDir}" || printSoftError "Could not return to previous directory" return 0 } @@ -2821,7 +2828,7 @@ updatePackageDB() printf "[\n]\n" > "${pdb}" fi - if [ $(unset CD_PATH; cd "${ZOPEN_PKGINSTALL}"; ls -A | wc -l) -eq 0 ]; then + if [ "$(unset CD_PATH; cd "${ZOPEN_PKGINSTALL}"; ls -A | wc -l)" -eq 0 ]; then printVerbose "No packages found to add to database [new install?]" return fi From ca64a583478fec6946ec39d115692e4a5b1e594f Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Fri, 28 Nov 2025 16:23:01 +0000 Subject: [PATCH 166/179] Accept first verb to save confusion --- bin/zopen | 70 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/bin/zopen b/bin/zopen index d7e62b22d..afb9e5914 100755 --- a/bin/zopen +++ b/bin/zopen @@ -101,37 +101,48 @@ subopts="" subcmd="" help=false version=false +verbFound=false -for arg in $*; do - case "${arg}" in - "alt"|"audit"|"build"|"clean"|"generate"|"init"|"install"|\ - "query"|"remove"|"update-cacert"|"info"|"usage"|"diagnostics"|\ - "whichproject"|"publish"|"list"|"upgrade"|"mirror"|"promote") - subcmd="zopen-${arg}" - ;; - "refresh") - subcmd='zopen-init' - subopts="${subopts} --refresh" - ;; - "config") - subcmd="zopen-${arg}-helper" - ;; - "--version") - if [ -z "${subcmd}" ]; then - subcmd='zopen-version' - version=true - else +while [ "$#" -gt 0 ]; do + arg=$1 || shift + if $verbFound; then + subopts="${subopts} ${arg}" + else + case "${arg}" in + "alt"|"audit"|"build"|"clean"|"generate"|"init"|"install"|\ + "query"|"remove"|"update-cacert"|"info"|"usage"|"diagnostics"|\ + "whichproject"|"publish"|"list"|"upgrade"|"mirror"|"promote") + subcmd="zopen-${arg}" + verbFound=true + ;; + "refresh") + subcmd='zopen-init' + subopts="${subopts} --refresh" + verbFound=true + ;; + "config") + subcmd="zopen-${arg}-helper" + verbFound=true + ;; + "--version") + if [ -z "${subcmd}" ]; then + subcmd='zopen-version' + version=true + else + subopts="${subopts} ${arg}" + fi + ;; + "help" | "--help" | "-?") + help=true + verbFound=true + ;; + *) + # let unknown stuff through subopts="${subopts} ${arg}" - fi - ;; - "help" | "--help" | "-?") - help=true - ;; - *) - # let unknown stuff through - subopts="${subopts} ${arg}" - ;; - esac + ;; + esac + fi + shift done if [ -z "${subcmd}" ]; then @@ -157,4 +168,5 @@ if ${version}; then fi fi +# shellcheck disable=SC2086 # We want globbing in this case ${subcmd} ${subopts} From c8f532d95d3f98ec7a7e9846895e94219df734f6 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Fri, 28 Nov 2025 16:24:19 +0000 Subject: [PATCH 167/179] Use wrapped jq command --- include/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common.sh b/include/common.sh index cd403066e..3b673db85 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2374,7 +2374,7 @@ spaceValidate(){ cacheBytes=$1 packageBytes=$2 - skip_size_check=$(jq -re '.skip_size_check' "${ZOPEN_JSON_CONFIG}") + skip_size_check=$(jqw -re '.skip_size_check' "${ZOPEN_JSON_CONFIG}") if "${skip_size_check}"; then printVerbose "Skipping size check due to skip_size_check=${skip_size_check}" return From 1adc9fed5e03e029a65f0fbb7dbbafe1c39bf740 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Fri, 28 Nov 2025 16:26:13 +0000 Subject: [PATCH 168/179] Ensure offline installer includes meta --- bin/zopen-init | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index c23bb27c9..0fecc4f41 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -656,12 +656,17 @@ installMeta() #Note that the releaseline is knwon as buildline in the metadata file! if ! bootstrapMetaReleaseline=$(jqw -r '.product.buildline' "${metadataFile}"); then printVerbose "Could not get releaseline/buildline from package metadata; using stable release" + metaReleaseLine="STABLE" + else + case "${bootstrapMetaReleaseline}" in + "STABLE"|"DEV") printVerbose "Valid releaseline '${bootstrapMetaReleaseline}' from metadata file" + metaReleaseLine="${bootstrapMetaReleaseline}" + ;; + *) printVerbose "Invalid releaseline '${bootstrapMetaReleaseline}'; defaulting to STABLE" + metaReleaseLine="STABLE" + ;; + esac fi - case "${bootstrapMetaReleaseline}" in - "STABLE"|"DEV") printVerbose "Valid releaseline '${bootstrapMetaReleaseline}' from metadata file" - metaReleaseLine="${bootstrapMetaReleaseline}";; - *) printVerbose "Invalid releaseline '${bootstrapMetaReleaseline}'; defaulting to STABLE" - esac "${MYDIR}"/zopen-install "${installOptions}" -y meta --release-line "${metaReleaseLine}" } @@ -710,7 +715,12 @@ installPrereqs() printInfo "- Sourcing environment" } -updateBootstrappedTools() { +# Used to ensure that there are initial versions of the core products available for use. +# - install/update the packages from the repo +# - in an offline/airgapped environment, this involves installing the bootstrapped paxes +# and copying over the meta that was used in the initial install +updateBootstrappedTools() +{ printInfo "- Installing/updating bootstrapped curl, jq, gpg, pinentry and meta to install updated versions (if available)" installOptions="--force" toolInstall=0 @@ -718,7 +728,7 @@ updateBootstrappedTools() { installOptions="${installOptions} --bypass-prereq-checks"; fi if ${refresh}; then - "${MYDIR}"/zopen-install "${installOptions}" -y curl jq pinentry gpg + "${MYDIR}"/zopen-install "${installOptions}" --yes curl jq pinentry gpg toolInstall=$? elif ${offlineInstall}; then printVerbose "As offline, no access to meta.pax so copy currently running" @@ -794,8 +804,9 @@ EOF fi done else + installMeta # Need to ensure meta is available # shellcheck disable=SC2086 # We want installOptions to be split - "${MYDIR}/zopen-install" ${installOptions} -y curl jq pinentry gpg meta + "${MYDIR}/zopen-install" ${installOptions} --yes curl jq pinentry gpg toolInstall=$? fi [ "${toolInstall}" -ne 0 ] && printError "Unable to install curl, jq, gpg, pinentry and/or meta; see previous errors and retry the initilisation using the '--re-init' parameter" @@ -888,9 +899,6 @@ generateZopenConfig ! ${offlineInstall} && updateCACert generateAnalyticsConfiguration [ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || createInitialPackageDB -if ! ${refresh} && ! ${offlineInstall}; then - installMeta - updateBootstrappedTools -fi +updateBootstrappedTools deinit From d9893f4d97cb4b711274750da2fcdfec238fc9f3 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Fri, 28 Nov 2025 16:27:37 +0000 Subject: [PATCH 169/179] Track offline install in config & simplify/reuse logic --- bin/zopen-init | 63 ++++++++++------------------------------------- include/common.sh | 35 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 50 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index 0fecc4f41..cf5374dfe 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -350,7 +350,6 @@ init() export ZOPEN_ROOTFS="${rootfs}" ZOPEN_CA_DIR="etc/pki/tls/certs" # Mimic location on some Linux distributions - ZOPEN_JSON_CONFIG="${ZOPEN_ROOTFS}/etc/zopen/config.json" certFileName="cacert.pem" } @@ -455,6 +454,7 @@ generateJsonConfiguration() autopkgpurge=$(jqw -r '.autopkgpurge' "${jsonConfig}") skip_broken=$(jqw -r '.skip_broken' "${jsonConfig}") skip_size_check=$(jqw -r '.skip_size_check' "${jsonConfig}") + offline=$(jqw -r '.offline' "${jsonConfig}") fi if ${releaselineDev}; then @@ -462,53 +462,13 @@ generateJsonConfiguration() releaseLine="DEV" printInfo "- Configured zopen to use development (DEV) releaseline packages where available" fi - case "${releaseLine}" in - "") releaseLine="STABLE" - printInfo "- Configured zopen to use stable releaseline packages" - ;; - "STABLE"|"DEV") :;; # Use as-is - *) printWarning "Current value for release line '${releaseLine}' invalid. Defaulting to 'STABLE'." - releaseLine="STABLE" - ;; - esac - case "${autocacheclean:=true}" in - "0"|"1"|[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee]) :;; # Valid value - "null") autocacheclean="true" # Default to true; jq found no entry (new config) - ;; - *) showConfigParmWarning "autocacheclean" "${autocacheclean}" "true|false" "true" - autocacheclean=true - - ;; - esac - case "${autopkgpurge:=false}" in - "0"|"1"|[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee]) :;; # Valid value - "null")autopkgpurge=false # Default to false; jq found no entry (new config) - ;; - *) showConfigParmWarning "autopkgpurge" "${autopkgpurge}" "true|false" "false" - autopkgpurge=false - ;; - esac - case "${skip_broken:=false}" in - "0"|"1"|[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee]) :;; # Valid value - "null") skip_broken=false # Default to false - ;; - *) showConfigParmWarning "skip_broken" "${skip_broken}" "true|false" "false" - skip_broken=false - ;; - esac - case "${skip_size_check:=false}" in - "0"|"1"|[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee]) :;; # Valid value - "null") skip_size_check=false # Default to false - ;; - *) showConfigParmWarning "skip_size_check" "${skip_size_check}" "true|false" "false" - skip_size_check=false - ;; - esac - case "${OVERRIDE_ZOS_TOOLS}" in - "true"|"false") :;; # Valid value - "null") OVERRIDE_ZOS_TOOLS=false ;; # No entry in config file - *) OVERRIDE_ZOS_TOOLS=false ;; # Bad envvar / empty value in config - esac + validateConfigValue "enum" "releaseLine" "$releaseLine" "STABLE" "DEV|STABLE" + validateConfigValue "boolean" "autocacheclean" "$autocacheclean" "true" + validateConfigValue "boolean" "autopkgpurge" "$autopkgpurge" "false" + validateConfigValue "boolean" "skip_broken" "$skip_broken" "false" + validateConfigValue "boolean" "skip_size_check" "$skip_size_check" "false" + validateConfigValue "boolean" "OVERRIDE_ZOS_TOOLS" "$OVERRIDE_ZOS_TOOLS" "false" + validateConfigValue "boolean" "offline" "$offline" "false" if [ ! -e "${jsonConfig}" ]; then printInfo "Generating global configuration" @@ -520,7 +480,8 @@ generateJsonConfiguration() "autocacheclean": ${autocacheclean}, "autopkgpurge": ${autopkgpurge}, "skip_broken": ${skip_broken}, - "skip_size_check": ${skip_size_check} + "skip_size_check": ${skip_size_check}, + "offline": ${offline} } zz else @@ -534,13 +495,15 @@ zz --arg autopkgpurge "${autopkgpurge}" \ --arg skip_broken "${skip_broken}" \ --arg skip_size_check "${skip_size_check}" \ + --arg offline "${offline}" \ ".release_line = \$releaseLine | .override_zos_tools = \$isoverrideZOSTools | .is_collecting_stats = \$isCollectingStats | .autocacheclean = \$autocacheclean | .autopkgpurge = \$autopkgpurge | .skip_broken = \$skip_broken | - .skip_size_check = \$skip_size_check + .skip_size_check = \$skip_size_check | + .offline = \$offline " "${jsonConfig}" > "${jsonConfigWorking}"; then printError "Errors updating existing configuration file. See previous errors for more information and retry command." fi diff --git a/include/common.sh b/include/common.sh index 3b673db85..e68e12da6 100755 --- a/include/common.sh +++ b/include/common.sh @@ -3131,6 +3131,41 @@ findSuggestion() { echo "${best_match}" } +validateConfigValue() +{ + type=$1 + key=$2 + value=$3 + default=$4 + enums=$5 + case "${type}" in + boolean) + case "${value:=$default}" in + "0"|"1"|[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee]) :;; # Valid value + "null") value="$default" # jq found no entry (new config?) + ;; + *) showConfigParmWarning "${key}" "${value}" "true|false" "${default}" + value="$default" + ;; + esac + ;; + enum) + if [ "${value:=$default}" = "null" ]; then + value="$default" # jq found no entry (new config?) + elif echo "${enums}" | awk -v RS='|' -v val="${value}" ' + { if (toupper(val) == toupper($0)) exit 0; } + END { exit 1 }'; then + : # Value value + else + showConfigParmWarning "${key}" "${value}" "${enums}" "${default}" + value="$default" + fi + ;; + *) assertFailed "Unknown config value type '${type}'" + esac + eval "$key=\"${value}\"" +} + # Map functions - no arrays or maps in POSIX.1 shell so improvise # Default separators DEFAULT_KVSEP="=" From dcc2de2d706d8c1d67c2275064ea092ae1742867 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Sat, 29 Nov 2025 16:01:52 +0000 Subject: [PATCH 170/179] Rewrite awk to not throw exit from END{..} --- include/common.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/common.sh b/include/common.sh index e68e12da6..0329fdb30 100755 --- a/include/common.sh +++ b/include/common.sh @@ -3153,8 +3153,9 @@ validateConfigValue() if [ "${value:=$default}" = "null" ]; then value="$default" # jq found no entry (new config?) elif echo "${enums}" | awk -v RS='|' -v val="${value}" ' - { if (toupper(val) == toupper($0)) exit 0; } - END { exit 1 }'; then + BEGIN {notfound=1} + { if (toupper(val) == toupper($0)) {notfound=0; exit 0;} } + END { exit notfound }'; then : # Value value else showConfigParmWarning "${key}" "${value}" "${enums}" "${default}" From 6bdddc5bbd2797b96d2edf812f85f80e5aa7f547 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Sat, 29 Nov 2025 17:34:29 +0000 Subject: [PATCH 171/179] EXtract function and refactor --- bin/zopen-init | 87 +++++++++++++++++++++++++------------------------- 1 file changed, 44 insertions(+), 43 deletions(-) diff --git a/bin/zopen-init b/bin/zopen-init index cf5374dfe..7279d16c7 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -604,8 +604,48 @@ zz fi fi } +installLocalMeta() +{ + metabindir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P) + ulzm="${rootfs}/usr/local/zopen/meta" + target="${ulzm}/meta-bootstrap" + mkdir -p "${target}" + printVerbose "Copying bare meta scripts from '${metabindir}' to '${target}" + cp -R "${metabindir}/../bin" "${target}" + cp -R "${metabindir}/../include" "${target}" + + printVerbose "Creating main symlink for meta" + ln -s "${ulzm}/meta-bootstrap" "${ulzm}/meta" + if ! runLogProgress "mergeIntoSystem \"meta\" \"${ulzm}/meta-bootstrap\" \"${rootfs}\"" \ + "Merging meta into symlink mesh" "Merged meta into symlink mesh"; then + printSoftError "Unexpected errors merging symlinks into mesh" + printError "Use zopen alt to select previous version to ensure known state" + fi + printVerbose "Adding bootstrapped meta to package db" + printVerbose "Creating basic metadata.json" + cat << EOF > "${target}/metadata.json" +{ + "version_scheme": "0.2", + "product": { + "name": "meta", + "version": "0.0.0.0-bootstrap", + "release": "$(date +%Y%m%d_%H%M%S)", + "summary": "meta on z/OS", + "repo": "https://github.com/zopencommunity/metaport", + "license": "https://github.com/zopencommunity/metaport/blob/main/patches/LICENSE", + "zopen_license": "https://github.com/zopencommunity/metaport/blob/main/LICENSE", + "categories": "utilities", + "size": "$(du -ks "${target}" | awk '{print $1 * 1024}')", + "build_dependencies": [], + "runtime_dependencies": [], + "system_prereqs": [] + } +} +EOF + addToInstallTracker "meta" +} -installMeta() +installRemoteMeta() { printHeader "Installing meta package" printVerbose "Determining if installing using a DEV meta" @@ -625,7 +665,7 @@ installMeta() "STABLE"|"DEV") printVerbose "Valid releaseline '${bootstrapMetaReleaseline}' from metadata file" metaReleaseLine="${bootstrapMetaReleaseline}" ;; - *) printVerbose "Invalid releaseline '${bootstrapMetaReleaseline}'; defaulting to STABLE" + *) printVerbose "Invalid releaseline '${bootstrapMetaReleaseline}'; defaulting to STABLE" metaReleaseLine="STABLE" ;; esac @@ -682,7 +722,7 @@ installPrereqs() # - install/update the packages from the repo # - in an offline/airgapped environment, this involves installing the bootstrapped paxes # and copying over the meta that was used in the initial install -updateBootstrappedTools() +updateBootstrappedTools() { printInfo "- Installing/updating bootstrapped curl, jq, gpg, pinentry and meta to install updated versions (if available)" installOptions="--force" @@ -694,45 +734,6 @@ updateBootstrappedTools() "${MYDIR}"/zopen-install "${installOptions}" --yes curl jq pinentry gpg toolInstall=$? elif ${offlineInstall}; then - printVerbose "As offline, no access to meta.pax so copy currently running" - metabindir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P) - ulzm="${rootfs}/usr/local/zopen/meta" - target="${ulzm}/meta-bootstrap" - mkdir -p "${target}" - printVerbose "Copying bare meta scripts from '${metabindir}' to '${target}" - cp -R "${metabindir}/../bin" "${target}" - cp -R "${metabindir}/../include" "${target}" - - printVerbose "Creating main symlink for meta" - ln -s "${ulzm}/meta-bootstrap" "${ulzm}/meta" - if ! runLogProgress "mergeIntoSystem \"meta\" \"${ulzm}/meta-bootstrap\" \"${rootfs}\"" \ - "Merging meta into symlink mesh" "Merged meta into symlink mesh"; then - printSoftError "Unexpected errors merging symlinks into mesh" - printError "Use zopen alt to select previous version to ensure known state" - fi - printVerbose "Adding bootstrapped meta to package db" - printVerbose "Creating basic metadata.json" - cat << EOF > "${target}/metadata.json" -{ - "version_scheme": "0.2", - "product": { - "name": "meta", - "version": "0.0.0.0-bootstrap", - "release": "$(date +%Y%m%d_%H%M%S)", - "summary": "meta on z/OS", - "repo": "https://github.com/zopencommunity/metaport", - "license": "https://github.com/zopencommunity/metaport/blob/main/patches/LICENSE", - "zopen_license": "https://github.com/zopencommunity/metaport/blob/main/LICENSE", - "categories": "utilities", - "size": "$(du -ks "${target}" | awk '{print $1 * 1024}')", - "build_dependencies": [], - "runtime_dependencies": [], - "system_prereqs": [] - } -} -EOF - addToInstallTracker "meta" - printVerbose "Installing bootstrap tools" PAX_LOCATIONS="$CURL_PAX_LOCATION,$JQ_PAX_LOCATION,${PINENTRY_PAX_LOCATION},${GPG_PAX_LOCATION}" @@ -767,7 +768,7 @@ EOF fi done else - installMeta # Need to ensure meta is available + installRemoteMeta # Need to ensure meta is available # shellcheck disable=SC2086 # We want installOptions to be split "${MYDIR}/zopen-install" ${installOptions} --yes curl jq pinentry gpg toolInstall=$? From 4245c330657a0bd9bc4a4a4f258fd0962a0eeefa Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Sat, 29 Nov 2025 17:36:36 +0000 Subject: [PATCH 172/179] Fix re-init logic --- bin/zopen-init | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/bin/zopen-init b/bin/zopen-init index 7279d16c7..592d26102 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -334,6 +334,7 @@ init() [ -n "${reinit}" ] && [ "y" = "${reinit}" ] }; then printInfo "- Re-initializing; zopen-config will be re-created" + reinitExisting=true backup="zopen-config.$(date +%Y%m%d%H%M%S)" bupdir=$(dirname "${configFile}")/zopen [ -e "${bupdir}" ] || { mkdir -p "${bupdir}" || printError "Unable to create directory '${bupdir}'"; } @@ -854,15 +855,29 @@ deinit() fi } +# Prompt/pre-fill config questions init +# Check and create if missing required file system layout generateFileSystem +# During install, install the pre-packaged required packages ! ${refresh} && installPrereqs +# On an install, ensure there is a bootstrapped meta to fallback to +# Note this stage +! (${refresh} || ${reinitExisting}) && installLocalMeta +# (Re)Create the config.json control file generateJsonConfiguration +# Create the default repository - even if offline to use as a template generateDefaultRepository +# Create the zopen-config configuration script generateZopenConfig +# Ensure the latest CA certificates are available ! ${offlineInstall} && updateCACert +# Create the analytics collector configuration generateAnalyticsConfiguration +# Create the installation tracking file; if not existing, recreate [ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || createInitialPackageDB +# Update the bootstrapped tools as needed - if offline, just install the +# shipped paxes updateBootstrappedTools deinit From 9414fe34410a8e355cea5dabb9aa46ad6181b29c Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Sat, 29 Nov 2025 17:36:59 +0000 Subject: [PATCH 173/179] Offline install tracker --- bin/zopen-init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/zopen-init b/bin/zopen-init index 592d26102..60f648312 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -469,7 +469,7 @@ generateJsonConfiguration() validateConfigValue "boolean" "skip_broken" "$skip_broken" "false" validateConfigValue "boolean" "skip_size_check" "$skip_size_check" "false" validateConfigValue "boolean" "OVERRIDE_ZOS_TOOLS" "$OVERRIDE_ZOS_TOOLS" "false" - validateConfigValue "boolean" "offline" "$offline" "false" + validateConfigValue "boolean" "offline" "$offlineInstall" "false" if [ ! -e "${jsonConfig}" ]; then printInfo "Generating global configuration" From d4fff2382007ab362fb03ee0381ea157d555b8c1 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 1 Dec 2025 15:41:39 +0000 Subject: [PATCH 174/179] Remove extra line --- bin/zopen-info | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/zopen-info b/bin/zopen-info index 35b022c34..5bc96f0f8 100755 --- a/bin/zopen-info +++ b/bin/zopen-info @@ -289,4 +289,3 @@ checkIfConfigLoaded updateCaches validatePackageList "${package}" printPackageInfo "${package}" "${remote_lookup}" -outputPackageInfo \ No newline at end of file From 02c24688e21004b01b0e2e5057f74569d6f963f4 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 2 Dec 2025 16:25:20 +0000 Subject: [PATCH 175/179] Offline installer stability changes --- include/common.sh | 68 ++++++++++++------- .../installPre/verifySignatureOfPax | 28 ++++---- 2 files changed, 57 insertions(+), 39 deletions(-) diff --git a/include/common.sh b/include/common.sh index 0329fdb30..e5b1b6b9d 100755 --- a/include/common.sh +++ b/include/common.sh @@ -2002,7 +2002,10 @@ getReleaseLineFromInstalledPkg(){ fi case "${installedReleaseline}" in "STABLE"|"DEV") printVerbose "Valid releaseline '${installedReleaseline}' from metadata file";; - *) printError "Invalid releaseline '${installedReleaseline}' for package '${repo}'" + "null") printVerbose "Explicit release line for package '${repo}' unknown; git clone perhaps." + installedReleaseline="UNKNOWN" + ;; + *) printError "Invalid releaseline '${installedReleaseline}' for package '${repo}'" ;; esac echo "${installedReleaseline}" return 0 @@ -2011,32 +2014,44 @@ getReleaseLineFromInstalledPkg(){ calculateReleaseLineMetadata() { repo="$1" + validatedReleaseLine="UNKNOWN" printDebug "No explicit version/tag/releaseline, checking for pre-existing package&releaseline" if isPackageActive "${repo}"; then - printVerbose "Package '${repo}' already installed; determining releaseline" + printVerbose "Package '${repo}' already installed; attempting to determine releaseline from metadata" if ! validatedReleaseLine=$(getReleaseLineFromInstalledPkg "${repo}"); then return 1 fi - else - printDebug "Checking for system-configured releaseline" - if [ -e "${ZOPEN_JSON_CONFIG}" ]; then - printDebug "Using v2 configuration: '${ZOPEN_JSON_CONFIG}'" - sysrelline=$(jq -re '.release_line' "${ZOPEN_JSON_CONFIG}") - elif [ -e "${ZOPEN_ROOTFS}/etc/zopen/releaseline" ] ; then - printDebug "Using legacy file-based config" - sysrelline=$(awk ' {print toupper($1)}') < "${ZOPEN_ROOTFS}/etc/zopen/releaseline" - fi - printDebug "Validating value: ${sysrelline}" - validatedReleaseLine=$(validateReleaseLine "${sysrelline}") - if [ -n "${validatedReleaseLine}" ]; then - printDebug "zopen system configured to use releaseline '${sysrelline}'; restricting to that releaseline" - else - printWarning "zopen misconfigured to use an unknown releaseline of '${sysrelline}'; defaulting to STABLE packages" - printWarning "Set the contents of '${ZOPEN_ROOTFS}/etc/zopen/releaseline' to a valid value to remove this message" - printWarning "Valid values are: DEV | STABLE" - validatedReleaseLine="STABLE" - fi fi + case "${validatedReleaseLine}" in + "UNKNOWN") + printVerbose "No specific releaseline from package itself. Checking for system-configured releaseline" + if [ -e "${ZOPEN_JSON_CONFIG}" ]; then + printDebug "Using v2 configuration: '${ZOPEN_JSON_CONFIG}'" + sysrelline=$(jq -re '.release_line' "${ZOPEN_JSON_CONFIG}") + elif [ -e "${ZOPEN_ROOTFS}/etc/zopen/releaseline" ] ; then + printDebug "Using legacy file-based config" + sysrelline=$(awk ' {print toupper($1)}') < "${ZOPEN_ROOTFS}/etc/zopen/releaseline" + else + printWarning "Cannot determine system or package release line (STABLE|DEV) for package '{repo}'" + printWarning "Using default of 'STABLE'" + validatedReleaseLine="STABLE" + fi + if [ -n "${sysrelline}" ]; then + printDebug "Validating value: ${sysrelline}" + validatedReleaseLine=$(validateReleaseLine "${sysrelline}") + if [ -n "${validatedReleaseLine}" ]; then + printVerbose "zopen system configured to use releaseline '${sysrelline}'; restricting to that releaseline" + else + printWarning "zopen misconfigured to use an unknown releaseline of '${sysrelline}'; defaulting to STABLE packages" + printWarning "Set the contents of '${ZOPEN_ROOTFS}/etc/zopen/releaseline' to a valid value to remove this message" + printWarning "Valid values are: DEV | STABLE" + validatedReleaseLine="STABLE" + fi + fi + ;; + STABLE|DEV) printVerbose "Using releaseline '${validatedReleaseLine}' for package '${repo}'" ;; + *) assertFailed "Invalid value for relaseline '${validatedReleaseLine}' calculated!" ;; + esac # We have some situations that could arise # 1. the port being installed has no releaseline tagging yet (ie. no releases tagged STABLE_* or DEV_*) @@ -2173,11 +2188,16 @@ createDependancyGraph() return 0 fi printDebug "Adding dependencies to install graph" - addToInstallGraph "dependancy" "${invalidPortAssetFile}" "${missing}" + if ! addToInstallGraph "dependancy" "${invalidPortAssetFile}" "${missing}"; then + return 1 + fi # Recurse in case the now-installing dependencies themselves have dependencies # Recursive dependencies should not break as the initial package will have been # marked for installation - createDependancyGraph "${invalidPortAssetFile}" + if ! createDependancyGraph "${invalidPortAssetFile}"; then + return 1 + fi + return 0 } # addToInstallGraph @@ -2195,7 +2215,7 @@ addToInstallGraph(){ printDebug "Adding ${pkgList} to install graph" for portRequested in ${pkgList}; do if ! getPortMetaData "${portRequested}" "${invalidPortAssetFile}"; then - continue + break fi ## Merge asset into JSON install list installList=$(echo "${installList}" | \ diff --git a/include/scriptlets/installPre/verifySignatureOfPax b/include/scriptlets/installPre/verifySignatureOfPax index 10e7d1a32..9420fb3ee 100644 --- a/include/scriptlets/installPre/verifySignatureOfPax +++ b/include/scriptlets/installPre/verifySignatureOfPax @@ -9,7 +9,11 @@ if ${skipverify}; then printWarning "Package validation with GPG explicitly skipped" return 0 fi -printInfo "- Performing GPG signature verification for '${pkg}' pax file" +printInfo "- Performing GPG signature verification for '${pkg}' pax file '${paxFile}'" +if ! [ -e "${metadataFile}" ]; then + printError "Metadata not found for '${pkg}'" + return 1 +fi metadataVersion=$(jq -r '.version_scheme' "${metadataFile}" 2>/dev/null) [ -n "${metadataVersion}" ] || printError "Metadata version could not be read from metadata file '${metadataFile}'" is_greater=$(echo "$metadataVersion > 0.1" | bc -l) @@ -34,6 +38,7 @@ extractFromMetaData(){ mdf=$2 if ! sRc=$(jq -e -r "${key}" "${mdf}"); then mdf="${ZOPEN_ROOTFS}/var/cache/zopen/$(basename "$3").json" + [ -e "${mdf}" ] || return 1 if ! sRc=$(jq -e -r "${key}" "${mdf}"); then return 1 fi @@ -42,20 +47,14 @@ extractFromMetaData(){ } # Extracting values and checking for errors -# If this is a local pax file, then there might not be a .product.pax in the -# metadata; check for a corresponding .json file in the cache. -# Local paxes need to be installed with --skipverify until they contain the -# .product.pax (and .signature and .public_key) entry -#if ! FILE_TO_VERIFY=$(jq -e -r '.product.pax' "${metadataFile}"); then -# printVerbose "Could not locate .product.pax in the pax's metadata. Checking cache..." -# metadataFile="${ZOPEN_ROOTFS}/var/cache/zopen/${paxFile}.json" -# if ! FILE_TO_VERIFY=$(jq -e -r '.product.pax' "${metadataFile}"); then -# printSoftError "Failed to extract 'pax' from ${metadataFile}" >&2 -# return 1 -# fi -#fi +# If this is a local pax file, then there should not be a .product.pax in the +# metadata. Local paxes need to be installed with --skipverify since do not +# contain the .product.pax and .signature and .public_key entries could be invalid +# and unreliable/untrustworthy if ! FILE_TO_VERIFY=$(extractFromMetaData ".product.pax" "${metadataFile}" "${paxFile}"); then - printSoftError "Failed to extract '.product.pax' from package metadata" >&2 + printWarning "Metadata not found for '${pkg}' - package validation cannot be performed" + printWarning "Use the --skip-verify parameter or configure the zopen environment" + printWarning "to enable local/offline installations without package verification." return 1 fi @@ -130,7 +129,6 @@ addCleanupTrapCmd "rm -rf ${KEYRING_FILE}~" # Also remove a potential lock file return 1 fi -# gpg_output=$(gpgCmd --no-default-keyring --keyring "${KEYRING_FILE}" --verify "${SIGNATURE_FILE}" "${FILE_TO_VERIFY}" 2>&1) gpg_output=$(gpgCmd --no-default-keyring --keyring "${KEYRING_FILE}" --verify "${SIGNATURE_FILE}" "${paxFile}" 2>&1) printVerbose "${gpg_output}" if ! echo "${gpg_output}" | grep -q "Good signature from"; then From 1936a784a1039ea637d08999533ef39e60e442c7 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Tue, 2 Dec 2025 16:26:04 +0000 Subject: [PATCH 176/179] Do not update tools on a meta refresh --- bin/zopen-init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/zopen-init b/bin/zopen-init index 60f648312..2cc09f0e7 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -878,6 +878,6 @@ generateAnalyticsConfiguration [ -e "${ZOPEN_ROOTFS}/var/lib/zopen/packageDB.json" ] || createInitialPackageDB # Update the bootstrapped tools as needed - if offline, just install the # shipped paxes -updateBootstrappedTools +! ${refresh} && updateBootstrappedTools deinit From 40cb5da993b57f3d62d4c457af4bcdcc4ed507f3 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 8 Dec 2025 15:49:21 +0000 Subject: [PATCH 177/179] Additional returns from cd --- bin/zopen-clean | 2 +- bin/zopen-init | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/zopen-clean b/bin/zopen-clean index c52ffe223..7182f868c 100755 --- a/bin/zopen-clean +++ b/bin/zopen-clean @@ -78,7 +78,7 @@ removeUnusedPackageVersions() counterfile=$(mktempfile "clean" ".rupv") addCleanupTrapCmd "rm -f ${counterfile}" - cd "${ZOPEN_PKGINSTALL}/${needle}" && find . -name "./*" -prune -type d > "${counterfile}" + (cd "${ZOPEN_PKGINSTALL}/${needle}" && find . -name "./*" -prune -type d > "${counterfile}") unusedCount=0 while read repo; do printVerbose "Parsing repo: '${repo}' as '${repo#./}'" diff --git a/bin/zopen-init b/bin/zopen-init index 2cc09f0e7..9268923b0 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -704,18 +704,19 @@ installPrereqs() done # Sourcing the environment from within the actual directories + curwd="$(pwd -P)" for tool in "curl" "jq" "pinentry" "gnupg"; do printVerbose "Sourcing environment to trigger any setup required for ${tool}" tooldir=$(ls "${rootfs}/boot" | grep ${tool} | head -1) [ -e "${rootfs}/boot/${tooldir}" ] || printError "Could not locate ${tool} directory '${rootfs}/boot/${tooldir}/' for bootstrap. Re-run 'zopen init' to retry" chmod -R 755 "${rootfs}/boot/${tooldir}" - curwd="${PWD}" cd "${rootfs}/boot/${tooldir}/" || printError "Could not access ${tool} bootstrap in directory '${rootfs}/boot/${tooldir}/'. Re-run 'zopen init' to retry" # shellcheck disable=SC1091 . ./.env cd "${curwd}" || printError "Could not change to ${curwd}. Re-run 'zopen init' to retry" done + cd "${curwd}" || printError "Could not return to ${curwd}." printInfo "- Sourcing environment" } From dfda0b3e20c5240e07ba736643ade28329bb6787 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 8 Dec 2025 15:49:38 +0000 Subject: [PATCH 178/179] Adding -x to debug --- bin/zopen-build | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/bin/zopen-build b/bin/zopen-build index f6319de4f..8d33daf3a 100755 --- a/bin/zopen-build +++ b/bin/zopen-build @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -x # # General purpose build script for zopen community ports - https://github.com/zopencommunity # @@ -760,6 +760,7 @@ setDepsEnv() deps="${ZOPEN_DEPS}" orig="${PWD}" + printVerbose "Saving original work directory '${orig}'" # Filter out duplicate deps deps=$(echo "${deps}" | xargs | tr ' ' '\n' | sort -u) for dep in ${deps}; do @@ -787,10 +788,13 @@ setDepsEnv() fi printVerbose "Setting up ${depdir} dependency environment" - cd "${depdir}" && . ./.env + curDir="$(pwd -P)" + cd "${depdir}" || printError "Failed to change to '${depdir}'" + . ./.env if [ $? -gt 0 ]; then printError "Failed to source ${depdir} .env" fi + cd "${curDir}" || printError "Failed to return to '${curDir}'" fi foundDep=true break @@ -812,10 +816,13 @@ setDepsEnv() fi depdir="${path}/${dep}/${dep}" printVerbose "Setting up upgraded ${depdir} dependency environment" - cd "${depdir}" && . ./.env + curDir="$(pwd -P)" + cd "${depdir}" || printError "Failed to change to '${depdir}'" + ./.env if [ $? -gt 0 ]; then printError "Failed to source ${depdir} .env" fi + cd "${curDir}" || printError "Failed to change back to '${curDir}'" versionPath="${depdir}/.version" if [ -r "${versionPath}" ]; then version=$(cat "${versionPath}") @@ -825,7 +832,8 @@ setDepsEnv() fi fi done - cd "${orig}" || exit 99 + cd "${orig}" || printError "Unable to change back to '${orig}'" + printVerbose "Changed back to directory '${orig}'" } setEnv() @@ -1043,6 +1051,7 @@ extractTarBall() fi rm -f "${tarballz}" tagTree "${sourcedir}" + curDir="$(pwd -P)" # TODO: Do we need to return here?? cd "${sourcedir}" || printError "Cannot cd to ${sourcedir}" # Clean up .git* files since we will be creating our own local git repo for applying patches From d1f9df84cebb345e614d62e6341bdc95baad6502 Mon Sep 17 00:00:00 2001 From: Devonian Teuchter Date: Mon, 8 Dec 2025 18:01:01 +0000 Subject: [PATCH 179/179] debug --- bin/zopen | 2 +- bin/zopen-alt | 2 +- bin/zopen-clean | 2 +- bin/zopen-init | 2 +- bin/zopen-install | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/zopen b/bin/zopen index afb9e5914..432b38c55 100755 --- a/bin/zopen +++ b/bin/zopen @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -x # # zopen wrapper script to allow single point of entry to zopen # tooling. diff --git a/bin/zopen-alt b/bin/zopen-alt index 08873ae0d..99418764f 100755 --- a/bin/zopen-alt +++ b/bin/zopen-alt @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -x # # "Alternatives" utility for zopen community - https://github.com/zopencommunity # diff --git a/bin/zopen-clean b/bin/zopen-clean index 7182f868c..78b157344 100755 --- a/bin/zopen-clean +++ b/bin/zopen-clean @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -x # # Cleanup utility for zopen community - https://github.com/zopencommunity # diff --git a/bin/zopen-init b/bin/zopen-init index 9268923b0..c7484351e 100755 --- a/bin/zopen-init +++ b/bin/zopen-init @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -x # Initialize zopen environment - https://github.com/zopencommunity # ME=$(basename "$0") diff --git a/bin/zopen-install b/bin/zopen-install index 7de84f03b..90c821621 100755 --- a/bin/zopen-install +++ b/bin/zopen-install @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -x # # Install utility for zopen community - https://github.com/zopencommunity #