feat: dnf module

modules.json

@@ -10,6 +10,7 @@
   "https://raw.githubusercontent.com/blue-build/modules/main/modules/gschema-overrides/module.yml",
   "https://raw.githubusercontent.com/blue-build/modules/main/modules/justfiles/module.yml",  
   "https://raw.githubusercontent.com/blue-build/modules/main/modules/rpm-ostree/module.yml",
+  "https://raw.githubusercontent.com/blue-build/modules/main/modules/dnf/module.yml",
   "https://raw.githubusercontent.com/blue-build/modules/main/modules/initramfs/module.yml",
   "https://raw.githubusercontent.com/blue-build/modules/main/modules/script/module.yml",
   "https://raw.githubusercontent.com/blue-build/modules/main/modules/signing/module.yml",

modules/dnf/README.md

@@ -0,0 +1,37 @@
+# `dnf`
+
+The [`dnf`](https://docs.fedoraproject.org/en-US/quick-docs/dnf/) module offers pseudo-declarative package and repository management using `dnf`.
+
+The module first downloads the repository files from URLs declared under `repos:` into `/etc/yum.repos.d/`. The magic string `%OS_VERSION%` can be substituted with the current VERSION_ID (major Fedora version), which can be used, for example, for pulling correct versions of repositories which have fixed Fedora version in the URL.
+
+You can also add repository files directly into your git repository if URLs are not provided. For example:
+```yml
+repos:
+   - my-repository.repo # copies in .repo file from files/dnf/my-repository.repo to /etc/yum.repos.d/
+```
+
+Specific COPR repositories can also be specified in `COPR user/project` format & is prefered over using direct COPR URL.
+
+If you use a repo that requires adding custom keys (eg. Brave Browser), you can import the keys by declaring the key URLs under `keys:`. The magic string acts the same as it does in `repos`.
+
+Then the module installs the packages declared under `install:` using `dnf -y install --refresh`, it removes the packages declared under `remove:` using `dnf -y remove`. If there are packages declared under both `install:` and `remove:` then removal is performed 1st & install 2nd.
+
+Installing RPM packages directly from a `http(s)` url that points to the RPM file is also supported, you can just put the URLs under `install:` and they'll be installed along with the other packages. The magic string `%OS_VERSION%` is substituted with the current VERSION_ID (major Fedora version) like with the `repos:` property.
+
+If an RPM is not available in a repository or as an URL, you can also install it directly from a file in your git repository. For example:
+```yml
+install:
+   - weird-package.rpm # tries to install files/dnf/weird-package.rpm
+```
+
+Additionally, the `dnf` module supports a fix for packages that install into `/opt/`. Installation for packages that install into folder names declared under `optfix:` are fixed using some symlinks. Directory path in `/opt/` for those packages should be provided in recipe, like in Example Configuration.
+
+There is an option to install & remove RPM groups if desired in `group-install:` & `group-remove:`. RPM groups removal & installation always run before packages removal & installation. To see the list of all available RPM groups, you can use `dnf group list` command.
+
+The module can also replace base RPM packages with packages from any repo. Under `replace:`, the module finds every pair of keys `- from-repo:` and `packages:`. (Multiple pairs are supported.) The module uses `- from-repo:` key to gather the repo for package replacement, then it replaces packages declared under `packages:` using the command `dnf -y distro-sync --refresh --repo "${repo}" "${packages}"`. The magic string `%OS_VERSION%` is substituted with the current VERSION_ID (major Fedora version) as already said above. You need to assure that you provided the repo in `repos:` before using replacement functionality. To gather the repo ID that you need to input, you can use `dnf repo list` command.
+
+:::note
+[Removed packages are still present in the underlying ostree repository](https://coreos.github.io/rpm-ostree/administrator-handbook/#removing-a-base-package), what `remove` does is kind of like hiding them from the system, it doesn't free up storage space.
+:::
+
+There is also an `install-weak-dependencies:` option to enable or disable installation of weak dependencies for every install operation. Weak dependencies are installed by default. Which kind of dependencies are considered weak can be seen [here](https://docs.fedoraproject.org/en-US/packaging-guidelines/WeakDependencies/).

modules/dnf/dnf.sh

@@ -0,0 +1,231 @@
+#!/usr/bin/env bash
+
+# Tell build process to exit if there are any errors.
+set -euo pipefail
+
+# Fail the build if dnf5 isn't installed
+if ! rpm -q dnf5 &>/dev/null; then
+  echo "ERROR: Main dependency 'dnf5' is not installed. Install 'dnf5' before using this module to solve this error."
+  exit 1
+fi
+
+# Fail the build if dnf5 plugins aren't installed
+if ! rpm -q dnf5-plugins &>/dev/null; then
+  echo "ERROR: Dependency 'dnf5-plugins' is not installed. It is needed for cleanly adding COPR repositories."
+  echo "       Install 'dnf5-plugins' before using this module to solve this error."
+  exit 1
+fi
+
+# Pull in repos
+get_json_array REPOS 'try .["repos"][]' "${1}"
+if [[ ${#REPOS[@]} -gt 0 ]]; then
+  echo "Adding repositories"
+  # Substitute %OS_VERSION% & remove newlines/whitespaces from all repo entries
+  for i in "${!REPOS[@]}"; do
+      repo="${REPOS[$i]}"
+      repo="${repo//%OS_VERSION%/${OS_VERSION}}"
+      REPOS[$i]="${repo//[$'\t\r\n ']}"
+      # Remove spaces/newlines for all repos other than COPR
+      if [[ "${repo}" != "COPR "* ]]; then
+        REPOS[$i]="${repo//[$'\t\r\n ']}"
+      else
+        REPOS[$i]="${repo}"
+      fi
+  done
+  # dnf config-manager & dnf copr don't support adding multiple repositories at once, hence why for/done loop is used
+  for repo in "${REPOS[@]}"; do
+      if [[ "${repo}" =~ ^https?:\/\/.* ]]; then
+        echo "Adding repository URL: '${repo}'"
+        dnf -y config-manager addrepo --from-repofile="${repo}"
+      elif [[ "${repo}" == *".repo" ]] && [[ -f "${CONFIG_DIRECTORY}/dnf/${repo}" ]]; then
+        echo "Adding repository file: '${repo##*/}'"
+        dnf -y config-manager addrepo --from-repofile="${CONFIG_DIRECTORY}/dnf/${repo}"
+      elif [[ "${repo}" == "COPR "* ]]; then
+        echo "Adding COPR repository: '${repo#COPR }'"
+        dnf -y copr enable "${repo#COPR }"
+      fi    
+  done
+fi
+
+# Install RPM keys if they are provided
+get_json_array KEYS 'try .["keys"][]' "${1}" 
+if [[ ${#KEYS[@]} -gt 0 ]]; then
+    echo "Adding keys"
+    for KEY in "${KEYS[@]}"; do
+        KEY="${KEY//%OS_VERSION%/${OS_VERSION}}"
+        rpm --import "${KEY//[$'\t\r\n ']}"
+    done
+fi
+
+# Create symlinks to fix packages that create directories in /opt
+get_json_array OPTFIX 'try .["optfix"][]' "${1}"
+if [[ ${#OPTFIX[@]} -gt 0 ]]; then
+    echo "Creating symlinks to fix packages that install to /opt"
+    # Create symlink for /opt to /var/opt since it is not created in the image yet
+    mkdir -p "/var/opt"
+    ln -snf "/var/opt" "/opt"
+    # Create symlinks for each directory specified in recipe.yml
+    for OPTPKG in "${OPTFIX[@]}"; do
+        OPTPKG="${OPTPKG%\"}"
+        OPTPKG="${OPTPKG#\"}"
+        mkdir -p "/usr/lib/opt/${OPTPKG}"
+        ln -s "../../usr/lib/opt/${OPTPKG}" "/var/opt/${OPTPKG}"
+        echo "Created symlinks for ${OPTPKG}"
+    done
+fi
+
+# Install & remove group packages
+get_json_array GROUP_INSTALL 'try .["group-install"][]' "${1}"
+get_json_array GROUP_REMOVE 'try .["group-remove"][]' "${1}"
+
+if [[ ${#GROUP_INSTALL[@]} -gt 0 && ${#GROUP_REMOVE[@]} -gt 0 ]]; then
+    echo "Removing & Installing RPM groups"
+    echo "Removing: ${GROUP_REMOVE[*]}"
+    echo "Installing: ${GROUP_INSTALL[*]}"
+    dnf -y group remove "${GROUP_REMOVE[@]}"
+    dnf -y group install --refresh "${GROUP_INSTALL[@]}"
+elif [[ ${#GROUP_INSTALL[@]} -gt 0 ]]; then
+    echo "Installing RPM groups"
+    echo "Installing: ${GROUP_INSTALL[*]}"
+    dnf -y group install --refresh "${GROUP_INSTALL[@]}"
+elif [[ ${#GROUP_REMOVE[@]} -gt 0 ]]; then
+    echo "Removing RPM groups"
+    echo "Removing: ${GROUP_REMOVE[*]}"
+    dnf -y remove "${GROUP_REMOVE[@]}"
+fi
+
+get_json_array INSTALL_PKGS 'try .["install"][]' "${1}"
+get_json_array REMOVE_PKGS 'try .["remove"][]' "${1}"
+
+CLASSIC_INSTALL=false
+HTTPS_INSTALL=false
+LOCAL_INSTALL=false
+
+# Sort classic, URL & local install packages
+if [[ ${#INSTALL_PKGS[@]} -gt 0 ]]; then
+  for i in "${!INSTALL_PKGS[@]}"; do
+      PKG="${INSTALL_PKGS[$i]}"
+      if [[ "${PKG}" =~ ^https?:\/\/.* ]]; then
+        INSTALL_PKGS[$i]="${PKG//%OS_VERSION%/${OS_VERSION}}"
+        HTTPS_INSTALL=true
+        HTTPS_PKGS+=("${INSTALL_PKGS[$i]}")
+      elif [[ ! "${PKG}" =~ ^https?:\/\/.* ]] && [[ -f "${CONFIG_DIRECTORY}/dnf/${PKG}" ]]; then
+        LOCAL_INSTALL=true
+        LOCAL_PKGS+=("${CONFIG_DIRECTORY}/dnf/${PKG}")
+      else
+        CLASSIC_INSTALL=true
+        CLASSIC_PKGS+=("${PKG}")
+      fi
+  done
+fi
+
+# Function to inform the user about which type of packages is he installing
+echo_rpm_install() {
+    if ${CLASSIC_INSTALL} && ! ${HTTPS_INSTALL} && ! ${LOCAL_INSTALL}; then
+      echo "Installing: ${CLASSIC_PKGS[*]}"
+    elif ! ${CLASSIC_INSTALL} && ${HTTPS_INSTALL} && ! ${LOCAL_INSTALL}; then
+      echo "Installing package(s) directly from URL: ${HTTPS_PKGS[*]}"
+    elif ! ${CLASSIC_INSTALL} && ! ${HTTPS_INSTALL} && ${LOCAL_INSTALL}; then
+      echo "Installing local package(s): ${LOCAL_PKGS[*]}"
+    elif ${CLASSIC_INSTALL} && ${HTTPS_INSTALL} && ! ${LOCAL_INSTALL}; then
+      echo "Installing: ${CLASSIC_PKGS[*]}"
+      echo "Installing package(s) directly from URL: ${HTTPS_PKGS[*]}"
+    elif ${CLASSIC_INSTALL} && ! ${HTTPS_INSTALL} && ${LOCAL_INSTALL}; then
+      echo "Installing: ${CLASSIC_PKGS[*]}"
+      echo "Installing local package(s): ${LOCAL_PKGS[*]}"
+    elif ! ${CLASSIC_INSTALL} && ${HTTPS_INSTALL} && ${LOCAL_INSTALL}; then
+      echo "Installing package(s) directly from URL: ${HTTPS_PKGS[*]}"    
+      echo "Installing local package(s): ${LOCAL_PKGS[*]}"
+    elif ${CLASSIC_INSTALL} && ${HTTPS_INSTALL} && ${LOCAL_INSTALL}; then
+      echo "Installing: ${CLASSIC_PKGS[*]}"
+      echo "Installing package(s) directly from URL: ${HTTPS_PKGS[*]}"
+      echo "Installing local package(s): ${LOCAL_PKGS[*]}"
+    fi
+}
+
+# Remove & install RPM packages
+if [[ ${#INSTALL_PKGS[@]} -gt 0 && ${#REMOVE_PKGS[@]} -gt 0 ]]; then
+    echo "Removing & Installing RPMs"
+    echo "Removing: ${REMOVE_PKGS[*]}"
+    echo_rpm_install
+    dnf -y remove "${REMOVE_PKGS[@]}"
+    dnf -y install --refresh "${INSTALL_PKGS[@]}"
+elif [[ ${#INSTALL_PKGS[@]} -gt 0 ]]; then
+    echo "Installing RPMs"
+    echo_rpm_install
+    dnf -y install --refresh "${INSTALL_PKGS[@]}"
+elif [[ ${#REMOVE_PKGS[@]} -gt 0 ]]; then
+    echo "Removing RPMs"
+    echo "Removing: ${REMOVE_PKGS[*]}"
+    dnf -y remove "${REMOVE_PKGS[@]}"
+fi
+
+get_json_array REPLACE 'try .["replace"][]' "$1"
+
+# Replace RPM packages from any repository
+if [[ ${#REPLACE[@]} -gt 0 ]]; then
+    for REPLACEMENT in "${REPLACE[@]}"; do
+
+        # Get repository
+        REPO=$(echo "${REPLACEMENT}" | jq -r 'try .["from-repo"]')
+        REPO="${REPO//%OS_VERSION%/${OS_VERSION}}"
+        REPO="${REPO//[$'\t\r\n ']}"
+
+        # Ensure repository is provided
+        if [[ "${REPO}" == "null" ]] || [[ -z "${REPO}" ]]; then
+            echo "ERROR: Key 'from-repo' was declared, but repository URL was not provided."
+            exit 1
+        fi
+
+        # Get packages to replace
+        get_json_array PACKAGES 'try .["packages"][]' "${REPLACEMENT}"
+
+        # Ensure packages are provided
+        if [[ ${#PACKAGES[@]} -eq 0 ]]; then
+            echo "ERROR: No packages were provided for repository '${REPO}'."
+            exit 1
+        fi
+
+        # Get if 'install-weak-dependencies' is provided
+        WEAK_DEPENDENCIES=$(echo "${REPLACEMENT}" | jq -r 'try .["install-weak-dependencies"]')
+
+        if [[ -z "${WEAK_DEPENDENCIES}" ]] || [[ "${WEAK_DEPENDENCIES}" == "null" ]] || [[ "${WEAK_DEPENDENCIES}" == "true" ]]; then
+          WEAK_DEPS_FLAG="--setopt=install_weak_deps=True"
+        elif [[ "${WEAK_DEPENDENCIES}" == "false" ]]; then
+          WEAK_DEPS_FLAG="--setopt=install_weak_deps=False"
+        fi
+
+        # Get if 'skip-unavailable-packages' is provided
+        SKIP_UNAVAILABLE=$(echo "${REPLACEMENT}" | jq -r 'try .["skip-unavailable-packages"]')
+
+        if [[ -z "${SKIP_UNAVAILABLE}" ]] || [[ "${SKIP_UNAVAILABLE}" == "null" ]] || [[ "${SKIP_UNAVAILABLE}" == "false" ]]; then
+          SKIP_UNAVAILABLE_FLAG=""
+        elif [[ "${SKIP_UNAVAILABLE}" == "true" ]]; then
+          SKIP_UNAVAILABLE_FLAG="--skip-unavailable"
+        fi
+
+        # Get if 'skip-broken-packages' is provided
+        SKIP_BROKEN=$(echo "${REPLACEMENT}" | jq -r 'try .["skip-broken-packages"]')
+
+        if [[ -z "${SKIP_BROKEN}" ]] || [[ "${SKIP_BROKEN}" == "null" ]] || [[ "${SKIP_BROKEN}" == "false" ]]; then
+          SKIP_BROKEN_FLAG=""
+        elif [[ "${SKIP_BROKEN}" == "true" ]]; then
+          SKIP_BROKEN_FLAG="--skip-broken"
+        fi       
+
+        # Get if 'allow-erasing-packages' is provided
+        ALLOW_ERASING=$(echo "${REPLACEMENT}" | jq -r 'try .["allow-erasing-packages"]')
+
+        if [[ -z "${ALLOW_ERASING}" ]] || [[ "${ALLOW_ERASING}" == "null" ]] || [[ "${ALLOW_ERASING}" == "false" ]]; then
+          ALLOW_ERASING_FLAG=""
+        elif [[ "${ALLOW_ERASING}" == "true" ]]; then
+          ALLOW_ERASING_FLAG="--allowerasing"
+        fi   
+
+        echo "Replacing packages from repository: '${REPO}'"
+        echo "Replacing: ${PACKAGES[*]}"
+
+        dnf -y "${WEAK_DEPS_FLAG}" distro-sync --refresh "${SKIP_UNAVAILABLE_FLAG}" "${SKIP_BROKEN_FLAG}" "${ALLOW_ERASING_FLAG}" --repo "${REPO}" "${PACKAGES[@]}"
+
+    done
+fi

modules/dnf/dnf.tsp

@@ -0,0 +1,47 @@
+import "@typespec/json-schema";
+using TypeSpec.JsonSchema;
+
+@jsonSchema("/modules/dnf.json")
+model DnfModule {
+    /** The dnf module offers pseudo-declarative package and repository management using dnf.
+     * https://blue-build.org/reference/modules/dnf/
+     */
+    type: "dnf";
+
+    /** List of links to .repo files to download into /etc/yum.repos.d/. */
+    repos?: Array<string>;
+
+    /** List of links to key files to import for installing from custom repositories. */
+    keys?: Array<string>;
+
+    /** List of folder names under /opt/ to enable for installing into. */
+    optfix?: Array<string>;
+
+    /** List of RPM groups to remove. */
+    "group-remove"?: Array<string>;
+
+    /** List of RPM groups to install. */
+    "group-install"?: Array<string>;
+
+    /** List of RPM packages to remove. */
+    remove?: Array<string>;
+
+    /** List of RPM packages to install. */
+    install?: Array<string>;
+
+    /** List of configurations for replacing packages from another repo. */
+    replace?: Array<{
+        /** URL to the source COPR repo for the new packages. */
+        "from-repo": string,
+        /** List of packages to replace using packages from the defined repo. */
+        packages: Array<string>,
+        /** Whether to install weak dependencies during the replacement or not. */
+        "install-weak-dependencies"?: boolean = true,
+        /** Whether to continue with the replacement if there are no packages available on the system to replace. */
+        "skip-unavailable-packages"?: boolean = false,
+        /** Whether to continue with the replacement if there are broken packages in the system during the replacement. */
+        "skip-broken-packages"?: boolean = false,
+        /** Whether to allow erasing (removal) of packages in case of dependency problems during the replacement. */
+        "allow-erasing-packages"?: boolean = false;        
+    }>;
+}

modules/dnf/module.yml

@@ -0,0 +1,31 @@
+name: dnf
+shortdesc: The dnf module offers pseudo-declarative package and repository management using dnf.
+example: |
+  type: dnf
+  install-weak-dependencies: true
+  repos:
+    - COPR atim/starship
+    - COPR trixieua/mutter-patched
+    - https://brave-browser-rpm-release.s3.brave.com/brave-browser.repo
+  keys:
+    - https://brave-browser-rpm-release.s3.brave.com/brave-core.asc
+  optfix:
+    - Tabby # needed because tabby installs into /opt/Tabby/
+    - brave.com
+  group-install: 
+    - cosmic-desktop
+    - cosmic-desktop-apps # Installs Cosmic desktop environment
+    - window-managers
+  install:
+    - starship
+    - brave-browser
+    - https://github.com/Eugeny/tabby/releases/download/v1.0.209/tabby-1.0.209-linux-x64.rpm
+  remove:
+    - firefox
+    - firefox-langpacks
+  replace:
+    - from-repo: copr:copr.fedorainfracloud.org:trixieua:mutter-patched
+      packages:
+        - mutter
+        - mutter-common
+        - gdm

modules/rpm-ostree/rpm-ostree.sh

 rm "/etc/yum.repos.d/${FILE_NAME}" 
 if [[ ${#REPOS[@]} -gt 0 ]]; then 
 INSTALL_PKGS[$i]="${PKG//%OS_VERSION%/${OS_VERSION}}" 
 LOCAL_INSTALL=true 
 elif ${CLASSIC_INSTALL} && ${HTTPS_INSTALL} && ! ${LOCAL_INSTALL}; then 
 rpm-ostree install "${HTTPS_PKGS[@]}" 
 rpm-ostree install "${LOCAL_PKGS[@]}" 
 rpm-ostree install "${HTTPS_PKGS[@]}" "${LOCAL_PKGS[@]}" 
 rm "/etc/yum.repos.d/${FILE_NAME}" 
@@ -45,7 +45,7 @@ if [[ ${#OPTFIX[@]} -gt 0 ]]; then
     echo "Creating symlinks to fix packages that install to /opt"
     # Create symlink for /opt to /var/opt since it is not created in the image yet
     mkdir -p "/var/opt"
-    ln -s "/var/opt"  "/opt"
+    ln -snf "/var/opt" "/opt"
     # Create symlinks for each directory specified in recipe.yml
     for OPTPKG in "${OPTFIX[@]}"; do
         OPTPKG="${OPTPKG%\"}"