fix: add push images in offline docs

Signed-off-by: redscholar <blacktiledhouse@gmail.com>

Author: redscholar

builtin/core/playbooks/upgrade_cluster.yaml

@@ -0,0 +1,91 @@
+---
+- hosts:
+    - all
+  gather_facts: true
+  roles:
+    - native/root
+
+- import_playbook: hook/pre_install.yaml
+
+# Load default variables and perform prechecks on all hosts
+- hosts:
+    - all
+  gather_facts: true
+  roles:
+    - defaults
+    - precheck
+
+# Download all required software and generate certificates on the localhost
+- hosts:
+    - localhost
+  roles:
+    - certs/init
+    - download
+
+# Initialize all nodes and install necessary software packages
+- hosts:
+    - etcd
+    - k8s_cluster
+    - image_registry
+    - nfs
+  roles:
+    - native
+
+# Upgrade external etcd cluster one by one (only when upgrade.etcd is set)
+- hosts:
+    - etcd
+  serial: 1
+  gather_facts: true
+  roles:
+    - role: etcd/backup
+      when:
+        - .etcd.deployment_type | eq "external"
+        - .upgrade.etcd
+        - .etcd.etcd_version | semverCompare (printf ">v%s" (index .etcd_install_version "stdout" "etcd Version"))
+    - role: etcd/prepare
+      when:
+        - .etcd.deployment_type | eq "external"
+        - .upgrade.etcd
+    - role: etcd/upgrade
+      when:
+        - .etcd.deployment_type | eq "external"
+        - .upgrade.etcd
+
+# Upgrade Kubernetes binaries and container runtime on all cluster nodes
+# cri sub-roles control their own behavior via upgrade.cri internally
+- hosts:
+    - k8s_cluster
+  gather_facts: true
+  roles:
+    - cri
+    - kubernetes/pre-kubernetes
+
+# Upgrade control plane nodes one by one using kubeadm upgrade
+- hosts:
+    - kube_control_plane
+  serial: 1
+  gather_facts: true
+  roles:
+    - role: kubernetes/upgrade-kubernetes
+      when: .kubernetes_install_ActiveState.stdout | eq "active"
+
+# Upgrade worker nodes using kubeadm upgrade node
+- hosts:
+    - kube_worker
+  gather_facts: true
+  roles:
+    - role: kubernetes/upgrade-kubernetes
+      when: .kubernetes_install_ActiveState.stdout | eq "active"
+
+# Upgrade network and storage components if their versions changed
+# Only when upgrade.cni / upgrade.storage_class is set
+- hosts:
+    - kube_control_plane|random
+  gather_facts: true
+  roles:
+    - role: cni
+      when: .upgrade.cni
+    - role: storageclass
+      when: .upgrade.storage_class
+
+- import_playbook: hook/post_install.yaml

builtin/core/roles/cri/containerd/tasks/main.yaml

@@ -1,4 +1,21 @@
 ---
+# Scenario 1: Create cluster or upgrading CRI together.
+# Full containerd installation/upgrade including backup, runc, binaries, config, and certs.
+- name: Containerd | Full containerd installation or upgrade
+  when:
+    - or (.kubernetes_install_LoadState.stdout | default "" | ne "loaded") (.upgrade.cri)
+  block:
+    - name: Containerd | Backup containerd configuration and binaries before upgrade
+      when:
+        - .kubernetes_install_LoadState.stdout | default "" | eq "loaded"
+      command: |
+        BACKUP_DIR="/etc/kubekey/backup/containerd/$(date +%Y%m%d-%H%M%S)"
+        mkdir -p "${BACKUP_DIR}"
+        cp /etc/containerd/config.toml "${BACKUP_DIR}/" 2>/dev/null || true
+        cp /usr/local/bin/containerd "${BACKUP_DIR}/" 2>/dev/null || true
+        cp /usr/local/bin/ctr "${BACKUP_DIR}/" 2>/dev/null || true
+        cp /usr/local/bin/runc "${BACKUP_DIR}/" 2>/dev/null || true
+        echo "✅ Containerd backup saved to ${BACKUP_DIR}"
 
 - name: Containerd | Sync Containerd binaries
   block:
@@ -71,3 +88,13 @@
   when: or .upgrade.cri (or (.containerd_install_version.error | empty | not) (.containerd_install_version.stdout | contains (printf " %s " .cri.containerd_version) | not))
   command: |
     systemctl daemon-reload && systemctl restart containerd.service && systemctl enable containerd.service
+
+# Scenario 2: Upgrading Kubernetes but NOT upgrading CRI.
+# Containerd is already installed and running, skip all installation steps.
+- name: Containerd | Skip containerd operations when not upgrading cri
+  when:
+    - .kubernetes_install_LoadState.stdout | default "" | eq "loaded"
+    - .upgrade.cri | not
+  debug:
+    msg: >-
+      ⏭️ Containerd is not marked for upgrade (upgrade.cri=false). Skipping containerd installation/update. If the installed containerd version is incompatible with Kubernetes {{ .kubernetes.kube_version }}, set --all or --set upgrade.cri=true to upgrade CRI.

builtin/core/roles/cri/crictl/tasks/main.yaml

@@ -5,7 +5,7 @@
   register: crictl_install_version
 
 - name: Crictl | Install and configure crictl if not present or version mismatch
-  when: or (.crictl_install_version.error | empty | not) (.crictl_install_version.stdout | ne (printf "crictl version %s" .cri.crictl_version))
+  when: or (.upgrade.cri) (.crictl_install_version.error | empty | not) (.crictl_install_version.stdout | ne (printf "crictl version %s" .cri.crictl_version))
   block:
     - name: Crictl | Copy crictl binary archive to the remote node
       copy:

builtin/core/roles/cri/docker/tasks/cridockerd.yaml

@@ -1,11 +1,54 @@
 ---
+# cri-dockerd is only needed for Kubernetes >= v1.24.0 (dockershim removal)
+- name: Cridockerd | Skip when Kubernetes version is below v1.24.0
+  when: .kubernetes.kube_version | semverCompare "<v1.24.0"
+  debug:
+    msg: "cri-dockerd is not required for Kubernetes < v1.24.0, skipping."
+
 - name: Cridockerd | Check if cri-dockerd is installed on the system
+  when: .kubernetes.kube_version | semverCompare ">=v1.24.0"
   ignore_errors: true
   command: cri-dockerd --version
   register: cridockerd_install_version
 
-- name: Cridockerd | Install and configure cri-dockerd if not present or version mismatch
-  when: or (.cridockerd_install_version.error | empty | not) (.cridockerd_install_version.stdout | hasPrefix (printf "cri-dockerd %s " .cri.cridockerd_version) | not)
+# When upgrade.cri=true or create cluster: install or upgrade cri-dockerd if version mismatch
+- name: Cridockerd | Install or upgrade cri-dockerd
+  when:
+    - .kubernetes.kube_version | semverCompare ">=v1.24.0"
+    - or (.kubernetes_install_LoadState.stdout | default "" | ne "loaded") (.upgrade.cri)
+    - or (.cridockerd_install_version.error | empty | not) (.cridockerd_install_version.stdout | hasPrefix (printf "cri-dockerd %s " .cri.cridockerd_version) | not)
+  block:
+    - name: Cridockerd | Backup cri-dockerd binary before upgrade
+      when: .kubernetes_install_LoadState.stdout | default "" | eq "loaded"
+      command: |
+        BACKUP_DIR="/etc/kubekey/backup/cridockerd/$(date +%Y%m%d-%H%M%S)"
+        mkdir -p "${BACKUP_DIR}"
+        cp /usr/local/bin/cri-dockerd "${BACKUP_DIR}/" 2>/dev/null || true
+        echo "✅ cri-dockerd backup saved to ${BACKUP_DIR}"
+    - name: Cridockerd | Copy cri-dockerd binary archive to the remote node
+      copy:
+        src: >-
+          {{ .binary_dir }}/cri-dockerd/{{ .cri.cridockerd_version }}/{{ .binary_type }}/cri-dockerd-{{ .cri.cridockerd_version | default "" | trimPrefix "v" }}.{{ .binary_type }}.tgz
+        dest: >-
+          {{ .tmp_dir }}/cri-dockerd-{{ .cri.cridockerd_version | default "" | trimPrefix "v" }}.{{ .binary_type }}.tgz
+    - name: Cridockerd | Extract cri-dockerd binary to /usr/local/bin
+      command: |
+        tar -xvf {{ .tmp_dir }}/cri-dockerd-{{ .cri.cridockerd_version | default "" | trimPrefix "v" }}.{{ .binary_type }}.tgz --strip-components=1 -C /usr/local/bin/
+    - name: Cridockerd | Generate cri-dockerd systemd service file
+      template:
+        src: cri-dockerd.service
+        dest: /etc/systemd/system/cri-dockerd.service
+    - name: Cridockerd | Start and enable the cri-dockerd service
+      command: |
+        systemctl daemon-reload && systemctl start cri-dockerd.service && systemctl enable cri-dockerd.service
+
+# When upgrade.cri=false and K8s is being upgraded: only install cri-dockerd if missing (not upgrade)
+- name: Cridockerd | Install cri-dockerd if missing when not upgrading cri
+  when:
+    - .kubernetes.kube_version | semverCompare ">=v1.24.0"
+    - .kubernetes_install_LoadState.stdout | default "" | eq "loaded"
+    - .upgrade.cri | not
+    - .cridockerd_install_version.error | empty | not
   block:
     - name: Cridockerd | Copy cri-dockerd binary archive to the remote node
       copy:

builtin/core/roles/cri/docker/tasks/main.yaml

@@ -1,3 +1,22 @@
+# Scenario 2: Create cluster or upgrading CRI together.
+# Perform full Docker installation/upgrade (backup, install, certs).
+# cri-dockerd is handled inside the block for K8s >= v1.24.0.
+- name: Docker | Full Docker installation or upgrade
+  when:
+    - or (.kubernetes_install_LoadState.stdout | default "" | ne "loaded") (.upgrade.cri)
+  block:
+    - name: Docker | Backup Docker configuration and binaries before upgrade
+      when:
+        - .kubernetes_install_LoadState.stdout | default "" | eq "loaded"
+      command: |
+        BACKUP_DIR="/etc/kubekey/backup/docker/$(date +%Y%m%d-%H%M%S)"
+        mkdir -p "${BACKUP_DIR}"
+        cp /etc/docker/daemon.json "${BACKUP_DIR}/" 2>/dev/null || true
+        cp /usr/local/bin/docker "${BACKUP_DIR}/" 2>/dev/null || true
+        cp /usr/local/bin/dockerd "${BACKUP_DIR}/" 2>/dev/null || true
+        cp /usr/local/bin/ctr "${BACKUP_DIR}/" 2>/dev/null || true
+        echo "✅ Docker backup saved to ${BACKUP_DIR}"
+
 ---
 # sync docker binary
 - name: Docker | Sync Docker binaries

builtin/core/roles/defaults/defaults/main/01-cluster_require.yaml

@@ -89,4 +89,21 @@ cluster_require:
     v1.33: v3.5.21-0
     v1.34.0: v3.5.21-0
     v1.34.1: v3.5.21-0
-    v1.34: v3.5.24-0
+    v1.34: v3.5.24-0
+
+  # Kubernetes upgrade path: the highest recommended patch version for each minor release.
+  # When a user specifies a minor version (e.g., v1.23) for upgrade, the corresponding
+  # patch version below will be used as the target version.
+  kube_upgrade_path:
+    v1.23: v1.23.17
+    v1.24: v1.24.17
+    v1.25: v1.25.16
+    v1.26: v1.26.15
+    v1.27: v1.27.16
+    v1.28: v1.28.15
+    v1.29: v1.29.15
+    v1.30: v1.30.14
+    v1.31: v1.31.14
+    v1.32: v1.32.13
+    v1.33: v1.33.7
+    v1.34: v1.34.3

builtin/core/roles/defaults/defaults/main/01-main.yaml

@@ -44,9 +44,31 @@ delete:
 # This is typically used with --with-data flag or delete.data: true
   data: false
 
+# Upgrade component toggles.
+# When running 'kk upgrade cluster', only kubelet/kubeadm are upgraded by default.
+# Set individual fields to true (or use --all / --set) to upgrade additional components.
 upgrade:
-  # Whether to modify CRI configuration file
+# When upgrading the cluster, also upgrade the container runtime (CRI), such as Docker or containerd.
   cri: false
 
+# When upgrading the cluster, also upgrade the etcd cluster.
+  etcd: false
+
+# When upgrading the cluster, also upgrade CoreDNS and NodeLocalDNS.
+  dns: false
+
+# When upgrading the cluster, also upgrade the private image registry (such as Harbor or registry).
+# This is typically used in conjunction with nodes defined in inventory.groups.image_registry.
+  image_registry: false
+
+# When upgrading the cluster, also upgrade the CNI plugin (e.g., Calico, Cilium, Flannel, Kube-OVN).
+  cni: false
+
+# When upgrading the cluster, also upgrade the storage class provisioner (e.g., localpv-provisioner, nfs-provisioner).
+  storage_class: false
+
+# When upgrading the cluster, also upgrade the NFS server.
+  nfs: false
+
 # image_manifests: List of container images to be synchronized to the private registry
 image_manifests: []

builtin/core/roles/defaults/tasks/main.yaml

@@ -43,6 +43,16 @@
       ignore_errors: true
       command: kubelet --version
       register: kubernetes_install_version
+    - name: Defaults | Get installed containerd version
+      ignore_errors: true
+      command: |
+        containerd --version 2>/dev/null | awk '{print $3}' || true
+      register: containerd_current_version
+    - name: Defaults | Get installed docker version
+      ignore_errors: true
+      command: |
+        docker version --format '{{ .Server.Version }}' 2>/dev/null || true
+      register: docker_current_version
 
 - name: Defaults | Gather ETCD service status
   when: .groups.etcd | has .inventory_hostname

builtin/core/roles/etcd/prepare/tasks/main.yaml

@@ -31,16 +31,6 @@
             {{- end -}}
             {{ $needUnInstalled | toJson }}            
 
-- name: Prepare | Validate installed etcd version
-  when: .etcd_install_LoadState.stdout | eq "loaded"
-  block:
-    - name: Prepare | Ensure target etcd version is not lower than installed version
-      when: .etcd_install_LoadState.stdout | eq "loaded"
-      assert:
-        that: .etcd.etcd_version | semverCompare (printf ">=v%s" (index .etcd_install_version "stdout" "etcd Version"))
-        fail_msg: >-
-          Installed etcd version: {{ index .etcd_install_version "stdout" "etcd Version" }} is lower than target etcd version: {{ .etcd.etcd_version }}
-
 - name: Prepare | Distribute etcd package for install or upgrade
   when: >-
     or 

builtin/core/roles/kubernetes/upgrade-kubernetes/tasks/main.yaml

@@ -0,0 +1,61 @@
+---
+- name: UpgradeKubernetes | Backup Kubernetes configurations and binaries before upgrade
+  command: |
+    BACKUP_DIR="/etc/kubekey/backup/kubernetes/$(date +%Y%m%d-%H%M%S)"
+    mkdir -p "${BACKUP_DIR}"
+    cp /etc/kubernetes/kubeadm-config.yaml "${BACKUP_DIR}/" 2>/dev/null || true
+    cp /usr/local/bin/kubeadm "${BACKUP_DIR}/" 2>/dev/null || true
+    cp /usr/local/bin/kubelet "${BACKUP_DIR}/" 2>/dev/null || true
+    cp /usr/local/bin/kubectl "${BACKUP_DIR}/" 2>/dev/null || true
+    echo "✅ Kubernetes backup saved to ${BACKUP_DIR}"
+
+- name: UpgradeKubernetes | Fail if kubelet service is not active
+  assert:
+    that: .kubernetes_install_ActiveState.stdout | eq "active"
+    fail_msg: >-
+      The kubelet service must be running and active before upgrade.
+
+- name: UpgradeKubernetes | Ensure kubeadm configuration exists locally on control plane
+  when: .groups.kube_control_plane | default list | has .inventory_hostname
+  command: |
+    if [ ! -f /etc/kubernetes/kubeadm-config.yaml ] || ! grep -q 'ClusterConfiguration' /etc/kubernetes/kubeadm-config.yaml 2>/dev/null; then
+      mkdir -p /etc/kubernetes
+      kubectl get cm kubeadm-config -n kube-system -o=jsonpath='{.data.ClusterConfiguration}' > /etc/kubernetes/kubeadm-config.yaml
+    fi
+
+- name: UpgradeKubernetes | Upgrade Kubernetes cluster on first control plane node
+  when:
+    - .groups.kube_control_plane | default list | has .inventory_hostname
+    - .init_kubernetes_node | eq .inventory_hostname
+  command: |
+    # In Kubernetes, the ImagePullCheck for the pause image uses a hardcoded tag.
+    # Adding --ignore-preflight-errors=ImagePull allows customization of the pause image reference.
+    /usr/local/bin/kubeadm upgrade apply {{ .kubernetes.kube_version }} --config=/etc/kubernetes/kubeadm-config.yaml --yes --ignore-preflight-errors=ImagePull
+
+- name: UpgradeKubernetes | Upgrade node configuration on other control plane nodes
+  when:
+    - .groups.kube_control_plane | default list | has .inventory_hostname
+    - .init_kubernetes_node | eq .inventory_hostname | not
+  command: |
+    /usr/local/bin/kubeadm upgrade node --ignore-preflight-errors=ImagePull
+
+- name: UpgradeKubernetes | Upgrade node configuration on worker nodes
+  when: .groups.kube_worker | default list | has .inventory_hostname
+  command: |
+    /usr/local/bin/kubeadm upgrade node --ignore-preflight-errors=ImagePull
+
+- name: UpgradeKubernetes | Restart kubelet service after upgrade
+  command: |
+    systemctl daemon-reload && systemctl restart kubelet.service
+
+- name: UpgradeKubernetes | Wait for kubelet to be ready after restart
+  command: |
+    for ((i=1; i<=30; i++)); do
+      if systemctl is-active kubelet.service >/dev/null 2>&1; then
+        echo "✅ kubelet is active"
+        exit 0
+      fi
+      sleep 5
+    done
+    echo "❌ kubelet is not active within 150 seconds"
+    exit 1