E2E: Use kind instead of minikube

Minikube is having troubles starting sometimes. It was nice to work with since the VM could easily be attached to the same network as the BMH VMs, but it is possible to work around that also with kind. Signed-off-by: Lennart Jern <[email protected]>
metal3-io · Jan 23, 2025 · 3916baa · 3916baa
1 parent e8e6433
commit 3916baa
Show file tree

Hide file tree

Showing 25 changed files with 175 additions and 165 deletions.
diff --git a/.gitignore b/.gitignore
@@ -28,6 +28,7 @@ go.work.sum
 *~
 *.tmp
 .DS_Store
+.zed*
 
 # Tilt files.
 .tiltbuild

diff --git a/config/overlays/e2e-release-0.6/ironic.env b/config/overlays/e2e-release-0.6/ironic.env
@@ -1,3 +1,3 @@
-DEPLOY_KERNEL_URL=http://192.168.222.199:6180/images/ironic-python-agent.kernel
-DEPLOY_RAMDISK_URL=http://192.168.222.199:6180/images/ironic-python-agent.initramfs
-IRONIC_ENDPOINT=https://192.168.222.199:6385/v1/
+DEPLOY_KERNEL_URL=http://192.168.222.1:6180/images/ironic-python-agent.kernel
+DEPLOY_RAMDISK_URL=http://192.168.222.1:6180/images/ironic-python-agent.initramfs
+IRONIC_ENDPOINT=https://192.168.222.1:6385/v1/
diff --git a/config/overlays/e2e-release-0.8/ironic.env b/config/overlays/e2e-release-0.8/ironic.env
@@ -1,3 +1,3 @@
-DEPLOY_KERNEL_URL=http://192.168.222.199:6180/images/ironic-python-agent.kernel
-DEPLOY_RAMDISK_URL=http://192.168.222.199:6180/images/ironic-python-agent.initramfs
-IRONIC_ENDPOINT=https://192.168.222.199:6385/v1/
+DEPLOY_KERNEL_URL=http://192.168.222.1:6180/images/ironic-python-agent.kernel
+DEPLOY_RAMDISK_URL=http://192.168.222.1:6180/images/ironic-python-agent.initramfs
+IRONIC_ENDPOINT=https://192.168.222.1:6385/v1/
diff --git a/config/overlays/e2e-release-0.9/ironic.env b/config/overlays/e2e-release-0.9/ironic.env
@@ -1,3 +1,3 @@
-DEPLOY_KERNEL_URL=http://192.168.222.199:6180/images/ironic-python-agent.kernel
-DEPLOY_RAMDISK_URL=http://192.168.222.199:6180/images/ironic-python-agent.initramfs
-IRONIC_ENDPOINT=https://192.168.222.199:6385/v1/
+DEPLOY_KERNEL_URL=http://192.168.222.1:6180/images/ironic-python-agent.kernel
+DEPLOY_RAMDISK_URL=http://192.168.222.1:6180/images/ironic-python-agent.initramfs
+IRONIC_ENDPOINT=https://192.168.222.1:6385/v1/
diff --git a/config/overlays/e2e/ironic.env b/config/overlays/e2e/ironic.env
@@ -1,3 +1,3 @@
-DEPLOY_KERNEL_URL=http://192.168.222.199:6180/images/ironic-python-agent.kernel
-DEPLOY_RAMDISK_URL=http://192.168.222.199:6180/images/ironic-python-agent.initramfs
-IRONIC_ENDPOINT=https://192.168.222.199:6385/v1/
+DEPLOY_KERNEL_URL=http://192.168.222.1:6180/images/ironic-python-agent.kernel
+DEPLOY_RAMDISK_URL=http://192.168.222.1:6180/images/ironic-python-agent.initramfs
+IRONIC_ENDPOINT=https://192.168.222.1:6385/v1/
diff --git a/hack/ci-e2e.sh b/hack/ci-e2e.sh
@@ -33,25 +33,24 @@ echo "BMO_E2E_EMULATOR=${BMO_E2E_EMULATOR}"
 export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/ironic.yaml"
 export E2E_BMCS_CONF_FILE="${REPO_ROOT}/test/e2e/config/bmcs-${BMC_PROTOCOL}.yaml"
 
-LOAD_LOCAL_IRONIC="${LOAD_LOCAL_IRONIC:-false}"
-
 case "${GINKGO_FOCUS:-}" in
   *upgrade*)
     export DEPLOY_IRONIC="false"
     export DEPLOY_BMO="false"
     export DEPLOY_CERT_MANAGER="false"
     export GINKGO_NODES=1
+    # Not really existing cluster, but the upgrade test creates its own clusters.
+    export USE_EXISTING_CLUSTER="true"
     ;;
   *)
     export GINKGO_SKIP="${GINKGO_SKIP:-upgrade}"
     ;;
 esac
-export USE_EXISTING_CLUSTER="true"
 
 # Ensure requirements are installed
 "${REPO_ROOT}/hack/e2e/ensure_go.sh"
 export PATH="/usr/local/go/bin:${PATH}"
-"${REPO_ROOT}/hack/e2e/ensure_minikube.sh"
+"${REPO_ROOT}/hack/e2e/ensure_kind.sh"
 "${REPO_ROOT}/hack/e2e/ensure_htpasswd.sh"
 # CAPI test framework uses kubectl in the background
 "${REPO_ROOT}/hack/e2e/ensure_kubectl.sh"
@@ -60,42 +59,15 @@ export PATH="/usr/local/go/bin:${PATH}"
 # Build the container image with e2e tag (used in tests)
 IMG=quay.io/metal3-io/baremetal-operator:e2e make docker
 
-# Set up minikube
-minikube start --driver=kvm2
-
 virsh -c qemu:///system net-define "${REPO_ROOT}/hack/e2e/net.xml"
 virsh -c qemu:///system net-start baremetal-e2e
-# Attach baremetal-e2e interface to minikube with specific mac.
-# This will give minikube a known reserved IP address that we can use for Ironic
-virsh -c qemu:///system attach-interface --domain minikube --mac="52:54:00:6c:3c:01" \
-  --model virtio --source baremetal-e2e --type network --config
-
-# Restart minikube to apply the changes
-minikube stop
-## Following loop is to avoid minikube restart issue
-## https://github.com/kubernetes/minikube/issues/14456
-while ! minikube start; do sleep 30; done
-
-# Load the BMO e2e image into it
-# minikube image load quay.io/metal3-io/baremetal-operator:e2e
-# Temporary workaround for https://github.com/kubernetes/minikube/issues/18021
-docker image save -o /tmp/bmo-e2e.tar quay.io/metal3-io/baremetal-operator:e2e
-minikube image load /tmp/bmo-e2e.tar
-rm /tmp/bmo-e2e.tar
-if [[ "${LOAD_LOCAL_IRONIC}" == "true" ]]; then
-    echo "Saving local ironic image!"
-    docker image save -o \
-        "/tmp/ironic-e2e.tar" "quay.io/metal3-io/ironic:local"
-    minikube image load /tmp/ironic-e2e.tar
-    rm /tmp/ironic-e2e.tar
-fi
 
 # This IP is defined by the network we created above.
 IP_ADDRESS="192.168.222.1"
 
 # This IP is also defined by the network above, and is used consistently in all of
 # our e2e overlays
-export IRONIC_PROVISIONING_IP="192.168.222.199"
+export IRONIC_PROVISIONING_IP="${IP_ADDRESS}"
 
 # Build vbmctl
 make build-vbmctl
@@ -114,7 +86,7 @@ if [[ "${BMO_E2E_EMULATOR}" == "vbmc" ]]; then
     address=$(echo "${bmc}" | jq -r '.address')
     hostName=$(echo "${bmc}" | jq -r '.hostName')
     vbmc_port="${address##*:}"
-    "${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${hostName}" "${vbmc_port}"
+    "${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${hostName}" "${vbmc_port}" "${IP_ADDRESS}"
   done
 
 elif [[ "${BMO_E2E_EMULATOR}" == "sushy-tools" ]]; then

diff --git a/hack/clean-e2e.sh b/hack/clean-e2e.sh
@@ -3,7 +3,7 @@
 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 cd "${REPO_ROOT}" || exit 1
 
-minikube delete
+kind delete cluster
 docker rm -f vbmc
 docker rm -f image-server-e2e
 docker rm -f sushy-tools

diff --git a/hack/e2e/ensure_kind.sh b/hack/e2e/ensure_kind.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -eux
+
+USR_LOCAL_BIN="/usr/local/bin"
+MINIMUM_KIND_VERSION=v0.26.0
+
+# Ensure the kind tool exists and is a viable version, or installs it
+verify_kind_version()
+{
+    # If kind is not available on the path, get it
+    if ! [ -x "$(command -v kind)" ]; then
+        if [[ "${OSTYPE}" == "linux-gnu" ]]; then
+            echo "kind not found, installing"
+            curl -LO "https://kind.sigs.k8s.io/dl/${MINIMUM_KIND_VERSION}/kind-linux-amd64"
+            sudo install kind "${USR_LOCAL_BIN}/kind"
+        else
+            echo "Missing required binary in path: kind"
+            return 2
+        fi
+    fi
+
+    local kind_version
+    IFS=" " read -ra kind_version <<< "$(kind version)"
+    if [[ "${MINIMUM_KIND_VERSION}" != $(echo -e "${MINIMUM_KIND_VERSION}\n${kind_version[1]}" | sort -s -t. -k 1,1 -k 2,2n -k 3,3n | head -n1) ]]; then
+        cat << EOF
+Detected kind version: ${kind_version[2]}.
+Requires ${MINIMUM_KIND_VERSION} or greater.
+Please install ${MINIMUM_KIND_VERSION} or later.
+EOF
+        return 2
+    fi
+}
+
+verify_kind_version
diff --git a/hack/e2e/ensure_minikube.sh b/hack/e2e/ensure_minikube.sh
diff --git a/hack/e2e/net.xml b/hack/e2e/net.xml
@@ -9,8 +9,7 @@
   <ip address='192.168.222.1' netmask='255.255.255.0'>
     <dhcp>
       <range start='192.168.222.2' end='192.168.222.199'/>
-      <host mac='52:54:00:6c:3c:01' name='minikube' ip='192.168.222.199'/>
-      <bootp file='http://192.168.222.199:6180/boot.ipxe'/>
+      <bootp file='http://192.168.222.1:6180/boot.ipxe'/>
     </dhcp>
   </ip>
 </network>
diff --git a/ironic-deployment/components/tls/certificate.yaml b/ironic-deployment/components/tls/certificate.yaml
@@ -12,8 +12,6 @@ metadata:
 spec:
   commonName: ironic-ca
   isCA: true
-  ipAddresses:
-  - IRONIC_HOST_IP
   issuerRef:
     kind: Issuer
     name: selfsigned-issuer

diff --git a/ironic-deployment/overlays/e2e-release-24.1/ironic_bmo_configmap.env b/ironic-deployment/overlays/e2e-release-24.1/ironic_bmo_configmap.env
@@ -1,6 +1,7 @@
 HTTP_PORT=6180
-PROVISIONING_IP=192.168.222.199
-CACHEURL=http://192.168.222.199/images
+PROVISIONING_INTERFACE=eth0
+CACHEURL=http://192.168.222.1/images
+IRONIC_EXTERNAL_CALLBACK_URL=https://192.168.222.1:6385
 IRONIC_KERNEL_PARAMS=console=ttyS0
 IRONIC_INSPECTOR_VLAN_INTERFACES=all
 USE_IRONIC_INSPECTOR=false
diff --git a/ironic-deployment/overlays/e2e-release-24.1/kustomization.yaml b/ironic-deployment/overlays/e2e-release-24.1/kustomization.yaml
@@ -16,7 +16,24 @@ configMapGenerator:
   behavior: create
 
 patches:
-  - path: ironic-patch.yaml
+- path: ironic-patch.yaml
+# The TLS component adds certificates but it cannot know the exact IPs of our environment.
+# Here we patch the certificates to have the correct IPs.
+# - 192.168.222.1: management computer IP, forwarded to ironic inside kind
+- patch: |-
+    - op: replace
+      path: /spec/ipAddresses/0
+      value: 192.168.222.1
+  target:
+    kind: Certificate
+    name: ironic-cert
+- patch: |-
+    - op: replace
+      path: /spec/ipAddresses/0
+      value: 192.168.222.1
+  target:
+    kind: Certificate
+    name: ironic-cacert
 
 images:
 - name: quay.io/metal3-io/ironic
@@ -28,18 +45,3 @@ secretGenerator:
   behavior: create
   envs:
   - ironic-htpasswd
-
-replacements:
-  # Replace IRONIC_HOST_IP in certificates with the PROVISIONING_IP from the configmap
-  - source:
-      kind: ConfigMap
-      name: ironic-bmo-configmap
-      fieldPath: .data.PROVISIONING_IP
-    targets:
-      - select:
-          version: v1
-          group: cert-manager.io
-          kind: Certificate
-          name:
-        fieldPaths:
-          - .spec.ipAddresses.0
diff --git a/ironic-deployment/overlays/e2e-release-25.0/ironic_bmo_configmap.env b/ironic-deployment/overlays/e2e-release-25.0/ironic_bmo_configmap.env
@@ -1,6 +1,7 @@
 HTTP_PORT=6180
-PROVISIONING_IP=192.168.222.199
-CACHEURL=http://192.168.222.199/images
+PROVISIONING_INTERFACE=eth0
+CACHEURL=http://192.168.222.1/images
+IRONIC_EXTERNAL_CALLBACK_URL=https://192.168.222.1:6385
 IRONIC_KERNEL_PARAMS=console=ttyS0
 IRONIC_INSPECTOR_VLAN_INTERFACES=all
 USE_IRONIC_INSPECTOR=false
diff --git a/ironic-deployment/overlays/e2e-release-25.0/kustomization.yaml b/ironic-deployment/overlays/e2e-release-25.0/kustomization.yaml
@@ -17,6 +17,23 @@ configMapGenerator:
 
 patches:
 - path: ironic-patch.yaml
+# The TLS component adds certificates but it cannot know the exact IPs of our environment.
+# Here we patch the certificates to have the correct IPs.
+# - 192.168.222.1: management computer IP, forwarded to ironic inside kind
+- patch: |-
+    - op: replace
+      path: /spec/ipAddresses/0
+      value: 192.168.222.1
+  target:
+    kind: Certificate
+    name: ironic-cert
+- patch: |-
+    - op: replace
+      path: /spec/ipAddresses/0
+      value: 192.168.222.1
+  target:
+    kind: Certificate
+    name: ironic-cacert
 
 images:
 - name: quay.io/metal3-io/ironic
@@ -28,18 +45,3 @@ secretGenerator:
   behavior: create
   envs:
   - ironic-htpasswd
-
-replacements:
-# Replace IRONIC_HOST_IP in certificates with the PROVISIONING_IP from the configmap
-- source:
-    kind: ConfigMap
-    name: ironic-bmo-configmap
-    fieldPath: .data.PROVISIONING_IP
-  targets:
-  - select:
-      version: v1
-      group: cert-manager.io
-      kind: Certificate
-      name:
-    fieldPaths:
-    - .spec.ipAddresses.0
diff --git a/ironic-deployment/overlays/e2e-release-26.0/ironic_bmo_configmap.env b/ironic-deployment/overlays/e2e-release-26.0/ironic_bmo_configmap.env
@@ -1,6 +1,7 @@
 HTTP_PORT=6180
-PROVISIONING_IP=192.168.222.199
-CACHEURL=http://192.168.222.199/images
+PROVISIONING_INTERFACE=eth0
+CACHEURL=http://192.168.222.1/images
+IRONIC_EXTERNAL_CALLBACK_URL=https://192.168.222.1:6385
 IRONIC_KERNEL_PARAMS=console=ttyS0
 IRONIC_INSPECTOR_VLAN_INTERFACES=all
 USE_IRONIC_INSPECTOR=false
diff --git a/ironic-deployment/overlays/e2e-release-26.0/kustomization.yaml b/ironic-deployment/overlays/e2e-release-26.0/kustomization.yaml
@@ -17,6 +17,23 @@ configMapGenerator:
 
 patches:
 - path: ironic-patch.yaml
+# The TLS component adds certificates but it cannot know the exact IPs of our environment.
+# Here we patch the certificates to have the correct IPs.
+# - 192.168.222.1: management computer IP, forwarded to ironic inside kind
+- patch: |-
+    - op: replace
+      path: /spec/ipAddresses/0
+      value: 192.168.222.1
+  target:
+    kind: Certificate
+    name: ironic-cert
+- patch: |-
+    - op: replace
+      path: /spec/ipAddresses/0
+      value: 192.168.222.1
+  target:
+    kind: Certificate
+    name: ironic-cacert
 
 images:
 - name: quay.io/metal3-io/ironic
@@ -28,18 +45,3 @@ secretGenerator:
   behavior: create
   envs:
   - ironic-htpasswd
-
-replacements:
-# Replace IRONIC_HOST_IP in certificates with the PROVISIONING_IP from the configmap
-- source:
-    kind: ConfigMap
-    name: ironic-bmo-configmap
-    fieldPath: .data.PROVISIONING_IP
-  targets:
-  - select:
-      version: v1
-      group: cert-manager.io
-      kind: Certificate
-      name:
-    fieldPaths:
-    - .spec.ipAddresses.0
diff --git a/ironic-deployment/overlays/e2e/ironic_bmo_configmap.env b/ironic-deployment/overlays/e2e/ironic_bmo_configmap.env
@@ -1,6 +1,7 @@
 HTTP_PORT=6180
-PROVISIONING_IP=192.168.222.199
-CACHEURL=http://192.168.222.199/images
+PROVISIONING_INTERFACE=eth0
+CACHEURL=http://192.168.222.1/images
+IRONIC_EXTERNAL_CALLBACK_URL=https://192.168.222.1:6385
 IRONIC_KERNEL_PARAMS=console=ttyS0
 IRONIC_INSPECTOR_VLAN_INTERFACES=all
 USE_IRONIC_INSPECTOR=false
-Original file line number
+Diff line change
@@ Expand Up / @@ -28,6 +28,7 @@ go.work.sum @@
     *~
     *.tmp
     .DS_Store
+    .zed*
     # Tilt files.
     .tiltbuild
@@ Expand Down @@