From 352ab758946949c74a029d38c7c27b70e4233b97 Mon Sep 17 00:00:00 2001 From: Lennart Jern Date: Wed, 22 Jan 2025 07:58:57 +0000 Subject: [PATCH 1/4] E2E: Clean up unused overlays Remove BMO overlays no longer used in the e2e tests. Drop upgrade from ironic-24.0 as it is out of support and not needed to be tested anymore. Signed-off-by: Lennart Jern --- config/overlays/e2e-release-0.4/ironic.env | 4 -- .../e2e-release-0.4/kustomization.yaml | 38 ------------- config/overlays/e2e-release-0.5/ironic.env | 3 - .../e2e-release-0.5/kustomization.yaml | 36 ------------ .../fixture-release-0.4/kustomization.yaml | 21 ------- .../fixture-release-0.5/kustomization.yaml | 21 ------- hack/ci-e2e.sh | 19 ------- .../ironic-patch.yaml | 10 ---- .../ironic_bmo_configmap.env | 6 -- .../kustomization.yaml | 55 ------------------- .../e2e-with-inspector/ironic-patch.yaml | 10 ---- .../ironic_bmo_configmap.env | 7 --- .../e2e-with-inspector/kustomization.yaml | 51 ----------------- test/e2e/config/ironic.yaml | 6 -- 14 files changed, 287 deletions(-) delete mode 100644 config/overlays/e2e-release-0.4/ironic.env delete mode 100644 config/overlays/e2e-release-0.4/kustomization.yaml delete mode 100644 config/overlays/e2e-release-0.5/ironic.env delete mode 100644 config/overlays/e2e-release-0.5/kustomization.yaml delete mode 100644 config/overlays/fixture-release-0.4/kustomization.yaml delete mode 100644 config/overlays/fixture-release-0.5/kustomization.yaml delete mode 100644 ironic-deployment/overlays/e2e-release-24.0-with-inspector/ironic-patch.yaml delete mode 100644 ironic-deployment/overlays/e2e-release-24.0-with-inspector/ironic_bmo_configmap.env delete mode 100644 ironic-deployment/overlays/e2e-release-24.0-with-inspector/kustomization.yaml delete mode 100644 ironic-deployment/overlays/e2e-with-inspector/ironic-patch.yaml delete mode 100644 ironic-deployment/overlays/e2e-with-inspector/ironic_bmo_configmap.env delete mode 100644 ironic-deployment/overlays/e2e-with-inspector/kustomization.yaml diff --git a/config/overlays/e2e-release-0.4/ironic.env b/config/overlays/e2e-release-0.4/ironic.env deleted file mode 100644 index ea9d07501a..0000000000 --- a/config/overlays/e2e-release-0.4/ironic.env +++ /dev/null @@ -1,4 +0,0 @@ -DEPLOY_KERNEL_URL=http://192.168.222.199:6180/images/ironic-python-agent.kernel -DEPLOY_RAMDISK_URL=http://192.168.222.199:6180/images/ironic-python-agent.initramfs -IRONIC_ENDPOINT=https://192.168.222.199:6385/v1/ -IRONIC_INSPECTOR_ENDPOINT=https://192.168.222.199:5050/v1/ diff --git a/config/overlays/e2e-release-0.4/kustomization.yaml b/config/overlays/e2e-release-0.4/kustomization.yaml deleted file mode 100644 index 10c4705ec6..0000000000 --- a/config/overlays/e2e-release-0.4/kustomization.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: baremetal-operator-system -resources: -- https://github.com/metal3-io/baremetal-operator/config/tls?ref=release-0.4&timeout=120s -components: -- ../../components/basic-auth -configMapGenerator: -- name: ironic - behavior: merge - envs: - - ironic.env -patches: -- patch: | - # Don't try to pull again the pre-loaded image - - op: replace - path: /spec/template/spec/containers/0/imagePullPolicy - value: IfNotPresent - target: - kind: Deployment - name: controller-manager -images: -- name: quay.io/metal3-io/baremetal-operator - newTag: release-0.4 -# We cannot use suffix hashes since the kustomizations we build on -# cannot be aware of what suffixes we add. -generatorOptions: - disableNameSuffixHash: true -# NOTE: These credentials are generated automatically in hack/ci-e2e.sh -secretGenerator: -- name: ironic-credentials - files: - - username=ironic-username - - password=ironic-password -- name: ironic-inspector-credentials - files: - - username=ironic-inspector-username - - password=ironic-inspector-password diff --git a/config/overlays/e2e-release-0.5/ironic.env b/config/overlays/e2e-release-0.5/ironic.env deleted file mode 100644 index 0187ae15e4..0000000000 --- a/config/overlays/e2e-release-0.5/ironic.env +++ /dev/null @@ -1,3 +0,0 @@ -DEPLOY_KERNEL_URL=http://192.168.222.199:6180/images/ironic-python-agent.kernel -DEPLOY_RAMDISK_URL=http://192.168.222.199:6180/images/ironic-python-agent.initramfs -IRONIC_ENDPOINT=https://192.168.222.199:6385/v1/ diff --git a/config/overlays/e2e-release-0.5/kustomization.yaml b/config/overlays/e2e-release-0.5/kustomization.yaml deleted file mode 100644 index 5c26ca29fa..0000000000 --- a/config/overlays/e2e-release-0.5/kustomization.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: baremetal-operator-system -resources: -- https://github.com/metal3-io/baremetal-operator/config/overlays/basic-auth_tls?ref=release-0.5&timeout=120s -configMapGenerator: -- name: ironic - behavior: create - envs: - - ironic.env -patches: -- patch: | - # Don't try to pull again the pre-loaded image - - op: replace - path: /spec/template/spec/containers/0/imagePullPolicy - value: IfNotPresent - target: - kind: Deployment - name: controller-manager -images: -- name: quay.io/metal3-io/baremetal-operator - newTag: release-0.5 -# We cannot use suffix hashes since the kustomizations we build on -# cannot be aware of what suffixes we add. -generatorOptions: - disableNameSuffixHash: true -# NOTE: These credentials are generated automatically in hack/ci-e2e.sh -secretGenerator: -- name: ironic-credentials - files: - - username=ironic-username - - password=ironic-password -- name: ironic-inspector-credentials - files: - - username=ironic-inspector-username - - password=ironic-inspector-password diff --git a/config/overlays/fixture-release-0.4/kustomization.yaml b/config/overlays/fixture-release-0.4/kustomization.yaml deleted file mode 100644 index 5c6f1efafa..0000000000 --- a/config/overlays/fixture-release-0.4/kustomization.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=release-0.4 -- https://github.com/metal3-io/baremetal-operator/config/default?ref=release-0.4 -patches: -- patch: | - # Enable test mode (fixture provider instead of ironic) - - op: add - path: /spec/template/spec/containers/0/args/- - value: --test-mode - # Don't try to pull again the pre-loaded image - - op: replace - path: /spec/template/spec/containers/0/imagePullPolicy - value: IfNotPresent - target: - kind: Deployment - name: controller-manager -images: -- name: quay.io/metal3-io/baremetal-operator - newTag: release-0.4 diff --git a/config/overlays/fixture-release-0.5/kustomization.yaml b/config/overlays/fixture-release-0.5/kustomization.yaml deleted file mode 100644 index 4600b60d9d..0000000000 --- a/config/overlays/fixture-release-0.5/kustomization.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=release-0.5 -- https://github.com/metal3-io/baremetal-operator/config/default?ref=release-0.5 -patches: -- patch: | - # Enable test mode (fixture provider instead of ironic) - - op: add - path: /spec/template/spec/containers/0/args/- - value: --test-mode - # Don't try to pull again the pre-loaded image - - op: replace - path: /spec/template/spec/containers/0/imagePullPolicy - value: IfNotPresent - target: - kind: Deployment - name: controller-manager -images: -- name: quay.io/metal3-io/baremetal-operator - newTag: release-0.5 diff --git a/hack/ci-e2e.sh b/hack/ci-e2e.sh index 5643778146..5d55b982f8 100755 --- a/hack/ci-e2e.sh +++ b/hack/ci-e2e.sh @@ -186,8 +186,6 @@ BMO_OVERLAYS=( ) IRONIC_OVERLAYS=( "${REPO_ROOT}/ironic-deployment/overlays/e2e" - "${REPO_ROOT}/ironic-deployment/overlays/e2e-with-inspector" - "${REPO_ROOT}/ironic-deployment/overlays/e2e-release-24.0-with-inspector" "${REPO_ROOT}/ironic-deployment/overlays/e2e-release-24.1" "${REPO_ROOT}/ironic-deployment/overlays/e2e-release-25.0" "${REPO_ROOT}/ironic-deployment/overlays/e2e-release-26.0" @@ -195,22 +193,14 @@ IRONIC_OVERLAYS=( IRONIC_USERNAME="$(uuidgen)" IRONIC_PASSWORD="$(uuidgen)" -IRONIC_INSPECTOR_USERNAME="$(uuidgen)" -IRONIC_INSPECTOR_PASSWORD="$(uuidgen)" # These must be exported so that envsubst can pick them up below export IRONIC_USERNAME export IRONIC_PASSWORD -export IRONIC_INSPECTOR_USERNAME -export IRONIC_INSPECTOR_PASSWORD for overlay in "${BMO_OVERLAYS[@]}"; do echo "${IRONIC_USERNAME}" > "${overlay}/ironic-username" echo "${IRONIC_PASSWORD}" > "${overlay}/ironic-password" - if [[ "${overlay}" =~ release-0\.[1-5]$ ]]; then - echo "${IRONIC_INSPECTOR_USERNAME}" > "${overlay}/ironic-inspector-username" - echo "${IRONIC_INSPECTOR_PASSWORD}" > "${overlay}/ironic-inspector-password" - fi done for overlay in "${IRONIC_OVERLAYS[@]}"; do @@ -218,15 +208,6 @@ for overlay in "${IRONIC_OVERLAYS[@]}"; do "${overlay}/ironic-htpasswd" envsubst < "${REPO_ROOT}/ironic-deployment/components/basic-auth/ironic-auth-config-tpl" > \ "${overlay}/ironic-auth-config" - - if [[ "${overlay}" =~ -with-inspector ]]; then - IRONIC_INSPECTOR_AUTH_CONFIG_TPL="/tmp/ironic-inspector-auth-config-tpl" - curl -o "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" https://raw.githubusercontent.com/metal3-io/baremetal-operator/release-0.5/ironic-deployment/components/basic-auth/ironic-inspector-auth-config-tpl - envsubst < "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" > \ - "${overlay}/ironic-inspector-auth-config" - echo "INSPECTOR_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_INSPECTOR_USERNAME}" \ - "${IRONIC_INSPECTOR_PASSWORD}")" > "${overlay}/ironic-inspector-htpasswd" - fi done # We need to gather artifacts/logs before exiting also if there are errors diff --git a/ironic-deployment/overlays/e2e-release-24.0-with-inspector/ironic-patch.yaml b/ironic-deployment/overlays/e2e-release-24.0-with-inspector/ironic-patch.yaml deleted file mode 100644 index 098d21e109..0000000000 --- a/ironic-deployment/overlays/e2e-release-24.0-with-inspector/ironic-patch.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ironic -spec: - template: - spec: - containers: - - name: ironic-dnsmasq - $patch: delete diff --git a/ironic-deployment/overlays/e2e-release-24.0-with-inspector/ironic_bmo_configmap.env b/ironic-deployment/overlays/e2e-release-24.0-with-inspector/ironic_bmo_configmap.env deleted file mode 100644 index cd099090c2..0000000000 --- a/ironic-deployment/overlays/e2e-release-24.0-with-inspector/ironic_bmo_configmap.env +++ /dev/null @@ -1,6 +0,0 @@ -HTTP_PORT=6180 -PROVISIONING_IP=192.168.222.199 -CACHEURL=http://192.168.222.199/images -IRONIC_FAST_TRACK=true -IRONIC_KERNEL_PARAMS=console=ttyS0 -IRONIC_INSPECTOR_VLAN_INTERFACES=all diff --git a/ironic-deployment/overlays/e2e-release-24.0-with-inspector/kustomization.yaml b/ironic-deployment/overlays/e2e-release-24.0-with-inspector/kustomization.yaml deleted file mode 100644 index 104be2fe63..0000000000 --- a/ironic-deployment/overlays/e2e-release-24.0-with-inspector/kustomization.yaml +++ /dev/null @@ -1,55 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: baremetal-operator-system -resources: -- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=release-0.5&timeout=120s -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=release-0.5&timeout=120s - -components: -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=release-0.5&timeout=120s -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=release-0.5&timeout=120s - -configMapGenerator: -- envs: - - ironic_bmo_configmap.env - name: ironic-bmo-configmap - behavior: create - -patches: - - path: ironic-patch.yaml - -images: -- name: quay.io/metal3-io/ironic - newTag: release-24.0 - -# NOTE: These credentials are generated automatically in hack/ci-e2e.sh -secretGenerator: -- name: ironic-htpasswd - behavior: create - files: - - htpasswd=ironic-htpasswd -- name: ironic-inspector-htpasswd - behavior: create - files: - - htpasswd=ironic-inspector-htpasswd -- name: ironic-auth-config - files: - - auth-config=ironic-auth-config -- name: ironic-inspector-auth-config - files: - - auth-config=ironic-inspector-auth-config - -replacements: - # Replace IRONIC_HOST_IP in certificates with the PROVISIONING_IP from the configmap - - source: - kind: ConfigMap - name: ironic-bmo-configmap - fieldPath: .data.PROVISIONING_IP - targets: - - select: - version: v1 - group: cert-manager.io - kind: Certificate - name: - fieldPaths: - - .spec.ipAddresses.0 diff --git a/ironic-deployment/overlays/e2e-with-inspector/ironic-patch.yaml b/ironic-deployment/overlays/e2e-with-inspector/ironic-patch.yaml deleted file mode 100644 index 098d21e109..0000000000 --- a/ironic-deployment/overlays/e2e-with-inspector/ironic-patch.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ironic -spec: - template: - spec: - containers: - - name: ironic-dnsmasq - $patch: delete diff --git a/ironic-deployment/overlays/e2e-with-inspector/ironic_bmo_configmap.env b/ironic-deployment/overlays/e2e-with-inspector/ironic_bmo_configmap.env deleted file mode 100644 index 65624598de..0000000000 --- a/ironic-deployment/overlays/e2e-with-inspector/ironic_bmo_configmap.env +++ /dev/null @@ -1,7 +0,0 @@ -HTTP_PORT=6180 -PROVISIONING_IP=192.168.222.199 -CACHEURL=http://192.168.222.199/images -IRONIC_FAST_TRACK=true -IRONIC_KERNEL_PARAMS=console=ttyS0 -IRONIC_INSPECTOR_VLAN_INTERFACES=all -USE_IRONIC_INSPECTOR=true diff --git a/ironic-deployment/overlays/e2e-with-inspector/kustomization.yaml b/ironic-deployment/overlays/e2e-with-inspector/kustomization.yaml deleted file mode 100644 index d1009417ca..0000000000 --- a/ironic-deployment/overlays/e2e-with-inspector/kustomization.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: baremetal-operator-system -resources: -- ../../../config/namespace -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/base?ref=release-0.5&timeout=120s - -components: -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/basic-auth?ref=release-0.5&timeout=120s -- https://github.com/metal3-io/baremetal-operator/ironic-deployment/components/tls?ref=release-0.5&timeout=120s - -configMapGenerator: -- envs: - - ironic_bmo_configmap.env - name: ironic-bmo-configmap - behavior: create - -patches: -- path: ironic-patch.yaml - -# NOTE: These credentials are generated automatically in hack/ci-e2e.sh -secretGenerator: -- name: ironic-htpasswd - behavior: create - envs: - - ironic-htpasswd -- name: ironic-inspector-htpasswd - behavior: create - envs: - - ironic-inspector-htpasswd -- name: ironic-auth-config - files: - - auth-config=ironic-auth-config -- name: ironic-inspector-auth-config - files: - - auth-config=ironic-inspector-auth-config - -replacements: -# Replace IRONIC_HOST_IP in certificates with the PROVISIONING_IP from the configmap -- source: - kind: ConfigMap - name: ironic-bmo-configmap - fieldPath: .data.PROVISIONING_IP - targets: - - select: - version: v1 - group: cert-manager.io - kind: Certificate - name: - fieldPaths: - - .spec.ipAddresses.0 diff --git a/test/e2e/config/ironic.yaml b/test/e2e/config/ironic.yaml index e0a7f336ee..1352a7b17e 100644 --- a/test/e2e/config/ironic.yaml +++ b/test/e2e/config/ironic.yaml @@ -79,12 +79,6 @@ bmoIronicUpgradeSpecs: initIronicKustomization: "../../ironic-deployment/overlays/e2e" upgradeEntityKustomization: "../../config/overlays/e2e" upgradeEntityName: "bmo" -- deployIronic: true - deployBMO: true - initBMOKustomization: "../../config/overlays/e2e-release-0.9" - initIronicKustomization: "../../ironic-deployment/overlays/e2e-release-24.0-with-inspector" - upgradeEntityKustomization: "../../ironic-deployment/overlays/e2e" - upgradeEntityName: "ironic" - deployIronic: true deployBMO: true initBMOKustomization: "../../config/overlays/e2e-release-0.9" From ace7702046732fe6643df67e6ae2a669475664dc Mon Sep 17 00:00:00 2001 From: Lennart Jern Date: Tue, 21 Jan 2025 15:54:09 +0200 Subject: [PATCH 2/4] E2E: Use kind instead of minikube Minikube is having troubles starting sometimes. It was nice to work with since the VM could easily be attached to the same network as the BMH VMs, but it is possible to work around that also with kind. Signed-off-by: Lennart Jern --- .gitignore | 1 + config/overlays/e2e-release-0.6/ironic.env | 6 ++-- config/overlays/e2e-release-0.8/ironic.env | 6 ++-- config/overlays/e2e-release-0.9/ironic.env | 6 ++-- config/overlays/e2e/ironic.env | 6 ++-- hack/ci-e2e.sh | 36 ++----------------- hack/clean-e2e.sh | 2 +- hack/e2e/ensure_kind.sh | 35 ++++++++++++++++++ hack/e2e/ensure_minikube.sh | 33 ----------------- hack/e2e/net.xml | 3 +- .../components/tls/certificate.yaml | 2 -- .../e2e-release-24.1/ironic_bmo_configmap.env | 5 +-- .../e2e-release-24.1/kustomization.yaml | 34 +++++++++--------- .../e2e-release-25.0/ironic_bmo_configmap.env | 5 +-- .../e2e-release-25.0/kustomization.yaml | 32 +++++++++-------- .../e2e-release-26.0/ironic_bmo_configmap.env | 5 +-- .../e2e-release-26.0/kustomization.yaml | 32 +++++++++-------- .../overlays/e2e/ironic_bmo_configmap.env | 5 +-- .../overlays/e2e/kustomization.yaml | 25 ++++++------- test/e2e/config/fixture.yaml | 3 ++ test/e2e/config/ironic.yaml | 29 +++++++++++++++ test/e2e/e2e_config.go | 4 +++ test/e2e/e2e_suite_test.go | 5 +-- test/e2e/upgrade_test.go | 23 ++++++------ test/go.mod | 2 +- tools/bmh_test/vm2vbmc.sh | 3 +- 26 files changed, 182 insertions(+), 166 deletions(-) create mode 100755 hack/e2e/ensure_kind.sh delete mode 100755 hack/e2e/ensure_minikube.sh diff --git a/.gitignore b/.gitignore index e4b21587a0..92aea3d84b 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ go.work.sum *~ *.tmp .DS_Store +.zed* # Tilt files. .tiltbuild diff --git a/config/overlays/e2e-release-0.6/ironic.env b/config/overlays/e2e-release-0.6/ironic.env index 0187ae15e4..44147ae0f3 100644 --- a/config/overlays/e2e-release-0.6/ironic.env +++ b/config/overlays/e2e-release-0.6/ironic.env @@ -1,3 +1,3 @@ -DEPLOY_KERNEL_URL=http://192.168.222.199:6180/images/ironic-python-agent.kernel -DEPLOY_RAMDISK_URL=http://192.168.222.199:6180/images/ironic-python-agent.initramfs -IRONIC_ENDPOINT=https://192.168.222.199:6385/v1/ +DEPLOY_KERNEL_URL=http://192.168.222.1:6180/images/ironic-python-agent.kernel +DEPLOY_RAMDISK_URL=http://192.168.222.1:6180/images/ironic-python-agent.initramfs +IRONIC_ENDPOINT=https://192.168.222.1:6385/v1/ diff --git a/config/overlays/e2e-release-0.8/ironic.env b/config/overlays/e2e-release-0.8/ironic.env index 0187ae15e4..44147ae0f3 100644 --- a/config/overlays/e2e-release-0.8/ironic.env +++ b/config/overlays/e2e-release-0.8/ironic.env @@ -1,3 +1,3 @@ -DEPLOY_KERNEL_URL=http://192.168.222.199:6180/images/ironic-python-agent.kernel -DEPLOY_RAMDISK_URL=http://192.168.222.199:6180/images/ironic-python-agent.initramfs -IRONIC_ENDPOINT=https://192.168.222.199:6385/v1/ +DEPLOY_KERNEL_URL=http://192.168.222.1:6180/images/ironic-python-agent.kernel +DEPLOY_RAMDISK_URL=http://192.168.222.1:6180/images/ironic-python-agent.initramfs +IRONIC_ENDPOINT=https://192.168.222.1:6385/v1/ diff --git a/config/overlays/e2e-release-0.9/ironic.env b/config/overlays/e2e-release-0.9/ironic.env index 0187ae15e4..44147ae0f3 100644 --- a/config/overlays/e2e-release-0.9/ironic.env +++ b/config/overlays/e2e-release-0.9/ironic.env @@ -1,3 +1,3 @@ -DEPLOY_KERNEL_URL=http://192.168.222.199:6180/images/ironic-python-agent.kernel -DEPLOY_RAMDISK_URL=http://192.168.222.199:6180/images/ironic-python-agent.initramfs -IRONIC_ENDPOINT=https://192.168.222.199:6385/v1/ +DEPLOY_KERNEL_URL=http://192.168.222.1:6180/images/ironic-python-agent.kernel +DEPLOY_RAMDISK_URL=http://192.168.222.1:6180/images/ironic-python-agent.initramfs +IRONIC_ENDPOINT=https://192.168.222.1:6385/v1/ diff --git a/config/overlays/e2e/ironic.env b/config/overlays/e2e/ironic.env index 0187ae15e4..44147ae0f3 100644 --- a/config/overlays/e2e/ironic.env +++ b/config/overlays/e2e/ironic.env @@ -1,3 +1,3 @@ -DEPLOY_KERNEL_URL=http://192.168.222.199:6180/images/ironic-python-agent.kernel -DEPLOY_RAMDISK_URL=http://192.168.222.199:6180/images/ironic-python-agent.initramfs -IRONIC_ENDPOINT=https://192.168.222.199:6385/v1/ +DEPLOY_KERNEL_URL=http://192.168.222.1:6180/images/ironic-python-agent.kernel +DEPLOY_RAMDISK_URL=http://192.168.222.1:6180/images/ironic-python-agent.initramfs +IRONIC_ENDPOINT=https://192.168.222.1:6385/v1/ diff --git a/hack/ci-e2e.sh b/hack/ci-e2e.sh index 5d55b982f8..b3100f5ee5 100755 --- a/hack/ci-e2e.sh +++ b/hack/ci-e2e.sh @@ -33,8 +33,6 @@ echo "BMO_E2E_EMULATOR=${BMO_E2E_EMULATOR}" export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/ironic.yaml" export E2E_BMCS_CONF_FILE="${REPO_ROOT}/test/e2e/config/bmcs-${BMC_PROTOCOL}.yaml" -LOAD_LOCAL_IRONIC="${LOAD_LOCAL_IRONIC:-false}" - case "${GINKGO_FOCUS:-}" in *upgrade*) export DEPLOY_IRONIC="false" @@ -46,12 +44,11 @@ case "${GINKGO_FOCUS:-}" in export GINKGO_SKIP="${GINKGO_SKIP:-upgrade}" ;; esac -export USE_EXISTING_CLUSTER="true" # Ensure requirements are installed "${REPO_ROOT}/hack/e2e/ensure_go.sh" export PATH="/usr/local/go/bin:${PATH}" -"${REPO_ROOT}/hack/e2e/ensure_minikube.sh" +"${REPO_ROOT}/hack/e2e/ensure_kind.sh" "${REPO_ROOT}/hack/e2e/ensure_htpasswd.sh" # CAPI test framework uses kubectl in the background "${REPO_ROOT}/hack/e2e/ensure_kubectl.sh" @@ -60,42 +57,15 @@ export PATH="/usr/local/go/bin:${PATH}" # Build the container image with e2e tag (used in tests) IMG=quay.io/metal3-io/baremetal-operator:e2e make docker -# Set up minikube -minikube start --driver=kvm2 - virsh -c qemu:///system net-define "${REPO_ROOT}/hack/e2e/net.xml" virsh -c qemu:///system net-start baremetal-e2e -# Attach baremetal-e2e interface to minikube with specific mac. -# This will give minikube a known reserved IP address that we can use for Ironic -virsh -c qemu:///system attach-interface --domain minikube --mac="52:54:00:6c:3c:01" \ - --model virtio --source baremetal-e2e --type network --config - -# Restart minikube to apply the changes -minikube stop -## Following loop is to avoid minikube restart issue -## https://github.com/kubernetes/minikube/issues/14456 -while ! minikube start; do sleep 30; done - -# Load the BMO e2e image into it -# minikube image load quay.io/metal3-io/baremetal-operator:e2e -# Temporary workaround for https://github.com/kubernetes/minikube/issues/18021 -docker image save -o /tmp/bmo-e2e.tar quay.io/metal3-io/baremetal-operator:e2e -minikube image load /tmp/bmo-e2e.tar -rm /tmp/bmo-e2e.tar -if [[ "${LOAD_LOCAL_IRONIC}" == "true" ]]; then - echo "Saving local ironic image!" - docker image save -o \ - "/tmp/ironic-e2e.tar" "quay.io/metal3-io/ironic:local" - minikube image load /tmp/ironic-e2e.tar - rm /tmp/ironic-e2e.tar -fi # This IP is defined by the network we created above. IP_ADDRESS="192.168.222.1" # This IP is also defined by the network above, and is used consistently in all of # our e2e overlays -export IRONIC_PROVISIONING_IP="192.168.222.199" +export IRONIC_PROVISIONING_IP="${IP_ADDRESS}" # Build vbmctl make build-vbmctl @@ -114,7 +84,7 @@ if [[ "${BMO_E2E_EMULATOR}" == "vbmc" ]]; then address=$(echo "${bmc}" | jq -r '.address') hostName=$(echo "${bmc}" | jq -r '.hostName') vbmc_port="${address##*:}" - "${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${hostName}" "${vbmc_port}" + "${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${hostName}" "${vbmc_port}" "${IP_ADDRESS}" done elif [[ "${BMO_E2E_EMULATOR}" == "sushy-tools" ]]; then diff --git a/hack/clean-e2e.sh b/hack/clean-e2e.sh index 17c8f7f010..40ab5d10fb 100755 --- a/hack/clean-e2e.sh +++ b/hack/clean-e2e.sh @@ -3,7 +3,7 @@ REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. cd "${REPO_ROOT}" || exit 1 -minikube delete +kind delete cluster docker rm -f vbmc docker rm -f image-server-e2e docker rm -f sushy-tools diff --git a/hack/e2e/ensure_kind.sh b/hack/e2e/ensure_kind.sh new file mode 100755 index 0000000000..ed6af17b27 --- /dev/null +++ b/hack/e2e/ensure_kind.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -eux + +USR_LOCAL_BIN="/usr/local/bin" +MINIMUM_KIND_VERSION=v0.26.0 + +# Ensure the kind tool exists and is a viable version, or installs it +verify_kind_version() +{ + # If kind is not available on the path, get it + if ! [ -x "$(command -v kind)" ]; then + if [[ "${OSTYPE}" == "linux-gnu" ]]; then + echo "kind not found, installing" + curl -LO "https://kind.sigs.k8s.io/dl/${MINIMUM_KIND_VERSION}/kind-linux-amd64" + sudo install kind "${USR_LOCAL_BIN}/kind" + else + echo "Missing required binary in path: kind" + return 2 + fi + fi + + local kind_version + IFS=" " read -ra kind_version <<< "$(kind version)" + if [[ "${MINIMUM_KIND_VERSION}" != $(echo -e "${MINIMUM_KIND_VERSION}\n${kind_version[1]}" | sort -s -t. -k 1,1 -k 2,2n -k 3,3n | head -n1) ]]; then + cat << EOF +Detected kind version: ${kind_version[2]}. +Requires ${MINIMUM_KIND_VERSION} or greater. +Please install ${MINIMUM_KIND_VERSION} or later. +EOF + return 2 + fi +} + +verify_kind_version diff --git a/hack/e2e/ensure_minikube.sh b/hack/e2e/ensure_minikube.sh deleted file mode 100755 index c4c48f9938..0000000000 --- a/hack/e2e/ensure_minikube.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -USR_LOCAL_BIN="/usr/local/bin" -OS=$(go env GOOS) -ARCH=$(go env GOARCH) -MINIMUM_MINIKUBE_VERSION=v1.33.0 - -verify_minikube_version() { - if ! [ -x "$(command -v minikube)" ]; then - if [[ "${OSTYPE}" == "linux-gnu" ]]; then - echo "minikube not found, installing" - curl -LO "https://storage.googleapis.com/minikube/releases/${MINIMUM_MINIKUBE_VERSION}/minikube-${OS}-${ARCH}" - sudo install minikube-linux-amd64 "${USR_LOCAL_BIN}/minikube" - else - echo "Missing required binary in path: minikube" - return 2 - fi - fi - local minikube_version - minikube_version="$(minikube version --short)" - if [[ "${MINIMUM_MINIKUBE_VERSION}" != $(echo -e "${MINIMUM_MINIKUBE_VERSION}\n${minikube_version}" | sort -V | head -n1) ]]; then - cat << EOF -Detected minikube version: ${minikube_version}. -Requires ${MINIMUM_MINIKUBE_VERSION} or greater. -Please install ${MINIMUM_MINIKUBE_VERSION} or later. -EOF - return 2 - fi -} - -verify_minikube_version diff --git a/hack/e2e/net.xml b/hack/e2e/net.xml index 71fdad47e8..6039360a57 100644 --- a/hack/e2e/net.xml +++ b/hack/e2e/net.xml @@ -9,8 +9,7 @@ - - + diff --git a/ironic-deployment/components/tls/certificate.yaml b/ironic-deployment/components/tls/certificate.yaml index d8a83c7aa7..2e576873db 100644 --- a/ironic-deployment/components/tls/certificate.yaml +++ b/ironic-deployment/components/tls/certificate.yaml @@ -12,8 +12,6 @@ metadata: spec: commonName: ironic-ca isCA: true - ipAddresses: - - IRONIC_HOST_IP issuerRef: kind: Issuer name: selfsigned-issuer diff --git a/ironic-deployment/overlays/e2e-release-24.1/ironic_bmo_configmap.env b/ironic-deployment/overlays/e2e-release-24.1/ironic_bmo_configmap.env index a0840f4859..9dc4c4bf68 100644 --- a/ironic-deployment/overlays/e2e-release-24.1/ironic_bmo_configmap.env +++ b/ironic-deployment/overlays/e2e-release-24.1/ironic_bmo_configmap.env @@ -1,6 +1,7 @@ HTTP_PORT=6180 -PROVISIONING_IP=192.168.222.199 -CACHEURL=http://192.168.222.199/images +PROVISIONING_INTERFACE=eth0 +CACHEURL=http://192.168.222.1/images +IRONIC_EXTERNAL_CALLBACK_URL=https://192.168.222.1:6385 IRONIC_KERNEL_PARAMS=console=ttyS0 IRONIC_INSPECTOR_VLAN_INTERFACES=all USE_IRONIC_INSPECTOR=false diff --git a/ironic-deployment/overlays/e2e-release-24.1/kustomization.yaml b/ironic-deployment/overlays/e2e-release-24.1/kustomization.yaml index 1936f375f6..0bf9aa3827 100644 --- a/ironic-deployment/overlays/e2e-release-24.1/kustomization.yaml +++ b/ironic-deployment/overlays/e2e-release-24.1/kustomization.yaml @@ -16,7 +16,24 @@ configMapGenerator: behavior: create patches: - - path: ironic-patch.yaml +- path: ironic-patch.yaml +# The TLS component adds certificates but it cannot know the exact IPs of our environment. +# Here we patch the certificates to have the correct IPs. +# - 192.168.222.1: management computer IP, forwarded to ironic inside kind +- patch: |- + - op: replace + path: /spec/ipAddresses/0 + value: 192.168.222.1 + target: + kind: Certificate + name: ironic-cert +- patch: |- + - op: replace + path: /spec/ipAddresses/0 + value: 192.168.222.1 + target: + kind: Certificate + name: ironic-cacert images: - name: quay.io/metal3-io/ironic @@ -28,18 +45,3 @@ secretGenerator: behavior: create envs: - ironic-htpasswd - -replacements: - # Replace IRONIC_HOST_IP in certificates with the PROVISIONING_IP from the configmap - - source: - kind: ConfigMap - name: ironic-bmo-configmap - fieldPath: .data.PROVISIONING_IP - targets: - - select: - version: v1 - group: cert-manager.io - kind: Certificate - name: - fieldPaths: - - .spec.ipAddresses.0 diff --git a/ironic-deployment/overlays/e2e-release-25.0/ironic_bmo_configmap.env b/ironic-deployment/overlays/e2e-release-25.0/ironic_bmo_configmap.env index a0840f4859..9dc4c4bf68 100644 --- a/ironic-deployment/overlays/e2e-release-25.0/ironic_bmo_configmap.env +++ b/ironic-deployment/overlays/e2e-release-25.0/ironic_bmo_configmap.env @@ -1,6 +1,7 @@ HTTP_PORT=6180 -PROVISIONING_IP=192.168.222.199 -CACHEURL=http://192.168.222.199/images +PROVISIONING_INTERFACE=eth0 +CACHEURL=http://192.168.222.1/images +IRONIC_EXTERNAL_CALLBACK_URL=https://192.168.222.1:6385 IRONIC_KERNEL_PARAMS=console=ttyS0 IRONIC_INSPECTOR_VLAN_INTERFACES=all USE_IRONIC_INSPECTOR=false diff --git a/ironic-deployment/overlays/e2e-release-25.0/kustomization.yaml b/ironic-deployment/overlays/e2e-release-25.0/kustomization.yaml index 8a84c06107..2c6839b1d8 100644 --- a/ironic-deployment/overlays/e2e-release-25.0/kustomization.yaml +++ b/ironic-deployment/overlays/e2e-release-25.0/kustomization.yaml @@ -17,6 +17,23 @@ configMapGenerator: patches: - path: ironic-patch.yaml +# The TLS component adds certificates but it cannot know the exact IPs of our environment. +# Here we patch the certificates to have the correct IPs. +# - 192.168.222.1: management computer IP, forwarded to ironic inside kind +- patch: |- + - op: replace + path: /spec/ipAddresses/0 + value: 192.168.222.1 + target: + kind: Certificate + name: ironic-cert +- patch: |- + - op: replace + path: /spec/ipAddresses/0 + value: 192.168.222.1 + target: + kind: Certificate + name: ironic-cacert images: - name: quay.io/metal3-io/ironic @@ -28,18 +45,3 @@ secretGenerator: behavior: create envs: - ironic-htpasswd - -replacements: -# Replace IRONIC_HOST_IP in certificates with the PROVISIONING_IP from the configmap -- source: - kind: ConfigMap - name: ironic-bmo-configmap - fieldPath: .data.PROVISIONING_IP - targets: - - select: - version: v1 - group: cert-manager.io - kind: Certificate - name: - fieldPaths: - - .spec.ipAddresses.0 diff --git a/ironic-deployment/overlays/e2e-release-26.0/ironic_bmo_configmap.env b/ironic-deployment/overlays/e2e-release-26.0/ironic_bmo_configmap.env index a0840f4859..9dc4c4bf68 100644 --- a/ironic-deployment/overlays/e2e-release-26.0/ironic_bmo_configmap.env +++ b/ironic-deployment/overlays/e2e-release-26.0/ironic_bmo_configmap.env @@ -1,6 +1,7 @@ HTTP_PORT=6180 -PROVISIONING_IP=192.168.222.199 -CACHEURL=http://192.168.222.199/images +PROVISIONING_INTERFACE=eth0 +CACHEURL=http://192.168.222.1/images +IRONIC_EXTERNAL_CALLBACK_URL=https://192.168.222.1:6385 IRONIC_KERNEL_PARAMS=console=ttyS0 IRONIC_INSPECTOR_VLAN_INTERFACES=all USE_IRONIC_INSPECTOR=false diff --git a/ironic-deployment/overlays/e2e-release-26.0/kustomization.yaml b/ironic-deployment/overlays/e2e-release-26.0/kustomization.yaml index d6d52e8d4a..ae5bc1f6a0 100644 --- a/ironic-deployment/overlays/e2e-release-26.0/kustomization.yaml +++ b/ironic-deployment/overlays/e2e-release-26.0/kustomization.yaml @@ -17,6 +17,23 @@ configMapGenerator: patches: - path: ironic-patch.yaml +# The TLS component adds certificates but it cannot know the exact IPs of our environment. +# Here we patch the certificates to have the correct IPs. +# - 192.168.222.1: management computer IP, forwarded to ironic inside kind +- patch: |- + - op: replace + path: /spec/ipAddresses/0 + value: 192.168.222.1 + target: + kind: Certificate + name: ironic-cert +- patch: |- + - op: replace + path: /spec/ipAddresses/0 + value: 192.168.222.1 + target: + kind: Certificate + name: ironic-cacert images: - name: quay.io/metal3-io/ironic @@ -28,18 +45,3 @@ secretGenerator: behavior: create envs: - ironic-htpasswd - -replacements: -# Replace IRONIC_HOST_IP in certificates with the PROVISIONING_IP from the configmap -- source: - kind: ConfigMap - name: ironic-bmo-configmap - fieldPath: .data.PROVISIONING_IP - targets: - - select: - version: v1 - group: cert-manager.io - kind: Certificate - name: - fieldPaths: - - .spec.ipAddresses.0 diff --git a/ironic-deployment/overlays/e2e/ironic_bmo_configmap.env b/ironic-deployment/overlays/e2e/ironic_bmo_configmap.env index d4aba53758..9be192df9c 100644 --- a/ironic-deployment/overlays/e2e/ironic_bmo_configmap.env +++ b/ironic-deployment/overlays/e2e/ironic_bmo_configmap.env @@ -1,6 +1,7 @@ HTTP_PORT=6180 -PROVISIONING_IP=192.168.222.199 -CACHEURL=http://192.168.222.199/images +PROVISIONING_INTERFACE=eth0 +CACHEURL=http://192.168.222.1/images +IRONIC_EXTERNAL_CALLBACK_URL=https://192.168.222.1:6385 IRONIC_KERNEL_PARAMS=console=ttyS0 IRONIC_INSPECTOR_VLAN_INTERFACES=all USE_IRONIC_INSPECTOR=false diff --git a/ironic-deployment/overlays/e2e/kustomization.yaml b/ironic-deployment/overlays/e2e/kustomization.yaml index 56f575d74b..2fc2a18570 100644 --- a/ironic-deployment/overlays/e2e/kustomization.yaml +++ b/ironic-deployment/overlays/e2e/kustomization.yaml @@ -17,6 +17,16 @@ configMapGenerator: patches: - path: ironic-patch.yaml +# The TLS component adds certificates but it cannot know the exact IPs of our environment. +# Here we patch the certificates to have the correct IPs. +# - 192.168.222.1: management computer IP, forwarded to ironic inside kind +- patch: |- + - op: replace + path: /spec/ipAddresses/0 + value: 192.168.222.1 + target: + kind: Certificate + name: ironic-cert # TODO(lentzi90): Remove once issue resolved: # https://github.com/metal3-io/baremetal-operator/issues/2110 @@ -31,18 +41,3 @@ secretGenerator: files: - htpasswd=ironic-htpasswd type: Opaque - -replacements: -# Replace IRONIC_HOST_IP in certificates with the PROVISIONING_IP from the configmap -- source: - kind: ConfigMap - name: ironic-bmo-configmap - fieldPath: .data.PROVISIONING_IP - targets: - - select: - version: v1 - group: cert-manager.io - kind: Certificate - name: - fieldPaths: - - .spec.ipAddresses.0 diff --git a/test/e2e/config/fixture.yaml b/test/e2e/config/fixture.yaml index 7a4a099c3f..1ec13e738d 100644 --- a/test/e2e/config/fixture.yaml +++ b/test/e2e/config/fixture.yaml @@ -25,6 +25,9 @@ variables: # This setting is for the separate cluster that are used for upgrade tests UPGRADE_DEPLOY_CERT_MANAGER: "true" + # We can create separate clusters for each upgrade test with fixture. + # Since there is no ironic involved, we do not risk having any port collisions. + UPGRADE_USE_EXISTING_CLUSTER: "false" IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img" ISO_IMAGE_URL: "http://192.168.222.1/cirros.iso" diff --git a/test/e2e/config/ironic.yaml b/test/e2e/config/ironic.yaml index 1352a7b17e..9d05da1cb7 100644 --- a/test/e2e/config/ironic.yaml +++ b/test/e2e/config/ironic.yaml @@ -2,6 +2,8 @@ images: # Use locally built e2e images - name: quay.io/metal3-io/baremetal-operator:e2e loadBehavior: tryLoad +# - name: quay.io/metal3-io/ironic:local +# loadBehavior: tryLoad # Save some time and network by using cached images if available - name: quay.io/metal3-io/baremetal-operator:release-0.6 loadBehavior: tryLoad @@ -27,6 +29,11 @@ variables: # This setting is for the separate cluster that are used for upgrade tests UPGRADE_DEPLOY_CERT_MANAGER: "true" + # We run upgrade tests separately, so they can use the default kind cluster + # since there are no other tests running at the same time. + # Creating separate upgrade clusters would cause port collisions with the default kind cluster. + # We also configure these jobs to not deploy BMO/Ironic so that can be handled by the upgrade test. + UPGRADE_USE_EXISTING_CLUSTER: "true" IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img" ISO_IMAGE_URL: "http://192.168.222.1/minimal_linux_live-v2.iso" @@ -60,37 +67,59 @@ intervals: default/wait-secret-deletion: ["1m", "1s"] default/wait-connect-ssh: ["2m", "10s"] +kindExtraPortMappings: +# Expose Ironic ports so they are reachable outside of kind +- containerPort: 6180 + hostPort: 6180 + listenAddress: "0.0.0.0" + protocol: TCP +- containerPort: 6385 + hostPort: 6385 + listenAddress: "0.0.0.0" + protocol: TCP +- containerPort: 5050 + hostPort: 5050 + listenAddress: "0.0.0.0" + protocol: TCP + + bmoIronicUpgradeSpecs: +# Upgrade BMO 0.6 -> main - deployIronic: true deployBMO: true initBMOKustomization: "../../config/overlays/e2e-release-0.6" initIronicKustomization: "../../ironic-deployment/overlays/e2e" upgradeEntityKustomization: "../../config/overlays/e2e" upgradeEntityName: "bmo" +# Upgrade BMO 0.8 -> main - deployIronic: true deployBMO: true initBMOKustomization: "../../config/overlays/e2e-release-0.8" initIronicKustomization: "../../ironic-deployment/overlays/e2e" upgradeEntityKustomization: "../../config/overlays/e2e" upgradeEntityName: "bmo" +# Upgrade BMO 0.9 -> main - deployIronic: true deployBMO: true initBMOKustomization: "../../config/overlays/e2e-release-0.9" initIronicKustomization: "../../ironic-deployment/overlays/e2e" upgradeEntityKustomization: "../../config/overlays/e2e" upgradeEntityName: "bmo" +# Upgrade Ironic 24.1 -> latest | BMO 0.9 - deployIronic: true deployBMO: true initBMOKustomization: "../../config/overlays/e2e-release-0.9" initIronicKustomization: "../../ironic-deployment/overlays/e2e-release-24.1" upgradeEntityKustomization: "../../ironic-deployment/overlays/e2e" upgradeEntityName: "ironic" +# Upgrade Ironic 25.0 -> latest | BMO 0.9 - deployIronic: true deployBMO: true initBMOKustomization: "../../config/overlays/e2e-release-0.9" initIronicKustomization: "../../ironic-deployment/overlays/e2e-release-25.0" upgradeEntityKustomization: "../../ironic-deployment/overlays/e2e" upgradeEntityName: "ironic" +# Upgrade Ironic 26.0 -> latest | BMO 0.9 - deployIronic: true deployBMO: true initBMOKustomization: "../../config/overlays/e2e-release-0.9" diff --git a/test/e2e/e2e_config.go b/test/e2e/e2e_config.go index c79215b396..5b7e44f564 100644 --- a/test/e2e/e2e_config.go +++ b/test/e2e/e2e_config.go @@ -14,6 +14,7 @@ import ( "github.com/pkg/errors" "gopkg.in/yaml.v2" "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/kind/pkg/apis/config/v1alpha4" ) // LoadImageBehavior indicates the behavior when loading an image. @@ -63,6 +64,9 @@ type Config struct { // BMOIronicUpgradeSpecs BMOIronicUpgradeSpecs []BMOIronicUpgradeInput `yaml:"bmoIronicUpgradeSpecs,omitempty"` + + // Extra port mappings for the kind cluster + KindExtraPortMappings []v1alpha4.PortMapping `yaml:"kindExtraPortMappings,omitempty"` } // LoadE2EConfig loads the configuration for the e2e test environment. diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index b6e2b61494..26b8d78322 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -153,8 +153,9 @@ var _ = SynchronizedBeforeSuite(func() []byte { kubeconfigPath = GetKubeconfigPath() } else { clusterProvider = bootstrap.CreateKindBootstrapClusterAndLoadImages(ctx, bootstrap.CreateKindBootstrapClusterAndLoadImagesInput{ - Name: "bmo-e2e", - Images: e2eConfig.Images, + Name: "bmo-e2e", + Images: e2eConfig.Images, + ExtraPortMappings: e2eConfig.KindExtraPortMappings, }) Expect(clusterProvider).ToNot(BeNil(), "Failed to create a cluster") kubeconfigPath = clusterProvider.GetKubeconfigPath() diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index 07c6e24ab7..0848c94dd2 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -173,14 +173,14 @@ const hardwareDetailsRelease04 = ` ` // RunUpgradeTest tests upgrade from an older version of BMO or Ironic --> main branch version with the following steps: -// - Initiate the cluster with an the older version of either BMO or Ironic, and the latest Ironic/BMO version that is suitable with it -// - Create a new namespace, and in it a BMH object with "disabled" annotation. -// - Wait until the BMH gets to "available" state. Because of the "disabled" annotation, it won't get further provisioned. -// - Upgrade BMO/Ironic to latest version. -// - Patch the BMH object with proper specs, so that it could be provisioned. -// - If the BMH is successfully provisioned, it means the upgraded BMO/Ironic recognized that BMH, hence the upgrade succeeded. +// - Initiate the cluster with an the older version of either BMO or Ironic, and the latest Ironic/BMO version that is suitable with it +// - Create a new namespace, and in it a BMH object with "disabled" annotation. +// - Wait until the BMH gets to "available" state. Because of the "disabled" annotation, it won't get further provisioned. +// - Upgrade BMO/Ironic to latest version. +// - Patch the BMH object with proper specs, so that it could be provisioned. +// - If the BMH is successfully provisioned, it means the upgraded BMO/Ironic recognized that BMH, hence the upgrade succeeded. +// // The function returns the namespace object, with its cancelFunc. These can be used to clean up the created resources. - func RunUpgradeTest(ctx context.Context, input *BMOIronicUpgradeInput, upgradeClusterProxy framework.ClusterProxy) (*corev1.Namespace, context.CancelFunc) { bmoIronicNamespace := "baremetal-operator-system" initBMOKustomization := input.InitBMOKustomization @@ -359,14 +359,15 @@ var _ = Describe("Upgrade", Label("optional", "upgrade"), func() { var kubeconfigPath string upgradeClusterName := "bmo-e2e-upgrade" - if useExistingCluster { + if e2eConfig.GetBoolVariable("UPGRADE_USE_EXISTING_CLUSTER") { kubeconfigPath = GetKubeconfigPath() } else { By("Creating a separate cluster for upgrade tests") upgradeClusterName = fmt.Sprintf("bmo-e2e-upgrade-%d", GinkgoParallelProcess()) upgradeClusterProvider := bootstrap.CreateKindBootstrapClusterAndLoadImages(ctx, bootstrap.CreateKindBootstrapClusterAndLoadImagesInput{ - Name: upgradeClusterName, - Images: e2eConfig.Images, + Name: upgradeClusterName, + Images: e2eConfig.Images, + ExtraPortMappings: e2eConfig.KindExtraPortMappings, }) Expect(upgradeClusterProvider).ToNot(BeNil(), "Failed to create a cluster") kubeconfigPath = upgradeClusterProvider.GetKubeconfigPath() @@ -399,9 +400,11 @@ var _ = Describe("Upgrade", Label("optional", "upgrade"), func() { } }) DescribeTable("", + // Test function that runs for each table entry func(ctx context.Context, input *BMOIronicUpgradeInput) { namespace, cancelWatches = RunUpgradeTest(ctx, input, upgradeClusterProxy) }, + // Description function that generates test descriptions func(ctx context.Context, input *BMOIronicUpgradeInput) string { var upgradeFromKustomization string upgradeEntityName := input.UpgradeEntityName diff --git a/test/go.mod b/test/go.mod index 1443e6bdbc..3dd366cad6 100644 --- a/test/go.mod +++ b/test/go.mod @@ -21,6 +21,7 @@ require ( sigs.k8s.io/cluster-api v1.9.3 sigs.k8s.io/cluster-api/test v1.9.3 sigs.k8s.io/controller-runtime v0.19.4 + sigs.k8s.io/kind v0.25.0 sigs.k8s.io/kustomize/api v0.19.0 sigs.k8s.io/kustomize/kyaml v0.19.0 ) @@ -153,7 +154,6 @@ require ( sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect sigs.k8s.io/gateway-api v1.1.0 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect - sigs.k8s.io/kind v0.25.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/tools/bmh_test/vm2vbmc.sh b/tools/bmh_test/vm2vbmc.sh index bfbfbc5c26..c2be2ecf7a 100755 --- a/tools/bmh_test/vm2vbmc.sh +++ b/tools/bmh_test/vm2vbmc.sh @@ -4,8 +4,9 @@ set -eux NAME="${1:?}" VBMC_PORT="${2:?}" +VBMC_ADDRESS="${3:-"::"}" # Add the BareMetalHost VM to VBMC -docker exec vbmc vbmc add "${NAME}" --port "${VBMC_PORT}" --libvirt-uri "qemu:///system" +docker exec vbmc vbmc add "${NAME}" --port "${VBMC_PORT}" --address "${VBMC_ADDRESS}" --libvirt-uri "qemu:///system" docker exec vbmc vbmc start "${NAME}" docker exec vbmc vbmc list From a4cab903a77d215b9cc55b7a4dead984db538a83 Mon Sep 17 00:00:00 2001 From: Lennart Jern Date: Wed, 22 Jan 2025 10:30:43 +0000 Subject: [PATCH 3/4] Fixture: Upload artifacts also on failure Signed-off-by: Lennart Jern --- .github/workflows/e2e-fixture-test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-fixture-test.yml b/.github/workflows/e2e-fixture-test.yml index 03f7d8fd21..7c612e2efa 100644 --- a/.github/workflows/e2e-fixture-test.yml +++ b/.github/workflows/e2e-fixture-test.yml @@ -24,8 +24,8 @@ jobs: - name: Install libvirt run: | - sudo apt-get update - sudo apt-get install -y libvirt-dev + sudo apt-get update + sudo apt-get install -y libvirt-dev - name: Build BMO e2e Docker Image env: @@ -40,6 +40,7 @@ jobs: run: make test-e2e - name: Upload artifacts + if: ${{ !cancelled() }} uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: name: artifacts-fixture.tar.gz From 2f49b5724ecff86d4f527ae0aeaf1d3b4e5e11a8 Mon Sep 17 00:00:00 2001 From: Lennart Jern Date: Wed, 22 Jan 2025 10:51:52 +0000 Subject: [PATCH 4/4] E2E: Delete BMHs prior to namespace deletion Sometimes the fixture tests hit the timeout for namespace deletion. The BMO logs indicate that BMO is trying to create new objects while the namespace is terminating. For example HardwareDetails. To avoid this, I think we can trigger deletion of the BMHs before we delete the namespace. We are running a bit close to the 1m deadline on successful runs in the re-inspection test. I believe this is explained by an extra reconcile loop when the hardwaredetails are updated because of the inspection. No other fixture test is close to this deadline normally. Signed-off-by: Lennart Jern --- test/e2e/common.go | 16 ++++++++++++++++ test/e2e/config/fixture.yaml | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/test/e2e/common.go b/test/e2e/common.go index 880b69bd51..59ff1850a9 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -84,6 +84,18 @@ func WaitForBmhInProvisioningState(ctx context.Context, input WaitForBmhInProvis }, intervals...).Should(Succeed()) } +// DeleteBmhsInNamespace deletes all BMHs in the given namespace. +func DeleteBmhsInNamespace(ctx context.Context, deleter client.Client, namespace string) { + bmh := metal3api.BareMetalHost{} + opts := client.DeleteAllOfOptions{ + ListOptions: client.ListOptions{ + Namespace: namespace, + }, + } + err := deleter.DeleteAllOf(ctx, &bmh, &opts) + Expect(err).NotTo(HaveOccurred(), "Unable to delete BMHs") +} + // WaitForBmhDeletedInput is the input for WaitForBmhDeleted. type WaitForBmhDeletedInput struct { Client client.Client @@ -134,6 +146,10 @@ func WaitForNamespaceDeleted(ctx context.Context, input WaitForNamespaceDeletedI } func cleanup(ctx context.Context, clusterProxy framework.ClusterProxy, namespace *corev1.Namespace, cancelWatches context.CancelFunc, intervals ...interface{}) { + // Trigger deletion of BMHs before deleting the namespace. + // This way there should be no risk of BMO getting stuck trying to progress + // and create HardwareDetails or similar, while the namespace is terminating. + DeleteBmhsInNamespace(ctx, clusterProxy.GetClient(), namespace.Name) framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{ Deleter: clusterProxy.GetClient(), Name: namespace.Name, diff --git a/test/e2e/config/fixture.yaml b/test/e2e/config/fixture.yaml index 1ec13e738d..33b87435ec 100644 --- a/test/e2e/config/fixture.yaml +++ b/test/e2e/config/fixture.yaml @@ -44,7 +44,7 @@ intervals: inspection/wait-available: ["1m", "1s"] external-inspection/wait-available: ["5s", "1ms"] default/wait-deployment: ["5m", "1s"] - default/wait-namespace-deleted: ["1m", "1s"] + default/wait-namespace-deleted: ["2m", "1s"] ironic/wait-deployment: ["10m", "2s"] default/wait-registration: ["5s", "5ms"] default/wait-provisioning: ["1m", "10ms"]