From 8f433fb93ffc2cf434bcc715725835b3eae94bf6 Mon Sep 17 00:00:00 2001 From: Rohan Kumar Date: Fri, 1 Aug 2025 16:20:47 +0530 Subject: [PATCH 1/2] ci : test k6 install and load test execution in e2e workflow - Add another Makefile target for load test execution - Execute this target in e2e test script Signed-off-by: Rohan Kumar --- .ci/openshift_e2e.sh | 2 + Makefile | 5 + test/load/devworkspace_load_test.js | 484 ++++++++++++++++++++++++++++ test/load/runk6.sh | 341 ++++++++++++++++++++ 4 files changed, 832 insertions(+) create mode 100644 test/load/devworkspace_load_test.js create mode 100644 test/load/runk6.sh diff --git a/.ci/openshift_e2e.sh b/.ci/openshift_e2e.sh index 9b9b175b2..c55418f42 100755 --- a/.ci/openshift_e2e.sh +++ b/.ci/openshift_e2e.sh @@ -72,4 +72,6 @@ make install export CLEAN_UP_AFTER_SUITE="false" make test_e2e bumpLogs + +make test_load ARGS="--mode operator --max-vus 250 --separate-namespaces false --test-duration-minutes 25 --dwo-namespace devworkspace-controller --logs-dir ${ARTIFACT_DIR}/load-testing-logs" make uninstall diff --git a/Makefile b/Makefile index b81b8187a..315f17d68 100644 --- a/Makefile +++ b/Makefile @@ -115,6 +115,11 @@ test_e2e_debug: mkdir -p /tmp/artifacts dlv test --listen=:2345 --headless=true --api-version=2 ./test/e2e/cmd/workspaces_test.go -- --ginkgo.fail-fast --ginkgo.junit-report=/tmp/artifacts/junit-workspaces-operator.xml +test_load: + @echo "Starting Load Testing Script..." && \ + bash ./test/load/runk6.sh $(ARGS) && \ + echo "Done" + ### manager: Build manager binary manager: generate fmt vet go build -o bin/manager main.go diff --git a/test/load/devworkspace_load_test.js b/test/load/devworkspace_load_test.js new file mode 100644 index 000000000..45c54ab0f --- /dev/null +++ b/test/load/devworkspace_load_test.js @@ -0,0 +1,484 @@ +// +// Copyright (c) 2019-2025 Red Hat, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import http from 'k6/http'; +import {check, sleep} from 'k6'; +import {Trend, Counter} from 'k6/metrics'; +import {htmlReport} from "https://raw.githubusercontent.com/benc-uk/k6-reporter/main/dist/bundle.js"; +import {textSummary} from "https://jslib.k6.io/k6-summary/0.0.1/index.js"; + +const inCluster = __ENV.IN_CLUSTER === 'true'; +const apiServer = inCluster ? `https://kubernetes.default.svc` : __ENV.KUBE_API; +const token = inCluster ? open('/var/run/secrets/kubernetes.io/serviceaccount/token') : __ENV.KUBE_TOKEN; +const useSeparateNamespaces = __ENV.SEPARATE_NAMESPACES === "true"; +const operatorNamespace = __ENV.DWO_NAMESPACE || 'openshift-operators'; +const externalDevWorkspaceLink = __ENV.DEVWORKSPACE_LINK || ''; +const shouldCreateAutomountResources = (__ENV.CREATE_AUTOMOUNT_RESOURCES || 'false') === 'true'; +const maxVUs = Number(__ENV.MAX_VUS || 50); +const devWorkspaceReadyTimeout = Number(__ENV.DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS || 600); +const autoMountConfigMapName = 'dwo-load-test-automount-configmap'; +const autoMountSecretName = 'dwo-load-test-automount-secret'; +const labelType = "test-type"; +const labelKey = "load-test"; +const loadTestDurationInMinutes = __ENV.TEST_DURATION_IN_MINUTES || "25"; +const loadTestNamespace = __ENV.LOAD_TEST_NAMESPACE || "loadtest-devworkspaces"; + +const headers = { + Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', +}; + +export const options = { + scenarios: { + create_and_delete_devworkspaces: { + executor: 'ramping-vus', + startVUs: 0, + stages: generateLoadTestStages(maxVUs), + gracefulRampDown: '1m', + }, + final_cleanup: { + executor: 'per-vu-iterations', + vus: 1, + iterations: 1, + startTime: `${loadTestDurationInMinutes}m`, + exec: 'final_cleanup', + }, + }, thresholds: { + 'checks': ['rate>0.95'], + 'devworkspace_create_duration': ['p(95)<15000'], + 'devworkspace_delete_duration': ['p(95)<10000'], + 'devworkspace_ready_duration': ['p(95)<60000'], + 'devworkspace_ready_failed': ['count<5'], + 'operator_cpu_violations': ['count==0'], + 'operator_mem_violations': ['count==0'], + }, insecureSkipTLSVerify: true, // trust self-signed certs like in CRC +}; + +const devworkspaceCreateDuration = new Trend('devworkspace_create_duration'); +const devworkspaceReady = new Counter('devworkspace_ready'); +const devworkspaceDeleteDuration = new Trend('devworkspace_delete_duration'); +const devworkspaceReadyDuration = new Trend('devworkspace_ready_duration'); +const devworkspaceReadyFailed = new Counter('devworkspace_ready_failed'); +const operatorCpu = new Trend('average_operator_cpu'); // in milli cores +const operatorMemory = new Trend('average_operator_memory'); // in Mi +const devworkspacesCreated = new Counter('devworkspace_create_count'); +const operatorCpuViolations = new Counter('operator_cpu_violations'); +const operatorMemViolations = new Counter('operator_mem_violations'); + +const maxCpuMillicores = 250; +const maxMemoryBytes = 200 * 1024 * 1024; + +export function setup() { + if (shouldCreateAutomountResources) { + createNewAutomountConfigMap(); + createNewAutomountSecret(); + } +} + +export default function () { + const vuId = __VU; + const iteration = __ITER; + const crName = `dw-test-${vuId}-${iteration}`; + const namespace = useSeparateNamespaces + ? `load-test-ns-${__VU}-${__ITER}` + : loadTestNamespace; + + if (!apiServer) { + throw new Error('KUBE_API env var is required'); + } + try { + if (useSeparateNamespaces) { + createNewNamespace(namespace); + } + const devWorkspaceCreated = createNewDevWorkspace(namespace, vuId, iteration); + if (devWorkspaceCreated) { + waitUntilDevWorkspaceIsReady(vuId, crName, namespace); + deleteDevWorkspace(crName, namespace); + } + } catch (error) { + console.error(`Load test for ${vuId}-${iteration} failed:`, error.message); + } +} + +export function final_cleanup() { + if (useSeparateNamespaces) { + deleteAllSeparateNamespaces(); + } else { + deleteAllDevWorkspacesInCurrentNamespace(); + } + + if (shouldCreateAutomountResources) { + deleteConfigMap(); + deleteSecret(); + } +} + +export function handleSummary(data) { + const allowed = ['devworkspace_create_count', 'devworkspace_create_duration', 'devworkspace_delete_duration', 'devworkspace_ready_duration', 'devworkspace_ready', 'devworkspace_ready_failed', 'operator_cpu_violations', 'operator_mem_violations', 'average_operator_cpu', 'average_operator_memory']; + + const filteredData = JSON.parse(JSON.stringify(data)); + for (const key of Object.keys(filteredData.metrics)) { + if (!allowed.includes(key)) { + delete filteredData.metrics[key]; + } + } + + let loadTestSummaryReport = { + stdout: textSummary(filteredData, {indent: ' ', enableColors: true}) + } + // Only generate HTML report when running outside the cluster + if (!inCluster) { + loadTestSummaryReport["devworkspace-load-test-report.html"] = htmlReport(data, { + title: "DevWorkspace Operator Load Test Report (HTTP)", + }); + } + return loadTestSummaryReport; +} + +function createNewDevWorkspace(namespace, vuId, iteration) { + const baseUrl = `${apiServer}/apis/workspace.devfile.io/v1alpha2/namespaces/${namespace}/devworkspaces`; + + const manifest = generateDevWorkspaceToCreate(vuId, iteration, namespace); + + const payload = JSON.stringify(manifest); + + const createStart = Date.now(); + const createRes = http.post(baseUrl, payload, {headers}); + check(createRes, { + 'DevWorkspace created': (r) => r.status === 201 || r.status === 409, + }); + + if (createRes.status !== 201 && createRes.status !== 409) { + console.error(`[VU ${vuId}] Failed to create DevWorkspace: ${createRes.status}, ${createRes.body}`); + return false; + } + devworkspaceCreateDuration.add(Date.now() - createStart); + devworkspacesCreated.add(1); + return true; +} + +function waitUntilDevWorkspaceIsReady(vuId, crName, namespace) { + const dwUrl = `${apiServer}/apis/workspace.devfile.io/v1alpha2/namespaces/${namespace}/devworkspaces/${crName}`; + const readyStart = Date.now(); + let isReady = false; + let attempts = 0; + const pollWaitInterval = 5; + const maxAttempts = devWorkspaceReadyTimeout / pollWaitInterval; + let res = {}; + + while (!isReady && attempts < maxAttempts) { + res = http.get(`${dwUrl}`, {headers}); + + if (res.status === 200) { + try { + const body = JSON.parse(res.body); + const phase = body?.status?.phase; + if (phase === 'Ready' || phase === 'Running') { + isReady = true; + break; + } else if (phase === 'Failing' || phase === 'Failed' || phase === 'Error') { + isReady = false; + break; + } + } catch (e) { + console.error(`GET [VU ${vuId}] Failed to parse DevWorkspace from API: ${res.body} : ${e.message}`); + } + } + + checkDevWorkspaceOperatorMetrics(); + sleep(pollWaitInterval); + attempts++; + } + + if (res.status === 200) { + if (isReady) { + devworkspaceReady.add(1); + devworkspaceReadyDuration.add(Date.now() - readyStart); + } else { + devworkspaceReadyFailed.add(1); + } + } +} + +function deleteDevWorkspace(crName, namespace) { + const dwUrl = `${apiServer}/apis/workspace.devfile.io/v1alpha2/namespaces/${namespace}/devworkspaces/${crName}`; + const deleteStart = Date.now(); + const delRes = http.del(`${dwUrl}`, null, {headers}); + devworkspaceDeleteDuration.add(Date.now() - deleteStart); + + check(delRes, { + 'DevWorkspace deleted or not found': (r) => r.status === 200 || r.status === 404, + }); +} + +function checkDevWorkspaceOperatorMetrics() { + const metricsUrl = `${apiServer}/apis/metrics.k8s.io/v1beta1/namespaces/${operatorNamespace}/pods`; + const res = http.get(metricsUrl, {headers}); + + + check(res, { + 'Fetched pod metrics successfully': (r) => r.status === 200, + }); + + if (res.status !== 200) { + console.warn(`[DWO METRICS] Unable to fetch DevWorkspace Operator metrics from Kubernetes, got ${res.status}`); + return; + } + + const data = JSON.parse(res.body); + const operatorPods = data.items.filter(p => p.metadata.name.includes("devworkspace-controller")); + + for (const pod of operatorPods) { + const container = pod.containers[0]; // assuming single container + const name = pod.metadata.name; + + const cpu = parseCpuToMillicores(container.usage.cpu); + const memory = parseMemoryToBytes(container.usage.memory); + + operatorCpu.add(cpu); + operatorMemory.add(memory / 1024 / 1024); + + const cpuOk = cpu <= maxCpuMillicores; + const memOk = memory <= maxMemoryBytes; + + if (!cpuOk) { + operatorCpuViolations.add(1); + } + if (!memOk) { + operatorMemViolations.add(1); + } + + check(null, { + [`[${name}] CPU < ${maxCpuMillicores}m`]: () => cpuOk, + [`[${name}] Memory < ${Math.round(maxMemoryBytes / 1024 / 1024)}Mi`]: () => memOk, + }); + } +} + +function createNewNamespace(namespaceName) { + const url = `${apiServer}/api/v1/namespaces`; + + const namespaceObj = { + apiVersion: 'v1', + kind: 'Namespace', + metadata: { + name: namespaceName, + labels: { + [labelKey]: labelType + } + } + } + const res = http.post(url, JSON.stringify(namespaceObj), {headers}); + + if (res.status !== 201 && res.status !== 409) { + throw new Error(`Failed to create Namespace: ${res.status} - ${namespaceName}`); + } +} + +function createNewAutomountConfigMap() { + const url = `${apiServer}/api/v1/namespaces/${loadTestNamespace}/configmaps`; + + const configMapManifest = { + apiVersion: 'v1', kind: 'ConfigMap', metadata: { + name: autoMountConfigMapName, namespace: loadTestNamespace, labels: { + 'controller.devfile.io/mount-to-devworkspace': 'true', 'controller.devfile.io/watch-configmap': 'true', + }, annotations: { + 'controller.devfile.io/mount-path': '/etc/config/dwo-load-test-configmap', + 'controller.devfile.io/mount-access-mode': '0644', + 'controller.devfile.io/mount-as': 'file', + }, + }, data: { + 'test.key': 'test-value', + }, + }; + + const res = http.post(url, JSON.stringify(configMapManifest), {headers}); + + if (res.status !== 201 && res.status !== 409) { + throw new Error(`Failed to create automount ConfigMap: ${res.status} - ${res.body}`); + } + console.log("Created automount configMap : " + autoMountConfigMapName); +} + +function createNewAutomountSecret() { + const manifest = { + apiVersion: 'v1', kind: 'Secret', metadata: { + name: autoMountSecretName, namespace: loadTestNamespace, labels: { + 'controller.devfile.io/mount-to-devworkspace': 'true', 'controller.devfile.io/watch-secret': 'true', + }, annotations: { + 'controller.devfile.io/mount-path': `/etc/secret/dwo-load-test-secret`, + 'controller.devfile.io/mount-as': 'file', + }, + }, type: 'Opaque', data: { + 'secret.key': __ENV.SECRET_VALUE_BASE64 || 'dGVzdA==', // base64-encoded 'test' + }, + }; + + const res = http.post(`${apiServer}/api/v1/namespaces/${loadTestNamespace}/secrets`, JSON.stringify(manifest), {headers}); + if (res.status !== 201 && res.status !== 409) { + throw new Error(`Failed to create automount Secret: ${res.status} - ${res.body}`); + } +} + +function deleteConfigMap() { + const url = `${apiServer}/api/v1/namespaces/${loadTestNamespace}/configmaps/${autoMountConfigMapName}`; + const res = http.del(url, null, { headers }); + if (res.status !== 200 && res.status !== 404) { + console.warn(`[CLEANUP] Failed to delete ConfigMap ${autoMountConfigMapName}: ${res.status}`); + } +} + +function deleteSecret() { + const url = `${apiServer}/api/v1/namespaces/${loadTestNamespace}/secrets/${autoMountSecretName}`; + const res = http.del(url, null, {headers}); + if (res.status !== 200 && res.status !== 404) { + console.warn(`[CLEANUP] Failed to delete Secret ${autoMountSecretName}: ${res.status}`); + } +} + +function deleteNamespace(name) { + const delRes = http.del(`${apiServer}/api/v1/namespaces/${name}`, null, {headers}); + if (delRes.status !== 200 && delRes.status !== 404) { + console.warn(`[CLEANUP] Failed to delete Namespace ${name}: ${delRes.status}`); + } +} + +function deleteAllDevWorkspacesInCurrentNamespace() { + const deleteByLabelSelectorUrl = `${apiServer}/apis/workspace.devfile.io/v1alpha2/namespaces/${loadTestNamespace}/devworkspaces?labelSelector=${labelKey}%3D${labelType}`; + console.log(`[CLEANUP] Deleting all DevWorkspaces in ${loadTestNamespace} containing label ${labelKey}=${labelType}`); + + const res = http.del(deleteByLabelSelectorUrl, null, {headers}); + if (res.status !== 200) { + console.error(`[CLEANUP] Failed to delete DevWorkspaces: ${res.status}`); + } +} + +function deleteAllSeparateNamespaces() { + const getNamespacesByLabel = `${apiServer}/api/v1/namespaces?labelSelector=${labelKey}%3D${labelType}`; + console.log(`[CLEANUP] Deleting all Namespaces containing label ${labelKey}=${labelType}`); + + const res = http.get(getNamespacesByLabel, {headers}); + if (res.status !== 200) { + console.error(`[CLEANUP] Failed to list DevWorkspaces: ${res.status}`); + return; + } + + const body = JSON.parse(res.body); + if (!body.items || !Array.isArray(body.items)) return; + + for (const item of body.items) { + deleteNamespace(item.metadata.name); + } +} + +function createOpinionatedDevWorkspace() { + return { + apiVersion: "workspace.devfile.io/v1alpha2", kind: "DevWorkspace", metadata: { + name: "minimal-dw", + namespace: loadTestNamespace, + labels: { + [labelKey]: labelType + } + }, spec: { + started: true, template: { + attributes: { + "controller.devfile.io/storage-type": "ephemeral", + }, components: [{ + name: "dev", container: { + image: "registry.access.redhat.com/ubi9/ubi-micro:9.6-1752751762", + command: ["sleep", "3600"], + imagePullPolicy: "IfNotPresent", + memoryLimit: "64Mi", + memoryRequest: "32Mi", + cpuLimit: "200m", + cpuRequest: "100m" + }, + },], + }, + }, + }; +} + +function parseJSONResponseToDevWorkspace(response) { + let devWorkspace; + try { + devWorkspace = response.json(); + } catch (e) { + throw new Error(`[DW CREATE] Failed to parse JSON : ${response.body}: ${e.message}`); + } + return devWorkspace; +} + +function downloadAndParseExternalWorkspace(externalDevWorkspaceLink) { + let manifest; + if (externalDevWorkspaceLink) { + const res = http.get(externalDevWorkspaceLink); + + if (res.status !== 200) { + throw new Error(`[DW CREATE] Failed to fetch JSON content from ${externalDevWorkspaceLink}, got ${res.status}`); + } + manifest = parseJSONResponseToDevWorkspace(res); + } + + return manifest; +} + +function generateDevWorkspaceToCreate(vuId, iteration, namespace) { + const name = `dw-test-${vuId}-${iteration}`; + let devWorkspace = {}; + if (externalDevWorkspaceLink.length > 0) { + devWorkspace = downloadAndParseExternalWorkspace(externalDevWorkspaceLink); + } else { + devWorkspace = createOpinionatedDevWorkspace(); + } + devWorkspace.metadata.name = name; + devWorkspace.metadata.namespace = namespace; + devWorkspace.metadata.labels = { + [labelKey]: labelType + } + return devWorkspace; +} + +function generateLoadTestStages(max) { + const stageDefinitions = [ + { percent: 0.25, target: Math.floor(max * 0.25) }, + { percent: 0.25, target: Math.floor(max * 0.5) }, + { percent: 0.2, target: Math.floor(max * 0.75) }, + { percent: 0.15, target: max }, + { percent: 0.10, target: Math.floor(max * 0.5) }, + { percent: 0.05, target: 0 }, + ]; + + return stageDefinitions.map(({ percent, target }) => ({ + duration: `${Math.round(loadTestDurationInMinutes * percent)}m`, + target, + })); +} + +function parseMemoryToBytes(memStr) { + if (memStr.endsWith("Ki")) return parseInt(memStr) * 1024; + if (memStr.endsWith("Mi")) return parseInt(memStr) * 1024 * 1024; + if (memStr.endsWith("Gi")) return parseInt(memStr) * 1024 * 1024 * 1024; + if (memStr.endsWith("n")) return parseInt(memStr) / 1e9; + if (memStr.endsWith("u")) return parseInt(memStr) / 1e6; + if (memStr.endsWith("m")) return parseInt(memStr) / 1e3; + return parseInt(memStr); // bytes +} + +function parseCpuToMillicores(cpuStr) { + if (cpuStr.endsWith("n")) return Math.round(parseInt(cpuStr) / 1e6); + if (cpuStr.endsWith("u")) return Math.round(parseInt(cpuStr) / 1e3); + if (cpuStr.endsWith("m")) return parseInt(cpuStr); + return Math.round(parseFloat(cpuStr) * 1000); +} \ No newline at end of file diff --git a/test/load/runk6.sh b/test/load/runk6.sh new file mode 100644 index 000000000..2bb16f88b --- /dev/null +++ b/test/load/runk6.sh @@ -0,0 +1,341 @@ +#!/bin/bash + +#!/bin/bash + +MODE="binary" # or 'operator' +LOAD_TEST_NAMESPACE="loadtest-devworkspaces" +DWO_NAMESPACE="openshift-operators" +SA_NAME="k6-devworkspace-tester" +CLUSTERROLE_NAME="k6-devworkspace-role" +ROLEBINDING_NAME="k6-devworkspace-binding" +CONFIGMAP_NAME="k6-test-script" +K6_CR_NAME="k6-test-run" +K6_SCRIPT="test/load/devworkspace_load_test.js" +K6_OPERATOR_VERSION="v0.0.22" +DEVWORKSPACE_LINK="https://gist.githubusercontent.com/rohanKanojia/ecf625afaf3fe817ac7d1db78bd967fc/raw/8c30c0370444040105ca45cd4ac0f7062a644bb7/dw-minimal.json" +MAX_VUS="100" +DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="1200" +SEPARATE_NAMESPACES="false" +CREATE_AUTOMOUNT_RESOURCES="false" +LOGS_DIR="logs" +TEST_DURATION_IN_MINUTES="25" +MIN_KUBECTL_VERSION="1.24.0" +MIN_CURL_VERSION="7.0.0" +MIN_K6_VERSION="1.1.0" + +# ----------- Main Execution Flow ----------- +main() { + parse_arguments "$@" + check_prerequisites + create_namespace + create_rbac + start_background_watchers + + if [[ "$MODE" == "operator" ]]; then + install_k6_operator + create_k6_configmap + delete_existing_testruns + create_k6_test_run + wait_for_test_completion + fetch_test_logs + elif [[ "$MODE" == "binary" ]]; then + generate_token_and_api_url + run_k6_binary_test + else + echo "โŒ Invalid mode: $MODE" + exit 1 + fi + stop_background_watchers + delete_namespace +} + +# ----------- Helper Functions ----------- +print_help() { + cat < Mode to run the script (default: operator) + --max-vus Number of virtual users for k6 (default: 100) + --separate-namespaces Use separate namespaces for workspaces (default: false) + --devworkspace-ready-timeout-seconds Timeout in seconds for workspace to become ready (default: 1200) + --devworkspace-link DevWorkspace link (default: empty, opinionated DevWorkspace is created) + --create-automount-resources Whether to create automount resources (default: false) + --dwo-namespace DevWorkspace Operator namespace (default: loadtest-devworkspaces) + --logs-dir Directory name where DevWorkspace and event logs would be dumped + --test-duration-minutes Duration in minutes for which to run load tests (default: 25 minutes) + -h, --help Show this help message +EOF +} + +parse_arguments() { + while [[ $# -gt 0 ]]; do + case "$1" in + --mode) + MODE="$2"; shift 2;; + --max-vus) + MAX_VUS="$2"; shift 2;; + --separate-namespaces) + SEPARATE_NAMESPACES="$2"; shift 2;; + --devworkspace-ready-timeout-seconds) + DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="$2"; shift 2;; + --devworkspace-link) + DEVWORKSPACE_LINK="$2"; shift 2;; + --create-automount-resources) + CREATE_AUTOMOUNT_RESOURCES="$2"; shift 2;; + --dwo-namespace) + LOAD_TEST_NAMESPACE="$2"; shift 2;; + --logs-dir) + LOGS_DIR="$2"; shift 2;; + --test-duration-minutes) + TEST_DURATION_IN_MINUTES="$2"; shift 2;; + -h|--help) + print_help; exit 0;; + *) + echo "โŒ Unknown option: $1" + print_help; exit 1;; + esac + done +} + +create_namespace() { + echo "๐Ÿ”ง Creating Namespace: $LOAD_TEST_NAMESPACE" + cat <> "${LOGS_DIR}/${TIMESTAMP}_events.log" 2>&1 & + PID_EVENTS_WATCH=$! + + kubectl get dw --watch --all-namespaces \ + >> "${LOGS_DIR}/${TIMESTAMP}_dw_watch.log" 2>&1 & + PID_DW_WATCH=$! +} + +stop_background_watchers() { + echo "๐Ÿ›‘ Stopping background watchers..." + kill "$PID_EVENTS_WATCH" "$PID_DW_WATCH" 2>/dev/null || true +} + +install_k6_operator() { + echo "๐Ÿ“ฆ Installing k6 operator..." + curl -L "https://raw.githubusercontent.com/grafana/k6-operator/refs/tags/${K6_OPERATOR_VERSION}/bundle.yaml" | kubectl apply -f - + echo "โณ Waiting until k6 operator deployment is ready..." + kubectl rollout status deployment/k6-operator-controller-manager -n k6-operator-system --timeout=300s +} + +create_k6_configmap() { + echo "๐Ÿงฉ Creating ConfigMap from script file: $K6_SCRIPT" + kubectl create configmap "$CONFIGMAP_NAME" \ + --from-file=script.js="$K6_SCRIPT" \ + --namespace "$LOAD_TEST_NAMESPACE" \ + --dry-run=client -o yaml | kubectl apply -f - +} + +delete_existing_testruns() { + echo "๐Ÿงน Deleting any existing K6 TestRun resources in namespace: $LOAD_TEST_NAMESPACE" + kubectl delete testrun --all -n "$LOAD_TEST_NAMESPACE" || true +} + +create_k6_test_run() { + echo "๐Ÿš€ Creating K6 TestRun custom resource..." + cat </dev/null) + + if [[ "$stage" == "finished" ]]; then + echo "TestRun $K6_CR_NAME is finished." + break + fi + + if (( SECONDS >= end )); then + echo "Timeout waiting for TestRun $K6_CR_NAME to finish." + exit 1 + fi + + sleep "$INTERVAL" + done +} + +fetch_test_logs() { + K6_TEST_POD=$(kubectl get pod -l k6_cr=$K6_CR_NAME,runner=true -n "${LOAD_TEST_NAMESPACE}" -o jsonpath='{.items[0].metadata.name}') + echo "๐Ÿ“œ Fetching logs from completed K6 test pod: $K6_TEST_POD" + kubectl logs "$K6_TEST_POD" -n "$LOAD_TEST_NAMESPACE" +} + +check_prerequisites() { + echo "๐Ÿ” Checking prerequisites..." + + check_command "kubectl" "$MIN_KUBECTL_VERSION" + check_command "curl" "$MIN_CURL_VERSION" + + if [[ "$MODE" == "binary" ]]; then + check_command "k6" "$MIN_K6_VERSION" + fi +} + +check_command() { + local cmd="$1" + local min_version="$2" + local version + + if ! command -v "$cmd" &>/dev/null; then + echo "โŒ Required command '$cmd' not found in PATH." + exit 1 + fi + + case "$cmd" in + kubectl) + version=$(kubectl version --client -o json | jq -r '.clientVersion.gitVersion' | sed 's/^v//') + ;; + curl) + version=$($cmd --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -n1) + ;; + k6) + version=$($cmd version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+') + ;; + *) + version="0.0.0" + ;; + esac + + if ! version_gte "$version" "$min_version"; then + echo "โŒ $cmd version $version is less than required $min_version" + exit 1 + else + echo "โœ… $cmd version $version (>= $min_version)" + fi +} + +version_gte() { + [ "$(printf '%s\n' "$2" "$1" | sort -V | head -n1)" = "$2" ] +} + +run_k6_binary_test() { + echo "๐Ÿš€ Running k6 load test..." + IN_CLUSTER='false' \ + KUBE_TOKEN="${KUBE_TOKEN}" \ + KUBE_API="${KUBE_API}" \ + DWO_NAMESPACE="${DWO_NAMESPACE}" \ + CREATE_AUTOMOUNT_RESOURCES="${CREATE_AUTOMOUNT_RESOURCES}" \ + SEPARATE_NAMESPACES="${SEPARATE_NAMESPACES}" \ + LOAD_TEST_NAMESPACE="${LOAD_TEST_NAMESPACE}" \ + DEVWORKSPACE_LINK="${DEVWORKSPACE_LINK}" \ + MAX_VUS="${MAX_VUS}" \ + TEST_DURATION_IN_MINUTES="${TEST_DURATION_IN_MINUTES}" \ + DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="${DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS}" \ + k6 run "${K6_SCRIPT}" + exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "โš ๏ธ k6 load test failed with exit code $exit_code. Proceeding to cleanup." + fi + return 0 +} + +main "$@" From 5f351dbddc4d852f27825cb0432bae536ad3209c Mon Sep 17 00:00:00 2001 From: Rohan Kumar Date: Tue, 26 Aug 2025 02:22:05 +0530 Subject: [PATCH 2/2] add options for --max-devworkspaces and --delete-devworkspace-after-ready Signed-off-by: Rohan Kumar --- .ci/openshift_e2e.sh | 5 ++-- test/load/devworkspace_load_test.js | 29 +++++++++++++++++- test/load/runk6.sh | 46 ++++++++++++++++++++++++++++- 3 files changed, 76 insertions(+), 4 deletions(-) diff --git a/.ci/openshift_e2e.sh b/.ci/openshift_e2e.sh index c55418f42..b70fc4d93 100755 --- a/.ci/openshift_e2e.sh +++ b/.ci/openshift_e2e.sh @@ -73,5 +73,6 @@ export CLEAN_UP_AFTER_SUITE="false" make test_e2e bumpLogs -make test_load ARGS="--mode operator --max-vus 250 --separate-namespaces false --test-duration-minutes 25 --dwo-namespace devworkspace-controller --logs-dir ${ARTIFACT_DIR}/load-testing-logs" -make uninstall +make test_load ARGS="--mode operator --max-vus 500 --max-devworkspaces 1500 --delete-devworkspace-after-ready false --separate-namespaces false --test-duration-minutes 25 --dwo-namespace devworkspace-controller --logs-dir ${ARTIFACT_DIR}/load-testing-logs" + +#make uninstall diff --git a/test/load/devworkspace_load_test.js b/test/load/devworkspace_load_test.js index 45c54ab0f..c5aa21056 100644 --- a/test/load/devworkspace_load_test.js +++ b/test/load/devworkspace_load_test.js @@ -23,10 +23,12 @@ const inCluster = __ENV.IN_CLUSTER === 'true'; const apiServer = inCluster ? `https://kubernetes.default.svc` : __ENV.KUBE_API; const token = inCluster ? open('/var/run/secrets/kubernetes.io/serviceaccount/token') : __ENV.KUBE_TOKEN; const useSeparateNamespaces = __ENV.SEPARATE_NAMESPACES === "true"; +const deleteDevWorkspaceAfterReady = __ENV.DELETE_DEVWORKSPACE_AFTER_READY === "true"; const operatorNamespace = __ENV.DWO_NAMESPACE || 'openshift-operators'; const externalDevWorkspaceLink = __ENV.DEVWORKSPACE_LINK || ''; const shouldCreateAutomountResources = (__ENV.CREATE_AUTOMOUNT_RESOURCES || 'false') === 'true'; const maxVUs = Number(__ENV.MAX_VUS || 50); +const maxDevWorkspaces = Number(__ENV.MAX_DEVWORKSPACES || -1); const devWorkspaceReadyTimeout = Number(__ENV.DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS || 600); const autoMountConfigMapName = 'dwo-load-test-automount-configmap'; const autoMountSecretName = 'dwo-load-test-automount-secret'; @@ -87,6 +89,12 @@ export function setup() { } export default function () { + if (maxDevWorkspaces > 0) { + const totalDevWorkspaces = getDevWorkspacesFromApiServer().length; + if (totalDevWorkspaces > maxDevWorkspaces) { + return; + } + } const vuId = __VU; const iteration = __ITER; const crName = `dw-test-${vuId}-${iteration}`; @@ -104,7 +112,9 @@ export default function () { const devWorkspaceCreated = createNewDevWorkspace(namespace, vuId, iteration); if (devWorkspaceCreated) { waitUntilDevWorkspaceIsReady(vuId, crName, namespace); - deleteDevWorkspace(crName, namespace); + if (deleteDevWorkspaceAfterReady) { + deleteDevWorkspace(crName, namespace); + } } } catch (error) { console.error(`Load test for ${vuId}-${iteration} failed:`, error.message); @@ -434,6 +444,23 @@ function downloadAndParseExternalWorkspace(externalDevWorkspaceLink) { return manifest; } +function getDevWorkspacesFromApiServer() { + const basePath = useSeparateNamespaces + ? `${apiServer}/apis/workspace.devfile.io/v1alpha2/devworkspaces` + : `${apiServer}/apis/workspace.devfile.io/v1alpha2/namespaces/${loadTestNamespace}/devworkspaces`; + + const url = `${basePath}?labelSelector=${labelKey}%3D${labelType}`; + const res = http.get(url, { headers }); + + if (res.status !== 200) { + console.error(`Failed to fetch DevWorkspaces: ${res.status} ${res.body}`); + return []; + } + + const body = JSON.parse(res.body); + return body.items.map((dw) => dw.metadata.name); +} + function generateDevWorkspaceToCreate(vuId, iteration, namespace) { const name = `dw-test-${vuId}-${iteration}`; let devWorkspace = {}; diff --git a/test/load/runk6.sh b/test/load/runk6.sh index 2bb16f88b..9b9a3629a 100644 --- a/test/load/runk6.sh +++ b/test/load/runk6.sh @@ -16,6 +16,8 @@ DEVWORKSPACE_LINK="https://gist.githubusercontent.com/rohanKanojia/ecf625afaf3fe MAX_VUS="100" DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="1200" SEPARATE_NAMESPACES="false" +DELETE_DEVWORKSPACE_AFTER_READY="true" +MAX_DEVWORKSPACES="-1" CREATE_AUTOMOUNT_RESOURCES="false" LOGS_DIR="logs" TEST_DURATION_IN_MINUTES="25" @@ -57,7 +59,9 @@ Usage: $0 [options] Options: --mode Mode to run the script (default: operator) --max-vus Number of virtual users for k6 (default: 100) + --max-devworkspaces Maximum number of DevWorkspaces to create (by default, it's not specified) --separate-namespaces Use separate namespaces for workspaces (default: false) + --delete-devworkspace-after-ready Delete DevWorkspace once it becomes Ready (default: true) --devworkspace-ready-timeout-seconds Timeout in seconds for workspace to become ready (default: 1200) --devworkspace-link DevWorkspace link (default: empty, opinionated DevWorkspace is created) --create-automount-resources Whether to create automount resources (default: false) @@ -77,6 +81,10 @@ parse_arguments() { MAX_VUS="$2"; shift 2;; --separate-namespaces) SEPARATE_NAMESPACES="$2"; shift 2;; + --max-devworkspaces) + MAX_DEVWORKSPACES="$2"; shift 2;; + --delete-devworkspace-after-ready) + DELETE_DEVWORKSPACE_AFTER_READY="$2"; shift 2;; --devworkspace-ready-timeout-seconds) DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="$2"; shift 2;; --devworkspace-link) @@ -173,11 +181,40 @@ start_background_watchers() { kubectl get dw --watch --all-namespaces \ >> "${LOGS_DIR}/${TIMESTAMP}_dw_watch.log" 2>&1 & PID_DW_WATCH=$! + + log_failed_devworkspaces & + PID_FAILED_DW_POLL=$! +} + +log_failed_devworkspaces() { + echo "๐Ÿ“„ Starting periodic failed DevWorkspaces report (every 10s)..." + + POLL_INTERVAL=10 # in seconds + ITERATIONS=$(((TEST_DURATION_IN_MINUTES * 60) / POLL_INTERVAL)) + + for ((i = 0; i < ITERATIONS; i++)); do + OUTPUT=$(kubectl get devworkspaces --all-namespaces -o json | jq -r ' + .items[] + | select(.status.phase == "Failed") + | [ + .metadata.namespace, + .metadata.name, + .status.phase, + (.status.message // "No message") + ] + | @csv') + + if [ -n "$OUTPUT" ]; then + echo "$OUTPUT" > "${LOGS_DIR}/dw_failure_report.csv" + fi + + sleep "$POLL_INTERVAL" + done } stop_background_watchers() { echo "๐Ÿ›‘ Stopping background watchers..." - kill "$PID_EVENTS_WATCH" "$PID_DW_WATCH" 2>/dev/null || true + kill "$PID_EVENTS_WATCH" "$PID_DW_WATCH" "$PID_FAILED_DW_POLL" 2>/dev/null || true } install_k6_operator() { @@ -235,6 +272,10 @@ spec: value: '${TEST_DURATION_IN_MINUTES}' - name: DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS value: '${DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS}' + - name: DELETE_DEVWORKSPACE_AFTER_READY + value: '${DELETE_DEVWORKSPACE_AFTER_READY}' + - name: MAX_DEVWORKSPACES + value: '${MAX_DEVWORKSPACES}' EOF } @@ -330,6 +371,8 @@ run_k6_binary_test() { MAX_VUS="${MAX_VUS}" \ TEST_DURATION_IN_MINUTES="${TEST_DURATION_IN_MINUTES}" \ DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="${DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS}" \ + DELETE_DEVWORKSPACE_AFTER_READY="${DELETE_DEVWORKSPACE_AFTER_READY}" \ + MAX_DEVWORKSPACES="${MAX_DEVWORKSPACES}" \ k6 run "${K6_SCRIPT}" exit_code=$? if [ $exit_code -ne 0 ]; then @@ -338,4 +381,5 @@ run_k6_binary_test() { return 0 } +trap stop_background_watchers EXIT main "$@"