From 916a456317272dfd8654285bbf06c7f20c1ff36f Mon Sep 17 00:00:00 2001 From: ytimocin Date: Tue, 19 Nov 2024 09:41:19 -0800 Subject: [PATCH] Updating the Running Cloud Functional Tests and adding the Running Non-cloud Functional Tests docs Signed-off-by: ytimocin --- ...s.md => running-cloud-functional-tests.md} | 32 ++--- .../running-noncloud-functional-tests.md | 119 ++++++++++++++++++ ...t.sh => execute_cloud_functional_tests.sh} | 8 +- test/execute_noncloud_functional_tests.sh | 20 +++ 4 files changed, 160 insertions(+), 19 deletions(-) rename docs/contributing/contributing-code/contributing-code-tests/{running-functional-tests.md => running-cloud-functional-tests.md} (75%) create mode 100644 docs/contributing/contributing-code/contributing-code-tests/running-noncloud-functional-tests.md rename test/{executeFunctionalTest.sh => execute_cloud_functional_tests.sh} (87%) create mode 100755 test/execute_noncloud_functional_tests.sh diff --git a/docs/contributing/contributing-code/contributing-code-tests/running-functional-tests.md b/docs/contributing/contributing-code/contributing-code-tests/running-cloud-functional-tests.md similarity index 75% rename from docs/contributing/contributing-code/contributing-code-tests/running-functional-tests.md rename to docs/contributing/contributing-code/contributing-code-tests/running-cloud-functional-tests.md index b730e90062..f042b841cd 100644 --- a/docs/contributing/contributing-code/contributing-code-tests/running-functional-tests.md +++ b/docs/contributing/contributing-code/contributing-code-tests/running-cloud-functional-tests.md @@ -1,6 +1,6 @@ -# Running Radius functional tests +# Running Radius Cloud Functional Tests -You can find the functional tests under `./test/functional`. A functional test (in our terminology) is a test that interacts with real hosting environments (Kubernetes), deploys real applications and resources, and covers realistic or simulated user scenarios. +You can find the functional tests under `./test/functional-portable`. A functional test (in our terminology) is a test that interacts with real hosting environments (Kubernetes), deploys real applications and resources, and covers realistic or simulated user scenarios. These tests verify whether: @@ -9,7 +9,7 @@ These tests verify whether: ## Running via GitHub workflow -These tests automatically run for every PR in the `functional-tests.yml` github workflow. +These tests automatically run for every PR in the `functional-test-cloud.yml` and `functional-test-noncloud.yml` github workflows. The `functional-test-cloud.yml` workflow requires an approval from one of the maintainers or approvers of Radius. We do not run these tests for commits to `main` or tags since they might block the build if they fail. @@ -23,7 +23,7 @@ For each PR we run the following set of steps: - Run tests - Delete any cloud resources that were created -We have a separate scheduled job (`purge-test-resources.yaml`) that will delete cloud resources that are left behind. This can happen when the test run is cancelled or times out. +We have two separate scheduled jobs (`purge-aws-test-resources.yaml` and `purge-azure-test-resources.yaml`) that will delete cloud resources that are left behind. This can happen when the test run is cancelled or times out. ## Configuration @@ -49,26 +49,28 @@ As much as possible, the tests use product functionality such as the Radius CLI 7. Publish Terraform test recipes by running `make publish-test-terraform-recipes` 8. Change the visibility of the published packages to 'public' -> ⚠️ The tests assume the Kubernetes namespace in use is `default`. If your environment is set up differently you will see -> test failures. - +> ⚠️ The tests assume the Kubernetes namespace in use is `default`. If your environment is set up differently you will see test failures. > ⚠️ If you set environment variables for functional tests you may need to restart VS Code or other editors for them to take effect. ### Run 1. Run: - ```sh - .{workspace}/radius/test/executeFunctionalTest.sh - ``` +```sh + .{workspace}/radius/test/execute_cloud_functional_tests.sh +``` + +When you're running locally with this configuration, the tests will use your locally selected Radius Environment and your local copy of `rad`. The `execute_cloud_functional_tests.sh` script creates the Azure resources and exports the values to be used in the functional test and runs: + +```sh + make test-functional-all-cloud +``` -When you're running locally with this configuration, the tests will use your locally selected Radius Environment and your local copy of `rad`. The executeFunctionalTest.sh scripts creates the azure resources and exports the values to be used in the functional test and runs: +Which in turn runs these tests: ```sh - make test-functional-corerp - make test-functional-msgrp - make test-functional-daprrp - make test-functional-datastoresrp + make test-functional-ucp-cloud + make test-functional-corerp-cloud ``` You can also run/debug individual tests from VSCode. diff --git a/docs/contributing/contributing-code/contributing-code-tests/running-noncloud-functional-tests.md b/docs/contributing/contributing-code/contributing-code-tests/running-noncloud-functional-tests.md new file mode 100644 index 0000000000..8fd7060f3b --- /dev/null +++ b/docs/contributing/contributing-code/contributing-code-tests/running-noncloud-functional-tests.md @@ -0,0 +1,119 @@ +# Running Radius Non-cloud Functional Tests + +You can find the functional tests under `./test/functional-portable`. A functional test (in our terminology) is a test that interacts with real hosting environments (Kubernetes), deploys real applications and resources, and covers realistic or simulated user scenarios. + +These tests verify whether: + +- That Radius Environment can be created successfully. +- That Bicep templates of sample applications can be deployed to the Radius Environment. + +## Running via GitHub workflow + +These tests automatically run for every PR in the `functional-test-cloud.yml` and `functional-test-noncloud.yml` github workflows. The `functional-test-cloud.yml` workflow requires an approval from one of the maintainers or approvers of Radius. + +We do not run these tests for commits to `main` or tags since they might block the build if they fail. + +### How this works + +For each PR we run the following set of steps: + +- Build Radius and publish test assets +- For each group of tests: + - Create a Kubernetes cluster and install the build + - Run tests + - Delete any cloud resources that were created + +We have two separate scheduled jobs (`purge-aws-test-resources.yaml` and `purge-azure-test-resources.yaml`) that will delete cloud resources that are left behind. This can happen when the test run is cancelled or times out. + +## Configuration + +These tests use your local Kubernetes credentials, and Radius Environment for testing. In a GitHub workflow, our automation makes the CI environment resemble a real user scenario. This way we test a setup that is close to what users will have in the real world. + +As much as possible, the tests use product functionality such as the Radius CLI configuration and local KubeConfig to detect settings. + +## Running the tests locally + +### Prerequisites + +1. Place `rad` on your path +1. Make sure `rad-bicep` is downloaded (`rad bicep download`) +1. Make sure your [local dev environment is setup](../contributing-code-control-plane/running-controlplane-locally.md) +1. Log into your Github account and [Generate PAT](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) +1. Log-in to the container registry of your Github organization. + + `export CR_PAT=` + + `echo $CR_PAT | docker login ghcr.io -u --password-stdin` + +1. Publish Bicep test recipes by running `BICEP_RECIPE_REGISTRY= make publish-test-bicep-recipes` +1. Publish Terraform test recipes by running `make publish-test-terraform-recipes` +1. Change the visibility of the published packages to 'public' + +> ⚠️ The tests assume the Kubernetes namespace in use is `default`. If your environment is set up differently you will see test failures. +> ⚠️ If you set environment variables for functional tests you may need to restart VS Code or other editors for them to take effect. + +### Run + +1. Run: + +```sh + .{workspace}/radius/test/execute_noncloud_functional_tests.sh +``` + +When you're running locally with this configuration, the tests will use your locally selected Radius Environment and your local copy of `rad`. The `execute_noncloud_functional_tests.sh` script runs: + +```sh + make test-functional-all-noncloud +``` + +Which in turn runs these tests: + +```sh + make test-functional-ucp-noncloud + make test-functional-kubernetes-noncloud + make test-functional-corerp-noncloud + make test-functional-cli-noncloud + make test-functional-msgrp-noncloud + make test-functional-daprrp-noncloud + make test-functional-datastoresrp-noncloud + make test-functional-samples-noncloud +``` + +You can also run/debug individual tests from VSCode. + +### Tips + +> 💡 If you make changes to recipes, make sure to re-run the _publish test recipe_ step from prerequisites. +> 💡 Make sure the packages published to your organization have their visibility set to "public" +> 💡 If you make changes to the `rad` CLI make sure to copy it to your path. + +### Seeing log output + +Some of these tests take a few minutes to run since they interact with cloud resources. You should configure VSCode to output verbose output so you can see the progress. + +Open your VSCode `settings.json` with the command `Preferences: Open Settings (JSON)` and configure the following options: + +```json +{ + ... + "go.testTimeout": "60m", + "go.testFlags": [ + "-v" + ], +} +``` + +### Using Codelens (VSCode) + +VSCode will start a child process when you execute a `'run test'/'debug test'` codelens action (see image for example). If you are using this to run functional tests, this process may not resolve `rad` correctly. You can specify environment variables for codelens using `settings.json`: + +```json +{ + ... + "go.testEnvVars": { + "RAD_PATH": "${workspaceFolder}/dist/linux_amd64/release" + }, +} +``` + +![Screenshot of VS Code Codelens UI](./vscode_debug_test.png) diff --git a/test/executeFunctionalTest.sh b/test/execute_cloud_functional_tests.sh similarity index 87% rename from test/executeFunctionalTest.sh rename to test/execute_cloud_functional_tests.sh index e2e1a5880a..f5b2cb0289 100755 --- a/test/executeFunctionalTest.sh +++ b/test/execute_cloud_functional_tests.sh @@ -20,6 +20,8 @@ usage() { exit 1 } +echo -e "Running Radius functional tests with cloud resources\n" + if [ $# -lt 1 ]; then usage fi @@ -31,7 +33,5 @@ resp=$(az deployment group create --resource-group $resourcegroup --template-fil cat resp export AZURE_COSMOS_MONGODB_ACCOUNT_ID=$(jq -r '.properties.outputs.cosmosMongoAccountID.value' <<<"${resp}") -make test-functional-corerp -make test-functional-msgrp -make test-functional-daprrp -make test-functional-datastoresrp +# We probably need AWS_ACCOUNT_ID and AWS_REGION as well +make test-functional-all-cloud diff --git a/test/execute_noncloud_functional_tests.sh b/test/execute_noncloud_functional_tests.sh new file mode 100755 index 0000000000..e201a9ba3c --- /dev/null +++ b/test/execute_noncloud_functional_tests.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# ------------------------------------------------------------ +# Copyright 2023 The Radius Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------ +echo -e "Running Radius functional tests with non-cloud resources\n" + +make test-functional-all-noncloud