Added CI to run ceph tests on ns aws using minio

Signed-off-by: Aayush Chouhan <[email protected]>

Author: aayushchouhan09

.github/workflows/ceph-ns-aws-s3-tests.yaml

@@ -0,0 +1,34 @@
+name: Ceph NS AWS S3 Tests
+on: [workflow_call]
+
+jobs:
+  ceph-s3-tests:
+    name: Ceph NS AWS S3 Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    permissions:
+      actions: read         # download-artifact
+      contents: read        # required for actions/checkout
+    steps:
+      - name: Checkout noobaa-core
+        uses: actions/checkout@v4
+        with:
+          repository: 'noobaa/noobaa-core'
+          path: 'noobaa-core'
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: noobaa-tester
+          path: /tmp
+
+      - name: Load image
+        run: docker load --input /tmp/noobaa-tester.tar
+
+      - name: Run Ceph NS AWS s3-tests
+        run: |
+          set -x
+          cd ./noobaa-core
+          mkdir -p logs/ceph-ns-aws-test-logs
+          chmod 777 logs/ceph-ns-aws-test-logs
+          make test-cephs3-ns-aws -o tester

.github/workflows/run-pr-tests.yaml

@@ -32,6 +32,10 @@ jobs:
     needs: build-noobaa-image
     uses: ./.github/workflows/ceph-s3-tests.yaml
 
+  ceph-ns-aws-s3-tests:
+    needs: build-noobaa-image
+    uses: ./.github/workflows/ceph-ns-aws-s3-tests.yaml
+
   ceph-nsfs-s3-tests:
     needs: build-noobaa-image
     uses: ./.github/workflows/ceph-nsfs-s3-tests.yaml

Makefile

@@ -111,6 +111,17 @@ MINT_MOCK_ACCESS_KEY="aaaaaaaaaaaaaEXAMPLE"
 MINT_MOCK_SECRET_KEY="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaEXAMPLE"
 MINT_NOOBAA_HTTP_ENDPOINT_PORT=6001
 
+####################
+# MINIO VARIABLES #
+###################
+
+MINIO_IMAGE?="quay.io/minio/minio"
+AWS_CLI_IMAGE?="amazon/aws-cli"
+MINIO_PORT?=9000
+MINIO_USER?="miniotest"
+MINIO_PASSWORD?="miniotest123"
+MINIO_TEST_BUCKET?="minio-test-bucket"
+
 ###############
 # BUILD LOCAL #
 ###############
@@ -352,6 +363,19 @@ test-cephs3: tester
 	@$(call remove_docker_network)
 .PHONY: test-cephs3
 
+test-cephs3-ns-aws: tester
+	@echo "\033[1;34mRunning ceph-s3 tests with namespace store pointing to s3 bucket in Minio\033[0m"
+	@$(call create_docker_network)
+	@$(call run_minio)
+	@$(call run_postgres)
+	@echo "\033[1;34mRunning tests\033[0m"
+	$(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" --env "USE_NAMESPACE_RESOURCE=true" --env "MINIO_ENDPOINT=http://coretest-minio-$(GIT_COMMIT)-$(NAME_POSTFIX):$(MINIO_PORT)" --env "MINIO_USER=$(MINIO_USER)" --env "MINIO_PASSWORD=$(MINIO_PASSWORD)" --env "MINIO_TEST_BUCKET=$(MINIO_TEST_BUCKET)" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/external_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh"
+	@$(call stop_noobaa)
+	@$(call stop_postgres)
+	@$(call stop_minio)
+	@$(call remove_docker_network)
+.PHONY: test-cephs3-ns-aws
+
 test-warp: tester
 	@echo "\033[1;34mRunning warp tests with Postgres.\033[0m"
 	@$(call create_docker_network)
@@ -581,3 +605,27 @@ define stop_blob_mock
 	fi
 	@echo "\033[1;32mBlob mock server stop done.\033[0m"
 endef
+
+#########
+# MINIO #
+#########
+
+define run_minio
+	@echo "\033[1;34mRunning Minio container\033[0m"
+	$(CONTAINER_ENGINE) run -d $(CPUSET) --network noobaa-net --name coretest-minio-$(GIT_COMMIT)-$(NAME_POSTFIX) -p $(MINIO_PORT):$(MINIO_PORT) --env "MINIO_ROOT_USER=$(MINIO_USER)" --env "MINIO_ROOT_PASSWORD=$(MINIO_PASSWORD)" $(MINIO_IMAGE) server /data --address ":$(MINIO_PORT)"
+	@echo "\033[1;34mWaiting for Minio to start..\033[0m"
+	sleep 20
+	@echo "\033[1;34mCreating test bucket in Minio..\033[0m"
+	$(CONTAINER_ENGINE) run --rm --network noobaa-net --env "AWS_ACCESS_KEY_ID=$(MINIO_USER)" --env "AWS_SECRET_ACCESS_KEY=$(MINIO_PASSWORD)" --env "AWS_DEFAULT_REGION=us-east-1" $(AWS_CLI_IMAGE) s3 mb s3://$(MINIO_TEST_BUCKET) --endpoint-url http://coretest-minio-$(GIT_COMMIT)-$(NAME_POSTFIX):$(MINIO_PORT)
+	@echo "\033[1;34mVerifying bucket creation..\033[0m"
+	$(CONTAINER_ENGINE) run --rm --network noobaa-net --env "AWS_ACCESS_KEY_ID=$(MINIO_USER)" --env "AWS_SECRET_ACCESS_KEY=$(MINIO_PASSWORD)" --env "AWS_DEFAULT_REGION=us-east-1" $(AWS_CLI_IMAGE) s3 ls --endpoint-url http://coretest-minio-$(GIT_COMMIT)-$(NAME_POSTFIX):$(MINIO_PORT)
+	@echo "\033[1;32mRun Minio done.\033[0m"
+endef
+
+define stop_minio
+	@echo "\033[1;34mStopping/removing Minio container\033[0m"
+	$(call disconnect_container_from_noobaa_network, coretest-minio-$(GIT_COMMIT)-$(NAME_POSTFIX))
+	$(CONTAINER_ENGINE) stop coretest-minio-$(GIT_COMMIT)-$(NAME_POSTFIX)
+	$(CONTAINER_ENGINE) rm coretest-minio-$(GIT_COMMIT)-$(NAME_POSTFIX)
+	@echo "\033[1;32mStop Minio done.\033[0m"
+endef

src/test/external_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh

@@ -34,7 +34,13 @@ export MGMT_ADDR=wss://${NOOBAA_MGMT_SERVICE_HOST:-localhost}:${NOOBAA_MGMT_SERV
 export BG_ADDR=wss://localhost:5445
 export HOSTED_AGENTS_ADDR=wss://localhost:5446
 
-export CEPH_TEST_LOGS_DIR=/logs/ceph-test-logs
+# set logs directory based on test type
+# later we will add more test types and logs directories on the basis of namespace resource type
+if [ "${USE_NAMESPACE_RESOURCE}" = "true" ]; then
+    export CEPH_TEST_LOGS_DIR=/logs/ceph-ns-aws-test-logs
+else
+    export CEPH_TEST_LOGS_DIR=/logs/ceph-test-logs
+fi
 
 export CONFIG_JS_OBJECT_SDK_BUCKET_CACHE_EXPIRY_MS=0 # Needed for disabling cache for ceph cors test and maybe some more
 export CONFIG_JS_allow_anonymous_access_in_test=true # Needed for allowing anon access for tests using ACL='public-read-write'