make file commands to test locally and workflow changes

Signed-off-by: chahatsagarmain <[email protected]>
kubeflow · Feb 4, 2025 · 3ec9555 · 3ec9555
1 parent 3059f7c
commit 3ec9555
Show file tree

Hide file tree

Showing 214 changed files with 13,753 additions and 8,862 deletions.
diff --git a/.github/actions/kfp-cluster/action.yml b/.github/actions/kfp-cluster/action.yml
@@ -19,8 +19,8 @@ runs:
 
     - name: Build images
       shell: bash
-      run: ./.github/resources/scripts/build-images.sh
+      run: make build-images
 
     - name: Deploy KFP
       shell: bash
-      run: ./.github/resources/scripts/deploy-kfp.sh
+      run: make deploy-kfp
diff --git a/.github/actions/kfp-tekton-cluster/action.yml b/.github/actions/kfp-tekton-cluster/action.yml
@@ -14,8 +14,8 @@ runs:
 
     - name: Build images
       shell: bash
-      run: ./.github/resources/scripts/build-images.sh
+      run: make build-images
 
     - name: Deploy KFP
       shell: bash
-      run: ./.github/resources/scripts/deploy-kfp-tekton.sh
+      run: make deploy-kfp-tekton
diff --git a/.github/resources/manifests/argo/kustomization.yaml b/.github/resources/manifests/argo/kustomization.yaml
@@ -5,13 +5,13 @@ resources:
 - ../../../../manifests/kustomize/env/platform-agnostic
 
 images:
-- name: gcr.io/ml-pipeline/api-server
+- name: ghcr.io/kubeflow/kfp-api-server
   newName: kind-registry:5000/apiserver
   newTag: latest
-- name: gcr.io/ml-pipeline/persistenceagent
+- name: ghcr.io/kubeflow/kfp-persistence-agent
   newName: kind-registry:5000/persistenceagent
   newTag: latest
-- name: gcr.io/ml-pipeline/scheduledworkflow
+- name: ghcr.io/kubeflow/kfp-scheduled-workflow-controller
   newName: kind-registry:5000/scheduledworkflow
   newTag: latest
 

diff --git a/.github/resources/manifests/tekton/kustomization.yaml b/.github/resources/manifests/tekton/kustomization.yaml
@@ -14,13 +14,13 @@ resources:
 # when application is deleted.
 
 images:
-- name: gcr.io/ml-pipeline/api-server
+- name: ghcr.io/kubeflow/kfp-api-server
   newName: kind-registry:5000/apiserver
   newTag: latest
-- name: gcr.io/ml-pipeline/persistenceagent
+- name: ghcr.io/kubeflow/kfp-persistence-agent
   newName: kind-registry:5000/persistenceagent
   newTag: latest
-- name: gcr.io/ml-pipeline/scheduledworkflow
+- name: ghcr.io/kubeflow/kfp-scheduled-workflow-controller
   newName: kind-registry:5000/scheduledworkflow
   newTag: latest
 - name: '*/aipipeline/tekton-exithandler-controller'

diff --git a/.github/resources/scripts/build-images.sh b/.github/resources/scripts/build-images.sh
@@ -25,35 +25,35 @@ EXIT_CODE=0
 
 docker system prune -a -f
 
-docker build -q -t "${REGISTRY}/apiserver:${TAG}" -f backend/Dockerfile . && docker push "${REGISTRY}/apiserver:${TAG}" || EXIT_CODE=$?
+docker build --progress=plain -t "${REGISTRY}/apiserver:${TAG}" -f backend/Dockerfile . && docker push "${REGISTRY}/apiserver:${TAG}" || EXIT_CODE=$?
 if [[ $EXIT_CODE -ne 0 ]]
 then
   echo "Failed to build apiserver image."
   exit $EXIT_CODE
 fi
 
-docker build -q -t "${REGISTRY}/persistenceagent:${TAG}" -f backend/Dockerfile.persistenceagent . && docker push "${REGISTRY}/persistenceagent:${TAG}" || EXIT_CODE=$?
+docker build --progress=plain -t "${REGISTRY}/persistenceagent:${TAG}" -f backend/Dockerfile.persistenceagent . && docker push "${REGISTRY}/persistenceagent:${TAG}" || EXIT_CODE=$?
 if [[ $EXIT_CODE -ne 0 ]]
 then
   echo "Failed to build persistenceagent image."
   exit $EXIT_CODE
 fi
 
-docker build -q -t "${REGISTRY}/scheduledworkflow:${TAG}" -f backend/Dockerfile.scheduledworkflow . && docker push "${REGISTRY}/scheduledworkflow:${TAG}" || EXIT_CODE=$?
+docker build --progress=plain -t "${REGISTRY}/scheduledworkflow:${TAG}" -f backend/Dockerfile.scheduledworkflow . && docker push "${REGISTRY}/scheduledworkflow:${TAG}" || EXIT_CODE=$?
 if [[ $EXIT_CODE -ne 0 ]]
 then
   echo "Failed to build scheduledworkflow image."
   exit $EXIT_CODE
 fi
 
-docker build -q -t "${REGISTRY}/driver:${TAG}" -f backend/Dockerfile.driver . && docker push "${REGISTRY}/driver:${TAG}" || EXIT_CODE=$?
+docker build --progress=plain -t "${REGISTRY}/driver:${TAG}" -f backend/Dockerfile.driver . && docker push "${REGISTRY}/driver:${TAG}" || EXIT_CODE=$?
 if [[ $EXIT_CODE -ne 0 ]]
 then
   echo "Failed to build driver image."
   exit $EXIT_CODE
 fi
 
-docker build -q -t "${REGISTRY}/launcher:${TAG}" -f backend/Dockerfile.launcher . && docker push "${REGISTRY}/launcher:${TAG}" || EXIT_CODE=$?
+docker build --progress=plain -t "${REGISTRY}/launcher:${TAG}" -f backend/Dockerfile.launcher . && docker push "${REGISTRY}/launcher:${TAG}" || EXIT_CODE=$?
 if [[ $EXIT_CODE -ne 0 ]]
 then
   echo "Failed to build launcher image."

diff --git a/.github/resources/scripts/kfp-readiness/wait_for_pods.py b/.github/resources/scripts/kfp-readiness/wait_for_pods.py
@@ -13,6 +13,17 @@
 config.load_kube_config()
 v1 = client.CoreV1Api()
 
+def log_pods():
+    pods = v1.list_namespaced_pod(namespace=namespace)
+
+    for pod in pods.items:
+        try:
+            logging.info(
+                f"---- Pod {namespace}/{pod.metadata.name} logs ----\n"
+                + v1.read_namespaced_pod_log(pod.metadata.name, namespace)
+            )
+        except client.exceptions.ApiException:
+            continue
 
 def get_pod_statuses():
     pods = v1.list_namespaced_pod(namespace=namespace)
@@ -74,6 +85,8 @@ def check_pods(calm_time=10, timeout=600, retries_after_ready=5):
         logging.info(f"Pods are still stabilizing. Retrying in {calm_time} seconds...")
         time.sleep(calm_time)
     else:
+        log_pods()
+
         raise Exception("Pods did not stabilize within the timeout period.")
 
     logging.info("Final pod statuses:")

diff --git a/.github/workflows/backend-visualization.yml b/.github/workflows/backend-visualization.yml
@@ -23,4 +23,4 @@ jobs:
           python-version: '3.9'
 
       - name: Run tests
-        run: ./test/presubmit-backend-visualization.sh
+        run: make test-backend-visualization-test
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
@@ -40,31 +40,24 @@ jobs:
           python-version: '3.9'
       - name: Install sdk
         run: |
-          python3 -m venv .venv
-          . .venv/bin/activate
-          pip install -e sdk/python
+          make setup-backend-test
       - name: Create KFP cluster
         uses: ./.github/actions/kfp-tekton-cluster
       - name: "flip coin test"
         run: |
-          . .venv/bin/activate
-          TEST_SCRIPT="test-flip-coin.sh" ./.github/resources/scripts/e2e-test.sh
+          make test-backend-test-flip-coin
       - name: "static loop test"
         run: |
-          . .venv/bin/activate
-          TEST_SCRIPT="test-static-loop.sh" ./.github/resources/scripts/e2e-test.sh
+          make test-backend-test-static-loop
       - name: "dynamic loop test"
         run: |
-          . .venv/bin/activate
-          TEST_SCRIPT="test-dynamic-loop.sh" ./.github/resources/scripts/e2e-test.sh
+          make test-backend-test-dynamic-loop
       - name: "use env"
         run: |
-          . .venv/bin/activate
-          TEST_SCRIPT="test-env.sh" ./.github/resources/scripts/e2e-test.sh
+          make test-backend-test-env
       - name: "use volume"
         run: |
-          . .venv/bin/activate
-          TEST_SCRIPT="test-volume.sh" ./.github/resources/scripts/e2e-test.sh
+          make test-backend-test-volume
       - name: Collect test results
         if: always()
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml
@@ -46,7 +46,7 @@ jobs:
           mkdir -p ./pr
           echo ${{ github.event.pull_request.number }} >> ./pr/pr_number          
           echo ${{ github.event.action }} >> ./pr/event_action
-      - uses: actions/upload-artifact@v3.1.0
+      - uses: actions/upload-artifact@v4
         with:
           name: pr
           path: pr/
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
@@ -37,12 +37,8 @@ jobs:
         with:
           k8s_version: ${{ matrix.k8s_version }}
 
-      - name: Forward API port
-        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
-
-      - name: Initialization tests v1
-        working-directory: ./backend/test/initialization
-        run: go test -v ./... -namespace kubeflow -args -runIntegrationTests=true
+      - name: Forward Port and initialize tests v1
+        run: make test-e2e-initialization-tests-v1
 
       - name: Collect test results
         if: always()
@@ -71,13 +67,8 @@ jobs:
         with:
           k8s_version: ${{ matrix.k8s_version }}
 
-
-      - name: Forward API port
-        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
-
       - name: Initialization tests v2
-        working-directory: ./backend/test/v2/initialization
-        run: go test -v ./... -namespace kubeflow -args -runIntegrationTests=true
+        run: make test-e2e-initialization-tests-v2
 
       - name: Collect test results
         if: always()
@@ -106,12 +97,8 @@ jobs:
         with:
           k8s_version: ${{ matrix.k8s_version }}
 
-      - name: Forward API port
-        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
-
       - name: API integration tests v1
-        working-directory: ./backend/test/integration
-        run: go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true
+        run:  make test-e2e-api-integration-tests-v1
 
       - name: Collect test results
         if: always()
@@ -140,13 +127,9 @@ jobs:
         with:
           k8s_version: ${{ matrix.k8s_version }}
 
-      - name: Forward API port
-        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
-
       - name: API integration tests v2
-        working-directory: ./backend/test/v2/integration
-        run: go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true
-
+        run: make test-e2e-api-integration-tests-v2
+
       - name: Collect test results
         if: always()
         uses: actions/upload-artifact@v4
@@ -174,19 +157,9 @@ jobs:
         with:
           k8s_version: ${{ matrix.k8s_version }}
 
-      - name: Forward API port
-        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
-
-      - name: Forward Frontend port
-        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline-ui" 3000 3000
-
-      - name: Build frontend integration tests image
-        working-directory: ./test/frontend-integration-test
-        run: docker build . -t kfp-frontend-integration-test:local
-
-      - name: Frontend integration tests
-        run: docker run --net=host kfp-frontend-integration-test:local --remote-run true
-
+      - name: Forward API port and run integration test 
+        run: make test-e2e-frontend-integration-test
+
       - name: Collect test results
         if: always()
         uses: actions/upload-artifact@v4
@@ -214,18 +187,9 @@ jobs:
         with:
           k8s_version: ${{ matrix.k8s_version }}
 
-      - name: Forward API port
-        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
-
-      - name: Install prerequisites
-        run: pip3 install -r ./test/sample-test/requirements.txt
-
-      - name: Basic sample tests - sequential
-        run:  python3 ./test/sample-test/sample_test_launcher.py sample_test run_test --namespace kubeflow --test-name sequential --results-gcs-dir output
-
-      - name: Basic sample tests - exit_handler
-        run: python3 ./test/sample-test/sample_test_launcher.py sample_test run_test --namespace kubeflow --test-name exit_handler --expected-result failed --results-gcs-dir output
-
+      - name: Forward API port and run integration tests
+        run: make test-e2e-basic-sample-tests
+
       - name: Collect test results
         if: always()
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml
@@ -20,16 +20,13 @@ jobs:
       uses: actions/checkout@v2
 
     - name: Set up Node.js
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
-        node-version: '14'
-
-    - name: Clean npm cache
-      run: npm cache clean --force
-
-    - name: Install dependencies
-      run: cd ./frontend && npm ci
+        node-version: '22'
+
+    - name: Setup frontend test
+      run: make setup-frontend-test
 
     - name: Run Frontend Tests
-      run: cd ./frontend && npm run test:ci
+      run: make test-frontend
 
diff --git a/.github/workflows/gcpc-modules-tests.yml b/.github/workflows/gcpc-modules-tests.yml
@@ -21,36 +21,8 @@ jobs:
         with:
           python-version: 3.9
 
-      - name: apt-get update
-        run: sudo apt-get update
-
-      - name: Install protobuf-compiler
-        run: sudo apt-get install protobuf-compiler -y
-
-      - name: Install setuptools
-        run: | 
-          pip3 install setuptools
-          pip3 freeze
-
-      - name: Install Wheel
-        run: pip3 install wheel==0.42.0
-
-      - name: Install python sdk 
-        run: pip install sdk/python
-
-      - name: Generate API proto files
-        working-directory: ./api
-        run: make clean python
-
-      - name: Install kfp-pipeline-spec from source
-        run: | 
-          python3 -m pip install api/v2alpha1/python
-
-      - name: Install google-cloud component 
-        run: pip install components/google-cloud
-
-      - name: Install Pytest
-        run: pip install $(grep 'pytest==' sdk/python/requirements-dev.txt)
+      - name: Install requirements
+        run: make setup-grpc-modules-test
 
       - name: Run test
-        run: pytest ./test/gcpc-tests/run_all_gcpc_modules.py
+        run: make test-grpc-modules
diff --git a/.github/workflows/image-builds.yml b/.github/workflows/image-builds.yml
@@ -77,20 +77,43 @@ jobs:
         include:
           - image: kfp-api-server
             dockerfile: backend/Dockerfile
+            context: .
           - image: kfp-frontend
             dockerfile: frontend/Dockerfile
+            context: .
           - image: kfp-persistence-agent
             dockerfile: backend/Dockerfile.persistenceagent
+            context: .
           - image: kfp-scheduled-workflow-controller
             dockerfile: backend/Dockerfile.scheduledworkflow
+            context: .
           - image: kfp-viewer-crd-controller
             dockerfile: backend/Dockerfile.viewercontroller
+            context: .
           - image: kfp-visualization-server
             dockerfile: backend/Dockerfile.visualization
+            context: .
           - image: kfp-launcher
             dockerfile: backend/Dockerfile.launcher
+            context: .
           - image: kfp-driver
             dockerfile: backend/Dockerfile.driver
+            context: .
+          - image: kfp-cache-deployer
+            dockerfile: backend/src/cache/deployer/Dockerfile
+            context: .
+          - image: kfp-cache-server
+            dockerfile: backend/Dockerfile.cacheserver
+            context: .
+          - image: kfp-metadata-writer
+            dockerfile: backend/metadata_writer/Dockerfile
+            context: .
+          - image: kfp-metadata-envoy
+            dockerfile: third_party/metadata_envoy/Dockerfile
+            context: .
+          - image: kfp-inverse-proxy-agent
+            dockerfile: proxy/Dockerfile
+            context: ./proxy
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -144,7 +167,7 @@ jobs:
         uses: docker/build-push-action@v6
         if: steps.check_tag.outcome == 'success'
         with:
-          context: .
+          context: ${{ matrix.context }}
           file: ${{ matrix.dockerfile }}
           push: true
           tags: ${{ steps.meta.outputs.tags }}