Merge pull request #230 from Ontos-AI/main #338
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and deploy API services | |
| on: | |
| push: | |
| branches: | |
| - staging | |
| release: | |
| types: | |
| - published | |
| workflow_dispatch: | |
| inputs: | |
| service: | |
| description: Service to build | |
| required: false | |
| type: choice | |
| options: | |
| - all | |
| - api | |
| - worker | |
| env: | |
| ACR_REGISTRY: ${{ secrets.ALIYUN_ACR_REGISTRY }} | |
| ACR_NAMESPACE: ${{ secrets.ALIYUN_ACR_NAMESPACE }} | |
| ECR_REGISTRY: 107424103509.dkr.ecr.us-east-1.amazonaws.com | |
| ECR_REPOSITORY: knowhere | |
| AWS_EKS_PROD_CLUSTER_NAME: ${{ secrets.AWS_EKS_PROD_CLUSTER_NAME }} | |
| AWS_EKS_PROD_REGION: ${{ secrets.AWS_EKS_PROD_REGION }} | |
| jobs: | |
| build-and-publish: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| strategy: | |
| matrix: | |
| service: [api, worker] | |
| include: | |
| - service: api | |
| dockerfile: deploy/docker/Dockerfile.api | |
| image_name: knowhere-backend | |
| ecr_repo_name: knowhere-backend | |
| - service: worker | |
| dockerfile: deploy/docker/Dockerfile.worker | |
| image_name: knowhere-worker | |
| ecr_repo_name: knowhere-worker | |
| steps: | |
| - name: Validate semantic release tag | |
| if: ${{ github.event_name == 'release' }} | |
| shell: bash | |
| run: | | |
| release_tag="${{ github.event.release.tag_name }}" | |
| semver_pattern='^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z.-]+)?$' | |
| if [[ ! "$release_tag" =~ $semver_pattern ]]; then | |
| echo "::error::Knowhere API releases must use semantic version tags like v1.0.1. Date-based tags such as 2026.06.18.1 or v2026.06.18.1 are not supported." | |
| exit 1 | |
| fi | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Decide build context | |
| id: context | |
| shell: bash | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| environment="staging" | |
| elif [ "${{ github.event_name }}" = "release" ]; then | |
| environment="prod" | |
| elif [[ "${{ github.ref }}" == refs/tags/* ]]; then | |
| environment="prod" | |
| else | |
| environment="staging" | |
| fi | |
| should_push="true" | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${{ github.event.inputs.service }}" ] && [ "${{ github.event.inputs.service }}" != "all" ] && [ "${{ github.event.inputs.service }}" != "${{ matrix.service }}" ]; then | |
| should_build="false" | |
| else | |
| should_build="true" | |
| fi | |
| echo "environment=$environment" >> "$GITHUB_OUTPUT" | |
| echo "should_push=$should_push" >> "$GITHUB_OUTPUT" | |
| echo "should_build=$should_build" >> "$GITHUB_OUTPUT" | |
| - name: Generate image tags | |
| if: steps.context.outputs.should_build == 'true' | |
| id: tags | |
| shell: bash | |
| run: | | |
| environment="${{ steps.context.outputs.environment }}" | |
| short_sha="${GITHUB_SHA::8}" | |
| if [ "${{ github.event_name }}" = "release" ]; then | |
| git_tag="${{ github.event.release.tag_name }}" | |
| image_tag="${git_tag}-${environment}" | |
| elif [[ "${{ github.ref }}" == refs/tags/* ]]; then | |
| git_tag="${GITHUB_REF#refs/tags/}" | |
| image_tag="${git_tag}-${environment}" | |
| elif [ "${{ github.event_name }}" = "pull_request" ]; then | |
| image_tag="pr-${{ github.event.pull_request.number }}-${short_sha}" | |
| else | |
| image_tag="${environment}-${short_sha}" | |
| fi | |
| latest_tag="${environment}-latest" | |
| echo "image_tag=$image_tag" >> "$GITHUB_OUTPUT" | |
| echo "latest_tag=$latest_tag" >> "$GITHUB_OUTPUT" | |
| - name: Prepare registry tags | |
| if: steps.context.outputs.should_build == 'true' | |
| id: images | |
| shell: bash | |
| run: | | |
| image_name="$(echo "${{ matrix.image_name }}" | tr '[:upper:]' '[:lower:]')" | |
| image_tag="${{ steps.tags.outputs.image_tag }}" | |
| latest_tag="${{ steps.tags.outputs.latest_tag }}" | |
| echo "local_image=${image_name}:${image_tag}" >> "$GITHUB_OUTPUT" | |
| if [ "${{ steps.context.outputs.should_push }}" = "true" ] && \ | |
| [ -n "${{ env.ACR_REGISTRY }}" ] && \ | |
| [ -n "${{ env.ACR_NAMESPACE }}" ] && \ | |
| [ -n "${{ secrets.ALIYUN_ACR_USERNAME }}" ] && \ | |
| [ -n "${{ secrets.ALIYUN_ACR_PASSWORD }}" ]; then | |
| acr_namespace="$(echo "${{ env.ACR_NAMESPACE }}" | tr '[:upper:]' '[:lower:]')" | |
| echo "acr_image=${{ env.ACR_REGISTRY }}/${acr_namespace}/${image_name}:${image_tag}" >> "$GITHUB_OUTPUT" | |
| echo "acr_latest=${{ env.ACR_REGISTRY }}/${acr_namespace}/${image_name}:${latest_tag}" >> "$GITHUB_OUTPUT" | |
| fi | |
| if [ "${{ steps.context.outputs.should_push }}" = "true" ] && \ | |
| [ -n "${{ secrets.AWS_ACCESS_KEY_ID }}" ] && \ | |
| [ -n "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ]; then | |
| echo "ecr_image=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}/${{ matrix.ecr_repo_name }}:${image_tag}" >> "$GITHUB_OUTPUT" | |
| echo "ecr_latest=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}/${{ matrix.ecr_repo_name }}:${latest_tag}" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Build tag list | |
| if: steps.context.outputs.should_build == 'true' | |
| id: tag-list | |
| shell: bash | |
| run: | | |
| { | |
| echo 'tags<<EOF' | |
| if [ "${{ steps.context.outputs.should_push }}" != "true" ]; then | |
| echo "${{ steps.images.outputs.local_image }}" | |
| fi | |
| if [ -n "${{ steps.images.outputs.acr_image }}" ]; then | |
| echo "${{ steps.images.outputs.acr_image }}" | |
| echo "${{ steps.images.outputs.acr_latest }}" | |
| fi | |
| if [ -n "${{ steps.images.outputs.ecr_image }}" ]; then | |
| echo "${{ steps.images.outputs.ecr_image }}" | |
| echo "${{ steps.images.outputs.ecr_latest }}" | |
| fi | |
| echo 'EOF' | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Set up Docker Buildx | |
| if: steps.context.outputs.should_build == 'true' | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to ACR | |
| if: steps.context.outputs.should_build == 'true' && steps.images.outputs.acr_image != '' | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.ACR_REGISTRY }} | |
| username: ${{ secrets.ALIYUN_ACR_USERNAME }} | |
| password: ${{ secrets.ALIYUN_ACR_PASSWORD }} | |
| - name: Configure AWS credentials | |
| if: steps.context.outputs.should_build == 'true' && steps.images.outputs.ecr_image != '' | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 | |
| - name: Login to ECR | |
| if: steps.context.outputs.should_build == 'true' && steps.images.outputs.ecr_image != '' | |
| shell: bash | |
| run: | | |
| aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${{ env.ECR_REGISTRY }} | |
| - name: Build image | |
| if: steps.context.outputs.should_build == 'true' | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: ${{ matrix.dockerfile }} | |
| push: ${{ steps.context.outputs.should_push == 'true' }} | |
| tags: ${{ steps.tag-list.outputs.tags }} | |
| build-args: | | |
| ENVIRONMENT=${{ steps.context.outputs.environment }} | |
| APP_VERSION=${{ steps.tags.outputs.image_tag }} | |
| GIT_COMMIT=${{ github.sha }} | |
| BUILD_TIME=${{ github.event.head_commit.timestamp || github.event.repository.updated_at }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: Summarize build | |
| if: steps.context.outputs.should_build == 'true' | |
| shell: bash | |
| run: | | |
| echo "Service: ${{ matrix.service }}" | |
| echo "Environment: ${{ steps.context.outputs.environment }}" | |
| echo "Push enabled: ${{ steps.context.outputs.should_push }}" | |
| echo "Primary tag: ${{ steps.tags.outputs.image_tag }}" | |
| deploy: | |
| runs-on: ubuntu-latest | |
| needs: build-and-publish | |
| if: github.event_name != 'pull_request' | |
| permissions: | |
| contents: read | |
| strategy: | |
| matrix: | |
| service: [api, worker] | |
| include: | |
| - service: api | |
| ecr_repo_name: knowhere-backend | |
| - service: worker | |
| ecr_repo_name: knowhere-worker | |
| steps: | |
| - name: Decide deployment context | |
| id: context | |
| shell: bash | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| environment="staging" | |
| elif [ "${{ github.event_name }}" = "release" ]; then | |
| environment="prod" | |
| elif [[ "${{ github.ref }}" == refs/tags/* ]]; then | |
| environment="prod" | |
| elif [ "${{ github.ref }}" = "refs/heads/staging" ]; then | |
| environment="staging" | |
| else | |
| environment="staging" | |
| should_deploy="false" | |
| fi | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${{ github.event.inputs.service }}" ] && [ "${{ github.event.inputs.service }}" != "all" ] && [ "${{ github.event.inputs.service }}" != "${{ matrix.service }}" ]; then | |
| should_deploy="false" | |
| elif [ -z "${should_deploy:-}" ]; then | |
| should_deploy="true" | |
| fi | |
| if [ -z "${{ secrets.AWS_ACCESS_KEY_ID }}" ] || \ | |
| [ -z "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ] || \ | |
| [ -z "${{ env.AWS_EKS_PROD_CLUSTER_NAME }}" ] || \ | |
| [ -z "${{ env.AWS_EKS_PROD_REGION }}" ]; then | |
| if [ "$environment" = "prod" ] && [ "$should_deploy" = "true" ]; then | |
| echo "::error::Production deployment credentials are not configured." | |
| exit 1 | |
| fi | |
| should_deploy="false" | |
| fi | |
| if [ "$environment" = "prod" ]; then | |
| namespace="knowhere-prod" | |
| else | |
| namespace="knowhere-staging" | |
| fi | |
| short_sha="${GITHUB_SHA::8}" | |
| if [ "${{ github.event_name }}" = "release" ]; then | |
| git_tag="${{ github.event.release.tag_name }}" | |
| image_tag="${git_tag}-${environment}" | |
| elif [[ "${{ github.ref }}" == refs/tags/* ]]; then | |
| git_tag="${GITHUB_REF#refs/tags/}" | |
| image_tag="${git_tag}-${environment}" | |
| else | |
| image_tag="${environment}-${short_sha}" | |
| fi | |
| echo "environment=$environment" >> "$GITHUB_OUTPUT" | |
| echo "namespace=$namespace" >> "$GITHUB_OUTPUT" | |
| echo "should_deploy=$should_deploy" >> "$GITHUB_OUTPUT" | |
| echo "image_uri=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}/${{ matrix.ecr_repo_name }}:${image_tag}" >> "$GITHUB_OUTPUT" | |
| - name: Configure AWS credentials | |
| if: steps.context.outputs.should_deploy == 'true' | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_EKS_PROD_REGION }} | |
| - name: Update kubeconfig | |
| if: steps.context.outputs.should_deploy == 'true' | |
| shell: bash | |
| run: | | |
| aws eks update-kubeconfig \ | |
| --name "${{ env.AWS_EKS_PROD_CLUSTER_NAME }}" \ | |
| --region "${{ env.AWS_EKS_PROD_REGION }}" | |
| - name: Roll out service on AWS EKS | |
| if: steps.context.outputs.should_deploy == 'true' | |
| shell: bash | |
| run: | | |
| deployment_name="knowhere-${{ matrix.service }}" | |
| container_name="${{ matrix.service }}" | |
| namespace="${{ steps.context.outputs.namespace }}" | |
| image_uri="${{ steps.context.outputs.image_uri }}" | |
| echo "Deploying ${deployment_name} to ${namespace} with ${image_uri}" | |
| kubectl set image "deployment/${deployment_name}" \ | |
| "${container_name}=${image_uri}" \ | |
| --namespace "${namespace}" | |
| if [ "${{ matrix.service }}" = "worker" ]; then | |
| kubectl patch "deployment/${deployment_name}" \ | |
| --namespace "${namespace}" \ | |
| --type='strategic' \ | |
| -p='{"spec":{"template":{"spec":{"containers":[{"name":"worker","readinessProbe":{"exec":{"command":["python","-c","from shared.services.worker_health import assert_worker_healthy; assert_worker_healthy()"]}},"livenessProbe":{"exec":{"command":["python","-c","from shared.services.worker_health import assert_worker_healthy; assert_worker_healthy()"]}}}]}}}}' | |
| fi | |
| kubectl rollout status "deployment/${deployment_name}" \ | |
| --namespace "${namespace}" \ | |
| --timeout=300s | |
| - name: Summarize deployment | |
| if: steps.context.outputs.should_deploy == 'true' | |
| shell: bash | |
| run: | | |
| echo "Service: ${{ matrix.service }}" | |
| echo "Environment: ${{ steps.context.outputs.environment }}" | |
| echo "Namespace: ${{ steps.context.outputs.namespace }}" | |
| echo "Image: ${{ steps.context.outputs.image_uri }}" | |
| release: | |
| name: Attach deployment release assets | |
| runs-on: ubuntu-latest | |
| needs: deploy | |
| if: >- | |
| ${{ github.event_name == 'release' && | |
| github.event.action == 'published' }} | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout deployed source | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.ref }} | |
| fetch-depth: 0 | |
| - name: Resolve release metadata | |
| id: release | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| release_tag="${{ github.event.release.tag_name }}" | |
| deployed_sha="$(git rev-parse HEAD)" | |
| echo "release_tag=${release_tag}" >> "${GITHUB_OUTPUT}" | |
| echo "deployed_sha=${deployed_sha}" >> "${GITHUB_OUTPUT}" | |
| - name: Create source archive | |
| env: | |
| RELEASE_TAG: ${{ steps.release.outputs.release_tag }} | |
| DEPLOYED_SHA: ${{ steps.release.outputs.deployed_sha }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| mkdir -p release-assets | |
| git archive --format=zip --output="release-assets/knowhere-api-${RELEASE_TAG}-source.zip" "${DEPLOYED_SHA}" | |
| cat > release-assets/build-info.json <<EOF | |
| { | |
| "repository": "${GITHUB_REPOSITORY}", | |
| "release_tag": "${RELEASE_TAG}", | |
| "deployed_commit": "${DEPLOYED_SHA}", | |
| "workflow_run_id": "${GITHUB_RUN_ID}", | |
| "workflow_run_url": "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | |
| } | |
| EOF | |
| - name: Upload release assets | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| RELEASE_TAG: ${{ steps.release.outputs.release_tag }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| gh release upload "${RELEASE_TAG}" \ | |
| release-assets/knowhere-api-${RELEASE_TAG}-source.zip \ | |
| release-assets/build-info.json \ | |
| --repo "${GITHUB_REPOSITORY}" \ | |
| --clobber |