feat: setting up integration CICD actions

.github/workflows/deploy-main.yml

@@ -0,0 +1,140 @@
+name: deploy-main
+
+on:
+  # deploy only when version tags are pushed
+  push:
+    tags:
+      - 'v*'
+
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'Git ref (tag/branch/commit) to deploy'
+        required: false
+        default: ''
+
+concurrency:
+  group: deploy-prod-${{ github.ref_name }}-${{ github.job }}
+  cancel-in-progress: false
+
+jobs:
+  # PROD (staging-sst-01)
+  deploy-staging:
+    if: ${{ startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' }}
+    name: Deploy DAB to staging_sst_01 instance
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11.11'
+
+      - name: Install uv and Databricks CLI
+        run: |
+          python -m pip install --upgrade pip
+          pip install uv
+          curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh
+          databricks -v
+        shell: bash
+
+      - name: Install project dependencies (uv)
+        run: |
+          uv venv
+          uv pip install .
+
+      - name: Configure Databricks env (Dev)
+        env:
+          DATABRICKS_HOST: ${{ secrets.DATABRICKS_DEV_HOST }}
+          DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_DEV_CLIENT_ID }}
+          DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_DEV_CLIENT_SECRET }}
+        run: |
+          databricks version
+
+      - name: Deploy bundle (prod target -> staging_sst_01)
+         working-directory: pipelines/pdp
+
+        env:
+          DATABRICKS_HOST: ${{ secrets.DATABRICKS_DEV_HOST }}
+          DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_DEV_CLIENT_ID }}
+          DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_DEV_CLIENT_SECRET }}
+          DB_WORKSPACE:         ${{ secrets.STAGING_DB_WORKSPACE }}
+          SA_EXECUTER:          ${{ secrets.STAGING_SERVICE_ACCOUNT_EXECUTER }}
+          DS_RUN_AS:            ${{ secrets.STAGING_DS_RUN_AS }}
+          GROUP_TO_MANAGE:      ${{ secrets.GROUP_TO_MANAGE }}
+          DATAKIND_EMAIL:       ${{ secrets.DATAKIND_EMAIL }}
+        run: |
+          echo "Deploying DAB to prod for tag $GITHUB_REF_NAME..."
+          databricks bundle validate --target=prod
+          databricks bundle deploy \
+            --target=prod \
+            --var="DB_workspace=$DB_WORKSPACE" \
+            --var="service_account_executer=$SA_EXECUTER" \
+            --var="ds_run_as=$DS_RUN_AS" \
+            --var="databricks_institution_name=midway_uni" \
+            --var="datakind_group_to_manage_workflow=$GROUP_TO_MANAGE" \
+            --var="datakind_notification_email=$DATAKIND_EMAIL" \
+            --var="git_tag=${GITHUB_REF_NAME}"
+
+  # DEV (dev-sst-02)
+  deploy-dev:
+    if: ${{ startsWith(github.ref, 'refs/tags/v')
+
+    name: Deploy DAB to dev_sst_02 instance
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11.11'
+
+      - name: Install uv and Databricks CLI
+        run: |
+          python -m pip install --upgrade pip
+          pip install uv
+          curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh
+          databricks -v
+        shell: bash
+
+      - name: Install project dependencies (uv)
+        run: |
+          uv venv
+          uv pip install .
+
+      - name: Configure Databricks env (Dev)
+        env:
+          DATABRICKS_HOST: ${{ secrets.DATABRICKS_DEV_HOST }}
+          DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_DEV_CLIENT_ID }}
+          DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_DEV_CLIENT_SECRET }}
+        run: |
+          databricks version
+
+      - name: Deploy bundle (prod target -> dev_sst_02)
+        working-directory: pipelines/pdp
+        env:
+          DATABRICKS_HOST: ${{ secrets.DATABRICKS_DEV_HOST }}
+          DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_DEV_CLIENT_ID }} 
+          DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_DEV_CLIENT_SECRET }}
+          DB_WORKSPACE:         secrets.DEV_DB_WORKSPACE
+          SA_EXECUTER:          ${{ secrets.DEV_SERVICE_ACCOUNT_EXECUTER }}
+          DS_RUN_AS:            ${{ secrets.DEV_DS_RUN_AS }}
+          GROUP_TO_MANAGE:      ${{ secrets.GROUP_TO_MANAGE }}
+          DATAKIND_EMAIL:       ${{ secrets.DATAKIND_EMAIL }}
+        run: |
+          echo "Deploying PROD target to dev_sst_02 for tag ${GITHUB_REF_NAME}..."
+          databricks bundle deploy \
+            --target=prod \
+            --var="DB_workspace=$DB_WORKSPACE" \
+            --var="service_account_executer=$SA_EXECUTER" \
+            --var="ds_run_as=$DS_RUN_AS" \
+            --var="databricks_institution_name=synthetic" \
+            --var="datakind_group_to_manage_workflow=$GROUP_TO_MANAGE" \
+            --var="datakind_notification_email=$DATAKIND_EMAIL" \
+            --var="git_tag=${GITHUB_REF_NAME}" \

.github/workflows/release-integration.yml

@@ -0,0 +1,108 @@
+name: release-branch-ci-dev
+
+on:
+  # run when release branch is created from git flow
+  push:
+    branches:
+      - 'release/*'
+
+concurrency:
+  group: release-branch-ci-dev-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  dev-train-infer:
+    name: Train + Inference on Dev (release branch)
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    env:
+      CONFIG_FILE: config.toml
+      DB_RUN_ID_PREFIX: release_${{ github.sha }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with: { fetch-depth: 0 }
+
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.11.11" }
+
+      - name: Install uv and Databricks CLI
+        run: |
+          python -m pip install --upgrade pip
+          pip install uv
+          curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh
+          databricks -v
+        shell: bash
+
+      - name: Install project dependencies (uv)
+        run: |
+          uv venv
+          uv pip install .
+        shell: bash
+
+      - name: Configure Databricks env (Dev)
+        env:
+          DATABRICKS_HOST: ${{ secrets.DATABRICKS_DEV_HOST }}
+          DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_DEV_CLIENT_ID }}
+          DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_DEV_CLIENT_SECRET }}
+        run: |
+          databricks version
+
+      - name: Deploy bundle to Dev (pin to current commit)
+        working-directory: pipelines/pdp
+        env:
+          DATABRICKS_HOST: ${{ secrets.DATABRICKS_DEV_HOST }}
+          DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_DEV_CLIENT_ID }} 
+          DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_DEV_CLIENT_SECRET }}
+          DB_WORKSPACE: ${{ secrets.DEV_DB_WORKSPACE }}
+          SA_EXECUTER: ${{ secrets.DEV_SERVICE_ACCOUNT_EXECUTER }}
+          DS_RUN_AS: ${{ secrets.DEV_DS_RUN_AS }}
+          GROUP_TO_MANAGE: ${{ secrets.GROUP_TO_MANAGE }}
+          DATAKIND_EMAIL: ${{ secrets.DATAKIND_EMAIL }}
+        run: |
+          databricks bundle deploy \
+            --target=dev \
+            --var "git_commit=${GITHUB_SHA}" \
+            --var="DB_workspace=$DB_WORKSPACE" \
+            --var="service_account_executer=$SA_EXECUTER" \
+            --var="ds_run_as=$DS_RUN_AS" \
+            --var "databricks_institution_name=synthetic_integration" \
+            --var="datakind_group_to_manage_workflow=$GROUP_TO_MANAGE" \
+            --var="datakind_notification_email=$DATAKIND_EMAIL" \
+
+      - name: TRAIN (Dev – develop-release-check)
+        working-directory: pipelines/pdp
+        env:
+          DATABRICKS_HOST:         ${{ secrets.DATABRICKS_DEV_HOST }}
+          DATABRICKS_CLIENT_ID:     ${{ secrets.DATABRICKS_DEV_CLIENT_ID }}
+          DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_DEV_CLIENT_SECRET }}
+        run: |
+          databricks bundle run github_sourced_pdp_training_pipeline --target dev \
+            --var "git_commit=${GITHUB_SHA}" \
+            --var "DB_workspace=${{ secrets.DEV_DB_WORKSPACE }}" \
+            --var "service_account_executer=${{ secrets.DEV_SERVICE_ACCOUNT_EXECUTER }}" \
+            --var "ds_run_as=${{ secrets.DEV_DS_RUN_AS }}" \
+            --var "datakind_group_to_manage_workflow=${{ secrets.GROUP_TO_MANAGE }}" \
+            --var "databricks_institution_name=synthetic_integration" \
+            --var "datakind_notification_email=${{ secrets.DATAKIND_EMAIL }}" \
+            --var "DK_CC_EMAIL=${{ secrets.DATAKIND_EMAIL }}" \
+            --params config_file_name="$CONFIG_FILE",job_type=training,db_run_id="${DB_RUN_ID_PREFIX}_train"
+
+      - name: INFER (Dev – develop-release-check)
+        working-directory: pipelines/pdp
+        env:
+          DATABRICKS_HOST:         ${{ secrets.DATABRICKS_DEV_HOST }}
+          DATABRICKS_CLIENT_ID:    ${{ secrets.DATABRICKS_DEV_CLIENT_ID }}
+          DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_DEV_CLIENT_SECRET }}
+        run: |
+          databricks bundle run edvise_github_sourced_pdp_inference_pipeline --target dev \
+            --var "git_commit=${GITHUB_SHA}" \
+            --var "DB_workspace=${{ secrets.DEV_DB_WORKSPACE }}" \
+            --var "service_account_executer=${{ secrets.DEV_SERVICE_ACCOUNT_EXECUTER }}" \
+            --var "ds_run_as=${{ secrets.DEV_DS_RUN_AS }}" \
+            --var "datakind_group_to_manage_workflow=${{ secrets.GROUP_TO_MANAGE }}" \
+            --var "databricks_institution_name=synthetic_integration" \
+            --var "datakind_notification_email=${{ secrets.DATAKIND_EMAIL }}" \
+            --var "DK_CC_EMAIL=${{ secrets.DATAKIND_EMAIL }}" \
+            --params "model_name=synthetic_integration_retention_2_year_time_first_within_cohort,model_type=h2o,config_file_name=${CONFIG_FILE},job_type=inference,db_run_id=${DB_RUN_ID_PREFIX}_inference"
+

.github/workflows/weekly-cleanup.yml

@@ -0,0 +1,62 @@
+name: weekly-cleanup
+
+on:
+  schedule:
+    - cron: "10 16 1-7 * 1"  # First Monday of every month at 16:10 UTC
+  workflow_dispatch: {}
+
+jobs:
+  cleanup-synthetic:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'schedule' && 'develop' || github.ref_name }}
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.11.11" }
+
+      - name: Install uv and Databricks CLI
+        run: |
+          python -m pip install --upgrade pip
+          pip install uv
+          curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh
+
+      - name: Install project deps
+        run: |
+          uv venv
+          uv pip install .
+
+      - name: Deploy bundle (dev)
+        working-directory: pipelines/pdp
+        env:
+          DATABRICKS_HOST:         ${{ secrets.DATABRICKS_DEV_HOST }}
+          DATABRICKS_CLIENT_ID:     ${{ secrets.DATABRICKS_DEV_CLIENT_ID }}
+          DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_DEV_CLIENT_SECRET }}
+        run: |
+          databricks bundle deploy --target dev \
+            --var "git_commit=${GITHUB_SHA}" \
+            --var "DB_workspace=${{ secrets.DEV_DB_WORKSPACE }}" \
+            --var "databricks_institution_name=synthetic_integration" \
+            --var "datakind_notification_email=${{ secrets.DATAKIND_EMAIL }}" \
+            --var "service_account_executer=${{ secrets.DEV_SERVICE_ACCOUNT_EXECUTER }}" \
+            --var "ds_run_as=${{ secrets.DEV_DS_RUN_AS }}" \
+            --var "datakind_group_to_manage_workflow=${{ secrets.GROUP_TO_MANAGE }}"
+
+      - name: CLEANUP
+        working-directory: pipelines/pdp
+        env:
+          DATABRICKS_HOST:         ${{ secrets.DATABRICKS_DEV_HOST }}
+          DATABRICKS_CLIENT_ID:     ${{ secrets.DATABRICKS_DEV_CLIENT_ID }}
+          DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_DEV_CLIENT_SECRET }}
+        run: |
+          databricks bundle run edvise_cleanup_synthetic --target dev \
+            --var "git_commit=${GITHUB_SHA}" \
+            --var "DB_workspace=${{ secrets.DEV_DB_WORKSPACE }}" \
+            --var "databricks_institution_name=synthetic_integration" \
+            --var "datakind_notification_email=${{ secrets.DATAKIND_EMAIL }}" \
+            --var "service_account_executer=${{ secrets.DEV_SERVICE_ACCOUNT_EXECUTER }}" \
+            --var "ds_run_as=${{ secrets.DEV_DS_RUN_AS }}" \
+            --var "datakind_group_to_manage_workflow=${{ secrets.GROUP_TO_MANAGE }}" \
+            --params dry_run=false,delete_models=true,delete_experiments=true,retention_days=30,databricks_institution_name=synthetic_integration