Introducing Run Examples check in PRs

.github/actions/ollama/action.yml

@@ -0,0 +1,48 @@
+name: 'Ollama Setup'
+description: 'Composte action for Ollama set up in GH environment'
+runs:
+  using: 'composite'
+  steps:
+    - name: Remove unnecessary files
+      shell: bash
+      run: |
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+      # Set up Ollama
+    - name: Install Ollama and start server
+      shell: bash
+      run: |
+        curl -fsSL https://ollama.com/install.sh | sudo -E sh
+
+    - name: Pull models in examples/
+      shell: bash
+      run: |
+        ollama pull granite3.2:2b
+        ollama pull granite3.2:8b
+        ollama pull mxbai-embed-large
+        ollama list
+
+    - name: Check that all required models are available
+      shell: bash
+      run: |
+        models=("mxbai-embed-large" "granite3.2:2b" "granite3.2:8b")
+        missing=0
+        for model in "${models[@]}"; do
+          if ! ollama list | awk 'NR>1 {print $1}' | grep -q "$model"; then
+            echo "❌ Model $model is missing!"
+            missing=1
+          fi
+        done
+
+        if [ "$missing" -eq 1 ]; then
+          exit 1
+        else
+          echo "✅ All expected models are available."
+        fi
+
+    - name: Wait for Ollama server
+      shell: bash
+      run: |
+        sleep 5
+        time curl -i http://localhost:11434

.github/actions/run-examples/action.yml

@@ -0,0 +1,109 @@
+name: 'Ollama Setup'
+description: 'Composte action to set up Run Examples'
+inputs:
+  python-version:
+    description: 'Python version'
+    required: true
+    default: '3.11'
+  runner-os:
+    description: 'Runner OS'
+    required: true
+  repository:
+    description: 'Repository name this pull request is initiated from'
+    required: false
+  head-ref:
+    description: 'Head ref of the repo'
+    required: false
+    default: 'main'
+  token: 
+    description: 'Github token'
+    required: false 
+  update-results:
+    description: 'Whether to update the results for this run. Must be false for nightly runs'
+    required: true
+  check:
+    description: 'Files to patch tests/test_examples_run.yaml with. These are the PDL files that the test will run against. Defaults to all PDL files.'
+    required: false
+    default: '[]'
+runs:
+  using: 'composite'
+  steps:
+    # # Set up Ollama
+    - uses: ./.github/actions/ollama
+
+    # Configure Run Examples environment
+    - uses: actions/checkout@v4
+      with:
+        token: ${{ inputs.token }}
+        ref: ${{ inputs.head-ref }}
+        repository: ${{ inputs.repository }}
+        fetch-depth: 0
+
+    - name: Patch tests/test_examples_run.yaml check with modified files
+      shell: bash 
+      run: |
+        yq -i '.check = (${{ inputs.check }})' tests/test_examples_run.yaml
+
+    - name: Print test Run Examples config
+      shell: bash
+      run: cat tests/test_examples_run.yaml
+
+    # Run tests
+    - name: Set up Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+    - name: Cache pip
+      uses: actions/cache@v4
+      with:
+        # This path is specific to Ubuntu
+        path: ${{ env.pythonLocation }}
+        # Look to see if there is a cache hit for the setup file
+        key: ${{ inputs.runner-os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
+        restore-keys: |
+          ${{ inputs.runner-os }}-pip-new3
+          ${{ inputs.runner-os }}-new3
+    - name: Install dependencies
+      shell: bash
+      run: pip install --upgrade --upgrade-strategy eager .[all]
+    - name: Pip list packages
+      shell: bash
+      run: pip list
+    - name: Run Pytest
+      shell: bash
+      run: |
+        cat tests/test_examples_run.yaml        
+        (
+          set +e
+          py.test -v --capture=tee-sys -rfE -s tests/test_examples_run.py --disable-pytest-warnings
+          EXIT_CODE=$?
+
+          if [ $EXIT_CODE -eq 0 ]; then
+            echo "TEST_RESULT=PASSED" >> $GITHUB_ENV
+          else
+            echo "TEST_RESULT=FAILED" >> $GITHUB_ENV
+          fi
+        )
+
+    # Commit the results if update results
+    - name: Push new results to branch
+      shell: bash
+      if: ${{ inputs.update-results == 'true' }}
+      run: |
+        git config --local user.name github-actions[bot]
+        git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com
+        git status
+        git pull origin ${{ inputs.head-ref }}
+        git add tests/results/
+        git diff --cached --quiet || git commit -s -m "github-actions[bot]: Run examples: updated result files on your behalf"
+        # git push origin ${{ inputs.head-ref }}
+        git push https://x-access-token:${{ inputs.token }}@github.com/${{ inputs.repository }} HEAD:${{ inputs.head-ref }}
+
+    - name: Check if pytest passed
+      shell: bash
+      run: |
+        if [ "$TEST_RESULT" == "PASSED" ]; then
+          exit 0
+        else
+            exit 1
+        fi

.github/workflows/run-examples-prep.yml

@@ -0,0 +1,47 @@
+---
+name: Run examples on modified PDL files
+on: [pull_request]
+jobs:
+  tests:
+    name: Execution tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.11', '3.12', '3.13']
+
+    steps:
+    # Detect modified PDL files, includes Add, Modified, but not Deleted
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Detect all PDL files that were changed or added
+      id: changed-pdl-files
+      uses: tj-actions/changed-files@6cb76d07bee4c9772c6882c06c37837bf82a04d3 # v46
+      with:
+        files: |
+          **.pdl
+        json: "true"
+        escape_json: "false"
+    - name: List PDL files that were modified or added and append to test_examples_run.yaml
+      run: |
+        echo ${{ steps.changed-pdl-files.outputs.all_changed_files }}
+        if [ ${{ steps.changed-pdl-files.outputs.all_changed_files_count }} -gt 0 ]; then
+          echo "early-stop=false" >> $GITHUB_ENV
+        else
+          echo "No file need to be checked, skipping all subsequent tests."
+          echo "early-stop=true" >> $GITHUB_ENV
+        fi
+
+    - name: Run examples 
+      uses:  ./.github/actions/run-examples
+      # Only run if there are modified PDL files
+      if:  env.early-stop == 'false'
+      with: 
+        python-version: ${{ matrix.python-version }}
+        runner-os: ${{ runner.os }}
+        repository: ${{ github.event.pull_request.head.repo.full_name }}
+        head-ref: ${{ github.event.pull_request.head.ref }}
+        token: ${{ github.token }}
+        update-results: 'false'
+        check: ${{ steps.changed-pdl-files.outputs.all_changed_files }}

.github/workflows/run-examples.yml

@@ -6,7 +6,6 @@ on:
     - cron:  '0 1 * * *'
   workflow_dispatch:
 
-
 jobs:
   tests:
     name: Execution tests
@@ -15,93 +14,17 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ['3.11', '3.12', '3.13']
-
     steps:
-
-      # Free up some disk space
-    - name: Remove unnecessary files
-      run: |
-        sudo rm -rf /usr/share/dotnet
-        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-
-      # Set up Ollama
-    - name: Install Ollama and start server
-      shell: bash
-      run: |
-        curl -fsSL https://ollama.com/install.sh | sudo -E sh
-
-    - name: Pull models in examples/
-      shell: bash
-      run: |
-        ollama pull granite3.2:2b
-        ollama pull granite3.2:8b
-        ollama pull mxbai-embed-large
-        ollama list
-
-    - name: Check that all required models are available
-      shell: bash
-      run: |
-        models=("mxbai-embed-large" "granite3.2:2b" "granite3.2:8b")
-        missing=0
-        for model in "${models[@]}"; do
-          if ! ollama list | awk 'NR>1 {print $1}' | grep -q "$model"; then
-            echo "❌ Model $model (or substring) is missing!"
-            missing=1
-          fi
-        done
-
-        if [ "$missing" -eq 1 ]; then
-          exit 1
-        else
-          echo "✅ All expected models are available."
-        fi
-
-    - name: Wait for Ollama server
-      shell: bash
-      run: |
-        sleep 10
-        time curl -i http://localhost:11434
-
-    # Run tests
-    - uses: actions/checkout@v4
-      with:
-        ref: ${{ github.head_ref }}
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Cache pip
-      uses: actions/cache@v4
-      with:
-        # This path is specific to Ubuntu
-        path: ${{ env.pythonLocation }}
-        # Look to see if there is a cache hit for the setup file
-        key: ${{ runner.os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-new3
-          ${{ runner.os }}-new3
-    - name: Install dependencies
-      run: pip install --upgrade --upgrade-strategy eager .[all]
-    - name: pip list packages
-      run: pip list
-    - name: show pip dependencies
-      run: |
-        pip install pipdeptree
-        pipdeptree -fl
-    - name: run tests
-      env:
-        WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }}
-        WATSONX_APIKEY: ${{ secrets.WATSONX_APIKEY }}
-        WATSONX_URL: ${{ secrets.WATSONX_URL }}
-        REPLICATE_API_TOKEN: ${{ secrets.REPLICATE_API_TOKEN }}
-        OLLAMA_GHACTIONS_RESULTS: true
-      run: py.test -v --capture=tee-sys -rfE -s tests/test_examples_run.py
-    - name: Update example result files (if any) generated from Ollama running on GH Actions
-      if: matrix.python-version == '3.11'
-      run: |
-        git config --local user.name github-actions[bot]
-        git config --local user.email "${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com"
-        git status
-        git add tests/results/
-        git diff --cached --quiet || git commit -S -s -m "github-actions[bot]: Updated results file when running examples on $(date)"
-        git push
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+      - uses: ./.github/actions/run-examples
+        with:
+          python-version: ${{ matrix.python-version }}
+          runner-os: ${{ runner.os }}
+          repository: ${{ github.repository }}
+          head-ref: ${{ github.head_ref }}
+          token: ${{ github.token }}
+          update-results: 'false'
+          check: '[]'   # Empty list means run against all PDL programs
+