Introducing Run Examples check in PRs

.github/actions/ollama/action.yml

@@ -0,0 +1,48 @@
+name: 'Ollama Setup'
+description: 'Composte action for Ollama set up in GH environment'
+runs:
+  using: 'composite'
+  steps:
+    - name: Remove unnecessary files
+      shell: bash
+      run: |
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+      # Set up Ollama
+    - name: Install Ollama and start server
+      shell: bash
+      run: |
+        curl -fsSL https://ollama.com/install.sh | sudo -E sh
+
+    - name: Pull models in examples/
+      shell: bash
+      run: |
+        ollama pull granite3.2:2b
+        ollama pull granite3.2:8b
+        ollama pull mxbai-embed-large
+        ollama list
+
+    - name: Check that all required models are available
+      shell: bash
+      run: |
+        models=("mxbai-embed-large" "granite3.2:2b" "granite3.2:8b")
+        missing=0
+        for model in "${models[@]}"; do
+          if ! ollama list | awk 'NR>1 {print $1}' | grep -q "$model"; then
+            echo "❌ Model $model is missing!"
+            missing=1
+          fi
+        done
+
+        if [ "$missing" -eq 1 ]; then
+          exit 1
+        else
+          echo "✅ All expected models are available."
+        fi
+
+    - name: Wait for Ollama server
+      shell: bash
+      run: |
+        sleep 5
+        time curl -i http://localhost:11434

.github/actions/run-examples/action.yml

@@ -0,0 +1,109 @@
+name: 'Ollama Setup'
+description: 'Composte action to set up Run Examples'
+inputs:
+  python-version:
+    description: 'Python version'
+    required: true
+    default: '3.11'
+  runner-os:
+    description: 'Runner OS'
+    required: true
+  repository:
+    description: 'Repository name this pull request is initiated from'
+    required: false
+  head-ref:
+    description: 'Head ref of the repo'
+    required: false
+    default: 'main'
+  token: 
+    description: 'Github token'
+    required: false 
+  update-results:
+    description: 'Whether to update the results for this run. Must be false for nightly runs'
+    required: true
+  check:
+    description: 'Files to patch tests/test_examples_run.yaml with. These are the PDL files that the test will run against. Defaults to all PDL files.'
+    required: false
+    default: '[]'
+runs:
+  using: 'composite'
+  steps:
+    # # Set up Ollama
+    - uses: ./.github/actions/ollama
+
+    # Configure Run Examples environment
+    - uses: actions/checkout@v4
+      with:
+        token: ${{ inputs.token }}
+        ref: ${{ inputs.head-ref }}
+        repository: ${{ inputs.repository }}
+        fetch-depth: 0
+
+    - name: Patch tests/test_examples_run.yaml check with modified files
+      shell: bash 
+      run: |
+        yq -i '.check = (${{ inputs.check }})' tests/test_examples_run.yaml
+
+    - name: Print test Run Examples config
+      shell: bash
+      run: cat tests/test_examples_run.yaml
+
+    # Run tests
+    - name: Set up Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+    - name: Cache pip
+      uses: actions/cache@v4
+      with:
+        # This path is specific to Ubuntu
+        path: ${{ env.pythonLocation }}
+        # Look to see if there is a cache hit for the setup file
+        key: ${{ inputs.runner-os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
+        restore-keys: |
+          ${{ inputs.runner-os }}-pip-new3
+          ${{ inputs.runner-os }}-new3
+    - name: Install dependencies
+      shell: bash
+      run: pip install --upgrade --upgrade-strategy eager .[all]
+    - name: Pip list packages
+      shell: bash
+      run: pip list
+    - name: Run Pytest
+      shell: bash
+      run: |
+        cat tests/test_examples_run.yaml        
+        (
+          set +e
+          py.test -v --capture=tee-sys -rfE -s tests/test_examples_run.py --disable-pytest-warnings
+          EXIT_CODE=$?
+
+          if [ $EXIT_CODE -eq 0 ]; then
+            echo "TEST_RESULT=PASSED" >> $GITHUB_ENV
+          else
+            echo "TEST_RESULT=FAILED" >> $GITHUB_ENV
+          fi
+        )
+
+    # Commit the results if update results
+    - name: Push new results to branch
+      shell: bash
+      if: ${{ inputs.update-results == 'true' }}
+      run: |
+        git config --local user.name github-actions[bot]
+        git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com
+        git status
+        git pull origin ${{ inputs.head-ref }}
+        git add tests/results/
+        git diff --cached --quiet || git commit -s -m "github-actions[bot]: Run examples: updated result files on your behalf"
+        # git push origin ${{ inputs.head-ref }}
+        git push https://x-access-token:${{ inputs.token }}@github.com/${{ inputs.repository }} HEAD:${{ inputs.head-ref }}
+
+    - name: Check if pytest passed
+      shell: bash
+      run: |
+        if [ "$TEST_RESULT" == "PASSED" ]; then
+          exit 0
+        else
+            exit 1
+        fi

.github/workflows/run-examples-prep.yml

@@ -0,0 +1,174 @@
+---
+name: Run examples on modified PDL files
+on: [pull_request]
+jobs:
+  tests:
+    name: Execution tests
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    # strategy:
+    #   fail-fast: false
+    #   max-parallel: 1
+    #   matrix:
+    #     python-version: ['3.11', '3.12', '3.13']
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.ref }}
+        repository: ${{ github.event.pull_request.head.repo.full_name }}
+        token: ${{ secrets.PDL_PAT }}
+        fetch-depth: 0
+
+    - name: Push the new run-examples yaml config 
+      shell: bash 
+      run: |
+        echo "This is a test: $(date)" > test-file.txt
+        git config --local user.name github-actions[bot]
+        git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com
+        git remote set-url fork-remote https://${{ secrets.PDL_PAT }}@github.com/${{ github.event.pull_request.head.repo.full_name }}.git
+        git status
+        git pull fork-remote ${{ github.head_ref }}
+        git add test-file.txt
+        git diff --cached --quiet || git commit -s -m "github-actions[bot]: Simple test for PAT"
+        git push fork-remote ${{ github.head_ref }}
+
+    # # Detect modified PDL files, includes Add, Modified, but not Deleted
+    # - uses: actions/checkout@v4
+    #   with:
+    #     fetch-depth: 0
+    # - name: Detect all PDL files that were changed or added
+    #   id: changed-pdl-files
+    #   uses: tj-actions/changed-files@6cb76d07bee4c9772c6882c06c37837bf82a04d3 # v46
+    #   with:
+    #     files: |
+    #       **.pdl
+    #     json: "true"
+    #     escape_json: "false"
+    # - name: List PDL files that were modified or added and append to test_examples_run
+    #   run: |
+    #     echo ${{ steps.changed-pdl-files.outputs.all_changed_files }}
+    #     if [ ${{ steps.changed-pdl-files.outputs.all_changed_files_count }} -gt 0 ]; then
+    #       echo "early-stop=false" >> $GITHUB_ENV
+    #     else
+    #       echo "No file need to be checked, skipping all subsequent tests."
+    #       echo "early-stop=true" >> $GITHUB_ENV
+    #     fi
+
+    # # Run tests if there are modified PDL files
+    # - name: Check if update results automatically by GitHub Actions Bot 
+    #   if: ${{ env.early-stop == 'false' }}
+    #   id: parse-run-examples-config
+    #   shell: bash
+    #   run: |
+    #     value=$(yq '.update_results' tests/test_examples_run.yaml | tr '[:upper:]' '[:lower:]')
+    #     echo "value=$value"
+    #     echo "update_results=$value" >> $GITHUB_ENV
+
+    # # Set up Ollama
+    # # - name: Cache Ollama model files 
+    # #   if: ${{ env.early-stop == 'false' }}
+    # #   uses: actions/cache@v4
+    # #   with:
+    # #     path: /usr/share/ollama/.ollama/models
+    # #     key: ${{ runner.os }}-build-ollama-cache
+    # #     restore-keys: |
+    # #       ${{ runner.os }}-build-
+    # #       ${{ runner.os }}
+    # - uses: ./.github/actions/ollama
+    #   if: ${{ env.early-stop == 'false' }}
+
+    # # Set up Run Examples environment 
+    # - name: Set up Python ${{ matrix.python-version }}
+    #   uses: actions/setup-python@v5
+    #   if: ${{ env.early-stop == 'false' }}
+    #   with:
+    #     python-version: ${{ matrix.python-version }}
+    # - name: Cache pip
+    #   uses: actions/cache@v4
+    #   if: ${{ env.early-stop == 'false' }}
+    #   with:
+    #     # This path is specific to Ubuntu
+    #     path: ${{ env.pythonLocation }}
+    #     # Look to see if there is a cache hit for the setup file
+    #     key: ${{ runner.os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
+    #     restore-keys: |
+    #       ${{ runner.os }}-pip-new3
+    #       ${{ runner.os }}-new3
+    # - name: Install dependencies
+    #   if: ${{ env.early-stop == 'false' }}
+    #   shell: bash
+    #   run: pip install --upgrade --upgrade-strategy eager .[all]
+    # - name: Pip list packages
+    #   if: ${{ env.early-stop == 'false' }}
+    #   shell: bash
+    #   run: pip list
+    # - name: Patch tests/test_examples_run.yaml check with modified files
+    #   if: ${{ env.early-stop == 'false' }}
+    #   shell: bash 
+    #   run: |
+    #     yq -i '.check = (${{ steps.changed-pdl-files.outputs.all_changed_files }})' tests/test_examples_run.yaml
+    # - name: View Run Examples config
+    #   if: ${{ env.early-stop == 'false' }}
+    #   shell: bash
+    #   run: cat tests/test_examples_run.yaml
+    # - name: Run Pytest
+    #   if: ${{ env.early-stop == 'false' }}
+    #   shell: bash
+    #   run: |    
+    #     (
+    #       set +e
+    #       py.test -v --capture=tee-sys -rfE -s tests/test_examples_run.py --disable-pytest-warnings
+    #       EXIT_CODE=$?
+
+    #       if [ $EXIT_CODE -eq 0 ]; then
+    #         echo "TEST_RESULT=PASSED" >> $GITHUB_ENV
+    #       else
+    #         echo "TEST_RESULT=FAILED" >> $GITHUB_ENV
+    #       fi
+    #     )
+
+    # # Commit the results if update results
+    # - name: Push new results to branch
+    #   shell: bash
+    #   if: ${{ env.early-stop == 'false' && env.update_results == 'true' }}
+    #   run: |
+    #     git config --local user.name github-actions[bot]
+    #     git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com
+    #     git status
+    #     git pull 
+    #     git add tests/results/
+    #     git diff --cached --quiet || git commit -s -m "github-actions[bot]: Run examples: updated result files on your behalf"
+    #     git push 
+
+    # # Patch config with update_results to false and commit to PR
+    # - name: Patch tests/test_examples_run.yaml with update_results to False, only for the last job in the series
+    #   if: ${{ env.early-stop == 'false' && env.update_results == 'true' && matrix.python-version == '3.13' }}
+    #   shell: bash 
+    #   run: |
+    #     yq -i '.update_results = ('false')' tests/test_examples_run.yaml
+    # - name: Push the new run-examples yaml config 
+    #   if: ${{ env.early-stop == 'false' && env.update_results == 'true' && matrix.python-version == '3.13' }}
+    #   shell: bash 
+    #   run: |
+    #     git config --local user.name github-actions[bot]
+    #     git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com
+    #     git remote set-url origin https://${{ secrets.PDL_PAT }}@github.com/${{ github.event.pull_request.head.repo.full_name }}.git
+    #     git status
+    #     git pull origin ${{ github.head_ref }}
+    #     git add tests/test_examples_run.yaml
+    #     git diff --cached --quiet || git commit -s -m "github-actions[bot]: Run examples: reverting update_result to false on your behalf"
+    #     git push origin ${{ github.head_ref }}
+
+    # - name: Check if pytest passed
+    #   shell: bash
+    #   if: ${{ env.early-stop == 'false' }}
+    #   run: |
+    #     if [ "$TEST_RESULT" == "PASSED" ]; then
+    #       exit 0
+    #     else
+    #         exit 1
+    #     fi
+
+