Report example runs as integration test coverage (#639)

* Report example runs as integration test coverage run_example.py automatically runs constructor with coverage if the coverage Python module is importable. * Move coverage reporting from recipe to action to send more meta-data * Enable branch coverage in addition to line coverage * Correct dependency * Report with token as more reliable
conda · Feb 7, 2023 · 3ecd704 · 3ecd704
1 parent 224bab9
commit 3ecd704
Show file tree

Hide file tree

Showing 4 changed files with 38 additions and 10 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -71,16 +71,20 @@ jobs:
           conda create -p ../conda conda-build conda-verify
       - name: Build the package
         env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
           PYTHONIOENCODING: utf-8
           # Uncomment to run within conda build
           # RUN_EXAMPLES: "1"
         run: |
           source $CONDA/etc/profile.d/conda.sh
           conda activate ../conda
           export CODECOV_COMMIT=$(git rev-parse --verify HEAD)
+          CODECOV_FOLDER=${PWD} \
           CONDA_BLD_PATH="${{ runner.temp }}/conda-bld" \
             conda build conda.recipe --python=${{ matrix.pyver }}
+      - uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: unit
       - name: Upload the packages as artifact
         if: github.event_name == 'push'
         uses: actions/upload-artifact@v3
@@ -93,7 +97,7 @@ jobs:
         run: |
           source $CONDA/etc/profile.d/conda.sh
           CONDA_BLD_PATH="${{ runner.temp }}/conda-bld" \
-            conda create -n constructor -c local --strict-channel-priority constructor
+            conda create -n constructor -c local --strict-channel-priority constructor coverage
           conda activate constructor
           set -x
           installed_channel=$(conda list constructor --json | jq -r '.[].channel')
@@ -139,19 +143,25 @@ jobs:
           fi
       - name: Run examples and prepare artifacts
         run: |
+          rm -rf coverage.json
           source $CONDA/etc/profile.d/conda.sh
           conda activate constructor
           mkdir -p examples_artifacts/
           python scripts/run_examples.py \
             --keep-artifacts=examples_artifacts/ \
             --conda-exe="${CONSTRUCTOR_CONDA_EXE}"
+          coverage json
       - name: Test with conda-libmamba-solver
         run: |
           source $CONDA/etc/profile.d/conda.sh
           conda activate constructor
           conda install -yq conda-libmamba-solver
           conda list
           CONDA_SOLVER=libmamba CONDA_VERBOSITY=1 constructor examples/noconda/ --output-dir=examples_artifacts/
+      - uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: integration
       - name: Upload the example installers as artifacts
         if: github.event_name == 'pull_request' && matrix.pyver == '3.9'
         uses: actions/upload-artifact@v3

diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml
@@ -15,8 +15,7 @@ build:
   entry_points:
     - constructor = constructor.main:main
   script_env:
-    - CODECOV_TOKEN
-    - CODECOV_COMMIT
+    - CODECOV_FOLDER
     - RUN_EXAMPLES
 
 requirements:
@@ -46,13 +45,13 @@ test:
     - pillow >=3.1
     - pytest
     - pytest-cov
-    - codecov
   commands:
     - pip check
-    - pytest -v --cov=constructor tests
-    - coverage run --append -m constructor -V
-    - if [%CODECOV_TOKEN%] neq [] codecov --commit %CODECOV_COMMIT% # [win]
-    - if [ -n "$CODECOV_TOKEN" ]; then codecov --commit $CODECOV_COMMIT; fi # [unix]
+    - pytest -v --cov-branch --cov=constructor tests
+    - coverage run --branch --append -m constructor -V
+    - coverage json
+    - if [%CODECOV_FOLDER%] neq [] copy coverage.json "%CODECOV_FOLDER%" # [win]
+    - if [ -n "$CODECOV_FOLDER" ]; then cp coverage.json "$CODECOV_FOLDER"; fi # [unix]
     - coverage report
     - if [%RUN_EXAMPLES%] neq [] python scripts/run_examples.py      # [win]
     - if [ -n "$RUN_EXAMPLES" ]; then python scripts/run_examples.py; fi  # [unix]

diff --git a/news/639-int-coverage b/news/639-int-coverage
@@ -0,0 +1,19 @@
+### Enhancements
+
+* <news item>
+
+### Bug fixes
+
+* <news item>
+
+### Deprecations
+
+* <news item>
+
+### Docs
+
+* <news item>
+
+### Other
+
+* Report example runs as integration coverage. (#639)
diff --git a/scripts/run_examples.py b/scripts/run_examples.py
@@ -15,7 +15,7 @@
 
 try:
     import coverage  # noqa
-    COV_CMD = ['coverage', 'run', '--append', '-m']
+    COV_CMD = ['coverage', 'run', '--branch', '--append', '-m']
 except ImportError:
     COV_CMD = []