From e4e2ef150641c63c52eac929b927624165013c8e Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Fri, 13 Mar 2020 17:45:39 -0500 Subject: [PATCH 01/14] Add tests in pyhf-validation-root-base Docker image * Uses the 'container' option c.f. - https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idcontainer - https://github.com/github/actions-cheat-sheet/blob/master/actions-cheat-sheet.adoc#workflow-syntax --- .github/workflows/ci.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b344c2..f8bb457 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,27 @@ jobs: file: ./coverage.xml flags: unittests + test-ROOT-6_20-Py-3_7: + + runs-on: ubuntu-latest + container: pyhf/pyhf-validation-root-base:root6.20.00-python3.7 + + steps: + - uses: actions/checkout@master + - name: Install utilities + run: | + apt-get -qq -y update + apt-get -qq -y install git + git init . + - name: Install dependencies + run: | + python -m pip install --upgrade --no-cache-dir pip setuptools wheel + python -m pip install -q --no-cache-dir -e .[complete] + python -m pip list + - name: Test with pytest + run: | + python -m pytest -r sx tests + docker: runs-on: ubuntu-latest From a9896ecbcd2ce0ab11ed09c7a6e3322c270849d6 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Sat, 14 Mar 2020 00:51:53 -0500 Subject: [PATCH 02/14] Add comment explaining 'git init .' --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8bb457..fed6d79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,6 +58,8 @@ jobs: steps: - uses: actions/checkout@master - name: Install utilities + # git init done as the .git directory seems + # to be missing after the checkout run: | apt-get -qq -y update apt-get -qq -y install git From 9e5723c4b77b88dc13a9b5a8eb1959582b3c4fd7 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Sun, 15 Mar 2020 17:34:42 -0500 Subject: [PATCH 03/14] Add JSON to workspace converter script --- scripts/JSON_to_workspace.sh | 74 ++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 scripts/JSON_to_workspace.sh diff --git a/scripts/JSON_to_workspace.sh b/scripts/JSON_to_workspace.sh new file mode 100644 index 0000000..780867b --- /dev/null +++ b/scripts/JSON_to_workspace.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +set -e + +function JSON_to_workspace() { + # 1: the analysis name + # 2: the url of the JSON workspaces + # 3: the background only workspace + # 4: the signal workspace patch + # 5: the patched background + signal workspace + local ANALYSIS_NAME=$1 + local HEPDATA_URL=$2 + local BACKGROUND_ONLY=$3 + local SIGNAL_PATCH=$4 + local JSON_WORKSPACE=$5 + + if [[ ! -d "${ANALYSIS_NAME}" ]]; then + mkdir "${ANALYSIS_NAME}" + fi + + # Download from HEPData + curl -sL -o "${ANALYSIS_NAME}/workspaces.tar.gz" "${HEPDATA_URL}" + # Unpack tarball + if [[ ! -d "${ANALYSIS_NAME}/workspaces" ]]; then + mkdir "${ANALYSIS_NAME}/workspaces" + fi + tar xvzf "${ANALYSIS_NAME}/workspaces.tar.gz" -C "${ANALYSIS_NAME}/workspaces" + + # Create a full signal+background workspace + jsonpatch \ + "${ANALYSIS_NAME}/workspaces/${BACKGROUND_ONLY}" \ + "${ANALYSIS_NAME}/workspaces/${SIGNAL_PATCH}" > \ + "${ANALYSIS_NAME}/workspaces/${JSON_WORKSPACE}" + + # Convert to ROOT + XML + if [[ ! -d "${ANALYSIS_NAME}/xml" ]]; then + mkdir "${ANALYSIS_NAME}/xml" + fi + rm -rf "${ANALYSIS_NAME}/xml"/* + pyhf json2xml \ + --output-dir "${ANALYSIS_NAME}/xml" \ + "${ANALYSIS_NAME}/workspaces/${JSON_WORKSPACE}" + + # Generate ROOT workspace + hist2workspace "${ANALYSIS_NAME}/xml"/FitConfig.xml +} + +function main() { + # 1: the analysis name + # 2: the url of the JSON workspaces + # 3: the background only workspace + # 4: the signal workspace patch + # 5: the patched background + signal workspace + local ANALYSIS_NAME=$1 + local HEPDATA_URL=$2 + local BACKGROUND_ONLY=$3 + local SIGNAL_PATCH=$4 + local JSON_WORKSPACE=$5 + + JSON_to_workspace "${ANALYSIS_NAME}" "${HEPDATA_URL}" "${BACKGROUND_ONLY}" "${SIGNAL_PATCH}" "${JSON_WORKSPACE}" + + # bash tests/test_suite.sh \ + # "${ANALYSIS_NAME}/xml"/config/FitConfig_combined_NormalMeasurement_model.root \ + # "${ANALYSIS_NAME}/workspaces/${JSON_WORKSPACE}" +} + +main "$@" || exit 1 + +# main \ +# sbottom \ +# https://doi.org/10.17182/hepdata.89408.v1/r2 \ +# RegionA/BkgOnly.json \ +# RegionA/patch.sbottom_1300_205_60.json \ +# RegionA/sbottom_1300_205_60.json From 7334751c3a71171a8cc540b5b324f5a2c7f2f3a1 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Sun, 15 Mar 2020 17:35:12 -0500 Subject: [PATCH 04/14] Add JSON to workspace converter test to CI --- .github/workflows/ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fed6d79..3069780 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,17 @@ jobs: - name: Test with pytest run: | python -m pytest -r sx tests + - name: Test workspace comparison + run: | + bash scripts/JSON_to_workspace.sh \ + sbottom \ + https://doi.org/10.17182/hepdata.89408.v1/r2 \ + RegionA/BkgOnly.json \ + RegionA/patch.sbottom_1300_205_60.json \ + RegionA/sbottom_1300_205_60.json + bash tests/test_suite.sh \ + sbottom/xml/config/FitConfig_combined_NormalMeasurement_model.root \ + sbottom/workspaces/RegionA/sbottom_1300_205_60.json docker: From dda8ac537e29f5d858dd40a8e599734147143919 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Sun, 15 Mar 2020 22:33:31 -0500 Subject: [PATCH 05/14] Remove commented out run examples --- scripts/JSON_to_workspace.sh | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/scripts/JSON_to_workspace.sh b/scripts/JSON_to_workspace.sh index 780867b..e8828b2 100644 --- a/scripts/JSON_to_workspace.sh +++ b/scripts/JSON_to_workspace.sh @@ -58,17 +58,6 @@ function main() { local JSON_WORKSPACE=$5 JSON_to_workspace "${ANALYSIS_NAME}" "${HEPDATA_URL}" "${BACKGROUND_ONLY}" "${SIGNAL_PATCH}" "${JSON_WORKSPACE}" - - # bash tests/test_suite.sh \ - # "${ANALYSIS_NAME}/xml"/config/FitConfig_combined_NormalMeasurement_model.root \ - # "${ANALYSIS_NAME}/workspaces/${JSON_WORKSPACE}" } main "$@" || exit 1 - -# main \ -# sbottom \ -# https://doi.org/10.17182/hepdata.89408.v1/r2 \ -# RegionA/BkgOnly.json \ -# RegionA/patch.sbottom_1300_205_60.json \ -# RegionA/sbottom_1300_205_60.json From efe8c61533dac45536b190504cb14acc4feffdc3 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Sun, 15 Mar 2020 22:39:16 -0500 Subject: [PATCH 06/14] Simplify logic on dir creation --- scripts/JSON_to_workspace.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/JSON_to_workspace.sh b/scripts/JSON_to_workspace.sh index e8828b2..2778d05 100644 --- a/scripts/JSON_to_workspace.sh +++ b/scripts/JSON_to_workspace.sh @@ -33,10 +33,10 @@ function JSON_to_workspace() { "${ANALYSIS_NAME}/workspaces/${JSON_WORKSPACE}" # Convert to ROOT + XML - if [[ ! -d "${ANALYSIS_NAME}/xml" ]]; then - mkdir "${ANALYSIS_NAME}/xml" + if [[ -d "${ANALYSIS_NAME}/xml" ]]; then + rm -rf "${ANALYSIS_NAME}/xml" fi - rm -rf "${ANALYSIS_NAME}/xml"/* + mkdir "${ANALYSIS_NAME}/xml" pyhf json2xml \ --output-dir "${ANALYSIS_NAME}/xml" \ "${ANALYSIS_NAME}/workspaces/${JSON_WORKSPACE}" From b6b8d8fbf7eadc739a7467045323d66783a5a7fa Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Sun, 15 Mar 2020 23:27:53 -0500 Subject: [PATCH 07/14] split out workspace generation and testing --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3069780..ba0547b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,7 @@ jobs: - name: Test with pytest run: | python -m pytest -r sx tests - - name: Test workspace comparison + - name: Generate ROOT workspace from JSON run: | bash scripts/JSON_to_workspace.sh \ sbottom \ @@ -80,6 +80,8 @@ jobs: RegionA/BkgOnly.json \ RegionA/patch.sbottom_1300_205_60.json \ RegionA/sbottom_1300_205_60.json + - name: Test workspace comparison + run: | bash tests/test_suite.sh \ sbottom/xml/config/FitConfig_combined_NormalMeasurement_model.root \ sbottom/workspaces/RegionA/sbottom_1300_205_60.json From 59b370ce640bd67800570020490c7fe7143c5249 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Mon, 16 Mar 2020 00:39:35 -0500 Subject: [PATCH 08/14] Fix ordering for replacement the replacement is of ROOT named parameters to be named like pyhf parameters --- scripts/compare_fitted_nuisance.py | 2 +- scripts/compare_nuisance.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/compare_fitted_nuisance.py b/scripts/compare_fitted_nuisance.py index e59e716..a4c0c83 100644 --- a/scripts/compare_fitted_nuisance.py +++ b/scripts/compare_fitted_nuisance.py @@ -94,7 +94,7 @@ def exhaust_argset(s): pyhf_param = ( param.replace("alpha_", "") .replace("gamma_stat_", "staterror_") - .replace("lumi", "Lumi") + .replace("Lumi", "lumi") .replace("_bin", "") ) diff --git a/scripts/compare_nuisance.py b/scripts/compare_nuisance.py index 06dc43b..7701000 100644 --- a/scripts/compare_nuisance.py +++ b/scripts/compare_nuisance.py @@ -35,8 +35,7 @@ def exhaust_argset(s): pars_root = [ sub.replace("alpha_", "") .replace("gamma_stat_", "staterror_") - .replace("gamma_stat_", "staterror_") - .replace("lumi", "Lumi") + .replace("Lumi", "lumi") .replace("_bin", "") for sub in pars ] From 5bb1375b6c773e9de6cd4ebe87ec7371acdb005c Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Fri, 27 Mar 2020 18:24:54 -0500 Subject: [PATCH 09/14] Lint shell scripts with shellcheck --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba0547b..f0f1d5e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,8 +62,12 @@ jobs: # to be missing after the checkout run: | apt-get -qq -y update - apt-get -qq -y install git + apt-get -qq -y install git shellcheck git init . + - name: Lint with Shellcheck + run: | + shellcheck tests/*.sh + shellcheck scripts/*.sh - name: Install dependencies run: | python -m pip install --upgrade --no-cache-dir pip setuptools wheel From 1390f9904f21d394165a796fde7a2f8359fa3d65 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Fri, 27 Mar 2020 22:35:43 -0500 Subject: [PATCH 10/14] Drop redundant install of git Git is already installed in the base image --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0f1d5e..148df94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: # to be missing after the checkout run: | apt-get -qq -y update - apt-get -qq -y install git shellcheck + apt-get -qq -y install shellcheck git init . - name: Lint with Shellcheck run: | From 546fd4af007728026a5520a2baf3ce367f0dbb20 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Fri, 10 Apr 2020 18:18:10 -0500 Subject: [PATCH 11/14] Use root6.22.02 release tag significantly smaller --- .github/workflows/ci.yml | 2 +- .github/workflows/publish.yml | 4 ++-- docker/Dockerfile | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 148df94..c81eedf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,7 +53,7 @@ jobs: test-ROOT-6_20-Py-3_7: runs-on: ubuntu-latest - container: pyhf/pyhf-validation-root-base:root6.20.00-python3.7 + container: pyhf/pyhf-validation-root-base:root6.22.02-python3.8 steps: - uses: actions/checkout@master diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e38768b..3a7b5c6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -22,7 +22,7 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} repository: pyhf/pyhf-validation dockerfile: docker/Dockerfile - tags: latest,root6.20.00,root6.20.00-python3.7 + tags: latest,root6.22.02,root6.22.02-python3.8 - name: Build and Publish to Registry with Release Tag if: startsWith(github.ref, 'refs/tags/') uses: docker/build-push-action@v1 @@ -31,5 +31,5 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} repository: pyhf/pyhf-validation dockerfile: docker/Dockerfile - tags: latest,latest-stable,root6.20.00,root6.20.00-python3.7 + tags: latest,latest-stable,root6.22.02,root6.22.02-python3.8 tag_with_ref: true diff --git a/docker/Dockerfile b/docker/Dockerfile index 722d398..d97d4a4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM pyhf/pyhf-validation-root-base:root6.20.00-python3.7 as base +FROM pyhf/pyhf-validation-root-base:root6.22.02-python3.8 as base FROM base as builder COPY . /code From 13cb617ada0be753d624e5ef4295eb9ce5a11a0c Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Tue, 17 Nov 2020 20:45:36 -0600 Subject: [PATCH 12/14] Correct name --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c81eedf..90ed31a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,7 @@ jobs: file: ./coverage.xml flags: unittests - test-ROOT-6_20-Py-3_7: + test-ROOT-6_22-Py-3_8: runs-on: ubuntu-latest container: pyhf/pyhf-validation-root-base:root6.22.02-python3.8 From 23ade2af1a597f2ed0126ee642c46f8de9bc9448 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Tue, 17 Nov 2020 21:45:43 -0600 Subject: [PATCH 13/14] Update to use modern pyhf CLI tooling --- .github/workflows/ci.yml | 8 +++--- scripts/JSON_to_workspace.sh | 52 +++++++++++++++++------------------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90ed31a..327c402 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,11 +79,11 @@ jobs: - name: Generate ROOT workspace from JSON run: | bash scripts/JSON_to_workspace.sh \ - sbottom \ - https://doi.org/10.17182/hepdata.89408.v1/r2 \ + sbottom-pallet \ + https://www.hepdata.net/record/resource/1408070?view=true \ RegionA/BkgOnly.json \ - RegionA/patch.sbottom_1300_205_60.json \ - RegionA/sbottom_1300_205_60.json + sbottom_1300_205_60 \ + RegionA/workspace_sbottom_1300_205_60.json - name: Test workspace comparison run: | bash tests/test_suite.sh \ diff --git a/scripts/JSON_to_workspace.sh b/scripts/JSON_to_workspace.sh index 2778d05..f0b8458 100644 --- a/scripts/JSON_to_workspace.sh +++ b/scripts/JSON_to_workspace.sh @@ -3,61 +3,57 @@ set -e function JSON_to_workspace() { - # 1: the analysis name + # 1: the analysis pallet name # 2: the url of the JSON workspaces # 3: the background only workspace - # 4: the signal workspace patch + # 4: the name of the signal patch # 5: the patched background + signal workspace - local ANALYSIS_NAME=$1 + local PALLET_NAME=$1 local HEPDATA_URL=$2 local BACKGROUND_ONLY=$3 - local SIGNAL_PATCH=$4 + local PATCH_NAME=$4 local JSON_WORKSPACE=$5 - if [[ ! -d "${ANALYSIS_NAME}" ]]; then - mkdir "${ANALYSIS_NAME}" - fi - # Download from HEPData - curl -sL -o "${ANALYSIS_NAME}/workspaces.tar.gz" "${HEPDATA_URL}" - # Unpack tarball - if [[ ! -d "${ANALYSIS_NAME}/workspaces" ]]; then - mkdir "${ANALYSIS_NAME}/workspaces" - fi - tar xvzf "${ANALYSIS_NAME}/workspaces.tar.gz" -C "${ANALYSIS_NAME}/workspaces" + pyhf contrib download "${HEPDATA_URL}" "${PALLET_NAME}" # Create a full signal+background workspace + workspace_dir="${BACKGROUND_ONLY%/*}" + pyhf patchset extract \ + --name "${PATCH_NAME}" \ + --output-file "${PALLET_NAME}/${workspace_dir}/${PATCH_NAME}.json" \ + "${PALLET_NAME}/${workspace_dir}/patchset.json" jsonpatch \ - "${ANALYSIS_NAME}/workspaces/${BACKGROUND_ONLY}" \ - "${ANALYSIS_NAME}/workspaces/${SIGNAL_PATCH}" > \ - "${ANALYSIS_NAME}/workspaces/${JSON_WORKSPACE}" + "${PALLET_NAME}/${BACKGROUND_ONLY}" \ + "${PALLET_NAME}/${workspace_dir}/${PATCH_NAME}.json" > \ + "${PALLET_NAME}/${JSON_WORKSPACE}" # Convert to ROOT + XML - if [[ -d "${ANALYSIS_NAME}/xml" ]]; then - rm -rf "${ANALYSIS_NAME}/xml" + if [[ -d "${PALLET_NAME}/xml" ]]; then + rm -rf "${PALLET_NAME}/xml" fi - mkdir "${ANALYSIS_NAME}/xml" + mkdir "${PALLET_NAME}/xml" pyhf json2xml \ - --output-dir "${ANALYSIS_NAME}/xml" \ - "${ANALYSIS_NAME}/workspaces/${JSON_WORKSPACE}" + --output-dir "${PALLET_NAME}/xml" \ + "${PALLET_NAME}/${JSON_WORKSPACE}" # Generate ROOT workspace - hist2workspace "${ANALYSIS_NAME}/xml"/FitConfig.xml + hist2workspace "${PALLET_NAME}/xml/FitConfig.xml" } function main() { - # 1: the analysis name + # 1: the analysis pallet name # 2: the url of the JSON workspaces # 3: the background only workspace - # 4: the signal workspace patch + # 4: the name of the signal patch # 5: the patched background + signal workspace - local ANALYSIS_NAME=$1 + local PALLET_NAME=$1 local HEPDATA_URL=$2 local BACKGROUND_ONLY=$3 - local SIGNAL_PATCH=$4 + local PATCH_NAME=$4 local JSON_WORKSPACE=$5 - JSON_to_workspace "${ANALYSIS_NAME}" "${HEPDATA_URL}" "${BACKGROUND_ONLY}" "${SIGNAL_PATCH}" "${JSON_WORKSPACE}" + JSON_to_workspace "${PALLET_NAME}" "${HEPDATA_URL}" "${BACKGROUND_ONLY}" "${PATCH_NAME}" "${JSON_WORKSPACE}" } main "$@" || exit 1 From 2ae3c60b0cf191efcd13268b9555eff2e296ab92 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Tue, 17 Nov 2020 22:11:21 -0600 Subject: [PATCH 14/14] Correct paths --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 327c402..45f4f89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade --no-cache-dir pip setuptools wheel - python -m pip install -q --no-cache-dir -e .[complete] + python -m pip install --quiet --no-cache-dir --editable .[complete] python -m pip list - name: Test with pytest run: | @@ -87,8 +87,8 @@ jobs: - name: Test workspace comparison run: | bash tests/test_suite.sh \ - sbottom/xml/config/FitConfig_combined_NormalMeasurement_model.root \ - sbottom/workspaces/RegionA/sbottom_1300_205_60.json + sbottom-pallet/xml/config/FitConfig_combined_NormalMeasurement_model.root \ + sbottom-pallet/RegionA/workspace_sbottom_1300_205_60.json docker: