From ae7c3827ca0dc89a1357e618cef1f57a72ff83fd Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Mon, 21 Oct 2024 17:52:30 -0400 Subject: [PATCH 01/88] Create test_env.yml --- .github/workflows/test_env.yml | 42 ++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/test_env.yml diff --git a/.github/workflows/test_env.yml b/.github/workflows/test_env.yml new file mode 100644 index 00000000..8cc07018 --- /dev/null +++ b/.github/workflows/test_env.yml @@ -0,0 +1,42 @@ +name: create_test_conda_env + +on: [push] + +jobs: + build-linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '>=3.9' + + - name: Add conda to system path + run: | + # $CONDA is an env var pointing to root of miniconda dir + echo $CONDA/bin >> $GITHUB_PATH + + - name: Create fre-workflows environment + run: | + # create environment containing all dependencies + # the env cannot be explicitly activated in github CI/CD + echo "MY LOCATION" + pwd + echo "MY LOCATION" + conda env create -f environment.yml --name fre-workflows + + # add conda env's executables to github's PATH equiv. + echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH + + - name: in fre-workflows environment, cylc lint + continue-on-error: true + run: | + # lint .cylc workflow files + cylc lint -v + + - name: in fre-workflows environment, root directory pytest + continue-on-error: true + run: | + # pytest unittests for other things in this repository + pytest -v -v -rx ./tests From 864c1c5dd7c35e8a39391f1e70c504b4b2189fd9 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Mon, 21 Oct 2024 17:57:56 -0400 Subject: [PATCH 02/88] Any podman or docker? --- .github/workflows/test_env.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_env.yml b/.github/workflows/test_env.yml index 8cc07018..91297e30 100644 --- a/.github/workflows/test_env.yml +++ b/.github/workflows/test_env.yml @@ -23,8 +23,10 @@ jobs: # the env cannot be explicitly activated in github CI/CD echo "MY LOCATION" pwd - echo "MY LOCATION" - conda env create -f environment.yml --name fre-workflows + echo "MY LOCATION" + docker --help + podman --help + conda create --name fre-workflows-env # add conda env's executables to github's PATH equiv. echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH From 4440a63025c9a1662dbe2fbbe9fcba5426044317 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Mon, 21 Oct 2024 18:04:58 -0400 Subject: [PATCH 03/88] clone HPC-ME tripleP branch --- .github/workflows/test_env.yml | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test_env.yml b/.github/workflows/test_env.yml index 91297e30..5a8e9650 100644 --- a/.github/workflows/test_env.yml +++ b/.github/workflows/test_env.yml @@ -24,21 +24,14 @@ jobs: echo "MY LOCATION" pwd echo "MY LOCATION" - docker --help - podman --help - conda create --name fre-workflows-env + #podman --help + #conda create --name fre-workflows-env # add conda env's executables to github's PATH equiv. - echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH + #echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH - - name: in fre-workflows environment, cylc lint - continue-on-error: true + - name: clone branch run: | - # lint .cylc workflow files - cylc lint -v - - - name: in fre-workflows environment, root directory pytest - continue-on-error: true - run: | - # pytest unittests for other things in this repository - pytest -v -v -rx ./tests + git clone --single-branch --branch TripleP https://gitlab.gfdl.noaa.gov/fre/HPC-ME.git + pwd + ls From ddbce330dc0224600956ce23fa091eaa70a6c7f3 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Mon, 21 Oct 2024 18:17:30 -0400 Subject: [PATCH 04/88] Update test_env.yml --- .github/workflows/test_env.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_env.yml b/.github/workflows/test_env.yml index 5a8e9650..e0a4443c 100644 --- a/.github/workflows/test_env.yml +++ b/.github/workflows/test_env.yml @@ -24,6 +24,7 @@ jobs: echo "MY LOCATION" pwd echo "MY LOCATION" + ls #podman --help #conda create --name fre-workflows-env @@ -32,6 +33,5 @@ jobs: - name: clone branch run: | - git clone --single-branch --branch TripleP https://gitlab.gfdl.noaa.gov/fre/HPC-ME.git + #git clone --single-branch --branch TripleP https://gitlab.gfdl.noaa.gov/fre/HPC-ME.git pwd - ls From 6c56993f2d332d4f05a1911926832081d9b8e584 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Mon, 21 Oct 2024 18:18:04 -0400 Subject: [PATCH 05/88] Add Dockerfile-ppp to test container build --- Dockerfile-ppp | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 Dockerfile-ppp diff --git a/Dockerfile-ppp b/Dockerfile-ppp new file mode 100644 index 00000000..4c2d6ac8 --- /dev/null +++ b/Dockerfile-ppp @@ -0,0 +1,34 @@ +FROM condaforge/mambaforge:24.1.2-0 +LABEL maintainer "Ciheim Brown" + +## Description ## +# condaforge is based on a stripped down ubuntu image. We need some extra bits for frerun + fremake +#### + +# apt installs to /usr/bin/ +RUN apt update \ + && apt -y install uuid-runtime time csh python bc + +# Put runscript in container and make it executable +COPY runscript.sh /exec/runscript.sh +RUN chmod +x /exec/runscript.sh + +# Set up conda environment directory for cylc workflow +RUN mkdir /opt/conda/cylc-flow-tools +COPY cylc-flow-tools.yaml /opt/conda/cylc-flow-tools +#COPY macro.py /tmp + +# Finding + replacing the problematic macro.py file with the corrected version +RUN conda env create --file /opt/conda/cylc-flow-tools/cylc-flow-tools.yaml -p /opt/conda/cylc-flow-tools +# && /bin/bash -c 'export macro=$(find /opt/conda/cylc-flow-tools/lib -wholename */site-packages/metomi/rose) \ +# && mv -f /tmp/macro.py $macro/macro.py' \ +# && git config --global --add safe.directory /mnt2 + +RUN IFS="" \ + && pv=$(python --version) \ + && pv1=$(echo $pv | cut -d ' ' -f2) \ + && conda config --add pkgs_dirs /opt/conda/cylc-flow-tools/lib/python$pv1/site-packages + +RUN conda install urwid==2.* + +ENTRYPOINT ["/bin/bash"] From eae85d4d58b11d54e7de997f8c97ea1109187f6c Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Mon, 21 Oct 2024 18:18:54 -0400 Subject: [PATCH 06/88] For Dockerfile test --- cylc-flow-tools.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 cylc-flow-tools.yaml diff --git a/cylc-flow-tools.yaml b/cylc-flow-tools.yaml new file mode 100644 index 00000000..57dbf87f --- /dev/null +++ b/cylc-flow-tools.yaml @@ -0,0 +1,16 @@ +name: cylc-flow-tools +channels: + - conda-forge + - NOAA-GFDL +dependencies: + - hsm + - fre-nctools + - nco + - cdo + - metomi-rose + - rsync + - make + - vim + - fre-cli ==2024.01 +# - hdf5 + - fre-python-tools From 18e8cb441b7d98f4ceb3beb27fd61428553b92aa Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 15:14:20 -0400 Subject: [PATCH 07/88] try wf dispatch things --- .github/workflows/test_cloud_runner.yml | 30 ++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 98633ae1..b20c44f5 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -1,11 +1,14 @@ -name: run_ppp +name: test_cloud_runner on: - pull_request: - branches: - - main # triggers pipeline on pull requests to merge into main - push: - branches: - - main # triggers pipeline on push to main + workflow_dispatch: + inputs: + fre-cli-repo: + description: 'NOAA-GFDL/fre-cli' + required: true + fre-cli-branch: + description: 'fre-cli branch to check out' + required: true + default: 'main' # cancel running jobs if theres a newer push concurrency: @@ -16,7 +19,18 @@ jobs: test-cloud: runs-on: self-hosted steps: - - uses: actions/checkout@v4 + - name: Test workflow_dispatch + uses: actions/checkout@v4 + with: + repository: ${{ github.event.inputs.fre-cli-repo }} + ref: ${{ github.event.inputs.fre-cli-branch }} + #path: '/contrib/container-test' + - name: check + run: | + echo "did it do stuff?" + pwd + ls fre-cli + exit 1 - name: Run ppp-container run: | # Print pwd then run script From 6fcf33bcd9da979f4daa00f5fc7b33223d7ff59f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 15:15:40 -0400 Subject: [PATCH 08/88] troubleshoot --- .github/workflows/test_cloud_runner.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index b20c44f5..e8d21a10 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -28,6 +28,7 @@ jobs: - name: check run: | echo "did it do stuff?" + echo "menial change" pwd ls fre-cli exit 1 From 3b17eaccc691bb7066e98e26e56c6129da528bb4 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 15:18:32 -0400 Subject: [PATCH 09/88] try options --- .github/workflows/test_cloud_runner.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index e8d21a10..debd2a59 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -3,10 +3,14 @@ on: workflow_dispatch: inputs: fre-cli-repo: - description: 'NOAA-GFDL/fre-cli' + description: 'NOAA-GFDL/fre-cli repo' required: true + type: choice + default: 'NOAA-GFDL/fre-cli' + options: + - NOAA-GFDL/fre-cli fre-cli-branch: - description: 'fre-cli branch to check out' + description: 'Input which fre-cli branch to check out' required: true default: 'main' From 257a1a03b6e51073fd580fd61a4a182c5673a55e Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 15:22:08 -0400 Subject: [PATCH 10/88] fix choices --- .github/workflows/test_cloud_runner.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index debd2a59..e7c9ae87 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -8,7 +8,7 @@ on: type: choice default: 'NOAA-GFDL/fre-cli' options: - - NOAA-GFDL/fre-cli + - 'NOAA-GFDL/fre-cli' fre-cli-branch: description: 'Input which fre-cli branch to check out' required: true @@ -21,7 +21,7 @@ concurrency: jobs: test-cloud: - runs-on: self-hosted + runs-on: ubuntu-latest steps: - name: Test workflow_dispatch uses: actions/checkout@v4 @@ -35,6 +35,8 @@ jobs: echo "menial change" pwd ls fre-cli + cd fre-cli + git branch exit 1 - name: Run ppp-container run: | From 1ce58687ed314db78b6da2a1bf5f449082b38342 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 15:25:13 -0400 Subject: [PATCH 11/88] little change to see if triggered wf works --- .github/workflows/test_cloud_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index e7c9ae87..f8e04758 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -32,7 +32,7 @@ jobs: - name: check run: | echo "did it do stuff?" - echo "menial change" + echo "menial change ahhh " pwd ls fre-cli cd fre-cli From 981fdf2751b892abaa5d3377d258af7ca86c8e94 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 15:28:05 -0400 Subject: [PATCH 12/88] comments --- .github/workflows/test_cloud_runner.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index f8e04758..d8533602 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -34,9 +34,9 @@ jobs: echo "did it do stuff?" echo "menial change ahhh " pwd - ls fre-cli - cd fre-cli - git branch + ls + #cd fre-cli + #git branch exit 1 - name: Run ppp-container run: | From 6eacb3cfc5c250ffac1eef2f5c577dc53100477f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 15:33:39 -0400 Subject: [PATCH 13/88] check out current repo so existing files still present --- .github/workflows/test_cloud_runner.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index d8533602..d85f39fb 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -23,6 +23,8 @@ jobs: test-cloud: runs-on: ubuntu-latest steps: + - name: Checkout current repository + uses: actions/checkout@v4 - name: Test workflow_dispatch uses: actions/checkout@v4 with: From 07564948426999dcbb0e1750d169187385db3fa3 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 15:40:47 -0400 Subject: [PATCH 14/88] add path --- .github/workflows/test_cloud_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index d85f39fb..b46e75b8 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -30,7 +30,7 @@ jobs: with: repository: ${{ github.event.inputs.fre-cli-repo }} ref: ${{ github.event.inputs.fre-cli-branch }} - #path: '/contrib/container-test' + path: './fre-cli-test/' - name: check run: | echo "did it do stuff?" From 1e54cda9687aa7a9dc2b58abcf0b681cfb03bf3c Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 15:42:23 -0400 Subject: [PATCH 15/88] see if checking out branch was done correctly --- .github/workflows/test_cloud_runner.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index b46e75b8..190ecd0b 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -37,8 +37,8 @@ jobs: echo "menial change ahhh " pwd ls - #cd fre-cli - #git branch + cd fre-cli-test + git branch exit 1 - name: Run ppp-container run: | From ca6fb5f84c905f618038c510be352341fb9c7f5f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 15:44:08 -0400 Subject: [PATCH 16/88] ls --- .github/workflows/test_cloud_runner.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 190ecd0b..2d1e1a00 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -36,8 +36,9 @@ jobs: echo "did it do stuff?" echo "menial change ahhh " pwd - ls + #ls cd fre-cli-test + ls git branch exit 1 - name: Run ppp-container From ac3cbe72a5a7c35a6390d2b600c4146db5b208e8 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 16:00:54 -0400 Subject: [PATCH 17/88] update runner and runscript for addition of runner --- .github/workflows/test_cloud_runner.yml | 13 ++++++++----- for_gh_runner/runscript.sh | 5 +++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 2d1e1a00..48735d89 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -21,7 +21,7 @@ concurrency: jobs: test-cloud: - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Checkout current repository uses: actions/checkout@v4 @@ -30,15 +30,18 @@ jobs: with: repository: ${{ github.event.inputs.fre-cli-repo }} ref: ${{ github.event.inputs.fre-cli-branch }} - path: './fre-cli-test/' + path: '/contrib/container-test/fre-cli-test/' - name: check run: | echo "did it do stuff?" - echo "menial change ahhh " + echo "current dir" pwd - #ls - cd fre-cli-test + #ls + echo "go into fre-cli" + cd /contrib/container-test/fre-cli-test/ + echo "contents of fre-cli" ls + ls fre/mkmf git branch exit 1 - name: Run ppp-container diff --git a/for_gh_runner/runscript.sh b/for_gh_runner/runscript.sh index 1d76943d..e71a82ab 100755 --- a/for_gh_runner/runscript.sh +++ b/for_gh_runner/runscript.sh @@ -25,6 +25,11 @@ conda activate /app/cylc-flow-tools #update fre-cli env? #pip install --upgrade fre-cli #conda env update -f ./for_gh_runner/cylc-flow-tools.yaml +cd /contrib/container-test/fre-cli-test/fre-cli +pip install . +export PATH=/mnt/.local/bin:$PATH +cd - +fre app regrid --help get_user_input () { # User input From 8a36fe7a1036c4b9e13188cbceb6bb32e595fd12 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 16:06:07 -0400 Subject: [PATCH 18/88] fix path --- .github/workflows/test_cloud_runner.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 48735d89..ff7a8043 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -30,7 +30,7 @@ jobs: with: repository: ${{ github.event.inputs.fre-cli-repo }} ref: ${{ github.event.inputs.fre-cli-branch }} - path: '/contrib/container-test/fre-cli-test/' + path: '/contrib/container-test' - name: check run: | echo "did it do stuff?" @@ -38,7 +38,7 @@ jobs: pwd #ls echo "go into fre-cli" - cd /contrib/container-test/fre-cli-test/ + cd /contrib/container-test/fre-cli echo "contents of fre-cli" ls ls fre/mkmf From 75ffaa08bf249ecf98f15a14381a3c0a08b758ff Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 16:09:15 -0400 Subject: [PATCH 19/88] fix path again --- .github/workflows/test_cloud_runner.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index ff7a8043..4e1d30b5 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -30,7 +30,7 @@ jobs: with: repository: ${{ github.event.inputs.fre-cli-repo }} ref: ${{ github.event.inputs.fre-cli-branch }} - path: '/contrib/container-test' + path: './' - name: check run: | echo "did it do stuff?" @@ -38,7 +38,7 @@ jobs: pwd #ls echo "go into fre-cli" - cd /contrib/container-test/fre-cli + cd fre-cli echo "contents of fre-cli" ls ls fre/mkmf From 717e1435b24c16e6751b36df6163df7dfb257a5d Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 16:11:07 -0400 Subject: [PATCH 20/88] update --- .github/workflows/test_cloud_runner.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 4e1d30b5..079cb21c 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -36,7 +36,8 @@ jobs: echo "did it do stuff?" echo "current dir" pwd - #ls + echo "contents of current fre-wf dir" + ls echo "go into fre-cli" cd fre-cli echo "contents of fre-cli" From 8c7d2a42f0897da51f88dae02846faa3cfd0a307 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 16:12:31 -0400 Subject: [PATCH 21/88] update path --- .github/workflows/test_cloud_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 079cb21c..e1e76d7e 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -30,7 +30,7 @@ jobs: with: repository: ${{ github.event.inputs.fre-cli-repo }} ref: ${{ github.event.inputs.fre-cli-branch }} - path: './' + path: './fre-cli' - name: check run: | echo "did it do stuff?" From 0b72ce9817ea51d225c678f4df768e24d9204eff Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 16:15:57 -0400 Subject: [PATCH 22/88] remove check, run workflow --- .github/workflows/test_cloud_runner.yml | 28 ++++++++++++------------- for_gh_runner/runscript.sh | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index e1e76d7e..ca8af3a3 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -31,20 +31,20 @@ jobs: repository: ${{ github.event.inputs.fre-cli-repo }} ref: ${{ github.event.inputs.fre-cli-branch }} path: './fre-cli' - - name: check - run: | - echo "did it do stuff?" - echo "current dir" - pwd - echo "contents of current fre-wf dir" - ls - echo "go into fre-cli" - cd fre-cli - echo "contents of fre-cli" - ls - ls fre/mkmf - git branch - exit 1 +# - name: check +# run: | +# echo "did it do stuff?" +# echo "current dir" +# pwd +# echo "contents of current fre-wf dir" +# ls +# echo "go into fre-cli" +# cd fre-cli +# echo "contents of fre-cli" +# ls +# ls fre/mkmf +# git branch +# exit 1 - name: Run ppp-container run: | # Print pwd then run script diff --git a/for_gh_runner/runscript.sh b/for_gh_runner/runscript.sh index e71a82ab..ea137a71 100755 --- a/for_gh_runner/runscript.sh +++ b/for_gh_runner/runscript.sh @@ -25,7 +25,7 @@ conda activate /app/cylc-flow-tools #update fre-cli env? #pip install --upgrade fre-cli #conda env update -f ./for_gh_runner/cylc-flow-tools.yaml -cd /contrib/container-test/fre-cli-test/fre-cli +cd fre-cli pip install . export PATH=/mnt/.local/bin:$PATH cd - From 05d75d9f4f0553a670b46037d01639a7d04bc86a Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 16:27:39 -0400 Subject: [PATCH 23/88] add --- .github/workflows/test_cloud_runner.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index ca8af3a3..a8b69067 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -31,6 +31,7 @@ jobs: repository: ${{ github.event.inputs.fre-cli-repo }} ref: ${{ github.event.inputs.fre-cli-branch }} path: './fre-cli' + submodules: true # - name: check # run: | # echo "did it do stuff?" From b1a8e616477c782e2e92200d4ce8a44122f65691 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 16:31:42 -0400 Subject: [PATCH 24/88] remove do_timeavgs --- for_gh_runner/yaml_workflow/settings.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/for_gh_runner/yaml_workflow/settings.yaml b/for_gh_runner/yaml_workflow/settings.yaml index 57da79e6..fc86ebe3 100644 --- a/for_gh_runner/yaml_workflow/settings.yaml +++ b/for_gh_runner/yaml_workflow/settings.yaml @@ -14,7 +14,6 @@ postprocess: pp_stop: *EXP_AMIP_END pp_grid_spec: *GRID_SPEC96 switches: - do_timeavgs: True clean_work: True do_refinediag: False do_atmos_plevel_masking: False From 9d636ae9cd49ee004bb4b2cdd5ac3fb71f514159 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 17 Sep 2025 16:55:50 -0400 Subject: [PATCH 25/88] revert changes --- for_gh_runner/yaml_workflow/settings.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/for_gh_runner/yaml_workflow/settings.yaml b/for_gh_runner/yaml_workflow/settings.yaml index fc86ebe3..57da79e6 100644 --- a/for_gh_runner/yaml_workflow/settings.yaml +++ b/for_gh_runner/yaml_workflow/settings.yaml @@ -14,6 +14,7 @@ postprocess: pp_stop: *EXP_AMIP_END pp_grid_spec: *GRID_SPEC96 switches: + do_timeavgs: True clean_work: True do_refinediag: False do_atmos_plevel_masking: False From 3482de56ee8e51354854f1bc71e99fd3fa9691f7 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Wed, 17 Sep 2025 17:05:31 -0400 Subject: [PATCH 26/88] Update test_cloud_runner.yml --- .github/workflows/test_cloud_runner.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index a8b69067..e3da4223 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -32,20 +32,6 @@ jobs: ref: ${{ github.event.inputs.fre-cli-branch }} path: './fre-cli' submodules: true -# - name: check -# run: | -# echo "did it do stuff?" -# echo "current dir" -# pwd -# echo "contents of current fre-wf dir" -# ls -# echo "go into fre-cli" -# cd fre-cli -# echo "contents of fre-cli" -# ls -# ls fre/mkmf -# git branch -# exit 1 - name: Run ppp-container run: | # Print pwd then run script From 56f39bc7ed31973b479626b50b0a299e82734aed Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Tue, 21 Oct 2025 10:06:53 -0400 Subject: [PATCH 27/88] Update bind paths in test_cloud_runner.yml to my own --- .github/workflows/test_cloud_runner.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 78604ed5..770f6839 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -59,10 +59,10 @@ jobs: cwd=$(pwd) echo $cwd gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar - bind_paths="--bind /contrib/container-test/ppp-setup/:/mnt:rw - --bind /contrib/container-test/history:/mnt/history:ro - --bind /contrib/container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro - --bind /contrib/container-test" + bind_paths="--bind /contrib/dana-container-test/ppp-setup/:/mnt:rw + --bind /contrib2/c96L65/history:/mnt/history:ro + --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro + --bind /contrib/dana-container-test" container_path=/contrib/container-test/ppp-wf.sif runscript_path=${cwd}/for_gh_runner/runscript.sh From 2b3de9ae131193d14c71c935cfa53034f1c8e0cf Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Tue, 21 Oct 2025 10:13:19 -0400 Subject: [PATCH 28/88] Create ppp-setup directory in pipeline Updated bind paths to use dynamic output directory. --- .github/workflows/test_cloud_runner.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 770f6839..d2cb6095 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -58,8 +58,20 @@ jobs: echo "Currently in:" cwd=$(pwd) echo $cwd + + # Set up output directory based on which runner picks up the job + # This is to ensure there are no race conditions when accessing + # the setup/output folder mounted to /mnt if there are multiple + # runners. + runner_folder=$(echo $cwd | cut -d/ -f3) + # workflow runner number + wfrunner=${runner_folder: -1} + output_dir=/contrib/dana-container-test/ppp-setup-$wfrunner + mkdir -p $output_dir + # Set up binds and required paths + gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar - bind_paths="--bind /contrib/dana-container-test/ppp-setup/:/mnt:rw + bind_paths="--bind $output_dir/:/mnt:rw --bind /contrib2/c96L65/history:/mnt/history:ro --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro --bind /contrib/dana-container-test" From f51e863e81300247a788a51421d9efa76d23098b Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Tue, 21 Oct 2025 10:44:17 -0400 Subject: [PATCH 29/88] Check I'm where I want to be --- for_gh_runner/runscript.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/for_gh_runner/runscript.sh b/for_gh_runner/runscript.sh index fcc12e3d..eb112b5b 100755 --- a/for_gh_runner/runscript.sh +++ b/for_gh_runner/runscript.sh @@ -24,6 +24,7 @@ conda activate /app/cylc-flow-tools # update fre-cli env with specific branch development cd fre-cli +pwd pip install . export PATH=/mnt/.local/bin:$PATH cd - From 3d8588179716511b64b812c89495b4be510a7aa4 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Tue, 21 Oct 2025 11:12:49 -0400 Subject: [PATCH 30/88] Update path --- .github/workflows/test_cloud_runner.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index d2cb6095..8aac54d3 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -66,10 +66,11 @@ jobs: runner_folder=$(echo $cwd | cut -d/ -f3) # workflow runner number wfrunner=${runner_folder: -1} - output_dir=/contrib/dana-container-test/ppp-setup-$wfrunner + + output_dir=/contrib/dana-container-test/ppp-setup-$runner_folder mkdir -p $output_dir - # Set up binds and required paths + # Set up binds and required paths gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar bind_paths="--bind $output_dir/:/mnt:rw --bind /contrib2/c96L65/history:/mnt/history:ro @@ -89,7 +90,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: fre-workflow-logs - path: /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log + path: /contrib/dana-container-test/ppp-setup*/cylc-run/test_pp__ptest__ttest/log if-no-files-found: error - name: Print pp-starter successes or failures From 6f1d8bdafc78daeb50097fd4f29bfbf952839fde Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Wed, 22 Oct 2025 12:22:57 -0400 Subject: [PATCH 31/88] Update paths in test_cloud_runner - update path to history files on cloud (accessing files in bucket is more efficient in workflow) - update output_dir name --- .github/workflows/test_cloud_runner.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 587b6804..63982a8a 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -48,16 +48,13 @@ jobs: # runners. runner_folder=$(echo $cwd | cut -d/ -f3) - # workflow runner number - wfrunner=${runner_folder: -1} - - output_dir=/contrib/container-test/ppp-setup-$wfrunner + output_dir=/contrib/container-test/ppp-setup-$runner_folder mkdir -p $output_dir # Set up binds and required paths gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar bind_paths="--bind $output_dir/:/mnt:rw - --bind /contrib/container-test/history:/mnt/history:ro + --bind /contrib2/c96L65/history:/mnt/history:ro --bind /contrib/container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro --bind /contrib/container-test" From d28cbb2529c43e8029f6f1bfa32426541e6ee33a Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Wed, 22 Oct 2025 12:31:56 -0400 Subject: [PATCH 32/88] Fix ppp-setup paths for each task using environment variable Updated paths to use OUTPUT_DIR environment variable for log file access --- .github/workflows/test_cloud_runner.yml | 41 +++++++++++++------------ 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 63982a8a..fc609c10 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -51,6 +51,9 @@ jobs: output_dir=/contrib/container-test/ppp-setup-$runner_folder mkdir -p $output_dir + # Write environment variable to GITHUB_ENV to make it accessible for other steps + echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV + # Set up binds and required paths gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar bind_paths="--bind $output_dir/:/mnt:rw @@ -71,18 +74,18 @@ jobs: uses: actions/upload-artifact@v4 with: name: fre-workflow-logs - path: /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log + path: ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log if-no-files-found: error - name: Print pp-starter successes or failures run: | set +e # Successes - success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/pp-starter/0[1-3]/job.status) + success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/pp-starter/0[1-3]/job.status) exit_status_1=$? # Failures - failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/pp-starter/0[1-3]/job.status) + failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/pp-starter/0[1-3]/job.status) exit_status_2=$? if [ $exit_status_1 -eq 0 ]; then @@ -109,11 +112,11 @@ jobs: run: | set +e # Successes - success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history/0[1-3]/job.status) + success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history/0[1-3]/job.status) exit_status_1=$? # Failures - failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history/0[1-3]/job.status) + failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history/0[1-3]/job.status) exit_status_2=$? if [ $exit_status_1 -eq 0 ]; then @@ -141,11 +144,11 @@ jobs: run: | set +e # Successes - success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/regrid-xy*/0[1-3]/job.status) + success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/regrid-xy*/0[1-3]/job.status) exit_status_1=$? # Failures - failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/regrid-xy*/0[1-3]/job.status) + failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/regrid-xy*/0[1-3]/job.status) exit_status_2=$? if [ $exit_status_1 -eq 0 ]; then @@ -172,11 +175,11 @@ jobs: run: | set +e # Successes - success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/remap-pp-components*/0[1-3]/job.status) + success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/remap-pp-components*/0[1-3]/job.status) exit_status_1=$? # Failures - failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/remap-pp-components*/0[1-3]/job.status) + failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/remap-pp-components*/0[1-3]/job.status) exit_status_2=$? if [ $exit_status_1 -eq 0 ]; then @@ -203,11 +206,11 @@ jobs: run: | set +e # Successes - success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/make-timeavgs*/0[1-3]/job.status) + success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/make-timeavgs*/0[1-3]/job.status) exit_status_1=$? # Failures - failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/make-timeavgs*/0[1-3]/job.status) + failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/make-timeavgs*/0[1-3]/job.status) exit_status_2=$? if [ $exit_status_1 -eq 0 ]; then @@ -234,11 +237,11 @@ jobs: run: | set +e # Successes - success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/rename-split-to-pp*/0[1-3]/job.status) + success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/rename-split-to-pp*/0[1-3]/job.status) exit_status_1=$? # Failures - failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/rename-split-to-pp*/0[1-3]/job.status) + failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/rename-split-to-pp*/0[1-3]/job.status) exit_status_2=$? if [ $exit_status_1 -eq 0 ]; then @@ -265,11 +268,11 @@ jobs: run: | set +e # Successes - success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/split-netcdf*/0[1-3]/job.status) + success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/split-netcdf*/0[1-3]/job.status) exit_status_1=$? # Failures - failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/split-netcdf*/0[1-3]/job.status) + failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/split-netcdf*/0[1-3]/job.status) exit_status_2=$? if [ $exit_status_1 -eq 0 ]; then @@ -296,11 +299,11 @@ jobs: run: | set +e # Successes - success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/combine-timeavgs*/0[1-3]/job.status) + success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/combine-timeavgs*/0[1-3]/job.status) exit_status_1=$? # Failures - failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/combine-timeavgs*/0[1-3]/job.status) + failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/combine-timeavgs*/0[1-3]/job.status) exit_status_2=$? if [ $exit_status_1 -eq 0 ]; then @@ -327,11 +330,11 @@ jobs: run: | set +e # Successes - success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/clean*/0[1-3]/job.status) + success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/clean*/0[1-3]/job.status) exit_status_1=$? # Failures - failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/clean*/0[1-3]/job.status) + failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/clean*/0[1-3]/job.status) exit_status_2=$? if [ $exit_status_1 -eq 0 ]; then From b90cd99d067a96679ea9f7346749c462d4b26d99 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Thu, 23 Oct 2025 01:33:57 -0400 Subject: [PATCH 33/88] Update settings yml path in model yaml --- for_gh_runner/yaml_workflow/model.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/for_gh_runner/yaml_workflow/model.yaml b/for_gh_runner/yaml_workflow/model.yaml index c92b08df..023a74f2 100644 --- a/for_gh_runner/yaml_workflow/model.yaml +++ b/for_gh_runner/yaml_workflow/model.yaml @@ -13,8 +13,8 @@ fre_properties: experiments: - name: "test_pp" + settings: "settings.yaml" pp: - - "settings.yaml" - "pp1.yaml" # analysis: # - "an1.yaml" From f73597d8976622d17f1a64e8911c0f351618dd1e Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 7 Nov 2025 11:31:21 -0500 Subject: [PATCH 34/88] Remove concurrency if we want multiple runners to run simultaneously --- .github/workflows/test_cloud_runner.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 3a497205..d4953277 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -28,9 +28,9 @@ on: - main # triggers pipeline on push to main # cancel running jobs if theres a newer push -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +#concurrency: +# group: ${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true jobs: test-cloud: From 4eff06384c7c49137616a084c11822d7da4b2d62 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 13 Nov 2025 14:56:42 -0500 Subject: [PATCH 35/88] just push anything up for the changes --- .github/workflows/test_cloud_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index d4953277..ef66812a 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -78,7 +78,7 @@ jobs: output_dir=/contrib/container-test/ppp-setup-$runner_folder mkdir -p $output_dir - # Write environment variable to GITHUB_ENV to make it accessible for other steps + ## Write environment variable to GITHUB_ENV to make it accessible for other steps echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV # Set up binds and required paths From a8aad3fb51101822981e5be695c5479e87c754b0 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Thu, 13 Nov 2025 15:10:15 -0500 Subject: [PATCH 36/88] Update output directory and bind paths in workflow --- .github/workflows/test_cloud_runner.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 0b3cdea4..6663bcb1 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -75,18 +75,19 @@ jobs: # the setup/output folder mounted to /mnt if there are multiple # runners. runner_folder=$(echo $cwd | cut -d/ -f3) - output_dir=/contrib/container-test/ppp-setup-$runner_folder + output_dir=/contrib/dana-container-test/ppp-setup-$runner_folder mkdir -p $output_dir ## Write environment variable to GITHUB_ENV to make it accessible for other steps echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV - + echo "OUTPUT_DIR: {output_dir}" + exit 1 # Set up binds and required paths gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar bind_paths="--bind $output_dir/:/mnt:rw - --bind /contrib2/c96L65/history:/mnt/history:ro - --bind /contrib/container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro - --bind /contrib/container-test" + --bind /contrib2/c96L65_TEST/history:/mnt/history:ro + --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro + --bind /contrib/dana-container-test" container_path=/contrib/container-test/ppp-wf.sif runscript_path=${cwd}/for_gh_runner/runscript.sh From 4930cd6506a0c4f8a846555db76e7e8cab092659 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Thu, 13 Nov 2025 15:12:56 -0500 Subject: [PATCH 37/88] Fix output directory variable in workflow --- .github/workflows/test_cloud_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 6663bcb1..1e3fc915 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -80,7 +80,7 @@ jobs: ## Write environment variable to GITHUB_ENV to make it accessible for other steps echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV - echo "OUTPUT_DIR: {output_dir}" + echo "OUTPUT_DIR: $output_dir" exit 1 # Set up binds and required paths gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar From 6b4f71dae0a2e709619b45939fb35eb9b34bee93 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Thu, 13 Nov 2025 15:16:59 -0500 Subject: [PATCH 38/88] Correct formatting in test_cloud_runner.yml Fix indentation and formatting issues in YAML file. --- .github/workflows/test_cloud_runner.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 1e3fc915..425cfc2b 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -75,6 +75,8 @@ jobs: # the setup/output folder mounted to /mnt if there are multiple # runners. runner_folder=$(echo $cwd | cut -d/ -f3) + echo "runner folder: $runner_folder" + output_dir=/contrib/dana-container-test/ppp-setup-$runner_folder mkdir -p $output_dir @@ -82,6 +84,7 @@ jobs: echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV echo "OUTPUT_DIR: $output_dir" exit 1 + # Set up binds and required paths gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar bind_paths="--bind $output_dir/:/mnt:rw From 17bbc2d9a85aeb90c6485c537bbd5f0f3d4ef91d Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Thu, 13 Nov 2025 15:20:12 -0500 Subject: [PATCH 39/88] Fix runner folder extraction in workflow --- .github/workflows/test_cloud_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 425cfc2b..0f0635d0 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -74,7 +74,7 @@ jobs: # This is to ensure there are no race conditions when accessing # the setup/output folder mounted to /mnt if there are multiple # runners. - runner_folder=$(echo $cwd | cut -d/ -f3) + runner_folder=$(echo $cwd | cut -d/ -f4) echo "runner folder: $runner_folder" output_dir=/contrib/dana-container-test/ppp-setup-$runner_folder From 6cfc3dde638574351209e0275e33199a9116c191 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Thu, 13 Nov 2025 15:22:20 -0500 Subject: [PATCH 40/88] Clean up GitHub Actions workflow configuration Removed concurrency settings and adjusted output directory echo. --- .github/workflows/test_cloud_runner.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 0f0635d0..898b917c 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -27,11 +27,6 @@ on: branches: - main # triggers pipeline on push to main -# cancel running jobs if theres a newer push -#concurrency: -# group: ${{ github.workflow }}-${{ github.ref }} -# cancel-in-progress: true - jobs: test-cloud: runs-on: self-hosted @@ -82,8 +77,7 @@ jobs: ## Write environment variable to GITHUB_ENV to make it accessible for other steps echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV - echo "OUTPUT_DIR: $output_dir" - exit 1 + echo "OUTPUT DIR: $output_dir" # Set up binds and required paths gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar From 00f6bc55e8f78059674845ea081fc5a45f15518b Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Thu, 13 Nov 2025 15:26:31 -0500 Subject: [PATCH 41/88] use RUNNER_NAME variable --- .github/workflows/test_cloud_runner.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 898b917c..16e2fe3c 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -69,15 +69,16 @@ jobs: # This is to ensure there are no race conditions when accessing # the setup/output folder mounted to /mnt if there are multiple # runners. - runner_folder=$(echo $cwd | cut -d/ -f4) - echo "runner folder: $runner_folder" +# runner_folder=$(echo $cwd | cut -d/ -f4) + echo "runner folder: $RUNNER_NAME" #github has this available?? - output_dir=/contrib/dana-container-test/ppp-setup-$runner_folder + output_dir=/contrib/dana-container-test/ppp-setup-$RUNNER_NAME mkdir -p $output_dir ## Write environment variable to GITHUB_ENV to make it accessible for other steps echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV echo "OUTPUT DIR: $output_dir" + exit 1 # Set up binds and required paths gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar From b26633a4ac1f1bb71a9c40754add0a258f60e3e9 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Thu, 13 Nov 2025 16:00:13 -0500 Subject: [PATCH 42/88] check if I can get runner name --- .github/workflows/test_cloud_runner.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 16e2fe3c..515bf940 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -69,8 +69,7 @@ jobs: # This is to ensure there are no race conditions when accessing # the setup/output folder mounted to /mnt if there are multiple # runners. -# runner_folder=$(echo $cwd | cut -d/ -f4) - echo "runner folder: $RUNNER_NAME" #github has this available?? + runner_folder=$(echo $cwd | cut -d/ -f4) output_dir=/contrib/dana-container-test/ppp-setup-$RUNNER_NAME mkdir -p $output_dir From eeec8dd6bb5a97580a326448e41900949d24a507 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Thu, 13 Nov 2025 16:02:05 -0500 Subject: [PATCH 43/88] runner_folder giving issues --- .github/workflows/test_cloud_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 515bf940..00e8fe3e 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -71,7 +71,7 @@ jobs: # runners. runner_folder=$(echo $cwd | cut -d/ -f4) - output_dir=/contrib/dana-container-test/ppp-setup-$RUNNER_NAME + output_dir=/contrib/dana-container-test/ppp-setup-$runner_folder mkdir -p $output_dir ## Write environment variable to GITHUB_ENV to make it accessible for other steps From 3a8d23aec597e673ae2126d121b25b9f16f11efd Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Thu, 13 Nov 2025 16:04:22 -0500 Subject: [PATCH 44/88] Remove exit command from test_cloud_runner.yml Removed exit command to allow workflow to continue. --- .github/workflows/test_cloud_runner.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 00e8fe3e..b5418bcb 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -77,7 +77,6 @@ jobs: ## Write environment variable to GITHUB_ENV to make it accessible for other steps echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV echo "OUTPUT DIR: $output_dir" - exit 1 # Set up binds and required paths gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar From 345f43e22d3d9bee25330cbda4c3a5aa845e8f47 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 14 Nov 2025 09:33:11 -0500 Subject: [PATCH 45/88] Update test_cloud_runner.yml --- .github/workflows/test_cloud_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index b5418bcb..6e6eff3a 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -81,7 +81,7 @@ jobs: # Set up binds and required paths gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar bind_paths="--bind $output_dir/:/mnt:rw - --bind /contrib2/c96L65_TEST/history:/mnt/history:ro + --bind /contrib2/c96L65/history:/mnt/history:ro --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro --bind /contrib/dana-container-test" From 6eec07c4870d40319d922b1cdd7f8e9044d35acb Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 14 Nov 2025 09:46:56 -0500 Subject: [PATCH 46/88] Update test_cloud_runner.yml --- .github/workflows/test_cloud_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 6e6eff3a..b5418bcb 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -81,7 +81,7 @@ jobs: # Set up binds and required paths gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar bind_paths="--bind $output_dir/:/mnt:rw - --bind /contrib2/c96L65/history:/mnt/history:ro + --bind /contrib2/c96L65_TEST/history:/mnt/history:ro --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro --bind /contrib/dana-container-test" From 730d8763d792138506925c643e7c0d1408078083 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 14 Nov 2025 09:54:39 -0500 Subject: [PATCH 47/88] Try to associate hist_dir with runner --- .github/workflows/test_cloud_runner.yml | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index b5418bcb..435220c8 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -77,14 +77,25 @@ jobs: ## Write environment variable to GITHUB_ENV to make it accessible for other steps echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV echo "OUTPUT DIR: $output_dir" - - # Set up binds and required paths - gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar - bind_paths="--bind $output_dir/:/mnt:rw - --bind /contrib2/c96L65_TEST/history:/mnt/history:ro - --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro - --bind /contrib/dana-container-test" + # Set up binds and required paths + if "-1" in runner_folder: + echo "runner folder 1" + gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar + bind_paths="--bind $output_dir/:/mnt:rw + --bind /contrib2/c96L65_TEST/history:/mnt/history:ro + --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro + --bind /contrib/dana-container-test" + exit 1 + elif "-2" in runner_folder: + echo "in runner_folder 2" + gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar + bind_paths="--bind $output_dir/:/mnt:rw + --bind /contrib2/c96L65/history:/mnt/history:ro + --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro + --bind /contrib/dana-container-test" + exit 1 + container_path=/contrib/container-test/ppp-wf.sif runscript_path=${cwd}/for_gh_runner/runscript.sh From cbeea22f78830df0f302354f749618b70b929577 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 14 Nov 2025 11:32:29 -0500 Subject: [PATCH 48/88] Update test_cloud_runner.yml with cases for bind paths --- .github/workflows/test_cloud_runner.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 435220c8..69f0143b 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -79,23 +79,27 @@ jobs: echo "OUTPUT DIR: $output_dir" # Set up binds and required paths - if "-1" in runner_folder: +# if "-1" in runner_folder: + case "$runner_folder" in + *"-1") echo "runner folder 1" gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar bind_paths="--bind $output_dir/:/mnt:rw --bind /contrib2/c96L65_TEST/history:/mnt/history:ro --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro --bind /contrib/dana-container-test" - exit 1 - elif "-2" in runner_folder: + exit 1;; +# elif "-2" in runner_folder: + *"-2") echo "in runner_folder 2" gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar bind_paths="--bind $output_dir/:/mnt:rw --bind /contrib2/c96L65/history:/mnt/history:ro --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro --bind /contrib/dana-container-test" - exit 1 - + exit 1;; + esac + container_path=/contrib/container-test/ppp-wf.sif runscript_path=${cwd}/for_gh_runner/runscript.sh From ef216f625feb9c1afde0cde2381d2fd77520c1d9 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 14 Nov 2025 11:41:47 -0500 Subject: [PATCH 49/88] Fix spacing --- .github/workflows/test_cloud_runner.yml | 39 ++++++++++++------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 69f0143b..2a3af7c1 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -77,28 +77,27 @@ jobs: ## Write environment variable to GITHUB_ENV to make it accessible for other steps echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV echo "OUTPUT DIR: $output_dir" - # Set up binds and required paths -# if "-1" in runner_folder: case "$runner_folder" in - *"-1") - echo "runner folder 1" - gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar - bind_paths="--bind $output_dir/:/mnt:rw - --bind /contrib2/c96L65_TEST/history:/mnt/history:ro - --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro - --bind /contrib/dana-container-test" - exit 1;; -# elif "-2" in runner_folder: - *"-2") - echo "in runner_folder 2" - gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar - bind_paths="--bind $output_dir/:/mnt:rw - --bind /contrib2/c96L65/history:/mnt/history:ro - --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro - --bind /contrib/dana-container-test" - exit 1;; - esac + *"-1") + echo "runner folder 1" + gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar + bind_paths="--bind $output_dir/:/mnt:rw + --bind /contrib2/c96L65_TEST/history:/mnt/history:ro + --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro + --bind /contrib/dana-container-test" + exit 1 + ;; + *"-2") + echo "in runner_folder 2" + gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar + bind_paths="--bind $output_dir/:/mnt:rw + --bind /contrib2/c96L65/history:/mnt/history:ro + --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro + --bind /contrib/dana-container-test" + exit 1 + ;; + esac container_path=/contrib/container-test/ppp-wf.sif runscript_path=${cwd}/for_gh_runner/runscript.sh From 9ae8a094e7ab8a9d677ba98830850b273b10200d Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 14 Nov 2025 11:50:01 -0500 Subject: [PATCH 50/88] Check right bind_paths --- .github/workflows/test_cloud_runner.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 2a3af7c1..10564da9 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -86,6 +86,7 @@ jobs: --bind /contrib2/c96L65_TEST/history:/mnt/history:ro --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro --bind /contrib/dana-container-test" + echo $bind_paths exit 1 ;; *"-2") @@ -95,6 +96,7 @@ jobs: --bind /contrib2/c96L65/history:/mnt/history:ro --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro --bind /contrib/dana-container-test" + echo $bind_paths exit 1 ;; esac From d9b2f8801129826b2b58e639031ac1de641fbe66 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 14 Nov 2025 12:07:42 -0500 Subject: [PATCH 51/88] Clean up test_cloud_runner.yml --- .github/workflows/test_cloud_runner.yml | 52 ++++++++++++------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 10564da9..91ed4296 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -62,47 +62,47 @@ jobs: run: | # Print pwd then run script echo "Currently in:" - cwd=$(pwd) - echo $cwd + echo $pwd # Set up output directory based on which runner picks up the job # This is to ensure there are no race conditions when accessing # the setup/output folder mounted to /mnt if there are multiple # runners. - runner_folder=$(echo $cwd | cut -d/ -f4) + runner_folder=$(echo $pwd | cut -d/ -f4) - output_dir=/contrib/dana-container-test/ppp-setup-$runner_folder - mkdir -p $output_dir - - ## Write environment variable to GITHUB_ENV to make it accessible for other steps - echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV - echo "OUTPUT DIR: $output_dir" - # Set up binds and required paths case "$runner_folder" in *"-1") - echo "runner folder 1" - gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar - bind_paths="--bind $output_dir/:/mnt:rw - --bind /contrib2/c96L65_TEST/history:/mnt/history:ro - --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro - --bind /contrib/dana-container-test" - echo $bind_paths - exit 1 + # Create the output directory associated with the runner that picked up the job + output_dir=/contrib/dana-container-test/ppp-setup-$runner_folder + mkdir -p $output_dir + + # Write environment variable to GITHUB_ENV to make it accessible for other steps + echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV + echo "OUTPUT DIR: $output_dir" ;; *"-2") echo "in runner_folder 2" - gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar - bind_paths="--bind $output_dir/:/mnt:rw - --bind /contrib2/c96L65/history:/mnt/history:ro - --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro - --bind /contrib/dana-container-test" - echo $bind_paths - exit 1 + # Create the output directory associated with the runner that picked up the job + output_dir=/contrib/dana-container-test/ppp-setup-$runner_folder + mkdir -p $output_dir + + # Write environment variable to GITHUB_ENV to make it accessible for other steps + echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV + echo "OUTPUT DIR: $output_dir" ;; esac + # Set up binds and required paths + gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar + bind_paths="--bind $output_dir/:/mnt:rw + --bind /contrib2/c96L65_TEST/history:/mnt/history:ro + --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro + --bind /contrib/dana-container-test" + echo $bind_paths + exit 1 + container_path=/contrib/container-test/ppp-wf.sif - runscript_path=${cwd}/for_gh_runner/runscript.sh + runscript_path=${pwd}/for_gh_runner/runscript.sh singularity exec --writable-tmpfs ${bind_paths} ${container_path} ${runscript_path} From 86380faf746e2a72f264f9c19073670087990771 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 14 Nov 2025 12:12:53 -0500 Subject: [PATCH 52/88] Update test_cloud_runner.yml --- .github/workflows/test_cloud_runner.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 91ed4296..296927ee 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -62,13 +62,13 @@ jobs: run: | # Print pwd then run script echo "Currently in:" - echo $pwd + echo ${PWD} # Set up output directory based on which runner picks up the job # This is to ensure there are no race conditions when accessing # the setup/output folder mounted to /mnt if there are multiple # runners. - runner_folder=$(echo $pwd | cut -d/ -f4) + runner_folder=$(echo ${PWD} | cut -d/ -f4) case "$runner_folder" in *"-1") @@ -102,14 +102,10 @@ jobs: exit 1 container_path=/contrib/container-test/ppp-wf.sif - runscript_path=${pwd}/for_gh_runner/runscript.sh + runscript_path=${PWD}/for_gh_runner/runscript.sh singularity exec --writable-tmpfs ${bind_paths} ${container_path} ${runscript_path} - # Can potentially do this to get list of tasks - #task_list=$(cylc list test_pp__ptest__ttest) - #task1=$(echo $task_list | cut -d ' ' -f1) .... - - name: Upload workflow-run log files uses: actions/upload-artifact@v4 with: From fe7daecb7030789b15e5ef047b714e2c89fdff1c Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 14 Nov 2025 12:26:42 -0500 Subject: [PATCH 53/88] Update test_cloud_runner.yml --- .github/workflows/test_cloud_runner.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 296927ee..0f2427b7 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -81,7 +81,6 @@ jobs: echo "OUTPUT DIR: $output_dir" ;; *"-2") - echo "in runner_folder 2" # Create the output directory associated with the runner that picked up the job output_dir=/contrib/dana-container-test/ppp-setup-$runner_folder mkdir -p $output_dir @@ -93,12 +92,14 @@ jobs: esac # Set up binds and required paths + # If we ave mulitple runners - this will need to be in cases as we need different history + # files to access each time. + # when accessing the same location - the workflow kind of hangs and takes a while to run gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar bind_paths="--bind $output_dir/:/mnt:rw --bind /contrib2/c96L65_TEST/history:/mnt/history:ro --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro --bind /contrib/dana-container-test" - echo $bind_paths exit 1 container_path=/contrib/container-test/ppp-wf.sif From 726698c84c22f545f7c2cee1f469d549cb089590 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 14 Nov 2025 13:03:40 -0500 Subject: [PATCH 54/88] Fix spelling --- .github/workflows/test_cloud_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 0f2427b7..98588c67 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -92,7 +92,7 @@ jobs: esac # Set up binds and required paths - # If we ave mulitple runners - this will need to be in cases as we need different history + # If we have mulitple runners - this will need to be in cases as we need different history # files to access each time. # when accessing the same location - the workflow kind of hangs and takes a while to run gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar From 55487bcf38a64638aa136eafc14e304af0f00f2e Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 14 Nov 2025 13:04:09 -0500 Subject: [PATCH 55/88] Clean --- .github/workflows/test_cloud_runner.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 98588c67..b73e8c05 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -100,8 +100,6 @@ jobs: --bind /contrib2/c96L65_TEST/history:/mnt/history:ro --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro --bind /contrib/dana-container-test" - exit 1 - container_path=/contrib/container-test/ppp-wf.sif runscript_path=${PWD}/for_gh_runner/runscript.sh From a494e89dd027702a3e9038796eceb5d5b80dff0f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 14 Nov 2025 13:21:13 -0500 Subject: [PATCH 56/88] Remove uneccessary files --- Dockerfile-ppp | 34 ---------------------------------- cylc-flow-tools.yaml | 16 ---------------- 2 files changed, 50 deletions(-) delete mode 100644 Dockerfile-ppp delete mode 100644 cylc-flow-tools.yaml diff --git a/Dockerfile-ppp b/Dockerfile-ppp deleted file mode 100644 index 4c2d6ac8..00000000 --- a/Dockerfile-ppp +++ /dev/null @@ -1,34 +0,0 @@ -FROM condaforge/mambaforge:24.1.2-0 -LABEL maintainer "Ciheim Brown" - -## Description ## -# condaforge is based on a stripped down ubuntu image. We need some extra bits for frerun + fremake -#### - -# apt installs to /usr/bin/ -RUN apt update \ - && apt -y install uuid-runtime time csh python bc - -# Put runscript in container and make it executable -COPY runscript.sh /exec/runscript.sh -RUN chmod +x /exec/runscript.sh - -# Set up conda environment directory for cylc workflow -RUN mkdir /opt/conda/cylc-flow-tools -COPY cylc-flow-tools.yaml /opt/conda/cylc-flow-tools -#COPY macro.py /tmp - -# Finding + replacing the problematic macro.py file with the corrected version -RUN conda env create --file /opt/conda/cylc-flow-tools/cylc-flow-tools.yaml -p /opt/conda/cylc-flow-tools -# && /bin/bash -c 'export macro=$(find /opt/conda/cylc-flow-tools/lib -wholename */site-packages/metomi/rose) \ -# && mv -f /tmp/macro.py $macro/macro.py' \ -# && git config --global --add safe.directory /mnt2 - -RUN IFS="" \ - && pv=$(python --version) \ - && pv1=$(echo $pv | cut -d ' ' -f2) \ - && conda config --add pkgs_dirs /opt/conda/cylc-flow-tools/lib/python$pv1/site-packages - -RUN conda install urwid==2.* - -ENTRYPOINT ["/bin/bash"] diff --git a/cylc-flow-tools.yaml b/cylc-flow-tools.yaml deleted file mode 100644 index 57dbf87f..00000000 --- a/cylc-flow-tools.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: cylc-flow-tools -channels: - - conda-forge - - NOAA-GFDL -dependencies: - - hsm - - fre-nctools - - nco - - cdo - - metomi-rose - - rsync - - make - - vim - - fre-cli ==2024.01 -# - hdf5 - - fre-python-tools From 374131a7f30c22928ca423f051702b7db1846298 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 14 Nov 2025 13:24:55 -0500 Subject: [PATCH 57/88] Remove extra test script (for own fork) --- .github/workflows/test_env.yml | 37 ---------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 .github/workflows/test_env.yml diff --git a/.github/workflows/test_env.yml b/.github/workflows/test_env.yml deleted file mode 100644 index e0a4443c..00000000 --- a/.github/workflows/test_env.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: create_test_conda_env - -on: [push] - -jobs: - build-linux: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '>=3.9' - - - name: Add conda to system path - run: | - # $CONDA is an env var pointing to root of miniconda dir - echo $CONDA/bin >> $GITHUB_PATH - - - name: Create fre-workflows environment - run: | - # create environment containing all dependencies - # the env cannot be explicitly activated in github CI/CD - echo "MY LOCATION" - pwd - echo "MY LOCATION" - ls - #podman --help - #conda create --name fre-workflows-env - - # add conda env's executables to github's PATH equiv. - #echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH - - - name: clone branch - run: | - #git clone --single-branch --branch TripleP https://gitlab.gfdl.noaa.gov/fre/HPC-ME.git - pwd From 620a63dfef711317972c021dc4602b7bb9915016 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 14 Nov 2025 13:26:03 -0500 Subject: [PATCH 58/88] Fix runscript --- for_gh_runner/runscript.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/for_gh_runner/runscript.sh b/for_gh_runner/runscript.sh index 890779dd..7e4005ce 100755 --- a/for_gh_runner/runscript.sh +++ b/for_gh_runner/runscript.sh @@ -24,11 +24,9 @@ conda activate /app/cylc-flow-tools # update fre-cli env with specific branch development cd fre-cli -pwd pip install . export PATH=/mnt/.local/bin:$PATH cd - -fre app --help get_user_input () { # User input From a6d1f0917e63a3f07adad26320038a82e7a87f1b Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 14 Nov 2025 13:28:54 -0500 Subject: [PATCH 59/88] Make not dana specific --- .github/workflows/test_cloud_runner.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index f4010b61..6bf5be88 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -69,11 +69,14 @@ jobs: # the setup/output folder mounted to /mnt if there are multiple # runners. runner_folder=$(echo ${PWD} | cut -d/ -f4) - + + # If we have mulitple runners - bind paths will also need to be in cases as we need + # different history files to access each time. + # when accessing the same location - the workflow kind of hangs and takes a while to run case "$runner_folder" in *"-1") # Create the output directory associated with the runner that picked up the job - output_dir=/contrib/dana-container-test/ppp-setup-$runner_folder + output_dir=/contrib/container-test/ppp-setup-$runner_folder mkdir -p $output_dir # Write environment variable to GITHUB_ENV to make it accessible for other steps @@ -82,7 +85,7 @@ jobs: ;; *"-2") # Create the output directory associated with the runner that picked up the job - output_dir=/contrib/dana-container-test/ppp-setup-$runner_folder + output_dir=/contrib/container-test/ppp-setup-$runner_folder mkdir -p $output_dir # Write environment variable to GITHUB_ENV to make it accessible for other steps @@ -92,14 +95,11 @@ jobs: esac # Set up binds and required paths - # If we have mulitple runners - this will need to be in cases as we need different history - # files to access each time. - # when accessing the same location - the workflow kind of hangs and takes a while to run gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar bind_paths="--bind $output_dir/:/mnt:rw - --bind /contrib2/c96L65_TEST/history:/mnt/history:ro - --bind /contrib/dana-container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro - --bind /contrib/dana-container-test" + --bind /contrib2/c96L65/history:/mnt/history:ro + --bind /contrib/container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro + --bind /contrib/container-test" container_path=/contrib/container-test/ppp-wf.sif runscript_path=${PWD}/for_gh_runner/runscript.sh From ae7078396339525359296ee34c86ce53385f0eb8 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 14 Nov 2025 13:48:04 -0500 Subject: [PATCH 60/88] Fix cut call --- .github/workflows/test_cloud_runner.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 6bf5be88..75c5b367 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -68,22 +68,13 @@ jobs: # This is to ensure there are no access issues when using # the setup/output folder mounted to /mnt if there are multiple # runners. - runner_folder=$(echo ${PWD} | cut -d/ -f4) + runner_folder=$(echo ${PWD} | cut -d/ -f3) # If we have mulitple runners - bind paths will also need to be in cases as we need # different history files to access each time. # when accessing the same location - the workflow kind of hangs and takes a while to run case "$runner_folder" in - *"-1") - # Create the output directory associated with the runner that picked up the job - output_dir=/contrib/container-test/ppp-setup-$runner_folder - mkdir -p $output_dir - - # Write environment variable to GITHUB_ENV to make it accessible for other steps - echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV - echo "OUTPUT DIR: $output_dir" - ;; - *"-2") + *"-aws2") # Create the output directory associated with the runner that picked up the job output_dir=/contrib/container-test/ppp-setup-$runner_folder mkdir -p $output_dir @@ -92,6 +83,15 @@ jobs: echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV echo "OUTPUT DIR: $output_dir" ;; + #*"-aws") + # # Create the output directory associated with the runner that picked up the job + # output_dir=/contrib/container-test/ppp-setup-$runner_folder + # mkdir -p $output_dir + # + # # Write environment variable to GITHUB_ENV to make it accessible for other steps + # echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV + # echo "OUTPUT DIR: $output_dir" + #;; esac # Set up binds and required paths From 2868d27d01513e8060e79adecce29018d0fadce9 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 14 Nov 2025 13:52:51 -0500 Subject: [PATCH 61/88] Test something --- .github/workflows/test_cloud_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 75c5b367..9866e805 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -96,7 +96,7 @@ jobs: # Set up binds and required paths gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar - bind_paths="--bind $output_dir/:/mnt:rw + bind_paths="--bind /contrib/container-test/ppp-setup/:/mnt:rw --bind /contrib2/c96L65/history:/mnt/history:ro --bind /contrib/container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro --bind /contrib/container-test" From 161a155ea9b25d2f82d731a0f0a09111fffee3c0 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 14 Nov 2025 13:56:28 -0500 Subject: [PATCH 62/88] WHOOPS - fix path name --- .github/workflows/test_cloud_runner.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 9866e805..41c321e1 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -74,7 +74,7 @@ jobs: # different history files to access each time. # when accessing the same location - the workflow kind of hangs and takes a while to run case "$runner_folder" in - *"-aws2") + *"RUNNER2") # Create the output directory associated with the runner that picked up the job output_dir=/contrib/container-test/ppp-setup-$runner_folder mkdir -p $output_dir @@ -96,7 +96,7 @@ jobs: # Set up binds and required paths gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar - bind_paths="--bind /contrib/container-test/ppp-setup/:/mnt:rw + bind_paths="--bind $output_dir/:/mnt:rw --bind /contrib2/c96L65/history:/mnt/history:ro --bind /contrib/container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro --bind /contrib/container-test" From d145dcd8764a1a9b9a0af4e63867a209c98e9d99 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 14 Nov 2025 13:59:28 -0500 Subject: [PATCH 63/88] Update test_cloud_runner.yml --- .github/workflows/test_cloud_runner.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 41c321e1..d8127b84 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -61,8 +61,7 @@ jobs: - name: Run ppp-container run: | # Print pwd then run script - echo "Currently in:" - echo ${PWD} + echo "Currently in: ${PWD}" # Set up output directory based on which runner picks up the job # This is to ensure there are no access issues when using From dfffdb80ca224b0532f967eb53ff3ee99904d1e2 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Tue, 2 Dec 2025 15:39:30 -0500 Subject: [PATCH 64/88] Test symlinking history dir Test symbolic link creation for history dir to be in home location (not bucket) --- .github/workflows/test_cloud_runner.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index d8127b84..71068f20 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -81,6 +81,14 @@ jobs: # Write environment variable to GITHUB_ENV to make it accessible for other steps echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV echo "OUTPUT DIR: $output_dir" + + # Create symbolic link to history/gridspec files in home + historyfile_linkdir=/home/Dana.Singh/workflow-run-$runner_folder + mkdir -p $historyfile_linkdir + + # try to 2 questionable locations first - to see if that matters at all + #ln -s /contrib2/c96L65/history $historyfile_linkdir + ln -s /contrib/container-test/history $historyfile_linkdir ;; #*"-aws") # # Create the output directory associated with the runner that picked up the job @@ -96,7 +104,7 @@ jobs: # Set up binds and required paths gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar bind_paths="--bind $output_dir/:/mnt:rw - --bind /contrib2/c96L65/history:/mnt/history:ro + --bind ${historyfile_linkdir}/history:/mnt/history:ro --bind /contrib/container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro --bind /contrib/container-test" container_path=/contrib/container-test/ppp-wf.sif From f09ad6a3dedd8e35c053f3a356da44cb4f849d66 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Tue, 2 Dec 2025 16:43:50 -0500 Subject: [PATCH 65/88] Change symbolic link to a hard link for history file --- .github/workflows/test_cloud_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 71068f20..a13f11fb 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -88,7 +88,7 @@ jobs: # try to 2 questionable locations first - to see if that matters at all #ln -s /contrib2/c96L65/history $historyfile_linkdir - ln -s /contrib/container-test/history $historyfile_linkdir + ln /contrib/container-test/history $historyfile_linkdir ;; #*"-aws") # # Create the output directory associated with the runner that picked up the job From 92858935f3d87ec6ede27a66d031853d0c4a92b7 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Tue, 2 Dec 2025 16:46:29 -0500 Subject: [PATCH 66/88] Update symlink creation for history files --- .github/workflows/test_cloud_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index a13f11fb..ecd46d8d 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -88,7 +88,7 @@ jobs: # try to 2 questionable locations first - to see if that matters at all #ln -s /contrib2/c96L65/history $historyfile_linkdir - ln /contrib/container-test/history $historyfile_linkdir + ln /contrib/container-test/history/* $historyfile_linkdir ;; #*"-aws") # # Create the output directory associated with the runner that picked up the job From b63d7eb0d4288c510ffb6af5682682032a694616 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Tue, 2 Dec 2025 16:51:20 -0500 Subject: [PATCH 67/88] Update history file linking method in workflow Changed hard link to symbolic link for history file. --- .github/workflows/test_cloud_runner.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index ecd46d8d..51d1a7ba 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -87,8 +87,8 @@ jobs: mkdir -p $historyfile_linkdir # try to 2 questionable locations first - to see if that matters at all - #ln -s /contrib2/c96L65/history $historyfile_linkdir - ln /contrib/container-test/history/* $historyfile_linkdir + ln -s /contrib2/c96L65/history $historyfile_linkdir ## symbolic link + #ln /contrib/container-test/history/* $historyfile_linkdir/history ##hard link ;; #*"-aws") # # Create the output directory associated with the runner that picked up the job From cbfbbce637a2813f492c81e3f20c9560cf5ea288 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Tue, 2 Dec 2025 16:59:55 -0500 Subject: [PATCH 68/88] Remove directories if they exist --- .github/workflows/test_cloud_runner.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 51d1a7ba..f9da0561 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -76,16 +76,18 @@ jobs: *"RUNNER2") # Create the output directory associated with the runner that picked up the job output_dir=/contrib/container-test/ppp-setup-$runner_folder + historyfile_linkdir=/home/Dana.Singh/workflow-run-$runner_folder + + rm -rf $output_dir + rm -rf $historyfile_linkdir + mkdir -p $output_dir + mkdir -p $historyfile_linkdir # Write environment variable to GITHUB_ENV to make it accessible for other steps echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV echo "OUTPUT DIR: $output_dir" - # Create symbolic link to history/gridspec files in home - historyfile_linkdir=/home/Dana.Singh/workflow-run-$runner_folder - mkdir -p $historyfile_linkdir - # try to 2 questionable locations first - to see if that matters at all ln -s /contrib2/c96L65/history $historyfile_linkdir ## symbolic link #ln /contrib/container-test/history/* $historyfile_linkdir/history ##hard link From daf39a69b4545e25bb514f11b7a2919ef6fc7f04 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 4 Dec 2025 15:17:26 -0500 Subject: [PATCH 69/88] Add clean step --- .github/workflows/test_cloud_runner.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index b6666f24..6353bae5 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -434,6 +434,10 @@ jobs: exit 2 fi + - name: Clean up output folder + run: rm -rf ${{ env.OUTPUT_DIR }} + ## will this work/conflict with cylc-run that was installed (with other symlinks) + ## Keep in here for now as a just in case reference ## # - name: Print any succeeded tasks # run: | From f0cb47e27b4f009c22a082743d45b5127a9fcc62 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 4 Dec 2025 15:24:34 -0500 Subject: [PATCH 70/88] Update pipeline --- .github/workflows/test_cloud_runner.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 1780eaae..de4f217b 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -237,11 +237,11 @@ jobs: run: | set +e # Successes - success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/data-catalog-final/0[1-3]/job.status) + success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/data-catalog-final/0[1-3]/job.status) exit_status_1=$? # Failures - failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/data-catalog-final/0[1-3]/job.status) + failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/data-catalog-final/0[1-3]/job.status) exit_status_2=$? if [ $exit_status_1 -eq 0 ]; then From 9e23c263a7348b95616885efd8cf8c8afb988eaa Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 4 Dec 2025 15:37:22 -0500 Subject: [PATCH 71/88] Update runner name --- .github/workflows/test_cloud_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index de4f217b..0622d31b 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -73,7 +73,7 @@ jobs: # different history files to access each time. # when accessing the same location - the workflow kind of hangs and takes a while to run case "$runner_folder" in - *"RUNNER2") + *"RUNNER1") # Create the output directory associated with the runner that picked up the job output_dir=/contrib/container-test/ppp-setup-$runner_folder historyfile_linkdir=/home/Dana.Singh/workflow-run-$runner_folder From abe72f7e8670685e918393b45b8fab441a50fc58 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Dec 2025 12:44:31 -0500 Subject: [PATCH 72/88] Test runner folder name is correct --- .github/workflows/test_cloud_runner.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 0622d31b..8e863209 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -7,7 +7,7 @@ on: required: false type: string fre-workflows-branch: - description: 'If on fork: Input branch you are using on your fork.' + description: 'If using fork: Input branch you are using on your fork.' required: false default: 'main' type: string @@ -68,7 +68,7 @@ jobs: # the setup/output folder mounted to /mnt if there are multiple # runners. runner_folder=$(echo ${PWD} | cut -d/ -f3) - + exit 0 # If we have mulitple runners - bind paths will also need to be in cases as we need # different history files to access each time. # when accessing the same location - the workflow kind of hangs and takes a while to run From feb3fc131ad66df0d874873ec81dc1fcd22a14bc Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Dec 2025 12:50:58 -0500 Subject: [PATCH 73/88] Add echo --- .github/workflows/test_cloud_runner.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 8e863209..2693b52a 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -68,6 +68,7 @@ jobs: # the setup/output folder mounted to /mnt if there are multiple # runners. runner_folder=$(echo ${PWD} | cut -d/ -f3) + echo "runner_folder: ${runner_folder}" exit 0 # If we have mulitple runners - bind paths will also need to be in cases as we need # different history files to access each time. From cfc6eff2742bf1e142c0b77a92a18fedc599b299 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Dec 2025 12:53:06 -0500 Subject: [PATCH 74/88] Update parsing path --- .github/workflows/test_cloud_runner.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 2693b52a..ac6db9e6 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -67,8 +67,8 @@ jobs: # This is to ensure there are no access issues when using # the setup/output folder mounted to /mnt if there are multiple # runners. - runner_folder=$(echo ${PWD} | cut -d/ -f3) - echo "runner_folder: ${runner_folder}" + runner_dirname=$(echo ${PWD} | cut -d/ -f4) + echo "runner_folder: ${runner_dirname}" exit 0 # If we have mulitple runners - bind paths will also need to be in cases as we need # different history files to access each time. From 6402a793d5a5dfa870a6425e021166ffdbd194e5 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Dec 2025 13:23:20 -0500 Subject: [PATCH 75/88] Update script --- .github/workflows/test_cloud_runner.yml | 69 +++++++++++-------------- 1 file changed, 30 insertions(+), 39 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index ac6db9e6..9169e1bc 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -68,47 +68,38 @@ jobs: # the setup/output folder mounted to /mnt if there are multiple # runners. runner_dirname=$(echo ${PWD} | cut -d/ -f4) - echo "runner_folder: ${runner_dirname}" - exit 0 - # If we have mulitple runners - bind paths will also need to be in cases as we need - # different history files to access each time. - # when accessing the same location - the workflow kind of hangs and takes a while to run - case "$runner_folder" in - *"RUNNER1") - # Create the output directory associated with the runner that picked up the job - output_dir=/contrib/container-test/ppp-setup-$runner_folder - historyfile_linkdir=/home/Dana.Singh/workflow-run-$runner_folder - - rm -rf $output_dir - rm -rf $historyfile_linkdir - - mkdir -p $output_dir - mkdir -p $historyfile_linkdir - - # Write environment variable to GITHUB_ENV to make it accessible for other steps - echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV - echo "OUTPUT DIR: $output_dir" - - # try to 2 questionable locations first - to see if that matters at all - ln -s /contrib2/c96L65/history $historyfile_linkdir ## symbolic link - #ln /contrib/container-test/history/* $historyfile_linkdir/history ##hard link - ;; - #*"-aws") - # # Create the output directory associated with the runner that picked up the job - # output_dir=/contrib/container-test/ppp-setup-$runner_folder - # mkdir -p $output_dir - # - # # Write environment variable to GITHUB_ENV to make it accessible for other steps - # echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV - # echo "OUTPUT DIR: $output_dir" - #;; - esac + echo "Runner that picked up the job: ${runner_dirname}" - # Set up binds and required paths + # Define output directory, grid spec file name, and directory where + # data will be linked gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar - bind_paths="--bind $output_dir/:/mnt:rw - --bind ${historyfile_linkdir}/history:/mnt/history:ro - --bind /contrib/container-test/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro + output_dir=/contrib/container-test/ppp-setup-$runner_folder + data_linkdir=/home/Dana.Singh/workflow-run-$runner_folder + + # Remove if paths previously exist + rm -rf $output_dir + rm -rf $data_linkdir + + ## Create data link directory and output directory + #mkdir -p $output_dir + #mkdir -p $historyfile_linkdir + + # try to 2 questionable locations first - to see if that matters at all + ln -s /contrib2/c96L65/history $data_linkdir ## symbolic link + ln -s /contrib/container-test/grid_spec/${gridspec_file} $data_linkdir + + # Write environment variable to GITHUB_ENV to make it accessible for other steps + echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV + echo "OUTPUT DIR: $output_dir" + echo "HISTORY FILE LOCATION: $data_linkdir/history" + echo "GRID SPEC LOCATION: $data_linkdir/grid_spec" + + exit 0 + + # Set up binds and required paths + bind_paths="--bind ${output_dir}/:/mnt:rw + --bind ${data_linkdir}/history:/mnt/history:ro + --bind ${data_linkdir}/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro --bind /contrib/container-test" container_path=/contrib/container-test/ppp-wf.sif runscript_path=${PWD}/for_gh_runner/runscript.sh From 33befd2f0894cd3e8d4dfc3c11947c02b1133637 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Dec 2025 13:26:48 -0500 Subject: [PATCH 76/88] fix variable --- .github/workflows/test_cloud_runner.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 9169e1bc..a5bae011 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -73,8 +73,8 @@ jobs: # Define output directory, grid spec file name, and directory where # data will be linked gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar - output_dir=/contrib/container-test/ppp-setup-$runner_folder - data_linkdir=/home/Dana.Singh/workflow-run-$runner_folder + output_dir=/contrib/container-test/ppp-setup-${runner_folder} + data_linkdir=/home/Dana.Singh/workflow-run-${runner_folder} # Remove if paths previously exist rm -rf $output_dir @@ -85,8 +85,8 @@ jobs: #mkdir -p $historyfile_linkdir # try to 2 questionable locations first - to see if that matters at all - ln -s /contrib2/c96L65/history $data_linkdir ## symbolic link - ln -s /contrib/container-test/grid_spec/${gridspec_file} $data_linkdir + ln -s /contrib2/c96L65/history ${data_linkdir} ## symbolic link + ln -s /contrib/container-test/grid_spec/${gridspec_file} ${data_linkdir} # Write environment variable to GITHUB_ENV to make it accessible for other steps echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV From e89a6ddf726c7464f9e4bea3bdba4ac0c7bd772f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Dec 2025 13:31:19 -0500 Subject: [PATCH 77/88] See what paths are --- .github/workflows/test_cloud_runner.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index a5bae011..9457c6ee 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -75,7 +75,9 @@ jobs: gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar output_dir=/contrib/container-test/ppp-setup-${runner_folder} data_linkdir=/home/Dana.Singh/workflow-run-${runner_folder} - + echo "OUTPUT DIR: $output_dir" + echo "DATA DIR: $data_linkdir" + exit 0 # Remove if paths previously exist rm -rf $output_dir rm -rf $data_linkdir From 742a7f849cafcfa143880ea2f45bce013ad5a5f7 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Dec 2025 13:36:35 -0500 Subject: [PATCH 78/88] Fix variable name --- .github/workflows/test_cloud_runner.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 9457c6ee..8eb8a812 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -67,14 +67,14 @@ jobs: # This is to ensure there are no access issues when using # the setup/output folder mounted to /mnt if there are multiple # runners. - runner_dirname=$(echo ${PWD} | cut -d/ -f4) + runner_foldername=$(echo ${PWD} | cut -d/ -f4) echo "Runner that picked up the job: ${runner_dirname}" # Define output directory, grid spec file name, and directory where # data will be linked gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar - output_dir=/contrib/container-test/ppp-setup-${runner_folder} - data_linkdir=/home/Dana.Singh/workflow-run-${runner_folder} + output_dir=/contrib/container-test/ppp-setup-${runner_foldername} + data_linkdir=/home/Dana.Singh/workflow-run-${runner_foldername} echo "OUTPUT DIR: $output_dir" echo "DATA DIR: $data_linkdir" exit 0 From 47ba6ab5e9f2852f8fec4386d41a5c35acc2f0fd Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Dec 2025 13:44:12 -0500 Subject: [PATCH 79/88] did this work? --- .github/workflows/test_cloud_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 8eb8a812..5962b43c 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -95,7 +95,7 @@ jobs: echo "OUTPUT DIR: $output_dir" echo "HISTORY FILE LOCATION: $data_linkdir/history" echo "GRID SPEC LOCATION: $data_linkdir/grid_spec" - + echo "RUNNER NAME: $RUNNER_NAME" exit 0 # Set up binds and required paths From 91cd2f8490b213c07f7a70618ee35c4bca808908 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Dec 2025 13:47:03 -0500 Subject: [PATCH 80/88] wrong place --- .github/workflows/test_cloud_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 5962b43c..abd3e210 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -77,6 +77,7 @@ jobs: data_linkdir=/home/Dana.Singh/workflow-run-${runner_foldername} echo "OUTPUT DIR: $output_dir" echo "DATA DIR: $data_linkdir" + echo "RUNNER NAME: $RUNNER_NAME" exit 0 # Remove if paths previously exist rm -rf $output_dir @@ -95,7 +96,6 @@ jobs: echo "OUTPUT DIR: $output_dir" echo "HISTORY FILE LOCATION: $data_linkdir/history" echo "GRID SPEC LOCATION: $data_linkdir/grid_spec" - echo "RUNNER NAME: $RUNNER_NAME" exit 0 # Set up binds and required paths From f7057404e9eb15a373970938296712d6918c44f0 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Dec 2025 14:07:24 -0500 Subject: [PATCH 81/88] update --- .github/workflows/test_cloud_runner.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index abd3e210..649925b4 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -67,18 +67,17 @@ jobs: # This is to ensure there are no access issues when using # the setup/output folder mounted to /mnt if there are multiple # runners. - runner_foldername=$(echo ${PWD} | cut -d/ -f4) - echo "Runner that picked up the job: ${runner_dirname}" + #runner_foldername=$(echo ${PWD} | cut -d/ -f4) + echo "Runner that picked up the job: ${RUNNER_NAME}" # Define output directory, grid spec file name, and directory where # data will be linked gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar - output_dir=/contrib/container-test/ppp-setup-${runner_foldername} - data_linkdir=/home/Dana.Singh/workflow-run-${runner_foldername} + output_dir=/contrib/container-test/output-${RUNNER_NAME} + data_linkdir=/home/Dana.Singh/workflow-data-${RUNNER_NAME} echo "OUTPUT DIR: $output_dir" echo "DATA DIR: $data_linkdir" - echo "RUNNER NAME: $RUNNER_NAME" - exit 0 + # Remove if paths previously exist rm -rf $output_dir rm -rf $data_linkdir @@ -87,8 +86,8 @@ jobs: #mkdir -p $output_dir #mkdir -p $historyfile_linkdir - # try to 2 questionable locations first - to see if that matters at all - ln -s /contrib2/c96L65/history ${data_linkdir} ## symbolic link + ## Create symbolic links (for if runners are used at same time) + ln -s /contrib2/c96L65/history ${data_linkdir} ln -s /contrib/container-test/grid_spec/${gridspec_file} ${data_linkdir} # Write environment variable to GITHUB_ENV to make it accessible for other steps From c8c148a279a1f815e49ca7807da9d1309b6c4e77 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Dec 2025 14:29:55 -0500 Subject: [PATCH 82/88] create dirs before trying to use them --- .github/workflows/test_cloud_runner.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 649925b4..20d8b645 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -64,10 +64,8 @@ jobs: echo "Currently in: ${PWD}" # Set up output directory based on which runner picks up the job - # This is to ensure there are no access issues when using - # the setup/output folder mounted to /mnt if there are multiple - # runners. - #runner_foldername=$(echo ${PWD} | cut -d/ -f4) + # This is to ensure there are no accessibility issues when using + # multiple runners. echo "Runner that picked up the job: ${RUNNER_NAME}" # Define output directory, grid spec file name, and directory where @@ -83,12 +81,13 @@ jobs: rm -rf $data_linkdir ## Create data link directory and output directory - #mkdir -p $output_dir - #mkdir -p $historyfile_linkdir + mkdir -p $output_dir + mkdir -p $data_linkdir + mkdir -p $data_linkdir/grid_spec ## Create symbolic links (for if runners are used at same time) ln -s /contrib2/c96L65/history ${data_linkdir} - ln -s /contrib/container-test/grid_spec/${gridspec_file} ${data_linkdir} + ln -s /contrib/container-test/grid_spec/${gridspec_file} ${data_linkdir}/grid_spec # Write environment variable to GITHUB_ENV to make it accessible for other steps echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV From 18143466e2a454785eb4e313a1ea69ceba30bd8c Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Dec 2025 14:34:05 -0500 Subject: [PATCH 83/88] remove exit --- .github/workflows/test_cloud_runner.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 20d8b645..5ecd97a4 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -94,7 +94,6 @@ jobs: echo "OUTPUT DIR: $output_dir" echo "HISTORY FILE LOCATION: $data_linkdir/history" echo "GRID SPEC LOCATION: $data_linkdir/grid_spec" - exit 0 # Set up binds and required paths bind_paths="--bind ${output_dir}/:/mnt:rw From 991f216e9cc53f446020dfca03cd48d5941a8f26 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Dec 2025 15:37:15 -0500 Subject: [PATCH 84/88] use test dir for my testing --- .github/workflows/test_cloud_runner.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 5ecd97a4..b3e3b72f 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -76,9 +76,10 @@ jobs: echo "OUTPUT DIR: $output_dir" echo "DATA DIR: $data_linkdir" - # Remove if paths previously exist - rm -rf $output_dir - rm -rf $data_linkdir + ## Remove if paths previously exist + #rm -rf $output_dir + #rm -rf $data_linkdir/history + #rm -rf $data_linkdir/grid_spec ## Create data link directory and output directory mkdir -p $output_dir @@ -86,7 +87,7 @@ jobs: mkdir -p $data_linkdir/grid_spec ## Create symbolic links (for if runners are used at same time) - ln -s /contrib2/c96L65/history ${data_linkdir} + ln -s /contrib2/c96L65_TEST/history ${data_linkdir} ln -s /contrib/container-test/grid_spec/${gridspec_file} ${data_linkdir}/grid_spec # Write environment variable to GITHUB_ENV to make it accessible for other steps From e6f088ba7674e1c1795ad4dc6f2c55d6cac85dad Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Dec 2025 16:12:41 -0500 Subject: [PATCH 85/88] clean file --- .github/workflows/test_cloud_runner.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index d669f998..ad6e8a98 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -73,8 +73,6 @@ jobs: gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar output_dir=/contrib/container-test/output-${RUNNER_NAME} data_linkdir=/home/Dana.Singh/workflow-data-${RUNNER_NAME} - echo "OUTPUT DIR: $output_dir" - echo "DATA DIR: $data_linkdir" ## Remove if paths previously exist #rm -rf $output_dir @@ -87,14 +85,15 @@ jobs: mkdir -p $data_linkdir/grid_spec ## Create symbolic links (for if runners are used at same time) - ln -s /contrib2/c96L65_TEST/history ${data_linkdir} + ln -s /contrib2/c96L65/history ${data_linkdir} ln -s /contrib/container-test/grid_spec/${gridspec_file} ${data_linkdir}/grid_spec # Write environment variable to GITHUB_ENV to make it accessible for other steps echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV echo "OUTPUT DIR: $output_dir" - echo "HISTORY FILE LOCATION: $data_linkdir/history" - echo "GRID SPEC LOCATION: $data_linkdir/grid_spec" + echo "DATA DIR: $data_linkdir" + echo "HISTORY FILE DIR: $data_linkdir/history" + echo "GRID SPEC DIR: $data_linkdir/grid_spec" # Set up binds and required paths bind_paths="--bind ${output_dir}/:/mnt:rw From 1802e37869a2d84b00693be99fdbefa667fe731f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Dec 2025 16:29:00 -0500 Subject: [PATCH 86/88] try forcing symlink --- .github/workflows/test_cloud_runner.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index ad6e8a98..8dcc728b 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -84,9 +84,9 @@ jobs: mkdir -p $data_linkdir mkdir -p $data_linkdir/grid_spec - ## Create symbolic links (for if runners are used at same time) - ln -s /contrib2/c96L65/history ${data_linkdir} - ln -s /contrib/container-test/grid_spec/${gridspec_file} ${data_linkdir}/grid_spec + ## Create symbolic links for data (for if runners are used at same time) + ln -sf /contrib2/c96L65_TEST/history ${data_linkdir} + ln -sf /contrib/container-test/grid_spec/${gridspec_file} ${data_linkdir}/grid_spec # Write environment variable to GITHUB_ENV to make it accessible for other steps echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV From a5e3e8a5d1a740fd26de25fd357eb3621dfa574c Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 23 Dec 2025 15:28:00 -0500 Subject: [PATCH 87/88] Update comments --- .github/workflows/test_cloud_runner.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 8dcc728b..aa1ede4e 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -74,7 +74,7 @@ jobs: output_dir=/contrib/container-test/output-${RUNNER_NAME} data_linkdir=/home/Dana.Singh/workflow-data-${RUNNER_NAME} - ## Remove if paths previously exist + ## Remove if paths previously exist - this might actually delete them from the host as well #rm -rf $output_dir #rm -rf $data_linkdir/history #rm -rf $data_linkdir/grid_spec @@ -103,6 +103,7 @@ jobs: container_path=/contrib/container-test/ppp-wf.sif runscript_path=${PWD}/for_gh_runner/runscript.sh + # Run the container singularity exec --writable-tmpfs ${bind_paths} ${container_path} ${runscript_path} - name: Upload workflow-run log files @@ -504,9 +505,9 @@ jobs: exit 2 fi + ## I think this will work (without cylc clean) because the OUTPUT_DIR is bind mounted from the host - name: Clean up output folder run: rm -rf ${{ env.OUTPUT_DIR }} - ## will this work/conflict with cylc-run that was installed (with other symlinks) ## Keep in here for now as a just in case reference ## # - name: Print any succeeded tasks From aa41c38d315cf6c864969b69150712fd29129901 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 26 Dec 2025 11:41:19 -0500 Subject: [PATCH 88/88] Clean --- .github/workflows/test_cloud_runner.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index aa1ede4e..b8b900f6 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -84,14 +84,13 @@ jobs: mkdir -p $data_linkdir mkdir -p $data_linkdir/grid_spec - ## Create symbolic links for data (for if runners are used at same time) + ## Create symbolic links for data (for if multiple runners are used at same time) ln -sf /contrib2/c96L65_TEST/history ${data_linkdir} ln -sf /contrib/container-test/grid_spec/${gridspec_file} ${data_linkdir}/grid_spec # Write environment variable to GITHUB_ENV to make it accessible for other steps echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV echo "OUTPUT DIR: $output_dir" - echo "DATA DIR: $data_linkdir" echo "HISTORY FILE DIR: $data_linkdir/history" echo "GRID SPEC DIR: $data_linkdir/grid_spec"