From 231d1792e3ac3c251cc3d2ce9200812a2c9fa336 Mon Sep 17 00:00:00 2001 From: Meagan Herbold Date: Thu, 7 Aug 2025 12:16:53 -0700 Subject: [PATCH 1/9] added omp_num_threads to slurm args for sampler runs only --- picca_bookkeeper/bookkeeper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/picca_bookkeeper/bookkeeper.py b/picca_bookkeeper/bookkeeper.py index 4933117..3874c27 100644 --- a/picca_bookkeeper/bookkeeper.py +++ b/picca_bookkeeper/bookkeeper.py @@ -3055,6 +3055,7 @@ def get_sampler_tasker( "job-name": job_name, "output": str(self.paths.fits_path / f"logs/{job_name}-%j.out"), "error": str(self.paths.fits_path / f"logs/{job_name}-%j.err"), + "OMP_NUM_THREADS": 1 } slurm_header_args = DictUtils.merge_dicts( From 903ab87cf3aa32bcb6a6779d6e60b8fc09271656 Mon Sep 17 00:00:00 2001 From: Meagan Herbold Date: Fri, 8 Aug 2025 13:31:05 -0700 Subject: [PATCH 2/9] added example configs for data release baselines --- .../configs/data/iron_FS_sampler_test.yaml | 40 +++++++++++++++++++ Examples/configs/data/jura_BAO_baseline.yaml | 19 +++++++++ Examples/configs/data/kibo_BAO_baseline.yaml | 21 ++++++++++ Examples/configs/data/loa_BAO_baseline.yaml | 19 +++++++++ Examples/configs/data/loa_FS_baseline.yaml | 28 +++++++++++++ 5 files changed, 127 insertions(+) create mode 100644 Examples/configs/data/iron_FS_sampler_test.yaml create mode 100644 Examples/configs/data/jura_BAO_baseline.yaml create mode 100644 Examples/configs/data/kibo_BAO_baseline.yaml create mode 100644 Examples/configs/data/loa_BAO_baseline.yaml create mode 100644 Examples/configs/data/loa_FS_baseline.yaml diff --git a/Examples/configs/data/iron_FS_sampler_test.yaml b/Examples/configs/data/iron_FS_sampler_test.yaml new file mode 100644 index 0000000..e3353af --- /dev/null +++ b/Examples/configs/data/iron_FS_sampler_test.yaml @@ -0,0 +1,40 @@ +general: + conda environment: /global/homes/m/mherbold/.conda/envs/full_shape_bookkeeper + system: slurm_perlmutter + slurm args: + mail-type: fail + mail-user: herbold.9@osu.edu + defaults file: Iron_FS.yaml +data: + bookkeeper dir: /pscratch/sd/m/mherbold/tests/bookkeeper/full_shape/iron_baseline_test + healpix data: /global/cfs/cdirs/desi/spectro/redux/iron/healpix/ + catalog: /global/cfs/cdirs/desicollab/science/lya/y1-kp6/iron-baseline/catalogs/QSO_cat_iron_main_dark_healpix_zlya-altbal_zwarn_cut_20230918.fits +delta extraction: + calib: 1 + calib region: ciii + use existing: /pscratch/sd/m/mherbold/tests/bookkeeper/full_shape/iron_baseline_test +correlations: + computed distortions: + general: /pscratch/sd/m/mherbold/tests/bookkeeper/full_shape/iron_baseline_test/correlations/results +fits: + extra args: + vega_main: + general: + Polychord: + num_live: 448 + num_repeats: 20 + path: + do_clustering: True + resume: True + boost_posterior: 5 + smooth_covariance: + general: + block-types: auto cross # use to omit B-region + slurm args: + run_vega_mpi: + general: + time: "08:00:00" + nodes: 7 + ntasks-per-node: 64 + sampler environment: /global/homes/m/mherbold/.conda/envs/full_shape_bookkeeper + # need to specify for now, default is: /global/common/software/desi/users/acuceu/stable_vega/activate_vega.sh diff --git a/Examples/configs/data/jura_BAO_baseline.yaml b/Examples/configs/data/jura_BAO_baseline.yaml new file mode 100644 index 0000000..bcb8189 --- /dev/null +++ b/Examples/configs/data/jura_BAO_baseline.yaml @@ -0,0 +1,19 @@ +general: + conda environment: /global/common/software/desi/users/cramireez/conda/envs/picca.marseille + system: slurm_perlmutter + slurm args: + mail-type: fail + mail-user: cramireez@ifae.es + defaults file: Jurav1 +data: + bookkeeper dir: /global/cfs/cdirs/desicollab/science/lya/y3/jura/validation_tests/baseline + healpix data: /dvs_ro/cfs/cdirs/desi/spectro/redux/jura/healpix + catalog: /global/cfs/cdirs/desicollab/science/lya/y3/jura/catalogs/QSO_cat_jura_main_dark_healpix_v1_zwarn_cut.fits +delta extraction: + calib: 1 + calib region: ciii + dla: /global/cfs/cdirs/desicollab/science/lya/y3/jura/catalogs/dlacat-20240618-conf05-nhi203-snr3.fits + bal: /global/cfs/cdirs/desicollab/science/lya/y3/jura/catalogs/QSO_cat_jura_main_dark_healpix_v1-bal.fits + mask file: /global/cfs/cdirs/desicollab/science/lya/y3/jura/deltas/delta-lya-1-0/continuum_fitting_mask.txt +correlations: {} +fits: {} diff --git a/Examples/configs/data/kibo_BAO_baseline.yaml b/Examples/configs/data/kibo_BAO_baseline.yaml new file mode 100644 index 0000000..2480c72 --- /dev/null +++ b/Examples/configs/data/kibo_BAO_baseline.yaml @@ -0,0 +1,21 @@ +general: + conda environment: /global/common/software/desi/users/iprafols/loa_unblind/conda/loa-2-0-2-0 + system: slurm_perlmutter + slurm args: + mail-type: fail + mail-user: cramirez@ifae.es + defaults file: /global/cfs/cdirs/desi/science/lya/y3/loa/validation_tests/3-0-0-0/configs/loa-3-0-0-0.yaml +data: + bookkeeper dir: /global/cfs/cdirs/desi/science/lya/y3/loa/validation_tests/3-0-0-0/variations_delta/only_qso_targets + healpix data: /global/cfs/cdirs/desi/spectro/redux/loa/healpix/ + catalog: /global/cfs/cdirs/desi/science/lya/y3/loa/catalogs/QSO_cat_loa_main_dark_healpix_only_qso_targets_v2.fits +delta extraction: + calib: 1 + calib region: ciii + dla: /global/cfs/cdirs/desi/science/lya/y3/loa/catalogs/loa_gp_and_tmp_or_gp_and_cnn_snr_2_cat_20241217.fits + bal: /global/cfs/cdirs/desi/science/lya/y3/loa/catalogs/QSO_cat_loa_main_dark_healpix_v2-altbal-20241115.fits + mask file: /global/cfs/cdirs/desi/science/lya/y3/loa/deltas/delta-lya-3-0/continuum_fitting_mask.txt +correlations: + computed distortions: + general: /global/cfs/cdirs/desi/science/lya/y3/loa/validation_tests/3-0-0-0/baseline/correlations/results +fits: {} diff --git a/Examples/configs/data/loa_BAO_baseline.yaml b/Examples/configs/data/loa_BAO_baseline.yaml new file mode 100644 index 0000000..fe4bf65 --- /dev/null +++ b/Examples/configs/data/loa_BAO_baseline.yaml @@ -0,0 +1,19 @@ +general: + conda environment: /global/common/software/desi/users/iprafols/loa_unblind/conda/loa-2-0-2-0 + system: slurm_perlmutter + slurm args: + mail-type: fail + mail-user: cramirez@ifae.es + defaults file: /global/cfs/cdirs/desi/science/lya/y3/loa/validation_tests/3-0-0-0/configs/loa-3-0-0-0.yaml +data: + bookkeeper dir: /global/cfs/cdirs/desi/science/lya/y3/loa/validation_tests/3-0-0-0/baseline + healpix data: /global/cfs/cdirs/desi/spectro/redux/loa/healpix/ + catalog: /global/cfs/cdirs/desi/survey/catalogs/DA2/QSO/loa/QSO_cat_loa_main_dark_healpix_v2.fits +delta extraction: + calib: 1 + calib region: ciii + dla: /global/cfs/cdirs/desi/science/lya/y3/loa/catalogs/loa_gp_and_tmp_or_gp_and_cnn_snr_2_cat_20241217.fits + bal: /global/cfs/cdirs/desi/science/lya/y3/loa/catalogs/QSO_cat_loa_main_dark_healpix_v2-altbal-20241115.fits + mask file: /global/cfs/cdirs/desi/science/lya/y3/loa/deltas/delta-lya-3-0/continuum_fitting_mask.txt +correlations: {} +fits: {} diff --git a/Examples/configs/data/loa_FS_baseline.yaml b/Examples/configs/data/loa_FS_baseline.yaml new file mode 100644 index 0000000..ccfac72 --- /dev/null +++ b/Examples/configs/data/loa_FS_baseline.yaml @@ -0,0 +1,28 @@ +general: + conda environment: /global/homes/m/mherbold/.conda/envs/full_shape_bookkeeper + system: slurm_perlmutter + slurm args: + mail-type: fail + mail-user: herbold.9@osu.edu + defaults file: /global/cfs/cdirs/desi/science/lya/y3-fs/validation-tests/v1-0-0-0/configs/loa_fs_v1.yaml + +data: + bookkeeper dir: /global/cfs/cdirs/desi/science/lya/y3-fs/validation-tests/v1-0-0-0/baseline + healpix data: /global/cfs/cdirs/desi/spectro/redux/loa/healpix/ + catalog: /global/cfs/cdirs/desi/survey/catalogs/DA2/QSO/loa/QSO_cat_loa_main_dark_healpix_v2.fits + +delta extraction: + calib: 1 + calib region: ciii + dla: /global/cfs/cdirs/desi/science/lya/y3/loa/catalogs/loa_gp_and_tmp_or_gp_and_cnn_snr_2_cat_20241217.fits + bal: /global/cfs/cdirs/desi/science/lya/y3/loa/catalogs/QSO_cat_loa_main_dark_healpix_v2-altbal-20241115.fits + mask file: /global/cfs/cdirs/desi/science/lya/y3/loa/deltas/delta-lya-3-0/continuum_fitting_mask.txt + +correlations: {} + +fits: + extra args: + smooth_covariance: + general: + block-types: auto cross # us to omit B-region + \ No newline at end of file From 39bd2c0578b4f748cefdaf5c38cb09caa1461c89 Mon Sep 17 00:00:00 2001 From: m-herbold <134715487+m-herbold@users.noreply.github.com> Date: Fri, 15 Aug 2025 10:08:20 -0400 Subject: [PATCH 3/9] Update README.md updated vega sampler instructions --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index 1a81121..4cb864e 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,11 @@ cd vega pip install -e ``` +If using the Vega sampler, install MPI: +```bash +MPICC="cc -shared" pip install --force-reinstall --no-cache-dir --no-binary=mpi4py mpi4py +``` + To get the most up-to-date version of the bookkeper, clone the repo: ```bash git clone https://github.com/cramirezpe/picca_bookkeeper.git picca_bookkeeper @@ -104,3 +109,23 @@ To run on mocks, use as a defulat file ``quickquasars``, ``raw`` or ``True``. T ``` picca_bookkeeper_run_full_analysis \path_to_mock_config.yaml ``` + +## Run the Vega Sampler +To run the sampler, ensure that MPI is properly installed with NERSC specific settings in your conda environment: +```bash +MPICC="cc -shared" pip install --force-reinstall --no-cache-dir --no-binary=mpi4py mpi4py +``` +Note: for the bookkeeper config file, you will need to specify your conda environment again. (Otherwise the bookkeeper will default to sourcing a separate Vega installation.) +```bash +general: + conda environment: /path/to/conda_environment + +... + +fits: + sampler environment: /path/to/conda_environment +``` +Then to run from the terminal (including arguments like ``--skip-sent`` if skipping deltas, etc.): +```bash +picca_bookkeeper_run_sampler /path_to_config.yaml +``` From 8f8b155dad879e17bac689ac1f3a81b01521ddbb Mon Sep 17 00:00:00 2001 From: m-herbold <134715487+m-herbold@users.noreply.github.com> Date: Fri, 15 Aug 2025 10:12:32 -0400 Subject: [PATCH 4/9] Update pyproject.toml updated package version dependencies --- pyproject.toml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d636140..fae17f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,16 +12,16 @@ build-backend = "poetry.core.masonry.api" [tool.poetry.dependencies] python = ">=3.10" -astropy = "^6.1.6" -fitsio = "^1.2.4" -getdist = "^1.5.4" +astropy = "^6.1.7" +fitsio = "^1.2.6" +getdist = "^1.6.4" healpy = "^1.18.0" -matplotlib = "^3.9.2" -numpy = ">=1.22,<2.1" +matplotlib = ">=3.10" +numpy = "^1.24.4" pandas = "^2.2.3" -scipy = "^1.14.1" +scipy = "^1.15.3" tabulate = "^0.9.0" -importlib-resources = "^6.4.5" +importlib-resources = "^6.5.2" [tool.poetry.scripts] picca_bookkeeper_run_delta_extraction = "picca_bookkeeper.scripts.run_delta_extraction:main" From 6ed6eaefe98669aa4e84fe758f9d12ec03ae7f30 Mon Sep 17 00:00:00 2001 From: m-herbold <134715487+m-herbold@users.noreply.github.com> Date: Fri, 15 Aug 2025 10:24:09 -0400 Subject: [PATCH 5/9] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4cb864e..9cab6ce 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Activate new environment, and install a stable version of Picca: ```bash conda activate evn_name -pip install picca --version=9.13.0 +pip install picca==9.13.0 ``` Install Vega: @@ -19,7 +19,7 @@ git clone https://github.com/andreicuceu/vega.git Vega cd vega -pip install -e +pip install -e . ``` If using the Vega sampler, install MPI: From 4f936715e69061c5d8977a83e7a73ee6adb8f52f Mon Sep 17 00:00:00 2001 From: m-herbold <134715487+m-herbold@users.noreply.github.com> Date: Fri, 15 Aug 2025 10:45:33 -0400 Subject: [PATCH 6/9] Update pyproject.toml --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index fae17f5..87ec56b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,8 @@ pandas = "^2.2.3" scipy = "^1.15.3" tabulate = "^0.9.0" importlib-resources = "^6.5.2" +iminuit = ">=2.31.1" +gitpython = ">=3.1.44" [tool.poetry.scripts] picca_bookkeeper_run_delta_extraction = "picca_bookkeeper.scripts.run_delta_extraction:main" From 3d08ab91cf56c2aee4ffd3fa30d7749700abb87e Mon Sep 17 00:00:00 2001 From: Meagan Herbold Date: Fri, 15 Aug 2025 07:52:18 -0700 Subject: [PATCH 7/9] updated example config for sampler --- Examples/configs/data/iron_FS_sampler_test.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Examples/configs/data/iron_FS_sampler_test.yaml b/Examples/configs/data/iron_FS_sampler_test.yaml index e3353af..7b5e607 100644 --- a/Examples/configs/data/iron_FS_sampler_test.yaml +++ b/Examples/configs/data/iron_FS_sampler_test.yaml @@ -21,20 +21,21 @@ fits: vega_main: general: Polychord: - num_live: 448 + num_live: 512 num_repeats: 20 path: do_clustering: True resume: True - boost_posterior: 5 + boost_posterior: 3 smooth_covariance: general: - block-types: auto cross # use to omit B-region + block-types: auto cross slurm args: run_vega_mpi: general: time: "08:00:00" - nodes: 7 + nodes: 6 ntasks-per-node: 64 sampler environment: /global/homes/m/mherbold/.conda/envs/full_shape_bookkeeper - # need to specify for now, default is: /global/common/software/desi/users/acuceu/stable_vega/activate_vega.sh + # need to specify, same as general: conda environment: + # default is currently: /global/common/software/desi/users/acuceu/stable_vega/activate_vega.sh From 700b9c26c21ebd805e8ffa136068d26019a0e61a Mon Sep 17 00:00:00 2001 From: Meagan Herbold Date: Fri, 15 Aug 2025 07:52:53 -0700 Subject: [PATCH 8/9] updated example config for sampler --- Examples/configs/data/iron_FS_sampler_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Examples/configs/data/iron_FS_sampler_test.yaml b/Examples/configs/data/iron_FS_sampler_test.yaml index 7b5e607..08789c2 100644 --- a/Examples/configs/data/iron_FS_sampler_test.yaml +++ b/Examples/configs/data/iron_FS_sampler_test.yaml @@ -30,6 +30,7 @@ fits: smooth_covariance: general: block-types: auto cross + # note: specify 'auto cross' to just run A region (omit B) slurm args: run_vega_mpi: general: From 1bda1b3b8712d8e7e42035f60630b441d6373a44 Mon Sep 17 00:00:00 2001 From: Meagan Herbold Date: Fri, 15 Aug 2025 08:02:34 -0700 Subject: [PATCH 9/9] updated tests --- .../guadalupe_sampler/fits/scripts/run_vega_sampler.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_files/guadalupe_sampler/fits/scripts/run_vega_sampler.sh b/tests/test_files/guadalupe_sampler/fits/scripts/run_vega_sampler.sh index 7c5fc7a..d733480 100644 --- a/tests/test_files/guadalupe_sampler/fits/scripts/run_vega_sampler.sh +++ b/tests/test_files/guadalupe_sampler/fits/scripts/run_vega_sampler.sh @@ -9,6 +9,7 @@ #SBATCH --job-name vega_sampler #SBATCH --output /picca_bookkeeper/tests/test_files/output/results/fits/logs/vega_sampler-%j.out #SBATCH --error /picca_bookkeeper/tests/test_files/output/results/fits/logs/vega_sampler-%j.err +export OMP_NUM_THREADS=1 module load python source /global/common/software/desi/users/acuceu/stable_vega/activate_vega.sh