diff --git a/Examples/configs/data/iron_FS_sampler_test.yaml b/Examples/configs/data/iron_FS_sampler_test.yaml new file mode 100644 index 00000000..08789c22 --- /dev/null +++ b/Examples/configs/data/iron_FS_sampler_test.yaml @@ -0,0 +1,42 @@ +general: + conda environment: /global/homes/m/mherbold/.conda/envs/full_shape_bookkeeper + system: slurm_perlmutter + slurm args: + mail-type: fail + mail-user: herbold.9@osu.edu + defaults file: Iron_FS.yaml +data: + bookkeeper dir: /pscratch/sd/m/mherbold/tests/bookkeeper/full_shape/iron_baseline_test + healpix data: /global/cfs/cdirs/desi/spectro/redux/iron/healpix/ + catalog: /global/cfs/cdirs/desicollab/science/lya/y1-kp6/iron-baseline/catalogs/QSO_cat_iron_main_dark_healpix_zlya-altbal_zwarn_cut_20230918.fits +delta extraction: + calib: 1 + calib region: ciii + use existing: /pscratch/sd/m/mherbold/tests/bookkeeper/full_shape/iron_baseline_test +correlations: + computed distortions: + general: /pscratch/sd/m/mherbold/tests/bookkeeper/full_shape/iron_baseline_test/correlations/results +fits: + extra args: + vega_main: + general: + Polychord: + num_live: 512 + num_repeats: 20 + path: + do_clustering: True + resume: True + boost_posterior: 3 + smooth_covariance: + general: + block-types: auto cross + # note: specify 'auto cross' to just run A region (omit B) + slurm args: + run_vega_mpi: + general: + time: "08:00:00" + nodes: 6 + ntasks-per-node: 64 + sampler environment: /global/homes/m/mherbold/.conda/envs/full_shape_bookkeeper + # need to specify, same as general: conda environment: + # default is currently: /global/common/software/desi/users/acuceu/stable_vega/activate_vega.sh diff --git a/Examples/configs/data/jura_BAO_baseline.yaml b/Examples/configs/data/jura_BAO_baseline.yaml new file mode 100644 index 00000000..bcb81895 --- /dev/null +++ b/Examples/configs/data/jura_BAO_baseline.yaml @@ -0,0 +1,19 @@ +general: + conda environment: /global/common/software/desi/users/cramireez/conda/envs/picca.marseille + system: slurm_perlmutter + slurm args: + mail-type: fail + mail-user: cramireez@ifae.es + defaults file: Jurav1 +data: + bookkeeper dir: /global/cfs/cdirs/desicollab/science/lya/y3/jura/validation_tests/baseline + healpix data: /dvs_ro/cfs/cdirs/desi/spectro/redux/jura/healpix + catalog: /global/cfs/cdirs/desicollab/science/lya/y3/jura/catalogs/QSO_cat_jura_main_dark_healpix_v1_zwarn_cut.fits +delta extraction: + calib: 1 + calib region: ciii + dla: /global/cfs/cdirs/desicollab/science/lya/y3/jura/catalogs/dlacat-20240618-conf05-nhi203-snr3.fits + bal: /global/cfs/cdirs/desicollab/science/lya/y3/jura/catalogs/QSO_cat_jura_main_dark_healpix_v1-bal.fits + mask file: /global/cfs/cdirs/desicollab/science/lya/y3/jura/deltas/delta-lya-1-0/continuum_fitting_mask.txt +correlations: {} +fits: {} diff --git a/Examples/configs/data/kibo_BAO_baseline.yaml b/Examples/configs/data/kibo_BAO_baseline.yaml new file mode 100644 index 00000000..2480c72d --- /dev/null +++ b/Examples/configs/data/kibo_BAO_baseline.yaml @@ -0,0 +1,21 @@ +general: + conda environment: /global/common/software/desi/users/iprafols/loa_unblind/conda/loa-2-0-2-0 + system: slurm_perlmutter + slurm args: + mail-type: fail + mail-user: cramirez@ifae.es + defaults file: /global/cfs/cdirs/desi/science/lya/y3/loa/validation_tests/3-0-0-0/configs/loa-3-0-0-0.yaml +data: + bookkeeper dir: /global/cfs/cdirs/desi/science/lya/y3/loa/validation_tests/3-0-0-0/variations_delta/only_qso_targets + healpix data: /global/cfs/cdirs/desi/spectro/redux/loa/healpix/ + catalog: /global/cfs/cdirs/desi/science/lya/y3/loa/catalogs/QSO_cat_loa_main_dark_healpix_only_qso_targets_v2.fits +delta extraction: + calib: 1 + calib region: ciii + dla: /global/cfs/cdirs/desi/science/lya/y3/loa/catalogs/loa_gp_and_tmp_or_gp_and_cnn_snr_2_cat_20241217.fits + bal: /global/cfs/cdirs/desi/science/lya/y3/loa/catalogs/QSO_cat_loa_main_dark_healpix_v2-altbal-20241115.fits + mask file: /global/cfs/cdirs/desi/science/lya/y3/loa/deltas/delta-lya-3-0/continuum_fitting_mask.txt +correlations: + computed distortions: + general: /global/cfs/cdirs/desi/science/lya/y3/loa/validation_tests/3-0-0-0/baseline/correlations/results +fits: {} diff --git a/Examples/configs/data/loa_BAO_baseline.yaml b/Examples/configs/data/loa_BAO_baseline.yaml new file mode 100644 index 00000000..fe4bf65f --- /dev/null +++ b/Examples/configs/data/loa_BAO_baseline.yaml @@ -0,0 +1,19 @@ +general: + conda environment: /global/common/software/desi/users/iprafols/loa_unblind/conda/loa-2-0-2-0 + system: slurm_perlmutter + slurm args: + mail-type: fail + mail-user: cramirez@ifae.es + defaults file: /global/cfs/cdirs/desi/science/lya/y3/loa/validation_tests/3-0-0-0/configs/loa-3-0-0-0.yaml +data: + bookkeeper dir: /global/cfs/cdirs/desi/science/lya/y3/loa/validation_tests/3-0-0-0/baseline + healpix data: /global/cfs/cdirs/desi/spectro/redux/loa/healpix/ + catalog: /global/cfs/cdirs/desi/survey/catalogs/DA2/QSO/loa/QSO_cat_loa_main_dark_healpix_v2.fits +delta extraction: + calib: 1 + calib region: ciii + dla: /global/cfs/cdirs/desi/science/lya/y3/loa/catalogs/loa_gp_and_tmp_or_gp_and_cnn_snr_2_cat_20241217.fits + bal: /global/cfs/cdirs/desi/science/lya/y3/loa/catalogs/QSO_cat_loa_main_dark_healpix_v2-altbal-20241115.fits + mask file: /global/cfs/cdirs/desi/science/lya/y3/loa/deltas/delta-lya-3-0/continuum_fitting_mask.txt +correlations: {} +fits: {} diff --git a/Examples/configs/data/loa_FS_baseline.yaml b/Examples/configs/data/loa_FS_baseline.yaml new file mode 100644 index 00000000..ccfac727 --- /dev/null +++ b/Examples/configs/data/loa_FS_baseline.yaml @@ -0,0 +1,28 @@ +general: + conda environment: /global/homes/m/mherbold/.conda/envs/full_shape_bookkeeper + system: slurm_perlmutter + slurm args: + mail-type: fail + mail-user: herbold.9@osu.edu + defaults file: /global/cfs/cdirs/desi/science/lya/y3-fs/validation-tests/v1-0-0-0/configs/loa_fs_v1.yaml + +data: + bookkeeper dir: /global/cfs/cdirs/desi/science/lya/y3-fs/validation-tests/v1-0-0-0/baseline + healpix data: /global/cfs/cdirs/desi/spectro/redux/loa/healpix/ + catalog: /global/cfs/cdirs/desi/survey/catalogs/DA2/QSO/loa/QSO_cat_loa_main_dark_healpix_v2.fits + +delta extraction: + calib: 1 + calib region: ciii + dla: /global/cfs/cdirs/desi/science/lya/y3/loa/catalogs/loa_gp_and_tmp_or_gp_and_cnn_snr_2_cat_20241217.fits + bal: /global/cfs/cdirs/desi/science/lya/y3/loa/catalogs/QSO_cat_loa_main_dark_healpix_v2-altbal-20241115.fits + mask file: /global/cfs/cdirs/desi/science/lya/y3/loa/deltas/delta-lya-3-0/continuum_fitting_mask.txt + +correlations: {} + +fits: + extra args: + smooth_covariance: + general: + block-types: auto cross # us to omit B-region + \ No newline at end of file diff --git a/README.md b/README.md index 1a811218..9cab6ce7 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Activate new environment, and install a stable version of Picca: ```bash conda activate evn_name -pip install picca --version=9.13.0 +pip install picca==9.13.0 ``` Install Vega: @@ -19,7 +19,12 @@ git clone https://github.com/andreicuceu/vega.git Vega cd vega -pip install -e +pip install -e . +``` + +If using the Vega sampler, install MPI: +```bash +MPICC="cc -shared" pip install --force-reinstall --no-cache-dir --no-binary=mpi4py mpi4py ``` To get the most up-to-date version of the bookkeper, clone the repo: @@ -104,3 +109,23 @@ To run on mocks, use as a defulat file ``quickquasars``, ``raw`` or ``True``. T ``` picca_bookkeeper_run_full_analysis \path_to_mock_config.yaml ``` + +## Run the Vega Sampler +To run the sampler, ensure that MPI is properly installed with NERSC specific settings in your conda environment: +```bash +MPICC="cc -shared" pip install --force-reinstall --no-cache-dir --no-binary=mpi4py mpi4py +``` +Note: for the bookkeeper config file, you will need to specify your conda environment again. (Otherwise the bookkeeper will default to sourcing a separate Vega installation.) +```bash +general: + conda environment: /path/to/conda_environment + +... + +fits: + sampler environment: /path/to/conda_environment +``` +Then to run from the terminal (including arguments like ``--skip-sent`` if skipping deltas, etc.): +```bash +picca_bookkeeper_run_sampler /path_to_config.yaml +``` diff --git a/picca_bookkeeper/bookkeeper.py b/picca_bookkeeper/bookkeeper.py index 4933117a..3874c277 100644 --- a/picca_bookkeeper/bookkeeper.py +++ b/picca_bookkeeper/bookkeeper.py @@ -3055,6 +3055,7 @@ def get_sampler_tasker( "job-name": job_name, "output": str(self.paths.fits_path / f"logs/{job_name}-%j.out"), "error": str(self.paths.fits_path / f"logs/{job_name}-%j.err"), + "OMP_NUM_THREADS": 1 } slurm_header_args = DictUtils.merge_dicts( diff --git a/pyproject.toml b/pyproject.toml index d6361407..87ec56b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,16 +12,18 @@ build-backend = "poetry.core.masonry.api" [tool.poetry.dependencies] python = ">=3.10" -astropy = "^6.1.6" -fitsio = "^1.2.4" -getdist = "^1.5.4" +astropy = "^6.1.7" +fitsio = "^1.2.6" +getdist = "^1.6.4" healpy = "^1.18.0" -matplotlib = "^3.9.2" -numpy = ">=1.22,<2.1" +matplotlib = ">=3.10" +numpy = "^1.24.4" pandas = "^2.2.3" -scipy = "^1.14.1" +scipy = "^1.15.3" tabulate = "^0.9.0" -importlib-resources = "^6.4.5" +importlib-resources = "^6.5.2" +iminuit = ">=2.31.1" +gitpython = ">=3.1.44" [tool.poetry.scripts] picca_bookkeeper_run_delta_extraction = "picca_bookkeeper.scripts.run_delta_extraction:main" diff --git a/tests/test_files/guadalupe_sampler/fits/scripts/run_vega_sampler.sh b/tests/test_files/guadalupe_sampler/fits/scripts/run_vega_sampler.sh index 7c5fc7ac..d733480b 100644 --- a/tests/test_files/guadalupe_sampler/fits/scripts/run_vega_sampler.sh +++ b/tests/test_files/guadalupe_sampler/fits/scripts/run_vega_sampler.sh @@ -9,6 +9,7 @@ #SBATCH --job-name vega_sampler #SBATCH --output /picca_bookkeeper/tests/test_files/output/results/fits/logs/vega_sampler-%j.out #SBATCH --error /picca_bookkeeper/tests/test_files/output/results/fits/logs/vega_sampler-%j.err +export OMP_NUM_THREADS=1 module load python source /global/common/software/desi/users/acuceu/stable_vega/activate_vega.sh