diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..fcb281f2c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +# See https://pre-commit.com for more information +repos: +- repo: https://github.com/google/yamlfmt + rev: v0.17.2 + hooks: + - id: yamlfmt + #exclude: diff --git a/.readthedocs.yml b/.readthedocs.yml index 5c95d6c8d..0b970c0da 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,7 +12,7 @@ build: - graphviz os: ubuntu-22.04 tools: - python: "3.7" + python: "3.10" # Build documentation in the docs/ directory with Sphinx sphinx: diff --git a/.yamlfmt.yaml b/.yamlfmt.yaml new file mode 100644 index 000000000..2d15a2b66 --- /dev/null +++ b/.yamlfmt.yaml @@ -0,0 +1,6 @@ + +formatter: + indent: 4 + retain_line_breaks: true + scan_folded_as_literal: true + trim_trailing_whitespace: true diff --git a/configs/components/amip/amip.yaml b/configs/components/amip/amip.yaml index b13680bac..cbe7474eb 100644 --- a/configs/components/amip/amip.yaml +++ b/configs/components/amip/amip.yaml @@ -17,7 +17,7 @@ amip: ## clean_command: rm -rf bin; cd src; make clean; cd .. comp_command: rm -rf bin; mkdir bin; cd src; make ; cd .. ; cp bin/amip-forcing.exe ./bin/amip - git-repository: https://gitlab.dkrz.de/ec-earth/amip-forcing.git + git-repository: https://gitlab.dkrz.de/ec-earth/amip-forcing-ece4.git install_bins: bin/amip ## ## Directories @@ -60,16 +60,16 @@ amip: namelist.amip: "${namelist_dir}/namelist.amip" input_in_work: - sst_forcing: "comb_input4MIPs_SSTsAndSeaIce_PAMIP_pdSST_pdSIC_gn.nc" - sic_forcing: "comb_input4MIPs_SSTsAndSeaIce_PAMIP_pdSST_pdSIC_gn.nc" + sst_forcing: "tosbcs_input4MIPs_SSTsAndSeaIce_CMIP_PCMDI-AMIP-1-1-0_gs1x1_187001-201512.nc" + sic_forcing: "siconcbcs_input4MIPs_SSTsAndSeaIce_CMIP_PCMDI-AMIP-1-1-0_gs1x1_187001-201512.nc" input_files: sst_forcing: sst_forcing sic_forcing: sic_forcing input_sources: - sst_forcing: "${input_dir}/comb_input4MIPs_SSTsAndSeaIce_PAMIP_pdSST_pdSIC_gn.nc" - sic_forcing: "${input_dir}/comb_input4MIPs_SSTsAndSeaIce_PAMIP_pdSST_pdSIC_gn.nc" + sst_forcing: "${input_dir}/tosbcs_input4MIPs_SSTsAndSeaIce_CMIP_PCMDI-AMIP-1-1-0_gs1x1_187001-201512.nc" + sic_forcing: "${input_dir}/siconcbcs_input4MIPs_SSTsAndSeaIce_CMIP_PCMDI-AMIP-1-1-0_gs1x1_187001-201512.nc" coupling_fields: "[[amip_fields_send-->FIELD]]": @@ -79,7 +79,7 @@ amip: namelist.amip: NAMAMIP: RunLengthSec: "${runtime_seconds}" - TimeStepSec: "84600" + TimeStepSec: "86400" FixYear: "0" StartYear: "1850" FileListSST: ${input_in_work.sst_forcing} diff --git a/configs/components/fesom/ESM_auto_tripy.yaml b/configs/components/fesom/ESM_auto_tripy.yaml new file mode 100644 index 000000000..93e81810e --- /dev/null +++ b/configs/components/fesom/ESM_auto_tripy.yaml @@ -0,0 +1,385 @@ +#___i1lvl Parameter___ +tripyrun_name: "${expid}" +do_papermill: True +do_parallel: false +input_paths: + - ${outdata_path} +input_names: + - "${output_file}" +append_run_name_to_save_path: False +notebook_dir: ${base_dir}/${expid}/analysis/fesom +figure_dir: ${base_dir}/${expid}/viz/fesom + + +#ref_path: /albedo/work/projects/p_fesom/pscholz/project_TRR181/trr181_tke_ctrl_ck0.1/5/ +#ref_name: $TKE$' + +mesh_path: ${mesh_dir} +save_path: "${base_dir}/${expid}/viz/fesom" + +which_clim: 'phc3' +clim_path: '/work/ab0246/a270092/input/fesom2/hydrography/phc3.0/phc3.0_annual.nc' + +ncol: 2 +box: None + +n_cycl: None +do_allcycl: False +year: [${start_year}, ${next_year}] +depth: None +proj: 'rob' +save_dpi: 150 + + +#_______________________________________________________________________________ +# mesh plots +hmesh: + #___2lvl Parameter___ + proj: 'ortho' + box: [-30,45] + do_mesh: True + do_plt: 'tpc' + do_lsm: 'bluemarble' + #___Variables________ + None: + proj: 'nears' + box: [-30, 45, 25] + nresol: +#_______________________________________________________________________________ +# horizontal plots +hslice: + #___2lvl Parameter___ + depths: [100, 1000] + + #___Variables________ + temp: + #___3lvl Parameter___ + cstr: 'blue2red' + #cstr: 'cmocean.thermal' + salt: + #___3lvl Parameter___ + cstr: 'cmocean.haline' + #sigma2: + ##___3lvl Parameter___ + #cstr: 'grads' + Kv: + #___3lvl Parameter___ + do_rescale: 'log10' + cstr: 'wbgyr' + crange: [1.0e-6, 1.0e-1, 1.0e-3] + #N2: + ##___3lvl Parameter___ + #cstr: 'wbgyr' + ssh: + #___3lvl Parameter___ + depths: None + cstr: 'grads' + cref: 0 + MLD2: + #___3lvl Parameter___ + depths: None + cstr: 'rygbw' + crange: [-3000, 0, -1500] + +#_______________________________________________________________________________ +# northern/southern stereographic plots +hslice_np: + #___2lvl Parameter___ + months: [3, 9] + proj: 'nps' + box: [-180,180,40,90] + #___Variables________ + a_ice: + #___3lvl Parameter___ + cstr: 'wbgyr' + crange: [0,1.0,0.5] + m_ice: + #___3lvl Parameter___ + cstr: 'wbgyr' + crange: [0.0, 5.0, 2.5] + MLD2: + #___3lvl Parameter___ + months: [3, 9] + cstr: 'rygbw' + crange: [-3500, 0, -1700] + +hslice_sp: + #___2lvl Parameter___ + months: [9, 3] + proj: 'sps' + box: [-180,180,-90,-50] + #___Variables________ + a_ice: + #___3lvl Parameter___ + cstr: 'wbgyr' + crange: [0,1.0,0.5] + m_ice: + #___3lvl Parameter___ + cstr: 'wbgyr' + crange: [0.0, 1.5, 0.75] + MLD2: + #___3lvl Parameter___ + cstr: 'rygbw' + crange: [-3500, 0, -1700] + +#_______________________________________________________________________________ +# horizontal temp/salt climatology anomaly +hslice_clim: + #___2lvl Parameter___ + depths: [100, 1000] + #___Variables________ + temp: + #___3lvl Parameter___ + cstr: 'blue2red' + crange: [-4, 4, 0] + salt: + #___3lvl Parameter___ + cstr: 'blue2red' + #crange: [-0.5, 0.5, 0] +#_______________________________________________________________________________ +# hovmöller plots absolute or with respect to reference run for different regions +hovm: + #___2lvl Parameter___ + box_regions: + - mpas_region/North_Atlantic_Ocean.shp + - mpas_region/Southern_Ocean.shp + - ocean_basins/Arctic_Basin.shp + - mpas_region/Greenland_Sea.shp + - mpas_region/Irminger_Sea.shp + - mpas_region/Norwegian_Sea.shp + - mpas_region/Labrador_Sea.shp + - mpas_region/North_Pacific_Ocean.shp + - mpas_region/South_Pacific_Ocean.shp + - mpas_region/Southern_Ocean.shp + - mpas_region/Western_Weddell_Sea.shp + year: [${initial_year}, ${next_year}] + #___Variables________ + temp: + salt: + +#_______________________________________________________________________________ +# hovmöller climatological anomaly for different regions +hovm_clim: + #___2lvl Parameter___ + box_regions: + - mpas_region/North_Atlantic_Ocean.shp + - mpas_region/Southern_Ocean.shp + #- ocean_basins/Arctic_Basin.shp + #- mpas_region/Greenland_Sea.shp + #- mpas_region/Irminger_Sea.shp + #- mpas_region/Norwegian_Sea.shp + #- mpas_region/Labrador_Sea.shp + #- mpas_region/North_Pacific_Ocean.shp + #- mpas_region/South_Pacific_Ocean.shp + #- mpas_region/Southern_Ocean.shp + #- mpas_region/Western_Weddell_Sea.shp + year: [${initial_year}, ${next_year}] + #___Variables________ + temp: + salt: + +#_______________________________________________________________________________ +# plot vertical profile of 3d variables averaged over certain regions +vprofile: + #___2lvl Parameter___ + box_regions: + - ocean_basins/Arctic_Basin.shp + - ocean_basins/Eurasian_Basin.shp + - mpas_region/Canada_Basin.shp + - mpas_region/North_Atlantic_Ocean.shp + - mpas_region/Greenland_Sea.shp + - mpas_region/Irminger_Sea.shp + - mpas_region/Norwegian_Sea.shp + - mpas_region/Labrador_Sea.shp + - mpas_region/North_Pacific_Ocean.shp + - mpas_region/South_Pacific_Ocean.shp + - mpas_region/Southern_Ocean.shp + - mpas_region/Western_Weddell_Sea.shp + ncol: 4 + do_clim: True + #___Variables________ + temp: + salt: + Kv: + #___3lvl Parameter___ + do_rescale: 'log10' + +#_______________________________________________________________________________ +# plot vertical profile of 3d temperature/salinity climatology anomaly averaged +# over certain regions +vprofile_clim: + #___2lvl Parameter___ + box_regions: + - ocean_basins/Arctic_Basin.shp + - ocean_basins/Eurasian_Basin.shp + - mpas_region/Canada_Basin.shp + - mpas_region/North_Atlantic_Ocean.shp + - mpas_region/Greenland_Sea.shp + - mpas_region/Irminger_Sea.shp + - mpas_region/Norwegian_Sea.shp + - mpas_region/Labrador_Sea.shp + - mpas_region/North_Pacific_Ocean.shp + - mpas_region/South_Pacific_Ocean.shp + - mpas_region/Southern_Ocean.shp + - mpas_region/Western_Weddell_Sea.shp + ncol: 4 + #___Variables________ + temp: + salt: + + +#_______________________________________________________________________________ +#plot vertical mean cross sections, arbitrary oriented transect + +transect: + #___2lvl Parameter___ + transects: + - [[ -66, -66],[-67,-55], 'Drake Passage'] + - [[ -30, -30],[-80, 90], '30°W, 80°S-90°N'] + - [[-170, -170],[-80, 90], '170°W, 80°S-90°N'] + #- [[-56.5,-52.1,-49.8,-49.5,-49.0,-47.6,-45.7,-44.0,-42.8,-37.8,-35.2,-30.6,-27.9,-21.0,-12.3,-9.2,-4.0], [ 51.0, 52.7, 53.6, 55.0, 56.5, 59.0, 59.8, 60.4, 60.0, 59.6, 59.2, 58.9, 58.0, 58.0, 57.5,57.0,57.0], 'OSNAP'] + #- [[-56.5,-52.1,-49.8,-49.5,-49.0,-47.6,-45.7,-44.0], [ 51.0, 52.7, 53.6, 55.0, 56.5, 59.0, 59.8, 60.4], 'OSNAP-West'] + #- [[-44.0,-42.8,-37.8,-35.2,-30.6,-27.9,-21.0,-12.3,-9.2,-4.0], [ 60.4, 60.0, 59.6, 59.2, 58.9, 58.0, 58.0, 57.5,57.0,57.0], 'OSNAP-East'] + #___Variables________ + temp: + #___3lvl Parameter___ + cstr: 'cmocean.thermal' + salt: + #___3lvl Parameter___ + cstr: 'cmocean.haline' + Kv: + #___3lvl Parameter___ + cstr: 'wbgyr' + do_rescale: 'log10' + crange: [-1.0e-6, 1.0e-0, 1.0e-3] + #KvN2: + ##___3lvl Parameter___ + #cstr: 'blue2red' + #do_rescale: 'slog10' + #crange: [-1.0e-8, 1.0e-8, 1.0e-12] + +#_______________________________________________________________________________ +# plot vertical clim anomaly mean cross sections, arbitrary oriented transect +transect_clim: + #___2lvl Parameter___ + transects: + - [[ -66, -66],[-67,-55], 'Drake Passage'] + - [[ -30, -30],[-80, 90], '30°W, 80°S-90°N'] + - [[-170, -170],[-80, 90], '170°W, 80°S-90°N'] + #- [[-56.5,-52.1,-49.8,-49.5,-49.0,-47.6,-45.7,-44.0,-42.8,-37.8,-35.2,-30.6,-27.9,-21.0,-12.3,-9.2,-4.0], [ 51.0, 52.7, 53.6, 55.0, 56.5, 59.0, 59.8, 60.4, 60.0, 59.6, 59.2, 58.9, 58.0, 58.0, 57.5,57.0,57.0], 'OSNAP'] + #- [[-56.5,-52.1,-49.8,-49.5,-49.0,-47.6,-45.7,-44.0], [ 51.0, 52.7, 53.6, 55.0, 56.5, 59.0, 59.8, 60.4], 'OSNAP-West'] + #- [[-44.0,-42.8,-37.8,-35.2,-30.6,-27.9,-21.0,-12.3,-9.2,-4.0], [ 60.4, 60.0, 59.6, 59.2, 58.9, 58.0, 58.0, 57.5,57.0,57.0], 'OSNAP-East'] + #___Variables________ + temp: + #___3lvl Parameter___ + cstr: 'blue2red' + crange: [-4, 4, 0] + salt: + #___3lvl Parameter___ + cstr: 'blue2red' + crange: [-0.5, 0.5, 0] + sigma2: + #___3lvl Parameter___ + cstr: 'blue2red' + +#______________________________________________________________________________ +# plot vertical section of volume transport through transect +transect_transp: + #___2lvl Parameter___ + transects: + - [[ -66, -66],[-67,-55], 'Drake Passage'] + #- [[ -30, -30],[-80, 90], '30°W, 80°S-90°N'] + #- [[-170, -170],[-80, 90], '170°W, 80°S-90°N'] + #- [[-56.5,-52.1,-49.8,-49.5,-49.0,-47.6,-45.7,-44.0,-42.8,-37.8,-35.2,-30.6,-27.9,-21.0,-12.3,-9.2,-4.0], [ 51.0, 52.7, 53.6, 55.0, 56.5, 59.0, 59.8, 60.4, 60.0, 59.6, 59.2, 58.9, 58.0, 58.0, 57.5,57.0,57.0], 'OSNAP'] + #- [[-56.5,-52.1,-49.8,-49.5,-49.0,-47.6,-45.7,-44.0], [ 51.0, 52.7, 53.6, 55.0, 56.5, 59.0, 59.8, 60.4], 'OSNAP-West'] + #- [[-44.0,-42.8,-37.8,-35.2,-30.6,-27.9,-21.0,-12.3,-9.2,-4.0], [ 60.4, 60.0, 59.6, 59.2, 58.9, 58.0, 58.0, 57.5,57.0,57.0], 'OSNAP-East'] + plt_contb: False + cstr: 'blue2red' + +#______________________________________________________________________________ +# plot time series of volume transport through transect +transect_transp_t: + #___2lvl Parameter___ + transects: + - [[ -66, -66],[-67,-55], 'Drake Passage'] + #- [[-56.5,-52.1,-49.8,-49.5,-49.0,-47.6,-45.7,-44.0,-42.8,-37.8,-35.2,-30.6,-27.9,-21.0,-12.3,-9.2,-4.0], [ 51.0, 52.7, 53.6, 55.0, 56.5, 59.0, 59.8, 60.4, 60.0, 59.6, 59.2, 58.9, 58.0, 58.0, 57.5,57.0,57.0], 'OSNAP'] + year: [${initial_year}, ${next_year}] #VARIABLE!!!! + do_allcycl: False + do_concat: False + +##_______________________________________________________________________________ +## plot zonally averaged section +#transect_zmean: + #___2lvl Parameter___ + box_regions: + - global + - ocean_basins/Atlantic_Basin.shp + - ocean_basins/Pacific_Basin.shp + - ocean_basins/Indian_Basin.shp + #___Variables________ + temp: + #___3lvl Parameter___ + cstr: 'cmocean.thermal' + salt: + #___3lvl Parameter___ + cstr: 'cmocean.haline' + sigma2: + #___3lvl Parameter___ + cstr: 'cmocean.dense' + Kv: + #___3lvl Parameter___ + do_rescale: 'log10' + cstr: 'wbgyr' + crange: [1.0e-6, 1.0e-0, 1.0e-4] + #N2: + ##___3lvl Parameter___ + #do_rescale: 'log10' + #cstr: 'wbgyr' + #crange: [1.0e-8, 1.0e-4, 1.0e-6] + #KvN2: + ##___3lvl Parameter___ + #do_rescale: 'slog10' + #cstr: 'blue2red' + #crange: [1.0e-8, 1.0e-8, 1.0e-12] + +# plot climatology bias of zonally averaged section +transect_zmean_clim: + #___2lvl Parameter___ + box_regions: + - global + - ocean_basins/Atlantic_Basin.shp + - ocean_basins/Pacific_Basin.shp + - ocean_basins/Indian_Basin.shp + ##___Variables________ + temp: + #___3lvl Parameter___ + crange: [-2.0,2.0,0] + cstr: 'blue2red' + salt: + #___3lvl Parameter___ + crange: [-0.75,0.75,0] + cstr: 'blue2red' + sigma2: + #___3lvl Parameter___ + crange: [-0.35,0.35,0] + cstr: 'blue2red' + +#_______________________________________________________________________________ +# plot meridional overturning streamfunction based on z-coordiante vertical +# velocity for different basins +zmoc: + #___2lvl Parameter___ + + #___Variables________ + gmoc: + amoc: + ipmoc: + +# plot time series of maximum AMOC upper circulation cell +zmoc_t: + #___2lvl Parameter___ + which_lats: [26, 40, 'max'] + year: [${initial_year}, ${next_year}] #VARIABLE!!! + do_allcycl: False + #___Variables________ + amoc: diff --git a/configs/components/fesom/Subjob_tripy.sh b/configs/components/fesom/Subjob_tripy.sh new file mode 100755 index 000000000..66e663a2a --- /dev/null +++ b/configs/components/fesom/Subjob_tripy.sh @@ -0,0 +1,182 @@ +#!/bin/bash -e +############################################################################### +# Automatic run of TriPyView as a post job of esm-runscripts +# Original code by: Simon Porrmann +# Cleanup and testing by: Paul Gierz +# +# March 2025 +# AWI Bremerhaven +############################################################################### + +echo "################################################################################" +echo " _____ _ ___ __ ___ " +echo " |_ _| _(_) _ \_ \ \ / (_)_____ __ __ " +echo " | || '_| | _/ || \ V /| / -_) V V / " +echo " |_||_| |_|_| \_, |\_/ |_\___|\_/\_/ " +echo " |__/ " +echo "################################################################################" +echo "" +echo "$(date):: Starting Tripyview Job" +echo "" + +outdata_path=$1 +start_date=$2 +next_date=$3 +initial_date=$4 +final_date=$5 +mesh_dir=$6 +base_dir=$7 +expid=$8 +conda_module=$9 +vis_sync=${10} +vis_server=${11} +vis_publicdir=${12} + +# Change dates to years +start_year=$(date -d "$start_date" +%Y) +next_year=$(date -d "$next_date" +%Y) +initial_year=$(date -d "$initial_date" +%Y) +final_year=$(date -d "$final_date" +%Y) + +output_file="${expid}_tripyrun_${start_year}.yaml" +target_dir="${base_dir}/${expid}/config/fesom" + +export mesh_dir +export outdata_path +export start_year +export next_year +export initial_year +export final_year +export base_dir +export expid +export output_file +export target_dir + + +echo "$(date):: Configuration Values:" +echo "$(date):: ---------------------" +echo "$(date):: outdata_path=$outdata_path" +echo "$(date):: start_date=$start_date" +echo "$(date):: next_date=$next_date" +echo "$(date):: initial_date=$initial_date" +echo "$(date):: final_date=$final_date" +echo "$(date):: mesh_dir=$mesh_dir" +echo "$(date):: base_dir=$base_dir" +echo "$(date):: expid=$expid" +echo "$(date):: output_file=$output_file" +echo "$(date):: target_dir=$target_dir" +echo "$(date):: conda_module=$conda_module" +echo "$(date):: vis_sync=$vis_sync" +echo "$(date):: vis_server=$vis_server" +echo "$(date):: vis_publicdir=$vis_publicdir" +echo "$(date):: ---------------------" + +function activate_env() { + # https://www.shellcheck.net/wiki/SC1091 + # Disable Not following on the source line + # shellcheck disable=SC1091 + source "$(conda info --base)/etc/profile.d/conda.sh" + conda activate esm-tools_auto_tripyview + echo "$(date):: Now in environment ${CONDA_DEFAULT_ENV} (under ${CONDA_PREFIX})" + echo "$(date):: Prepending PATH (by default, conda activate appends...)" + export PATH="${CONDA_PREFIX}/bin:${PATH}" + echo "$(date):: $(which -a python)" + echo "$(date):: PATH=${PATH}" +} + +function check_conda_availability() { + module load $conda_module + if ! command -v conda &> /dev/null; then + echo "$(date):: Conda is not installed. Please install conda or miniconda" + exit 1 + fi +} + +function activate_or_install_tripyview_environment() { + # https://www.shellcheck.net/wiki/SC1091 + # Disable Not following on the source line + # shellcheck disable=SC1091 + # Überprüfen, ob das Environment "esm-tools_auto_tripyview" existiert + if conda info --envs | grep -q "^esm-tools_auto_tripyview "; then + echo "$(date):: esm-tools_auto_tripyview already exists. Activating now..." + activate_env + else + echo "$(date):: esm-tools_auto_tripyview env DOES NOT exist. Installing..." + source "$(conda info --base)/etc/profile.d/conda.sh" + conda create -y --name esm-tools_auto_tripyview python=3.9 + activate_env + mkdir -p "${base_dir}/${expid}/src" + cd "${base_dir}/${expid}/src" || exit 1 + git clone https://github.com/fesom/tripyview + conda install -c conda-forge libstdcxx-ng + cd tripyview + pip install -e . + fi +} + +function setup_target_directory() { + # Create folder for Tripy configs and results in the experiment folder + if [ ! -d "$target_dir" ]; then + echo "$(date):: Directory $target_dir not existing. Creating..." + mkdir -p "$target_dir" + fi +} + +function prepare_output_file() { + # go to Directory + # working with envsubst (comes from GNU-gettext bib. Installed on OS and most linux distributions). Creating the Tripyview YAML for the current run. + cd "${base_dir}/${expid}/scripts/fesom" || exit 1 + envsubst < ESM_auto_tripy.yaml > "$output_file" + mv -v "$output_file" "$target_dir" + ls -l "${target_dir}/${output_file}" + echo "$(date):: Prepared ${output_file} for tripyrun" +} + +function execute_tripyrun() { + # Run the new created Tripy config + echo "$(date):: Starting tripyrun" + cd "$target_dir" || exit 1 + # Sanity check: + echo "$(date):: $(which python)" + echo "$(date):: $(python -VV)" + python -c "import tripyview; print(tripyview.__file__); print(tripyview.__version__)" + # Go! + tripyrun "$output_file" +} + +function sync_to_vis_server() { + # Convert to lowercase for case-insensitive boolean comparison + if [ "${vis_sync,,}" == "true" ]; then + echo "$(date):: Syncing visualization results to ${USER}@${vis_server}:${vis_publicdir}" + + # Create remote directories if they don't exist + echo "$(date):: Creating remote directories" + ssh "${USER}@${vis_server}" "mkdir -p ${vis_publicdir}/analysis ${vis_publicdir}/viz" + + # Sync analysis and viz directories + echo "$(date):: Syncing analysis directory" + rsync -avzhP "${base_dir}/${expid}/analysis/fesom" "${USER}@${vis_server}:${vis_publicdir}/analysis/" + + echo "$(date):: Syncing viz directory" + rsync -avzhP "${base_dir}/${expid}/viz/fesom" "${USER}@${vis_server}:${vis_publicdir}/viz/" + + echo "$(date):: Sync complete" + else + echo "$(date):: Visualization sync disabled (vis_sync=${vis_sync})" + fi +} + + +function main() { + check_conda_availability + activate_or_install_tripyview_environment + setup_target_directory + prepare_output_file + execute_tripyrun + sync_to_vis_server +} + +main + + + diff --git a/configs/components/fesom/fesom-2.1.yaml b/configs/components/fesom/fesom-2.1.yaml index d8783fa89..4aeea8dbe 100644 --- a/configs/components/fesom/fesom-2.1.yaml +++ b/configs/components/fesom/fesom-2.1.yaml @@ -469,8 +469,6 @@ fesom: bin_sources: fesom: "${bin_dir}/${comp_executable}" - - config_files: config: config forcing: forcing @@ -612,6 +610,7 @@ fesom: further_reading: - fesom/fesom.forcing.yaml + - fesom/fesom.tripyview.yaml # ------------------------ # RECIPE diff --git a/configs/components/fesom/fesom-2.5.yaml b/configs/components/fesom/fesom-2.5.yaml index 7533df54f..d6d4d69ba 100644 --- a/configs/components/fesom/fesom-2.5.yaml +++ b/configs/components/fesom/fesom-2.5.yaml @@ -645,6 +645,7 @@ fesom: further_reading: - fesom/fesom.forcing.yaml - fesom/fesom.env.yaml + - fesom/fesom.tripyview.yaml # ------------------------ # RECIPE @@ -659,4 +660,3 @@ fesom: - "copy_stuff_back_from_work" - "copy_all_results_to_exp" - "clean_run_dir" - diff --git a/configs/components/fesom/fesom-2.6.yaml b/configs/components/fesom/fesom-2.6.yaml index efe10aea2..59fef2c8a 100644 --- a/configs/components/fesom/fesom-2.6.yaml +++ b/configs/components/fesom/fesom-2.6.yaml @@ -1,13 +1,12 @@ # FESOM2 YAML CONFIGURATION FILE # - fesom: model: fesom branch: "2.6" version: "2.6" type: ocean - comp_command: mkdir -p build; cd build; cmake -DCMAKE_INSTALL_PREFIX=../ ..; make install -j `nproc --all` + comp_command: mkdir -p build; cd build; cmake -DENABLE_OPENMP=ON -DCMAKE_INSTALL_PREFIX=../ ..; make install -j `nproc --all` clean_command: ${defaults.clean_command} # MA This variable needs to be set true in every version of fesom-2.1 (choose_ block @@ -26,7 +25,7 @@ fesom: choose_version: "2.6": - branch: "2.6.6" + branch: "2.6.10" "2.6-paleodyn": branch: "2.6.7" @@ -46,6 +45,16 @@ fesom: "2.6.9": branch: "2.6.9" + "2.6.10": + branch: "2.6.10" + + "2.6.11": + branch: "2.6.11" + + "2.6-main": + branch: "main" + namelist_dir: "${model_dir}/config/" + destination: "fesom-2.6" install_bins: bin/fesom.x git-repository: @@ -109,45 +118,104 @@ fesom: - namelist.tra - namelist.transit + whichEVP: 0 # 0=EVP, 1=mEVP, 2=aEVP + # See: Kimmritz et. al 2015 & 2016: https://doi.org/10.1016/j.jcp.2015.04.051, https://doi.org/10.1016/j.ocemod.2016.03.004 choose_resolution: CORE2: nx: 126858 mesh_dir: "${fesom.mesh_base_dir}/core2/" nproc: 288 + whichEVP: 0 + alpha_evp: 250 # constant that control numerical stability of mEVP. + beta_evp: 250 # constant that control numerical stability of mEVP. + CORE3: + nx: 211567 + mesh_dir: "${fesom.mesh_base_dir}/core3/" + nproc: 128 + whichEVP: 1 + alpha_evp: 300 # constant that control numerical stability of mEVP. + beta_evp: 300 # constant that control numerical stability of mEVP. pi: nx: 3140 mesh_dir: "${fesom.mesh_base_dir}/pi/" nproc: 4 + whichEVP: 0 + alpha_evp: 250 # constant that control numerical stability of mEVP. + beta_evp: 250 # constant that control numerical stability of mEVP. LGM: nx: 95239 mesh_dir: "/home/a/a270064/bb1029/inputs/mesh_glac1d_m1_512/" nproc: 1024 + whichEVP: 0 + alpha_evp: 250 # constant that control numerical stability of mEVP. + beta_evp: 250 # constant that control numerical stability of mEVP. GLOB: nx: 830305 + whichEVP: 1 + alpha_evp: 250 # constant that control numerical stability of mEVP. + beta_evp: 250 # constant that control numerical stability of mEVP. CAVCORE2: nx: 72411 mesh_dir: "/work/ollie/pscholz/mesh_fesom2.0/cavity_72k/" nproc: 288 + whichEVP: 0 + alpha_evp: 250 # constant that control numerical stability of mEVP. + beta_evp: 250 # constant that control numerical stability of mEVP. PI_ICEv2: nx: 132273 + whichEVP: 0 + alpha_evp: 250 # constant that control numerical stability of mEVP. + beta_evp: 250 # constant that control numerical stability of mEVP. D3: nx: 3160340 - DARS: - nx: 3160340 - DART: - nx: 3160340 + whichEVP: 1 + alpha_evp: 500 # constant that control numerical stability of mEVP. + beta_evp: 500 # constant that control numerical stability of mEVP. orca25: nx: 912469 + whichEVP: 1 + alpha_evp: 500 # constant that control numerical stability of mEVP. + beta_evp: 500 # constant that control numerical stability of mEVP. + DART: + nx: 3160340 + whichEVP: 1 + alpha_evp: 500 # constant that control numerical stability of mEVP. + beta_evp: 500 # constant that control numerical stability of mEVP. + DARS: + nx: 3160340 + whichEVP: 1 + alpha_evp: 500 # constant that control numerical stability of mEVP. + beta_evp: 500 # constant that control numerical stability of mEVP. + DARS2: + nx: 3146761 + whichEVP: 1 + alpha_evp: 500 # constant that control numerical stability of mEVP. + beta_evp: 500 # constant that control numerical stability of mEVP. ROS2: nx: 26306560 + whichEVP: 1 + alpha_evp: 900 # constant that control numerical stability of mEVP. + beta_evp: 900 # constant that control numerical stability of mEVP. HR: nx: 1306775 + whichEVP: 1 + alpha_evp: 500 # constant that control numerical stability of mEVP. + beta_evp: 500 # constant that control numerical stability of mEVP. jane: nx: 33348172 + whichEVP: 1 + alpha_evp: 900 # constant that control numerical stability of mEVP. + beta_evp: 900 # constant that control numerical stability of mEVP. SO3: nx: 11087062 + whichEVP: 1 + alpha_evp: 800 # constant that control numerical stability of mEVP. + beta_evp: 800 # constant that control numerical stability of mEVP. Arc01_fesom2: nx: 753179 + whichEVP: 1 + alpha_evp: 500 # constant that control numerical stability of mEVP. + beta_evp: 500 # constant that control numerical stability of mEVP. restart_in_files: @@ -311,7 +379,6 @@ fesom: felem: "felem" #num_non_melted_icb_file: "num_non_melted_icb_file" - log_files: clock: clock mesh_diag: mesh_diag @@ -460,6 +527,11 @@ fesom: runoff_data_source: ${runoff_data_source} sss_data_source: ${sss_data_source} + namelist.ice: + ice_dyn: + whichEVP: ${whichEVP} + alpha_evp: ${alpha_evp} + beta_evp: ${beta_evp} use_landice_water: False choose_general.iterative_coupling: @@ -515,8 +587,6 @@ fesom: bin: "${bin_dir}/${comp_executable}" lib: "${fesom.model_dir}/build/lib/libfesom.so" - - config_files: config: config cvmix: cvmix @@ -566,6 +636,8 @@ fesom: grid: feom v_feom: grid: feom + FCO2_feom: + grid: feom taux_oce: grid: feom tauy_oce: @@ -594,6 +666,12 @@ fesom: grid: feom enth_oce: grid: feom + u10w_oce: + grid: feom + v10w_oce: + grid: feom + XCO2_oce: + grid: feom wiso_fields: @@ -636,6 +714,7 @@ fesom: further_reading: - fesom/fesom.forcing.yaml - fesom/fesom.env.yaml + - fesom/fesom.tripyview.yaml # ------------------------ # RECIPE diff --git a/configs/components/fesom/fesom-2.7.yaml b/configs/components/fesom/fesom-2.7.yaml new file mode 100644 index 000000000..359ff823f --- /dev/null +++ b/configs/components/fesom/fesom-2.7.yaml @@ -0,0 +1,719 @@ +# FESOM2 YAML CONFIGURATION FILE +# +fesom: + model: fesom + branch: "2.7.0" + version: "2.7" + type: ocean + + comp_command: mkdir -p build; cd build; cmake -DENABLE_OPENMP=ON -DCMAKE_INSTALL_PREFIX=../ ..; make install -j `nproc --all` + clean_command: ${defaults.clean_command} + + # MA This variable needs to be set true in every version of fesom-2.1 (choose_ block + # below) whose source code contains the icebergs implementation + with_icb: false + use_icebergs: false + use_icesheet_coupling: false + + wiso_code: false + with_wiso: false + with_recom: false + + required_plugins: + - "git+https://github.com/esm-tools-plugins/tar_binary_restarts" + + choose_version: + "2.7": + branch: "2.7.0" + "2.7.2": + branch: "2.7.2" + "2.7.3": + branch: "2.7.3" + "2.7-main": + branch: "main" + #namelist_dir: "${model_dir}/config/" + + destination: "fesom-2.7" + install_bins: bin/fesom.x + git-repository: + - https://github.com/FESOM/fesom2.git + contact: "jan.streffing(at)awi.de, patrick.scholz(at)awi.de miguel.andres-martinez(at)awi.de, paul.gierz(at)awi.de" + + restart_rate: "12" + restart_unit: "m" + restart_first: 12 + restart_flag: "last" + + tar_binary_restarts: True + + time_step: 1800 + resolution: CORE2 + + comp_executable: fesom.x + executable: fesom + + mesh_rotated: false + old_mesh_format: false + with_part_format: false + time_dimension: "time" + + pool_dir: "${computer.pool_directories.pool}" + + choose_computer.name: + levante: + pool_dir: "${computer.pool_directories.pool}/AWICM/" + climate_data_dir: "${pool_dir}/FESOM2/INITIAL/phc3.0/" + forcing_data_dir: "${pool_dir}/FESOM2/FORCING/" + mesh_base_dir: "${pool_dir}/FESOM2/MESHES_FESOM2.1/" + albedo: + climate_data_dir: "${pool_dir}/FESOM/hydrography/" + mesh_base_dir: "${pool_dir}/FESOM/meshes_default/" + + setup_dir: "${model_dir}" + bin_dir: "${setup_dir}/bin" + climate_data_dir: "${pool_dir}/FESOM2/hydrography/phc3.0/" + forcing_data_dir: "${pool_dir}/forcing/" + mesh_base_dir: "${pool_dir}/FESOM2/meshes/" + ini_data_dir: "${pool_dir}/pool-data/" + namelist_dir: "${esm_namelist_dir}/fesom2/${version}/" + + opbnd_dir: "somepath" + tide_forcing_dir: "somepath" + + steps_per_day: "$(( 86400 / ${time_step} ))" + + asforcing: JRA55 + + namelists: + - namelist.config + - namelist.cvmix + - namelist.dyn + - namelist.forcing + - namelist.ice + - namelist.icepack + - namelist.io + - namelist.oce + - namelist.tra + - namelist.transit + + whichEVP: 0 # 0=EVP, 1=mEVP, 2=aEVP + # See: Kimmritz et. al 2015 & 2016: https://doi.org/10.1016/j.jcp.2015.04.051, https://doi.org/10.1016/j.ocemod.2016.03.004 + choose_resolution: + CORE2: + nx: 126858 + mesh_dir: "${fesom.mesh_base_dir}/core2/" + nproc: 288 + whichEVP: 0 + alpha_evp: 250 # constant that control numerical stability of mEVP. + beta_evp: 250 # constant that control numerical stability of mEVP. + CORE3: + nx: 211567 + mesh_dir: "${fesom.mesh_base_dir}/core3/" + nproc: 128 + whichEVP: 1 + alpha_evp: 300 # constant that control numerical stability of mEVP. + beta_evp: 300 # constant that control numerical stability of mEVP. + pi: + nx: 3140 + mesh_dir: "${fesom.mesh_base_dir}/pi/" + nproc: 4 + whichEVP: 0 + alpha_evp: 250 # constant that control numerical stability of mEVP. + beta_evp: 250 # constant that control numerical stability of mEVP. + LGM: + nx: 95239 + mesh_dir: "/home/a/a270064/bb1029/inputs/mesh_glac1d_m1_512/" + nproc: 1024 + whichEVP: 0 + alpha_evp: 250 # constant that control numerical stability of mEVP. + beta_evp: 250 # constant that control numerical stability of mEVP. + GLOB: + nx: 830305 + whichEVP: 1 + alpha_evp: 250 # constant that control numerical stability of mEVP. + beta_evp: 250 # constant that control numerical stability of mEVP. + CAVCORE2: + nx: 72411 + mesh_dir: "/work/ollie/pscholz/mesh_fesom2.0/cavity_72k/" + nproc: 288 + whichEVP: 0 + alpha_evp: 250 # constant that control numerical stability of mEVP. + beta_evp: 250 # constant that control numerical stability of mEVP. + PI_ICEv2: + nx: 132273 + whichEVP: 0 + alpha_evp: 250 # constant that control numerical stability of mEVP. + beta_evp: 250 # constant that control numerical stability of mEVP. + D3: + nx: 3160340 + whichEVP: 1 + alpha_evp: 500 # constant that control numerical stability of mEVP. + beta_evp: 500 # constant that control numerical stability of mEVP. + orca25: + nx: 912469 + whichEVP: 1 + alpha_evp: 500 # constant that control numerical stability of mEVP. + beta_evp: 500 # constant that control numerical stability of mEVP. + DART: + nx: 3160340 + whichEVP: 1 + alpha_evp: 500 # constant that control numerical stability of mEVP. + beta_evp: 500 # constant that control numerical stability of mEVP. + DARS: + nx: 3160340 + whichEVP: 1 + alpha_evp: 500 # constant that control numerical stability of mEVP. + beta_evp: 500 # constant that control numerical stability of mEVP. + DARS2: + nx: 3146761 + whichEVP: 1 + alpha_evp: 500 # constant that control numerical stability of mEVP. + beta_evp: 500 # constant that control numerical stability of mEVP. + ROS2: + nx: 26306560 + whichEVP: 1 + alpha_evp: 900 # constant that control numerical stability of mEVP. + beta_evp: 900 # constant that control numerical stability of mEVP. + HR: + nx: 1306775 + whichEVP: 1 + alpha_evp: 500 # constant that control numerical stability of mEVP. + beta_evp: 500 # constant that control numerical stability of mEVP. + jane: + nx: 33348172 + whichEVP: 1 + alpha_evp: 900 # constant that control numerical stability of mEVP. + beta_evp: 900 # constant that control numerical stability of mEVP. + SO3: + nx: 11087062 + whichEVP: 1 + alpha_evp: 800 # constant that control numerical stability of mEVP. + beta_evp: 800 # constant that control numerical stability of mEVP. + Arc01_fesom2: + nx: 753179 + whichEVP: 1 + alpha_evp: 500 # constant that control numerical stability of mEVP. + beta_evp: 500 # constant that control numerical stability of mEVP. + + + restart_in_files: + par_oce_restart: par_oce_restart + par_ice_restart: par_ice_restart + + restart_in_in_work: + par_oce_restart: fesom.${parent_date!syear}.oce.restart/*.nc + par_ice_restart: fesom.${parent_date!syear}.ice.restart/*.nc + fesom_raw_restart_info: fesom_raw_restart/*.info + fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_bin_restart_info: fesom_bin_restart/*.info + fesom_bin_restart: fesom_bin_restart/np${nproc}/* + wiso_restart: fesom.${parent_date!syear}.wiso.restart.nc + icb_restart: iceberg.restart #.${parent_date!syear!month} + icb_restart_ISM: iceberg.restart.ISM + restart_in_sources: + par_oce_restart: fesom.${parent_date!syear}.oce.restart/*.nc + par_ice_restart: fesom.${parent_date!syear}.ice.restart/*.nc + fesom_raw_restart_info: fesom_raw_restart/*.info + fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_bin_restart_info: fesom_bin_restart/*.info + fesom_bin_restart: fesom_bin_restart/np${nproc}/* + wiso_restart: fesom.${parent_date!syear}.wiso.restart.nc + icb_restart: iceberg.restart #.${parent_date!syear!month} + icb_restart_ISM: iceberg.restart.ISM + + restart_out_files: + par_oce_restart: par_oce_restart + par_ice_restart: par_ice_restart + fesom_raw_restart_info: fesom_raw_restart_info + fesom_raw_restart: fesom_raw_restart + fesom_bin_restart_info: fesom_bin_restart_info + fesom_bin_restart: fesom_bin_restart + + restart_out_in_work: + par_oce_restart: fesom.${end_date!syear}.oce.restart/*.nc + par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc + fesom_raw_restart_info: fesom_raw_restart/*.info + fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_raw_restart_tar: fesom_raw_restart.tar.* + fesom_bin_restart_info: fesom_bin_restart/*.info + fesom_bin_restart: fesom_bin_restart/np${nproc}/* + fesom_bin_restart_tar: fesom_bin_restart.tar.* + wiso_restart: fesom.${end_date!syear}.wiso.restart.nc + icb_restart: iceberg.restart #.${parent_date!syear} + icb_restart_ISM: iceberg.restart.ISM + restart_out_sources: + par_oce_restart: fesom.${end_date!syear}.oce.restart/*.nc + par_ice_restart: fesom.${end_date!syear}.ice.restart/*.nc + fesom_raw_restart_info: fesom_raw_restart/*.info + fesom_raw_restart: fesom_raw_restart/np${nproc}/*.dump + fesom_raw_restart_tar: fesom_raw_restart.tar.* + fesom_bin_restart_info: fesom_bin_restart/*.info + fesom_bin_restart: fesom_bin_restart/np${nproc}/* + fesom_bin_restart_tar: fesom_bin_restart.tar.* + wiso_restart: fesom.${end_date!syear}.wiso.restart.nc + icb_restart: iceberg.restart #.${parent_date!syear} + icb_restart_ISM: iceberg.restart.ISM + + outdata_sources: + fesom: "*.fesom.*.nc" + outdata_targets: + fesom: "*.fesom.*.nc" + + choose_with_icb: + True: + required_plugins: + - "git+https://github.com/esm-tools-plugins/fesom_icb_pism" + + prepcompute_recipe: + - "compile_model" + - "_show_simulation_info" + - "create_new_files" + - "create_empty_folders" + - "prepare_coupler_files" + - "assemble" + - "log_used_files" + - "wait_for_iterative_coupling" + - "prep_icebergs" # Plugin + - "copy_files_to_thisrun" + - "modify_namelists" + - "apply_iceberg_calving_to_namelists" # Plugin + - "modify_files" + - "copy_files_to_work" + - "report_missing_files" + - "compute_and_log_file_checksums" + - "_write_finalized_config" + - "database_entry" + + use_icebergs: True # turns on icebergs + use_icesheet_coupling: True # initializes new icebergs every restart + update_icebergs: True + + icb_inputs: [longitude,latitude,length,height,scaling,felem] + + add_restart_in_files: + icb_restart: icb_restart + icb_restart_ISM: icb_restart_ISM + + add_restart_out_files: + icb_restart: icb_restart + icb_restart_ISM: icb_restart_ISM + + add_outdata_targets: + buoys_track: "buoys_track.nc_${run_datestamp}" + + add_outdata_sources: + buoys_track: "buoys_track.nc" + + choose_general.iterative_coupling: + True: + choose_general.chunk_number: + 1: + disch_file: "${ini_disch_file}" + "*": + disch_file: "${general.experiment_couple_dir}/latest_discharge.nc" + '*': + use_landice_water: False + + choose_use_icesheet_coupling: + true: + choose_general.run_number: + 1: + iceberg_dir: "${ini_iceberg_dir}" + "*": + iceberg_dir: "${general.experiment_couple_dir}" + + add_input_files: + num_non_melted_icb_file: "num_non_melted_icb_file" + + "*": + iceberg_dir: "${ini_iceberg_dir}" + + choose_update_icebergs: + true: + add_input_sources: + "[[icb_inputs-->ICB_INPUT]]": ${iceberg_dir}/icb_ICB_INPUT.dat + num_non_melted_icb_file: "${iceberg_dir}/num_non_melted_icb_file" + + false: + add_input_sources: + "[[icb_inputs-->ICB_INPUT]]": ${iceberg_dir}/icb_ICB_INPUT.dat_@YEAR@0101-@YEAR@1231 + num_non_melted_icb_file: "${iceberg_dir}/num_non_melted_icb_file_@YEAR@0101-@YEAR@1231" + + add_input_in_work: + latitude: "icb_latitude.dat" + longitude: "icb_longitude.dat" + length: "icb_length.dat" + height: "icb_height.dat" + scaling: "icb_scaling.dat" + felem: "icb_felem.dat" + num_non_melted_icb_file: "num_non_melted_icb_file" + + add_input_files: + latitude: "latitude" + longitude: "longitude" + length: "length" + height: "height" + scaling: "scaling" + felem: "felem" + #num_non_melted_icb_file: "num_non_melted_icb_file" + + log_files: + clock: clock + mesh_diag: mesh_diag + + log_in_work: + clock: fesom.clock + mesh_diag: fesom.mesh.diag.nc + + log_sources: + clock: fesom.clock + mesh_diag: fesom.mesh.diag.nc + + file_movements: + fesom_raw_restart_in: + all_directions: move + fesom_raw_restart_out: + all_directions: move + fesom_raw_restart_info_in: + all_directions: move + fesom_raw_restart_info_out: + all_directions: move + fesom_raw_restart_tar_out: + all_directions: move + fesom_bin_restart_in: + all_directions: move + fesom_bin_restart_out: + all_directions: move + fesom_bin_restart_info_in: + all_directions: move + fesom_bin_restart_info_out: + all_directions: move + fesom_bin_restart_tar_out: + all_directions: move + + choose_tar_binary_restarts: + True: + add_restart_out_files: + fesom_raw_restart_tar: fesom_raw_restart_tar + + # Is it a branchoff experiment? + branchoff: "$(( ${lresume} and ${general.run_number}==1 ))" + choose_branchoff: + # Makes sure the new in the namelist make an earlier date than that of + # of the clock to avoid cold starts for branching off experiments + true: + daynew: "${startday}" + yearnew: "$(( ${initial_date!syear} - 1 ))" + false: + daynew: "${initial_date!sdoy}" + yearnew: "${initial_date!syear}" + add_restart_in_files: + fesom_raw_restart_info: fesom_raw_restart_info + fesom_raw_restart: fesom_raw_restart + fesom_bin_restart_info: fesom_bin_restart_info + fesom_bin_restart: fesom_bin_restart + + + + namelist_changes: + namelist.config: + clockinit: + daynew: "${daynew}" + yearnew: "${yearnew}" + calendar: + include_fleapyear: "${leapyear}" + paths: + MeshPath: "${mesh_dir}" + ClimateDataPath: "${climate_data_dir}" + ResultPath: "${work_dir}" + timestep: + step_per_day: "${steps_per_day}" + run_length: "${restart_rate}" + run_length_unit: "${restart_unit}" + restart_log: + restart_length: "${restart_rate}" + restart_length_unit: "${restart_unit}" + inout: + restartflag: "${restart_flag}" + output_length: "${restart_rate}" + output_length_unit: "${restart_unit}" + restart_length: "${restart_rate}" + run_config: + lwiso: "${with_wiso}" + icebergs: + use_icebergs: "${use_icebergs}" + use_icesheet_coupling: "${use_icesheet_coupling}" + steps_per_ib_step: 8 + + + namelist.forcing: + + forcing_exchange_coeff: + Ce_atm_oce: ${Ce_atm_oce} + Ch_atm_oce: ${Ch_atm_oce} + Cd_atm_oce: ${Cd_atm_oce} + Ce_atm_ice: ${Ce_atm_ice} + Ch_atm_ice: ${Ch_atm_ice} + Cd_atm_ice: ${Cd_atm_ice} + + forcing_bulk: + AOMIP_drag_coeff: ${AOMIP_drag_coeff} + ncar_bulk_formulae: ${ncar_bulk_formulae} + ncar_bulk_z_wind: ${ncar_bulk_z_wind} + ncar_bulk_z_tair: ${ncar_bulk_z_tair} + ncar_bulk_z_shum: ${ncar_bulk_z_shum} + + nam_sbc: + nm_xwind_file: "${forcing_data_dir}/${forcing_folder}/${xwind_prefix}" + nm_ywind_file: "${forcing_data_dir}/${forcing_folder}/${ywind_prefix}" + nm_humi_file: "${forcing_data_dir}/${forcing_folder}/${humi_prefix}" + nm_qsr_file: "${forcing_data_dir}/${forcing_folder}/${qsr_prefix}" + nm_qlw_file: "${forcing_data_dir}/${forcing_folder}/${qlw_prefix}" + nm_tair_file: "${forcing_data_dir}/${forcing_folder}/${tair_prefix}" + nm_prec_file: "${forcing_data_dir}/${forcing_folder}/${prec_prefix}" + nm_snow_file: "${forcing_data_dir}/${forcing_folder}/${snow_prefix}" + nm_mslp_file: "${forcing_data_dir}/${forcing_folder}/${mslp_prefix}" + + nm_xwind_var: ${nm_xwind_var} + nm_ywind_var: ${nm_ywind_var} + nm_humi_var: ${nm_humi_var} + nm_qsr_var: ${nm_qsr_var} + nm_qlw_var: ${nm_qlw_var} + nm_tair_var: ${nm_tair_var} + nm_prec_var: ${nm_prec_var} + nm_snow_var: ${nm_snow_var} + nm_mslp_var: ${nm_mslp_var} + + nm_nc_iyear: ${nm_nc_iyear} + nm_nc_imm: ${nm_nc_imm} + nm_nc_idd: ${nm_nc_idd} + nm_nc_freq: ${nm_nc_freq} + nm_nc_tmid: ${nm_nc_tmid} + l_xwind: ${l_xwind} + l_ywind: ${l_ywind} + l_humi: ${l_humi} + l_qsr: ${l_qsr} + l_qlw: ${l_qlw} + l_tair: ${l_tair} + l_prec: ${l_prec} + l_mslp: ${l_mslp} + l_cloud: ${l_cloud} + l_snow: ${l_snow} + + nm_runoff_file: "${forcing_data_dir}/${forcing_folder}/${runoff_file}" + nm_sss_data_file: "${forcing_data_dir}/${forcing_folder}/${sss_data_file}" + + runoff_data_source: ${runoff_data_source} + sss_data_source: ${sss_data_source} + namelist.ice: + ice_dyn: + whichEVP: ${whichEVP} + alpha_evp: ${alpha_evp} + beta_evp: ${beta_evp} + + with_cavity: false + choose_with_cavity: + true: + add_namelist_changes: + namelist.config: + run_config: + use_cavity: true + + use_landice_water: False + choose_general.iterative_coupling: + true: + add_output_targets: + fesom: "*.fesom.*.nc" + + choose_general.chunk_number: + 1: + use_landice_water: False + "*": + use_landice_water: True + + fwf_path: "${general.experiment_couple_dir}" + + add_namelist_changes: + namelist.forcing: + land_ice: + use_landice_water: "${use_landice_water}" + landice_start_mon: 1 + landice_end_mon: 12 + fwf_path: "${fwf_path}" + '*': + use_landice_water: False + + lasttime: "$(( 86400 - ${time_step}))" + currentday: "${current_date!sdoy}" + choose_currentday: + "1": + starttime: "0.0000000000000" + startday: 1 + '*': + starttime: "86400.0000000000" + startday: "$(( ${start_date!sdoy} - 1 ))" + + choose_lresume: + false: + create_config: + fesom.clock: + - "<--append-- 0.0000000000000 ${initial_date!sdoy} ${initial_date!syear}" + - "<--append-- 0.0000000000000 ${initial_date!sdoy} ${initial_date!syear}" + true: + create_config: + fesom.clock: + - "<--append-- ${lasttime} ${parent_date!sdoy} ${parent_date!syear}" + - "<--append-- ${starttime} ${startday} ${start_date!syear}" + + bin_in_work: + bin: "${executable}" + lib: "lib/fesom/libfesom.so" + + bin_sources: + bin: "${bin_dir}/${comp_executable}" + lib: "${fesom.model_dir}/build/lib/libfesom.so" + + + + config_files: + config: config + cvmix: cvmix + dyn: dyn + forcing: forcing + ice: ice + icepack: icepack + io: io + oce: oce + tra: tra + transit: transit + + config_sources: + config: "${namelist_dir}/namelist.config" + cvmix: "${namelist_dir}/namelist.cvmix" + dyn: "${namelist_dir}/namelist.dyn" + forcing: "${namelist_dir}/namelist.forcing" + ice: "${namelist_dir}/namelist.ice" + icepack: "${namelist_dir}/namelist.icepack" + io: "${namelist_dir}/namelist.io" + oce: "${namelist_dir}/namelist.oce" + tra: "${namelist_dir}/namelist.tra" + transit: "${namelist_dir}/namelist.transit" + + config_in_work: + config: "namelist.config" + cvmix: "namelist.cvmix" + dyn: "namelist.dyn" + forcing: "namelist.forcing" + ice: "namelist.ice" + icepack: "namelist.icepack" + io: "namelist.io" + oce: "namelist.oce" + tra: "namelist.tra" + transit: "namelist.transit" + + coupling_fields: + sst_feom: + grid: feom + sit_feom: + grid: feom + sie_feom: + grid: feom + snt_feom: + grid: feom + u_feom: + grid: feom + v_feom: + grid: feom + FCO2_feom: + grid: feom + taux_oce: + grid: feom + tauy_oce: + grid: feom + taux_ico: + grid: feom + tauy_ico: + grid: feom + prec_oce: + grid: feom + snow_oce: + grid: feom + evap_oce: + grid: feom + subl_oce: + grid: feom + heat_oce: + grid: feom + heat_ico: + grid: feom + heat_swo: + grid: feom + hydr_oce: + grid: feom + calv_oce: + grid: feom + enth_oce: + grid: feom + u10w_oce: + grid: feom + v10w_oce: + grid: feom + XCO2_oce: + grid: feom + + + wiso_fields: + - o18w_oce + - hdow_oce + - o16w_oce + - o18i_oce + - hdoi_oce + - o16i_oce + - w1_oce + - w2_oce + - w3_oce + - i1_oce + - i2_oce + - i3_oce + + + choose_with_wiso: + True: + add_restart_in_files: + wiso_restart: wiso_restart + add_restart_out_files: + wiso_restart: wiso_restart + add_coupling_fields: + "[[wiso_fields-->FIELD]]": + grid: feom + + + grids: + feom: + name: feom + nx: ${nx} + ny: 1 + oasis_grid_type: "U" + oyac_grid_type: "GC" + oyac_io_per_node: 4 + number_of_overlapping_points: 0 # oasis P-value + + + further_reading: + - fesom/fesom.forcing.yaml + - fesom/fesom.env.yaml + + # ------------------------ + # RECIPE + # ------------------------ + tidy_recipe: + - "tidy_coupler" + - "tar_binary_restarts" # Plugin + - "assemble" + - "check_for_unknown_files" + - "log_used_files" + - "throw_away_some_infiles" + - "copy_stuff_back_from_work" + - "copy_all_results_to_exp" + - "clean_run_dir" + diff --git a/configs/components/fesom/fesom.forcing.yaml b/configs/components/fesom/fesom.forcing.yaml index e5c430edc..28fd30db4 100644 --- a/configs/components/fesom/fesom.forcing.yaml +++ b/configs/components/fesom/fesom.forcing.yaml @@ -1,6 +1,66 @@ -# WARNING: To be use only with FESOM 2.1 +# WARNING: To be use only with FESOM 2.1 or higher versions fesom: choose_asforcing: + + CORE1: + leapyear: false + forcing_folder: "CORE1" + + Ce_atm_oce: 1.75e-3 + Ch_atm_oce: 1.75e-3 + Cd_atm_oce: 1.0e-3 + Ce_atm_ice: 1.75e-3 + Ch_atm_ice: 1.75e-3 + Cd_atm_ice: 1.2e-3 + + AOMIP_drag_coeff: False + ncar_bulk_formulae: True + ncar_bulk_z_wind: 10.0 + ncar_bulk_z_tair: 10.0 + ncar_bulk_z_shum: 10.0 + + xwind_prefix: "u_10.15JUNE2009" + ywind_prefix: "v_10.15JUNE2009" + humi_prefix: "q_10.15JUNE2009" + qsr_prefix: "ncar_rad.15JUNE2009" + qlw_prefix: "ncar_rad.15JUNE2009" + tair_prefix: "t_10.15JUNE2009" + prec_prefix: "ncar_precip.15JUNE2009" + snow_prefix: "ncar_precip.15JUNE2009" + mslp_prefix: "slp.15JUNE2009" + + nm_xwind_var: "U_10_MOD" + nm_ywind_var: "V_10_MOD" + nm_humi_var: "Q_10_MOD" + nm_qsr_var: "SWDN_MOD" + nm_qlw_var: "LWDN_MOD" + nm_tair_var: "T_10_MOD" + nm_prec_var: "RAIN" + nm_snow_var: "SNOW" + nm_mslp_var: "SLP" + + nm_nc_iyear: 1900 + nm_nc_imm: 1 + nm_nc_idd: 1 + nm_nc_freq: 1 + nm_nc_tmid: 1 + l_xwind: True + l_ywind: True + l_humi: True + l_qsr: True + l_qlw: True + l_tair: True + l_prec: True + l_mslp: False + l_cloud: False + l_snow: True + + runoff_file: "runoff.15JUNE2009.nc" + sss_data_file: "PHC2_salx.nc" + + runoff_data_source: 'CORE2' + sss_data_source: 'CORE2' + CORE2: leapyear: false forcing_folder: "CORE2" diff --git a/configs/components/fesom/fesom.tripyview.yaml b/configs/components/fesom/fesom.tripyview.yaml new file mode 100644 index 000000000..5f947038b --- /dev/null +++ b/configs/components/fesom/fesom.tripyview.yaml @@ -0,0 +1,96 @@ +# ============================================================================= +# FESOM TRIPYVIEW AUTOMATIC VISUALIZATION CONFIGURATION +# ============================================================================= +# +# TriPyView is an automatic visualization and analysis tool for FESOM output. +# This configuration enables post-processing visualization as a subjob after +# each model run completes. +# +# USAGE +# ----- +# To enable TriPyView in your experiment, add this to your runscript's +# further_reading section: +# +# fesom: +# further_reading: +# - fesom/fesom.tripyview +# +# CONFIGURATION OPTIONS +# --------------------- +# vis_server : Hostname of the visualization server to sync results to +# vis_publicdir : Public directory path on the visualization server +# vis_sync : Enable/disable syncing to visualization server (true/false) +# +# EXAMPLE USAGE +# ------------- +# For Levante (DKRZ), the configuration is automatically applied. For other +# systems, you can override in your runscript: +# +# fesom: +# further_reading: +# - fesom/fesom.tripyview +# vis_server: your.server.hostname +# vis_publicdir: /path/to/public/directory +# vis_sync: true +# +# WHAT IT DOES +# ------------ +# The tripyview subjob automatically: +# - Installs TriPyView in a dedicated conda environment (first run only) +# - Generates visualization configuration for each model year +# - Executes TriPyView to create plots and analysis outputs +# - Syncs results to a viewing server (if enabled) +# +# OUTPUT LOCATIONS +# ---------------- +# - Configs : ${base_dir}/${expid}/config/fesom/ +# - Results : ${base_dir}/${expid}/analysis/fesom/ +# ${base_dir}/${expid}/viz/fesom/ +# +# REQUIREMENTS +# ------------ +# - Conda available on the system +# - TriPyView: https://github.com/fesom/tripyview +# - For syncing: SSH access to the visualization server +# +# ============================================================================= + +fesom: + vis_project: ${general.account} + vis_user: !ENV ${USER} + vis_server: NOT_SET + vis_publicdir: NOT_SET + vis_sync: false + tripyview_conda_module: "NOT_SET" + + choose_computer.name: + levante: + vis_server: modvis.dkrz.de + vis_publicdir: /var/www/projects/${fesom.vis_project}/${fesom.vis_user}/${general.expid} + vis_sync: true + tripyview_conda_module: "python3" # Provides conda: https://tinyurl.com/3887k99t + albedo: + tripyview_conda_module: "conda" + + scripts_files: + Tripy1: Tripy1 + Tripy2: Tripy2 + scripts_sources: + Tripy1: "${esm_function_dir}/components/fesom/Subjob_tripy.sh" + Tripy2: "${esm_function_dir}/components/fesom/ESM_auto_tripy.yaml" + scripts_targets: + Tripy1: "${general.experiment_scripts_dir}/${model}/Subjob_tripy.sh" + Tripy2: "${general.experiment_scripts_dir}/${model}/ESM_auto_tripy.yaml" + + workflow: + next_run_triggered_by: tidy + subjobs: + Tripy_Fesom: + batch_or_shell: batch + order_in_cluster: concurrent + run_on_queue: ${computer.partitions.pp.name} + run_after: tidy + script_dir: ${base_dir}/${expid}/scripts/fesom + submit_to_batch_system: true + script: "Subjob_tripy.sh ${fesom.experiment_outdata_dir} ${general.start_date} ${general.next_date} ${general.initial_date} ${general.final_date} ${fesom.mesh_dir} ${general.base_dir} ${general.expid} ${fesom.tripyview_conda_module} ${fesom.vis_sync} ${fesom.vis_server} ${fesom.vis_publicdir}" + nproc: 1 diff --git a/configs/components/lpj_guess/lpj_guess.cmip.yaml b/configs/components/lpj_guess/lpj_guess.cmip.yaml new file mode 100644 index 000000000..6ce7b67ae --- /dev/null +++ b/configs/components/lpj_guess/lpj_guess.cmip.yaml @@ -0,0 +1,51 @@ +# LPJ-GUESS CMIP Configuration +# +# This file provides CMIP-specific defaults for LPJ-GUESS. +# Set lpj_guess.mip to "cmip6" or "cmip7" to select the appropriate defaults. +# Individual settings can still be overridden in the runscript. + +lpj_guess: + # Default MIP version and scenario + mip: "cmip7" + scenario: piControl + + # CO2 forcing file paths + co2_file_cmip6: ${lpj_guess.input_dir}/cmip6_forcing/mole_fraction_of_carbon_dioxide_in_air_input4MIPs_lpjg.nc + co2_file_cmip7: ${lpj_guess.input_dir}/cmip7_forcing/co2_input4MIPs_GHGConcentrations_CMIP_CR-CMIP-1-0-0_gm_1750-2022.nc + co2_workname_cmip6: mole_fraction_of_carbon_dioxide_in_air_input4MIPs_lpjg.nc + co2_workname_cmip7: co2_input4MIPs_GHGConcentrations_CMIP_CR-CMIP-1-0-0_gm_1750-2022.nc + + choose_mip: + "cmip6": + co2_source_file: ${co2_file_cmip6} + co2_work_file: ${co2_workname_cmip6} + ndep_cmip_dir: "" + use_luh3_input: 0 # CMIP6 uses LUH2 text files + + "cmip7": + co2_source_file: ${co2_file_cmip7} + co2_work_file: ${co2_workname_cmip7} + + ndep_cmip_dir: "../ndep/" + + use_luh3_input: 1 # CMIP7 uses LUH3 NetCDF files + luh3_input_dir: ${lpj_guess.input_dir}/land_use/regridded/ + use_luh3_yearly_transitions: 0 + file_luh3_transitions_dir: "${lpj_guess.input_dir}/land_use/transitions_yearly" + file_luh3_transitions: "${lpj_guess.input_dir}/land_use/regridded/multiple-transitions_input4MIPs_landState_CMIP_UofMD-landState-3-1-1_gn_0850-2023_${lpj_guess.mygrid}.nc" + file_luh3_states: "${lpj_guess.input_dir}/land_use/regridded/multiple-states_input4MIPs_landState_CMIP_UofMD-landState-3-1-1_gn_0850-2024_${lpj_guess.mygrid}.nc" + file_luh3_management: "${lpj_guess.input_dir}/land_use/regridded/multiple-management_input4MIPs_landState_CMIP_UofMD-landState-3-1-1_gn_0850-2024_${lpj_guess.mygrid}.nc" + run_landcover: 1 + run_natural: 1 + run_crop: 1 + run_pasture: 1 + run_barren: 0 # Disabled for LUH3 - calculated as residual internally + run_urban: 1 + run_peatland: 1 # Disabled for now. + firemodel: BLAZE + + choose_scenario: + "piControl": + fixed_CO2: 1850 + fixed_ndep: 1850 + fixed_LU: 1850 diff --git a/configs/components/lpj_guess/lpj_guess.yaml b/configs/components/lpj_guess/lpj_guess.yaml new file mode 100644 index 000000000..46fb2ecbf --- /dev/null +++ b/configs/components/lpj_guess/lpj_guess.yaml @@ -0,0 +1,352 @@ +# LPJ-GUESS 4 +# +# For more information about the extended YAML syntax, please consult: +# https://esm-tools.readthedocs.io/en/latest/yaml.html#esm-tools-extended-yaml-syntax +# +# For more information about the ESM-Tools feature variables available, please consult: +# https://esm-tools.readthedocs.io/en/latest/esm_variables.html#esm-tools-variables + +lpj_guess: + model: lpj_guess + version: 4.1.2 + + required_plugins: + - "git+https://github.com/JanStreffing/lpjg2nc2" + + further_reading: + - lpj_guess/lpj_guess.cmip.yaml + + metadata: + Institute: Lund Universitiy + Description: Dynamic vegetation model + Authors: Ben Smith + Publications: + - "Smith, B., Prentice, I.C. & Sykes, M.T. 2001. Representation of vegetation dynamics in the modelling of terrestrial ecosystems: comparing two contrasting approaches within European climate space. Global Ecology & Biogeography 10: 621-637. " + - "Sitch, S., Smith, B., Prentice, I.C., Arneth, A., Bondeau, A., Cramer, W., Kaplan, J., Levis, S., Lucht, W., Sykes, M., Thonicke, K. & Venevsky, S. 2003. Evaluation of ecosystem dynamics, plant geography and terrestrial carbon cycling in the LPJ Dynamic Global Vegetation Model. Global Change Biology 9: 161-185. " + - "Martín Belda, D., Anthoni, P., Wårlind, D., Olin, S., Schurgers, G., Tang, J., Smith, B., & Arneth, A. 2022. LPJ-GUESS/LSMv1.0: A next generation Land Surface Model with high ecological realism. Geoscientific Model Development: " + - "Nord, J., Anthoni, P., Gregor, K., Gustafson, A., Hantson, S., Lindeskog, M., Meyer, B., Miller, P., Nieradzik, L., Olin, S., Papastefanou, P., Smith, B., Tang, J., Wårlind, D., & and past LPJ-GUESS contributors. (2021). LPJ-GUESS Release v4.1.1 model code (4.1.1). Zenodo. " + License: Mozilla Public License Version 2.0 + + + git-repository: https://gitlab.dkrz.de/ec-earth/lpj-guess4.1.1.git + destination: lpj_guess + branch: main + comp_command: "mkdir -p build; cd build; cmake ..; make -j8" + + setup_dir: "${model_dir}" + bin_dir: "${setup_dir}/bin" + install_bins: "build/guess" + clean_command: "rm -rf build" + executable: guess_TL255 + execution_command: ${executable} ${runscript_name} + #resolution: TL159 #if it's not set i dosen't work! + resolution: TL255 #if it's not set i dosen't work! + TM5: 0 + + # Enumeration for parallel run directories (run1, run2, ..., runN) + nproc: 8 + proc_list_entry: '[str(x) for x in range(1,${nproc}+1)]' + proc_list: $(( ${proc_list_entry} )) + + choose_version: + 4.1.1: + runscript_name: run_trunk_4_1.ins + git-repository: https://gitlab.dkrz.de/ec-earth/lpj-guess4.1.1.git + branch: main + 4.1.2: + runscript_name: run_coupled_4_1_2.ins + git-repository: https://git.smhi.se/e8891/lpjg-4.1.git + branch: lpj_guess_awiesm3 + install_bins: build/guess + executable: guess + 4.1.3: + runscript_name: run_coupled_4_1_2.ins + git-repository: https://git.smhi.se/jan.streffing/lpjg-4.1.git + branch: 4.1.3 + install_bins: build/guess + executable: guess + + + # Automatically select the source code for esm_master if the command is operating + # with a coupled setup + model_dir: ${general.model_dir} + + choose_computer.name: + levante: + pool_dir: "/work/ab0246/a270092/input/" + + # Some recommended defaults (restart for old experiment yes or no?) + lresume: false + time_step: 86400 + nyear_spinup: 10 + + # Fire model selection: BLAZE, GLOBFIRM, NOFIRE + # Set to BLAZE to enable SIMFIRE/BLAZE fire module + firemodel: NOFIRE + + choose_lresume: + false: + restart_num: 0 + true: + restart_num: 1 + + # Some commonly used variables in other ESM-Tools components (but optional) + input_dir: ${pool_dir}/lpj-guess/ + out_date_folder: "${end_date!syear!smonth!sday}" + + # CMIP N deposition directory (empty string to disable, otherwise full path with trailing /) + ndep_input_dir: ${input_dir}/ndep/${resolution}/ + + # Output mode: + # esm_tools_copy: slow but reliable, many files + # collect_txt: combile into one run1 folder for ec2cmor2 few files, fast + # convert_netcdf (future) + output_mode: collect_txt + choose_output_mode: + esm_tools_copy: + # Define files to ignore + ignore_files: + outputs: outputs + log: log + ignore_in_work: + # Ignore output files with the same name + outputs: "output/*.out" + # Ignore log files + log: "*.log" + # Ignore input directories (created during setup) + ndep_dir: "ndep" + landuse_dir: "landuse" + ignore_sources: + outputs: "output/*.out" + log: "*.log" + + # Output data - enumerate over all run directories to preserve structure + # Uses proc_list enumeration (defined above via nproc) + outdata_files: + "lpjg_run_[[proc_list-->PROC_NUM]]": "lpjg_run_PROC_NUM" + outdata_in_work: + "lpjg_run_[[proc_list-->PROC_NUM]]": "runPROC_NUM/output/*.out" + outdata_sources: + "lpjg_run_[[proc_list-->PROC_NUM]]": "runPROC_NUM/*" + + collect_txt: + # In collect_txt mode, esm_tools does NOT copy output files. + # Instead, a separate slurm job combines runX/output/*.out into + # single .out files per pattern (for ec2cmor2 preprocessing). + combine_njobs: 8 + + ignore_files: + outputs: outputs + log: log + ignore_in_work: + outputs: "output/*.out" + log: "*.log" + ndep_dir: "ndep" + landuse_dir: "landuse" + ignore_sources: + outputs: "output/*.out" + log: "*.log" + + outdata_files: {} + outdata_in_work: {} + outdata_sources: {} + + scripts_files: + combine1: combine1 + scripts_sources: + combine1: "${esm_function_dir}/components/lpj_guess/subjob_combine.sh" + scripts_targets: + combine1: "${general.experiment_scripts_dir}/${model}/subjob_combine.sh" + + workflow: + next_run_triggered_by: tidy + subjobs: + Combine_LPJG: + batch_or_shell: batch + order_in_cluster: concurrent + run_on_queue: ${computer.partitions.pp.name} + run_after: tidy + script_dir: ${base_dir}/${expid}/scripts/lpj_guess + submit_to_batch_system: true + script: "subjob_combine.sh ${general.thisrun_dir}/work ${lpj_guess.experiment_outdata_dir} ${general.run_datestamp} ${lpj_guess.combine_njobs}" + nproc: 1 + + namelist_dir: ${general.esm_namelist_dir}/lpj_guess/ + gridlist_dir: ${input_dir}/gridlists/ + forcing_dir: ${input_dir}/climate_forcing_data/ + + #gridlist: "full_global" # sets up grid cell list =/ setup_name!! + gridlist: "home" # sets up grid cell list =/ setup_name!! + setup_name: global # sets up all the model parameters + + + # File dictionaries + config_sources: + runscript: ${namelist_dir}/run_trunk_4_1.j2 + forcing_path_file: ${namelist_dir}/paths_forcing.j2 + config_in_work: + runscript: ${runscript_name} + forcing_path_file: paths_forcing.ins + + bin_sources: + bin_tag: ${model_dir}/bin/${executable} + + lpjg_parent_state: $((${parent_date!syear}+1)) + lpjg_end_state: $((${end_date!syear}+1)) + + restart_in_files: + state: state + restart_in_in_work: + state: lpjg_state_${lpjg_parent_state}/* + restart_in_sources: + state: lpjg_state_${lpjg_parent_state}/* + restart_out_files: + state: state + restart_out_in_work: + state: lpjg_state_${lpjg_end_state}/* + restart_out_sources: + state: lpjg_state_${lpjg_end_state}/* + + # Soil type file + slt_dir: ${input_dir}/slt/ + + + # SIMFIRE/BLAZE fire input directory + # Files: simfire_monthly_ba.nc (12 months), annual_population_density.nc (1850-2025) + fire_input_dir: ${input_dir}/fire/${resolution}/ + + # N deposition forcing files - link from input to work/ndep with renamed filenames + forcing_files: + soil_layer_type: soil_layer_type + drynhx: drynhx + drynoy: drynoy + wetnhx: wetnhx + wetnoy: wetnoy + peat: peat + simfire_ba: simfire_ba + simfire_popdens: simfire_popdens + forcing_sources: + soil_layer_type: ${slt_dir}/slt_${resolution}.nc + drynhx: ${ndep_input_dir}/drynhx_${resolution}_hist_d1.nc + drynoy: ${ndep_input_dir}/drynoy_${resolution}_hist_d1.nc + wetnhx: ${ndep_input_dir}/wetnhx_${resolution}_hist_d1.nc + wetnoy: ${ndep_input_dir}/wetnoy_${resolution}_hist_d1.nc + peat: ${input_dir}/peat/${resolution}_peat_frac.txt + simfire_ba: ${fire_input_dir}/simfire_monthly_ba.nc + simfire_popdens: ${fire_input_dir}/annual_population_density.nc + "soil_layer_type_[[proc_list-->PROC_NUM]]": ${slt_dir}/slt_${resolution}.nc + "drynhx_[[proc_list-->PROC_NUM]]": ${ndep_input_dir}/drynhx_${resolution}_hist_d1.nc + "drynoy_[[proc_list-->PROC_NUM]]": ${ndep_input_dir}/drynoy_${resolution}_hist_d1.nc + "wetnhx_[[proc_list-->PROC_NUM]]": ${ndep_input_dir}/wetnhx_${resolution}_hist_d1.nc + "wetnoy_[[proc_list-->PROC_NUM]]": ${ndep_input_dir}/wetnoy_${resolution}_hist_d1.nc + forcing_in_work: + soil_layer_type: slt_${resolution}.nc + drynhx: ndep/drynhx2.nc + drynoy: ndep/drynoy2.nc + wetnhx: ndep/wetnhx2.nc + wetnoy: ndep/wetnoy2.nc + peat: landuse/peat_frac.txt + simfire_ba: fire/simfire_monthly_ba.nc + simfire_popdens: fire/annual_population_density.nc + "soil_layer_type_[[proc_list-->PROC_NUM]]": runPROC_NUM/slt_${resolution}.nc + "drynhx_[[proc_list-->PROC_NUM]]": runPROC_NUM/ndep/drynhx2.nc + "drynoy_[[proc_list-->PROC_NUM]]": runPROC_NUM/ndep/drynoy2.nc + "wetnhx_[[proc_list-->PROC_NUM]]": runPROC_NUM/ndep/wetnhx2.nc + "wetnoy_[[proc_list-->PROC_NUM]]": runPROC_NUM/ndep/wetnoy2.nc + + choose_resolution: + # Note: + # Tco grids only work with version 43r3. + TCO95: + nx: 40320 + ny: 1 + oasis_grid_name: "096" + res_number: 95 + res_number_tl: "95_4" + truncation: "TCO" + TL159: + nx: 35718 + ny: 1 + oasis_grid_name: "080" + res_number: 159 + res_number_tl: "159l_2" + truncation: "TL" + TCO159: + nx: 108160 + ny: 1 + oasis_grid_name: "160" + res_number: 159 + res_number_tl: "159_4" + truncation: "TCO" + TCO199: + nx: 167200 + ny: 1 + oasis_grid_name: "200" + res_number: 199 + res_number_tl: "199_4" + truncation: "TCO" + TL255: + nx: 88838 + ny: 1 + oasis_grid_name: "128" + res_number: 255 + res_number_tl: "255l_2" + truncation: "TL" + TCO319: + nx: 421120 + ny: 1 + oasis_grid_name: 320 + res_number: 319 + res_number_tl: "319_4" + truncation: "TCO" + TCO399: + nx: 654400 + ny: 1 + oasis_grid_name: 400 + res_number: 399 + res_number_tl: "399_4" + truncation: "TCO" + TCO639: + nx: 1661440 + ny: 1 + oasis_grid_name: 640 + res_number: 639 + res_number_tl: "639_4" + truncation: "TCO" + TCO1279: + nx: 6599680 + ny: 1 + oasis_grid_name: 128 + res_number: 1279 + res_number_tl: "1279_4" + truncation: "TCO" + TL511: + nx: 348528 + ny: 1 + oasis_grid_name: "256" + res_number: 511 + res_number_tl: "511l_2" + truncation: "TL" + TL799: + nx: 843490 + ny: 1 + oasis_grid_name: "400" + res_number: 799 + res_number_tl: "799l_2" + truncation: "TL" + TL1279: + nx: 2140702 + ny: 1 + oasis_grid_name: "640" + res_number: 799 + res_number_tl: "1279l_2" + truncation: "TL" + + +computer: + add_export_vars: + myfortrancompiler: $FC + OASIS_ANCHOR: ../../../oasis + NETCDF_C_DIR: $NETCDFROOT + NETCDF_F_DIR: $NETCDFFROOT + OASIS_DIR: ../../../oasis/arch_ecearth/ diff --git a/configs/components/lpj_guess/subjob_combine.sh b/configs/components/lpj_guess/subjob_combine.sh new file mode 100755 index 000000000..4fa3f00f8 --- /dev/null +++ b/configs/components/lpj_guess/subjob_combine.sh @@ -0,0 +1,51 @@ +#!/bin/bash -e +############################################################################### +# Combine LPJ-GUESS parallel run output files into single .out files +# for ec2cmor2 preprocessing. +# +# Called as an esm_tools subjob after tidy. +# Requires: pip package lpjg2nc2 (provides combine_runs.py) +# +# Feb 2026 +# AWI Bremerhaven +############################################################################### + +echo "################################################################################" +echo " LPJ-GUESS combine_runs (ec2cmor2 prep) " +echo "################################################################################" +echo "" +echo "$(date):: Starting LPJ-GUESS output combination" +echo "" + +thisrun_work_dir=$1 +outdata_dir=$2 +run_datestamp=$3 +njobs=$4 + +# Construct paths +input_path="${thisrun_work_dir}" +output_path="${outdata_dir}/${run_datestamp}/run1" + +echo "$(date):: Configuration:" +echo "$(date):: input_path = ${input_path}" +echo "$(date):: output_path = ${output_path}" +echo "$(date):: run_datestamp = ${run_datestamp}" +echo "$(date):: njobs = ${njobs}" + +# Create output directory +mkdir -p "${output_path}" + +# Run combine_runs.py with --no-split +echo "$(date):: Running combine_runs.py --no-split" +combine_runs "${input_path}" --no-split -o "${output_path}" -j "${njobs}" +rc=$? + +if [ $rc -eq 0 ]; then + echo "$(date):: Successfully combined LPJ-GUESS output" + echo "$(date):: Output in: ${output_path}" +else + echo "$(date):: ERROR: combine_runs failed with exit code ${rc}" + exit $rc +fi + +echo "$(date):: Done" diff --git a/configs/components/oasis3mct/oasis3mct.yaml b/configs/components/oasis3mct/oasis3mct.yaml index cc35bf6e2..e3f8d651f 100644 --- a/configs/components/oasis3mct/oasis3mct.yaml +++ b/configs/components/oasis3mct/oasis3mct.yaml @@ -59,6 +59,7 @@ oasis3mct: - '4.0-awicm-3.2' - '5.0-smhi' - '5.0-geomar' + - '5.1' - '6.0-YAC' choose_version: '2.8-paleodyn': @@ -83,11 +84,12 @@ oasis3mct: branch: awicm-3.1 4.0-awicm-3.2: branch: awicm-3.2 + comp_command: 'mkdir -p build; cd build; cmake ..; make -j 1; mkdir -p ../include/; cp lib/psmile/libpsmile.a lib/psmile/mct/libmct.a lib/psmile/mct/mpeu/libmpeu.a lib/psmile/scrip/libscrip.a ../lib; cd ..; find . -type f \( -name "*.o" -o -name "*.mod" \) ! -path "./include/*" -exec cp -t include/ {} +; mkdir -p arch_ecearth; ln -fs ../lib ../include arch_ecearth/' 4.0-awicm-frontiers: branch: awicm-3-frontiers 5.0-smhi: - branch: main_dkrz - git-repository: https://gitlab.dkrz.de/ec-earth/oasis3mct5-ece + branch: cray_ftn_fixes + git-repository: https://git.smhi.se/jan.streffing/oasis3-mct-5 comp_command: 'mkdir -p build; cd build; cmake ..; make -j 1; mkdir -p ../include/; cp lib/psmile/libpsmile.a lib/psmile/mct/libmct.a lib/psmile/mct/mpeu/libmpeu.a lib/psmile/scrip/libscrip.a ../lib; cd ..; find . -type f \( -name "*.o" -o -name "*.mod" \) ! -path "./include/*" -exec cp -t include/ {} +; mkdir -p arch_ecearth; ln -fs ../lib ../include arch_ecearth/' 5.0-geomar: # this version is taken directly from the CERFACS gitlab @@ -106,6 +108,10 @@ oasis3mct: clean_command: 'rm -rf INSTALL_OASIS.ESMTOOLS' # this is how we should do it but it does not work #clean_command: 'export ESM_OASIS_DIR=${model_dir} ; cd util/make_dir/ ; make -f TopMakefileOasis3 realclean ; cd ../../ ; rm -rf build lib/libpsmile.a lib/libscrip.a lib/libmct.a lib/libmpeu.a' + '5.1': + branch: '5.1' + git-repository: https://git.smhi.se/jan.streffing/oasis3-mct-5 + comp_command: 'mkdir -p build; cd build; cmake ..; make -j 1; mkdir -p ../include/; cp lib/psmile/libpsmile.a lib/psmile/mct/libmct.a lib/psmile/mct/mpeu/libmpeu.a lib/psmile/scrip/libscrip.a ../lib; cd ..; find . -type f \( -name "*.o" -o -name "*.mod" \) ! -path "./include/*" -exec cp -t include/ {} +; mkdir -p arch_ecearth; ln -fs ../lib ../include arch_ecearth/' ec-earth: branch: ec-earth-version foci: diff --git a/configs/components/oifs/append_to_namelist.sh b/configs/components/oifs/append_to_namelist.sh new file mode 100755 index 000000000..eb0bed7eb --- /dev/null +++ b/configs/components/oifs/append_to_namelist.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Usage: ./append_namelist.sh input.txt target.f90 + +input_file="$1" +target_file="$2" + +echo "Appending $input_file to $target_file" +echo "" >> "$target_file" +cat "$input_file" >> "$target_file" +echo "" >> "$target_file" diff --git a/configs/components/oifs/oifs.yaml b/configs/components/oifs/oifs.yaml index 8a4e42ae1..4848e44a4 100644 --- a/configs/components/oifs/oifs.yaml +++ b/configs/components/oifs/oifs.yaml @@ -1,21 +1,26 @@ # #OIFS standalone YAML FILE # - +#general: # missing when doing esm_master get-oifs-48r1 +# with_co2_oce_coupling: false +# with_co2_veg_coupling: false +# oifs: model: oifs executable: oifs - execution_command: ${executable} -e ${oifs.input_expid} + execution_command: ${executable} -v ecmwf -e ${oifs.input_expid} version: 43r3 major_version: 43r3 source_code_permissions: 700 type: atmosphere with_xios: false # TODO: Change to true when the run bug is fixed (compile bug was fixed) with_nest1: false + xml_dir: '' required_plugins: - "git+https://github.com/esm-tools-plugins/postprocess" - "git+https://github.com/esm-tools-plugins/preprocess" + - "git+https://github.com/JanStreffing/append_to_namelist" description: | The OpenIFS atmosphere model based on ECMWF IFS @@ -52,10 +57,8 @@ oifs: namelist_dir: ${general.esm_namelist_dir}/oifs/${version}/ cmip5_data_dir: ${input_dir}/../cmip5_data cmip6_data_dir: ${input_dir}/../cmip6_data - cmip7_data_dir: ${input_dir}/../cmip7_data cmip5_data_dir_nml: ${cmip5_data_dir} cmip6_data_dir_nml: ${cmip6_data_dir} - cmip7_data_dir_nml: ${cmip7_data_dir} restart_dir: "" in_date_folder: "${parent_date!syear!smonth!sday}" out_date_folder: "${end_date!syear!smonth!sday}" @@ -114,6 +117,8 @@ oifs: - 43r3-v1 - 43r3-v2 - 48r1 + - 48r1v1 + - 48r1v2 choose_version: 40r1: @@ -325,14 +330,11 @@ oifs: add_include_models: - oasis3mct requires: - #- oasis3mct-4.0 - #- xios-2.5_r1910_oifs - oasis3mct-5.0-geomar - xios-trunk major_version: 43r3 48r1: branch: 'main' - # --build-type Debug comp_command: "./openifs-bundle create; ./openifs-bundle build --threads=32 --openifs-only --without-double-precision --install; chmod -R ${source_code_permissions} .; git config core.fileMode false; mv install/bin/ifsMASTER.SP install/bin/OpenIFS" clean_command: rm -rf build install executable: OpenIFS @@ -341,6 +343,26 @@ oifs: destination: oifs-48r1 with_xios: false major_version: 48r1 + 48r1v1: + branch: '48r1v1' + comp_command: "./openifs-bundle create; ./openifs-bundle build --threads=32 --openifs-only --without-double-precision --install; chmod -R ${source_code_permissions} .; git config core.fileMode false; mv install/bin/ifsMASTER.SP install/bin/OpenIFS" + clean_command: rm -rf build install + executable: OpenIFS + git-repository: https://git.smhi.se/jan.streffing/oifs48r1.git + install_bins: install/bin/OpenIFS + destination: oifs-48r1 + with_xios: false + major_version: 48r1 + 48r1v2: + branch: '48r1v2' + comp_command: "./openifs-bundle create; ./openifs-bundle build --threads=32 --openifs-only --without-double-precision --install; chmod -R ${source_code_permissions} .; git config core.fileMode false; mv install/bin/ifsMASTER.SP install/bin/OpenIFS" + clean_command: rm -rf build install + executable: OpenIFS + git-repository: https://git.smhi.se/jan.streffing/oifs48r1.git + install_bins: install/bin/OpenIFS + destination: oifs-48r1 + with_xios: false + major_version: 48r1 clean_command: rm -rf make/esm comp_command: cd make; ../fcm/bin/fcm make -v -j8 -f oifs.cfg git-repository: https://gitlab.dkrz.de/modular_esm/oifs-40r1.git @@ -353,7 +375,7 @@ oifs: nlev: ${levels_number} prepifs_expid: gu5a prepifs_startdate: 19820101 - mip: "cmip5" + mip: "cmip7" o3_scheme: "default" scenario: "amip-prepifs" output: "default" @@ -580,7 +602,8 @@ oifs: NAMCT0: LFDBOP: "${lfdbop}" LSMSSIG: "${lsmssig}" - + NFRPOS: "${output_step_freq}" + NFRHIS: "${output_step_freq}" lresume: false restart_rate: 1 @@ -824,6 +847,7 @@ oifs: # XIOS output oifsnc: ${input_expid}_*_${start_date!syear!smonth!sday}_*.nc atm: atm_* + atmos: atmos.* outdata_sources: # old grib based output @@ -834,6 +858,7 @@ oifs: # XIOS output oifsnc: ${input_expid}_*_${start_date!syear!smonth!sday}_*.nc atm: atm_* + atmos: atmos.* #====================================================================================== @@ -858,6 +883,7 @@ oifs: outdata_files: oifsnc: oifsnc atm: atm + atmos: atmos 0: outdata_files: ICMGG: ICMGG @@ -982,7 +1008,7 @@ oifs: YI_NL%LMASSFIX: ".true." YL_NL%LMASSFIX: ".true." YQ_NL%LMASSFIX: ".true." - 3: #DestinE 48r1 settings + 3: add_namelist_changes: fort.4: NAMDYN: @@ -1006,6 +1032,11 @@ oifs: YR_NL%LINTLIN: ".true." YS_NL%BETAMFBC: -1 YS_NL%LINTLIN: ".true." + YS_NL%LMASSFIX: ".true." + YR_NL%LMASSFIX: ".true." + YI_NL%LMASSFIX: ".true." + YL_NL%LMASSFIX: ".true." + YQ_NL%LMASSFIX: ".true." choose_cloudfixer: @@ -1119,8 +1150,8 @@ oifs: NAMRES: NFRRES: 1 NRESTS(:): - - -1 - - -${oifs.next_step} + - 1 + - ${oifs.next_step} NAMPAR0: NPROC: ${oifs.nproc} NAMRIP: @@ -1131,8 +1162,9 @@ oifs: NAMPPC: LRSACC: ".true." # reset accumulation of fluxes further_reading: + - oifs/oifs48.tuning.yaml - oifs/oifs48.env.yaml - #- oifs/oifs48.cmip.yaml + - oifs/oifs48.cmip.yaml # If 1, use Coddington spectrum which reduced stratospheric warm bias @@ -1339,27 +1371,15 @@ oifs: # OUTPUT CONFIG #====================================================================================== + output_step_freq: $(( 3600 / ${time_step} * 6 )) choose_output: - "default": - add_namelist_changes: - fort.4: - NAMCT0: - NFRPOS: "$(( 3600 / ${time_step} * 6 ))" - NFRHIS: "$(( 3600 / ${time_step} * 6 ))" - + "1hr": + output_step_freq: $(( 3600 / ${time_step} * 1 )) + "3hr": + output_step_freq: $(( 3600 / ${time_step} * 3 )) "12hr": - add_namelist_changes: - fort.4: - NAMCT0: - NFRPOS: 12 - NFRHIS: 12 + output_step_freq: $(( 3600 / ${time_step} * 12 )) - "1hr": - add_namelist_changes: - fort.4: - NAMCT0: - NFRPOS: "$(( 3600 / ${time_step} * 1 ))" - NFRHIS: "$(( 3600 / ${time_step} * 1 ))" #====================================================================================== # PREPROCESS: eternal runs #====================================================================================== @@ -1369,7 +1389,6 @@ oifs: # as general.run_number, but for branchoff experiments should be 0 so that the 3rd # preprocessing option is used, instead of the options 1 or 2. branchoff: false - ini_pseudo_initial_date: "${initial_date}" #${prev_run.general.start_date}" eternal_run_number: "$(( 0 if (${lresume} and ${general.run_number} == 1) or (${general.run_number} == 2 and ${prev_run.oifs.branchoff}) @@ -1388,6 +1407,8 @@ oifs: # # When we branch off from an existing experiment, we must set ini_pseudo_initial_date # in the runscript to inform ESM-Tools what initial_date it should trick OpenIFS it is. + # ini_pseudo_initial_date is set automatically to prev_run.general.start_date a few + # lines below, but can also be set manually from the runscript. # For example, a piControl may start from 1850 and run to 2850. # A historical run would then start from 2850 of the existing run, but we would set # start_date for the new run to be 1850 and also trick OpenIFS to think initial_date is 1849. @@ -1399,6 +1420,7 @@ oifs: choose_eternal_run_number: 0: # If we are branching off from an existing run + ini_pseudo_initial_date: "${prev_run.general.start_date}" pseudo_initial_date: "${ini_pseudo_initial_date}" start_ndays_source: "${prev_run.oifs.next_ndays}" branchoff: true @@ -1497,6 +1519,44 @@ oifs: method: "${preprocess_method}" type: shell + append_to_namelist: + append_to_namelist_shell: + method: "${append_to_namelist_method}" + type: shell + + append_to_namelist_method: "" + choose_general.with_co2_oce_coupling: + true: + choose_general.with_co2_veg_coupling: + true: + append_to_namelist_method: " + ${general.esm_function_dir}/components/oifs/append_to_namelist.sh + ${namelist_dir}/namcompo_emis_co2_oce.nml + ${thisrun_config_dir}/fort.4; + ${general.esm_function_dir}/components/oifs/append_to_namelist.sh + ${namelist_dir}/namcompo_emis_co2_veg.nml + ${thisrun_config_dir}/fort.4; + ${general.esm_function_dir}/components/oifs/append_to_namelist.sh + ${namelist_dir}/namcompo_emis_co2_terminator.nml + ${thisrun_config_dir}/fort.4" + false: + append_to_namelist_method: " + ${general.esm_function_dir}/components/oifs/append_to_namelist.sh + ${namelist_dir}/namcompo_emis_co2_oce.nml + ${thisrun_config_dir}/fort.4; + ${general.esm_function_dir}/components/oifs/append_to_namelist.sh + ${namelist_dir}/namcompo_emis_co2_terminator.nml + ${thisrun_config_dir}/fort.4" + false: + choose_general.with_co2_veg_coupling: + true: + append_to_namelist_method: " + ${general.esm_function_dir}/components/oifs/append_to_namelist.sh + ${namelist_dir}/namcompo_emis_co2_veg.nml + ${thisrun_config_dir}/fort.4; + ${general.esm_function_dir}/components/oifs/append_to_namelist.sh + ${namelist_dir}/namcompo_emis_co2_terminator.nml + ${thisrun_config_dir}/fort.4" #====================================================================================== # POSTPROCESS @@ -1543,6 +1603,7 @@ oifs: - "write_env" - "preprocess" - "modify_namelists" + - "append_to_namelist" - "modify_files" - "copy_files_to_work" - "report_missing_files" diff --git a/configs/components/oifs/oifs48.cmip.yaml b/configs/components/oifs/oifs48.cmip.yaml index 8c88cdbd3..2c2840a5d 100644 --- a/configs/components/oifs/oifs48.cmip.yaml +++ b/configs/components/oifs/oifs48.cmip.yaml @@ -1,29 +1,69 @@ -choose_oifs.mip: - "cmip5": - add_namelist_changes: - fort.4: - NAMECECMIP: - CMIP5DATADIR: "${cmip5_data_dir_nml}" - "cmip6": - add_namelist_changes: - fort.4: - NAMECECMIP: - LCMIP6: ".true." - CMIP6DATADIR: "${cmip6_data_dir_nml}" - choose_scenario: - "piControl": - add_namelist_changes: - fort.4: - NAMECECMIP: - NCMIPFIXYR: 1850 - "SSP5-8.5": - add_namelist_changes: - fort.4: - NAMECECMIP: - SCENARIONAME: "SSP5-8.5" - "cmip7": - add_namelist_changes: - fort.4: - NAMECECMIP: - LCMIP7: ".true." - CMIP7DATADIR: "${cmip7_data_dir_nml}" +oifs: + cmip7_data_dir: ${input_dir}/cmip7-data + choose_mip: + "cmip5": + add_namelist_changes: + fort.4: + NAMECECMIP: + CMIP5DATADIR: "${oifs.cmip5_data_dir_nml}" + "cmip6": + add_namelist_changes: + fort.4: + NAMECECMIP: + LCMIP6: ".true." + CMIP6DATADIR: "${oifs.cmip6_data_dir_nml}" + NAERAD: + CO3DATADIR: "${oifs.cmip6_data_dir_nml}/" + choose_scenario: + "piControl": + add_namelist_changes: + fort.4: + NAMECECMIP: + NCMIPFIXYR: 1850 + "SSP5-8.5": + add_namelist_changes: + fort.4: + NAMECECMIP: + SCENARIONAME: "SSP5-8.5" + "SSP3-7.0": + add_namelist_changes: + fort.4: + NAMECECMIP: + SCENARIONAME: "SSP3-7.0" + "SSP2-4.5": + add_namelist_changes: + fort.4: + NAMECECMIP: + SCENARIONAME: "SSP2-4.5" + "SSP1-2.6": + add_namelist_changes: + fort.4: + NAMECECMIP: + SCENARIONAME: "SSP1-2.6" + "cmip7": + choose_scenario: + "piControl": + spinup_year: 1850 + add_namelist_changes: + fort.4: + NAMECECMIP: + NCMIPFIXYR: '${spinup_year}' + NAERAD: + LCMIP_STRATAER_CMIP7: True + LCMIP_STRATAER_BCKGD: False + NCMIPFIXYR: '${spinup_year}' + add_namelist_changes: + fort.4: + NAMECECMIP: + LCMIP7: True + LCMIP6: False + CMIP7DATADIR: '${cmip7_data_dir}' + LMACV2SP: True + LMACV2SP_CCNF: True + NAERAD: + NO3CMIP: 7 + CO3DATADIR: '${cmip7_data_dir}' + CCMIP_STRAT_FULL: '/work/ab0246/a270092/input/oifs-48r1/cmip7-data/strat_aer/CMIP7_stratospheric_aerosols_opt_EC_EARTH_historical_1750-2023_2-2-1_${levels}.nc' + CCMIP_STRAT_FULL_CLIM: '/work/ab0246/a270092/input/oifs-48r1/cmip7-data/strat_aer/CMIP7_stratospheric_aerosols_opt_EC_EARTH_climatology2-2-1_${levels}.nc' + NAMMCC: + LRDALB: True diff --git a/configs/components/oifs/oifs48.env.yaml b/configs/components/oifs/oifs48.env.yaml index 049f54da0..8a318563a 100644 --- a/configs/components/oifs/oifs48.env.yaml +++ b/configs/components/oifs/oifs48.env.yaml @@ -4,8 +4,8 @@ computer: choose_computer.name: levante: - compiler_mpi: gnu11_ompi4 - iolibraries: system_gnu_libs + compiler_mpi: intel2022_openmpi + iolibraries: system_intel_libs load_python: True add_export_vars: DATA: "${general.work_dir}" @@ -14,7 +14,16 @@ computer: OIFS_LOGFILE: '${oifs.model_dir}/build/oifs_test_log.txt' DR_HOOK_IGNORE_SIGNALS: '-1' OIFS_DATA_DIR: "/p/project/chhb19/streffing1/input/oifs-48r1/" - MKL_CBWR: AUTO,STRICT + # MKL_CBWR: AUTO,STRICT # DISABLED: kills performance by disabling vectorization + OMP_SCHEDULE: STATIC + OMP_STACKSIZE: 512M + # Intel MKL performance optimizations + MKL_NUM_THREADS: "1" + MKL_DYNAMIC: "FALSE" + # Intel MPI collective optimizations + I_MPI_ADJUST_ALLREDUCE: "3" + I_MPI_ADJUST_BCAST: "1" + I_MPI_DEBUG: "0" juwels: compiler_mpi: intel2025_psmpi @@ -26,7 +35,32 @@ computer: GRIB_SAMPLES_PATH: '${oifs.model_dir}/build/share/eccodes/ifs_samples/grib1_mlgrib2' OIFS_LOGFILE: '${oifs.model_dir}/build/oifs_test_log.txt' DR_HOOK_IGNORE_SIGNALS: '-1' + OMP_SCHEDULE: STATIC + OMP_STACKSIZE: 512M + albedo: + add_module_actions: + - "unload eccodes" + add_export_vars: + MKL_CBWR: AUTO,STRICT + DATA: "${general.work_dir}" + LD_LIBRARY_PATH[(3)]: '$LD_LIBRARY_PATH:${thisrun_work_dir}/lib/oifs/' + GRIB_SAMPLES_PATH: '${oifs.model_dir}/build/share/eccodes/ifs_samples/grib1_mlgrib2' + OIFS_LOGFILE: '${oifs.model_dir}/build/oifs_test_log.txt' + DR_HOOK_IGNORE_SIGNALS: '-1' + OMP_SCHEDULE: STATIC + OMP_STACKSIZE: 512M + + aleph: + add_export_vars: + DATA: "${general.work_dir}" + LD_LIBRARY_PATH[(3)]: '$LD_LIBRARY_PATH:${thisrun_work_dir}/lib/oifs/' + PATH: '$PATH:${oifs.model_dir}/install/bin/' + GRIB_SAMPLES_PATH: '${oifs.model_dir}/build/share/eccodes/ifs_samples/grib1_mlgrib2' + OIFS_LOGFILE: '${oifs.model_dir}/build/oifs_test_log.txt' + DR_HOOK_IGNORE_SIGNALS: '-1' + OMP_SCHEDULE: STATIC + OMP_STACKSIZE: 512M '*': add_export_vars: MKL_CBWR: AUTO,STRICT @@ -35,5 +69,7 @@ computer: GRIB_SAMPLES_PATH: '${oifs.model_dir}/build/share/eccodes/ifs_samples/grib1_mlgrib2' OIFS_LOGFILE: '${oifs.model_dir}/build/oifs_test_log.txt' DR_HOOK_IGNORE_SIGNALS: '-1' + OMP_SCHEDULE: STATIC + OMP_STACKSIZE: 512M diff --git a/configs/components/oifs/oifs48.tuning.yaml b/configs/components/oifs/oifs48.tuning.yaml new file mode 100644 index 000000000..72038c55e --- /dev/null +++ b/configs/components/oifs/oifs48.tuning.yaml @@ -0,0 +1,15 @@ +oifs: + add_namelist_changes: + fort.4: + NAERAD: + LAER3D: False + NRADFR: -3 + NDECOLAT: 0 + NAMGWWMS: + GGAUSSB: -0.5 + choose_resolution: + '*': + add_add_namelist_changes: + fort.4: + NAMCLDP: + RVICE: 0.16 diff --git a/configs/components/rnfmap/rnfmap.env.yaml b/configs/components/rnfmap/rnfmap.env.yaml index b809759b6..c6e08144c 100644 --- a/configs/components/rnfmap/rnfmap.env.yaml +++ b/configs/components/rnfmap/rnfmap.env.yaml @@ -26,6 +26,10 @@ computer: add_export_vars: RNF_FFLAGS: '"-fdefault-real-8 -march=core-avx2 -mtune=core-avx2 -O3 -fopenmp -g -fbacktrace -fconvert=big-endian"' RNF_CFLAG: '"-march=core-avx2 -mtune=core-avx2 -O3 -fopenmp -g -fbacktrace -fno-opt-report -fpe0"' + gnu2024_ompi2024: + add_export_vars: + RNF_FFLAGS: '"-fdefault-real-8 -march=core-avx2 -mtune=core-avx2 -O3 -fopenmp -g -fbacktrace -fconvert=big-endian"' + RNF_CFLAG: '"-march=core-avx2 -mtune=core-avx2 -O3 -fopenmp -g -fbacktrace -fno-opt-report -fpe0"' '*': add_export_vars: RNF_FFLAGS: '"-r8 -fp-model precise -align array32byte -O3 -qopenmp -g -traceback -convert big_endian -march=core-avx2 -mtune=core-avx2"' @@ -66,8 +70,15 @@ computer: RNF_NETCDFF_INCLUDE: '"-I$NETCDFFROOT/include"' RNF_NETCDFF_LIB: '"-L$NETCDFFROOT/lib -lnetcdff"' # compilers and compile switches - RNF_FFLAGS: '"-r8 -fp-model precise -align array32byte -O3 -qopenmp -g -traceback -convert big_endian -march=core-avx2 -mtune=core-avx2"' # -qoverride-limits -fast-transcendentals -m64 -fma -pc64"' - RNF_CFLAGS: '"-fp-model precise -O3 -g -traceback -qopt-report=0 -fpe0 -qopenmp -march=core-avx2 -mtune=core-avx2"' # -qoverride-limits -fast-transcendentals -m64 -fma -pc64"' + choose_computer.compiler_suite: + gcc: + add_export_vars: + RNF_FFLAGS: '"-fdefault-real-8 -march=znver2 -mtune=znver2 -O3 -fopenmp -g -fbacktrace -fconvert=big-endian -fallow-argument-mismatch"' + RNF_CFLAGS: '"-march=znver2 -mtune=znver2 -O3 -fopenmp -g"' + intel-oneapi: + add_export_vars: + RNF_FFLAGS: '"-r8 -fp-model precise -align array32byte -O3 -qopenmp -g -traceback -convert big_endian -march=core-avx2 -mtune=core-avx2"' + RNF_CFLAGS: '"-fp-model precise -O3 -g -traceback -qopt-report=0 -fpe0 -qopenmp -march=core-avx2 -mtune=core-avx2"' aleph: diff --git a/configs/components/rnfmap/rnfmap.yaml b/configs/components/rnfmap/rnfmap.yaml index fd85c252c..fc8a57408 100644 --- a/configs/components/rnfmap/rnfmap.yaml +++ b/configs/components/rnfmap/rnfmap.yaml @@ -29,6 +29,7 @@ rnfmap: - foci211 - v1.0 - v1.1 + - v1.2 runoff_file: '${input_dir}/runoff_maps${runoff_maps_name}.nc' @@ -67,6 +68,12 @@ rnfmap: - rnfmap/rnfmap.env.yaml destination: rnfmap runoff_file: '${input_dir}/runoff_maps${runoff_maps_name}_v1.1.nc' + v1.2: + branch: '1.2' + further_reading: + - rnfmap/rnfmap.env.yaml + destination: rnfmap + runoff_file: '${input_dir}/runoff_maps${runoff_maps_name}_v1.1.nc' agrif: branch: agrif diff --git a/configs/components/xios/xios.env.yaml b/configs/components/xios/xios.env.yaml index d48f15c65..1f6ac4b74 100644 --- a/configs/components/xios/xios.env.yaml +++ b/configs/components/xios/xios.env.yaml @@ -1,6 +1,5 @@ computer: choose_computer.name: - ollie: + levante: add_module_actions: - - "unload netcdf" - - "load netcdf/4.4.1.1_intel_mpi" + - "load perl" diff --git a/configs/components/xios/xios.yaml b/configs/components/xios/xios.yaml index 234cc6858..21e555714 100644 --- a/configs/components/xios/xios.yaml +++ b/configs/components/xios/xios.yaml @@ -34,6 +34,7 @@ xios: - 2.5_r1910_ogcm - 2.5_smhi - 2.5_ece + - 2.5.1 branch: esm-tools @@ -97,6 +98,8 @@ xios: choose_computer.compiler_mpi: gnu2023_ompi2023: archfile: ESMTOOLS_generic_oasis_gcc + gnu2024_ompi2024: + archfile: ESMTOOLS_generic_oasis_gcc "*": archfile: ESMTOOLS_generic_oasis_intel use_oasis: --use_oasis oasis3_mct @@ -135,12 +138,55 @@ xios: choose_computer.compiler_mpi: gnu2023_ompi2023: archfile: ESMTOOLS_generic_oasis_gcc + gnu2024_ompi2024: + archfile: ESMTOOLS_generic_oasis_gcc "*": foo: bar use_oasis: --use_oasis oasis3_mct branch: main comp_command: export XIOS_TOPLEVEL=${model_dir}; ./make_xios --arch ${archfile} --netcdf_lib netcdf4_par ${use_oasis} --job 24 --prod; cp bin/xios_server.exe bin/xios.x # XIOS with oasis dependencies and oasis support + 2.5.1: + git-repository: https://gitlab.dkrz.de/ec-earth/xios-2.5-ece.git + choose_computer.useMPI: # JS: This entire block is a missnomer. It's not the MPI thats imporant but the compiler cray ftn vs intel. + cray_mpich: + archfile: ESMTOOLS_generic_oasis_cray + intelmpi: + archfile: ESMTOOLS_generic_oasis_intel + parastationmpi: + archfile: ESMTOOLS_generic_oasis_intel + "*": + foo: bar + # MA: yet another block, this time for albedo. We cannot fix this now, but in the + # future we should adopt the compiler suite logic for the other computers. Already + # included in ESM-Tools clickup + choose_computer.compiler_suite: + gcc: + archfile: ESMTOOLS_generic_oasis_gcc + intel-oneapi: + archfile: ESMTOOLS_levante_oasis_intel + "*": + foo: bar + # JS: On levante this switch has yet another name. Again only the compiler part matters. + choose_computer.name: + levante: + choose_computer.compiler_mpi: + gnu11_ompi4: + archfile: ESMTOOLS_generic_oasis_gcc + "*": + archfile: ESMTOOLS_levante_oasis_intel + juwels: + choose_computer.compiler_mpi: + gnu2023_ompi2023: + archfile: ESMTOOLS_generic_oasis_gcc + gnu2024_ompi2024: + archfile: ESMTOOLS_generic_oasis_gcc + "*": + foo: bar + use_oasis: --use_oasis oasis3_mct + branch: 2.5.1 + comp_command: export XIOS_TOPLEVEL=${model_dir}; ./make_xios --arch ${archfile} --netcdf_lib netcdf4_par ${use_oasis} --job 24 --prod; cp bin/xios_server.exe bin/xios.x + # XIOS with oasis dependencies and oasis support "2.5-yac": git-repository: https://gitlab.dkrz.de/ec-earth/xios-2.5.git choose_computer.useMPI: # JS: This entire block is a missnomer. It's not the MPI thats imporant but the compiler cray ftn vs intel. @@ -370,5 +416,5 @@ xios: bin_sources: server: ${bin_dir}/xios.x - further_reading: - - xios/xios.env.yaml + #further_reading: + # - xios/xios.env.yaml diff --git a/configs/couplings/awicm3_v3.4.0/awicm3_v3.4.0.yaml b/configs/couplings/awicm3_v3.4.0/awicm3_v3.4.0.yaml new file mode 100644 index 000000000..2c2f1ed2e --- /dev/null +++ b/configs/couplings/awicm3_v3.4.0/awicm3_v3.4.0.yaml @@ -0,0 +1,10 @@ +components: +- xios-2.5.1 +- rnfmap-v1.2 +- oifs-48r1v1 +- fesom-2.7.2 +- oasis3mct-5.1 +coupling_changes: +- sed -i 's/xios-2.5+/xios/g' oifs-48r1/bundle.yml +- sed -i 's/xios\/build_ecearth/xios/g' oifs-48r1/bundle.yml +- sed -i 's/oasis3-mct-5.2/oasis/g' oifs-48r1/bundle.yml diff --git a/configs/couplings/awicm3_v3.4.1/awicm3_v3.4.1.yaml b/configs/couplings/awicm3_v3.4.1/awicm3_v3.4.1.yaml new file mode 100644 index 000000000..0bb496d65 --- /dev/null +++ b/configs/couplings/awicm3_v3.4.1/awicm3_v3.4.1.yaml @@ -0,0 +1,10 @@ +components: +- xios-2.5.1 +- rnfmap-v1.2 +- oifs-48r1v2 +- fesom-2.7.3 +- oasis3mct-5.1 +coupling_changes: +- sed -i 's/xios-2.5+/xios/g' oifs-48r1/bundle.yml +- sed -i 's/xios\/build_ecearth/xios/g' oifs-48r1/bundle.yml +- sed -i 's/oasis3-mct-5.2/oasis/g' oifs-48r1/bundle.yml diff --git a/configs/couplings/awiesm3_v3.4.0/awiesm3_v3.4.0.yaml b/configs/couplings/awiesm3_v3.4.0/awiesm3_v3.4.0.yaml new file mode 100644 index 000000000..8219f418d --- /dev/null +++ b/configs/couplings/awiesm3_v3.4.0/awiesm3_v3.4.0.yaml @@ -0,0 +1,11 @@ +components: +- xios-2.5.1 +- rnfmap-v1.2 +- oifs-48r1v1 +- fesom-2.7.2 +- oasis3mct-5.1 +- lpj_guess-4.1.3 +coupling_changes: +- sed -i 's/xios-2.5+/xios/g' oifs-48r1/bundle.yml +- sed -i 's/xios\/build_ecearth/xios/g' oifs-48r1/bundle.yml +- sed -i 's/oasis3-mct-5.2/oasis/g' oifs-48r1/bundle.yml diff --git a/configs/couplings/awiesm3_v3.4.1/awiesm3_v3.4.1.yaml b/configs/couplings/awiesm3_v3.4.1/awiesm3_v3.4.1.yaml new file mode 100644 index 000000000..97f6d6367 --- /dev/null +++ b/configs/couplings/awiesm3_v3.4.1/awiesm3_v3.4.1.yaml @@ -0,0 +1,11 @@ +components: +- xios-2.5.1 +- rnfmap-v1.2 +- oifs-48r1v2 +- fesom-2.7.3 +- oasis3mct-5.1 +- lpj_guess-4.1.3 +coupling_changes: +- sed -i 's/xios-2.5+/xios/g' oifs-48r1/bundle.yml +- sed -i 's/xios\/build_ecearth/xios/g' oifs-48r1/bundle.yml +- sed -i 's/oasis3-mct-5.2/oasis/g' oifs-48r1/bundle.yml diff --git a/configs/couplings/fesom-2.6-main+oifs-43r3-awicm-3.2.4+xios-2.5/fesom-2.6-main+oifs-43r3-awicm-3.2.4+xios-2.5.yaml b/configs/couplings/fesom-2.6-main+oifs-43r3-awicm-3.2.4+xios-2.5/fesom-2.6-main+oifs-43r3-awicm-3.2.4+xios-2.5.yaml new file mode 100644 index 000000000..2552de8ff --- /dev/null +++ b/configs/couplings/fesom-2.6-main+oifs-43r3-awicm-3.2.4+xios-2.5/fesom-2.6-main+oifs-43r3-awicm-3.2.4+xios-2.5.yaml @@ -0,0 +1,11 @@ +components: +- xios-2.5 +- rnfmap-awicm-3.1 +- oifs-43r3-awicm-3.2.4 +- fesom-2.6.8 +- oasis3mct-4.0-awicm-3.2 +coupling_changes: +- sed -i '/COUPLENEMOECE = /s/.TRUE./.FALSE./g' oifs-43r3/src/ifs/module/yommcc.F90 +- sed -i '/COUPLEFESOM2 = /s/.FALSE./.TRUE./g' oifs-43r3/src/ifs/module/yommcc.F90 +- sed -i '/COUPLENEMOFOCI = /s/.TRUE./.FALSE./g' oifs-43r3/src/ifs/module/yommcc.F90 + diff --git a/configs/couplings/fesom-2.6.11+echam-6.3.05p2-awiesm-2.1/fesom-2.6.11+echam-6.3.05p2-awiesm-2.1.yaml b/configs/couplings/fesom-2.6.11+echam-6.3.05p2-awiesm-2.1/fesom-2.6.11+echam-6.3.05p2-awiesm-2.1.yaml new file mode 100644 index 000000000..f0b942de3 --- /dev/null +++ b/configs/couplings/fesom-2.6.11+echam-6.3.05p2-awiesm-2.1/fesom-2.6.11+echam-6.3.05p2-awiesm-2.1.yaml @@ -0,0 +1,8 @@ +components: +- echam-6.3.05p2-awiesm-2.1 +- fesom-2.6.11 +- oasis3mct-2.8-paleodyn +coupling_changes: +- sed -i '/set(FESOM_COUPLED/s/OFF/ON/g' fesom-2.6/CMakeLists.txt +- sed -i '/ECHAM6_COUPLED/s/OFF/ON/g' echam-6.3.05p2/config/mh-linux +- sed -ir '/..FC_DEFINE}__cpl_mpiom/s/..FC_DEFINE}__cpl_mpiom//g' echam-6.3.05p2/configure.ac diff --git a/configs/couplings/fesom-2.7+oifs-48r1+xios-2.5+lpj_guess-4.1.2/fesom-2.7+oifs-48r1+xios-2.5+lpj_guess-4.1.2.yaml b/configs/couplings/fesom-2.7+oifs-48r1+xios-2.5+lpj_guess-4.1.2/fesom-2.7+oifs-48r1+xios-2.5+lpj_guess-4.1.2.yaml new file mode 100644 index 000000000..0fc843387 --- /dev/null +++ b/configs/couplings/fesom-2.7+oifs-48r1+xios-2.5+lpj_guess-4.1.2/fesom-2.7+oifs-48r1+xios-2.5+lpj_guess-4.1.2.yaml @@ -0,0 +1,11 @@ +components: +- xios-2.5-ece +- rnfmap-v1.1 +- oifs-48r1 +- fesom-2.7 +- oasis3mct-5.0-smhi +- lpj_guess-4.1.2 +coupling_changes: +- sed -i 's/xios-2.5+/xios/g' oifs-48r1/bundle.yml +- sed -i 's/xios\/build_ecearth/xios/g' oifs-48r1/bundle.yml +- sed -i 's/oasis3-mct-5.2/oasis/g' oifs-48r1/bundle.yml diff --git a/configs/couplings/fesom-2.6.9+oifs-48r1+xios-2.5/fesom-2.6.9+oifs-48r1+xios-2.5.yaml b/configs/couplings/fesom-2.7+oifs-48r1+xios-2.5/fesom-2.7+oifs-48r1+xios-2.5.yaml similarity index 98% rename from configs/couplings/fesom-2.6.9+oifs-48r1+xios-2.5/fesom-2.6.9+oifs-48r1+xios-2.5.yaml rename to configs/couplings/fesom-2.7+oifs-48r1+xios-2.5/fesom-2.7+oifs-48r1+xios-2.5.yaml index 7757a0b88..23ef2b695 100644 --- a/configs/couplings/fesom-2.6.9+oifs-48r1+xios-2.5/fesom-2.6.9+oifs-48r1+xios-2.5.yaml +++ b/configs/couplings/fesom-2.7+oifs-48r1+xios-2.5/fesom-2.7+oifs-48r1+xios-2.5.yaml @@ -2,7 +2,7 @@ components: - xios-2.5-ece - rnfmap-v1.1 - oifs-48r1 -- fesom-2.6.9 +- fesom-2.7 - oasis3mct-5.0-smhi coupling_changes: - sed -i 's/xios-2.5+/xios/g' oifs-48r1/bundle.yml diff --git a/configs/couplings/lpj_guess-4.1.2+oasis/lpj_guess-4.1.2+oasis.yaml b/configs/couplings/lpj_guess-4.1.2+oasis/lpj_guess-4.1.2+oasis.yaml new file mode 100644 index 000000000..0d6387062 --- /dev/null +++ b/configs/couplings/lpj_guess-4.1.2+oasis/lpj_guess-4.1.2+oasis.yaml @@ -0,0 +1,3 @@ +components: +- oasis3mct-5.0-smhi +- lpj_guess-4.1.2 diff --git a/configs/couplings/lpjg-spinup_v3.4.0/lpjg-spinup_v3.4.0.yaml b/configs/couplings/lpjg-spinup_v3.4.0/lpjg-spinup_v3.4.0.yaml new file mode 100644 index 000000000..82db6a8b7 --- /dev/null +++ b/configs/couplings/lpjg-spinup_v3.4.0/lpjg-spinup_v3.4.0.yaml @@ -0,0 +1,3 @@ +components: +- lpj_guess-4.1.3 +- oasis3mct-5.1 diff --git a/configs/esm_software/esm_runscripts/esm_plugins.yaml b/configs/esm_software/esm_runscripts/esm_plugins.yaml index 47158d88e..d0e2c472f 100644 --- a/configs/esm_software/esm_runscripts/esm_plugins.yaml +++ b/configs/esm_software/esm_runscripts/esm_plugins.yaml @@ -65,7 +65,7 @@ core: observe: - - "init_monitor_file" + - "init_observe_logs" - "wait_and_observe" - "wake_up_call" diff --git a/configs/esm_software/esm_runscripts/esm_runscripts.yaml b/configs/esm_software/esm_runscripts/esm_runscripts.yaml index 8e425a98e..09a29c313 100644 --- a/configs/esm_software/esm_runscripts/esm_runscripts.yaml +++ b/configs/esm_software/esm_runscripts/esm_runscripts.yaml @@ -58,7 +58,7 @@ choose_job_type: observe: recipe: - - "init_monitor_file" + - "init_observe_logs" - "wait_and_observe" - "wake_up_call" diff --git a/configs/machines/albedo.yaml b/configs/machines/albedo.yaml index 157a6e3e8..fcdae7e20 100644 --- a/configs/machines/albedo.yaml +++ b/configs/machines/albedo.yaml @@ -81,7 +81,7 @@ computer: compiler_module: gcc/12.1.0 mkl_module: intel-oneapi-mkl/2022.1.0 mkl_root: /albedo/soft/sw/spack-sw/intel-oneapi-mkl/2022.1.0-akthm3n/mkl/2022.1.0 - mpi_module: openmpi/4.1.3-gcc12.1.0 + mpi_module: openmpi/4.1.3 hdf5_module: hdf5/1.12.2-openmpi4.1.3-gcc12.1.0 netcdfc_module: netcdf-c/4.8.1-openmpi4.1.3-gcc12.1.0 netcdff_module: netcdf-fortran/4.5.4-openmpi4.1.3-gcc12.1.0 @@ -105,6 +105,7 @@ computer: compiler_module: intel-oneapi-compilers/2022.1.0 mkl_module: intel-oneapi-mkl/2022.1.0-gcc12.1.0 mkl_root: /albedo/soft/sw/spack-sw/intel-oneapi-mkl/2022.1.0-akthm3n/mkl/2022.1.0 + tbb_root: /albedo/soft/sw/spack-sw/intel-oneapi-mkl/2022.1.0-akthm3n/tbb/latest mpi_module: openmpi/4.1.3-intel2021.6.0 hdf5_module: hdf5/1.12.2-openmpi4.1.3-intel2021.6.0 netcdfc_module: netcdf-c/4.8.1-openmpi4.1.3-intel2021.6.0 @@ -124,10 +125,14 @@ computer: NETCDFROOT: "/albedo/soft/sw/spack-sw/netcdf-c/4.8.1-vvxxdc3/" NETCDFFROOT: "/albedo/soft/sw/spack-sw/netcdf-fortran/4.5.4-uwfs3bu/" ECCODESROOT: "/albedo/soft/sw/spack-sw/eccodes/2.25.0-vn2k575/" + # TBB library + TBBROOT: "/albedo/soft/sw/spack-sw/intel-oneapi-mkl/2022.1.0-akthm3n/tbb/latest" module_actions: # Ensure a clean environment before start: - "purge" + # Build systems + - "load cmake" # Compilers: - "load ${compiler_module}" - "load ${mkl_module}" @@ -140,6 +145,7 @@ computer: - "load ${netcdff_module}" - "load ${netcdfcxx_module}" - "load ${eccodes_module}" + - "load libaec" # General Tools: - "load cdo/2.0.5" - "load nco/5.0.1" @@ -191,6 +197,7 @@ computer: LAPACK_LIB: "'-L${mkl_root} -lmkl_intel_lp64 -lmkl_core -lmkl_sequential -lm -ldl'" LD_LIBRARY_PATH[(1)]: ${mkl_root}/lib/intel64:$LD_LIBRARY_PATH + LD_LIBRARY_PATH[(2)]: ${tbb_root}/lib/intel64/gcc4.8:$LD_LIBRARY_PATH PATH[(1)]: "$PERL5LIB/../bin:$PATH" PATH[(2)]: "$PATH:$ECCODESROOT/bin" #$HDF5ROOT/bin:$NETCDFFROOT/bin:$NETCDFROOT/bin:$ECCODESROOT/bin:$PATH diff --git a/configs/machines/aleph.yaml b/configs/machines/aleph.yaml index 266f681fc..839e92394 100644 --- a/configs/machines/aleph.yaml +++ b/configs/machines/aleph.yaml @@ -55,6 +55,7 @@ computer: mpicc: cc cc: cc cxx: CC + c++_lib: cray-c++-rts module_actions: - "unload craype" @@ -63,14 +64,13 @@ computer: - "load pbs" - "load cray-mpich/7.7.3" - "load craype-x86-skylake" - - "load cmake/3.14.0" + - "load cmake/3.28.3" - "load cray-hdf5-parallel/1.10.2.0" - "load cray-netcdf-hdf5parallel/4.6.1.3" - "load cdo/1.9.5" - "load fftw/2.1.5.9" - "load nco/4.9.4" - "load proj4/5.1.0" - - "load python/3.9.1" - "list" export_vars: @@ -84,6 +84,8 @@ computer: NETCDF_CXX_INCLUDE_DIRECTORIES: $NETCDFROOT/include NETCDF_CXX_LIBRARIES: $NETCDFROOT/lib + AEC_ROOT: /proj/awi/software/HPC_LIBRARIES/PrgEnv-cray-6.0.4/libs + # For OASIS3-MCT5 from CERFACS OASIS_NETCDF: $NETCDF_DIR OASIS_NETCDFF: $NETCDF_DIR @@ -113,6 +115,8 @@ computer: # Important for eccodes to work CRAYPE_LINK_TYPE: dynamic + LD_LIBRARY_PATH: $AEC_ROOT/lib64:$LD_LIBRARY_PATH + label_format: '"[@MODEL@]%l:"' launcher_flags_per_component: "-n @nproc@ -N @nproc_per_node@ -d @cpus_per_proc@ env OMP_NUM_THREADS=@omp_num_threads@ diff --git a/configs/machines/juwels.yaml b/configs/machines/juwels.yaml index 9415727ef..9ef4f95af 100644 --- a/configs/machines/juwels.yaml +++ b/configs/machines/juwels.yaml @@ -132,6 +132,25 @@ computer: MPIROOT: "\"$($FC -show | perl -lne 'm{ -I(.*?)/include } and print $1')\"" MPI_LIB: "\"$($FC -show |sed -e 's/^[^ ]*//' -e 's/-[I][^ ]*//g')\"" + gnu2024_ompi2024: + add_module_actions: + - "load Stages/2024" + - "load GCC" + - "load OpenMPI" + - "load CMake" + - "load Python" + - "load imkl" + - "load Perl" + - "load git" + add_export_vars: + FC: mpifort + F77: mpifort + MPIFC: mpifort + FCFLAGS: -free + CC: mpicc + CXX: mpic++ + MPIROOT: "\"$($FC -show | perl -lne 'm{ -I(.*?)/include } and print $1')\"" + MPI_LIB: "\"$($FC -show |sed -e 's/^[^ ]*//' -e 's/-[I][^ ]*//g')\"" choose_iolibraries: diff --git a/configs/machines/levante.yaml b/configs/machines/levante.yaml index ad777760b..f566c569c 100644 --- a/configs/machines/levante.yaml +++ b/configs/machines/levante.yaml @@ -157,6 +157,8 @@ computer: MPI_LIB: "\"$(mpif90 -show |sed -e 's/^[^ ]*//' -e 's/-[I][^ ]*//g')\"" intel2022_impi2021: + launcher_flags: "-l --mpi=pmi2" # Fix for Intel MPI PMI2_Job_GetId errors + python_version: "python3" # ESM-Tools vars necessary for ``system_libs`` case mod_netcdf_c: netcdf-c/4.8.1-intel-oneapi-mpi-2021.5.0-intel-2021.5.0 mod_netcdf_f: netcdf-fortran/4.5.3-intel-oneapi-mpi-2021.5.0-intel-2021.5.0 @@ -177,6 +179,7 @@ computer: MPI_LIB: "\"$(mpiifort -show |sed -e 's/^[^ ]*//' -e 's/-[I][^ ]*//g')\"" intel2022_openmpi: + python_version: "python3" # ESM-Tools vars necessary for ``system_libs`` case mod_netcdf_c: netcdf-c/4.8.1-openmpi-4.1.2-intel-2021.5.0 mod_netcdf_f: netcdf-fortran/4.5.3-openmpi-4.1.2-intel-2021.5.0 @@ -343,16 +346,20 @@ computer: LD_RUN_PATH: $LD_LIBRARY_PATH AEC_ROOT: /sw/spack-levante/libaec-1.0.5-gij7yv aec_ROOT: /sw/spack-levante/libaec-1.0.5-gij7yv + UCXROOT: /sw/spack-levante/ucx-1.12.0-nlas56 HDF5ROOT: /sw/spack-levante/hdf5-1.12.1-tvymb5 HDF5_C_INCLUDE_DIRECTORIES: $HDF5ROOT/include HDF5_ROOT: $HDF5ROOT - MPIROOT: "\"$(mpif90 -show | perl -lne 'm{ -I(.*?)/include } and print $1')\"" NetCDF_C_ROOT: /sw/spack-levante/netcdf-c-4.8.1-2k3cmu NetCDF_Fortran_ROOT: /sw/spack-levante/netcdf-fortran-4.5.3-k6xq5g NETCDF_PATH: $NetCDF_C_ROOT NETCDF_ROOT: $NetCDF_Fortran_ROOT NETCDFROOT: $NetCDF_C_ROOT NETCDFFROOT: $NetCDF_Fortran_ROOT + NETCDF_Fortran_INCLUDE_DIRECTORIES: $NETCDFFROOT/include + NETCDF_C_INCLUDE_DIRECTORIES: $NETCDFROOT/include + NETCDF_CXX_INCLUDE_DIRECTORIES: $NETCDFROOT/include + NETCDF_CXX_LIBRARIES: $NETCDFROOT/lib TBBROOT: /sw/spack-levante/intel-oneapi-compilers-2022.0.1-an2cbq/tbb/2021.5.0 TBBMALLOC_DIR: $TBBROOT/lib/intel64/gcc4.8 MPI_HOME: /sw/spack-levante/openmpi-4.1.2-yfwe6t @@ -360,7 +367,7 @@ computer: mpi_ROOT: /sw/spack-levante/openmpi-4.1.2-yfwe6t MPI_DIR: /sw/spack-levante/openmpi-4.1.2-yfwe6t mpi_DIR: /sw/spack-levante/openmpi-4.1.2-yfwe6t - LD_LIBRARY_PATH: $MPI_HOME/lib:$NetCDF_C_ROOT/lib:$NetCDF_Fortran_ROOT/lib:$HDF5ROOT/lib:$AEC_ROOT/lib:$TBBMALLOC_DIR:$LD_LIBRARY_PATH + LD_LIBRARY_PATH: $MPI_HOME/lib:$NetCDF_C_ROOT/lib:$NetCDF_Fortran_ROOT/lib:$HDF5ROOT/lib:$AEC_ROOT/lib:$UCXROOT/lib:$TBBMALLOC_DIR:$LD_LIBRARY_PATH LD_LIBRARY_PATH[(0)]: /sw/spack-levante/intel-oneapi-mkl-2022.0.1-ttdktf/mkl/2022.0.1/lib/intel64:$LD_LIBRARY_PATH system_intel_impi_libs: #Copy paste from DE IFS env.sh @@ -379,13 +386,16 @@ computer: HDF5ROOT: /sw/spack-levante/hdf5-1.12.1-jmeuy3 HDF5_C_INCLUDE_DIRECTORIES: $HDF5ROOT/include HDF5_ROOT: $HDF5ROOT - MPIROOT: "\"$(mpif90 -show | perl -lne 'm{ -I(.*?)/include } and print $1')\"" NetCDF_C_ROOT: /sw/spack-levante/netcdf-c-4.8.1-7dq6g2 NetCDF_Fortran_ROOT: /sw/spack-levante/netcdf-fortran-4.5.3-r5r3ev NETCDF_PATH: $NetCDF_C_ROOT NETCDF_ROOT: $NetCDF_Fortran_ROOT NETCDFROOT: $NetCDF_C_ROOT NETCDFFROOT: $NetCDF_Fortran_ROOT + NETCDF_Fortran_INCLUDE_DIRECTORIES: $NETCDFFROOT/include + NETCDF_C_INCLUDE_DIRECTORIES: $NETCDFROOT/include + NETCDF_CXX_INCLUDE_DIRECTORIES: $NETCDFROOT/include + NETCDF_CXX_LIBRARIES: $NETCDFROOT/lib TBBROOT: /sw/spack-levante/intel-oneapi-compilers-2022.0.1-an2cbq/tbb/2021.5.0 TBBMALLOC_DIR: $TBBROOT/lib/intel64/gcc4.8 MPI_HOME: /sw/spack-levante/openmpi-4.1.2-yfwe6t diff --git a/configs/setups/awicm3/awicm3.yaml b/configs/setups/awicm3/awicm3.yaml index 7b2c0e075..f7cb03bc1 100644 --- a/configs/setups/awicm3/awicm3.yaml +++ b/configs/setups/awicm3/awicm3.yaml @@ -9,6 +9,8 @@ general: lresume: False coupled_setup: True + with_co2_oce_coupling: False + with_co2_veg_coupling: False include_models: - fesom @@ -26,6 +28,8 @@ general: - 'v3.2.1' - 'v3.2.2' - 'v3.3.0' + - 'v3.4.0' + - 'v3.4.1' - 'develop' - 'frontiers-xios' @@ -100,10 +104,26 @@ general: - xios oasis_with_yac: False xios_interp_order: 1 + v3.4.0: + major_version: v3.4 + couplings: + - awicm3_v3.4.0 + add_include_models: + - xios + oasis_with_yac: False + xios_interp_order: 1 + v3.4.1: + major_version: v3.4 + couplings: + - awicm3_v3.4.0 + add_include_models: + - xios + oasis_with_yac: False + xios_interp_order: 1 develop: major_version: v3.4 couplings: - - fesom-2.6.9+oifs-48r1+xios-2.5 + - fesom-2.7+oifs-48r1+xios-2.5 add_include_models: - xios oasis_with_yac: False @@ -121,6 +141,7 @@ general: postprocessing: false post_time: "00:05:00" compute_time: "00:30:00" + parallel_file_movements: 'threads' reusable_filetypes: [input, bin, src] @@ -130,7 +151,7 @@ general: ollie: pool_dir: "/work/ollie/jstreffi/input" juwels: - pool_dir: "/p/project/chhb19/jstreffi/input" + pool_dir: "/p/project/chhb19/streffing1/input" aleph: pool_dir: "/proj/awi/input" blogin: @@ -147,7 +168,6 @@ general: # Load general environment for the whole coupled setup dr_hook_ignore_signals: -1 - ######################################################################################### ########### necessary changes to submodels compared to standalone setups ################ ######################################################################################### @@ -241,7 +261,8 @@ oifs: "v3.4": version: "48r1" with_xios: True - comp_command: "./openifs-bundle create; ./openifs-bundle build --threads=32 --openifs-only --without-double-precision --with-cplng2 --with-oifs-xios --install; mv install/bin/ifsMASTER.SP install/bin/OpenIFS" + build_type: 'release' + comp_command: "./openifs-bundle create; ./openifs-bundle build --threads=32 --openifs-only --without-double-precision --with-cplng2 --with-oifs-xios --build-type=${build_type} --install; mv install/bin/ifsMASTER.SP install/bin/OpenIFS" namelist_dir: ${general.esm_namelist_dir}/oifs/${version}/awicm3/v3.4 mip: "cmip6" wam: true @@ -304,7 +325,7 @@ oifs: levels: L91 res_level: ${resolution}${levels} - awicm3_fields: [A_Qns_oce, A_Qs_all, A_Precip_liquid, A_Precip_solid, A_Evap, A_Subl, A_Q_ice, A_TauX_oce, A_TauY_oce, A_TauX_ice, A_TauY_ice, A_SST, A_Ice_frac, A_Snow_thickness, A_Ice_temp, A_Ice_albedo,A_CurX,A_CurY] + awicm3_fields: [A_Qns_oce, A_Qs_all, A_Precip_liquid, A_Precip_solid, A_Evap, A_Subl, A_Q_ice, A_TauX_oce, A_TauY_oce, A_TauX_ice, A_TauY_ice, A_SST, A_Ice_frac, A_Snow_thickness, A_Ice_temp, A_Ice_albedo, A_CurX, A_CurY, A_WindX, A_WindY] coupling_fields: "[[awicm3_fields-->FIELD]]": @@ -356,13 +377,13 @@ fesom: nproc: 144 TL159_CORE2: nproc: 72 + build_type: "Release" # Available options: Release, Debug, Bit + comp_command: mkdir -p build; cd build; cmake -DOIFS_COUPLED=ON -DFESOM_COUPLED=ON -DENABLE_OPENMP=ON -DOPENMP_REPRODUCIBLE=OFF -DDISABLE_MULTITHREADING=OFF -DDISABLE_PARALLEL_RESTART_READ=ON ${yac_flag} -DCMAKE_BUILD_TYPE=${build_type} -DCMAKE_INSTALL_PREFIX=../ ..; make install -j `nproc --all` choose_general.oasis_with_yac: True: - comp_command: mkdir -p build; cd build; cmake -DOIFS_COUPLED=ON -DFESOM_COUPLED=ON -DENABLE_OPENMP=ON -DOASIS_WITH_YAC=ON -DCMAKE_INSTALL_PREFIX=../ ..; make install -j `nproc --all` + yac_flag: "-DOASIS_WITH_YAC=ON" False: - comp_command: mkdir -p build; cd build; cmake -DOIFS_COUPLED=ON -DFESOM_COUPLED=ON -DENABLE_OPENMP=ON -DDISABLE_MULTITHREADING=OFF -DCMAKE_INSTALL_PREFIX=../ ..; make install -j `nproc --all` - "*": - comp_command: "missing" + yac_flag: "" missing_owy: "$(( 'oasis_with_yac' not in config['general'] ))" # TODO: move this to the block above when the missing option is included in choose_ blocks choose_missing_owy: @@ -383,6 +404,7 @@ fesom: restart_unit: "d" leapyear: True + tar_binary_restarts: false # Move in the future to fesom-2.0.yaml @@ -465,25 +487,23 @@ fesom: albi: 0.6 albim: 0.43 "v3.4": - version: "2.6.9" + version: "2.7" append_icmcl: "" add_namelist_changes: - namelist.config: - ale_def: - which_ALE: 'zstar' - namelist.oce: - oce_dyn: - SPP: true - k_gm_max: 3000 - k_gm_rampmax: 40.0 - k_gm_rampmin: 30 - scaling_ferreira: true - scaling_fesom14: true namelist.tra: tracer_phys: surf_relax_s: "${surf_relax_s}" - use_momix: false - kv0_const: false + namelist.config: + geometry: + metric_factor_zero: True + ale_def: + which_ale: 'linfs' + restart_log: + raw_restart_length_unit: "off" + bin_restart_length_unit: "off" + namelist.ice: + ice_therm: + use_meltponds: true "frontiers-xios": model_tag: "frontiers" @@ -539,6 +559,7 @@ xios: regular_res_lat: 192 regular_res_lon: 400 config_sources: + context_ifs: ${xml_dir}/context_ifs.xml.j2 domain_def: ${xml_dir}/domain_def.xml.j2 file_def: ${xml_dir}/file_def.xml.j2 choose_general.major_version: @@ -712,7 +733,7 @@ oasis3mct: - 'A_SST:A_Ice_frac:A_Snow_thickness:A_Ice_temp:A_Ice_albedo <--gauswgt_i-- sst_feom:sie_feom:snt_feom:ist_feom:sia_feom' - 'A_CurX:A_CurY <--gauswgt_i-- u_feom:v_feom' 'v3.4': - mct_version: 5.0 + mct_version: "5.0" input_dir: ${pool_dir}/cy${oifs.version}/${oifs.resolution}-${fesom.resolution}/ rstos_file_name: 'rstos' rstas_file_name: 'rstas' @@ -724,7 +745,8 @@ oasis3mct: - 'prec_oce:snow_oce:evap_oce:subl_oce <--gauswgt_c-- A_Precip_liquid:A_Precip_solid:A_Evap:A_Subl' - 'heat_ico <--gauswgt_gss-- A_Q_ice' - 'taux_oce:tauy_oce:taux_ico:tauy_ico <--bicubic-- A_TauX_oce:A_TauY_oce:A_TauX_ice:A_TauY_ice' - - 'R_Runoff_atm:R_Calving_atm <--gauswgt_glo-- A_Runoff:A_Calving' + - 'u10w_oce:v10w_oce <--bicubic-- A_WindX:A_WindY' + - 'R_Runoff_atm:R_Calving_atm <--gauswgt_c-- A_Runoff:A_Calving' - 'hydr_oce:calv_oce <--gauswgt_c-- R_Runoff_oce:R_Calving_oce' rstos.nc: - 'A_SST:A_Ice_frac:A_Snow_thickness:A_Ice_temp:A_Ice_albedo <--gauswgt_i-- sst_feom:sie_feom:snt_feom:ist_feom:sia_feom' @@ -973,9 +995,6 @@ oasis3mct: computer: - # Activates the taskset approach for heterogeneous parallelization, by default - # since hetjob approach doesn't seem to work (second run showing mpi/pmix - # problems) taskset: true merge_component_envs: compile: True @@ -984,11 +1003,14 @@ computer: v3.4: choose_name: juwels: + taskset: false iolibraries: system_libs - compiler_mpi: gnu2023_ompi2023 + compiler_mpi: gnu2024_ompi2024 levante: - iolibraries: system_gnu_libs - compiler_mpi: gnu11_ompi4 + iolibraries: system_intel_libs + compiler_mpi: intel2022_openmpi + albedo: + compiler_suite: gcc '*': choose_name: juwels: diff --git a/configs/setups/awiesm/awiesm.yaml b/configs/setups/awiesm/awiesm.yaml index 583f87aaf..9e00f8751 100644 --- a/configs/setups/awiesm/awiesm.yaml +++ b/configs/setups/awiesm/awiesm.yaml @@ -106,6 +106,11 @@ general: - fesom-2.6.7+echam-6.3.05p2-awiesm-2.1 fesom_Dflags: "-DFESOM_COUPLED=ON -DCMAKE_INSTALL_PREFIX=../" with_recom: false + '2.6.2': + couplings: + - fesom-2.6.11+echam-6.3.05p2-awiesm-2.1 + fesom_Dflags: "-DFESOM_COUPLED=ON -DCMAKE_INSTALL_PREFIX=../" + with_recom: false '2.1-paleodyn-2.5': couplings: - fesom-2.5+echam-6.3.05p2-awiesm-2.1 @@ -350,6 +355,14 @@ fesom: namelist.tra: tracer_phys: surf_relax_s: 0.0 + "2.6.2": + version: "2.6.11" + ALE_scheme: 'zstar' + namelist_dir: ${fesom.model_dir}/config/ + add_namelist_changes: + namelist.tra: + tracer_phys: + surf_relax_s: 0.0 "2.1-paleodyn-2.5": version: "2.5-paleodyn" ALE_scheme: 'zstar' diff --git a/configs/setups/awiesm3/awiesm3.yaml b/configs/setups/awiesm3/awiesm3.yaml new file mode 100644 index 000000000..2ded615d6 --- /dev/null +++ b/configs/setups/awiesm3/awiesm3.yaml @@ -0,0 +1,1013 @@ +######################################################################################### +######################### AWIESM3 YAML CONFIGURATION FILE ############################### +######################################################################################### +general: + + model: awiesm3 + + model_dir: ${esm_master_dir}/awiesm3-${general.version} + + lresume: false + coupled_setup: true + + include_models: + - fesom + - oifs + - rnfmap + - lpj_guess + - oasis3mct + + available_versions: + - 'v3.4.0' + - 'v3.4.1' + - 'develop' + - 'develop-cc' + + choose_version: + v3.4.0: + major_version: v3.4 + couplings: + - awiesm3_v3.4.0 + add_include_models: + - xios + oasis_with_yac: false + xios_interp_order: 1 + with_co2_tracer: false + with_co2_oce_coupling: false + with_co2_veg_coupling: false + v3.4.1: + major_version: v3.4 + couplings: + - awiesm3_v3.4.1 + add_include_models: + - xios + oasis_with_yac: false + xios_interp_order: 1 + with_co2_tracer: false + with_co2_oce_coupling: false + with_co2_veg_coupling: false + develop: + major_version: v3.4 + couplings: + - fesom-2.7+oifs-48r1+xios-2.5+lpj_guess-4.1.2 + add_include_models: + - xios + oasis_with_yac: false + xios_interp_order: 1 + with_co2_tracer: false + with_co2_oce_coupling: false + with_co2_veg_coupling: false + + develop-cc: + major_version: v3.4 + couplings: + - fesom-2.7+oifs-48r1+xios-2.5+lpj_guess-4.1.2 + add_include_models: + - xios + oasis_with_yac: false + xios_interp_order: 1 + with_co2_tracer: true + with_co2_oce_coupling: true + with_co2_veg_coupling: true + + version: "v3.4" + resolution: ${oifs.resolution}_${fesom.resolution} + postprocessing: false + post_time: "00:05:00" + compute_time: "00:30:00" + parallel_file_movements: 'threads' + + reusable_filetypes: [input, bin, src] + + with_co2_coupling: false + + choose_computer.name: + levante: + pool_dir: "/work/ab0246/a270092/input/" + juwels: + pool_dir: "/p/project/chhb19/jstreffi/input" + aleph: + pool_dir: "/proj/awi/input" + albedo: + pool_dir: "/albedo/pool/" + "*": + pool_dir: "TO-BE-DEFINED-BY-THE-USER" + + # Load compute recipe from oifs.yaml + prepcompute_recipe: ${oifs.prepcompute_recipe} + tidy_recipe: ${fesom.tidy_recipe} + + # Load general environment for the whole coupled setup + dr_hook_ignore_signals: -1 + required_plugins: + - "git+https://github.com/JanStreffing/ocp-tool/" + +######################################################################################### +########### necessary changes to submodels compared to standalone setups ################ +######################################################################################### + +oifs: + version: "48r1" + + add_prepcompute_recipe: ["oasis_rmp_rst_to_input"] + + pool_dir: ${general.pool_dir} + model_dir: ${general.model_dir}/oifs-${oifs.version} + setup_dir: ${general.model_dir} + restart_firstlast: "last" + restart_rate: 12 + ocean_resolution: "${fesom.resolution}" + + scenario: "historical" + mip: "cmip7" + cmip7_cmor_output: "" + + input_expid: awi3 # This cannot be longer than 4 characters + input_dir: ${pool_dir}/oifs-${version} + rtables_dir: ${input_dir}/rtables/ + forcing_dir: ${input_dir}/${version}/ifsdata/ + ifsdata_dir: ${input_dir}/${version}/ifsdata/ + + choose_general.major_version: + "v3.4": + with_xios: true + build_type: 'release' + comp_command: "./openifs-bundle create; ./openifs-bundle build --threads=32 + --openifs-only --without-double-precision --with-cplng2 --with-oifs-xios + --build-type=${build_type} --install; chmod -R ${source_code_permissions} .; + git config core.fileMode false; mv install/bin/ifsMASTER.SP install/bin/OpenIFS" + namelist_dir: ${general.esm_namelist_dir}/oifs/${version}/awicm3/v3.4 + wam: true + add_add_namelist_changes: + fort.4: + NAMCT0: + LXIOS: true + NAEPHY: + LECURR: true + + + cmip6_data_dir_nml: ${input_dir}/cmip6-data + cmip5_data_dir_nml: ${input_dir}/cmip5-data + + # nproc must be not a prime number for OpenIFS parallelisation to work + # (This is just one of the many mysteries of the IFS...) + # + # We want to use 1 CPU for runoff mapper + # 287 fulfills this requirement and 288 divides well with 36 and 48 + # (287 = 41 * 7) + # so that the configuration fills up the compute nodes + # 575 would also be ok (575 = 23 * 25) + # Note: If we don't use MPI/OMP we don't need full nodes for oifs. + nproc: 36 + omp_num_threads: 2 + + perturb: 0 + ensemble_id: 1 + + restart_type: "ifs_default" + + resolution: TCO95 + levels: L91 + + add_namelist_changes: + fort.4: + NAMECECFG: + ECE_CPL_NEMO_LIM: .false. + ECE_CPL_AMIP: .false. + ECE_CPL_FESOM_FESIM: .true. + ECE_CPL_LPJG: .true. + + res_level: ${resolution}${levels} + atm_oce_fields: [A_Qns_oce, A_Qs_all, A_Precip_liquid, A_Precip_solid, A_Evap, + A_Subl, A_Q_ice, A_TauX_oce, A_TauY_oce, A_TauX_ice, A_TauY_ice, A_SST, A_Ice_frac, + A_Snow_thickness, A_Ice_temp, A_Ice_albedo, A_CurX, A_CurY, A_WindX, A_WindY, + A_FCO2_oce, A_XCO2_oce] + atm_veg_fields: [LAILVeg, LAIHVeg, FracLVeg, FracHVeg, TypeHVeg, TypeLVeg, T2MVeg, + TPVeg, SDVeg, SDensVeg, SoilTVeg.L001, SoilTVeg.L002, SoilTVeg.L003, SoilTVeg.L004, + SoilMVeg.L001, SoilMVeg.L002, SoilMVeg.L003, SoilMVeg.L004, SSRVeg, SLRVeg, + SHUMVeg, PRESVeg, WSPDVeg, TMINVeg, TMAXVeg, XCO2Veg, FCO2NAT, FCO2ANT, FCO2NPP] + coupling_fields: + "[[atm_oce_fields-->FIELD]]": + grid: atma + "[[atm_veg_fields-->FIELD]]": + grid: atma + A_Runoff: + grid: atmr + A_Calving: + grid: atmr + + input_files: + ICMGG_INIT: ICMGG_INIT + ICMGG_INIUA: ICMGG_INIUA + ICMSH_INIT: ICMSH_INIT + + prepifs_dir: ${input_dir}/${res_level}/ + tl_o3_data_dir: ${input_dir}/${version}/climate/ + ICMGG_INIT_name: "_${fesom.resolution}" + + # CO2 tracer is turned on, if at least one of the couplings for co2 is active + + choose_general.with_co2_oce_coupling: + true: + add_add_add_add_add_namelist_changes: + fort.4: + NAMECECFG: + ECE_CPL_FESOM_RECOM: true + + choose_general.with_co2_veg_coupling: + true: + add_add_add_add_add_add_namelist_changes: + fort.4: + NAMECECFG: + ECE_CPL_LPJG_CO2: true + + choose_general.with_co2_tracer: + true: + branch: 'lpj_guess_implementation+co2' + namelist_changes: + fort.4: + NAMGFL: + NGHG: 1 + YGHG_NL(1)%CNAME: "CO2_GHG" + YGHG_NL(1)%IGRBCODE: 210061 + YGHG_NL(1)%LADV: TRUE + YGHG_NL(1)%LADV5: TRUE + YGHG_NL(1)%LREQOUT: TRUE + YGHG_NL(1)%LMASSFIX: TRUE + YGHG_NL(1)%LQM: FALSE + YGHG_NL(1)%LQM3D: TRUE + YGHG_NL(1)%LNEGFIX: FALSE + YGHG_NL(1)%BETAMFBC: 2 + YGHG_NL(1)%LASSIM: TRUE + NMFDIAGLEV: 2 + NOPTVFE: 1 + + NAMCOMPO: + LCHEM_DIA: .false. + LCHEM_DDFLX: .false. + LCOMPO_DDFLX_DIR: .false. + LCHEM_TROPO: .false. + LCOMPO_DCDD: .false. + NAEPHY: + LNEEONLINE: False + LWETONLINE: False + LBFASCO2: False + + +fesom: + version: "2.7" + time_step: 1800 + resolution: CORE2 + lresume: false + nproc: 72 + omp_num_threads: 1 + choose_general.with_co2_oce_coupling: + true: + branch: fesom2.6_recom_awiesm3_co2_coupling + recom_setting: 2p3z2d + recom_comp_flag: -DRECOM_COUPLED=ON + add_namelists: + - namelist.recom + add_config_files: + recom: recom + add_config_sources: + recom: "${namelist_dir}/namelist.recom.${recom_setting}" + tra: "${namelist_dir}/namelist.tra.recom.${recom_setting}" + io: "${namelist_dir}/namelist.io.recom.${recom_setting}" + add_config_in_work: + recom: "namelist.recom" + recom_input_path: '/work/ab0246/a270092/input/fesom2/recom/' + add_add_namelist_changes: + namelist.recom: + nam_rsbc: + nm_fe_data_file: '${recom_input_path}/${resolution}/DustClimMonthlyAlbani.nc' + nm_aen_data_file: '${recom_input_path}/${resolution}/AeolianNitrogenDep.nc' + nm_river_data_file: '${recom_input_path}/${resolution}/RiverineInput.nc' + nm_erosion_data_file: '${recom_input_path}/${resolution}/ErosionInput.nc' + nm_co2_data_file: '${recom_input_path}/MonthlyAtmCO2_gcb2023.nc' # not used but searched for by bin. TODO: test removing. + pavariables: + REcoMDataPath: '${recom_input_path}/${resolution}' + false: + recom_comp_flag: "" + + build_type: Release #Available options: Debug Release RelWithDebInfo MinSizeRel + comp_command: mkdir -p build; cd build; cmake -DOIFS_COUPLED=ON -DFESOM_COUPLED=ON + -DENABLE_OPENMP=ON -DOPENMP_REPRODUCIBLE=OFF -DDISABLE_MULTITHREADING=OFF -DDISABLE_PARALLEL_RESTART_READ=ON ${recom_comp_flag} -DCMAKE_BUILD_TYPE=${build_type} -DDISABLE_MULTITHREADING=OFF + ${yac_flag} -DCMAKE_INSTALL_PREFIX=../ ..; make install + -j `nproc --all` + choose_general.oasis_with_yac: + true: + yac_flag: "-DOASIS_WITH_YAC=ON" + false: + yac_flag: "" + + missing_owy: "$(( 'oasis_with_yac' not in config['general'] ))" # TODO: move this to the block above when the missing option is included in choose_ blocks + choose_missing_owy: + true: + error: + "Missing variable": + message: "Please define ``general.oasis_with_yac`` boolean in + ``awicm3.yaml``" + ask_user_to_continue: false + + tide_forcing_dir: "" + forcing_data_dir: "" + model_dir: ${general.model_dir}/${fesom.destination} + setup_dir: ${general.model_dir} + + pool_dir: "${general.pool_dir}/fesom2/" + climate_data_dir: "${pool_dir}/hydrography/" + + restart_unit: "d" + leapyear: true + tar_binary_restarts: false + + mesh_dir: "${pool_dir}/$(( '${resolution}'.lower() ))/" + + namelist_changes: + namelist.config: + restart_log: + restart_length_unit: "${restart_unit}" + restart_length: "${restart_rate}" + + choose_general.major_version: + "v3.4": + add_namelist_changes: + namelist.config: + geometry: + metric_factor_zero: True + ale_def: + which_ale: 'linfs' + restart_log: + raw_restart_length_unit: "off" + bin_restart_length_unit: "off" + namelist.tra: + tracer_phys: + surf_relax_s: "0.0" + namelist.ice: + ice_therm: + use_meltponds: true + + + # Remove ICMCL_INIT forcing, which is only required for standalone simulations (form v3.2 onwards contains seasonal leaf area index, veg albedo etc.) + remove_forcing_files: + - ICMCL_INIT + remove_namelist_changes.namelist.config.inout: # Not present in the fesom-bounded namelist but in the fesom-2.0.yaml + - restart_length + - restart_length_unit + remove_namelist_changes.namelist.config.paths: + - ForcingDataPath + - OpbndPath + - TideForcingPath + # Remove all namelist changes for namelist.forcing defined in fesom-2.0.yaml + remove_namelist_changes: + - namelist.forcing + + + coupling_freq_in_steps: $((${oasis3mct.coupling_time_step} / ${fesom.time_step})) + + remove_config_files: + - diag + + atm_oce_fields: [ist_feom, sia_feom] + + coupling_fields: + "[[atm_oce_fields-->FIELD]]": + grid: feom + + +# It is possible for the esm_tools to modify the fesom io namelist, in case you don't want to make your own copy and edit that. +# By default this option is turned off, and thus the follwing two lines are commented out. +# further_reading: +# - fesom/fesom.io.yaml + +lpj_guess: + version: "4.1.2" + build_type: Release # Available options: Debug, Release + #comp_command: "mkdir -p build; cd build; cmake .. -DOASIS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_Fortran_FLAGS='-g -ggdb -fcheck=all -fbacktrace -fbounds-check -finit-real=snan -ffpe-trap=invalid,zero,overflow' -DCMAKE_C_FLAGS='-g -O0 -fsanitize=address'; make -j8; cd .." + comp_command: "mkdir -p build; cd build; cmake .. -DOASIS=ON -DCMAKE_BUILD_TYPE=${build_type}; make -j8; cd .." + execution_command: ${executable} -parallel ${runscript_name} + + daily_output: true + + # Automatically set mygrid to match OIFS resolution + mygrid: ${oifs.resolution} + + veg_atm_fields: [GUE_LLAI, GUE_HLAI, GUE_FRAL, GUE_FRAH, GUE_TYPH, GUE_TYPL, T2M_LPJG, + TPRE_LPJ, SNOC_LPJ, SNOD_LPJ, ST1L_LPJ, ST2L_LPJ, ST3L_LPJ, ST4L_LPJ, SM1L_LPJ, + SM2L_LPJ, SM3L_LPJ, SM4L_LPJ, SWNR_LPJ, LWNR_LPJ, SHUM_LPJ, SPRE_LPJ, WSPD_LPJ, + TMIN_LPJ, TMAX_LPJ, GUE_CNPP, GUE_CANT, GUE_CNAT, CO2_LPJG] + + coupling_fields: + "[[veg_atm_fields-->FIELD]]": + grid: ${lpj_guess.mygrid}-land + + resolution: ${oifs.resolution} + grids: + ${resolution}-land: + name: ${resolution}-land + nx: "${nx}" + ny: "${ny}" + oasis_grid_type: "D" + oyac_grid_type: "LL" + oyac_io_per_node: 4 + lpjg: + name: lpjg + nx: "${nx}" + ny: "${ny}" + oasis_grid_type: "D" + oyac_grid_type: "LL" + oyac_io_per_node: 4 + TL159-land: + name: TL159-land + nx: "${nx}" + ny: "${ny}" + oasis_grid_type: "D" + oyac_grid_type: "LL" + oyac_io_per_node: 4 + TL255-land: + name: TL255-land + nx: "${nx}" + ny: "${ny}" + oasis_grid_type: "D" + oyac_grid_type: "LL" + oyac_io_per_node: 4 + TCO95-land: + name: TCO95-land + nx: "${nx}" + ny: "${ny}" + oasis_grid_type: "D" + oyac_grid_type: "LL" + oyac_io_per_node: 4 + TCO319-land: + name: TCO319-land + nx: "${nx}" + ny: "${ny}" + oasis_grid_type: "D" + oyac_grid_type: "LL" + oyac_io_per_node: 4 + + + nproc: 128 + proc_list_entry: '[str(x) for x in range(1,${nproc}+1)]' + proc_list: $(( ${proc_list_entry} )) + + # run_year: Chunk start year - used for restart file naming + run_year: ${start_date!syear} + # initial_year: Experiment start year - for continuous output years in spinup + initial_year: ${initial_date!syear} + run_month: 1 + run_day: 1 + runtime_seconds: "$(( ${next_date} - ${start_date} ))" + ndays_per_run: "$(( ${runtime_seconds} / 86400 ))" + co2_coupling: 0 + fourxCO2: false + onepcCO2: false + fixed_CO2: -1 + fixed_ndep: -1 + fixed_LU: -1 + nyear_spinup: 1 + freenyears: 0 + + + + config_sources: + tmp_yaml_file: ${namelist_dir}/lpjg_steps.yaml.j2 + ins_files: ${namelist_dir}/*.ins + lpjg_output_ins: ${namelist_dir}/lpjg_default_cmip7_output.ins.j2 + "all_ins_[[proc_list-->PROC_NUM]]": ${namelist_dir}/*.ins + "guess_ins_[[proc_list-->PROC_NUM]]": ${namelist_dir}/guess.ins.j2 + "ecearth_ins_[[proc_list-->PROC_NUM]]": ${namelist_dir}/ecearth.ins.j2 + "globsoil_ins_[[proc_list-->PROC_NUM]]": ${namelist_dir}/global_soiln.ins + "lpjg_output_ins_[[proc_list-->PROC_NUM]]": ${namelist_dir}/lpjg_default_cmip7_output.ins.j2 + "tmp_yaml_file_[[proc_list-->PROC_NUM]]": ${namelist_dir}/lpjg_steps.yaml.j2 + + "mock_[[proc_list-->PROC_NUM]]": ${lpj_guess.input_dir}/dirmaker + + config_in_work: + tmp_yaml_file: lpjg_steps.yaml + ins_files: /*.ins + lpjg_output_ins: lpjg_output.ins + "all_ins_[[proc_list-->PROC_NUM]]": runPROC_NUM/*.ins + "guess_ins_[[proc_list-->PROC_NUM]]": runPROC_NUM/guess.ins + "ecearth_ins_[[proc_list-->PROC_NUM]]": runPROC_NUM/ecearth.ins + "globsoil_ins_[[proc_list-->PROC_NUM]]": runPROC_NUM/global_soiln.ins + "lpjg_output_ins_[[proc_list-->PROC_NUM]]": runPROC_NUM/lpjg_output.ins + "tmp_yaml_file_[[proc_list-->PROC_NUM]]": runPROC_NUM/lpjg_steps.yaml + + "mock_[[proc_list-->PROC_NUM]]": runPROC_NUM/output/stdo_stde.log + + + forcing_files: + soil_layer_type: soil_layer_type + CO2: CO2 + + forcing_sources: + soil_layer_type: ${lpj_guess.slt_dir}/slt_${lpj_guess.resolution}_${fesom.resolution}.nc + CO2: ${lpj_guess.co2_source_file} + "soil_layer_type_[[proc_list-->PROC_NUM]]": ${lpj_guess.slt_dir}/slt_${lpj_guess.resolution}_${fesom.resolution}.nc + "CO2_[[proc_list-->PROC_NUM]]": ${lpj_guess.co2_source_file} + + forcing_in_work: + # Work directory files use old naming (slt_ATMO.nc) that LPJ-GUESS binary expects + soil_layer_type: slt_${mygrid}.nc + CO2: ${lpj_guess.co2_work_file} + "soil_layer_type_[[proc_list-->PROC_NUM]]": runPROC_NUM/slt_${mygrid}.nc + "CO2_[[proc_list-->PROC_NUM]]": runPROC_NUM/${lpj_guess.co2_work_file} + + # Create empty folders for landuse and fire in work + create_folders: + - ${work_dir}/landuse + - ${work_dir}/fire + + choose_general.run_number: + 1: + input_sources: + vegin: ${oasis3mct.input_dir}/vegin.nc + "VegInit_[[proc_list-->PROC_NUM]]": ${oasis3mct.input_dir}/vegin.nc + input_in_work: + vegin: vegin.nc + "VegInit_[[proc_list-->PROC_NUM]]": runPROC_NUM/vegin.nc + choose_general.with_co2_veg_coupling: + true: + add_input_sources: + rst_co2_av: ${oasis3mct.input_dir}/rst_co2_av.nc + "rst_co2_avInit_[[proc_list-->PROC_NUM]]": ${oasis3mct.input_dir}/rst_co2_av.nc + add_in_work: + rst_co2_av: rst_co2_av.nc + "rst_co2_avInit_[[proc_list-->PROC_NUM]]": runPROC_NUM/rst_co2_av.nc + + choose_general.with_co2_veg_coupling: + true: + co2_coupling: 1 + + +xios: + xml_dir: "${general.esm_namelist_dir}/oifs/${oifs.version}/xios/" + interpolation_order: 1 + compression_level: 1 + truncation: "${oifs.truncation}" + choose_truncation: + TCO: + regular_res_lat: "$(( (${oifs.res_number}+1)*2 ))" #For TCO95->192 + regular_res_lon: "$(( (${oifs.res_number})*4+20 ))" #For TCO95->400 + TL: + regular_res_lat: "$(( (${oifs.res_number}+1)*2 ))" + regular_res_lon: "$(( (${oifs.res_number}+1)*4 ))" + cmip7_cmor_output: "${oifs.cmip7_cmor_output}" + choose_cmip7_cmor_output: + "cmip7_spinup": + config_sources: + context_ifs: ${xml_dir}/context_ifs.xml.j2 + domain_def: ${xml_dir}/domain_def.xml.j2 + file_def: ${xml_dir}/file_def_oifs_cmip7_spinup.xml.j2 + field_def: ${xml_dir}/field_def_cmip7.xml + "*": + config_sources: + context_ifs: ${xml_dir}/context_ifs.xml.j2 + domain_def: ${xml_dir}/domain_def.xml.j2 + file_def: ${xml_dir}/file_def.xml.j2 + add_config_files: + context_ifs: context_ifs + ifs_xml: ifs_xml + domain_def: domain_def + field_def: field_def + file_def: file_def + axis_def: axis_def + grid_def: grid_def + + nproc: 1 + omp_num_threads: ${computer.partitions.compute.cores_per_node} + +rnfmap: + pool_dir: ${general.pool_dir} + runoff_maps_name: _${fesom.resolution} + version: v1.1 + + nproc: 1 + omp_num_threads: ${computer.partitions.compute.cores_per_node} + time_step: ${oasis3mct.coupling_time_step} + + runoff_method: awicm3-default + + coupling_fields: + "[[runoff_fields_send-->FIELD]]": + grid: rnfo + + grids: + rnfa: + name: RnfA + nx: 512 + ny: 256 + oasis_grid_type: "LR" + oyac_grid_type: "LL" + oyac_io_per_node: 4 + + rnfo: + name: RnfO + nx: 512 + ny: 256 + oasis_grid_type: "LR" + oyac_grid_type: "LL" + oyac_io_per_node: 4 + + input_files: + runoff_maps: runoff_maps + + +######################################################################################### + +# Settings for OASIS +oasis3mct: + model_dir: ${general.model_dir}/oasis + pool_dir: ${general.pool_dir}/oasis/ + norestart: "T" + + process_ordering: #look into it..... + - fesom + - oifs + - lpj_guess + - rnfmap + + required_plugins: + - "git+https://github.com/esm-tools-plugins/oasis_rmp_rst_to_input" + + # lresume needs to be true to allow usage of flxatmos and sstocean + # "restart" files in initial coupling time step + # but only if the general model setup is a cold start (i.e. general.lresume = false) + lresume: true + ini_parent_date: "20590101" # not used, but needs to be present + ini_parent_exp_id: "khw0030" # not used, but needs to be present + ini_restart_dir: "${input_dir}" + + output_exchanged_vars: false + + time_step: 7200 #separate timestep needed for atmosphere-ocean in the future LASZLO + + a2o_lag: "${lag}" + a2v_lag: "${oifs.time_step}" + a2r_lag: "${lag}" + o2a_lag: "${lag}" + o2r_lag: 0 + r2a_lag: "${lag}" + v2a_lag: "${lpj_guess.time_step}" + a2o_seq: 2 + o2a_seq: 2 + o2r_seq: 1 + r2a_seq: 2 + a2v_seq: 2 + v2a_seq: 2 + + coupling_time_step: "${oasis3mct.time_step}" + choose_general.major_version: + 'v3.4': + add_process_ordering: + - xios + coupling_target_fields: + rstas.nc: + - 'heat_oce:heat_swo <--gauswgt_c-- A_Qns_oce:A_Qs_all' + - 'prec_oce:snow_oce:evap_oce:subl_oce <--gauswgt_c-- A_Precip_liquid:A_Precip_solid:A_Evap:A_Subl' + - 'heat_ico <--gauswgt_gss-- A_Q_ice' + - 'taux_oce:tauy_oce:taux_ico:tauy_ico <--bicubic-- A_TauX_oce:A_TauY_oce:A_TauX_ice:A_TauY_ice' + - 'u10w_oce:v10w_oce <--bicubic-- A_WindX:A_WindY' + - 'R_Runoff_atm:R_Calving_atm <--gauswgt_c-- A_Runoff:A_Calving' + - 'hydr_oce:calv_oce <--gauswgt_c-- R_Runoff_oce:R_Calving_oce' + rstos.nc: + - 'A_SST:A_Ice_frac:A_Snow_thickness:A_Ice_temp:A_Ice_albedo <--gauswgt_i-- + sst_feom:sie_feom:snt_feom:ist_feom:sia_feom' + - 'A_CurX:A_CurY <--gauswgt_i-- u_feom:v_feom' + vegin.nc: + - 'LAILVeg:LAIHVeg:FracLVeg:FracHVeg:TypeLVeg:TypeHVeg <--loctrans_avg-- + GUE_LLAI:GUE_HLAI:GUE_TYPH:GUE_FRAH:GUE_TYPL:GUE_FRAL' + - 'T2M_LPJG:TPRE_LPJ:SNOC_LPJ:SNOD_LPJ <--loctrans_avg-- T2MVeg:TPVeg:SDVeg:SDensVeg' + - 'ST1L_LPJ:ST2L_LPJ:ST3L_LPJ:ST4L_LPJ <--loctrans_avg-- SoilTVeg.L001:SoilTVeg.L002:SoilTVeg.L003:SoilTVeg.L004' + - 'SM1L_LPJ:SM2L_LPJ:SM3L_LPJ:SM4L_LPJ <--loctrans_avg-- SoilMVeg.L001:SoilMVeg.L002:SoilMVeg.L003:SoilMVeg.L004' + - 'SWNR_LPJ:LWNR_LPJ:SHUM_LPJ:SPRE_LPJ:WSPD_LPJ <--loctrans_avg-- + SSRVeg:SLRVeg:SHUMVeg:PRESVeg:WSPDVeg' + - 'TMIN_LPJ <--loctrans_min-- TMINVeg' + - 'TMAX_LPJ <--loctrans_max-- TMAXVeg' + choose_general.with_co2_veg_coupling: + true: + add_coupling_target_fields: + rst_co2_av.nc: + - 'CO2_LPJG <--loctrans_avg-- XCO2Veg' + - 'FCO2NAT:FCO2ANT:FCO2NPP <--loctrans_avg-- GUE_CNAT:GUE_CANT:GUE_CNPP' + add_merge_recv_restarts: + - rst_co2_av.nc # CO2 vegetation coupling (bidirectional LOCTRANS) + choose_general.with_co2_oce_coupling: + true: + add_coupling_target_fields: + rst_co2_ao.nc: + - 'XCO2_oce <--gauswgt_c-- A_XCO2_oce' + - 'A_FCO2_oce <--gauswgt_gss-- FCO2_feom' + + # Specify which restart files need bidirectional merging + # Use this for restart files with LOCTRANS+EXPOUT in both directions + # where both send and receive use the same restart file name in namcouple + merge_recv_restarts: + - vegin.nc # OpenIFS <-> lpjg bidirectional LOCTRANS + + mct_version: "5.0" + input_dir: ${oasis3mct.pool_dir}/cy${oifs.version}/${oifs.resolution}-${fesom.resolution}/ + rstos_file_name: 'rstos' + rstas_file_name: 'rstas' + vegin_file_name: 'vegin' + rst_co2_ao_name: 'rst_co2_ao' + rst_co2_av_name: 'rst_co2_av' + + land_atmo_coupling_time_step: 86400 + coupling_directions: + 'atma->feom': + lag: ${a2o_lag} + seq: ${a2o_seq} + 'atmr->rnfa': + lag: ${r2a_lag} + seq: ${r2a_seq} + 'rnfo->feom': + lag: ${o2r_lag} + seq: ${o2r_seq} + 'feom->atma': + lag: ${o2a_lag} + seq: ${o2a_seq} + '${lpj_guess.mygrid}-land->atma': + lag: ${v2a_lag} + seq: ${v2a_seq} + coupling_time_step: ${land_atmo_coupling_time_step} + lag_overwrite: true + 'atma->${lpj_guess.mygrid}-land': + lag: ${a2v_lag} + seq: ${a2v_seq} + coupling_time_step: ${land_atmo_coupling_time_step} + lag_overwrite: true + + + coupling_methods: + bicubic: + time_transformation: average + remapping: + bicubic: + search_bin: latitude + nb_of_search_bins: 15 + + bicubic_glb: + time_transformation: average + remapping: + bicubic: + search_bin: latitude + nb_of_search_bins: 15 + postprocessing: + conserv: + method: glbpos + algorithm: opt + bicubic_gss: + time_transformation: average + remapping: + bicubic: + search_bin: latitude + nb_of_search_bins: 15 + postprocessing: + conserv: + method: gsspos + algorithm: opt + gauswgt_c: + time_transformation: conserv + remapping: + gauswgt: + search_bin: latitude + nb_of_search_bins: 1 + nb_of_neighbours: 25 + weight: "0.1" + postprocessing: + conserv: + method: glbpos + algorithm: opt + gauswgt_gss: + time_transformation: conserv + remapping: + gauswgt: + search_bin: latitude + nb_of_search_bins: 1 + nb_of_neighbours: 25 + weight: "0.1" + postprocessing: + conserv: + method: gsspos + algorithm: opt + gauswgt_i: + time_transformation: instant + remapping: + gauswgt: + search_bin: latitude + nb_of_search_bins: 1 + nb_of_neighbours: 25 + weight: "0.1" + gauswgt_glo: + time_transformation: conserv + remapping: + gauswgt: + search_bin: latitude + nb_of_search_bins: 1 + nb_of_neighbours: 25 + weight: "0.1" + postprocessing: + conserv: + method: global + loctrans_avg: + time_transformation: average + remapping: + loctrans: + loctrans_min: + time_transformation: t_min + remapping: + loctrans: + loctrans_max: + time_transformation: t_max + remapping: + loctrans: + oyac_1stcon_nnn_stocha: + oyac: true + time_transformation: stocha + remapping: + conserv: + order: first + intersect_norm: fracarea + nnn: + weighting: avg + nb_of_neighbours: 1 + oyac_1stcon_nnn_i: + oyac: true + time_transformation: instant + remapping: + conserv: + order: first + intersect_norm: fracarea + nnn: + weighting: avg + nb_of_neighbours: 1 + oyac_gauswgt_c: + oyac: true + time_transformation: conserv + remapping: + nnn: + weighting: gauss + nb_of_neighbours: 9 + oyac_avg_c: + oyac: true + time_transformation: conserv + remapping: + nnn: + weighting: avg + nb_of_neighbours: 9 + oyac_gauswgt_i: + oyac: true + time_transformation: instant + remapping: + nnn: + weighting: gauss + nb_of_neighbours: 9 + oyac_hcsbb_nnn_i: + oyac: true + time_transformation: instant + remapping: + hcsbb: + lorem: ipsum + nnn: + weighting: avg + nb_of_neighbours: 1 + oyac_spmap_nnn_i: + oyac: true + time_transformation: instant + remapping: + spmap: + weighting: dist + spread_dist: 3 + nnn: + weighting: avg + nb_of_neighbours: 1 + + + input_files: {} + + input_sources: + rmp: ${input_dir}/${fesom.nproc}/rmp_*.nc + areas: ${input_dir}/areas.nc + masks: ${input_dir}/masks.nc + grids: ${input_dir}/grids.nc + + input_in_work: + rmp: rmp_*.nc + areas: areas.nc + masks: masks.nc + grids: grids.nc + + restart_in_files: + rmp: rmp + rstas.nc: rstas.nc + rstos.nc: rstos.nc + vegin.nc: vegin.nc + + restart_in_in_work: + rmp: rmp_*.nc + rstas.nc: rstas.nc + rstos.nc: rstos.nc + vegin.nc: vegin.nc + + restart_in_sources: + rmp: rmp_*.nc + rstas.nc: rstas.nc + rstos.nc: rstos.nc + vegin.nc: vegin.nc + + restart_out_files: + rmp: rmp + rstas.nc: rstas.nc + rstos.nc: rstos.nc + vegin.nc: vegin.nc + + restart_out_in_work: + rmp: rmp_*.nc + rstas.nc: rstas.nc + rstos.nc: rstos.nc + vegin.nc: vegin.nc + + restart_out_sources: + rmp: rmp_*.nc + rstas.nc: rstas.nc + rstos.nc: rstos.nc + vegin.nc: vegin.nc + + choose_general.with_co2_veg_coupling: + true: + add_restart_in_files: + rst_co2_av.nc: rst_co2_av.nc + add_restart_in_in_work: + rst_co2_av.nc: rst_co2_av.nc + add_restart_out_files: + rst_co2_av.nc: rst_co2_av.nc + add_restart_out_in_work: + rst_co2_av.nc: rst_co2_av.nc + add_restart_out_sources: + rst_co2_av.nc: rst_co2_av.nc + choose_general.with_co2_oce_coupling: + true: + add_restart_in_files: + rst_co2_ao.nc: rst_co2_ao.nc + add_restart_in_in_work: + rst_co2_ao.nc: rst_co2_ao.nc + add_restart_out_files: + rst_co2_ao.nc: rst_co2_ao.nc + add_restart_out_in_work: + rst_co2_ao.nc: rst_co2_ao.nc + add_restart_out_sources: + rst_co2_ao.nc: rst_co2_ao.nc + + remove_config_files: + - cf + + choose_lresume: + false: + lag: 0 + choose_general.run_number: + 1: + add_input_files: + rmp: rmp + areas: areas + masks: masks + grids: grids + true: + lag: "${time_step}" + # If ini_restart_dir is not specified in the runscript and, therefore, + # it is not a branched off experiment adds the /fesom.nproc/ to the + # sources when the run_number is 1 + load_restart_from_pool: "$(( os.path.realpath('${input_dir}') == os.path.realpath('${ini_restart_dir}') + and ${general.run_number} == 1 ))" + choose_load_restart_from_pool: + true: + add_restart_in_sources: + rmp: ${input_dir}/${fesom.nproc}/rmp_*.nc + rstas.nc: ${input_dir}/${fesom.nproc}/${rstas_file_name}.nc + rstos.nc: ${input_dir}/${fesom.nproc}/${rstos_file_name}.nc + choose_general.with_co2_oce_coupling: + true: + add_restart_in_sources: + rst_co2_ao.nc: ${input_dir}/${rst_co2_ao_name}.nc + +computer: + # Activates the taskset approach for heterogeneous parallelization, by default + # since hetjob approach doesn't seem to work (second run showing mpi/pmix + # problems) + taskset: true + merge_component_envs: + compile: true + + choose_general.major_version: + v3.4: + choose_name: + juwels: + iolibraries: system_libs + compiler_mpi: gnu2023_ompi2023 + levante: + iolibraries: system_intel_libs + compiler_mpi: intel2022_openmpi + '*': + foo: bar diff --git a/configs/setups/lpjg-spinup/lpjg-spinup.yaml b/configs/setups/lpjg-spinup/lpjg-spinup.yaml new file mode 100644 index 000000000..81e02f394 --- /dev/null +++ b/configs/setups/lpjg-spinup/lpjg-spinup.yaml @@ -0,0 +1,196 @@ +######################################################################################### +######################### AWIESM3 YAML CONFIGURATION FILE ############################### +######################################################################################### +general: + + model: lpjg-spinup + + model_dir: ${esm_master_dir}/lpj_guess-${general.version} + + lresume: false + coupled_setup: true + + include_models: + - lpj_guess + - oasis3mct + + available_versions: + - 'v3.4.0' + - 'develop' + + choose_version: + v3.4.0: + couplings: + - lpjg-spinup_v3.4.0 + major_version: v3.4 + oasis_with_yac: false + with_co2_tracer: false + with_co2_oce_coupling: false + with_co2_veg_coupling: false + develop: + couplings: + - lpj_guess-4.1.2+oasis + major_version: v3.4 + oasis_with_yac: false + with_co2_tracer: false + with_co2_oce_coupling: false + with_co2_veg_coupling: false + + version: "v3.4" + resolution: TL255 + postprocessing: false + post_time: "00:05:00" + compute_time: "00:30:00" + parallel_file_movements: 'threads' + + reusable_filetypes: [input, bin, src] + + with_co2_coupling: false + + choose_computer.name: + levante: + pool_dir: "/work/ab0246/a270092/input" + juwels: + pool_dir: "/p/project/chhb19/jstreffi/input" + aleph: + pool_dir: "/proj/awi/input" + albedo: + pool_dir: "/albedo/pool/" + "*": + pool_dir: "TO-BE-DEFINED-BY-THE-USER" + + # Load general environment for the whole coupled setup + dr_hook_ignore_signals: -1 + +######################################################################################### +########### necessary changes to submodels compared to standalone setups ################ +######################################################################################### + + +lpj_guess: + version: "4.1.2" + #comp_command: "mkdir -p build; cd build; cmake .. -DOASIS=ON -DCMAKE_BUILD_TYPE=DEBUG; make -j8; cd .." #original comp_command. + comp_command: "mkdir -p build; cd build; cmake .. -DOASIS=ON -DCMAKE_BUILD_TYPE=RELEASE; + make -j8; cd .." #without debug. + #comp_command: "mkdir -p build; cd build; cmake .. -DOASIS=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_Fortran_FLAGS='-g -ggdb -fcheck=all -fbacktrace -fbounds-check -finit-real=snan -ffpe-trap=invalid,zero,overflow' -DCMAKE_C_FLAGS='-g -O0 -fsanitize=address'; make -j8; cd .." #debuging comp_command + execution_command: ${executable} -parallel ${runscript_name} + + resolution: TL255 + fesom_resolution: CORE2 + openifsspinup: true + daily_output: false + + # Disable CMIP N-Dep file reading for spinup - use pre-industrial constant values instead + ndep_cmip_dir: "" + + nproc: 128 + proc_list_entry: '[str(x) for x in range(1,${nproc}+1)]' + proc_list: $(( ${proc_list_entry} )) + + # run_year: Chunk start year - used for restart file naming + run_year: ${start_date!syear} + # initial_year: Experiment start year - used for continuous output years + initial_year: ${initial_date!syear} + run_month: 1 + run_day: 1 + runtime_seconds: "$(( ${next_date} - ${start_date} ))" + ndays_per_run: "$(( ${runtime_seconds} / 86400 ))" + co2_coupling: 0 + fourxCO2: f + onepcCO2: f + fixed_CO2: -1 + fixed_ndep: -1 + fixed_LU: -1 + # nyear_spinup and freenyears should only apply to the first leg. + # For continuation legs, disable the internal spinup to avoid + # re-running N-free years and causing soil C discontinuities. + choose_general.run_number: + 1: + nyear_spinup: ${nyear} + # freenyears: N limitation delay years (must be <= nyear_spinup) + # Set equal to nyear_spinup for full N limitation delay during spinup + freenyears: ${nyear} + "*": + nyear_spinup: 1 + freenyears: 0 + + config_sources: + tmp_yaml_file: ${namelist_dir}/lpjg_steps.yaml.j2 + ins_files: ${namelist_dir}/*.ins + lpjg_output_ins: ${namelist_dir}/lpjg_default_cmip7_output.ins.j2 + "all_ins_[[proc_list-->PROC_NUM]]": ${namelist_dir}/*.ins + "guess_ins_[[proc_list-->PROC_NUM]]": ${namelist_dir}/guess.ins.j2 + "ecearth_ins_[[proc_list-->PROC_NUM]]": ${namelist_dir}/ecearth.ins.j2 + "globsoil_ins_[[proc_list-->PROC_NUM]]": ${namelist_dir}/global_soiln.ins + "lpjg_output_ins_[[proc_list-->PROC_NUM]]": ${namelist_dir}/lpjg_default_cmip7_output.ins.j2 + "tmp_yaml_file_[[proc_list-->PROC_NUM]]": ${namelist_dir}/lpjg_steps.yaml.j2 + "mock_[[proc_list-->PROC_NUM]]": ${lpj_guess.input_dir}/dirmaker + + config_in_work: + tmp_yaml_file: lpjg_steps.yaml + ins_files: /*.ins + lpjg_output_ins: lpjg_output.ins + "all_ins_[[proc_list-->PROC_NUM]]": runPROC_NUM/*.ins + "guess_ins_[[proc_list-->PROC_NUM]]": runPROC_NUM/guess.ins + "ecearth_ins_[[proc_list-->PROC_NUM]]": runPROC_NUM/ecearth.ins + "globsoil_ins_[[proc_list-->PROC_NUM]]": runPROC_NUM/global_soiln.ins + "lpjg_output_ins_[[proc_list-->PROC_NUM]]": runPROC_NUM/lpjg_output.ins + "tmp_yaml_file_[[proc_list-->PROC_NUM]]": runPROC_NUM/lpjg_steps.yaml + + "mock_[[proc_list-->PROC_NUM]]": runPROC_NUM/output/stdo_stde.log + + forcing_sources: + oifs_forcing: ${lpj_guess.input_dir}/oifs_forcing/AM04_atm_cmip6_1d_1990-1999_${lpj_guess.resolution}_${lpj_guess.fesom_resolution}.nc + soil_layer_type: ${lpj_guess.slt_dir}/slt_${lpj_guess.resolution}_${lpj_guess.fesom_resolution}.nc + + forcing_in_work: + oifs_forcing: ifs_spinup_forcing/AM04_atm_cmip6_1d_1990-1999_fast_lpjgforcing.nc + soil_layer_type: slt_${lpj_guess.resolution}.nc + + add_forcing_files: + oifs_forcing: oifs_forcing + + # Create empty folders for landuse and fire in work + create_folders: + - ${work_dir}/landuse + - ${work_dir}/fire + + # Disable output copying for spinup runs + outdata_files: {} + + + +oasis3mct: + input_dir: ${oasis3mct.pool_dir}/oasis/cy48r1/${lpj_guess.resolution}-${lpj_guess.fesom_resolution} + process_ordering: + - lpj_guess + + input_sources: + areas: ${input_dir}/areas.nc + masks: ${input_dir}/masks.nc + grids: ${input_dir}/grids.nc + + input_in_work: + areas: areas.nc + masks: masks.nc + grids: grids.nc + +computer: + # Activates the taskset approach for heterogeneous parallelization, by default + # since hetjob approach doesn't seem to work (second run showing mpi/pmix + # problems) + taskset: true + merge_component_envs: + compile: true + + choose_general.major_version: + v3.4: + choose_name: + juwels: + iolibraries: system_libs + compiler_mpi: gnu2023_ompi2023 + levante: + iolibraries: system_gnu_libs + compiler_mpi: gnu11_ompi4 + '*': + foo: bar diff --git a/configs/setups/oifsamip/oifsamip.yaml b/configs/setups/oifsamip/oifsamip.yaml index 115c6c74c..b31c010bf 100644 --- a/configs/setups/oifsamip/oifsamip.yaml +++ b/configs/setups/oifsamip/oifsamip.yaml @@ -9,6 +9,8 @@ general: lresume: False coupled_setup: True + with_co2_oce_coupling: False + with_co2_veg_coupling: False include_models: - amip @@ -33,12 +35,13 @@ general: postprocessing: false post_time: "00:05:00" compute_time: "00:30:00" + parallel_file_movements: 'threads' reusable_filetypes: [input, bin, src] choose_computer.name: juwels: - pool_dir: "/p/project/chhb19/jstreffi/input" + pool_dir: "/p/project/chhb19/streffing1/input" aleph: pool_dir: "/proj/awi/input" levante: @@ -115,6 +118,7 @@ oifs: ECE_CPL_AMIP: ".true." NAMMCC: LNEMOLIMCUR: ".false." + LNEMOSICOUP: ".false." NAEPHY: LECURR: ".false." #NALBEDOSCHEME: 3 @@ -195,6 +199,7 @@ xios: compression_level: 1 add_config_sources: + context_ifs: ${xml_dir}/context_ifs.xml.j2 domain_def: ${xml_dir}/domain_def.xml.j2 file_def: ${xml_dir}/file_def.xml.j2 diff --git a/couplings/fesom/coupling_ice2fesomUKK.functions b/couplings/fesom/coupling_ice2fesomUKK.functions index d87a3cb37..dbafbaa8d 100644 --- a/couplings/fesom/coupling_ice2fesomUKK.functions +++ b/couplings/fesom/coupling_ice2fesomUKK.functions @@ -76,31 +76,39 @@ function ice2fesom { } function build_submesh { - echo "UKK13" + echo "====================================================================" + echo "[BUILD_SUBMESH] Starting dynamic submesh generation" + echo "====================================================================" SUBMESH_DIR=submesh_${CHUNK_END_DATE_fesom}/ - echo "submesh $SUBMESH_DIR" + echo "[BUILD_SUBMESH] Target submesh directory: ${SUBMESH_DIR}" MAXMESH_DIR_fesom=$MAX_MESH - echo $MESH_DIR + echo "[BUILD_SUBMESH] Current MESH_DIR: ${MESH_DIR}" + echo "[BUILD_SUBMESH] Maximum mesh directory: ${MAXMESH_DIR_fesom}" mkdir -p $SUBMESH_DIR rm -rf $SUBMESH_DIR/* + echo "[BUILD_SUBMESH] Step 1: Regridding ice geometry to ocean resolution..." cdo -setmisstoc,5 -remapbil,T512grid ${COUPLE_DIR}/latest_ice_geometry.nc tmp1.nc cdo -remapnn,core2_griddes_nodes.nc tmp1.nc tmp2.nc ncks -O -d time,0 tmp2.nc ${COUPLE_DIR}/latest_ice_geometry_for_fesom.nc rm tmp1.nc tmp2.nc + echo "[BUILD_SUBMESH] Step 2: Generating mesh flags from PISM ice geometry..." python3 mesh_flags.py > output.txt 2>&1 + echo "[BUILD_SUBMESH] Step 3: Copying mesh text files and compiling Fortran submesh tool..." cp ${MESH_DIR_fesom}/*.txt . - gfortran -o fesom_submesh.x fesom_submesh_UKK.F90 > lala + gfortran -o fesom_submesh.x fesom_submesh_UKK.F90 > compile_submesh.log 2>&1 rm -rf ${COUPLE_DIR}/${SUBMESH_DIR}/* - echo UKK111 ./fesom_submesh.x ${MAXMESH_DIR_fesom} ${COUPLE_DIR}/${SUBMESH_DIR}/ + echo "[BUILD_SUBMESH] Step 4: Running mesh reduction algorithm..." + echo "[BUILD_SUBMESH] Command: ./fesom_submesh.x ${MAXMESH_DIR_fesom} ${COUPLE_DIR}/${SUBMESH_DIR}/" ./fesom_submesh.x ${MAXMESH_DIR_fesom} ${COUPLE_DIR}/${SUBMESH_DIR}/ rm -rf ${COUPLE_DIR}/latest_submesh - echo ${MESH_DIR_fesom} - echo $PATH - echo "singularity run -B ${COUPLE_DIR}:/scif/data --app mesh-partition oras://registry.hzdr.de/paul.gierz/meshtools-creator:dev --mesh-dir ${SUBMESH_DIR} --n-part 288 --use-cavity" + echo "[BUILD_SUBMESH] Mesh directory (FESOM): ${MESH_DIR_fesom}" + echo "[BUILD_SUBMESH] Step 5: Running mesh partitioning with Singularity..." + echo "[BUILD_SUBMESH] Container: paul.gierz/meshtools-creator:dev" + echo "[BUILD_SUBMESH] Partitions: 288, Using cavity: yes" singularity run -B ${COUPLE_DIR}:/scif/data --app mesh-partition oras://registry.hzdr.de/paul.gierz/meshtools-creator:dev --mesh-dir ${SUBMESH_DIR} --n-part 288 --use-cavity - echo "UKK999" + echo "[BUILD_SUBMESH] Step 6: Creating mesh grid description file..." singularity run -B ${COUPLE_DIR}:/scif/data --app mesh-griddes oras://registry.hzdr.de/paul.gierz/meshtools-creator:dev --mesh-dir ${SUBMESH_DIR} --ofile mesh_cst.nc rm tmp*.nc mv ${COUPLE_DIR}/mesh_cst.nc ${COUPLE_DIR}/${SUBMESH_DIR}/ @@ -108,7 +116,8 @@ function build_submesh { mesh_dir=${COUPLE_DIR}/${SUBMESH_DIR} if [ -f "${mesh_dir}/cavity_nlvls.out" ]; then - echo "Found cavity_nlvls.out -> building OCE_flag mask" + echo "[BUILD_SUBMESH] Step 7: Processing ice shelf cavity levels..." + echo "[BUILD_SUBMESH] Found cavity_nlvls.out -> building OCE_flag mask" # Convert text to NetCDF cdo -L -f nc -b F64 input,${mesh_dir}/mesh_cst.nc cavity_nlvls_tmp.nc < ${mesh_dir}/cavity_nlvls.out @@ -119,7 +128,8 @@ if [ -f "${mesh_dir}/cavity_nlvls.out" ]; then # Merge mask into mesh cdo -L -f nc -b F64 -merge ${mesh_dir}/mesh_cst.nc OCE_flag_tmp.nc GLAC_flag_tmp.nc mesh_cst_tmp.nc else - echo "No cavity_nlvls.out -> OCE_flag is constant 1 everywhere" + echo "[BUILD_SUBMESH] Step 7: No cavity detected..." + echo "[BUILD_SUBMESH] No cavity_nlvls.out -> OCE_flag is constant 1 everywhere" # Create a constant mask variable OCE_flag = 1 on the mesh grid cdo -L -f nc -b F64 -setname,OCE_flag -const,1,${mesh_dir}/mesh_cst.nc OCE_flag_tmp.nc cdo -L -f nc -b F64 -setname,GLAC_flag -setctomiss,1 -const,1,${mesh_dir}/mesh_cst.nc GLAC_flag_tmp.nc @@ -130,9 +140,17 @@ fi cp mesh_cst_tmp.nc ${mesh_dir}/mesh_cst.nc +# Rename cavity_depth.out to pyfesom2-standard name +if [[ -f ${mesh_dir}/cavity_depth.out ]]; then + mv ${mesh_dir}/cavity_depth.out ${mesh_dir}/cavity_depth@node.out + echo "[BUILD_SUBMESH] Renamed cavity_depth.out -> cavity_depth@node.out" +fi + # clean up +echo "[BUILD_SUBMESH] Cleaning up temporary files..." rm *.out *_tmp.nc - +echo "[BUILD_SUBMESH] Submesh generation completed successfully!" +echo "====================================================================" } @@ -267,12 +285,14 @@ function iterative_coupling_pism_ocean_prepare_ocean_icebergmodel_forcing { COUNT_MAX=12 while [ ${counter} -lt ${COUNT_MAX} ] do - echo " * inside while loop " + echo "[ICEBERG_FORCING] Checking for PISM output file (attempt ${counter}/${COUNT_MAX})..." if [ -f ${COUPLE_DIR}/../outdata/pism/latest_ex_file_pism.nc ]; then + echo "[ICEBERG_FORCING] PISM output file found!" break fi - echo; echo " * File ${COUPLE_DIR}/../outdata/pism/latest_ex_file_pism.nc not found. Waiting for 10 seconds ..." + echo "[ICEBERG_FORCING] File not found: ${COUPLE_DIR}/../outdata/pism/latest_ex_file_pism.nc" + echo "[ICEBERG_FORCING] Waiting 10 seconds before retry..." sleep 10 counter=$((counter+1)) done @@ -285,9 +305,11 @@ function iterative_coupling_pism_ocean_prepare_ocean_icebergmodel_forcing { pism_discharge_file=${SPINUP_FILE_pism} fi - echo "CHUNK_SIZE_pism_standalone: ${CHUNK_SIZE_pism_standalone}" - echo "COUPLE_DIR: ${COUPLE_DIR}" - echo "PISM_DISCHARGE_FILE: ${pism_discharge_file}" + echo "[ICEBERG_FORCING] Configuration:" + echo "[ICEBERG_FORCING] CHUNK_SIZE_pism_standalone: ${CHUNK_SIZE_pism_standalone}" + echo "[ICEBERG_FORCING] COUPLE_DIR: ${COUPLE_DIR}" + echo "[ICEBERG_FORCING] PISM_DISCHARGE_FILE: ${pism_discharge_file}" + echo "[ICEBERG_FORCING] Processing iceberg discharge data with CDO..." cdo -s -timmean -selname,tendency_of_ice_amount_due_to_discharge \ -setgrid,${COUPLE_DIR}/ice.griddes ${pism_discharge_file} ${COUPLE_DIR}/latest_discharge.nc diff --git a/couplings/general/coupling_general.functions b/couplings/general/coupling_general.functions index 4818a2e42..371c47c0b 100644 --- a/couplings/general/coupling_general.functions +++ b/couplings/general/coupling_general.functions @@ -431,6 +431,12 @@ function iterative_coupling_pism_regrid_add_xy_array { else # This is the standard case: generate both float and double files, just in case something strange happens INITIAL_FILE_pism=${POOL_DIR_pism}/grids/${DOMAIN_pism}/pismr_${DOMAIN_pism}_${RES_pism}_xy.nc + if [[ -f ${COUPLE_DIR}/pismr_${DOMAIN_pism}_${RES_pism}_xy.nc ]] + then + INITIAL_FILE_pism=${COUPLE_DIR}/pismr_${DOMAIN_pism}_${RES_pism}_xy.nc + else + INITIAL_FILE_pism=${POOL_DIR_pism}/grids/${DOMAIN_pism}/pismr_${DOMAIN_pism}_${RES_pism}_xy.nc + fi PISM_GRID_xy_file_double=${PISM_GRID_xy_file_double:-xy_file_double.$(basename ${INITIAL_FILE_pism})} PISM_GRID_xy_file_float=${PISM_GRID_xy_file_float:-xy_file_float.$(basename ${INITIAL_FILE_pism})} pism_helpers_create_xy_axis ${INITIAL_FILE_pism} ${PISM_GRID_xy_file_double} ${PISM_GRID_xy_file_float} diff --git a/couplings/oifs/coupling_ice2oifs.functions b/couplings/oifs/coupling_ice2oifs.functions new file mode 100644 index 000000000..eeec87d45 --- /dev/null +++ b/couplings/oifs/coupling_ice2oifs.functions @@ -0,0 +1,130 @@ +#!/usr/bin/ksh + +function generate_ocp_config { + echo "[OCP_TOOL] Generating ocp-tool YAML configuration..." + OCP_CONFIG=${COUPLE_DIR}/ocp_coupling_config.yaml + + # Determine truncation type string for ocp-tool + if [[ "${OCP_OIFS_TRUNCATION}" == "TCO" ]]; then + TRUNC_TYPE="cubic-octahedral" + REDUCED_KEY="gaussian_grids_octahedral_reduced" + else + TRUNC_TYPE="linear" + REDUCED_KEY="gaussian_grids_linear_reduced" + fi + + cat > ${OCP_CONFIG} << EOF +atmosphere: + resolution_list: [${OCP_OIFS_RES_NUMBER}] + truncation_type: "${TRUNC_TYPE}" + experiment_name: "${OCP_OIFS_PREPIFS_EXPID}" + +ocean: + grid_name: "${OCP_FESOM_RES}" + has_ice_cavities: true + mesh_file: "${COUPLE_DIR}/${SUBMESH_DIR}/mesh.nc" + intermediate_resolution: "r360x181" + force_overwrite_griddes: true + +runoff: + manual_basin_removal: + - "caspian-sea" + - "black-sea" + +paths: + input: + fesom_mesh: "${COUPLE_DIR}/${SUBMESH_DIR}/" + gaussian_grids_full: "${OCP_POOL_DIR}/ocp-tool/gaussian_grids_full/" + ${REDUCED_KEY}: "${OCP_POOL_DIR}/ocp-tool/gaussian_grids_octahedral_reduced/" + openifs_default: "${OCP_POOL_DIR}/oifs-${OCP_OIFS_VERSION}/${OCP_OIFS_RES}${OCP_OIFS_LEVELS}/" + runoff_default: "${OCP_POOL_DIR}/ocp-tool/runoff_map_default/" + lpj_guess: "${OCP_POOL_DIR}/ocp-tool/lpj-guess/" + root_dir: "${COUPLE_DIR}" + +options: + verbose: true + parallel_workers: 4 + use_dask: true +EOF + + echo "[OCP_TOOL] Config written to: ${OCP_CONFIG}" + echo "[OCP_TOOL] truncation_type: ${TRUNC_TYPE}" + echo "[OCP_TOOL] resolution: ${OCP_OIFS_RES_NUMBER}" + echo "[OCP_TOOL] experiment: ${OCP_OIFS_PREPIFS_EXPID}" + echo "[OCP_TOOL] ocean grid: ${OCP_FESOM_RES}" + echo "[OCP_TOOL] fesom_mesh: ${COUPLE_DIR}/${SUBMESH_DIR}/" + echo "[OCP_TOOL] openifs_default: ${OCP_POOL_DIR}/oifs-${OCP_OIFS_VERSION}/${OCP_OIFS_RES}${OCP_OIFS_LEVELS}/" + echo "[OCP_TOOL] root_dir (output): ${COUPLE_DIR}" +} + + +function ice2oifs { + echo "====================================================================" + echo "[OCP_TOOL] Starting OpenIFS initial condition regeneration" + echo "====================================================================" + echo "[OCP_TOOL] Mesh dir: ${COUPLE_DIR}/${SUBMESH_DIR}/ (mesh.nc will be generated by ocp-tool)" + echo "[OCP_TOOL] Resolution: ${OCP_OIFS_RES} (${OCP_OIFS_TRUNCATION})" + echo "[OCP_TOOL] Ocean grid: ${OCP_FESOM_RES}" + echo "[OCP_TOOL] Pool dir: ${OCP_POOL_DIR}" + + # Validate required environment variables + local missing_vars="" + for var in OCP_POOL_DIR OCP_OIFS_RES OCP_OIFS_RES_NUMBER OCP_OIFS_TRUNCATION \ + OCP_OIFS_LEVELS OCP_OIFS_PREPIFS_EXPID OCP_OIFS_VERSION OCP_FESOM_RES \ + COUPLE_DIR SUBMESH_DIR; do + eval val=\$$var + if [[ -z "$val" ]]; then + missing_vars="${missing_vars} ${var}" + fi + done + if [[ -n "${missing_vars}" ]]; then + echo "[OCP_TOOL] ERROR: Missing required environment variables:${missing_vars}" + echo "[OCP_TOOL] Skipping ocp-tool processing." + return 1 + fi + + # Validate FESOM ASCII mesh files exist (ocp-tool generates mesh.nc from these via pyfesom2) + if [[ ! -f "${COUPLE_DIR}/${SUBMESH_DIR}/nod2d.out" ]]; then + echo "[OCP_TOOL] ERROR: nod2d.out not found at ${COUPLE_DIR}/${SUBMESH_DIR}/" + echo "[OCP_TOOL] ocp-tool needs ASCII mesh files (nod2d.out, elem2d.out, etc.) to generate mesh.nc" + echo "[OCP_TOOL] Skipping ocp-tool processing." + return 1 + fi + + # Generate the YAML config + generate_ocp_config + + # Save current directory and move to COUPLE_DIR + # (ocp-tool's create_outputdirs.py uses CWD-relative ./output/ paths) + local saved_dir=$(pwd) + cd ${COUPLE_DIR} + echo "[OCP_TOOL] Running ocp-tool from: $(pwd)" + + # Run ocp-tool (assumes ocp_tool is pip-installed for the python3 in PATH) + local start_time=$(date +%s) + python3 -m ocp_tool ${OCP_CONFIG} + local OCP_EXIT=$? + local end_time=$(date +%s) + local elapsed=$((end_time - start_time)) + + # Return to original directory + cd ${saved_dir} + + if [[ ${OCP_EXIT} -ne 0 ]]; then + echo "[OCP_TOOL] ERROR: ocp-tool exited with code ${OCP_EXIT}" + echo "[OCP_TOOL] Coupling will continue but OpenIFS files may be stale!" + else + echo "[OCP_TOOL] ocp-tool completed successfully in ${elapsed} seconds" + fi + + local OCP_OUTPUT_DIR="${COUPLE_DIR}/output/${OCP_OIFS_TRUNCATION}${OCP_OIFS_RES_NUMBER}_${OCP_FESOM_RES}" + echo "[OCP_TOOL] Output directory: ${OCP_OUTPUT_DIR}" + if [[ -d "${OCP_OUTPUT_DIR}" ]]; then + echo "[OCP_TOOL] Output contents:" + ls -la ${OCP_OUTPUT_DIR}/ + fi + + echo "====================================================================" + echo "[OCP_TOOL] OpenIFS initial condition regeneration finished" + echo "====================================================================" +} diff --git a/couplings/pism/coupling_ocean2pism.functions b/couplings/pism/coupling_ocean2pism.functions index f46ba93fe..32e66f407 100644 --- a/couplings/pism/coupling_ocean2pism.functions +++ b/couplings/pism/coupling_ocean2pism.functions @@ -12,7 +12,12 @@ function ocean2pism { INTERPOL_TYPE_OCE=${INTERPOL_TYPE_OCE:-"dis"} #Standard=dis : "bil" "con" "dis" "nn", "bil" doesn't work for FESOM WEIGHTS_OCE2ICE=${WEIGHTS_OCE2ICE:-weights_oce2ice.${INTERPOL_TYPE_OCE}.nc} - GRIDDES_ICE=${POOL_DIR_pism}/grids/${DOMAIN_pism}/${EXE_pism}_${DOMAIN_pism}_${RES_pism}.griddes + if [[ -f ${COUPLE_DIR}/ice.griddes ]] + then + GRIDDES_ICE=${COUPLE_DIR}/ice.griddes + else + GRIDDES_ICE=${POOL_DIR_pism}/grids/${DOMAIN_pism}/${EXE_pism}_${DOMAIN_pism}_${RES_pism}.griddes + fi GRIDDES_OCE=${GRIDDES_OCE:-ocean.griddes} PISM_OCEAN_FORCING_FILE=${COUPLE_DIR}/"ocean_forcing4pism_ver${VERSION_pism}.${INTERPOL_TYPE_OCE}.${CHUNK_DATE_TAG_pism}.nc" diff --git a/couplings/pism/coupling_pism2atmosphere.functions b/couplings/pism/coupling_pism2atmosphere.functions index 80005486d..6813224c2 100644 --- a/couplings/pism/coupling_pism2atmosphere.functions +++ b/couplings/pism/coupling_pism2atmosphere.functions @@ -333,15 +333,34 @@ function convert_ice_thk_diff_to_flux_over_pism_run { function generate_orography_anomaly_over_pism_run { echo ""; echo " * calculating surface height change over the last pism chunk" ifile1=${last_year_in_chunk_restart} - ifile2=${first_year_in_chunk_input} - ofile=${COUPLE_DIR}/ice_orog_difference.nc - cdo -s -sub \ - -chname,${OROG_VARNAME_pism},${OROG_ANOM_VARNAME_pism} \ - -expr,"${OROG_VARNAME_pism}=thk+topg" \ - $ifile1 \ - -expr,"${OROG_VARNAME_pism}=thk+topg" \ - $ifile2 \ - $ofile 2>> ${COUPLE_DIR}/cdo_stderr_pism2atm + + # LA: as before use difference between PISM input and PISM restart for orography change + if [[ "x${orog_reference_pism}" == "x0" ]]; then + ifile2=${first_year_in_chunk_input} + ofile=${COUPLE_DIR}/ice_orog_difference.nc + cdo -s -sub \ + -chname,${OROG_VARNAME_pism},${OROG_ANOM_VARNAME_pism} \ + -expr,"${OROG_VARNAME_pism}=thk+topg" \ + $ifile1 \ + -expr,"${OROG_VARNAME_pism}=thk+topg" \ + $ifile2 \ + $ofile 2>> ${COUPLE_DIR}/cdo_stderr_pism2atm + # LA: now new use difference between elev_lo (ECHAM orography) and PISM restart for orography change + elif [[ "x${orog_reference_pism}" == "x1" ]]; then + ifile2="${COUPLE_DIR}/elev_lo.nc" + ofile=${COUPLE_DIR}/ice_orog_difference.nc + cdo -s -sub \ + -chname,${OROG_VARNAME_pism},${OROG_ANOM_VARNAME_pism} \ + -expr,"${OROG_VARNAME_pism}=thk+topg" \ + $ifile1 \ + -selname,"orog" \ + $ifile2 \ + $ofile 2>> ${COUPLE_DIR}/cdo_stderr_pism2atm + else + echo " * choose 0 or 1" + echo " * EXIT " + exit 42 + fi ncks -O -x -v time,GEOSP $ofile tmp.nc 2>> ${COUPLE_DIR}/cdo_stderr_pism2atm mv tmp.nc $ofile diff --git a/couplings/pism/coupling_pism2esm.functions b/couplings/pism/coupling_pism2esm.functions index 847efc50b..f9290dab2 100644 --- a/couplings/pism/coupling_pism2esm.functions +++ b/couplings/pism/coupling_pism2esm.functions @@ -1,6 +1,11 @@ #!/usr/bin/ksh pism2esm() { + echo "====================================================================" + echo "[PISM2ESM] Starting PISM to ESM coupling" + echo "====================================================================" + echo "[PISM2ESM] Loaded modules:" echo $(module list) + echo "[PISM2ESM] Loading coupling functions..." . ${FUNCTION_PATH}/../general/general_lists.functions . ${FUNCTION_PATH}/../general/general_helpers.functions . ${FUNCTION_PATH}/../general/coupling_general.functions @@ -33,18 +38,22 @@ pism2esm() { # . ${FUNCTION_PATH}/coupling_pism2atmosphere.functions.origin . ${FUNCTION_PATH}/coupling_pism2ocean.functions - echo "UKK0********" - # export GEOMETRY_FILE="latest_ice_geometry.nc" - # export PATH2COUPLE=${COUPLE_DIR} + echo "[PISM2ESM] Extracting ice geometry variables from PISM output..." + echo "[PISM2ESM] Input file: ${OUTPUT_DIR_pism}/latest_ex_file_pism.nc" + echo "[PISM2ESM] Output file: ${COUPLE_DIR}/latest_ice_geometry.nc" if [[ -f ${OUTPUT_DIR_pism}/latest_ex_file_pism.nc ]]; then echo "cdo seltimestep,1 -expr,ice_subNN=(usurf-thk)*(mask==3); shelf_msk=(mask==3); ocean_msk=(mask==4); land_msk=(mask<=3); grounded_ice_mask=(mask==2); mask=mask; ${OUTPUT_DIR_pism}/latest_ex_file_pism.nc ${COUPLE_DIR}/latest_ice_geometry.nc" cdo seltimestep,1 -expr,"ice_subNN=(usurf-thk)*(mask==3); shelf_msk=(mask==3); ocean_msk=(mask==4); land_msk=(mask<=3); grounded_ice_mask=((mask==2)); mask=mask;" \ ${OUTPUT_DIR_pism}/latest_ex_file_pism.nc ${COUPLE_DIR}/latest_ice_geometry.nc + else + echo "[PISM2ESM] WARNING: PISM output file not found!" fi # env > env_pism2awiesm.txt #pism2ocean 2>> ${COUPLE_DIR}/cdo_stderr_pism2ocean - echo "*** Starting pis2atmosphere ***" + echo "[PISM2ESM] Calling pism2atmosphere coupling..." # pism2atmosphere 2>> ${COUPLE_DIR}/cdo_stderr_pism2awiesm #pism2ocean + echo "[PISM2ESM] PISM to ESM coupling completed" + echo "====================================================================" } diff --git a/couplings/pism/env_pism.py b/couplings/pism/env_pism.py index d2e2386db..62ad86d34 100644 --- a/couplings/pism/env_pism.py +++ b/couplings/pism/env_pism.py @@ -49,6 +49,17 @@ def prepare_environment(config): "TEMP2_BIAS_FILE": config[config["general"]["setup_name"]].get("temp2_bias_file"), "DOWNSCALING_LAPSE_RATE": config[config["general"]["setup_name"]].get("lapse_rate", -0.005), "DOWNSCALE_PRECIP": config[config["general"]["setup_name"]].get("downscale_precip", 1), + + # OCP-tool variables (for ice2oifs coupling) + "OCP_POOL_DIR": config["general"].get("pool_dir", ""), + "OCP_OIFS_RES": config.get("oifs", {}).get("resolution", ""), + "OCP_OIFS_RES_NUMBER": config.get("oifs", {}).get("res_number", ""), + "OCP_OIFS_TRUNCATION": config.get("oifs", {}).get("truncation", ""), + "OCP_OIFS_LEVELS": config.get("oifs", {}).get("levels", ""), + "OCP_OIFS_PREPIFS_EXPID": config.get("oifs", {}).get("prepifs_expid", ""), + "OCP_OIFS_INPUT_EXPID": config.get("oifs", {}).get("input_expid", ""), + "OCP_OIFS_VERSION": config.get("oifs", {}).get("version", ""), + "OCP_FESOM_RES": config.get("fesom", {}).get("resolution", ""), } print (environment_dict) return environment_dict diff --git a/couplings/pism/mesh_flags.py b/couplings/pism/mesh_flags.py index f57dd888c..d169014c3 100644 --- a/couplings/pism/mesh_flags.py +++ b/couplings/pism/mesh_flags.py @@ -3,21 +3,30 @@ import xarray as xr import cftime from netCDF4 import Dataset -print("BLABLA") + +print("====================================================================") +print("[MESH_FLAGS] Starting mesh flag generation from PISM ice geometry") +print("====================================================================") + pathname = os.getenv('PATH2COUPLE') filename = os.getenv('GEOMETRY_FILE') #Mmesh_folder = os.getenv('MESHES') Mmesh_folder = os.getenv('MAX_MESH') -print(f"Pathname: {pathname}") -print(f"Filename: {filename}") -print(f"Meshes: {Mmesh_folder}") +print(f"[MESH_FLAGS] Configuration:") +print(f"[MESH_FLAGS] Coupling directory: {pathname}") +print(f"[MESH_FLAGS] Geometry file: {filename}") +print(f"[MESH_FLAGS] Maximum mesh directory: {Mmesh_folder}") # Read netCDF data fname = os.path.join(pathname,filename) sea_level=0.0 +print(f"[MESH_FLAGS] Reading PISM geometry file: {fname}") # Read mesh data (assuming text files) Mnode_fname = os.path.join(Mmesh_folder, 'nod2d.out') Mbed_fname = os.path.join(Mmesh_folder, 'aux3d.out') +print(f"[MESH_FLAGS] Reading FESOM mesh files:") +print(f"[MESH_FLAGS] Nodes: {Mnode_fname}") +print(f"[MESH_FLAGS] Bedrock: {Mbed_fname}") ds = xr.open_dataset(fname, decode_times=False) #print("BLABLA") #ds = ds.convert_calendar('standard', align_on='year') @@ -31,17 +40,23 @@ #ds = ds.convert_calendar('standard', align_on='year') #time_values = ds['time'].values mask = ds['mask'].values.flatten() -print("MASK") +print(f"[MESH_FLAGS] Loaded PISM mask data: {mask.shape[0]} nodes") +print(f"[MESH_FLAGS] Mask values present: {np.unique(mask)}") -cavity_depth = np.minimum(ds['ice_subNN'].values.flatten(),0) +print(f"[MESH_FLAGS] Computing ice shelf cavity depth...") +cavity_depth = np.minimum(ds['ice_subNN'].values.flatten(),0) +print(f"[MESH_FLAGS] Cavity depth range: [{cavity_depth.min():.1f}, {cavity_depth.max():.1f}] m") # Read Mnode_info #Mnode_info0 = np.loadtxt(f"{Mmesh_folder}{Mnode_fname}") # Load the ASCII file as a flat array +print(f"[MESH_FLAGS] Reading FESOM maximum mesh node information...") with open(Mnode_fname, 'r') as f: # Read the number from the first line Mnode_num = int(f.readline().strip()) +print(f"[MESH_FLAGS] Total nodes in maximum mesh: {Mnode_num}") # Read Mbed_info +print(f"[MESH_FLAGS] Reading bedrock elevation data...") Mbed_info0 = np.loadtxt(f"{Mbed_fname}") Mvert_num = int(Mbed_info0[0]) Mvert_lev = Mbed_info0[1:Mvert_num+1] @@ -52,28 +67,44 @@ #Mnode_lat = Mnode_info[:, 2] Mbed_info = -np.abs(Mbed_info) +print(f"[MESH_FLAGS] Processing topography and masks...") # Prepare arrays to write out topo_raw = Mbed_info.copy() #condition=((Mbed_info > sea_level) & (mask < 4)) # not ocean and bedrock above sea level +print(f"[MESH_FLAGS] Applying mask conditions:") condition=((Mbed_info - cavity_depth) > sea_level) # excl grounded ice shelf even if it is floating in PISM. +print(f"[MESH_FLAGS] Nodes above sea level: {np.sum(condition)}") topo_raw[condition] = 9999 +print(f"[MESH_FLAGS] Grounded ice nodes (mask==2): {np.sum(mask == 2)}") topo_raw[mask == 2] = 9999 # grounded ice in PISM +print(f"[MESH_FLAGS] Creating node classification flags:") Mnode_pism = np.ones(Mnode_num) -Mnode_pism[mask == 3] = 1 -Mnode_pism[mask == 4] = 1 -Mnode_pism[mask == 2] = 0 -Mnode_pism[mask == 1] = 0 -Mnode_pism[mask == 5] = 2 -Mnode_pism[condition] = -1 +Mnode_pism[mask == 3] = 1 # Ice shelf +Mnode_pism[mask == 4] = 1 # Open ocean +Mnode_pism[mask == 2] = 0 # Grounded ice +Mnode_pism[mask == 1] = 0 # Ice-free land +Mnode_pism[mask == 5] = 2 # Special case +Mnode_pism[condition] = -1 # Above sea level + +print(f"[MESH_FLAGS] Ice shelf nodes (flag=1, mask=3): {np.sum(mask == 3)}") +print(f"[MESH_FLAGS] Open ocean nodes (flag=1, mask=4): {np.sum(mask == 4)}") +print(f"[MESH_FLAGS] Grounded ice nodes (flag=0, mask=2): {np.sum(mask == 2)}") +print(f"[MESH_FLAGS] Ice-free land nodes (flag=0, mask=1): {np.sum(mask == 1)}") +print(f"[MESH_FLAGS] Special nodes (flag=2, mask=5): {np.sum(mask == 5)}") +print(f"[MESH_FLAGS] Excluded nodes (flag=-1): {np.sum(condition)}") #cavity_raw = np.zeros(Mnode_num) -print([Mbed_info, cavity_depth,Mnode_pism]) # Write out +print(f"[MESH_FLAGS] Writing output files:") np.savetxt('topo_raw.txt', topo_raw, fmt='%10f') +print(f"[MESH_FLAGS] ✓ topo_raw.txt ({len(topo_raw)} values)") np.savetxt('cavity_raw.txt', cavity_depth, fmt='%10f') +print(f"[MESH_FLAGS] ✓ cavity_raw.txt ({len(cavity_depth)} values)") np.savetxt('mask.txt', Mnode_pism, fmt='% d') +print(f"[MESH_FLAGS] ✓ mask.txt ({len(Mnode_pism)} values)") -print("Generate new bathymetry and iceshelf topography on maxmesh. Done!") +print("[MESH_FLAGS] Mesh flag generation completed successfully!") +print("====================================================================") diff --git a/couplings/pism/test.sh b/couplings/pism/test.sh index 0484120d0..f90f1cdb1 100755 --- a/couplings/pism/test.sh +++ b/couplings/pism/test.sh @@ -1,23 +1,88 @@ -#!/bin/ksh +#!/bin/bash +# Save script directory BEFORE sourcing env file (which overrides PWD) +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +echo "Script directory: ${SCRIPT_DIR}" + +# Check for clean flag +if [[ "$1" == "clean" ]] || [[ "$1" == "--clean" ]]; then + echo "[CLEAN] Removing all generated test files..." + rm -rf ${SCRIPT_DIR}/test_couple/ + rm -f ${SCRIPT_DIR}/stderr_awiesm ${SCRIPT_DIR}/stderr_fesom + echo "[CLEAN] Done. Run ./test.sh to start fresh test." + exit 0 +fi + +# Save python3 path and HOME before env file overwrites them +PYTHON3_DIR="$(dirname "$(which python3)")" +ORIG_HOME="${HOME}" + echo "Sourcing environments..." source ./env_pism2awiesm.txt + +# Restore HOME (env file sets it to a different user) +export HOME="${ORIG_HOME}" echo "env pism2awiesm" #source ./env_pism2ocean.txt echo "env pism2ocean" echo "...done" -export PATH="/sw/spack-levante/mambaforge-22.9.0-2-Linux-x86_64-kptncg/bin/:${PATH}" + +# Set COUPLE_DIR to a local writable directory +export COUPLE_DIR="${SCRIPT_DIR}/test_couple" +mkdir -p ${COUPLE_DIR} +echo "Using local COUPLE_DIR: ${COUPLE_DIR}" + +# Copy grid files if they don't exist +if [ ! -f ${COUPLE_DIR}/ice.griddes ]; then + echo "[SETUP] Copying required grid description files..." + cp /work/ab0246/a270122/lars_and_paul/experiments/test_env/couple/ice.griddes ${COUPLE_DIR}/ + cp /work/ab0246/a270122/lars_and_paul/experiments/test_env/couple/ocean.griddes ${COUPLE_DIR}/ +fi + +# Override OUTPUT_DIR_pism from env file to use readable directory +export OUTPUT_DIR_pism="/work/ab0246/a270122/lars_and_paul/experiments/test_env/outdata/pism" +echo "Reading PISM output from: ${OUTPUT_DIR_pism}" + +# Restore python3 to PATH (env file overwrites it) +export PATH="${PYTHON3_DIR}:${PATH}" export PATH="/sw/spack-levante/singularity-3.8.5-w53g5a/bin/:${PATH}" -export PATH="/usr/bin/:${PATH}" +export PATH="${PATH}:/usr/bin/" export PATH2COUPLE=${COUPLE_DIR} export MAX_MESH='/work/ba0989/a270124/PalModII/experiments/ICEBERGS/mesh_core2/' export CHANGE_OCEAN='1' +export ICE_TO_FESOM=1 +export SUBMESH_DIR="submesh_5007-12-31T00:00:00" echo $PATH -. /work/ab0246/a270122/lars_and_paul/esm_tools/couplings/pism/coupling_pism2esm.functions -. /work/ab0246/a270122/lars_and_paul/esm_tools/couplings/fesom/coupling_ice2fesomUKK.functions -echo "UKK ******" + +# Copy submesh for ocp-tool testing (needs mesh_cst.nc + nod2d.out, elem2d.out, etc.) +if [ ! -f ${COUPLE_DIR}/${SUBMESH_DIR}/mesh_cst.nc ]; then + echo "[SETUP] Copying submesh test data..." + mkdir -p ${COUPLE_DIR}/${SUBMESH_DIR} + cp -r /work/ab0246/a270122/lars_and_paul/experiments/test_env/couple/${SUBMESH_DIR}/* ${COUPLE_DIR}/${SUBMESH_DIR}/ +fi + +# OCP-tool environment variables (hardcoded for testing; in production these +# come from env_pism.py via ESM-Tools config) +export OCP_POOL_DIR="/work/ab0246/a270092/input" +export OCP_OIFS_RES="TCO95" +export OCP_OIFS_RES_NUMBER=95 +export OCP_OIFS_TRUNCATION="TCO" +export OCP_OIFS_LEVELS="L91" +export OCP_OIFS_PREPIFS_EXPID="ab45" +export OCP_OIFS_INPUT_EXPID="awi3" +export OCP_OIFS_VERSION="48r1" +export OCP_FESOM_RES="CORE2" + +# Source local updated coupling functions (use SCRIPT_DIR captured before PWD override) +. ${SCRIPT_DIR}/coupling_pism2esm.functions +. ${SCRIPT_DIR}/../fesom/coupling_ice2fesomUKK.functions +. ${SCRIPT_DIR}/../oifs/coupling_ice2oifs.functions + +echo "[TEST] Running pism2esm coupling..." pism2esm 2>> ./stderr_awiesm -echo "UKK1 ******" +echo "[TEST] Running ice2fesom coupling..." ice2fesom 2>> ./stderr_fesom +echo "[TEST] Running ice2oifs (ocp-tool) coupling..." +ice2oifs 2>> ./stderr_oifs #pism2ocean 2>> ./stderr_pism2ocean exit diff --git a/docs/conf.py b/docs/conf.py index b50c71b88..a8e2a575b 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -163,7 +163,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. diff --git a/docs/esm_master.rst b/docs/esm_master.rst index bc9cad002..97a8733b1 100644 --- a/docs/esm_master.rst +++ b/docs/esm_master.rst @@ -63,6 +63,11 @@ You can get further help with:: $ esm_master --help +Logging +------- + +``esm_master`` uses Loguru for console logging. To increase verbosity call ``esm_master`` with the ``-v`` or ``--verbose`` flag. + Configuring esm-master for Compile-Time Overrides ------------------------------------------------- diff --git a/docs/esm_runscripts.rst b/docs/esm_runscripts.rst index 60d55a678..c0a0e4b6e 100644 --- a/docs/esm_runscripts.rst +++ b/docs/esm_runscripts.rst @@ -9,7 +9,7 @@ Usage esm_runscripts [-h] [-d] [-v] [-e EXPID] [-c] [-P] [-j LAST_JOBTYPE] [-t TASK] [-p PID] [-x EXCLUDE] [-o ONLY] - [-r RESUME_FROM] [-U] + [-r RESUME_FROM] [-U] [-i INSPECT] runscript Arguments @@ -47,6 +47,8 @@ Optional arguments Description of the experiment folder), a filename or a directory name output the content of the file /directory if found in the last ``run_`` folder.) + ``--trace`` Enable ``TRACE``-level output to stdout. + ``--task-log-files`` Enable per-task log files on disk. =================================================================== ========================================================== @@ -427,3 +429,30 @@ ESM-Tools behave. To create a virtual environment with ESM-Tools installed in folder .venv_esmtools listed above and **not** from your user install directory. You should make **all** changes to the namelists and config files via your user runscript (:ref:`yaml:Changing Namelists`). This is recommended in all cases!!! + +Logging and verbosity +--------------------- + +``esm_runscripts`` uses Loguru-based logging with simple flags to control verbosity and +file logging. Logs are always written in the main run log ( +``//log/___.log``). For more log +granularity, it is possible to also set ``--task-log-files`` as a flag of +``esm_runscripts``, to write logs of each task to a separate file. You can use the +following ``esm_runscripts`` flags to control the logging behavior: + +* ``--trace``: enable ``TRACE``-level output to stdout. Prints very detailed + diagnostics and the parsed command-line config. +* ``-d``, ``--debug``: enable ``DEBUG``-level output to stdout (less detailed than + ``--trace``) and breakpoints. +* ``-v``, ``--verbose``: also enables ``DEBUG``-level output to stdout, without + breakpoints. +* ``--task-log-files``: enable per-task log files on disk. When enabled, + ``esm_runscripts`` writes each task's output to a file in the experiment's ``log`` + folder (``//log/____.log``). + To reduce the number of files, this option is turned off by default, but the logs + are always printed in the run log anyway. + +.. note:: + Because the logging starts before the parsing of the yaml files, it is not possible + to control the logging behavior from variables defined in the yamls. Only + command-line flags can control the logging behavior. diff --git a/docs/glossary.rst b/docs/glossary.rst new file mode 100644 index 000000000..3d20fe326 --- /dev/null +++ b/docs/glossary.rst @@ -0,0 +1,17 @@ +======== +Glossary +======== + +.. glossary:: + + chunk + A chunk is a set of :term:`runs` that are grouped together for execution. This allows + to group runs for offline coupling. + experiment + The whole simulation and computations performed under the same :term:`workflow`. + job + A job is a unit of work inside a :term:`workflow`. Sometimes also referred as a phase. + run + A run the set of computations and :term:`jobs` until a restart is performed, a cycle of within a :term:`experiment`. + workflow + A workflow is a series of :term:`jobs` that are executed during a run and their dependencies. diff --git a/docs/index.rst b/docs/index.rst index b711c1199..62ade1083 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,7 @@ Welcome to ESM Tools's documentation! esm_tests motd cookbook + glossary faq packages API diff --git a/docs/workflow.rst b/docs/workflow.rst index 04be36e06..5d8ce6e50 100644 --- a/docs/workflow.rst +++ b/docs/workflow.rst @@ -5,135 +5,347 @@ ESM Runscripts - Using the Workflow Manager Introduction ------------ -Starting with Release 6.0, esm_runscripts allows the user to define additional subjobs for data processing, arrange them in clusters, and set the order of execution -of these and the standard runjob parts in a flexible and short way from the runscript. This is applicable for both pre- and postprocessing, but especially useful -for iterative coupling jobs, like e.g. coupling pism to vilma (see below). In this section we explain the basic concept, and the keywords that have to be set in the -runscript to make use of this feature. +Starting with Release 6.0, ``esm_runscripts`` allows to define additional :term:`jobs` for e.g. data processing, coupling. +Such jobs can be arranged into job-clusters, and the order of execution can be set in a flexible and short way from the runscript. +This is applicable for both pre- and postprocessing, but especially useful for iterative coupling jobs, like e.g. coupling PISM to +VILMA (see below). In this section we explain the basic concept, describe the keywords that have to be set in the runscript in order +to make use of this feature, and give some examples on how to integrate pre- and postprocessing jobs and how to set up jobs for iterative +coupling. -Subjobs of a ``normal`` run ---------------------------- +Default jobs of a general model simulation run +---------------------------------------------- -Even before the addition of the workflow manager, the run jobs of esm_runscript were split into different subjobs, even though that was mostly hidden from the user's view. Before -Release 6.0, these subjobs were: - -:: - - compute --> tidy (incl. wait_and_observe + resubmit next run) - -Technically, ``wait_and_observe`` was part of the tidy job, as was the resubmission, including above only for the purpose of demonstrating the difference to the -new standard workflow, which is now (post-Release 6.0):: +The task of ``esm_runscript`` is split into different subjobs which are:: newrun --> prepcompute --> compute --> observe_compute --> tidy (+ resubmit next run) -Other than before adding the workflow manager, these standard subjobs are all separated and independant subjobs, each submitted (or started) by the previous subjob in one of three -ways (see below). The splitting of the old compute job into newrun, prepcompute and compute on one side, and tidy into observe and tidy, was necessary to enable -the user to insert coupling subjobs for iterative coupling at the correct places. Here is what each of the standard subjobs does: +These standard jobs are all separated and independent, each submitted (or started) by the previous job in one of three +ways (see below). Here is what each of the standard jobs do: +.. |warn| replace:: ⚠️ **Warning:** It needs to be the first job of any :term:`experiment`. -====================================================== ========================================================== -Subjob Function -====================================================== ========================================================== +====================================================== ============================================================= ======================== +Job Description Started by +====================================================== ============================================================= ======================== newrun Initializes a new experiment, only very basic stuff, like creating (empty) folders needed by any of the following - subjobs. NEEDS TO BE THE FIRST SUBJOB OF ANY EXPERIMENT. - prepcompute Prepares the compute job. All the (Python) functionality that + subjobs/jobs. + |warn| + prepcompute Prepares the compute job. All the (Python) functionality that newrun needs to be run, up to the job submission. Includes copying files, editing namelists, write batch scripts, etc. - compute Actual model integration, nothing else. No Python codes + compute Actual model integration, nothing else. No Python codes prepcompute via ``sbatch`` or other batch system command involved. - observe_compute Python job running at the same time as compute, checking if + observe_compute Python job running at the same time as compute, checking if ``sbatch``, started by its own ``esm_runscripts`` call in the ``.run`` script, after the ``compute`` job has been submitted with ``srun`` or other batch launcher. the compute job is still running, looking for some known errors for monitoring / job termination. - tidy Sorts the produced outputs, restarts and log files into + tidy Sorts the produced outputs, restarts and log files into observe_compute the correct folders, checks for missing and unknown files, builds coupler restart files if not present -====================================================== ========================================================== +====================================================== ============================================================= ======================== + +.. note:: + None of this has to be edited by the users. The above described workflow jobs form the default set of jobs needed to run any simulation. + Changing anyone of these jobs may lead `ESM-Tools` to fail. However, additional jobs can be added to this workflow, as described below, + to extend the default workflow. + +Inspect workflow jobs +--------------------- + +To inspect the workflow and workflow jobs that are defined by e.g. a choosen setup or in an already run simulation/experiment, you can +run esm_runscript with the ``-i`` (``--inspect``) option. This can be done for two different cases: + +- To inspect the workflow previous to running a certain experiment. For example, if you want to add a new workflow job, and need to know + which jobs are already defined in a choosen setup or model configuration:: + + esm_runscripts runscript.yaml -i workflow + +- To inspect a workflow from an experiment that has beed carried out already or created during a check-run (-c):: + + esm_runscripts runscript.yaml -e -i workflow + +It will display the workflow configuration showing the order of workflow jobss and their attributes and possible dependencies. This output +should help to find out the correct keyworkds to be set when integrating a new workflow job. + +**Example output**:: -It is important to understand that none of this has to be edited by the users, this is the default setup. + Workflow sequence (cluster [jobs]) + ---------------------------------- + prepcompute ['prepcompute'] -> compute ['compute'] -> tidy ['tidy'] -> prepcompute ['prepcompute'] and my_own_new_cluster ['my_new_last_job', 'my_second_new_job'] +.. _def_workflow_jobs: -Keywords available for defining additional data processing subjobs ------------------------------------------------------------------- +Defining additional workflow jobs +--------------------------------- -The workflow manager is intended to include shell scripted data processing jobs into the -esm_runscripts workflow, so several things have to be defined: +If it is necessary to complement the default workflow with simulation specific processing steps, this sequence of default workflow jobs can be +extended by adapting the runscipt or any component specific configuration files. The workflow manager will evaluate these additional jobss and +integrate them into the default sequence of the workflow. In order to integrate the additional jobs correctly, the following information about +this job needs to be given in the one of the yaml files: * Name of the script to be run * Name of the python script used for setting up the environment - * Name of the folder in which both of the above scripts can be found - * Information on how often the subjob should be called - * Information on between which other subjobs the new subjob should be inserted into the workflow - * In case it isn't clear: Which subjob should resubmit the next run. - -The keywords used to define that are: - -====================================================== ========================================================== -Keyword Function -====================================================== ========================================================== - workflow Chapter headline in a model's section, indicating that - alterations to the standard workflow will be defined here - subjob_clusters Section in the workflow chapter, containing the information - on additional subjob_clusters. A subjob_cluster is a - collection of subjobs run from the same batch script. Each - subjob needs to belong to one cluster, if none is defined, - each subjob will automatically get assigned to its own - cluster. Each entry in ``subjob_clusters`` is a dict, - with the outermost key being the (arbitrary) name of the - cluster. - subjobs Section in the workflow chapter, containing the information - on additional subjobs. - run_after / run_before Entry in spevifications of a subjob_cluster, to define - before or after which other cluster of the workflow this cluster - is supposed to run. Only one of the two should be specified. - Can also be used in the specifications of subjobs if these - subjobs get a corresponding cluster auto-assigned. - script: - script_dir: - call_function: - env_preparation: - next_run_triggered_by: -====================================================== ========================================================== + * Path to the directory in which both of the above scripts can be found + * Information on how often the job should be called + * Information where in the workflow the new job needs to be inserted + * In case it isn't clear, which job should resubmit the next run. + +In general, a workflow can be defined in the runscript or in any component configuration file. But there are some restrictions to the +definition that needs to be taken into account: + * The name of each job needs to be unique. Otherwise, an exception error will be raised. + * The names of the default jobs are not allowed to be used for any new jobs. This will also cause an exception error during runtime. + * Settings in the runscript will overwrite settings in other config files. (See also + :ref:`yaml_hierarchy:Hierarchy of YAML configuration files`.) + +Keywords to define a new workflow job +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +To provide the information about a new job the following keywords and mappings (key/value pairs) are available (keywords that are indicated +with ``< >`` need to be adapted by the user): + +====================================================== ============ =========================== ========================================================== +Keyword Mandatory (Default) values Function +====================================================== ============ =========================== ========================================================== + ``workflow`` yes -- Chapter headline in a runscript or configuration section, + indicating that alterations to the standard workflow + will be defined here. + + ``subjobs`` yes user defined string Section within the ``workflow`` chapter that containes new + additional workflow jobs. + + ```` yes user defined string Section within the ``subjobs`` section for each new job. + The name of the new job needs to be unique. See also further + explenation in :ref:`def_workflow_jobs` + + ``run_after: `` or ``run_before: `` no default: last job in Key/value entry in each ``job`` section. + (default) workflow This mapping defines the (default or user) job of the + (e.g. tidy) workflow after or before the new job should be executed. + Only one of the two should be specified. + + ``submit_to_batch_system: `` no **false**, true Key/value entry in each ``job`` section. + This mapping defines if the (default or user) job is + submitted to batch system or not. + + ``run_on_queue: `` no None Key/value entry in each ``job`` section. + This mapping defines to which queue (name) the job + should be submitted to. + + ``batch_or_shell: `` no **shell**, batch Key/value entry in each ``job`` section. + This mapping defines if the (default or user) job is submitted + as batch job or as shell script. + This attribute will be overwritten depending on ``submit_to_batch_system`` + + ``cluster: `` no Job name Key/value entry in each ``job`` section. Jobs + that have the same entry in ``cluster`` will be run + from the same batch script. + + ``order_in_cluster: `` no **sequential**, concurrent Key/value entry in each ``job`` section. This mapping + defines how jobs in the same ```` should be run. + Concurrent or serial. + + ``script: `` yes None Key/value entry in each ``job`` section. + This mapping defines the name of the script that is going + to be executed during the new workflow job. + + ``script_dir: `` yes None Key/value entry in each ``job`` section. + This mapping defines the path to the script set by the variable + ``