diff --git a/configs/components/amip/amip.yaml b/configs/components/amip/amip.yaml index d6a523d28..81406cc07 100644 --- a/configs/components/amip/amip.yaml +++ b/configs/components/amip/amip.yaml @@ -17,7 +17,7 @@ amip: ## clean_command: rm -rf bin; cd src; make clean; cd .. comp_command: rm -rf bin; mkdir bin; cd src; make ; cd .. ; cp bin/amip-forcing.exe ./bin/amip - git-repository: https://gitlab.dkrz.de/ec-earth/amip-forcing.git + git-repository: https://gitlab.dkrz.de/ec-earth/amip-forcing-ece4.git install_bins: bin/amip ## ## Directories @@ -41,6 +41,8 @@ amip: namelists: - "namelist.amip" + mip: "cmip7" + choose_resolution: 360x180: nx: 360 @@ -59,18 +61,30 @@ amip: config_sources: namelist.amip: "${namelist_dir}/namelist.amip" - input_in_work: - sst_forcing: "tosbcs_input4MIPs_SSTsAndSeaIce_CMIP_PCMDI-AMIP-1-1-0_gs1x1_187001-201512.nc" - sic_forcing: "siconcbcs_input4MIPs_SSTsAndSeaIce_CMIP_PCMDI-AMIP-1-1-0_gs1x1_187001-201512.nc" + choose_mip: + "cmip6": + input_in_work: + sst_forcing: "tosbcs_input4MIPs_SSTsAndSeaIce_CMIP_PCMDI-AMIP-1-1-0_gs1x1_187001-201512.nc" + sic_forcing: "siconcbcs_input4MIPs_SSTsAndSeaIce_CMIP_PCMDI-AMIP-1-1-0_gs1x1_187001-201512.nc" + input_sources: + sst_forcing: "${input_dir}/tosbcs_input4MIPs_SSTsAndSeaIce_CMIP_PCMDI-AMIP-1-1-0_gs1x1_187001-201512.nc" + sic_forcing: "${input_dir}/siconcbcs_input4MIPs_SSTsAndSeaIce_CMIP_PCMDI-AMIP-1-1-0_gs1x1_187001-201512.nc" + sstvarname: "tosbcs" + siconcvarname: "siconcbcs" + "cmip7": + input_in_work: + sst_forcing: "tos_input4MIPs_SSTsAndSeaIce_CMIP_PCMDI-AMIP-1-1-10_gn_187001-202212.nc" + sic_forcing: "siconc_input4MIPs_SSTsAndSeaIce_CMIP_PCMDI-AMIP-1-1-10_gn_187001-202212.nc" + input_sources: + sst_forcing: "${input_dir}/tos_input4MIPs_SSTsAndSeaIce_CMIP_PCMDI-AMIP-1-1-10_gn_187001-202212.nc" + sic_forcing: "${input_dir}/siconc_input4MIPs_SSTsAndSeaIce_CMIP_PCMDI-AMIP-1-1-10_gn_187001-202212.nc" + sstvarname: "tos" + siconcvarname: "siconc" input_files: sst_forcing: sst_forcing sic_forcing: sic_forcing - input_sources: - sst_forcing: "${input_dir}/tosbcs_input4MIPs_SSTsAndSeaIce_CMIP_PCMDI-AMIP-1-1-0_gs1x1_187001-201512.nc" - sic_forcing: "${input_dir}/siconcbcs_input4MIPs_SSTsAndSeaIce_CMIP_PCMDI-AMIP-1-1-0_gs1x1_187001-201512.nc" - coupling_fields: "[[amip_fields_send-->FIELD]]": grid: AMIP @@ -79,11 +93,13 @@ amip: namelist.amip: NAMAMIP: RunLengthSec: "${runtime_seconds}" - TimeStepSec: "84600" + TimeStepSec: "86400" FixYear: "0" StartYear: "1850" FileListSST: ${input_in_work.sst_forcing} FileListSIC: ${input_in_work.sic_forcing} + sstvarname: ${sstvarname} + siconcvarname: ${siconcvarname} grids: AMIP: diff --git a/configs/components/oasis3mct/oasis3mct.yaml b/configs/components/oasis3mct/oasis3mct.yaml index 5077d1fd4..1ada6e545 100644 --- a/configs/components/oasis3mct/oasis3mct.yaml +++ b/configs/components/oasis3mct/oasis3mct.yaml @@ -87,8 +87,8 @@ oasis3mct: 4.0-awicm-frontiers: branch: awicm-3-frontiers 5.0-smhi: - branch: main - git-repository: https://git.smhi.se/ec-earth/vendor/oasis/oasis3-mct-5.git + branch: local_combined_fixes + git-repository: https://git.smhi.se/jan.streffing/oasis3-mct-5 comp_command: 'mkdir -p build; cd build; cmake ..; make -j 1; mkdir -p ../include/; cp lib/psmile/libpsmile.a lib/psmile/mct/libmct.a lib/psmile/mct/mpeu/libmpeu.a lib/psmile/scrip/libscrip.a ../lib; cd ..; find . -type f \( -name "*.o" -o -name "*.mod" \) ! -path "./include/*" -exec cp -t include/ {} +; mkdir -p arch_ecearth; ln -fs ../lib ../include arch_ecearth/' 5.0-geomar: # this version is taken directly from the CERFACS gitlab diff --git a/configs/components/oifs/oifs.yaml b/configs/components/oifs/oifs.yaml index e848a80bc..621e34b2f 100644 --- a/configs/components/oifs/oifs.yaml +++ b/configs/components/oifs/oifs.yaml @@ -1526,7 +1526,7 @@ oifs: #====================================================================================== # RECIPE #====================================================================================== - +general: prepcompute_recipe: - "compile_model" - "_show_simulation_info" diff --git a/configs/components/oifs/oifs48.env.yaml b/configs/components/oifs/oifs48.env.yaml index 049f54da0..cab244bb1 100644 --- a/configs/components/oifs/oifs48.env.yaml +++ b/configs/components/oifs/oifs48.env.yaml @@ -4,8 +4,8 @@ computer: choose_computer.name: levante: - compiler_mpi: gnu11_ompi4 - iolibraries: system_gnu_libs + compiler_mpi: intel2022_openmpi + iolibraries: system_intel_libs load_python: True add_export_vars: DATA: "${general.work_dir}" @@ -14,7 +14,16 @@ computer: OIFS_LOGFILE: '${oifs.model_dir}/build/oifs_test_log.txt' DR_HOOK_IGNORE_SIGNALS: '-1' OIFS_DATA_DIR: "/p/project/chhb19/streffing1/input/oifs-48r1/" - MKL_CBWR: AUTO,STRICT + # MKL_CBWR: AUTO,STRICT # DISABLED: kills performance by disabling vectorization + OMP_SCHEDULE: STATIC + OMP_STACKSIZE: 512M + # Intel MKL performance optimizations + MKL_NUM_THREADS: "1" + MKL_DYNAMIC: "FALSE" + # Intel MPI collective optimizations + I_MPI_ADJUST_ALLREDUCE: "3" + I_MPI_ADJUST_BCAST: "1" + I_MPI_DEBUG: "0" juwels: compiler_mpi: intel2025_psmpi @@ -26,7 +35,30 @@ computer: GRIB_SAMPLES_PATH: '${oifs.model_dir}/build/share/eccodes/ifs_samples/grib1_mlgrib2' OIFS_LOGFILE: '${oifs.model_dir}/build/oifs_test_log.txt' DR_HOOK_IGNORE_SIGNALS: '-1' + OMP_SCHEDULE: STATIC + OMP_STACKSIZE: 512M + albedo: + add_module_actions: + - "unload eccodes" + add_export_vars: + MKL_CBWR: AUTO,STRICT + DATA: "${general.work_dir}" + LD_LIBRARY_PATH[(3)]: '$LD_LIBRARY_PATH:${thisrun_work_dir}/lib/oifs/' + GRIB_SAMPLES_PATH: '${oifs.model_dir}/build/share/eccodes/ifs_samples/grib1_mlgrib2' + OIFS_LOGFILE: '${oifs.model_dir}/build/oifs_test_log.txt' + DR_HOOK_IGNORE_SIGNALS: '-1' + + aleph: + add_export_vars: + DATA: "${general.work_dir}" + LD_LIBRARY_PATH[(3)]: '$LD_LIBRARY_PATH:${thisrun_work_dir}/lib/oifs/' + PATH: '$PATH:${oifs.model_dir}/install/bin/' + GRIB_SAMPLES_PATH: '${oifs.model_dir}/build/share/eccodes/ifs_samples/grib1_mlgrib2' + OIFS_LOGFILE: '${oifs.model_dir}/build/oifs_test_log.txt' + DR_HOOK_IGNORE_SIGNALS: '-1' + OMP_SCHEDULE: STATIC + OMP_STACKSIZE: 512M '*': add_export_vars: MKL_CBWR: AUTO,STRICT @@ -35,5 +67,7 @@ computer: GRIB_SAMPLES_PATH: '${oifs.model_dir}/build/share/eccodes/ifs_samples/grib1_mlgrib2' OIFS_LOGFILE: '${oifs.model_dir}/build/oifs_test_log.txt' DR_HOOK_IGNORE_SIGNALS: '-1' + OMP_SCHEDULE: STATIC + OMP_STACKSIZE: 512M diff --git a/configs/components/xios/xios.env.yaml b/configs/components/xios/xios.env.yaml index d48f15c65..1f6ac4b74 100644 --- a/configs/components/xios/xios.env.yaml +++ b/configs/components/xios/xios.env.yaml @@ -1,6 +1,5 @@ computer: choose_computer.name: - ollie: + levante: add_module_actions: - - "unload netcdf" - - "load netcdf/4.4.1.1_intel_mpi" + - "load perl" diff --git a/configs/machines/levante.yaml b/configs/machines/levante.yaml index ad777760b..f566c569c 100644 --- a/configs/machines/levante.yaml +++ b/configs/machines/levante.yaml @@ -157,6 +157,8 @@ computer: MPI_LIB: "\"$(mpif90 -show |sed -e 's/^[^ ]*//' -e 's/-[I][^ ]*//g')\"" intel2022_impi2021: + launcher_flags: "-l --mpi=pmi2" # Fix for Intel MPI PMI2_Job_GetId errors + python_version: "python3" # ESM-Tools vars necessary for ``system_libs`` case mod_netcdf_c: netcdf-c/4.8.1-intel-oneapi-mpi-2021.5.0-intel-2021.5.0 mod_netcdf_f: netcdf-fortran/4.5.3-intel-oneapi-mpi-2021.5.0-intel-2021.5.0 @@ -177,6 +179,7 @@ computer: MPI_LIB: "\"$(mpiifort -show |sed -e 's/^[^ ]*//' -e 's/-[I][^ ]*//g')\"" intel2022_openmpi: + python_version: "python3" # ESM-Tools vars necessary for ``system_libs`` case mod_netcdf_c: netcdf-c/4.8.1-openmpi-4.1.2-intel-2021.5.0 mod_netcdf_f: netcdf-fortran/4.5.3-openmpi-4.1.2-intel-2021.5.0 @@ -343,16 +346,20 @@ computer: LD_RUN_PATH: $LD_LIBRARY_PATH AEC_ROOT: /sw/spack-levante/libaec-1.0.5-gij7yv aec_ROOT: /sw/spack-levante/libaec-1.0.5-gij7yv + UCXROOT: /sw/spack-levante/ucx-1.12.0-nlas56 HDF5ROOT: /sw/spack-levante/hdf5-1.12.1-tvymb5 HDF5_C_INCLUDE_DIRECTORIES: $HDF5ROOT/include HDF5_ROOT: $HDF5ROOT - MPIROOT: "\"$(mpif90 -show | perl -lne 'm{ -I(.*?)/include } and print $1')\"" NetCDF_C_ROOT: /sw/spack-levante/netcdf-c-4.8.1-2k3cmu NetCDF_Fortran_ROOT: /sw/spack-levante/netcdf-fortran-4.5.3-k6xq5g NETCDF_PATH: $NetCDF_C_ROOT NETCDF_ROOT: $NetCDF_Fortran_ROOT NETCDFROOT: $NetCDF_C_ROOT NETCDFFROOT: $NetCDF_Fortran_ROOT + NETCDF_Fortran_INCLUDE_DIRECTORIES: $NETCDFFROOT/include + NETCDF_C_INCLUDE_DIRECTORIES: $NETCDFROOT/include + NETCDF_CXX_INCLUDE_DIRECTORIES: $NETCDFROOT/include + NETCDF_CXX_LIBRARIES: $NETCDFROOT/lib TBBROOT: /sw/spack-levante/intel-oneapi-compilers-2022.0.1-an2cbq/tbb/2021.5.0 TBBMALLOC_DIR: $TBBROOT/lib/intel64/gcc4.8 MPI_HOME: /sw/spack-levante/openmpi-4.1.2-yfwe6t @@ -360,7 +367,7 @@ computer: mpi_ROOT: /sw/spack-levante/openmpi-4.1.2-yfwe6t MPI_DIR: /sw/spack-levante/openmpi-4.1.2-yfwe6t mpi_DIR: /sw/spack-levante/openmpi-4.1.2-yfwe6t - LD_LIBRARY_PATH: $MPI_HOME/lib:$NetCDF_C_ROOT/lib:$NetCDF_Fortran_ROOT/lib:$HDF5ROOT/lib:$AEC_ROOT/lib:$TBBMALLOC_DIR:$LD_LIBRARY_PATH + LD_LIBRARY_PATH: $MPI_HOME/lib:$NetCDF_C_ROOT/lib:$NetCDF_Fortran_ROOT/lib:$HDF5ROOT/lib:$AEC_ROOT/lib:$UCXROOT/lib:$TBBMALLOC_DIR:$LD_LIBRARY_PATH LD_LIBRARY_PATH[(0)]: /sw/spack-levante/intel-oneapi-mkl-2022.0.1-ttdktf/mkl/2022.0.1/lib/intel64:$LD_LIBRARY_PATH system_intel_impi_libs: #Copy paste from DE IFS env.sh @@ -379,13 +386,16 @@ computer: HDF5ROOT: /sw/spack-levante/hdf5-1.12.1-jmeuy3 HDF5_C_INCLUDE_DIRECTORIES: $HDF5ROOT/include HDF5_ROOT: $HDF5ROOT - MPIROOT: "\"$(mpif90 -show | perl -lne 'm{ -I(.*?)/include } and print $1')\"" NetCDF_C_ROOT: /sw/spack-levante/netcdf-c-4.8.1-7dq6g2 NetCDF_Fortran_ROOT: /sw/spack-levante/netcdf-fortran-4.5.3-r5r3ev NETCDF_PATH: $NetCDF_C_ROOT NETCDF_ROOT: $NetCDF_Fortran_ROOT NETCDFROOT: $NetCDF_C_ROOT NETCDFFROOT: $NetCDF_Fortran_ROOT + NETCDF_Fortran_INCLUDE_DIRECTORIES: $NETCDFFROOT/include + NETCDF_C_INCLUDE_DIRECTORIES: $NETCDFROOT/include + NETCDF_CXX_INCLUDE_DIRECTORIES: $NETCDFROOT/include + NETCDF_CXX_LIBRARIES: $NETCDFROOT/lib TBBROOT: /sw/spack-levante/intel-oneapi-compilers-2022.0.1-an2cbq/tbb/2021.5.0 TBBMALLOC_DIR: $TBBROOT/lib/intel64/gcc4.8 MPI_HOME: /sw/spack-levante/openmpi-4.1.2-yfwe6t diff --git a/configs/setups/oifsamip/oifsamip.yaml b/configs/setups/oifsamip/oifsamip.yaml index 736e4a40c..094ff468c 100644 --- a/configs/setups/oifsamip/oifsamip.yaml +++ b/configs/setups/oifsamip/oifsamip.yaml @@ -43,7 +43,6 @@ general: pool_dir: "/proj/awi/input" levante: pool_dir: "/work/ab0246/a270092/input" - # Load compute recipe from oifs.yaml compute_recipe: ${oifs.compute_recipe} @@ -75,6 +74,7 @@ oifs: setup_dir: ${general.model_dir} restart_firstlast: "last" restart_rate: 12 + with_xios: true mip: "cmip6" scenario: "historical" @@ -107,8 +107,6 @@ oifs: namelist_dir: ${general.esm_namelist_dir}/oifs/${version}/awicm3/v3.4 add_namelist_changes: fort.4: - NAMCT0: - LXIOS: TRUE NAMECECFG: ECE_CPL_NEMO_LIM: ".false." ECE_CPL_FESOM_FESIM: ".false." @@ -120,7 +118,7 @@ oifs: LECURR: ".false." #NALBEDOSCHEME: 3 - + cmip7_data_dir_nml: ${input_dir}/cmip7-data cmip6_data_dir_nml: ${input_dir}/cmip6-data cmip5_data_dir_nml: ${input_dir}/cmip5-data @@ -163,8 +161,65 @@ oifs: prepifs_dir: ${input_dir}/${res_level}/ tl_o3_data_dir: ${input_dir}/${version}/climate/ - ICMGG_INIT_name: "" - + ICMGG_INIT_name: "_AMIP" + + file_movements: + forcing: + all_directions: link + restart_in: + all_directions: link + restart_out: + all_directions: move + rcf_in: + all_directions: copy + waminfo_in: + all_directions: copy + outdata: + all_directions: move + + #====================================================================================== + # OUTPUT FILES + #====================================================================================== + + outdata_in_work: + ICMGG: ICMGG${oifs.input_expid}+* + ICMSH: ICMSH${oifs.input_expid}+* + ICMUA: ICMUA${oifs.input_expid}+* + MPP: MPP* + # XIOS output + oifsnc: ${input_expid}_*_${start_date!syear!smonth!sday}_*.nc + atm: atm_* + + outdata_sources: + # old grib based output + ICMGG: ${out_date_folder}/ICMGG${oifs.input_expid}+* + ICMSH: ${out_date_folder}/ICMSH${oifs.input_expid}+* + ICMUA: ${out_date_folder}/ICMUA${oifs.input_expid}+* + MPP: ${out_date_folder}/MPP* + # XIOS output + oifsnc: ${input_expid}_*_${start_date!syear!smonth!sday}_*.nc + atm: atm_* + + #====================================================================================== + # XIOS (Parallel output via XML IO server) + #====================================================================================== + + # Using XIOS? + choose_with_xios: + 1: + include_models: + - xios + add_namelist_changes: + fort.4: + NAMCT0: + LXIOS: ".true." + outdata_files: + oifsnc: oifsnc + atm: atm + 0: + outdata_files: + ICMGG: ICMGG + ICMSH: ICMSH amip: diff --git a/namelists/amip/namelist.amip b/namelists/amip/namelist.amip index 086da3036..8fda9d5bf 100755 --- a/namelists/amip/namelist.amip +++ b/namelists/amip/namelist.amip @@ -1,12 +1,14 @@ &namamip runlengthsec = 86400 - timestepsec = 84600 + timestepsec = 86400 startyear = 1850 startmonth = 1 startday = 1 fixyear = 1850 filelistsst = 'comb_input4MIPs_SSTsAndSeaIce_PAMIP_pdSST_pdSIC_gn.nc' filelistsic = 'comb_input4MIPs_SSTsAndSeaIce_PAMIP_pdSST_pdSIC_gn.nc' + sstvarname = 'sstbcs' + siconcvarname = 'siconcbcs' ldebug = .true. linterpolate = .true. / diff --git a/runscripts/oifsamip/oifsamip-cy43-ollie-TCO159L91.yaml b/runscripts/oifsamip/oifsamip-cy43-ollie-TCO159L91.yaml index d50156e0f..0cbbf466e 100644 --- a/runscripts/oifsamip/oifsamip-cy43-ollie-TCO159L91.yaml +++ b/runscripts/oifsamip/oifsamip-cy43-ollie-TCO159L91.yaml @@ -27,4 +27,4 @@ oifs: oasis3mct: lresume: true # Set to false to generate the rst files for first leg - time_step: 84600 + time_step: 86400 diff --git a/runscripts/oifsamip/oifsamip-cy43-ollie-TCO159L91_inital.yaml b/runscripts/oifsamip/oifsamip-cy43-ollie-TCO159L91_inital.yaml index 64a3a811c..d5a5b4d50 100644 --- a/runscripts/oifsamip/oifsamip-cy43-ollie-TCO159L91_inital.yaml +++ b/runscripts/oifsamip/oifsamip-cy43-ollie-TCO159L91_inital.yaml @@ -27,4 +27,4 @@ oifs: oasis3mct: lresume: false # Set to false to generate the rst files for first leg - time_step: 84600 + time_step: 86400 diff --git a/runscripts/oifsamip/oifsamip-cy48-levante-TCO95L91.yaml b/runscripts/oifsamip/oifsamip-cy48-levante-TCO95L91.yaml index f65c01b65..463f25233 100644 --- a/runscripts/oifsamip/oifsamip-cy48-levante-TCO95L91.yaml +++ b/runscripts/oifsamip/oifsamip-cy48-levante-TCO95L91.yaml @@ -42,7 +42,7 @@ oifs: oasis3mct: lresume: false # Set to false to generate the rst files for first leg - time_step: 84600 + time_step: 86400 xios: with_model: oifs diff --git a/runscripts/oifsamip/oifsamip-cy48-levante-TCO95L91_inital.yaml b/runscripts/oifsamip/oifsamip-cy48-levante-TCO95L91_inital.yaml index 5e0e24e78..15b1ea72d 100644 --- a/runscripts/oifsamip/oifsamip-cy48-levante-TCO95L91_inital.yaml +++ b/runscripts/oifsamip/oifsamip-cy48-levante-TCO95L91_inital.yaml @@ -28,7 +28,7 @@ oifs: oasis3mct: lresume: false # Set to false to generate the rst files for first leg - time_step: 84600 + time_step: 86400 xios: with_model: oifs