From db095fb33ad37cd1b2d633a388b9c25000d4b16c Mon Sep 17 00:00:00 2001 From: Paul Mullowney Date: Thu, 2 Oct 2025 21:08:00 -0500 Subject: [PATCH] Adding roctx markers and nvtx --- driver/ecrad_ifs_driver.F90 | 22 ++++++++ driver/ecrad_ifs_driver_blocked.F90 | 10 ++++ ifs/radiation_scheme.F90 | 48 ++++++++++++++++ radiation/radiation_ifs_rrtm.F90 | 83 +++++++++++++++++++++++++++- radiation/radiation_interface.F90 | 49 ++++++++++++++++ radiation/radiation_mcica_acc_lw.F90 | 13 +++++ radiation/radiation_mcica_acc_sw.F90 | 13 +++++ 7 files changed, 237 insertions(+), 1 deletion(-) diff --git a/driver/ecrad_ifs_driver.F90 b/driver/ecrad_ifs_driver.F90 index d042d674..2e8e7552 100644 --- a/driver/ecrad_ifs_driver.F90 +++ b/driver/ecrad_ifs_driver.F90 @@ -70,6 +70,10 @@ program ecrad_ifs_driver #ifdef HAVE_NVTX use nvtx #endif +#ifdef HAVE_ROCTX + use roctx_profiling, only: roctxstartrange, roctxendrange + use iso_c_binding, only: c_null_char +#endif implicit none @@ -381,6 +385,9 @@ program ecrad_ifs_driver #ifdef HAVE_NVTX call nvtxStartRange("ecrad_offload") #endif +#ifdef HAVE_ROCTX + call roctxStartRange("ecrad_offload"//c_null_char) +#endif #if defined(_OPENACC) !$ACC DATA COPYIN(yradiation, yradiation%rad_config, single_level, thermodynamics, & @@ -420,6 +427,9 @@ program ecrad_ifs_driver #ifdef HAVE_NVTX call nvtxEndRange #endif +#ifdef HAVE_ROCTX + call roctxEndRange +#endif ! -------------------------------------------------------- ! Section 4b: Call radiation_scheme per block @@ -442,6 +452,9 @@ program ecrad_ifs_driver #ifdef HAVE_NVTX call nvtxStartRange("ecrad_it") #endif +#ifdef HAVE_ROCTX + call roctxStartRange("ecrad_it"//c_null_char) +#endif ! Compute number of blocks to process nblock = (driver_config%iendcol - driver_config%istartcol & @@ -493,6 +506,12 @@ program ecrad_ifs_driver & ) end do !$OMP END PARALLEL DO +#ifdef HAVE_NVTX + call nvtxEndRange +#endif +#ifdef HAVE_ROCTX + call roctxEndRange +#endif end do #ifdef DEBUG @@ -537,6 +556,9 @@ program ecrad_ifs_driver #ifdef HAVE_NVTX call nvtxEndRange #endif +#ifdef HAVE_ROCTX + call roctxEndRange +#endif #ifndef NO_OPENMP if (driver_config%nrepeat > driver_config%nwarmup) then diff --git a/driver/ecrad_ifs_driver_blocked.F90 b/driver/ecrad_ifs_driver_blocked.F90 index 72d885ad..b547eb78 100644 --- a/driver/ecrad_ifs_driver_blocked.F90 +++ b/driver/ecrad_ifs_driver_blocked.F90 @@ -72,6 +72,10 @@ program ecrad_ifs_driver #ifdef HAVE_NVTX use nvtx #endif +#ifdef HAVE_ROCTX + use roctx_profiling, only: roctxstartrange, roctxendrange + use iso_c_binding, only: c_null_char +#endif implicit none @@ -400,6 +404,9 @@ program ecrad_ifs_driver #ifdef HAVE_NVTX call nvtxStartRange("ecrad_it") #endif +#ifdef HAVE_ROCTX + call roctxStartRange("ecrad_it"//c_null_char) +#endif #ifdef COPY_ASYNC #if defined(OMPGPU) @@ -568,6 +575,9 @@ program ecrad_ifs_driver #ifdef HAVE_NVTX call nvtxEndRange #endif +#ifdef HAVE_ROCTX + call roctxEndRange +#endif end do diff --git a/ifs/radiation_scheme.F90 b/ifs/radiation_scheme.F90 index ec6e8452..a042e8b8 100644 --- a/ifs/radiation_scheme.F90 +++ b/ifs/radiation_scheme.F90 @@ -93,6 +93,10 @@ SUBROUTINE RADIATION_SCHEME & #ifdef HAVE_NVTX USE NVTX #endif +#ifdef HAVE_ROCTX +USE ROCTX_PROFILING, ONLY: ROCTXSTARTRANGE, ROCTXENDRANGE +USE ISO_C_BINDING, ONLY: C_NULL_CHAR +#endif IMPLICIT NONE @@ -321,6 +325,9 @@ SUBROUTINE RADIATION_SCHEME & #ifdef HAVE_NVTX call nvtxStartRange("allocate") #endif +#ifdef HAVE_ROCTX +call roctxStartRange("allocate"//c_null_char) +#endif CALL SINGLE_LEVEL%ALLOCATE(KLON, YRERAD%NSW, YRERAD%NLWEMISS, & & USE_SW_ALBEDO_DIRECT=.TRUE.) CALL THERMODYNAMICS%ALLOCATE(KLON, KLEV, USE_H2O_SAT=.TRUE.) @@ -338,6 +345,9 @@ SUBROUTINE RADIATION_SCHEME & #ifdef HAVE_NVTX call nvtxEndRange #endif +#ifdef HAVE_ROCTX +call roctxEndRange +#endif #if defined(_OPENACC) || defined(OMPGPU) LLACC = .TRUE. @@ -395,6 +405,9 @@ SUBROUTINE RADIATION_SCHEME & #ifdef HAVE_NVTX call nvtxStartRange("thermodynamics setup") #endif +#ifdef HAVE_ROCTX +call roctxStartRange("thermodynamics setup"//c_null_char) +#endif ! Set thermodynamic profiles: simply copy over the half-level ! pressure and temperature !$ACC PARALLEL DEFAULT(NONE) ASYNC(1) IF(LLACC) & @@ -460,6 +473,10 @@ SUBROUTINE RADIATION_SCHEME & call nvtxEndRange call nvtxStartRange("single level setup") #endif +#ifdef HAVE_ROCTX +call roctxEndRange +call roctxStartRange("single level setup"//c_null_char) +#endif ! Set single-level fileds SINGLE_LEVEL%SOLAR_IRRADIANCE = PSOLAR_IRRADIANCE @@ -580,6 +597,10 @@ SUBROUTINE RADIATION_SCHEME & call nvtxEndRange call nvtxStartRange("cloud setup") #endif +#ifdef HAVE_ROCTX +call roctxEndRange +call roctxStartRange("cloud setup"//c_null_char) +#endif ! Set cloud fields !$ACC PARALLEL DEFAULT(PRESENT) ASYNC(1) IF(LLACC) @@ -719,6 +740,10 @@ SUBROUTINE RADIATION_SCHEME & call nvtxEndRange call nvtxStartRange("aerosol setup") #endif +#ifdef HAVE_ROCTX +call roctxEndRange +call roctxStartRange("aerosol setup"//c_null_char) +#endif ! Copy over aerosol mass mixing ratio IF (.NOT. LL_USE_TEGEN_AEROSOLS) THEN @@ -804,6 +829,10 @@ SUBROUTINE RADIATION_SCHEME & call nvtxEndRange call nvtxStartRange("gas setup") #endif +#ifdef HAVE_ROCTX +call roctxEndRange +call roctxStartRange("gas setup"//c_null_char) +#endif ! Insert gas mixing ratios (mol mol-1 or kg kg-1) CALL GAS%PUT(GAS, IH2O, IMASSMIXINGRATIO, PQ, LACC=LLACC) @@ -824,11 +853,17 @@ SUBROUTINE RADIATION_SCHEME & #ifdef HAVE_NVTX call nvtxEndRange #endif +#ifdef HAVE_ROCTX +call roctxEndRange +#endif ! Call radiation scheme #ifdef HAVE_NVTX call nvtxStartRange("radiation") #endif +#ifdef HAVE_ROCTX +call roctxStartRange("radiation"//c_null_char) +#endif CALL RADIATION(KLON, KLEV, KIDIA, KFDIA, RAD_CONFIG,& & SINGLE_LEVEL, THERMODYNAMICS, GAS, YLCLOUD, AEROSOL, FLUX) @@ -836,6 +871,9 @@ SUBROUTINE RADIATION_SCHEME & #ifdef HAVE_NVTX call nvtxEndRange #endif +#ifdef HAVE_ROCTX +call roctxEndRange +#endif ! Check fluxes are within physical bounds IF (YRERAD%NDUMPBADINPUTS /= 0 & @@ -919,6 +957,9 @@ SUBROUTINE RADIATION_SCHEME & #ifdef HAVE_NVTX call nvtxStartRange("compute fluxes") #endif +#ifdef HAVE_ROCTX +call roctxStartRange("compute fluxes"//c_null_char) +#endif ! Compute required output fluxes ! First the net fluxes !$ACC PARALLEL DEFAULT(PRESENT) ASYNC(1) IF(LLACC) @@ -1020,6 +1061,10 @@ SUBROUTINE RADIATION_SCHEME & call nvtxEndRange call nvtxStartRange("cleanup") #endif +#ifdef HAVE_ROCTX +call roctxEndRange +call roctxStartRange("cleanup"//c_null_char) +#endif !$ACC END DATA !$OMP TARGET EXIT DATA MAP(DELETE:ZRE_LIQUID_UM, ZRE_ICE_UM, ZDECORR_LEN_KM,ZLAYER_MASS) @@ -1047,6 +1092,9 @@ SUBROUTINE RADIATION_SCHEME & #ifdef HAVE_NVTX call nvtxEndRange #endif +#ifdef HAVE_ROCTX +call roctxEndRange +#endif END ASSOCIATE diff --git a/radiation/radiation_ifs_rrtm.F90 b/radiation/radiation_ifs_rrtm.F90 index b55ecaac..e35c3a51 100644 --- a/radiation/radiation_ifs_rrtm.F90 +++ b/radiation/radiation_ifs_rrtm.F90 @@ -233,6 +233,13 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & use radiation_thermodynamics, only : thermodynamics_type use radiation_single_level, only : single_level_type use radiation_gas +#ifdef HAVE_NVTX + use nvtx +#endif +#ifdef HAVE_ROCTX + use roctx_profiling, only: roctxstartrange, roctxendrange + use iso_c_binding, only: c_null_char +#endif integer, intent(in) :: ncol ! number of columns integer, intent(in) :: nlev ! number of model levels @@ -374,6 +381,13 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & #include "srtm_setcoef.intfb.h" #include "srtm_gas_optical_depth.intfb.h" +#ifdef HAVE_NVTX + call nvtxStartRange("radiation::ifs_rrtm_gas_optics") +#endif +#ifdef HAVE_ROCTX + call roctxStartRange("radiation::ifs_rrtm_gas_optics"//c_null_char) +#endif + if (lhook) call dr_hook('radiation_ifs_rrtm:gas_optics',0,hook_handle) do_sw = (config%do_sw .and. config%i_gas_model_sw == IGasModelIFSRRTMG) @@ -467,6 +481,12 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & ! Warning: O2 is hard-coded within the following function so the ! user-provided concentrations of this gas are ignored for both ! the longwave and shortwave +#ifdef HAVE_NVTX + call nvtxStartRange("radiation::rrtm_prepare_gases") +#endif +#ifdef HAVE_ROCTX + call roctxStartRange("radiation::rrtm_prepare_gases"//c_null_char) +#endif CALL RRTM_PREPARE_GASES & & ( istartcol, iendcol, ncol, nlev, & & thermodynamics%pressure_hl(:,istartlev:iendlev+1), & @@ -485,9 +505,20 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & & gas%mixing_ratio(:,istartlev:iendlev,IO3), & & ZCOLDRY, ZWBRODL,ZWKL, ZWX, & & ZPAVEL , ZTAVEL , ZPZ , ZTZ, IREFLECT) +#ifdef HAVE_NVTX + call nvtxEndRange +#endif +#ifdef HAVE_ROCTX + call roctxEndRange +#endif if (do_lw) then - +#ifdef HAVE_NVTX + call nvtxStartRange("radiation::rrtm_setcoef_140GP") +#endif +#ifdef HAVE_ROCTX + call roctxStartRange("radiation::rrtm_setcoef_140GP"//c_null_char) +#endif CALL RRTM_SETCOEF_140GP & &( istartcol, iendcol, nlev , ZCOLDRY , ZWBRODL , ZWKL , & & ZFAC00 , ZFAC01 , ZFAC10 , ZFAC11 , ZFORFAC,ZFORFRAC,INDFOR, JP, JT, JT1 , & @@ -497,6 +528,12 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & & ZRAT_H2OCO2, ZRAT_H2OCO2_1, ZRAT_H2OO3, ZRAT_H2OO3_1, & & ZRAT_H2ON2O, ZRAT_H2ON2O_1, ZRAT_H2OCH4, ZRAT_H2OCH4_1, & & ZRAT_N2OCO2, ZRAT_N2OCO2_1, ZRAT_O3CO2, ZRAT_O3CO2_1) +#ifdef HAVE_NVTX + call nvtxEndRange +#endif +#ifdef HAVE_ROCTX + call roctxEndRange +#endif !$ACC PARALLEL DEFAULT(NONE) ASYNC(1) !$ACC LOOP GANG VECTOR COLLAPSE(3) @@ -509,6 +546,13 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & end do !$ACC END PARALLEL +#ifdef HAVE_NVTX + call nvtxStartRange("radiation::rrtm_gas_optical_depth") +#endif +#ifdef HAVE_ROCTX + call roctxStartRange("radiation::rrtm_gas_optical_depth"//c_null_char) +#endif + CALL RRTM_GAS_OPTICAL_DEPTH & &( istartcol, iendcol, nlev, ZOD_LW, ZPAVEL, ZCOLDRY, ZCOLBRD, ZWX ,& & ZTAUAERL, ZFAC00 , ZFAC01, ZFAC10 , ZFAC11 , ZFORFAC,ZFORFRAC,INDFOR, & @@ -519,6 +563,12 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & & ZRAT_H2OCO2, ZRAT_H2OCO2_1, ZRAT_H2OO3, ZRAT_H2OO3_1, & & ZRAT_H2ON2O, ZRAT_H2ON2O_1, ZRAT_H2OCH4, ZRAT_H2OCH4_1, & & ZRAT_N2OCO2, ZRAT_N2OCO2_1, ZRAT_O3CO2, ZRAT_O3CO2_1) +#ifdef HAVE_NVTX + call nvtxEndRange +#endif +#ifdef HAVE_ROCTX + call roctxEndRange +#endif if (present(lw_albedo)) then @@ -601,6 +651,12 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & if (do_sw) then +#ifdef HAVE_NVTX + call nvtxStartRange("radiation::srtm_sefcoeff") +#endif +#ifdef HAVE_ROCTX + call roctxStartRange("radiation::srtm_sefcoeff"//c_null_char) +#endif CALL SRTM_SETCOEF & & ( istartcol, iendcol, nlev,& & ZPAVEL , ZTAVEL,& @@ -611,6 +667,12 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & & ZFAC00 , ZFAC01 , ZFAC10 , ZFAC11,& & JP , JT , JT1 , single_level%cos_sza(istartcol:iendcol) & & ) +#ifdef HAVE_NVTX + call nvtxEndRange +#endif +#ifdef HAVE_ROCTX + call roctxEndRange +#endif ! SRTM_GAS_OPTICAL_DEPTH will not initialize profiles when the sun ! is below the horizon, so we do it here @@ -634,6 +696,13 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & end do !$ACC END PARALLEL +#ifdef HAVE_NVTX + call nvtxStartRange("radiation::srtm_gas_optical_depth") +#endif +#ifdef HAVE_ROCTX + call roctxStartRange("radiation::srtm_gas_optical_depth"//c_null_char) +#endif + CALL SRTM_GAS_OPTICAL_DEPTH & &( istartcol, iendcol , nlev , ZONEMINUS_ARRAY,& & single_level%cos_sza(istartcol:iendcol), ILAYTROP,& @@ -642,6 +711,12 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & & ZFAC00 , ZFAC01 , ZFAC10 , ZFAC11 ,& & JP , JT , JT1 ,& & ZOD_SW , ZSSA_SW , ZINCSOL ) +#ifdef HAVE_NVTX + call nvtxEndRange +#endif +#ifdef HAVE_ROCTX + call roctxEndRange +#endif ! Scale the incoming solar per band, if requested if (config%use_spectral_solar_scaling) then @@ -724,6 +799,12 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & !$ACC WAIT !$ACC END DATA +#ifdef HAVE_NVTX + call nvtxEndRange +#endif +#ifdef HAVE_ROCTX + call roctxEndRange +#endif if (lhook) call dr_hook('radiation_ifs_rrtm:gas_optics',1,hook_handle) diff --git a/radiation/radiation_interface.F90 b/radiation/radiation_interface.F90 index 97cbf5ec..cc757b02 100644 --- a/radiation/radiation_interface.F90 +++ b/radiation/radiation_interface.F90 @@ -242,6 +242,14 @@ subroutine radiation(ncol, nlev, istartcol, iendcol, config, & use radiation_general_cloud_optics, only : general_cloud_optics use radiation_aerosol_optics, only : add_aerosol_optics +#ifdef HAVE_NVTX + use nvtx +#endif +#ifdef HAVE_ROCTX + use roctx_profiling, only: roctxstartrange, roctxendrange + use iso_c_binding, only: c_null_char +#endif + ! Inputs integer, intent(in) :: ncol ! number of columns integer, intent(in) :: nlev ! number of model levels @@ -331,6 +339,12 @@ subroutine radiation(ncol, nlev, istartcol, iendcol, config, & ! Input arrays arranged in order of increasing pressure / ! decreasing height: progress normally +#ifdef HAVE_NVTX + call nvtxStartRange("radiation::get_albedos") +#endif +#ifdef HAVE_ROCTX + call roctxStartRange("radiation::get_albedos"//c_null_char) +#endif ! Extract surface albedos at each gridpoint call single_level%get_albedos(single_level, & @@ -338,6 +352,15 @@ subroutine radiation(ncol, nlev, istartcol, iendcol, config, & & sw_albedo_direct, sw_albedo_diffuse, & & lw_albedo) +#ifdef HAVE_NVTX + !$ACC WAIT(1) + call nvtxEndRange + call nvtxStartRange("radiation::gas_optics") +#endif +#ifdef HAVE_ROCTX + call roctxEndRange + call roctxStartRange("radiation::gas_optics"//c_null_char) +#endif ! Compute gas absorption optical depth in shortwave and ! longwave, shortwave single scattering albedo (i.e. fraction of ! extinction due to Rayleigh scattering), Planck functions and @@ -366,6 +389,15 @@ subroutine radiation(ncol, nlev, istartcol, iendcol, config, & & incoming_sw=incoming_sw) end if end if +#ifdef HAVE_NVTX + !$ACC WAIT(1) + call nvtxEndRange + call nvtxStartRange("radiation::cloud_optics") +#endif +#ifdef HAVE_ROCTX + call roctxEndRange + call roctxStartRange("radiation::cloud_optics"//c_null_char) +#endif if (config%do_clouds) then ! Crop the cloud fraction to remove clouds that have too small @@ -396,6 +428,15 @@ subroutine radiation(ncol, nlev, istartcol, iendcol, config, & end if end if ! do_clouds +#ifdef HAVE_NVTX + !$ACC WAIT(1) + call nvtxEndRange + call nvtxStartRange("radiation::aerosol_optics") +#endif +#ifdef HAVE_ROCTX + call roctxEndRange + call roctxStartRange("radiation::aerosol_optics"//c_null_char) +#endif if (config%use_aerosols) then if (config%i_gas_model_sw == IGasModelMonochromatic) then ! call add_aerosol_optics_mono(nlev,istartcol,iendcol, & @@ -427,6 +468,14 @@ subroutine radiation(ncol, nlev, istartcol, iendcol, config, & !$ACC END PARALLEL end if +#ifdef HAVE_NVTX + !$ACC WAIT(1) + call nvtxEndRange +#endif +#ifdef HAVE_ROCTX + call roctxEndRange +#endif + ! For diagnostic purposes, save these intermediate variables to ! a NetCDF file if (config%do_save_radiative_properties) then diff --git a/radiation/radiation_mcica_acc_lw.F90 b/radiation/radiation_mcica_acc_lw.F90 index 8367cdee..1c25636e 100644 --- a/radiation/radiation_mcica_acc_lw.F90 +++ b/radiation/radiation_mcica_acc_lw.F90 @@ -58,6 +58,10 @@ subroutine solver_mcica_acc_lw(nlev,istartcol,iendcol, & use radiation_cloud_generator_acc, only: cloud_generator_acc use radiation_cloud_cover, only : beta2alpha, MaxCloudFrac +#ifdef HAVE_NVTX + use nvtx +#endif + implicit none ! Inputs @@ -165,6 +169,10 @@ subroutine solver_mcica_acc_lw(nlev,istartcol,iendcol, & real(jphook) :: hook_handle +#ifdef HAVE_NVTX + call nvtxStartRange("radiation::radiation::mcica_acc_lw") +#endif + if (lhook) call dr_hook('radiation_mcica_acc_lw:solver_mcica_acc_lw',0,hook_handle) if (.not. config%do_clear) then @@ -575,6 +583,11 @@ subroutine solver_mcica_acc_lw(nlev,istartcol,iendcol, & if (lhook) call dr_hook('radiation_mcica_acc_lw:solver_mcica_acc_lw',1,hook_handle) +#ifdef HAVE_NVTX + !$ACC WAIT(1) + call nvtxEndRange +#endif + end subroutine solver_mcica_acc_lw end module radiation_mcica_acc_lw diff --git a/radiation/radiation_mcica_acc_sw.F90 b/radiation/radiation_mcica_acc_sw.F90 index c50db4ce..5c2da07e 100644 --- a/radiation/radiation_mcica_acc_sw.F90 +++ b/radiation/radiation_mcica_acc_sw.F90 @@ -59,6 +59,10 @@ subroutine solver_mcica_acc_sw(nlev,istartcol,iendcol, & use radiation_cloud_generator_acc, only: cloud_generator_acc use radiation_cloud_cover, only : beta2alpha, MaxCloudFrac +#ifdef HAVE_NVTX + use nvtx +#endif + implicit none ! Inputs @@ -160,6 +164,10 @@ subroutine solver_mcica_acc_sw(nlev,istartcol,iendcol, & real(jphook) :: hook_handle +#ifdef HAVE_NVTX + call nvtxStartRange("radiation::radiation::mcica_acc_sw") +#endif + if (lhook) call dr_hook('radiation_mcica_acc_sw:solver_mcica_acc_sw',0,hook_handle) if (.not. config%do_clear) then @@ -576,6 +584,11 @@ subroutine solver_mcica_acc_sw(nlev,istartcol,iendcol, & if (lhook) call dr_hook('radiation_mcica_acc_sw:solver_mcica_acc_sw',1,hook_handle) +#ifdef HAVE_NVTX + !$ACC WAIT(1) + call nvtxEndRange +#endif + end subroutine solver_mcica_acc_sw end module radiation_mcica_acc_sw