diff --git a/ifsrrtm/rrtm_gas_optical_depth.F90 b/ifsrrtm/rrtm_gas_optical_depth.F90 index 534b44f4..8772d0b4 100644 --- a/ifsrrtm/rrtm_gas_optical_depth.F90 +++ b/ifsrrtm/rrtm_gas_optical_depth.F90 @@ -213,6 +213,9 @@ SUBROUTINE RRTM_GAS_OPTICAL_DEPTH(KIDIA,KFDIA,KLEV,POD,PAVEL, PCOLDRY,PCOLBRD,PW !- Loop over g-channels. #if defined(OMPGPU) !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) +#if defined(__amdflang__) +!$OMP TILE SIZES(1,256,1) +#endif DO JLEV = 1, KLEV DO JLON = KIDIA, KFDIA DO JI = 1, JPGPT @@ -220,8 +223,15 @@ SUBROUTINE RRTM_GAS_OPTICAL_DEPTH(KIDIA,KFDIA,KLEV,POD,PAVEL, PCOLDRY,PCOLBRD,PW ENDDO ENDDO ENDDO +#if defined(__amdflang__) +!$OMP END TILE +#endif !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO #else +!$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) +#if defined(__amdflang__) +!$OMP INTERCHANGE PERMUTATION(1,3,2) +#endif !$ACC PARALLEL DEFAULT(NONE) ASYNC(1) !$ACC LOOP GANG VECTOR TILE(1,8,32) DO JLEV = 1, KLEV @@ -233,6 +243,7 @@ SUBROUTINE RRTM_GAS_OPTICAL_DEPTH(KIDIA,KFDIA,KLEV,POD,PAVEL, PCOLDRY,PCOLBRD,PW ENDDO ENDDO !$ACC END PARALLEL +!$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO #endif ! ----------------------------------------------------------------- diff --git a/radiation/radiation_cloud.F90 b/radiation/radiation_cloud.F90 index 9b1b2d58..3eec8c6f 100644 --- a/radiation/radiation_cloud.F90 +++ b/radiation/radiation_cloud.F90 @@ -819,6 +819,24 @@ subroutine crop_cloud_fraction(this, istartcol, iendcol, & nlev = size(this%fraction,2) +#if defined(OMPGPU) + !$OMP TARGET ENTER DATA MAP(ALLOC:sum_mixing_ratio) + do jlev = 1,nlev + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO + do jcol = istartcol,iendcol + sum_mixing_ratio(jcol) = 0.0_jprb + do jh = 1, this%ntype + sum_mixing_ratio(jcol) = sum_mixing_ratio(jcol) + this%mixing_ratio(jcol,jlev,jh) + end do + if (this%fraction(jcol,jlev) < cloud_fraction_threshold & + & .or. sum_mixing_ratio(jcol) < cloud_mixing_ratio_threshold) then + this%fraction(jcol,jlev) = 0.0_jprb + end if + end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO + end do + !$OMP TARGET EXIT DATA MAP(DELETE:sum_mixing_ratio) +#else !$ACC PARALLEL DEFAULT(PRESENT) CREATE(sum_mixing_ratio) ASYNC(1) !$ACC LOOP SEQ do jlev = 1,nlev @@ -842,6 +860,7 @@ subroutine crop_cloud_fraction(this, istartcol, iendcol, & end do end do !$ACC END PARALLEL +#endif if (lhook) call dr_hook('radiation_cloud:crop_cloud_fraction',1,hook_handle) diff --git a/radiation/radiation_flux.F90 b/radiation/radiation_flux.F90 index e3147739..a71b53fc 100644 --- a/radiation/radiation_flux.F90 +++ b/radiation/radiation_flux.F90 @@ -433,6 +433,10 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) real(jphook) :: hook_handle +#if defined(OMPGPU) + integer :: istart, iend, ig + real(jprb) :: s1, s2 +#endif if (lhook) call dr_hook('radiation_flux:calc_surface_spectral',0,hook_handle) #if defined(_OPENACC) || defined(OMPGPU) @@ -457,6 +461,12 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) & + this%sw_dn_direct_surf_band(:,jcol) end do else + +#if defined(OMPGPU) + istart = lbound(this%sw_dn_surf_band,1) + iend = ubound(this%sw_dn_surf_band,1) +#endif + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO !$ACC PARALLEL DEFAULT(PRESENT) NUM_GANGS(iendcol-istartcol+1) NUM_WORKERS(1) & !$ACC VECTOR_LENGTH(32*((config%n_g_sw-1)/32+1)) ASYNC(1) !$ACC LOOP GANG @@ -467,11 +477,20 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) call indexed_sum(this%sw_dn_diffuse_surf_g(:,jcol), & & config%i_band_from_reordered_g_sw, & & this%sw_dn_surf_band(:,jcol)) +#if defined(OMPGPU) + do ig = istart, iend + this%sw_dn_surf_band(ig,jcol) & + & = this%sw_dn_surf_band(ig,jcol) & + & + this%sw_dn_direct_surf_band(ig,jcol) + end do +#else this%sw_dn_surf_band(:,jcol) & & = this%sw_dn_surf_band(:,jcol) & & + this%sw_dn_direct_surf_band(:,jcol) +#endif end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO end if if (config%do_clear) then @@ -488,6 +507,11 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) & + this%sw_dn_direct_surf_clear_band(:,jcol) end do else +#if defined(OMPGPU) + istart = lbound(this%sw_dn_surf_clear_band,1) + iend = ubound(this%sw_dn_surf_clear_band,1) +#endif + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO !$ACC PARALLEL DEFAULT(PRESENT) NUM_GANGS(iendcol-istartcol+1) NUM_WORKERS(1) & !$ACC VECTOR_LENGTH(32*(config%n_g_sw-1)/32+1) ASYNC(1) !$ACC LOOP GANG @@ -498,11 +522,20 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) call indexed_sum(this%sw_dn_diffuse_surf_clear_g(:,jcol), & & config%i_band_from_reordered_g_sw, & & this%sw_dn_surf_clear_band(:,jcol)) +#if defined(OMPGPU) + do ig = istart, iend + this%sw_dn_surf_clear_band(ig,jcol) & + & = this%sw_dn_surf_clear_band(ig,jcol) & + & + this%sw_dn_direct_surf_clear_band(ig,jcol) + end do +#else this%sw_dn_surf_clear_band(:,jcol) & & = this%sw_dn_surf_clear_band(:,jcol) & & + this%sw_dn_direct_surf_clear_band(:,jcol) +#endif end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO end if end if @@ -511,6 +544,7 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) ! Fluxes in bands required for canopy radiative transfer if (config%do_sw .and. config%do_canopy_fluxes_sw) then if (config%use_canopy_full_spectrum_sw) then + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC PARALLEL DEFAULT(PRESENT) ASYNC(1) !$ACC LOOP GANG VECTOR COLLAPSE(2) do jcol = istartcol,iendcol @@ -520,6 +554,7 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) end do end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO else if (config%do_nearest_spectral_sw_albedo) then !$ACC DATA CREATE(i_albedo_from_reordered_g_sw) !$ACC PARALLEL DEFAULT(PRESENT) ASYNC(1) @@ -536,6 +571,7 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) & i_albedo_from_reordered_g_sw, & & this%sw_dn_diffuse_surf_canopy, istartcol, iendcol) else + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO !$ACC PARALLEL DEFAULT(PRESENT) NUM_GANGS(iendcol-istartcol+1) NUM_WORKERS(1) & !$ACC VECTOR_LENGTH(32*(config%n_g_sw-1)/32+1) ASYNC(1) !$ACC LOOP GANG @@ -548,6 +584,7 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) & this%sw_dn_diffuse_surf_canopy(:,jcol)) end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO end if !$ACC END DATA else @@ -555,6 +592,7 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) ! this%sw_dn_[direct_]surf_band to be defined, i.e. ! config%do_surface_sw_spectral_flux == .true. nalbedoband = size(config%sw_albedo_weights,1) + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC PARALLEL DEFAULT(PRESENT) ASYNC(1) & !$ACC PRESENT(this%sw_dn_diffuse_surf_canopy, this%sw_dn_direct_surf_canopy, & !$ACC config%sw_albedo_weights) @@ -565,9 +603,27 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) this%sw_dn_direct_surf_canopy (jalbedoband,jcol) = 0.0_jprb end do end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO + + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC LOOP GANG VECTOR COLLAPSE(2) do jcol = istartcol, iendcol do jalbedoband = 1,nalbedoband +#if defined(OMPGPU) + s1 = 0 + s2 = 0 + do jband = 1,config%n_bands_sw + if (config%sw_albedo_weights(jalbedoband,jband) /= 0.0_jprb) then + ! Initially, "diffuse" is actually "total" + s1 = s1 + config%sw_albedo_weights(jalbedoband,jband) & + & * this%sw_dn_surf_band(jband,jcol) + s2 = s2 + config%sw_albedo_weights(jalbedoband,jband) & + & * this%sw_dn_direct_surf_band(jband,jcol) + end if + end do + this%sw_dn_diffuse_surf_canopy(jalbedoband,jcol) = this%sw_dn_diffuse_surf_canopy(jalbedoband,jcol) + s1 + this%sw_dn_direct_surf_canopy(jalbedoband,jcol) = this%sw_dn_direct_surf_canopy(jalbedoband,jcol) + s2 +#else !$ACC LOOP SEQ do jband = 1,config%n_bands_sw if (config%sw_albedo_weights(jalbedoband,jband) /= 0.0_jprb) then @@ -582,8 +638,11 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) & * this%sw_dn_direct_surf_band(jband,jcol) end if end do +#endif end do end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC LOOP GANG VECTOR COLLAPSE(2) do jcol = istartcol,iendcol do jalbedoband = 1,nalbedoband @@ -594,12 +653,14 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) end do end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO end if end if ! do_canopy_fluxes_sw if (config%do_lw .and. config%do_canopy_fluxes_lw) then if (config%use_canopy_full_spectrum_lw) then + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC PARALLEL DEFAULT(PRESENT) ASYNC(1) !$ACC LOOP GANG VECTOR COLLAPSE(2) do jcol = istartcol,iendcol @@ -608,7 +669,32 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) end do end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO else if (config%do_nearest_spectral_lw_emiss) then +#if defined (OMPGPU) + !$OMP TARGET ENTER DATA MAP(alloc:i_emiss_from_reordered_g_lw) + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO + do jg = 1,config%n_g_lw + i_emiss_from_reordered_g_lw(jg) = config%i_emiss_from_band_lw(config%i_band_from_reordered_g_lw(jg)) + end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO + + if (use_indexed_sum_vec) then + call indexed_sum_vec(this%lw_dn_surf_g, & + & i_emiss_from_reordered_g_lw, & + & this%lw_dn_surf_canopy, istartcol, iendcol) + ! & config%i_emiss_from_band_lw(config%i_band_from_reordered_g_lw), & + else + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO + do jcol = istartcol,iendcol + call indexed_sum(this%lw_dn_surf_g(:,jcol), & + & i_emiss_from_reordered_g_lw, & + & this%lw_dn_surf_canopy(:,jcol)) + end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO + end if + !$OMP TARGET EXIT DATA MAP(delete:i_emiss_from_reordered_g_lw) +#else !$ACC DATA CREATE(i_emiss_from_reordered_g_lw) !$ACC PARALLEL DEFAULT(PRESENT) ASYNC(1) !$ACC LOOP GANG VECTOR @@ -635,7 +721,9 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) !$ACC END PARALLEL end if !$ACC END DATA +#endif else + !$OMP TARGET ENTER DATA MAP(ALLOC: lw_dn_surf_band) !$ACC DATA CREATE(lw_dn_surf_band) ASYNC(1) ! Compute fluxes in each longwave emissivity interval using ! weights; first sum over g points to get the values in bands @@ -644,6 +732,7 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) & config%i_band_from_reordered_g_lw, & & lw_dn_surf_band, istartcol, iendcol) else + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO !$ACC PARALLEL DEFAULT(PRESENT) NUM_GANGS(iendcol-istartcol+1) NUM_WORKERS(1) & !$ACC VECTOR_LENGTH(32*(config%n_g_lw-1)/32+1) ASYNC(1) !$ACC LOOP GANG @@ -653,8 +742,10 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) & lw_dn_surf_band(:,jcol)) end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO end if nalbedoband = size(config%lw_emiss_weights,1) + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC PARALLEL DEFAULT(PRESENT) & !$ACC PRESENT(this%lw_dn_surf_canopy, config%lw_emiss_weights) ASYNC(1) !$ACC LOOP GANG VECTOR COLLAPSE(2) @@ -663,22 +754,38 @@ subroutine calc_surface_spectral(this, config, istartcol, iendcol) this%lw_dn_surf_canopy(jalbedoband,jcol) = 0.0_jprb end do end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC LOOP GANG VECTOR COLLAPSE(2) do jcol = istartcol,iendcol do jalbedoband = 1,nalbedoband +#if defined(OMPGPU) + s1 = 0 + do jband = 1,config%n_bands_lw + if (config%lw_emiss_weights(jalbedoband,jband) /= 0.0_jprb) then + s1 = s1 + config%lw_emiss_weights(jalbedoband,jband) & + & * lw_dn_surf_band(jband,jcol) + end if + end do + this%lw_dn_surf_canopy(jalbedoband,jcol) & + & = this%lw_dn_surf_canopy(jalbedoband,jcol) + s1 +#else !$ACC LOOP SEQ do jband = 1,config%n_bands_lw - if (config%lw_emiss_weights(jalbedoband,jband) /= 0.0_jprb) then - this%lw_dn_surf_canopy(jalbedoband,jcol) & - & = this%lw_dn_surf_canopy(jalbedoband,jcol) & - & + config%lw_emiss_weights(jalbedoband,jband) & - & * lw_dn_surf_band(jband,jcol) - end if - end do + if (config%lw_emiss_weights(jalbedoband,jband) /= 0.0_jprb) then + this%lw_dn_surf_canopy(jalbedoband,jcol) & + & = this%lw_dn_surf_canopy(jalbedoband,jcol) & + & + config%lw_emiss_weights(jalbedoband,jband) & + & * lw_dn_surf_band(jband,jcol) + end if + end do +#endif end do end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO !$ACC END DATA + !$OMP TARGET EXIT DATA MAP(DELETE: lw_dn_surf_band) end if end if @@ -752,7 +859,6 @@ subroutine calc_toa_spectral(this, config, istartcol, iendcol) end if if (config%do_lw .and. config%do_toa_spectral_flux) then - if (use_indexed_sum_vec) then call indexed_sum_vec(this%lw_up_toa_g, & & config%i_band_from_reordered_g_lw, & @@ -891,7 +997,7 @@ end subroutine add_indexed_sum !--------------------------------------------------------------------- ! As "add_indexed_sum" but this version overwrites existing contents ! of "dest" - pure subroutine indexed_sum(source, ind, dest) + subroutine indexed_sum(source, ind, dest) real(jprb), intent(in) :: source(:) integer, intent(in) :: ind(:) @@ -901,18 +1007,32 @@ pure subroutine indexed_sum(source, ind, dest) !$ACC ROUTINE VECTOR - dest = 0.0 + istart = lbound(dest,1) + iend = ubound(dest,1) +#if defined(OMPGPU) + do jg = istart, iend + dest(jg) = 0.0 + end do +#else + dest = 0.0 +#endif + !! Nested parallelism does yet work with amdflang. Leave + !! these commented out for now. PJM 10/2/2025 istart = lbound(source,1) iend = ubound(source,1) + !!$OMP PARALLEL DO PRIVATE(ig) !$ACC LOOP VECTOR PRIVATE(ig) do jg = istart, iend ig = ind(jg) + !!$OMP ATOMIC UPDATE !$ACC ATOMIC UPDATE dest(ig) = dest(ig) + source(jg) !$ACC END ATOMIC + !!$END OMP ATOMIC end do + !!$OMP END PARALLEL DO end subroutine indexed_sum diff --git a/radiation/radiation_gas_constants.F90 b/radiation/radiation_gas_constants.F90 index e04bc70b..3b4a4acd 100644 --- a/radiation/radiation_gas_constants.F90 +++ b/radiation/radiation_gas_constants.F90 @@ -38,7 +38,7 @@ module radiation_gas_constants integer, parameter :: INO2 = 12 integer, parameter :: NMaxGases = 12 !$ACC DECLARE COPYIN(NMaxGases) - + !!$OMP DECLARE TARGET(NMaxGases) ! Molar masses (g mol-1) of dry air and the various gases above real(jprb), parameter :: AirMolarMass = 28.970_jprb real(jprb), parameter, dimension(0:NMaxGases) :: GasMolarMass = (/ & diff --git a/radiation/radiation_ifs_rrtm.F90 b/radiation/radiation_ifs_rrtm.F90 index e35c3a51..dd04d430 100644 --- a/radiation/radiation_ifs_rrtm.F90 +++ b/radiation/radiation_ifs_rrtm.F90 @@ -403,6 +403,36 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & call radiation_abort() endif #endif + !$OMP TARGET ENTER DATA MAP(ALLOC:incoming_sw_scale, & + !$OMP ZOD_LW, ZOD_SW, ZSSA_SW, ZINCSOL, & + !$OMP ZCOLMOL, ZCOLDRY, ZWBRODL, ZCOLBRD, ZWKL, & + !$OMP ZWX, & + !$OMP ZTAUAERL, & + !$OMP ZFAC00, ZFAC01, ZFAC10, ZFAC11, & + !$OMP ZFORFAC, ZFORFRAC, INDFOR, & + !$OMP INDMINOR, ZSCALEMINOR, ZSCALEMINORN2, ZMINORFRAC, & + !$OMP ZRAT_H2OCO2,ZRAT_H2OCO2_1, & + !$OMP ZRAT_H2OO3 ,ZRAT_H2OO3_1, & + !$OMP ZRAT_H2ON2O,ZRAT_H2ON2O_1, & + !$OMP ZRAT_H2OCH4,ZRAT_H2OCH4_1, & + !$OMP ZRAT_N2OCO2,ZRAT_N2OCO2_1, & + !$OMP ZRAT_O3CO2 ,ZRAT_O3CO2_1, & + !$OMP JP, JT, JT1, & + !$OMP ZONEMINUS_ARRAY, & + !$OMP ZCOLH2O, ZCOLCO2, ZCOLO3, ZCOLN2O, ZCOLCH4, ZCOLO2, & + !$OMP ZCO2MULT, & + !$OMP ILAYTROP, ILAYSWTCH, ILAYLOW, & + !$OMP ZPAVEL, ZTAVEL, ZPZ, ZTZ, & + !$OMP ZSELFFAC, ZSELFFRAC, & + !$OMP INDSELF, & + !$OMP ZPFRAC, & + !$OMP IREFLECT, & + !$OMP pressure_fl, temperature_fl) +#if defined(__amdflang) && defined(OMPGPU) + !$OMP TARGET DATA MAP(PRESENT, ALLOC: config, single_level, thermodynamics, gas, & + !$OMP od_lw, od_sw, ssa_sw, lw_albedo, planck_hl, lw_emission, & + !$OMP incoming_sw) +#endif !$ACC DATA CREATE(incoming_sw_scale, & !$ACC ZOD_LW, ZOD_SW, ZSSA_SW, ZINCSOL, & @@ -441,15 +471,18 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & ZPI = 2.0_jprb*ASIN(1.0_jprb) ZFLUXFAC = ZPI * 1.E+4 ZONEMINUS = 1.0_jprb - 1.0e-6_jprb + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO !$ACC PARALLEL DEFAULT(NONE) ASYNC(1) !$ACC LOOP GANG VECTOR do jcol= istartcol,iendcol ZONEMINUS_ARRAY(jcol) = ZONEMINUS end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO ! Are full level temperature and pressure available in thermodynmics? If not, interpolate. if (thermodynamics%rrtm_pass_temppres_fl) then + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC PARALLEL DEFAULT(NONE) ASYNC(1) !$ACC LOOP GANG VECTOR COLLAPSE(2) do jlev=1,nlev @@ -459,7 +492,9 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & end do end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO else + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC PARALLEL DEFAULT(NONE) ASYNC(1) !$ACC LOOP GANG VECTOR COLLAPSE(2) do jlev=1,nlev @@ -473,6 +508,7 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & end do end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO end if ! Check we have gas mixing ratios in the right units @@ -535,6 +571,7 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & call roctxEndRange #endif + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) !$ACC PARALLEL DEFAULT(NONE) ASYNC(1) !$ACC LOOP GANG VECTOR COLLAPSE(3) do jg = 1,JPBAND @@ -545,6 +582,7 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & end do end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO #ifdef HAVE_NVTX call nvtxStartRange("radiation::rrtm_gas_optical_depth") @@ -587,6 +625,7 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & ! lw_emission at this point is actually the planck function of ! the surface + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC PARALLEL DEFAULT(NONE) ASYNC(1) !$ACC LOOP GANG VECTOR COLLAPSE(2) do jcol = istartcol,iendcol @@ -595,6 +634,7 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & end do end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO else ! Longwave emission has already been computed if (config%use_canopy_full_spectrum_lw) then @@ -634,6 +674,7 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & end do else ! G points have not been reordered + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) !$ACC PARALLEL DEFAULT(NONE) ASYNC(1) !$ACC LOOP GANG COLLAPSE(3) do jcol = istartcol,iendcol @@ -645,6 +686,7 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & end do end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO end if end if @@ -676,6 +718,7 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & ! SRTM_GAS_OPTICAL_DEPTH will not initialize profiles when the sun ! is below the horizon, so we do it here + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) !$ACC PARALLEL DEFAULT(NONE) ASYNC(1) !$ACC LOOP GANG VECTOR COLLAPSE(3) do jg = 1, JPGPT_SW @@ -687,6 +730,8 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & end do end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC PARALLEL DEFAULT(NONE) ASYNC(1) !$ACC LOOP GANG VECTOR COLLAPSE(2) do jg = 1, JPGPT_SW @@ -695,6 +740,7 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & end do end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO #ifdef HAVE_NVTX call nvtxStartRange("radiation::srtm_gas_optical_depth") @@ -720,6 +766,7 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & ! Scale the incoming solar per band, if requested if (config%use_spectral_solar_scaling) then + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC PARALLEL DEFAULT(NONE) ASYNC(1) !$ACC LOOP GANG VECTOR COLLAPSE(2) do jg = 1,JPGPT_SW @@ -729,12 +776,14 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & end do end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO end if ! Scaling factor to ensure that the total solar irradiance is as ! requested. Note that if the sun is below the horizon then ! ZINCSOL will be zero. if (present(incoming_sw)) then + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO !$ACC PARALLEL DEFAULT(NONE) ASYNC(1) !$ACC LOOP GANG VECTOR do jcol = istartcol,iendcol @@ -747,6 +796,7 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & end if end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO end if if (config%i_solver_sw == ISolverSpartacus) then @@ -769,6 +819,29 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & end do else ! G points have not been reordered +#if defined(OMPGPU) + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) +#if defined(__amdflang__) + !$OMP TILE SIZES(1,256,1) +#endif + do jlev = 1,nlev + do jcol = istartcol,iendcol + do jg = 1,config%n_g_sw + ! Check for negative optical depth + od_sw (jg,nlev+1-jlev,jcol) = max(config%min_gas_od_sw, ZOD_SW(jcol,jlev,jg)) + ssa_sw(jg,nlev+1-jlev,jcol) = ZSSA_SW(jcol,jlev,jg) + end do + end do + end do +#if defined(__amdflang__) + !$OMP END TILE +#endif + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO +#else + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) +#if defined(__amdflang__) + !$OMP INTERCHANGE PERMUTATION(2,1,3) +#endif !$ACC PARALLEL DEFAULT(NONE) ASYNC(1) !$ACC LOOP GANG VECTOR TILE(4,1,32) do jcol = istartcol,iendcol @@ -781,8 +854,10 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & end do end do !$ACC END PARALLEL - + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO +#endif if (present(incoming_sw)) then + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC PARALLEL DEFAULT(NONE) ASYNC(1) !$ACC LOOP GANG VECTOR COLLAPSE(2) do jcol = istartcol,iendcol @@ -791,6 +866,7 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & end do end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO end if end if @@ -799,13 +875,41 @@ subroutine gas_optics(ncol,nlev,istartcol,iendcol, & !$ACC WAIT !$ACC END DATA + !$OMP TARGET EXIT DATA MAP(DELETE: incoming_sw_scale, & + !$OMP ZOD_LW, ZOD_SW, ZSSA_SW, ZINCSOL, & + !$OMP ZCOLMOL, ZCOLDRY, ZWBRODL, ZCOLBRD, ZWKL, & + !$OMP ZWX, & + !$OMP ZTAUAERL, & + !$OMP ZFAC00, ZFAC01, ZFAC10, ZFAC11, & + !$OMP ZFORFAC, ZFORFRAC, INDFOR, & + !$OMP INDMINOR, ZSCALEMINOR, ZSCALEMINORN2, ZMINORFRAC, & + !$OMP ZRAT_H2OCO2,ZRAT_H2OCO2_1, & + !$OMP ZRAT_H2OO3 ,ZRAT_H2OO3_1, & + !$OMP ZRAT_H2ON2O,ZRAT_H2ON2O_1, & + !$OMP ZRAT_H2OCH4,ZRAT_H2OCH4_1, & + !$OMP ZRAT_N2OCO2,ZRAT_N2OCO2_1, & + !$OMP ZRAT_O3CO2 ,ZRAT_O3CO2_1, & + !$OMP JP, JT, JT1, & + !$OMP ZONEMINUS_ARRAY, & + !$OMP ZCOLH2O, ZCOLCO2, ZCOLO3, ZCOLN2O, ZCOLCH4, ZCOLO2, & + !$OMP ZCO2MULT, & + !$OMP ILAYTROP, ILAYSWTCH, ILAYLOW, & + !$OMP ZPAVEL, ZTAVEL, ZPZ, ZTZ, & + !$OMP ZSELFFAC, ZSELFFRAC, & + !$OMP INDSELF, & + !$OMP ZPFRAC, & + !$OMP IREFLECT, & + !$OMP pressure_fl, temperature_fl) +#if defined(__amdflang) && defined(OMPGPU) + !$OMP END TARGET DATA +#endif + #ifdef HAVE_NVTX call nvtxEndRange #endif #ifdef HAVE_ROCTX call roctxEndRange #endif - if (lhook) call dr_hook('radiation_ifs_rrtm:gas_optics',1,hook_handle) end subroutine gas_optics @@ -839,6 +943,16 @@ subroutine planck_function_atmos(nlev,istartcol,iendcol, & real(jprb), dimension(config%n_g_lw,nlev+1,istartcol:iendcol), intent(out) :: & & planck_hl +#if defined(OMPGPU) + ! Planck function values per band + real(jprb), dimension(istartcol:iendcol, config%n_bands_lw,nlev+1) :: planck_store + + ! Look-up table variables for Planck function + real(jprb), dimension(istartcol:iendcol,nlev+1) :: frac + integer, dimension(istartcol:iendcol,nlev+1) :: ind + + real(jprb) :: planck_tmp(istartcol:iendcol,config%n_g_lw,nlev+1) +#else ! Planck function values per band real(jprb), dimension(istartcol:iendcol,nlev+1, config%n_bands_lw) :: planck_store @@ -846,6 +960,8 @@ subroutine planck_function_atmos(nlev,istartcol,iendcol, & real(jprb), dimension(istartcol:iendcol,nlev+1) :: frac integer, dimension(istartcol:iendcol,nlev+1) :: ind + real(jprb) :: planck_tmp(istartcol:iendcol,config%n_g_lw) +#endif ! Temperature (K) of a half-level real(jprb) :: temperature @@ -868,10 +984,84 @@ subroutine planck_function_atmos(nlev,istartcol,iendcol, & ! Work out interpolations: for each half level, the index of the ! lowest interpolation bound, and the fraction into interpolation ! interval +#if defined(OMPGPU) + !$OMP TARGET ENTER DATA MAP(ALLOC:planck_store, frac, ind, planck_tmp) + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) PRIVATE(temperature) + do jlev = 1,nlev+1 + do jcol = istartcol,iendcol + temperature = thermodynamics%temperature_hl(jcol,jlev+ilevoffset) + if (temperature < 339.0_jprb .and. temperature >= 160.0_jprb) then + ! Linear interpolation between -113 and 66 degC + ind(jcol,jlev) = int(temperature - 159.0_jprb) + frac(jcol,jlev) = temperature - int(temperature) + else if(temperature >= 339.0_jprb) then + ! Extrapolation above 66 degC + ind(jcol,jlev) = 180 + frac(jcol,jlev) = temperature - 339.0_jprb + else + ! Cap below -113 degC (to avoid possible negative Planck + ! function values) + ind(jcol,jlev) = 1 + frac(jcol,jlev) = 0.0_jprb + end if + end do + end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO + + ! Calculate Planck functions per band + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) PRIVATE(factor) + do jlev = 1,nlev+1 + do jband = 1,config%n_bands_lw + do jcol = istartcol,iendcol + factor = zfluxfac * delwave(jband) + planck_store(jcol,jband,jlev) = factor & + & * (totplnk(ind(jcol,jlev),jband) & + & + frac(jcol,jlev)*(totplnk(ind(jcol,jlev)+1,jband)-totplnk(ind(jcol,jlev),jband))) + end do + end do + end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO + + ! G points have not been reordered + ! Top-of-atmosphere half level - note that PFRAC is on model + ! levels not half levels + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) PRIVATE(iband) + do jg = 1,config%n_g_lw + do jcol = istartcol,iendcol + iband = config%i_band_from_g_lw(jg) + planck_hl(jg,1,jcol) = planck_store(jcol,iband,1) * PFRAC(jcol,jg,nlev) + end do + end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO + + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) PRIVATE(iband) + do jlev = 2,nlev+1 + do jg = 1,config%n_g_lw + do jcol = istartcol,iendcol + iband = config%i_band_from_g_lw(jg) + planck_tmp(jcol,jg,jlev) = planck_store(jcol,iband,jlev) * PFRAC(jcol,jg,nlev+2-jlev) + end do + end do + end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO + + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(3) + do jlev = 2,nlev+1 + do jcol = istartcol,iendcol + do jg = 1,config%n_g_lw + planck_hl(jg,jlev,jcol) = planck_tmp(jcol,jg,jlev) + end do + end do + end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO + !$OMP TARGET EXIT DATA MAP(DELETE:planck_store, frac, ind, planck_tmp) + +#else !$ACC PARALLEL DEFAULT(NONE) CREATE(planck_store, frac, ind) PRESENT(config, thermodynamics, PFRAC, & !$ACC planck_hl) ASYNC(1) !$ACC LOOP SEQ do jlev = 1,nlev+1 + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO PRIVATE(temperature) !$ACC LOOP GANG(STATIC:1) VECTOR PRIVATE(temperature) do jcol = istartcol,iendcol temperature = thermodynamics%temperature_hl(jcol,jlev+ilevoffset) @@ -958,8 +1148,8 @@ subroutine planck_function_atmos(nlev,istartcol,iendcol, & #endif !$ACC END PARALLEL - !$ACC WAIT +#endif if (lhook) call dr_hook('radiation_ifs_rrtm:planck_function_atmos',1,hook_handle) @@ -1011,8 +1201,10 @@ subroutine planck_function_surf(istartcol, iendcol, config, temperature, PFRAC, if (lhook) call dr_hook('radiation_ifs_rrtm:planck_function_surf',0,hook_handle) ZFLUXFAC = 2.0_jprb*ASIN(1.0_jprb) * 1.0e4_jprb + !$OMP TARGET ENTER DATA MAP(ALLOC:planck_store, frac, ind) ! Work out surface interpolations + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO PRIVATE(Tsurf) !$ACC PARALLEL DEFAULT(NONE) CREATE(planck_store, frac, ind) PRESENT(config, temperature, PFRAC, planck_surf) & !$ACC ASYNC(1) !$ACC LOOP GANG(STATIC:1) VECTOR PRIVATE(Tsurf) @@ -1033,7 +1225,21 @@ subroutine planck_function_surf(istartcol, iendcol, config, temperature, PFRAC, frac(jcol) = 0.0_jprb end if end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO +#if defined(OMPGPU) + ! Calculate Planck functions per band + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) PRIVATE(factor) + do jband = 1,config%n_bands_lw + do jcol = istartcol,iendcol + factor = zfluxfac * delwave(jband) + planck_store(jcol,jband) = factor & + & * (totplnk(ind(jcol),jband) & + & + frac(jcol)*(totplnk(ind(jcol)+1,jband)-totplnk(ind(jcol),jband))) + end do + end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO +#else ! Calculate Planck functions per band !$ACC LOOP SEQ PRIVATE(factor) do jband = 1,config%n_bands_lw @@ -1045,6 +1251,7 @@ subroutine planck_function_surf(istartcol, iendcol, config, temperature, PFRAC, & + frac(jcol)*(totplnk(ind(jcol)+1,jband)-totplnk(ind(jcol),jband))) end do end do +#endif if (config%i_solver_lw == ISolverSpartacus) then ! We need to rearrange the gas optics info in memory: reordering @@ -1053,6 +1260,17 @@ subroutine planck_function_surf(istartcol, iendcol, config, temperature, PFRAC, ! the spectrum that are optically thin for gases) and reorder ! in pressure since the the functions above treat pressure ! decreasing with increasing index. +#if defined(OMPGPU) + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) PRIVATE(iband, ig) + do jgreorder = 1,config%n_g_lw + do jcol = istartcol,iendcol + iband = config%i_band_from_reordered_g_lw(jgreorder) + ig = config%i_g_from_reordered_g_lw(jgreorder) + planck_surf(jgreorder,jcol) = planck_store(jcol,iband) * PFRAC(jcol,ig) + end do + end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO +#else !$ACC LOOP SEQ PRIVATE(iband, ig) do jgreorder = 1,config%n_g_lw iband = config%i_band_from_reordered_g_lw(jgreorder) @@ -1062,8 +1280,19 @@ subroutine planck_function_surf(istartcol, iendcol, config, temperature, PFRAC, planck_surf(jgreorder,jcol) = planck_store(jcol,iband) * PFRAC(jcol,ig) end do end do +#endif else ! G points have not been reordered +#if defined(OMPGPU) + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) PRIVATE(iband) + do jg = 1,config%n_g_lw + do jcol = istartcol,iendcol + iband = config%i_band_from_g_lw(jg) + planck_surf(jg,jcol) = planck_store(jcol,iband) * PFRAC(jcol,jg) + end do + end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO +#else !$ACC LOOP SEQ PRIVATE(iband) do jg = 1,config%n_g_lw iband = config%i_band_from_g_lw(jg) @@ -1072,10 +1301,12 @@ subroutine planck_function_surf(istartcol, iendcol, config, temperature, PFRAC, planck_surf(jg,jcol) = planck_store(jcol,iband) * PFRAC(jcol,jg) end do end do +#endif end if !$ACC END PARALLEL !$ACC WAIT + !$OMP TARGET EXIT DATA MAP(DELETE:planck_store, frac, ind) if (lhook) call dr_hook('radiation_ifs_rrtm:planck_function_surf',1,hook_handle) diff --git a/radiation/radiation_single_level.F90 b/radiation/radiation_single_level.F90 index 0b931898..4ef61f1e 100644 --- a/radiation/radiation_single_level.F90 +++ b/radiation/radiation_single_level.F90 @@ -270,6 +270,7 @@ subroutine get_albedos(this, istartcol, iendcol, config, & if (lhook) call dr_hook('radiation_single_level:get_albedos',0,hook_handle) !$ACC DATA CREATE(sw_albedo_band, lw_albedo_band) ASYNC(1) + !$OMP TARGET ENTER DATA MAP(ALLOC: sw_albedo_band, lw_albedo_band) if (config%do_sw) then ! Albedos/emissivities are stored in single_level in their own @@ -294,6 +295,7 @@ subroutine get_albedos(this, istartcol, iendcol, config, & call radiation_abort() end if + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC PARALLEL DEFAULT(PRESENT) ASYNC(1) !$ACC LOOP SEQ do jband = 1,config%n_bands_sw @@ -302,14 +304,33 @@ subroutine get_albedos(this, istartcol, iendcol, config, & sw_albedo_band(jcol,jband) = 0.0_jprb end do end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO +#if defined(OMPGPU) + !!$OMP TARGET DATA MAP(PRESENT, ALLOC: config, sw_albedo_band, this%sw_albedo, config%sw_albedo_weights) + do jalbedoband = 1,nalbedoband + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) + do jband = 1,config%n_bands_sw + do jcol = istartcol,iendcol + sw_albedo_band(jcol,jband) & + & = sw_albedo_band(jcol,jband) & + & + config%sw_albedo_weights(jalbedoband,jband) & + & * this%sw_albedo(jcol, jalbedoband) + end do + end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO + end do + !!$OMP END TARGET DATA +#else !$ACC LOOP SEQ do jband = 1,config%n_bands_sw !$ACC LOOP SEQ do jalbedoband = 1,nalbedoband #ifndef _OPENACC +#else if (config%sw_albedo_weights(jalbedoband,jband) /= 0.0_jprb) then #endif + !$ACC LOOP GANG(STATIC:1) VECTOR do jcol = istartcol,iendcol sw_albedo_band(jcol,jband) & @@ -318,15 +339,18 @@ subroutine get_albedos(this, istartcol, iendcol, config, & & * this%sw_albedo(jcol, jalbedoband) end do #ifndef _OPENACC +#else end if #endif end do end do +#endif !OMPGPU -#ifndef _OPENACC +#if !defined(_OPENACC) && !defined(OMPGPU) sw_albedo_diffuse = transpose(sw_albedo_band(istartcol:iendcol, & & config%i_band_from_reordered_g_sw)) #else + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC LOOP GANG(STATIC:1) VECTOR do jcol = istartcol,iendcol !$ACC LOOP SEQ @@ -335,8 +359,11 @@ subroutine get_albedos(this, istartcol, iendcol, config, & & config%i_band_from_reordered_g_sw(jg)) end do end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO #endif + if (allocated(this%sw_albedo_direct)) then + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC LOOP SEQ do jband = 1,config%n_bands_sw !$ACC LOOP GANG(STATIC:1) VECTOR @@ -344,12 +371,28 @@ subroutine get_albedos(this, istartcol, iendcol, config, & sw_albedo_band(jcol,jband) = 0.0_jprb end do end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO +#if defined(OMPGPU) + do jalbedoband = 1,nalbedoband + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) + do jband = 1,config%n_bands_sw + do jcol = istartcol,iendcol + sw_albedo_band(jcol,jband) & + & = sw_albedo_band(jcol,jband) & + & + config%sw_albedo_weights(jalbedoband,jband) & + & * this%sw_albedo_direct(jcol, jalbedoband) + end do + end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO + end do +#else !$ACC LOOP SEQ do jband = 1,config%n_bands_sw !$ACC LOOP SEQ do jalbedoband = 1,nalbedoband #ifndef _OPENACC +#else if (config%sw_albedo_weights(jalbedoband,jband) /= 0.0_jprb) then #endif !$ACC LOOP GANG(STATIC:1) VECTOR @@ -360,24 +403,30 @@ subroutine get_albedos(this, istartcol, iendcol, config, & & * this%sw_albedo_direct(jcol, jalbedoband) end do #ifndef _OPENACC +#else end if #endif end do end do -#ifndef _OPENACC +#endif !OMPGPU + +#if !defined(_OPENACC) && !defined(OMPGPU) sw_albedo_direct = transpose(sw_albedo_band(istartcol:iendcol, & & config%i_band_from_reordered_g_sw)) #else - !$ACC LOOP GANG(STATIC:1) VECTOR - do jcol = istartcol,iendcol - !$ACC LOOP SEQ - do jg = 1,config%n_g_sw - sw_albedo_direct(jg,jcol) = sw_albedo_band(jcol, & - & config%i_band_from_reordered_g_sw(jg)) + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) + !$ACC LOOP GANG(STATIC:1) VECTOR + do jcol = istartcol,iendcol + !$ACC LOOP SEQ + do jg = 1,config%n_g_sw + sw_albedo_direct(jg,jcol) = sw_albedo_band(jcol, & + & config%i_band_from_reordered_g_sw(jg)) + end do end do - end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO #endif else + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC LOOP GANG(STATIC:1) VECTOR do jcol = istartcol,iendcol !$ACC LOOP SEQ @@ -385,8 +434,9 @@ subroutine get_albedos(this, istartcol, iendcol, config, & sw_albedo_direct(jg,jcol) = sw_albedo_diffuse(jg,jcol) end do end do + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO end if - !$ACC END PARALLEL + !$ACC END PARALLEL else ! Albedos mapped less accurately to ecRad spectral bands if (maxval(config%i_albedo_from_band_sw) > size(this%sw_albedo,2)) then @@ -449,10 +499,11 @@ subroutine get_albedos(this, istartcol, iendcol, config, & & maxval(config%i_emiss_from_band_lw), ' bands' call radiation_abort() end if -#ifndef _OPENACC +#if !defined(_OPENACC) && !defined(OMPGPU) lw_albedo = 1.0_jprb - transpose(this%lw_emissivity(istartcol:iendcol, & & config%i_emiss_from_band_lw(config%i_band_from_reordered_g_lw))) #else + !$OMP TARGET TEAMS DISTRIBUTE PARALLEL DO COLLAPSE(2) !$ACC PARALLEL DEFAULT(PRESENT) ASYNC(1) !$ACC LOOP GANG VECTOR COLLAPSE(2) do jcol = istartcol,iendcol @@ -462,12 +513,14 @@ subroutine get_albedos(this, istartcol, iendcol, config, & end do end do !$ACC END PARALLEL + !$OMP END TARGET TEAMS DISTRIBUTE PARALLEL DO #endif end if end if !$ACC WAIT !$ACC END DATA + !$OMP TARGET EXIT DATA MAP(DELETE: sw_albedo_band, lw_albedo_band) if (lhook) call dr_hook('radiation_single_level:get_albedos',1,hook_handle)