diff --git a/src/fesom_module.F90 b/src/fesom_module.F90 index 3d17360a2..8a3f4f2d6 100755 --- a/src/fesom_module.F90 +++ b/src/fesom_module.F90 @@ -446,12 +446,13 @@ subroutine fesom_init(fesom_total_nsteps) !$ACC ENTER DATA CREATE (f%tracers%data, f%tracers%work) do tr_num=1, f%tracers%num_tracers !$ACC ENTER DATA CREATE (f%tracers%data(tr_num)%values, f%tracers%data(tr_num)%valuesAB) + !$ACC ENTER DATA CREATE (f%tracers%data(tr_num)%valuesold) !$ACC ENTER DATA CREATE (f%tracers%data(tr_num)%tra_adv_ph, f%tracers%data(tr_num)%tra_adv_pv) end do - !$ACC ENTER DATA CREATE (f%tracers%work%fct_ttf_min, f%tracers%work%fct_ttf_max, f%tracers%work%fct_plus, f%tracers%work%fct_minus) & - !$ACC CREATE (f%tracers%work%adv_flux_hor, f%tracers%work%adv_flux_ver, f%tracers%work%fct_LO) & - !$ACC CREATE (f%tracers%work%del_ttf_advvert, f%tracers%work%del_ttf_advhoriz, f%tracers%work%edge_up_dn_grad) & - !$ACC CREATE (f%tracers%work%del_ttf) + !$ACC ENTER DATA CREATE (f%tracers%work%fct_ttf_min, f%tracers%work%fct_ttf_max, f%tracers%work%fct_plus, f%tracers%work%fct_minus) + !$ACC ENTER DATA CREATE (f%tracers%work%adv_flux_hor, f%tracers%work%adv_flux_ver, f%tracers%work%fct_LO) + !$ACC ENTER DATA CREATE (f%tracers%work%del_ttf_advvert, f%tracers%work%del_ttf_advhoriz, f%tracers%work%edge_up_dn_grad) + !$ACC ENTER DATA CREATE (tr_xy, tr_z, relax2clim, Sclim, Tclim) end subroutine fesom_init @@ -721,11 +722,13 @@ subroutine fesom_finalize() !$ACC EXIT DATA DELETE (f%ice) do tr_num=1, f%tracers%num_tracers !$ACC EXIT DATA DELETE (f%tracers%data(tr_num)%values, f%tracers%data(tr_num)%valuesAB) + !$ACC EXIT DATA DELETE (f%tracers%data(tr_num)%valuesold) end do !$ACC EXIT DATA DELETE (f%tracers%work%fct_ttf_min, f%tracers%work%fct_ttf_max, f%tracers%work%fct_plus, f%tracers%work%fct_minus) !$ACC EXIT DATA DELETE (f%tracers%work%adv_flux_hor, f%tracers%work%adv_flux_ver, f%tracers%work%fct_LO) !$ACC EXIT DATA DELETE (f%tracers%work%del_ttf_advvert, f%tracers%work%del_ttf_advhoriz, f%tracers%work%edge_up_dn_grad) !$ACC EXIT DATA DELETE (f%tracers%work%del_ttf) + !$ACC EXIT DATA DELETE (tr_xy, tr_z, relax2clim, Sclim, Tclim) !$ACC EXIT DATA DELETE (f%tracers%data, f%tracers%work) !$ACC EXIT DATA DELETE (f%dynamics%w, f%dynamics%w_e, f%dynamics%uv) !$ACC EXIT DATA DELETE (f%dynamics, f%tracers) @@ -831,4 +834,4 @@ subroutine fesom_finalize() ! call clock_finish end subroutine fesom_finalize -end module fesom_module +end module fesom_module \ No newline at end of file diff --git a/src/oce_ale_tracer.F90 b/src/oce_ale_tracer.F90 index 9142fc3d8..b3ea5912b 100644 --- a/src/oce_ale_tracer.F90 +++ b/src/oce_ale_tracer.F90 @@ -224,8 +224,8 @@ subroutine solve_tracers_ale(ice, dynamics, tracers, partit, mesh) ! advect tracers if (flag_debug .and. mype==0) print *, achar(27)//'[37m'//' --> call adv_tracers_ale'//achar(27)//'[0m' !here update only those initialized in the init_tracers. (values, valuesAB, edge_up_dn_grad, ...) - !$ACC UPDATE DEVICE(tracers%data(tr_num)%values, tracers%data(tr_num)%valuesAB) & - !$ACC DEVICE(tracers%work%edge_up_dn_grad) !!& + !!$ACC UPDATE DEVICE(tracers%data(tr_num)%values, tracers%data(tr_num)%valuesAB, tracers%data(tr_num)%valuesold) + !$ACC UPDATE DEVICE(tracers%work%edge_up_dn_grad) !!& ! it will update del_ttf with contributions from horizontal and vertical advection parts (del_ttf_advhoriz and del_ttf_advvert) !$ACC wait(1) call do_oce_adv_tra(dt, UV, Wvel, Wvel_i, Wvel_e, tr_num, dynamics, tracers, partit, mesh) @@ -1666,4 +1666,4 @@ FUNCTION bc_surface(n, id, sval, nzmin, partit, mesh, sst, sss, aice) END SELECT RETURN -end function bc_surface +end function bc_surface \ No newline at end of file diff --git a/src/oce_tracer_mod.F90 b/src/oce_tracer_mod.F90 index 1a0fe05d4..2fb963fe0 100755 --- a/src/oce_tracer_mod.F90 +++ b/src/oce_tracer_mod.F90 @@ -25,8 +25,9 @@ SUBROUTINE init_tracers_AB(tr_num, tracers, partit, mesh) type(t_tracer), intent(inout), target :: tracers integer :: n,nz -#ifdef ENABLE_OPENACC -!$ACC parallel loop collapse(2) default(present) !!!async(1) +#ifndef ENABLE_OPENACC +#else +!$ACC PARALLEL LOOP COLLAPSE(2) DEFAULT(PRESENT) #endif do n=1, partit%myDim_nod2D+partit%eDim_nod2D do nz=1, mesh%nl-1 @@ -36,51 +37,89 @@ SUBROUTINE init_tracers_AB(tr_num, tracers, partit, mesh) tracers%work%del_ttf_advvert (nz, n) = 0.0_WP end do end do -#ifdef ENABLE_OPENACC -!$ACC end parallel loop +#ifndef ENABLE_OPENACC +#else +!$ACC END PARALLEL LOOP #endif + + ! AB interpolation + if (tracers%data(tr_num)%AB_order==2) then +#ifndef ENABLE_OPENACC !$OMP PARALLEL DO - do n=1, partit%myDim_nod2D+partit%eDim_nod2D - ! AB interpolation - if (tracers%data(tr_num)%AB_order==2) then +#else +!ACC PARALLEL LOOP DEFAULT(PRESENT) +#endif + do n=1, partit%myDim_nod2D+partit%eDim_nod2D tracers%data(tr_num)%valuesAB(:, n) =-(0.5_WP+epsilon)*tracers%data(tr_num)%valuesold(1, :, n)+(1.5_WP+epsilon)*tracers%data(tr_num)%values(:, n) - elseif (tracers%data(tr_num)%AB_order==3) then + end do +#ifndef ENABLE_OPENACC +!$OMP END PARALLEL DO +#else +!ACC END PARALLEL LOOP +#endif + elseif (tracers%data(tr_num)%AB_order==3) then +#ifndef ENABLE_OPENACC +!$OMP PARALLEL DO +#else +!ACC PARALLEL LOOP DEFAULT(PRESENT) +#endif + do n=1, partit%myDim_nod2D+partit%eDim_nod2D tracers%data(tr_num)%valuesAB(:, n) =5.0_WP*tracers%data(tr_num)%valuesold(2, :, n)-16.0_WP*tracers%data(tr_num)%valuesold(1, :, n)+23.0_WP*tracers%data(tr_num)%values(:, n) tracers%data(tr_num)%valuesAB(:, n) =tracers%data(tr_num)%valuesAB(:, n)/12.0_WP - else - write(*,*) - print *, achar(27)//'[33m' - write(*,*) '____________________________________________________________________' - write(*,*) ' ERROR: Adams-Bashfort tracer order must be 2 or 3, others are not' - write(*,*) ' supported!' - write(*,*) ' your tracers%data(tr_num)%AB_order =', tracers%data(tr_num)%AB_order - write(*,*) ' ' - write(*,*) ' --> check your namelist.tra !!!' - write(*,*) ' &tracer_general' - write(*,*) ' ...' - write(*,*) ' AB_order = 2' - write(*,*) ' / ' - write(*,*) '____________________________________________________________________' - print *, achar(27)//'[0m' - write(*,*) - call par_ex(partit%MPI_COMM_FESOM, partit%mype, 0) - end if - end do + end do +#ifndef ENABLE_OPENACC !$OMP END PARALLEL DO +#else +!ACC END PARALLEL LOOP +#endif + else + write(*,*) + print *, achar(27)//'[33m' + write(*,*) '____________________________________________________________________' + write(*,*) ' ERROR: Adams-Bashfort tracer order must be 2 or 3, others are not' + write(*,*) ' supported!' + write(*,*) ' your tracers%data(tr_num)%AB_order =', tracers%data(tr_num)%AB_order + write(*,*) ' ' + write(*,*) ' --> check your namelist.tra !!!' + write(*,*) ' &tracer_general' + write(*,*) ' ...' + write(*,*) ' AB_order = 2' + write(*,*) ' / ' + write(*,*) '____________________________________________________________________' + print *, achar(27)//'[0m' + write(*,*) + call par_ex(partit%MPI_COMM_FESOM, partit%mype, 0) + end if if (tracers%data(tr_num)%AB_order==2) then +#ifndef ENABLE_OPENACC !$OMP PARALLEL DO +#else +!ACC PARALLEL LOOP DEFAULT(PRESENT) +#endif do n=1, partit%myDim_nod2d+partit%eDim_nod2D tracers%data(tr_num)%valuesold(1, :, n)=tracers%data(tr_num)%values(:, n) end do +#ifndef ENABLE_OPENACC !$OMP END PARALLEL DO +#else +!ACC END PARALLEL LOOP +#endif elseif (tracers%data(tr_num)%AB_order==3) then +#ifndef ENABLE_OPENACC !$OMP PARALLEL DO +#else +!ACC PARALLEL LOOP DEFAULT(PRESENT) +#endif do n=1, partit%myDim_nod2d+partit%eDim_nod2D tracers%data(tr_num)%valuesold(2, :, n)=tracers%data(tr_num)%valuesold(1, :, n) tracers%data(tr_num)%valuesold(1, :, n)=tracers%data(tr_num)%values(:, n) end do +#ifndef ENABLE_OPENACC !$OMP END PARALLEL DO +#else +!ACC END PARALLEL LOOP +#endif end if if (flag_debug .and. partit%mype==0) print *, achar(27)//'[38m'//' --> call tracer_gradient_elements'//achar(27)//'[0m' @@ -127,7 +166,12 @@ SUBROUTINE tracer_gradient_elements(ttf, partit, mesh) #include "associate_mesh_def.h" #include "associate_part_ass.h" #include "associate_mesh_ass.h" + +#ifndef ENABLE_OPENACC !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, elnodes, nz, nzmin, nzmax) +#else +!ACC PARALLEL LOOP DEFAULT(PRESENT) PRIVATE(elnodes) +#endif DO elem=1, myDim_elem2D elnodes=elem2D_nodes(:,elem) nzmin = ulevels(elem) @@ -138,7 +182,11 @@ SUBROUTINE tracer_gradient_elements(ttf, partit, mesh) tr_xy(2,nz, elem)=sum(gradient_sca(4:6,elem)*ttf(nz,elnodes)) END DO END DO +#ifndef ENABLE_OPENACC !$OMP END PARALLEL DO +#else +!ACC END PARALLEL LOOP +#endif END SUBROUTINE tracer_gradient_elements ! ! @@ -163,7 +211,11 @@ SUBROUTINE tracer_gradient_z(ttf, partit, mesh) #include "associate_mesh_def.h" #include "associate_part_ass.h" #include "associate_mesh_ass.h" +#ifndef ENABLE_OPENACC !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(n, nz, nzmin, nzmax, dz) +#else +!ACC PARALLEL LOOP DEFAULT(PRESENT) +#endif DO n=1, myDim_nod2D+eDim_nod2D nzmax=nlevels_nod2D(n) nzmin=ulevels_nod2D(n) @@ -174,7 +226,11 @@ SUBROUTINE tracer_gradient_z(ttf, partit, mesh) tr_z(nzmin, n)=0.0_WP tr_z(nzmax, n)=0.0_WP END DO +#ifndef ENABLE_OPENACC !$OMP END PARALLEL DO +#else +!ACC END PARALLEL LOOP +#endif END SUBROUTINE tracer_gradient_z ! ! @@ -202,7 +258,12 @@ SUBROUTINE relax_to_clim(tr_num, tracers, partit, mesh) trarr=>tracers%data(tr_num)%values(:,:) if ((clim_relax>1.0e-8_WP).and.(tracers%data(tr_num)%ID==1)) then +#ifndef ENABLE_OPENACC !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(n, nzmin, nzmax) +#else +!ACC UPDATE DEVICE(relax2clim, Tclim) +!ACC PARALLEL LOOP DEFAULT(PRESENT) +#endif DO n=1, myDim_nod2D nzmin = ulevels_nod2D(n) nzmax = nlevels_nod2D(n) @@ -211,17 +272,30 @@ SUBROUTINE relax_to_clim(tr_num, tracers, partit, mesh) trarr(nzmin:nzmax-1,n)=trarr(nzmin:nzmax-1,n)+& relax2clim(n)*dt*(Tclim(nzmin:nzmax-1,n)-trarr(nzmin:nzmax-1,n)) END DO +#ifndef ENABLE_OPENACC !$OMP END PARALLEL DO +#else +!ACC END PARALLEL LOOP +#endif END if if ((clim_relax>1.0e-8_WP).and.(tracers%data(tr_num)%ID==2)) then +#ifndef ENABLE_OPENACC !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(n, nzmin, nzmax) +#else +!ACC UPDATE DEVICE(Sclim) +!ACC PARALLEL LOOP DEFAULT(PRESENT) +#endif DO n=1, myDim_nod2D nzmin = ulevels_nod2D(n) nzmax = nlevels_nod2D(n) trarr(nzmin:nzmax-1,n)=trarr(nzmin:nzmax-1,n)+& relax2clim(n)*dt*(Sclim(nzmin:nzmax-1,n)-trarr(nzmin:nzmax-1,n)) END DO +#ifndef ENABLE_OPENACC !$OMP END PARALLEL DO +#else +!ACC END PARALLEL LOOP +#endif END IF END SUBROUTINE relax_to_clim END MODULE o_tracers \ No newline at end of file