Skip to content

Phase 1 OpenACC #732

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
suvarchal merged 4 commits into from
Jul 1, 2025
Merged

Phase 1 OpenACC #732

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6004818
Removed redundant creation of del_ttf in GPU memory and organised the…
Jun 30, 2025
a094723
Added Phase1 changes, compiling and running.
Jun 30, 2025
ade0193
Updated changes so we can solve the openmp CI test
Jul 1, 2025
fbea8c5
Merge branch 'main' into openacc_Phase_1
suvarchal Jul 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions src/fesom_module.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -446,12 +446,13 @@ subroutine fesom_init(fesom_total_nsteps)
!$ACC ENTER DATA CREATE (f%tracers%data, f%tracers%work)
do tr_num=1, f%tracers%num_tracers
!$ACC ENTER DATA CREATE (f%tracers%data(tr_num)%values, f%tracers%data(tr_num)%valuesAB)
!$ACC ENTER DATA CREATE (f%tracers%data(tr_num)%valuesold)
!$ACC ENTER DATA CREATE (f%tracers%data(tr_num)%tra_adv_ph, f%tracers%data(tr_num)%tra_adv_pv)
end do
!$ACC ENTER DATA CREATE (f%tracers%work%fct_ttf_min, f%tracers%work%fct_ttf_max, f%tracers%work%fct_plus, f%tracers%work%fct_minus) &
!$ACC CREATE (f%tracers%work%adv_flux_hor, f%tracers%work%adv_flux_ver, f%tracers%work%fct_LO) &
!$ACC CREATE (f%tracers%work%del_ttf_advvert, f%tracers%work%del_ttf_advhoriz, f%tracers%work%edge_up_dn_grad) &
!$ACC CREATE (f%tracers%work%del_ttf)
!$ACC ENTER DATA CREATE (f%tracers%work%fct_ttf_min, f%tracers%work%fct_ttf_max, f%tracers%work%fct_plus, f%tracers%work%fct_minus)
!$ACC ENTER DATA CREATE (f%tracers%work%adv_flux_hor, f%tracers%work%adv_flux_ver, f%tracers%work%fct_LO)
!$ACC ENTER DATA CREATE (f%tracers%work%del_ttf_advvert, f%tracers%work%del_ttf_advhoriz, f%tracers%work%edge_up_dn_grad)
!$ACC ENTER DATA CREATE (tr_xy, tr_z, relax2clim, Sclim, Tclim)
end subroutine fesom_init


Expand Down Expand Up @@ -721,11 +722,13 @@ subroutine fesom_finalize()
!$ACC EXIT DATA DELETE (f%ice)
do tr_num=1, f%tracers%num_tracers
!$ACC EXIT DATA DELETE (f%tracers%data(tr_num)%values, f%tracers%data(tr_num)%valuesAB)
!$ACC EXIT DATA DELETE (f%tracers%data(tr_num)%valuesold)
end do
!$ACC EXIT DATA DELETE (f%tracers%work%fct_ttf_min, f%tracers%work%fct_ttf_max, f%tracers%work%fct_plus, f%tracers%work%fct_minus)
!$ACC EXIT DATA DELETE (f%tracers%work%adv_flux_hor, f%tracers%work%adv_flux_ver, f%tracers%work%fct_LO)
!$ACC EXIT DATA DELETE (f%tracers%work%del_ttf_advvert, f%tracers%work%del_ttf_advhoriz, f%tracers%work%edge_up_dn_grad)
!$ACC EXIT DATA DELETE (f%tracers%work%del_ttf)
!$ACC EXIT DATA DELETE (tr_xy, tr_z, relax2clim, Sclim, Tclim)
!$ACC EXIT DATA DELETE (f%tracers%data, f%tracers%work)
!$ACC EXIT DATA DELETE (f%dynamics%w, f%dynamics%w_e, f%dynamics%uv)
!$ACC EXIT DATA DELETE (f%dynamics, f%tracers)
Expand Down Expand Up @@ -831,4 +834,4 @@ subroutine fesom_finalize()
! call clock_finish
end subroutine fesom_finalize

end module fesom_module
end module fesom_module
6 changes: 3 additions & 3 deletions src/oce_ale_tracer.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,8 +224,8 @@ subroutine solve_tracers_ale(ice, dynamics, tracers, partit, mesh)
! advect tracers
if (flag_debug .and. mype==0) print *, achar(27)//'[37m'//' --> call adv_tracers_ale'//achar(27)//'[0m'
!here update only those initialized in the init_tracers. (values, valuesAB, edge_up_dn_grad, ...)
!$ACC UPDATE DEVICE(tracers%data(tr_num)%values, tracers%data(tr_num)%valuesAB) &
!$ACC DEVICE(tracers%work%edge_up_dn_grad) !!&
!!$ACC UPDATE DEVICE(tracers%data(tr_num)%values, tracers%data(tr_num)%valuesAB, tracers%data(tr_num)%valuesold)
!$ACC UPDATE DEVICE(tracers%work%edge_up_dn_grad) !!&
! it will update del_ttf with contributions from horizontal and vertical advection parts (del_ttf_advhoriz and del_ttf_advvert)
!$ACC wait(1)
call do_oce_adv_tra(dt, UV, Wvel, Wvel_i, Wvel_e, tr_num, dynamics, tracers, partit, mesh)
Expand Down Expand Up @@ -1666,4 +1666,4 @@ FUNCTION bc_surface(n, id, sval, nzmin, partit, mesh, sst, sss, aice)
END SELECT
RETURN

end function bc_surface
end function bc_surface
128 changes: 101 additions & 27 deletions src/oce_tracer_mod.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,9 @@ SUBROUTINE init_tracers_AB(tr_num, tracers, partit, mesh)
type(t_tracer), intent(inout), target :: tracers
integer :: n,nz

#ifdef ENABLE_OPENACC
!$ACC parallel loop collapse(2) default(present) !!!async(1)
#ifndef ENABLE_OPENACC
#else
!$ACC PARALLEL LOOP COLLAPSE(2) DEFAULT(PRESENT)
#endif
do n=1, partit%myDim_nod2D+partit%eDim_nod2D
do nz=1, mesh%nl-1
Expand All @@ -36,51 +37,89 @@ SUBROUTINE init_tracers_AB(tr_num, tracers, partit, mesh)
tracers%work%del_ttf_advvert (nz, n) = 0.0_WP
end do
end do
#ifdef ENABLE_OPENACC
!$ACC end parallel loop
#ifndef ENABLE_OPENACC
#else
!$ACC END PARALLEL LOOP
#endif

! AB interpolation
if (tracers%data(tr_num)%AB_order==2) then
#ifndef ENABLE_OPENACC
!$OMP PARALLEL DO
do n=1, partit%myDim_nod2D+partit%eDim_nod2D
! AB interpolation
if (tracers%data(tr_num)%AB_order==2) then
#else
!ACC PARALLEL LOOP DEFAULT(PRESENT)
#endif
do n=1, partit%myDim_nod2D+partit%eDim_nod2D
tracers%data(tr_num)%valuesAB(:, n) =-(0.5_WP+epsilon)*tracers%data(tr_num)%valuesold(1, :, n)+(1.5_WP+epsilon)*tracers%data(tr_num)%values(:, n)
elseif (tracers%data(tr_num)%AB_order==3) then
end do
#ifndef ENABLE_OPENACC
!$OMP END PARALLEL DO
#else
!ACC END PARALLEL LOOP
#endif
elseif (tracers%data(tr_num)%AB_order==3) then
#ifndef ENABLE_OPENACC
!$OMP PARALLEL DO
#else
!ACC PARALLEL LOOP DEFAULT(PRESENT)
#endif
do n=1, partit%myDim_nod2D+partit%eDim_nod2D
tracers%data(tr_num)%valuesAB(:, n) =5.0_WP*tracers%data(tr_num)%valuesold(2, :, n)-16.0_WP*tracers%data(tr_num)%valuesold(1, :, n)+23.0_WP*tracers%data(tr_num)%values(:, n)
tracers%data(tr_num)%valuesAB(:, n) =tracers%data(tr_num)%valuesAB(:, n)/12.0_WP
else
write(*,*)
print *, achar(27)//'[33m'
write(*,*) '____________________________________________________________________'
write(*,*) ' ERROR: Adams-Bashfort tracer order must be 2 or 3, others are not'
write(*,*) ' supported!'
write(*,*) ' your tracers%data(tr_num)%AB_order =', tracers%data(tr_num)%AB_order
write(*,*) ' '
write(*,*) ' --> check your namelist.tra !!!'
write(*,*) ' &tracer_general'
write(*,*) ' ...'
write(*,*) ' AB_order = 2'
write(*,*) ' / '
write(*,*) '____________________________________________________________________'
print *, achar(27)//'[0m'
write(*,*)
call par_ex(partit%MPI_COMM_FESOM, partit%mype, 0)
end if
end do
end do
#ifndef ENABLE_OPENACC
!$OMP END PARALLEL DO
#else
!ACC END PARALLEL LOOP
#endif
else
write(*,*)
print *, achar(27)//'[33m'
write(*,*) '____________________________________________________________________'
write(*,*) ' ERROR: Adams-Bashfort tracer order must be 2 or 3, others are not'
write(*,*) ' supported!'
write(*,*) ' your tracers%data(tr_num)%AB_order =', tracers%data(tr_num)%AB_order
write(*,*) ' '
write(*,*) ' --> check your namelist.tra !!!'
write(*,*) ' &tracer_general'
write(*,*) ' ...'
write(*,*) ' AB_order = 2'
write(*,*) ' / '
write(*,*) '____________________________________________________________________'
print *, achar(27)//'[0m'
write(*,*)
call par_ex(partit%MPI_COMM_FESOM, partit%mype, 0)
end if

if (tracers%data(tr_num)%AB_order==2) then
#ifndef ENABLE_OPENACC
!$OMP PARALLEL DO
#else
!ACC PARALLEL LOOP DEFAULT(PRESENT)
#endif
do n=1, partit%myDim_nod2d+partit%eDim_nod2D
tracers%data(tr_num)%valuesold(1, :, n)=tracers%data(tr_num)%values(:, n)
end do
#ifndef ENABLE_OPENACC
!$OMP END PARALLEL DO
#else
!ACC END PARALLEL LOOP
#endif
elseif (tracers%data(tr_num)%AB_order==3) then
#ifndef ENABLE_OPENACC
!$OMP PARALLEL DO
#else
!ACC PARALLEL LOOP DEFAULT(PRESENT)
#endif
do n=1, partit%myDim_nod2d+partit%eDim_nod2D
tracers%data(tr_num)%valuesold(2, :, n)=tracers%data(tr_num)%valuesold(1, :, n)
tracers%data(tr_num)%valuesold(1, :, n)=tracers%data(tr_num)%values(:, n)
end do
#ifndef ENABLE_OPENACC
!$OMP END PARALLEL DO
#else
!ACC END PARALLEL LOOP
#endif
end if

if (flag_debug .and. partit%mype==0) print *, achar(27)//'[38m'//' --> call tracer_gradient_elements'//achar(27)//'[0m'
Expand Down Expand Up @@ -127,7 +166,12 @@ SUBROUTINE tracer_gradient_elements(ttf, partit, mesh)
#include "associate_mesh_def.h"
#include "associate_part_ass.h"
#include "associate_mesh_ass.h"

#ifndef ENABLE_OPENACC
!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(elem, elnodes, nz, nzmin, nzmax)
#else
!ACC PARALLEL LOOP DEFAULT(PRESENT) PRIVATE(elnodes)
#endif
DO elem=1, myDim_elem2D
elnodes=elem2D_nodes(:,elem)
nzmin = ulevels(elem)
Expand All @@ -138,7 +182,11 @@ SUBROUTINE tracer_gradient_elements(ttf, partit, mesh)
tr_xy(2,nz, elem)=sum(gradient_sca(4:6,elem)*ttf(nz,elnodes))
END DO
END DO
#ifndef ENABLE_OPENACC
!$OMP END PARALLEL DO
#else
!ACC END PARALLEL LOOP
#endif
END SUBROUTINE tracer_gradient_elements
!
!
Expand All @@ -163,7 +211,11 @@ SUBROUTINE tracer_gradient_z(ttf, partit, mesh)
#include "associate_mesh_def.h"
#include "associate_part_ass.h"
#include "associate_mesh_ass.h"
#ifndef ENABLE_OPENACC
!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(n, nz, nzmin, nzmax, dz)
#else
!ACC PARALLEL LOOP DEFAULT(PRESENT)
#endif
DO n=1, myDim_nod2D+eDim_nod2D
nzmax=nlevels_nod2D(n)
nzmin=ulevels_nod2D(n)
Expand All @@ -174,7 +226,11 @@ SUBROUTINE tracer_gradient_z(ttf, partit, mesh)
tr_z(nzmin, n)=0.0_WP
tr_z(nzmax, n)=0.0_WP
END DO
#ifndef ENABLE_OPENACC
!$OMP END PARALLEL DO
#else
!ACC END PARALLEL LOOP
#endif
END SUBROUTINE tracer_gradient_z
!
!
Expand Down Expand Up @@ -202,7 +258,12 @@ SUBROUTINE relax_to_clim(tr_num, tracers, partit, mesh)
trarr=>tracers%data(tr_num)%values(:,:)

if ((clim_relax>1.0e-8_WP).and.(tracers%data(tr_num)%ID==1)) then
#ifndef ENABLE_OPENACC
!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(n, nzmin, nzmax)
#else
!ACC UPDATE DEVICE(relax2clim, Tclim)
!ACC PARALLEL LOOP DEFAULT(PRESENT)
#endif
DO n=1, myDim_nod2D
nzmin = ulevels_nod2D(n)
nzmax = nlevels_nod2D(n)
Expand All @@ -211,17 +272,30 @@ SUBROUTINE relax_to_clim(tr_num, tracers, partit, mesh)
trarr(nzmin:nzmax-1,n)=trarr(nzmin:nzmax-1,n)+&
relax2clim(n)*dt*(Tclim(nzmin:nzmax-1,n)-trarr(nzmin:nzmax-1,n))
END DO
#ifndef ENABLE_OPENACC
!$OMP END PARALLEL DO
#else
!ACC END PARALLEL LOOP
#endif
END if
if ((clim_relax>1.0e-8_WP).and.(tracers%data(tr_num)%ID==2)) then
#ifndef ENABLE_OPENACC
!$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(n, nzmin, nzmax)
#else
!ACC UPDATE DEVICE(Sclim)
!ACC PARALLEL LOOP DEFAULT(PRESENT)
#endif
DO n=1, myDim_nod2D
nzmin = ulevels_nod2D(n)
nzmax = nlevels_nod2D(n)
trarr(nzmin:nzmax-1,n)=trarr(nzmin:nzmax-1,n)+&
relax2clim(n)*dt*(Sclim(nzmin:nzmax-1,n)-trarr(nzmin:nzmax-1,n))
END DO
#ifndef ENABLE_OPENACC
!$OMP END PARALLEL DO
#else
!ACC END PARALLEL LOOP
#endif
END IF
END SUBROUTINE relax_to_clim
END MODULE o_tracers