Skip to content

[PULL REQUEST] Adjoint run skeleton #71

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
TerribleNews wants to merge 14 commits into from
Closed

[PULL REQUEST] Adjoint run skeleton #71

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
93cd922
Added template files to geos-chem run directory
Oct 13, 2020
d535301
Forward and reverse models compile and run.
Dec 4, 2020
b9ef5b7
Updated create run directory for adjoint
Dec 7, 2020
79642a4
Trying to fix submodule branch problem
Dec 7, 2020
9d672c8
Merge branch 'main' of github.com:geoschem/GCHP into main
Dec 12, 2020
725b93d
Forward and reverse models compile and run.
Dec 4, 2020
45c5176
Added adjoint flags to cmake options
Dec 31, 2020
c386764
Merge branch 'add_ems_adj' of github.com:TerribleNews/GCHPctm_adj int…
Dec 31, 2020
4b48f39
Fixed CO2 run dir in GEOS-Chem submodule
Mar 9, 2021
b8501c8
Moved submodules to forked versions
Mar 10, 2021
e114221
Updated run files to match MPI configuration of runConfig.sh
Mar 30, 2021
27b67ff
Merge branch 'add_ems_adj' of github.com:TerribleNews/GCHPctm_adj int…
Mar 30, 2021
e53eb70
Wrapped debug printing in ifdef ADJOINT blocks
Apr 5, 2021
3685f4e
Moved adjoint-only code into ifdef ADJOINT block
Apr 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[submodule "src/MAPL"]
path = src/MAPL
url = https://github.com/geoschem/MAPL
url = https://github.com/TerribleNews/MAPL
[submodule "src/GMAO_Shared"]
path = src/GMAO_Shared
url = https://github.com/geoschem/GMAO_Shared
Expand All @@ -15,13 +15,13 @@
url = https://github.com/geoschem/FMS.git
[submodule "src/GCHP_GridComp/FVdycoreCubed_GridComp"]
path = src/GCHP_GridComp/FVdycoreCubed_GridComp
url = https://github.com/geoschem/FVdycoreCubed_GridComp.git
url = https://github.com/TerribleNews/FVdycoreCubed_GridComp.git
[submodule "src/GCHP_GridComp/GEOSChem_GridComp/geos-chem"]
path = src/GCHP_GridComp/GEOSChem_GridComp/geos-chem
url = https://github.com/geoschem/geos-chem.git
url = https://github.com/TerribleNews/geos-chem.git
[submodule "src/GCHP_GridComp/HEMCO_GridComp/HEMCO"]
path = src/GCHP_GridComp/HEMCO_GridComp/HEMCO
url = https://github.com/geoschem/HEMCO.git
url = https://github.com/TerribleNews/HEMCO.git
[submodule "src/yaFyaml"]
path = src/yaFyaml
url = https://github.com/geoschem/yaFyaml.git
Expand Down
6 changes: 6 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,3 +115,9 @@ set_target_properties(

PROPERTIES EXCLUDE_FROM_ALL TRUE
)

target_compile_definitions(Baselibs INTERFACE
$<$<BOOL:${ADJOINT}>:ADJOINT>
$<$<BOOL:${REVERSE_OPERATORS}>:REVERSE_OPERATORS>
""
)
2 changes: 1 addition & 1 deletion ESMA_cmake
Open in desktop
Submodule ESMA_cmake updated 5 files
+1 −1 FindBaselibs.cmake
+0 −15 FindESMF.cmake
+1 −1 GNU.cmake
+4 −5 Intel.cmake
+1 −0 esma.cmake
2 changes: 1 addition & 1 deletion src/GCHP_GridComp/FVdycoreCubed_GridComp
Open in desktop
Submodule FVdycoreCubed_GridComp updated 2 files
+34 −0 AdvCore_GridCompMod.F90
+4 −4 CubeToLatLon.F90
262 changes: 260 additions & 2 deletions src/GCHP_GridComp/GCHP_GridCompMod.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,10 @@ subroutine SetServices ( GC, RC )

type (ESMF_Config) :: CF
logical :: am_I_Root
#ifdef ADJOINT
character(len=ESMF_MAXSTR) :: ModelPhase
logical :: isAdjoint
#endif

!=============================================================================

Expand All @@ -115,7 +119,24 @@ subroutine SetServices ( GC, RC )
RC=STATUS )
_VERIFY(STATUS)

!BOP
#ifdef ADJOINT
CALL ESMF_ConfigGetAttribute( CF, ModelPhase, &
Label = "MODEL_PHASE:",&
Default="FORWARD", &
__RC__ )
isAdjoint = .false.
if (trim(ModelPhase) == 'ADJOINT') &
isAdjoint = .true.

#ifdef REVERSE_OPERATORS
IF (.not. isAdjoint) THEN
IF (MAPL_Am_I_Root()) &
WRITE(*,*) ' Forward run, adding children in standard order. ' // ModelPhase
#else
WRITE(*,*) ' Adding children in standard order. MODEL_PHASE: ' // ModelPhase
#endif

#endif

! !IMPORT STATE:

Expand All @@ -139,6 +160,29 @@ subroutine SetServices ( GC, RC )
RC=STATUS)
_VERIFY(STATUS)

#ifdef ADJOINT
#ifdef REVERSE_OPERATORS
ELSE
IF (MAPL_Am_I_Root()) &
WRITE(*,*) ' Adjoint run, adding children in reverse order. '
! Add dynamics
ADV = MAPL_AddChild(GC, NAME='DYNAMICS', SS=AtmosAdvSetServices, &
RC=STATUS)
_VERIFY(STATUS)

! Add chemistry
CHEM = MAPL_AddChild(GC, NAME='GCHPchem', SS=AtmosChemSetServices, &
RC=STATUS)
_VERIFY(STATUS)

! Add component for deriving variables for other components
ECTM = MAPL_AddChild(GC, NAME='GCHPctmEnv' , SS=EctmSetServices, &
RC=STATUS)
_VERIFY(STATUS)
ENDIF
#endif
#endif

! Set internal connections between the children`s IMPORTS and EXPORTS
! -------------------------------------------------------------------
!BOP
Expand Down Expand Up @@ -274,11 +318,27 @@ subroutine Initialize ( GC, IMPORT, EXPORT, CLOCK, RC )
call MAPL_GridCreate( GC, rc=status )
_VERIFY(STATUS)

#ifdef ADJOINT
if (MAPL_Am_I_Root()) THEN
WRITE(*,*) 'Before Generic Init'
endif
#endif

! Try to locate a grid cell. This should only ever work on one PET



! Call Initialize for every Child
!--------------------------------
call MAPL_GenericInitialize ( GC, IMPORT, EXPORT, CLOCK, __RC__ )
_VERIFY(STATUS)

#ifdef ADJOINT
if (MAPL_Am_I_Root()) THEN
WRITE(*,*) 'After Generic Init'
endif
#endif

call MAPL_TimerOn(STATE,"TOTAL")
! call MAPL_TimerOn(STATE,"INITIALIZE")

Expand Down Expand Up @@ -353,7 +413,11 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC )
integer :: I, L
integer :: IM, JM, LM
real :: DT


character(len=ESMF_MAXSTR) :: ModelPhase
logical :: isAdjoint
logical, save :: firstRun = .true.

!=============================================================================

! Begin...
Expand All @@ -377,6 +441,18 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC )
_VERIFY(STATUS)
endif

#ifdef ADJOINT
! Check if this is an adjoint run
CALL ESMF_ConfigGetAttribute( CF, ModelPhase, &
Label = "MODEL_PHASE:",&
Default="FORWARD", &
__RC__ )
isAdjoint = .false.
if (trim(ModelPhase) == 'ADJOINT') &
isAdjoint = .true.
#endif


! Start timers
!-------------
call MAPL_TimerOn(STATE,"TOTAL")
Expand All @@ -401,6 +477,21 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC )
call ESMF_ConfigGetAttribute(CF, DT, Label="RUN_DT:" , RC=STATUS)
_VERIFY(STATUS)

#ifdef ADJOINT
!------------------------------------------------------------
! If we're doing the adoint, we should be running these in
! reverse order. Possibly not the Environment module?
! In cany case, this isn't working yet, so disable it for now
!------------------------------------------------------------
#ifdef REVERSE_OPERATORS
IF (.not. isAdjoint) THEN
#else
IF (.true.) THEN
#endif
if (MAPL_Am_I_Root()) THEN
WRITE(*,*) 'Not reversing high-level operators'
endif
#endif
! Cinderella Component: to derive variables for other components
!---------------------

Expand Down Expand Up @@ -493,10 +584,177 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC )
'GCHP, after GEOS-Chem: ', RC=STATUS )
_VERIFY(STATUS)
endif
#ifdef ADJOINT
ELSE
if (MAPL_Am_I_Root()) THEN
WRITE(*,*) 'Reversing high-level operators'
endif

IF (firstRun) THEN
! Cinderella Component: to derive variables for other components
!---------------------

if ( MemDebugLevel > 0 ) THEN
call ESMF_VMBarrier(VM, RC=STATUS)
_VERIFY(STATUS)
call MAPL_MemUtilsWrite(VM, &
'GIGC, before GEOS_ctmE: ', RC=STATUS )
_VERIFY(STATUS)
endif

call MAPL_TimerOn ( STATE, GCNames(ECTM) )
call ESMF_GridCompRun ( GCS(ECTM), &
importState = GIM(ECTM), &
exportState = GEX(ECTM), &
clock = CLOCK, &
userRC = STATUS )
_VERIFY(STATUS)

call MAPL_TimerOff( STATE, GCNames(ECTM) )

if ( MemDebugLevel > 0 ) THEN
call ESMF_VMBarrier(VM, RC=STATUS)
_VERIFY(STATUS)
call MAPL_MemUtilsWrite(VM, &
'GIGC, after GEOS_ctmE: ', RC=STATUS )
_VERIFY(STATUS)
endif

! Dynamics & Advection
!------------------
! SDE 2017-02-18: This needs to run even if transport is off, as it is
! responsible for the pressure level edge arrays. It already has an internal
! switch ("AdvCore_Advection") which can be used to prevent any actual
! transport taking place by bypassing the advection calculation.

if ( MemDebugLevel > 0 ) THEN
call ESMF_VMBarrier(VM, RC=STATUS)
_VERIFY(STATUS)
call MAPL_MemUtilsWrite(VM, &
'GIGC, before Advection: ', RC=STATUS )
_VERIFY(STATUS)
endif

call MAPL_TimerOn ( STATE, GCNames(ADV) )
call ESMF_GridCompRun ( GCS(ADV), &
importState = GIM(ADV), &
exportState = GEX(ADV), &
clock = CLOCK, &
userRC = STATUS );
_VERIFY(STATUS)
call MAPL_GenericRunCouplers (STATE, ADV, CLOCK, RC=STATUS );
_VERIFY(STATUS)
call MAPL_TimerOff( STATE, GCNames(ADV) )

if ( MemDebugLevel > 0 ) THEN
call ESMF_VMBarrier(VM, RC=STATUS)
_VERIFY(STATUS)
call MAPL_MemUtilsWrite(VM, &
'GIGC, after Advection: ', RC=STATUS )
_VERIFY(STATUS)
endif

ENDIF
! Chemistry
!------------------

if ( MemDebugLevel > 0 ) THEN
call ESMF_VMBarrier(VM, RC=STATUS)
_VERIFY(STATUS)
call MAPL_MemUtilsWrite(VM, &
'GIGC, before GEOS-Chem: ', RC=STATUS )
_VERIFY(STATUS)
endif

call MAPL_TimerOn ( STATE, GCNames(CHEM) )
call ESMF_GridCompRun ( GCS(CHEM), &
importState = GIM(CHEM), &
exportState = GEX(CHEM), &
clock = CLOCK, &
userRC = STATUS );
_VERIFY(STATUS)
call MAPL_GenericRunCouplers (STATE, CHEM, CLOCK, RC=STATUS );
_VERIFY(STATUS)
call MAPL_TimerOff(STATE,GCNames(CHEM))

if ( MemDebugLevel > 0 ) THEN
call ESMF_VMBarrier(VM, RC=STATUS)
_VERIFY(STATUS)
call MAPL_MemUtilsWrite(VM, &
'GIGC, after GEOS-Chem: ', RC=STATUS )
_VERIFY(STATUS)
endif

! Cinderella Component: to derive variables for other components
!---------------------

if ( MemDebugLevel > 0 ) THEN
call ESMF_VMBarrier(VM, RC=STATUS)
_VERIFY(STATUS)
call MAPL_MemUtilsWrite(VM, &
'GIGC, before GEOS_ctmE: ', RC=STATUS )
_VERIFY(STATUS)
endif

call MAPL_TimerOn ( STATE, GCNames(ECTM) )
call ESMF_GridCompRun ( GCS(ECTM), &
importState = GIM(ECTM), &
exportState = GEX(ECTM), &
clock = CLOCK, &
userRC = STATUS )
_VERIFY(STATUS)

call MAPL_TimerOff( STATE, GCNames(ECTM) )

if ( MemDebugLevel > 0 ) THEN
call ESMF_VMBarrier(VM, RC=STATUS)
_VERIFY(STATUS)
call MAPL_MemUtilsWrite(VM, &
'GIGC, after GEOS_ctmE: ', RC=STATUS )
_VERIFY(STATUS)
endif

! Dynamics & Advection
!------------------
! SDE 2017-02-18: This needs to run even if transport is off, as it is
! responsible for the pressure level edge arrays. It already has an internal
! switch ("AdvCore_Advection") which can be used to prevent any actual
! transport taking place by bypassing the advection calculation.

if ( MemDebugLevel > 0 ) THEN
call ESMF_VMBarrier(VM, RC=STATUS)
_VERIFY(STATUS)
call MAPL_MemUtilsWrite(VM, &
'GIGC, before Advection: ', RC=STATUS )
_VERIFY(STATUS)
endif

call MAPL_TimerOn ( STATE, GCNames(ADV) )
call ESMF_GridCompRun ( GCS(ADV), &
importState = GIM(ADV), &
exportState = GEX(ADV), &
clock = CLOCK, &
userRC = STATUS );
_VERIFY(STATUS)
call MAPL_GenericRunCouplers (STATE, ADV, CLOCK, RC=STATUS );
_VERIFY(STATUS)
call MAPL_TimerOff( STATE, GCNames(ADV) )

if ( MemDebugLevel > 0 ) THEN
call ESMF_VMBarrier(VM, RC=STATUS)
_VERIFY(STATUS)
call MAPL_MemUtilsWrite(VM, &
'GIGC, after Advection: ', RC=STATUS )
_VERIFY(STATUS)
endif

ENDIF
#endif
call MAPL_TimerOff(STATE,"RUN")
call MAPL_TimerOff(STATE,"TOTAL")

firstRun = .false.

_RETURN(ESMF_SUCCESS)

end subroutine Run
Expand Down
Loading