diff --git a/.gitmodules b/.gitmodules index 6e91cc7b2..1dd2a4748 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "src/MAPL"] path = src/MAPL - url = https://github.com/geoschem/MAPL + url = https://github.com/TerribleNews/MAPL [submodule "src/GMAO_Shared"] path = src/GMAO_Shared url = https://github.com/geoschem/GMAO_Shared @@ -15,13 +15,13 @@ url = https://github.com/geoschem/FMS.git [submodule "src/GCHP_GridComp/FVdycoreCubed_GridComp"] path = src/GCHP_GridComp/FVdycoreCubed_GridComp - url = https://github.com/geoschem/FVdycoreCubed_GridComp.git + url = https://github.com/TerribleNews/FVdycoreCubed_GridComp.git [submodule "src/GCHP_GridComp/GEOSChem_GridComp/geos-chem"] path = src/GCHP_GridComp/GEOSChem_GridComp/geos-chem - url = https://github.com/geoschem/geos-chem.git + url = https://github.com/TerribleNews/geos-chem.git [submodule "src/GCHP_GridComp/HEMCO_GridComp/HEMCO"] path = src/GCHP_GridComp/HEMCO_GridComp/HEMCO - url = https://github.com/geoschem/HEMCO.git + url = https://github.com/TerribleNews/HEMCO.git [submodule "src/yaFyaml"] path = src/yaFyaml url = https://github.com/geoschem/yaFyaml.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 52092f3dd..397fb66d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,3 +115,9 @@ set_target_properties( PROPERTIES EXCLUDE_FROM_ALL TRUE ) + +target_compile_definitions(Baselibs INTERFACE + $<$:ADJOINT> + $<$:REVERSE_OPERATORS> + "" +) diff --git a/ESMA_cmake b/ESMA_cmake index 702d94de9..a4362bfcd 160000 --- a/ESMA_cmake +++ b/ESMA_cmake @@ -1 +1 @@ -Subproject commit 702d94de9100a050031bf799ad9eed7996098568 +Subproject commit a4362bfcdf84b278e67a4f4a1d2cc2dd50d077a4 diff --git a/src/GCHP_GridComp/FVdycoreCubed_GridComp b/src/GCHP_GridComp/FVdycoreCubed_GridComp index 4ff5e9249..d87af81f4 160000 --- a/src/GCHP_GridComp/FVdycoreCubed_GridComp +++ b/src/GCHP_GridComp/FVdycoreCubed_GridComp @@ -1 +1 @@ -Subproject commit 4ff5e9249dc51c19c84fbda9bb1a13e7e630912d +Subproject commit d87af81f4244b5cda3d8da13b10933db075c7583 diff --git a/src/GCHP_GridComp/GCHP_GridCompMod.F90 b/src/GCHP_GridComp/GCHP_GridCompMod.F90 index 16fb3406f..da4be39fe 100644 --- a/src/GCHP_GridComp/GCHP_GridCompMod.F90 +++ b/src/GCHP_GridComp/GCHP_GridCompMod.F90 @@ -91,6 +91,10 @@ subroutine SetServices ( GC, RC ) type (ESMF_Config) :: CF logical :: am_I_Root +#ifdef ADJOINT + character(len=ESMF_MAXSTR) :: ModelPhase + logical :: isAdjoint +#endif !============================================================================= @@ -115,7 +119,24 @@ subroutine SetServices ( GC, RC ) RC=STATUS ) _VERIFY(STATUS) -!BOP +#ifdef ADJOINT + CALL ESMF_ConfigGetAttribute( CF, ModelPhase, & + Label = "MODEL_PHASE:",& + Default="FORWARD", & + __RC__ ) + isAdjoint = .false. + if (trim(ModelPhase) == 'ADJOINT') & + isAdjoint = .true. + +#ifdef REVERSE_OPERATORS + IF (.not. isAdjoint) THEN + IF (MAPL_Am_I_Root()) & + WRITE(*,*) ' Forward run, adding children in standard order. ' // ModelPhase +#else + WRITE(*,*) ' Adding children in standard order. MODEL_PHASE: ' // ModelPhase +#endif + +#endif ! !IMPORT STATE: @@ -139,6 +160,29 @@ subroutine SetServices ( GC, RC ) RC=STATUS) _VERIFY(STATUS) +#ifdef ADJOINT +#ifdef REVERSE_OPERATORS + ELSE + IF (MAPL_Am_I_Root()) & + WRITE(*,*) ' Adjoint run, adding children in reverse order. ' + ! Add dynamics + ADV = MAPL_AddChild(GC, NAME='DYNAMICS', SS=AtmosAdvSetServices, & + RC=STATUS) + _VERIFY(STATUS) + + ! Add chemistry + CHEM = MAPL_AddChild(GC, NAME='GCHPchem', SS=AtmosChemSetServices, & + RC=STATUS) + _VERIFY(STATUS) + + ! Add component for deriving variables for other components + ECTM = MAPL_AddChild(GC, NAME='GCHPctmEnv' , SS=EctmSetServices, & + RC=STATUS) + _VERIFY(STATUS) + ENDIF +#endif +#endif + ! Set internal connections between the children`s IMPORTS and EXPORTS ! ------------------------------------------------------------------- !BOP @@ -274,11 +318,27 @@ subroutine Initialize ( GC, IMPORT, EXPORT, CLOCK, RC ) call MAPL_GridCreate( GC, rc=status ) _VERIFY(STATUS) +#ifdef ADJOINT + if (MAPL_Am_I_Root()) THEN + WRITE(*,*) 'Before Generic Init' + endif +#endif + + ! Try to locate a grid cell. This should only ever work on one PET + + + ! Call Initialize for every Child !-------------------------------- call MAPL_GenericInitialize ( GC, IMPORT, EXPORT, CLOCK, __RC__ ) _VERIFY(STATUS) +#ifdef ADJOINT + if (MAPL_Am_I_Root()) THEN + WRITE(*,*) 'After Generic Init' + endif +#endif + call MAPL_TimerOn(STATE,"TOTAL") ! call MAPL_TimerOn(STATE,"INITIALIZE") @@ -353,7 +413,11 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) integer :: I, L integer :: IM, JM, LM real :: DT - + + character(len=ESMF_MAXSTR) :: ModelPhase + logical :: isAdjoint + logical, save :: firstRun = .true. + !============================================================================= ! Begin... @@ -377,6 +441,18 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) _VERIFY(STATUS) endif +#ifdef ADJOINT + ! Check if this is an adjoint run + CALL ESMF_ConfigGetAttribute( CF, ModelPhase, & + Label = "MODEL_PHASE:",& + Default="FORWARD", & + __RC__ ) + isAdjoint = .false. + if (trim(ModelPhase) == 'ADJOINT') & + isAdjoint = .true. +#endif + + ! Start timers !------------- call MAPL_TimerOn(STATE,"TOTAL") @@ -401,6 +477,21 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) call ESMF_ConfigGetAttribute(CF, DT, Label="RUN_DT:" , RC=STATUS) _VERIFY(STATUS) +#ifdef ADJOINT + !------------------------------------------------------------ + ! If we're doing the adoint, we should be running these in + ! reverse order. Possibly not the Environment module? + ! In cany case, this isn't working yet, so disable it for now + !------------------------------------------------------------ +#ifdef REVERSE_OPERATORS + IF (.not. isAdjoint) THEN +#else + IF (.true.) THEN +#endif + if (MAPL_Am_I_Root()) THEN + WRITE(*,*) 'Not reversing high-level operators' + endif +#endif ! Cinderella Component: to derive variables for other components !--------------------- @@ -493,10 +584,177 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) 'GCHP, after GEOS-Chem: ', RC=STATUS ) _VERIFY(STATUS) endif +#ifdef ADJOINT + ELSE + if (MAPL_Am_I_Root()) THEN + WRITE(*,*) 'Reversing high-level operators' + endif + + IF (firstRun) THEN + ! Cinderella Component: to derive variables for other components + !--------------------- + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, before GEOS_ctmE: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + call MAPL_TimerOn ( STATE, GCNames(ECTM) ) + call ESMF_GridCompRun ( GCS(ECTM), & + importState = GIM(ECTM), & + exportState = GEX(ECTM), & + clock = CLOCK, & + userRC = STATUS ) + _VERIFY(STATUS) + + call MAPL_TimerOff( STATE, GCNames(ECTM) ) + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, after GEOS_ctmE: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + ! Dynamics & Advection + !------------------ + ! SDE 2017-02-18: This needs to run even if transport is off, as it is + ! responsible for the pressure level edge arrays. It already has an internal + ! switch ("AdvCore_Advection") which can be used to prevent any actual + ! transport taking place by bypassing the advection calculation. + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, before Advection: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + call MAPL_TimerOn ( STATE, GCNames(ADV) ) + call ESMF_GridCompRun ( GCS(ADV), & + importState = GIM(ADV), & + exportState = GEX(ADV), & + clock = CLOCK, & + userRC = STATUS ); + _VERIFY(STATUS) + call MAPL_GenericRunCouplers (STATE, ADV, CLOCK, RC=STATUS ); + _VERIFY(STATUS) + call MAPL_TimerOff( STATE, GCNames(ADV) ) + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, after Advection: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + ENDIF + ! Chemistry + !------------------ + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, before GEOS-Chem: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + call MAPL_TimerOn ( STATE, GCNames(CHEM) ) + call ESMF_GridCompRun ( GCS(CHEM), & + importState = GIM(CHEM), & + exportState = GEX(CHEM), & + clock = CLOCK, & + userRC = STATUS ); + _VERIFY(STATUS) + call MAPL_GenericRunCouplers (STATE, CHEM, CLOCK, RC=STATUS ); + _VERIFY(STATUS) + call MAPL_TimerOff(STATE,GCNames(CHEM)) + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, after GEOS-Chem: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + ! Cinderella Component: to derive variables for other components + !--------------------- + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, before GEOS_ctmE: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + call MAPL_TimerOn ( STATE, GCNames(ECTM) ) + call ESMF_GridCompRun ( GCS(ECTM), & + importState = GIM(ECTM), & + exportState = GEX(ECTM), & + clock = CLOCK, & + userRC = STATUS ) + _VERIFY(STATUS) + + call MAPL_TimerOff( STATE, GCNames(ECTM) ) + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, after GEOS_ctmE: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + ! Dynamics & Advection + !------------------ + ! SDE 2017-02-18: This needs to run even if transport is off, as it is + ! responsible for the pressure level edge arrays. It already has an internal + ! switch ("AdvCore_Advection") which can be used to prevent any actual + ! transport taking place by bypassing the advection calculation. + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, before Advection: ', RC=STATUS ) + _VERIFY(STATUS) + endif + call MAPL_TimerOn ( STATE, GCNames(ADV) ) + call ESMF_GridCompRun ( GCS(ADV), & + importState = GIM(ADV), & + exportState = GEX(ADV), & + clock = CLOCK, & + userRC = STATUS ); + _VERIFY(STATUS) + call MAPL_GenericRunCouplers (STATE, ADV, CLOCK, RC=STATUS ); + _VERIFY(STATUS) + call MAPL_TimerOff( STATE, GCNames(ADV) ) + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, after Advection: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + ENDIF +#endif call MAPL_TimerOff(STATE,"RUN") call MAPL_TimerOff(STATE,"TOTAL") + firstRun = .false. + _RETURN(ESMF_SUCCESS) end subroutine Run diff --git a/src/GCHP_GridComp/GCHPctmEnv_GridComp/GCHPctmEnv_GridCompMod.F90 b/src/GCHP_GridComp/GCHPctmEnv_GridComp/GCHPctmEnv_GridCompMod.F90 index 4d2d34ca7..af026acdd 100755 --- a/src/GCHP_GridComp/GCHPctmEnv_GridComp/GCHPctmEnv_GridCompMod.F90 +++ b/src/GCHP_GridComp/GCHPctmEnv_GridComp/GCHPctmEnv_GridCompMod.F90 @@ -541,6 +541,11 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) integer :: i, j real(r8) :: PSDry0, PSDry1, PEdge_Bot, PEdge_Top +#ifdef ADJOINT + logical, save :: firstRun = .true. + integer :: reverseTime +#endif + ! Get the target components name and set-up traceback handle. ! ----------------------------------------------------------- call ESMF_GridCompGet ( GC, name=COMP_NAME, Grid=esmfGrid, RC=STATUS ) @@ -561,6 +566,18 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) _VERIFY(STATUS) dt = ndt +#ifdef ADJOINT + call MAPL_GetResource( ggState, reverseTime, 'REVERSE_TIME:', default=0, RC=STATUS ) + _VERIFY(STATUS) + IF ( MAPL_Am_I_Root() ) THEN + WRITE(*,*) ' GIGCenv REVERSE_TIME: ', reverseTime + ENDIF + IF ( reverseTime .eq. 1) THEN + WRITE(*,*) ' GIGCenv swapping timestep sign.' + dt = -dt + ENDIF +#endif + ! Get to the imports... ! --------------------- call MAPL_GetPointer ( IMPORT, PS0, 'PS1', RC=STATUS ) @@ -676,8 +693,15 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) ! Use dry pressure at the start of the timestep to calculate mass ! fluxes. GMAO method uses mid-step UC, VC and PLE? PLEr8 = 1.00d0*(DryPLE0r8) +#ifdef ADJOINT + if (.not. firstRun) THEN +#endif call fv_computeMassFluxes(UCr8, VCr8, PLEr8, & MFXr8, MFYr8, CXr8, CYr8, dt) +#ifdef ADJOINT + endif + firstRun = .false. +#endif !DEALLOCATE( UCr8, VCr8, PLEr8, PLE0, PLE1, DryPLE0, DryPLE1 ) DEALLOCATE( UCr8, VCr8, PLEr8, UC, VC) diff --git a/src/GCHP_GridComp/GEOSChem_GridComp/CMakeLists.txt b/src/GCHP_GridComp/GEOSChem_GridComp/CMakeLists.txt index e6bb932e0..86da2082d 100755 --- a/src/GCHP_GridComp/GEOSChem_GridComp/CMakeLists.txt +++ b/src/GCHP_GridComp/GEOSChem_GridComp/CMakeLists.txt @@ -22,6 +22,8 @@ add_subdirectory(geos-chem EXCLUDE_FROM_ALL) target_compile_definitions(GEOSChemBuildProperties INTERFACE ESMF_ EXTERNAL_GRID NC_HAS_COMPRESSION MODEL_GCHPCTM MODEL_GCHP $<$:USE_REAL8> $<$:RRTMG> + $<$:ADJOINT> + $<$:REVERSE_OPERATORS> ) target_link_libraries(GEOSChemBuildProperties INTERFACE diff --git a/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem b/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem index 0d390164f..5891c0a91 160000 --- a/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem +++ b/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem @@ -1 +1 @@ -Subproject commit 0d390164f3441816defd05519f28dfde574a1cfa +Subproject commit 5891c0a91d88e27ef9d25a6222418111fb019bb1 diff --git a/src/GCHP_GridComp/HEMCO_GridComp/CMakeLists.txt b/src/GCHP_GridComp/HEMCO_GridComp/CMakeLists.txt index 0a2cf04c9..82e7ca677 100644 --- a/src/GCHP_GridComp/HEMCO_GridComp/CMakeLists.txt +++ b/src/GCHP_GridComp/HEMCO_GridComp/CMakeLists.txt @@ -1,3 +1,9 @@ # HEMCO_GridComp/CMakeLists.txt add_subdirectory(HEMCO EXCLUDE_FROM_ALL) + +target_compile_definitions(HEMCOBuildProperties INTERFACE + $<$:ADJOINT> + $<$:REVERSE_OPERATORS> + "" +) diff --git a/src/GCHP_GridComp/HEMCO_GridComp/HEMCO b/src/GCHP_GridComp/HEMCO_GridComp/HEMCO index 11d4f4057..057363277 160000 --- a/src/GCHP_GridComp/HEMCO_GridComp/HEMCO +++ b/src/GCHP_GridComp/HEMCO_GridComp/HEMCO @@ -1 +1 @@ -Subproject commit 11d4f40571caa2d7de23c504f0c312ab1f54ded9 +Subproject commit 057363277fc0820ee7d4828ccbd23c4308b9992b diff --git a/src/MAPL b/src/MAPL index a9590432b..020d7e403 160000 --- a/src/MAPL +++ b/src/MAPL @@ -1 +1 @@ -Subproject commit a9590432be58889e4277253c7872d982980e4bb3 +Subproject commit 020d7e4038af9a5dbcbf9c4bc38771ca76be1c97