From f1da9f4dc19caadf35e376d9b484c21bda21487b Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 25 Apr 2022 13:56:58 -0500 Subject: [PATCH 01/29] Add smartredis to MPAS-Ocean build --- components/mpas-framework/Makefile | 11 ++++++++++- components/mpas-ocean/src/Makefile | 3 ++- components/mpas-ocean/src/build_options.mk | 1 + components/mpas-ocean/src/mode_forward/Makefile | 2 +- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/components/mpas-framework/Makefile b/components/mpas-framework/Makefile index c7804a511102..9e9a892d7c24 100644 --- a/components/mpas-framework/Makefile +++ b/components/mpas-framework/Makefile @@ -517,6 +517,7 @@ CPPINCLUDES = FCINCLUDES = LIBS = + # # If user has indicated a PIO2 library, define USE_PIO2 pre-processor macro # @@ -605,6 +606,14 @@ endif LIBS += -L$(PNETCDF)/$(PNETCDFLIBLOC) -lpnetcdf endif +#ifeq "$(USE_SMARTSIM)" "true" +#override LIBS += -L$(SMARTREDIS_PATH)/lib +#override LIBS += -lhiredis.a +#override LIBS += -lredis++.a +#override LIBS += -lsmartredis.so +#override LIBS += -lsmartredis-fortran.so +#endif + ifeq "$(USE_LAPACK)" "true" ifndef LAPACK $(error LAPACK is not set. Please set LAPACK to the LAPACK install directory when USE_LAPACK=true) @@ -1005,7 +1014,7 @@ drver: $(AUTOCLEAN_DEPS) framework dycore mpas: $(AUTOCLEAN_DEPS) framework dycore drver - $(LINKER) $(LDFLAGS) -o $(EXE_NAME) $(FWPATH)/driver/*.o -L$(FWPATH) -Lsrc -ldycore -lops -lframework $(LIBS) -I./external/esmf_time_f90 -L$(FWPATH)/external/esmf_time_f90 -lesmf_time + $(LINKER) $(LDFLAGS) -o $(EXE_NAME) $(FWPATH)/driver/*.o -L$(FWPATH) -Lsrc -ldycore -lops -lframework $(LIBS) -I./external/esmf_time_f90 -L$(FWPATH)/external/esmf_time_f90 -lesmf_time -L$(SMARTREDIS_PATH)/lib -lhiredis -lredis++ -lsmartredis-fortran framework: cd $(FWPATH); $(MAKE) \ diff --git a/components/mpas-ocean/src/Makefile b/components/mpas-ocean/src/Makefile index 822ed12d5409..6b5970446fe1 100644 --- a/components/mpas-ocean/src/Makefile +++ b/components/mpas-ocean/src/Makefile @@ -1,4 +1,4 @@ -.SUFFIXES: .F .c .o +.SUFFIXES: .F .F90 .c .o ifeq "$(EXCLUDE_INIT_MODE)" "true" CPPLOCALFLAGS = -DEXCLUDE_INIT_MODE @@ -9,6 +9,7 @@ FRAMEWORK_DIR = ../../../mpas-framework/src OCEAN_SHARED_INCLUDES = -I$(FRAMEWORK_DIR)/framework -I$(FRAMEWORK_DIR)/external/esmf_time_f90 -I$(FRAMEWORK_DIR)/operators OCEAN_SHARED_INCLUDES += -I$(PWD)/shared -I$(PWD)/analysis_members -I$(PWD)/mode_forward -I$(PWD)/mode_analysis OCEAN_SHARED_INCLUDES += -I$(PWD)/BGC -I$(PWD)/MARBL/include -I$(PWD)/cvmix/src/shared -I$(PWD)/gotm/build -I$(PWD)/gotm/build/modules -I$(PWD)/SHTNS -I$(PWD)/ppr/src +OCEAN_SHARED_INCLUDES += -I$(SMARTREDIS_PATH)/include ifneq "$(EXCLUDE_INIT_MODE)" "true" OCEAN_SHARED_INCLUDES += -I$(PWD)/mode_init endif diff --git a/components/mpas-ocean/src/build_options.mk b/components/mpas-ocean/src/build_options.mk index 8e854d2db2ff..d35830b61ad0 100644 --- a/components/mpas-ocean/src/build_options.mk +++ b/components/mpas-ocean/src/build_options.mk @@ -11,6 +11,7 @@ FCINCLUDES += -I$(ROOT_DIR)/BGC FCINCLUDES += -I$(ROOT_DIR)/MARBL/include FCINCLUDES += -I$(ROOT_DIR)/gotm/build/modules FCINCLUDES += -I$(ROOT_DIR)/ppr/src +FCINCLUDES += -I$(SMARTREDIS_PATH)/include override CPPFLAGS += -DCORE_OCEAN report_builds: diff --git a/components/mpas-ocean/src/mode_forward/Makefile b/components/mpas-ocean/src/mode_forward/Makefile index 8fc3b5c86515..fd2385ad350a 100644 --- a/components/mpas-ocean/src/mode_forward/Makefile +++ b/components/mpas-ocean/src/mode_forward/Makefile @@ -1,4 +1,4 @@ -.SUFFIXES: .F .o +.SUFFIXES: .F .o .F90 OBJS = mpas_ocn_forward_mode.o \ mpas_ocn_time_integration.o \ From 0c8da57f497cbdc0524310ea01984f04763f6bcb Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 25 Apr 2022 09:39:56 -0500 Subject: [PATCH 02/29] Add client initialization to forward mode --- .../src/mode_forward/mpas_ocn_forward_mode.F | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index adb19ac4ce81..e68e18a08938 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -95,6 +95,8 @@ module ocn_forward_mode use ocn_constants use ocn_config +! use smartredis_client, only: client_type + implicit none private @@ -103,6 +105,8 @@ module ocn_forward_mode ocn_forward_mode_finalize, & ocn_forward_mode_setup_clock +!# include "../../../mpas-framework/src/framework/enum_fortran.inc" + !*********************************************************************** contains @@ -143,6 +147,8 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ !----------------------------------------------------------------- integer :: err_tmp ! local error codes from subroutine calls + integer(kind=enum_kind) :: err_client + character(len=100) :: log_string real (kind=RKIND) :: & dt, &! time step in seconds @@ -172,6 +178,7 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ alarmTimeStep ! alarm interval in time manager form character (len=StrKIND) :: units + !type(client_type) :: client ! End preamble !------------- @@ -202,6 +209,20 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ ! Currently only support one block per task block => domain % blocklist + call get_environment_variable('SSDB',log_string) + call mpas_log_write('SSDB='//TRIM(log_string)) + if (len(TRIM(log_string)) < 5) then + call mpas_log_write('No SSDB')!, & + else + err_client = client%initialize(.false.) + if (err_client /= SRNoError) then + write(log_string,*) 'Error initializing redis client:',err_client + call mpas_log_write(log_string)!, & + !MPAS_LOG_ERR, masterOnly=.true., flushNow=.true.) + !ierr = ior(ierr, 1) + endif + endif + !----------------------------------------------------------------- ! Initialize data from external file input !----------------------------------------------------------------- From 211e400ed20ff9e180e8fddccd596d25a8f8d807 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 28 Feb 2022 11:42:04 -0700 Subject: [PATCH 03/29] Use a modified copy of enum_fortran.inc in mpas_framework --- .../src/framework/enum_fortran.inc | 79 +++++++++++++++++++ .../src/mode_forward/mpas_ocn_forward_mode.F | 2 +- 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 components/mpas-framework/src/framework/enum_fortran.inc diff --git a/components/mpas-framework/src/framework/enum_fortran.inc b/components/mpas-framework/src/framework/enum_fortran.inc new file mode 100644 index 000000000000..0555b0901462 --- /dev/null +++ b/components/mpas-framework/src/framework/enum_fortran.inc @@ -0,0 +1,79 @@ +! BSD 2-Clause License +! +! Copyright (c) 2021, Hewlett Packard Enterprise +! All rights reserved. +! +! Redistribution and use in source and binary forms, with or without +! modification, are permitted provided that the following conditions are met: +! +! 1. Redistributions of source code must retain the above copyright notice, this +! list of conditions and the following disclaimer. +! +! 2. Redistributions in binary form must reproduce the above copyright notice, +! this list of conditions and the following disclaimer in the documentation +! and/or other materials provided with the distribution. +! +! THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +! AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +! IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +! DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +! FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +! DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +! SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +! CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +! OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +! OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +! Dummy enum, used to set the integer kind used +integer, parameter :: enum_kind = 4 + +! The following enums have analogues to the C and C++ clients. Fortran does not haven named enums, but the name:value +! parameters must be the same. By convention negative enums, represent items that are not supported in the Fortran +! client. + +! TensorType +enum, bind(c) + enumerator :: tensor_invalid = -1 + enumerator :: tensor_dbl = 1 + enumerator :: tensor_flt = 2 + enumerator :: tensor_int8 = 3 + enumerator :: tensor_int16 = 4 + enumerator :: tensor_int32 = 5 + enumerator :: tensor_int64 = 6 + enumerator :: tensor_uint8 = -1 + enumerator :: tensor_uint16 = -1 +end enum + +! MemoryLayout​ +enum, bind(c) + enumerator :: c_invalid = -1 + enumerator :: c_nested = 1 + enumerator :: c_contiguous = 2 + enumerator :: c_fortran_nested = 3 + enumerator :: c_fortran_contiguous = 4 +end enum + +! MetadataType​ +enum, bind(c) + enumerator :: meta_invalid = -1 + enumerator :: meta_dbl = 1 + enumerator :: meta_flt = 2 + enumerator :: meta_int32 = 3 + enumerator :: meta_int64 = 4 + enumerator :: meta_uint32 = -1 + enumerator :: meta_uint64 = -1 + enumerator :: meta_string = 7 +end enum + +! SRError +enum, bind(c) + enumerator :: SRNoError = 0 + enumerator :: SRBadAllocError = 1 + enumerator :: SRDatabaseError = 2 + enumerator :: SRInternalError = 3 + enumerator :: SRRuntimeError = 4 + enumerator :: SRParameterError = 5 + enumerator :: SRTimeoutError = 6 + enumerator :: SRKeyError = 7 + enumerator :: SRInvalidError = -1 +end enum diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index e68e18a08938..15056b19624c 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -105,7 +105,7 @@ module ocn_forward_mode ocn_forward_mode_finalize, & ocn_forward_mode_setup_clock -!# include "../../../mpas-framework/src/framework/enum_fortran.inc" +# include "../../../mpas-framework/src/framework/enum_fortran.inc" !*********************************************************************** From 2a09863fa10baf96122862f4e51276ee93c0b926 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 28 Mar 2022 10:15:34 -0600 Subject: [PATCH 04/29] Use streams for smartredis io --- components/mpas-ocean/src/Registry.xml | 11 +++++ .../src/mode_forward/mpas_ocn_forward_mode.F | 45 ++++++++++++++++++- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 601d2d6184f0..95b00c65beb4 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -1821,6 +1821,17 @@ + + + Date: Tue, 12 Jul 2022 14:56:04 -0700 Subject: [PATCH 05/29] Add F90 files to mode_forward --- components/mpas-ocean/src/mode_forward/Makefile | 6 ++++-- .../mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/src/mode_forward/Makefile b/components/mpas-ocean/src/mode_forward/Makefile index fd2385ad350a..60bf40814ef7 100644 --- a/components/mpas-ocean/src/mode_forward/Makefile +++ b/components/mpas-ocean/src/mode_forward/Makefile @@ -21,8 +21,10 @@ mpas_ocn_time_integration_split.o: mpas_ocn_forward_mode.o: mpas_ocn_time_integration.o \ mpas_ocn_time_integration_rk4.o \ mpas_ocn_time_integration_si.o \ - mpas_ocn_time_integration_split.o - + mpas_ocn_time_integration_split.o \ + $(SMARTREDIS_PATH)/../src/fortran/fortran_c_interop.F90 \ + $(SMARTREDIS_PATH)/../src/fortran/dataset.F90 \ + $(SMARTREDIS_PATH)/../src/fortran/client.F90 clean: $(RM) *.o *.mod *.f90 diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index 39bf1033b8c9..d2babe49a1d5 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -232,6 +232,8 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ call mpas_log_write(log_string)!, & !MPAS_LOG_ERR, masterOnly=.true., flushNow=.true.) !ierr = ior(ierr, 1) + else + call mpas_log_write('Client initialized') endif endif !call MPAS_stream_mgr_begin_iteration(domain%streamManager, 'smartsend', ierr) From c2457bfd455e7b54b49f1158eaf9ff1e564f3b9f Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Wed, 13 Jul 2022 10:59:16 -0700 Subject: [PATCH 06/29] Add commented notes/code about smartredis comms --- .../src/mode_forward/mpas_ocn_forward_mode.F | 73 +++++++++++-------- 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index d2babe49a1d5..d020b5365a66 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -158,6 +158,7 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ real (kind=c_float), pointer :: & sendArrayReal0d ! pointer to float to send to orchestrator + real (kind=RKIND), dimension(:), pointer :: RossbyRadiusCell real (kind=c_float), dimension(:), pointer :: & sendArrayReal1d ! pointer to array to send to orchestrator @@ -236,35 +237,6 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ call mpas_log_write('Client initialized') endif endif - !call MPAS_stream_mgr_begin_iteration(domain%streamManager, 'smartsend', ierr) - !call MPAS_stream_mgr_get_next_field(domain%streamManager, 'smartsend', smartFieldName, isActive) - !if (isActive) then - ! call mpas_pool_get_field_info(domain%blocklist%allFields, smartFieldName, fieldInfo) - ! WRITE(log_string,*) smartFieldName, 'has type', fieldInfo%fieldType - ! call mpas_log_write(log_string) - ! if ( fieldInfo % fieldType == MPAS_POOL_REAL ) then - ! call mpas_pool_get_field(domain%blocklist%allFields, smartFieldName, sendArrayReal1d) - ! WRITE(log_string,*) smartFieldName, 'retrieved from stream', sendArrayReal1d - ! err_client = client%put_tensor("send_array", sendArrayReal1d, shape(sendArrayReal1d)) - ! if (err_client /= SRNoError) then - ! WRITE(log_string,*) 'Error put tensor:', smartFieldName - ! call mpas_log_write(log_string, MPAS_LOG_ERR, masterOnly=.true., flushNow=.true.) - ! ierr = ior(ierr, 1) - ! endif - ! endif - !else - ! WRITE(log_string,*) 'smartsend field', smartFieldName, 'is not active' - ! call mpas_log_write(log_string) - !endif - ! TODO add similar loop for stream smartrecv - ! err_client = client%unpack_tensor("send_array", recvArrayReal1d, shape(recvArrayReal1d)) - ! if (err_client /= SRNoError) then - ! WRITE(log_string,*) 'Error unpack tensor:', smartFieldName - ! call mpas_log_write(log_string, MPAS_LOG_ERR, masterOnly=.true., flushNow=.true.) - ! ierr = ior(ierr, 1) - ! endif - !call mpas_get_package(domain%packages,'smartPKG',smartPKG) - !----------------------------------------------------------------- ! Initialize data from external file input @@ -570,6 +542,49 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ MPAS_LOG_ERR, masterOnly=.true., flushNow=.true.) return endif + call mpas_timer_start('smartsend_init') + !Note: these commands will get the field and not field%array + !> call mpas_pool_get_field(domain%blocklist%allFields, RossbyRadius, RossbyRadius) + !> call mpas_pool_get_field(domain%blocklist%allFields, RossbyRadius, sendArrayReal1d) + !call mpas_pool_get_subpool(domain % blocklist % structs, 'diagnostics', diagnosticsPool) + !call mpas_pool_get_array(diagnosticsPool, 'RossbyRadiusCell', & + ! RossbyRadiusCell) + ! TODO determine whether RKIND arrays can be assigned into c_float arrays + ! sendArrayReal1d is a c_float array and RossbyRadius is a RKIND array + !WRITE(log_string,*) smartFieldName, 'retrieved from stream', sendArrayReald + !err_client = client%put_tensor("send_array", sendArrayReal1d, shape(sendArrayReal1d)) + + ! TODO LATER: Move variable names into a stream and ensure that the array dimensions of + ! the variable being sent to the database node match that of the source array + !call MPAS_stream_mgr_begin_iteration(domain%streamManager, 'smartsend', ierr) + !call MPAS_stream_mgr_get_next_field(domain%streamManager, 'smartsend', smartFieldName, isActive) + !if (isActive) then + ! call mpas_pool_get_field_info(domain%blocklist%allFields, smartFieldName, fieldInfo) + ! WRITE(log_string,*) smartFieldName, 'has type', fieldInfo%fieldType + ! call mpas_log_write(log_string) + ! if ( fieldInfo % fieldType == MPAS_POOL_REAL ) then + ! call mpas_pool_get_field(domain%blocklist%allFields, smartFieldName, sendArrayReal1d) + ! WRITE(log_string,*) smartFieldName, 'retrieved from stream', sendArrayReal1d + ! err_client = client%put_tensor("send_array", sendArrayReal1d, shape(sendArrayReal1d)) + ! if (err_client /= SRNoError) then + ! WRITE(log_string,*) 'Error put tensor:', smartFieldName + ! call mpas_log_write(log_string, MPAS_LOG_ERR, masterOnly=.true., flushNow=.true.) + ! ierr = ior(ierr, 1) + ! endif + ! endif + !else + ! WRITE(log_string,*) 'smartsend field', smartFieldName, 'is not active' + ! call mpas_log_write(log_string) + !endif + ! TODO add similar loop for stream smartrecv + ! err_client = client%unpack_tensor("send_array", recvArrayReal1d, shape(recvArrayReal1d)) + ! if (err_client /= SRNoError) then + ! WRITE(log_string,*) 'Error unpack tensor:', smartFieldName + ! call mpas_log_write(log_string, MPAS_LOG_ERR, masterOnly=.true., flushNow=.true.) + ! ierr = ior(ierr, 1) + ! endif + + call mpas_timer_stop('smartsend_init') !-------------------------------------------------------------------- From 1ec22c481fb6e76b7081d16eefecfd6b2cd44777 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Wed, 27 Jul 2022 10:39:56 -0700 Subject: [PATCH 07/29] Updates to forward mode data calls --- .../src/mode_forward/mpas_ocn_forward_mode.F | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index d020b5365a66..6f9aa1dfb455 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -543,19 +543,21 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ return endif call mpas_timer_start('smartsend_init') - !Note: these commands will get the field and not field%array - !> call mpas_pool_get_field(domain%blocklist%allFields, RossbyRadius, RossbyRadius) - !> call mpas_pool_get_field(domain%blocklist%allFields, RossbyRadius, sendArrayReal1d) - !call mpas_pool_get_subpool(domain % blocklist % structs, 'diagnostics', diagnosticsPool) - !call mpas_pool_get_array(diagnosticsPool, 'RossbyRadiusCell', & - ! RossbyRadiusCell) + + ! Since we use diagnostics_variables module, we do not need to fetch variables from pools + call mpas_log_write('Start smartsend for RossbyRadius') + err_client = client%put_tensor("send_array", RossbyRadius, shape(RossbyRadius)) + call mpas_log_write('End smartsend for RossbyRadius') + ! TODO determine whether RKIND arrays can be assigned into c_float arrays + ! - according to Andrew Shao, this should be unnecessary ! sendArrayReal1d is a c_float array and RossbyRadius is a RKIND array - !WRITE(log_string,*) smartFieldName, 'retrieved from stream', sendArrayReald - !err_client = client%put_tensor("send_array", sendArrayReal1d, shape(sendArrayReal1d)) + !Note: these commands will get the field and not field%array ! TODO LATER: Move variable names into a stream and ensure that the array dimensions of ! the variable being sent to the database node match that of the source array + ! Loop through variable names in stream and get from allFields rather + ! than hard-code variables in diagnosticsPool !call MPAS_stream_mgr_begin_iteration(domain%streamManager, 'smartsend', ierr) !call MPAS_stream_mgr_get_next_field(domain%streamManager, 'smartsend', smartFieldName, isActive) !if (isActive) then From 1f0d58221b1e23d284380639973d70acfdd48d87 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Thu, 25 Aug 2022 16:33:27 -0700 Subject: [PATCH 08/29] Test send/recv arrays at init --- .../src/mode_forward/mpas_ocn_forward_mode.F | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index 6f9aa1dfb455..92cb70686064 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -23,6 +23,7 @@ module ocn_forward_mode use mpas_kind_types use mpas_derived_types use mpas_pool_routines + #use mpas_client_routines use mpas_stream_manager use mpas_timekeeping use mpas_dmpar @@ -155,18 +156,11 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ maxDensity, &! local max of mesh density maxDensity_global ! global max of mesh density - real (kind=c_float), pointer :: & - sendArrayReal0d ! pointer to float to send to orchestrator - - real (kind=RKIND), dimension(:), pointer :: RossbyRadiusCell - real (kind=c_float), dimension(:), pointer :: & - sendArrayReal1d ! pointer to array to send to orchestrator - - real (kind=RKIND), pointer :: & - recvArrayReal0d ! pointer to float to recv from orchestrator - - real (kind=RKIND), dimension(:), pointer :: & - recvArrayReal1d ! pointer to array to recv from orchestrator + ! Arrays to send/receive from orchestrator + real (kind=RKIND) :: & + sendArrayReal0d, recvArrayReal0d + real (kind=RKIND), dimension(:), allocatable :: & + sendArrayReal1d, recvArrayReal1d real (kind=RKIND), dimension(:), pointer :: & meshDensity ! pointer to mesh density array @@ -545,9 +539,24 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ call mpas_timer_start('smartsend_init') ! Since we use diagnostics_variables module, we do not need to fetch variables from pools - call mpas_log_write('Start smartsend for RossbyRadius') - err_client = client%put_tensor("send_array", RossbyRadius, shape(RossbyRadius)) - call mpas_log_write('End smartsend for RossbyRadius') + call mpas_log_write('Start smartsend at init') + allocate(sendArrayReal1d(10)) + sendArrayReal1d(:) = 8.0_RKIND + err_client = client%put_tensor("init_array", dummy_array, shape(dummy_array)) + if (err_client /= SRNoError) then + write(log_string,*) 'client%put_tensor failed with err:',err_client + call mpas_log_write(log_string) + else + call mpas_log_write('client%put_tensor successful')!TODO test pointer + err_client = client%unpack_tensor("send_array", recv_array_real_64, shape(recv_array_real_64)) + if (err_client /= SRNoError) then + write(log_string,*) 'client%unpack_tensor failed with err:',err_client + call mpas_log_write(log_string) + else + call mpas_log_write('client%unpack_tensor successful') + endif + endif + call mpas_log_write('End smartsend for dummy_array') ! TODO determine whether RKIND arrays can be assigned into c_float arrays ! - according to Andrew Shao, this should be unnecessary From 9152af3c7f15ec7932b8c4505f51c5790e48b867 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Thu, 25 Aug 2022 16:34:35 -0700 Subject: [PATCH 09/29] Add other client io --- .../src/mode_forward/mpas_ocn_forward_mode.F | 89 +++++++++---------- 1 file changed, 43 insertions(+), 46 deletions(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index 92cb70686064..8ac7dc72b97a 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -542,58 +542,22 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ call mpas_log_write('Start smartsend at init') allocate(sendArrayReal1d(10)) sendArrayReal1d(:) = 8.0_RKIND - err_client = client%put_tensor("init_array", dummy_array, shape(dummy_array)) + err_client = client%put_tensor("init_send_array", sendArrayReal1d, shape(sendArrayReal1d)) if (err_client /= SRNoError) then write(log_string,*) 'client%put_tensor failed with err:',err_client call mpas_log_write(log_string) else call mpas_log_write('client%put_tensor successful')!TODO test pointer - err_client = client%unpack_tensor("send_array", recv_array_real_64, shape(recv_array_real_64)) - if (err_client /= SRNoError) then - write(log_string,*) 'client%unpack_tensor failed with err:',err_client - call mpas_log_write(log_string) - else - call mpas_log_write('client%unpack_tensor successful') - endif endif - call mpas_log_write('End smartsend for dummy_array') - - ! TODO determine whether RKIND arrays can be assigned into c_float arrays - ! - according to Andrew Shao, this should be unnecessary - ! sendArrayReal1d is a c_float array and RossbyRadius is a RKIND array - - !Note: these commands will get the field and not field%array - ! TODO LATER: Move variable names into a stream and ensure that the array dimensions of - ! the variable being sent to the database node match that of the source array - ! Loop through variable names in stream and get from allFields rather - ! than hard-code variables in diagnosticsPool - !call MPAS_stream_mgr_begin_iteration(domain%streamManager, 'smartsend', ierr) - !call MPAS_stream_mgr_get_next_field(domain%streamManager, 'smartsend', smartFieldName, isActive) - !if (isActive) then - ! call mpas_pool_get_field_info(domain%blocklist%allFields, smartFieldName, fieldInfo) - ! WRITE(log_string,*) smartFieldName, 'has type', fieldInfo%fieldType - ! call mpas_log_write(log_string) - ! if ( fieldInfo % fieldType == MPAS_POOL_REAL ) then - ! call mpas_pool_get_field(domain%blocklist%allFields, smartFieldName, sendArrayReal1d) - ! WRITE(log_string,*) smartFieldName, 'retrieved from stream', sendArrayReal1d - ! err_client = client%put_tensor("send_array", sendArrayReal1d, shape(sendArrayReal1d)) - ! if (err_client /= SRNoError) then - ! WRITE(log_string,*) 'Error put tensor:', smartFieldName - ! call mpas_log_write(log_string, MPAS_LOG_ERR, masterOnly=.true., flushNow=.true.) - ! ierr = ior(ierr, 1) - ! endif - ! endif - !else - ! WRITE(log_string,*) 'smartsend field', smartFieldName, 'is not active' - ! call mpas_log_write(log_string) - !endif - ! TODO add similar loop for stream smartrecv - ! err_client = client%unpack_tensor("send_array", recvArrayReal1d, shape(recvArrayReal1d)) - ! if (err_client /= SRNoError) then - ! WRITE(log_string,*) 'Error unpack tensor:', smartFieldName - ! call mpas_log_write(log_string, MPAS_LOG_ERR, masterOnly=.true., flushNow=.true.) - ! ierr = ior(ierr, 1) - ! endif + err_client = client%unpack_tensor("init_recv_array", recvArrayReal0d, shape(recvArrayReal0d)) + if (err_client /= SRNoError) then + write(log_string,*) 'client%unpack_tensor failed with err:',err_client + call mpas_log_write(log_string) + else + write(log_string,*) 'client%unpack_tensor succeeded:',recvArrayReal0d + call mpas_log_write(log_string) + endif + call mpas_log_write('End smartsend at init') call mpas_timer_stop('smartsend_init') @@ -869,6 +833,39 @@ function ocn_forward_mode_run(domain) result(ierr)!{{{ call mpas_stream_mgr_write(domain % streamManager, streamID='restart', ierr=ierr) call mpas_timer_stop('io_write') + !Note: these commands will get the field and not field%array + ! TODO LATER: Move variable names into a stream and ensure that the array dimensions of + ! the variable being sent to the database node match that of the source array + ! Loop through variable names in stream and get from allFields rather + ! than hard-code variables in diagnosticsPool + !call MPAS_stream_mgr_begin_iteration(domain%streamManager, 'smartsend', ierr) + !call MPAS_stream_mgr_get_next_field(domain%streamManager, 'smartsend', smartFieldName, isActive) + !if (isActive) then + ! call mpas_pool_get_field_info(domain%blocklist%allFields, smartFieldName, fieldInfo) + ! WRITE(log_string,*) smartFieldName, 'has type', fieldInfo%fieldType + ! call mpas_log_write(log_string) + ! if ( fieldInfo % fieldType == MPAS_POOL_REAL ) then + ! call mpas_pool_get_field(domain%blocklist%allFields, smartFieldName, sendArrayReal1d) + ! WRITE(log_string,*) smartFieldName, 'retrieved from stream', sendArrayReal1d + ! err_client = client%put_tensor("send_array", sendArrayReal1d, shape(sendArrayReal1d)) + ! if (err_client /= SRNoError) then + ! WRITE(log_string,*) 'Error put tensor:', smartFieldName + ! call mpas_log_write(log_string, MPAS_LOG_ERR, masterOnly=.true., flushNow=.true.) + ! ierr = ior(ierr, 1) + ! endif + ! endif + !else + ! WRITE(log_string,*) 'smartsend field', smartFieldName, 'is not active' + ! call mpas_log_write(log_string) + !endif + ! TODO add similar loop for stream smartrecv + ! err_client = client%unpack_tensor("send_array", recvArrayReal1d, shape(recvArrayReal1d)) + ! if (err_client /= SRNoError) then + ! WRITE(log_string,*) 'Error unpack tensor:', smartFieldName + ! call mpas_log_write(log_string, MPAS_LOG_ERR, masterOnly=.true., flushNow=.true.) + ! ierr = ior(ierr, 1) + ! endif + call mpas_timer_start('reset_io_alarms') call mpas_stream_mgr_reset_alarms(domain % streamManager, streamID='restart', ierr=ierr) From ad7fa9720b143d6856e9eb2e8d21a3e85eeb5048 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Thu, 25 Aug 2022 16:53:53 -0700 Subject: [PATCH 10/29] Run script on client --- .../src/mode_forward/mpas_ocn_forward_mode.F | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index 8ac7dc72b97a..611caa52eb1c 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -23,7 +23,7 @@ module ocn_forward_mode use mpas_kind_types use mpas_derived_types use mpas_pool_routines - #use mpas_client_routines + !use mpas_client_routines use mpas_stream_manager use mpas_timekeeping use mpas_dmpar @@ -150,6 +150,8 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ integer :: err_tmp ! local error codes from subroutine calls integer(kind=enum_kind) :: err_client character(len=100) :: log_string + character(len=*), parameter :: script_key = 'test_client_recv' + character(len=*), parameter :: script_file = '../../test_client_recv.py' real (kind=RKIND) :: & dt, &! time step in seconds @@ -230,6 +232,20 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ else call mpas_log_write('Client initialized') endif + err_client = client%set_script_from_file(script_key, "CPU", script_file) + if (err_client /= SRNoError) then + write(log_string,*) 'Error setting client script:',err_client + call mpas_log_write(log_string) + else + call mpas_log_write('Client script set') + endif + err_client = client%run_script(script_key, "init_send", inputs, outputs) + if (err_client /= SRNoError) then + write(log_string,*) 'Error running client script:',err_client + call mpas_log_write(log_string) + else + call mpas_log_write('Client script run') + endif endif !----------------------------------------------------------------- From ea20d130f0e9b83658b2e4f8e1465244e5fe05a3 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Thu, 25 Aug 2022 16:54:20 -0700 Subject: [PATCH 11/29] Add python script with different function defs --- components/mpas-ocean/test_client_recv.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 components/mpas-ocean/test_client_recv.py diff --git a/components/mpas-ocean/test_client_recv.py b/components/mpas-ocean/test_client_recv.py new file mode 100644 index 000000000000..caeda9339b80 --- /dev/null +++ b/components/mpas-ocean/test_client_recv.py @@ -0,0 +1,15 @@ +from smartredis import Client +from smartsim import Experiment +from smartsim.database import Orchestrator +from smartsim.log import get_logger, log_to_file +import time +import os + +client = Client(address="10.128.4.24:6379", cluster=False) +key_found = client.poll_key("send_array", 200, 10000) +print('key_found',key_found) +if key_found: + dummy_array = client.get_tensor("send_array") + print('dummy_array=',dummy_array) +else: + print('key not found') From 34b426f78fcdcda0af7fc7b3b1a2aa8118846a22 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 26 Aug 2022 14:04:10 -0700 Subject: [PATCH 12/29] Run analysis script from mpas-o --- .../mode_forward/data_processing_script.txt | 20 +++++++++++++++++++ .../src/mode_forward/test_client_recv.txt | 2 ++ components/mpas-ocean/test_client_recv.py | 19 ++++-------------- components/mpas-ocean/test_client_recv.txt | 3 +++ 4 files changed, 29 insertions(+), 15 deletions(-) create mode 100644 components/mpas-ocean/src/mode_forward/data_processing_script.txt create mode 100644 components/mpas-ocean/src/mode_forward/test_client_recv.txt create mode 100644 components/mpas-ocean/test_client_recv.txt diff --git a/components/mpas-ocean/src/mode_forward/data_processing_script.txt b/components/mpas-ocean/src/mode_forward/data_processing_script.txt new file mode 100644 index 000000000000..6ea6861df7f7 --- /dev/null +++ b/components/mpas-ocean/src/mode_forward/data_processing_script.txt @@ -0,0 +1,20 @@ + +def pre_process_3ch(image): + mean = torch.zeros(1).float().to(image.device) + std = torch.zeros(1).float().to(image.device) + mean[0] = 0.1307 + std[0] = 0.3081 + mean = mean.unsqueeze(1).unsqueeze(1) + std = std.unsqueeze(1).unsqueeze(1) + temp = image.float().div(28).permute(1, 0) + return temp.sub(mean).div(std).unsqueeze(0) + +def post_process(output): + return output.max(1)[1] + + +def pre_process(inp): + mean = torch.zeros(1).float().to(inp.device) + mean[0] = 2.0 + temp = inp.float() * mean + return temp diff --git a/components/mpas-ocean/src/mode_forward/test_client_recv.txt b/components/mpas-ocean/src/mode_forward/test_client_recv.txt new file mode 100644 index 000000000000..57a79087918b --- /dev/null +++ b/components/mpas-ocean/src/mode_forward/test_client_recv.txt @@ -0,0 +1,2 @@ +def init_send(inp): + return [8.0] diff --git a/components/mpas-ocean/test_client_recv.py b/components/mpas-ocean/test_client_recv.py index caeda9339b80..dca6bf080b78 100644 --- a/components/mpas-ocean/test_client_recv.py +++ b/components/mpas-ocean/test_client_recv.py @@ -1,15 +1,4 @@ -from smartredis import Client -from smartsim import Experiment -from smartsim.database import Orchestrator -from smartsim.log import get_logger, log_to_file -import time -import os - -client = Client(address="10.128.4.24:6379", cluster=False) -key_found = client.poll_key("send_array", 200, 10000) -print('key_found',key_found) -if key_found: - dummy_array = client.get_tensor("send_array") - print('dummy_array=',dummy_array) -else: - print('key not found') +def init_send(inputs): + #temp = inputs.float() * 2.0 + temp = [2.0] + return temp diff --git a/components/mpas-ocean/test_client_recv.txt b/components/mpas-ocean/test_client_recv.txt new file mode 100644 index 000000000000..00e8b54e21b2 --- /dev/null +++ b/components/mpas-ocean/test_client_recv.txt @@ -0,0 +1,3 @@ + +def init_send(inp): + return [8.0] From f0473aaade3a21e1bad8e49451ccfc14a1969948 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 26 Aug 2022 14:09:59 -0700 Subject: [PATCH 13/29] Add db launch script --- components/mpas-ocean/launch_db_for_mpaso.py | 49 ++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 components/mpas-ocean/launch_db_for_mpaso.py diff --git a/components/mpas-ocean/launch_db_for_mpaso.py b/components/mpas-ocean/launch_db_for_mpaso.py new file mode 100644 index 000000000000..1246be07df1c --- /dev/null +++ b/components/mpas-ocean/launch_db_for_mpaso.py @@ -0,0 +1,49 @@ +from smartredis import Client +from smartsim import Experiment +from smartsim.database import Orchestrator +from smartsim.log import get_logger, log_to_file +import time +import os + +MPASO_DIR="/global/cscratch1/sd/cbegeman/MPAS-Ocean-test-case-output/smartsim/ocean/baroclinic_channel/10km/default" + +print('Create Experiment object') +exp = Experiment("mpas-ocean_simulation", launcher="slurm") +print('Create database') +port = 6379 +db = exp.create_database(db_nodes=1, + batch=True, + port=port, + time="00:30:00", + account="e3sm", + partition="standard", + batch_args={"C":"haswell"}) + +if os.path.exists('{}/db_debug.log'.format(MPASO_DIR)): + os.system('rm {}/db_debug.log'.format(MPASO_DIR)) +log_to_file('{}/db_debug.log'.format(MPASO_DIR)) +logger = get_logger('db_launcher') + +# define how simulation should be executed +print('Set run settings') +exp.generate(db, overwrite=True) +print('Start experiment') +exp.start(db) + +os.environ['SSDB'] = f"{db.hosts[0]}:{port}" +logger.debug('SSDB={}'.format(os.environ.get('SSDB'))) +print(f"{db.hosts[0]}:{port}") + +print(exp.get_status(db)) + +time.sleep(25*60) +#client = Client(address=f"{db.hosts[0]}:{port}", cluster=False) +#print('client initialized') +#key_found = client.poll_key("send_array", 200, 10000) +#print('key_found',key_found) +#if key_found: +# dummy_array = client.get_tensor("send_array").astype(real) +# print('dummy_array=',dummy_array) +#else: +# print('key not found') +exp.stop(db) From ec0401753102ab84c0dda08ca844e83a51aa2c5b Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 26 Aug 2022 14:10:30 -0700 Subject: [PATCH 14/29] Add scripts to sync db with mpas-o --- components/mpas-ocean/environ.sh | 7 ++++++ components/mpas-ocean/get_ssdb.py | 14 +++++++++++ components/mpas-ocean/mpas_client_script.py | 25 +++++++++++++++++++ components/mpas-ocean/run_mpaso_and_script.sh | 15 +++++++++++ 4 files changed, 61 insertions(+) create mode 100644 components/mpas-ocean/environ.sh create mode 100644 components/mpas-ocean/get_ssdb.py create mode 100644 components/mpas-ocean/mpas_client_script.py create mode 100644 components/mpas-ocean/run_mpaso_and_script.sh diff --git a/components/mpas-ocean/environ.sh b/components/mpas-ocean/environ.sh new file mode 100644 index 000000000000..9ae7a8c23595 --- /dev/null +++ b/components/mpas-ocean/environ.sh @@ -0,0 +1,7 @@ +export SMARTREDIS_PATH=/global/cscratch1/sd/$USER/smartredis-v0.3.0/install/lib +export LD_LIBRARY_PATH=${SMARTREDIS_PATH}:$LD_LIBRARY_PATH +export SMARTREDIS_DEBUG_LEVEL=VERBOSE +export SSDB=None + + +source load_compass_env.sh diff --git a/components/mpas-ocean/get_ssdb.py b/components/mpas-ocean/get_ssdb.py new file mode 100644 index 000000000000..cfd4a5e7f4ca --- /dev/null +++ b/components/mpas-ocean/get_ssdb.py @@ -0,0 +1,14 @@ +import shutil, time, sys, os, subprocess, fileinput + +with open('db_debug.log') as f: + lines = f.readlines() + for line in lines: + if 'SSDB' in line: + SSDB = line.split('=')[1] +os.environ['SSDB'] = SSDB +print(f'SSDB={os.environ.get("SSDB")}') + +with fileinput.FileInput(f'environ.sh', inplace=True, + backup='.bak') as f: + for line in f: + print(line.replace('export SSDB=None', f'export SSDB={SSDB}'), end='') diff --git a/components/mpas-ocean/mpas_client_script.py b/components/mpas-ocean/mpas_client_script.py new file mode 100644 index 000000000000..888e8f221b38 --- /dev/null +++ b/components/mpas-ocean/mpas_client_script.py @@ -0,0 +1,25 @@ +from smartredis import Client +from smartsim import Experiment +from smartsim.database import Orchestrator +from smartsim.log import get_logger, log_to_file +import time +import os +import numpy + +#with open('db_debug.log') as f: +# lines = f.readlines() +# for line in lines: +# if 'SSDB' in line: +# SSDB = line.split('=')[1] +SSDB=os.environ.get("SSDB") +print(f'SSDB in client script {SSDB}') +client = Client(address=SSDB, cluster=False) +recv_array = 20*numpy.ones(1) +err = client.put_tensor("init_recv", recv_array) +key_found = client.poll_key("init_send", 200, 10000) +print('key_found',key_found) +if key_found: + dummy_array = client.get_tensor("init_send") + print('init_send_array=',dummy_array[0]) +else: + print('key not found') diff --git a/components/mpas-ocean/run_mpaso_and_script.sh b/components/mpas-ocean/run_mpaso_and_script.sh new file mode 100644 index 000000000000..8ce98d08955e --- /dev/null +++ b/components/mpas-ocean/run_mpaso_and_script.sh @@ -0,0 +1,15 @@ +CASE_DIR=/global/cscratch1/sd/cbegeman/MPAS-Ocean-test-case-output/smartsim/ocean/baroclinic_channel/10km/default +MODEL_DIR=/global/homes/c/cbegeman/E3SM-new/components/mpas-ocean +conda activate smartsim +cp $MODEL_DIR/environ.sh $CASE_DIR/. +cd $CASE_DIR +python $MODEL_DIR/get_ssdb.py +echo "completed get_ssdb" +source $CASE_DIR/environ.sh +echo $SSDB +echo "running compass" +compass run & + +echo "running client script" +conda activate smartsim +python $MODEL_DIR/mpas_client_script.py & From 8d8b2be714704b08a4de1725bc4269769d6d3549 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 29 Aug 2022 11:33:34 -0700 Subject: [PATCH 15/29] Test namelist change in forward mode --- .../src/mode_forward/mpas_ocn_forward_mode.F | 103 ++++++++++++------ 1 file changed, 69 insertions(+), 34 deletions(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index 611caa52eb1c..33275882500a 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -23,7 +23,6 @@ module ocn_forward_mode use mpas_kind_types use mpas_derived_types use mpas_pool_routines - !use mpas_client_routines use mpas_stream_manager use mpas_timekeeping use mpas_dmpar @@ -150,8 +149,11 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ integer :: err_tmp ! local error codes from subroutine calls integer(kind=enum_kind) :: err_client character(len=100) :: log_string - character(len=*), parameter :: script_key = 'test_client_recv' - character(len=*), parameter :: script_file = '../../test_client_recv.py' + character(len=*), parameter :: script_key = 'init_script' + character(len=*), parameter :: script_file = '/global/homes/c/cbegeman/E3SM-new/components/mpas-ocean/src/mode_forward/test_client_recv.txt' + character(len=255) :: in_key, script_out_key, out_key + + character(len=255), dimension(1) :: inputs, outputs real (kind=RKIND) :: & dt, &! time step in seconds @@ -188,6 +190,7 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ character (len=StrKIND) :: units type(client_type) :: client + logical :: is_error ! End preamble !------------- @@ -232,21 +235,52 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ else call mpas_log_write('Client initialized') endif - err_client = client%set_script_from_file(script_key, "CPU", script_file) + + write(log_string,*) 'Before client: config_mom_del2', config_mom_del2 + call mpas_log_write(log_string) + in_key = 'init_send' + script_out_key = 'init_recv' + out_key = 'init_recv' + inputs(1) = in_key + outputs(1) = script_out_key + + allocate(sendArrayReal1d(1), recvArrayReal1d(1)) + sendArrayReal1d(:) = 2.0_RKIND + recvArrayReal1d(:) = 0.0_RKIND + err_client = client%put_tensor(in_key, sendArrayReal1d, shape(sendArrayReal1d)) if (err_client /= SRNoError) then - write(log_string,*) 'Error setting client script:',err_client + write(log_string,*) 'Error put tensor:',err_client call mpas_log_write(log_string) else - call mpas_log_write('Client script set') + call mpas_log_write('Put tensor') endif - err_client = client%run_script(script_key, "init_send", inputs, outputs) + !err_client = client%set_script_from_file(script_key, "CPU", script_file) + !if (err_client /= SRNoError) then + ! write(log_string,*) 'Error setting client script:',err_client + ! call mpas_log_write(log_string) + !else + ! call mpas_log_write('Client script set') + !endif + !err_client = client%run_script(script_key, "pre_process", inputs, outputs) + !if (err_client /= SRNoError) then + ! write(log_string,*) 'Error running client script:',err_client + ! call mpas_log_write(log_string) + ! is_error = client%SR_error_parser(err_client) + !else + !call mpas_log_write('Client script run') + err_client = client%unpack_tensor(outputs(1), recvArrayReal1d, shape(recvArrayReal1d)) if (err_client /= SRNoError) then - write(log_string,*) 'Error running client script:',err_client + write(log_string,*) 'Error unpacking tensor init_recv:',err_client call mpas_log_write(log_string) else - call mpas_log_write('Client script run') + write(log_string,*) 'init_recv:',recvArrayReal1d(1) + call mpas_log_write(log_string) endif + !endif endif + config_mom_del2 = recvArrayReal1d(1) + write(log_string,*) 'After client: config_mom_del2', config_mom_del2 + call mpas_log_write(log_string) !----------------------------------------------------------------- ! Initialize data from external file input @@ -552,30 +586,28 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ MPAS_LOG_ERR, masterOnly=.true., flushNow=.true.) return endif - call mpas_timer_start('smartsend_init') - - ! Since we use diagnostics_variables module, we do not need to fetch variables from pools - call mpas_log_write('Start smartsend at init') - allocate(sendArrayReal1d(10)) - sendArrayReal1d(:) = 8.0_RKIND - err_client = client%put_tensor("init_send_array", sendArrayReal1d, shape(sendArrayReal1d)) - if (err_client /= SRNoError) then - write(log_string,*) 'client%put_tensor failed with err:',err_client - call mpas_log_write(log_string) - else - call mpas_log_write('client%put_tensor successful')!TODO test pointer - endif - err_client = client%unpack_tensor("init_recv_array", recvArrayReal0d, shape(recvArrayReal0d)) - if (err_client /= SRNoError) then - write(log_string,*) 'client%unpack_tensor failed with err:',err_client - call mpas_log_write(log_string) - else - write(log_string,*) 'client%unpack_tensor succeeded:',recvArrayReal0d - call mpas_log_write(log_string) - endif - call mpas_log_write('End smartsend at init') - - call mpas_timer_stop('smartsend_init') + !call mpas_timer_start('smartsend_init') + + !! Since we use diagnostics_variables module, we do not need to fetch variables from pools + !call mpas_log_write('Start smartsend at init') + !err_client = client%put_tensor("init_send_array", sendArrayReal1d, shape(sendArrayReal1d)) + !if (err_client /= SRNoError) then + ! write(log_string,*) 'client%put_tensor failed with err:',err_client + ! call mpas_log_write(log_string) + !else + ! call mpas_log_write('client%put_tensor successful')!TODO test pointer + !endif + !err_client = client%unpack_tensor("init_recv_array", recvArrayReal0d, shape(recvArrayReal0d)) + !if (err_client /= SRNoError) then + ! write(log_string,*) 'client%unpack_tensor failed with err:',err_client + ! call mpas_log_write(log_string) + !else + ! write(log_string,*) 'client%unpack_tensor succeeded:',recvArrayReal0d + ! call mpas_log_write(log_string) + !endif + !call mpas_log_write('End smartsend at init') + + !call mpas_timer_stop('smartsend_init') !-------------------------------------------------------------------- @@ -948,9 +980,12 @@ end function ocn_forward_mode_run!}}} function ocn_forward_mode_finalize(domain) result(iErr)!{{{ type (domain_type), intent(inout) :: domain - + character(len=100) :: log_string integer :: ierr + write(log_string,*) 'Finalize: config_mom_del2', config_mom_del2 + call mpas_log_write(log_string) + ! destroy the ocean mesh structure call ocn_meshDestroy(ierr) From 686a595643c6e9cef2e77e42f317b60a33599475 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 29 Aug 2022 11:34:25 -0700 Subject: [PATCH 16/29] Test batch client (nodes>=3) --- components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index 33275882500a..b4864eb5cb8c 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -226,7 +226,7 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ if (len(TRIM(log_string)) < 5) then call mpas_log_write('No SSDB')!, & else - err_client = client%initialize(.false.) + err_client = client%initialize(.true.) if (err_client /= SRNoError) then write(log_string,*) 'Error initializing redis client:',err_client call mpas_log_write(log_string)!, & From 5d9c285023063b41e5f185fd224e70da2ec868ae Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Tue, 30 Aug 2022 10:25:57 -0700 Subject: [PATCH 17/29] Add dataset to mpas_forward_mode --- .../src/mode_forward/mpas_ocn_forward_mode.F | 75 +++++++++++++++++-- 1 file changed, 69 insertions(+), 6 deletions(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index b4864eb5cb8c..72c1b03d349c 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -30,6 +30,7 @@ module ocn_forward_mode use mpas_log use mpas_decomp use mpas_log +! use mpas_smart_routines use ocn_analysis_driver use ocn_init_routines @@ -96,6 +97,7 @@ module ocn_forward_mode use ocn_config use smartredis_client, only: client_type + use smartredis_dataset, only: dataset_type implicit none private @@ -150,7 +152,8 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ integer(kind=enum_kind) :: err_client character(len=100) :: log_string character(len=*), parameter :: script_key = 'init_script' - character(len=*), parameter :: script_file = '/global/homes/c/cbegeman/E3SM-new/components/mpas-ocean/src/mode_forward/test_client_recv.txt' + character(len=*), parameter :: script_file = & + 'test_client_recv.txt' character(len=255) :: in_key, script_out_key, out_key character(len=255), dimension(1) :: inputs, outputs @@ -165,7 +168,9 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ sendArrayReal0d, recvArrayReal0d real (kind=RKIND), dimension(:), allocatable :: & sendArrayReal1d, recvArrayReal1d - + real (kind=RKIND), dimension(:,:), allocatable :: & + sendArrayReal2d, recvArrayReal2d + real (kind=RKIND) :: meta_scalar real (kind=RKIND), dimension(:), pointer :: & meshDensity ! pointer to mesh density array @@ -190,6 +195,7 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ character (len=StrKIND) :: units type(client_type) :: client + type(dataset_type) :: dataset logical :: is_error ! End preamble @@ -226,7 +232,7 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ if (len(TRIM(log_string)) < 5) then call mpas_log_write('No SSDB')!, & else - err_client = client%initialize(.true.) + err_client = client%initialize(.false.) if (err_client /= SRNoError) then write(log_string,*) 'Error initializing redis client:',err_client call mpas_log_write(log_string)!, & @@ -245,15 +251,20 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ outputs(1) = script_out_key allocate(sendArrayReal1d(1), recvArrayReal1d(1)) + allocate(sendArrayReal2d(1,1), recvArrayReal2d(1,1)) sendArrayReal1d(:) = 2.0_RKIND recvArrayReal1d(:) = 0.0_RKIND + sendArrayReal2d(1,1) = 2.0_RKIND + recvArrayReal2d(1,1) = 0.0_RKIND err_client = client%put_tensor(in_key, sendArrayReal1d, shape(sendArrayReal1d)) if (err_client /= SRNoError) then write(log_string,*) 'Error put tensor:',err_client call mpas_log_write(log_string) + is_error = client%SR_error_parser(err_client) else call mpas_log_write('Put tensor') endif + !err_client = client%set_script_from_file(script_key, "CPU", script_file) !if (err_client /= SRNoError) then ! write(log_string,*) 'Error setting client script:',err_client @@ -272,15 +283,67 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ if (err_client /= SRNoError) then write(log_string,*) 'Error unpacking tensor init_recv:',err_client call mpas_log_write(log_string) + is_error = client%SR_error_parser(err_client) else write(log_string,*) 'init_recv:',recvArrayReal1d(1) call mpas_log_write(log_string) + config_mom_del2 = recvArrayReal1d(1) + write(log_string,*) 'After client: config_mom_del2', config_mom_del2 + call mpas_log_write(log_string) + endif + !endif + + err_client = dataset%initialize("example_fortran_dataset") + if (err_client .ne. SRNoError) then + call mpas_log_write('dataset initialization failed') + else + call mpas_log_write('Dataset initialized') + endif + + ! Add a tensor to the dataset and verify that we can retrieve it + err_client = dataset%add_tensor("dataset_send", sendArrayReal2d, & + shape(sendArrayReal2d)) + if (err_client .ne. SRNoError) then + call mpas_log_write('dataset%add_tensor failed') + else + call mpas_log_write('Dataset tensor added') + endif + err_client = dataset%unpack_dataset_tensor( & + "dataset_send",recvArrayReal2d, shape(recvArrayReal2d)) + if (err_client .ne. SRNoError) then + write(log_string,*) 'dataset%unpack_dataset_tensor failed:',err_client + call mpas_log_write(log_string) + else + write(log_string,*) 'Dataset tensor received:',recvArrayReal2d(1,1) + call mpas_log_write(log_string) endif + + ! Add metascalars to the dataset and verify that we can retrieve them + err_client = dataset%add_meta_scalar('config_mom_del2', config_mom_del2) + if (err_client .ne. SRNoError) then + write(log_string,*) 'dataset%add_meta_scalar failed:',err_client + call mpas_log_write(log_string) + else + call mpas_log_write('Dataset metascalar added') + endif + + !err_client = dataset%get_meta_scalars('config_mom_del2', meta_scalar) + !if (err_client .ne. SRNoError) then + ! write(log_string,*) 'dataset%get_meta_scalar failed:',err_client + ! call mpas_log_write(log_string) + !else + ! write(log_string,*) 'Dataset metascalar received:',meta_scalar + ! call mpas_log_write(log_string) !endif + err_client = client%put_dataset(dataset) + if (err_client .ne. SRNoError) then + write(log_string,*) 'client%put_dataset failed:',err_client + call mpas_log_write(log_string) + else + call mpas_log_write('Dataset put') + endif + endif - config_mom_del2 = recvArrayReal1d(1) - write(log_string,*) 'After client: config_mom_del2', config_mom_del2 - call mpas_log_write(log_string) !----------------------------------------------------------------- ! Initialize data from external file input From 8dd8c7a64d4b87f0cdeb52b2d49d288efad35107 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Tue, 30 Aug 2022 12:15:30 -0700 Subject: [PATCH 18/29] Fixup mpas_client_script --- components/mpas-ocean/mpas_client_script.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/mpas-ocean/mpas_client_script.py b/components/mpas-ocean/mpas_client_script.py index 888e8f221b38..bcd02fb458f9 100644 --- a/components/mpas-ocean/mpas_client_script.py +++ b/components/mpas-ocean/mpas_client_script.py @@ -23,3 +23,9 @@ print('init_send_array=',dummy_array[0]) else: print('key not found') + +mpas_dataset = client.get_dataset('example_fortran_dataset') +mpas_tensor = mpas_dataset.get_tensor('dataset_send') +print(f'dataset_send tensor={mpas_tensor[0,0]}') +config_mom_del2 = mpas_dataset.get_meta_scalars('config_mom_del2') +print(f'dataset config_mom_del2={config_mom_del2}') From de1de4304f7b85bd94a08f5578477b062461b76f Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Tue, 30 Aug 2022 10:26:54 -0700 Subject: [PATCH 19/29] Migrate routines to mpas-framework --- components/mpas-framework/Makefile | 15 +- .../mpas-framework/src/framework/Makefile | 6 + .../src/framework/mpas_smart_routines.F | 143 ++++++++++++++++++ .../src/mode_forward/mpas_ocn_forward_mode.F | 60 +++----- 4 files changed, 181 insertions(+), 43 deletions(-) create mode 100644 components/mpas-framework/src/framework/mpas_smart_routines.F diff --git a/components/mpas-framework/Makefile b/components/mpas-framework/Makefile index 9e9a892d7c24..050242d2071a 100644 --- a/components/mpas-framework/Makefile +++ b/components/mpas-framework/Makefile @@ -606,13 +606,14 @@ endif LIBS += -L$(PNETCDF)/$(PNETCDFLIBLOC) -lpnetcdf endif -#ifeq "$(USE_SMARTSIM)" "true" -#override LIBS += -L$(SMARTREDIS_PATH)/lib -#override LIBS += -lhiredis.a -#override LIBS += -lredis++.a -#override LIBS += -lsmartredis.so -#override LIBS += -lsmartredis-fortran.so -#endif +ifeq "$(USE_SMARTSIM)" "true" + CPPINCLUDES += -I$(SMARTREDIS_PATH)/include + FCINCLUDES += -I$(SMARTREDIS_PATH)/include + LIBS += -L$(SMARTREDIS_PATH)/lib + LIBS += -lhiredis.a + LIBS += -lredis++.a + LIBS += -lsmartredis-fortran.a +endif ifeq "$(USE_LAPACK)" "true" ifndef LAPACK diff --git a/components/mpas-framework/src/framework/Makefile b/components/mpas-framework/src/framework/Makefile index 96976d054ff3..d71a7737e4c1 100644 --- a/components/mpas-framework/src/framework/Makefile +++ b/components/mpas-framework/src/framework/Makefile @@ -32,6 +32,7 @@ OBJS = mpas_kind_types.o \ mpas_domain_routines.o \ mpas_field_routines.o \ mpas_pool_routines.o \ + mpas_smart_routines.o \ xml_stream_parser.o \ regex_matching.o \ mpas_field_accessor.o \ @@ -110,6 +111,11 @@ mpas_forcing.o: mpas_derived_types.o mpas_timekeeping.o mpas_stream_manager.o mp mpas_c_interfacing.o: +mpas_smart_routines.o: mpas_derived_types.o mpas_kind_types.o mpas_pool_routines.o mpas_field_routines.o mpas_threading.o mpas_log.o \ + $(SMARTREDIS_PATH)/../src/fortran/fortran_c_interop.F90 \ + $(SMARTREDIS_PATH)/../src/fortran/dataset.F90 \ + $(SMARTREDIS_PATH)/../src/fortran/client.F90 + xml_stream_parser.o: xml_stream_parser.c $(CC) $(CFLAGS) $(CPPFLAGS) $(CPPINCLUDES) -I../external/ezxml -c xml_stream_parser.c diff --git a/components/mpas-framework/src/framework/mpas_smart_routines.F b/components/mpas-framework/src/framework/mpas_smart_routines.F new file mode 100644 index 000000000000..9d635a530b55 --- /dev/null +++ b/components/mpas-framework/src/framework/mpas_smart_routines.F @@ -0,0 +1,143 @@ +! Copyright (c) 2013, Los Alamos National Security, LLC (LANS) +! and the University Corporation for Atmospheric Research (UCAR). +! +! Unless noted otherwise source code is licensed under the BSD license. +! Additional copyright and license information can be found in the LICENSE file +! distributed with this code, or at http://mpas-dev.github.com/license.html +! +module mpas_smart_routines + +#define COMMA , +#define STREAM_DEBUG_WRITE(M) call mpas_log_write(M) + +#ifdef SINGLE_PRECISION +#define REAL_IO_TYPE MPAS_IO_REAL +#else +#define REAL_IO_TYPE MPAS_IO_DOUBLE +#endif + + use mpas_kind_types + use mpas_derived_types + use mpas_pool_routines + use mpas_field_routines + use mpas_threading + use mpas_log + use smartredis_client, only: client_type + + implicit none + + public :: mpas_smart_init + +# include "enum_fortran.inc" + + contains + + subroutine mpas_smart_init(client, ierr) + + implicit none + + type(client_type), intent(out) :: client + integer, intent(out), optional :: ierr + + character(len=100) :: log_string + integer(kind=enum_kind) :: err_client + + call get_environment_variable('SSDB',log_string) + call mpas_log_write('SSDB='//TRIM(log_string)) + if (len(TRIM(log_string)) < 5) then + call mpas_log_write('FWK: No SSDB')!, & + else + err_client = client%initialize(.false.) + if (err_client /= SRNoError) then + write(log_string,*) 'FWK: Error initializing redis client:',err_client + call mpas_log_write(log_string) + ierr = ior(ierr, 1) + else + call mpas_log_write('FWK: Client initialized') + endif + endif + + end subroutine mpas_smart_init + + !subroutine mpas_smart_stream_send(client, stream, ierr) + + ! implicit none + + ! type(client_type), intent(in) :: client + ! !type (MPAS_streamManager_type), intent(inout) :: manager + ! !character (len=*), intent(in), optional :: streamID + ! type (MPAS_Stream_type), intent(in) :: stream + ! !type (MPAS_stream_list_type), pointer :: stream_cursor + ! integer, intent(out), optional :: ierr + + ! type (field_list_type), pointer :: field_cursor + ! type (field1dReal), pointer :: field_1d_ptr + ! !call MPAS_stream_list_query(manager % streams, streamID, stream_cursor, ierr=ierr)) + ! ! + ! ! Loop over fields in the stream + ! ! + ! field_cursor => stream % fieldList + ! do while (associated(field_cursor)) + ! if (field_cursor % field_type == FIELD_1D_REAL) then + ! field_1d_ptr = field_cursor % real1dField + ! call mpas_smart_field_send_1d(client, field_1d_ptr, ierr) + ! else + ! call mpas_log_write('field is not of type 1D REAL') + ! endif + ! end do + + !end subroutine mpas_smart_stream_send + + subroutine mpas_smart_field_send(client, domain, fieldName, ierr) + + type(client_type), intent(in) :: client + type (domain_type), intent(in) :: domain + character(len=100), intent(in) :: fieldName + integer, intent(out), optional :: ierr + + type (field1dReal), pointer :: field_ptr + character(len=100) :: log_string + integer(kind=enum_kind) :: err_client + + call mpas_pool_get_field_info(domain%blocklist%allFields, & + smartFieldName, fieldInfo) + WRITE(log_string,*) smartFieldName, 'has type', fieldInfo%fieldType + call mpas_log_write(log_string) + if ( fieldInfo % fieldType == MPAS_POOL_REAL ) then + call mpas_pool_get_field(domain%blocklist%allFields, & + smartFieldName, sendArrayReal1d) + WRITE(log_string,*) smartFieldName, 'retrieved from stream', & + sendArrayReal1d + call mpas_smart_field_send_1d(client, field_ptr, ierr) + endif + + end subroutine mpas_smart_field_send + + subroutine mpas_smart_field_send_1d(client, field_ptr, ierr) + + implicit none + + type(client_type), intent(in) :: client + type (field1dReal), pointer, intent(in) :: field_ptr + integer, intent(out), optional :: ierr + + character(len=100) :: log_string + integer(kind=enum_kind) :: err_client + real (kind=RKIND), dimension(:), allocatable :: field_temp + integer, dimension(1) :: field_dims + + call mpas_pool_get_dimension(field_ptr % block % dimensions, & + trim(field_ptr % dimNames(1), dim1) + allocate(field_temp(field_dims(1))) + field_temp = field_ptr % array + + err_client = client%put_tensor("send_array", field_temp, & + shape(sendArrayReal1d)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'Error put tensor:', smartFieldName + call mpas_log_write(log_string) + ierr = ior(ierr, 1) + endif + + end subroutine mpas_smart_field_send +end module mpas_smart_routines diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index 72c1b03d349c..195a73183dc7 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -30,7 +30,7 @@ module ocn_forward_mode use mpas_log use mpas_decomp use mpas_log -! use mpas_smart_routines + use mpas_smart_routines use ocn_analysis_driver use ocn_init_routines @@ -107,8 +107,6 @@ module ocn_forward_mode ocn_forward_mode_finalize, & ocn_forward_mode_setup_clock -# include "../../../mpas-framework/src/framework/enum_fortran.inc" - !*********************************************************************** contains @@ -170,6 +168,10 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ sendArrayReal1d, recvArrayReal1d real (kind=RKIND), dimension(:,:), allocatable :: & sendArrayReal2d, recvArrayReal2d + real (kind=RKIND), dimension(:,:,:), allocatable :: & + sendArrayReal3d, recvArrayReal3d + real (kind=RKIND), dimension(:,:,:,:), allocatable :: & + sendArrayReal4d, recvArrayReal4d real (kind=RKIND) :: meta_scalar real (kind=RKIND), dimension(:), pointer :: & meshDensity ! pointer to mesh density array @@ -227,20 +229,12 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ ! Currently only support one block per task block => domain % blocklist - call get_environment_variable('SSDB',log_string) - call mpas_log_write('SSDB='//TRIM(log_string)) - if (len(TRIM(log_string)) < 5) then - call mpas_log_write('No SSDB')!, & + call mpas_smart_init(client, ierr) + if (ierr /= 0.0_RKIND) then + write(log_string,*) 'OCN: Error initializing redis client:',err_client + call mpas_log_write(log_string) else - err_client = client%initialize(.false.) - if (err_client /= SRNoError) then - write(log_string,*) 'Error initializing redis client:',err_client - call mpas_log_write(log_string)!, & - !MPAS_LOG_ERR, masterOnly=.true., flushNow=.true.) - !ierr = ior(ierr, 1) - else - call mpas_log_write('Client initialized') - endif + call mpas_log_write('OCN Client initialized') write(log_string,*) 'Before client: config_mom_del2', config_mom_del2 call mpas_log_write(log_string) @@ -774,6 +768,11 @@ function ocn_forward_mode_run(domain) result(ierr)!{{{ type (MPAS_timeInterval_type) :: timeStep + logical :: validField + logical :: isActive + character(len=StrKind) :: fieldName + character(len=StrKIND) :: log_string + ierr = 0 ! Eventually, dt should be domain specific @@ -949,26 +948,15 @@ function ocn_forward_mode_run(domain) result(ierr)!{{{ ! the variable being sent to the database node match that of the source array ! Loop through variable names in stream and get from allFields rather ! than hard-code variables in diagnosticsPool - !call MPAS_stream_mgr_begin_iteration(domain%streamManager, 'smartsend', ierr) - !call MPAS_stream_mgr_get_next_field(domain%streamManager, 'smartsend', smartFieldName, isActive) - !if (isActive) then - ! call mpas_pool_get_field_info(domain%blocklist%allFields, smartFieldName, fieldInfo) - ! WRITE(log_string,*) smartFieldName, 'has type', fieldInfo%fieldType - ! call mpas_log_write(log_string) - ! if ( fieldInfo % fieldType == MPAS_POOL_REAL ) then - ! call mpas_pool_get_field(domain%blocklist%allFields, smartFieldName, sendArrayReal1d) - ! WRITE(log_string,*) smartFieldName, 'retrieved from stream', sendArrayReal1d - ! err_client = client%put_tensor("send_array", sendArrayReal1d, shape(sendArrayReal1d)) - ! if (err_client /= SRNoError) then - ! WRITE(log_string,*) 'Error put tensor:', smartFieldName - ! call mpas_log_write(log_string, MPAS_LOG_ERR, masterOnly=.true., flushNow=.true.) - ! ierr = ior(ierr, 1) - ! endif - ! endif - !else - ! WRITE(log_string,*) 'smartsend field', smartFieldName, 'is not active' - ! call mpas_log_write(log_string) - !endif + call MPAS_stream_mgr_begin_iteration(domain%streamManager, & + 'smartsend', ierr) + do while (MPAS_stream_mgr_get_next_field(domain%streamManager, & + 'smartsend', fieldName))!, isActive) + call mpas_smart_field_send(domain, fieldName, ierr) + !WRITE(log_string,*) 'smartsend field', fieldName, & + ! 'is not active' + !call mpas_log_write(log_string) + end do ! TODO add similar loop for stream smartrecv ! err_client = client%unpack_tensor("send_array", recvArrayReal1d, shape(recvArrayReal1d)) ! if (err_client /= SRNoError) then From 7901fc92a0739d8aa1667ddbcdde39c1b0bd7459 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Wed, 31 Aug 2022 06:34:31 -0700 Subject: [PATCH 20/29] Support send tensor and send config to client --- .../src/framework/mpas_smart_routines.F | 331 +++++++++++++++--- components/mpas-ocean/mpas_client_script.py | 4 +- .../src/mode_forward/mpas_ocn_forward_mode.F | 57 +-- 3 files changed, 308 insertions(+), 84 deletions(-) diff --git a/components/mpas-framework/src/framework/mpas_smart_routines.F b/components/mpas-framework/src/framework/mpas_smart_routines.F index 9d635a530b55..589221fa63df 100644 --- a/components/mpas-framework/src/framework/mpas_smart_routines.F +++ b/components/mpas-framework/src/framework/mpas_smart_routines.F @@ -25,18 +25,29 @@ module mpas_smart_routines use smartredis_client, only: client_type implicit none + save public :: mpas_smart_init # include "enum_fortran.inc" +! --- Private module variables --- + type(client_type) :: client + + interface mpas_smart_field_client_send + module procedure mpas_smart_field_client_send_1d + module procedure mpas_smart_field_client_send_2d + module procedure mpas_smart_field_client_send_3d + module procedure mpas_smart_field_client_send_4d + end interface + contains - subroutine mpas_smart_init(client, ierr) + subroutine mpas_smart_init(client_out, ierr) - implicit none + !implicit none - type(client_type), intent(out) :: client + type(client_type), intent(out) :: client_out integer, intent(out), optional :: ierr character(len=100) :: log_string @@ -49,95 +60,299 @@ subroutine mpas_smart_init(client, ierr) else err_client = client%initialize(.false.) if (err_client /= SRNoError) then - write(log_string,*) 'FWK: Error initializing redis client:',err_client + write(log_string,*) 'FWK: Error initializing redis client:', & + err_client call mpas_log_write(log_string) ierr = ior(ierr, 1) else call mpas_log_write('FWK: Client initialized') + client_out = client endif endif end subroutine mpas_smart_init - !subroutine mpas_smart_stream_send(client, stream, ierr) - - ! implicit none - - ! type(client_type), intent(in) :: client - ! !type (MPAS_streamManager_type), intent(inout) :: manager - ! !character (len=*), intent(in), optional :: streamID - ! type (MPAS_Stream_type), intent(in) :: stream - ! !type (MPAS_stream_list_type), pointer :: stream_cursor - ! integer, intent(out), optional :: ierr - - ! type (field_list_type), pointer :: field_cursor - ! type (field1dReal), pointer :: field_1d_ptr - ! !call MPAS_stream_list_query(manager % streams, streamID, stream_cursor, ierr=ierr)) - ! ! - ! ! Loop over fields in the stream - ! ! - ! field_cursor => stream % fieldList - ! do while (associated(field_cursor)) - ! if (field_cursor % field_type == FIELD_1D_REAL) then - ! field_1d_ptr = field_cursor % real1dField - ! call mpas_smart_field_send_1d(client, field_1d_ptr, ierr) - ! else - ! call mpas_log_write('field is not of type 1D REAL') - ! endif - ! end do - - !end subroutine mpas_smart_stream_send - - subroutine mpas_smart_field_send(client, domain, fieldName, ierr) - - type(client_type), intent(in) :: client - type (domain_type), intent(in) :: domain - character(len=100), intent(in) :: fieldName + subroutine mpas_smart_database_init(ierr) + + !implicit none + integer, intent(out), optional :: ierr - type (field1dReal), pointer :: field_ptr character(len=100) :: log_string integer(kind=enum_kind) :: err_client + if (err_client /= SRNoError) then + write(log_string,*) 'FWK: Error initializing redis database:', & + err_client + call mpas_log_write(log_string) + ierr = ior(ierr, 1) + else + call mpas_log_write('FWK: Database initialized') + endif + + end subroutine mpas_smart_database_init + +! -------------------------------------------------------------------- +! SUBROUTINE mpas_smart_stream_send +! Send the stream to a client. +! -------------------------------------------------------------------- +! subroutine mpas_smart_stream_send(stream, ierr) + +! implicit none + +! !type (MPAS_streamManager_type), intent(inout) :: manager +! !character (len=*), intent(in), optional :: streamID +! type (MPAS_Stream_type), intent(in) :: stream +! !type (MPAS_stream_list_type), pointer :: stream_cursor +! integer, intent(out), optional :: ierr + +! type (field_list_type), pointer :: field_cursor +! type (field1dReal), pointer :: field_1d_ptr +! !call MPAS_stream_list_query(manager % streams, streamID, stream_cursor, ierr=ierr)) +! ! +! ! Loop over fields in the stream +! ! +! field_cursor => stream % fieldList +! do while (associated(field_cursor)) +! if (field_cursor % field_type == FIELD_1D_REAL) then +! field_1d_ptr = field_cursor % real1dField +! call mpas_smart_field_send_1d(client, field_1d_ptr, ierr) +! else +! call mpas_log_write('field is not of type 1D REAL') +! endif +! end do + +! end subroutine mpas_smart_stream_send + +! -------------------------------------------------------------------- +! SUBROUTINE mpas_smart_stream_database_send +! Send the stream to a database. +! TODO: if decomposed and blockWrite, send each block to a +! different database +! -------------------------------------------------------------------- + +! subroutine mpas_smart_stream_database_send(stream, database_list) +! +! type (field_list_type), pointer :: field_cursor + +! field_cursor => stream % fieldList +! do while (associated(field_cursor)) + +! if (field_cursor % isDecomposed) then +! ! Gather field from across multiple blocks +! field_2dreal_ptr => field_cursor % real2dField +! i = 1 +! do while (associated(field_2dreal_ptr)) +! i = i + ownedSize +! if ( .not. stream % blockWrite ) then +! field_2dreal_ptr => field_2dreal_ptr % next +! else +! nullify(field_2dreal_ptr) +! end if +! end do +! end if +! if ( .not. stream % blockWrite ) then + +! end subroutine mpas_smart_database_send + + subroutine mpas_smart_config_send(domain, configName, ierr) + + type (domain_type), intent(in) :: domain + character(len=*), intent(in) :: configName + integer, intent(out), optional :: ierr + + type (mpas_pool_field_info_type) :: fieldInfo + real (kind=RKIND), pointer :: config_ptr + real (kind=RKIND), dimension(1) :: config_temp + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + + call mpas_pool_get_config(domain % configs, & + trim(configName), & + config_ptr) + config_temp(1) = config_ptr + err_client = client%put_tensor(trim(configName), & + config_temp, shape(config_temp)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'FWK: Error put config:', trim(configName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'FWK: Put config:', trim(configName) + call mpas_log_write(trim(log_string)) + endif + + end subroutine mpas_smart_config_send + + subroutine mpas_smart_field_send(domain, fieldName, ierr) + + type (domain_type), intent(in) :: domain + character(len=StrKIND), intent(in) :: fieldName + integer, intent(out), optional :: ierr + + type (mpas_pool_field_info_type) :: fieldInfo + type (field1DReal), pointer :: field_real1d_ptr + type (field2DReal), pointer :: field_real2d_ptr + type (field3DReal), pointer :: field_real3d_ptr + type (field4DReal), pointer :: field_real4d_ptr + character(len=StrKIND) :: log_string + call mpas_pool_get_field_info(domain%blocklist%allFields, & - smartFieldName, fieldInfo) - WRITE(log_string,*) smartFieldName, 'has type', fieldInfo%fieldType + fieldName, fieldInfo) call mpas_log_write(log_string) if ( fieldInfo % fieldType == MPAS_POOL_REAL ) then - call mpas_pool_get_field(domain%blocklist%allFields, & - smartFieldName, sendArrayReal1d) - WRITE(log_string,*) smartFieldName, 'retrieved from stream', & - sendArrayReal1d - call mpas_smart_field_send_1d(client, field_ptr, ierr) + if ( fieldInfo % nDims == 1 ) then + call mpas_pool_get_field(domain%blocklist%allFields, & + trim(fieldName), field_real1d_ptr) + call mpas_log_write(trim(fieldName)//' retrieved') + call mpas_smart_field_client_send(field_real1d_ptr, ierr) + elseif ( fieldInfo % nDims == 2 ) then + call mpas_pool_get_field(domain%blocklist%allFields, & + trim(fieldName), field_real2d_ptr) + call mpas_log_write(trim(fieldName)//' retrieved') + call mpas_smart_field_client_send(field_real2d_ptr, ierr) + elseif ( fieldInfo % nDims == 3 ) then + call mpas_pool_get_field(domain%blocklist%allFields, & + trim(fieldName), field_real3d_ptr) + call mpas_log_write(trim(fieldName)//' retrieved') + call mpas_smart_field_client_send(field_real3d_ptr, ierr) + elseif ( fieldInfo % nDims == 4 ) then + call mpas_pool_get_field(domain%blocklist%allFields, & + trim(fieldName), field_real4d_ptr) + call mpas_log_write(trim(fieldName)//' retrieved') + call mpas_smart_field_client_send(field_real4d_ptr, ierr) + else + call mpas_log_write(trim(fieldName)// & + ' is not of a supported dimension') + endif + else + call mpas_log_write(trim(fieldName)//' is not real') endif end subroutine mpas_smart_field_send - subroutine mpas_smart_field_send_1d(client, field_ptr, ierr) - - implicit none +! INTERFACE mpas_smart_field_client_send +! TODO add optional argument to provide a different key + subroutine mpas_smart_field_client_send_1d(field_ptr, ierr) - type(client_type), intent(in) :: client type (field1dReal), pointer, intent(in) :: field_ptr integer, intent(out), optional :: ierr - character(len=100) :: log_string + character(len=StrKIND) :: log_string integer(kind=enum_kind) :: err_client real (kind=RKIND), dimension(:), allocatable :: field_temp integer, dimension(1) :: field_dims - call mpas_pool_get_dimension(field_ptr % block % dimensions, & - trim(field_ptr % dimNames(1), dim1) + field_dims = field_ptr % dimSizes + !call mpas_pool_get_dimension(field_ptr % block % dimensions, & + ! trim(field_ptr % dimNames(1)), field_dims(1)) allocate(field_temp(field_dims(1))) field_temp = field_ptr % array - err_client = client%put_tensor("send_array", field_temp, & - shape(sendArrayReal1d)) + err_client = client%put_tensor(trim(field_ptr % fieldName), & + field_temp, & + shape(field_temp)) if (err_client /= SRNoError) then - WRITE(log_string,*) 'Error put tensor:', smartFieldName - call mpas_log_write(log_string) + WRITE(log_string,*) 'FWK: Error put tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'FWK: Put tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) endif + deallocate(field_temp) + + end subroutine mpas_smart_field_client_send_1d + + subroutine mpas_smart_field_client_send_2d(field_ptr, ierr) + + type (field2dReal), pointer, intent(in) :: field_ptr + integer, intent(out), optional :: ierr + + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + real (kind=RKIND), dimension(:,:), allocatable :: field_temp + integer, dimension(2) :: field_dims + + field_dims = field_ptr % dimSizes + allocate(field_temp(field_dims(1), field_dims(2))) + field_temp = field_ptr % array + + err_client = client%put_tensor(trim(field_ptr % fieldName), & + field_temp, & + shape(field_temp)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'Error put tensor:', & + trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'Put tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + endif + deallocate(field_temp) + + end subroutine mpas_smart_field_client_send_2d + + subroutine mpas_smart_field_client_send_3d(field_ptr, ierr) + + type (field3dReal), pointer, intent(in) :: field_ptr + integer, intent(out), optional :: ierr + + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + real (kind=RKIND), dimension(:,:,:), allocatable :: field_temp + integer, dimension(3) :: field_dims + + field_dims = field_ptr % dimSizes + allocate(field_temp(field_dims(1), field_dims(2), field_dims(3))) + field_temp = field_ptr % array + + err_client = client%put_tensor(trim(field_ptr % fieldName), & + field_temp, & + shape(field_temp)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'Error put tensor:', & + trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'Put tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + endif + deallocate(field_temp) + + end subroutine mpas_smart_field_client_send_3d + + subroutine mpas_smart_field_client_send_4d(field_ptr, ierr) + + type (field4dReal), pointer, intent(in) :: field_ptr + integer, intent(out), optional :: ierr + + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + real (kind=RKIND), dimension(:,:,:,:), allocatable :: field_temp + integer, dimension(4) :: field_dims + + field_dims = field_ptr % dimSizes + allocate(field_temp(field_dims(1), field_dims(2), field_dims(3), & + field_dims(4))) + field_temp = field_ptr % array + + err_client = client%put_tensor(trim(field_ptr % fieldName), & + field_temp, & + shape(field_temp)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'Error put tensor:', & + trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'Put tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + endif + deallocate(field_temp) + + end subroutine mpas_smart_field_client_send_4d - end subroutine mpas_smart_field_send end module mpas_smart_routines diff --git a/components/mpas-ocean/mpas_client_script.py b/components/mpas-ocean/mpas_client_script.py index bcd02fb458f9..3b3f4eade79d 100644 --- a/components/mpas-ocean/mpas_client_script.py +++ b/components/mpas-ocean/mpas_client_script.py @@ -16,10 +16,10 @@ client = Client(address=SSDB, cluster=False) recv_array = 20*numpy.ones(1) err = client.put_tensor("init_recv", recv_array) -key_found = client.poll_key("init_send", 200, 10000) +key_found = client.poll_key("config_mom_del2", 20, 10000) print('key_found',key_found) if key_found: - dummy_array = client.get_tensor("init_send") + dummy_array = client.get_tensor("config_mom_del2") print('init_send_array=',dummy_array[0]) else: print('key not found') diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index 195a73183dc7..0b43d0ad6933 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -151,8 +151,8 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ character(len=100) :: log_string character(len=*), parameter :: script_key = 'init_script' character(len=*), parameter :: script_file = & - 'test_client_recv.txt' - character(len=255) :: in_key, script_out_key, out_key + 'data_processing_script.txt' + character(len=255) :: script_in_key, script_out_key character(len=255), dimension(1) :: inputs, outputs @@ -236,13 +236,15 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ else call mpas_log_write('OCN Client initialized') + ! Send config option to client write(log_string,*) 'Before client: config_mom_del2', config_mom_del2 call mpas_log_write(log_string) - in_key = 'init_send' - script_out_key = 'init_recv' - out_key = 'init_recv' - inputs(1) = in_key - outputs(1) = script_out_key + call mpas_smart_config_send(domain, 'config_mom_del2', ierr) + if (ierr /= 0.0_RKIND) then + call mpas_log_write('Error put config') + else + call mpas_log_write('Put config') + endif allocate(sendArrayReal1d(1), recvArrayReal1d(1)) allocate(sendArrayReal2d(1,1), recvArrayReal2d(1,1)) @@ -250,29 +252,36 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ recvArrayReal1d(:) = 0.0_RKIND sendArrayReal2d(1,1) = 2.0_RKIND recvArrayReal2d(1,1) = 0.0_RKIND - err_client = client%put_tensor(in_key, sendArrayReal1d, shape(sendArrayReal1d)) + script_in_key = 'init_send_script' + script_out_key = 'init_recv_script' + inputs(1) = script_in_key + outputs(1) = script_out_key + allocate(sendArrayReal4d(1,1,28,28)) + call random_number(sendArrayReal4d) + !sendArrayReal4d(:,:,:,:) = 2.0_RKIND + err_client = client%put_tensor(script_in_key, sendArrayReal4d, shape(sendArrayReal4d)) if (err_client /= SRNoError) then - write(log_string,*) 'Error put tensor:',err_client + write(log_string,*) 'Error put tensor for script:',err_client + call mpas_log_write(log_string) + else + call mpas_log_write('Put tensor for script set') + endif + err_client = client%set_script_from_file(script_key, "CPU", script_file) + if (err_client /= SRNoError) then + write(log_string,*) 'Error setting client script:',err_client + call mpas_log_write(log_string) + else + call mpas_log_write('Client script set') + endif + err_client = client%run_script(script_key, "pre_process", inputs, outputs) + if (err_client /= SRNoError) then + write(log_string,*) 'Error running client script:',err_client call mpas_log_write(log_string) is_error = client%SR_error_parser(err_client) else - call mpas_log_write('Put tensor') + call mpas_log_write('Client script run') endif - !err_client = client%set_script_from_file(script_key, "CPU", script_file) - !if (err_client /= SRNoError) then - ! write(log_string,*) 'Error setting client script:',err_client - ! call mpas_log_write(log_string) - !else - ! call mpas_log_write('Client script set') - !endif - !err_client = client%run_script(script_key, "pre_process", inputs, outputs) - !if (err_client /= SRNoError) then - ! write(log_string,*) 'Error running client script:',err_client - ! call mpas_log_write(log_string) - ! is_error = client%SR_error_parser(err_client) - !else - !call mpas_log_write('Client script run') err_client = client%unpack_tensor(outputs(1), recvArrayReal1d, shape(recvArrayReal1d)) if (err_client /= SRNoError) then write(log_string,*) 'Error unpacking tensor init_recv:',err_client From 7cbf1effb6ac95d131b0368921280f18552c18e6 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Fri, 2 Sep 2022 12:37:17 -0700 Subject: [PATCH 21/29] Test run script --- .../src/mode_forward/mpas_ocn_forward_mode.F | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index 0b43d0ad6933..0cca7adbb643 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -275,11 +275,19 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ endif err_client = client%run_script(script_key, "pre_process", inputs, outputs) if (err_client /= SRNoError) then - write(log_string,*) 'Error running client script:',err_client + write(log_string,*) 'Error running script pre_process:',err_client call mpas_log_write(log_string) is_error = client%SR_error_parser(err_client) else - call mpas_log_write('Client script run') + call mpas_log_write('Pre_process script run') + endif + err_client = client%run_script(script_key, "post_process", inputs, outputs) + if (err_client /= SRNoError) then + write(log_string,*) 'Error running script post_process:',err_client + call mpas_log_write(log_string) + is_error = client%SR_error_parser(err_client) + else + call mpas_log_write('Post_process script run') endif err_client = client%unpack_tensor(outputs(1), recvArrayReal1d, shape(recvArrayReal1d)) From 8f9f518f01681dcfdfc9a75fe534ccdb7bea1858 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Wed, 14 Sep 2022 14:10:44 -0700 Subject: [PATCH 22/29] Add smart client recv interface --- .../src/framework/mpas_smart_routines.F | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) diff --git a/components/mpas-framework/src/framework/mpas_smart_routines.F b/components/mpas-framework/src/framework/mpas_smart_routines.F index 589221fa63df..33154e1eded3 100644 --- a/components/mpas-framework/src/framework/mpas_smart_routines.F +++ b/components/mpas-framework/src/framework/mpas_smart_routines.F @@ -41,6 +41,13 @@ module mpas_smart_routines module procedure mpas_smart_field_client_send_4d end interface + interface mpas_smart_field_client_recv + module procedure mpas_smart_field_client_recv_1d + module procedure mpas_smart_field_client_recv_2d + module procedure mpas_smart_field_client_recv_3d + module procedure mpas_smart_field_client_recv_4d + end interface + contains subroutine mpas_smart_init(client_out, ierr) @@ -231,6 +238,54 @@ subroutine mpas_smart_field_send(domain, fieldName, ierr) end subroutine mpas_smart_field_send + subroutine mpas_smart_field_recv(domain, fieldName, ierr) + + type (domain_type), intent(in) :: domain + character(len=StrKIND), intent(in) :: fieldName + integer, intent(out), optional :: ierr + + type (mpas_pool_field_info_type) :: fieldInfo + type (field1DReal), pointer :: field_real1d_ptr + type (field2DReal), pointer :: field_real2d_ptr + type (field3DReal), pointer :: field_real3d_ptr + type (field4DReal), pointer :: field_real4d_ptr + character(len=StrKIND) :: log_string + + call mpas_pool_get_field_info(domain%blocklist%allFields, & + fieldName, fieldInfo) + call mpas_log_write(log_string) + if ( fieldInfo % fieldType == MPAS_POOL_REAL ) then + if ( fieldInfo % nDims == 1 ) then + call mpas_pool_get_field(domain%blocklist%allFields, & + trim(fieldName), field_real1d_ptr) + call mpas_log_write(trim(fieldName)//' retrieved') + call mpas_smart_field_client_recv(field_real1d_ptr, ierr) + elseif ( fieldInfo % nDims == 2 ) then + call mpas_pool_get_field(domain%blocklist%allFields, & + trim(fieldName), field_real2d_ptr) + call mpas_log_write(trim(fieldName)//' retrieved') + call mpas_smart_field_client_recv(field_real2d_ptr, ierr) + elseif ( fieldInfo % nDims == 3 ) then + call mpas_pool_get_field(domain%blocklist%allFields, & + trim(fieldName), field_real3d_ptr) + call mpas_log_write(trim(fieldName)//' retrieved') + call mpas_smart_field_client_recv(field_real3d_ptr, ierr) + elseif ( fieldInfo % nDims == 4 ) then + call mpas_pool_get_field(domain%blocklist%allFields, & + trim(fieldName), field_real4d_ptr) + call mpas_log_write(trim(fieldName)//' retrieved') + call mpas_smart_field_client_recv(field_real4d_ptr, ierr) + else + call mpas_log_write(trim(fieldName)// & + ' is not of a supported dimension') + endif + else + call mpas_log_write(trim(fieldName)//' is not real') + endif + + end subroutine mpas_smart_field_recv + + ! INTERFACE mpas_smart_field_client_send ! TODO add optional argument to provide a different key subroutine mpas_smart_field_client_send_1d(field_ptr, ierr) @@ -355,4 +410,126 @@ subroutine mpas_smart_field_client_send_4d(field_ptr, ierr) end subroutine mpas_smart_field_client_send_4d +! INTERFACE mpas_smart_field_client_recv +! TODO add optional argument to provide a different key + subroutine mpas_smart_field_client_send_1d(field_ptr, ierr) + + type (field1dReal), pointer, intent(in) :: field_ptr + integer, intent(out), optional :: ierr + + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + real (kind=RKIND), dimension(:), allocatable :: field_temp + integer, dimension(1) :: field_dims + + field_dims = field_ptr % dimSizes + allocate(field_temp(field_dims(1))) + + err_client = client%unpack_tensor(trim(field_ptr % fieldName), & + field_temp, & + shape(field_temp)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'FWK: Error unpack tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'FWK: Unpack tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + endif + field_ptr % array = field_temp + deallocate(field_temp) + + end subroutine mpas_smart_field_client_recv_1d + + subroutine mpas_smart_field_client_recv_2d(field_ptr, ierr) + + type (field2dReal), pointer, intent(in) :: field_ptr + integer, intent(out), optional :: ierr + + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + real (kind=RKIND), dimension(:,:), allocatable :: field_temp + integer, dimension(2) :: field_dims + + field_dims = field_ptr % dimSizes + allocate(field_temp(field_dims(1), field_dims(2))) + + err_client = client%unpack_tensor(trim(field_ptr % fieldName), & + field_temp, & + shape(field_temp)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'Error unpack tensor:', & + trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'FWK: unpack tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + endif + field_ptr % array = field_temp + deallocate(field_temp) + + end subroutine mpas_smart_field_client_recv_2d + + subroutine mpas_smart_field_client_recv_3d(field_ptr, ierr) + + type (field3dReal), pointer, intent(in) :: field_ptr + integer, intent(out), optional :: ierr + + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + real (kind=RKIND), dimension(:,:,:), allocatable :: field_temp + integer, dimension(3) :: field_dims + + field_dims = field_ptr % dimSizes + allocate(field_temp(field_dims(1), field_dims(2), field_dims(3))) + + err_client = client%unpack_tensor(trim(field_ptr % fieldName), & + field_temp, & + shape(field_temp)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'Error unpack tensor:', & + trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'FWK: unpack tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + endif + field_ptr % array = field_temp + deallocate(field_temp) + + end subroutine mpas_smart_field_client_recv_3d + + subroutine mpas_smart_field_client_recv_4d(field_ptr, ierr) + + type (field4dReal), pointer, intent(in) :: field_ptr + integer, intent(out), optional :: ierr + + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + real (kind=RKIND), dimension(:,:,:,:), allocatable :: field_temp + integer, dimension(4) :: field_dims + + field_dims = field_ptr % dimSizes + allocate(field_temp(field_dims(1), field_dims(2), field_dims(3), & + field_dims(4))) + + err_client = client%unpack_tensor(trim(field_ptr % fieldName), & + field_temp, & + shape(field_temp)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'Error unpack tensor:', & + trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'Unpack tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + endif + field_ptr % array = field_temp + deallocate(field_temp) + + end subroutine mpas_smart_field_client_recv_4d + end module mpas_smart_routines From 43ee4a7382e28cd4caa80010612538e50989eccd Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Wed, 14 Sep 2022 14:34:54 -0700 Subject: [PATCH 23/29] Clean-up smartsim calls in forward mode --- .../src/mode_forward/mpas_ocn_forward_mode.F | 112 ++++++++---------- 1 file changed, 48 insertions(+), 64 deletions(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index 0cca7adbb643..9dc1ba38d405 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -229,23 +229,20 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ ! Currently only support one block per task block => domain % blocklist - call mpas_smart_init(client, ierr) - if (ierr /= 0.0_RKIND) then - write(log_string,*) 'OCN: Error initializing redis client:',err_client - call mpas_log_write(log_string) - else - call mpas_log_write('OCN Client initialized') - - ! Send config option to client - write(log_string,*) 'Before client: config_mom_del2', config_mom_del2 - call mpas_log_write(log_string) - call mpas_smart_config_send(domain, 'config_mom_del2', ierr) + if (config_use_smartsim) then + call mpas_smart_init(client, ierr) if (ierr /= 0.0_RKIND) then - call mpas_log_write('Error put config') - else - call mpas_log_write('Put config') + call mpas_log_write('Error initializing smartredis client', & + MPAS_LOG_CRIT) endif + ! ---- Example: Send config option to client ---- + ! configName = 'config_mom_del2' + ! call mpas_smart_config_send(domain, configName, ierr) + ! if (ierr /= 0.0_RKIND) then + ! call mpas_log_write('Error: Smartsim put config at initialization') + ! endif + allocate(sendArrayReal1d(1), recvArrayReal1d(1)) allocate(sendArrayReal2d(1,1), recvArrayReal2d(1,1)) sendArrayReal1d(:) = 2.0_RKIND @@ -266,42 +263,28 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ else call mpas_log_write('Put tensor for script set') endif - err_client = client%set_script_from_file(script_key, "CPU", script_file) - if (err_client /= SRNoError) then - write(log_string,*) 'Error setting client script:',err_client - call mpas_log_write(log_string) - else - call mpas_log_write('Client script set') - endif - err_client = client%run_script(script_key, "pre_process", inputs, outputs) - if (err_client /= SRNoError) then - write(log_string,*) 'Error running script pre_process:',err_client - call mpas_log_write(log_string) - is_error = client%SR_error_parser(err_client) - else - call mpas_log_write('Pre_process script run') - endif - err_client = client%run_script(script_key, "post_process", inputs, outputs) - if (err_client /= SRNoError) then - write(log_string,*) 'Error running script post_process:',err_client - call mpas_log_write(log_string) - is_error = client%SR_error_parser(err_client) - else - call mpas_log_write('Post_process script run') - endif - - err_client = client%unpack_tensor(outputs(1), recvArrayReal1d, shape(recvArrayReal1d)) - if (err_client /= SRNoError) then - write(log_string,*) 'Error unpacking tensor init_recv:',err_client - call mpas_log_write(log_string) - is_error = client%SR_error_parser(err_client) - else - write(log_string,*) 'init_recv:',recvArrayReal1d(1) - call mpas_log_write(log_string) - config_mom_del2 = recvArrayReal1d(1) - write(log_string,*) 'After client: config_mom_del2', config_mom_del2 - call mpas_log_write(log_string) - endif + !err_client = client%set_script_from_file(script_key, "CPU", script_file) + !if (err_client /= SRNoError) then + ! write(log_string,*) 'Error setting client script:',err_client + ! call mpas_log_write(log_string) + !else + ! call mpas_log_write('Client script set') + !endif + !err_client = client%run_script(script_key, "pre_process", inputs, outputs) + !if (err_client /= SRNoError) then + ! write(log_string,*) 'Error running script pre_process:',err_client + ! call mpas_log_write(log_string) + ! is_error = client%SR_error_parser(err_client) + !else + ! call mpas_log_write('Pre_process script run') + !endif + !err_client = client%run_script(script_key, "post_process", inputs, outputs) + !if (err_client /= SRNoError) then + ! write(log_string,*) 'Error running script post_process:',err_client + ! call mpas_log_write(log_string) + ! is_error = client%SR_error_parser(err_client) + !else + ! call mpas_log_write('Post_process script run') !endif err_client = dataset%initialize("example_fortran_dataset") @@ -965,22 +948,26 @@ function ocn_forward_mode_run(domain) result(ierr)!{{{ ! the variable being sent to the database node match that of the source array ! Loop through variable names in stream and get from allFields rather ! than hard-code variables in diagnosticsPool + call mpas_timer_start('smartsim i/o') call MPAS_stream_mgr_begin_iteration(domain%streamManager, & 'smartsend', ierr) do while (MPAS_stream_mgr_get_next_field(domain%streamManager, & - 'smartsend', fieldName))!, isActive) + 'smartsend', fieldName)) call mpas_smart_field_send(domain, fieldName, ierr) - !WRITE(log_string,*) 'smartsend field', fieldName, & - ! 'is not active' - !call mpas_log_write(log_string) + if (ierr /= 0.0_RKIND) then + call mpas_log_write('MPAS I/O error in smartsend stream') + endif end do - ! TODO add similar loop for stream smartrecv - ! err_client = client%unpack_tensor("send_array", recvArrayReal1d, shape(recvArrayReal1d)) - ! if (err_client /= SRNoError) then - ! WRITE(log_string,*) 'Error unpack tensor:', smartFieldName - ! call mpas_log_write(log_string, MPAS_LOG_ERR, masterOnly=.true., flushNow=.true.) - ! ierr = ior(ierr, 1) - ! endif + call MPAS_stream_mgr_begin_iteration(domain%streamManager, & + 'smartrecv', ierr) + do while (MPAS_stream_mgr_get_next_field(domain%streamManager, & + 'smartrecv', fieldName)) + call mpas_smart_field_recv(domain, fieldName, ierr) + if (ierr /= 0.0_RKIND) then + call mpas_log_write('MPAS I/O error in smartrecv stream') + endif + end do + call mpas_timer_stop('smartsim i/o') call mpas_timer_start('reset_io_alarms') @@ -1051,9 +1038,6 @@ function ocn_forward_mode_finalize(domain) result(iErr)!{{{ character(len=100) :: log_string integer :: ierr - write(log_string,*) 'Finalize: config_mom_del2', config_mom_del2 - call mpas_log_write(log_string) - ! destroy the ocean mesh structure call ocn_meshDestroy(ierr) From b0658123e542128e6151a407383d0fe24815f1f2 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Wed, 14 Sep 2022 14:37:24 -0700 Subject: [PATCH 24/29] Add config option for smartsim --- components/mpas-ocean/src/Registry.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/mpas-ocean/src/Registry.xml b/components/mpas-ocean/src/Registry.xml index 95b00c65beb4..ce361bed9353 100644 --- a/components/mpas-ocean/src/Registry.xml +++ b/components/mpas-ocean/src/Registry.xml @@ -164,6 +164,10 @@ possible_values="Any positive integer value greater than 0." mode="forward;analysis;init" /> + Date: Wed, 14 Sep 2022 14:39:23 -0700 Subject: [PATCH 25/29] Add smart config recv routine --- .../src/framework/mpas_smart_routines.F | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/components/mpas-framework/src/framework/mpas_smart_routines.F b/components/mpas-framework/src/framework/mpas_smart_routines.F index 33154e1eded3..e9b258d67cfe 100644 --- a/components/mpas-framework/src/framework/mpas_smart_routines.F +++ b/components/mpas-framework/src/framework/mpas_smart_routines.F @@ -191,6 +191,35 @@ subroutine mpas_smart_config_send(domain, configName, ierr) end subroutine mpas_smart_config_send + subroutine mpas_smart_config_recv(domain, configName, ierr) + + type (domain_type), intent(in) :: domain + character(len=*), intent(in) :: configName + integer, intent(out), optional :: ierr + + type (mpas_pool_field_info_type) :: fieldInfo + real (kind=RKIND), pointer :: config_ptr + real (kind=RKIND), dimension(1) :: config_temp + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + + call mpas_pool_get_config(domain % configs, & + trim(configName), & + config_ptr) + err_client = client%unpack_tensor(trim(configName), & + config_temp, shape(config_temp)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'FWK: Error unpack config:', trim(configName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'FWK: unpack config:', trim(configName) + call mpas_log_write(trim(log_string)) + endif + config_ptr = config_temp(1) + + end subroutine mpas_smart_config_recv + subroutine mpas_smart_field_send(domain, fieldName, ierr) type (domain_type), intent(in) :: domain @@ -412,7 +441,7 @@ end subroutine mpas_smart_field_client_send_4d ! INTERFACE mpas_smart_field_client_recv ! TODO add optional argument to provide a different key - subroutine mpas_smart_field_client_send_1d(field_ptr, ierr) + subroutine mpas_smart_field_client_recv_1d(field_ptr, ierr) type (field1dReal), pointer, intent(in) :: field_ptr integer, intent(out), optional :: ierr From c79733835b67d00e59887698ac1700c3cf282b5f Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Wed, 14 Sep 2022 15:08:23 -0700 Subject: [PATCH 26/29] Add dataset routines --- .../src/framework/mpas_smart_routines.F | 431 +++++++++++++++++- .../src/mode_forward/mpas_ocn_forward_mode.F | 58 +-- 2 files changed, 446 insertions(+), 43 deletions(-) diff --git a/components/mpas-framework/src/framework/mpas_smart_routines.F b/components/mpas-framework/src/framework/mpas_smart_routines.F index e9b258d67cfe..5461d3ff81ec 100644 --- a/components/mpas-framework/src/framework/mpas_smart_routines.F +++ b/components/mpas-framework/src/framework/mpas_smart_routines.F @@ -23,6 +23,7 @@ module mpas_smart_routines use mpas_threading use mpas_log use smartredis_client, only: client_type + use smartredis_dataset, only: dataset_type implicit none save @@ -48,6 +49,20 @@ module mpas_smart_routines module procedure mpas_smart_field_client_recv_4d end interface + interface mpas_smart_dataset_add_field_real + module procedure mpas_smart_dataset_add_field_real_1d + module procedure mpas_smart_dataset_add_field_real_2d + module procedure mpas_smart_dataset_add_field_real_3d + module procedure mpas_smart_dataset_add_field_real_4d + end interface + + interface mpas_smart_dataset_recv_field_real + module procedure mpas_smart_dataset_recv_field_real_1d + module procedure mpas_smart_dataset_recv_field_real_2d + module procedure mpas_smart_dataset_recv_field_real_3d + module procedure mpas_smart_dataset_recv_field_real_4d + end interface + contains subroutine mpas_smart_init(client_out, ierr) @@ -79,15 +94,16 @@ subroutine mpas_smart_init(client_out, ierr) end subroutine mpas_smart_init - subroutine mpas_smart_database_init(ierr) - - !implicit none + subroutine mpas_smart_dataset_init(dataset_name, dataset, ierr) + character(len=strKIND), intent(in) :: dataset_name + type(dataset_type), intent(out) :: dataset integer, intent(out), optional :: ierr character(len=100) :: log_string integer(kind=enum_kind) :: err_client + err_client = dataset%initialize(dataset_name) if (err_client /= SRNoError) then write(log_string,*) 'FWK: Error initializing redis database:', & err_client @@ -97,7 +113,25 @@ subroutine mpas_smart_database_init(ierr) call mpas_log_write('FWK: Database initialized') endif - end subroutine mpas_smart_database_init + end subroutine mpas_smart_dataset_init + + subroutine mpas_smart_dataset_put(dataset, ierr) + + type(dataset_type), intent(in) :: dataset + integer, intent(out), optional :: ierr + + integer(kind=enum_kind) :: err_client + + err_client = client%put_dataset(dataset) + if (err_client /= SRNoError) then + call mpas_log_write('FWK: Error putting redis dataset: $i', & + intArgs=(/err_client/)) + ierr = ior(ierr, 1) + else + call mpas_log_write('FWK: Dataset put') + endif + + end subroutine mpas_smart_dataset_put ! -------------------------------------------------------------------- ! SUBROUTINE mpas_smart_stream_send @@ -314,6 +348,161 @@ subroutine mpas_smart_field_recv(domain, fieldName, ierr) end subroutine mpas_smart_field_recv + subroutine mpas_smart_dataset_config_send(dataset, domain, configName, ierr) + + type(dataset_type), intent(in) :: dataset + type (domain_type), intent(in) :: domain + character(len=*), intent(in) :: configName + integer, intent(out), optional :: ierr + + type (mpas_pool_field_info_type) :: fieldInfo + real (kind=RKIND), pointer :: config_ptr + real (kind=RKIND), dimension(1) :: config_temp + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + + call mpas_pool_get_config(domain % configs, & + trim(configName), & + config_ptr) + config_temp(1) = config_ptr + err_client = dataset%add_meta_scalar(trim(configName), & + config_temp) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'FWK: Error put config in dataset:', trim(configName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'FWK: Put config in dataset:', trim(configName) + call mpas_log_write(trim(log_string)) + endif + + end subroutine mpas_smart_dataset_config_send + + subroutine mpas_smart_dataset_config_recv(dataset, domain, configName, ierr) + + type(dataset_type), intent(in) :: dataset + type (domain_type), intent(in) :: domain + character(len=*), intent(in) :: configName + integer, intent(out), optional :: ierr + + type (mpas_pool_field_info_type) :: fieldInfo + real (kind=RKIND), pointer :: config_ptr + real (kind=RKIND), dimension(1) :: config_temp + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + + call mpas_pool_get_config(domain % configs, & + trim(configName), & + config_ptr) + config_temp(1) = config_ptr + err_client = dataset%get_meta_scalar(trim(configName), & + config_temp) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'FWK: Error put config in dataset:', trim(configName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'FWK: Put config in dataset:', trim(configName) + call mpas_log_write(trim(log_string)) + endif + + end subroutine mpas_smart_dataset_config_recv + + subroutine mpas_smart_dataset_add_field(dataset, domain, fieldName, ierr) + + type(dataset_type), intent(in) :: dataset + type (domain_type), intent(in) :: domain + character(len=StrKIND), intent(in) :: fieldName + integer, intent(out), optional :: ierr + + type (mpas_pool_field_info_type) :: fieldInfo + type (field1DReal), pointer :: field_real1d_ptr + type (field2DReal), pointer :: field_real2d_ptr + type (field3DReal), pointer :: field_real3d_ptr + type (field4DReal), pointer :: field_real4d_ptr + character(len=StrKIND) :: log_string + + call mpas_pool_get_field_info(domain%blocklist%allFields, & + fieldName, fieldInfo) + call mpas_log_write(log_string) + if ( fieldInfo % fieldType == MPAS_POOL_REAL ) then + if ( fieldInfo % nDims == 1 ) then + call mpas_pool_get_field(domain%blocklist%allFields, & + trim(fieldName), field_real1d_ptr) + call mpas_log_write(trim(fieldName)//' retrieved') + call mpas_smart_dataset_add_field_real(dataset, field_real1d_ptr, ierr) + elseif ( fieldInfo % nDims == 2 ) then + call mpas_pool_get_field(domain%blocklist%allFields, & + trim(fieldName), field_real2d_ptr) + call mpas_log_write(trim(fieldName)//' retrieved') + call mpas_smart_dataset_add_field_real(dataset, field_real2d_ptr, ierr) + elseif ( fieldInfo % nDims == 3 ) then + call mpas_pool_get_field(domain%blocklist%allFields, & + trim(fieldName), field_real3d_ptr) + call mpas_log_write(trim(fieldName)//' retrieved') + call mpas_smart_dataset_add_field_real(dataset, field_real3d_ptr, ierr) + elseif ( fieldInfo % nDims == 4 ) then + call mpas_pool_get_field(domain%blocklist%allFields, & + trim(fieldName), field_real4d_ptr) + call mpas_log_write(trim(fieldName)//' retrieved') + call mpas_smart_dataset_add_field_real(dataset, field_real4d_ptr, ierr) + else + call mpas_log_write(trim(fieldName)// & + ' is not of a supported dimension') + endif + else + call mpas_log_write(trim(fieldName)//' is not real') + endif + + end subroutine mpas_smart_dataset_add_field + + subroutine mpas_smart_dataset_recv_field(dataset, domain, fieldName, ierr) + + type(dataset_type), intent(in) :: dataset + type (domain_type), intent(in) :: domain + character(len=StrKIND), intent(in) :: fieldName + integer, intent(out), optional :: ierr + + type (mpas_pool_field_info_type) :: fieldInfo + type (field1DReal), pointer :: field_real1d_ptr + type (field2DReal), pointer :: field_real2d_ptr + type (field3DReal), pointer :: field_real3d_ptr + type (field4DReal), pointer :: field_real4d_ptr + character(len=StrKIND) :: log_string + + call mpas_pool_get_field_info(domain%blocklist%allFields, & + fieldName, fieldInfo) + call mpas_log_write(log_string) + if ( fieldInfo % fieldType == MPAS_POOL_REAL ) then + if ( fieldInfo % nDims == 1 ) then + call mpas_pool_get_field(domain%blocklist%allFields, & + trim(fieldName), field_real1d_ptr) + call mpas_log_write(trim(fieldName)//' retrieved') + call mpas_smart_dataset_recv_field_real(dataset, field_real1d_ptr, ierr) + elseif ( fieldInfo % nDims == 2 ) then + call mpas_pool_get_field(domain%blocklist%allFields, & + trim(fieldName), field_real2d_ptr) + call mpas_log_write(trim(fieldName)//' retrieved') + call mpas_smart_dataset_recv_field_real(dataset, field_real2d_ptr, ierr) + elseif ( fieldInfo % nDims == 3 ) then + call mpas_pool_get_field(domain%blocklist%allFields, & + trim(fieldName), field_real3d_ptr) + call mpas_log_write(trim(fieldName)//' retrieved') + call mpas_smart_dataset_recv_field_real(dataset, field_real3d_ptr, ierr) + elseif ( fieldInfo % nDims == 4 ) then + call mpas_pool_get_field(domain%blocklist%allFields, & + trim(fieldName), field_real4d_ptr) + call mpas_log_write(trim(fieldName)//' retrieved') + call mpas_smart_dataset_recv_field_real(dataset, field_real4d_ptr, ierr) + else + call mpas_log_write(trim(fieldName)// & + ' is not of a supported dimension') + endif + else + call mpas_log_write(trim(fieldName)//' is not real') + endif + + end subroutine mpas_smart_dataset_recv_field ! INTERFACE mpas_smart_field_client_send ! TODO add optional argument to provide a different key @@ -561,4 +750,238 @@ subroutine mpas_smart_field_client_recv_4d(field_ptr, ierr) end subroutine mpas_smart_field_client_recv_4d +! INTERFACE mpas_smart_dataset_add_field_real + subroutine mpas_smart_dataset_add_field_real_1d(dataset, field_ptr, ierr) + + type(dataset_type), intent(in) :: dataset + type (field1dReal), pointer, intent(in) :: field_ptr + integer, intent(out), optional :: ierr + + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + real (kind=RKIND), dimension(:), allocatable :: field_temp + integer, dimension(1) :: field_dims + + field_dims = field_ptr % dimSizes + allocate(field_temp(field_dims(1))) + + err_client = dataset%add_tensor(trim(field_ptr % fieldName), & + field_temp, & + shape(field_temp)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'FWK: Error dataset add tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'FWK: Added tensor to dataset:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + endif + deallocate(field_temp) + + end subroutine mpas_smart_dataset_add_field_real_1d + + subroutine mpas_smart_dataset_add_field_real_2d(dataset, field_ptr, ierr) + + type(dataset_type), intent(in) :: dataset + type (field2dReal), pointer, intent(in) :: field_ptr + integer, intent(out), optional :: ierr + + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + real (kind=RKIND), dimension(:,:), allocatable :: field_temp + integer, dimension(2) :: field_dims + + field_dims = field_ptr % dimSizes + allocate(field_temp(field_dims(1),field_dims(2))) + + err_client = dataset%add_tensor(trim(field_ptr % fieldName), & + field_temp, & + shape(field_temp)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'FWK: Error dataset add tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'FWK: Added tensor to dataset:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + endif + deallocate(field_temp) + + end subroutine mpas_smart_dataset_add_field_real_2d + + subroutine mpas_smart_dataset_add_field_real_3d(dataset, field_ptr, ierr) + + type(dataset_type), intent(in) :: dataset + type (field3dReal), pointer, intent(in) :: field_ptr + integer, intent(out), optional :: ierr + + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + real (kind=RKIND), dimension(:,:,:), allocatable :: field_temp + integer, dimension(3) :: field_dims + + field_dims = field_ptr % dimSizes + allocate(field_temp(field_dims(1),field_dims(2),field_dims(3))) + + err_client = dataset%add_tensor(trim(field_ptr % fieldName), & + field_temp, & + shape(field_temp)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'FWK: Error dataset add tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'FWK: Added tensor to dataset:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + endif + deallocate(field_temp) + + end subroutine mpas_smart_dataset_add_field_real_3d + + subroutine mpas_smart_dataset_add_field_real_4d(dataset, field_ptr, ierr) + + type(dataset_type), intent(in) :: dataset + type (field4dReal), pointer, intent(in) :: field_ptr + integer, intent(out), optional :: ierr + + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + real (kind=RKIND), dimension(:,:,:,:), allocatable :: field_temp + integer, dimension(4) :: field_dims + + field_dims = field_ptr % dimSizes + allocate(field_temp(field_dims(1),field_dims(2),field_dims(3),field_dims(4))) + + err_client = dataset%add_tensor(trim(field_ptr % fieldName), & + field_temp, & + shape(field_temp)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'FWK: Error dataset add tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'FWK: Added tensor to dataset:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + endif + deallocate(field_temp) + + end subroutine mpas_smart_dataset_add_field_real_4d + +! INTERFACE mpas_smart_dataset_recv_field_real + subroutine mpas_smart_dataset_recv_field_real_1d(dataset, field_ptr, ierr) + + type(dataset_type), intent(in) :: dataset + type (field1dReal), pointer, intent(in) :: field_ptr + integer, intent(out), optional :: ierr + + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + real (kind=RKIND), dimension(:), allocatable :: field_temp + integer, dimension(1) :: field_dims + + field_dims = field_ptr % dimSizes + allocate(field_temp(field_dims(1))) + + err_client = dataset%unpack_dataset_tensor(trim(field_ptr % fieldName), & + field_temp, & + shape(field_temp)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'FWK: Error dataset unpack tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'FWK: Added tensor to dataset:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + endif + deallocate(field_temp) + + end subroutine mpas_smart_dataset_recv_field_real_1d + + subroutine mpas_smart_dataset_recv_field_real_2d(dataset, field_ptr, ierr) + + type(dataset_type), intent(in) :: dataset + type (field2dReal), pointer, intent(in) :: field_ptr + integer, intent(out), optional :: ierr + + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + real (kind=RKIND), dimension(:,:), allocatable :: field_temp + integer, dimension(2) :: field_dims + + field_dims = field_ptr % dimSizes + allocate(field_temp(field_dims(1),field_dims(2))) + + err_client = dataset%unpack_dataset_tensor(trim(field_ptr % fieldName), & + field_temp, & + shape(field_temp)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'FWK: Error dataset unpack tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'FWK: Added tensor to dataset:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + endif + deallocate(field_temp) + + end subroutine mpas_smart_dataset_recv_field_real_2d + + subroutine mpas_smart_dataset_recv_field_real_3d(dataset, field_ptr, ierr) + + type(dataset_type), intent(in) :: dataset + type (field3dReal), pointer, intent(in) :: field_ptr + integer, intent(out), optional :: ierr + + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + real (kind=RKIND), dimension(:,:,:), allocatable :: field_temp + integer, dimension(3) :: field_dims + + field_dims = field_ptr % dimSizes + allocate(field_temp(field_dims(1),field_dims(2),field_dims(3))) + + err_client = dataset%unpack_dataset_tensor(trim(field_ptr % fieldName), & + field_temp, & + shape(field_temp)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'FWK: Error dataset unpack tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'FWK: Added tensor to dataset:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + endif + deallocate(field_temp) + + end subroutine mpas_smart_dataset_recv_field_real_3d + + subroutine mpas_smart_dataset_recv_field_real_4d(dataset, field_ptr, ierr) + + type(dataset_type), intent(in) :: dataset + type (field4dReal), pointer, intent(in) :: field_ptr + integer, intent(out), optional :: ierr + + character(len=StrKIND) :: log_string + integer(kind=enum_kind) :: err_client + real (kind=RKIND), dimension(:,:,:,:), allocatable :: field_temp + integer, dimension(4) :: field_dims + + field_dims = field_ptr % dimSizes + allocate(field_temp(field_dims(1),field_dims(2),field_dims(3),field_dims(4))) + + err_client = dataset%unpack_dataset_tensor(trim(field_ptr % fieldName), & + field_temp, & + shape(field_temp)) + if (err_client /= SRNoError) then + WRITE(log_string,*) 'FWK: Error dataset unpack tensor:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + ierr = ior(ierr, 1) + else + WRITE(log_string,*) 'FWK: Added tensor to dataset:', trim(field_ptr % fieldName) + call mpas_log_write(trim(log_string)) + endif + deallocate(field_temp) + + end subroutine mpas_smart_dataset_recv_field_real_4d + end module mpas_smart_routines diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index 9dc1ba38d405..7fa9546c2ae7 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -149,6 +149,7 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ integer :: err_tmp ! local error codes from subroutine calls integer(kind=enum_kind) :: err_client character(len=100) :: log_string + character(len=StrKIND) :: datasetName, fieldName character(len=*), parameter :: script_key = 'init_script' character(len=*), parameter :: script_file = & 'data_processing_script.txt' @@ -231,7 +232,7 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ if (config_use_smartsim) then call mpas_smart_init(client, ierr) - if (ierr /= 0.0_RKIND) then + if (ierr /= 0) then call mpas_log_write('Error initializing smartredis client', & MPAS_LOG_CRIT) endif @@ -239,7 +240,7 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ ! ---- Example: Send config option to client ---- ! configName = 'config_mom_del2' ! call mpas_smart_config_send(domain, configName, ierr) - ! if (ierr /= 0.0_RKIND) then + ! if (ierr /= 0) then ! call mpas_log_write('Error: Smartsim put config at initialization') ! endif @@ -287,49 +288,28 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ ! call mpas_log_write('Post_process script run') !endif - err_client = dataset%initialize("example_fortran_dataset") - if (err_client .ne. SRNoError) then - call mpas_log_write('dataset initialization failed') - else - call mpas_log_write('Dataset initialized') - endif + datasetName = 'example_fortran_dataset' + call mpas_smart_dataset_init(datasetName, dataset, ierr) + fieldName = 'ssh' ! Add a tensor to the dataset and verify that we can retrieve it - err_client = dataset%add_tensor("dataset_send", sendArrayReal2d, & - shape(sendArrayReal2d)) - if (err_client .ne. SRNoError) then + call mpas_smart_dataset_add_field(dataset, domain, fieldName, ierr) + if (ierr /= 0) then call mpas_log_write('dataset%add_tensor failed') - else - call mpas_log_write('Dataset tensor added') endif - err_client = dataset%unpack_dataset_tensor( & - "dataset_send",recvArrayReal2d, shape(recvArrayReal2d)) - if (err_client .ne. SRNoError) then - write(log_string,*) 'dataset%unpack_dataset_tensor failed:',err_client - call mpas_log_write(log_string) - else - write(log_string,*) 'Dataset tensor received:',recvArrayReal2d(1,1) - call mpas_log_write(log_string) + call mpas_smart_dataset_recv_field(dataset, domain, fieldName, ierr) + if (ierr /= 0) then + call mpas_log_write('dataset%unpack_dataset_tensor failed') endif - - ! Add metascalars to the dataset and verify that we can retrieve them - err_client = dataset%add_meta_scalar('config_mom_del2', config_mom_del2) - if (err_client .ne. SRNoError) then - write(log_string,*) 'dataset%add_meta_scalar failed:',err_client - call mpas_log_write(log_string) - else - call mpas_log_write('Dataset metascalar added') + call mpas_smart_dataset_config_send(dataset, domain, configName, ierr) + if (ierr /= 0) then + call mpas_log_write('dataset%add_meta_scalar failed') endif - - !err_client = dataset%get_meta_scalars('config_mom_del2', meta_scalar) - !if (err_client .ne. SRNoError) then - ! write(log_string,*) 'dataset%get_meta_scalar failed:',err_client - ! call mpas_log_write(log_string) - !else - ! write(log_string,*) 'Dataset metascalar received:',meta_scalar - ! call mpas_log_write(log_string) - !endif - err_client = client%put_dataset(dataset) + call mpas_smart_dataset_config_recv(dataset, domain, configName, ierr) + if (ierr /= 0) then + call mpas_log_write('dataset%get_meta_scalar failed') + endif + call mpas_smart_dataset_put(dataset, ierr) if (err_client .ne. SRNoError) then write(log_string,*) 'client%put_dataset failed:',err_client call mpas_log_write(log_string) From a089bfa9ad7cd9d9f64bc9c1c329ded270c4f5b0 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 10 Oct 2022 07:31:24 -0700 Subject: [PATCH 27/29] Fixup dataset, client routines --- .../src/framework/mpas_smart_routines.F | 22 +++++++++---------- components/mpas-ocean/mpas_client_script.py | 16 +++++++++----- .../src/mode_forward/mpas_ocn_forward_mode.F | 5 +++-- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/components/mpas-framework/src/framework/mpas_smart_routines.F b/components/mpas-framework/src/framework/mpas_smart_routines.F index 5461d3ff81ec..1b93de5de860 100644 --- a/components/mpas-framework/src/framework/mpas_smart_routines.F +++ b/components/mpas-framework/src/framework/mpas_smart_routines.F @@ -28,7 +28,7 @@ module mpas_smart_routines implicit none save - public :: mpas_smart_init + public :: mpas_smart_client_init, mpas_smart_dataset_init # include "enum_fortran.inc" @@ -65,7 +65,7 @@ module mpas_smart_routines contains - subroutine mpas_smart_init(client_out, ierr) + subroutine mpas_smart_client_init(client_out, ierr) !implicit none @@ -92,7 +92,7 @@ subroutine mpas_smart_init(client_out, ierr) endif endif - end subroutine mpas_smart_init + end subroutine mpas_smart_client_init subroutine mpas_smart_dataset_init(dataset_name, dataset, ierr) @@ -366,7 +366,7 @@ subroutine mpas_smart_dataset_config_send(dataset, domain, configName, ierr) config_ptr) config_temp(1) = config_ptr err_client = dataset%add_meta_scalar(trim(configName), & - config_temp) + config_ptr) if (err_client /= SRNoError) then WRITE(log_string,*) 'FWK: Error put config in dataset:', trim(configName) call mpas_log_write(trim(log_string)) @@ -387,15 +387,14 @@ subroutine mpas_smart_dataset_config_recv(dataset, domain, configName, ierr) type (mpas_pool_field_info_type) :: fieldInfo real (kind=RKIND), pointer :: config_ptr - real (kind=RKIND), dimension(1) :: config_temp + real (kind=RKIND), dimension(:), pointer :: config_temp character(len=StrKIND) :: log_string integer(kind=enum_kind) :: err_client call mpas_pool_get_config(domain % configs, & trim(configName), & config_ptr) - config_temp(1) = config_ptr - err_client = dataset%get_meta_scalar(trim(configName), & + err_client = dataset%get_meta_scalars(trim(configName), & config_temp) if (err_client /= SRNoError) then WRITE(log_string,*) 'FWK: Error put config in dataset:', trim(configName) @@ -405,6 +404,7 @@ subroutine mpas_smart_dataset_config_recv(dataset, domain, configName, ierr) WRITE(log_string,*) 'FWK: Put config in dataset:', trim(configName) call mpas_log_write(trim(log_string)) endif + config_ptr = config_temp(1) end subroutine mpas_smart_dataset_config_recv @@ -526,7 +526,7 @@ subroutine mpas_smart_field_client_send_1d(field_ptr, ierr) field_temp, & shape(field_temp)) if (err_client /= SRNoError) then - WRITE(log_string,*) 'FWK: Error put tensor:', trim(field_ptr % fieldName) + WRITE(log_string,*) 'FWK: Error put tensor:', trim(field_ptr % fieldName), err_client call mpas_log_write(trim(log_string)) ierr = ior(ierr, 1) else @@ -556,7 +556,7 @@ subroutine mpas_smart_field_client_send_2d(field_ptr, ierr) shape(field_temp)) if (err_client /= SRNoError) then WRITE(log_string,*) 'Error put tensor:', & - trim(field_ptr % fieldName) + trim(field_ptr % fieldName), err_client call mpas_log_write(trim(log_string)) ierr = ior(ierr, 1) else @@ -586,7 +586,7 @@ subroutine mpas_smart_field_client_send_3d(field_ptr, ierr) shape(field_temp)) if (err_client /= SRNoError) then WRITE(log_string,*) 'Error put tensor:', & - trim(field_ptr % fieldName) + trim(field_ptr % fieldName), err_client call mpas_log_write(trim(log_string)) ierr = ior(ierr, 1) else @@ -617,7 +617,7 @@ subroutine mpas_smart_field_client_send_4d(field_ptr, ierr) shape(field_temp)) if (err_client /= SRNoError) then WRITE(log_string,*) 'Error put tensor:', & - trim(field_ptr % fieldName) + trim(field_ptr % fieldName), err_client call mpas_log_write(trim(log_string)) ierr = ior(ierr, 1) else diff --git a/components/mpas-ocean/mpas_client_script.py b/components/mpas-ocean/mpas_client_script.py index 3b3f4eade79d..257e898aa41f 100644 --- a/components/mpas-ocean/mpas_client_script.py +++ b/components/mpas-ocean/mpas_client_script.py @@ -14,18 +14,22 @@ SSDB=os.environ.get("SSDB") print(f'SSDB in client script {SSDB}') client = Client(address=SSDB, cluster=False) -recv_array = 20*numpy.ones(1) -err = client.put_tensor("init_recv", recv_array) -key_found = client.poll_key("config_mom_del2", 20, 10000) + +#recv_array = 20*numpy.ones(1) +#err = client.put_tensor("init_recv", recv_array) + +# Test smartsend client recv +key_found = client.poll_key("ssh", 20, 10000) print('key_found',key_found) if key_found: - dummy_array = client.get_tensor("config_mom_del2") - print('init_send_array=',dummy_array[0]) + dummy_array = client.get_tensor("ssh") + print('smartsend =',dummy_array[0]) else: print('key not found') +# Test dataset capabilities mpas_dataset = client.get_dataset('example_fortran_dataset') -mpas_tensor = mpas_dataset.get_tensor('dataset_send') +mpas_tensor = mpas_dataset.get_tensor('ssh') print(f'dataset_send tensor={mpas_tensor[0,0]}') config_mom_del2 = mpas_dataset.get_meta_scalars('config_mom_del2') print(f'dataset config_mom_del2={config_mom_del2}') diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F index 7fa9546c2ae7..7b399c133f4c 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_forward_mode.F @@ -149,7 +149,7 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ integer :: err_tmp ! local error codes from subroutine calls integer(kind=enum_kind) :: err_client character(len=100) :: log_string - character(len=StrKIND) :: datasetName, fieldName + character(len=StrKIND) :: datasetName, configName, fieldName character(len=*), parameter :: script_key = 'init_script' character(len=*), parameter :: script_file = & 'data_processing_script.txt' @@ -231,7 +231,7 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ block => domain % blocklist if (config_use_smartsim) then - call mpas_smart_init(client, ierr) + call mpas_smart_client_init(client, ierr) if (ierr /= 0) then call mpas_log_write('Error initializing smartredis client', & MPAS_LOG_CRIT) @@ -301,6 +301,7 @@ function ocn_forward_mode_init(domain, startTimeStamp) result(ierr)!{{{ if (ierr /= 0) then call mpas_log_write('dataset%unpack_dataset_tensor failed') endif + configName = 'config_mom_del2' call mpas_smart_dataset_config_send(dataset, domain, configName, ierr) if (ierr /= 0) then call mpas_log_write('dataset%add_meta_scalar failed') From f88bca277480fda68d93e025a77901d97f8e752c Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 10 Oct 2022 13:50:07 -0700 Subject: [PATCH 28/29] Add error parser to smart routines --- components/mpas-framework/src/framework/mpas_smart_routines.F | 1 + 1 file changed, 1 insertion(+) diff --git a/components/mpas-framework/src/framework/mpas_smart_routines.F b/components/mpas-framework/src/framework/mpas_smart_routines.F index 1b93de5de860..c2145b67c762 100644 --- a/components/mpas-framework/src/framework/mpas_smart_routines.F +++ b/components/mpas-framework/src/framework/mpas_smart_routines.F @@ -85,6 +85,7 @@ subroutine mpas_smart_client_init(client_out, ierr) write(log_string,*) 'FWK: Error initializing redis client:', & err_client call mpas_log_write(log_string) + err_client = client%SR_error_parser(err_client) ierr = ior(ierr, 1) else call mpas_log_write('FWK: Client initialized') From 137fce214c97eabda7924a5ee690155948019e78 Mon Sep 17 00:00:00 2001 From: Carolyn Begeman Date: Mon, 10 Oct 2022 13:50:31 -0700 Subject: [PATCH 29/29] Update helper scripts --- components/mpas-ocean/get_ssdb.py | 2 +- components/mpas-ocean/launch_db_for_mpaso.py | 9 ++++----- components/mpas-ocean/run_mpaso_and_script.sh | 3 ++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/mpas-ocean/get_ssdb.py b/components/mpas-ocean/get_ssdb.py index cfd4a5e7f4ca..429d654e09e3 100644 --- a/components/mpas-ocean/get_ssdb.py +++ b/components/mpas-ocean/get_ssdb.py @@ -1,6 +1,6 @@ import shutil, time, sys, os, subprocess, fileinput -with open('db_debug.log') as f: +with open('/global/homes/c/cbegeman/E3SM-new/components/mpas-ocean/db_debug.log') as f: lines = f.readlines() for line in lines: if 'SSDB' in line: diff --git a/components/mpas-ocean/launch_db_for_mpaso.py b/components/mpas-ocean/launch_db_for_mpaso.py index 1246be07df1c..48d66cb684ef 100644 --- a/components/mpas-ocean/launch_db_for_mpaso.py +++ b/components/mpas-ocean/launch_db_for_mpaso.py @@ -5,8 +5,7 @@ import time import os -MPASO_DIR="/global/cscratch1/sd/cbegeman/MPAS-Ocean-test-case-output/smartsim/ocean/baroclinic_channel/10km/default" - +BUILD_DIR="/global/homes/c/cbegeman/E3SM-new/components/mpas-ocean/" print('Create Experiment object') exp = Experiment("mpas-ocean_simulation", launcher="slurm") print('Create database') @@ -19,9 +18,9 @@ partition="standard", batch_args={"C":"haswell"}) -if os.path.exists('{}/db_debug.log'.format(MPASO_DIR)): - os.system('rm {}/db_debug.log'.format(MPASO_DIR)) -log_to_file('{}/db_debug.log'.format(MPASO_DIR)) +if os.path.exists('{}/db_debug.log'.format(BUILD_DIR)): + os.system('rm {}/db_debug.log'.format(BUILD_DIR)) +log_to_file('{}/db_debug.log'.format(BUILD_DIR)) logger = get_logger('db_launcher') # define how simulation should be executed diff --git a/components/mpas-ocean/run_mpaso_and_script.sh b/components/mpas-ocean/run_mpaso_and_script.sh index 8ce98d08955e..a3090f249e03 100644 --- a/components/mpas-ocean/run_mpaso_and_script.sh +++ b/components/mpas-ocean/run_mpaso_and_script.sh @@ -1,7 +1,8 @@ -CASE_DIR=/global/cscratch1/sd/cbegeman/MPAS-Ocean-test-case-output/smartsim/ocean/baroclinic_channel/10km/default +CASE_DIR=/global/cscratch1/sd/cbegeman/MPAS-Ocean-test-case-output/smartsim/ocean/baroclinic_channel/10km/default/ MODEL_DIR=/global/homes/c/cbegeman/E3SM-new/components/mpas-ocean conda activate smartsim cp $MODEL_DIR/environ.sh $CASE_DIR/. +cp $CASE_DIR/db_debug.log $MODEL_DIR/. cd $CASE_DIR python $MODEL_DIR/get_ssdb.py echo "completed get_ssdb"