diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index 5801f71e2..ec72581fa 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -30,6 +30,7 @@ set(BaseSources "StructureFactory.cpp" "ModelArray.cpp" "ModelArraySlice.cpp" + "ModelArrayStore.cpp" "ModelComponent.cpp" "ModelMetadata.cpp" "NetcdfMetadataConfiguration.cpp" @@ -61,6 +62,10 @@ if(ENABLE_MPI) list(APPEND NextsimSourcesTemp "${ParallelNetCDFSources}") endif() +if(WITH_KOKKOS) + list(APPEND NextsimSourcesTemp "${CMAKE_CURRENT_SOURCE_DIR}/kokkos/KokkosUtils.cpp") +endif() + set(NextsimSources "${NextsimSourcesTemp}" PARENT_SCOPE) set(NextsimIncludeDirs diff --git a/core/src/FieldAdvection.cpp b/core/src/FieldAdvection.cpp index bd10a1d68..aff7a71f5 100644 --- a/core/src/FieldAdvection.cpp +++ b/core/src/FieldAdvection.cpp @@ -20,4 +20,13 @@ ModelArray& FieldAdvection::advectField( return field; } +#ifdef USE_KOKKOS +void FieldAdvection::advectField( + const DeviceViewMA& field, const TimestepTime& tst, double lowerLimit, double upperLimit) +{ + Module::getImplementation().advectField( + tst.step.seconds(), field, lowerLimit, upperLimit); +} +#endif + } /* namespace Nextsim */ diff --git a/core/src/Model.cpp b/core/src/Model.cpp index ec10187cb..f37e8ebc4 100644 --- a/core/src/Model.cpp +++ b/core/src/Model.cpp @@ -169,6 +169,8 @@ Model::HelpMap& Model::getHelpRecursive(HelpMap& map, bool getAll) void Model::run() { + assert(Nextsim::ModelComponent::getStore().checkAllRegistered()); + try { iterator.run(); } catch (const std::exception& e) { diff --git a/core/src/ModelArrayStore.cpp b/core/src/ModelArrayStore.cpp new file mode 100644 index 000000000..d0bee674f --- /dev/null +++ b/core/src/ModelArrayStore.cpp @@ -0,0 +1,103 @@ +/*! + * @author Robert Jendersie + */ + +#include "include/ModelArrayStore.hpp" +#include "include/Logged.hpp" +#include +namespace Nextsim { + +#ifdef USE_KOKKOS + +// double ConstDeviceModelArray::operator[](DeviceIndex i) const { return m_deviceView(i, 0); } +ConstDeviceModelArray::operator ConstDeviceViewMA() const { return m_deviceView; } + +// double& DeviceModelArray::operator[](DeviceIndex i) const { return m_deviceView(i, 0); } +DeviceModelArray::operator const DeviceViewMA&() const { return m_deviceView; } + +/*************************************************************/ +HostViewMA ModelArrayStore::ExtModelArray::hostView() +{ + assert(modelArray.trueSize() > 0 && "ModelArray is allocated"); + return makeKokkosHostView(modelArray.getDataRef()); +} + +const DeviceViewMA& ModelArrayStore::ExtModelArray::deviceView() +{ + assert(modelArray.trueSize() > 0 && "ModelArray is allocated"); + if (!m_deviceModelArray.m_deviceView.is_allocated()) { + m_deviceModelArray.m_deviceView = makeKokkosDeviceView(name, modelArray.getDataRef()); + } + + return m_deviceModelArray.m_deviceView; +} + +DeviceModelArray& ModelArrayStore::ExtModelArray::deviceModelArray() +{ + // handles initialization + deviceView(); + return m_deviceModelArray; +} +#endif + +/*************************************************************/ +ModelArrayStore::ExtModelArrayFlagged::ExtModelArrayFlagged( + const std::string& name, bool _isReadWrite, bool _isRegistered) + : isReadWrite(_isReadWrite) + , isRegistered(_isRegistered) +{ + extModelArray.name = name; +} + +/*************************************************************/ +std::unordered_map ModelArrayStore::getAllData() const +{ + std::unordered_map dataMap; + + for (const auto& [name, extArrFlagged] : store) { + dataMap.emplace(name, &extArrFlagged.extModelArray.modelArray); + } + + return dataMap; +} + +bool ModelArrayStore::checkAllRegistered() const +{ + bool b = true; + for (const auto& [_, extArrFlagged] : store) { + if (!extArrFlagged.isRegistered) { + Logged::warning("Field \"" + extArrFlagged.extModelArray.name + + "\" was not properly initialized by a registering ModelArrayAccessor.\n"); + b = false; + } + } + + return b; +} + +ModelArrayStore::ExtModelArray& ModelArrayStore::getRW(const std::string& field) +{ + if (auto it = store.find(field); it != store.end()) { + ExtModelArrayFlagged& extArrFlagged = it->second; + if (!extArrFlagged.isReadWrite) { + if (extArrFlagged.isRegistered) { + throw std::logic_error( + "Trying to access the read-only ModelArray \"" + field + "\" as read-write."); + } else { + // promote to read-write because we don't now the true restriction yet + extArrFlagged.isReadWrite = true; + } + } + return extArrFlagged.extModelArray; + } + + // Regular emplace would be fine here since we know that it does not exist but try_emplace + // has a more ergonomic signature. + return store.try_emplace(field, field, RW, false).first->second.extModelArray; +} + +ModelArrayStore::ExtModelArray& ModelArrayStore::getRO(const std::string& field) +{ + return store.try_emplace(field, field, RO, false).first->second.extModelArray; +} +} \ No newline at end of file diff --git a/core/src/ModelComponent.cpp b/core/src/ModelComponent.cpp index 2fbfec1b6..c9c8b6fcf 100644 --- a/core/src/ModelComponent.cpp +++ b/core/src/ModelComponent.cpp @@ -8,6 +8,11 @@ namespace Nextsim { size_t ModelComponent::nOcean = 0; std::vector ModelComponent::oceanIndex; +bool ModelComponent::columnPhysicsStoreIsDestroyed; // initialized to 0 because of static lifetime + +#if USE_KOKKOS +KokkosDeviceMapView ModelComponent::oceanIndexDevice; +#endif ModelComponent::ModelComponent() { @@ -38,6 +43,10 @@ void ModelComponent::setOceanMask(const ModelArray& mask) oceanIndex[iOceanIndex++] = i; } } + +#if USE_KOKKOS + makeOceanIndexDevice(); +#endif } // Fills the nOcean and OceanIndex variables for the zero land case @@ -53,6 +62,10 @@ void ModelComponent::noLandMask() for (size_t i = 0; i < ModelArray::size(ModelArray::Type::H); ++i) { oceanIndex[i] = i; } + +#if USE_KOKKOS + makeOceanIndexDevice(); +#endif } ModelArray ModelComponent::mask(const ModelArray& data, const double missingValue) @@ -67,4 +80,20 @@ ModelArray ModelComponent::mask(const ModelArray& data, const double missingValu const ModelArray& ModelComponent::oceanMask() { return oceanMaskSingleton(); } +#if USE_KOKKOS +void ModelComponent::makeOceanIndexDevice() +{ + // some tests don't need Kokkos + if (!Kokkos::is_initialized()) { + return; + } + + std::vector buf(oceanIndex.begin(), oceanIndex.end()); + oceanIndexDevice = makeKokkosDeviceViewMap("oceanIndex", buf, MakeViewOptions::ALWAYS_COPY); + Finalizer::registerUnique(destroyOceanIndex); +} + +void ModelComponent::destroyOceanIndex() { oceanIndexDevice.assign_data(nullptr); } +#endif + } /* namespace Nextsim */ diff --git a/core/src/PrognosticData.cpp b/core/src/PrognosticData.cpp index 140d6d7cc..0916940e9 100644 --- a/core/src/PrognosticData.cpp +++ b/core/src/PrognosticData.cpp @@ -7,7 +7,6 @@ #include "include/FieldAdvection.hpp" #include "include/Finalizer.hpp" -#include "include/ModelArrayRef.hpp" #include "include/NextsimModule.hpp" #include "include/gridNames.hpp" @@ -23,18 +22,14 @@ static constexpr bool checkFieldsFastDefault = true; PrognosticData::PrognosticData() : m_dt(1) - , hice(ModelArray::AdvectionType, { 0, 50 }) - , cice(ModelArray::AdvectionType, { 0, 1 }) - , damage(ModelArray::AdvectionType, { 0, 1 }) - , hsnow(ModelArray::AdvectionType, { 0, 10 }) + , hiceAccessor(getStore(), RW, ModelArray::AdvectionType, std::pair(0.0, 50.0)) + , ciceAccessor(getStore(), RW, ModelArray::AdvectionType, std::pair(0.0, 1.0)) + , damageAccessor(getStore(), RW, ModelArray::AdvectionType, std::pair(0.0, 1.0)) + , hsnowAccessor(getStore(), RW, ModelArray::AdvectionType, std::pair(0.0, 10.0)) , pAtmBdy(nullptr) , pOcnBdy(nullptr) , pDynamics(nullptr) { - getStore().registerArray(Shared::DAMAGE, &damage, RW); - getStore().registerArray(Shared::H_ICE_DG, &hice, RW); - getStore().registerArray(Shared::C_ICE_DG, &cice, RW); - getStore().registerArray(Shared::H_SNOW_DG, &hsnow, RW); } void PrognosticData::configure() @@ -64,8 +59,8 @@ void PrognosticData::configure() } } else if (checkFast) { addChecks({ - { "thickness", &hice }, - { "concentration", &cice }, + { "thickness", &hiceAccessor.getHostRO() }, + { "concentration", &ciceAccessor.getHostRO() }, }); } } @@ -82,6 +77,10 @@ void copyMeanComponent(const ModelArray& source, ModelArray& sink) void PrognosticData::setData(const ModelState::DataMap& ms) { + AdvectedField& hice = hiceAccessor.getHostRW(); + AdvectedField& cice = ciceAccessor.getHostRW(); + AdvectedField& hsnow = hsnowAccessor.getHostRW(); + AdvectedField& damage = damageAccessor.getHostRW(); if (ms.count(maskName)) { setOceanMask(ms.at(maskName)); @@ -154,9 +153,9 @@ ModelState PrognosticData::getStatePrognostic() const { ModelState state = { { { maskName, ModelArray(oceanMask()) }, // make a copy - { hiceName, hice }, - { ciceName, cice }, - { hsnowName, hsnow }, + { hiceName, hiceAccessor.getHostRO() }, + { ciceName, ciceAccessor.getHostRO() }, + { hsnowName, hsnowAccessor.getHostRO() }, }, ModelComponent::getConfiguration() }; diff --git a/core/src/discontinuousgalerkin/include/ModelArrayDetails.hpp b/core/src/discontinuousgalerkin/include/ModelArrayDetails.hpp index 589b4a389..19ca9d32f 100644 --- a/core/src/discontinuousgalerkin/include/ModelArrayDetails.hpp +++ b/core/src/discontinuousgalerkin/include/ModelArrayDetails.hpp @@ -25,7 +25,7 @@ enum class Type { enum class Base { Cell, Vertex }; -static const Type AdvectionType = Type::DG; +static constexpr Type AdvectionType = Type::DG; static ModelArray HField() { return ModelArray(Type::H); } static ModelArray VertexField() { return ModelArray(Type::VERTEX); } diff --git a/core/src/include/FieldAdvection.hpp b/core/src/include/FieldAdvection.hpp index 963a2a12d..9964116e2 100644 --- a/core/src/include/FieldAdvection.hpp +++ b/core/src/include/FieldAdvection.hpp @@ -11,6 +11,10 @@ #include "include/ModelArray.hpp" #include "include/Time.hpp" +#ifdef USE_KOKKOS +#include "include/ModelArrayStore.hpp" // needed for DeviceViewMA +#endif + namespace Nextsim { class FieldAdvection { @@ -26,6 +30,18 @@ class FieldAdvection { static ModelArray& advectField(ModelArray& field, const TimestepTime& tst, double lowerLimit = -std::numeric_limits::infinity(), double upperLimit = std::numeric_limits::infinity()); + +#ifdef USE_KOKKOS + static void advectField(const DeviceViewMA& field, const TimestepTime& tst, + double lowerLimit = -std::numeric_limits::infinity(), + double upperLimit = std::numeric_limits::infinity()); + +/* template + static void advectField(ExecSpace execSpace, const DeviceViewMA& field, const TimestepTime& tst, + double lowerLimit = -std::numeric_limits::infinity(), + double upperLimit = std::numeric_limits::infinity()) + {}*/ +#endif }; } /* namespace Nextsim */ diff --git a/core/src/include/KernelAlternatives.hpp b/core/src/include/KernelAlternatives.hpp new file mode 100644 index 000000000..12ce421f5 --- /dev/null +++ b/core/src/include/KernelAlternatives.hpp @@ -0,0 +1,51 @@ +/*! + * @author Robert Jendersie + */ + +#ifndef KERNEL_ALTERNATIVES_HPP +#define KERNEL_ALTERNATIVES_HPP + +#ifdef USE_KOKKOS +#include "../kokkos/include/KokkosUtils.hpp" +#include +#else +#include +#include +#endif + +namespace Nextsim { + +// namespace Utils contains standard math functions (sin, sqrt, min, ...) +#ifdef USE_KOKKOS +namespace Utils = Kokkos; +#else +namespace Utils = std; +#endif + +// KERNEL_IMPL_FUNCTION annotates functions that need to be callable inside a kernel +#ifdef USE_KOKKOS +#define KERNEL_IMPL_FUNCTION KOKKOS_IMPL_FUNCTION +#else +// no special annotation needed +#define KERNEL_IMPL_FUNCTION +#endif + +// KERNEL_LAMBDA capture list of a lambda used in a overElements call +#ifdef USE_KOKKOS +#define OVER_ELEMENTS_LAMBDA KOKKOS_LAMBDA +using ElementIndex = DeviceIndex; +#else +#define OVER_ELEMENTS_LAMBDA [&] +using ElementIndex = size_t; +#endif + +// Execution space that is used to signal async operations. +#ifdef USE_KOKKOS +using DefaultExecutionSpace = Kokkos::DefaultExecutionSpace; +#else +struct ExecutionSpaceDummy { }; +using DefaultExecutionSpace = ExecutionSpaceDummy; +#endif +} + +#endif /* KERNEL_ALTERNATIVES_HPP */ \ No newline at end of file diff --git a/core/src/include/ModelArrayAccessor.hpp b/core/src/include/ModelArrayAccessor.hpp new file mode 100644 index 000000000..61c6c4f89 --- /dev/null +++ b/core/src/include/ModelArrayAccessor.hpp @@ -0,0 +1,183 @@ +/*! + * @author Robert Jendersie + */ + +#ifndef MODELARRAYACCESSOR_HPP +#define MODELARRAYACCESSOR_HPP + +#include "ModelArrayStore.hpp" + +namespace Nextsim { + +template class ModelArrayAccessor; + +template class ModelArrayAccessor { +public: + ModelArrayAccessor(ModelArrayStore& store) + : target(store.getRO(fieldName)) + { + } + + const ModelArray& getHostRO() const + { +#ifdef USE_KOKKOS + if (target.syncState == SyncState::DEVICE_CHANGED) { + Kokkos::deep_copy(target.hostView(), target.deviceView()); + target.syncState = SyncState::SYNCED; + } +#endif + return target.modelArray; + } + +#ifdef USE_KOKKOS + template const ModelArray& getHostRO(ExecSpace execSpace) const + { + if (target.syncState == SyncState::DEVICE_CHANGED) { + Kokkos::deep_copy(execSpace, target.hostView(), target.deviceView()); + target.syncState = SyncState::SYNCED; + } + return target.modelArray; + } + + // returns a copy because target.deviceView has mutable data + ConstDeviceViewMA getDeviceRO() const + { + assert(target.modelArray.trueSize() > 0 && "ModelArray is allocated"); + + DeviceViewMA deviceView = target.deviceView(); + if (target.syncState == SyncState::HOST_CHANGED) { + Kokkos::deep_copy(deviceView, target.hostView()); + target.syncState = SyncState::SYNCED; + } + + return deviceView; + } + + template ConstDeviceViewMA getDeviceRO(ExecSpace execSpace) const + { + assert(target.modelArray.trueSize() > 0 && "ModelArray is allocated"); + + DeviceViewMA deviceView = target.deviceView(); + if (target.syncState == SyncState::HOST_CHANGED) { + Kokkos::deep_copy(execSpace, deviceView, target.hostView()); + target.syncState = SyncState::SYNCED; + } + + return deviceView; + } +#endif + + // decltype(auto) for perfect forwarding of either copy or const reference + decltype(auto) getAutoRO() const + { +#ifdef USE_KOKKOS + return ConstDeviceModelArray(getDeviceRO()); +#else + return getHostRO(); +#endif + } + template decltype(auto) getAutoRO(ExecSpace execSpace) const + { +#ifdef USE_KOKKOS + return ConstDeviceModelArray(getDeviceRO(execSpace)); +#else + return getHostRO(); +#endif + } + +protected: + // for the RW version + ModelArrayAccessor(ModelArrayStore::ExtModelArray& _target) + : target(_target) + { + } + // lifetime and persistent address are enforced by ModelArrayStore + ModelArrayStore::ExtModelArray& target; +}; + +template +class ModelArrayAccessor : public ModelArrayAccessor { + using Base = ModelArrayAccessor; + +public: + ModelArrayAccessor(ModelArrayStore& store) + : Base(store.getRW(fieldName)) + { + } + + template + ModelArrayAccessor(ModelArrayStore& store, bool isReadWriteExternal, Args&&... args) + // using ModelArrayAccessor directly instead of Base here leads to a compiler + // error in ModelArrayAccessor_test.cpp + : Base(store.registerArray(fieldName, isReadWriteExternal, std::forward(args)...)) + { + } + + ModelArray& getHostRW() + { +#ifdef USE_KOKKOS + if (this->target.syncState == SyncState::DEVICE_CHANGED) { + Kokkos::deep_copy(this->target.hostView(), this->target.deviceView()); + } + this->target.syncState = SyncState::HOST_CHANGED; +#endif + + return this->target.modelArray; + } + +#ifdef USE_KOKKOS + template ModelArray& getHostRW(ExecSpace execSpace) + { + if (this->target.syncState == SyncState::DEVICE_CHANGED) { + Kokkos::deep_copy(execSpace, this->target.hostView(), this->target.deviceView()); + } + this->target.syncState = SyncState::HOST_CHANGED; + + return this->target.modelArray; + } + + const DeviceViewMA& getDeviceRW() + { + const DeviceViewMA& deviceView = this->target.deviceView(); + if (this->target.syncState == SyncState::HOST_CHANGED) + Kokkos::deep_copy(deviceView, this->target.hostView()); + this->target.syncState = SyncState::DEVICE_CHANGED; + + return deviceView; + } + + template const DeviceViewMA& getDeviceRW(ExecSpace execSpace) + { + const DeviceViewMA& deviceView = this->target.deviceView(); + if (this->target.syncState == SyncState::HOST_CHANGED) + Kokkos::deep_copy(execSpace, deviceView, this->target.hostView()); + this->target.syncState = SyncState::DEVICE_CHANGED; + + return deviceView; + } +#endif + + auto& getAutoRW() + { +#ifdef USE_KOKKOS + // sync buffers + getDeviceRW(); + return this->target.deviceModelArray(); +#else + return getHostRW(); +#endif + } + template auto& getAutoRW(ExecSpace execSpace) + { +#ifdef USE_KOKKOS + // sync buffers + getDeviceRW(execSpace); + return this->target.deviceModelArray(); +#else + return getHostRW(); +#endif + } +}; +} + +#endif /* MODELARRAYACCESSOR_HPP */ \ No newline at end of file diff --git a/core/src/include/ModelArrayStore.hpp b/core/src/include/ModelArrayStore.hpp new file mode 100644 index 000000000..a1c50a32a --- /dev/null +++ b/core/src/include/ModelArrayStore.hpp @@ -0,0 +1,141 @@ +/*! + * @author Robert Jendersie + */ + +#ifndef MASTORE_HPP +#define MASTORE_HPP + +#include "../kokkos/include/KokkosModelArray.hpp" +#include "ModelArray.hpp" +#include "ModelArrayRef.hpp" // for RW,RO globals + +#include +#include + +struct TextTag; + +namespace Nextsim { + +#ifdef USE_KOKKOS + +// Wrapper for Kokkos views with semantics closer to ModelArray +class ConstDeviceModelArray { +public: + ConstDeviceModelArray(ConstDeviceViewMA deviceView) + : m_deviceView(std::move(deviceView)) + { + } + + KOKKOS_IMPL_FUNCTION double operator[](DeviceIndex i) const { return m_deviceView(i, 0); } + + operator ConstDeviceViewMA() const; + +private: + ConstDeviceViewMA m_deviceView; + + friend class ModelArrayStore; +}; +class DeviceModelArray { +public: + // DeviceModelArray(DeviceViewMA deviceView) : m_deviceView(std::move(m_deviceView)) {} + + KOKKOS_IMPL_FUNCTION double& operator[](DeviceIndex i) const { return m_deviceView(i, 0); } + + operator const DeviceViewMA&() const; + +private: + DeviceModelArray() = default; + DeviceViewMA m_deviceView; + + friend class ModelArrayStore; +}; + +enum struct SyncState { SYNCED, HOST_CHANGED, DEVICE_CHANGED }; +#endif + +class ModelArrayStore { +public: + std::unordered_map getAllData() const; + + // Checks that all known fields have been properly registered with an Accessor that provided the + // constructor arguments. + // @return True iff all fields are registered. + bool checkAllRegistered() const; + +private: + struct ExtModelArray { + std::string name; + ModelArray modelArray; + +#ifdef USE_KOKKOS + SyncState syncState = SyncState::SYNCED; + // not a simple data member because the host buffer owned by ModelArray can be overwritten + HostViewMA hostView(); + // handles lazy initialization for the device buffer + const DeviceViewMA& deviceView(); + DeviceModelArray& deviceModelArray(); + + private: + DeviceModelArray m_deviceModelArray; +#endif + }; + + struct ExtModelArrayFlagged { + ExtModelArrayFlagged(const std::string& name, bool _isReadWrite, bool _isRegistered); + + bool isReadWrite; + bool isRegistered; + ExtModelArray extModelArray; + }; + + template + ExtModelArray& registerArray(const std::string& field, bool isReadWrite, Args&&... args) + { + auto it = store.find(field); + if (it != store.end()) { + ExtModelArrayFlagged& extArrFlagged = it->second; + if (extArrFlagged.isReadWrite == RW && isReadWrite == RO) { + throw std::logic_error("Registering ModelArray \"" + field + + "\" as read-only but at least one accessor requested read-write."); + } + + if (extArrFlagged.isRegistered) { + // Double registration unfortunately has to be allowed because two instances + // of each ModelComponent are created which try to register the same fields. + // This loophole can be used to get a RW access for a field that should be RO. + // throw std::logic_error( + // "Registering ModelArray \"" + field + "\" but it was already registered."); + if (extArrFlagged.isReadWrite != isReadWrite) { + throw std::logic_error("Registering ModelArray \"" + field + + "\" but it was already registered with different access restrictions."); + } + } else { + extArrFlagged.isReadWrite = isReadWrite; + extArrFlagged.isRegistered = true; + } + } else { + // Regular emplace would be fine here since we know that it does not exist but + // try_emplace has a more ergonomic signature. + it = store.try_emplace(field, field, isReadWrite, true).first; + } + + ExtModelArray& extArr = it->second.extModelArray; + + // destroy and construct inplace to keep pointers valid + extArr.modelArray.~ModelArray(); + new (&extArr.modelArray) ModelArray(std::forward(args)...); + + return extArr; + } + + ExtModelArray& getRW(const std::string& field); + ExtModelArray& getRO(const std::string& field); + + std::unordered_map store; + + template friend class ModelArrayAccessor; +}; + +} + +#endif /* MARSTORE_HPP */ diff --git a/core/src/include/ModelComponent.hpp b/core/src/include/ModelComponent.hpp index f4772c2b4..e348cacf0 100644 --- a/core/src/include/ModelComponent.hpp +++ b/core/src/include/ModelComponent.hpp @@ -6,10 +6,10 @@ #ifndef MODELCOMPONENT_HPP #define MODELCOMPONENT_HPP +#include "include/Finalizer.hpp" #include "include/Logged.hpp" #include "include/MissingData.hpp" -#include "include/ModelArrayRef.hpp" -#include "include/ModelArrayReferenceStore.hpp" +#include "include/ModelArrayStore.hpp" #include "include/ModelState.hpp" #include "include/OutputSpec.hpp" #include "include/TextTag.hpp" @@ -132,7 +132,7 @@ namespace CouplingFields { /*! * A class encapsulating a component of the model. It also provide a method of * communicating data between ModelComponents using enums, static arrays of - * pointers and the ModelArrayRef class. + * pointers and the ModelArrayAccessor class. */ class ModelComponent { public: @@ -180,9 +180,37 @@ class ModelComponent { virtual ModelState getStatePrognostic() const { return { {}, getConfiguration() }; } /*! - * @brief Returns the ModelArrayRef backing store for column physics fields. + * @brief Returns the ModelArrayAccessor backing store for column physics fields. */ - static ModelArrayReferenceStore& getStore() { return columnPhysicsStore(); } + static ModelArrayStore& getStore() { return *columnPhysicsStore(); } + + // needs to be public because it calls a device function but should be used as if protected +#if USE_KOKKOS + template inline static void overElementsDevice(Fn fn) + { + // the static member can not be captured directly + const auto oceanIndexLocal = oceanIndexDevice; + Kokkos::parallel_for( + "overElements", nOcean, KOKKOS_LAMBDA(const DeviceIndex i) { fn(oceanIndexLocal[i]); }); + } +#endif + + template inline static void overElementsAuto(Fn&& fn) + { +#if USE_KOKKOS + overElementsDevice(std::forward(fn)); +#else + overElements(std::forward(fn)); +#endif + } + + template static void overElements(Fn fn) + { +#pragma omp parallel for + for (size_t i = 0; i < nOcean; ++i) { + fn(oceanIndex[i]); + } + } protected: inline static void overElements(IteratedFn fn, const TimestepTime& tst) @@ -231,14 +259,36 @@ class ModelComponent { } private: - static ModelArrayReferenceStore& columnPhysicsStore() + static std::unique_ptr& columnPhysicsStore() + { + static std::unique_ptr storePtr; + + if (!storePtr) { + if (columnPhysicsStoreIsDestroyed) { + Logged::warning("Trying to access the ModelArray store after it was destroyed."); + } else { + storePtr = std::make_unique(); + Finalizer::registerUnique(destroyModelArrayStore); + } + } + return storePtr; + } + static void destroyModelArrayStore() { - static ModelArrayReferenceStore store; - return store; + columnPhysicsStore().reset(nullptr); + columnPhysicsStoreIsDestroyed = true; } static size_t nOcean; static std::vector oceanIndex; + static bool columnPhysicsStoreIsDestroyed; + +#if USE_KOKKOS + static void makeOceanIndexDevice(); + static void destroyOceanIndex(); + static KokkosDeviceMapView oceanIndexDevice; + // static Kokkos:: +#endif }; } /* namespace Nextsim */ diff --git a/core/src/include/PrognosticData.hpp b/core/src/include/PrognosticData.hpp index f4a2651e8..a2fd29e13 100644 --- a/core/src/include/PrognosticData.hpp +++ b/core/src/include/PrognosticData.hpp @@ -56,10 +56,10 @@ class PrognosticData : public CheckingModelComponent, public Configured hiceAccessor; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor damageAccessor; + ModelArrayAccessor hsnowAccessor; // cell averaged snow thickness IAtmosphereBoundary* pAtmBdy; IOceanBoundary* pOcnBdy; diff --git a/core/src/include/constants.hpp b/core/src/include/constants.hpp index 2f94d8b69..57cfdeb59 100644 --- a/core/src/include/constants.hpp +++ b/core/src/include/constants.hpp @@ -5,6 +5,8 @@ #ifndef SRC_INCLUDE_CONSTANTS_HPP #define SRC_INCLUDE_CONSTANTS_HPP +#include "include/KernelAlternatives.hpp" + //! General physical constants of the Earth and universe namespace PhysicalConstants { @@ -129,21 +131,27 @@ const double TfOcean = -1.8; namespace Nextsim { //! Convert a temperature from ˚C to K -inline double kelvin(double celsius) { return celsius + Water::Tf; } +KERNEL_IMPL_FUNCTION inline double kelvin(double celsius) { return celsius + Water::Tf; } //! Convert a temperature from K to ˚C -inline double celsius(double kelvin) { return kelvin - Water::Tf; } +KERNEL_IMPL_FUNCTION inline double celsius(double kelvin) { return kelvin - Water::Tf; } //! Convert an angle from radians to degrees -inline double degrees(double radians) { return radians * PhysicalConstants::rad2deg; } +KERNEL_IMPL_FUNCTION inline double degrees(double radians) +{ + return radians * PhysicalConstants::rad2deg; +} //! Convert an angle from degrees to radians -inline double radians(double degrees) { return degrees * PhysicalConstants::deg2rad; } +KERNEL_IMPL_FUNCTION inline double radians(double degrees) +{ + return degrees * PhysicalConstants::deg2rad; +} //! Convert a pressure from Pa to mbar -inline double mbar(double pascals) { return pascals / 100; } +KERNEL_IMPL_FUNCTION inline double mbar(double pascals) { return pascals / 100; } //! Convert a pressure from mbar to Pa -inline double pascals(double mbar) { return mbar * 100; } +KERNEL_IMPL_FUNCTION inline double pascals(double mbar) { return mbar * 100; } } #endif /* SRC_INCLUDE_CONSTANTS_HPP */ diff --git a/dynamics/src/kokkos/KokkosUtils.cpp b/core/src/kokkos/KokkosUtils.cpp similarity index 99% rename from dynamics/src/kokkos/KokkosUtils.cpp rename to core/src/kokkos/KokkosUtils.cpp index be9f7c0bc..4e1ec9ede 100644 --- a/dynamics/src/kokkos/KokkosUtils.cpp +++ b/core/src/kokkos/KokkosUtils.cpp @@ -24,4 +24,4 @@ DeviceBitset makeKokkosDeviceBitset(const std::vector& buf) return bitsetDevice; } -} \ No newline at end of file +} \ No newline at end of file diff --git a/core/src/kokkos/include/KokkosModelArray.hpp b/core/src/kokkos/include/KokkosModelArray.hpp new file mode 100644 index 000000000..315e3bf4a --- /dev/null +++ b/core/src/kokkos/include/KokkosModelArray.hpp @@ -0,0 +1,23 @@ +/*! + * @author Robert Jendersie + */ + +// by also guarding for USE_KOKKOS this header can be safely included even +// when Kokkos is not enabled +#if !defined(KOKKOSMODELARRAY_HPP) && defined(USE_KOKKOS) +#define KOKKOSMODELARRAY_HPP + +#include "../../include/ModelArray.hpp" +#include "KokkosUtils.hpp" + +namespace Nextsim { + +// kokkos views compatible with ModelArray +using DeviceViewMA = KokkosDeviceView; +using ConstDeviceViewMA = ConstKokkosDeviceView; +using HostViewMA = KokkosHostView; +using ConstHostViewMA = ConstKokkosHostView; + +} + +#endif // KOKKOSMODELARRAY_HPP \ No newline at end of file diff --git a/dynamics/src/kokkos/include/KokkosTimer.hpp b/core/src/kokkos/include/KokkosTimer.hpp similarity index 85% rename from dynamics/src/kokkos/include/KokkosTimer.hpp rename to core/src/kokkos/include/KokkosTimer.hpp index e4b653290..bc4055d26 100644 --- a/dynamics/src/kokkos/include/KokkosTimer.hpp +++ b/core/src/kokkos/include/KokkosTimer.hpp @@ -5,8 +5,13 @@ #include #endif +#include + namespace Nextsim { +// The KokkosTimer introduces fences to measure the actual execution time for async operations. This +// has a non-trivial overhead so fewer timers should be used for accurate measurements of larger +// code segments. constexpr bool DETAILED_MEASUREMENTS = true; template class KokkosTimer { diff --git a/dynamics/src/kokkos/include/KokkosUtils.hpp b/core/src/kokkos/include/KokkosUtils.hpp similarity index 88% rename from dynamics/src/kokkos/include/KokkosUtils.hpp rename to core/src/kokkos/include/KokkosUtils.hpp index 3f6733c74..db38b04de 100644 --- a/dynamics/src/kokkos/include/KokkosUtils.hpp +++ b/core/src/kokkos/include/KokkosUtils.hpp @@ -2,7 +2,9 @@ * @author Robert Jendersie */ -#ifndef KOKKOSUTILS_HPP +// by also guarding for USE_KOKKOS this header can be safely included even +// when Kokkos is not enabled +#if !defined(KOKKOSUTILS_HPP) && defined(USE_KOKKOS) #define KOKKOSUTILS_HPP #include @@ -66,24 +68,28 @@ namespace Details { } -// We can't specialize just for Eigen::Matrix because it needs to work with classes inheriting from -// Eigen::Matrix -template -using KokkosDeviceView +// Kokkos view types that are compatible (type, compile-time size, storage order, constness) with +// Eigen matrices. We can't specialize just for Eigen::Matrix because it needs to work with classes +// inheriting from Eigen::Matrix like DGVector +template +using KokkosEigenView = Kokkos::View::Type, - typename Details::ToKokkosLayout::Type>; -template -using ConstKokkosDeviceView + typename Details::ToKokkosLayout::Type, Args...>; +template +using ConstKokkosEigenView = Kokkos::View::Type, typename Details::ToKokkosLayout::Type>; + +template using KokkosDeviceView = KokkosEigenView; +template using ConstKokkosDeviceView = ConstKokkosEigenView; +template +using KokkosHostView + = KokkosEigenView>; template -using KokkosHostView = - typename Kokkos::View::Type, - typename Details::ToKokkosLayout::Type, Kokkos::HostSpace, - Kokkos::MemoryTraits>; +using ConstKokkosHostView + = ConstKokkosEigenView>; /*! * @brief Creates a host view compatible with an Eigen matrix. @@ -95,6 +101,7 @@ using KokkosHostView = */ template auto makeKokkosHostView(EigenMat& mat) { + // const_cast is necessary because Eigen only gives const access to the underlying pointer if constexpr (EigenMat::RowsAtCompileTime == 1 || EigenMat::ColsAtCompileTime == 1) { return KokkosHostView( const_cast(mat.data()), mat.rows() * mat.cols()); @@ -102,6 +109,14 @@ template auto makeKokkosHostView(EigenMat& mat) return KokkosHostView( const_cast(mat.data()), mat.rows(), mat.cols()); } +// const overload +template auto makeKokkosHostView(const EigenMat& mat) +{ + if constexpr (EigenMat::RowsAtCompileTime == 1 || EigenMat::ColsAtCompileTime == 1) { + return ConstKokkosHostView(mat.data(), mat.rows() * mat.cols()); + } + return ConstKokkosHostView(mat.data(), mat.rows(), mat.cols()); +} /// Options for the creation of views based on existing data. enum struct MakeViewOptions { diff --git a/core/src/modules/DiagnosticOutputModule/SimpleOutput.cpp b/core/src/modules/DiagnosticOutputModule/SimpleOutput.cpp index 44d68e3ee..d4a90701c 100644 --- a/core/src/modules/DiagnosticOutputModule/SimpleOutput.cpp +++ b/core/src/modules/DiagnosticOutputModule/SimpleOutput.cpp @@ -5,7 +5,6 @@ #include "include/SimpleOutput.hpp" #include "include/Logged.hpp" -#include "include/ModelArrayRef.hpp" #include "include/StructureFactory.hpp" #include diff --git a/core/src/modules/DynamicsModule/BBMDynamics.cpp b/core/src/modules/DynamicsModule/BBMDynamics.cpp index 7d75a2a24..e852580fd 100644 --- a/core/src/modules/DynamicsModule/BBMDynamics.cpp +++ b/core/src/modules/DynamicsModule/BBMDynamics.cpp @@ -91,13 +91,6 @@ void BBMDynamics::setData(const ModelState::DataMap& ms) { IDynamics::setData(ms); - // Set the DG field data. Needs to be done before initialise() because the Kokkos kernel - // requires the actual vectors to init its views. - kernel.setDGArray(hiceName, hiceDG.allComponents()); - kernel.setDGArray(ciceName, ciceDG.allComponents()); - kernel.setDGArray(hsnowName, hsnowDG.allComponents()); - kernel.setDGArray(damageName, damage.allComponents()); - const bool isSpherical = checkSpherical(ms); ModelArray coords = ms.at(coordsName); @@ -107,8 +100,8 @@ void BBMDynamics::setData(const ModelState::DataMap& ms) // TODO: Some encoding of the periodic edge boundary conditions kernel.initialise(coords, isSpherical, ms.at(maskName)); - uice = ms.at(uName); - vice = ms.at(vName); + uiceAccessor.getHostRW() = ms.at(uName); + viceAccessor.getHostRW() = ms.at(vName); // Set the data in the kernel arrays. // Required data @@ -137,12 +130,23 @@ void BBMDynamics::update(const TimestepTime& tst) { std::cout << tst.start << std::endl; + // set dg fields + // Needs to be done every step even so the field references do not change to ensure that the + // write accesses are registered and that needed host-device data transfers can take place. + kernel.setDGArray(hiceName, hiceDGAccessor.getAutoRW()); + kernel.setDGArray(ciceName, ciceDGAccessor.getAutoRW()); + kernel.setDGArray(hsnowName, hsnowDGAccessor.getAutoRW()); + kernel.setDGArray(damageName, damageAccessor.getAutoRW()); + // set the forcing velocities - kernel.setData(uWindName, uwind); - kernel.setData(vWindName, vwind); - kernel.setData(uOceanName, uocean); - kernel.setData(vOceanName, vocean); - kernel.setData(sshName, ssh); + static KokkosTimer timer("setData"); + timer.start(); + kernel.setData(uWindName, uwindAccessor.getHostRO()); + kernel.setData(vWindName, vwindAccessor.getHostRO()); + kernel.setData(uOceanName, uoceanAccessor.getHostRO()); + kernel.setData(vOceanName, voceanAccessor.getHostRO()); + kernel.setData(sshName, sshAccessor.getHostRO()); + timer.stop(); /* * Ice velocity components are stored in the dynamics, and not changed by the model outside the @@ -151,16 +155,19 @@ void BBMDynamics::update(const TimestepTime& tst) kernel.update(tst); - uice = kernel.getDG0Data(uName); - vice = kernel.getDG0Data(vName); + static KokkosTimer timerGet("getData"); + timerGet.start(); + uiceAccessor.getHostRW() = kernel.getDG0Data(uName); + viceAccessor.getHostRW() = kernel.getDG0Data(vName); - taux = kernel.getDG0Data(uIOStressName); - tauy = kernel.getDG0Data(vIOStressName); + tauxAccessor.getHostRW() = kernel.getDG0Data(uIOStressName); + tauyAccessor.getHostRW() = kernel.getDG0Data(vIOStressName); - shear = kernel.getDG0Data(shearName); - divergence = kernel.getDG0Data(divergenceName); - sigmaI = kernel.getDG0Data(sigmaIName); - sigmaII = kernel.getDG0Data(sigmaIIName); + shearAccessor.getHostRW() = kernel.getDG0Data(shearName); + divergenceAccessor.getHostRW() = kernel.getDG0Data(divergenceName); + sigmaIAccessor.getHostRW() = kernel.getDG0Data(sigmaIName); + sigmaIIAccessor.getHostRW() = kernel.getDG0Data(sigmaIIName); + timerGet.stop(); } void BBMDynamics::prepareAdvection() { kernel.prepareAdvection(); } @@ -171,6 +178,14 @@ void BBMDynamics::advectField( kernel.advectField(timestep, field, lowerLimit, upperLimit); } +#ifdef USE_KOKKOS +void BBMDynamics::advectField( + double timestep, const DeviceViewMA& field, double lowerLimit, double upperLimit) +{ + kernel.advectDGVFieldDevice(timestep, field, lowerLimit, upperLimit); +} +#endif + BBMDynamics::HelpMap& BBMDynamics::getHelpText(HelpMap& map, bool getAll) { map["BBMDynamics"] = { diff --git a/core/src/modules/DynamicsModule/MEVPDynamics.cpp b/core/src/modules/DynamicsModule/MEVPDynamics.cpp index 17629cabc..5759296fb 100644 --- a/core/src/modules/DynamicsModule/MEVPDynamics.cpp +++ b/core/src/modules/DynamicsModule/MEVPDynamics.cpp @@ -77,12 +77,6 @@ void MEVPDynamics::setData(const ModelState::DataMap& ms) { IDynamics::setData(ms); - // Set the DG field data. Needs to be done before initialise() because the Kokkos kernel - // requires the actual vectors to init its views. - kernel.setDGArray(hiceName, hiceDG.allComponents()); - kernel.setDGArray(ciceName, ciceDG.allComponents()); - kernel.setDGArray(hsnowName, hsnowDG.allComponents()); - const bool isSpherical = checkSpherical(ms); ModelArray coords = ms.at(coordsName); @@ -93,8 +87,8 @@ void MEVPDynamics::setData(const ModelState::DataMap& ms) // TODO: Some encoding of the periodic edge boundary conditions kernel.initialise(coords, isSpherical, ms.at(maskName)); - uice = ms.at(uName); - vice = ms.at(vName); + uiceAccessor.getHostRW() = ms.at(uName); + viceAccessor.getHostRW() = ms.at(vName); // Set the data in the kernel arrays. for (const auto& fieldName : namedFields) { @@ -106,25 +100,32 @@ void MEVPDynamics::update(const TimestepTime& tst) { std::cout << tst.start << std::endl; + // set dg fields + // Needs to be done every step even so the field references do not change to ensure that the + // write accesses are registered and that needed host-device data transfers can take place. + kernel.setDGArray(hiceName, hiceDGAccessor.getAutoRW()); + kernel.setDGArray(ciceName, ciceDGAccessor.getAutoRW()); + kernel.setDGArray(hsnowName, hsnowDGAccessor.getAutoRW()); + // set the forcing velocities - kernel.setData(uWindName, uwind); - kernel.setData(vWindName, vwind); - kernel.setData(uOceanName, uocean); - kernel.setData(vOceanName, vocean); - kernel.setData(sshName, ssh); + kernel.setData(uWindName, uwindAccessor.getAutoRO()); + kernel.setData(vWindName, vwindAccessor.getAutoRO()); + kernel.setData(uOceanName, uoceanAccessor.getAutoRO()); + kernel.setData(vOceanName, voceanAccessor.getAutoRO()); + kernel.setData(sshName, sshAccessor.getHostRO()); kernel.update(tst); - uice = kernel.getDG0Data(uName); - vice = kernel.getDG0Data(vName); + uiceAccessor.getHostRW() = kernel.getDG0Data(uName); + viceAccessor.getHostRW() = kernel.getDG0Data(vName); - taux = kernel.getDG0Data(uIOStressName); - tauy = kernel.getDG0Data(vIOStressName); + tauxAccessor.getHostRW() = kernel.getDG0Data(uIOStressName); + tauyAccessor.getHostRW() = kernel.getDG0Data(vIOStressName); - shear = kernel.getDG0Data(shearName); - divergence = kernel.getDG0Data(divergenceName); - sigmaI = kernel.getDG0Data(sigmaIName); - sigmaII = kernel.getDG0Data(sigmaIIName); + shearAccessor.getHostRW() = kernel.getDG0Data(shearName); + divergenceAccessor.getHostRW() = kernel.getDG0Data(divergenceName); + sigmaIAccessor.getHostRW() = kernel.getDG0Data(sigmaIName); + sigmaIIAccessor.getHostRW() = kernel.getDG0Data(sigmaIIName); } void MEVPDynamics::advectField( @@ -133,6 +134,14 @@ void MEVPDynamics::advectField( kernel.advectField(timestep, field, lowerLimit, upperLimit); } +#ifdef USE_KOKKOS +void MEVPDynamics::advectField( + double timestep, const DeviceViewMA& field, double lowerLimit, double upperLimit) +{ + kernel.advectDGVFieldDevice(timestep, field, lowerLimit, upperLimit); +} +#endif + MEVPDynamics::HelpMap& MEVPDynamics::getHelpText(HelpMap& map, bool getAll) { map["MEVPDynamics"] = { diff --git a/core/src/modules/DynamicsModule/include/BBMDynamics.hpp b/core/src/modules/DynamicsModule/include/BBMDynamics.hpp index de99043d8..849e25767 100644 --- a/core/src/modules/DynamicsModule/include/BBMDynamics.hpp +++ b/core/src/modules/DynamicsModule/include/BBMDynamics.hpp @@ -30,6 +30,12 @@ class BBMDynamics : public IDynamics, public Configured { double lowerLimit = -std::numeric_limits::infinity(), double upperLimit = std::numeric_limits::infinity()) override; +#ifdef USE_KOKKOS + void advectField(double timestep, const DeviceViewMA& field, + double lowerLimit = -std::numeric_limits::infinity(), + double upperLimit = std::numeric_limits::infinity()) override; +#endif + void setData(const ModelState::DataMap&) override; void configure() override; ConfigMap getConfiguration() const override; diff --git a/core/src/modules/DynamicsModule/include/DummyDynamics.hpp b/core/src/modules/DynamicsModule/include/DummyDynamics.hpp index 956010a1e..f7cdaa2c8 100644 --- a/core/src/modules/DynamicsModule/include/DummyDynamics.hpp +++ b/core/src/modules/DynamicsModule/include/DummyDynamics.hpp @@ -23,6 +23,9 @@ class DummyDynamics : public IDynamics { { IDynamics::setData(state); + HField& uice = uiceAccessor.getHostRW(); + HField& vice = viceAccessor.getHostRW(); + // Set the ice velocity to zero so that we don't trip the PrognosticData field checks. uice = 0.; vice = 0.; diff --git a/core/src/modules/DynamicsModule/include/FreeDriftDynamics.hpp b/core/src/modules/DynamicsModule/include/FreeDriftDynamics.hpp index 0fa9ff05f..a72c8ee06 100644 --- a/core/src/modules/DynamicsModule/include/FreeDriftDynamics.hpp +++ b/core/src/modules/DynamicsModule/include/FreeDriftDynamics.hpp @@ -38,13 +38,13 @@ class FreeDriftDynamics : public IDynamics { std::cout << tst.start << std::endl; // set the forcing velocities - kernel.setData(uOceanName, uocean); - kernel.setData(vOceanName, vocean); + kernel.setData(uOceanName, uoceanAccessor.getHostRO()); + kernel.setData(vOceanName, voceanAccessor.getHostRO()); kernel.update(tst); - uice = kernel.getDG0Data(uName); - vice = kernel.getDG0Data(vName); + uiceAccessor.getHostRW() = kernel.getDG0Data(uName); + viceAccessor.getHostRW() = kernel.getDG0Data(vName); } void setData(const ModelState::DataMap& ms) override @@ -53,9 +53,9 @@ class FreeDriftDynamics : public IDynamics { // Set the DG field data. Needs to be done before initialise() because the Kokkos kernel // requires the actual vectors to init its views. - kernel.setDGArray(hiceName, hiceDG.allComponents()); - kernel.setDGArray(ciceName, ciceDG.allComponents()); - kernel.setDGArray(hsnowName, hsnowDG.allComponents()); + kernel.setDGArray(hiceName, hiceDGAccessor.getHostRW()); + kernel.setDGArray(ciceName, ciceDGAccessor.getHostRW()); + kernel.setDGArray(hsnowName, hsnowDGAccessor.getHostRW()); bool isSpherical = checkSpherical(ms); @@ -70,10 +70,6 @@ class FreeDriftDynamics : public IDynamics { for (const auto& fieldName : namedFields) { kernel.setData(fieldName, ms.at(fieldName)); } - - // Set the DG field data - kernel.setDGArray(hiceName, hiceDG.allComponents()); - kernel.setDGArray(ciceName, ciceDG.allComponents()); } void advectField(double timestep, ModelArray& field, diff --git a/core/src/modules/DynamicsModule/include/MEVPDynamics.hpp b/core/src/modules/DynamicsModule/include/MEVPDynamics.hpp index b82a0e9c8..e8b4f06d9 100644 --- a/core/src/modules/DynamicsModule/include/MEVPDynamics.hpp +++ b/core/src/modules/DynamicsModule/include/MEVPDynamics.hpp @@ -10,9 +10,9 @@ #include "include/IDamageHealing.hpp" #include "include/IDynamics.hpp" -#include "kokkos/include/KokkosMEVPDynamicsKernel.hpp" #include "include/MEVPDynamicsKernel.hpp" #include "include/NextsimModule.hpp" +#include "kokkos/include/KokkosMEVPDynamicsKernel.hpp" #include "include/ModelArray.hpp" #include "include/ModelComponent.hpp" @@ -37,6 +37,12 @@ class MEVPDynamics : public IDynamics, public Configured { double lowerLimit = -std::numeric_limits::infinity(), double upperLimit = std::numeric_limits::infinity()) override; +#ifdef USE_KOKKOS + void advectField(double timestep, const DeviceViewMA& field, + double lowerLimit = -std::numeric_limits::infinity(), + double upperLimit = std::numeric_limits::infinity()) override; +#endif + void setData(const ModelState::DataMap&) override; void configure() override; ConfigMap getConfiguration() const override; diff --git a/core/src/modules/include/IDynamics.hpp b/core/src/modules/include/IDynamics.hpp index 2be8c7ffd..854418bbf 100644 --- a/core/src/modules/include/IDynamics.hpp +++ b/core/src/modules/include/IDynamics.hpp @@ -5,6 +5,7 @@ #ifndef IDYNAMICS_HPP #define IDYNAMICS_HPP +#include "include/ModelArrayAccessor.hpp" #include "include/ModelComponent.hpp" #include "include/Time.hpp" #include "include/gridNames.hpp" @@ -19,46 +20,38 @@ class IDynamics : public ModelComponent { { } IDynamics(bool usesDamageIn) - : uice(ModelArray::Type::H) - , vice(ModelArray::Type::H) - , taux(ModelArray::Type::H) - , tauy(ModelArray::Type::H) - , shear(ModelArray::Type::H) - , divergence(ModelArray::Type::H) - , sigmaI(ModelArray::Type::H) - , sigmaII(ModelArray::Type::H) - , damage(getStore()) - , uwind(getStore()) - , vwind(getStore()) - , uocean(getStore()) - , vocean(getStore()) - , ssh(getStore()) + : uiceAccessor(getStore(), RO, ModelArray::Type::H) + , viceAccessor(getStore(), RO, ModelArray::Type::H) + , tauxAccessor(getStore(), RO, ModelArray::Type::H) + , tauyAccessor(getStore(), RO, ModelArray::Type::H) + , shearAccessor(getStore(), RO, ModelArray::Type::H) + , divergenceAccessor(getStore(), RO, ModelArray::Type::H) + , sigmaIAccessor(getStore(), RO, ModelArray::Type::H) + , sigmaIIAccessor(getStore(), RO, ModelArray::Type::H) + , damageAccessor(getStore()) + , uwindAccessor(getStore()) + , vwindAccessor(getStore()) + , uoceanAccessor(getStore()) + , voceanAccessor(getStore()) + , sshAccessor(getStore()) , m_usesDamage(usesDamageIn) - , hiceDG(getStore()) - , ciceDG(getStore()) - , hsnowDG(getStore()) + , hiceDGAccessor(getStore()) + , ciceDGAccessor(getStore()) + , hsnowDGAccessor(getStore()) { - getStore().registerArray(Protected::DIV, &divergence, RO); - getStore().registerArray(Protected::ICE_U, &uice, RO); - getStore().registerArray(Protected::ICE_V, &vice, RO); - getStore().registerArray(Protected::IO_STRESS_X, &taux, RO); - getStore().registerArray(Protected::IO_STRESS_Y, &tauy, RO); - getStore().registerArray(Protected::SHEAR, &shear, RO); - getStore().registerArray(Protected::SIGMAI, &sigmaI, RO); - getStore().registerArray(Protected::SIGMAII, &sigmaII, RO); } virtual ~IDynamics() = default; ModelState getStatePrognostic() const override { ModelState state = { { - { uName, uice }, - { vName, vice }, + { uName, uiceAccessor.getHostRO() }, + { vName, viceAccessor.getHostRO() }, }, getConfiguration() }; if (m_usesDamage) { - ModelState::DataMap damageState = { { damageName, damage } }; + ModelState::DataMap damageState = { { damageName, damageAccessor.getHostRO() } }; state.merge(damageState); } @@ -68,14 +61,14 @@ class IDynamics : public ModelComponent { ModelState getStateDiagnostic() const override { ModelState state = { { - { uIOStressName, taux }, - { vIOStressName, tauy }, - { uName, uice }, - { vName, vice }, - { shearName, shear }, - { divergenceName, divergence }, - { sigmaIName, sigmaI }, - { sigmaIIName, sigmaII }, + { uIOStressName, tauxAccessor.getHostRO() }, + { vIOStressName, tauyAccessor.getHostRO() }, + { uName, uiceAccessor.getHostRO() }, + { vName, viceAccessor.getHostRO() }, + { shearName, shearAccessor.getHostRO() }, + { divergenceName, divergenceAccessor.getHostRO() }, + { sigmaIName, sigmaIAccessor.getHostRO() }, + { sigmaIIName, sigmaIIAccessor.getHostRO() }, }, {} }; return state.merge(getStatePrognostic()); @@ -84,11 +77,17 @@ class IDynamics : public ModelComponent { std::string getName() const override { return "IDynamics"; } virtual void setData(const ModelState::DataMap& ms) override { + HField& uice = uiceAccessor.getHostRW(); uice.resize(); + HField& vice = viceAccessor.getHostRW(); vice.resize(); + HField& shear = shearAccessor.getHostRW(); shear.resize(); + HField& divergence = divergenceAccessor.getHostRW(); divergence.resize(); + HField& sigmaI = sigmaIAccessor.getHostRW(); sigmaI.resize(); + HField& sigmaII = sigmaIIAccessor.getHostRW(); sigmaII.resize(); } @@ -105,37 +104,45 @@ class IDynamics : public ModelComponent { { } +#ifdef USE_KOKKOS + virtual void advectField(double timestep, const DeviceViewMA& field, + double lowerLimit = -std::numeric_limits::infinity(), + double upperLimit = std::numeric_limits::infinity()) + { + } +#endif + virtual void prepareAdvection() = 0; protected: // Shared ice velocity arrays - HField uice; - HField vice; + ModelArrayAccessor uiceAccessor; + ModelArrayAccessor viceAccessor; // Ice-ocean stress (for the coupler, mostly) - HField taux; - HField tauy; + ModelArrayAccessor tauxAccessor; + ModelArrayAccessor tauyAccessor; // Diagnostic outputs of shear, divergence and the stress invariants - HField shear; - HField divergence; - HField sigmaI; - HField sigmaII; + ModelArrayAccessor shearAccessor; + ModelArrayAccessor divergenceAccessor; + ModelArrayAccessor sigmaIAccessor; + ModelArrayAccessor sigmaIIAccessor; // References to the DG0 finite volume data arrays - ModelArrayRef damage; + ModelArrayAccessor damageAccessor; // References to the forcing velocity arrays - ModelArrayRef uwind; - ModelArrayRef vwind; - ModelArrayRef uocean; - ModelArrayRef vocean; - ModelArrayRef ssh; + ModelArrayAccessor uwindAccessor; + ModelArrayAccessor vwindAccessor; + ModelArrayAccessor uoceanAccessor; + ModelArrayAccessor voceanAccessor; + ModelArrayAccessor sshAccessor; // Does this implementation of the dynamics use damage? bool m_usesDamage; // Store the h_ice and c_ice DG fields here, rather than in the kernel. - ModelArrayRef hiceDG; - ModelArrayRef ciceDG; - ModelArrayRef hsnowDG; + ModelArrayAccessor hiceDGAccessor; + ModelArrayAccessor ciceDGAccessor; + ModelArrayAccessor hsnowDGAccessor; /* * Checks and returns if the provided data map is spherical diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt index 20cbb0bd0..84e32891c 100644 --- a/core/test/CMakeLists.txt +++ b/core/test/CMakeLists.txt @@ -432,8 +432,12 @@ else() target_include_directories(testModelArrayRef PRIVATE ${MODEL_INCLUDE_DIR}) target_link_libraries(testModelArrayRef PRIVATE nextsimlib doctest::doctest) + add_executable(testModelArrayAccessor "ModelArrayAccessor_test.cpp") + target_include_directories(testModelArrayAccessor PRIVATE ${MODEL_INCLUDE_DIR}) + target_link_libraries(testModelArrayAccessor PRIVATE nextsimlib doctest::doctest) + # PrognosticData (and hopefully that alone) requires code from the physics tree - add_executable(testPrognosticData "PrognosticData_test.cpp" "DynamicsModuleForPDtest.cpp") + add_executable(testPrognosticData "PrognosticData_test.cpp" "DynamicsModuleForPDtest.cpp" "MainTest.cpp") target_include_directories( testPrognosticData PRIVATE diff --git a/core/test/ConfigOutput_test.cpp b/core/test/ConfigOutput_test.cpp index 3d8f83424..dc6b4f820 100644 --- a/core/test/ConfigOutput_test.cpp +++ b/core/test/ConfigOutput_test.cpp @@ -16,7 +16,7 @@ #include "include/Finalizer.hpp" #include "include/IStructure.hpp" #include "include/ModelArray.hpp" -#include "include/ModelArrayRef.hpp" +#include "include/ModelArrayAccessor.hpp" #include "include/ModelComponent.hpp" #include "include/ModelMetadata.hpp" #include "include/ModelState.hpp" @@ -89,10 +89,19 @@ TEST_CASE("Test periodic output") Module::setImplementation("Nextsim::ParametricGrid"); - HField hice(ModelArray::Type::H); - HField cice(ModelArray::Type::H); - HField hsnow(ModelArray::Type::H); - HField tsurf(ModelArray::Type::H); + ModelArrayAccessor hiceAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + ModelArrayAccessor ciceAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + ModelArrayAccessor hsnowAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + ModelArrayAccessor tsurfAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + + HField& hice = hiceAccessor.getHostRW(); + HField& cice = ciceAccessor.getHostRW(); + HField& hsnow = hsnowAccessor.getHostRW(); + HField& tsurf = tsurfAccessor.getHostRW(); // An internal diagnostic field, not made available through the data store HField topMelt(ModelArray::Type::H); @@ -109,11 +118,6 @@ TEST_CASE("Test periodic output") tsurf = 0.; topMelt = 0.; - ModelComponent::getStore().registerArray(Shared::H_ICE_DG, &hice); - ModelComponent::getStore().registerArray(Shared::C_ICE_DG, &cice); - ModelComponent::getStore().registerArray(Shared::H_SNOW_DG, &hsnow); - ModelComponent::getStore().registerArray(Protected::T_SURF, &tsurf); - // Set up the coordinates, but use arrays filled with zeros HField latlonData(ModelArray::Type::H); latlonData = 0.; diff --git a/core/test/DynamicsModuleForPDtest.cpp b/core/test/DynamicsModuleForPDtest.cpp index f49d5df78..1e1217f99 100644 --- a/core/test/DynamicsModuleForPDtest.cpp +++ b/core/test/DynamicsModuleForPDtest.cpp @@ -20,6 +20,14 @@ template <> const Module::Map& Module::f return theMap; } +// Needed so that the finalizer works correctly because getGenerationFunction() looks up the +// generator function under this name. For some reason the PDTestDynamics module is only registered +// in a build without debug symbols, so this issue only appears in a release build. +template <> std::string Module::getDefaultImplementationName() +{ + return PDTESTDYNAMICS; +} + template <> Module::Fn& Module::getGenerationFunction() { static Fn thePtr = functionMap().at(PDTESTDYNAMICS); diff --git a/core/test/MainTest.cpp b/core/test/MainTest.cpp new file mode 100644 index 000000000..577d6909b --- /dev/null +++ b/core/test/MainTest.cpp @@ -0,0 +1,29 @@ +#define DOCTEST_CONFIG_IMPLEMENT + +#include "include/Finalizer.hpp" +#include +#ifdef USE_KOKKOS +#include +#endif + +// Main routine for tests that use Kokkos in some way. +int main(int argc, char** argv) +{ +#ifdef USE_KOKKOS + Kokkos::initialize(argc, argv); +#endif + + doctest::Context ctx; + ctx.applyCommandLine(argc, argv); + + int test_result = ctx.run(); + + // cleanup resources including Kokkos allocated buffers before finalizing Kokkos to prevent + // errors + Nextsim::Finalizer::finalize(); +#ifdef USE_KOKKOS + Kokkos::finalize(); +#endif + + return test_result; +} diff --git a/core/test/ModelArrayAccessor_test.cpp b/core/test/ModelArrayAccessor_test.cpp new file mode 100644 index 000000000..4625498d7 --- /dev/null +++ b/core/test/ModelArrayAccessor_test.cpp @@ -0,0 +1,354 @@ +/*! + * @author Tim Spain + * @author Robert Jendersie + */ + +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN +#include + +#include "../src/include/ModelArrayAccessor.hpp" +#include "../src/include/ModelArrayStore.hpp" + +namespace Nextsim { + +class MiniModelComponent { +public: + static constexpr TextTag H_ICE0 = { "H_ICE0" }; + static constexpr TextTag SW_IN = { "SW_IN" }; + static constexpr TextTag H_ICE = { "H_ICE" }; +}; + +class AtmIn : public MiniModelComponent { +public: + AtmIn(ModelArrayStore& store) + : hiceAccessor(store, RW) + , swinAccessor(store, RW) + { + } + void configure() + { + hiceAccessor.getHostRW().resize(); + swinAccessor.getHostRW().resize(); + } + void setData(const std::vector& values) + { + hiceAccessor.getHostRW() = values[0]; + swinAccessor.getHostRW() = values[1]; + } + +private: + ModelArrayAccessor hiceAccessor; + ModelArrayAccessor swinAccessor; +}; + +class IceThermo : public MiniModelComponent { +public: + IceThermo(ModelArrayStore& store) + : hiceAccessor(store) + { + } + + void update(int tStep) + { + ModelArray& hice = hiceAccessor.getHostRW(); + hice[0] *= (1. + tStep) / tStep; + } + +private: + ModelArrayAccessor hiceAccessor; +}; + +class IceCalc : public MiniModelComponent { +public: + IceCalc(ModelArrayStore& store) + : hiceAccessor(store, RW) + , hice0Accessor(store) + , thermo(store) + { + } + void configure() { hiceAccessor.getHostRW().resize(); } + void update(int tStep) + { + ModelArray& hice = hiceAccessor.getHostRW(); + const ModelArray& hice0 = hice0Accessor.getHostRO(); + hice[0] = hice0[0]; + thermo.update(tStep); + } + void getData(double& dataOut) + { + const ModelArray& hice = hiceAccessor.getHostRO(); + dataOut = hice[0]; + } + +private: + ModelArrayAccessor hiceAccessor; + ModelArrayAccessor hice0Accessor; + + IceThermo thermo; +}; + +TEST_SUITE_BEGIN("[ModelArrayAccessor]"); +TEST_CASE("Accessing the data") +{ + // create store on stack so that test cases do not influence each other + ModelArrayStore store; + + AtmIn atmIn(store); + double hice0 = 0.56; + double swin = 311; + ModelArray::setDimensions(ModelArray::Type::H, { 1, 1 }); + atmIn.configure(); + atmIn.setData({ hice0, swin }); + + IceCalc iceCalc(store); + iceCalc.configure(); + int tStep = 40; + iceCalc.update(tStep); + + double hicef; + iceCalc.getData(hicef); + double target = hice0 * (1. + tStep) / tStep; + REQUIRE(hicef == doctest::Approx(target).epsilon(1e-8)); +} + +TEST_CASE("(Not) getting write access to a read only field") +{ + ModelArrayStore store; + ModelArrayAccessor hice0SrcAccessor(store, RO); + HField& hice0Src = hice0SrcAccessor.getHostRW(); + hice0Src.resize(); + hice0Src[0] = 1.0; + REQUIRE_THROWS_AS( + (ModelArrayAccessor(store)), std::logic_error); + + // inverted initialization order: register after accessor was created + ModelArrayAccessor hiceAccessor(store); + REQUIRE_THROWS_AS( + (ModelArrayAccessor(store, RO)), std::logic_error); +} + +static const double targetFlux = 320; +// "inline" here prevents warning from -Wsubobject-linkage +// internal linkage which causes the warning would only be a problem if CouplEr was used elsewhere +inline constexpr TextTag sw_in = { "sw_in" }; + +class CouplEr { +public: + CouplEr(ModelArrayStore& bs) + : swFluxAccessor(bs) + { + } + void update() + { + ModelArray& swFlux = swFluxAccessor.getHostRW(); + swFlux[0] = targetFlux; + } + +private: + ModelArrayAccessor swFluxAccessor; +}; + +class CouplIn : public MiniModelComponent { +public: + CouplIn(ModelArrayStore& store) + : hiceAccessor(store, RW) + , swinAccessor(store, RW) + , coupler(store) + { + } + void configure() + { + hiceAccessor.getHostRW().resize(); + swinAccessor.getHostRW().resize(); + } + void setData() + { + ModelArray& hice = hiceAccessor.getHostRW(); + ModelArray& swin = swinAccessor.getHostRW(); + hice[0] = 0.5; + swin[0] = 350; + } + void update() { coupler.update(); } + +private: + ModelArrayAccessor hiceAccessor; + ModelArrayAccessor swinAccessor; + CouplEr coupler; +}; + +TEST_CASE("Accessing the data two ways") +{ + ModelArrayStore store; + + CouplIn couplIn(store); + ModelArray::setDimensions(ModelArray::Type::H, { 1, 1 }); + couplIn.configure(); + ModelArrayAccessor swinAccessor(store); + couplIn.setData(); + + const ModelArray& swin = swinAccessor.getHostRO(); + REQUIRE(swin[0] != targetFlux); + couplIn.update(); + REQUIRE(swin[0] == targetFlux); +} + +/* + * This functionality is specific to ModelArrayRef and no longer needed. + */ +// TEST_CASE("Test component 0-only operations") +// { +// ModelArrayStore store; +// +// static constexpr TextTag DG_SRC = { "DG_SRC" }; +// ModelArray::setDimension(ModelArray::Dimension::DG, 2); +// ModelArrayAccessor dgSrcAccessor(store, RO, ModelArray::Type::DG); +// ModelArray& dgSrc = dgSrcAccessor.getHostRW(); +// dgSrc.resize(); +// dgSrc = 5.; +// +// ModelArrayAccessor dgRefAccessor(store); +// const ModelArray& dgRef = dgRefAccessor.getHostRO(); +// ModelArray argument(ModelArray::Type::H); +// argument.resize(); +// argument = 3.; +// ModelArray sum = dgRef + argument; +// REQUIRE(sum.getType() == ModelArray::Type::H); +// REQUIRE(sum(0, 0) == 5. + 3.); +// +// ModelArray difference = dgRef - argument; +// REQUIRE(difference.getType() == ModelArray::Type::H); +// REQUIRE(difference(0, 0) == 5. - 3.); +// +// ModelArray product = dgRef * argument; +// REQUIRE(product.getType() == ModelArray::Type::H); +// REQUIRE(product(0, 0) == 5. * 3.); +// +// ModelArray ratio = dgRef / argument; +// REQUIRE(ratio.getType() == ModelArray::Type::H); +// REQUIRE(ratio(0, 0) == 5. / 3.); +// +// double scalar = 3.; +// ModelArray sumScalar = dgRef + scalar; +// REQUIRE(sumScalar.getType() == ModelArray::Type::H); +// REQUIRE(sumScalar(0, 0) == 5. + 3.); +// +// ModelArray differenceScalar = dgRef - scalar; +// REQUIRE(differenceScalar.getType() == ModelArray::Type::H); +// REQUIRE(differenceScalar(0, 0) == 5. - 3.); +// +// ModelArray productScalar = dgRef * scalar; +// REQUIRE(productScalar.getType() == ModelArray::Type::H); +// REQUIRE(productScalar(0, 0) == 5. * 3.); +// +// ModelArray ratioScalar = dgRef / scalar; +// REQUIRE(ratioScalar.getType() == ModelArray::Type::H); +// REQUIRE(ratioScalar(0, 0) == 5. / 3.); +// +// static constexpr TextTag RW_SRC = { "RW_SRC" }; +// ModelArrayAccessor rwRefAccessor(store, RW); +// const ModelArray& rwRef = rwRefAccessor.getHostRW(); +// argument = 7.; +// ModelArray sumRW = rwRef + argument; +// REQUIRE(sumRW.getType() == ModelArray::Type::H); +// REQUIRE(sumRW(0, 0) == 5. + 7.); +// +// ModelArray differenceRW = rwRef - argument; +// REQUIRE(differenceRW.getType() == ModelArray::Type::H); +// REQUIRE(differenceRW(0, 0) == 5. - 7.); +// +// ModelArray productRW = rwRef * argument; +// REQUIRE(productRW.getType() == ModelArray::Type::H); +// REQUIRE(productRW(0, 0) == 5. * 7.); +// +// ModelArray ratioRW = rwRef / argument; +// REQUIRE(ratioRW.getType() == ModelArray::Type::H); +// REQUIRE(ratioRW(0, 0) == 5. / 7.); +// +// scalar = 7.; +// ModelArray sumRWScalar = rwRef + scalar; +// REQUIRE(sumRWScalar.getType() == ModelArray::Type::H); +// REQUIRE(sumRWScalar(0, 0) == 5. + 7.); +// +// ModelArray differenceRWScalar = rwRef - scalar; +// REQUIRE(differenceRWScalar.getType() == ModelArray::Type::H); +// REQUIRE(differenceRWScalar(0, 0) == 5. - 7.); +// +// ModelArray productRWScalar = rwRef * scalar; +// REQUIRE(productRWScalar.getType() == ModelArray::Type::H); +// REQUIRE(productRWScalar(0, 0) == 5. * 7.); +// +// ModelArray ratioRWScalar = rwRef / scalar; +// REQUIRE(ratioRWScalar.getType() == ModelArray::Type::H); +// REQUIRE(ratioRWScalar(0, 0) == 5. / 7.); +// } + +TEST_CASE("Full component access") +{ + ModelArrayStore store; + + const size_t nx = 5; + const size_t ny = 7; + const size_t nDG = 3; + ModelArray::setDimension(ModelArray::Dimension::X, nx); + ModelArray::setDimension(ModelArray::Dimension::Y, ny); + ModelArray::setDimension(ModelArray::Dimension::DG, nDG); + ModelArray dgSrc(ModelArray::Type::DG); + dgSrc.resize(); + dgSrc = 5.; + static constexpr TextTag DG_SRC = { "DG_SRC" }; + ModelArrayAccessor dgRefAccessor(store, RO, ModelArray::Type::DG); + ModelArrayAccessor dgRefConstAccessor(store); + const ModelArray& dgRef = dgRefConstAccessor.getHostRO(); + const ModelArray::DataType& eArray = dgRef.data(); + REQUIRE(eArray.rows() == nx * ny); + REQUIRE(eArray.cols() == nDG); +} + +#ifdef USE_KOKKOS +TEST_CASE("Host device data syncing") +{ + Kokkos::initialize(Kokkos::InitializationSettings {}); + // scope to limit the lifetime of the ModelArrayStore + { + ModelArrayStore store; + + ModelArray::setDimensions(ModelArray::Type::H, { 2, 3 }); + + constexpr int IDX = 5; + + // init on host + ModelArrayAccessor hiceSrcAccessor(store, RW); + { + ModelArray& hice = hiceSrcAccessor.getHostRW(); + hice.resize(); + hice = 2.0; + hice[IDX] = 5.0; + } + // do work on device + { + ModelArrayAccessor hiceDstAccessor(store); + const DeviceViewMA& hiceDevice = hiceDstAccessor.getDeviceRW(); + Kokkos::parallel_for( + "updateDevice", hiceDevice.extent(0), + KOKKOS_LAMBDA(const DeviceIndex i) { hiceDevice(i, 0) += i == IDX ? -1.0 : 1.0; }); + } + // check results on host + { + const ModelArray& hice = hiceSrcAccessor.getHostRO(); + for (size_t i = 0; i < hice.size(); ++i) { + if (i == IDX) { + CHECK(hice[i] == 4.0); + } else { + CHECK(hice[i] == 3.0); + } + } + } + } + + Kokkos::finalize(); +} +#endif + +TEST_SUITE_END(); + +}; diff --git a/core/test/ModelComponent_test.cpp b/core/test/ModelComponent_test.cpp index eb55fa815..35bb7f29f 100644 --- a/core/test/ModelComponent_test.cpp +++ b/core/test/ModelComponent_test.cpp @@ -5,8 +5,8 @@ #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN #include -#include "include/ModelArrayRef.hpp" #include "include/ModelComponent.hpp" +#include "include/ModelArrayAccessor.hpp" #include @@ -30,55 +30,53 @@ class Module1 : public ModelComponent { class ModuleSupplyAndWait : public ModelComponent { public: ModuleSupplyAndWait() - : hice(ModelArray::HField()) - , cice_ref(getStore()) + : hiceAccessor(getStore(), RO) + , cice_refAccessor(getStore()) { - getStore().registerArray(Shared::H_ICE_DG, &hice, RO); } - void setData(const ModelState::DataMap& ms) override { hice[0] = hiceData; } + void setData(const ModelState::DataMap& ms) override { hiceAccessor.getHostRW()[0] = hiceData; } std::string getName() const override { return "SupplyAndWait"; } ModelState getStatePrognostic() const override { return { { - { "hice", hice }, + { "hice", hiceAccessor.getHostRO() }, }, {} }; } const double hiceData = 1.2; - double data() { return hice[0]; } - double refData() { return cice_ref[0]; } + double data() { return hiceAccessor.getHostRO()[0]; } + double refData() { return cice_refAccessor.getHostRO()[0]; } private: - HField hice; - ModelArrayRef cice_ref; + ModelArrayAccessor hiceAccessor; + ModelArrayAccessor cice_refAccessor; }; class ModuleRequestAndSupply : public ModelComponent { public: ModuleRequestAndSupply() - : cice(ModelArray::HField()) - , hice_ref(getStore()) + : ciceAccessor(getStore(), RO) + , hice_refAccessor(getStore()) { - getStore().registerArray(Shared::C_ICE_DG, &cice, RO); } - void setData(const ModelState::DataMap& ms) override { cice[0] = ciceData; } + void setData(const ModelState::DataMap& ms) override { ciceAccessor.getHostRW()[0] = ciceData; } std::string getName() const override { return "SupplyAndWait"; } ModelState getStatePrognostic() const override { return { { - { "cice", cice }, + { "cice", ciceAccessor.getHostRO() }, }, {} }; } const double ciceData = 0.6; - double data() { return cice[0]; } - double refData() { return hice_ref[0]; } + double data() { return ciceAccessor.getHostRO()[0]; } + double refData() { return hice_refAccessor.getHostRO()[0]; } private: - HField cice; - ModelArrayRef hice_ref; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor hice_refAccessor; }; TEST_SUITE_BEGIN("ModelComponent"); @@ -95,57 +93,55 @@ TEST_CASE("Test array registration") class ModuleSemiShared : public ModelComponent { public: ModuleSemiShared() - : qic(ModelArray::HField()) - , qio_ref(getStore()) + : qicAccessor(getStore(), RW) + , qio_refAccessor(getStore()) { - getStore().registerArray(Shared::Q_IC, &qic, RW); } - void setData(const ModelState::DataMap& ms) override { qic[0] = qicData; } + void setData(const ModelState::DataMap& ms) override { qicAccessor.getHostRW()[0] = qicData; } std::string getName() const override { return "SemiShared"; } ModelState getStatePrognostic() const override { return { { - { "qic", qic }, + { "qic", qicAccessor.getHostRO() }, }, {} }; } const double qicData = 123; - double data() { return qic[0]; } - double refData() { return qio_ref[0]; } + double data() { return qicAccessor.getHostRO()[0]; } + double refData() { return qio_refAccessor.getHostRO()[0]; } private: - HField qic; - ModelArrayRef qio_ref; + ModelArrayAccessor qicAccessor; + ModelArrayAccessor qio_refAccessor; }; class ModuleShared : public ModelComponent { public: ModuleShared() - : qio(ModelArray::HField()) - , qic_ref(getStore()) + : qioAccessor(getStore(), RW) + , qic_refAccessor(getStore()) { - getStore().registerArray(Shared::Q_IO, &qio, RW); } - void setData(const ModelState::DataMap& ms) override { qio[0]; } + void setData(const ModelState::DataMap& ms) override { /*qio[0]; */ } std::string getName() const override { return "Shared"; } ModelState getStatePrognostic() const override { return { { - { "qio", qio }, + { "qio", qioAccessor.getHostRO() }, }, {} }; } const double qioData = 234; const double qicData = 246; - double data() { return qio[0]; } - double& refData() { return qic_ref[0]; } - void setRefData() { qic_ref[0] = qicData; } + double data() { return qioAccessor.getHostRO()[0]; } + double& refData() { return qic_refAccessor.getHostRW()[0]; } + void setRefData() { qic_refAccessor.getHostRW()[0] = qicData; } private: - HField qio; - ModelArrayRef qic_ref; + ModelArrayAccessor qioAccessor; + ModelArrayAccessor qic_refAccessor; }; TEST_CASE("Shared and semi-protected arrays") diff --git a/core/test/PrognosticData_test.cpp b/core/test/PrognosticData_test.cpp index 35c7b22a3..8db7850c3 100644 --- a/core/test/PrognosticData_test.cpp +++ b/core/test/PrognosticData_test.cpp @@ -2,7 +2,6 @@ * @author Tim Spain */ -#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN #include #include "include/PrognosticData.hpp" @@ -53,13 +52,15 @@ TEST_CASE("PrognosticData call order test") void updateBefore(const TimestepTime& tst) override { UnescoFreezing uf; - sst = -1.; + sstAccessor.getHostRW() = -1.; + HField& sss = sssAccessor.getHostRW(); sss = 32.; + HField& mld = mldAccessor.getHostRW(); mld = 10.25; - tf = uf(sss[0]); - cpml = Water::cp * Water::rho * mld[0]; - u = 0; - v = 0; + tfAccessor.getHostRW() = uf(sss[0]); + cpmlAccessor.getHostRW() = Water::cp * Water::rho * mld[0]; + uAccessor.getHostRW() = 0; + vAccessor.getHostRW() = 0; } void updateAfter(const TimestepTime& tst) override { } } ocnBdy; @@ -85,7 +86,8 @@ TEST_CASE("PrognosticData call order test") TimestepTime tst = { TimePoint("2000-01-01T00:00:00Z"), Duration("P0-0T0:10:0") }; pData.update(tst); - ModelArrayRef qow(ModelComponent::getStore()); + ModelArrayAccessor qowAccessor(ModelComponent::getStore()); + const HField& qow = qowAccessor.getHostRO(); double prec = 1e-5; // Correct value diff --git a/dynamics/src/CGDynamicsKernel.cpp b/dynamics/src/CGDynamicsKernel.cpp index 2c846eaa0..7758f2226 100644 --- a/dynamics/src/CGDynamicsKernel.cpp +++ b/dynamics/src/CGDynamicsKernel.cpp @@ -201,9 +201,9 @@ void CGDynamicsKernel::computeGradientOfSeaSurfaceHeight( yGradSeaSurfaceHeight = vGrad; } else { // outer nodes - size_t icg1 = 0; #pragma omp parallel for for (size_t iy = 0; iy <= smesh->ny; ++iy) { + size_t icg1 = (smesh->nx + 1) * iy; size_t icg2 = (2 * smesh->nx + 1) * 2 * iy; for (size_t ix = 0; ix <= smesh->nx; ++ix, ++icg1, icg2 += 2) { xGradSeaSurfaceHeight(icg2) = uGrad(icg1); diff --git a/dynamics/src/include/DynamicsKernel.hpp b/dynamics/src/include/DynamicsKernel.hpp index 1941e930e..72ed83cf0 100644 --- a/dynamics/src/include/DynamicsKernel.hpp +++ b/dynamics/src/include/DynamicsKernel.hpp @@ -27,7 +27,6 @@ #include #include #include -#include namespace Nextsim { @@ -92,7 +91,6 @@ template class DynamicsKernel { */ virtual void setData(const std::string& name, const ModelArray& data) { - // Special cases: hice, cice, (damage, stress) <- not yet implemented if (name == hiceName || name == ciceName || name == hsnowName) { throw std::runtime_error(std::string("Use setDGArray() to set the data for ") + name); @@ -140,6 +138,23 @@ template class DynamicsKernel { } } + // in combination with getAutoRO() this should do the automatic switch + // problem: buffer is only marked dirty at the begining but the data is changed in every update + // solutions: + // * provide accessor instead + // * do set data in every update (error prone since it is easily forgotten) + // * take fields as arguments to kernel update + // virtual void setDGArray(const std::string& name, const DeviceViewAdvect& dgData); + // bad interface: copy can't be avoided, no overload possible + // virtual DeviceViewMA getDG0Data(const std::string& name) const; + // alternative: + // virtual void getDG0Data(const std::string& name, const DeviceViewMA& destination) const; + // other thinks to consider: + // * should make datatransfers in kernel and mirrored views obsolete + // * to completely eliminate host views have to also port computeSeaSurfaceHeight? + // * no data transfer capability in kernel would it more difficult to debug / develop new + // features + virtual void update(const TimestepTime& tst) { ++stepNumber; } /*! diff --git a/dynamics/src/include/ParametricMap.hpp b/dynamics/src/include/ParametricMap.hpp index 36e806225..1e9b634d8 100644 --- a/dynamics/src/include/ParametricMap.hpp +++ b/dynamics/src/include/ParametricMap.hpp @@ -85,9 +85,8 @@ template class ParametricMomentumMap { * * Very similar to divS1 and divS2 but working in CG(1) vectors */ - std::vector, - Eigen::aligned_allocator>> - dX_SSH, dY_SSH; + using DSSHMatrix = Eigen::Matrix; + std::vector> dX_SSH, dY_SSH; /*! * These matrices realize the integration of (E, \grad phi) scaled with the diff --git a/dynamics/src/kokkos/KokkosBrittleCGDynamicsKernel.cpp b/dynamics/src/kokkos/KokkosBrittleCGDynamicsKernel.cpp index 03b2d8ea7..cc637b5cb 100644 --- a/dynamics/src/kokkos/KokkosBrittleCGDynamicsKernel.cpp +++ b/dynamics/src/kokkos/KokkosBrittleCGDynamicsKernel.cpp @@ -3,8 +3,8 @@ */ #include "include/KokkosBrittleCGDynamicsKernel.hpp" +#include "../../../core/src/kokkos/include/KokkosTimer.hpp" #include "include/KokkosDGLimit.hpp" -#include "include/KokkosTimer.hpp" #include namespace Nextsim { @@ -41,8 +41,8 @@ void KokkosBrittleCGDynamicsKernel::initialise( std::tie(avgUHost, avgUDevice) = makeKokkosDualView("avgU", this->avgU); std::tie(avgVHost, avgVDevice) = makeKokkosDualView("avgV", this->avgV); - std::tie(damageHost, damageDevice) - = makeKokkosDualView("damage", static_cast&>(this->damage)); + // std::tie(damageHost, damageDevice) + // = makeKokkosDualView("damage", static_cast&>(this->damage)); } /*************************************************************/ @@ -81,27 +81,11 @@ void KokkosBrittleCGDynamicsKernel::update(const TimestepTime& tst) static KokkosTimer timerDivergence("divGPU"); static KokkosTimer timerMomentum("momentumGPU"); static KokkosTimer timerBoundary("bcGPU"); - static KokkosTimer timerUpload("uploadGPU"); - static KokkosTimer timerDownload("downloadGPU"); static KokkosTimer timerAdvection("advectionGPU"); static KokkosTimer timerPrepIt("prepItGPU"); - timerUpload.start(); // explicit execution space enables asynchronous execution auto execSpace = Kokkos::DefaultExecutionSpace(); - // uDevice, vDevice are already copied to the device in KokkosCGDynamicsKernel::prepareAdvection - Kokkos::deep_copy(execSpace, this->uOceanDevice, this->uOceanHost); - Kokkos::deep_copy(execSpace, this->vOceanDevice, this->vOceanHost); - - Kokkos::deep_copy(execSpace, this->uAtmosDevice, this->uAtmosHost); - Kokkos::deep_copy(execSpace, this->vAtmosDevice, this->vAtmosHost); - - Kokkos::deep_copy(execSpace, this->hiceDevice, this->hiceHost); - Kokkos::deep_copy(execSpace, this->ciceDevice, this->ciceHost); - Kokkos::deep_copy(execSpace, this->hsnowDevice, this->hsnowHost); - - Kokkos::deep_copy(execSpace, this->damageDevice, this->damageHost); - timerUpload.stop(); const FloatType dt = tst.step.seconds(); timerAdvection.start(); @@ -161,22 +145,6 @@ void KokkosBrittleCGDynamicsKernel::update(const TimestepTime& tst) Base::updateIceOceanStressDevice(this->uIceOceanStressDevice, this->vIceOceanStressDevice, this->avgUDevice, this->avgVDevice, this->uOceanDevice, this->vOceanDevice, this->params, this->cosOceanAngle, this->sinOceanAngle); - // not needed on the host because the fields are only used in getDG0Data - // Kokkos::deep_copy(execSpace, this->uIceOceanStressHost, this->uIceOceanStressDevice); - // Kokkos::deep_copy(execSpace, this->vIceOceanStressHost, this->vIceOceanStressDevice); - - timerDownload.start(); - Kokkos::deep_copy(execSpace, this->uHost, this->uDevice); - Kokkos::deep_copy(execSpace, this->vHost, this->vDevice); - - Kokkos::deep_copy(execSpace, this->hiceHost, this->hiceDevice); - Kokkos::deep_copy(execSpace, this->ciceHost, this->ciceDevice); - Kokkos::deep_copy(execSpace, this->hsnowHost, this->hsnowDevice); - Kokkos::deep_copy(execSpace, this->damageHost, this->damageDevice); - /* Kokkos::deep_copy(execSpace, this->s11Host, this->s11Device); - Kokkos::deep_copy(execSpace, this->s12Host, this->s12Device); - Kokkos::deep_copy(execSpace, this->s22Host, this->s22Device);*/ - timerDownload.stop(); // Finally, do the base class update DynamicsKernel::update(tst); @@ -189,10 +157,22 @@ void KokkosBrittleCGDynamicsKernel::setData( if (name == damageName) { throw std::runtime_error(std::string("Use setDGArray() to set the data for ") + name); } else { - CGDynamicsKernel::setData(name, data); + KokkosCGDynamicsKernel::setData(name, data); } } +template +void KokkosBrittleCGDynamicsKernel::setData( + const std::string& name, const ConstDeviceViewMA& data) +{ + if (name == damageName) { + throw std::runtime_error(std::string("Use setDGArray() to set the data for ") + name); + } else { + KokkosCGDynamicsKernel::setData(name, data); + } +} + +/* template void KokkosBrittleCGDynamicsKernel::setDGArray( const std::string& name, ModelArray::DataType& dgData) @@ -202,6 +182,17 @@ void KokkosBrittleCGDynamicsKernel::setDGArray( } else { CGDynamicsKernel::setDGArray(name, dgData); } +}*/ + +template +void KokkosBrittleCGDynamicsKernel::setDGArray( + const std::string& name, const KokkosDeviceView& dgData) +{ + if (name == damageName) { + damageDevice = dgData; + } else { + KokkosCGDynamicsKernel::setDGArray(name, dgData); + } } template diff --git a/dynamics/src/kokkos/KokkosCGDynamicsKernel.cpp b/dynamics/src/kokkos/KokkosCGDynamicsKernel.cpp index b84d19197..347b06db8 100644 --- a/dynamics/src/kokkos/KokkosCGDynamicsKernel.cpp +++ b/dynamics/src/kokkos/KokkosCGDynamicsKernel.cpp @@ -3,12 +3,12 @@ */ #include "include/KokkosCGDynamicsKernel.hpp" +#include "../../../core/src/kokkos/include/KokkosTimer.hpp" +#include "include/KokkosDGLimit.hpp" #include "include/KokkosDGTransport.hpp" #include "include/KokkosInterpolations.hpp" #include "include/KokkosMesh.hpp" -#include "include/KokkosDGLimit.hpp" #include "include/KokkosSlopeLimiter.hpp" -#include "include/KokkosTimer.hpp" namespace Nextsim { /*************************************************************/ @@ -19,8 +19,7 @@ KokkosCGDynamicsKernel::KokkosCGDynamicsKernel(const DynamicsParame } // defined explicitly to enable pimpl with unique ptr -template -KokkosCGDynamicsKernel::~KokkosCGDynamicsKernel() = default; +template KokkosCGDynamicsKernel::~KokkosCGDynamicsKernel() = default; /*************************************************************/ template @@ -40,6 +39,8 @@ void KokkosCGDynamicsKernel::initialise( = makeKokkosDualView("xGradSeaSurfaceHeight", this->xGradSeaSurfaceHeight); std::tie(yGradSeaSurfaceHeightHost, yGradSeaSurfaceHeightDevice) = makeKokkosDualView("yGradSeaSurfaceHeight", this->yGradSeaSurfaceHeight); + std::tie(seaSurfaceHeightHost, seaSurfaceHeightDevice) + = makeKokkosDualView("seaSurfaceHeight", this->seaSurfaceHeight); std::tie(dStressXHost, dStressXDevice) = makeKokkosDualView("dStressX", this->dStressX); std::tie(dStressYHost, dStressYDevice) = makeKokkosDualView("dStressY", this->dStressY); @@ -68,6 +69,8 @@ void KokkosCGDynamicsKernel::initialise( tempData.resize_by_mesh(*this->smesh); std::tie(tempDataAdvectHost, tempDataAdvectDevice) = makeKokkosDualView("tempDataAdvect", (this->tempDataAdvect)); + ModelArray tempDataMA; + tempDataMADevice = makeKokkosDeviceView("tempDataMA", tempDataMA.data()); assert(this->pmap); divS1Device = makeKokkosDeviceViewMap("divS1", this->pmap->divS1, MakeViewOptions::DEVICE_COPY); @@ -79,14 +82,13 @@ void KokkosCGDynamicsKernel::initialise( = makeKokkosDeviceViewMap("iMgradY", this->pmap->iMgradY, MakeViewOptions::DEVICE_COPY); iMMDevice = makeKokkosDeviceViewMap("iMM", this->pmap->iMM, MakeViewOptions::DEVICE_COPY); - // needed for stress and momentum - std::tie(hiceHost, hiceDevice) - = makeKokkosDualView("hice", static_cast&>(this->hice)); - std::tie(ciceHost, ciceDevice) - = makeKokkosDualView("cice", static_cast&>(this->cice)); - std::tie(hsnowHost, hsnowDevice) - = makeKokkosDualView("hsnow", static_cast&>(this->hsnow)); - + // this fields are given from outside the kernel now with setDGVector + // std::tie(hiceHost, hiceDevice) + // = makeKokkosDualView("hice", static_cast&>(this->hice)); + // std::tie(ciceHost, ciceDevice) + // = makeKokkosDualView("cice", static_cast&>(this->cice)); + // std::tie(hsnowHost, hsnowDevice) + // = makeKokkosDualView("hsnow", static_cast&>(this->hsnow)); PSIAdvectDevice = makeKokkosDeviceView( "PSI", PSI, MakeViewOptions::DEVICE_COPY); @@ -94,7 +96,7 @@ void KokkosCGDynamicsKernel::initialise( "PSI", PSI, MakeViewOptions::DEVICE_COPY); lumpedCGMassDevice = makeKokkosDeviceView( - "lumpedcgmass", this->pmap->lumpedcgmass, MakeViewOptions::DEVICE_COPY); + "lumpedCGMass", this->pmap->lumpedcgmass, MakeViewOptions::DEVICE_COPY); iMJwPSIDevice = makeKokkosDeviceViewMap("iMJwPSI", this->pmap->iMJwPSI, MakeViewOptions::DEVICE_COPY); iMJwPSIAdvectDevice = makeKokkosDeviceViewMap( @@ -122,6 +124,28 @@ void KokkosCGDynamicsKernel::initialise( *this->smesh, *this->meshData, *cG2DGAdvectInterpolator); slopeLimiterDevice = std::make_unique>(*this->smesh, *this->meshData); + + if constexpr (DGadvection != 1 || CGdegree != 1) { + dG2CGFirstOrderInterpolator + = std::make_unique>(*this->smesh); + _dXSSHDevice + = makeKokkosDeviceViewMap("dXSSH", this->pmap->dX_SSH, MakeViewOptions::DEVICE_COPY); + _dYSSHDevice + = makeKokkosDeviceViewMap("dYSSH", this->pmap->dY_SSH, MakeViewOptions::DEVICE_COPY); + _lumpedCG1MassDevice = makeKokkosDeviceView( + "lumpedCG1Mass", this->pmap->lumpedcg1mass, MakeViewOptions::DEVICE_COPY); + _uGradDevice = DeviceViewCG1("uGrad", _lumpedCG1MassDevice.extent(0)); + _vGradDevice = DeviceViewCG1("vGrad", _lumpedCG1MassDevice.extent(0)); + } + + namedCGFields = { + { uName, uDevice }, + { vName, vDevice }, + { uWindName, uAtmosDevice }, + { vWindName, vAtmosDevice }, + { uOceanName, uOceanDevice }, + { vOceanName, vOceanDevice }, + }; } /*************************************************************/ @@ -131,58 +155,115 @@ ModelArray KokkosCGDynamicsKernel::getDG0Data(const std::string& na if (name == shearName) { computeShearDevice(tempDataAdvectDevice, e11Device, e12Device, e22Device, PSIStressDevice, meshData->landMaskDevice, iMJwPSIAdvectDevice); - Kokkos::deep_copy(this->tempDataAdvectHost, this->tempDataAdvectDevice); HField data(ModelArray::Type::H); - return DGModelArray::dg2ma(tempDataAdvect, data); + return dG2MA(data, tempDataAdvectDevice); } else if (name == divergenceName) { computeTensorInvariantIDevice(tempDataDevice, e11Device, e12Device, e22Device); Kokkos::deep_copy(this->tempDataHost, this->tempDataDevice); HField data(ModelArray::Type::H); - return DGModelArray::dg2ma(tempData, data); + return dG2MA(data, tempDataAdvectDevice); } else if (name == sigmaIName) { computeTensorInvariantIDevice(tempDataDevice, s11Device, s12Device, s22Device); - Kokkos::deep_copy(this->tempDataHost, this->tempDataDevice); HField data(ModelArray::Type::H); - return DGModelArray::dg2ma(tempData, data); + return dG2MA(data, tempDataAdvectDevice); } else if (name == sigmaIIName) { computeTensorInvariantIIDevice(tempDataDevice, s11Device, s12Device, s22Device); - Kokkos::deep_copy(this->tempDataHost, this->tempDataDevice); HField data(ModelArray::Type::H); - return DGModelArray::dg2ma(tempData, data); + return dG2MA(data, tempDataAdvectDevice); } else if (name == uName) { (*cG2DGAdvectInterpolator)(tempDataAdvectDevice, uDevice); - Kokkos::deep_copy(tempDataAdvectHost, tempDataAdvectDevice); - ModelArray data(ModelArray::Type::U); - return DGModelArray::dg2ma(tempDataAdvect, data); + HField data(ModelArray::Type::U); + return dG2MA(data, tempDataAdvectDevice); } else if (name == vName) { (*cG2DGAdvectInterpolator)(tempDataAdvectDevice, vDevice); - Kokkos::deep_copy(tempDataAdvectHost, tempDataAdvectDevice); - ModelArray data(ModelArray::Type::V); - return DGModelArray::dg2ma(tempDataAdvect, data); + HField data(ModelArray::Type::V); + return dG2MA(data, tempDataAdvectDevice); } else if (name == uIOStressName) { (*cG2DGAdvectInterpolator)(tempDataAdvectDevice, uIceOceanStressDevice); - Kokkos::deep_copy(tempDataAdvectHost, tempDataAdvectDevice); - ModelArray data(ModelArray::Type::U); - return DGModelArray::dg2ma(tempDataAdvect, data); + HField data(ModelArray::Type::U); + return dG2MA(data, tempDataAdvectDevice); } else if (name == vIOStressName) { (*cG2DGAdvectInterpolator)(tempDataAdvectDevice, vIceOceanStressDevice); - Kokkos::deep_copy(tempDataAdvectHost, tempDataAdvectDevice); - ModelArray data(ModelArray::Type::V); - return DGModelArray::dg2ma(tempDataAdvect, data); + HField data(ModelArray::Type::V); + return dG2MA(data, tempDataAdvectDevice); } else { return CGDynamicsKernel::getDG0Data(name); } } +/*************************************************************/ +template +void KokkosCGDynamicsKernel::setData(const std::string& name, const ModelArray& data) +{ + // Just copy to device and use the other setter so that we don't have to treat every field + // differently. This is much faster than directly copying to the destination if there is + // a stride involved. + // const auto& [dataHost, dataDevice] = makeKokkosDualView(name + "Temp", data.data()); + assert(data.components(0).size() == 1 && "Expecting only dg(0) fields in setData()"); + const auto dataHost = makeKokkosHostView(data.data()); + Kokkos::deep_copy(tempDataMADevice, dataHost); + setData(name, tempDataMADevice); +} + +template +void KokkosCGDynamicsKernel::setData( + const std::string& name, const ConstDeviceViewMA& data) +{ + // Special cases: hice, cice + if (name == hiceName || name == ciceName || name == hsnowName) { + throw std::runtime_error(std::string("Use setDGArray() to set the data for ") + name); + } else if (name == sshName) { + kokkosMA2DG<1>(seaSurfaceHeightDevice, data); + } else if (auto it = namedCGFields.find(name); it != namedCGFields.end()) { + mA2CG(it->second, data); + } else { + throw std::runtime_error(std::string("Trying to setData() for the unknown field ") + name); + } +} + +/*************************************************************/ +template +void KokkosCGDynamicsKernel::setDGArray( + const std::string& name, ModelArray::DataType& dgData) +{ + // There are different problems depending on the configuration. + if constexpr (IS_GPU_EXEC_SPACE) { + // If the kernel runs on the device it would be necessary to copy every DGVector set with + // this function back after the update. It makes more sense to handle host-device transfers + // outside of the kernel where the ModelArrayStore tracks the state. + throw std::runtime_error("Setting the buffers with setDGArray(ModelArray::DataType&) does " + "not work properly with device execution " + "because the updated fields are not transferred back."); + } else { + // To implement this function we just have to update the correct Kokkos views, i.e. + // std::tie(hiceHost, hiceDevice) = makeKokkosDualView("hice", dgData); + // However, there is currently no scenario where this function is needed. + // As long as Kokkos is enabled, the view based setDGArray() will be used, even without + // device execution. + throw std::runtime_error( + "Setting the buffers with setDGArray(ModelArray::DataType&) is currently" + "not implemented for the Kokkos kernel."); + } +} + +template +void KokkosCGDynamicsKernel::setDGArray( + const std::string& name, const KokkosDeviceView& dgData) +{ + if (name == hiceName) { + hiceDevice = dgData; + } else if (name == ciceName) { + ciceDevice = dgData; + } else if (name == hsnowName) { + hsnowDevice = dgData; + } +} + /*************************************************************/ template void KokkosCGDynamicsKernel::prepareAdvection() { static KokkosTimer timerPrepAdvection("prepareAdvection"); - //CGDynamicsKernel::prepareAdvection(); timerPrepAdvection.start(); - auto execSpace = Kokkos::DefaultExecutionSpace(); - Kokkos::deep_copy(execSpace, this->uDevice, this->uHost); - Kokkos::deep_copy(execSpace, this->vDevice, this->vHost); dGTransportDevice->prepareAdvection(uDevice, vDevice); timerPrepAdvection.stop(); } @@ -239,18 +320,139 @@ void KokkosCGDynamicsKernel::advectDGVFieldDevice( } /*************************************************************/ +template void compare(const std::string& name, const Mat& m1, const Mat& m2) +{ + FloatType normRef = m1.norm(); + FloatType normDiff = (m1 - m2).norm(); + std::cout << name << " - abs: " << normDiff << ", rel: " << normDiff / normRef + << ", norm: " << normRef; + Eigen::Index maxIndex; + const FloatType maxVal = (m1 - m2).cwiseAbs().maxCoeff(&maxIndex); + std::cout << ", max diff: " << maxVal << " at " << maxIndex << " abs(" << m1(maxIndex) << "-" + << m2(maxIndex) << ")" << std::endl; +} + template void KokkosCGDynamicsKernel::updateGradientOfSeaSurfaceHeight() { - // Reinit the gradient of the sea surface height. Not done by DataMap as seaSurfaceHeight is - // always dG(0). Currently done on CPU because their are no dependencies on other - // computations and the cost is small (<3% of dynamics with a single thread). - this->computeGradientOfSeaSurfaceHeight(this->seaSurfaceHeight); + // capturing structured bindings is C++20 so we can't just do + // const auto& [dG2CGInterpolater, dXSSHDevice, dYSSHDevice, lumpedCG1MassDevice] = + const auto precomputedMaps = [this]() { + if constexpr (DGadvection == 1 && CGdegree == 1) { + return std::tie(dG2CGAdvectInterpolator, divS1Device, divS2Device, lumpedCGMassDevice); + } else { + return std::tie( + dG2CGFirstOrderInterpolator, _dXSSHDevice, _dYSSHDevice, _lumpedCG1MassDevice); + } + }(); + const auto& dG2CGInterpolater = std::get<0>(precomputedMaps); + const auto& dXSSHDevice = std::get<1>(precomputedMaps); + const auto& dYSSHDevice = std::get<2>(precomputedMaps); + const auto& lumpedCG1MassDevice = std::get<3>(precomputedMaps); + auto execSpace = Kokkos::DefaultExecutionSpace(); - Kokkos::deep_copy( - execSpace, this->xGradSeaSurfaceHeightDevice, this->xGradSeaSurfaceHeightHost); - Kokkos::deep_copy( - execSpace, this->yGradSeaSurfaceHeightDevice, this->yGradSeaSurfaceHeightHost); + Kokkos::deep_copy(execSpace, this->seaSurfaceHeightDevice, this->seaSurfaceHeightHost); + + const DeviceViewCG1 cgSeaSurfaceHeightDevice + = DeviceViewCG1("cgSeaSurfaceHeight", lumpedCG1MassDevice.extent(0)); + (*dG2CGInterpolater)(cgSeaSurfaceHeightDevice, seaSurfaceHeightDevice); + + const auto gradientFields = [&]() { + if constexpr (CGdegree == 1) { + return std::tie(xGradSeaSurfaceHeightDevice, yGradSeaSurfaceHeightDevice); + } else { + return std::tie(_uGradDevice, _vGradDevice); + } + }(); + const DeviceViewCG1& uGradDevice = std::get<0>(gradientFields); + const DeviceViewCG1& vGradDevice = std::get<1>(gradientFields); + Kokkos::deep_copy(execSpace, uGradDevice, 0.0); + Kokkos::deep_copy(execSpace, vGradDevice, 0.0); + + const DeviceIndex nx = this->smesh->nx; + const DeviceIndex ny = this->smesh->ny; + + Kokkos::parallel_for( + "computeCGSeaSurfaceGrads", nx * ny, KOKKOS_LAMBDA(const DeviceIndex eid) { + const DeviceIndex cy = eid / nx; //!< y-index of element + const DeviceIndex cx = eid % nx; //!< x-index of element + const DeviceIndex cg1id = cy * (nx + 1) + cx; //!< lower/left Index in cg vector + + // get local CG nodes + const Eigen::Vector locCGSSH = { cgSeaSurfaceHeightDevice(cg1id), + cgSeaSurfaceHeightDevice(cg1id + 1), cgSeaSurfaceHeightDevice(cg1id + nx + 1), + cgSeaSurfaceHeightDevice(cg1id + nx + 1 + 1) }; + + const auto dXSSH = makeEigenMap(dXSSHDevice); + const auto dYSSH = makeEigenMap(dYSSHDevice); + + // compute grad + const Eigen::Vector tx = dXSSH[eid] * locCGSSH; + const Eigen::Vector ty = dYSSH[eid] * locCGSSH; + + // add global vector + Kokkos::atomic_sub(&uGradDevice(cg1id), tx(0)); + Kokkos::atomic_sub(&uGradDevice(cg1id + 1), tx(1)); + Kokkos::atomic_sub(&uGradDevice(cg1id + nx + 1), tx(2)); + Kokkos::atomic_sub(&uGradDevice(cg1id + nx + 1 + 1), tx(3)); + Kokkos::atomic_sub(&vGradDevice(cg1id), ty(0)); + Kokkos::atomic_sub(&vGradDevice(cg1id + 1), ty(1)); + Kokkos::atomic_sub(&vGradDevice(cg1id + nx + 1), ty(2)); + Kokkos::atomic_sub(&vGradDevice(cg1id + nx + 1 + 1), ty(3)); + }); + + // scale with mass + const DeviceIndex cg1Row = nx + 1; + // const DeviceIndex cg1rowRed = nx - 1; + const auto makeScaleGradFn = [&](const DeviceViewCG1& gradDevice) { + return KOKKOS_LAMBDA(const DeviceIndex i) { gradDevice(i) /= lumpedCG1MassDevice(i); }; + }; + + Kokkos::parallel_for("scaleUGrad", uGradDevice.extent(0), makeScaleGradFn(uGradDevice)); + Kokkos::parallel_for("scaleVGrad", vGradDevice.extent(0), makeScaleGradFn(vGradDevice)); + + // correct boundary (just extend in last elements) + // Corners are handled impliclty. By treating them like regular nodes they recieve the + // values of the inner diagonal neighbors during the second (y) update. + const DeviceIndex topLeft = ny * cg1Row; + const auto makeExtendBoundaryXFn = [&](const DeviceViewCG1& gradDevice) { + return KOKKOS_LAMBDA(const DeviceIndex i) + { + gradDevice(i) = gradDevice(i + cg1Row); + gradDevice(topLeft + i) = gradDevice(topLeft + i - cg1Row); + }; + }; + + const auto makeExtendBoundaryYFn = [&](const DeviceViewCG1& gradDevice) { + return KOKKOS_LAMBDA(const DeviceIndex i) + { + const DeviceIndex j = i * cg1Row; + gradDevice(j) = gradDevice(j + 1); + gradDevice(j + cg1Row - 1) = gradDevice(j + cg1Row - 1 - 1); + }; + }; + + Kokkos::parallel_for("extendCornersUGradX", nx + 1, makeExtendBoundaryXFn(uGradDevice)); + Kokkos::parallel_for("extendCornersUGradY", ny + 1, makeExtendBoundaryYFn(uGradDevice)); + + Kokkos::parallel_for("extendCornersVGradX", nx + 1, makeExtendBoundaryXFn(vGradDevice)); + Kokkos::parallel_for("extendCornersVGradY", ny + 1, makeExtendBoundaryYFn(vGradDevice)); + + // Interpolate to CG2 + // If we have CGdegree == 1 we are already finished since the computations where done + // directly in the destination (xGradSeaSurfaceHeightDevice, yGradSeaSurfaceHeightDevice). + if constexpr (CGdegree == 2) { + Kokkos::deep_copy(execSpace, xGradSeaSurfaceHeightDevice, 0.0); + Kokkos::deep_copy(execSpace, yGradSeaSurfaceHeightDevice, 0.0); + Interpolations::kokkosCG12CG2(xGradSeaSurfaceHeightDevice, uGradDevice, nx, ny); + Interpolations::kokkosCG12CG2(yGradSeaSurfaceHeightDevice, vGradDevice, nx, ny); + } + + // gradients are only used internally so they don't have to be synced + /* Kokkos::deep_copy( + execSpace, this->xGradSeaSurfaceHeightHost, xGradSeaSurfaceHeightDevice); + Kokkos::deep_copy( + execSpace, this->yGradSeaSurfaceHeightHost, yGradSeaSurfaceHeightDevice);*/ } /*************************************************************/ @@ -548,6 +750,18 @@ void KokkosCGDynamicsKernel::computeTensorInvariantIIDevice( }); } +/*************************************************************/ +template +ModelArray& KokkosCGDynamicsKernel::dG2MA( + ModelArray& ma, const ConstDeviceViewAdvect& dg) const +{ + kokkosDG2MA(tempDataMADevice, dg); + const auto maView = makeKokkosHostView(ma.getDataRef()); + Kokkos::deep_copy(maView, tempDataMADevice); + + return ma; +} + /*************************************************************/ // because ParametricMomentumMap::iMJwPSIAdvect does not properly depend on DGadvection we // can only build this version since the switch is implemented in compile-time we dont really need diff --git a/dynamics/src/kokkos/KokkosInterpolations.cpp b/dynamics/src/kokkos/KokkosInterpolations.cpp index c94a564fd..a6e774030 100644 --- a/dynamics/src/kokkos/KokkosInterpolations.cpp +++ b/dynamics/src/kokkos/KokkosInterpolations.cpp @@ -173,5 +173,45 @@ namespace Interpolations { template class KokkosDG2CGInterpolator<2, 3>; template class KokkosDG2CGInterpolator<2, 6>; template class KokkosDG2CGInterpolator<2, 8>; + + /*************************************************************/ + void kokkosCG12CG2(const KokkosDeviceView>& dest, + const ConstKokkosDeviceView>& src, const DeviceIndex nx, const DeviceIndex ny) + { + assert(src.extent(0) == (nx + 1) * (ny + 1)); + assert(dest.extent(0) == (2 * nx + 1) * (2 * ny + 1)); + + const DeviceIndex cg1Row = nx + 1; + Kokkos::parallel_for( + "CG12CG2", src.extent(0), KOKKOS_LAMBDA(const DeviceIndex icg1) { + const DeviceIndex iy = icg1 / cg1Row; + const DeviceIndex ix = icg1 % cg1Row; + const DeviceIndex icg2 = (2 * nx + 1) * 2 * iy + 2 * ix; + + // outer nodes + const FloatType dof0 = src(icg1); + dest(icg2) = dof0; + + // along horizontal lines + FloatType dof1; + if (ix < nx) { + dof1 = src(icg1 + 1); + dest(icg2 + 1) = 0.5 * (dof0 + dof1); + } + + // along vertical lines + FloatType dof2; + if (iy < ny) { + dof2 = src(icg1 + cg1Row); + dest(icg2 + 2 * nx + 1) = 0.5 * (dof0 + dof2); + } + + // midpoints + if (ix < nx && iy < ny) { + const FloatType dof3 = src(icg1 + cg1Row + 1); + dest(icg2 + 2 * nx + 1 + 1) = 0.25 * (dof0 + dof1 + dof2 + dof3); + } + }); + } } } \ No newline at end of file diff --git a/dynamics/src/kokkos/KokkosMEVPDynamicsKernel.cpp b/dynamics/src/kokkos/KokkosMEVPDynamicsKernel.cpp index 44c05d23f..c26757987 100644 --- a/dynamics/src/kokkos/KokkosMEVPDynamicsKernel.cpp +++ b/dynamics/src/kokkos/KokkosMEVPDynamicsKernel.cpp @@ -4,7 +4,7 @@ #include "include/KokkosMEVPDynamicsKernel.hpp" #include "include/KokkosMesh.hpp" -#include "include/KokkosTimer.hpp" +#include "../../../core/src/kokkos/include/KokkosTimer.hpp" #include namespace Nextsim { @@ -49,36 +49,19 @@ void KokkosMEVPDynamicsKernel::update(const TimestepTime& tst) static KokkosTimer timerDivergence("divGPU"); static KokkosTimer timerMomentum("momentumGPU"); static KokkosTimer timerBoundary("bcGPU"); - static KokkosTimer timerUpload("uploadGPU"); - static KokkosTimer timerDownload("downloadGPU"); static KokkosTimer timerAdvection("advectionGPU"); static KokkosTimer timerPrepIt("prepItGPU"); - timerUpload.start(); // explicit execution space enables asynchronous execution auto execSpace = Kokkos::DefaultExecutionSpace(); - Kokkos::deep_copy(execSpace, this->uDevice, this->uHost); - Kokkos::deep_copy(execSpace, this->vDevice, this->vHost); - // uDevice, vDevice are already copied to the device in KokkosCGDynamicsKernel::prepareAdvection - Kokkos::deep_copy(execSpace, this->u0DeviceMut, this->uDevice); - Kokkos::deep_copy(execSpace, this->v0DeviceMut, this->vDevice); - - Kokkos::deep_copy(execSpace, this->uOceanDevice, this->uOceanHost); - Kokkos::deep_copy(execSpace, this->vOceanDevice, this->vOceanHost); - - Kokkos::deep_copy(execSpace, this->uAtmosDevice, this->uAtmosHost); - Kokkos::deep_copy(execSpace, this->vAtmosDevice, this->vAtmosHost); - - Kokkos::deep_copy(execSpace, this->hiceDevice, this->hiceHost); - Kokkos::deep_copy(execSpace, this->ciceDevice, this->ciceHost); - Kokkos::deep_copy(execSpace, this->hsnowDevice, this->hsnowHost); - timerUpload.stop(); timerAdvection.start(); this->advectDynamicsFields(tst.step.seconds()); timerAdvection.stop(); timerPrepIt.start(); + Kokkos::deep_copy(execSpace, this->u0DeviceMut, this->uDevice); + Kokkos::deep_copy(execSpace, this->v0DeviceMut, this->vDevice); Base::prepareIterationDevice(this->cgHDevice, this->cgADevice, this->hiceDevice, this->ciceDevice, *this->dG2CGAdvectInterpolator); this->updateGradientOfSeaSurfaceHeight(); @@ -128,18 +111,6 @@ void KokkosMEVPDynamicsKernel::update(const TimestepTime& tst) Base::updateIceOceanStressDevice(this->uIceOceanStressDevice, this->vIceOceanStressDevice, this->uDevice, this->vDevice, this->uOceanDevice, this->vOceanDevice, this->params, this->cosOceanAngle, this->sinOceanAngle); - // not needed on the host because the fields are only used in getDG0Data - // Kokkos::deep_copy(execSpace, this->uIceOceanStressHost, this->uIceOceanStressDevice); - // Kokkos::deep_copy(execSpace, this->vIceOceanStressHost, this->vIceOceanStressDevice); - - timerDownload.start(); - Kokkos::deep_copy(execSpace, this->uHost, this->uDevice); - Kokkos::deep_copy(execSpace, this->vHost, this->vDevice); - - Kokkos::deep_copy(execSpace, this->hiceHost, this->hiceDevice); - Kokkos::deep_copy(execSpace, this->ciceHost, this->ciceDevice); - Kokkos::deep_copy(execSpace, this->hsnowHost, this->hsnowDevice); - timerDownload.stop(); // Finally, do the base class update DynamicsKernel::update(tst); diff --git a/dynamics/src/kokkos/KokkosSlopeLimiter.cpp b/dynamics/src/kokkos/KokkosSlopeLimiter.cpp index 079057672..457456d01 100644 --- a/dynamics/src/kokkos/KokkosSlopeLimiter.cpp +++ b/dynamics/src/kokkos/KokkosSlopeLimiter.cpp @@ -109,7 +109,7 @@ void KokkosSlopeLimiter::computeAlphas(const DeviceViewDG1& alpha, const Con // relative indices of the four vertices in minV/maxV const Kokkos::Array cgIndices = { 0, 1, nx + 1, nx + 2 }; - assert(alpha.extend(0) == nx * ny); + assert(alpha.extent(0) == nx * ny); Kokkos::parallel_for( "computeAlphas", alpha.extent(0), KOKKOS_LAMBDA(const DeviceIndex c) { const DeviceIndex cx = c % nx; @@ -139,7 +139,7 @@ void KokkosSlopeLimiter::computeAlphasX(const DeviceViewDG1& alphaX, // relative indices of the four vertices in minV/maxV const Kokkos::Array cgIndices = { 0, 1, nx + 1, nx + 2 }; - assert(alphaX.extend(0) == nx * ny); + assert(alphaX.extent(0) == nx * ny); Kokkos::parallel_for( "computeAlphasX", alphaX.extent(0), KOKKOS_LAMBDA(const DeviceIndex c) { const DeviceIndex cx = c % nx; @@ -167,7 +167,7 @@ void KokkosSlopeLimiter::computeAlphasY(const DeviceViewDG1& alphaY, // relative indices of the four vertices in minV/maxV const Kokkos::Array cgIndices = { 0, 1, nx + 1, nx + 2 }; - assert(alphaX.extend(0) == nx * ny); + assert(alphaY.extent(0) == nx * ny); Kokkos::parallel_for( "computeAlphasY", alphaY.extent(0), KOKKOS_LAMBDA(const DeviceIndex c) { const DeviceIndex cx = c % nx; diff --git a/dynamics/src/kokkos/include/KokkosBrittleCGDynamicsKernel.hpp b/dynamics/src/kokkos/include/KokkosBrittleCGDynamicsKernel.hpp index be07f4b78..af975dc84 100644 --- a/dynamics/src/kokkos/include/KokkosBrittleCGDynamicsKernel.hpp +++ b/dynamics/src/kokkos/include/KokkosBrittleCGDynamicsKernel.hpp @@ -36,8 +36,11 @@ class KokkosBrittleCGDynamicsKernel : public KokkosCGDynamicsKernel void update(const TimestepTime& tst) override; // expose additional fields - void setData(const std::string& name, const ModelArray& data) override; - void setDGArray(const std::string& name, ModelArray::DataType& dgData) override; + void setData(const std::string& name, const ModelArray& data); + void setData(const std::string& name, const ConstDeviceViewMA& data); + //void setDGArray(const std::string& name, ModelArray::DataType& dgData) override; + using Base::setDGArray; + void setDGArray(const std::string& name, const DeviceViewMA& dgData) override; static void updateMomentumDevice(const DeviceViewCG& uDevice, const DeviceViewCG& vDevice, const DeviceViewCG& avgUDevice, const DeviceViewCG& avgVDevice, @@ -79,7 +82,7 @@ class KokkosBrittleCGDynamicsKernel : public KokkosCGDynamicsKernel HostViewCG avgVHost; DeviceViewAdvect damageDevice; - HostViewAdvect damageHost; + // HostViewAdvect damageHost; }; } /* namespace Nextsim */ diff --git a/dynamics/src/kokkos/include/KokkosCGDynamicsKernel.hpp b/dynamics/src/kokkos/include/KokkosCGDynamicsKernel.hpp index 90ff01498..0396bb146 100644 --- a/dynamics/src/kokkos/include/KokkosCGDynamicsKernel.hpp +++ b/dynamics/src/kokkos/include/KokkosCGDynamicsKernel.hpp @@ -5,8 +5,10 @@ #ifndef KOKKOSCGDYNAMICSKERNEL_HPP #define KOKKOSCGDYNAMICSKERNEL_HPP +#include "../../../core/src/kokkos/include/KokkosModelArray.hpp" +#include "../../../core/src/kokkos/include/KokkosUtils.hpp" #include "../../include/CGDynamicsKernel.hpp" -#include "KokkosUtils.hpp" +#include "KokkosDGModelArray.hpp" namespace Nextsim { @@ -24,11 +26,17 @@ template constexpr int NGP_DG = ((DG == 8) || (DG == 6)) ? 3 : (DG == 3 template class KokkosCGDynamicsKernel : public CGDynamicsKernel { public: + using Base = CGDynamicsKernel; // common types for Kokkos buffers // cG components using DeviceViewCG = KokkosDeviceView>; using HostViewCG = KokkosHostView>; using ConstDeviceViewCG = ConstKokkosDeviceView>; + using DeviceViewCG1 = KokkosDeviceView>; + using ConstDeviceViewCG1 = ConstKokkosDeviceView>; + + using DeviceViewDG1 = KokkosDeviceView>; + using HostViewDG1 = KokkosHostView>; // strain and stress components using DeviceViewStress = KokkosDeviceView>; @@ -60,6 +68,8 @@ template class KokkosCGDynamicsKernel : public CGDynamicsKerne typename ParametricMomentumMap::GaussMapMatrix>; using GaussMapAdvectDevice = KokkosDeviceMapView< typename ParametricMomentumMap::GaussMapAdvectMatrix>; + using DSSHDevice + = KokkosDeviceMapView::DSSHMatrix>; KokkosCGDynamicsKernel(const DynamicsParameters& params); // still defaulted but explicitly defined in the source file to allow for pimpl with unique_ptr @@ -69,6 +79,15 @@ template class KokkosCGDynamicsKernel : public CGDynamicsKerne ModelArray getDG0Data(const std::string& name) const override; + // The host variant is needed in IDynamics:setData where data does not come out of the + // ModelArrayStore. + void setData(const std::string& name, const ModelArray& data) override; + // Use this to directly set the data of kernel's internal device buffers. + virtual void setData(const std::string& name, const ConstDeviceViewMA& data); + void setDGArray(const std::string& name, ModelArray::DataType& dgData) override; + virtual void setDGArray( + const std::string& name, const KokkosDeviceView& dgData); + void prepareAdvection() override; void advectDynamicsFields(double timestep) override; DGVector& advectDGVField(double timestep, DGVector& field, @@ -130,6 +149,22 @@ template class KokkosCGDynamicsKernel : public CGDynamicsKerne // currently not used void updateMomentum(const TimestepTime& tst) override { } + // copy data from a ModelArray view to a cG-field on device + template + void mA2CG(const DeviceViewCG& dest, + const ConstKokkosEigenView& src) const + { + Kokkos::deep_copy(tempDataAdvectDevice, 0.0); + kokkosMA2DG(tempDataAdvectDevice, src); + (*dG2CGAdvectInterpolator)(dest, tempDataAdvectDevice); + } + + // copy data from a dg field on device to a host ModelArray + ModelArray& dG2MA(ModelArray& ma, const ConstDeviceViewAdvect& dg) const; + + // named fields for setData + std::unordered_map namedCGFields; + // cG (velocity) components DeviceViewCG uDevice; HostViewCG uHost; @@ -145,6 +180,8 @@ template class KokkosCGDynamicsKernel : public CGDynamicsKerne HostViewCG xGradSeaSurfaceHeightHost; DeviceViewCG yGradSeaSurfaceHeightDevice; HostViewCG yGradSeaSurfaceHeightHost; + DeviceViewDG1 seaSurfaceHeightDevice; + HostViewDG1 seaSurfaceHeightHost; DeviceViewCG dStressXDevice; HostViewCG dStressXHost; @@ -187,6 +224,7 @@ template class KokkosCGDynamicsKernel : public CGDynamicsKerne DGVector tempDataAdvect; DeviceViewAdvect tempDataAdvectDevice; HostViewAdvect tempDataAdvectHost; + DeviceViewMA tempDataMADevice; // precomputed parametric map DivMapDevice divS1Device; @@ -199,11 +237,11 @@ template class KokkosCGDynamicsKernel : public CGDynamicsKerne // data that is needed by the child classes implementing stress and momentum DeviceViewAdvect hiceDevice; - HostViewAdvect hiceHost; + // HostViewAdvect hiceHost; DeviceViewAdvect ciceDevice; - HostViewAdvect ciceHost; + // HostViewAdvect ciceHost; DeviceViewAdvect hsnowDevice; - HostViewAdvect hsnowHost; + // HostViewAdvect hsnowHost; // constant matrices also need to be available on the GPU PSIAdvectView PSIAdvectDevice; @@ -224,6 +262,15 @@ template class KokkosCGDynamicsKernel : public CGDynamicsKerne dG2CGAdvectInterpolator; std::unique_ptr> dGTransportDevice; std::unique_ptr> slopeLimiterDevice; + + // sea surface height is always computed in first order + // parts are only initialized if cG2DGAdvectInterpolator has a different order from <1,1> + std::unique_ptr> dG2CGFirstOrderInterpolator; + DSSHDevice _dXSSHDevice; + DSSHDevice _dYSSHDevice; + ConstDeviceViewCG1 _lumpedCG1MassDevice; + DeviceViewCG1 _uGradDevice; + DeviceViewCG1 _vGradDevice; }; } diff --git a/dynamics/src/kokkos/include/KokkosDGLimit.hpp b/dynamics/src/kokkos/include/KokkosDGLimit.hpp index 70be5aff6..91dd96887 100644 --- a/dynamics/src/kokkos/include/KokkosDGLimit.hpp +++ b/dynamics/src/kokkos/include/KokkosDGLimit.hpp @@ -5,7 +5,7 @@ #ifndef __KOKKOSDGLIMIT_HPP #define __KOKKOSDGLIMIT_HPP -#include "KokkosUtils.hpp" +#include "../../../core/src/kokkos/include/KokkosUtils.hpp" #include "../../include/dgVector.hpp" namespace Nextsim { diff --git a/dynamics/src/kokkos/include/KokkosDGModelArray.hpp b/dynamics/src/kokkos/include/KokkosDGModelArray.hpp new file mode 100644 index 000000000..4ee562008 --- /dev/null +++ b/dynamics/src/kokkos/include/KokkosDGModelArray.hpp @@ -0,0 +1,77 @@ +/*! + * @author Robert Jendersie + */ + +// by also guarding for USE_KOKKOS this header can be safely included even +// when Kokkos is not enabled +#if !defined(KOKKOSDGMODELARRAY_HPP) && defined(USE_KOKKOS) +#define KOKKOSDGMODELARRAY_HPP + +#include "../../include/dgVector.hpp" +#include "include/ModelArray.hpp" + +namespace Nextsim { + +/*! + * @brief Copy data from an ModelArray to a DGVector through Kokkos views. + * + * @details Source and destination do not have to be located in the same memory space but strided + * host -> device transfers can be extremely slow. + * + * @param dg The destination DGVector view. + * @param ma The source ModelArray view. + */ +template +void kokkosMA2DG(const KokkosEigenView, ArgsDG...>& dg, + const ConstKokkosEigenView& ma) +{ + // N == 1 needs different treatment at compile-time because the corresponding Kokkos::View has + // only one dimension. + if constexpr (N == 1) { + assert(ma.extent(1) == 1); + // views need to have the same rank so we squeeze 1-sized component dimension + const auto firstCompMA = Kokkos::subview(ma, Kokkos::ALL(), 0); + Kokkos::deep_copy(dg, firstCompMA); + } else if (N == ma.extent(1)) { + Kokkos::deep_copy(dg, ma); + } else { + assert(ma.extent(1) == 1); + // Assign only to the 0th component. Use a range to keep the dimension. + const auto firstCompDG = Kokkos::subview(dg, Kokkos::ALL(), std::make_pair(0, 1)); + Kokkos::deep_copy(firstCompDG, ma); + } +} + +/*! + * @brief Copy data from an DGVector to a ModelArray through Kokkos views. + * + * @details Source and destination do not have to be located in the same memory space but strided + * host -> device transfers can be extremely slow. + * + * @param ma The destination ModelArray view. + * @param dg The source DGVector view. + */ +template +void kokkosDG2MA(const KokkosEigenView& ma, + const ConstKokkosEigenView, ArgsDG...>& dg) +{ + // N == 1 needs different treatment at compile-time because the corresponding Kokkos::View has + // only one dimension. + if constexpr (N == 1) { + assert(ma.extent(1) == 1); + // views need to have the same rank so we squeeze 1-sized component dimension + const auto firstCompMA = Kokkos::subview(ma, Kokkos::ALL(), 0); + Kokkos::deep_copy(firstCompMA, dg); + } else if (N == ma.extent(1)) { + Kokkos::deep_copy(ma, dg); + } else { + assert(ma.extent(1) == 1); + // Assign only to the 0th component. Use a range to keep the dimension. + const auto firstCompDG = Kokkos::subview(dg, Kokkos::ALL(), std::make_pair(0, 1)); + Kokkos::deep_copy(ma, firstCompDG); + } +} + +} + +#endif // KOKKOSDGMODELARRAY_HPP \ No newline at end of file diff --git a/dynamics/src/kokkos/include/KokkosInterpolations.hpp b/dynamics/src/kokkos/include/KokkosInterpolations.hpp index 608cef524..ab1e35254 100644 --- a/dynamics/src/kokkos/include/KokkosInterpolations.hpp +++ b/dynamics/src/kokkos/include/KokkosInterpolations.hpp @@ -5,10 +5,10 @@ #ifndef __KOKKOSINTERPOLATIONS_HPP #define __KOKKOSINTERPOLATIONS_HPP +#include "../../../core/src/kokkos/include/KokkosUtils.hpp" #include "../include/cgVector.hpp" #include "../include/codeGenerationDGinGauss.hpp" #include "../include/dgVector.hpp" -#include "KokkosUtils.hpp" // #include "../include/ParametricMesh.hpp" namespace Nextsim { @@ -45,6 +45,10 @@ namespace Interpolations { DeviceIndex nx; DeviceIndex ny; }; + + // Interpolate CG1 to CG2. This is a free function because it does not use precomputed maps. + void kokkosCG12CG2(const KokkosDeviceView>& dest, + const ConstKokkosDeviceView>& src, const DeviceIndex nx, const DeviceIndex ny); } } diff --git a/dynamics/src/kokkos/include/KokkosMEVPDynamicsKernel.hpp b/dynamics/src/kokkos/include/KokkosMEVPDynamicsKernel.hpp index 681c50d75..c3b127903 100644 --- a/dynamics/src/kokkos/include/KokkosMEVPDynamicsKernel.hpp +++ b/dynamics/src/kokkos/include/KokkosMEVPDynamicsKernel.hpp @@ -9,7 +9,7 @@ #include "../../include/VPParameters.hpp" #include "KokkosCGDynamicsKernel.hpp" -#include "KokkosUtils.hpp" +#include "../../../core/src/kokkos/include/KokkosUtils.hpp" namespace Nextsim { diff --git a/dynamics/src/kokkos/include/KokkosMesh.hpp b/dynamics/src/kokkos/include/KokkosMesh.hpp index 1a5292264..294309921 100644 --- a/dynamics/src/kokkos/include/KokkosMesh.hpp +++ b/dynamics/src/kokkos/include/KokkosMesh.hpp @@ -6,7 +6,7 @@ #define KOKKOSPARAMETRICMESH_HPP #include "../../include/ParametricMesh.hpp" -#include "KokkosUtils.hpp" +#include "../../../core/src/kokkos/include/KokkosUtils.hpp" namespace Nextsim { diff --git a/dynamics/src/kokkos/include/KokkosSlopeLimiter.hpp b/dynamics/src/kokkos/include/KokkosSlopeLimiter.hpp index cbb5a607c..792eb6595 100644 --- a/dynamics/src/kokkos/include/KokkosSlopeLimiter.hpp +++ b/dynamics/src/kokkos/include/KokkosSlopeLimiter.hpp @@ -6,7 +6,7 @@ #define __KOKKOSSLOPELIMITER_HPP #include "KokkosMesh.hpp" -#include "KokkosUtils.hpp" +#include "../../../core/src/kokkos/include/KokkosUtils.hpp" #include "ParametricMesh.hpp" #include "cgVector.hpp" #include "dgVector.hpp" diff --git a/physics/src/IceGrowth.cpp b/physics/src/IceGrowth.cpp index 0cabe574e..1cea9b5fe 100644 --- a/physics/src/IceGrowth.cpp +++ b/physics/src/IceGrowth.cpp @@ -20,11 +20,11 @@ static const std::map keyMap = { }; IceGrowth::IceGrowth() - : hice(getStore()) - , cice(getStore()) - , hsnow(getStore()) - , qow(getStore()) - , deltaHi(getStore()) + : hiceAccessor(getStore()) + , ciceAccessor(getStore()) + , hsnowAccessor(getStore()) + , qowAccessor(getStore()) + , deltaHiAccessor(getStore()) { } diff --git a/physics/src/SlabOcean.cpp b/physics/src/SlabOcean.cpp index acb1bcb21..f04be0cd1 100644 --- a/physics/src/SlabOcean.cpp +++ b/physics/src/SlabOcean.cpp @@ -7,6 +7,9 @@ #include "include/constants.hpp" #include "include/gridNames.hpp" +#include "include/KernelAlternatives.hpp" +#include "kokkos/include/KokkosTimer.hpp" + #include #include @@ -27,11 +30,6 @@ void SlabOcean::configure() { relaxationTimeT = Configured::getConfiguration(keyMap.at(TIMET_KEY), defaultRelaxationTime); relaxationTimeS = Configured::getConfiguration(keyMap.at(TIMES_KEY), defaultRelaxationTime); - - getStore().registerArray(Protected::SLAB_QDW, &qdw, RO); - getStore().registerArray(Protected::SLAB_FDW, &fdw, RO); - getStore().registerArray(Protected::SLAB_SST, &sstSlab, RO); - getStore().registerArray(Protected::SLAB_SSS, &sssSlab, RO); } ConfigMap SlabOcean::getConfiguration() const @@ -45,8 +43,8 @@ ConfigMap SlabOcean::getConfiguration() const ModelState SlabOcean::getStatePrognostic() const { return { { - { sstName, sstSlab }, - { sssName, sssSlab }, + { sstName, sstSlabAccessor.getHostRO() }, + { sssName, sssSlabAccessor.getHostRO() }, }, getConfiguration() }; } @@ -54,8 +52,8 @@ ModelState SlabOcean::getStatePrognostic() const ModelState SlabOcean::getStateDiagnostic() const { ModelState state = { { - { "Q_slab", qdw }, - { "F_slab", fdw }, + { "Q_slab", qdwAccessor.getHostRO() }, + { "F_slab", fdwAccessor.getHostRO() }, }, {} }; @@ -77,36 +75,69 @@ SlabOcean::HelpMap& SlabOcean::getHelpText(HelpMap& map, bool getAll) void SlabOcean::setData(const ModelState::DataMap& ms) { + HField& qdw = qdwAccessor.getHostRW(); qdw.resize(); + HField& fdw = fdwAccessor.getHostRW(); fdw.resize(); + HField& sstSlab = sstSlabAccessor.getHostRW(); sstSlab.resize(); + HField& sssSlab = sssSlabAccessor.getHostRW(); sssSlab.resize(); } void SlabOcean::update(const TimestepTime& tst) { - dt = tst.step.seconds(); - overElements( - [this](const size_t i, const TimestepTime& tsTime) { this->updateElement(i, tsTime); }, - tst); -} - -void SlabOcean::updateElement(size_t i, const TimestepTime& tst) -{ - // Slab SST update - qdw[i] = (sstExt[i] - sst[i]) * cpml[i] / relaxationTimeT; - sstSlab[i] = sst[i] - dt * (qswNet[i] + qNoSun[i] - qdw[i]) / cpml[i]; - - // Slab SSS update - const double arealDensity = cpml[i] / Water::cp; // density times depth, or cpml divided by cp - // This is simplified compared to the finiteelement.cpp calculation - // Fdw = delS * mld * physical::rhow /(timeS*M_sss[i] - ddt*delS) where delS = sssSlab - sssExt - fdw[i] = (1 - sssExt[i] / sss[i]) * arealDensity / relaxationTimeS; - - // Mass per unit area after all the changes in water volume - // Clamp the denominator to be at least 1 m deep, i.e. at least Water::rho kg m⁻² - const double denominator = std::max(arealDensity - (fwFlux[i] - fdw[i]) * dt, Water::rhoOcean); - sssSlab[i] = sss[i] + (sss[i] * fwFlux[i] - fdw[i] * dt) / denominator; + static KokkosTimer timer("SlabOcean"); + static KokkosTimer timerCopy("SlabOceanCopy"); + static KokkosTimer timerUpdate("SlabOceanUpdate"); + + timer.start(); + timerCopy.start(); + + auto execSpace = DefaultExecutionSpace(); + auto& sstSlab = sstSlabAccessor.getAutoRW(execSpace); + auto& fdw = fdwAccessor.getAutoRW(execSpace); + auto& qdw = qdwAccessor.getAutoRW(execSpace); + auto& sssSlab = sssSlabAccessor.getAutoRW(execSpace); + const auto& fwFlux = fwFluxAccessor.getAutoRO(execSpace); + const auto& sssExt = sssExtAccessor.getAutoRO(execSpace); + const auto& sst = sstAccessor.getAutoRO(execSpace); + const auto& sstExt = sstExtAccessor.getAutoRO(execSpace); + const auto& qswNet = qswNetAccessor.getAutoRO(execSpace); + const auto& sss = sssAccessor.getAutoRO(execSpace); + const auto& qNoSun = qNoSunAccessor.getAutoRO(execSpace); + const auto& cpml = cpmlAccessor.getAutoRO(execSpace); + + const double dt = tst.step.seconds(); + const double relaxationTimeT = SlabOcean::relaxationTimeT; + const double relaxationTimeS = SlabOcean::relaxationTimeS; + timerCopy.stop(); + + timerUpdate.start(); + overElementsAuto(OVER_ELEMENTS_LAMBDA(const ElementIndex i) { + // Slab SST update + qdw[i] = (sstExt[i] - sst[i]) * cpml[i] / relaxationTimeT; + sstSlab[i] = sst[i] - dt * (qswNet[i] + qNoSun[i] - qdw[i]) / cpml[i]; + + // Slab SSS update + const double arealDensity + = cpml[i] / Water::cp; // density times depth, or cpml divided by cp + // This is simplified compared to the finiteelement.cpp calculation + // Fdw = delS * mld * physical::rhow /(timeS*M_sss[i] - ddt*delS) where delS = sssSlab - + // sssExt + fdw[i] = (1 - sssExt[i] / sss[i]) * arealDensity / relaxationTimeS; + + // the compiler does not like a global constant appearing in the argument list of max + // "identifier "Water::rhoOcean" is undefined in device code" + const double rhoOcean = Water::rhoOcean; + // Mass per unit area after all the changes in water volume + // Clamp the denominator to be at least 1 m deep, i.e. at least Water::rho kg m⁻² + const double denominator + = Utils::max(arealDensity - (fwFlux[i] - fdw[i]) * dt, rhoOcean); + sssSlab[i] = sss[i] + (sss[i] * fwFlux[i] - fdw[i] * dt) / denominator; + }); + timerUpdate.stop(); + timer.stop(); } } /* namespace Nextsim */ diff --git a/physics/src/include/IceGrowth.hpp b/physics/src/include/IceGrowth.hpp index 93596f7d0..4040d77ba 100644 --- a/physics/src/include/IceGrowth.hpp +++ b/physics/src/include/IceGrowth.hpp @@ -11,7 +11,6 @@ #include "include/IIceThermodynamics.hpp" #include "include/ILateralIceSpread.hpp" #include "include/IceMinima.hpp" -#include "include/ModelArrayRef.hpp" #include "include/ModelComponent.hpp" #include "include/Time.hpp" @@ -53,11 +52,13 @@ class IceGrowth : public ModelComponent, public Configured { std::unique_ptr iHealing; // Data fields - ModelArrayRef hice; // Timestep initial cell averaged ice thickness, m - ModelArrayRef hsnow; // Timestep initial cell averaged snow thickness, m - ModelArrayRef cice; // Timestep initial ice concentration - ModelArrayRef qow; // open water heat flux, from FluxCalculation - ModelArrayRef deltaHi; // New ice thickness this timestep, m + ModelArrayAccessor + hiceAccessor; // Timestep initial cell averaged ice thickness, m + ModelArrayAccessor + hsnowAccessor; // Timestep initial cell averaged snow thickness, m + ModelArrayAccessor ciceAccessor; // Timestep initial ice concentration + ModelArrayAccessor qowAccessor; // open water heat flux, from FluxCalculation + ModelArrayAccessor deltaHiAccessor; // New ice thickness this timestep, m bool doThermo = true; // Perform any thermodynamics calculations at all }; diff --git a/physics/src/include/SlabOcean.hpp b/physics/src/include/SlabOcean.hpp index 10d2930a2..8f82fe334 100644 --- a/physics/src/include/SlabOcean.hpp +++ b/physics/src/include/SlabOcean.hpp @@ -7,7 +7,7 @@ #include "include/Configured.hpp" #include "include/ModelArray.hpp" -#include "include/ModelArrayRef.hpp" +#include "include/ModelArrayAccessor.hpp" #include "include/ModelComponent.hpp" namespace Nextsim { @@ -20,20 +20,20 @@ namespace Nextsim { */ class SlabOcean : public ModelComponent, public Configured { public: - SlabOcean(ModelArrayReferenceStore& coupingArrays) - : qdw(ModelArray::Type::H) - , fdw(ModelArray::Type::H) - , sstSlab(ModelArray::Type::H) - , sssSlab(ModelArray::Type::H) - , sstExt(getStore()) - , sssExt(getStore()) - , sst(getStore()) - , sss(getStore()) - , cpml(getStore()) - , qswNet(coupingArrays) - , qNoSun(coupingArrays) - , fwFlux(coupingArrays) - , sFlux(coupingArrays) + SlabOcean(ModelArrayStore& couplingArrays) + : qdwAccessor(getStore(), RO, ModelArray::Type::H) + , fdwAccessor(getStore(), RO, ModelArray::Type::H) + , sstSlabAccessor(getStore(), RO, ModelArray::Type::H) + , sssSlabAccessor(getStore(), RO, ModelArray::Type::H) + , sstExtAccessor(getStore()) + , sssExtAccessor(getStore()) + , sstAccessor(getStore()) + , sssAccessor(getStore()) + , cpmlAccessor(getStore()) + , qswNetAccessor(couplingArrays) + , qNoSunAccessor(couplingArrays) + , fwFluxAccessor(couplingArrays) + , sFluxAccessor(couplingArrays) { } @@ -60,29 +60,25 @@ class SlabOcean : public ModelComponent, public Configured { private: // Owned shared fields - HField qdw; - HField fdw; - HField sstSlab; - HField sssSlab; + ModelArrayAccessor qdwAccessor; + ModelArrayAccessor fdwAccessor; + ModelArrayAccessor sstSlabAccessor; + ModelArrayAccessor sssSlabAccessor; // Input fields - ModelArrayRef sstExt; - ModelArrayRef sssExt; - ModelArrayRef sst; - ModelArrayRef sss; - ModelArrayRef cpml; - ModelArrayRef qswNet; - ModelArrayRef qNoSun; - ModelArrayRef fwFlux; - ModelArrayRef sFlux; + ModelArrayAccessor sstExtAccessor; + ModelArrayAccessor sssExtAccessor; + ModelArrayAccessor sstAccessor; + ModelArrayAccessor sssAccessor; + ModelArrayAccessor cpmlAccessor; + ModelArrayAccessor qswNetAccessor; + ModelArrayAccessor qNoSunAccessor; + ModelArrayAccessor fwFluxAccessor; + ModelArrayAccessor sFluxAccessor; // TODO ModelArrayRef to assimilation flux double relaxationTimeT = defaultRelaxationTime; double relaxationTimeS = defaultRelaxationTime; - - double dt; - - void updateElement(size_t i, const TimestepTime& tst); }; } /* namespace Nextsim */ diff --git a/physics/src/modules/AtmosphereBoundaryModule/BenchmarkAtmosphere.cpp b/physics/src/modules/AtmosphereBoundaryModule/BenchmarkAtmosphere.cpp index 5676677e1..e9fc49167 100644 --- a/physics/src/modules/AtmosphereBoundaryModule/BenchmarkAtmosphere.cpp +++ b/physics/src/modules/AtmosphereBoundaryModule/BenchmarkAtmosphere.cpp @@ -14,13 +14,13 @@ void BenchmarkAtmosphere::setData(const ModelState::DataMap& ms) IAtmosphereBoundary::setData(ms); BenchmarkCoordinates::setData(); // Constant, zero fluxes in the atmosphere - qia = 0.; - dqia_dt = 0.; - qow = 0.; - subl = 0.; - snow = 0.; - rain = 0.; - evap = 0.; + qiaAccessor.getHostRW() = 0.; + dqia_dtAccessor.getHostRW() = 0.; + qowAccessor.getHostRW() = 0.; + sublAccessor.getHostRW() = 0.; + snowAccessor.getHostRW() = 0.; + rainAccessor.getHostRW() = 0.; + evapAccessor.getHostRW() = 0.; } void BenchmarkAtmosphere::update(const TimestepTime& tst) @@ -59,6 +59,9 @@ void BenchmarkAtmosphere::update(const TimestepTime& tst) const ModelArray& xPrime = BenchmarkCoordinates::x() - x0; const ModelArray& yPrime = BenchmarkCoordinates::y() - y0; + UField& uwind = uwindAccessor.getHostRW(); + VField& vwind = vwindAccessor.getHostRW(); + // Perform the rest of the calculation per element for (size_t j = 0; j < BenchmarkCoordinates::ny(); ++j) { for (size_t i = 0; i < BenchmarkCoordinates::nx(); ++i) { diff --git a/physics/src/modules/AtmosphereBoundaryModule/ConfiguredAtmosphere.cpp b/physics/src/modules/AtmosphereBoundaryModule/ConfiguredAtmosphere.cpp index 0548535c3..965646913 100644 --- a/physics/src/modules/AtmosphereBoundaryModule/ConfiguredAtmosphere.cpp +++ b/physics/src/modules/AtmosphereBoundaryModule/ConfiguredAtmosphere.cpp @@ -45,14 +45,14 @@ const static std::map keyMap = { }; ConfiguredAtmosphere::ConfiguredAtmosphere() - : fluxImpl(0) + : tairAccessor(getStore(), RO, ModelArray::Type::H) + , tdewAccessor(getStore(), RO, ModelArray::Type::H) + , pairAccessor(getStore(), RO, ModelArray::Type::H) + , sw_inAccessor(getStore(), RO, ModelArray::Type::H) + , lw_inAccessor(getStore(), RO, ModelArray::Type::H) + , windAccessor(getStore(), RO, ModelArray::Type::H) + , fluxImpl(nullptr) { - getStore().registerArray(Protected::T_AIR, &tair, RO); - getStore().registerArray(Protected::DEW_2M, &tdew, RO); - getStore().registerArray(Protected::P_AIR, &pair, RO); - getStore().registerArray(Protected::SW_IN, &sw_in, RO); - getStore().registerArray(Protected::LW_IN, &lw_in, RO); - getStore().registerArray(Protected::WIND_SPEED, &wind, RO); } ConfigurationHelp::HelpMap& ConfiguredAtmosphere::getHelpRecursive(HelpMap& map, bool getAll) @@ -118,11 +118,17 @@ void ConfiguredAtmosphere::setData(const ModelState::DataMap& dm) { IAtmosphereBoundary::setData(dm); + HField& tair = tairAccessor.getHostRW(); tair.resize(); + HField& tdew = tdewAccessor.getHostRW(); tdew.resize(); + HField& pair = pairAccessor.getHostRW(); pair.resize(); + HField& sw_in = sw_inAccessor.getHostRW(); sw_in.resize(); + HField& lw_in = lw_inAccessor.getHostRW(); lw_in.resize(); + HField& wind = windAccessor.getHostRW(); wind.resize(); tair = tair0; @@ -130,12 +136,12 @@ void ConfiguredAtmosphere::setData(const ModelState::DataMap& dm) pair = pair0; sw_in = sw0; lw_in = lw0; - snow = snowfall0; - rain = rain0; + snowAccessor.getHostRW() = snowfall0; + rainAccessor.getHostRW() = rain0; wind = std::hypot(uWind0, vWind0); - uwind = uWind0; - vwind = vWind0; + uwindAccessor.getHostRW() = uWind0; + vwindAccessor.getHostRW() = vWind0; fluxImpl->setData(dm); } diff --git a/physics/src/modules/AtmosphereBoundaryModule/ConstantAtmosphereBoundary.cpp b/physics/src/modules/AtmosphereBoundaryModule/ConstantAtmosphereBoundary.cpp index 9e6c4a567..fc749431a 100644 --- a/physics/src/modules/AtmosphereBoundaryModule/ConstantAtmosphereBoundary.cpp +++ b/physics/src/modules/AtmosphereBoundaryModule/ConstantAtmosphereBoundary.cpp @@ -14,16 +14,16 @@ ConstantAtmosphereBoundary::ConstantAtmosphereBoundary() void ConstantAtmosphereBoundary::setData(const ModelState::DataMap& ms) { // Directly set the array values - qia = 305.288; // Pulled from IceGrowth_test.cpp: New Ice Formation - dqia_dt = 4.5036; - qow = 307.546; - subl = 0.; // Seems unlikely… - snow = 0.; - rain = 0.; - evap = 0; // somehow... - uwind = 0; - vwind = 0; - penSW = 0.; + qiaAccessor.getHostRW() = 305.288; // Pulled from IceGrowth_test.cpp: New Ice Formation + dqia_dtAccessor.getHostRW() = 4.5036; + qowAccessor.getHostRW() = 307.546; + sublAccessor.getHostRW() = 0.; // Seems unlikely… + snowAccessor.getHostRW() = 0.; + rainAccessor.getHostRW() = 0.; + evapAccessor.getHostRW() = 0; // somehow... + uwindAccessor.getHostRW() = 0; + vwindAccessor.getHostRW() = 0; + penSWAccessor.getHostRW() = 0.; } void ConstantAtmosphereBoundary::update(const TimestepTime& tst) diff --git a/physics/src/modules/AtmosphereBoundaryModule/ERA5Atmosphere.cpp b/physics/src/modules/AtmosphereBoundaryModule/ERA5Atmosphere.cpp index ddd49f8db..271b5ed90 100644 --- a/physics/src/modules/AtmosphereBoundaryModule/ERA5Atmosphere.cpp +++ b/physics/src/modules/AtmosphereBoundaryModule/ERA5Atmosphere.cpp @@ -21,19 +21,13 @@ static const std::map keyMap = { ERA5Atmosphere::ERA5Atmosphere() : fluxImpl(nullptr) - , tair(ModelArray::Type::H, { -100, 100 }) - , tdew(ModelArray::Type::H, { -100, 100 }) - , pair(ModelArray::Type::H, { 500e2, 2000e2 }) - , sw_in(ModelArray::Type::H, { -1e-6, 1e4 }) - , lw_in(ModelArray::Type::H, { -1e-6, 1e4 }) - , wind(ModelArray::Type::H, { 0, 100 }) + , tairAccessor(getStore(), RO, ModelArray::Type::H, std::pair(-100.0, 100.0)) + , tdewAccessor(getStore(), RO, ModelArray::Type::H, std::pair(-100.0, 100.0)) + , pairAccessor(getStore(), RO, ModelArray::Type::H, std::pair(500e2, 2000e2)) + , sw_inAccessor(getStore(), RO, ModelArray::Type::H, std::pair(-1e-6, 1e4)) + , lw_inAccessor(getStore(), RO, ModelArray::Type::H, std::pair(-1e-6, 1e4)) + , windAccessor(getStore(), RO, ModelArray::Type::H, std::pair(0.0, 100.0)) { - getStore().registerArray(Protected::T_AIR, &tair, RO); - getStore().registerArray(Protected::DEW_2M, &tdew, RO); - getStore().registerArray(Protected::P_AIR, &pair, RO); - getStore().registerArray(Protected::SW_IN, &sw_in, RO); - getStore().registerArray(Protected::LW_IN, &lw_in, RO); - getStore().registerArray(Protected::WIND_SPEED, &wind, RO); } ConfigurationHelp::HelpMap& ERA5Atmosphere::getHelpRecursive(HelpMap& map, bool getAll) @@ -57,12 +51,12 @@ void ERA5Atmosphere::configure() tryConfigure(fluxImpl); addChecks({ - { "tair", &tair }, - { "tdew", &tdew }, - { "pair", &pair }, - { "sw_in", &sw_in }, - { "lw_in", &lw_in }, - { "wind", &wind }, + { "tair", &tairAccessor.getHostRO() }, + { "tdew", &tdewAccessor.getHostRO() }, + { "pair", &pairAccessor.getHostRO() }, + { "sw_in", &sw_inAccessor.getHostRO() }, + { "lw_in", &lw_inAccessor.getHostRO() }, + { "wind", &windAccessor.getHostRO() }, }); } @@ -80,16 +74,16 @@ void ERA5Atmosphere::update(const TimestepTime& tst) = { "tair", "dew2m", "pair", "sw_in", "lw_in", "wind_speed", "u", "v" }; ModelState state = ParaGridIO::readForcingTimeStatic(forcings, tst.start, filePath); - tair = state.data.at("tair"); - tdew = state.data.at("dew2m"); - pair = state.data.at("pair"); - sw_in = state.data.at("sw_in"); - lw_in = state.data.at("lw_in"); - wind = state.data.at("wind_speed"); - uwind = state.data.at("u"); - vwind = state.data.at("v"); - snow = 0; // FIXME get snow data - rain = 0; // FIXME get rain data + tairAccessor.getHostRW() = state.data.at("tair"); + tdewAccessor.getHostRW() = state.data.at("dew2m"); + pairAccessor.getHostRW() = state.data.at("pair"); + sw_inAccessor.getHostRW() = state.data.at("sw_in"); + lw_inAccessor.getHostRW() = state.data.at("lw_in"); + windAccessor.getHostRW() = state.data.at("wind_speed"); + uwindAccessor.getHostRW() = state.data.at("u"); + vwindAccessor.getHostRW() = state.data.at("v"); + snowAccessor.getHostRW() = 0; // FIXME get snow data + rainAccessor.getHostRW() = 0; // FIXME get rain data fluxImpl->update(tst); diff --git a/physics/src/modules/AtmosphereBoundaryModule/FluxConfiguredAtmosphere.cpp b/physics/src/modules/AtmosphereBoundaryModule/FluxConfiguredAtmosphere.cpp index c70f826eb..607afd7bd 100644 --- a/physics/src/modules/AtmosphereBoundaryModule/FluxConfiguredAtmosphere.cpp +++ b/physics/src/modules/AtmosphereBoundaryModule/FluxConfiguredAtmosphere.cpp @@ -94,27 +94,27 @@ ConfigMap FluxConfiguredAtmosphere::getConfiguration() const void FluxConfiguredAtmosphere::setData(const ModelState::DataMap& dm) { IAtmosphereBoundary::setData(dm); - qia = qia0; - dqia_dt = dqia_dt0; - qow = qow0; - subl = subl0; - snow = snowfall0; - rain = rain0; - evap = evap0; - uwind = u0; - vwind = v0; + qiaAccessor.getHostRW() = qia0; + dqia_dtAccessor.getHostRW() = dqia_dt0; + qowAccessor.getHostRW() = qow0; + sublAccessor.getHostRW() = subl0; + snowAccessor.getHostRW() = snowfall0; + rainAccessor.getHostRW() = rain0; + evapAccessor.getHostRW() = evap0; + uwindAccessor.getHostRW() = u0; + vwindAccessor.getHostRW() = v0; // Not configured here: - penSW = 0; // Penetrating shortwave radiation - tauXOW = 0; // x(east)-ward open ocean stress, Pa - tauYOW = 0; // y(north)-ward open ocean stress, Pa + penSWAccessor.getHostRW() = 0; // Penetrating shortwave radiation + tauXOWAccessor.getHostRW() = 0; // x(east)-ward open ocean stress, Pa + tauYOWAccessor.getHostRW() = 0; // y(north)-ward open ocean stress, Pa } void FluxConfiguredAtmosphere::update(const TimestepTime& tst) { /* The open water heat flux is reset by the thermodynamics, so that the (slab) ocean doesn't * super cool. Therefore, we have to reset it here on every update */ - qow = qow0; + qowAccessor.getHostRW() = qow0; } } /* namespace Nextsim */ diff --git a/physics/src/modules/AtmosphereBoundaryModule/MU71Atmosphere.cpp b/physics/src/modules/AtmosphereBoundaryModule/MU71Atmosphere.cpp index c576405d0..d45261a76 100644 --- a/physics/src/modules/AtmosphereBoundaryModule/MU71Atmosphere.cpp +++ b/physics/src/modules/AtmosphereBoundaryModule/MU71Atmosphere.cpp @@ -51,9 +51,9 @@ MU71Atmosphere::HelpMap& MU71Atmosphere::getHelpRecursive(HelpMap& map, bool get MU71Atmosphere::MU71Atmosphere() : iIceAlbedoImpl(nullptr) - , tsurf(getStore()) - , hsnow(getStore()) - , cice(getStore()) + , tsurfAccessor(getStore()) + , hsnowAccessor(getStore()) + , ciceAccessor(getStore()) , q_sw(monthlyCubicBSpline(swTable)) , q_lw(monthlyCubicBSpline(lwTable)) , q_sh(monthlyCubicBSpline(shTable)) @@ -68,42 +68,87 @@ void MU71Atmosphere::update(const Nextsim::TimestepTime& tst) isLeap = ((tst.start.gmtime()->tm_year % 4 == 0) && (tst.start.gmtime()->tm_year % 100 != 0)) || (tst.start.gmtime()->tm_year % 400 == 0); + HField& qow = qowAccessor.getHostRW(); + VField& vwind = vwindAccessor.getHostRW(); + HField& snow = snowAccessor.getHostRW(); + HField& dqia_dt = dqia_dtAccessor.getHostRW(); + HField& evap = evapAccessor.getHostRW(); + HField& rain = rainAccessor.getHostRW(); + HField& subl = sublAccessor.getHostRW(); + HField& qia = qiaAccessor.getHostRW(); + HField& penSW = penSWAccessor.getHostRW(); + UField& uwind = uwindAccessor.getHostRW(); + const AdvectedField& cice = ciceAccessor.getHostRO(); + const AdvectedField& hsnow = hsnowAccessor.getHostRO(); + const AdvectedField& tsurf = tsurfAccessor.getHostRO(); + overElements( - [this](size_t i, const TimestepTime& tsTime) { this->calculateElement(i, tsTime); }, tst); + [&](size_t i, const TimestepTime& tsTime) { + const double Tsurf_K = kelvin(tsurf[i]); + + double albedoValue, i0; + double sw_in = convFactor * q_sw(dayOfYear, isLeap); + const double hs = cice[i] > 0 ? hsnow[i] / cice[i] : 0.; + std::tie(albedoValue, i0) = iIceAlbedoImpl->surfaceShortWaveBalance(tsurf[i], hs, m_I0); + double qsw = -sw_in * (1. - albedoValue) * (1. - i0); + penSW[i] = sw_in * (1. - albedoValue) * i0; + qia[i] = -convFactor + * (q_sh(dayOfYear, isLeap) + q_lh(dayOfYear, isLeap) + q_lw(dayOfYear, isLeap)) + // LW is tabulated + black body radiation + + Ice::epsilon * PhysicalConstants::sigma * std::pow(Tsurf_K, 4) + qsw; + + // Just the derivative of the black body radiation + dqia_dt[i] = 4. * Ice::epsilon * PhysicalConstants::sigma * std::pow(Tsurf_K, 3); + + // Only snowfall if we're not melting + if ((hs > 0 && tsurf[i] < 0.) || (hs == 0 && tsurf[i] < -Ice::s * Water::mu)) + snow[i] = snowfall(); + else + snow[i] = 0.; + + // Not needed/specified by M&U '71 + qow[i] = 0.; + subl[i] = 0.; + rain[i] = 0.; + evap[i] = 0.; + uwind[i] = 0.; + vwind[i] = 0.; + }, + tst); } -void MU71Atmosphere::calculateElement(size_t i, const TimestepTime& tst) -{ - const double Tsurf_K = kelvin(tsurf[i]); - - double albedoValue, i0; - double sw_in = convFactor * q_sw(dayOfYear, isLeap); - const double hs = cice[i] > 0 ? hsnow[i] / cice[i] : 0.; - std::tie(albedoValue, i0) = iIceAlbedoImpl->surfaceShortWaveBalance(tsurf[i], hs, m_I0); - double qsw = -sw_in * (1. - albedoValue) * (1. - i0); - penSW[i] = sw_in * (1. - albedoValue) * i0; - qia[i] = -convFactor - * (q_sh(dayOfYear, isLeap) + q_lh(dayOfYear, isLeap) + q_lw(dayOfYear, isLeap)) - // LW is tabulated + black body radiation - + Ice::epsilon * PhysicalConstants::sigma * std::pow(Tsurf_K, 4) + qsw; - - // Just the derivative of the black body radiation - dqia_dt[i] = 4. * Ice::epsilon * PhysicalConstants::sigma * std::pow(Tsurf_K, 3); - - // Only snowfall if we're not melting - if ((hs > 0 && tsurf[i] < 0.) || (hs == 0 && tsurf[i] < -Ice::s * Water::mu)) - snow[i] = snowfall(); - else - snow[i] = 0.; - - // Not needed/specified by M&U '71 - qow[i] = 0.; - subl[i] = 0.; - rain[i] = 0.; - evap[i] = 0.; - uwind[i] = 0.; - vwind[i] = 0.; -} +// void MU71Atmosphere::calculateElement(size_t i, const TimestepTime& tst) +// { +// const double Tsurf_K = kelvin(tsurf[i]); +// +// double albedoValue, i0; +// double sw_in = convFactor * q_sw(dayOfYear, isLeap); +// const double hs = cice[i] > 0 ? hsnow[i] / cice[i] : 0.; +// std::tie(albedoValue, i0) = iIceAlbedoImpl->surfaceShortWaveBalance(tsurf[i], hs, m_I0); +// double qsw = -sw_in * (1. - albedoValue) * (1. - i0); +// penSW[i] = sw_in * (1. - albedoValue) * i0; +// qia[i] = -convFactor +// * (q_sh(dayOfYear, isLeap) + q_lh(dayOfYear, isLeap) + q_lw(dayOfYear, isLeap)) +// // LW is tabulated + black body radiation +// + Ice::epsilon * PhysicalConstants::sigma * std::pow(Tsurf_K, 4) + qsw; +// +// // Just the derivative of the black body radiation +// dqia_dt[i] = 4. * Ice::epsilon * PhysicalConstants::sigma * std::pow(Tsurf_K, 3); +// +// // Only snowfall if we're not melting +// if ((hs > 0 && tsurf[i] < 0.) || (hs == 0 && tsurf[i] < -Ice::s * Water::mu)) +// snow[i] = snowfall(); +// else +// snow[i] = 0.; +// +// // Not needed/specified by M&U '71 +// qow[i] = 0.; +// subl[i] = 0.; +// rain[i] = 0.; +// evap[i] = 0.; +// uwind[i] = 0.; +// vwind[i] = 0.; +// } // Snowfall according to M&U '71 (in m/s water equivalent) double MU71Atmosphere::snowfall() diff --git a/physics/src/modules/AtmosphereBoundaryModule/include/ConfiguredAtmosphere.hpp b/physics/src/modules/AtmosphereBoundaryModule/include/ConfiguredAtmosphere.hpp index 98710ead5..3b7bf639c 100644 --- a/physics/src/modules/AtmosphereBoundaryModule/include/ConfiguredAtmosphere.hpp +++ b/physics/src/modules/AtmosphereBoundaryModule/include/ConfiguredAtmosphere.hpp @@ -51,12 +51,12 @@ class ConfiguredAtmosphere : public IAtmosphereBoundary, public Configured tairAccessor; + ModelArrayAccessor tdewAccessor; + ModelArrayAccessor pairAccessor; + ModelArrayAccessor sw_inAccessor; + ModelArrayAccessor lw_inAccessor; + ModelArrayAccessor windAccessor; IFluxCalculation* fluxImpl; }; diff --git a/physics/src/modules/AtmosphereBoundaryModule/include/ERA5Atmosphere.hpp b/physics/src/modules/AtmosphereBoundaryModule/include/ERA5Atmosphere.hpp index d796be988..f8e829253 100644 --- a/physics/src/modules/AtmosphereBoundaryModule/include/ERA5Atmosphere.hpp +++ b/physics/src/modules/AtmosphereBoundaryModule/include/ERA5Atmosphere.hpp @@ -44,12 +44,12 @@ class ERA5Atmosphere : public IAtmosphereBoundary, public Configured tairAccessor; + ModelArrayAccessor tdewAccessor; + ModelArrayAccessor pairAccessor; + ModelArrayAccessor sw_inAccessor; + ModelArrayAccessor lw_inAccessor; + ModelArrayAccessor windAccessor; IFluxCalculation* fluxImpl; }; diff --git a/physics/src/modules/AtmosphereBoundaryModule/include/MU71Atmosphere.hpp b/physics/src/modules/AtmosphereBoundaryModule/include/MU71Atmosphere.hpp index 333e11c6b..341a797d1 100644 --- a/physics/src/modules/AtmosphereBoundaryModule/include/MU71Atmosphere.hpp +++ b/physics/src/modules/AtmosphereBoundaryModule/include/MU71Atmosphere.hpp @@ -48,9 +48,9 @@ class MU71Atmosphere : public IAtmosphereBoundary, public Configured tsurf; - ModelArrayRef hsnow; // cell-averaged value - ModelArrayRef cice; // cell-averaged value + ModelArrayAccessor tsurfAccessor; + ModelArrayAccessor hsnowAccessor; // cell-averaged value + ModelArrayAccessor ciceAccessor; // cell-averaged value /*! * @brief A function to calculate the snow fall according tu Maykut and Untersteiner (1971) diff --git a/physics/src/modules/DamageHealingModule/ConstantHealing.cpp b/physics/src/modules/DamageHealingModule/ConstantHealing.cpp index d548b4b89..fd740fcce 100644 --- a/physics/src/modules/DamageHealingModule/ConstantHealing.cpp +++ b/physics/src/modules/DamageHealingModule/ConstantHealing.cpp @@ -1,10 +1,12 @@ /*! * @author Einar Ólason + * @author Robert Jendersie */ #include "include/ConstantHealing.hpp" - #include "include/IceMinima.hpp" +#include "include/KernelAlternatives.hpp" +#include "kokkos/include/KokkosTimer.hpp" namespace Nextsim { @@ -41,35 +43,50 @@ ConstantHealing::HelpMap& ConstantHealing::getHelpRecursive(HelpMap& map, bool g * 2. Constant healing with a given time scale (tD) */ void ConstantHealing::update(const TimestepTime& tstep) { - overElements( - [this](size_t i, const TimestepTime& tsTime) { this->updateElement(i, tsTime); }, tstep); -} - -void ConstantHealing::updateElement(size_t i, const TimestepTime& tstep) -{ - // No ice, no healing - if (cice[i] <= IceMinima::c()) { - damage[i] = 1.; - return; - } - - // Only lateral growth contributes to healing, not melt(!) - double const lateralGrowth = std::max(0., deltaCi[i]); - - /* 1. Lateral ice formation - * A weighted average of the original damage, weighted by the old concentration, and the - * undamaged new ice damage (1), weighted by the concentration of new ice. */ - damage[i] = (damage[i] * (cice[i] - lateralGrowth) + lateralGrowth) / cice[i]; - - /* 2. Constant healing - * Damage healing using a constant timescale. Originally conceived as an exponential decay, but - * then revised to a linear one. */ - // This is what Sylvain and Pierre (Bouillon and Rampal, 2015) - // damage[i] += damage[i] * tstep.step / tD; - - // This is what Véro did (Dansereau et al., 2016) - damage[i] += tstep.step / tD; - damage[i] = std::min(1., damage[i]); + static KokkosTimer timer("ConstantHealing"); + static KokkosTimer timerCopy("ConstantHealingCopy"); + static KokkosTimer timerUpdate("ConstantHealingUpdate"); + + timer.start(); + timerCopy.start(); + auto execSpace = DefaultExecutionSpace(); + auto& damage = damageAccessor.getAutoRW(execSpace); + const auto& deltaCi = deltaCiAccessor.getAutoRO(execSpace); + const auto& cice = ciceAccessor.getAutoRO(execSpace); + + const double tD = ConstantHealing::tD; + const double cMin = IceMinima::c(); + const double dt = tstep.step.seconds(); + timerCopy.stop(); + + timerUpdate.start(); + overElementsAuto(OVER_ELEMENTS_LAMBDA(const ElementIndex i) { + // No ice, no healing + if (cice[i] <= cMin) { + damage[i] = 1.; + return; + } + + // Only lateral growth contributes to healing, not melt(!) + double const lateralGrowth = Utils::max(0., deltaCi[i]); + + /* 1. Lateral ice formation + * A weighted average of the original damage, weighted by the old concentration, and the + * undamaged new ice damage (1), weighted by the concentration of new ice. */ + damage[i] = (damage[i] * (cice[i] - lateralGrowth) + lateralGrowth) / cice[i]; + + /* 2. Constant healing + * Damage healing using a constant timescale. Originally conceived as an exponential + * decay, but then revised to a linear one. */ + // This is what Sylvain and Pierre (Bouillon and Rampal, 2015) + // damage[i] += damage[i] * tstep.step / tD; + + // This is what Véro did (Dansereau et al., 2016) + damage[i] += dt / tD; + damage[i] = Utils::min(1., damage[i]); + }); + timerUpdate.stop(); + timer.stop(); } } diff --git a/physics/src/modules/DamageHealingModule/include/ConstantHealing.hpp b/physics/src/modules/DamageHealingModule/include/ConstantHealing.hpp index 376acf61a..67c6d27db 100644 --- a/physics/src/modules/DamageHealingModule/include/ConstantHealing.hpp +++ b/physics/src/modules/DamageHealingModule/include/ConstantHealing.hpp @@ -33,7 +33,6 @@ class ConstantHealing : public IDamageHealing, public Configured 0 ? hsnow[i] / cice[i] : 0.; - std::tie(albedoValue, i0) = iIceAlbedoImpl->surfaceShortWaveBalance(tsurf[i], hs, m_I0); - Q_sw_ia[i] = -sw_in[i] * (1. - albedoValue) * (1. - i0); - const double extinction = 0.; // TODO: Replace with de Beer's law or a module - penSW[i] = sw_in[i] * (1. - albedoValue) * i0 * (1. - extinction); - Q_sw_base[i] = sw_in[i] * (1. - albedoValue) * i0 * extinction; - - // Longwave flux - Q_lw_ia[i] = stefanBoltzmannLaw(tsurf[i]) - lw_in[i]; - double dQlw_dT = 4 / kelvin(tsurf[i]) * stefanBoltzmannLaw(tsurf[i]); - - // Total flux - qia[i] = Q_lh_ia[i] + Q_sh_ia[i] + Q_sw_ia[i] + Q_lw_ia[i]; - - // Total temperature dependence of flux - dqia_dt[i] = dQlh_dT + dQsh_dT + dQlw_dT; -} +// void FiniteElementFluxes::calculateIce(size_t i, const TimestepTime& tst) +// { +// // Mass flux ice +// subl[i] = dragIce_t * rho_air[i] * windSpeed[i] * (sh_ice[i] - sh_air[i]); +// +// // Momentum flux is dealt with by the ice dynamics +// +// // Heat flux ice-atmosphere +// // Latent heat from sublimation +// Q_lh_ia[i] = subl[i] * latentHeatIce(tsurf[i]); +// double dmdot_dT = dragIce_t * rho_air[i] * windSpeed[i] * dshice_dT[i]; +// double dQlh_dT = latentHeatIce(tsurf[i]) * dmdot_dT; +// +// // Sensible heat flux +// Q_sh_ia[i] = dragIce_t * rho_air[i] * cp_air[i] * windSpeed[i] * (tsurf[i] - t_air[i]); +// double dQsh_dT = dragIce_t * rho_air[i] * cp_air[i] * windSpeed[i]; +// +// // Shortwave flux +// double albedoValue, i0; +// const double hs = cice[i] > 0 ? hsnow[i] / cice[i] : 0.; +// std::tie(albedoValue, i0) = iIceAlbedoImpl->surfaceShortWaveBalance(tsurf[i], hs, m_I0); +// Q_sw_ia[i] = -sw_in[i] * (1. - albedoValue) * (1. - i0); +// const double extinction = 0.; // TODO: Replace with de Beer's law or a module +// penSW[i] = sw_in[i] * (1. - albedoValue) * i0 * (1. - extinction); +// Q_sw_base[i] = sw_in[i] * (1. - albedoValue) * i0 * extinction; +// +// // Longwave flux +// Q_lw_ia[i] = stefanBoltzmannLaw(tsurf[i]) - lw_in[i]; +// double dQlw_dT = 4 / kelvin(tsurf[i]) * stefanBoltzmannLaw(tsurf[i]); +// +// // Total flux +// qia[i] = Q_lh_ia[i] + Q_sh_ia[i] + Q_sw_ia[i] + Q_lw_ia[i]; +// +// // Total temperature dependence of flux +// dqia_dt[i] = dQlh_dT + dQsh_dT + dQlw_dT; +// } void FiniteElementFluxes::update(const TimestepTime& tst) { @@ -202,23 +202,128 @@ void FiniteElementFluxes::update(const TimestepTime& tst) void FiniteElementFluxes::updateAtmosphere(const TimestepTime& tst) { + const AdvectedField& tsurf = tsurfAccessor.getHostRO(); + const HField& sss = sssAccessor.getHostRO(); + const HField& p_air = p_airAccessor.getHostRO(); + const HField& sst = sstAccessor.getHostRO(); + const HField& t_air = t_airAccessor.getHostRO(); + const HField& t_dew2 = t_dew2Accessor.getHostRO(); + overElements( - [this](size_t i, const TimestepTime& tsTime) { this->calculateAtmos(i, tsTime); }, tst); + [&](size_t i, const TimestepTime& tsTime) { + // Specific humidity of... + // ...the air + sh_air[i] = FiniteElementSpecHum::water()(t_dew2[i], p_air[i]); + // ...over the open ocean + sh_water[i] = FiniteElementSpecHum::water()(sst[i], p_air[i], sss[i]); + // ...over the ice + std::pair iceData + = FiniteElementSpecHum::ice().valueAndDerivative(tsurf[i], p_air[i]); + sh_ice[i] = iceData.first; + dshice_dT[i] = iceData.second; + // Density of the wet air + double Ra_wet = Air::Ra / (1 - sh_air[i] * (1 - Vapour::Ra / Air::Ra)); + rho_air[i] = p_air[i] / (Ra_wet * kelvin(t_air[i])); + // Heat capacity of the wet air + cp_air[i] = Air::cp + sh_air[i] * Vapour::cp; + }, + tst); } void FiniteElementFluxes::updateOW(const TimestepTime& tst) { + HField& tau_x_ow = tau_x_owAccessor.getHostRW(); + HField& tau_y_ow = tau_y_owAccessor.getHostRW(); + HField& Q_sw_ow = Q_sw_owAccessor.getHostRW(); + HField& qow = qowAccessor.getHostRW(); + HField& evap = evapAccessor.getHostRW(); + const HField& lw_in = lw_inAccessor.getHostRO(); + const HField& sst = sstAccessor.getHostRO(); + const HField& sw_in = sw_inAccessor.getHostRO(); + const HField& t_air = t_airAccessor.getHostRO(); + const UField& u_air = u_airAccessor.getHostRO(); + const VField& v_air = v_airAccessor.getHostRO(); + const HField& windSpeed = windSpeedAccessor.getHostRO(); + overElements( - [this](size_t i, const TimestepTime& tsTime) { this->calculateOW(i, tsTime); }, tst); + [&](size_t i, const TimestepTime& tsTime) { + // Mass flux from open water (evaporation) + evap[i] = dragOcean_q * rho_air[i] * windSpeed[i] * (sh_water[i] - sh_air[i]); + // Momentum flux from open water (drag pressure) + /* Drag the ocean experiences from the wind - still only used in the coupled case */ + tau_x_ow[i] = rho_air[i] * dragOcean_m(windSpeed[i]) * u_air[i] * windSpeed[i]; + tau_y_ow[i] = rho_air[i] * dragOcean_m(windSpeed[i]) * v_air[i] * windSpeed[i]; + + // Heat flux open water + // Latent heat from evaporation (and condensation) + Q_lh_ow[i] = evap[i] * latentHeatWater(sst[i]); + // Sensible heat + Q_sh_ow[i] = dragOcean_t * rho_air[i] * cp_air[i] * windSpeed[i] * (sst[i] - t_air[i]); + // Shortwave flux + Q_sw_ow[i] = -sw_in[i] * (1 - m_oceanAlbedo); + // Longwave flux + Q_lw_ow[i] = stefanBoltzmannLaw(sst[i]) - lw_in[i]; + // Total open water flux + qow[i] = Q_lh_ow[i] + Q_sh_ow[i] + Q_sw_ow[i] + Q_lw_ow[i]; + }, + tst); } void FiniteElementFluxes::updateIce(const TimestepTime& tst) { + HField& Q_sw_base = Q_sw_baseAccessor.getHostRW(); + HField& qia = qiaAccessor.getHostRW(); + HField& subl = sublAccessor.getHostRW(); + HField& penSW = penSWAccessor.getHostRW(); + HField& dqia_dt = dqia_dtAccessor.getHostRW(); + const AdvectedField& cice = ciceAccessor.getHostRO(); + const AdvectedField& hsnow = hsnowAccessor.getHostRO(); + const HField& lw_in = lw_inAccessor.getHostRO(); + const HField& sw_in = sw_inAccessor.getHostRO(); + const HField& t_air = t_airAccessor.getHostRO(); + const AdvectedField& tsurf = tsurfAccessor.getHostRO(); + const HField& windSpeed = windSpeedAccessor.getHostRO(); + iIceAlbedoImpl->setTime(tst.start); overElements( - [this](size_t i, const TimestepTime& tsTime) { this->calculateIce(i, tsTime); }, tst); + [&](size_t i, const TimestepTime& tsTime) { + // Mass flux ice + subl[i] = dragIce_t * rho_air[i] * windSpeed[i] * (sh_ice[i] - sh_air[i]); + + // Momentum flux is dealt with by the ice dynamics + + // Heat flux ice-atmosphere + // Latent heat from sublimation + Q_lh_ia[i] = subl[i] * latentHeatIce(tsurf[i]); + double dmdot_dT = dragIce_t * rho_air[i] * windSpeed[i] * dshice_dT[i]; + double dQlh_dT = latentHeatIce(tsurf[i]) * dmdot_dT; + + // Sensible heat flux + Q_sh_ia[i] = dragIce_t * rho_air[i] * cp_air[i] * windSpeed[i] * (tsurf[i] - t_air[i]); + double dQsh_dT = dragIce_t * rho_air[i] * cp_air[i] * windSpeed[i]; + + // Shortwave flux + double albedoValue, i0; + const double hs = cice[i] > 0 ? hsnow[i] / cice[i] : 0.; + std::tie(albedoValue, i0) = iIceAlbedoImpl->surfaceShortWaveBalance(tsurf[i], hs, m_I0); + Q_sw_ia[i] = -sw_in[i] * (1. - albedoValue) * (1. - i0); + const double extinction = 0.; // TODO: Replace with de Beer's law or a module + penSW[i] = sw_in[i] * (1. - albedoValue) * i0 * (1. - extinction); + Q_sw_base[i] = sw_in[i] * (1. - albedoValue) * i0 * extinction; + + // Longwave flux + Q_lw_ia[i] = stefanBoltzmannLaw(tsurf[i]) - lw_in[i]; + double dQlw_dT = 4 / kelvin(tsurf[i]) * stefanBoltzmannLaw(tsurf[i]); + + // Total flux + qia[i] = Q_lh_ia[i] + Q_sh_ia[i] + Q_sw_ia[i] + Q_lw_ia[i]; + + // Total temperature dependence of flux + dqia_dt[i] = dQlh_dT + dQsh_dT + dQlw_dT; + }, + tst); } - +/* void FiniteElementFluxes::calculateAtmos(size_t i, const TimestepTime& tst) { // Specific humidity of... @@ -236,7 +341,7 @@ void FiniteElementFluxes::calculateAtmos(size_t i, const TimestepTime& tst) rho_air[i] = p_air[i] / (Ra_wet * kelvin(t_air[i])); // Heat capacity of the wet air cp_air[i] = Air::cp + sh_air[i] * Vapour::cp; -} +}*/ double FiniteElementFluxes::latentHeatWater(double temperature) { diff --git a/physics/src/modules/FluxCalculationModule/include/FiniteElementFluxes.hpp b/physics/src/modules/FluxCalculationModule/include/FiniteElementFluxes.hpp index 5bb368cc9..c78b81b3f 100644 --- a/physics/src/modules/FluxCalculationModule/include/FiniteElementFluxes.hpp +++ b/physics/src/modules/FluxCalculationModule/include/FiniteElementFluxes.hpp @@ -10,7 +10,6 @@ #include "include/IFluxCalculation.hpp" #include "include/IIceAlbedo.hpp" #include "include/IIceOceanHeatFlux.hpp" -#include "include/ModelArrayRef.hpp" #include "include/ModelComponent.hpp" namespace Nextsim { @@ -33,19 +32,19 @@ class FiniteElementFluxes : public IFluxCalculation, public Configured sst; - ModelArrayRef sss; - ModelArrayRef t_air; - ModelArrayRef t_dew2; - ModelArrayRef p_air; - ModelArrayRef windSpeed; - ModelArrayRef u_air; - ModelArrayRef v_air; - ModelArrayRef hsnow; // cell-averaged value - ModelArrayRef cice; - ModelArrayRef tsurf; - ModelArrayRef sw_in; - ModelArrayRef lw_in; + ModelArrayAccessor sstAccessor; + ModelArrayAccessor sssAccessor; + ModelArrayAccessor t_airAccessor; + ModelArrayAccessor t_dew2Accessor; + ModelArrayAccessor p_airAccessor; + ModelArrayAccessor windSpeedAccessor; + ModelArrayAccessor u_airAccessor; + ModelArrayAccessor v_airAccessor; + ModelArrayAccessor hsnowAccessor; // cell-averaged value + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor tsurfAccessor; + ModelArrayAccessor sw_inAccessor; + ModelArrayAccessor lw_inAccessor; void calculateOW(size_t i, const TimestepTime& tst); void calculateIce(size_t i, const TimestepTime& tst); diff --git a/physics/src/modules/IceOceanHeatFluxModule/BasicIceOceanHeatFlux.cpp b/physics/src/modules/IceOceanHeatFluxModule/BasicIceOceanHeatFlux.cpp index 2b21891a2..c72629bc1 100644 --- a/physics/src/modules/IceOceanHeatFluxModule/BasicIceOceanHeatFlux.cpp +++ b/physics/src/modules/IceOceanHeatFluxModule/BasicIceOceanHeatFlux.cpp @@ -14,18 +14,32 @@ static inline double doOne(double tBot, double sst, double mlBulkCp, double time void BasicIceOceanHeatFlux::update(const TimestepTime& tst) { + HField& qio = qioAccessor.getHostRW(); + const AdvectedField& cice = ciceAccessor.getHostRO(); + const HField& mlBulkCp = mlBulkCpAccessor.getHostRO(); + const HField& sst = sstAccessor.getHostRO(); + const HField& tf = tfAccessor.getHostRO(); + overElements( - [this](size_t i, const TimestepTime& tsTime) { this->updateElement(i, tsTime); }, tst); + [&](size_t i, const TimestepTime& tsTime) { + // Use the timestep length as the relaxation time scale + if (cice[i] > 0.) { + qio[i] = doOne(tf[i], sst[i], mlBulkCp[i], tst.step.seconds()); + } else { + qio[i] = 0.; + } + }, + tst); } -void BasicIceOceanHeatFlux::updateElement(size_t i, const TimestepTime& tst) -{ - // Use the timestep length as the relaxation time scale - if (cice[i] > 0.) { - qio[i] = doOne(tf[i], sst[i], mlBulkCp[i], tst.step.seconds()); - } else { - qio[i] = 0.; - } -} +// void BasicIceOceanHeatFlux::updateElement(size_t i, const TimestepTime& tst) +// { +// // Use the timestep length as the relaxation time scale +// if (cice[i] > 0.) { +// qio[i] = doOne(tf[i], sst[i], mlBulkCp[i], tst.step.seconds()); +// } else { +// qio[i] = 0.; +// } +// } } /* namespace Nextsim */ diff --git a/physics/src/modules/IceOceanHeatFluxModule/include/BasicIceOceanHeatFlux.hpp b/physics/src/modules/IceOceanHeatFluxModule/include/BasicIceOceanHeatFlux.hpp index 73a1b36ac..7d3ce0548 100644 --- a/physics/src/modules/IceOceanHeatFluxModule/include/BasicIceOceanHeatFlux.hpp +++ b/physics/src/modules/IceOceanHeatFluxModule/include/BasicIceOceanHeatFlux.hpp @@ -15,7 +15,7 @@ class BasicIceOceanHeatFlux : public IIceOceanHeatFlux { public: BasicIceOceanHeatFlux() : IIceOceanHeatFlux() - , mlBulkCp(getStore()) + , mlBulkCpAccessor(getStore()) { } virtual ~BasicIceOceanHeatFlux() = default; @@ -26,7 +26,7 @@ class BasicIceOceanHeatFlux : public IIceOceanHeatFlux { void updateElement(size_t i, const TimestepTime&); protected: - ModelArrayRef mlBulkCp; + ModelArrayAccessor mlBulkCpAccessor; }; } /* namespace Nextsim */ diff --git a/physics/src/modules/IceThermodynamicsModule/ThermoIce0.cpp b/physics/src/modules/IceThermodynamicsModule/ThermoIce0.cpp index 7b9df9824..c531d1b88 100644 --- a/physics/src/modules/IceThermodynamicsModule/ThermoIce0.cpp +++ b/physics/src/modules/IceThermodynamicsModule/ThermoIce0.cpp @@ -7,6 +7,7 @@ #include "include/IFreezingPoint.hpp" #include "include/IceGrowth.hpp" #include "include/IceMinima.hpp" +#include "include/KernelAlternatives.hpp" #include "include/ModelArray.hpp" #include "include/NextsimModule.hpp" #include "include/constants.hpp" @@ -19,18 +20,157 @@ const double ThermoIce0::freezingPointIce = -Water::mu * Ice::s; ThermoIce0::ThermoIce0() : IIceThermodynamics() - , snowMelt(ModelArray::Type::H) - , topMelt(ModelArray::Type::H) - , botMelt(ModelArray::Type::H) - , qic(ModelArray::Type::H) + , snowMeltAccessor(getStore(), RO, ModelArray::Type::H) + , topMeltAccessor(getStore(), RO, ModelArray::Type::H) + , botMeltAccessor(getStore(), RO, ModelArray::Type::H) + , qicAccessor(getStore(), RW, ModelArray::Type::H) { + snowMeltAccessor.getHostRW().resize(); + topMeltAccessor.getHostRW().resize(); + botMeltAccessor.getHostRW().resize(); + qicAccessor.getHostRW().resize(); } void ThermoIce0::update(const TimestepTime& tsTime) { IIceThermodynamics::update(tsTime); - overElements( - [this](size_t i, const TimestepTime& tst) { this->calculateElement(i, tst); }, tsTime); + + auto execSpace = DefaultExecutionSpace(); + auto& tsurf = tsurfAccessor.getAutoRW(execSpace); + auto& deltaHi = deltaHiAccessor.getAutoRW(execSpace); + auto& hice = hiceAccessor.getAutoRW(execSpace); + auto& hsnow = hsnowAccessor.getAutoRW(execSpace); + auto& cice = ciceAccessor.getAutoRW(execSpace); + auto& qow = qowAccessor.getAutoRW(execSpace); + auto& qswBase = qswBaseAccessor.getAutoRW(execSpace); + auto& snowToIce = snowToIceAccessor.getAutoRW(execSpace); + auto& topMelt = topMeltAccessor.getAutoRW(execSpace); + auto& snowMelt = snowMeltAccessor.getAutoRW(execSpace); + auto& botMelt = botMeltAccessor.getAutoRW(execSpace); + auto& qic = qicAccessor.getAutoRW(execSpace); + const auto& sublim = sublimAccessor.getAutoRO(execSpace); + const auto& tf = tfAccessor.getAutoRO(execSpace); + const auto& snowfall = snowfallAccessor.getAutoRO(execSpace); + const auto& dQia_dt = dQia_dtAccessor.getAutoRO(execSpace); + const auto& qio = qioAccessor.getAutoRO(execSpace); + const auto& penSw = penSwAccessor.getAutoRO(execSpace); + const auto& qia = qiaAccessor.getAutoRO(execSpace); + + // static members can not be captured directly + const bool doFlooding = ThermoIce0::doFlooding; + const double freezingPointIce = ThermoIce0::freezingPointIce; + const double kappa_s = ThermoIce0::kappa_s; + const double cMin = IceMinima::c(); + const double hMin = IceMinima::h(); + const double dt = tsTime.step.seconds(); + + overElementsAuto(OVER_ELEMENTS_LAMBDA(const ElementIndex i) { + static const double bulkLHFusionSnow = Water::Lf * Ice::rhoSnow; + static const double bulkLHFusionIce = Water::Lf * Ice::rho; + + // If there is too little ice, do nothing and zero out the computed arrays + if (hice[i] <= hMin || cice[i] <= cMin) { + deltaHi[i] = 0.; + snowToIce[i] = 0.; + snowMelt[i] = 0.; + qswBase[i] = 0.; + + tsurf[i] = freezingPointIce; + + // Add to open water flux, since cice will be set to zero + qow[i] += (hice[i] * bulkLHFusionIce + hsnow[i] * bulkLHFusionSnow) / dt; + cice[i] = 0.; + hice[i] = 0.; + hsnow[i] = 0.; + + return; + } + + // Semtner's fudge factors for the zero-layer model + constexpr double beta = 0.4; + constexpr double gamma = 1.065; + + double hi = hice[i] / cice[i]; + const double oldHi = hi; + double hs = hsnow[i] / cice[i]; + // Create a reference to the local updated Tice value here to avoid having + // to write the array access expression out in full every time + double& tice_i = tsurf[i]; + double tice0 = tsurf[i]; + double k_lSlab = kappa_s * Ice::kappa / (kappa_s * hi + Ice::kappa * hs) * gamma; + qic[i] = k_lSlab * (tf[i] - tice0); + double remainingFlux = qic[i] - (qia[i] + (1. - beta) * penSw[i]); + tice_i = tice0 + remainingFlux / (k_lSlab + dQia_dt[i]); + + // Clamp the temperature of the ice to a maximum of the melting point + // of ice or snow + double meltingLimit = (hs > 0) ? 0 : freezingPointIce; + tice_i = Utils::min(meltingLimit, tice_i); + + // Top melt. Melting rate is non-positive. + double snowMeltRate = Utils::min(-remainingFlux, 0.) / bulkLHFusionSnow; + snowMelt[i] = snowMeltRate * dt; + double snowSublRate = sublim[i] / Ice::rhoSnow; + double nowSnow = hs + (snowMeltRate - snowSublRate) * dt; + // Use excess flux to melt ice. Non-positive value + double excessIceMelt = Utils::min(nowSnow, 0.) * bulkLHFusionSnow / bulkLHFusionIce; + // With the excess flux noted, clamp the snow thickness to a minimum of zero. + hs = Utils::max(nowSnow, 0.); + + // Bottom melt or growth + double iceBottomChange = (qic[i] - qio[i]) * dt / bulkLHFusionIce; + // Total thickness change + deltaHi[i] = excessIceMelt + iceBottomChange; + hi += deltaHi[i]; + + // Then add snowfall back on top if there's still ice + if (hi > 0.) + hs += snowfall[i] * dt / Ice::rhoSnow; + + // Amount of melting (only) at the top and bottom of the ice + topMelt[i] = Utils::min(excessIceMelt, 0.); + botMelt[i] = Utils::min(iceBottomChange, 0.); + // Snow to ice conversion + double iceDraught = (hi * Ice::rho + hs * Ice::rhoSnow) / Water::rhoOcean; + + if (doFlooding && iceDraught > hi) { + double snowDraught = iceDraught - hi; + snowToIce[i] = snowDraught; + hs -= snowDraught * Ice::rho / Ice::rhoSnow; + hi = iceDraught; + } else { + snowToIce[i] = 0; + } + + // Melt all ice if it is below minimum threshold + if (hi < hMin) { + if (deltaHi[i] < 0) { + double scaling = oldHi / deltaHi[i]; + topMelt[i] *= scaling; + botMelt[i] *= scaling; + } + + // No snow was converted to ice + snowToIce[i] = 0.; + + // Change in thickness is all of the old thickness + deltaHi[i] = -oldHi; + + // Add the melt flux to open water flux, since cice will be set to zero + qow[i] += cice[i] * (hi * bulkLHFusionIce + hs * bulkLHFusionSnow) / dt; + + // No ice, no snow and the surface temperature is the melting point of ice + cice[i] = 0.; + hice[i] = 0.; + hsnow[i] = 0.; + tsurf[i] = celsius(Ice::Tm); + } else { + // If there is still ice, we need to update the DG ice and snow thicknesses with the + // new values + hice[i] = hi * cice[i]; + hsnow[i] = hs * cice[i]; + } + }); } static const std::map keyMap = { @@ -47,10 +187,10 @@ ConfigMap ThermoIce0::getConfiguration() const { return { { keyMap.at(KS_KEY), k ModelState ThermoIce0::getStateDiagnostic() const { ModelState state = { { - { "snow_melt", snowMelt }, - { "top_melt", topMelt }, - { "bottom_melt", botMelt }, - { "Q_ic", qic }, + { "snow_melt", snowMeltAccessor.getHostRO() }, + { "top_melt", topMeltAccessor.getHostRO() }, + { "bottom_melt", botMeltAccessor.getHostRO() }, + { "Q_ic", qicAccessor.getHostRO() }, }, getConfiguration() }; @@ -80,119 +220,10 @@ void ThermoIce0::setData(const ModelState::DataMap& ms) { IIceThermodynamics::setData(ms); - snowMelt.resize(); - topMelt.resize(); - botMelt.resize(); - qic.resize(); -} - -void ThermoIce0::calculateElement(size_t i, const TimestepTime& tst) -{ - static const double bulkLHFusionSnow = Water::Lf * Ice::rhoSnow; - static const double bulkLHFusionIce = Water::Lf * Ice::rho; - - // If there is too little ice, do nothing and zero out the computed arrays - if (hice[i] <= IceMinima::h() || cice[i] <= IceMinima::c()) { - deltaHi[i] = 0.; - snowToIce[i] = 0.; - snowMelt[i] = 0.; - qswBase[i] = 0.; - - tsurf[i] = freezingPointIce; - - // Add to open water flux, since cice will be set to zero - qow[i] += (hice[i] * bulkLHFusionIce + hsnow[i] * bulkLHFusionSnow) / tst.step; - cice[i] = 0.; - hice[i] = 0.; - hsnow[i] = 0.; - - return; - } - - // Semtner's fudge factors for the zero-layer model - constexpr double beta = 0.4; - constexpr double gamma = 1.065; - - double hi = hice[i] / cice[i]; - const double oldHi = hi; - double hs = hsnow[i] / cice[i]; - // Create a reference to the local updated Tice value here to avoid having - // to write the array access expression out in full every time - double& tice_i = tsurf[i]; - double tice0 = tsurf[i]; - double k_lSlab = kappa_s * Ice::kappa / (kappa_s * hi + Ice::kappa * hs) * gamma; - qic[i] = k_lSlab * (tf[i] - tice0); - double remainingFlux = qic[i] - (qia[i] + (1. - beta) * penSw[i]); - tice_i = tice0 + remainingFlux / (k_lSlab + dQia_dt[i]); - - // Clamp the temperature of the ice to a maximum of the melting point - // of ice or snow - double meltingLimit = (hs > 0) ? 0 : freezingPointIce; - tice_i = std::min(meltingLimit, tice_i); - - // Top melt. Melting rate is non-positive. - double snowMeltRate = std::min(-remainingFlux, 0.) / bulkLHFusionSnow; - snowMelt[i] = snowMeltRate * tst.step; - double snowSublRate = sublim[i] / Ice::rhoSnow; - double nowSnow = hs + (snowMeltRate - snowSublRate) * tst.step; - // Use excess flux to melt ice. Non-positive value - double excessIceMelt = std::min(nowSnow, 0.) * bulkLHFusionSnow / bulkLHFusionIce; - // With the excess flux noted, clamp the snow thickness to a minimum of zero. - hs = std::max(nowSnow, 0.); - - // Bottom melt or growth - double iceBottomChange = (qic[i] - qio[i]) * tst.step / bulkLHFusionIce; - // Total thickness change - deltaHi[i] = excessIceMelt + iceBottomChange; - hi += deltaHi[i]; - - // Then add snowfall back on top if there's still ice - if (hi > 0.) - hs += snowfall[i] * tst.step / Ice::rhoSnow; - - // Amount of melting (only) at the top and bottom of the ice - topMelt[i] = std::min(excessIceMelt, 0.); - botMelt[i] = std::min(iceBottomChange, 0.); - // Snow to ice conversion - double iceDraught = (hi * Ice::rho + hs * Ice::rhoSnow) / Water::rhoOcean; - - if (doFlooding && iceDraught > hi) { - double snowDraught = iceDraught - hi; - snowToIce[i] = snowDraught; - hs -= snowDraught * Ice::rho / Ice::rhoSnow; - hi = iceDraught; - } else { - snowToIce[i] = 0; - } - - // Melt all ice if it is below minimum threshold - if (hi < IceMinima::h()) { - if (deltaHi[i] < 0) { - double scaling = oldHi / deltaHi[i]; - topMelt[i] *= scaling; - botMelt[i] *= scaling; - } - - // No snow was converted to ice - snowToIce[i] = 0.; - - // Change in thickness is all of the old thickness - deltaHi[i] = -oldHi; - - // Add the melt flux to open water flux, since cice will be set to zero - qow[i] += cice[i] * (hi * bulkLHFusionIce + hs * bulkLHFusionSnow) / tst.step; - - // No ice, no snow and the surface temperature is the melting point of ice - cice[i] = 0.; - hice[i] = 0.; - hsnow[i] = 0.; - tsurf[i] = celsius(Ice::Tm); - } else { - // If there is still ice, we need to update the DG ice and snow thicknesses with the new - // values - hice[i] = hi * cice[i]; - hsnow[i] = hs * cice[i]; - } + snowMeltAccessor.getHostRW().resize(); + topMeltAccessor.getHostRW().resize(); + botMeltAccessor.getHostRW().resize(); + qicAccessor.getHostRW().resize(); } } /* namespace Nextsim */ diff --git a/physics/src/modules/IceThermodynamicsModule/ThermoWinton.cpp b/physics/src/modules/IceThermodynamicsModule/ThermoWinton.cpp index b434a868d..79250d06a 100644 --- a/physics/src/modules/IceThermodynamicsModule/ThermoWinton.cpp +++ b/physics/src/modules/IceThermodynamicsModule/ThermoWinton.cpp @@ -1,5 +1,6 @@ /*! * @author Tim Spain + * @author Robert Jendersie */ #include "include/ThermoWinton.hpp" @@ -7,6 +8,8 @@ #include "include/constants.hpp" #include "include/gridNames.hpp" +#include "include/KernelAlternatives.hpp" +#include "kokkos/include/KokkosTimer.hpp" #include @@ -25,20 +28,20 @@ const std::string ThermoWinton::tBottomName = "tbottom"; ThermoWinton::ThermoWinton() : IIceThermodynamics() - , tInternal(ModelArray::AdvectionType) - , tBottom(ModelArray::AdvectionType) - , snowMelt(ModelArray::Type::H) - , topMelt(ModelArray::Type::H) - , botMelt(ModelArray::Type::H) - , sw_in(getStore()) - , subl(getStore()) + , tInternalAccessor(getStore(), RO, ModelArray::AdvectionType) + , tBottomAccessor(getStore(), RO, ModelArray::AdvectionType) + , snowMeltAccessor(getStore(), RO, ModelArray::Type::H) + , topMeltAccessor(getStore(), RO, ModelArray::Type::H) + , botMeltAccessor(getStore(), RO, ModelArray::Type::H) + , sw_inAccessor(getStore()) + , sublAccessor(getStore()) { - tInternal.resize(); - tBottom.resize(); - snowMelt.resize(); - topMelt.resize(); - botMelt.resize(); - snowToIce.resize(); + tInternalAccessor.getHostRW().resize(); + tBottomAccessor.getHostRW().resize(); + snowMeltAccessor.getHostRW().resize(); + topMeltAccessor.getHostRW().resize(); + botMeltAccessor.getHostRW().resize(); + snowToIceAccessor.getHostRW().resize(); } static const std::map keyMap = { @@ -63,9 +66,9 @@ ConfigMap ThermoWinton::getConfiguration() const ModelState ThermoWinton::getStateDiagnostic() const { ModelState state = { { - { "snow_melt", snowMelt }, - { "top_melt", topMelt }, - { "bottom_melt", botMelt }, + { "snow_melt", snowMeltAccessor.getHostRO() }, + { "top_melt", topMeltAccessor.getHostRO() }, + { "bottom_melt", botMeltAccessor.getHostRO() }, }, getConfiguration() }; @@ -76,8 +79,8 @@ ModelState ThermoWinton::getStateDiagnostic() const ModelState ThermoWinton::getStatePrognostic() const { ModelState state = { { - { tInteriorName, tInternal }, - { tBottomName, tBottom }, + { tInteriorName, tInternalAccessor.getHostRO() }, + { tBottomName, tBottomAccessor.getHostRO() }, }, getConfiguration() }; @@ -101,12 +104,14 @@ void ThermoWinton::setData(const ModelState::DataMap& state) { IIceThermodynamics::setData(state); - tInternal.resize(); - tBottom.resize(); - snowMelt.resize(); - topMelt.resize(); - botMelt.resize(); - snowToIce.resize(); + AdvectedField& tInternal = tInternalAccessor.getHostRW(); + AdvectedField& tBottom = tBottomAccessor.getHostRW(); + tInternalAccessor.getHostRW().resize(); + tBottomAccessor.getHostRW().resize(); + snowMeltAccessor.getHostRW().resize(); + topMeltAccessor.getHostRW().resize(); + botMeltAccessor.getHostRW().resize(); + snowToIceAccessor.getHostRW().resize(); /* If the internal temperature is not in the restart file, then we simply set it to the freezing * point of seawater. It's a safe approximation, and it seems the user doesn't really care! */ @@ -127,225 +132,10 @@ void ThermoWinton::setData(const ModelState::DataMap& state) } } -void ThermoWinton::update(const TimestepTime& tst) -{ - // Advect ice temperatures - IIceThermodynamics::update(tst); - FieldAdvection::advectField(tInternal, tst, IIceThermodynamics::minT, seaIceTf); - FieldAdvection::advectField(tBottom, tst, IIceThermodynamics::minT, seaIceTf); - // Perform the rest of the thermodynamics using the advected temperatures - overElements( - [this](const size_t i, const TimestepTime& tsTime) { calculateElement(i, tsTime); }, tst); -} - -void ThermoWinton::calculateElement(size_t i, const TimestepTime& tst) -{ - - static const double bulkLHFusionSnow = Water::Lf * Ice::rhoSnow; - static const double bulkLHFusionIce = Water::Lf * Ice::rho; - - // Don't do anything if there is no ice - if (cice[i] <= IceMinima::c() || hice[i] <= IceMinima::h()) { - - snowToIce[i] = 0.; - snowMelt[i] = 0.; - qswBase[i] = 0.; - - // Add to open water flux, since cice will be set to zero - qow[i] += (hice[i] * bulkLHFusionIce + hsnow[i] * bulkLHFusionSnow) / tst.step; - - deltaHi[i] = 0; - cice[i] = 0; - hice[i] = 0; - hsnow[i] = 0; - - tsurf[i] = seaIceTf; - tInternal[i] = seaIceTf; - tBottom[i] = seaIceTf; - - return; - } - - double tSurf = tsurf[i]; // surface temperature - double tUppr = tInternal[i]; // upper layer temperature - double tLowr = tBottom[i]; // lower layer temperature - double tBott = tf[i]; // freezing point of (local) seawater - - double hi = hice[i] / cice[i]; - double hs = hsnow[i] / cice[i]; - const double oldHi = hi; // Ice thickness at the start of the timestep - - double dt = tst.step.seconds(); - - double surfMelt = 0; // surface melting mass loss - // Calculate temperatures by solving the heat conduction equation - calculateTemps(tSurf, tUppr, tLowr, surfMelt, i, dt); - - // The ratio of ΔΗ_f T_f / c_p,ice is used a lot. Units are K² - const static double dHfTf_cp = Water::Lf * seaIceTf / Ice::cp; - - // Thickness changes - // ice - double h1 = hi / 2; - double h2 = hi / 2; - // Eqs. (1) and (25) - but I 've multiplied them with \rho_i (hence cVol), because it's missing - // in the paper - double e1 = cVol * (tUppr - seaIceTf) - bulkLHFusionIce * (1 - seaIceTf / tUppr); - double e2 = cVol * (tLowr - seaIceTf) - bulkLHFusionIce; - - // snow - hs += snowfall[i] / Ice::rhoSnow * dt; - // double accumulatedSnowThickness = snowfall[i] / Ice::rhoSnow * dt; - - // sublimation - // 4 cases - const double& subli = subl[i]; - double deltaSnow = subli * dt / Ice::rhoSnow; - double deltaIce1 = (deltaSnow - hs) * Ice::rhoSnow / Ice::rho; - double deltaIce2 = deltaIce1 - h1; - if (deltaSnow <= hs) { - // sublimation is less than or equal to the mass of snow - hs -= deltaSnow; - } else if (deltaIce1 <= h1) { - // sublimation minus sublimed snow is less than or equal to half the - // ice thickness - h1 -= deltaIce1; - hs = 0; - } else if (deltaIce2 <= h2) { - // sublimation minus sublimed snow is greater than half the ice - // thickness, but not all of it - h2 -= deltaIce2; - h1 = 0; - hs = 0; - } else { - // the snow and ice sublimates - double oceanEvapError = (deltaIce2 - h2) * Ice::rho / Water::rhoOcean; - // TODO: log the error - h2 = 0; - h1 = 0; - hs = 0; - } - // Sublimated ice counts as top melt - topMelt[i] = std::max(0., h1 + h2 - hi); // (23) - - // Bottom melt/freezing - double meltBottom = (qio[i] - 4 * Ice::kappa * (tBott - tLowr) / hi) * dt; - snowMelt[i] = 0; - if (meltBottom <= 0.) { - // Freezing - // Eq. (25) - but I 've multiplied them with \rho_i (hence cVol), because it's missing in - // the paper - double eBot = cVol * (tBott - seaIceTf) - bulkLHFusionIce; - deltaIce2 = meltBottom / eBot; // (24) - tLowr = (deltaIce2 * tBott + h2 * tLowr) / (deltaIce2 + h2); // (26) - h2 += deltaIce2; - } else { - // Melting - // Eqs. (31)-(32) with added division with \rho_i (and \rho_s for 32) - deltaIce2 = -std::min(-meltBottom / e2, h2); - deltaIce1 = -std::min(std::max(-(meltBottom + e2 * h2) / e1, 0.), h1); - snowMelt[i] - = -std::min(std::max((meltBottom + e2 * h2 + e1 * h1) / bulkLHFusionSnow, 0.), hs); - - // If everything melts we need to put heat back into the ocean - if (h2 + h1 + hs - deltaIce2 - deltaIce1 - snowMelt[i] <= 0.) { - // (34) - with added multiplication of rhoi and rhos and division with dt - qow[i] -= cice[i] * std::max(meltBottom - bulkLHFusionSnow * hs + e1 * h1 + e2 * h2, 0.) - / dt; - } - - hs += snowMelt[i]; - h1 += deltaIce1; - h2 += deltaIce2; - botMelt[i] += deltaIce1 + deltaIce2; - } - - // Melting at the surface - // Do we really need an assertion here? - // assert(surfMelt >= 0); - // Eqs. (27)-(29) with division of \rho_i and \rho_s - snowMelt[i] -= std::min(surfMelt * dt / bulkLHFusionSnow, hs); - deltaIce1 = -std::min(std::max(-(surfMelt * dt - bulkLHFusionSnow * hs) / e1, 0.), h1); - deltaIce2 - = -std::min(std::max(-(surfMelt * dt - bulkLHFusionSnow * hs + e1 * h1) / e2, 0.), h2); - - // If everything melts we need to put heat back into the ocean - // Eq (30) - with multiplication of rhoi and rhos and division with dt - if (h2 + h1 + hs - deltaIce2 - deltaIce1 - snowMelt[i] <= 0.) { - qow[i] -= cice[i] * std::max(surfMelt * dt - bulkLHFusionSnow * hs + e1 * h1 + e2 * h2, 0.) - / dt; - } - - hs += snowMelt[i]; - h1 += deltaIce1; - h2 += deltaIce2; - topMelt[i] += deltaIce1 + deltaIce2; - - // Snow to ice conversion - double freeboard = (hi * (Water::rhoOcean - Ice::rho) - hs * Ice::rhoSnow) / Water::rhoOcean; - if (doFlooding && freeboard < 0.) { - hs += std::min(freeboard * Ice::rho / Ice::rhoSnow, 0.); // (35) using += - deltaIce1 = std::max(-freeboard, 0.); // (36) - double f1 = 1 - deltaIce1 / (deltaIce1 + h1); // Fraction of new ice in the upper layer - double tBar = f1 * (tUppr + dHfTf_cp / tUppr) + (1 - f1) * seaIceTf; // (39) - tUppr = (tBar - std::sqrt(tBar * tBar - 4 * dHfTf_cp)) / 2; // (38) - h1 += deltaIce1; - snowToIce[i] += deltaIce1; - } - - // Add up the half-layer thicknesses - hi = h1 + h2; - // Adjust the temperatures to evenly divide the ice - if (h2 > h1) { - // Lower layer ice is added to the upper layer - double f1 = h1 / hi * 2; - double tBar = f1 * (tUppr + dHfTf_cp / tUppr) + (1 - f1) * tLowr; // (39) - // The upper layer temperature changes - tUppr = (tBar - std::sqrt(tBar * tBar - 4 * dHfTf_cp)) / 2; // (38) - } else { - // Upper layer ice is added to the lower layer - double f1 = (2 * h1 - hi) / hi; - // Lower layer temperature changes - tLowr = f1 * (tUppr + dHfTf_cp / tUppr) + (1 - f1) * tLowr; // (40) - // Melt from top and bottom if the lower layer temperature is too high - if (tLowr > seaIceTf) { - double deltaMelt = hi / 4 * Ice::cp * (tLowr - seaIceTf) * tUppr - / (Ice::Lf * tUppr + (Ice::cp * tUppr - Ice::Lf) * (seaIceTf - tUppr)); - topMelt[i] -= deltaMelt; - botMelt[i] -= deltaMelt; - hi -= 2 * deltaMelt; - tLowr = seaIceTf; - } - } - deltaHi[i] = hi - oldHi; - - // Remove very small ice thickness - // The fluxes are already sorted - if (hi < IceMinima::h()) { - if (deltaHi[i] < 0) { - topMelt[i] *= oldHi / deltaHi[i]; - botMelt[i] *= oldHi / deltaHi[i]; - } - snowToIce[i] = 0; - - deltaHi[i] = -oldHi; - cice[i] = 0; - - tSurf = seaIceTf; - tUppr = seaIceTf; - tLowr = seaIceTf; - } - - tsurf[i] = tSurf; - tInternal[i] = tUppr; - tBottom[i] = tLowr; - - hice[i] = hi * cice[i]; - hsnow[i] = hs * cice[i]; -} - -void ThermoWinton::calculateTemps( - double& tSurf, double& tUppr, double& tLowr, double& mSurf, size_t i, double dt) +KERNEL_IMPL_FUNCTION static void calculateTemps(double& tSurf, double& tUppr, double& tLowr, + double& mSurf, const double cice, const double dQia_dt, const double hice, const double hsnow, + const double penSw, const double qia, const double tf, double dt, const double cVol, + const double seaIceTf, const double kappa_s) { /* * Internal temperatures @@ -353,26 +143,26 @@ void ThermoWinton::calculateTemps( * finally T2 Numers in parentheses refer to equations in the paper */ - const double hi = hice[i] / cice[i]; - const double hs = hsnow[i] / cice[i]; - const double tBase = tf[i]; // Freezing point of seawater with the local salinity + const double hi = hice / cice; + const double hs = hsnow / cice; + const double tBase = tf; // Freezing point of seawater with the local salinity const double tMelt = (hs > 0) ? 0 : seaIceTf; // Melting point at the surface // First some coefficients based on temperatures from the previous time step double k12 = 4 * Ice::kappa * kappa_s / (kappa_s * hi + 4 * Ice::kappa * hs); // (5) - double a = qia[i] - tSurf * dQia_dt[i]; // (7) - double b = dQia_dt[i]; // (8) + double a = qia - tSurf * dQia_dt; // (7) + double b = dQia_dt; // (8) double k32 = 2 * Ice::kappa / hi; // (10) double a1 = hi * cVol / (2 * dt) + k32 * (4 * dt * k32 + hi * cVol) / (6 * dt * k32 + hi * cVol) + b * k12 / (k12 + b); // (16) - double b1 = -hi * (cVol * tUppr + Ice::Lf * Ice::rho * seaIceTf / tUppr) / (2 * dt) - penSw[i] + double b1 = -hi * (cVol * tUppr + Ice::Lf * Ice::rho * seaIceTf / tUppr) / (2 * dt) - penSw - k32 * (4 * dt * k32 * tBase + hi * cVol * tLowr) / (6 * dt * k32 + hi * cVol) + a * k12 / (k12 + b); // (17) double c1 = hi * Ice::Lf * Ice::rho * seaIceTf / (2 * dt); // (18) // Updated surface and mid-ice temperatures - tUppr = -(b1 + std::sqrt(b1 * b1 - 4 * a1 * c1)) / (2 * a1); // (21) + tUppr = -(b1 + Utils::sqrt(b1 * b1 - 4 * a1 * c1)) / (2 * a1); // (21) tSurf = (k12 * tUppr - a) / (k12 + b); // (6) // Is the surface melting? if (tSurf > tMelt) { @@ -381,14 +171,271 @@ void ThermoWinton::calculateTemps( // apply the change to the *1 parameters, rather than recalculating in full a1 += k12 - k12 * b / (k12 + b); // (19) simplified using += b1 -= k12 * tSurf + a * k12 / (k12 + b); // (20) simplified using -= - tUppr = -(b1 + std::sqrt(b1 * b1 - 4 * a1 * c1)) / (2 * a1); // (21) + tUppr = -(b1 + Utils::sqrt(b1 * b1 - 4 * a1 * c1)) / (2 * a1); // (21) // Surface melt mSurf = k12 * (tUppr - tSurf) - (a + b * tSurf); // (22) } // update the lower T based on the new value of the upper T (eq 15) - tLowr = (2 * dt * k32 * (tUppr + 2 * tf[i]) + hi * cVol * tLowr) / (6 * dt * k32 + hi * cVol); + tLowr = (2 * dt * k32 * (tUppr + 2 * tf) + hi * cVol * tLowr) / (6 * dt * k32 + hi * cVol); +} + +void ThermoWinton::update(const TimestepTime& tst) +{ + static KokkosTimer timer("ThermoWinton"); + static KokkosTimer timerAdvect("ThermoWintonAdvection"); + static KokkosTimer timerCopy("ThermoWintonCopy"); + static KokkosTimer timerUpdate("ThermoWintonUpdate"); + + timer.start(); + timerAdvect.start(); + // Advect ice temperatures + IIceThermodynamics::update(tst); + auto execSpace = DefaultExecutionSpace(); + auto& tBottom = tBottomAccessor.getAutoRW(execSpace); + auto& tInternal = tInternalAccessor.getAutoRW(execSpace); + FieldAdvection::advectField(tInternal, tst, IIceThermodynamics::minT, seaIceTf); + FieldAdvection::advectField(tBottom, tst, IIceThermodynamics::minT, seaIceTf); + timerAdvect.stop(); + + timerCopy.start(); + // explicit execution space enables asynchronous execution + auto& hsnow = hsnowAccessor.getAutoRW(execSpace); + auto& botMelt = botMeltAccessor.getAutoRW(execSpace); + auto& hice = hiceAccessor.getAutoRW(execSpace); + auto& topMelt = topMeltAccessor.getAutoRW(execSpace); + auto& qswBase = qswBaseAccessor.getAutoRW(execSpace); + auto& snowMelt = snowMeltAccessor.getAutoRW(execSpace); + auto& tsurf = tsurfAccessor.getAutoRW(execSpace); + auto& deltaHi = deltaHiAccessor.getAutoRW(execSpace); + auto& qow = qowAccessor.getAutoRW(execSpace); + auto& snowToIce = snowToIceAccessor.getAutoRW(execSpace); + auto& cice = ciceAccessor.getAutoRW(execSpace); + const auto& dQia_dt = dQia_dtAccessor.getAutoRO(execSpace); + const auto& penSw = penSwAccessor.getAutoRO(execSpace); + const auto& qia = qiaAccessor.getAutoRO(execSpace); + const auto& qio = qioAccessor.getAutoRO(execSpace); + const auto& snowfall = snowfallAccessor.getAutoRO(execSpace); + const auto& subl = sublAccessor.getAutoRO(execSpace); + const auto& tf = tfAccessor.getAutoRO(execSpace); + + // static members can not be captured directly + const double cVol = ThermoWinton::cVol; + const bool doFlooding = ThermoWinton::doFlooding; + const double seaIceTf = ThermoWinton::seaIceTf; + const double kappa_s = ThermoWinton::kappa_s; + const double cMin = IceMinima::c(); + const double hMin = IceMinima::h(); + const double dt = tst.step.seconds(); + timerCopy.stop(); + + timerUpdate.start(); + overElementsAuto(OVER_ELEMENTS_LAMBDA (const ElementIndex i) { + static const double bulkLHFusionSnow = Water::Lf * Ice::rhoSnow; + static const double bulkLHFusionIce = Water::Lf * Ice::rho; + + // Don't do anything if there is no ice + if (cice[i] <= cMin || hice[i] <= hMin) { + + snowToIce[i] = 0.; + snowMelt[i] = 0.; + qswBase[i] = 0.; + + // Add to open water flux, since cice will be set to zero + qow[i] += (hice[i] * bulkLHFusionIce + hsnow[i] * bulkLHFusionSnow) / dt; + + deltaHi[i] = 0; + cice[i] = 0; + hice[i] = 0; + hsnow[i] = 0; + + tsurf[i] = seaIceTf; + tInternal[i] = seaIceTf; + tBottom[i] = seaIceTf; + + return; + } + + double tSurf = tsurf[i]; // surface temperature + double tUppr = tInternal[i]; // upper layer temperature + double tLowr = tBottom[i]; // lower layer temperature + double tBott = tf[i]; // freezing point of (local) seawater + + double hi = hice[i] / cice[i]; + double hs = hsnow[i] / cice[i]; + const double oldHi = hi; // Ice thickness at the start of the timestep + + double surfMelt = 0; // surface melting mass loss + // Calculate temperatures by solving the heat conduction equation + calculateTemps(tSurf, tUppr, tLowr, surfMelt, cice[i], dQia_dt[i], hice[i], hsnow[i], + penSw[i], qia[i], tf[i], dt, cVol, seaIceTf, kappa_s); + + // The ratio of ΔΗ_f T_f / c_p,ice is used a lot. Units are K² + const static double dHfTf_cp = Water::Lf * seaIceTf / Ice::cp; + + // Thickness changes + // ice + double h1 = hi / 2; + double h2 = hi / 2; + // Eqs. (1) and (25) - but I 've multiplied them with \rho_i (hence cVol), because it's + // missing in the paper + double e1 = cVol * (tUppr - seaIceTf) - bulkLHFusionIce * (1 - seaIceTf / tUppr); + double e2 = cVol * (tLowr - seaIceTf) - bulkLHFusionIce; + + // snow + hs += snowfall[i] / Ice::rhoSnow * dt; + // double accumulatedSnowThickness = snowfall[i] / Ice::rhoSnow * dt; + + // sublimation + // 4 cases + const double& subli = subl[i]; + double deltaSnow = subli * dt / Ice::rhoSnow; + double deltaIce1 = (deltaSnow - hs) * Ice::rhoSnow / Ice::rho; + double deltaIce2 = deltaIce1 - h1; + if (deltaSnow <= hs) { + // sublimation is less than or equal to the mass of snow + hs -= deltaSnow; + } else if (deltaIce1 <= h1) { + // sublimation minus sublimed snow is less than or equal to half the + // ice thickness + h1 -= deltaIce1; + hs = 0; + } else if (deltaIce2 <= h2) { + // sublimation minus sublimed snow is greater than half the ice + // thickness, but not all of it + h2 -= deltaIce2; + h1 = 0; + hs = 0; + } else { + // the snow and ice sublimates + // double oceanEvapError = (deltaIce2 - h2) * Ice::rho / Water::rhoOcean; + // TODO: log the error + h2 = 0; + h1 = 0; + hs = 0; + } + // Sublimated ice counts as top melt + topMelt[i] = Utils::max(0., h1 + h2 - hi); // (23) + + // Bottom melt/freezing + double meltBottom = (qio[i] - 4 * Ice::kappa * (tBott - tLowr) / hi) * dt; + snowMelt[i] = 0; + if (meltBottom <= 0.) { + // Freezing + // Eq. (25) - but I 've multiplied them with \rho_i (hence cVol), because it's + // missing in the paper + double eBot = cVol * (tBott - seaIceTf) - bulkLHFusionIce; + deltaIce2 = meltBottom / eBot; // (24) + tLowr = (deltaIce2 * tBott + h2 * tLowr) / (deltaIce2 + h2); // (26) + h2 += deltaIce2; + } else { + // Melting + // Eqs. (31)-(32) with added division with \rho_i (and \rho_s for 32) + deltaIce2 = -Utils::min(-meltBottom / e2, h2); + deltaIce1 = -Utils::min(Utils::max(-(meltBottom + e2 * h2) / e1, 0.), h1); + snowMelt[i] = -Utils::min( + Utils::max((meltBottom + e2 * h2 + e1 * h1) / bulkLHFusionSnow, 0.), hs); + + // If everything melts we need to put heat back into the ocean + if (h2 + h1 + hs - deltaIce2 - deltaIce1 - snowMelt[i] <= 0.) { + // (34) - with added multiplication of rhoi and rhos and division with dt + qow[i] -= cice[i] + * Utils::max(meltBottom - bulkLHFusionSnow * hs + e1 * h1 + e2 * h2, 0.) / dt; + } + + hs += snowMelt[i]; + h1 += deltaIce1; + h2 += deltaIce2; + botMelt[i] += deltaIce1 + deltaIce2; + } + + // Melting at the surface + // Do we really need an assertion here? + // assert(surfMelt >= 0); + // Eqs. (27)-(29) with division of \rho_i and \rho_s + snowMelt[i] -= Utils::min(surfMelt * dt / bulkLHFusionSnow, hs); + deltaIce1 = -Utils::min(Utils::max(-(surfMelt * dt - bulkLHFusionSnow * hs) / e1, 0.), h1); + deltaIce2 = -Utils::min( + Utils::max(-(surfMelt * dt - bulkLHFusionSnow * hs + e1 * h1) / e2, 0.), h2); + + // If everything melts we need to put heat back into the ocean + // Eq (30) - with multiplication of rhoi and rhos and division with dt + if (h2 + h1 + hs - deltaIce2 - deltaIce1 - snowMelt[i] <= 0.) { + qow[i] -= cice[i] + * Utils::max(surfMelt * dt - bulkLHFusionSnow * hs + e1 * h1 + e2 * h2, 0.) / dt; + } + + hs += snowMelt[i]; + h1 += deltaIce1; + h2 += deltaIce2; + topMelt[i] += deltaIce1 + deltaIce2; + + // Snow to ice conversion + double freeboard + = (hi * (Water::rhoOcean - Ice::rho) - hs * Ice::rhoSnow) / Water::rhoOcean; + if (doFlooding && freeboard < 0.) { + hs += Utils::min(freeboard * Ice::rho / Ice::rhoSnow, 0.); // (35) using += + deltaIce1 = Utils::max(-freeboard, 0.); // (36) + double f1 = 1 - deltaIce1 / (deltaIce1 + h1); // Fraction of new ice in the upper layer + double tBar = f1 * (tUppr + dHfTf_cp / tUppr) + (1 - f1) * seaIceTf; // (39) + tUppr = (tBar - Utils::sqrt(tBar * tBar - 4 * dHfTf_cp)) / 2; // (38) + h1 += deltaIce1; + snowToIce[i] += deltaIce1; + } + + // Add up the half-layer thicknesses + hi = h1 + h2; + // Adjust the temperatures to evenly divide the ice + if (h2 > h1) { + // Lower layer ice is added to the upper layer + double f1 = h1 / hi * 2; + double tBar = f1 * (tUppr + dHfTf_cp / tUppr) + (1 - f1) * tLowr; // (39) + // The upper layer temperature changes + tUppr = (tBar - Utils::sqrt(tBar * tBar - 4 * dHfTf_cp)) / 2; // (38) + } else { + // Upper layer ice is added to the lower layer + double f1 = (2 * h1 - hi) / hi; + // Lower layer temperature changes + tLowr = f1 * (tUppr + dHfTf_cp / tUppr) + (1 - f1) * tLowr; // (40) + // Melt from top and bottom if the lower layer temperature is too high + if (tLowr > seaIceTf) { + double deltaMelt = hi / 4 * Ice::cp * (tLowr - seaIceTf) * tUppr + / (Ice::Lf * tUppr + (Ice::cp * tUppr - Ice::Lf) * (seaIceTf - tUppr)); + topMelt[i] -= deltaMelt; + botMelt[i] -= deltaMelt; + hi -= 2 * deltaMelt; + tLowr = seaIceTf; + } + } + deltaHi[i] = hi - oldHi; + + // Remove very small ice thickness + // The fluxes are already sorted + if (hi < hMin) { + if (deltaHi[i] < 0) { + topMelt[i] *= oldHi / deltaHi[i]; + botMelt[i] *= oldHi / deltaHi[i]; + } + snowToIce[i] = 0; + + deltaHi[i] = -oldHi; + cice[i] = 0; + + tSurf = seaIceTf; + tUppr = seaIceTf; + tLowr = seaIceTf; + } + + tsurf[i] = tSurf; + tInternal[i] = tUppr; + tBottom[i] = tLowr; + + hice[i] = hi * cice[i]; + hsnow[i] = hs * cice[i]; + }); + timerUpdate.stop(); + timer.stop(); } } /* namespace Nextsim */ diff --git a/physics/src/modules/IceThermodynamicsModule/include/ThermoIce0.hpp b/physics/src/modules/IceThermodynamicsModule/include/ThermoIce0.hpp index 0599d308e..b2077562c 100644 --- a/physics/src/modules/IceThermodynamicsModule/include/ThermoIce0.hpp +++ b/physics/src/modules/IceThermodynamicsModule/include/ThermoIce0.hpp @@ -8,7 +8,7 @@ #include "include/Configured.hpp" #include "include/IIceThermodynamics.hpp" -#include "include/ModelArrayRef.hpp" + namespace Nextsim { //! A class implementing IIceThermodynamics as the ThermoIce0 model. @@ -33,12 +33,18 @@ class ThermoIce0 : public IIceThermodynamics, public Configured { void update(const TimestepTime& tsTime) override; private: - void calculateElement(size_t i, const TimestepTime& tst); + // local namespace to prevent conflicts with other thermodynamics implementations + struct Private { + static inline constexpr TextTag T_SNOW_MELT = "T_SNOW_MELT"; + static inline constexpr TextTag T_TOP_MELT = "T_TOP_MELT"; + static inline constexpr TextTag T_BOT_MELT = "T_BOT_MELT"; + }; + // private owned + ModelArrayAccessor snowMeltAccessor; + ModelArrayAccessor topMeltAccessor; + ModelArrayAccessor botMeltAccessor; - HField snowMelt; - HField topMelt; - HField botMelt; - HField qic; + ModelArrayAccessor qicAccessor; static const double freezingPointIce; static double kappa_s; diff --git a/physics/src/modules/IceThermodynamicsModule/include/ThermoWinton.hpp b/physics/src/modules/IceThermodynamicsModule/include/ThermoWinton.hpp index 7ac8d05d4..0048e8098 100644 --- a/physics/src/modules/IceThermodynamicsModule/include/ThermoWinton.hpp +++ b/physics/src/modules/IceThermodynamicsModule/include/ThermoWinton.hpp @@ -8,7 +8,6 @@ #include "include/Configured.hpp" #include "include/IIceThermodynamics.hpp" -#include "include/ModelArrayRef.hpp" namespace Nextsim { @@ -41,23 +40,28 @@ class ThermoWinton : public IIceThermodynamics, public Configured static const std::string tBottomName; private: - void calculateElement(size_t i, const TimestepTime& tst); + // local namespace to prevent conflicts with other thermodynamics implementations + struct Private { + static inline constexpr TextTag T_INTERNAL = "T_INTERNAL"; + static inline constexpr TextTag T_BOTTOM = "T_BOTTOM"; + static inline constexpr TextTag T_SNOW_MELT = "T_SNOW_MELT"; + static inline constexpr TextTag T_TOP_MELT = "T_TOP_MELT"; + static inline constexpr TextTag T_BOT_MELT = "T_BOT_MELT"; + }; + // private owned + ModelArrayAccessor tInternalAccessor; + ModelArrayAccessor tBottomAccessor; + ModelArrayAccessor snowMeltAccessor; + ModelArrayAccessor topMeltAccessor; + ModelArrayAccessor botMeltAccessor; - AdvectedField tInternal; - AdvectedField tBottom; - HField snowMelt; - HField topMelt; - HField botMelt; - ModelArrayRef sw_in; - ModelArrayRef subl; + ModelArrayAccessor sw_inAccessor; + ModelArrayAccessor sublAccessor; static const double cVol; static bool doFlooding; static const double seaIceTf; static double kappa_s; - - void calculateTemps( - double& tSurf, double& tMidt, double& tBotn, double& mSurf, size_t i, double dt); }; } /* namespace Nextsim */ diff --git a/physics/src/modules/LateralIceSpreadModule/HiblerSpread.cpp b/physics/src/modules/LateralIceSpreadModule/HiblerSpread.cpp index 6b15e3cc4..8aafc10aa 100644 --- a/physics/src/modules/LateralIceSpreadModule/HiblerSpread.cpp +++ b/physics/src/modules/LateralIceSpreadModule/HiblerSpread.cpp @@ -1,8 +1,11 @@ /*! * @author Tim Spain + * @author Robert Jendersie */ #include "include/HiblerSpread.hpp" +#include "kokkos/include/KokkosTimer.hpp" +#include "include/KernelAlternatives.hpp" #include "include/IceMinima.hpp" #include "include/constants.hpp" @@ -51,9 +54,24 @@ HiblerSpread::HelpMap& HiblerSpread::getHelpRecursive(HelpMap& map, bool getAll) return getHelpText(map, getAll); } -double HiblerSpread::freeze(const double newIce) { return newIce / h0; } +/*! + * Updates the freezing of open water for the timestep. + * + * @param tStep The object containing the timestep start and duration times. + * @param newIce The positive change in ice thickness this timestep. + * @param deltaCFreeze The change in concentration due to freezing. + */ +KERNEL_IMPL_FUNCTION double freeze(const double newIce, const double h0) { return newIce / h0; } -double HiblerSpread::melt(const double deltaHi, const double cice, const double hice) +/*! + * Updates the lateral melting of ice for the timestep. + * + * @param tStep The object containing the timestep start and duration times. + * @param deltaHi The change in ice thickness this timestep. + * @param cice The ice concentration. + * @param hice The ice-average ice thickness. + */ +KERNEL_IMPL_FUNCTION double melt(double deltaHi, double cice, double hice, double phiM) { /* We only decrease the concentration if the ice is melting, if the ice cover is not 100%, and * if there's ice there in the first place. */ @@ -65,61 +83,91 @@ double HiblerSpread::melt(const double deltaHi, const double cice, const double } } -void HiblerSpread::newIceFormation(size_t i, const TimestepTime& tst) -{ - // Flux cooling the ocean from open water - // TODO Add assimilation fluxes here - double coolingFlux = qow[i]; - // Temperature change of the mixed layer during this timestep - double deltaTml = -coolingFlux / mixedLayerBulkHeatCapacity[i] * tst.step; - // Initial temperature - double t0 = sst[i]; - // Freezing point temperature - double tf0 = tf[i]; - // Final temperature - double t1 = t0 + deltaTml; - - // deal with cooling below the freezing point - if (t1 < tf0) { - // Heat lost cooling the mixed layer to freezing point - double sensibleFlux = (tf0 - t0) / deltaTml * coolingFlux; - // Any heat beyond that is latent heat forming new ice - double latentFlux = coolingFlux - sensibleFlux; - - qow[i] = sensibleFlux; - newice[i] = latentFlux * tst.step * (1 - cice[i]) / (Ice::Lf * Ice::rho); - } else { - newice[i] = 0; - } -} - -void HiblerSpread::lateralIceSpread(size_t i, const TimestepTime& tstep) +void HiblerSpread::update(const TimestepTime& tstep) { - const double deltaCMelt = melt(deltaHi[i], cice[i], hice[i]); - const double deltaCFreeze = freeze(newice[i]); - - deltaCIce[i] = deltaCFreeze + deltaCMelt; - cice[i] = (hice[i] > 0 || newice[i] > 0) ? cice[i] + deltaCIce[i] : 0; - if (cice[i] >= IceMinima::c()) { - // The updated ice thickness must conserve volume - hice[i] += newice[i]; - if (deltaCIce[i] < 0) { - /* Snow is lost if the concentration decreases, and energy is returned to the ocean. - * We reduce the snow volume by a "slice" of snow with the dimensions hs * deltaCIce. */ - const double hs = hsnow[i] / (cice[i] - deltaCIce[i]); - qow[i] -= deltaCIce[i] * hs * Water::Lf * Ice::rhoSnow / tstep.step; - hsnow[i] += hs * deltaCIce[i]; - } // else: Snow volume is conserved, so no change to hsnow[i] - } -} - -void HiblerSpread::applyLimits(size_t i, const TimestepTime& tstep) -{ - if (cice[i] < IceMinima::c() || hice[i] < IceMinima::h()) { - qow[i] += Water::Lf * (hice[i] * Ice::rho + hsnow[i] * Ice::rhoSnow) / tstep.step; - hice[i] = 0; - cice[i] = 0; - hsnow[i] = 0; - } + static KokkosTimer timer("HiblerSpread"); + static KokkosTimer timerCopy("HiblerSpreadCopy"); + static KokkosTimer timerUpdate("HiblerSpreadUpdate"); + timer.start(); + + timerCopy.start(); + auto execSpace = DefaultExecutionSpace(); + auto& hsnow = hsnowAccessor.getAutoRW(execSpace); + auto& qow = qowAccessor.getAutoRW(execSpace); + auto& cice = ciceAccessor.getAutoRW(execSpace); + auto& newice = newiceAccessor.getAutoRW(execSpace); + auto& hice = hiceAccessor.getAutoRW(execSpace); + auto& deltaCIce = deltaCIceAccessor.getAutoRW(execSpace); + const auto& mixedLayerBulkHeatCapacity = mixedLayerBulkHeatCapacityAccessor.getAutoRO(execSpace); + const auto& deltaHi = deltaHiAccessor.getAutoRO(execSpace); + const auto& tf = tfAccessor.getAutoRO(execSpace); + const auto& sst = sstAccessor.getAutoRO(execSpace); + + // static members can not be captured directly + const double h0 = HiblerSpread::h0; + const double phiM = HiblerSpread::phiM; + const double dt = tstep.step.seconds(); + const double cMin = IceMinima::c(); + const double hMin = IceMinima::h(); + timerCopy.stop(); + + timerUpdate.start(); + overElementsAuto( + OVER_ELEMENTS_LAMBDA (const ElementIndex i) { + // newIceFormation + // Flux cooling the ocean from open water + // TODO Add assimilation fluxes here + double coolingFlux = qow[i]; + // Temperature change of the mixed layer during this timestep + double deltaTml = -coolingFlux / mixedLayerBulkHeatCapacity[i] * dt; + // Initial temperature + double t0 = sst[i]; + // Freezing point temperature + double tf0 = tf[i]; + // Final temperature + double t1 = t0 + deltaTml; + + // deal with cooling below the freezing point + if (t1 < tf0) { + // Heat lost cooling the mixed layer to freezing point + double sensibleFlux = (tf0 - t0) / deltaTml * coolingFlux; + // Any heat beyond that is latent heat forming new ice + double latentFlux = coolingFlux - sensibleFlux; + + qow[i] = sensibleFlux; + newice[i] = latentFlux * dt * (1 - cice[i]) / (Ice::Lf * Ice::rho); + } else { + newice[i] = 0; + } + + // lateralIceSpread + const double deltaCMelt = melt(deltaHi[i], cice[i], hice[i], phiM); + const double deltaCFreeze = freeze(newice[i], h0); + + deltaCIce[i] = deltaCFreeze + deltaCMelt; + cice[i] = (hice[i] > 0 || newice[i] > 0) ? cice[i] + deltaCIce[i] : 0; + if (cice[i] >= cMin) { + // The updated ice thickness must conserve volume + hice[i] += newice[i]; + if (deltaCIce[i] < 0) { + /* Snow is lost if the concentration decreases, and energy is returned + * to the ocean. We reduce the snow volume by a "slice" of snow with the + * dimensions hs * deltaCIce. */ + const double hs = hsnow[i] / (cice[i] - deltaCIce[i]); + qow[i] -= deltaCIce[i] * hs * Water::Lf * Ice::rhoSnow / dt; + hsnow[i] += hs * deltaCIce[i]; + } // else: Snow volume is conserved, so no change to hsnow[i] + } + + // applyLimits + if (cice[i] < cMin || hice[i] < hMin) { + qow[i] += Water::Lf * (hice[i] * Ice::rho + hsnow[i] * Ice::rhoSnow) / dt; + hice[i] = 0; + cice[i] = 0; + hsnow[i] = 0; + } + }); + timerUpdate.stop(); + timer.stop(); } } diff --git a/physics/src/modules/LateralIceSpreadModule/include/HiblerSpread.hpp b/physics/src/modules/LateralIceSpreadModule/include/HiblerSpread.hpp index f74e98f7e..a66373eee 100644 --- a/physics/src/modules/LateralIceSpreadModule/include/HiblerSpread.hpp +++ b/physics/src/modules/LateralIceSpreadModule/include/HiblerSpread.hpp @@ -8,6 +8,8 @@ #include "include/Configured.hpp" #include "include/ILateralIceSpread.hpp" +#include "include/IceMinima.hpp" +#include "include/constants.hpp" namespace Nextsim { @@ -16,10 +18,10 @@ class HiblerSpread : public ILateralIceSpread, public Configured { public: HiblerSpread() : ILateralIceSpread() - , hice(getStore()) - , mixedLayerBulkHeatCapacity(getStore()) - , sst(getStore()) - , tf(getStore()) + , hiceAccessor(getStore()) + , mixedLayerBulkHeatCapacityAccessor(getStore()) + , sstAccessor(getStore()) + , tfAccessor(getStore()) { } virtual ~HiblerSpread() = default; @@ -37,50 +39,16 @@ class HiblerSpread : public ILateralIceSpread, public Configured { static HelpMap& getHelpText(HelpMap& map, bool getAll); static HelpMap& getHelpRecursive(HelpMap&, bool getAll); - void update(const TimestepTime& tstep) override - { - overElements( - [this](size_t i, const TimestepTime& tst) { this->updateWrapper(i, tst); }, tstep); - } - + void update(const TimestepTime& tstep) override; private: - void updateWrapper(size_t i, const TimestepTime& tst) - { - newIceFormation(i, tst); - lateralIceSpread(i, tst); - applyLimits(i, tst); - } - - /*! - * Updates the freezing of open water for the timestep. - * - * @param tStep The object containing the timestep start and duration times. - * @param newIce The positive change in ice thickness this timestep. - * @param deltaCFreeze The change in concentration due to freezing. - */ - double freeze(double newIce); - - /*! - * Updates the lateral melting of ice for the timestep. - * - * @param tStep The object containing the timestep start and duration times. - * @param deltaHi The change in ice thickness this timestep. - * @param cice The ice concentration. - * @param hice The ice-average ice thickness. - */ - double melt(double deltaHi, double cice, double hice); - void newIceFormation(size_t i, const TimestepTime& tst); - void lateralIceSpread(size_t i, const TimestepTime& tstep); - void applyLimits(size_t i, const TimestepTime& tstep); - static double h0; static double phiM; - ModelArrayRef - mixedLayerBulkHeatCapacity; // J K⁻¹ m⁻², from atmospheric state - ModelArrayRef sst; // sea surface temperature, ˚C - ModelArrayRef tf; // ocean freezing point, ˚C - ModelArrayRef hice; // Timestep initial true ice thickness, m + ModelArrayAccessor + mixedLayerBulkHeatCapacityAccessor; // J K⁻¹ m⁻², from atmospheric state + ModelArrayAccessor sstAccessor; // sea surface temperature, ˚C + ModelArrayAccessor tfAccessor; // ocean freezing point, ˚C + ModelArrayAccessor hiceAccessor; // Timestep initial true ice thickness, m }; } diff --git a/physics/src/modules/OceanBoundaryModule/BenchmarkOcean.cpp b/physics/src/modules/OceanBoundaryModule/BenchmarkOcean.cpp index fff440d56..6507890b6 100644 --- a/physics/src/modules/OceanBoundaryModule/BenchmarkOcean.cpp +++ b/physics/src/modules/OceanBoundaryModule/BenchmarkOcean.cpp @@ -15,13 +15,14 @@ void BenchmarkOcean::setData(const ModelState::DataMap& ms) BenchmarkCoordinates::setData(); // The material parameters of the ocean are fixed to ensure no // thermodynamics occurs - sst = -1.; - sss = 32; - tf = -1.8; + sstAccessor.getHostRW() = -1.; + sssAccessor.getHostRW() = 32; + tfAccessor.getHostRW() = -1.8; + ModelArray& mld = mldAccessor.getHostRW(); mld = 10.; - cpml = Water::rho * Water::cp * mld[0]; - qio = 0; - ssh = 0.; + cpmlAccessor.getHostRW() = Water::rho * Water::cp * mld[0]; + qioAccessor.getHostRW() = 0; + sshAccessor.getHostRW() = 0.; // The time and length scales of the current generation function // constexpr double L = 512000.; // Size of the domain in km @@ -29,8 +30,8 @@ void BenchmarkOcean::setData(const ModelState::DataMap& ms) // constexpr double T = 8 * 86400; // 8 days in seconds // The currents are constant wrt time and space - u = vMaxOcean * (2 * BenchmarkCoordinates::fy() - 1); - v = vMaxOcean * (1 - 2 * BenchmarkCoordinates::fx()); + uAccessor.getHostRW() = vMaxOcean * (2 * BenchmarkCoordinates::fy() - 1); + vAccessor.getHostRW() = vMaxOcean * (1 - 2 * BenchmarkCoordinates::fx()); } } /* namespace Nextsim */ diff --git a/physics/src/modules/OceanBoundaryModule/ConfiguredOcean.cpp b/physics/src/modules/OceanBoundaryModule/ConfiguredOcean.cpp index b0a687310..c7e408212 100644 --- a/physics/src/modules/OceanBoundaryModule/ConfiguredOcean.cpp +++ b/physics/src/modules/OceanBoundaryModule/ConfiguredOcean.cpp @@ -7,7 +7,6 @@ #include "include/Finalizer.hpp" #include "include/IFreezingPoint.hpp" #include "include/IIceOceanHeatFlux.hpp" -#include "include/ModelArrayRef.hpp" #include "include/NextsimModule.hpp" #include "include/constants.hpp" @@ -35,8 +34,10 @@ static const std::map keyMap = { }; ConfiguredOcean::ConfiguredOcean() - : sstExt(ModelArray::Type::H) - , sssExt(ModelArray::Type::H) + : sstExtAccessor(getStore(), RO, ModelArray::Type::H) + , sssExtAccessor(getStore(), RO, ModelArray::Type::H) + , sstSlabAccessor(getStore()) + , sssSlabAccessor(getStore()) , slabOcean(m_couplingArrays) { } @@ -72,8 +73,6 @@ void ConfiguredOcean::configure() v0 = Configured::getConfiguration(keyMap.at(CURRENTV_KEY), v0); // set the external SS* arrays as part of configuration, as opposed to at construction as normal - getStore().registerArray(Protected::EXT_SST, &sstExt, RO); - getStore().registerArray(Protected::EXT_SSS, &sssExt, RO); slabOcean.configure(); @@ -106,20 +105,23 @@ ModelState ConfiguredOcean::getStateDiagnostic() const void ConfiguredOcean::setData(const ModelState::DataMap& ms) { IOceanBoundary::setData(ms); + HField& sstExt = sstExtAccessor.getHostRW(); sstExt.resize(); + HField& sssExt = sssExtAccessor.getHostRW(); sssExt.resize(); sstExt = sst0; sssExt = sss0; + HField& mld = mldAccessor.getHostRW(); mld = mld0; - u = u0; - v = v0; - tf = Module::getImplementation()(sssExt[0]); - cpml = Water::rho * Water::cp * mld[0]; + uAccessor.getHostRW() = u0; + vAccessor.getHostRW() = v0; + tfAccessor.getHostRW() = Module::getImplementation()(sssExt[0]); + cpmlAccessor.getHostRW() = Water::rho * Water::cp * mld[0]; /* It's only the SSH gradient which has an effect, so being able to set a constant SSH is * useless. */ - ssh = 0.; + sshAccessor.getHostRW() = 0.; slabOcean.setData(ms); @@ -135,7 +137,7 @@ void ConfiguredOcean::updateAfter(const TimestepTime& tst) { mergeFluxes(tst); slabOcean.update(tst); - sst = ModelArrayRef(getStore()); - sss = ModelArrayRef(getStore()); + sstAccessor.getHostRW().component() = sstSlabAccessor.getHostRO().component(); + sssAccessor.getHostRW().component() = sssSlabAccessor.getHostRO().component(); } } /* namespace Nextsim */ diff --git a/physics/src/modules/OceanBoundaryModule/ConstantOceanBoundary.cpp b/physics/src/modules/OceanBoundaryModule/ConstantOceanBoundary.cpp index 5a531c295..cd6abaccd 100644 --- a/physics/src/modules/OceanBoundaryModule/ConstantOceanBoundary.cpp +++ b/physics/src/modules/OceanBoundaryModule/ConstantOceanBoundary.cpp @@ -21,16 +21,17 @@ void ConstantOceanBoundary::setData(const ModelState::DataMap& ms) IOceanBoundary::setData(ms); // Directly set the array values - sss = 32.; - u = 0; - v = 0; + sssAccessor.getHostRW() = 32.; + uAccessor.getHostRW() = 0; + vAccessor.getHostRW() = 0; + HField& mld = mldAccessor.getHostRW(); mld = 10.; double tf32 = -1.751; // Hand calculated from S = 32 using UNESCO - tf = tf32; - sst = tf32; // Tf == SST ensures that there is no ice-ocean heat flux - cpml = Water::cp * Water::rho * mld; - qio = 0.; - ssh = 0.; + tfAccessor.getHostRW() = tf32; + sstAccessor.getHostRW() = tf32; // Tf == SST ensures that there is no ice-ocean heat flux + cpmlAccessor.getHostRW() = Water::cp * Water::rho * mld; + qioAccessor.getHostRW() = 0.; + sshAccessor.getHostRW() = 0.; } void ConstantOceanBoundary::updateBefore(const TimestepTime& tst) diff --git a/physics/src/modules/OceanBoundaryModule/FluxConfiguredOcean.cpp b/physics/src/modules/OceanBoundaryModule/FluxConfiguredOcean.cpp index 3709892fd..21c70cbe4 100644 --- a/physics/src/modules/OceanBoundaryModule/FluxConfiguredOcean.cpp +++ b/physics/src/modules/OceanBoundaryModule/FluxConfiguredOcean.cpp @@ -78,18 +78,20 @@ ConfigMap FluxConfiguredOcean::getConfiguration() const void FluxConfiguredOcean::setData(const ModelState::DataMap& ms) { IOceanBoundary::setData(ms); - qio = qio0; - sst = sst0; + qioAccessor.getHostRW() = qio0; + sstAccessor.getHostRW() = sst0; + HField& sss = sssAccessor.getHostRW(); sss = sss0; + HField& mld = mldAccessor.getHostRW(); mld = mld0; - u = u0; - v = v0; - tf = Module::getImplementation()(sss[0]); - cpml = Water::rho * Water::cp * mld[0]; + uAccessor.getHostRW() = u0; + vAccessor.getHostRW() = v0; + tfAccessor.getHostRW() = Module::getImplementation()(sss[0]); + cpmlAccessor.getHostRW() = Water::rho * Water::cp * mld[0]; /* It's only the SSH gradient which has an effect, so being able to set a constant SSH is * useless. */ - ssh = 0.; + sshAccessor.getHostRW() = 0.; } } /* namespace Nextsim */ diff --git a/physics/src/modules/OceanBoundaryModule/TOPAZOcean.cpp b/physics/src/modules/OceanBoundaryModule/TOPAZOcean.cpp index 045529ad1..d21f89b4a 100644 --- a/physics/src/modules/OceanBoundaryModule/TOPAZOcean.cpp +++ b/physics/src/modules/OceanBoundaryModule/TOPAZOcean.cpp @@ -22,8 +22,10 @@ static const std::map keyMap = { }; TOPAZOcean::TOPAZOcean() - : sstExt(ModelArray::Type::H, { -5, 50 }) - , sssExt(ModelArray::Type::H, { 0, 50 }) + : sstExtAccessor(getStore(), RO, ModelArray::Type::H, std::pair(-5.0, 50.0)) + , sssExtAccessor(getStore(), RO, ModelArray::Type::H, std::pair(0.0, 50.0)) + , sstSlabAccessor(getStore()) + , sssSlabAccessor(getStore()) , slabOcean(m_couplingArrays) { } @@ -52,9 +54,6 @@ void TOPAZOcean::configure() filePath = Configured::getConfiguration(keyMap.at(FILEPATH_KEY), std::string()); slabOcean.configure(); - - getStore().registerArray(Protected::EXT_SST, &sstExt, RO); - getStore().registerArray(Protected::EXT_SSS, &sssExt, RO); } ConfigMap TOPAZOcean::getConfiguration() const { return { { keyMap.at(FILEPATH_KEY), filePath } }; } @@ -64,19 +63,26 @@ void TOPAZOcean::updateBefore(const TimestepTime& tst) std::set forcings = { sstName, sssName, mldName, uName, vName, sshName }; ModelState state = ParaGridIO::readForcingTimeStatic(forcings, tst.start, filePath); - sstExt = state.data.at(sstName); - sssExt = state.data.at(sssName); + sstExtAccessor.getHostRW() = state.data.at(sstName); + sssExtAccessor.getHostRW() = state.data.at(sssName); + HField& mld = mldAccessor.getHostRW(); mld = state.data.at(mldName); - u = state.data.at(uName); - v = state.data.at(vName); + uAccessor.getHostRW() = state.data.at(uName); + vAccessor.getHostRW() = state.data.at(vName); + HField& ssh = sshAccessor.getHostRW(); if (state.data.count(sshName)) { ssh = state.data.at(sshName); } else { ssh = 0.; } - cpml = Water::rhoOcean * Water::cp * mld; - overElements([this](size_t i, const TimestepTime& tsTime) { this->updateTf(i, tsTime); }, tst); + cpmlAccessor.getHostRW() = Water::rhoOcean * Water::cp * mld; + + // Update the freezing point + const HField& sss = sssAccessor.getHostRO(); + HField& tf = tfAccessor.getHostRW(); + overElements( + [&](size_t i, const TimestepTime& tsTime) { tf[i] = (*pFreezingPoint)(sss[i]); }, tst); Module::getImplementation().update(tst); } @@ -85,8 +91,8 @@ void TOPAZOcean::updateAfter(const TimestepTime& tst) { mergeFluxes(tst); slabOcean.update(tst); - sst = ModelArrayRef(getStore()); - sss = ModelArrayRef(getStore()); + sstAccessor.getHostRW().component() = sstSlabAccessor.getHostRO().component(); + sssAccessor.getHostRW().component() = sssSlabAccessor.getHostRO().component(); try { checkFields(); @@ -113,7 +119,9 @@ void TOPAZOcean::setData(const ModelState::DataMap& ms) { IOceanBoundary::setData(ms); + HField& sstExt = sstExtAccessor.getHostRW(); sstExt.resize(); + HField& sssExt = sssExtAccessor.getHostRW(); sssExt.resize(); addChecks({ @@ -124,6 +132,7 @@ void TOPAZOcean::setData(const ModelState::DataMap& ms) slabOcean.setData(ms); } -void TOPAZOcean::updateTf(size_t i, const TimestepTime& tst) { tf[i] = (*pFreezingPoint)(sss[i]); } +// void TOPAZOcean::updateTf(size_t i, const TimestepTime& tst) { tf[i] = (*pFreezingPoint)(sss[i]); +// } } /* namespace Nextsim */ diff --git a/physics/src/modules/OceanBoundaryModule/UniformOcean.cpp b/physics/src/modules/OceanBoundaryModule/UniformOcean.cpp index 081bac873..55ad527cf 100644 --- a/physics/src/modules/OceanBoundaryModule/UniformOcean.cpp +++ b/physics/src/modules/OceanBoundaryModule/UniformOcean.cpp @@ -13,18 +13,20 @@ namespace Nextsim { void UniformOcean::setData(const ModelState::DataMap& ms) { IOceanBoundary::setData(ms); - sst = sst0; + sstAccessor.getHostRW() = sst0; + HField& sss = sssAccessor.getHostRW(); sss = sss0; + HField& mld = mldAccessor.getHostRW(); mld = mld0; - u = u0; - v = v0; - tf = Module::getImplementation()(sss[0]); - cpml = Water::rho * Water::cp * mld[0]; - qio = qio0; + uAccessor.getHostRW() = u0; + vAccessor.getHostRW() = v0; + tfAccessor.getHostRW() = Module::getImplementation()(sss[0]); + cpmlAccessor.getHostRW() = Water::rho * Water::cp * mld[0]; + qioAccessor.getHostRW() = qio0; /* It's only the SSH gradient which has an effect, so being able to set a constant SSH is * useless. */ - ssh = 0.; + sshAccessor.getHostRW() = 0.; } UniformOcean& UniformOcean::setSST(double sstIn) diff --git a/physics/src/modules/OceanBoundaryModule/include/ConfiguredOcean.hpp b/physics/src/modules/OceanBoundaryModule/include/ConfiguredOcean.hpp index 31b7fca3c..d8e2b6511 100644 --- a/physics/src/modules/OceanBoundaryModule/include/ConfiguredOcean.hpp +++ b/physics/src/modules/OceanBoundaryModule/include/ConfiguredOcean.hpp @@ -49,8 +49,11 @@ class ConfiguredOcean : public IOceanBoundary, public Configured sstExtAccessor; + ModelArrayAccessor sssExtAccessor; + + ModelArrayAccessor sstSlabAccessor; + ModelArrayAccessor sssSlabAccessor; // We need a slab ocean in this implementation SlabOcean slabOcean; diff --git a/physics/src/modules/OceanBoundaryModule/include/TOPAZOcean.hpp b/physics/src/modules/OceanBoundaryModule/include/TOPAZOcean.hpp index b228d9645..0478c21f1 100644 --- a/physics/src/modules/OceanBoundaryModule/include/TOPAZOcean.hpp +++ b/physics/src/modules/OceanBoundaryModule/include/TOPAZOcean.hpp @@ -49,8 +49,11 @@ class TOPAZOcean : public IOceanBoundary, public Configured { // be static. static std::string filePath; - HField sstExt; - HField sssExt; + ModelArrayAccessor sstExtAccessor; + ModelArrayAccessor sssExtAccessor; + + ModelArrayAccessor sstSlabAccessor; + ModelArrayAccessor sssSlabAccessor; SlabOcean slabOcean; diff --git a/physics/src/modules/include/IAtmosphereBoundary.hpp b/physics/src/modules/include/IAtmosphereBoundary.hpp index 90f3c0d4e..579e7c01d 100644 --- a/physics/src/modules/include/IAtmosphereBoundary.hpp +++ b/physics/src/modules/include/IAtmosphereBoundary.hpp @@ -4,7 +4,7 @@ */ #include "include/CheckingModelComponent.hpp" -#include "include/ModelArrayRef.hpp" +#include "include/ModelArrayAccessor.hpp" #include "include/Time.hpp" #ifndef IATMOSPHEREBOUNDARY_HPP @@ -16,55 +16,48 @@ namespace Nextsim { class IAtmosphereBoundary : public CheckingModelComponent { public: IAtmosphereBoundary() - : qia(ModelArray::Type::H, { -1e4, 1e4 }) - , dqia_dt(ModelArray::Type::H) - , qow(ModelArray::Type::H, { -1e4, 1e4 }) - , subl(ModelArray::Type::H) - , snow(ModelArray::Type::H, { 0, 1e-3 }) - , rain(ModelArray::Type::H, { 0, 1e-3 }) - , evap(ModelArray::Type::H, { -1e-3, 1e-3 }) - , uwind(ModelArray::Type::U, { -100, 100 }) - , vwind(ModelArray::Type::V, { -100, 100 }) - , penSW(ModelArray::Type::H) - , tauXOW(ModelArray::Type::H, { -10, 10 }) - , tauYOW(ModelArray::Type::H, { -10, 10 }) + : qiaAccessor(getStore(), RW, ModelArray::Type::H, std::pair(-1e4, 1e4)) + , dqia_dtAccessor(getStore(), RW, ModelArray::Type::H) + , qowAccessor(getStore(), RW, ModelArray::Type::H, std::pair(-1e4, 1e4)) + , sublAccessor(getStore(), RW, ModelArray::Type::H) + , snowAccessor(getStore(), RO, ModelArray::Type::H, std::pair(0.0, 1e-3)) + , rainAccessor(getStore(), RO, ModelArray::Type::H, std::pair(0.0, 1e-3)) + , evapAccessor(getStore(), RW, ModelArray::Type::H, std::pair(-1e-3, 1e-3)) + , uwindAccessor(getStore(), RO, ModelArray::Type::U, std::pair(-100.0, 100.0)) + , vwindAccessor(getStore(), RO, ModelArray::Type::V, std::pair(-100.0, 100.0)) + , penSWAccessor(getStore(), RW, ModelArray::Type::H) + , tauXOWAccessor(getStore(), RW, ModelArray::Type::H, std::pair(-10.0, 10.0)) + , tauYOWAccessor(getStore(), RW, ModelArray::Type::H, std::pair(-10.0, 10.0)) { - m_couplingArrays.registerArray(CouplingFields::SUBL, &subl, RW); - m_couplingArrays.registerArray(CouplingFields::SNOW, &snow, RW); - m_couplingArrays.registerArray(CouplingFields::RAIN, &rain, RW); - m_couplingArrays.registerArray(CouplingFields::EVAP, &evap, RW); - m_couplingArrays.registerArray(CouplingFields::WIND_U, &uwind, RW); - m_couplingArrays.registerArray(CouplingFields::WIND_V, &vwind, RW); - - getStore().registerArray(Shared::Q_IA, &qia, RW); - getStore().registerArray(Shared::DQIA_DT, &dqia_dt, RW); - getStore().registerArray(Shared::Q_OW, &qow, RW); - getStore().registerArray(Shared::SUBLIM, &subl, RW); - getStore().registerArray(Shared::OW_STRESS_X, &tauXOW, RW); - getStore().registerArray(Shared::OW_STRESS_Y, &tauYOW, RW); - getStore().registerArray(Protected::SNOW, &snow, RO); - getStore().registerArray(Shared::EVAP, &evap, RW); - getStore().registerArray(Shared::RAIN, &rain, RO); - getStore().registerArray(Protected::WIND_U, &uwind, RO); - getStore().registerArray(Protected::WIND_V, &vwind, RO); - getStore().registerArray(Shared::Q_PEN_SW, &penSW, RW); } virtual ~IAtmosphereBoundary() = default; std::string getName() const override { return "IAtmosphereBoundary"; } void setData(const ModelState::DataMap& ms) override { + HField& qia = qiaAccessor.getHostRW(); qia.resize(); + HField& dqia_dt = dqia_dtAccessor.getHostRW(); dqia_dt.resize(); + HField& qow = qowAccessor.getHostRW(); qow.resize(); + HField& subl = sublAccessor.getHostRW(); subl.resize(); + HField& snow = snowAccessor.getHostRW(); snow.resize(); + HField& rain = rainAccessor.getHostRW(); rain.resize(); + HField& evap = evapAccessor.getHostRW(); evap.resize(); + UField& uwind = uwindAccessor.getHostRW(); uwind.resize(); + VField& vwind = vwindAccessor.getHostRW(); vwind.resize(); + HField& penSW = penSWAccessor.getHostRW(); penSW.resize(); + HField& tauXOW = tauXOWAccessor.getHostRW(); tauXOW.resize(); + HField& tauYOW = tauYOWAccessor.getHostRW(); tauYOW.resize(); addChecks({ @@ -82,22 +75,24 @@ class IAtmosphereBoundary : public CheckingModelComponent { virtual void update(const TimestepTime& tst) { } protected: - ModelArrayReferenceStore& couplingArrays() { return m_couplingArrays; } + ModelArrayStore& couplingArrays() { return m_couplingArrays; } - HField qia; - HField dqia_dt; - HField qow; - HField subl; - HField snow; - HField rain; - HField evap; - UField uwind; - VField vwind; - HField penSW; - HField tauXOW; // x(east)-ward open ocean stress, Pa - HField tauYOW; // y(north)-ward open ocean stress, Pa + ModelArrayStore m_couplingArrays; - ModelArrayReferenceStore m_couplingArrays; + ModelArrayAccessor qiaAccessor; + ModelArrayAccessor dqia_dtAccessor; + ModelArrayAccessor qowAccessor; + ModelArrayAccessor sublAccessor; + ModelArrayAccessor snowAccessor; + ModelArrayAccessor rainAccessor; + ModelArrayAccessor evapAccessor; + ModelArrayAccessor uwindAccessor; + ModelArrayAccessor vwindAccessor; + ModelArrayAccessor penSWAccessor; + ModelArrayAccessor + tauXOWAccessor; // x(east)-ward open ocean stress, Pa + ModelArrayAccessor + tauYOWAccessor; // y(north)-ward open ocean stress, Pa }; } // namespace Nextsim diff --git a/physics/src/modules/include/IDamageHealing.hpp b/physics/src/modules/include/IDamageHealing.hpp index c8bc32832..986c48370 100644 --- a/physics/src/modules/include/IDamageHealing.hpp +++ b/physics/src/modules/include/IDamageHealing.hpp @@ -6,7 +6,7 @@ #ifndef IDAMAGEHEALING_HPP #define IDAMAGEHEALING_HPP -#include "include/ModelArrayRef.hpp" +#include "include/ModelArrayAccessor.hpp" #include "include/ModelComponent.hpp" namespace Nextsim { @@ -26,15 +26,15 @@ class IDamageHealing : public ModelComponent { protected: IDamageHealing() - : cice(getStore()) - , deltaCi(getStore()) - , damage(getStore()) + : ciceAccessor(getStore()) + , deltaCiAccessor(getStore()) + , damageAccessor(getStore()) { } - ModelArrayRef cice; // From prognostic data - ModelArrayRef deltaCi; // From LateralIceSpread - ModelArrayRef damage; // From prognostic data + ModelArrayAccessor ciceAccessor; // From prognostic data + ModelArrayAccessor deltaCiAccessor; // From LateralIceSpread + ModelArrayAccessor damageAccessor; // From prognostic data }; } /* namespace Nextsim */ diff --git a/physics/src/modules/include/IFluxCalculation.hpp b/physics/src/modules/include/IFluxCalculation.hpp index 1f2876c99..67237e111 100644 --- a/physics/src/modules/include/IFluxCalculation.hpp +++ b/physics/src/modules/include/IFluxCalculation.hpp @@ -7,7 +7,7 @@ #define IFLUXCALCULATION_HPP #include "include/Configured.hpp" -#include "include/ModelArrayRef.hpp" +#include "include/ModelArrayAccessor.hpp" #include "include/ModelComponent.hpp" #include "include/ModelState.hpp" @@ -16,16 +16,16 @@ namespace Nextsim { class IFluxCalculation : public ModelComponent { public: IFluxCalculation() - : qow(getStore()) - , evap(getStore()) - , subl(getStore()) - , qia(getStore()) - , penSW(getStore()) - , dqia_dt(getStore()) - , Q_sw_ow(getStore()) - , Q_sw_base(getStore()) - , tau_x_ow(getStore()) - , tau_y_ow(getStore()) + : qowAccessor(getStore()) + , evapAccessor(getStore()) + , sublAccessor(getStore()) + , qiaAccessor(getStore()) + , penSWAccessor(getStore()) + , dqia_dtAccessor(getStore()) + , Q_sw_owAccessor(getStore()) + , Q_sw_baseAccessor(getStore()) + , tau_x_owAccessor(getStore()) + , tau_y_owAccessor(getStore()) { } virtual ~IFluxCalculation() = default; @@ -44,16 +44,16 @@ class IFluxCalculation : public ModelComponent { protected: // All fluxes are positive upwards, including incident radiation fluxes // The flux fields are owned by IAtmosphereBoundary - ModelArrayRef qow; - ModelArrayRef evap; - ModelArrayRef subl; - ModelArrayRef qia; - ModelArrayRef penSW; - ModelArrayRef dqia_dt; - ModelArrayRef Q_sw_ow; - ModelArrayRef Q_sw_base; - ModelArrayRef tau_x_ow; - ModelArrayRef tau_y_ow; + ModelArrayAccessor qowAccessor; + ModelArrayAccessor evapAccessor; + ModelArrayAccessor sublAccessor; + ModelArrayAccessor qiaAccessor; + ModelArrayAccessor penSWAccessor; + ModelArrayAccessor dqia_dtAccessor; + ModelArrayAccessor Q_sw_owAccessor; + ModelArrayAccessor Q_sw_baseAccessor; + ModelArrayAccessor tau_x_owAccessor; + ModelArrayAccessor tau_y_owAccessor; }; } #endif /* IFLUXCALCULATION_HPP */ diff --git a/physics/src/modules/include/IIceOceanHeatFlux.hpp b/physics/src/modules/include/IIceOceanHeatFlux.hpp index 7bf936435..7ef71ec7c 100644 --- a/physics/src/modules/include/IIceOceanHeatFlux.hpp +++ b/physics/src/modules/include/IIceOceanHeatFlux.hpp @@ -7,7 +7,7 @@ #define IICEOCEANHEATFLUX_HPP #include "include/ModelArray.hpp" -#include "include/ModelArrayRef.hpp" +#include "include/ModelArrayAccessor.hpp" #include "include/ModelComponent.hpp" #include "include/Time.hpp" @@ -17,10 +17,10 @@ namespace Nextsim { class IIceOceanHeatFlux : public ModelComponent { public: IIceOceanHeatFlux() - : sst(getStore()) - , tf(getStore()) - , cice(getStore()) - , qio(getStore()) + : sstAccessor(getStore()) + , tfAccessor(getStore()) + , ciceAccessor(getStore()) + , qioAccessor(getStore()) { } virtual ~IIceOceanHeatFlux() = default; @@ -37,11 +37,11 @@ class IIceOceanHeatFlux : public ModelComponent { virtual void update(const TimestepTime&) = 0; protected: - ModelArrayRef sst; - ModelArrayRef tf; - ModelArrayRef cice; + ModelArrayAccessor sstAccessor; + ModelArrayAccessor tfAccessor; + ModelArrayAccessor ciceAccessor; - ModelArrayRef qio; + ModelArrayAccessor qioAccessor; }; } #endif /* IICEOCEANHEATFLUX_HPP_ */ diff --git a/physics/src/modules/include/IIceThermodynamics.hpp b/physics/src/modules/include/IIceThermodynamics.hpp index 4c9654db4..4004be70a 100644 --- a/physics/src/modules/include/IIceThermodynamics.hpp +++ b/physics/src/modules/include/IIceThermodynamics.hpp @@ -8,7 +8,7 @@ #include "include/ConfigurationHelp.hpp" #include "include/FieldAdvection.hpp" #include "include/ModelArray.hpp" -#include "include/ModelArrayRef.hpp" +#include "include/ModelArrayAccessor.hpp" #include "include/ModelArraySlice.hpp" #include "include/ModelComponent.hpp" #include "include/Slice.hpp" @@ -16,6 +16,11 @@ #include "include/gridNames.hpp" namespace Nextsim { + +namespace Private { + inline constexpr TextTag SNOW_TO_ICE = "SNOW_TO_ICE"; +} + //! An interface class to update the ice thermodynamics. class IIceThermodynamics : public ModelComponent { public: @@ -24,9 +29,11 @@ class IIceThermodynamics : public ModelComponent { std::string getName() const override { return "IceThermodynamics"; } void setData(const ModelState::DataMap& ms) override { + AdvectedField& tsurf = tsurfAccessor.getHostRW(); tsurf.resize(); + HField& deltaHi = deltaHiAccessor.getHostRW(); deltaHi.resize(); - snowToIce.resize(); + snowToIceAccessor.getHostRW().resize(); /* If the surface temperature is not in the restart file, then we simply set it to the * zero. It's a safe approximation, and it seems the user doesn't @@ -42,7 +49,7 @@ class IIceThermodynamics : public ModelComponent { ModelState getStatePrognostic() const override { return { { - { tsurfName, tsurf }, + { tsurfName, tsurfAccessor.getHostRO() }, }, getConfiguration() }; } @@ -50,8 +57,8 @@ class IIceThermodynamics : public ModelComponent { ModelState getStateDiagnostic() const override { ModelState state = { { - { "delta_H_ice", deltaHi }, - { "snow_to_ice", snowToIce }, + { "delta_H_ice", deltaHiAccessor.getHostRO() }, + { "snow_to_ice", snowToIceAccessor.getHostRO() }, }, getConfiguration() }; state.merge(getStatePrognostic()); @@ -66,55 +73,59 @@ class IIceThermodynamics : public ModelComponent { */ virtual void update(const TimestepTime& tsTime) { - FieldAdvection::advectField(tsurf, tsTime, minT, 0.); +#ifdef USE_KOKKOS + FieldAdvection::advectField(tsurfAccessor.getDeviceRW(), tsTime, minT, 0.); +#else + FieldAdvection::advectField(tsurfAccessor.getHostRW(), tsTime, minT, 0.); +#endif } inline static std::string getKappaSConfigKey() { return "nextsim_thermo.ks"; } protected: IIceThermodynamics() - : tsurf(ModelArray::AdvectionType) - , deltaHi(ModelArray::Type::H) - , snowToIce(ModelArray::Type::H) - , hice(getStore()) - , cice(getStore()) - , hsnow(getStore()) - , qic(getStore()) - , qio(getStore()) - , qow(getStore()) - , qia(getStore()) - , dQia_dt(getStore()) - , penSw(getStore()) - , sublim(getStore()) - , tf(getStore()) - , snowfall(getStore()) - , sss(getStore()) - , qswBase(getStore()) + : tsurfAccessor(getStore(), RO, ModelArray::AdvectionType) + , deltaHiAccessor(getStore(), RW, ModelArray::Type::H) + , snowToIceAccessor(getStore(), RO, ModelArray::Type::H) + , hiceAccessor(getStore()) + , ciceAccessor(getStore()) + , hsnowAccessor(getStore()) + // , qicAccessor(getStore()) + , qioAccessor(getStore()) + , qowAccessor(getStore()) + , qiaAccessor(getStore()) + , dQia_dtAccessor(getStore()) + , penSwAccessor(getStore()) + , sublimAccessor(getStore()) + , tfAccessor(getStore()) + , snowfallAccessor(getStore()) + , sssAccessor(getStore()) + , qswBaseAccessor(getStore()) { - getStore().registerArray(Shared::DELTA_HICE, &deltaHi, RW); - getStore().registerArray(Protected::T_SURF, &tsurf, RO); } - ModelArrayRef hice; // From PrognosticData - ModelArrayRef cice; // From PrognosticData - ModelArrayRef hsnow; // From PrognosticData - ModelArrayRef - qic; // From IceTemperature. Conductive heat flux to the ice surface. - ModelArrayRef qswBase; // Short-wave flux through the base of the ice - ModelArrayRef qio; // From FluxCalculation - ModelArrayRef qow; // From FluxCalculation - ModelArrayRef qia; // From FluxCalculation - ModelArrayRef dQia_dt; // From FluxCalculation - ModelArrayRef penSw; // From FluxCalculation - ModelArrayRef sublim; // From AtmosphereState - ModelArrayRef tf; // Sea water freezing temperature - ModelArrayRef snowfall; // From ExternalData - ModelArrayRef sss; // From ExternalData (possibly PrognosticData) + ModelArrayAccessor hiceAccessor; // From PrognosticData + ModelArrayAccessor ciceAccessor; // From PrognosticData + ModelArrayAccessor hsnowAccessor; // From PrognosticData + // Q_IC is owned by ThermoIce0 and currently not used + // ModelArrayAccessor + // qicAccessor; // From IceTemperature. Conductive heat flux to the ice surface. + ModelArrayAccessor + qswBaseAccessor; // Short-wave flux through the base of the ice + ModelArrayAccessor qioAccessor; // From FluxCalculation + ModelArrayAccessor qowAccessor; // From FluxCalculation + ModelArrayAccessor qiaAccessor; // From FluxCalculation + ModelArrayAccessor dQia_dtAccessor; // From FluxCalculation + ModelArrayAccessor penSwAccessor; // From FluxCalculation + ModelArrayAccessor sublimAccessor; // From AtmosphereState + ModelArrayAccessor tfAccessor; // Sea water freezing temperature + ModelArrayAccessor snowfallAccessor; // From ExternalData + ModelArrayAccessor sssAccessor; // From ExternalData (possibly PrognosticData) // Owned, shared arrays - AdvectedField tsurf; - HField deltaHi; + ModelArrayAccessor tsurfAccessor; + ModelArrayAccessor deltaHiAccessor; // Owned, Module-private arrays - HField snowToIce; + ModelArrayAccessor snowToIceAccessor; constexpr static double minT = -90.0; }; diff --git a/physics/src/modules/include/ILateralIceSpread.hpp b/physics/src/modules/include/ILateralIceSpread.hpp index d7fdb025f..19e94b07b 100644 --- a/physics/src/modules/include/ILateralIceSpread.hpp +++ b/physics/src/modules/include/ILateralIceSpread.hpp @@ -7,7 +7,7 @@ #ifndef ILATERALICESPREAD_HPP #define ILATERALICESPREAD_HPP -#include "include/ModelArrayRef.hpp" +#include "include/ModelArrayAccessor.hpp" #include "include/ModelComponent.hpp" #include "include/Time.hpp" @@ -20,8 +20,11 @@ class ILateralIceSpread : public ModelComponent { std::string getName() const override { return "LateralIceSpread"; } void setData(const ModelState::DataMap& ms) override { + HField& deltaCIce = deltaCIceAccessor.getHostRW(); deltaCIce.resize(); + HField& newice = newiceAccessor.getHostRW(); newice.resize(); + HField& snowMelt = snowMeltAccessor.getHostRW(); snowMelt.resize(); /* @@ -43,29 +46,28 @@ class ILateralIceSpread : public ModelComponent { protected: ILateralIceSpread() - : deltaCIce(ModelArray::Type::H) - , newice(ModelArray::Type::H) - , snowMelt(ModelArray::Type::H) - , cice(getStore()) - , deltaHi(getStore()) - , hice(getStore()) - , hsnow(getStore()) - , qow(getStore()) + : deltaCIceAccessor(getStore(), RW, ModelArray::Type::H) + , newiceAccessor(getStore(), RW, ModelArray::Type::H) + , snowMeltAccessor(getStore(), RW, ModelArray::Type::H) + , ciceAccessor(getStore()) + , deltaHiAccessor(getStore()) + , hiceAccessor(getStore()) + , hsnowAccessor(getStore()) + , qowAccessor(getStore()) { - getStore().registerArray(Shared::DELTA_CICE, &deltaCIce, RW); - getStore().registerArray(Shared::HSNOW_MELT, &snowMelt, RW); - getStore().registerArray(Shared::NEW_ICE, &newice, RW); } - HField deltaCIce; // Change in ice concentration - HField newice; // New ice over open water this timestep, m - HField snowMelt; // Ocean to snow transfer of freshwater kg m⁻² + ModelArrayAccessor deltaCIceAccessor; // Change in ice concentration + ModelArrayAccessor + newiceAccessor; // New ice over open water this timestep, m + ModelArrayAccessor + snowMeltAccessor; // Ocean to snow transfer of freshwater kg m⁻² - ModelArrayRef cice; // From IceGrowth - ModelArrayRef deltaHi; // From Vertical Ice Growth - ModelArrayRef hice; // From IceGrowth - ModelArrayRef hsnow; // From Ice Growth? - ModelArrayRef qow; // From FluxCalculation + ModelArrayAccessor ciceAccessor; // From IceGrowth + ModelArrayAccessor deltaHiAccessor; // From Vertical Ice Growth + ModelArrayAccessor hiceAccessor; // From IceGrowth + ModelArrayAccessor hsnowAccessor; // From Ice Growth? + ModelArrayAccessor qowAccessor; // From FluxCalculation }; } /* namespace Nextsim */ diff --git a/physics/src/modules/include/IOceanBoundary.hpp b/physics/src/modules/include/IOceanBoundary.hpp index 7aa5bdfb3..386cd6d93 100644 --- a/physics/src/modules/include/IOceanBoundary.hpp +++ b/physics/src/modules/include/IOceanBoundary.hpp @@ -7,6 +7,7 @@ #define IOCEANBOUNDARY_HPP #include "include/CheckingModelComponent.hpp" +#include "include/ModelArrayAccessor.hpp" #include "include/constants.hpp" #include "include/gridNames.hpp" @@ -16,83 +17,74 @@ namespace Nextsim { class IOceanBoundary : public CheckingModelComponent { public: IOceanBoundary() - : qio(ModelArray::Type::H, { -1e8, 1e8 }) - , sst(ModelArray::Type::H, { -5, 50 }) - , sss(ModelArray::Type::H, { 0, 50 }) - , mld(ModelArray::Type::H, { 1e-3, 12e3 }) - , cpml(ModelArray::Type::H, { 0, 1e11 }) - , tf(ModelArray::Type::H, { -5, 0 }) - , u(ModelArray::Type::H, { -1, 1 }) - , v(ModelArray::Type::H, { -1, 1 }) - , ssh(ModelArray::Type::H, { -10, 10 }) - , qNoSun(ModelArray::Type::H, { -1e6, 1e6 }) - , qswNet(ModelArray::Type::H, { -1e3, 1e3 }) - , fwFlux(ModelArray::Type::H) - , sFlux(ModelArray::Type::H) - , qswow(ModelArray::Type::H, { -1e3, 1e-6 }) - , qswBase(ModelArray::Type::H, { -1e3, 1e-6 }) - , tauX(ModelArray::Type::H, { -10, 10 }) - , tauY(ModelArray::Type::H, { -10, 10 }) - , cice(getStore()) - , evap(getStore()) - , rain(getStore()) - , newIce(getStore()) - , deltaHice(getStore()) - , deltaSmelt(getStore()) - , qow(getStore()) - , tauXIO(getStore()) - , tauYIO(getStore()) - , tauXOW(getStore()) - , tauYOW(getStore()) + : qioAccessor(getStore(), RW, ModelArray::Type::H, std::pair(-1e8, 1e8)) + , sstAccessor(getStore(), RO, ModelArray::Type::H, std::pair(-5.0, 50.0)) + , sssAccessor(getStore(), RO, ModelArray::Type::H, std::pair(0.0, 50.0)) + , mldAccessor(getStore(), RO, ModelArray::Type::H, std::pair(1e-3, 12e3)) + , cpmlAccessor(getStore(), RO, ModelArray::Type::H, std::pair(0.0, 1e11)) + , tfAccessor(getStore(), RO, ModelArray::Type::H, std::pair(-5.0, 0.0)) + , uAccessor(getStore(), RO, ModelArray::Type::H, std::pair(-1.0, 1.0)) + , vAccessor(getStore(), RO, ModelArray::Type::H, std::pair(-1.0, 1.0)) + , sshAccessor(getStore(), RO, ModelArray::Type::H, std::pair(-10.0, 10.0)) + , qNoSunAccessor(m_couplingArrays, RO, ModelArray::Type::H, std::pair(-1e6, 1e6)) + , qswNetAccessor(m_couplingArrays, RO, ModelArray::Type::H, std::pair(-1e3, 1e3)) + , fwFluxAccessor(m_couplingArrays, RO, ModelArray::Type::H) + , sFluxAccessor(m_couplingArrays, RO, ModelArray::Type::H) + , qswowAccessor(getStore(), RW, ModelArray::Type::H, std::pair(-1e3, 1e-6)) + , qswBaseAccessor(getStore(), RW, ModelArray::Type::H, std::pair(-1e3, 1e-6)) + , tauXAccessor(m_couplingArrays, RO, ModelArray::Type::H, std::pair(-10.0, 10.0)) + , tauYAccessor(m_couplingArrays, RO, ModelArray::Type::H, std::pair(-10.0, 10.0)) + , ciceAccessor(getStore()) + , evapAccessor(getStore()) + , rainAccessor(getStore()) + , newIceAccessor(getStore()) + , deltaHiceAccessor(getStore()) + , deltaSmeltAccessor(getStore()) + , qowAccessor(getStore()) + , tauXIOAccessor(getStore()) + , tauYIOAccessor(getStore()) + , tauXOWAccessor(getStore()) + , tauYOWAccessor(getStore()) { - // Receive - m_couplingArrays.registerArray(CouplingFields::MLD, &mld, RW); - m_couplingArrays.registerArray(CouplingFields::OCEAN_U, &u, RW); - m_couplingArrays.registerArray(CouplingFields::OCEAN_V, &v, RW); - m_couplingArrays.registerArray(CouplingFields::SSH, &ssh, RW); - m_couplingArrays.registerArray(CouplingFields::SSS, &sss, RW); - m_couplingArrays.registerArray(CouplingFields::SST, &sst, RW); - // Send - m_couplingArrays.registerArray(CouplingFields::FWFLUX, &fwFlux, RO); - m_couplingArrays.registerArray(CouplingFields::O_STRESS_X, &tauX, RO); - m_couplingArrays.registerArray(CouplingFields::O_STRESS_Y, &tauY, RO); - m_couplingArrays.registerArray(CouplingFields::Q_SS_NO_SW, &qNoSun, RO); - m_couplingArrays.registerArray(CouplingFields::Q_SS_SW, &qswNet, RO); - m_couplingArrays.registerArray(CouplingFields::SFLUX, &sFlux, RO); - - getStore().registerArray(Shared::Q_IO, &qio, RW); - getStore().registerArray(Protected::SST, &sst, RO); - getStore().registerArray(Protected::SSS, &sss, RO); - getStore().registerArray(Protected::MLD, &mld, RO); - getStore().registerArray(Protected::ML_BULK_CP, &cpml, RO); - getStore().registerArray(Protected::TF, &tf, RO); - getStore().registerArray(Protected::OCEAN_U, &u, RO); - getStore().registerArray(Protected::OCEAN_V, &v, RO); - getStore().registerArray(Protected::SSH, &ssh, RO); - getStore().registerArray(Shared::Q_SW_OW, &qswow, RW); - getStore().registerArray(Shared::Q_SW_BASE, &qswBase, RW); } virtual ~IOceanBoundary() = default; std::string getName() const override { return "IOceanBoundary"; } void setData(const ModelState::DataMap& ms) override { + HField& qio = qioAccessor.getHostRW(); qio.resize(); + HField& sst = sstAccessor.getHostRW(); sst.resize(); + HField& sss = sssAccessor.getHostRW(); sss.resize(); + HField& mld = mldAccessor.getHostRW(); mld.resize(); + HField& cpml = cpmlAccessor.getHostRW(); cpml.resize(); + HField& tf = tfAccessor.getHostRW(); tf.resize(); + UField& u = uAccessor.getHostRW(); u.resize(); + VField& v = vAccessor.getHostRW(); v.resize(); + HField& ssh = sshAccessor.getHostRW(); ssh.resize(); + HField& qNoSun = qNoSunAccessor.getHostRW(); qNoSun.resize(); + HField& qswNet = qswNetAccessor.getHostRW(); qswNet.resize(); + HField& fwFlux = fwFluxAccessor.getHostRW(); fwFlux.resize(); + HField& sFlux = sFluxAccessor.getHostRW(); sFlux.resize(); + HField& qswow = qswowAccessor.getHostRW(); qswow.resize(); + HField& qswBase = qswBaseAccessor.getHostRW(); qswBase.resize(); + HField& tauX = tauXAccessor.getHostRW(); tauX.resize(); + HField& tauY = tauYAccessor.getHostRW(); tauY.resize(); addChecks({ @@ -143,73 +135,103 @@ class IOceanBoundary : public CheckingModelComponent { */ void mergeFluxes(const TimestepTime& tst) { - dt = tst.step.seconds(); - overElements([this](const size_t i, - const TimestepTime& tsTime) { this->mergeFluxesElement(i, tsTime); }, + assert(m_couplingArrays.checkAllRegistered()); + + const double dt = tst.step.seconds(); + HField& fwFlux = fwFluxAccessor.getHostRW(); + HField& qNoSun = qNoSunAccessor.getHostRW(); + HField& tauX = tauXAccessor.getHostRW(); + HField& tauY = tauYAccessor.getHostRW(); + HField& sFlux = sFluxAccessor.getHostRW(); + HField& qswNet = qswNetAccessor.getHostRW(); + const HField& tauXIO = tauXIOAccessor.getHostRO(); + const HField& cice = ciceAccessor.getHostRO(); + const HField& deltaHice = deltaHiceAccessor.getHostRO(); + const HField& rain = rainAccessor.getHostRO(); + const HField& sss = sssAccessor.getHostRO(); + const HField& qswow = qswowAccessor.getHostRO(); + const HField& tauYIO = tauYIOAccessor.getHostRO(); + const HField& evap = evapAccessor.getHostRO(); + const HField& qio = qioAccessor.getHostRO(); + const HField& newIce = newIceAccessor.getHostRO(); + const HField& tauYOW = tauYOWAccessor.getHostRO(); + const HField& qow = qowAccessor.getHostRO(); + const HField& qswBase = qswBaseAccessor.getHostRO(); + const HField& deltaSmelt = deltaSmeltAccessor.getHostRO(); + const HField& tauXOW = tauXOWAccessor.getHostRO(); + + overElements( + [&](const size_t i, const TimestepTime& tsTime) { + // Heat fluxes - partitioned in solar and non-solar + qswNet[i] = cice[i] * qswBase[i] + (1 - cice[i]) * qswow[i]; + qNoSun[i] = cice[i] * qio[i] + (1 - cice[i]) * qow[i] - qswNet[i]; + + // Mass fluxes - fresh water and salt + // ice volume change, both laterally and vertically + const double deltaIceVol = newIce[i] + deltaHice[i] * cice[i]; + // change in snow volume due to melting (should be < 0) + const double meltSnowVol = deltaSmelt[i] * cice[i]; + // Effective ice salinity is always less than or equal to the SSS, and here we use + // the right units too + const double effectiveIceSal = 1e-3 * std::min(Ice::s, sss[i]); + + // Positive flux is up! + fwFlux[i] + = ((1 - effectiveIceSal) * Ice::rho * deltaIceVol + Ice::rhoSnow * meltSnowVol) + / dt + + (evap[i] - rain[i]) * (1 - cice[i]); + sFlux[i] = effectiveIceSal * Ice::rho * deltaIceVol / dt; + + // Momentum fluxes + tauX[i] = cice[i] * tauXIO[i] + (1 - cice[i]) * tauXOW[i]; + tauY[i] = cice[i] * tauYIO[i] + (1 - cice[i]) * tauYOW[i]; + }, tst); } private: - double dt; - - void mergeFluxesElement(size_t i, const TimestepTime& tst) - { - // Heat fluxes - partitioned in solar and non-solar - qswNet[i] = cice[i] * qswBase[i] + (1 - cice[i]) * qswow[i]; - qNoSun[i] = cice[i] * qio[i] + (1 - cice[i]) * qow[i] - qswNet[i]; - - // Mass fluxes - fresh water and salt - // ice volume change, both laterally and vertically - const double deltaIceVol = newIce[i] + deltaHice[i] * cice[i]; - // change in snow volume due to melting (should be < 0) - const double meltSnowVol = deltaSmelt[i] * cice[i]; - // Effective ice salinity is always less than or equal to the SSS, and here we use the right - // units too - const double effectiveIceSal = 1e-3 * std::min(Ice::s, sss[i]); - - // Positive flux is up! - fwFlux[i] - = ((1 - effectiveIceSal) * Ice::rho * deltaIceVol + Ice::rhoSnow * meltSnowVol) / dt - + (evap[i] - rain[i]) * (1 - cice[i]); - sFlux[i] = effectiveIceSal * Ice::rho * deltaIceVol / dt; - - // Momentum fluxes - tauX[i] = cice[i] * tauXIO[i] + (1 - cice[i]) * tauXOW[i]; - tauY[i] = cice[i] * tauYIO[i] + (1 - cice[i]) * tauYOW[i]; - } - protected: - HField qio; // Ice-ocean heat flux, W m⁻² - HField sst; // Coupled or slab ocean sea surface temperature, ˚C - HField sss; // Coupled or slab ocean sea surface salinity, PSU - HField mld; // Mixed layer or slab ocean depth m - HField tf; // Freezing point of the mixed layer, ˚C - HField cpml; // Heat capacity of the mixed layer, J K⁻¹ m² - UField u; // x(east)-ward ocean current, m s⁻¹ - VField v; // y(north)-ward ocean current, m s⁻¹ - HField ssh; // sea surface height, m - HField qNoSun; // Net surface ocean heat flux, except short wave, W m⁻² - HField qswNet; // Net surface ocean shortwave flux, W m⁻² - HField fwFlux; // Net surface ocean fresh-water flux, kg m⁻² - HField sFlux; // Net surface ocean salt flux, kg m⁻² - HField qswow; // Shortwave flux in open water W m⁻² - HField qswBase; // Shortwave flux at the base of the ice W m⁻² - HField tauX; // x(east)-ward total ocean stress, Pa - HField tauY; // y(north)-ward total ocean stress, Pa - - ModelArrayReferenceStore m_couplingArrays; - - ModelArrayRef cice; - ModelArrayRef tauXIO; - ModelArrayRef tauYIO; - ModelArrayRef evap; - ModelArrayRef rain; - ModelArrayRef newIce; - ModelArrayRef deltaHice; - ModelArrayRef deltaSmelt; - ModelArrayRef qow; - ModelArrayRef tauXOW; - ModelArrayRef tauYOW; + ModelArrayStore m_couplingArrays; + + ModelArrayAccessor qioAccessor; // Ice-ocean heat flux, W m⁻² + ModelArrayAccessor + sstAccessor; // Coupled or slab ocean sea surface temperature, ˚C + ModelArrayAccessor + sssAccessor; // Coupled or slab ocean sea surface salinity, PSU + ModelArrayAccessor mldAccessor; // Mixed layer or slab ocean depth m + ModelArrayAccessor tfAccessor; // Freezing point of the mixed layer, ˚C + ModelArrayAccessor + cpmlAccessor; // Heat capacity of the mixed layer, J K⁻¹ m² + ModelArrayAccessor uAccessor; // x(east)-ward ocean current, m s⁻¹ + ModelArrayAccessor vAccessor; // y(north)-ward ocean current, m s⁻¹ + ModelArrayAccessor sshAccessor; // sea surface height, m + ModelArrayAccessor + qNoSunAccessor; // Net surface ocean heat flux, except short wave, W m⁻² + ModelArrayAccessor + qswNetAccessor; // Net surface ocean shortwave flux, W m⁻² + ModelArrayAccessor + fwFluxAccessor; // Net surface ocean fresh-water flux, kg m⁻² + ModelArrayAccessor + sFluxAccessor; // Net surface ocean salt flux, kg m⁻² + ModelArrayAccessor qswowAccessor; // Shortwave flux in open water W m⁻² + ModelArrayAccessor + qswBaseAccessor; // Shortwave flux at the base of the ice W m⁻² + ModelArrayAccessor + tauXAccessor; // x(east)-ward total ocean stress, Pa + ModelArrayAccessor + tauYAccessor; // y(north)-ward total ocean stress, Pa + + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor tauXIOAccessor; + ModelArrayAccessor tauYIOAccessor; + ModelArrayAccessor evapAccessor; + ModelArrayAccessor rainAccessor; + ModelArrayAccessor newIceAccessor; + ModelArrayAccessor deltaHiceAccessor; + ModelArrayAccessor deltaSmeltAccessor; + ModelArrayAccessor qowAccessor; + ModelArrayAccessor tauXOWAccessor; + ModelArrayAccessor tauYOWAccessor; }; } /* namespace Nextsim */ diff --git a/physics/test/BasicIceOceanFlux_test.cpp b/physics/test/BasicIceOceanFlux_test.cpp index 4961b8ae7..da3e8bec1 100644 --- a/physics/test/BasicIceOceanFlux_test.cpp +++ b/physics/test/BasicIceOceanFlux_test.cpp @@ -34,31 +34,30 @@ TEST_CASE("Melting conditions") class ProgData : public ModelComponent { public: ProgData() + : hiceAccessor(getStore(), RW) + , ciceAccessor(getStore(), RW) + , hsnowAccessor(getStore(), RW) { - getStore().registerArray(Shared::H_ICE_DG, &hice, RW); - getStore().registerArray(Shared::C_ICE_DG, &cice, RW); - getStore().registerArray(Shared::H_SNOW_DG, &hsnow, RW); } std::string getName() const override { return "ProgData"; } void setData(const ModelState::DataMap&) override { noLandMask(); - cice[0] = 0.5; - hice[0] = 0.1; // Here we are using the cell-averaged thicknesses - hsnow[0] = 0.01; + ciceAccessor.getHostRW()[0] = 0.5; + hiceAccessor.getHostRW()[0] = 0.1; // Here we are using the cell-averaged thicknesses + hsnowAccessor.getHostRW()[0] = 0.01; } - HField hice; - HField cice; - HField hsnow; + ModelArrayAccessor hiceAccessor; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor hsnowAccessor; } iceState; iceState.setData(ModelState().data); - HField qio; + ModelArrayAccessor qioAccessor(ModelComponent::getStore()); + HField& qio = qioAccessor.getHostRW(); qio.resize(); - ModelComponent::getStore().registerArray(Shared::Q_IO, &qio, RW); - TimestepTime tst = { TimePoint("2000-001"), Duration("P0-0T0:10:0") }; BasicIceOceanHeatFlux biohf; biohf.update(tst); @@ -78,38 +77,37 @@ TEST_CASE("Freezing conditions") class ProgData : public ModelComponent { public: - ProgData() + ProgData() // RO before the Accessor refactor but these fields have to be RW + : hiceAccessor(getStore(), RW) // RO + , ciceAccessor(getStore(), RW) // RO + , hsnowAccessor(getStore(), RW) // RO { - getStore().registerArray(Shared::H_ICE_DG, &hice, RO); - getStore().registerArray(Shared::C_ICE_DG, &cice, RO); - getStore().registerArray(Shared::H_SNOW_DG, &hsnow, RO); } std::string getName() const override { return "ProgData"; } void setData(const ModelState::DataMap&) override { noLandMask(); - cice[0] = 0.5; - hice[0] = 0.1; // Here we are using the cell-averaged thicknesses - hsnow[0] = 0.01; + ciceAccessor.getHostRW()[0] = 0.5; + hiceAccessor.getHostRW()[0] = 0.1; // Here we are using the cell-averaged thicknesses + hsnowAccessor.getHostRW()[0] = 0.01; } - HField hice; - HField cice; - HField hsnow; + ModelArrayAccessor hiceAccessor; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor hsnowAccessor; } iceState; iceState.setData(ModelState().data); - HField qio; - qio.resize(); - ModelComponent::getStore().registerArray(Shared::Q_IO, &qio, RW); + ModelArrayAccessor qioAccessor(ModelComponent::getStore()); + qioAccessor.getHostRW().resize(); TimestepTime tst = { TimePoint("2000-001"), Duration("P0-0T0:10:0") }; BasicIceOceanHeatFlux biohf; biohf.update(tst); double prec = 1e-5; - REQUIRE(qio[0] == doctest::Approx(73.9465).epsilon(prec)); + REQUIRE(qioAccessor.getHostRO()[0] == doctest::Approx(73.9465).epsilon(prec)); } TEST_SUITE_END(); diff --git a/physics/test/BenchmarkBoundaries_test.cpp b/physics/test/BenchmarkBoundaries_test.cpp index a1145e622..dbdb77b27 100644 --- a/physics/test/BenchmarkBoundaries_test.cpp +++ b/physics/test/BenchmarkBoundaries_test.cpp @@ -27,8 +27,10 @@ TEST_CASE("OceanTest") benchOcean.setData(ModelState::DataMap()); // Get the u and v arrays - ModelArrayRef uOcean(ModelComponent::getStore()); - ModelArrayRef vOcean(ModelComponent::getStore()); + ModelArrayAccessor uOceanAccessor(ModelComponent::getStore()); + const UField& uOcean = uOceanAccessor.getHostRO(); + ModelArrayAccessor vOceanAccessor(ModelComponent::getStore()); + const VField& vOcean = vOceanAccessor.getHostRO(); // Check the wind at an arbitrary point lies in a reasonable range size_t iTest = 50; size_t jTest = 40; @@ -60,21 +62,21 @@ TEST_CASE("AtmosphereTest") benchAtm.update(tst); // Get the u and v arrays - ModelArrayRef uWind(ModelComponent::getStore()); - ModelArrayRef vWind(ModelComponent::getStore()); + ModelArrayAccessor uWindAccessor(ModelComponent::getStore()); + ModelArrayAccessor vWindAccessor(ModelComponent::getStore()); // Check the wind at an arbitrary point lies in a reasonable range size_t iTest = 50; size_t jTest = 40; - double uTest = uWind(iTest, jTest); - double vTest = vWind(iTest, jTest); + const double uTest = uWindAccessor.getHostRO()(iTest, jTest); + const double vTest = vWindAccessor.getHostRO()(iTest, jTest); REQUIRE(uTest != 0.); REQUIRE(vTest != 0.); // Check that the cyclone is moving away (weakening) from the test point tst.start += tst.step; benchAtm.update(tst); - REQUIRE(fabs(uWind(iTest, jTest) < fabs(uTest))); - REQUIRE(fabs(vWind(iTest, jTest) < fabs(vTest))); + REQUIRE(fabs(uWindAccessor.getHostRO()(iTest, jTest) < fabs(uTest))); + REQUIRE(fabs(vWindAccessor.getHostRO()(iTest, jTest) < fabs(vTest))); } } diff --git a/physics/test/CMakeLists.txt b/physics/test/CMakeLists.txt index af5e6cbe2..cbe4e0555 100644 --- a/physics/test/CMakeLists.txt +++ b/physics/test/CMakeLists.txt @@ -23,12 +23,12 @@ if(NOT ENABLE_MPI) target_compile_definitions(testTOPAZOcn PRIVATE TEST_FILES_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}\") target_link_libraries(testTOPAZOcn PRIVATE nextsimlib doctest::doctest) - add_executable(testIceGrowth "IceGrowth_test.cpp") + add_executable(testIceGrowth "IceGrowth_test.cpp" "../../core/test/MainTest.cpp") target_include_directories(testIceGrowth PRIVATE "${ModulesRoot}/OceanBoundaryModule" "${CoreModulesRoot}/FreezingPointModule") target_link_libraries(testIceGrowth PRIVATE nextsimlib doctest::doctest) - add_executable(testThermoWinton "ThermoWintonTemperature_test.cpp") + add_executable(testThermoWinton "ThermoWintonTemperature_test.cpp" "../../core/test/MainTest.cpp") target_include_directories(testThermoWinton PRIVATE "${ModulesRoot}/IceThermodynamicsModule" "${ModulesRoot}/OceanBoundaryModule") target_link_libraries(testThermoWinton PRIVATE nextsimlib doctest::doctest) @@ -51,7 +51,7 @@ if(NOT ENABLE_MPI) target_include_directories(testSpecHum PRIVATE "${ModulesRoot}/FluxCalculationModule") target_link_libraries(testSpecHum PRIVATE nextsimlib doctest::doctest) - add_executable(testThermoIce0 "ThermoIce0_test.cpp") + add_executable(testThermoIce0 "ThermoIce0_test.cpp" "../../core/test/MainTest.cpp") target_include_directories(testThermoIce0 PRIVATE "${ModulesRoot}/IceThermodynamicsModule") target_link_libraries(testThermoIce0 PRIVATE nextsimlib doctest::doctest) @@ -62,7 +62,7 @@ if(NOT ENABLE_MPI) target_include_directories(testConstantOcn PRIVATE "${ModulesRoot}/OceanBoundaryModule") target_link_libraries(testConstantOcn PRIVATE nextsimlib doctest::doctest) - add_executable(testSlabOcn "SlabOcean_test.cpp") + add_executable(testSlabOcn "SlabOcean_test.cpp" "../../core/test/MainTest.cpp") target_include_directories(testSlabOcn PRIVATE "${CoreModulesRoot}/FreezingPointModule") target_link_libraries(testSlabOcn PRIVATE nextsimlib doctest::doctest) @@ -76,7 +76,7 @@ if(NOT ENABLE_MPI) "${ModulesRoot}/OceanBoundaryModule") target_link_libraries(testBenchmarkBoundaries PRIVATE nextsimlib doctest::doctest) - add_executable(testDamageHealing "DamageHealing_test.cpp") + add_executable(testDamageHealing "DamageHealing_test.cpp" "../../core/test/MainTest.cpp") target_include_directories(testDamageHealing PRIVATE "${ModulesRoot}/DamageHealingModule" "${ModulesRoot}/DamageHealingModule") target_link_libraries(testDamageHealing PRIVATE nextsimlib doctest::doctest) diff --git a/physics/test/ConstantOceanBoundary_test.cpp b/physics/test/ConstantOceanBoundary_test.cpp index 1f360eb40..a697bdf55 100644 --- a/physics/test/ConstantOceanBoundary_test.cpp +++ b/physics/test/ConstantOceanBoundary_test.cpp @@ -7,7 +7,6 @@ #include "include/ConstantOceanBoundary.hpp" -#include "include/ModelArrayRef.hpp" #include "include/ModelState.hpp" namespace Nextsim { @@ -17,14 +16,17 @@ TEST_CASE("ConstantOcean Qio calculation") { ModelArray::setDimensions(ModelArray::Type::H, { 1, 1 }); - HField cice(ModelArray::Type::H); + ModelArrayAccessor ciceAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + HField& cice = ciceAccessor.getHostRW(); cice = 1.0; // Need some ice if Qio is to be calculated - ModelComponent::getStore().registerArray(Shared::C_ICE_DG, &cice, RO); ConstantOceanBoundary cob; cob.setData(ModelState::DataMap()); cob.updateBefore(TimestepTime()); - ModelArrayRef qio(ModelComponent::getStore()); + + ModelArrayAccessor qioAccessor(ModelComponent::getStore()); + const HField& qio = qioAccessor.getHostRO(); REQUIRE(qio[0] != 0.); } diff --git a/physics/test/DamageHealing_test.cpp b/physics/test/DamageHealing_test.cpp index 00fed0e19..a118ddfed 100644 --- a/physics/test/DamageHealing_test.cpp +++ b/physics/test/DamageHealing_test.cpp @@ -2,7 +2,6 @@ * @author Einar Ólason */ -#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN #include #include "include/IDamageHealing.hpp" @@ -36,23 +35,23 @@ TEST_CASE("Thermodynamic healing") class PrognosticData : public ModelComponent { public: PrognosticData() + : ciceAccessor(getStore(), RO) + , deltaCiAccessor(getStore(), RO) + , damageAccessor(getStore(), RW) { - getStore().registerArray(Shared::DELTA_CICE, &deltaCi, RO); - getStore().registerArray(Shared::C_ICE_DG, &cice, RO); - getStore().registerArray(Shared::DAMAGE, &damage, RW); } std::string getName() const override { return "PrognosticData"; } void setData(const ModelState::DataMap&) override { noLandMask(); - cice = 0.5; - deltaCi = 0.0; + ciceAccessor.getHostRW() = 0.5; + deltaCiAccessor.getHostRW() = 0.0; } - HField cice; - HField deltaCi; - HField damage; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor deltaCiAccessor; + ModelArrayAccessor damageAccessor; } iceState; iceState.setData(ModelState().data); @@ -64,14 +63,16 @@ TEST_CASE("Thermodynamic healing") TimestepTime tst = { TimePoint("2000-001"), Duration("P0-1T00:00:00") }; double prec = 1e-8; - iceState.damage = 0.5; + // always go through the accessor because their are be updates in between + iceState.damageAccessor.getHostRW() = 0.5; iHealing->update(tst); - REQUIRE(iceState.damage[0] == doctest::Approx(0.55).epsilon(prec)); + REQUIRE(iceState.damageAccessor.getHostRO()[0] == doctest::Approx(0.55).epsilon(prec)); - iceState.damage = 0.99; + iceState.damageAccessor.getHostRW() = 0.99; iHealing->update(tst); - REQUIRE(iceState.damage[0] <= 1.); - REQUIRE(iceState.damage[0] == doctest::Approx(1.).epsilon(prec)); + const AdvectedField& damage = iceState.damageAccessor.getHostRO(); + REQUIRE(damage[0] <= 1.); + REQUIRE(damage[0] == doctest::Approx(1.).epsilon(prec)); } TEST_CASE("New ice formation") @@ -96,23 +97,23 @@ TEST_CASE("New ice formation") class PrognosticData : public ModelComponent { public: PrognosticData() + : ciceAccessor(getStore(), RO) + , deltaCiAccessor(getStore(), RO) + , damageAccessor(getStore(), RW) { - getStore().registerArray(Shared::DELTA_CICE, &deltaCi, RO); - getStore().registerArray(Shared::C_ICE_DG, &cice, RO); - getStore().registerArray(Shared::DAMAGE, &damage, RW); } std::string getName() const override { return "PrognosticData"; } void setData(const ModelState::DataMap&) override { noLandMask(); - cice = 0.5; - deltaCi = 0.1; + ciceAccessor.getHostRW() = 0.5; + deltaCiAccessor.getHostRW() = 0.1; } - HField cice; - HField deltaCi; - HField damage; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor deltaCiAccessor; + ModelArrayAccessor damageAccessor; } iceState; iceState.setData(ModelState().data); @@ -122,35 +123,36 @@ TEST_CASE("New ice formation") TimestepTime tst = { TimePoint("2000-001"), Duration("P0-1T00:00:00") }; double prec = 1e-8; - iceState.cice = 0.6; - iceState.deltaCi = 0.3; - iceState.damage = 0.; + // always go through the accessor because there are updates in between + iceState.ciceAccessor.getHostRW() = 0.6; + iceState.deltaCiAccessor.getHostRW() = 0.3; + iceState.damageAccessor.getHostRW() = 0.; iHealing->update(tst); - REQUIRE(iceState.damage[0] == doctest::Approx(0.55).epsilon(prec)); + REQUIRE(iceState.damageAccessor.getHostRO()[0] == doctest::Approx(0.55).epsilon(prec)); - iceState.cice = 0.6; - iceState.deltaCi = 0.3; - iceState.damage = 0.5; + iceState.ciceAccessor.getHostRW() = 0.6; + iceState.deltaCiAccessor.getHostRW() = 0.3; + iceState.damageAccessor.getHostRW() = 0.5; iHealing->update(tst); - REQUIRE(iceState.damage[0] == doctest::Approx(0.80).epsilon(prec)); + REQUIRE(iceState.damageAccessor.getHostRO()[0] == doctest::Approx(0.80).epsilon(prec)); - iceState.cice = 0.5; - iceState.deltaCi = 0.1; - iceState.damage = 0.5; + iceState.ciceAccessor.getHostRW() = 0.5; + iceState.deltaCiAccessor.getHostRW() = 0.1; + iceState.damageAccessor.getHostRW() = 0.5; iHealing->update(tst); - REQUIRE(iceState.damage[0] == doctest::Approx(0.65).epsilon(prec)); + REQUIRE(iceState.damageAccessor.getHostRO()[0] == doctest::Approx(0.65).epsilon(prec)); - iceState.cice = 1.; - iceState.deltaCi = 0.1; - iceState.damage = 1.; + iceState.ciceAccessor.getHostRW() = 1.; + iceState.deltaCiAccessor.getHostRW() = 0.1; + iceState.damageAccessor.getHostRW() = 1.; iHealing->update(tst); - REQUIRE(iceState.damage[0] <= 1.); - REQUIRE(iceState.damage[0] <= doctest::Approx(1.).epsilon(prec)); + REQUIRE(iceState.damageAccessor.getHostRO()[0] <= 1.); + REQUIRE(iceState.damageAccessor.getHostRO()[0] <= doctest::Approx(1.).epsilon(prec)); - iceState.cice = 0.5; - iceState.deltaCi = -0.5; - iceState.damage = 0.5; + iceState.ciceAccessor.getHostRW() = 0.5; + iceState.deltaCiAccessor.getHostRW() = -0.5; + iceState.damageAccessor.getHostRW() = 0.5; iHealing->update(tst); - REQUIRE(iceState.damage[0] == doctest::Approx(0.55).epsilon(prec)); + REQUIRE(iceState.damageAccessor.getHostRO()[0] == doctest::Approx(0.55).epsilon(prec)); } } diff --git a/physics/test/ERA5Atm_test.cpp b/physics/test/ERA5Atm_test.cpp index e04ff46cf..1b160a7f8 100644 --- a/physics/test/ERA5Atm_test.cpp +++ b/physics/test/ERA5Atm_test.cpp @@ -61,13 +61,13 @@ TEST_CASE("ERA5Atmosphere construction test") e5.configure(); e5.setFilePath(filePath); - ModelArrayRef tair(ModelComponent::getStore()); - ModelArrayRef tdew(ModelComponent::getStore()); - ModelArrayRef pair(ModelComponent::getStore()); - ModelArrayRef qswin(ModelComponent::getStore()); - ModelArrayRef qlwin(ModelComponent::getStore()); - ModelArrayRef wind(ModelComponent::getStore()); - ModelArrayRef u(ModelComponent::getStore()); + // ModelArrayAccessor tairAccessor(ModelComponent::getStore()); + // ModelArrayAccessor tdewAccessor(ModelComponent::getStore()); + ModelArrayAccessor pairAccessor(ModelComponent::getStore()); + // ModelArrayAccessor qswinAccessor(ModelComponent::getStore()); + // ModelArrayAccessor qlwinAccessor(ModelComponent::getStore()); + ModelArrayAccessor windAccessor(ModelComponent::getStore()); + ModelArrayAccessor uAccessor(ModelComponent::getStore()); TimePoint t1("2000-01-01T00:00:00Z"); TimestepTime tst = { t1, Duration(600) }; @@ -75,36 +75,54 @@ TEST_CASE("ERA5Atmosphere construction test") // Get the forcing fields at time 0 e5.update(tst); - REQUIRE(wind(0, 0) == 0.); - REQUIRE(wind(12, 12) == 12.012); - REQUIRE(wind(30, 20) == 20.030); - REQUIRE(pair(30, 20) == (1.01e5 + 20.030)); + { + const HField& wind = windAccessor.getHostRO(); + const HField& pair = pairAccessor.getHostRO(); + REQUIRE(wind(0, 0) == 0.); + REQUIRE(wind(12, 12) == 12.012); + REQUIRE(wind(30, 20) == 20.030); + REQUIRE(pair(30, 20) == (1.01e5 + 20.030)); + } TimePoint t2("2000-02-01T00:00:00Z"); e5.update({ t2, Duration(600) }); - REQUIRE(wind(0, 0) == 0. + 100.); - REQUIRE(wind(12, 12) == 12.012 + 100); - REQUIRE(wind(30, 20) == 20.030 + 100); - REQUIRE(pair(30, 20) == (1.01e5 + 20.030) + 1000); + { + const HField& wind = windAccessor.getHostRO(); + const HField& pair = pairAccessor.getHostRO(); + REQUIRE(wind(0, 0) == 0. + 100.); + REQUIRE(wind(12, 12) == 12.012 + 100); + REQUIRE(wind(30, 20) == 20.030 + 100); + REQUIRE(pair(30, 20) == (1.01e5 + 20.030) + 1000); + } TimePoint t12("2000-12-01T00:00:00Z"); e5.update({ t12, Duration(600) }); - REQUIRE(wind(0, 0) == 0. + 100. * 11); - REQUIRE(wind(12, 12) == 12.012 + 100 * 11); - REQUIRE(wind(30, 20) == 20.030 + 100 * 11); - REQUIRE(pair(30, 20) == (1.01e5 + 20.030) + 1000 * 11); + { + const HField& wind = windAccessor.getHostRO(); + const HField& pair = pairAccessor.getHostRO(); + REQUIRE(wind(0, 0) == 0. + 100. * 11); + REQUIRE(wind(12, 12) == 12.012 + 100 * 11); + REQUIRE(wind(30, 20) == 20.030 + 100 * 11); + REQUIRE(pair(30, 20) == (1.01e5 + 20.030) + 1000 * 11); + } // All times after the last time sample should use the last sample's data TimePoint t120("2010-01-01T00:00:00Z"); e5.update({ t120, Duration(600) }); - REQUIRE(wind(0, 0) == 0. + 100. * 11); - REQUIRE(wind(12, 12) == 12.012 + 100 * 11); - REQUIRE(wind(30, 20) == 20.030 + 100 * 11); - REQUIRE(pair(30, 20) == (1.01e5 + 20.030) + 1000 * 11); - REQUIRE(u(30, 20) == (10 + 20.030) + 10 * 11); + { + const HField& wind = windAccessor.getHostRO(); + const HField& pair = pairAccessor.getHostRO(); + const UField& u = uAccessor.getHostRO(); + + REQUIRE(wind(0, 0) == 0. + 100. * 11); + REQUIRE(wind(12, 12) == 12.012 + 100 * 11); + REQUIRE(wind(30, 20) == 20.030 + 100 * 11); + REQUIRE(pair(30, 20) == (1.01e5 + 20.030) + 1000 * 11); + REQUIRE(u(30, 20) == (10 + 20.030) + 10 * 11); + } std::filesystem::remove(filePath); } diff --git a/physics/test/FiniteElementFluxes_test.cpp b/physics/test/FiniteElementFluxes_test.cpp index 7b8bf7160..b5868f819 100644 --- a/physics/test/FiniteElementFluxes_test.cpp +++ b/physics/test/FiniteElementFluxes_test.cpp @@ -12,7 +12,6 @@ #include "include/ConfiguredModule.hpp" #include "include/IFreezingPoint.hpp" #include "include/ModelArray.hpp" -#include "include/ModelArrayRef.hpp" #include "include/ModelComponent.hpp" #include "include/NextsimModule.hpp" #include "include/Time.hpp" @@ -46,25 +45,33 @@ TEST_CASE("Melting conditions") class AtmosphereData : public ModelComponent { public: AtmosphereData() + : tairAccessor(getStore(), RO) + , tdewAccessor(getStore(), RO) + , pairAccessor(getStore(), RO) + , windSpeedAccessor(getStore(), RO) + , u_airAccessor(getStore(), RO) + , v_airAccessor(getStore(), RO) + , sw_inAccessor(getStore(), RO) + , lw_inAccessor(getStore(), RO) { - getStore().registerArray(Protected::T_AIR, &tair, RO); - getStore().registerArray(Protected::DEW_2M, &tdew, RO); - getStore().registerArray(Protected::P_AIR, &pair, RO); - getStore().registerArray(Protected::WIND_SPEED, &windSpeed, RO); - getStore().registerArray(Protected::WIND_U, &u_air, RO); - getStore().registerArray(Protected::WIND_V, &v_air, RO); - getStore().registerArray(Protected::SW_IN, &sw_in, RO); - getStore().registerArray(Protected::LW_IN, &lw_in, RO); } void setData(const ModelState::DataMap& state) override { + HField& tair = tairAccessor.getHostRW(); tair.resize(); + HField& tdew = tdewAccessor.getHostRW(); tdew.resize(); + HField& pair = pairAccessor.getHostRW(); pair.resize(); + HField& windSpeed = windSpeedAccessor.getHostRW(); windSpeed.resize(); + HField& u_air = u_airAccessor.getHostRW(); u_air.resize(); + HField& v_air = v_airAccessor.getHostRW(); v_air.resize(); + HField& sw_in = sw_inAccessor.getHostRW(); sw_in.resize(); + HField& lw_in = lw_inAccessor.getHostRW(); lw_in.resize(); tair = 3; @@ -79,14 +86,14 @@ TEST_CASE("Melting conditions") std::string getName() const override { return "AtmData"; } private: - HField tair; - HField tdew; - HField pair; - HField windSpeed; - HField u_air; - HField v_air; - HField sw_in; - HField lw_in; + ModelArrayAccessor tairAccessor; + ModelArrayAccessor tdewAccessor; + ModelArrayAccessor pairAccessor; + ModelArrayAccessor windSpeedAccessor; + ModelArrayAccessor u_airAccessor; + ModelArrayAccessor v_airAccessor; + ModelArrayAccessor sw_inAccessor; + ModelArrayAccessor lw_inAccessor; HField snowfall; } atmState; atmState.setData(ModelState().data); @@ -94,68 +101,52 @@ TEST_CASE("Melting conditions") class ProgData : public ModelComponent { public: ProgData() + : hiceAccessor(getStore(), RO) + , ciceAccessor(getStore(), RO) + , hsnowAccessor(getStore(), RO) + , tsurfAccessor(getStore(), RO) { - getStore().registerArray(Shared::H_ICE_DG, &hice, RO); - getStore().registerArray(Shared::C_ICE_DG, &cice, RO); - getStore().registerArray(Shared::H_SNOW_DG, &hsnow, RO); - getStore().registerArray(Protected::T_SURF, &tsurf, RO); } std::string getName() const override { return "ProgData"; } void setData(const ModelState::DataMap&) override { noLandMask(); - cice[0] = 0.5; - hice[0] = 0.1; // Here we are using the cell-averaged thicknesses - hsnow[0] = 0.01; - tsurf[0] = -1.; + ciceAccessor.getHostRW()[0] = 0.5; + hiceAccessor.getHostRW()[0] = 0.1; // Here we are using the cell-averaged thicknesses + hsnowAccessor.getHostRW()[0] = 0.01; + tsurfAccessor.getHostRW()[0] = -1.; } - HField hice; - HField cice; - HField hsnow; - HField tsurf; + ModelArrayAccessor hiceAccessor; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor hsnowAccessor; + ModelArrayAccessor tsurfAccessor; } iceState; iceState.setData(ModelState().data); - HField qow; - qow.resize(); - ModelComponent::getStore().registerArray(Shared::Q_OW, &qow, RW); + ModelArrayAccessor qowAccessor(ModelComponent::getStore(), RW); + qowAccessor.getHostRW().resize(); + ModelArrayAccessor qiaAccessor(ModelComponent::getStore(), RW); + qiaAccessor.getHostRW().resize(); + ModelArrayAccessor penSWAccessor(ModelComponent::getStore(), RW); + penSWAccessor.getHostRW().resize(); + ModelArrayAccessor qsw_owAccessor(ModelComponent::getStore(), RW); + qsw_owAccessor.getHostRW().resize(); + ModelArrayAccessor qsw_baseAccessor(ModelComponent::getStore(), RW); + qsw_baseAccessor.getHostRW().resize(); + ModelArrayAccessor dqia_dtAccessor(ModelComponent::getStore(), RW); + dqia_dtAccessor.getHostRW().resize(); + ModelArrayAccessor sublAccessor(ModelComponent::getStore(), RW); + sublAccessor.getHostRW().resize(); + ModelArrayAccessor evapAccessor(ModelComponent::getStore(), RW); + evapAccessor.getHostRW().resize(); + + ModelArrayAccessor tauXAccessor(ModelComponent::getStore(), RW); + tauXAccessor.getHostRW().resize(); + ModelArrayAccessor tauYAccessor(ModelComponent::getStore(), RW); + tauYAccessor.getHostRW().resize(); - HField qia; - qia.resize(); - ModelComponent::getStore().registerArray(Shared::Q_IA, &qia, RW); - - HField penSW; - penSW.resize(); - ModelComponent::getStore().registerArray(Shared::Q_PEN_SW, &penSW, RW); - - HField qsw_ow; - qsw_ow.resize(); - ModelComponent::getStore().registerArray(Shared::Q_SW_OW, &qsw_ow, RW); - - HField qsw_base; - qsw_base.resize(); - ModelComponent::getStore().registerArray(Shared::Q_SW_BASE, &qsw_base, RW); - - HField dqia_dt; - dqia_dt.resize(); - ModelComponent::getStore().registerArray(Shared::DQIA_DT, &dqia_dt, RW); - - HField subl; - subl.resize(); - ModelComponent::getStore().registerArray(Shared::SUBLIM, &subl, RW); - - HField evap; - evap.resize(); - ModelComponent::getStore().registerArray(Shared::EVAP, &evap, RW); - - HField tauX; - HField tauY; - tauX.resize(); - tauY.resize(); - ModelComponent::getStore().registerArray(Shared::OW_STRESS_X, &tauX, RW); - ModelComponent::getStore().registerArray(Shared::OW_STRESS_Y, &tauY, RW); TimestepTime tst = { TimePoint("2000-001"), Duration("P0-0T0:10:0") }; // OceanState is independently updated @@ -166,12 +157,12 @@ TEST_CASE("Melting conditions") fef.update(tst); double prec = 1e-5; - REQUIRE(qow[0] == doctest::Approx(-109.923).epsilon(prec)); - REQUIRE(qia[0] == doctest::Approx(-85.6364).epsilon(prec)); - REQUIRE(dqia_dt[0] == doctest::Approx(19.7016).epsilon(prec)); - REQUIRE(subl[0] == doctest::Approx(-7.3858e-06).epsilon(prec)); - REQUIRE(tauX[0] == doctest::Approx(1.89732e-2).epsilon(prec)); - REQUIRE(tauY[0] == doctest::Approx(2.52976e-2).epsilon(prec)); + REQUIRE(qowAccessor.getHostRO()[0] == doctest::Approx(-109.923).epsilon(prec)); + REQUIRE(qiaAccessor.getHostRO()[0] == doctest::Approx(-85.6364).epsilon(prec)); + REQUIRE(dqia_dtAccessor.getHostRO()[0] == doctest::Approx(19.7016).epsilon(prec)); + REQUIRE(sublAccessor.getHostRO()[0] == doctest::Approx(-7.3858e-06).epsilon(prec)); + REQUIRE(tauXAccessor.getHostRO()[0] == doctest::Approx(1.89732e-2).epsilon(prec)); + REQUIRE(tauYAccessor.getHostRO()[0] == doctest::Approx(2.52976e-2).epsilon(prec)); } TEST_CASE("Freezing conditions") @@ -196,25 +187,33 @@ TEST_CASE("Freezing conditions") class AtmosphereData : public ModelComponent { public: AtmosphereData() + : tairAccessor(getStore(), RO) + , tdewAccessor(getStore(), RO) + , pairAccessor(getStore(), RO) + , windSpeedAccessor(getStore(), RO) + , u_airAccessor(getStore(), RO) + , v_airAccessor(getStore(), RO) + , sw_inAccessor(getStore(), RO) + , lw_inAccessor(getStore(), RO) { - getStore().registerArray(Protected::T_AIR, &tair, RO); - getStore().registerArray(Protected::DEW_2M, &tdew, RO); - getStore().registerArray(Protected::P_AIR, &pair, RO); - getStore().registerArray(Protected::WIND_SPEED, &windSpeed, RO); - getStore().registerArray(Protected::WIND_U, &u_air, RO); - getStore().registerArray(Protected::WIND_V, &v_air, RO); - getStore().registerArray(Protected::SW_IN, &sw_in, RO); - getStore().registerArray(Protected::LW_IN, &lw_in, RO); } void setData(const ModelState::DataMap& state) override { + HField& tair = tairAccessor.getHostRW(); tair.resize(); + HField& tdew = tdewAccessor.getHostRW(); tdew.resize(); + HField& pair = pairAccessor.getHostRW(); pair.resize(); + HField& windSpeed = windSpeedAccessor.getHostRW(); windSpeed.resize(); + HField& u_air = u_airAccessor.getHostRW(); u_air.resize(); + HField& v_air = v_airAccessor.getHostRW(); v_air.resize(); + HField& sw_in = sw_inAccessor.getHostRW(); sw_in.resize(); + HField& lw_in = lw_inAccessor.getHostRW(); lw_in.resize(); tair = -12; tdew = -12; @@ -228,14 +227,14 @@ TEST_CASE("Freezing conditions") std::string getName() const override { return "AtmData"; } private: - HField tair; - HField tdew; - HField pair; - HField windSpeed; - HField u_air; - HField v_air; - HField sw_in; - HField lw_in; + ModelArrayAccessor tairAccessor; + ModelArrayAccessor tdewAccessor; + ModelArrayAccessor pairAccessor; + ModelArrayAccessor windSpeedAccessor; + ModelArrayAccessor u_airAccessor; + ModelArrayAccessor v_airAccessor; + ModelArrayAccessor sw_inAccessor; + ModelArrayAccessor lw_inAccessor; HField snowfall; } atmState; atmState.setData(ModelState().data); @@ -243,61 +242,47 @@ TEST_CASE("Freezing conditions") class ProgData : public ModelComponent { public: ProgData() + : hiceAccessor(getStore(), RO) + , ciceAccessor(getStore(), RO) + , hsnowAccessor(getStore(), RO) + , tsurfAccessor(getStore(), RO) { - getStore().registerArray(Shared::H_ICE_DG, &hice, RO); - getStore().registerArray(Shared::C_ICE_DG, &cice, RO); - getStore().registerArray(Shared::H_SNOW_DG, &hsnow, RO); - getStore().registerArray(Protected::T_SURF, &tsurf, RO); } std::string getName() const override { return "ProgData"; } void setData(const ModelState::DataMap&) override { noLandMask(); - cice[0] = 0.5; - hice[0] = 0.1; // Here we are using the cell-averaged thicknesses - hsnow[0] = 0.01; - tsurf[0] = -9.; + ciceAccessor.getHostRW()[0] = 0.5; + hiceAccessor.getHostRW()[0] = 0.1; // Here we are using the cell-averaged thicknesses + hsnowAccessor.getHostRW()[0] = 0.01; + tsurfAccessor.getHostRW()[0] = -9.; } - HField hice; - HField cice; - HField hsnow; - HField tsurf; + ModelArrayAccessor hiceAccessor; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor hsnowAccessor; + ModelArrayAccessor tsurfAccessor; } iceState; iceState.setData(ModelState().data); - HField qow; - qow.resize(); - ModelComponent::getStore().registerArray(Shared::Q_OW, &qow, RW); - - HField qia; - qia.resize(); - ModelComponent::getStore().registerArray(Shared::Q_IA, &qia, RW); - - HField penSW; - penSW.resize(); - ModelComponent::getStore().registerArray(Shared::Q_PEN_SW, &penSW, RW); - - HField dqia_dt; - dqia_dt.resize(); - ModelComponent::getStore().registerArray(Shared::DQIA_DT, &dqia_dt, RW); - - HField subl; - subl.resize(); - ModelComponent::getStore().registerArray(Shared::SUBLIM, &subl, RW); - - HField evap; - evap.resize(); - ModelComponent::getStore().registerArray(Shared::EVAP, &evap, RW); - - HField tauX; - HField tauY; - tauX.resize(); - tauY.resize(); - ModelComponent::getStore().registerArray(Shared::OW_STRESS_X, &tauX, RW); - ModelComponent::getStore().registerArray(Shared::OW_STRESS_Y, &tauY, RW); + ModelArrayAccessor qowAccessor(ModelComponent::getStore(), RW); + qowAccessor.getHostRW().resize(); + ModelArrayAccessor qiaAccessor(ModelComponent::getStore(), RW); + qiaAccessor.getHostRW().resize(); + ModelArrayAccessor penSWAccessor(ModelComponent::getStore(), RW); + penSWAccessor.getHostRW().resize(); + ModelArrayAccessor dqia_dtAccessor(ModelComponent::getStore(), RW); + dqia_dtAccessor.getHostRW().resize(); + ModelArrayAccessor sublAccessor(ModelComponent::getStore(), RW); + sublAccessor.getHostRW().resize(); + ModelArrayAccessor evapAccessor(ModelComponent::getStore(), RW); + evapAccessor.getHostRW().resize(); + ModelArrayAccessor tauXAccessor(ModelComponent::getStore(), RW); + tauXAccessor.getHostRW().resize(); + ModelArrayAccessor tauYAccessor(ModelComponent::getStore(), RW); + tauYAccessor.getHostRW().resize(); TimestepTime tst = { TimePoint("2000-001"), Duration("P0-0T0:10:0") }; // OceanState is independently updated @@ -308,12 +293,12 @@ TEST_CASE("Freezing conditions") fef.update(tst); double prec = 1e-5; - REQUIRE(qow[0] == doctest::Approx(143.266).epsilon(prec)); - REQUIRE(qia[0] == doctest::Approx(42.2955).epsilon(prec)); - REQUIRE(dqia_dt[0] == doctest::Approx(16.7615).epsilon(prec)); - REQUIRE(subl[0] == doctest::Approx(2.15132e-6).epsilon(prec)); - REQUIRE(tauX[0] == doctest::Approx(2.00279e-2).epsilon(prec)); - REQUIRE(tauY[0] == doctest::Approx(2.67038e-2).epsilon(prec)); + REQUIRE(qowAccessor.getHostRO()[0] == doctest::Approx(143.266).epsilon(prec)); + REQUIRE(qiaAccessor.getHostRO()[0] == doctest::Approx(42.2955).epsilon(prec)); + REQUIRE(dqia_dtAccessor.getHostRO()[0] == doctest::Approx(16.7615).epsilon(prec)); + REQUIRE(sublAccessor.getHostRO()[0] == doctest::Approx(2.15132e-6).epsilon(prec)); + REQUIRE(tauXAccessor.getHostRO()[0] == doctest::Approx(2.00279e-2).epsilon(prec)); + REQUIRE(tauYAccessor.getHostRO()[0] == doctest::Approx(2.67038e-2).epsilon(prec)); } TEST_SUITE_END(); diff --git a/physics/test/IceGrowth_test.cpp b/physics/test/IceGrowth_test.cpp index e809921a6..1fd29233a 100644 --- a/physics/test/IceGrowth_test.cpp +++ b/physics/test/IceGrowth_test.cpp @@ -2,7 +2,6 @@ * @author Tim Spain */ -#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN #include #include @@ -41,16 +40,16 @@ TEST_CASE("New ice formation") void setData(const ModelState::DataMap& ms) override { IAtmosphereBoundary::setData(ms); - qia = 305.288; - dqia_dt = 4.5036; - qow = 307.546; - subl = 0.; // Seems unlikely… - penSW = 0.; - snow = 0.; - rain = 0.; - evap = 0.; // Seems unlikely… - uwind = 0; - vwind = 0.; + qiaAccessor.getHostRW() = 305.288; + dqia_dtAccessor.getHostRW() = 4.5036; + qowAccessor.getHostRW() = 307.546; + sublAccessor.getHostRW() = 0.; // Seems unlikely… + penSWAccessor.getHostRW() = 0.; + snowAccessor.getHostRW() = 0.; + rainAccessor.getHostRW() = 0.; + evapAccessor.getHostRW() = 0.; // Seems unlikely… + uwindAccessor.getHostRW() = 0; + vwindAccessor.getHostRW() = 0.; } } atmBdy; atmBdy.setData(ModelState().data); @@ -58,24 +57,24 @@ TEST_CASE("New ice formation") class PrognosticData : public ModelComponent { public: PrognosticData() + : hiceAccessor(getStore(), RW) + , ciceAccessor(getStore(), RW) + , hsnowAccessor(getStore(), RW) { - getStore().registerArray(Shared::H_ICE_DG, &hice, RW); - getStore().registerArray(Shared::C_ICE_DG, &cice, RW); - getStore().registerArray(Shared::H_SNOW_DG, &hsnow, RW); } std::string getName() const override { return "PrognosticData"; } void setData(const ModelState::DataMap&) override { noLandMask(); - cice = 0.5; - hice = 0.1; // Cell averaged - hsnow = 0; // Cell averaged + ciceAccessor.getHostRW() = 0.5; + hiceAccessor.getHostRW() = 0.1; // Cell averaged + hsnowAccessor.getHostRW() = 0; // Cell averaged } - HField hice; - HField cice; - HField hsnow; + ModelArrayAccessor hiceAccessor; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor hsnowAccessor; } proData; proData.setData(ModelState().data); @@ -83,11 +82,11 @@ TEST_CASE("New ice formation") ocnBdy.setQio(124.689); ocnBdy.setData(ModelState().data); - HField damage(ModelArray::Type::H); - HField oldDamage(ModelArray::Type::H); - damage = 1; - ModelComponent::getStore().registerArray(Shared::DAMAGE, &damage, RW); - ModelComponent::getStore().registerArray(Protected::DAMAGE, &oldDamage, RO); + ModelArrayAccessor damageAccessor( + ModelComponent::getStore(), RW, ModelArray::Type::H); + ModelArrayAccessor oldDamageAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + damageAccessor.getHostRW() = 1; TimestepTime tst = { TimePoint("2000-001"), Duration("P0-1") }; IceGrowth ig; @@ -100,7 +99,8 @@ TEST_CASE("New ice formation") ocnBdy.updateBefore(tst); ig.update(tst); - ModelArrayRef newice(ModelComponent::getStore()); + ModelArrayAccessor newiceAccessor(ModelComponent::getStore()); + const HField& newice = newiceAccessor.getHostRO(); double prec = 1e-5; REQUIRE(newice[0] == doctest::Approx(0.0258264).epsilon(prec)); @@ -122,16 +122,16 @@ TEST_CASE("Melting conditions") void setData(const ModelState::DataMap& ms) override { IAtmosphereBoundary::setData(ms); - qia = -84.5952; - dqia_dt = 19.7016; - qow = -109.923; - subl = -7.3858e-06; - penSW = 0.; - snow = 0.; - rain = 0.; - evap = 0.; // Seems unlikely… - uwind = 0; - vwind = 0.; + qiaAccessor.getHostRW() = -84.5952; + dqia_dtAccessor.getHostRW() = 19.7016; + qowAccessor.getHostRW() = -109.923; + sublAccessor.getHostRW() = -7.3858e-06; + penSWAccessor.getHostRW() = 0.; + snowAccessor.getHostRW() = 0.; + rainAccessor.getHostRW() = 0.; + evapAccessor.getHostRW() = 0.; // Seems unlikely… + uwindAccessor.getHostRW() = 0; + vwindAccessor.getHostRW() = 0.; } std::string getName() const override { return "AtmosphericBoundary"; } } atmBdy; @@ -140,24 +140,24 @@ TEST_CASE("Melting conditions") class PrognosticData : public ModelComponent { public: PrognosticData() + : hiceAccessor(getStore(), RW) + , ciceAccessor(getStore(), RW) + , hsnowAccessor(getStore(), RW) { - getStore().registerArray(Shared::H_ICE_DG, &hice, RW); - getStore().registerArray(Shared::C_ICE_DG, &cice, RW); - getStore().registerArray(Shared::H_SNOW_DG, &hsnow, RW); } std::string getName() const override { return "PrognosticData"; } void setData(const ModelState::DataMap&) override { noLandMask(); - cice = 0.5; - hice = 0.1; // Cell averaged - hsnow = 0.01; // Cell averaged + ciceAccessor.getHostRW() = 0.5; + hiceAccessor.getHostRW() = 0.1; // Cell averaged + hsnowAccessor.getHostRW() = 0.01; // Cell averaged } - HField hice; - HField cice; - HField hsnow; + ModelArrayAccessor hiceAccessor; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor hsnowAccessor; } proData; proData.setData(ModelState().data); @@ -167,11 +167,11 @@ TEST_CASE("Melting conditions") ocnBdy.setQio(53717.8); ocnBdy.setData(ModelState().data); - HField damage(ModelArray::Type::H); - HField oldDamage(ModelArray::Type::H); - damage = 1; - ModelComponent::getStore().registerArray(Shared::DAMAGE, &damage, RW); - ModelComponent::getStore().registerArray(Protected::DAMAGE, &oldDamage, RO); + ModelArrayAccessor damageAccessor( + ModelComponent::getStore(), RW, ModelArray::Type::H); + ModelArrayAccessor oldDamageAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + damageAccessor.getHostRW() = 1; TimestepTime tst = { TimePoint("2000-001"), Duration("P0-0T0:10:0") }; IceGrowth ig; @@ -184,10 +184,14 @@ TEST_CASE("Melting conditions") ocnBdy.updateBefore(tst); ig.update(tst); - ModelArrayRef newice(ModelComponent::getStore()); - ModelArrayRef hice(ModelComponent::getStore()); - ModelArrayRef cice(ModelComponent::getStore()); - ModelArrayRef hsnow(ModelComponent::getStore()); + ModelArrayAccessor newiceAccessor(ModelComponent::getStore()); + const HField& newice = newiceAccessor.getHostRO(); + ModelArrayAccessor hiceAccessor(ModelComponent::getStore()); + const HField& hice = hiceAccessor.getHostRO(); + ModelArrayAccessor ciceAccessor(ModelComponent::getStore()); + const HField& cice = ciceAccessor.getHostRO(); + ModelArrayAccessor hsnowAccessor(ModelComponent::getStore()); + const HField& hsnow = hsnowAccessor.getHostRO(); double prec = 1e-5; // The thickness values from old NextSIM are cell-averaged. Perform that @@ -215,16 +219,16 @@ TEST_CASE("Freezing conditions") void setData(const ModelState::DataMap& ms) override { IAtmosphereBoundary::setData(ms); - qia = 42.2955; - dqia_dt = 16.7615; - qow = 143.266; - subl = 2.15132e-6; - penSW = 0.; - snow = 1e-3; - rain = 0.; - evap = -1e-3; // E-P = 0 - uwind = 0; - vwind = 0.; + qiaAccessor.getHostRW() = 42.2955; + dqia_dtAccessor.getHostRW() = 16.7615; + qowAccessor.getHostRW() = 143.266; + sublAccessor.getHostRW() = 2.15132e-6; + penSWAccessor.getHostRW() = 0.; + snowAccessor.getHostRW() = 1e-3; + rainAccessor.getHostRW() = 0.; + evapAccessor.getHostRW() = -1e-3; // E-P = 0 + uwindAccessor.getHostRW() = 0; + vwindAccessor.getHostRW() = 0.; } } atmBdy; atmBdy.setData(ModelState().data); @@ -232,24 +236,24 @@ TEST_CASE("Freezing conditions") class PrognosticData : public ModelComponent { public: PrognosticData() + : hiceAccessor(getStore(), RW) + , ciceAccessor(getStore(), RW) + , hsnowAccessor(getStore(), RW) { - getStore().registerArray(Shared::H_ICE_DG, &hice, RW); - getStore().registerArray(Shared::C_ICE_DG, &cice, RW); - getStore().registerArray(Shared::H_SNOW_DG, &hsnow, RW); } std::string getName() const override { return "PrognosticData"; } void setData(const ModelState::DataMap&) override { noLandMask(); - cice = 0.5; - hice = 0.1; // Cell averaged - hsnow = 0.01; // Cell averaged + ciceAccessor.getHostRW() = 0.5; + hiceAccessor.getHostRW() = 0.1; // Cell averaged + hsnowAccessor.getHostRW() = 0.01; // Cell averaged } - HField hice; - HField cice; - HField hsnow; + ModelArrayAccessor hiceAccessor; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor hsnowAccessor; } proData; proData.setData(ModelState().data); @@ -259,11 +263,11 @@ TEST_CASE("Freezing conditions") ocnBdy.setQio(73.9465); ocnBdy.setData(ModelState().data); - HField damage(ModelArray::Type::H); - HField oldDamage(ModelArray::Type::H); - damage = 1; - ModelComponent::getStore().registerArray(Shared::DAMAGE, &damage, RW); - ModelComponent::getStore().registerArray(Protected::DAMAGE, &oldDamage, RO); + ModelArrayAccessor damageAccessor( + ModelComponent::getStore(), RW, ModelArray::Type::H); + ModelArrayAccessor oldDamageAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + damageAccessor.getHostRW() = 1; TimestepTime tst = { TimePoint("2000-001"), Duration("P0-0T0:10:0") }; IceGrowth ig; @@ -276,10 +280,14 @@ TEST_CASE("Freezing conditions") ocnBdy.updateBefore(tst); ig.update(tst); - ModelArrayRef newice(ModelComponent::getStore()); - ModelArrayRef hice(ModelComponent::getStore()); - ModelArrayRef cice(ModelComponent::getStore()); - ModelArrayRef hsnow(ModelComponent::getStore()); + ModelArrayAccessor newiceAccessor(ModelComponent::getStore()); + const HField& newice = newiceAccessor.getHostRO(); + ModelArrayAccessor hiceAccessor(ModelComponent::getStore()); + const HField& hice = hiceAccessor.getHostRO(); + ModelArrayAccessor ciceAccessor(ModelComponent::getStore()); + const HField& cice = ciceAccessor.getHostRO(); + ModelArrayAccessor hsnowAccessor(ModelComponent::getStore()); + const HField& hsnow = hsnowAccessor.getHostRO(); double prec = 1e-5; @@ -308,16 +316,16 @@ TEST_CASE("Dummy ice") void setData(const ModelState::DataMap& ms) override { IAtmosphereBoundary::setData(ms); - qia = 0.; - dqia_dt = 0.; - qow = 0.; - subl = 0.; - penSW = 0.; - snow = 0.; - rain = 0.; - evap = 0.; // E-P = 0 - uwind = 0; - vwind = 0.; + qiaAccessor.getHostRW() = 0.; + dqia_dtAccessor.getHostRW() = 0.; + qowAccessor.getHostRW() = 0.; + sublAccessor.getHostRW() = 0.; + penSWAccessor.getHostRW() = 0.; + snowAccessor.getHostRW() = 0.; + rainAccessor.getHostRW() = 0.; + evapAccessor.getHostRW() = 0.; // E-P = 0 + uwindAccessor.getHostRW() = 0; + vwindAccessor.getHostRW() = 0.; } } atmBdy; atmBdy.setData(ModelState().data); @@ -331,24 +339,24 @@ TEST_CASE("Dummy ice") class PrognosticData : public ModelComponent { public: PrognosticData() + : hiceAccessor(getStore(), RW) + , ciceAccessor(getStore(), RW) + , hsnowAccessor(getStore(), RW) { - getStore().registerArray(Shared::H_ICE_DG, &hice, RW); - getStore().registerArray(Shared::C_ICE_DG, &cice, RW); - getStore().registerArray(Shared::H_SNOW_DG, &hsnow, RW); } std::string getName() const override { return "PrognosticData"; } void setData(const ModelState::DataMap&) override { noLandMask(); - cice = cice0; - hice = hice0; // Cell averaged - hsnow = hsnow0; // Cell averaged + ciceAccessor.getHostRW() = cice0; + hiceAccessor.getHostRW() = hice0; // Cell averaged + hsnowAccessor.getHostRW() = hsnow0; // Cell averaged } - HField hice; - HField cice; - HField hsnow; + ModelArrayAccessor hiceAccessor; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor hsnowAccessor; } proData; proData.setData(ModelState().data); @@ -356,11 +364,11 @@ TEST_CASE("Dummy ice") ocnBdy.setQio(0.); ocnBdy.setData(ModelState().data); - HField damage(ModelArray::Type::H); - HField oldDamage(ModelArray::Type::H); - damage = 1; - ModelComponent::getStore().registerArray(Shared::DAMAGE, &damage, RW); - ModelComponent::getStore().registerArray(Protected::DAMAGE, &oldDamage, RO); + ModelArrayAccessor damageAccessor( + ModelComponent::getStore(), RW, ModelArray::Type::H); + ModelArrayAccessor oldDamageAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + damageAccessor.getHostRW() = 1; TimestepTime tst = { TimePoint("2000-001"), Duration("P0-0T0:10:0") }; @@ -377,10 +385,14 @@ TEST_CASE("Dummy ice") // double prec = 1e-5; - ModelArrayRef newice(ModelComponent::getStore()); - ModelArrayRef hice(ModelComponent::getStore()); - ModelArrayRef cice(ModelComponent::getStore()); - ModelArrayRef hsnow(ModelComponent::getStore()); + ModelArrayAccessor newiceAccessor(ModelComponent::getStore()); + const HField& newice = newiceAccessor.getHostRO(); + ModelArrayAccessor hiceAccessor(ModelComponent::getStore()); + const HField& hice = hiceAccessor.getHostRO(); + ModelArrayAccessor ciceAccessor(ModelComponent::getStore()); + const HField& cice = ciceAccessor.getHostRO(); + ModelArrayAccessor hsnowAccessor(ModelComponent::getStore()); + const HField& hsnow = hsnowAccessor.getHostRO(); // The thickness values from old NextSIM are cell-averaged. Perform that // conversion here. @@ -411,16 +423,16 @@ TEST_CASE("Zero thickness") void setData(const ModelState::DataMap& ms) override { IAtmosphereBoundary::setData(ms); - qia = -84.5952; - dqia_dt = 19.7016; - qow = -109.923; - subl = -7.3858e-06; - penSW = 0.; - snow = 0.; - rain = 0.; - evap = 0.; // Seems unlikely… - uwind = 0; - vwind = 0.; + qiaAccessor.getHostRW() = -84.5952; + dqia_dtAccessor.getHostRW() = 19.7016; + qowAccessor.getHostRW() = -109.923; + sublAccessor.getHostRW() = -7.3858e-06; + penSWAccessor.getHostRW() = 0.; + snowAccessor.getHostRW() = 0.; + rainAccessor.getHostRW() = 0.; + evapAccessor.getHostRW() = 0.; // Seems unlikely… + uwindAccessor.getHostRW() = 0; + vwindAccessor.getHostRW() = 0.; } std::string getName() const override { return "AtmosphericBoundary"; } } atmBdy; @@ -429,24 +441,24 @@ TEST_CASE("Zero thickness") class PrognosticData : public ModelComponent { public: PrognosticData() + : hiceAccessor(getStore(), RW) + , ciceAccessor(getStore(), RW) + , hsnowAccessor(getStore(), RW) { - getStore().registerArray(Shared::H_ICE_DG, &hice, RW); - getStore().registerArray(Shared::C_ICE_DG, &cice, RW); - getStore().registerArray(Shared::H_SNOW_DG, &hsnow, RW); } std::string getName() const override { return "PrognosticData"; } void setData(const ModelState::DataMap&) override { noLandMask(); - cice = 0.5; - hice = 0.1; // Cell averaged - hsnow = 0.01; // Cell averaged + ciceAccessor.getHostRW() = 0.5; + hiceAccessor.getHostRW() = 0.1; // Cell averaged + hsnowAccessor.getHostRW() = 0.01; // Cell averaged } - HField hice; - HField cice; - HField hsnow; + ModelArrayAccessor hiceAccessor; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor hsnowAccessor; } proData; proData.setData(ModelState().data); @@ -454,20 +466,21 @@ TEST_CASE("Zero thickness") ocnBdy.setQio(53717.8); // 57 kW m⁻² to go from -1 to -1.75 over the whole mixed layer in 600 s ocnBdy.setData(ModelState().data); - HField damage(ModelArray::Type::H); - HField oldDamage(ModelArray::Type::H); - damage = 1; - ModelComponent::getStore().registerArray(Shared::DAMAGE, &damage, RW); - ModelComponent::getStore().registerArray(Protected::DAMAGE, &oldDamage, RO); + ModelArrayAccessor damageAccessor( + ModelComponent::getStore(), RW, ModelArray::Type::H); + ModelArrayAccessor oldDamageAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + damageAccessor.getHostRW() = 1; class ZeroThicknessIce : public IIceThermodynamics { void setData(const ModelState::DataMap&) override { } void update(const TimestepTime& tsTime) override { - deltaHi[0] = -hice[0]; + AdvectedField& hice = hiceAccessor.getHostRW(); + deltaHiAccessor.getHostRW()[0] = -hice[0]; hice[0] = 0; - tsurf[0] = 0; - snowToIce[0] = 0; + tsurfAccessor.getHostRW()[0] = 0; + snowToIceAccessor.getHostRW()[0] = 0; } }; Module::Module::setExternalImplementation( @@ -484,9 +497,12 @@ TEST_CASE("Zero thickness") ocnBdy.updateBefore(tst); ig.update(tst); - ModelArrayRef newice(ModelComponent::getStore()); - ModelArrayRef hice(ModelComponent::getStore()); - ModelArrayRef cice(ModelComponent::getStore()); + ModelArrayAccessor newiceAccessor(ModelComponent::getStore()); + const HField& newice = newiceAccessor.getHostRO(); + ModelArrayAccessor hiceAccessor(ModelComponent::getStore()); + const HField& hice = hiceAccessor.getHostRO(); + ModelArrayAccessor ciceAccessor(ModelComponent::getStore()); + const HField& cice = ciceAccessor.getHostRO(); // double prec = 1e-6; @@ -517,16 +533,16 @@ TEST_CASE("Turn off thermo") void setData(const ModelState::DataMap& ms) override { IAtmosphereBoundary::setData(ms); - qia = 42.2955; - dqia_dt = 16.7615; - qow = 143.266; - subl = 2.15132e-6; - penSW = 0.; - snow = 1e-3; - rain = 0.; - evap = -1e-3; // E-P = 0 - uwind = 0; - vwind = 0.; + qiaAccessor.getHostRW() = 42.2955; + dqia_dtAccessor.getHostRW() = 16.7615; + qowAccessor.getHostRW() = 143.266; + sublAccessor.getHostRW() = 2.15132e-6; + penSWAccessor.getHostRW() = 0.; + snowAccessor.getHostRW() = 1e-3; + rainAccessor.getHostRW() = 0.; + evapAccessor.getHostRW() = -1e-3; // E-P = 0 + uwindAccessor.getHostRW() = 0; + vwindAccessor.getHostRW() = 0.; } } atmBdy; atmBdy.setData(ModelState().data); @@ -534,24 +550,24 @@ TEST_CASE("Turn off thermo") class PrognosticData : public ModelComponent { public: PrognosticData() + : hiceAccessor(getStore(), RW) + , ciceAccessor(getStore(), RW) + , hsnowAccessor(getStore(), RW) { - getStore().registerArray(Shared::H_ICE_DG, &hice, RW); - getStore().registerArray(Shared::C_ICE_DG, &cice, RW); - getStore().registerArray(Shared::H_SNOW_DG, &hsnow, RW); } std::string getName() const override { return "PrognosticData"; } void setData(const ModelState::DataMap&) override { noLandMask(); - cice = 0.5; - hice = 0.1; // Cell averaged - hsnow = 0.01; // Cell averaged + ciceAccessor.getHostRW() = 0.5; + hiceAccessor.getHostRW() = 0.1; // Cell averaged + hsnowAccessor.getHostRW() = 0.01; // Cell averaged } - HField hice; - HField cice; - HField hsnow; + ModelArrayAccessor hiceAccessor; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor hsnowAccessor; } proData; proData.setData(ModelState().data); @@ -563,28 +579,30 @@ TEST_CASE("Turn off thermo") } void setData(const ModelState::DataMap& state) override { - qio = 73.9465; - sst = -1.75; - sss = 32.; - mld = 10.25; - u = 0.; - v = 0.; + qioAccessor.getHostRW() = 73.9465; + sstAccessor.getHostRW() = -1.75; + sssAccessor.getHostRW() = 32.; + mldAccessor.getHostRW() = 10.25; + uAccessor.getHostRW() = 0.; + vAccessor.getHostRW() = 0.; } void updateBefore(const TimestepTime& tst) override { UnescoFreezing uf; - cpml = Water::cp * Water::rho * mld; - tf = uf(sss[0]); + const HField& mld = mldAccessor.getHostRO(); + cpmlAccessor.getHostRW() = Water::cp * Water::rho * mld; + const HField& sss = sssAccessor.getHostRO(); + tfAccessor.getHostRW() = uf(sss[0]); } void updateAfter(const TimestepTime& tst) override { } } ocnBdy; ocnBdy.setData(ModelState().data); - HField damage(ModelArray::Type::H); - HField oldDamage(ModelArray::Type::H); - damage = 1; - ModelComponent::getStore().registerArray(Shared::DAMAGE, &damage, RW); - ModelComponent::getStore().registerArray(Protected::DAMAGE, &oldDamage, RO); + ModelArrayAccessor damageAccessor( + ModelComponent::getStore(), RW, ModelArray::Type::H); + ModelArrayAccessor oldDamageAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + damageAccessor.getHostRW() = 1; TimestepTime tst = { TimePoint("2000-001"), Duration("P0-0T0:10:0") }; IceGrowth ig; @@ -597,10 +615,14 @@ TEST_CASE("Turn off thermo") ocnBdy.updateBefore(tst); ig.update(tst); - ModelArrayRef newice(ModelComponent::getStore()); - ModelArrayRef hice(ModelComponent::getStore()); - ModelArrayRef cice(ModelComponent::getStore()); - ModelArrayRef hsnow(ModelComponent::getStore()); + ModelArrayAccessor newiceAccessor(ModelComponent::getStore()); + const HField& newice = newiceAccessor.getHostRO(); + ModelArrayAccessor hiceAccessor(ModelComponent::getStore()); + const HField& hice = hiceAccessor.getHostRO(); + ModelArrayAccessor ciceAccessor(ModelComponent::getStore()); + const HField& cice = ciceAccessor.getHostRO(); + ModelArrayAccessor hsnowAccessor(ModelComponent::getStore()); + const HField& hsnow = hsnowAccessor.getHostRO(); // double prec = 1e-5; diff --git a/physics/test/SlabOcean_test.cpp b/physics/test/SlabOcean_test.cpp index 59a52f9c5..6dc43245f 100644 --- a/physics/test/SlabOcean_test.cpp +++ b/physics/test/SlabOcean_test.cpp @@ -2,7 +2,6 @@ * @author Tim Spain */ -#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN #include #include "include/SlabOcean.hpp" @@ -25,68 +24,94 @@ TEST_CASE("Test Qdw") ModelArray::setDimensions(ModelArray::Type::H, { 1, 1 }); - ModelArrayReferenceStore couplingArrays; + ModelArrayStore couplingArrays; double tOffset = 0.001; // Supply the data to the slab ocean - HField sss(ModelArray::Type::H); + ModelArrayAccessor sssAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + HField& sss = sssAccessor.getHostRW(); sss = 32.; - ModelComponent::getStore().registerArray(Protected::SSS, &sss, RO); - HField sst(ModelArray::Type::H); + ModelArrayAccessor sstAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + HField& sst = sstAccessor.getHostRW(); sst = LinearFreezing()(sss[0]); - ModelComponent::getStore().registerArray(Protected::SST, &sst, RO); - HField mld(ModelArray::Type::H); + ModelArrayAccessor mldAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + HField& mld = mldAccessor.getHostRW(); mld = 6.48; - ModelComponent::getStore().registerArray(Protected::MLD, &mld, RO); - HField cpml(ModelArray::Type::H); + ModelArrayAccessor cpmlAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + HField& cpml = cpmlAccessor.getHostRW(); cpml = Water::cp * Water::rho * mld; - ModelComponent::getStore().registerArray(Protected::ML_BULK_CP, &cpml, RO); - HField cice(ModelArray::Type::H); + ModelArrayAccessor ciceAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + HField& cice = ciceAccessor.getHostRW(); double cice0 = 0.5; cice = cice0; - ModelComponent::getStore().registerArray(Shared::C_ICE_DG, &cice, RO); - HField data0(ModelArray::Type::H); + /* + ModelArrayAccessor data0Accessor( + couplingArrays, RO, ModelArray::Type::H); + HField& data0 = data0Accessor.getHostRW(); data0 = 0; - couplingArrays.registerArray(CouplingFields::Q_SS_NO_SW, &data0, RO); - couplingArrays.registerArray(CouplingFields::Q_SS_SW, &data0, RO); - couplingArrays.registerArray(CouplingFields::FWFLUX, &data0, RO); - couplingArrays.registerArray(CouplingFields::SFLUX, &data0, RO); + ModelArrayAccessor data1Accessor( + couplingArrays, RO, ModelArray::Type::H); + data1Accessor.getHostRW() = data0; + ModelArrayAccessor data2Accessor( + couplingArrays, RO, ModelArray::Type::H); + data2Accessor.getHostRW() = data0; + ModelArrayAccessor data3Accessor( + couplingArrays, RO, ModelArray::Type::H); + data3Accessor.getHostRW() = data0;*/ // External SS* data - HField sssExt(ModelArray::Type::H); + ModelArrayAccessor sssExtAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + HField& sssExt = sssExtAccessor.getHostRW(); sssExt = sss; - ModelComponent::getStore().registerArray(Protected::EXT_SSS, &sssExt, RO); - HField sstExt(ModelArray::Type::H); + ModelArrayAccessor sstExtAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + HField& sstExt = sstExtAccessor.getHostRW(); sstExt = sst + tOffset; - ModelComponent::getStore().registerArray(Protected::EXT_SST, &sstExt, RO); SlabOcean slabOcean(couplingArrays); slabOcean.configure(); slabOcean.update(tst); - ModelArrayRef qdw(ModelComponent::getStore()); + ModelArrayAccessor qdwAccessor(ModelComponent::getStore()); + const HField& qdw = qdwAccessor.getHostRO(); + double prec = 1e-8; REQUIRE(qdw[0] == doctest::Approx(tOffset * cpml[0] / SlabOcean::defaultRelaxationTime).epsilon(prec)); - ModelArrayRef sstSlab(ModelComponent::getStore()); - REQUIRE(sstSlab[0] != doctest::Approx(sst[0]).epsilon(prec / dt)); - REQUIRE(sstSlab[0] == doctest::Approx(sst[0] + dt * qdw[0] / cpml[0]).epsilon(prec)); + ModelArrayAccessor sstSlabAccessor(ModelComponent::getStore()); + // scope needed because we have to access sstSlab again after update + { + const HField& sstSlab = sstSlabAccessor.getHostRO(); + + REQUIRE(sstSlab[0] != doctest::Approx(sst[0]).epsilon(prec / dt)); + REQUIRE(sstSlab[0] == doctest::Approx(sst[0] + dt * qdw[0] / cpml[0]).epsilon(prec)); + } - HField qswNet(ModelArray::Type::H); + ModelArrayAccessor qswNetAccessor( + couplingArrays, RW, ModelArray::Type::H); + HField& qswNet = qswNetAccessor.getHostRW(); qswNet[0] = 15; - couplingArrays.registerArray(CouplingFields::Q_SS_SW, &qswNet, RW); - HField qNoSun(ModelArray::Type::H); + ModelArrayAccessor qNoSunAccessor( + couplingArrays, RW, ModelArray::Type::H); + HField& qNoSun = qNoSunAccessor.getHostRW(); qNoSun[0] = -17.5; - couplingArrays.registerArray(CouplingFields::Q_SS_NO_SW, &qNoSun, RW); + // Should not need to update anything else, as the slabOcean update only changes SLAB_SST slabOcean.update(tst); + const HField& sstSlab = sstSlabAccessor.getHostRO(); REQUIRE(sstSlab[0] == doctest::Approx(sst[0] - dt * (qswNet[0] + qNoSun[0] - qdw[0]) / cpml[0]).epsilon(prec)); } @@ -100,52 +125,68 @@ TEST_CASE("Test Fdw") ModelArray::setDimensions(ModelArray::Type::H, { 1, 1 }); - ModelArrayReferenceStore couplingArrays; + ModelArrayStore couplingArrays; double sOffset = 0.1; // Supply the data to the slab ocean - HField sss(ModelArray::Type::H); + ModelArrayAccessor sssAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + HField& sss = sssAccessor.getHostRW(); sss = 32.; - ModelComponent::getStore().registerArray(Protected::SSS, &sss, RO); - HField sst(ModelArray::Type::H); + ModelArrayAccessor sstAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + HField& sst = sstAccessor.getHostRW(); sst = LinearFreezing()(sss[0]); - ModelComponent::getStore().registerArray(Protected::SST, &sst, RO); - HField mld(ModelArray::Type::H); + ModelArrayAccessor mldAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + HField& mld = mldAccessor.getHostRW(); mld = 6.48; - ModelComponent::getStore().registerArray(Protected::MLD, &mld, RO); - HField cpml(ModelArray::Type::H); + ModelArrayAccessor cpmlAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + HField& cpml = cpmlAccessor.getHostRW(); cpml = Water::cp * Water::rho * mld; - ModelComponent::getStore().registerArray(Protected::ML_BULK_CP, &cpml, RO); - HField data0(ModelArray::Type::H); + /* + ModelArrayAccessor data0Accessor( + couplingArrays, RO, ModelArray::Type::H); + HField& data0 = data0Accessor.getHostRW(); data0 = 0; - couplingArrays.registerArray(CouplingFields::Q_SS_NO_SW, &data0, RO); - couplingArrays.registerArray(CouplingFields::Q_SS_SW, &data0, RO); - couplingArrays.registerArray(CouplingFields::FWFLUX, &data0, RO); - couplingArrays.registerArray(CouplingFields::SFLUX, &data0, RO); + ModelArrayAccessor data1Accessor( + couplingArrays, RO, ModelArray::Type::H); + data1Accessor.getHostRW() = data0; + ModelArrayAccessor data2Accessor( + couplingArrays, RO, ModelArray::Type::H); + data2Accessor.getHostRW() = data0; + ModelArrayAccessor data3Accessor( + couplingArrays, RO, ModelArray::Type::H); + data3Accessor.getHostRW() = data0;*/ // External SS* data - HField sssExt(ModelArray::Type::H); + ModelArrayAccessor sssExtAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + HField& sssExt = sssExtAccessor.getHostRW(); sssExt = sss + sOffset; - ModelComponent::getStore().registerArray(Protected::EXT_SSS, &sssExt, RO); - HField sstExt(ModelArray::Type::H); + ModelArrayAccessor sstExtAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + HField& sstExt = sstExtAccessor.getHostRW(); sstExt = sst; - ModelComponent::getStore().registerArray(Protected::EXT_SST, &sstExt, RO); SlabOcean slabOcean(couplingArrays); slabOcean.configure(); slabOcean.update(tst); - ModelArrayRef fdw(ModelComponent::getStore()); + ModelArrayAccessor fdwAccessor(ModelComponent::getStore()); + const HField& fdw = fdwAccessor.getHostRO(); + double prec = 1e-6; REQUIRE(fdw[0] == doctest::Approx( -sOffset / sss[0] * mld[0] * Water::rho / SlabOcean::defaultRelaxationTime) - .epsilon(prec)); + .epsilon(prec)); // Test that the finiteelement.cpp calculation of fdw is not being used double delS = -sOffset; double timeS = SlabOcean::defaultRelaxationTime; @@ -153,22 +194,25 @@ TEST_CASE("Test Fdw") double oldFdw = delS * mld[0] * Water::rho / (timeS * sss[0] - ddt * delS); REQUIRE(fdw[0] != doctest::Approx(oldFdw).epsilon(prec * 1e-6)); - ModelArrayRef sssSlab(ModelComponent::getStore()); + ModelArrayAccessor sssSlabAccessor(ModelComponent::getStore()); + const HField& sssSlab = sssSlabAccessor.getHostRO(); + REQUIRE(sssSlab[0] != doctest::Approx(sss[0]).epsilon(prec / dt)); REQUIRE(sssSlab[0] == doctest::Approx(sss[0] - (fdw[0] * dt) / (mld[0] * Water::rho + fdw[0] * dt)) - .epsilon(prec)); + .epsilon(prec)); - HField snowMeltFlux(ModelArray::Type::H); + ModelArrayAccessor snowMeltFluxAccessor( + couplingArrays, RW, ModelArray::Type::H); + HField& snowMeltFlux = snowMeltFluxAccessor.getHostRW(); double snowMelt = -1e-4; double snowMeltVol = snowMelt * Ice::rhoSnow; snowMeltFlux = snowMeltVol / dt; - couplingArrays.registerArray(CouplingFields::FWFLUX, &snowMeltFlux, RW); slabOcean.update(tst); REQUIRE(sssSlab[0] == doctest::Approx(sss[0] + (snowMeltVol - fdw[0] * dt) / (mld[0] * Water::rho - snowMeltVol + fdw[0] * dt)) - .epsilon(prec)); + .epsilon(prec)); } TEST_SUITE_END(); } /* namespace Nextsim */ diff --git a/physics/test/TOPAZOcn_test.cpp b/physics/test/TOPAZOcn_test.cpp index f53a03957..a1f77b079 100644 --- a/physics/test/TOPAZOcn_test.cpp +++ b/physics/test/TOPAZOcn_test.cpp @@ -41,20 +41,21 @@ TEST_CASE("TOPAZOcean test") topaz.configure(); topaz.setFilePath(filePath); - ModelArrayRef sst(ModelComponent::getStore()); - ModelArrayRef sss(ModelComponent::getStore()); - ModelArrayRef mld(ModelComponent::getStore()); - ModelArrayRef u(ModelComponent::getStore()); - ModelArrayRef v(ModelComponent::getStore()); - ModelArrayRef ssh(ModelComponent::getStore()); + ModelArrayAccessor sstAccessor(ModelComponent::getStore()); + ModelArrayAccessor sssAccessor(ModelComponent::getStore()); + ModelArrayAccessor mldAccessor(ModelComponent::getStore()); + ModelArrayAccessor uAccessor(ModelComponent::getStore()); + ModelArrayAccessor vAccessor(ModelComponent::getStore()); + ModelArrayAccessor sshAccessor(ModelComponent::getStore()); TimePoint t1("2000-01-01T00:00:00Z"); TimestepTime tst = { t1, Duration(600) }; // The Qio calculation requires c_ice data - HField cice(ModelArray::Type::H); + ModelArrayAccessor ciceAccessor( + ModelComponent::getStore(), RO, ModelArray::Type::H); + HField& cice = ciceAccessor.getHostRW(); cice = 0.; - ModelComponent::getStore().registerArray(Shared::C_ICE_DG, &cice, RO); // Get the forcing fields at time 0 topaz.updateBefore(tst); @@ -64,39 +65,71 @@ TEST_CASE("TOPAZOcean test") // Use this, rather than the literal 0.035045, as the two are not equal at double precision double targetFrac = 35 * 0.001 + 45 * 0.000001; - REQUIRE(sst(0, 0) == mdi); - REQUIRE(sst(32, 32) == -0.032032); - REQUIRE(sst(45, 35) == -(0 + targetFrac)); - REQUIRE(mld(45, 35) == (10 + targetFrac)); - REQUIRE(ssh(45, 35) == (20 + targetFrac)); + { + const HField& sst = sstAccessor.getHostRO(); + const HField& sss = sssAccessor.getHostRO(); + const HField& mld = mldAccessor.getHostRO(); + const HField& u = uAccessor.getHostRO(); + const HField& v = vAccessor.getHostRO(); + const HField& ssh = sshAccessor.getHostRO(); + REQUIRE(sst(0, 0) == mdi); + REQUIRE(sst(32, 32) == -0.032032); + REQUIRE(sst(45, 35) == -(0 + targetFrac)); + REQUIRE(mld(45, 35) == (10 + targetFrac)); + REQUIRE(ssh(45, 35) == (20 + targetFrac)); + } TimePoint t2("2000-02-01T00:00:00Z"); topaz.updateBefore({ t2, Duration(600) }); - REQUIRE(sst(0, 0) == mdi); - REQUIRE(sst(32, 32) == -0.032032 - 1); - REQUIRE(sst(45, 35) == -(0 + targetFrac) - 1); - REQUIRE(mld(45, 35) == (10 + targetFrac) + 1); - REQUIRE(ssh(45, 35) == (20 + targetFrac) + 1); + { + const HField& sst = sstAccessor.getHostRO(); + const HField& sss = sssAccessor.getHostRO(); + const HField& mld = mldAccessor.getHostRO(); + const HField& u = uAccessor.getHostRO(); + const HField& v = vAccessor.getHostRO(); + const HField& ssh = sshAccessor.getHostRO(); + REQUIRE(sst(0, 0) == mdi); + REQUIRE(sst(32, 32) == -0.032032 - 1); + REQUIRE(sst(45, 35) == -(0 + targetFrac) - 1); + REQUIRE(mld(45, 35) == (10 + targetFrac) + 1); + REQUIRE(ssh(45, 35) == (20 + targetFrac) + 1); + } TimePoint t12("2000-12-01T00:00:00Z"); topaz.updateBefore({ t12, Duration(600) }); - REQUIRE(sst(0, 0) == mdi); - REQUIRE(sst(32, 32) == -0.032032 - 11); - REQUIRE(sst(45, 35) == -(0 + targetFrac) - 11); - REQUIRE(mld(45, 35) == (10 + targetFrac) + 11); - REQUIRE(ssh(45, 35) == (20 + targetFrac) + 11); + { + const HField& sst = sstAccessor.getHostRO(); + const HField& sss = sssAccessor.getHostRO(); + const HField& mld = mldAccessor.getHostRO(); + const HField& u = uAccessor.getHostRO(); + const HField& v = vAccessor.getHostRO(); + const HField& ssh = sshAccessor.getHostRO(); + REQUIRE(sst(0, 0) == mdi); + REQUIRE(sst(32, 32) == -0.032032 - 11); + REQUIRE(sst(45, 35) == -(0 + targetFrac) - 11); + REQUIRE(mld(45, 35) == (10 + targetFrac) + 11); + REQUIRE(ssh(45, 35) == (20 + targetFrac) + 11); + } // All times after the last time sample should use the last sample's data TimePoint t120("2010-01-01T00:00:00Z"); topaz.updateBefore({ t120, Duration(600) }); - REQUIRE(sst(0, 0) == mdi); - REQUIRE(sst(32, 32) == -0.032032 - 11); - REQUIRE(sst(45, 35) == -(0 + targetFrac) - 11); - REQUIRE(mld(45, 35) == (10 + targetFrac) + 11); - REQUIRE(ssh(45, 35) == (20 + targetFrac) + 11); + { + const HField& sst = sstAccessor.getHostRO(); + const HField& sss = sssAccessor.getHostRO(); + const HField& mld = mldAccessor.getHostRO(); + const HField& u = uAccessor.getHostRO(); + const HField& v = vAccessor.getHostRO(); + const HField& ssh = sshAccessor.getHostRO(); + REQUIRE(sst(0, 0) == mdi); + REQUIRE(sst(32, 32) == -0.032032 - 11); + REQUIRE(sst(45, 35) == -(0 + targetFrac) - 11); + REQUIRE(mld(45, 35) == (10 + targetFrac) + 11); + REQUIRE(ssh(45, 35) == (20 + targetFrac) + 11); + } std::filesystem::remove(filePath); } diff --git a/physics/test/ThermoIce0_test.cpp b/physics/test/ThermoIce0_test.cpp index 0107b2499..0a50bfd93 100644 --- a/physics/test/ThermoIce0_test.cpp +++ b/physics/test/ThermoIce0_test.cpp @@ -2,7 +2,6 @@ * @author Tim Spain */ -#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN #include #include @@ -32,86 +31,73 @@ TEST_CASE("Threshold ice") class IceTemperatureData : public ModelComponent { public: IceTemperatureData() + : hiceAccessor(getStore(), RW) + , ciceAccessor(getStore(), RW) + , hsnowAccessor(getStore(), RW) + , sstAccessor(getStore(), RO) + , sssAccessor(getStore(), RO) + , tfAccessor(getStore(), RO) + , snowAccessor(getStore(), RO) + , mlbhcAccessor(getStore(), RO) + , qioAccessor(getStore(), RW) + , qowAccessor(getStore(), RW) + , qiaAccessor(getStore(), RW) + , dqia_dtAccessor(getStore(), RW) + , sublAccessor(getStore(), RW) + , penSWAccessor(getStore(), RW) { - getStore().registerArray(Shared::H_ICE_DG, &hice, RW); - getStore().registerArray(Shared::C_ICE_DG, &cice, RW); - getStore().registerArray(Shared::H_SNOW_DG, &hsnow, RW); - getStore().registerArray(Protected::SST, &sst, RO); - getStore().registerArray(Protected::SSS, &sss, RO); - getStore().registerArray(Protected::TF, &tf, RO); - getStore().registerArray(Protected::SNOW, &snow, RO); - getStore().registerArray(Protected::ML_BULK_CP, &mlbhc, RO); - getStore().registerArray(Shared::Q_IO, &qio, RW); - getStore().registerArray(Shared::Q_OW, &qow, RW); - getStore().registerArray(Shared::Q_IA, &qia, RW); - getStore().registerArray(Shared::DQIA_DT, &dqia_dt, RW); - getStore().registerArray(Shared::SUBLIM, &subl, RW); - getStore().registerArray(Shared::Q_PEN_SW, &penSW, RW); } std::string getName() const override { return "IceTemperatureData"; } void setData(const ModelState::DataMap&) override { - cice[0] = 0.99; + ciceAccessor.getHostRW()[0] = 0.99; + HField& hice = hiceAccessor.getHostRW(); hice[0] = 0.001; hice0[0] = hice[0]; - hsnow[0] = 0.; + hsnowAccessor.getHostRW()[0] = 0.; + HField& sss = sssAccessor.getHostRW(); sss[0] = 32.; - sst[0] = Module::getImplementation()(sss[0]); - snow[0] = 0.; - tf[0] = Module::getImplementation()(sss[0]); - mlbhc[0] = 4.29151e7; - qio[0] = 0.; - qow[0] = 0; - qia[0] = 0; - dqia_dt[0] = 0; - subl[0] = 0; - penSW[0] = 0; + sstAccessor.getHostRW()[0] = Module::getImplementation()(sss[0]); + snowAccessor.getHostRW()[0] = 0.; + tfAccessor.getHostRW()[0] = Module::getImplementation()(sss[0]); + mlbhcAccessor.getHostRW()[0] = 4.29151e7; + qioAccessor.getHostRW()[0] = 0.; + qowAccessor.getHostRW()[0] = 0; + qiaAccessor.getHostRW()[0] = 0; + dqia_dtAccessor.getHostRW()[0] = 0; + sublAccessor.getHostRW()[0] = 0; + penSWAccessor.getHostRW()[0] = 0; } - HField hice; + ModelArrayAccessor hiceAccessor; HField hice0; - HField cice; - HField hsnow; - HField sst; - HField sss; - HField tf; - HField snow; - HField mlbhc; // Mixed layer bulk heat capacity - HField qio; - HField qow; - HField qia; - HField dqia_dt; - HField subl; - HField penSW; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor hsnowAccessor; + ModelArrayAccessor sstAccessor; + ModelArrayAccessor sssAccessor; + ModelArrayAccessor tfAccessor; + ModelArrayAccessor snowAccessor; + ModelArrayAccessor + mlbhcAccessor; // Mixed layer bulk heat capacity + ModelArrayAccessor qioAccessor; + ModelArrayAccessor qowAccessor; + ModelArrayAccessor qiaAccessor; + ModelArrayAccessor dqia_dtAccessor; + ModelArrayAccessor sublAccessor; + ModelArrayAccessor penSWAccessor; } atmoState; atmoState.setData(ModelState::DataMap()); // Supply the atmospheric boundary arrays, without an entire // IAtmosphereBoundary implementation - HField qow; - qow.resize(); - ModelComponent::getStore().registerArray(Shared::Q_OW, &qow, RW); - - HField qia; - qia.resize(); - ModelComponent::getStore().registerArray(Shared::Q_IA, &qia, RW); - - HField dqia_dt; - dqia_dt.resize(); - ModelComponent::getStore().registerArray(Shared::DQIA_DT, &dqia_dt, RW); - - HField qic; + ModelArrayAccessor qicAccessor(ModelComponent::getStore(), RW); + HField& qic = qicAccessor.getHostRW(); qic.resize(); - ModelComponent::getStore().registerArray(Shared::Q_IC, &qic, RW); - HField subl; - subl.resize(); - ModelComponent::getStore().registerArray(Shared::SUBLIM, &subl, RW); - - HField qswbase; + ModelArrayAccessor qswbaseAccessor(ModelComponent::getStore(), RW); + HField& qswbase = qswbaseAccessor.getHostRW(); qswbase.resize(); - ModelComponent::getStore().registerArray(Shared::Q_SW_BASE, &qswbase, RW); // An implementation of IFluxCalculation that returns zero fluxes class FluxData : public IFluxCalculation { @@ -124,11 +110,11 @@ TEST_CASE("Threshold ice") void setData(const ModelState::DataMap&) override { - qow[0] = 0; - qia[0] = 0; - dqia_dt[0] = 0; - subl[0] = 0; - penSW[0] = 0; + qowAccessor.getHostRW()[0] = 0; + qiaAccessor.getHostRW()[0] = 0; + dqia_dtAccessor.getHostRW()[0] = 0; + sublAccessor.getHostRW()[0] = 0; + penSWAccessor.getHostRW()[0] = 0; } void update(const TimestepTime&) override { } @@ -145,10 +131,14 @@ TEST_CASE("Threshold ice") ti0t.setData({ { tsurfName, tSurf } }); ti0t.update(tst); - ModelArrayRef hice(ModelComponent::getStore()); + ModelArrayAccessor hiceAccessor(ModelComponent::getStore()); + const HField& hice = hiceAccessor.getHostRO(); + // So little ice should be reduced to zero REQUIRE(hice[0] == 0.); - ModelArrayRef cice(ModelComponent::getStore()); + ModelArrayAccessor ciceAccessor(ModelComponent::getStore()); + const HField& cice = ciceAccessor.getHostRO(); + REQUIRE(cice[0] == 0.); } TEST_SUITE_END(); diff --git a/physics/test/ThermoWintonTemperature_test.cpp b/physics/test/ThermoWintonTemperature_test.cpp index a6b9f1a0e..e5e1f56a3 100644 --- a/physics/test/ThermoWintonTemperature_test.cpp +++ b/physics/test/ThermoWintonTemperature_test.cpp @@ -2,7 +2,6 @@ * @author Tim Spain */ -#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN #include #include @@ -15,7 +14,6 @@ #include "include/IIceAlbedo.hpp" #include "include/IOceanBoundary.hpp" #include "include/ModelArray.hpp" -#include "include/ModelArrayRef.hpp" #include "include/ModelComponent.hpp" #include "include/NextsimModule.hpp" #include "include/Time.hpp" @@ -46,27 +44,27 @@ TEST_CASE("Melting conditions") class IceTemperatureData : public ModelComponent { public: IceTemperatureData() + : sw_inAccessor(getStore(), RO) + , hiceAccessor(getStore(), RW) + , ciceAccessor(getStore(), RW) + , hsnowAccessor(getStore(), RW) { - getStore().registerArray(Protected::SW_IN, &sw_in, RO); - getStore().registerArray(Shared::H_ICE_DG, &hice, RW); - getStore().registerArray(Shared::C_ICE_DG, &cice, RW); - getStore().registerArray(Shared::H_SNOW_DG, &hsnow, RW); } std::string getName() const override { return "IceTemperatureData"; } void setData(const ModelState::DataMap&) override { - cice[0] = 0.5; - hice[0] = 0.1; - hsnow[0] = 0.01; - sw_in[0] = -10.1675; // Net shortwave flux from incident 50 W/m^2 + ciceAccessor.getHostRW()[0] = 0.5; + hiceAccessor.getHostRW()[0] = 0.1; + hsnowAccessor.getHostRW()[0] = 0.01; + sw_inAccessor.getHostRW()[0] = -10.1675; // Net shortwave flux from incident 50 W/m^2 } - HField sw_in; + ModelArrayAccessor sw_inAccessor; - HField hice; - HField cice; - HField hsnow; + ModelArrayAccessor hiceAccessor; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor hsnowAccessor; } initCond; initCond.setData(ModelState().data); @@ -79,12 +77,12 @@ TEST_CASE("Melting conditions") void setData(const ModelState::DataMap& ms) override { IAtmosphereBoundary::setData(ms); - snow[0] = 0.00; - qow[0] = -109.923; - qia[0] = -85.6364; - dqia_dt[0] = 19.7016; - subl[0] = -7.3858e-06; - penSW[0] = 1.04125; + snowAccessor.getHostRW()[0] = 0.00; + qowAccessor.getHostRW()[0] = -109.923; + qiaAccessor.getHostRW()[0] = -85.6364; + dqia_dtAccessor.getHostRW()[0] = 19.7016; + sublAccessor.getHostRW()[0] = -7.3858e-06; + penSWAccessor.getHostRW()[0] = 1.04125; } } atmosState; atmosState.setData(ModelState().data); @@ -137,12 +135,12 @@ TEST_CASE("Freezing conditions") class IceTemperatureData : public ModelComponent { public: IceTemperatureData() + : snowAccessor(getStore(), RO) + , sw_inAccessor(getStore(), RO) + , hiceAccessor(getStore(), RW) + , ciceAccessor(getStore(), RW) + , hsnowAccessor(getStore(), RW) { - getStore().registerArray(Protected::SNOW, &snow, RO); - getStore().registerArray(Protected::SW_IN, &sw_in, RO); - getStore().registerArray(Shared::H_ICE_DG, &hice, RW); - getStore().registerArray(Shared::C_ICE_DG, &cice, RW); - getStore().registerArray(Shared::H_SNOW_DG, &hsnow, RW); } std::string getName() const override { return "IceTemperatureData"; } @@ -151,23 +149,23 @@ TEST_CASE("Freezing conditions") cice0[0] = 0.5; hice0[0] = 0.1; hsnow0[0] = 0.01; - snow[0] = 1e-3; - sw_in[0] = 0; + snowAccessor.getHostRW()[0] = 1e-3; + sw_inAccessor.getHostRW()[0] = 0; - hice = hice0; - cice = cice0; - hsnow = hsnow0; + hiceAccessor.getHostRW() = hice0; + ciceAccessor.getHostRW() = cice0; + hsnowAccessor.getHostRW() = hsnow0; } HField hice0; HField cice0; HField hsnow0; - HField snow; - HField sw_in; + ModelArrayAccessor snowAccessor; + ModelArrayAccessor sw_inAccessor; - HField hice; - HField cice; - HField hsnow; + ModelArrayAccessor hiceAccessor; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor hsnowAccessor; } atmoState; atmoState.setData(ModelState().data); @@ -180,12 +178,12 @@ TEST_CASE("Freezing conditions") void setData(const ModelState::DataMap& ms) override { IAtmosphereBoundary::setData(ms); - snow[0] = 0.00; - qow[0] = 143.266; - qia[0] = 42.2955; - dqia_dt[0] = 16.7615; - subl[0] = 2.15132e-6; - penSW[0] = 0.; + snowAccessor.getHostRW()[0] = 0.00; + qowAccessor.getHostRW()[0] = 143.266; + qiaAccessor.getHostRW()[0] = 42.2955; + dqia_dtAccessor.getHostRW()[0] = 16.7615; + sublAccessor.getHostRW()[0] = 2.15132e-6; + penSWAccessor.getHostRW()[0] = 0.; } } atmosState; atmosState.setData(ModelState().data); @@ -238,13 +236,12 @@ TEST_CASE("No ice do nothing") class IceTemperatureData : public ModelComponent { public: IceTemperatureData() + : snowAccessor(getStore(), RO) + , sw_inAccessor(getStore(), RO) + , hiceAccessor(getStore(), RW) + , ciceAccessor(getStore(), RW) + , hsnowAccessor(getStore(), RW) { - getStore().registerArray(Protected::SNOW, &snow, RO); - getStore().registerArray(Protected::SW_IN, &sw_in, RO); - - getStore().registerArray(Shared::H_ICE_DG, &hice, RW); - getStore().registerArray(Shared::C_ICE_DG, &cice, RW); - getStore().registerArray(Shared::H_SNOW_DG, &hsnow, RW); } std::string getName() const override { return "IceTemperatureData"; } @@ -253,23 +250,23 @@ TEST_CASE("No ice do nothing") cice0[0] = 0; hice0[0] = 0; hsnow0[0] = 0; - snow[0] = 0; - sw_in[0] = 0; + snowAccessor.getHostRW()[0] = 0; + sw_inAccessor.getHostRW()[0] = 0; - hice = hice0; - cice = cice0; - hsnow = hsnow0; + hiceAccessor.getHostRW() = hice0; + ciceAccessor.getHostRW() = cice0; + hsnowAccessor.getHostRW() = hsnow0; } HField hice0; HField cice0; HField hsnow0; - HField snow; - HField sw_in; + ModelArrayAccessor snowAccessor; + ModelArrayAccessor sw_inAccessor; - HField hice; - HField cice; - HField hsnow; + ModelArrayAccessor hiceAccessor; + ModelArrayAccessor ciceAccessor; + ModelArrayAccessor hsnowAccessor; } atmoState; atmoState.setData(ModelState().data); @@ -278,11 +275,12 @@ TEST_CASE("No ice do nothing") void setData(const ModelState::DataMap& ms) override { IOceanBoundary::setData(ms); - sst[0] = 1.75; + sstAccessor.getHostRW()[0] = 1.75; + HField& sss = sssAccessor.getHostRW(); sss[0] = 32.; - tf[0] = Module::getImplementation()(sss[0]); - cpml[0] = 4.29151e7; - qio[0] = 0; + tfAccessor.getHostRW()[0] = Module::getImplementation()(sss[0]); + cpmlAccessor.getHostRW()[0] = 4.29151e7; + qioAccessor.getHostRW()[0] = 0; } void updateBefore(const TimestepTime& tst) override { } void updateAfter(const TimestepTime& tst) override { } @@ -294,11 +292,11 @@ TEST_CASE("No ice do nothing") void setData(const ModelState::DataMap& ms) override { IAtmosphereBoundary::setData(ms); - snow[0] = 0.00; - qow[0] = 143.266; - qia[0] = 42.2955; - dqia_dt[0] = 16.7615; - subl[0] = 2.15132e-6; + snowAccessor.getHostRW()[0] = 0.00; + qowAccessor.getHostRW()[0] = 143.266; + qiaAccessor.getHostRW()[0] = 42.2955; + dqia_dtAccessor.getHostRW()[0] = 16.7615; + sublAccessor.getHostRW()[0] = 2.15132e-6; } } atmosState; atmosState.setData(ModelState().data); @@ -318,10 +316,12 @@ TEST_CASE("No ice do nothing") twin.configure(); twin.update(tst); - ModelArrayRef hice(ModelComponent::getStore()); - ModelArrayRef cice(ModelComponent::getStore()); + ModelArrayAccessor hiceAccessor(ModelComponent::getStore()); + const HField& hice = hiceAccessor.getHostRO(); + ModelArrayAccessor ciceAccessor(ModelComponent::getStore()); + const HField& cice = ciceAccessor.getHostRO(); -// double prec = 1e-5; + // double prec = 1e-5; REQUIRE(hice[0] == 0); REQUIRE(cice[0] == 0); diff --git a/physics/test/UniformOcean_test.cpp b/physics/test/UniformOcean_test.cpp index c7fffca52..44d9a72e6 100644 --- a/physics/test/UniformOcean_test.cpp +++ b/physics/test/UniformOcean_test.cpp @@ -31,14 +31,22 @@ TEST_CASE("UniformOcean construction") UniformOcean uniOcn(sstIn, sssIn, mldIn, uIn, vIn, qioIn); uniOcn.setData(ModelState::DataMap()); - ModelArrayRef sst(ModelComponent::getStore()); - ModelArrayRef sss(ModelComponent::getStore()); - ModelArrayRef mld(ModelComponent::getStore()); - ModelArrayRef u(ModelComponent::getStore()); - ModelArrayRef v(ModelComponent::getStore()); - ModelArrayRef qio(ModelComponent::getStore()); - ModelArrayRef cpml(ModelComponent::getStore()); - ModelArrayRef tf(ModelComponent::getStore()); + ModelArrayAccessor sstAccessor(ModelComponent::getStore()); + const HField& sst = sstAccessor.getHostRO(); + ModelArrayAccessor sssAccessor(ModelComponent::getStore()); + const HField& sss = sssAccessor.getHostRO(); + ModelArrayAccessor mldAccessor(ModelComponent::getStore()); + const HField& mld = mldAccessor.getHostRO(); + ModelArrayAccessor uAccessor(ModelComponent::getStore()); + const HField& u = uAccessor.getHostRO(); + ModelArrayAccessor vAccessor(ModelComponent::getStore()); + const HField& v = vAccessor.getHostRO(); + ModelArrayAccessor qioAccessor(ModelComponent::getStore()); + const HField& qio = qioAccessor.getHostRO(); + ModelArrayAccessor cpmlAccessor(ModelComponent::getStore()); + const HField& cpml = cpmlAccessor.getHostRO(); + ModelArrayAccessor tfAccessor(ModelComponent::getStore()); + const HField& tf = tfAccessor.getHostRO(); REQUIRE(sst[0] == sstIn); REQUIRE(sss[0] == sssIn); @@ -67,14 +75,22 @@ TEST_CASE("UniformOcean set functions") uniOcn.setSST(sstIn).setSSS(sssIn).setMLD(mldIn).setU(uIn).setV(vIn).setQio(qioIn); uniOcn.setData(ModelState::DataMap()); - ModelArrayRef sst(ModelComponent::getStore()); - ModelArrayRef sss(ModelComponent::getStore()); - ModelArrayRef mld(ModelComponent::getStore()); - ModelArrayRef u(ModelComponent::getStore()); - ModelArrayRef v(ModelComponent::getStore()); - ModelArrayRef qio(ModelComponent::getStore()); - ModelArrayRef cpml(ModelComponent::getStore()); - ModelArrayRef tf(ModelComponent::getStore()); + ModelArrayAccessor sstAccessor(ModelComponent::getStore()); + const HField& sst = sstAccessor.getHostRO(); + ModelArrayAccessor sssAccessor(ModelComponent::getStore()); + const HField& sss = sssAccessor.getHostRO(); + ModelArrayAccessor mldAccessor(ModelComponent::getStore()); + const HField& mld = mldAccessor.getHostRO(); + ModelArrayAccessor uAccessor(ModelComponent::getStore()); + const HField& u = uAccessor.getHostRO(); + ModelArrayAccessor vAccessor(ModelComponent::getStore()); + const HField& v = vAccessor.getHostRO(); + ModelArrayAccessor qioAccessor(ModelComponent::getStore()); + const HField& qio = qioAccessor.getHostRO(); + ModelArrayAccessor cpmlAccessor(ModelComponent::getStore()); + const HField& cpml = cpmlAccessor.getHostRO(); + ModelArrayAccessor tfAccessor(ModelComponent::getStore()); + const HField& tf = tfAccessor.getHostRO(); REQUIRE(sst[0] == sstIn); REQUIRE(sss[0] == sssIn);