Skip to content

Restricted Additive Schwarz with multiple layers of overlap #2369

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Restricted Additive Schwarz with multiple layers of overlap #2369

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e2d3036
Enable multiple overlap layers and restricted overlapping schwartz
andrthu Feb 7, 2020
0e3e072
Use parallel cell index set to seperate ghost/overlap cells.
andrthu Feb 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion CMakeLists_files.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,6 @@ list (APPEND PUBLIC_HEADER_FILES
opm/simulators/linalg/OwningBlockPreconditioner.hpp
opm/simulators/linalg/OwningTwoLevelPreconditioner.hpp
opm/simulators/linalg/ParallelOverlappingILU0.hpp
opm/simulators/linalg/ParallelRestrictedAdditiveSchwarz.hpp
opm/simulators/linalg/ParallelIstlInformation.hpp
opm/simulators/linalg/PressureSolverPolicy.hpp
opm/simulators/linalg/PressureTransferPolicy.hpp
Expand Down
13 changes: 13 additions & 0 deletions ebos/eclbasevanguard.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,13 +77,15 @@ NEW_PROP_TAG(EclOutputInterval);
NEW_PROP_TAG(IgnoreKeywords);
NEW_PROP_TAG(EnableExperiments);
NEW_PROP_TAG(EdgeWeightsMethod);
NEW_PROP_TAG(OverlapLayers);

SET_STRING_PROP(EclBaseVanguard, IgnoreKeywords, "");
SET_STRING_PROP(EclBaseVanguard, EclDeckFileName, "");
SET_INT_PROP(EclBaseVanguard, EclOutputInterval, -1); // use the deck-provided value
SET_BOOL_PROP(EclBaseVanguard, EnableOpmRstFile, false);
SET_BOOL_PROP(EclBaseVanguard, EclStrictParsing, false);
SET_INT_PROP(EclBaseVanguard, EdgeWeightsMethod, 1);
SET_INT_PROP(EclBaseVanguard, OverlapLayers, 1);

END_PROPERTIES

Expand Down Expand Up @@ -129,6 +131,8 @@ public:
"Use strict mode for parsing - all errors are collected before the applicaton exists.");
EWOMS_REGISTER_PARAM(TypeTag, int, EdgeWeightsMethod,
"Choose edge-weighing strategy: 0=uniform, 1=trans, 2=log(trans).");
EWOMS_REGISTER_PARAM(TypeTag, int, OverlapLayers,
"Number of overlap layers added between subdomains in the parallel grid. If set to larger then 1, Restricted Additive Schwarz (RAS) will be used as parallel preconditioner instead of the default Block-Jacobi method. Expect lower number of linear solver iterations at the cost of increased parallel overhead.");
}

/*!
Expand Down Expand Up @@ -261,6 +265,7 @@ public:

std::string fileName = EWOMS_GET_PARAM(TypeTag, std::string, EclDeckFileName);
edgeWeightsMethod_ = Dune::EdgeWeightMethod(EWOMS_GET_PARAM(TypeTag, int, EdgeWeightsMethod));
overlapLayers_ = EWOMS_GET_PARAM(TypeTag, int, OverlapLayers);

// Skip processing of filename if external deck already exists.
if (!externalDeck_)
Expand Down Expand Up @@ -449,6 +454,13 @@ public:
*/
Dune::EdgeWeightMethod edgeWeightsMethod() const
{ return edgeWeightsMethod_; }

/*
* \brief Parameter deciding the number of overlap layers between subdomains in parallel runs.
*/
int overlapLayers() const
{return overlapLayers_;}

/*!
* \brief Returns the name of the case.
*
Expand Down Expand Up @@ -608,6 +620,7 @@ private:
Opm::SummaryConfig* eclSummaryConfig_;

Dune::EdgeWeightMethod edgeWeightsMethod_;
int overlapLayers_;
};

template <class TypeTag>
Expand Down
3 changes: 2 additions & 1 deletion ebos/eclcpgridvanguard.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ public:
}

Dune::EdgeWeightMethod edgeWeightsMethod = this->edgeWeightsMethod();
int overlapLayers = this->overlapLayers();

// convert to transmissibility for faces
// TODO: grid_->numFaces() is not generic. use grid_->size(1) instead? (might
Expand Down Expand Up @@ -187,7 +188,7 @@ public:
//distribute the grid and switch to the distributed view.
{
const auto wells = this->schedule().getWellsatEnd();
defunctWellNames_ = std::get<1>(grid_->loadBalance(edgeWeightsMethod, &wells, faceTrans.data()));
defunctWellNames_ = std::get<1>(grid_->loadBalance(edgeWeightsMethod, &wells, faceTrans.data(), overlapLayers));
}
grid_->switchToDistributedView();

Expand Down
1 change: 0 additions & 1 deletion opm/simulators/linalg/CPRPreconditioner.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@
#include <opm/common/ErrorMacros.hpp>
#include <opm/common/Exceptions.hpp>

#include <opm/simulators/linalg/ParallelRestrictedAdditiveSchwarz.hpp>
#include <opm/simulators/linalg/ParallelOverlappingILU0.hpp>
namespace Opm
{
Expand Down
11 changes: 9 additions & 2 deletions opm/simulators/linalg/ISTLSolverEbos.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
#include <opm/simulators/linalg/MatrixBlock.hpp>
#include <opm/simulators/linalg/BlackoilAmg.hpp>
#include <opm/simulators/linalg/CPRPreconditioner.hpp>
#include <opm/simulators/linalg/ParallelRestrictedAdditiveSchwarz.hpp>
#include <opm/simulators/linalg/ParallelOverlappingILU0.hpp>
#include <opm/simulators/linalg/ExtractParallelGridInformationToISTL.hpp>
#include <opm/simulators/linalg/findOverlapRowsAndColumns.hpp>
Expand Down Expand Up @@ -257,6 +256,8 @@ class WellModelMatrixAdapter : public Dune::AssembledLinearOperator<M,X,Y>
const auto wellsForConn = simulator_.vanguard().schedule().getWellsatEnd();
const bool useWellConn = EWOMS_GET_PARAM(TypeTag, bool, MatrixAddWellContributions);

overlapLayers_ = EWOMS_GET_PARAM(TypeTag, int, OverlapLayers);

detail::setWellConnections(gridForConn, wellsForConn, useWellConn, wellConnectionsGraph_);
detail::findOverlapAndInterior(gridForConn, overlapRows_, interiorRows_);
if (gridForConn.comm().size() > 1) {
Expand Down Expand Up @@ -391,6 +392,11 @@ class WellModelMatrixAdapter : public Dune::AssembledLinearOperator<M,X,Y>
// Construct scalar product.
auto sp = Dune::createScalarProduct<Vector,POrComm>(parallelInformation_arg, category);

// If overlapLayers_ > 1 we are using Restricted Additive Schwarz and therefore need
// the residual at overlap DoFs.
if (overlapLayers_ > 1)
parallelInformation_arg.copyOwnerToAll(istlb, istlb);
Copy link
Member

@blattms blattms Feb 28, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please explain why this is only needed for more than one overlap layers. Shouldn't the right hand side always be consistent (all procs have the same values?)

Copy link
Member

@hnil hnil Apr 3, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The right hand side is normally unique, which is correct with only one overlap since this will give Dirichlet condition for the copy (?). I think the problem is that of some reason the residual is only calculated for the owner cells since this is the only needed for the original solvers. With this copying and the additional setting of zero for the copy attribute (i.e. "useless overlap") the right hand side is correct. The correct thing is to assemble for all owner and overlap (I would have prefered all) and then set it to zero (at the same time as the matrix elements is fixed to get correct Dirichlet condition). If amg solver is used I think a change in the redistribution function is needed to get correct behavoir if the coarse scale communicator has overlap attributes.


#if FLOW_SUPPORT_AMG // activate AMG if either flow_ebos is used or UMFPack is not available
if( parameters_.linear_solver_use_amg_ || parameters_.use_cpr_)
{
Expand Down Expand Up @@ -688,7 +694,7 @@ class WellModelMatrixAdapter : public Dune::AssembledLinearOperator<M,X,Y>
pattern[idx].insert(*wc);

// Add just a single element to ghost rows
if (elem.partitionType() != Dune::InteriorEntity)
if (elem.partitionType() == Dune::GhostEntity)
Copy link
Member

@blattms blattms Feb 28, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is totally unrelated to this PR but it just struck me when reading the diff:

Should we move the for-loop over the wellConnectionsGraph_ to else branch and explicitly use noGhostMat_->setrowsize(1);. That seems clearer and might prevent surprises if we distribute the wells.

{
noGhostMat_->setrowsize(idx, pattern[idx].size());
}
Expand Down Expand Up @@ -949,6 +955,7 @@ class WellModelMatrixAdapter : public Dune::AssembledLinearOperator<M,X,Y>
std::vector<int> overlapRows_;
std::vector<int> interiorRows_;
std::vector<std::set<int>> wellConnectionsGraph_;
int overlapLayers_;
FlowLinearSolverParameters parameters_;
Vector weights_;
bool scale_variables_;
Expand Down
2 changes: 1 addition & 1 deletion opm/simulators/linalg/ParallelOverlappingILU0.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -509,7 +509,7 @@ namespace Opm
if( j.index() == iIndex )
{
inv[ row ] = (*j);
break;
break;
}
else if ( j.index() >= i.index() )
{
Expand Down
228 changes: 0 additions & 228 deletions opm/simulators/linalg/ParallelRestrictedAdditiveSchwarz.hpp
View file
Open in desktop

This file was deleted.

4 changes: 2 additions & 2 deletions opm/simulators/linalg/findOverlapRowsAndColumns.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ namespace detail
template<class Grid, class W>
void setWellConnections(const Grid& grid, const W& wells, bool useWellConn, std::vector<std::set<int>>& wellGraph)
{
if ( grid.comm().size() > 1)
if ( grid.comm().size() > 1 )
{
Dune::CartesianIndexMapper< Grid > cartMapper( grid );
const int numCells = cartMapper.compressedSize(); // grid.numCells()
Expand Down Expand Up @@ -102,7 +102,7 @@ namespace detail
const auto& elem = *elemIt;
int lcell = lid.id(elem);

if (elem.partitionType() != Dune::InteriorEntity)
if (elem.partitionType() == Dune::GhostEntity)
{
//add row to list
overlapRows.push_back(lcell);
Expand Down