diff --git a/src/opt/mpi/include/rrange.cc.h b/src/opt/mpi/include/rrange.cc.h index af387b2..d67af1f 100644 --- a/src/opt/mpi/include/rrange.cc.h +++ b/src/opt/mpi/include/rrange.cc.h @@ -30,7 +30,9 @@ namespace CNORXZ mRange(in.mRange), mI(std::make_shared(mRange->local())), mK(std::make_shared(mRange->geom())), - mNRanks(getNumRanks()) + mNRanks(getNumRanks()), + mRankOffset(in.mRankOffset), + mRankFormat(in.mRankFormat) { *this = in.lex(); } @@ -42,6 +44,8 @@ namespace CNORXZ mI = std::make_shared(mRange->local()); mK = std::make_shared(mRange->geom()); mNRanks = getNumRanks(); + mRankOffset = in.mRankOffset; + mRankFormat = in.mRankFormat; *this = in.lex(); return *this; } @@ -271,19 +275,25 @@ namespace CNORXZ Vector lexs(mNRanks); MPI_Allgather(&lex, 1, MPI_UNSIGNED_LONG, lexs.data(), 1, MPI_UNSIGNED_LONG, MPI_COMM_WORLD); - SizeT root = 0; - for(; root != lexs.size(); ++root) { + SizeT root = mRankOffset; + //SizeT root = 0; + for(; root < lexs.size(); root += mRankFormat) { + //for(; root < lexs.size(); ++root) { if(lexs[root] != mI->lmax().val()){ break; } } - if(root == lexs.size()){ // metaPos not in rrange + if(root >= lexs.size()){ // metaPos not in rrange *this = lmax().val(); VCHECK(toString(metaPos)); assert(0); } else { - *mK = root; + assert((root-mRankOffset) % mRankFormat == 0); + const SizeT kx = (root-mRankOffset) / mRankFormat; + CXZ_ASSERT(kx < mK->lmax().val(), "invalid rank; check rankFormat!"); + *mK = kx; + //*mK = root; *mI = lexs[root]; (*this)(); } @@ -404,12 +414,27 @@ namespace CNORXZ return mK; } + template + void RIndex::setRankFormat(SizeT rankFormat) + { + mRankFormat = rankFormat; + CXZ_ASSERT(mNRanks % mRankFormat == 0, "rankFormat (" << mRankFormat + << ") does not divide total number of ranks (" << mNRanks << ")"); + } + + template + SizeT RIndex::rankFormat() const + { + return mRankFormat; + } + template decltype(auto) operator*(const Sptr>& a, const Sptr& b) { return iptrMul(a, b); } + /*=====================+ | RRangeFactory | +=====================*/ diff --git a/src/opt/mpi/include/rrange.h b/src/opt/mpi/include/rrange.h index 4691585..f68bfcc 100644 --- a/src/opt/mpi/include/rrange.h +++ b/src/opt/mpi/include/rrange.h @@ -158,6 +158,12 @@ namespace CNORXZ /** Get index indicating the current rank this index points to. */ Sptr rankI() const; + + /** Set the rank format. */ + void setRankFormat(SizeT rankFormat); + + /** Get the rank format. */ + SizeT rankFormat() const; private: SizeT mLex = 0; @@ -165,6 +171,8 @@ namespace CNORXZ Sptr mI; /**< Index on the local range of the THIS rank. */ Sptr mK; /**< Multi-index indicating the current rank. */ SizeT mNRanks; /**< Number of ranks; in general different from but dividable by mK's maximum. */ + SizeT mRankOffset = 0; /** < Offset in case this index only serves a sub-slice of ranks .*/ + SizeT mRankFormat = 1; /**< Frequency of ranks to be served by this index. */ }; template