diff --git a/src/opt/mpi/include/raindex.cc.h b/src/opt/mpi/include/raindex.cc.h new file mode 100644 index 0000000..4b30ca9 --- /dev/null +++ b/src/opt/mpi/include/raindex.cc.h @@ -0,0 +1,119 @@ +// -*- C++ -*- +/** + + @file opt/mpi/include/raindex.cc.h + @brief RAIndex template implementations. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + +**/ + +#ifndef __cxz_mpi_raindex_cc_h__ +#define __cxz_mpi_raindex_cc_h__ + +namespace CNOXRZ +{ + namespace mpi + { + template + RAIndex::RAIndex(const T* loc, const RangePtr& range, SizeT lexpos) : + RIndex(range, lexpos), + mLoc(loc), + mCur(i.rank()), + mThisRank(getRankNumber()) + { + setBufferSize(); + setBuffer(); + } + + + template + RAIndex::RAIndex(const T* loc, const RIndex& i) : + RIndex(i) + mLoc(loc), + mCur(i.rank()), + mThisRank(getRankNumber()) + { + setBufferSize(); + setBuffer(); + } + + + template + RAIndex::RAIndex(const T* loc, const RAIndex& i) : + RAIndex(i) + { + mLoc = loc; + setBuffer(); + } + + template + RAIndex RAIndex::operator+(Int n) const + { + RAIndex o = *this; + return o += n; + } + + template + RAIndex RAIndex::operator-(Int n) const + { + RAIndex o = *this; + return o -= n; + } + + template + const T& RAIndex::operator*() const + { + if(rank() != mThisRank){ + if(mCur != rank()){ + setBuffer(); + } + return mBuf[local()->pos() % mBufSize]; + } + else { + mLoc[local()->pos()]; + } + } + + template + const T* RAIndex::operator->() const + { + if(rank() != mThisRank){ + if(mCur != rank()){ + setBuffer(); + } + return mBuf + local()->pos() % mBufSize; + } + else { + return mLoc + local()->pos(); + } + } + + template + void RAIndex::setBufferSize() + { + // use the contiguous stride's size as buffer size: + mBufSize = 1; + const auto& df = deepFormat(); + for(SizeT i = 0; i != df.size(); ++i){ + if(df[i] == 1){ + mBufSize = deepMax()[i]; + } + } + } + + template + void RAIndex::setBuffer() + { + if(mBuf.size() != mBufSize){ + mBuf.resize(mBufSize); + } + // A Bcast alternative with const pointer to source would be better... + std::memcpy(mBuf.data(), mLoc + local()->pos() / mBufSize, mBufSize*sizeof(T)); + MPI_Bcast(mBuf.data(), mBufSize*sizeof(T), MPI_BYTE, static_cast(rank()), + MPI_COMM_WORLD ); + } + + } // namespace mpi +} // namespace CNOXRZ diff --git a/src/opt/mpi/include/raindex.h b/src/opt/mpi/include/raindex.h index 737662d..22a0bee 100644 --- a/src/opt/mpi/include/raindex.h +++ b/src/opt/mpi/include/raindex.h @@ -16,35 +16,49 @@ namespace CNORXZ { - - /** **** - Index for multi-ranked array. - This is meant to be a global index. For local iterations - a YIndex is sufficient. - @tparam T data type. - */ - template - class RAIndex : public RIndex + namespace mpi { - public: - typedef typename RIndex::IB IB; - DEFAULT_MEMBERS(RAIndex); - RAIndex(const T* loc, const RangePtr& range, SizeT lexpos = 0); - RAIndex(const T* loc, const RIndex& i); - RAIndex(const T* loc, const RAIndex& i); + /** **** + Index for multi-ranked array. + This is meant to be a global index, its position must be equal on each rank, + otherwise deadlocks will occur. This kind of index exists for + completeness (global array element access), it should not be used in + operations that require high performance. + For local iterations (which is what one usually does) a YIndex is sufficient. + @tparam T data type. + */ + template + class RAIndex : public RIndex + { + public: + typedef typename RIndex::IB IB; - RAIndex operator+(Int n) const; - RAIndex operator-(Int n) const; + DEFAULT_MEMBERS(RAIndex); + RAIndex(const T* loc, const RangePtr& range, SizeT lexpos = 0); + RAIndex(const T* loc, const RIndex& i); + RAIndex(const T* loc, const RAIndex& i); - const T& operator*() const; - const T& operator->() const; - - private: - const T* mLoc = nullptr; - Vector mBuf; // used if iterating over content on different rank - }; - -} + RAIndex operator+(Int n) const; + RAIndex operator-(Int n) const; + + const T& operator*() const; + const T* operator->() const; + + void setBufferSize(); + + private: + + void setBuffer(); + + const T* mLoc = nullptr; + Vector mBuf; // used if iterating over content on different rank + SizeT mCur; // current rank in the buffer + SizeT mBufSize; + SizeT mThisRank; + }; + + } // namespace mpi +} // namespace CNOXRZ #endif diff --git a/src/opt/mpi/include/rarray.cc.h b/src/opt/mpi/include/rarray.cc.h index d6dc04c..3d630b0 100644 --- a/src/opt/mpi/include/rarray.cc.h +++ b/src/opt/mpi/include/rarray.cc.h @@ -20,7 +20,8 @@ namespace CNORXZ template RCArray::RCArray(const Sptr> a, const RangePtr& geom) : mA(a), - mGeom(geom) + mGeom(geom), + mGlobal(RRangeFactory(a->range(),mGeom).create()) {} template @@ -53,22 +54,30 @@ namespace CNORXZ template T RCArray::operator[](const SPack& pack) const { + CXZ_ERROR("not implemented"); + return T(); } template template T RCArray::at(const SPack& pack) const { + CXZ_ERROR("not implemented"); + return T(); } template T RCArray::operator[](const DPack& pack) const { + CXZ_ERROR("not implemented"); + return T(); } template T RCArray::at(const DPack& pack) const { + CXZ_ERROR("not implemented"); + return T(); } template @@ -76,23 +85,31 @@ namespace CNORXZ Sptr> RCArray::sl(const IndexInterface& begin, const IndexInterface& end) const { + CXZ_ERROR("not implemented"); + return nullptr; } template template COpRoot RCArray::operator()(const Sptr& i) const { + CXZ_ERROR("not implemented"); + return COpRoot(); } template template inline decltype(auto) RCArray::operator()(const SPack& pack) const { + CXZ_ERROR("not implemented"); + return COpRoot(); } template inline decltype(auto) RCArray::operator()(const DPack& pack) const { + CXZ_ERROR("not implemented"); + return COpRoot(); } template @@ -110,27 +127,31 @@ namespace CNORXZ template RangePtr RCArray::range() const { - return RRangeFactory(mA->range(),mGeom).create(); + return mGlobal; } template const_iterator RCArray::begin() const { + return const_iterator(mA.data(), mGlobal); } template const_iterator RCArray::end() const { + return const_iterator(mA.data(), mGlobal, mGlobal->size()); } template const_iterator RCArray::cbegin() const { + return const_iterator(mA.data(), mGlobal); } template const_iterator RCArray::cend() const { + return const_iterator(mA.data(), mGlobal, mGlobal->size()); } template @@ -145,6 +166,11 @@ namespace CNORXZ return *mA; } + template + RangePtr RCArray::geom() const + { + return mGeom; + } } // namespace mpi } // namespace CNORXZ diff --git a/src/opt/mpi/include/rarray.h b/src/opt/mpi/include/rarray.h index 70ca945..bf82331 100644 --- a/src/opt/mpi/include/rarray.h +++ b/src/opt/mpi/include/rarray.h @@ -102,10 +102,14 @@ namespace CNORXZ /** Get local array object. */ const CArrayBase& local() const; - + + /** Get rank geometry. */ + RangePtr geom() const; + private: ObjHandle> mA; RangePtr mGeom; + RangePtr mGlobal; }; /** **** diff --git a/src/opt/mpi/tests/rarray_unit_test.cc b/src/opt/mpi/tests/rarray_unit_test.cc new file mode 100644 index 0000000..506f4f9 --- /dev/null +++ b/src/opt/mpi/tests/rarray_unit_test.cc @@ -0,0 +1,123 @@ +// -*- C++ -*- +/** + + @file opt/mpi/tests/rarray_unit_test.cc + @brief RArray unit tests. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + +**/ + +#include +#include + +#include "gtest/gtest.h" + +#include "cnorxz.h" +#include "cnorxz_mpi.h" +#include "test_numbers.h" +#include "rrange.cc.h" +#include "rarray.cc.h" + +namespace +{ + using namespace CNORXZ; + using Test::Numbers; + using namespace CNORXZ::mpi; + + class MPIEnv : public ::testing::Environment + { + public: + + MPIEnv(int argc, char** argv) : mArgc(argc), mArgv(argv) {} + + virtual ~MPIEnv() override {} + + virtual void SetUp() override + { + MPI_Init(&mArgc, &mArgv); + } + + virtual void TearDown() override + { + MPI_Finalize(); + } + + protected: + int mArgc; + char** mArgv; + }; + + + class RCArray_Test : public ::testing::Test + { + protected: + + RCArray_Test() + { + CXZ_ASSERT(getNumRanks() == 4, "exptected 4 ranks"); + Vector xs(12); + Vector ts(16); + for(SizeT i = 0; i != xs.size(); ++i){ + const Int x = static_cast(i) - static_cast(xs.size()/2); + xs[i] = x; + } + for(SizeT i = 0; i != ts.size(); ++i){ + const Int t = static_cast(i) - static_cast(ts.size()/2); + ts[i] = t; + } + mXRange = URangeFactory(xs).create(); + mTRange = URangeFactory(ts).create(); + Vector rs { mTRange, mXRange, mXRange, mXRange }; + mGRange = YRangeFactory(rs).create(); + RangePtr g1 = CRangeFactory(1).create(); + RangePtr g2 = CRangeFactory(2).create(); + Vector gs { g2, g1, g1, g2 }; + mGeom = YRangeFactory(gs).create(); + mRRange = rrange(mGRange, mGeom); + const SizeT size = ts.size()*xs.size()*xs.size()*xs.size(); + Vector vec = Numbers::get(0,size/4+10); + Vector data(size); + Vector mData(size*4); + const SizeT myrank = getRankNumber(); + for(SizeT i = 0; i != size; ++i){ + data[i] = vec[i] * vec[i+myrank] / vec[i+2*myrank]; + mData[i + size*myrank] = data[i]; + MPI_Bcast(mData.data() + size*i, size, MPI_DOUBLE, i, MPI_COMM_WORLD); + } + mLoc = std::make_shared>( mRRange->sub(1), data); + mA = RCArray(mLoc, mGeom); + } + + RangePtr mXRange; + RangePtr mTRange; + RangePtr mGRange; + RangePtr mGeom; + RangePtr mRRange; + Sptr> mLoc; + RCArray mA; + Vector mData; + }; + + TEST_F(RArray_Test, Basics) + { + EXPECT_EQ(mA.size(), mRRange->size()); + } + + TEST_F(RArray_Test, GlobalIterate) + { + const SizeT size = mRRange->sub(1)->size(); + auto e = mA.end(); + for(auto i = mA.begin(); i != e; ++i){ + EXPECT_EQ(*i, mData[i.rank()*size + i.local().pos()]); + } + } +} + +int main(int argc, char** argv) +{ + ::testing::InitGoogleTest(&argc, argv); + ::testing::AddGlobalTestEnvironment( new MPIEnv(argc, argv) ); + return RUN_ALL_TESTS(); +}