diff --git a/CMakeLists.txt b/CMakeLists.txt index 9499092..5ee9b3f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -123,6 +123,14 @@ else() endif() endif() +# CHECK LIBRARIES : mpi + +if(DEFINED ENABLE_mpi) + set(ENABLE_mpi ${ENABLE_mpi} CACHE BOOL "enable mpi") +else() + set(ENABLE_mpi TRUE CACHE BOOL "enable mpi") +endif() + # DEFINES add_definitions(-DVERSION="${VERSION}") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1d60ecb..4d753fc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,4 +12,8 @@ if(ENABLE_cereal) add_subdirectory(opt/cereal) endif() +if(ENABLE_mpi) + add_subdirectory(opt/mpi) +endif() + install(DIRECTORY include/ DESTINATION ${INSTALL_PATH}/include/cnorxz) diff --git a/src/include/ranges/yrange.h b/src/include/ranges/yrange.h index 06cb273..e85a2cf 100644 --- a/src/include/ranges/yrange.h +++ b/src/include/ranges/yrange.h @@ -311,6 +311,7 @@ namespace CNORXZ public: typedef RangeBase RB; typedef YIndex IndexType; + typedef typename IndexType::MetaType MetaType; friend YRangeFactory; diff --git a/src/opt/mpi/CMakeLists.txt b/src/opt/mpi/CMakeLists.txt new file mode 100644 index 0000000..6416c7c --- /dev/null +++ b/src/opt/mpi/CMakeLists.txt @@ -0,0 +1,16 @@ + +find_package(MPI REQUIRED) +if(MPI_FOUND) + include_directories(${MPI_C_INCLUDE_DIRS}) +else() + message(FATAL_ERROR "HDF5 not found") +endif() +message(STATUS "mpi lib = ${MPI_C_LIBRARIES}") +set(MPI_LIBS ${MPI_LIBRARIES}) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) + +add_subdirectory(lib) +add_subdirectory(tests) + +install(DIRECTORY include/ DESTINATION ${INSTALL_PATH}/include/cnorxz/mpi) diff --git a/src/opt/mpi/include/cnorxz_mpi.cc.h b/src/opt/mpi/include/cnorxz_mpi.cc.h new file mode 100644 index 0000000..7a2846c --- /dev/null +++ b/src/opt/mpi/include/cnorxz_mpi.cc.h @@ -0,0 +1,14 @@ +// -*- C++ -*- +/** + + @file opt/mpi/include/cnorxz_mpi.cc.h + @brief CNORXZ MPI template sources header + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ + +#include "mpi_wrappers.cc.h" +//#include "rop_types.cc.h" +#include "rrange.cc.h" diff --git a/src/opt/mpi/include/cnorxz_mpi.h b/src/opt/mpi/include/cnorxz_mpi.h new file mode 100644 index 0000000..dac771d --- /dev/null +++ b/src/opt/mpi/include/cnorxz_mpi.h @@ -0,0 +1,20 @@ +// -*- C++ -*- +/** + + @file opt/mpi/include/cnorxz_mpi.h + @brief CNORXZ MPI main header + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ + +#include "mpi_base.h" +#include "mpi_wrappers.h" +//#include "raindex.h" +//#include "rarray.h" +//#include "rop_types.h" +#include "rrange.h" +#include "typemap.h" + +#include "cnorxz_mpi.cc.h" diff --git a/src/opt/mpi/include/mpi_base.h b/src/opt/mpi/include/mpi_base.h index 678eddb..bcb8cf0 100644 --- a/src/opt/mpi/include/mpi_base.h +++ b/src/opt/mpi/include/mpi_base.h @@ -20,6 +20,14 @@ namespace CNORXZ { namespace mpi { + // class declarations + + template + class RRange; + + template + class RIndex; + // wrapper functions /** Get number of THIS rank. */ diff --git a/src/opt/mpi/include/mpi_wrappers.cc.h b/src/opt/mpi/include/mpi_wrappers.cc.h index 5979385..dce47ce 100644 --- a/src/opt/mpi/include/mpi_wrappers.cc.h +++ b/src/opt/mpi/include/mpi_wrappers.cc.h @@ -22,9 +22,9 @@ namespace CNORXZ template void BCast::bcast(T& d, SizeT root) { - static_assert( TypeMap::exists, "no bcast implementation for given type" ); + static_assert( Typemap::exists, "no bcast implementation for given type" ); const int ret = MPI_Bcast( reinterpret_cast(&d), 1, - TypeMap::value, MPI_COMM_WORLD ); + Typemap::value(), root, MPI_COMM_WORLD ); CXZ_ASSERT(ret == MPI_SUCCESS, "got bcast error = " << ret); return; } @@ -35,7 +35,7 @@ namespace CNORXZ { SizeT size = d.size(); const int ret = MPI_Bcast( reinterpret_cast(&size), 1, - MPI_UNSIGNED_LONG, MPI_COMM_WORLD ); + MPI_UNSIGNED_LONG, root, MPI_COMM_WORLD ); CXZ_ASSERT(ret == MPI_SUCCESS, "got bcast error = " << ret); if(size != d.size()){ d.resize(size); @@ -47,7 +47,7 @@ namespace CNORXZ } else { const int ret2 = MPI_Bcast( reinterpret_cast(d.data()), size, - TypeMap::value, MPI_COMM_WORLD ); + Typemap::value(), root, MPI_COMM_WORLD ); CXZ_ASSERT(ret2 == MPI_SUCCESS, "got bcast error = " << ret2); } } @@ -55,15 +55,15 @@ namespace CNORXZ template void BCast>::bcast(Arr& d, SizeT root) { - if constexpr( BCast::special ){ + if constexpr( BCast::special ){ for(auto& x: d){ bcast(x, root); } } else { const int ret = MPI_Bcast( reinterpret_cast(d.data()), N, - TypeMap::value, MPI_COMM_WORLD ); - CXZ_ASSERT(ret == MPI_SUCCESS, "got bcast error = " << ret2); + Typemap::value(), root, MPI_COMM_WORLD ); + CXZ_ASSERT(ret == MPI_SUCCESS, "got bcast error = " << ret); } } @@ -71,25 +71,25 @@ namespace CNORXZ { SizeT size = d.size(); const int ret = MPI_Bcast( reinterpret_cast(&size), 1, - MPI_UNSIGNED_LONG, MPI_COMM_WORLD ); + MPI_UNSIGNED_LONG, root, MPI_COMM_WORLD ); CXZ_ASSERT(ret == MPI_SUCCESS, "got bcast error = " << ret); if(size != d.size()){ d.resize(size); } - const int ret = MPI_Bcast( reinterpret_cast(d.data()), size, - MPI_CHAR, MPI_COMM_WORLD ); - CXZ_ASSERT(ret == MPI_SUCCESS, "got bcast error = " << ret); + const int ret2 = MPI_Bcast( reinterpret_cast(d.data()), size, + MPI_CHAR, root, MPI_COMM_WORLD ); + CXZ_ASSERT(ret2 == MPI_SUCCESS, "got bcast error = " << ret2); } template void BCast>::bcast(Tuple& d, SizeT root) { if constexpr( ( BCast::special or ... ) ){ - ifor<0,sizeof...(Ts)>( [&](auto i) { bcast( std::get(d), root ); }, NoF {} ); + iter<0,sizeof...(Ts)>( [&](auto i) { bcast( std::get(d), root ); }, NoF {} ); } else { const int ret = MPI_Bcast( reinterpret_cast(&d), sizeof(d), - MPI_BYTE, MPI_COMM_WORLD ); + MPI_BYTE, root, MPI_COMM_WORLD ); CXZ_ASSERT(ret == MPI_SUCCESS, "got bcast error = " << ret); } } @@ -97,7 +97,8 @@ namespace CNORXZ template void bcast(T& d, SizeT root) { - return BCast(d, root); + BCast::bcast(d, root); + return; } } // namespace mpi diff --git a/src/opt/mpi/include/mpi_wrappers.h b/src/opt/mpi/include/mpi_wrappers.h index caee97b..2bd00a1 100644 --- a/src/opt/mpi/include/mpi_wrappers.h +++ b/src/opt/mpi/include/mpi_wrappers.h @@ -12,8 +12,7 @@ #ifndef __cxz_mpi_wrappers_h__ #define __cxz_mpi_wrappers_h__ -#include "mpi.h" -#include "cnorxz.h" +#include "mpi_base.h" #include "typemap.h" namespace CNORXZ diff --git a/src/opt/mpi/include/rrange.cc.h b/src/opt/mpi/include/rrange.cc.h index d928457..d8c4063 100644 --- a/src/opt/mpi/include/rrange.cc.h +++ b/src/opt/mpi/include/rrange.cc.h @@ -14,6 +14,7 @@ #include "rrange.h" #include "mpi_wrappers.h" +#include "mpi_wrappers.cc.h" namespace CNORXZ { @@ -27,24 +28,24 @@ namespace CNORXZ template RIndex::RIndex(const RIndex& in) : mRange(in.mRange), - mI(std::make_shared(mRange->local())), - mK(std::make_shared(mRange->geom())) + mI(std::make_shared(mRange->local())), + mK(std::make_shared(mRange->geom())) { *this = in.lex(); } template - RIndex& RIndex::operator=(const RIndex& in) + RIndex& RIndex::operator=(const RIndex& in) { mRange = in.mRange; - mI = std::make_shared(mRange->local()); - mK = std::make_shared(mRange->geom()); + mI = std::make_shared(mRange->local()); + mK = std::make_shared(mRange->geom()); *this = in.lex(); return *this; } template - RIndex::RIndex(const RangePtr& global, SizeT lexpos = 0) : + RIndex::RIndex(const RangePtr& global, SizeT lexpos) : mRange(rangeCast(global)), mI(std::make_shared(mRange->local())), mK(std::make_shared(mRange->geom())) @@ -56,7 +57,7 @@ namespace CNORXZ RIndex& RIndex::operator=(SizeT pos) { IB::mPos = pos; // = lex - if(lexpos >= lmax().val()){ + if(pos >= lmax().val()){ IB::mPos = pmax().val(); return *this; } @@ -155,15 +156,15 @@ namespace CNORXZ } template - constexpr RIndex::decltype(auto) pmax() const + constexpr decltype(auto) RIndex::pmax() const { - return mK->lmax().val() * mI->lmax().val(); + return UPos(mK->lmax().val() * mI->lmax().val()); } template - constexpr RIndex::decltype(auto) lmax() const + constexpr decltype(auto) RIndex::lmax() const { - return mK->lmax().val() * mI->lmax().val(); + return UPos(mK->lmax().val() * mI->lmax().val()); } template @@ -173,7 +174,7 @@ namespace CNORXZ } template - MetaType RIndex::operator*() const + typename RIndex::MetaType RIndex::operator*() const { return meta(); } @@ -185,7 +186,7 @@ namespace CNORXZ } template - Sptr RIndex::range() const + Sptr::RangeType> RIndex::range() const { return mRange; } @@ -194,7 +195,7 @@ namespace CNORXZ template decltype(auto) RIndex::stepSize(const IndexId& id) const { - return mK->stepSize(id) * mI->lmax().val() + mI->stepSize(id); + return mK->stepSize(id) * mI->lmax() + mI->stepSize(id); } template @@ -202,16 +203,20 @@ namespace CNORXZ { const SizeT r = mK->lex(); String o; - broadcast(r, mI->stringMeta(), &o); + auto x = mI->stringMeta(); + bcast(x, r); return o; } template - MetaType RIndex::meta() const + typename RIndex::MetaType RIndex::meta() const { - const SizeT r = mK->lex(); MetaType o; - broadcast(r, mI->meta(), &o); + if constexpr(Typemap::exists){ + auto x = mI->meta(); + const SizeT r = mK->lex(); + bcast(x, r); + } return o; } @@ -253,7 +258,7 @@ namespace CNORXZ constexpr decltype(auto) RIndex::ifor(const Xpr& xpr, F&& f) const { CXZ_ERROR("not implemented"); - return 0; + return xpr; } template @@ -276,7 +281,7 @@ namespace CNORXZ } template - Sptr RIndex::local() const + Sptr RIndex::local() const { return mI; } @@ -299,8 +304,9 @@ namespace CNORXZ << ") does not match number of ranks ( = " << s << ")"); if constexpr(has_static_sub::value and has_static_sub::value) { - static_assert(typename RangeI::NR == typename RangeK::NR, - "ranges have to be of same dimension"); + constexpr SizeT NRI = RangeI::NR; + constexpr SizeT NRK = RangeK::NR; + static_assert(NRI == NRK, "ranges have to be of same dimension"); } else { CXZ_ASSERT(ri->dim() == rk->dim(), "ranges have to be of same dimension, got " @@ -311,7 +317,7 @@ namespace CNORXZ template void RRangeFactory::make() { - Vector key = { mRI->key(), mRK->key() }; + Vector key = { mRI->id(), mRK->id() }; const auto& info = typeid(RRange); mProd = this->fromCreated(info, key); if(mProd == nullptr) { @@ -397,7 +403,7 @@ namespace CNORXZ } template - const MetaType RRange::get(SizeT pos) const + const typename RRange::MetaType RRange::get(SizeT pos) const { return (this->begin()+pos)->meta(); } @@ -424,6 +430,12 @@ namespace CNORXZ mLocal(loc), mGeom(geom) {} + + template + Vector RRange::key() const + { + return Vector { mLocal->id(), mGeom->id() }; + } } // namespace mpi } // namespace CNORXZ diff --git a/src/opt/mpi/include/rrange.h b/src/opt/mpi/include/rrange.h index 80a9a53..522111c 100644 --- a/src/opt/mpi/include/rrange.h +++ b/src/opt/mpi/include/rrange.h @@ -12,7 +12,7 @@ #ifndef __cxz_mpi_rrange_h__ #define __cxz_mpi_rrange_h__ -#include "cnorxz.h" +#include "mpi_base.h" namespace CNORXZ { @@ -140,7 +140,7 @@ namespace CNORXZ SizeT rank() const; /** Get the local index on THIS rank. */ - Sptr local() const; + Sptr local() const; //!!! private: @@ -237,6 +237,7 @@ namespace CNORXZ Sptr mLocal; /**< Local range of THIS rank. */ Sptr mGeom; /**< Rank geometry range. */ + virtual Vector key() const override final; }; /** Create RRange from global range and given rank geometry. diff --git a/src/opt/mpi/include/typemap.h b/src/opt/mpi/include/typemap.h index b0e20c9..f5c66ee 100644 --- a/src/opt/mpi/include/typemap.h +++ b/src/opt/mpi/include/typemap.h @@ -18,42 +18,42 @@ namespace CNORXZ struct Typemap { static constexpr bool exists = true; - static constexpr MPI_Datatype value = MPI_INT; + static MPI_Datatype value() { return MPI_INT; } }; template <> struct Typemap { static constexpr bool exists = true; - static constexpr MPI_Datatype value = MPI_UNSIGNED; + static MPI_Datatype value() { return MPI_UNSIGNED; } }; template <> struct Typemap { static constexpr bool exists = true; - static constexpr MPI_Datatype value = MPI_LONG; + static MPI_Datatype value() { return MPI_LONG; } }; template <> struct Typemap { static constexpr bool exists = true; - static constexpr MPI_Datatype value = MPI_UNSIGNED_LONG; + static MPI_Datatype value() { return MPI_UNSIGNED_LONG; } }; template <> struct Typemap { static constexpr bool exists = true; - static constexpr MPI_Datatype value = MPI_DOUBLE; + static MPI_Datatype value() { return MPI_DOUBLE; } }; template <> struct Typemap { static constexpr bool exists = true; - static constexpr MPI_Datatype value = MPI_FLOAT; + static MPI_Datatype value() { return MPI_FLOAT; } }; // further !!! diff --git a/src/opt/mpi/lib/CMakeLists.txt b/src/opt/mpi/lib/CMakeLists.txt new file mode 100644 index 0000000..8d2e833 --- /dev/null +++ b/src/opt/mpi/lib/CMakeLists.txt @@ -0,0 +1,18 @@ + +set(libcnorxzmpi_a_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/mpi_base.cc + ${CMAKE_CURRENT_SOURCE_DIR}/rrange.cc + ) + +add_library(cnorxzmpi_obj OBJECT + ${libcnorxzmpi_a_SOURCES} + ) + +set_target_properties(cnorxzmpi_obj PROPERTIES POSITION_INDEPENDENT_CODE TRUE) + +add_library(cnorxzmpi SHARED + $ + ) +set_target_properties(cnorxzmpi PROPERTIES POSITION_INDEPENDENT_CODE TRUE) + +install(TARGETS cnorxzmpi LIBRARY DESTINATION ${INSTALL_PATH}/lib) diff --git a/src/opt/mpi/lib/rrange.cc b/src/opt/mpi/lib/rrange.cc index ff93029..31f2c99 100644 --- a/src/opt/mpi/lib/rrange.cc +++ b/src/opt/mpi/lib/rrange.cc @@ -10,6 +10,7 @@ **/ #include "rrange.h" +#include "rrange.cc.h" namespace CNORXZ { @@ -27,14 +28,16 @@ namespace CNORXZ "global range has to be of same dimension as geometry range, got " << global->dim() << " and " << geom->dim()); MArray o(geom); - YIndex k = geom->begin(); - YIndex ke = geom->end(); - auto mu = std::make_shared(geom->sub()); - MArray> rs(geom->sub()->range()); - rs(mu) = operation( [&](const SizeT i) { return rsplit(globla->sub(i), geom->sub(i)); } , xpr(mu) ); - Vector elem(geom->sub()->range()); + YIndex k(geom); + YIndex ke(geom, geom->size()); + auto mu = std::make_shared(geom->sub().range()); + MArray> rs(geom->sub().range()); + rs(mu) = operation( [&](const SizeT i) { return rsplit(global->sub(i), geom->sub(i)); } , xpr(mu) ); + Vector elem(geom->dim()); for(; k != ke; ++k){ - mu->ifor( operation( [&](const SizeT i){ elem[i] = rs[i][k->pack()[i]]; }, xpr(mu) ) )(); + mu->ifor( operation( [&](const SizeT i, const MArray& el){ + elem[i] = el[DIndex(k.pack()[i])]; + }, xpr(mu), rs(mu) ), NoF {} )(); o[k] = YRangeFactory(elem).create(); } return o; @@ -42,13 +45,13 @@ namespace CNORXZ else { CXZ_ASSERT(global->size() % geom->size() == 0, "global range has to be dividible by geometry range, got " - << global->size() << " and " << k->lmax().val()); + << global->size() << " and " << geom->size()); const SizeT n = global->size() / geom->size(); auto k = std::make_shared(geom); auto jb = global->begin(); auto je = global->begin(); MArray o(geom); - o(k) = operation( [&](const SizeT x){ jb = n*x; je = n*(x+1)-1; return jb.prange(je) } , xpr(k) ); + o(k) = operation( [&](const SizeT x){ jb = n*x; je = n*(x+1)-1; return jb.prange(je); } , xpr(k) ); return o; } } @@ -58,14 +61,15 @@ namespace CNORXZ const MArray rs = rsplit(global, geom); RangePtr o = nullptr; for(auto ri = rs.begin(); ri != rs.end(); ++ri){ - if(ri->lex() == getRangeNum()){ + if(ri.lex() == getRankNumber()){ o = *ri; } } assert(o); auto loc = rangeCast(global); auto geo = rangeCast(geom); - return RRangeFactory(loc, geom).create(); + RRangeFactory xx(loc, geo); + return RRangeFactory(loc, geo).create(); } diff --git a/src/opt/mpi/tests/CMakeLists.txt b/src/opt/mpi/tests/CMakeLists.txt new file mode 100644 index 0000000..eba47b0 --- /dev/null +++ b/src/opt/mpi/tests/CMakeLists.txt @@ -0,0 +1,7 @@ + +add_definitions(-DTEST_NUMBER_FILE="${CMAKE_SOURCE_DIR}/src/tests/numbers.txt") +include_directories(${CMAKE_SOURCE_DIR}/src/tests) +add_executable(mpirrutest rrange_unit_test.cc) +add_dependencies(mpirrutest cnorxz cnorxzmpi test_lib) +target_link_libraries(mpirrutest ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${MPI_LIBS} cnorxz cnorxzmpi test_lib) +add_test(NAME mpirrutest COMMAND mpirrutest) diff --git a/src/opt/mpi/tests/rrange_unit_test.cc b/src/opt/mpi/tests/rrange_unit_test.cc new file mode 100644 index 0000000..9c470e4 --- /dev/null +++ b/src/opt/mpi/tests/rrange_unit_test.cc @@ -0,0 +1,69 @@ +// -*- C++ -*- +/** + + @file opt/mpi/tests/rrange_unit_test.cc + @brief RRange unit tests. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + +**/ + +#include +#include + +#include "gtest/gtest.h" + +#include "cnorxz.h" +#include "cnorxz_mpi.h" +#include "test_numbers.h" +#include "rrange.cc.h" + +namespace +{ + using namespace CNORXZ; + using Test::Numbers; + using namespace CNORXZ::mpi; + + class RRange_Test : public ::testing::Test + { + protected: + + RRange_Test() + { + CXZ_ASSERT(getNumRanks() == 4, "exptected 4 ranks"); + Vector xs(12); + Vector ts(16); + mXRange = URangeFactory(xs).create(); + mTRange = URangeFactory(ts).create(); + Vector rs { mTRange, mXRange, mXRange, mXRange }; + mGRange = YRangeFactory(rs).create(); + RangePtr g1 = CRangeFactory(1).create(); + RangePtr g2 = CRangeFactory(2).create(); + Vector gs { g2, g1, g1, g2 }; + mGeom = YRangeFactory(gs).create(); + mRRange = rrange(mGRange, mGeom); + } + + RangePtr mXRange; + RangePtr mTRange; + RangePtr mGRange; + RangePtr mGeom; + RangePtr mRRange; + }; + + TEST_F(RRange_Test, Basics) + { + EXPECT_EQ(mRRange->size(), mGRange->size()); + } + +} + +int main(int argc, char** argv) +{ + ::testing::InitGoogleTest(&argc, argv); + MPI_Init(&argc, &argv); + const int ret = RUN_ALL_TESTS(); + MPI_Finalize(); + return ret; +}