diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4d753fc..33bc1eb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -16,4 +16,8 @@ if(ENABLE_mpi) add_subdirectory(opt/mpi) endif() +if(ENABLE_hdf5 AND ENABLE_mpi) + add_subdirectory(opt/hdf5-mpi) +endif() + install(DIRECTORY include/ DESTINATION ${INSTALL_PATH}/include/cnorxz) diff --git a/src/opt/hdf5-mpi/CMakeLists.txt b/src/opt/hdf5-mpi/CMakeLists.txt new file mode 100644 index 0000000..acf303f --- /dev/null +++ b/src/opt/hdf5-mpi/CMakeLists.txt @@ -0,0 +1,33 @@ + +find_package(HDF5 REQUIRED COMPONENTS C HL) +if(HDF5_FOUND) + include_directories(${HDF5_INCLUDE_DIRS}) +else() + message(FATAL_ERROR "HDF5 not found") +endif() +message(STATUS "hdf5 libs = ${HDF5_LIBRARIES}") +set(HDF5_LIBS ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES}) + +find_package(MPI REQUIRED) +if(MPI_FOUND) + include_directories(${MPI_C_INCLUDE_DIRS}) +else() + message(FATAL_ERROR "MPI not found") +endif() +message(STATUS "mpi lib = ${MPI_C_LIBRARIES}") +set(MPI_LIBS ${MPI_LIBRARIES}) + +if(HDF5_IS_PARALLEL) + include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../hdf5/include) + include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../mpi/include) + include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) + + add_subdirectory(lib) + #add_subdirectory(tests) + + install(CODE "execute_process(COMMAND sed -i \"s|CXZ_H5_MPI_BUILD_MODE 1|CXZ_H5_MPI_BUILD_MODE 0|g;\" ${CMAKE_CURRENT_SOURCE_DIR}/include/h5_mpi_base.h)") + install(DIRECTORY include/ DESTINATION ${INSTALL_PATH}/include/cnorxz/hdf5-mpi) + install(CODE "execute_process(COMMAND sed -i \"s|CXZ_H5_MPI_BUILD_MODE 0|CXZ_H5_MPI_BUILD_MODE 1|g;\" ${CMAKE_CURRENT_SOURCE_DIR}/include/h5_mpi_base.h)") +else() + message(WARNING "no parallel support in available HDF5 library") +endif() diff --git a/src/opt/hdf5-mpi/include/h5_mpi_base.h b/src/opt/hdf5-mpi/include/h5_mpi_base.h new file mode 100644 index 0000000..73e44a5 --- /dev/null +++ b/src/opt/hdf5-mpi/include/h5_mpi_base.h @@ -0,0 +1,19 @@ + +#ifndef __cxz_h5_mpi_base_h__ +#define __cxz_h5_mpi_base_h__ + +#define CXZ_H5_MPI_BUILD_MODE 1 + +#if CXZ_H5_MPI_BUILD_MODE + +#include "cnorxz_mpi.h" +#include "cnorxz_hdf5.h" + +#else + +#include "mpi/cnorxz_mpi.h" +#include "hdf5/cnorxz_hdf5.h" + +#endif + +#endif diff --git a/src/opt/hdf5-mpi/include/h5_rdataset.cc.h b/src/opt/hdf5-mpi/include/h5_rdataset.cc.h new file mode 100644 index 0000000..36a9185 --- /dev/null +++ b/src/opt/hdf5-mpi/include/h5_rdataset.cc.h @@ -0,0 +1,50 @@ +// -*- C++ -*- +/** + + @file opt/hdf5-mpi/include/h5_dataset.cc.h + @brief Implementation of template member functions of RDataset and SRDataset. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ + +#ifndef __cxz_h5_rdataset_cc_h__ +#define __cxz_h5_rdataset_cc_h__ + +#include "h5_rdataset.h" + +namespace CNORXZ +{ + namespace hdf5 + { + template + RDataset& RDataset::init(const mpi::RArray& data) + { + const hid_t tid = getTypeId(*data.data()); + if(data.begin().formatIsTrivial()){ + init(data.range(), tid, data.data()); + } + else { + CXZ_ERROR("Got array type with non-trivial format; non-contiguous data formats are not supported yet!"); + } + return *this; + } + + template + SRDataset::SRDataset(const String& name, const ContentBase* _parent) : + RDataset(name, _parent) + {} + + template + mpi::RArray SRDataset::read(const RangePtr& geom) const + { + RangePtr rr = mpi::rrange(mFileRange, geom); + mpi::RArray out(rr); + readbase(out.data(), rr, nullptr); + return out; + } + + } +} + diff --git a/src/opt/hdf5-mpi/include/h5_rdataset.h b/src/opt/hdf5-mpi/include/h5_rdataset.h index f07f621..5fafdf7 100644 --- a/src/opt/hdf5-mpi/include/h5_rdataset.h +++ b/src/opt/hdf5-mpi/include/h5_rdataset.h @@ -12,6 +12,13 @@ #ifndef __cxz_h5_rdataset_h__ #define __cxz_h5_rdataset_h__ +//#include "hdf5/h5_types.h" +//#include "hdf5/h5_content_base.h" +//#include "hdf5/h5_dataset.h" +//#include "mpi/mpi_base.h" + +#include "h5_mpi_base.h" + namespace CNORXZ { namespace hdf5 @@ -19,8 +26,7 @@ namespace CNORXZ /** **** Class to handle hdf5 datasets on multiple ranks. */ - template - class RDataset : public SDataset + class RDataset : public Dataset { public: DEFAULT_MEMBERS(RDataset); /**< Default constructors and assignments. */ @@ -30,27 +36,45 @@ namespace CNORXZ @param _parent Parent content object. */ RDataset(const String& name, const ContentBase* _parent); - + + virtual RDataset& initbase(const RangePtr& fileRange, hid_t type) override; + virtual RDataset& writebase(const RangePtr& writeRange, Sptr pos, + const void* data) override; + virtual void readbase(void* dest, RangePtr readrange, Sptr beg) const override; + /** Initalize the dataset. @param data Array containing the dataset. */ template - RDataset& init(const mpi::RArray& data); + Dataset& init(const mpi::RArray& data); - /** Read the dataset. - @return Array containing the dataset values. - */ - mpi::RArray read() const; - - /** Read a given subset of the dataset. - The subset needs to be hypercubic. - @param beg Index indicating the global begin edge of the hypercube. - @param end Index indicating the global end edge of the hypercube (inclusive). - @return Array containing the dataset values. - */ - template - mpi::RArray read(const IndexInterface& beg, const IndexInterface& end) const; - + private: + bool checkHaveParallel() const; }; + /** **** + Class to handle hdf5 datasets on multiple ranks, + the value type is assumed to be known at compile time. + @tparam T Dataset value type. + */ + template + class SRDataset : public RDataset + { + public: + DEFAULT_MEMBERS(SRDataset); /**< Default constructors and assignments. */ + + /** Construct the class. + @param name Dataset name. + @param _parent Parent content object. + */ + SRDataset(const String& name, const ContentBase* _parent); + + /** Read the dataset. + @param geom Geometry of the created array. + @return Array containing the dataset values. + */ + mpi::RArray read(const RangePtr& geom) const; + }; + } +} #endif diff --git a/src/opt/hdf5-mpi/include/h5_rfile.h b/src/opt/hdf5-mpi/include/h5_rfile.h new file mode 100644 index 0000000..9088304 --- /dev/null +++ b/src/opt/hdf5-mpi/include/h5_rfile.h @@ -0,0 +1,45 @@ +// -*- C++ -*- +/** + + @file opt/hdf5/include/h5_rfile.h + @brief RFile declaration. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ + +#ifndef __cxz_h5_rfile_h__ +#define __cxz_h5_rfile_h__ + +//#include "h5_file.h" +#include "h5_mpi_base.h" + +namespace CNORXZ +{ + namespace hdf5 + { + /** **** + Class to handle hdf5 file objects with parallel I/O. + Objects of this type usually serve as root object + so they don't have any parent. + */ + class RFile : public File + { + public: + DEFAULT_MEMBERS(RFile); /**< Default constructors and assignments. */ + + /** Construct the class. + @param fname Path to the hdf5 file to be handled. + @param _ro Open in read-only mode if true, otherwise have write access. + */ + RFile(const String& fname, bool _ro = true); + + /** Destructor. Release all involved hdf5 ids. */ + ~RFile(); + + }; + } +} + +#endif diff --git a/src/opt/hdf5-mpi/lib/CMakeLists.txt b/src/opt/hdf5-mpi/lib/CMakeLists.txt new file mode 100644 index 0000000..c0349f1 --- /dev/null +++ b/src/opt/hdf5-mpi/lib/CMakeLists.txt @@ -0,0 +1,18 @@ + +set(libcnorxzhdf5mpi_a_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/h5_rfile.cc + ${CMAKE_CURRENT_SOURCE_DIR}/h5_rdataset.cc + ) + +add_library(cnorxzhdf5mpi_obj OBJECT + ${libcnorxzhdf5mpi_a_SOURCES} + ) + +set_target_properties(cnorxzhdf5mpi_obj PROPERTIES POSITION_INDEPENDENT_CODE TRUE) + +add_library(cnorxzhdf5mpi SHARED + $ + ) +set_target_properties(cnorxzhdf5mpi PROPERTIES POSITION_INDEPENDENT_CODE TRUE) + +install(TARGETS cnorxzhdf5mpi LIBRARY DESTINATION ${INSTALL_PATH}/lib) diff --git a/src/opt/hdf5-mpi/lib/h5_rdataset.cc b/src/opt/hdf5-mpi/lib/h5_rdataset.cc new file mode 100644 index 0000000..622f2fb --- /dev/null +++ b/src/opt/hdf5-mpi/lib/h5_rdataset.cc @@ -0,0 +1,157 @@ +// -*- C++ -*- +/** + + @file opt/hdf5-mpi/lib/h5_rdataset.cc + @brief RDataset implementations. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ + +#include "h5_rdataset.h" +#include "h5_rfile.h" + +namespace CNORXZ +{ + namespace hdf5 + { + RDataset::RDataset(const String& name, const ContentBase* _parent) : + Dataset(name, _parent) + { + CXZ_ASSERT( checkHaveParallel(), "tried to open dataset in parallel mode while file" + << parent()->filename() << " was opened in serial mode"); + if(exists()){ + open(); + } + } + + RDataset& RDataset::initbase(const RangePtr& fileRange, hid_t type) + { + RangePtr fr = fileRange; + if(fr->stype() == "R"){ + const RangePtr local = fr->sub(1); + const RangePtr geom = fr->sub(0); + const SizeT ndims = local->dim(); + //CXZ_ASSERT(ndims == geom->dim(), "") + Vector rs(ndims); + for(SizeT i = 0; i != ndims; ++i){ + const SizeT ext = local->savesub(i)->size()*geom->savesub(i)->size(); + rs[i] = CRangeFactory(ext).create(); + } + fr = yrange(rs); + } + Dataset::initbase(fr, type); + return *this; + } + + RDataset& RDataset::writebase(const RangePtr& writeRange, Sptr pos, const void* data) + { + //bool todo = true; + RangePtr dr = writeRange; + if(dr->stype() == "R"){ + dr = writeRange->sub(1); + } + CXZ_ASSERT(dr->dim() == mFileRange->dim(), "dimension of data range (" + << dr->dim() << ") different from dimension of file range (" + << mFileRange->dim() << ")"); + Vector offset(mFileRange->dim()); + if(dr->stype() == "R"){ + mpi::RIndex idx(writeRange); + idx.localize(); + const SizeT rat = mpi::getNumRanks() / idx.rankI()->lmax().val(); + assert(rat == 1); // for now... + assert(mpi::getRankNumber() == idx.rankI()->lex()); + for(SizeT i = 0; i != offset.size(); ++i){ + offset[i] = idx.rankI()->pack().get(i)->lex() * dr->savesub(i)->size(); + } + } + if(pos){ + CXZ_ASSERT(pos->range()->dim() == mFileRange->dim(), "dimension of position index (" + << pos->range()->dim() << ") different from dimension of file range (" + << mFileRange->dim() << ")"); + for(SizeT i = 0; i != offset.size(); ++i){ + offset[i] += pos->pack().get(i)->lex(); + } + } + + Vector dims(dr->dim()); + for(SizeT i = 0; i != dims.size(); ++i){ + dims[i] = dr->sub(i)->size(); + } + H5Sselect_hyperslab(mFilespace, H5S_SELECT_SET, offset.data(), NULL, dims.data(), NULL); + const hid_t memspace = H5Screate_simple(dims.size(), dims.data(), NULL); + const hid_t xfer_plist_id = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(xfer_plist_id, H5FD_MPIO_COLLECTIVE); + H5Dwrite(mId, mType, memspace, mFilespace, xfer_plist_id, data); + H5Pclose(xfer_plist_id); + H5Sclose(memspace); + return *this; + } + + void Dataset::readbase(void* dest, RangePtr readRange, Sptr beg) const + { + RangePtr dr = readRange; + if(not dr){ + dr = mFileRange; + } + if(dr->stype() == "R"){ + dr = readRange->sub(1); + } + CXZ_ASSERT(dr->dim() == mFileRange->dim(), "dimension of data range (" + << dr->dim() << ") different from dimension of file range (" + << mFileRange->dim() << ")"); + Vector offset(mFileRange->dim()); + if(dr->stype() == "R"){ + mpi::RIndex idx(readRange); + idx.localize(); + const SizeT rat = mpi::getNumRanks() / idx.rankI()->lmax().val(); + assert(rat == 1); // for now... + assert(mpi::getRankNumber() == idx.rankI()->lex()); + for(SizeT i = 0; i != offset.size(); ++i){ + offset[i] = idx.rankI()->pack().get(i)->lex() * dr->savesub(i)->size(); + } + } + if(beg){ + CXZ_ASSERT(beg->range()->dim() == mFileRange->dim(), "dimension of position index (" + << beg->range()->dim() << ") different from dimension of file range (" + << mFileRange->dim() << ")"); + for(SizeT i = 0; i != offset.size(); ++i){ + offset[i] += beg->pack().get(i)->lex(); + } + } + + Vector dims(mFileRange->dim()); + for(SizeT i = 0; i != dims.size(); ++i){ + dims[i] = readRange->sub(i)->size(); + } + H5Sselect_hyperslab(mFilespace, H5S_SELECT_SET, offset.data(), NULL, dims.data(), NULL); + const hid_t mem_space_id = H5Screate_simple(static_cast(dims.size()), + dims.data(), nullptr); + const hid_t xfer_plist_id = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(xfer_plist_id, H5FD_MPIO_COLLECTIVE); + //MArray out(readRange); + const herr_t err = H5Dread(mId, mType, mem_space_id, mFilespace, xfer_plist_id, dest); + CXZ_ASSERT(err >= 0, "error while reading dataset '" << mName + << "', errorcode :" << err); + H5Pclose(xfer_plist_id); + H5Sclose(mem_space_id); + } + + bool RDataset::checkHaveParallel() const + { + const ContentBase* p = parent(); + while(p->type() != ContentType::FILE and p != nullptr){ + p = p->parent(); + } + if(not p){ + return false; + } + const RFile* fp = dynamic_cast(p); + if(not fp){ + return false; + } + return true; + } + } +} diff --git a/src/opt/hdf5-mpi/lib/h5_rfile.cc b/src/opt/hdf5-mpi/lib/h5_rfile.cc new file mode 100644 index 0000000..f6095e3 --- /dev/null +++ b/src/opt/hdf5-mpi/lib/h5_rfile.cc @@ -0,0 +1,33 @@ +// -*- C++ -*- +/** + + @file opt/hdf5/lib/h5_file.cc + @brief RFile implementations. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ + +#include "h5_rfile.h" +#include "mpi.h" + +namespace CNORXZ +{ + namespace hdf5 + { + RFile::RFile(const String& fname, bool _ro) : + File(fname, _ro) + { + mFAPL_id = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(mFAPL_id, MPI_COMM_WORLD, MPI_INFO_NULL); + } + + RFile::~RFile() + { + if(mFAPL_id){ + H5Pclose(mFAPL_id); + } + } + } +} diff --git a/src/opt/hdf5/include/h5_dataset.cc.h b/src/opt/hdf5/include/h5_dataset.cc.h index 2d11703..101578e 100644 --- a/src/opt/hdf5/include/h5_dataset.cc.h +++ b/src/opt/hdf5/include/h5_dataset.cc.h @@ -23,7 +23,7 @@ namespace CNORXZ { const hid_t tid = getTypeId(*data.data()); if(data.begin().formatIsTrivial()){ - init(data.range(), tid, data.data()); + initbase(data.range(), tid, data.data()); } else { CXZ_ERROR("Got array type with non-trivial format; non-contiguous data formats are not supported yet!"); diff --git a/src/opt/hdf5/include/h5_dataset.h b/src/opt/hdf5/include/h5_dataset.h index acd9cfe..bfd483b 100644 --- a/src/opt/hdf5/include/h5_dataset.h +++ b/src/opt/hdf5/include/h5_dataset.h @@ -48,14 +48,14 @@ namespace CNORXZ @param dataRange A potentially multi-dimensional range characterizing the dataset. @param type Data type id. */ - virtual Dataset& init(const RangePtr& dataRange, hid_t type); + virtual Dataset& initbase(const RangePtr& dataRange, hid_t type); /** Initalize the dataset. @param dataRange A potentially multi-dimensional range characterizing the dataset. @param type Data type id. @param data Pointer to raw data. */ - virtual Dataset& init(const RangePtr& dataRange, hid_t type, const void* data); + virtual Dataset& initbase(const RangePtr& dataRange, hid_t type, const void* data); /** Write data into dataset. @param dataRange A potentially multi-dimensional range characterizing the format of the data to be written. diff --git a/src/opt/hdf5/include/h5_file.h b/src/opt/hdf5/include/h5_file.h index 9ab01b2..8fb03ec 100644 --- a/src/opt/hdf5/include/h5_file.h +++ b/src/opt/hdf5/include/h5_file.h @@ -2,7 +2,7 @@ /** @file opt/hdf5/include/h5_file.h - @brief Group declaration. + @brief File declaration. Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de @@ -52,8 +52,9 @@ namespace CNORXZ */ bool ishdf5() const; - private: + protected: bool mRo = true; + hid_t mFAPL_id = H5P_DEFAULT; }; } } diff --git a/src/opt/hdf5/lib/h5_dataset.cc b/src/opt/hdf5/lib/h5_dataset.cc index 83f746a..f8e78eb 100644 --- a/src/opt/hdf5/lib/h5_dataset.cc +++ b/src/opt/hdf5/lib/h5_dataset.cc @@ -84,7 +84,7 @@ namespace CNORXZ return H5Lexists(mParent->id(), mName.c_str(), H5P_DEFAULT) > 0; } - Dataset& Dataset::init(const RangePtr& fileRange, hid_t type) + Dataset& Dataset::initbase(const RangePtr& fileRange, hid_t type) { CXZ_ASSERT(not isOpen(), "tried to initialize dataset that is already extisting"); mFileRange = fileRange; @@ -105,9 +105,9 @@ namespace CNORXZ return *this; } - Dataset& Dataset::init(const RangePtr& writeRange, hid_t type, const void* data) + Dataset& Dataset::initbase(const RangePtr& writeRange, hid_t type, const void* data) { - init(writeRange, type); + initbase(writeRange, type); writebase(writeRange, std::make_shared(mFileRange), data); return *this; } diff --git a/src/opt/hdf5/lib/h5_file.cc b/src/opt/hdf5/lib/h5_file.cc index 0730786..0703dd2 100644 --- a/src/opt/hdf5/lib/h5_file.cc +++ b/src/opt/hdf5/lib/h5_file.cc @@ -2,7 +2,7 @@ /** @file opt/hdf5/lib/h5_file.cc - @brief Group implementations. + @brief File implementations. Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de @@ -47,14 +47,14 @@ namespace CNORXZ if(mRo){ CXZ_ASSERT( ex == 1, "could not open file as read-only: '" << fn << "' does not exist'"); - mId = H5Fopen( fn.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT ); + mId = H5Fopen( fn.c_str(), H5F_ACC_RDONLY, mFAPL_id ); } else { if(ex == 1){ - mId = H5Fopen( fn.c_str(), H5F_ACC_RDWR, H5P_DEFAULT ); + mId = H5Fopen( fn.c_str(), H5F_ACC_RDWR, mFAPL_id ); } else { - mId = H5Fcreate( fn.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT ); + mId = H5Fcreate( fn.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, mFAPL_id ); } } CXZ_ASSERT( mId > 0, "error while opening file '" << fn << "'" );