From f597254d5d70937c566155ec3085a42a0dcbba36 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Wed, 30 Oct 2024 18:05:10 -0700 Subject: [PATCH] hdf5: dataset: some reorganization + group: add general add/get member functions --- src/opt/hdf5/include/h5_dataset.cc.h | 16 +++--- src/opt/hdf5/include/h5_dataset.h | 11 ++++- src/opt/hdf5/include/h5_group.cc.h | 18 +++++++ src/opt/hdf5/include/h5_group.h | 14 ++++++ src/opt/hdf5/lib/h5_dataset.cc | 73 +++++++++++++++++++--------- 5 files changed, 98 insertions(+), 34 deletions(-) diff --git a/src/opt/hdf5/include/h5_dataset.cc.h b/src/opt/hdf5/include/h5_dataset.cc.h index 74a3762..2d11703 100644 --- a/src/opt/hdf5/include/h5_dataset.cc.h +++ b/src/opt/hdf5/include/h5_dataset.cc.h @@ -22,8 +22,6 @@ namespace CNORXZ Dataset& Dataset::init(const ArrayBase& data) { const hid_t tid = getTypeId(*data.data()); - //VCHECK(tid); - //init(data.range(), tid); if(data.begin().formatIsTrivial()){ init(data.range(), tid, data.data()); } @@ -41,7 +39,7 @@ namespace CNORXZ template MArray SDataset::read() const { - MArray out(mDataRange); + MArray out(mFileRange); readbase(out.data(), nullptr, nullptr); return out; } @@ -50,12 +48,12 @@ namespace CNORXZ template MArray SDataset::read(const IndexInterface& idx) const { - CXZ_ASSERT(idx.dim() == mDataRange->dim(), "got index of inconsistent dimension, got" - << idx.dim() << ", expected " << mDataRange->dim()); + CXZ_ASSERT(idx.dim() == mFileRange->dim(), "got index of inconsistent dimension, got" + << idx.dim() << ", expected " << mFileRange->dim()); const RangePtr outrange = idx.range(); - CXZ_ASSERT(outrange->size() == mDataRange->size(), + CXZ_ASSERT(outrange->size() == mFileRange->size(), "got index of range of inconsistent size, expected " - << mDataRange->size() << ", got " << outrange->size()); + << mFileRange->size() << ", got " << outrange->size()); MArray out(outrange); readbase(out.data(), outrange, nullptr); return out; @@ -65,8 +63,8 @@ namespace CNORXZ template MArray SDataset::read(const IndexInterface& beg, const IndexInterface& end) const { - CXZ_ASSERT(beg.dim() == mDataRange->dim(), "got index of inconsistent dimension, got" - << beg.dim() << ", expected " << mDataRange->dim()); + CXZ_ASSERT(beg.dim() == mFileRange->dim(), "got index of inconsistent dimension, got" + << beg.dim() << ", expected " << mFileRange->dim()); const RangePtr outrange = beg.prange(end); MArray out(outrange); readbase(out.data(), outrange, toYptr(beg)); diff --git a/src/opt/hdf5/include/h5_dataset.h b/src/opt/hdf5/include/h5_dataset.h index ddd2079..acd9cfe 100644 --- a/src/opt/hdf5/include/h5_dataset.h +++ b/src/opt/hdf5/include/h5_dataset.h @@ -57,6 +57,13 @@ namespace CNORXZ */ virtual Dataset& init(const RangePtr& dataRange, hid_t type, const void* data); + /** Write data into dataset. + @param dataRange A potentially multi-dimensional range characterizing the format of the data to be written. + @param pos Position on target. + @param data Pointer to raw data to be writte. + */ + virtual Dataset& writebase(const RangePtr& dataRange, Sptr pos, const void* data); + /** Read the dataset. @param dest Pointer to destination. @param readrange Range of the destination data. @@ -78,14 +85,14 @@ namespace CNORXZ protected: Vector mkOff(const Sptr& beg) const; - RangePtr mDataRange; /**< The data range. */ + RangePtr mFileRange; /**< The range of the dataset. */ hid_t mType; /**< The data type identifier. */ hid_t mFilespace; /**< The hdf5 file space identifier. */ }; /** **** - Class to handle hdf5 datasets, the value type is known at compile time. + Class to handle hdf5 datasets, the value type is assumed to be known at compile time. @tparam T Dataset value type. */ template diff --git a/src/opt/hdf5/include/h5_group.cc.h b/src/opt/hdf5/include/h5_group.cc.h index e64fb38..64c6e3c 100644 --- a/src/opt/hdf5/include/h5_group.cc.h +++ b/src/opt/hdf5/include/h5_group.cc.h @@ -39,6 +39,13 @@ namespace CNORXZ return pos - beg; } + template + decltype(auto) Group::get(const String& name, F&& f) + { + auto i = this->getIndexTo(name); + return f(*i); + } + template Sptr> Group::getTable(const String& name, Tuple proto) { @@ -108,6 +115,17 @@ namespace CNORXZ return *this; } + template + Group& Group::add(const String& name, F&& f, const Args&... args) + { + CXZ_ASSERT(this->isOpen(), "tried to extend closed group"); + Vector nvec({name}); + mCont.extend( URangeFactory( nvec ).create() ); + auto ii = getIndexTo(name); + *ii = f(args...); + return *this; + } + template decltype(auto) Group::iter(F&& f) const { diff --git a/src/opt/hdf5/include/h5_group.h b/src/opt/hdf5/include/h5_group.h index 7fc4629..a4f1744 100644 --- a/src/opt/hdf5/include/h5_group.h +++ b/src/opt/hdf5/include/h5_group.h @@ -96,6 +96,13 @@ namespace CNORXZ */ const MArray& get() const; + /** Get object contained by this group. + @param name Object name. + @param f Function to interpret the object. + */ + template + decltype(auto) get(const String& name, F&& f); + /** Add a new group to this group. @param name Name of the created group. */ @@ -119,6 +126,13 @@ namespace CNORXZ template Group& addDataset(const String& name, const ArrayBase& data); + /** Add new object to this group. + @param name Object name. + @param f Function used to create the object. + */ + template + Group& add(const String& name, F&& f, const Args&... args); + /** Iterate over all group elements (const). @param f function object to be executed on each group element. */ diff --git a/src/opt/hdf5/lib/h5_dataset.cc b/src/opt/hdf5/lib/h5_dataset.cc index c944aeb..83f746a 100644 --- a/src/opt/hdf5/lib/h5_dataset.cc +++ b/src/opt/hdf5/lib/h5_dataset.cc @@ -53,7 +53,7 @@ namespace CNORXZ for(SizeT i = 0; i != ndims; ++i){ rs[i] = CRangeFactory(dims[i]).create(); } - mDataRange = yrange(rs); + mFileRange = yrange(rs); } return *this; } @@ -84,18 +84,18 @@ namespace CNORXZ return H5Lexists(mParent->id(), mName.c_str(), H5P_DEFAULT) > 0; } - Dataset& Dataset::init(const RangePtr& dataRange, hid_t type) + Dataset& Dataset::init(const RangePtr& fileRange, hid_t type) { CXZ_ASSERT(not isOpen(), "tried to initialize dataset that is already extisting"); - mDataRange = dataRange; + mFileRange = fileRange; const H5T_class_t tc = H5Tget_class(type); CXZ_ASSERT(tc != H5T_NO_CLASS, "id does not correspond to a data type"); // (did not found anythng better to check if type id is valid)... const hid_t dcpl_id = H5Pcreate(H5P_DATASET_CREATE); // TODO: all sub-ranges explicity!!!: - const SizeT ndim = dataRange->dim(); + const SizeT ndim = mFileRange->dim(); Vector exts(ndim); for(SizeT i = 0; i != ndim; ++i){ - exts[i] = static_cast( dataRange->sub(i)->size() ); + exts[i] = static_cast( mFileRange->sub(i)->size() ); } mFilespace = H5Screate_simple(ndim, exts.data(), NULL); mType = type; @@ -105,37 +105,64 @@ namespace CNORXZ return *this; } - Dataset& Dataset::init(const RangePtr& dataRange, hid_t type, const void* data) + Dataset& Dataset::init(const RangePtr& writeRange, hid_t type, const void* data) { - init(dataRange, type); - Vector dims(mDataRange->dim()); - for(SizeT i = 0; i != dims.size(); ++i){ - dims[i] = mDataRange->sub(i)->size(); - } - const hid_t memspace = H5Screate_simple(dims.size(), dims.data(), NULL); - H5Dwrite(mId, mType, memspace, mFilespace, H5P_DEFAULT, data); - H5Sclose(memspace); + init(writeRange, type); + writebase(writeRange, std::make_shared(mFileRange), data); return *this; } - void Dataset::readbase(void* dest, RangePtr readrange, Sptr beg) const + Dataset& Dataset::writebase(const RangePtr& writeRange, Sptr pos, const void* data) { - // TODO: Check if readrange is compatible with mDataRange!!! - if(not readrange){ - readrange = mDataRange; - } - Vector dims(readrange->dim()); + //CXZ_ERROR("TODO!!!"); + CXZ_ASSERT(writeRange->dim() == mFileRange->dim(), "dimension of data range (" + << writeRange->dim() << ") different from dimension of file range (" + << mFileRange->dim() << ")"); + Vector dims(writeRange->dim()); for(SizeT i = 0; i != dims.size(); ++i){ - dims[i] = readrange->sub(i)->size(); + dims[i] = writeRange->sub(i)->size(); + } + if(pos){ + CXZ_ASSERT(pos->range()->dim() == mFileRange->dim(), "dimension of position index (" + << pos->range()->dim() << ") different from dimension of file range (" + << mFileRange->dim() << ")"); + const Vector fpos = mkOff(pos); + H5Sselect_hyperslab(mFilespace, H5S_SELECT_SET, fpos.data(), NULL, dims.data(), NULL); + } + const hid_t memspace = H5Screate_simple(dims.size(), dims.data(), NULL); + const hid_t xfer_plist_id = H5Pcreate(H5P_DATASET_XFER); + H5Dwrite(mId, mType, memspace, mFilespace, xfer_plist_id, data); + H5Pclose(xfer_plist_id); + H5Sclose(memspace); + return *this; + } + + void Dataset::readbase(void* dest, RangePtr readRange, Sptr beg) const + { + // TODO: Check if readRange is compatible with mFileRange!!! + if(not readRange){ + readRange = mFileRange; + } + else { + CXZ_ASSERT(readRange->dim() == mFileRange->dim(), "dimension of data range (" + << readRange->dim() << ") different from dimension of file range (" + << mFileRange->dim() << ")"); + } + Vector dims(readRange->dim()); + for(SizeT i = 0; i != dims.size(); ++i){ + dims[i] = readRange->sub(i)->size(); } if(beg){ + CXZ_ASSERT(beg->range()->dim() == mFileRange->dim(), "dimension of position index (" + << beg->range()->dim() << ") different from dimension of file range (" + << mFileRange->dim() << ")"); const Vector fpos = mkOff(beg); H5Sselect_hyperslab(mFilespace, H5S_SELECT_SET, fpos.data(), NULL, dims.data(), NULL); } const hid_t mem_space_id = H5Screate_simple(static_cast(dims.size()), dims.data(), nullptr); const hid_t xfer_plist_id = H5Pcreate(H5P_DATASET_XFER); - //MArray out(readrange); + //MArray out(readRange); const herr_t err = H5Dread(mId, mType, mem_space_id, mFilespace, xfer_plist_id, dest); CXZ_ASSERT(err >= 0, "error while reading dataset '" << mName << "', errorcode :" << err); @@ -145,7 +172,7 @@ namespace CNORXZ const RangePtr& Dataset::dataRange() const { - return mDataRange; + return mFileRange; } Vector Dataset::mkOff(const Sptr& beg) const