From ca207d4013a3e91a3325eac956a5191c93270769 Mon Sep 17 00:00:00 2001
From: Christian Zimmermann <chizeta@f3l.de>
Date: Sun, 16 Mar 2025 23:51:58 -0700
Subject: [PATCH] WIP: rdataset: new read function
---
src/opt/hdf5_mpi/lib/h5_rdataset.cc | 111 ++++++++++++++++++++++++++++
1 file changed, 111 insertions(+)
diff --git a/src/opt/hdf5_mpi/lib/h5_rdataset.cc b/src/opt/hdf5_mpi/lib/h5_rdataset.cc
index 327fd2b..e499103 100644
--- a/src/opt/hdf5_mpi/lib/h5_rdataset.cc
+++ b/src/opt/hdf5_mpi/lib/h5_rdataset.cc
@@ -89,6 +89,117 @@ namespace CNORXZ
return *this;
}
+ void RDataset::readbase(void* dest, Sptr<YIndex> mbeg, Sptr<YIndex> mend,
+ Sptr<YIndex> fbeg) const
+ {
+ const SizeT D = mFileRange->dim();
+ const bool todo = mbeg != nullptr;
+ Vector<hsize_t> offset(D);
+ Vector<hsize_t> dims(D);
+ if(todo){
+ RangePtr dr = mbeg->range();
+ //const bool parallel = true;
+ CXZ_ASSERT(mend != nullptr, "no end edge given");
+ CXZ_ASSERT(beg->range()->dim() == D, "dimension of file index ("
+ << dr->dim() << ") different from dimension of file range ("
+ << mFileRange->dim() << ")");
+ CXZ_ASSERT(mbeg->range()->dim() == D, "dimension of data index ("
+ << dr->dim() << ") different from dimension of file range ("
+ << mFileRange->dim() << ")");
+ CXZ_ASSERT(mend->range()->dim() == D, "dimension of data index ("
+ << dr->dim() << ") different from dimension of file range ("
+ << mFileRange->dim() << ")");
+
+ for(SizeT i = 0; i != D; ++i){
+ const SizeT bi = mbeg->pack().get(i)->lex();
+ const SizeT ei = mend->pack().get(i)->lex();
+ const SizeT fgebi = fbeg->pack().get(i)->lex();
+ dims[i] = ei - bi + 1; // inclusive!
+ offset[i] = fbegi;
+ }
+ }
+ else {
+ for(SizeT i = 0; i != D; ++i){
+ dims[i] = 0;
+ offset[i] = 0;
+ }
+ }
+ H5Sselect_hyperslab(mFilespace, H5S_SELECT_SET, offset.data(), NULL, dims.data(), NULL);
+ const hid_t mem_space_id = H5Screate_simple(static_cast<hsize_t>(dims.size()),
+ dims.data(), nullptr);
+ const hid_t xfer_plist_id = H5Pcreate(H5P_DATASET_XFER);
+ H5Pset_dxpl_mpio(xfer_plist_id, H5FD_MPIO_COLLECTIVE);
+ const herr_t err = H5Dread(mId, mType, mem_space_id, mFilespace, xfer_plist_id, dest);
+ CXZ_ASSERT(err >= 0, "error while reading dataset '" << mName
+ << "', errorcode :" << err);
+ H5Pclose(xfer_plist_id);
+ H5Sclose(mem_space_id);
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ void RDataset::readbase(void* dest, Sptr<mpi::RIndex<YIndex,YIndex>> mbeg,
+ Sptr<mpi::RIndex<YIndex,YIndex>> mend, Sptr<YIndex> fbeg) const
+ {
+ const SizeT D = mFileRange->dim()
+ CXZ_ASSERT(beg->range()->dim() == D, "dimension of file index ("
+ << dr->dim() << ") different from dimension of file range ("
+ << mFileRange->dim() << ")");
+ CXZ_ASSERT(mbeg->range()->dim() == D, "dimension of data index ("
+ << dr->dim() << ") different from dimension of file range ("
+ << mFileRange->dim() << ")");
+ CXZ_ASSERT(mend->range()->dim() == D, "dimension of data index ("
+ << dr->dim() << ") different from dimension of file range ("
+ << mFileRange->dim() << ")");
+
+ Sptr<YIndex> mbegl = std::make_shared<YIndex>(mbeg->local());
+ Sptr<YIndex> mendl = std::make_shared<YIndex>(mend->local());
+ Sptr<YIndex> fbegl = std::make_shared<YIndex>(fbeg);
+
+ mbeg->localize();
+ mend->localize();
+ auto mbeg2 = std::make_shared<mpi::RIndex<YIndex,YIndex>>(mbeg->range());
+ mbeg2->synchronize();
+ auto rmy = mbeg2->rankI();
+ auto rb = mbeg->rankI();
+ auto re = mend->rankI();
+ auto lb = mbeg->local();
+ auto le = mend->local();
+
+ bool skip = false;
+ for(SizeT i = 0; i != offset.size(); ++i){
+ const SizeT rmyi = rmy->pack().get(i)->lex();
+ const SizeT rbi = rb->pack().get(i)->lex();
+ const SizeT rei = re->pack().get(i)->lex();
+ const SizeT lbi = lb->pack().get(i)->lex();
+ const SizeT lei = le->pack().get(i)->lex();
+ const SizeT Li = lb->pack().get(i)->range()->size();
+ const SizeT fgebi = fbeg->pack().get(i)->lex();
+
+ if(rei < rmyi or rbi > rmyi){
+ skip = true;
+ }
+ const SizeT lbegi = (rbi < rmyi) ? 0 : ( (rbi > rmyi) ? Li-1 : lbi );
+ const SizeT lendi = (rei < rmyi) ? 0 : ( (rei > rmyi) ? Li-1 : lei );
+
+ const SizeT offxi = Li - fbegi % Li + ( rmyi - rbi ) * Li;
+
+ *mbegl->pack().get(i) = lbegi;
+ *mendl->pack().get(i) = lendi;
+ *fbegl->pack().get(i) = (rbi == rmyi) ? fbegi : ( (rbi > rmyi) ? Li-1 : offxi );
+ }
+ (*mbegl)();
+ (*mendl)();
+ (*fbegl)();
+
+ if(skip){
+ // synchronous MPI read -> have to call this functio on all ranks
+ readbase(dest, nullptr, nullptr, fbegl);
+ }
+ else {
+ readbase(dest, mbegl, mendl, fbegl);
+ }
+ }
+
void RDataset::readbase(void* dest, RangePtr readRange, Sptr<YIndex> beg) const
{
RangePtr dr = readRange;