From 58fa19f51413bc9187ef2e17202fe50b45b62f58 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Tue, 24 Jan 2023 01:39:23 +0100 Subject: [PATCH] hdf5: preliminary append (init) table test --- src/include/ranges/urange.cc.h | 6 ++++++ src/include/ranges/urange.h | 1 + src/opt/hdf5/include/h5_content_base.h | 1 + src/opt/hdf5/include/h5_group.cc.h | 12 ++++++++++- src/opt/hdf5/include/h5_table.cc.h | 26 ++++++++++++++++-------- src/opt/hdf5/include/h5_table.h | 2 +- src/opt/hdf5/include/h5_types.cc.h | 18 ++++++++++++++++ src/opt/hdf5/include/h5_types.h | 4 +++- src/opt/hdf5/tests/h5_basic_unit_test.cc | 7 +++++++ 9 files changed, 66 insertions(+), 11 deletions(-) diff --git a/src/include/ranges/urange.cc.h b/src/include/ranges/urange.cc.h index db64eec..89a0bfe 100644 --- a/src/include/ranges/urange.cc.h +++ b/src/include/ranges/urange.cc.h @@ -243,6 +243,12 @@ namespace CNORXZ return mSpace[pos]; } + template + const MetaType* URange::get() const + { + return mSpace.data(); + } + template SizeT URange::getMeta(const MetaType& meta) const { diff --git a/src/include/ranges/urange.h b/src/include/ranges/urange.h index 5fdbcf2..745ced5 100644 --- a/src/include/ranges/urange.h +++ b/src/include/ranges/urange.h @@ -99,6 +99,7 @@ namespace CNORXZ virtual RangePtr extend(const RangePtr& r) const override final; const MetaType& get(SizeT pos) const; + const MetaType* get() const; SizeT getMeta(const MetaType& metaPos) const; private: diff --git a/src/opt/hdf5/include/h5_content_base.h b/src/opt/hdf5/include/h5_content_base.h index ce77097..dace4d6 100644 --- a/src/opt/hdf5/include/h5_content_base.h +++ b/src/opt/hdf5/include/h5_content_base.h @@ -4,6 +4,7 @@ #include "cnorxz.h" #include +#include namespace CNORXZ { diff --git a/src/opt/hdf5/include/h5_group.cc.h b/src/opt/hdf5/include/h5_group.cc.h index 4cb1bea..554bc44 100644 --- a/src/opt/hdf5/include/h5_group.cc.h +++ b/src/opt/hdf5/include/h5_group.cc.h @@ -19,7 +19,17 @@ namespace CNORXZ const RangePtr& fields) { CXZ_ASSERT(this->isOpen(), "tried to extend closed group"); - + Vector nvec({name}); + Vector dvec({DType(name)}); + auto extr = URangeFactory( nvec ).create(); + mCont.extend(extr); + auto ii = mCont.begin(); + ii.at(dvec); // 'at' returns YIndex&, so cannot use it inline... + auto tab = std::make_shared>(name, this, fields); + for(auto& d: data){ + tab->appendRecord(d); + } + *ii = tab; return *this; } diff --git a/src/opt/hdf5/include/h5_table.cc.h b/src/opt/hdf5/include/h5_table.cc.h index c148ad1..cc42228 100644 --- a/src/opt/hdf5/include/h5_table.cc.h +++ b/src/opt/hdf5/include/h5_table.cc.h @@ -31,20 +31,20 @@ namespace CNORXZ Tuple x; if(mRecords == nullptr) { - auto mOffsets = MArray( mFields, iter<0,N> - ( [&](auto i) { return &std::get(x) - &x; }, + mOffsets = MArray( mFields, iter<0,N> + ( [&](auto i) { return getTupleOffset(x, i); }, [](const auto&... e) { return Vector({e...}); }) ); - auto mSizes = MArray( mFields, iter<0,N> + mSizes = MArray( mFields, iter<0,N> ( [&](auto i) { return sizeof(std::get(x)); }, [](const auto&... e) { return Vector({e...}); }) ); - auto mTypes = MArray( mFields, iter<0,N> + mTypes = MArray( mFields, iter<0,N> ( [&](auto i) { return getTypeId(std::get(x)); }, [](const auto&... e) { return Vector({e...}); }) ); } else { iter<0,N>( [&](auto i) { CXZ_ASSERT - ( &std::get(x) - &x == mOffsets.data()[i], - "wrong offset for field " << i << ": " << &std::get(x) - &x + ( getTupleOffset(x, i) == mOffsets.data()[i], + "wrong offset for field " << i << ": " << getTupleOffset(x, i) << " vs " << mOffsets.data()[i] ); }, NoF{} ); iter<0,N>( [&](auto i) { CXZ_ASSERT ( sizeof(std::get(x)) == mSizes.data()[i], @@ -60,15 +60,25 @@ namespace CNORXZ template STable& STable::appendRecord(const Tuple& t) { + constexpr hsize_t chunk_size = sizeof(t); + constexpr Int compress = 0; RangePtr appr = CRangeFactory(1).create(); if(mRecords == nullptr){ mRecords = appr; + Vector fields(mFields->size()); + auto fr = std::dynamic_pointer_cast>(mFields); + for(auto fi = fr->begin(); fi != fr->end(); ++fi){ + fields[fi.lex()] = (*fi).c_str(); + } + H5TBmake_table(mName.c_str(), mParent->id(), mName.c_str(), mFields->size(), + mRecords->size(), sizeof(t), fields.data(), mOffsets.data(), + mTypes.data(), chunk_size, NULL, compress, &t); } else { mRecords = mRecords->extend(appr); + H5TBappend_records(mParent->id(), mName.c_str(), 1, sizeof(t), + mOffsets.data(), mSizes.data(), &t); } - H5TBappend_records(mParent->id(), mName, 1, sizeof(t), - mOffsets.data(), mSizes.data(), &t); return *this; } } diff --git a/src/opt/hdf5/include/h5_table.h b/src/opt/hdf5/include/h5_table.h index cb113dc..18a6476 100644 --- a/src/opt/hdf5/include/h5_table.h +++ b/src/opt/hdf5/include/h5_table.h @@ -26,7 +26,7 @@ namespace CNORXZ Table& appendRecord(SizeT n, const char* data); Table& readRecord(SizeT pos, SizeT n, char* data); - private: + protected: RangePtr mRecords; RangePtr mFields; // -> FIndex (position -> offset) MArray mSizes; diff --git a/src/opt/hdf5/include/h5_types.cc.h b/src/opt/hdf5/include/h5_types.cc.h index 81784f1..b7c93ac 100644 --- a/src/opt/hdf5/include/h5_types.cc.h +++ b/src/opt/hdf5/include/h5_types.cc.h @@ -2,10 +2,24 @@ #ifndef __cxz_h5_types_cc_h__ #define __cxz_h5_types_cc_h__ +#include "base/types.h" + namespace CNORXZ { namespace hdf5 { + template + SizeT getTupleOffset(const Tuple& t, CSizeT i) + { + const PtrId beg = reinterpret_cast(&t); + const PtrId pos = reinterpret_cast(&std::get(t)); + return pos - beg; + } + + /************** + * TypeId * + **************/ + template inline hid_t TypeId::get() { @@ -34,6 +48,10 @@ namespace CNORXZ return arrtype; } + /***************** + * getTypeId * + *****************/ + template hid_t getTypeId(T x) { diff --git a/src/opt/hdf5/include/h5_types.h b/src/opt/hdf5/include/h5_types.h index 7ecc4bf..6036760 100644 --- a/src/opt/hdf5/include/h5_types.h +++ b/src/opt/hdf5/include/h5_types.h @@ -6,6 +6,8 @@ namespace CNORXZ { namespace hdf5 { + template + SizeT getTupleOffset(const Tuple& t, CSizeT i); /************** * TypeId * @@ -31,7 +33,7 @@ namespace CNORXZ *****************/ template - hid_t getTypeId(const T& x); + hid_t getTypeId(T x); } } diff --git a/src/opt/hdf5/tests/h5_basic_unit_test.cc b/src/opt/hdf5/tests/h5_basic_unit_test.cc index 6535862..2ddeb6a 100644 --- a/src/opt/hdf5/tests/h5_basic_unit_test.cc +++ b/src/opt/hdf5/tests/h5_basic_unit_test.cc @@ -67,6 +67,13 @@ namespace h5f.addGroup("gr1"); h5f.addGroup("gr2"); EXPECT_EQ(h5f.get().size(), 2u); + + // move to separate test: + RangePtr rs = CRangeFactory(2).create(); + RangePtr fs = URangeFactory(Vector({"field1","second","real"})).create(); + Vector> v({ {0, -6, 3.141}, {3, -8, 0.789} }); + MArray> a(rs, std::move(v)); + std::dynamic_pointer_cast(h5f.get().data()[0])->addTable("tab1", a, fs); } TEST_F(Group_Test, Read)