chizeta/cnorxz
chizeta/cnorxz
0
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

add table read functions
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
Details

Browse source
This commit is contained in:
Christian Zimmermann 2024-01-28 22:00:02 +01:00
parent b2637802be
commit e3ad04e56c
6 changed files with 144 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
src/opt/hdf5/include/h5_table.cc.h
View file

@ -22,26 +22,25 @@ namespace CNORXZ
Table(name, _parent)
{
constexpr SizeT N = sizeof...(Ts);
if(mFields == nullptr){
if(mFields != nullptr){
CXZ_ASSERT(mFields->size() == N, "expected tuple of size = " << mFields->size()
<< ", got: " << N);
Tuple<Ts...> x;
iter<0,N>( [&](auto i) { CXZ_ASSERT
( getTupleOffset(x, i) == mOffsets.data()[i],
"wrong offset for field " << i << ": " << getTupleOffset(x, i)
<< " vs " << mOffsets.data()[i] ); }, NoF{} );
iter<0,N>( [&](auto i) { CXZ_ASSERT
( sizeof(std::get<i>(x)) == mSizes.data()[i],
"wrong size for field " << i << ": " << sizeof(std::get<i>(x))
( sizeof(tget<i>(x)) == mSizes.data()[i],
"wrong size for field " << i << ": " << sizeof(tget<i>(x))
<< " vs " << mSizes.data()[i] ); }, NoF{} );
iter<0,N>( [&](auto i) { CXZ_ASSERT
( getTypeId(std::get<i>(x)) == mTypes.data()[i],
"wrong type for field " << i << ": " << getTypeId(std::get<i>(x))
( H5Tget_class(getTypeId(tget<i>(x))) == mTypes.data()[i],
"wrong type for field " << i
<< ": " << H5Tget_class(getTypeId(tget<i>(x)))
<< " vs " << mTypes.data()[i] ); }, NoF{} );
}
else {
CXZ_ASSERT(mFields->size() == N, "expected tuple of size = " << mFields->size()
<< ", got: " << N);
}
}
template <typename... Ts>
@ -73,11 +72,11 @@ namespace CNORXZ
[](const auto&... e) { return Vector<SizeT>({e...}); }) );
mSizes = MArray<SizeT>
( mFields, iter<0,N>
( [&](auto i) { return sizeof(std::get<i>(x)); },
( [&](auto i) { return sizeof(tget<i>(x)); },
[](const auto&... e) { return Vector<SizeT>({e...}); }) );
mTypes = MArray<hid_t>
( mFields, iter<0,N>
( [&](auto i) { return getTypeId(std::get<i>(x)); },
( [&](auto i) { return getTypeId(tget<i>(x)); },
[](const auto&... e) { return Vector<hid_t>({e...}); }) );
return *this;
}
@ -96,10 +95,12 @@ namespace CNORXZ
}
template <typename... Ts>
MArray<Tuple<Ts...>> STable<Ts...>::read()
MArray<Tuple<Ts...>> STable<Ts...>::read() const
{
CXZ_ASSERT(isOpen(), "attempt to read table that has not been opened");
MArray<Tuple<Ts...>> out(mRecords);
//...!!!
H5TBread_table(mParent->id(), mName.c_str(), mTypesize, mOffsets.data(),
mSizes.data(), out.data());
return out;
}
}

9
src/opt/hdf5/include/h5_table.h
View file

@ -29,6 +29,8 @@ namespace CNORXZ
Table& initTable(SizeT n, const void* data, SizeT dsize, SizeT chunk_size);
Table& appendRecord(SizeT n, const void* data, SizeT dsize);
Table& readRecord(SizeT pos, SizeT n, char* data);
MArray<DType> read() const;
const RangePtr& fields() const;
const RangePtr& records() const;
@ -39,8 +41,11 @@ namespace CNORXZ
MArray<SizeT> mOffsets;
MArray<hid_t> mTypes;
hid_t mType = 0;
SizeT mTypesize = 0;
};
// caution: the memory ordering is the only thing that counts;
// std::tuple has REVERSE ordering!
template <typename... Ts>
class STable : public Table
{
@ -51,11 +56,11 @@ namespace CNORXZ
const Arr<String,sizeof...(Ts)>& fnames);
STable& initFields(const Arr<String,sizeof...(Ts)>& fnames);
STable& appendRecord(const Tuple<Ts...>& t);
STable& appendRecord(const MArray<Tuple<Ts...>>& t);
MArray<Tuple<Ts...>> read();
MArray<Tuple<Ts...>> read() const;
template <class F>
decltype(auto) iterRecords(F&& f) const;

22
src/opt/hdf5/include/h5_type_id.cc.h
View file

@ -8,11 +8,31 @@ namespace CNORXZ
{
namespace hdf5
{
template <typename... Ts>
decltype(auto) reverseTuple(const Tuple<Ts...>& t)
{
return iter<0,sizeof...(Ts)>
( [&](auto i) { return tget<i>(t); },
[](const auto&... e) { return std::make_tuple(e...); } );
}
template <SizeT I, typename... Ts>
constexpr const auto& tget(const Tuple<Ts...>& t)
{
return std::get<sizeof...(Ts)-I-1>(t);
}
template <SizeT I, typename... Ts>
constexpr auto& tget(Tuple<Ts...>& t)
{
return std::get<sizeof...(Ts)-I-1>(t);
}
template <SizeT N, typename... Ts>
SizeT getTupleOffset(const Tuple<Ts...>& t, CSizeT<N> i)
{
const PtrId beg = reinterpret_cast<PtrId>(&t);
const PtrId pos = reinterpret_cast<PtrId>(&std::get<i>(t));
const PtrId pos = reinterpret_cast<PtrId>(&tget<i>(t));
return pos - beg;
}

13
src/opt/hdf5/include/h5_type_id.h
View file

@ -2,10 +2,23 @@
#ifndef __cxz_h5_type_id_h__
#define __cxz_h5_type_id_h__
#include "base/types.h"
namespace CNORXZ
{
namespace hdf5
{
template <typename... Ts>
decltype(auto) reverseTuple(const Tuple<Ts...>& t);
template <typename... Ts>
using RTuple = typename std::remove_reference<decltype(reverseTuple(Tuple<Ts...>()))>::type;
template <SizeT I, typename... Ts>
constexpr const auto& tget(const Tuple<Ts...>& t);
template <SizeT I, typename... Ts>
constexpr auto& tget(Tuple<Ts...>& t);
template <SizeT N, typename... Ts>
SizeT getTupleOffset(const Tuple<Ts...>& t, CSizeT<N> i);

56
src/opt/hdf5/lib/h5_table.cc
View file

@ -21,9 +21,8 @@ namespace CNORXZ
}
Vector<SizeT> offsets(nfields);
Vector<SizeT> sizes(nfields);
SizeT typesize = 0;
H5TBget_field_info(mParent->id(), mName.c_str(), fieldsptr.data(), sizes.data(),
offsets.data(), &typesize);
offsets.data(), &mTypesize);
Vector<FieldID> fields(nfields);
for(SizeT i = 0; i != nfields; ++i){
fields[i].first = i;
@ -132,6 +131,59 @@ namespace CNORXZ
CXZ_ERROR("not implemented!!!");
return *this;
}
MArray<DType> Table::read() const
{
Vector<char> buf(mTypesize*mRecords->size());
H5TBread_table(mParent->id(), mName.c_str(), mTypesize, mOffsets.data(),
mSizes.data(), buf.data());
const hid_t dset_id = H5Dopen(mParent->id(), mName.c_str(), H5P_DEFAULT);
const hid_t type_id = H5Dget_type(dset_id);
MArray<DType> out(mRecords*mFields);
CIndex fi(mFields);
CIndex ri(mRecords);
const char* b = nullptr;
for(fi = 0; fi.lex() != mFields->size(); ++fi){
const SizeT off = mOffsets[fi];
const hid_t tp = H5Tget_member_type( type_id, fi.lex() );
const hid_t tc = H5Tget_class(tp);
const size_t ts = H5Tget_size(tp);
//const bool sig = H5Tget_sign(tp);
DType x;
switch(tc){
case H5T_INTEGER: {
if(ts == 4){
for(ri = 0, b = buf.data(); ri.lex() != mRecords->size(); ++ri, b += mTypesize){
const Int xi = *reinterpret_cast<const Int*>(b+off);
out[ri*fi] = DType(xi);
}
break;
}
else if(ts == 8){
for(ri = 0, b = buf.data(); ri.lex() != mRecords->size(); ++ri, b += mTypesize){
const LInt xi = *reinterpret_cast<const LInt*>(b+off);
out[ri*fi] = DType(xi);
}
break;
}
}
case H5T_FLOAT: {
if(ts == 8){
for(ri = 0, b = buf.data(); ri.lex() != mRecords->size(); ++ri, b += mTypesize){
const Double xi = *reinterpret_cast<const Double*>(b+off);
out[ri*fi] = DType(xi);
}
break;
}
}
default:
CXZ_ERROR("type " << tp << " not supported");
}
}
H5Tclose(type_id);
H5Dclose(dset_id);
return out;
}
const RangePtr& Table::fields() const
{

47
src/opt/hdf5/tests/h5_basic_unit_test.cc
View file

@ -39,24 +39,32 @@ namespace
Group_Test()
{
mFileName = testh5file;
//mGrps = { "gr1", "gr2", "foo", "bar", "moregroups" };
mFs = {"field1","second","real"};
Vector<Tuple<SizeT,Int,Double>> v
( { {0, -6, 3.141},
{3, -8, 0.789},
{34, 4, 10.009},
{2, -777, -9.77},
{321, 0, -0.003}
// Tuple has reverse (!) memory ordering:
Vector<Tuple<Double,Int,SizeT>> v
( { { 3.141, -6, 0 },
{ 0.789, -8, 3 },
{ 10.009, 4, 34 },
{ -9.77, -777, 2 },
{ -0.003, 0, 321 }
} );
RangePtr rs = CRangeFactory(v.size()).create();
mTabA = MArray<Tuple<SizeT,Int,Double>>(rs, std::move(v));
RangePtr fs = CRangeFactory(mFs.size()).create();
mTabA = MArray<Tuple<Double,Int,SizeT>>(rs, std::move(v));
mTabD = MArray<DType>(rs*fs);
CIndex i(rs);
CIndex j(fs);
for(i = 0; i.lex() != rs->size(); ++i){
iter<0,3>( [&](auto jj)
{ j = jj; mTabD[i*j] = DType(tget<jj>(v[i.lex()])); }, NoF{} );
}
}
String mFileName;
//Vector<String> mGrps;
Arr<String,3> mFs;
MArray<Tuple<SizeT,Int,Double>> mTabA;
MArray<Tuple<Double,Int,SizeT>> mTabA;
MArray<DType> mTabD;
};
@ -142,14 +150,29 @@ namespace
TEST_F(Group_Test, ReadTable)
{
typedef Tuple<SizeT,Int,Double> RecType;
typedef Tuple<Double,Int,SizeT> RecType;
File h5f(mFileName, true);
h5f.open();
auto tab = h5f.getGroup("gr1")->open().getTable("tab1", RecType());
EXPECT_EQ(tab->fields()->size(), 3u);
EXPECT_EQ(tab->records()->size(), 5u);
tab->open();
auto cont = tab->read();
EXPECT_EQ(cont.size(), mTabA.size());
CIndex i(mTabA.range());
for(; i.lex() != i.lmax().val(); ++i){
EXPECT_EQ( cont[i], mTabA[i] );
}
auto dtab = h5f.getGroup("gr1")->open().getTable("tab1");
auto dcont = dtab->read();
EXPECT_EQ(dcont.range()->dim(), 2u);
EXPECT_EQ(dcont.range()->sub(0)->size(), 5u);
EXPECT_EQ(dcont.range()->sub(1)->size(), 3u);
for(auto ai = dcont.begin(); ai != dcont.end(); ++ai){
EXPECT_EQ(dcont[ai].str(), mTabD[ai].str());
}
h5f.close();
}
}