chizeta/cnorxz
chizeta/cnorxz
0
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

improve table read / readRecord
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
Details

Browse source
This commit is contained in:
Christian Zimmermann 2024-01-29 23:54:25 +01:00
parent e3ad04e56c
commit f7c00eab8c
3 changed files with 97 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/opt/hdf5/include/h5_table.cc.h
View file

@ -89,7 +89,7 @@ namespace CNORXZ
initTable(1, &t, sizeof(t), sizeof(t)); initTable(1, &t, sizeof(t), sizeof(t));
} }
else { else {
Table::appendRecord(1, &t, sizeof(t)); Table::appendRecords(1, &t);
} }
return *this; return *this;
} }

8
src/opt/hdf5/include/h5_table.h
View file

@ -27,8 +27,9 @@ namespace CNORXZ
Table& initFieldNames(const Vector<String>& fnames); Table& initFieldNames(const Vector<String>& fnames);
Table& initTable(SizeT n, const void* data, SizeT dsize, SizeT chunk_size); Table& initTable(SizeT n, const void* data, SizeT dsize, SizeT chunk_size);
Table& appendRecord(SizeT n, const void* data, SizeT dsize); Table& appendRecords(SizeT n, const void* data);
Table& readRecord(SizeT pos, SizeT n, char* data); Table& readRecords(SizeT pos, SizeT n, char* data);
MArray<DType> readRecord(SizeT pos) const;
MArray<DType> read() const; MArray<DType> read() const;
const RangePtr& fields() const; const RangePtr& fields() const;
@ -42,6 +43,9 @@ namespace CNORXZ
MArray<hid_t> mTypes; MArray<hid_t> mTypes;
hid_t mType = 0; hid_t mType = 0;
SizeT mTypesize = 0; SizeT mTypesize = 0;
MArray<std::function<DType(const char*)>> mInterpret;
void mkTypes();
}; };
// caution: the memory ordering is the only thing that counts; // caution: the memory ordering is the only thing that counts;

140
src/opt/hdf5/lib/h5_table.cc
View file

@ -65,6 +65,7 @@ namespace CNORXZ
if(mId == 0){ if(mId == 0){
mId = H5Dopen(mParent->id(), mName.c_str(), H5P_DEFAULT); mId = H5Dopen(mParent->id(), mName.c_str(), H5P_DEFAULT);
mType = H5Dget_type(mId); mType = H5Dget_type(mId);
mkTypes();
} }
return *this; return *this;
} }
@ -110,78 +111,58 @@ namespace CNORXZ
for(auto fi = fr->begin(); fi != fr->end(); ++fi){ for(auto fi = fr->begin(); fi != fr->end(); ++fi){
fields[fi.lex()] = (*fi).second.c_str(); fields[fi.lex()] = (*fi).second.c_str();
} }
mTypesize = dsize;
const herr_t err = H5TBmake_table const herr_t err = H5TBmake_table
(mName.c_str(), mParent->id(), mName.c_str(), mFields->size(), mRecords->size(), dsize, (mName.c_str(), mParent->id(), mName.c_str(), mFields->size(), mRecords->size(), mTypesize,
fields.data(), mOffsets.data(), mTypes.data(), chunk_size, NULL, compress, data); fields.data(), mOffsets.data(), mTypes.data(), chunk_size, NULL, compress, data);
CXZ_ASSERT(err >= 0, "error while initialzing table: error code = " << err); CXZ_ASSERT(err >= 0, "error while initialzing table: error code = " << err);
return *this; return *this;
} }
Table& Table::appendRecord(SizeT n, const void* data, SizeT dsize) Table& Table::appendRecords(SizeT n, const void* data)
{ {
mRecords = mRecords->extend( CRangeFactory(1).create() ); mRecords = mRecords->extend( CRangeFactory(n).create() );
const herr_t err = H5TBappend_records(mParent->id(), mName.c_str(), n, dsize, const herr_t err = H5TBappend_records(mParent->id(), mName.c_str(), n, mTypesize,
mOffsets.data(), mSizes.data(), data); mOffsets.data(), mSizes.data(), data);
CXZ_ASSERT(err >= 0, "error while appending record to table: error code = " << err); CXZ_ASSERT(err >= 0, "error while appending record to table: error code = " << err);
return *this; return *this;
} }
Table& Table::readRecord(SizeT pos, SizeT n, char* data) Table& Table::readRecords(SizeT pos, SizeT n, char* data)
{ {
CXZ_ERROR("not implemented!!!"); H5TBread_records(mParent->id(), mName.c_str(), pos, n, mTypesize, mOffsets.data(),
mSizes.data(), data);
return *this; return *this;
} }
MArray<DType> Table::readRecord(SizeT pos) const
{
Vector<char> buf(mTypesize);
H5TBread_records(mParent->id(), mName.c_str(), pos, 1, mTypesize, mOffsets.data(),
mSizes.data(), buf.data());
MArray<DType> out(mFields);
auto fi = std::make_shared<CIndex>(mFields);
out(fi) = operation( [&](const SizeT& off, const std::function<DType(const char*)>& f){
return f( buf.data() + off );
}, mOffsets(fi), mInterpret(fi) );
return out;
}
MArray<DType> Table::read() const MArray<DType> Table::read() const
{ {
Vector<char> buf(mTypesize*mRecords->size()); Vector<char> buf(mTypesize*mRecords->size());
H5TBread_table(mParent->id(), mName.c_str(), mTypesize, mOffsets.data(), H5TBread_table(mParent->id(), mName.c_str(), mTypesize, mOffsets.data(),
mSizes.data(), buf.data()); mSizes.data(), buf.data());
const hid_t dset_id = H5Dopen(mParent->id(), mName.c_str(), H5P_DEFAULT);
const hid_t type_id = H5Dget_type(dset_id);
MArray<DType> out(mRecords*mFields); MArray<DType> out(mRecords*mFields);
CIndex fi(mFields); auto fi = std::make_shared<CIndex>(mFields);
CIndex ri(mRecords); auto ri = std::make_shared<CIndex>(mRecords);
const char* b = nullptr;
for(fi = 0; fi.lex() != mFields->size(); ++fi){ out(ri*fi) = operation( [&](const SizeT& off, const std::function<DType(const char*)>& f, const SizeT r){
const SizeT off = mOffsets[fi]; return f( buf.data() + r*mTypesize + off );
const hid_t tp = H5Tget_member_type( type_id, fi.lex() ); }, mOffsets(fi), mInterpret(fi), xpr(ri) );
const hid_t tc = H5Tget_class(tp);
const size_t ts = H5Tget_size(tp);
//const bool sig = H5Tget_sign(tp);
DType x;
switch(tc){
case H5T_INTEGER: {
if(ts == 4){
for(ri = 0, b = buf.data(); ri.lex() != mRecords->size(); ++ri, b += mTypesize){
const Int xi = *reinterpret_cast<const Int*>(b+off);
out[ri*fi] = DType(xi);
}
break;
}
else if(ts == 8){
for(ri = 0, b = buf.data(); ri.lex() != mRecords->size(); ++ri, b += mTypesize){
const LInt xi = *reinterpret_cast<const LInt*>(b+off);
out[ri*fi] = DType(xi);
}
break;
}
}
case H5T_FLOAT: {
if(ts == 8){
for(ri = 0, b = buf.data(); ri.lex() != mRecords->size(); ++ri, b += mTypesize){
const Double xi = *reinterpret_cast<const Double*>(b+off);
out[ri*fi] = DType(xi);
}
break;
}
}
default:
CXZ_ERROR("type " << tp << " not supported");
}
}
H5Tclose(type_id);
H5Dclose(dset_id);
return out; return out;
} }
@ -195,5 +176,64 @@ namespace CNORXZ
return mRecords; return mRecords;
} }
void Table::mkTypes()
{
mTypes = MArray<hid_t>(mFields);
auto i = std::make_shared<CIndex>(mFields);
mTypes(i) = operation( [&](const SizeT n){ return H5Tget_member_type( mType, n ); } , xpr(i) );
mInterpret = MArray<std::function<DType(const char*)>>(mFields);
mInterpret(i) = operation( [](const hid_t tid){
const hid_t tc = H5Tget_class(tid);
const size_t ts = H5Tget_size(tid);
const bool sig = H5Tget_sign(tid);
std::function<DType(const char*)> of;
switch(tc){
case H5T_INTEGER:{
switch(ts){
case 1:{ // 8-bit
of = sig ?
[](const char* d){ const int8_t x = *reinterpret_cast<const int8_t*>(d); return DType(static_cast<Int>(x)); } :
[](const char* d){ const uint8_t x = *reinterpret_cast<const uint8_t*>(d); return DType(static_cast<SizeT>(x)); };
break;
}
case 2:{ // 16-bit
of = sig ?
[](const char* d){ const int16_t x = *reinterpret_cast<const int16_t*>(d); return DType(static_cast<Int>(x)); } :
[](const char* d){ const uint16_t x = *reinterpret_cast<const uint16_t*>(d); return DType(static_cast<SizeT>(x)); };
break;
}
case 4:{ // 32-bit
of = sig ?
[](const char* d){ const int32_t x = *reinterpret_cast<const int32_t*>(d); return DType(static_cast<Int>(x)); } :
[](const char* d){ const uint32_t x = *reinterpret_cast<const uint32_t*>(d); return DType(static_cast<SizeT>(x)); };
break;
}
case 8:{ // 64-bit
of = sig ?
[](const char* d){ const int64_t x = *reinterpret_cast<const int64_t*>(d); return DType(static_cast<LInt>(x)); } :
[](const char* d){ const uint64_t x = *reinterpret_cast<const uint64_t*>(d); return DType(static_cast<SizeT>(x)); };
break;
}
default:
CXZ_ERROR("got integer of weird size: " << ts);
}
break;
}
case H5T_FLOAT:{
if(ts == 4) {
of = [](const char* d){ const float x = *reinterpret_cast<const float*>(d); return DType(static_cast<Double>(x)); };
}
else {
CXZ_ASSERT(ts == 8, "got float of weird size: " << ts);
of = [](const char* d){ const double x = *reinterpret_cast<const double*>(d); return DType(static_cast<Double>(x)); };
}
break;
}
default:
CXZ_ERROR("got unsupported type");
};
return of;
}, mTypes(i) );
}
} }
} }