chizeta/cnorxz
chizeta/cnorxz
0
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

hdf5: dataset
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
Details

Browse source
This commit is contained in:
Christian Zimmermann 2024-02-01 00:09:39 +01:00
parent e2e233b931
commit 6f3601daa4
9 changed files with 155 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/opt/hdf5/include/cnorxz_hdf5.cc.h
View file

@ -3,3 +3,4 @@
#include "h5_type_id.cc.h"
#include "h5_group.cc.h"
#include "h5_table.cc.h"
#include "h5_dataset.cc.h"

1
src/opt/hdf5/include/cnorxz_hdf5.h
View file

@ -3,6 +3,7 @@
#include "h5_file.h"
#include "h5_group.h"
#include "h5_table.h"
#include "h5_dataset.h"
#include "h5_type_id.h"
#include "cnorxz_hdf5.cc.h"

88
src/opt/hdf5/include/h5_dataset.cc.h
View file

@ -2,6 +2,8 @@
#ifndef __cxz_h5_dataset_cc_h__
#define __cxz_h5_dataset_cc_h__
#include "h5_dataset.h"
namespace CNORXZ
{
namespace hdf5
@ -9,9 +11,91 @@ namespace CNORXZ
template <typename T>
Dataset& Dataset::init(const ArrayBase<T>& data)
{
// !!!!
const hid_t tid = getTypeId(*data.data());
init(data.range(), tid);
if(data.begin().formatIsTrivial()){
Vector<hsize_t> dims(mDataRange->dim());
for(SizeT i = 0; i != dims.size(); ++i){
dims[i] = mDataRange->sub(i)->size();
}
const hid_t memspace = H5Screate_simple(dims.size(), dims.data(), NULL);
H5Dwrite(mId, mType, memspace, mFilespace, H5P_DEFAULT, data.data());
H5Sclose(memspace);
}
else {
CXZ_ERROR("IMPLEMENT!!!");
}
return *this;
}
template <typename T>
SDataset<T>::SDataset(const String& name, const ContentBase* _parent) :
Dataset(name, _parent)
{}
template <typename T>
MArray<T> SDataset<T>::read() const
{
Vector<hsize_t> dims(mDataRange->dim());
for(SizeT i = 0; i != dims.size(); ++i){
dims[i] = mDataRange->sub(i)->size();
}
const hid_t mem_space_id = H5Screate_simple(static_cast<hsize_t>(dims.size()),
dims.data(), nullptr);
const hid_t xfer_plist_id = H5Pcreate(H5P_DATASET_XFER);
MArray<T> out(mDataRange);
const herr_t err = H5Dread(mId, mType, mem_space_id, mFilespace, xfer_plist_id, out.data());
CXZ_ASSERT(err >= 0, "error while reading dataset '" << mName
<< "', errorcode :" << err);
H5Pclose(xfer_plist_id);
H5Sclose(mem_space_id);
return out;
}
template <typename T>
template <class I, typename M>
MArray<T> SDataset<T>::read(const IndexInterface<I,M>& beg, const IndexInterface<I,M>& end) const
{
CXZ_ASSERT(beg.dim() == mDataRange->dim(), "got index of inconsistent dimension, got"
<< beg.dim() << ", expected " << mDataRange->dim());
const RangePtr outrange = beg.prange(end);
Vector<hsize_t> dims(outrange->dim());
for(SizeT i = 0; i != dims.size(); ++i){
dims[i] = outrange->sub(i)->size();
}
const Vector<hsize_t> fpos = mkFPos(beg);
H5Sselect_hyperslab(mFilespace, H5S_SELECT_SET, fpos.data(), NULL, dims.data(), NULL);
const hid_t mem_space_id = H5Screate_simple(static_cast<hsize_t>(dims.size()),
dims.data(), nullptr);
const hid_t xfer_plist_id = H5Pcreate(H5P_DATASET_XFER);
MArray<T> out(outrange);
const herr_t err = H5Dread(mId, mType, mem_space_id, mFilespace, xfer_plist_id, out.data());
CXZ_ASSERT(err >= 0, "error while reading dataset '" << mName
<< "', errorcode :" << err);
H5Pclose(xfer_plist_id);
H5Sclose(mem_space_id);
return out;
}
template <typename T>
template <class I, typename M>
Vector<hsize_t> SDataset<T>::mkFPos(const IndexInterface<I,M>& beg) const
{
Vector<hsize_t> fpos(beg.dim());
if constexpr(has_static_sub<I>::value){
iter<0,index_dim<I>::value> ( [&](auto i) { fpos[i] = beg.pack().get(i).lex(); }, NoF{} );
}
else if constexpr(has_sub<I>::value){
for(SizeT i = 0; i != beg.dim(); ++i){
fpos[i] = beg.pack().get(i).lex();
}
}
else {
fpos[0] = beg.lex();
}
return fpos;
}
}
}

8
src/opt/hdf5/include/h5_dataset.h
View file

@ -47,6 +47,14 @@ namespace CNORXZ
SDataset(const String& name, const ContentBase* _parent);
MArray<T> read() const;
template <class I, typename M>
MArray<T> read(const IndexInterface<I,M>& beg, const IndexInterface<I,M>& end) const;
private:
template <class I, typename M>
Vector<hsize_t> mkFPos(const IndexInterface<I,M>& beg) const;
};
}
}

42
src/opt/hdf5/include/h5_group.cc.h
View file

@ -13,9 +13,9 @@ namespace CNORXZ
Sptr<STable<Ts...>> Group::getTable(const String& name, Tuple<Ts...> proto)
{
auto i = this->getIndexTo(name);
auto tab = std::dynamic_pointer_cast<Table>( *i );
CXZ_ASSERT(tab->type() == ContentType::TABLE,
CXZ_ASSERT((*i)->type() == ContentType::TABLE,
"element '" << name << "' is not of type TABLE");
auto tab = std::dynamic_pointer_cast<Table>( *i );
if(tab == nullptr){
auto stab = std::dynamic_pointer_cast<STable<Ts...>>(*i);
CXZ_ASSERT(stab != nullptr, "wrong format for table '" << name << "'");
@ -28,12 +28,37 @@ namespace CNORXZ
return stab;
}
}
template <typename T>
Group& Group::addData(const String& name, const ArrayBase<T>& data)
Sptr<SDataset<T>> Group::getDataset(const String& name, T proto)
{
auto i = this->getIndexTo(name);
CXZ_ASSERT((*i)->type() == ContentType::DSET,
"element '" << name << "' is not of type DSET");
auto dset = std::dynamic_pointer_cast<Dataset>( *i );
if(dset == nullptr){
auto sdset = std::dynamic_pointer_cast<SDataset<T>>(*i);
CXZ_ASSERT(sdset != nullptr, "wrong format for dataset '" << name << "'");
return sdset;
}
else {
(*i)->close();
auto sdset = std::make_shared<SDataset<T>>(name, this);
*i = sdset;
return sdset;
}
}
template <typename T>
Group& Group::addDataset(const String& name, const ArrayBase<T>& data)
{
CXZ_ASSERT(this->isOpen(), "tried to extend closed group");
CXZ_ERROR("not implemented!!!");
Vector<String> nvec({name});
mCont.extend( URangeFactory<String>( nvec ).create() );
auto ii = getIndexTo(name);
auto dset = std::make_shared<SDataset<T>>(name, this);
dset->init(data);
*ii = dset;
return *this;
}
@ -43,11 +68,8 @@ namespace CNORXZ
{
CXZ_ASSERT(this->isOpen(), "tried to extend closed group");
Vector<String> nvec({name});
Vector<DType> dvec({DType(name)});
auto extr = URangeFactory<String>( nvec ).create();
mCont.extend(extr);
auto ii = mCont.begin();
ii.at(dvec); // 'at' returns YIndex&, so cannot use it inline...
mCont.extend( URangeFactory<String>( nvec ).create() );
auto ii = getIndexTo(name);
auto tab = std::make_shared<STable<Ts...>>(name, this, fnames);
for(auto& d: data){
tab->appendRecord(d);

12
src/opt/hdf5/include/h5_group.h
View file

@ -28,20 +28,24 @@ namespace CNORXZ
const ContentPtr& get(const String& name) const;
Sptr<Group> getGroup(const String& name) const;
Sptr<Table> getTable(const String& name) const;
Sptr<Dataset> getDataset(const String& name) const;
template <typename... Ts>
Sptr<STable<Ts...>> getTable(const String& name, Tuple<Ts...> proto);
const MArray<ContentPtr>& get() const;
Group& addGroup(const String& name);
template <typename T>
Group& addData(const String& name, const ArrayBase<T>& data);
Sptr<SDataset<T>> getDataset(const String& name, T proto);
const MArray<ContentPtr>& get() const;
Group& addGroup(const String& name);
template <typename... Ts>
Group& addTable(const String& name, const ArrayBase<Tuple<Ts...>>& data,
const Arr<String,sizeof...(Ts)>& fnames);
template <typename T>
Group& addDataset(const String& name, const ArrayBase<T>& data);
template <class F>
decltype(auto) iter(F&& f) const;

9
src/opt/hdf5/include/h5_types.h
View file

@ -21,7 +21,14 @@ namespace CNORXZ
// definition -> h5_table.h
template <typename... Ts>
class STable;
// definition -> h5_dataset.h
class Dataset;
// definition -> h5_dataset.h
template <typename T>
class SDataset;
}
}

7
src/opt/hdf5/lib/h5_dataset.cc
View file

@ -44,10 +44,6 @@ namespace CNORXZ
rs[i] = CRangeFactory(dims[i]).create();
}
mDataRange = yrange(rs);
// use when reading!!!:
//hid_t mem_space_id = H5Screate_simple(static_cast<hsize_t>(ndims), dims.data(), nullptr);
//hid_t xfer_plist_id = H5Pcreate(H5P_DATASET_XFER);
}
return *this;
}
@ -55,7 +51,6 @@ namespace CNORXZ
Dataset& Dataset::close()
{
if(mId != 0){
//!!!
H5Sclose(mFilespace);
H5Tclose(mType);
H5Dclose(mId);
@ -81,7 +76,7 @@ namespace CNORXZ
Dataset& Dataset::init(const RangePtr& dataRange, hid_t type)
{
CXZ_ASSERT(not open(), "tried to initialize dataset that is already extisting");
CXZ_ASSERT(not isOpen(), "tried to initialize dataset that is already extisting");
const H5T_class_t tc = H5Tget_class(type);
CXZ_ASSERT(tc != H5T_NO_CLASS, "id does not correspond to a data type"); // (did not found anythng better to check if type id is valid)...
const hid_t dcpl_id = H5Pcreate(H5P_DATASET_CREATE);

12
src/opt/hdf5/lib/h5_group.cc
View file

@ -1,6 +1,7 @@
#include "h5_group.h"
#include "h5_table.h"
#include "h5_dataset.h"
namespace CNORXZ
{
@ -104,6 +105,14 @@ namespace CNORXZ
return std::dynamic_pointer_cast<Table>( table );
}
Sptr<Dataset> Group::getDataset(const String& name) const
{
auto dset = this->get(name);
CXZ_ASSERT(dset->type() == ContentType::DSET,
"element '" << name << "' is not of type DSET");
return std::dynamic_pointer_cast<Dataset>( dset );
}
const MArray<ContentPtr>& Group::get() const
{
CXZ_ASSERT(this->isOpen(), "tried to get content of closed group");
@ -186,8 +195,7 @@ namespace CNORXZ
*index = std::make_shared<Table>(sname, icd->parent);
}
else {
CXZ_ERROR("IMPLEMENT!!!");
//*index = std::make_shared<DSet>(sname, icd->parent);
*index = std::make_shared<Dataset>(sname, icd->parent);
}
break;
}