chizeta/cnorxz
chizeta/cnorxz
0
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Compare commits

...

3 commits
8b66ad04ba ... f597254d5d

Author SHA1 Message Date
Christian Zimmermann
f597254d5d hdf5: dataset: some reorganization + group: add general add/get member functions 2024-10-30 18:05:10 -07:00
Christian Zimmermann
1e4daf8691 ranges: add stype() 2024-10-30 18:04:10 -07:00
Christian Zimmermann
b4ab639747 hdf5: some reorganization of the code + new member functions 2024-10-29 00:56:17 -07:00
22 changed files with 245 additions and 71 deletions

1
src/include/ranges/crange.h
View file

@ -178,6 +178,7 @@ namespace CNORXZ
virtual String stringMeta(SizeT pos) const override final;
virtual const TypeInfo& type() const override final;
virtual const TypeInfo& metaType() const override final;
virtual String stype() const override final;
virtual RangePtr extend(const RangePtr& r) const override final;
/** return meta data at given position

6
src/include/ranges/mrange.cc.h
View file

@ -772,6 +772,12 @@ namespace CNORXZ
return (this->begin()+pos).stringMeta();
}
template <class... Ranges>
String MRange<Ranges...>::stype() const
{
return "M";
}
template <class... Ranges>
const TypeInfo& MRange<Ranges...>::type() const
{

1
src/include/ranges/mrange.h
View file

@ -373,6 +373,7 @@ namespace CNORXZ
virtual SizeT size() const override final;
virtual SizeT dim() const override final;
virtual String stringMeta(SizeT pos) const override final;
virtual String stype() const override final;
virtual const TypeInfo& type() const override final;
virtual const TypeInfo& metaType() const override final;
virtual RangePtr extend(const RangePtr& r) const override final;

6
src/include/ranges/prange.cc.h
View file

@ -320,6 +320,12 @@ namespace CNORXZ
return mRange->stringMeta( mParts[pos] );
}
template <class RangeT>
String PRange<RangeT>::stype() const
{
return "P";
}
template <class RangeT>
const TypeInfo& PRange<RangeT>::type() const
{

1
src/include/ranges/prange.h
View file

@ -193,6 +193,7 @@ namespace CNORXZ
virtual SizeT size() const override final;
virtual SizeT dim() const override final;
virtual String stringMeta(SizeT pos) const override final;
virtual String stype() const override final;
virtual const TypeInfo& type() const override final;
virtual const TypeInfo& metaType() const override final;
virtual RangePtr extend(const RangePtr& r) const override final;

27
src/include/ranges/range_base.h
View file

@ -118,7 +118,7 @@ namespace CNORXZ
*/
virtual MArray<RangePtr> sub() const;
/** Range size */
/** Range size. */
virtual SizeT size() const = 0;
/** Range dimension.
@ -128,42 +128,45 @@ namespace CNORXZ
*/
virtual SizeT dim() const = 0;
/** Range type */
/** Range type. */
virtual const TypeInfo& type() const = 0;
/** The type of the range's meta data */
/** The type of the range's meta data. */
virtual const TypeInfo& metaType() const = 0;
/** String identifier indicating the range's type. */
virtual String stype() const = 0;
/** Meta data in string format at lexicographic position pos.
@param pos size type argument
@param pos size type argument.
*/
virtual String stringMeta(SizeT pos) const = 0;
/** Return DIndex iterating over the range's parameter space.
@param pos index position, size type argument
@param pos index position, size type argument.
*/
virtual DIndex index(SizeT pos = 0) const = 0;
/** Create extended range.
Creates a new range consisting of the meta data spaces
of this range and the range r
@param r shared pointer to range used for extension
of this range and the range r.
@param r shared pointer to range used for extension.
*/
virtual RangePtr extend(const RangePtr& r) const = 0;
/** check if ranges are the same */
/** check if ranges are the same. */
bool operator==(const RangeBase& in) const;
/** check if ranges are not the same */
/** check if ranges are not the same. */
bool operator!=(const RangeBase& in) const;
/** return the range's uuid */
/** return the range's uuid. */
Uuid id() const;
/** return a DIndex pointing to the range's first element */
/** return a DIndex pointing to the range's first element. */
DIndex begin() const;
/** return a DIndex pointing to the element after the range's last element */
/** return a DIndex pointing to the element after the range's last element. */
DIndex end() const;
friend RangeFactoryBase;

6
src/include/ranges/srange.cc.h
View file

@ -294,6 +294,12 @@ namespace CNORXZ
return toString(mSpace[pos]);
}
template <typename MetaType, SizeT S>
String SRange<MetaType,S>::stype() const
{
return "S";
}
template <typename MetaType, SizeT S>
const TypeInfo& SRange<MetaType,S>::type() const
{

1
src/include/ranges/srange.h
View file

@ -192,6 +192,7 @@ namespace CNORXZ
virtual SizeT size() const override final;
virtual SizeT dim() const override final;
virtual String stringMeta(SizeT pos) const override final;
virtual String stype() const override final;
virtual const TypeInfo& type() const override final;
virtual const TypeInfo& metaType() const override final;
virtual RangePtr extend(const RangePtr& r) const override final;

6
src/include/ranges/urange.cc.h
View file

@ -326,6 +326,12 @@ namespace CNORXZ
return toString(this->get(pos));
}
template <typename MetaT>
String URange<MetaT>::stype() const
{
return "U";
}
template <typename MetaT>
const TypeInfo& URange<MetaT>::type() const
{

1
src/include/ranges/urange.h
View file

@ -181,6 +181,7 @@ namespace CNORXZ
virtual SizeT size() const override final;
virtual SizeT dim() const override final;
virtual String stringMeta(SizeT pos) const override final;
virtual String stype() const override final;
virtual const TypeInfo& type() const override final;
virtual const TypeInfo& metaType() const override final;
virtual RangePtr extend(const RangePtr& r) const override final;

6
src/include/ranges/wrange.cc.h
View file

@ -356,6 +356,12 @@ namespace CNORXZ
return toString(this->get(pos));
}
template <typename MetaT>
String WRange<MetaT>::stype() const
{
return "W";
}
template <typename MetaT>
const TypeInfo& WRange<MetaT>::type() const
{

1
src/include/ranges/wrange.h
View file

@ -184,6 +184,7 @@ namespace CNORXZ
virtual SizeT size() const override final;
virtual SizeT dim() const override final;
virtual String stringMeta(SizeT pos) const override final;
virtual String stype() const override final;
virtual const TypeInfo& type() const override final;
virtual const TypeInfo& metaType() const override final;
virtual RangePtr extend(const RangePtr& r) const override final;

1
src/include/ranges/yrange.h
View file

@ -320,6 +320,7 @@ namespace CNORXZ
virtual SizeT size() const override final;
virtual SizeT dim() const override final;
virtual String stringMeta(SizeT pos) const override final;
virtual String stype() const override final;
virtual const TypeInfo& type() const override final;
virtual const TypeInfo& metaType() const override final;
virtual RangePtr extend(const RangePtr& r) const override final;

5
src/lib/ranges/crange.cc
View file

@ -225,6 +225,11 @@ namespace CNORXZ
return toString(pos);
}
String CRange::stype() const
{
return "C";
}
const TypeInfo& CRange::type() const
{
return typeid(CRange);

5
src/lib/ranges/yrange.cc
View file

@ -628,6 +628,11 @@ namespace CNORXZ
}
return out;
}
String YRange::stype() const
{
return "Y";
}
const TypeInfo& YRange::type() const
{

76
src/opt/hdf5/include/h5_dataset.cc.h
View file

@ -22,16 +22,8 @@ namespace CNORXZ
Dataset& Dataset::init(const ArrayBase<T>& data)
{
const hid_t tid = getTypeId(*data.data());
//VCHECK(tid);
init(data.range(), tid);
if(data.begin().formatIsTrivial()){
Vector<hsize_t> dims(mDataRange->dim());
for(SizeT i = 0; i != dims.size(); ++i){
dims[i] = mDataRange->sub(i)->size();
}
const hid_t memspace = H5Screate_simple(dims.size(), dims.data(), NULL);
H5Dwrite(mId, mType, memspace, mFilespace, H5P_DEFAULT, data.data());
H5Sclose(memspace);
init(data.range(), tid, data.data());
}
else {
CXZ_ERROR("Got array type with non-trivial format; non-contiguous data formats are not supported yet!");
@ -47,19 +39,23 @@ namespace CNORXZ
template <typename T>
MArray<T> SDataset<T>::read() const
{
Vector<hsize_t> dims(mDataRange->dim());
for(SizeT i = 0; i != dims.size(); ++i){
dims[i] = mDataRange->sub(i)->size();
}
const hid_t mem_space_id = H5Screate_simple(static_cast<hsize_t>(dims.size()),
dims.data(), nullptr);
const hid_t xfer_plist_id = H5Pcreate(H5P_DATASET_XFER);
MArray<T> out(mDataRange);
const herr_t err = H5Dread(mId, mType, mem_space_id, mFilespace, xfer_plist_id, out.data());
CXZ_ASSERT(err >= 0, "error while reading dataset '" << mName
<< "', errorcode :" << err);
H5Pclose(xfer_plist_id);
H5Sclose(mem_space_id);
MArray<T> out(mFileRange);
readbase(out.data(), nullptr, nullptr);
return out;
}
template <typename T>
template <class I, typename M>
MArray<T> SDataset<T>::read(const IndexInterface<I,M>& idx) const
{
CXZ_ASSERT(idx.dim() == mFileRange->dim(), "got index of inconsistent dimension, got"
<< idx.dim() << ", expected " << mFileRange->dim());
const RangePtr outrange = idx.range();
CXZ_ASSERT(outrange->size() == mFileRange->size(),
"got index of range of inconsistent size, expected "
<< mFileRange->size() << ", got " << outrange->size());
MArray<T> out(outrange);
readbase(out.data(), outrange, nullptr);
return out;
}
@ -67,46 +63,28 @@ namespace CNORXZ
template <class I, typename M>
MArray<T> SDataset<T>::read(const IndexInterface<I,M>& beg, const IndexInterface<I,M>& end) const
{
CXZ_ASSERT(beg.dim() == mDataRange->dim(), "got index of inconsistent dimension, got"
<< beg.dim() << ", expected " << mDataRange->dim());
CXZ_ASSERT(beg.dim() == mFileRange->dim(), "got index of inconsistent dimension, got"
<< beg.dim() << ", expected " << mFileRange->dim());
const RangePtr outrange = beg.prange(end);
Vector<hsize_t> dims(outrange->dim());
for(SizeT i = 0; i != dims.size(); ++i){
dims[i] = outrange->sub(i)->size();
}
const Vector<hsize_t> fpos = mkFPos(beg);
H5Sselect_hyperslab(mFilespace, H5S_SELECT_SET, fpos.data(), NULL, dims.data(), NULL);
const hid_t mem_space_id = H5Screate_simple(static_cast<hsize_t>(dims.size()),
dims.data(), nullptr);
const hid_t xfer_plist_id = H5Pcreate(H5P_DATASET_XFER);
MArray<T> out(outrange);
const herr_t err = H5Dread(mId, mType, mem_space_id, mFilespace, xfer_plist_id, out.data());
CXZ_ASSERT(err >= 0, "error while reading dataset '" << mName
<< "', errorcode :" << err);
H5Pclose(xfer_plist_id);
H5Sclose(mem_space_id);
readbase(out.data(), outrange, toYptr(beg));
return out;
}
template <typename T>
template <class I, typename M>
Vector<hsize_t> SDataset<T>::mkFPos(const IndexInterface<I,M>& beg) const
Sptr<YIndex> SDataset<T>::toYptr(const IndexInterface<I,M>& beg) const
{
Vector<hsize_t> fpos(beg.dim());
if constexpr(has_static_sub<I>::value){
iter<0,index_dim<I>::value> ( [&](auto i) { fpos[i] = beg.THIS().pack().get(i)->lex(); }, NoF{} );
if constexpr(std::is_same<YIndex,I>::value){
return std::make_shared<YIndex>(beg.THIS());
}
else if constexpr(has_sub<I>::value){
for(SizeT i = 0; i != beg.dim(); ++i){
fpos[i] = beg.THIS().pack().get(i)->lex();
}
else if constexpr(has_sub<I>::value) {
return yindexPtr(beg.pack());
}
else {
fpos[0] = beg.lex();
return yindexPtr(DPack(std::make_shared<I>(beg)));
}
return fpos;
}
}
}

41
src/opt/hdf5/include/h5_dataset.h
View file

@ -48,8 +48,29 @@ namespace CNORXZ
@param dataRange A potentially multi-dimensional range characterizing the dataset.
@param type Data type id.
*/
Dataset& init(const RangePtr& dataRange, hid_t type);
virtual Dataset& init(const RangePtr& dataRange, hid_t type);
/** Initalize the dataset.
@param dataRange A potentially multi-dimensional range characterizing the dataset.
@param type Data type id.
@param data Pointer to raw data.
*/
virtual Dataset& init(const RangePtr& dataRange, hid_t type, const void* data);
/** Write data into dataset.
@param dataRange A potentially multi-dimensional range characterizing the format of the data to be written.
@param pos Position on target.
@param data Pointer to raw data to be writte.
*/
virtual Dataset& writebase(const RangePtr& dataRange, Sptr<YIndex> pos, const void* data);
/** Read the dataset.
@param dest Pointer to destination.
@param readrange Range of the destination data.
@param beg Position within the file space.
*/
virtual void readbase(void* dest, RangePtr readrange, Sptr<YIndex> beg) const;
/** Initalize the dataset.
@param data Array containing the dataset.
*/
@ -62,14 +83,16 @@ namespace CNORXZ
const RangePtr& dataRange() const;
protected:
RangePtr mDataRange; /**< The data range. */
Vector<hsize_t> mkOff(const Sptr<YIndex>& beg) const;
RangePtr mFileRange; /**< The range of the dataset. */
hid_t mType; /**< The data type identifier. */
hid_t mFilespace; /**< The hdf5 file space identifier. */
};
/** ****
Class to handle hdf5 datasets, the value type is known at compile time.
Class to handle hdf5 datasets, the value type is assumed to be known at compile time.
@tparam T Dataset value type.
*/
template <typename T>
@ -89,6 +112,14 @@ namespace CNORXZ
*/
MArray<T> read() const;
/** Read the dataset using range of given index.
The index position is ignored.
@param idx Index specifying the range type.
@return Array containing the dataset values.
*/
template <class I, typename M>
MArray<T> read(const IndexInterface<I,M>& idx) const;
/** Read a given subset of the dataset.
The subset needs to be hypercubic.
@param beg Index indicating the begin edge of the hypercube.
@ -100,8 +131,10 @@ namespace CNORXZ
private:
//template <class I, typename M>
//Vector<hsize_t> mkFPos(const IndexInterface<I,M>& beg) const;
template <class I, typename M>
Vector<hsize_t> mkFPos(const IndexInterface<I,M>& beg) const;
Sptr<YIndex> toYptr(const IndexInterface<I,M>& beg) const;
};
}
}

18
src/opt/hdf5/include/h5_group.cc.h
View file

@ -39,6 +39,13 @@ namespace CNORXZ
return pos - beg;
}
template <class F>
decltype(auto) Group::get(const String& name, F&& f)
{
auto i = this->getIndexTo(name);
return f(*i);
}
template <typename... Ts>
Sptr<STable<Ts...>> Group::getTable(const String& name, Tuple<Ts...> proto)
{
@ -108,6 +115,17 @@ namespace CNORXZ
return *this;
}
template <class F, typename... Args>
Group& Group::add(const String& name, F&& f, const Args&... args)
{
CXZ_ASSERT(this->isOpen(), "tried to extend closed group");
Vector<String> nvec({name});
mCont.extend( URangeFactory<String>( nvec ).create() );
auto ii = getIndexTo(name);
*ii = f(args...);
return *this;
}
template <class F>
decltype(auto) Group::iter(F&& f) const
{

14
src/opt/hdf5/include/h5_group.h
View file

@ -96,6 +96,13 @@ namespace CNORXZ
*/
const MArray<ContentPtr>& get() const;
/** Get object contained by this group.
@param name Object name.
@param f Function to interpret the object.
*/
template <class F>
decltype(auto) get(const String& name, F&& f);
/** Add a new group to this group.
@param name Name of the created group.
*/
@ -119,6 +126,13 @@ namespace CNORXZ
template <typename T>
Group& addDataset(const String& name, const ArrayBase<T>& data);
/** Add new object to this group.
@param name Object name.
@param f Function used to create the object.
*/
template <class F, typename... Args>
Group& add(const String& name, F&& f, const Args&... args);
/** Iterate over all group elements (const).
@param f function object to be executed on each group element.
*/

86
src/opt/hdf5/lib/h5_dataset.cc
View file

@ -53,7 +53,7 @@ namespace CNORXZ
for(SizeT i = 0; i != ndims; ++i){
rs[i] = CRangeFactory(dims[i]).create();
}
mDataRange = yrange(rs);
mFileRange = yrange(rs);
}
return *this;
}
@ -84,18 +84,18 @@ namespace CNORXZ
return H5Lexists(mParent->id(), mName.c_str(), H5P_DEFAULT) > 0;
}
Dataset& Dataset::init(const RangePtr& dataRange, hid_t type)
Dataset& Dataset::init(const RangePtr& fileRange, hid_t type)
{
CXZ_ASSERT(not isOpen(), "tried to initialize dataset that is already extisting");
mDataRange = dataRange;
mFileRange = fileRange;
const H5T_class_t tc = H5Tget_class(type);
CXZ_ASSERT(tc != H5T_NO_CLASS, "id does not correspond to a data type"); // (did not found anythng better to check if type id is valid)...
const hid_t dcpl_id = H5Pcreate(H5P_DATASET_CREATE);
// TODO: all sub-ranges explicity!!!:
const SizeT ndim = dataRange->dim();
const SizeT ndim = mFileRange->dim();
Vector<hsize_t> exts(ndim);
for(SizeT i = 0; i != ndim; ++i){
exts[i] = static_cast<hsize_t>( dataRange->sub(i)->size() );
exts[i] = static_cast<hsize_t>( mFileRange->sub(i)->size() );
}
mFilespace = H5Screate_simple(ndim, exts.data(), NULL);
mType = type;
@ -105,9 +105,83 @@ namespace CNORXZ
return *this;
}
Dataset& Dataset::init(const RangePtr& writeRange, hid_t type, const void* data)
{
init(writeRange, type);
writebase(writeRange, std::make_shared<YIndex>(mFileRange), data);
return *this;
}
Dataset& Dataset::writebase(const RangePtr& writeRange, Sptr<YIndex> pos, const void* data)
{
//CXZ_ERROR("TODO!!!");
CXZ_ASSERT(writeRange->dim() == mFileRange->dim(), "dimension of data range ("
<< writeRange->dim() << ") different from dimension of file range ("
<< mFileRange->dim() << ")");
Vector<hsize_t> dims(writeRange->dim());
for(SizeT i = 0; i != dims.size(); ++i){
dims[i] = writeRange->sub(i)->size();
}
if(pos){
CXZ_ASSERT(pos->range()->dim() == mFileRange->dim(), "dimension of position index ("
<< pos->range()->dim() << ") different from dimension of file range ("
<< mFileRange->dim() << ")");
const Vector<hsize_t> fpos = mkOff(pos);
H5Sselect_hyperslab(mFilespace, H5S_SELECT_SET, fpos.data(), NULL, dims.data(), NULL);
}
const hid_t memspace = H5Screate_simple(dims.size(), dims.data(), NULL);
const hid_t xfer_plist_id = H5Pcreate(H5P_DATASET_XFER);
H5Dwrite(mId, mType, memspace, mFilespace, xfer_plist_id, data);
H5Pclose(xfer_plist_id);
H5Sclose(memspace);
return *this;
}
void Dataset::readbase(void* dest, RangePtr readRange, Sptr<YIndex> beg) const
{
// TODO: Check if readRange is compatible with mFileRange!!!
if(not readRange){
readRange = mFileRange;
}
else {
CXZ_ASSERT(readRange->dim() == mFileRange->dim(), "dimension of data range ("
<< readRange->dim() << ") different from dimension of file range ("
<< mFileRange->dim() << ")");
}
Vector<hsize_t> dims(readRange->dim());
for(SizeT i = 0; i != dims.size(); ++i){
dims[i] = readRange->sub(i)->size();
}
if(beg){
CXZ_ASSERT(beg->range()->dim() == mFileRange->dim(), "dimension of position index ("
<< beg->range()->dim() << ") different from dimension of file range ("
<< mFileRange->dim() << ")");
const Vector<hsize_t> fpos = mkOff(beg);
H5Sselect_hyperslab(mFilespace, H5S_SELECT_SET, fpos.data(), NULL, dims.data(), NULL);
}
const hid_t mem_space_id = H5Screate_simple(static_cast<hsize_t>(dims.size()),
dims.data(), nullptr);
const hid_t xfer_plist_id = H5Pcreate(H5P_DATASET_XFER);
//MArray<T> out(readRange);
const herr_t err = H5Dread(mId, mType, mem_space_id, mFilespace, xfer_plist_id, dest);
CXZ_ASSERT(err >= 0, "error while reading dataset '" << mName
<< "', errorcode :" << err);
H5Pclose(xfer_plist_id);
H5Sclose(mem_space_id);
}
const RangePtr& Dataset::dataRange() const
{
return mDataRange;
return mFileRange;
}
Vector<hsize_t> Dataset::mkOff(const Sptr<YIndex>& beg) const
{
Vector<hsize_t> off(beg->dim());
for(SizeT i = 0; i != beg->dim(); ++i){
off[i] = beg->pack().get(i)->lex();
}
return off;
}
}

6
src/opt/mpi/include/rrange.cc.h
View file

@ -605,6 +605,12 @@ namespace CNORXZ
{
return (this->begin()+pos).stringMeta();
}
template <class RangeI, class RangeK>
String RRange<RangeI,RangeK>::stype() const
{
return "R";
}
template <class RangeI, class RangeK>
const TypeInfo& RRange<RangeI,RangeK>::type() const

1
src/opt/mpi/include/rrange.h
View file

@ -270,6 +270,7 @@ namespace CNORXZ
virtual SizeT size() const override final;
virtual SizeT dim() const override final;
virtual String stringMeta(SizeT pos) const override final;
virtual String stype() const override final;
virtual const TypeInfo& type() const override final;
virtual const TypeInfo& metaType() const override final;
virtual RangePtr extend(const RangePtr& r) const override final;