From e3f8fa17bac86f0f8c54fc039ed9347482a1a32d Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Sat, 30 Dec 2023 18:10:07 +0100 Subject: [PATCH 01/32] yrange/yindex: comments --- src/include/ranges/crange.h | 15 ++- src/include/ranges/yrange.h | 204 +++++++++++++++++++++++++++++++++--- 2 files changed, 194 insertions(+), 25 deletions(-) diff --git a/src/include/ranges/crange.h b/src/include/ranges/crange.h index 8e96d1a..cfef60b 100644 --- a/src/include/ranges/crange.h +++ b/src/include/ranges/crange.h @@ -2,8 +2,7 @@ /** @file include/ranges/crange.h - @brief ... - + @brief CRange and CIndex declaration. Copyright (c) 2022 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de @@ -80,7 +79,7 @@ namespace CNORXZ SizeT operator*() const; /** @copydoc IndexInterface::dim() */ - SizeT dim() const; // = 1 + SizeT dim() const; /** @copydoc IndexInterface::range() */ Sptr range() const; @@ -140,14 +139,14 @@ namespace CNORXZ public: typedef CRange oType; - /** construct and setup factory - @param size size of the range to be constructed + /** Construct and setup factory + @param size Size of the range to be constructed */ CRangeFactory(SizeT size); - /** construct and setup factory - @param size size of the range to be constructed - @param ref range the range to be constructed is related to + /** Construct and setup factory + @param size Size of the range to be constructed + @param ref Range the range to be constructed is related to */ CRangeFactory(SizeT size, RangePtr ref); diff --git a/src/include/ranges/yrange.h b/src/include/ranges/yrange.h index 48febe8..a5468b9 100644 --- a/src/include/ranges/yrange.h +++ b/src/include/ranges/yrange.h @@ -2,8 +2,7 @@ /** @file include/ranges/yrange.h - @brief ... - + @brief YRange and YIndex declaration Copyright (c) 2022 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de @@ -24,6 +23,13 @@ namespace CNORXZ { + /** **** + Specific index for YRanges. + + A YIndex is a multi-index which consists of a set of sub-indices + and a format. In the case the index is used to access data, this format + determines the linearized memory position for a given sub-index combination. + */ class YIndex : public IndexInterface> { public: @@ -32,59 +38,154 @@ namespace CNORXZ typedef Vector MetaType; INDEX_RANDOM_ACCESS_ITERATOR_DEFS(MetaType); + + /** Default constructor. */ YIndex() = default; + + /** Move constructor. */ YIndex(YIndex&& i) = default; + + /** Move assignment. */ YIndex& operator=(YIndex&& i) = default; - // no defaults: + + /** Copy constructor. + No default copy: Have to copy sub-index instances + */ YIndex(const YIndex& i); + + /** Copy assigment. + No default copy: Have to copy sub-index instances + */ YIndex& operator=(const YIndex& i); + /** Construct from sub-index pointers. + @param is Vector of XIndex pointers. + */ YIndex(const Vector& is); + + /** Construct from sub-index pointers, specify index format. + @param bs Index format (YFormat). + @param is Vector of XIndex pointers. + */ YIndex(const YFormat& bs, const Vector& is); + + /** Construct from a range and an initial lexicographic position + @param range Range to iterate over. + @param lexpos Initial lexicographic position. + */ YIndex(const RangePtr& range, SizeT lexpos = 0); + + /** Construct from a range and an initial lexicographic position, specify format. + @param range Range to iterate over. + @param bs Index format. + @param lexpos Initial lexicographic position. + */ YIndex(const RangePtr& range, const YFormat& bs, SizeT lexpos = 0); + /** @copydoc IndexInterface::operator=(SizeT) */ YIndex& operator=(SizeT lexpos); + + /** @copydoc IndexInterface::operator++() */ YIndex& operator++(); + + /** @copydoc IndexInterface::operator--() */ YIndex& operator--(); - YIndex operator+(Int n) const; // equivalent to applying n times ++ + + /** @copydoc IndexInterface::operator+() */ + YIndex operator+(Int n) const; + + /** @copydoc IndexInterface::operator-() */ YIndex operator-(Int n) const; + + /** @copydoc IndexInterface::operator-(CIndex) */ SizeT operator-(const YIndex& i) const; + + /** @copydoc IndexInterface::operator+=() */ YIndex& operator+=(Int n); + + /** @copydoc IndexInterface::operator-=() */ YIndex& operator-=(Int n); + /** @copydoc IndexInterface::lex() */ SizeT lex() const; + + /** @copydoc IndexInterface::pmax() */ UPos pmax() const; + + /** @copydoc IndexInterface::lmax() */ UPos lmax() const; + + /** @copydoc IndexInterface::id() */ IndexId<0> id() const; + /** @copydoc IndexInterface::operator*() */ Vector operator*() const; + /** @copydoc IndexInterface::dim() */ SizeT dim() const; + + /** @copydoc IndexInterface::range() */ Sptr range() const; + + /** @copydoc IndexInterface::stepSize() */ UPos stepSize(const IndexId<0> id) const; + /** @copydoc IndexInterface::stringMeta() */ String stringMeta() const; + + /** @copydoc IndexInterface::meta() */ Vector meta() const; + + /** @copydoc IndexInterface::at() */ YIndex& at(const Vector& meta); + /** @copydoc IndexInterface::prange() */ + RangePtr prange(const YIndex& last) const; + + /** @copydoc IndexInterface::deepFormat() */ + Vector deepFormat() const; + + /** @copydoc IndexInterface::deepMax() */ + Vector deepMax() const; + + /** @copydoc IndexInterface::reformat() */ + YIndex& reformat(const Vector& f, const Vector& s); + + /** @copydoc IndexInterface::ifor() */ DXpr ifor(const DXpr& xpr, NoF&& f) const; - YIndex& operator()(const Sptr& i); - YIndex& operator()(); - - const DPack& pack() const; - RangePtr prange(const YIndex& last) const; - Vector deepFormat() const; - Vector deepMax() const; - const YFormat& format() const; - const YFormat& lexFormat() const; - YIndex& setFormat(const YFormat& bs); - YIndex& reformat(const Vector& f, const Vector& s); - /** @copydoc IndexInterface::formatIsTrivial() */ bool formatIsTrivial() const; + /** Replace sub-index instances. + All linearized positions are updated accordingly. + @param i Pointer to YIndex which provides the new sub-index instance + */ + YIndex& operator()(const Sptr& i); + + /** Update all linearized positions. */ + YIndex& operator()(); + + /** Get all sub-indices + @return Pack of sub-indices + */ + const DPack& pack() const; + + /** Get index format. + @return The format. + */ + const YFormat& format() const; + + /** Get lexicographic (trivial) index format. + @return The lexicographic format. + */ + const YFormat& lexFormat() const; + + /** Set the index format. + @param bs The new format. + */ + YIndex& setFormat(const YFormat& bs); + private: inline Vector mkFormat() const; inline Vector mkLexFormat() const; @@ -107,26 +208,80 @@ namespace CNORXZ UPos mLMax = 0; }; + /** **** + Specialization: YIndex is a multi-index. + @see index_is_multi + */ template <> struct index_is_multi { static constexpr bool value = true; }; + /** **** + Specialization: YIndex has sub-indices. + @see has_sub + */ template <> struct has_sub { static constexpr bool value = true; }; + /** Create YIndex from an index pack assuming a trivial index format. + @param pack Dynamic index pack. + @return The created YIndex. + */ YIndex yindex(const DPack& pack); + + /** Create YIndex from sub-indices assuming a trivial index format. + @param is Vector of pointers to the sub-indices used in the YIndex. + @return The created YIndex. + */ YIndex yindex(const Vector& is); + + /** Create YIndex from an index pack assuming a trivial index format. + @param pack Dynamic index pack. + @return A shared pointer to the created YIndex. + */ Sptr yindexPtr(const DPack& is); + + /** Create YIndex from sub-indices assuming a trivial index format. + @param is Vector of pointers to the sub-indices used in the YIndex. + @return A shared pointer to the created YIndex. + */ Sptr yindexPtr(const Vector& is); + + /** Create YIndex from sub-indices. + @param is Vector of pointers to the sub-indices used in the YIndex. + @param bs Index format. + @return A shared pointer to the created YIndex. + */ Sptr yindexPtr(const Vector& bs, const Vector& is); + /** **** + Specific factory for YRange. + */ class YRangeFactory : public RangeFactoryBase { public: + + /** Construct and setup factory. + @param rvec Vector of ranges i.e. the sub-ranges the YRange consists of + */ YRangeFactory(const Vector& rvec); + + /** Construct and setup factory. + @param rvec Vector of ranges i.e. the sub-ranges the YRange consists of (move) + */ YRangeFactory(Vector&& rvec); + + /** Construct and setup factory. + @param rvec Vector of ranges i.e. the sub-ranges the YRange consists of + @param ref Range the range to be constructed is related to + */ YRangeFactory(const Vector& rvec, const RangePtr& ref); + + /** Construct and setup factory. + @param rvec Vector of ranges i.e. the sub-ranges the YRange consists of (move) + @param ref Range the range to be constructed is related to + */ YRangeFactory(Vector&& rvec, const RangePtr& ref); private: @@ -138,6 +293,13 @@ namespace CNORXZ }; + /** **** + Dynamic multi-dimensional range + + Dimension and sub-range types are determined at runtime + The size of the range is given by the product of the + sizes of all sub-ranges. + */ class YRange : public RangeInterface { public: @@ -169,8 +331,16 @@ namespace CNORXZ SERIALIZATION_FUNCTIONS_NOPUB; }; + /** Create YRange from sub-ranges. + @param rs Vector of pointers to the sub-ranges used by the YRange. + @return A shared pointer to the created YRange. + */ RangePtr yrange(const Vector& rs); - + + /** **** + Specialize RangeCast for casts to YRange. + @see RangeCast + */ template <> struct RangeCast { -- 2.45.2 From ab13d97b0381b6fa5ce99790c04ca39b92a7496f Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Tue, 2 Jan 2024 20:43:05 +0100 Subject: [PATCH 02/32] cmake: new version string --- CMakeLists.txt | 61 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d56df34..fb5a89f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,27 +2,44 @@ cmake_minimum_required(VERSION 3.0) project(cnorxz) +# LIB VERSION + +set(VERSION_PART "pre") +set(VERSION_TAG_HASH "6857e3fc7d0af25db3a925791d1cabc6342b930a") + +# OPTIONS + +option(SCALAR_BUILD "" OFF) + +# INCLUDES + include(cmake/check_avx.cmake) +# GIT VARIABLES + execute_process(COMMAND bash "-c" "git rev-parse HEAD" OUTPUT_VARIABLE GIT_HASH OUTPUT_STRIP_TRAILING_WHITESPACE) -execute_process(COMMAND bash "-c" "git tag -l --sort=refname 'v*' | tail -n1" OUTPUT_VARIABLE GIT_TAG OUTPUT_STRIP_TRAILING_WHITESPACE) -execute_process(COMMAND bash "-c" "git rev-list -n 1 ${GIT_TAG}" OUTPUT_VARIABLE GIT_TAG_HASH OUTPUT_STRIP_TRAILING_WHITESPACE) +execute_process(COMMAND bash "-c" "git rev-parse --abbrev-ref HEAD" OUTPUT_VARIABLE GIT_BRANCH OUTPUT_STRIP_TRAILING_WHITESPACE) + +# BUILD / CHECK VERSION STRING + message(STATUS "${GIT_HASH}") -option(RUN_PIPELINE "" OFF) -option(SCALAR_BUILD "" OFF) -if(RUN_PIPELINE) - set(VERSION "v0.0.0-test") -else() - message(STATUS "${GIT_TAG}") - message(STATUS "${GIT_TAG_HASH}") - string(SUBSTRING "${GIT_TAG}" 1 -1 VERSION) - if(NOT ("${GIT_HASH}" EQUAL "${GIT_TAG_HASH}")) +message(STATUS "${VERSION_PART}") +message(STATUS "${VERSION_TAG_HASH}") +set(VERSION "${VERSION_PART}") +if(NOT ("${GIT_BRANCH}" EQUAL "release")) + if(NOT ("${GIT_HASH}" EQUAL "${VERSION_TAG_HASH}")) string(SUBSTRING ${GIT_HASH} 0 7 GIT_HASH_SHORT) - set(VERSION "${VERSION}-${GIT_HASH_SHORT}") + set(VERSION "${VERSION}-${GIT_BRANCH}-${GIT_HASH_SHORT}") + endif() +else() + if(NOT ("${GIT_HASH}" EQUAL "${VERSION_TAG_HASH}")) + message(FATAL_ERROR "version mash-up, do not use, contact maintainer") endif() endif() message(STATUS "version = ${VERSION}") +# CHECK COMPILER + if(CMAKE_COMPILER_IS_GNUCXX) if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 7.0) message(FATAL_ERROR "require gcc version >= 7.0") @@ -31,10 +48,16 @@ else() message(WARNING "compiler ${CMAKE_CXX_COMPILER_ID} officially not supported") endif() +# FLAGS + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror -std=c++17 -Wpedantic -O2 -march=native -faligned-new -funroll-loops -fopenmp") +# TESTING + enable_testing() +# INSTALL PATH + if(IS_ABSOLUTE "${CMAKE_INSTALL_PREFIX}") set(INSTALL_PATH ${CMAKE_INSTALL_PREFIX}) else() @@ -43,6 +66,8 @@ else() endif() message(STATUS "found absolute install path '${INSTALL_PATH}'") +# SCALAR / INTRINSICS + if(NOT ${SCALAR_BUILD}) message(STATUS "check for intrinsics") check_avx() @@ -55,6 +80,8 @@ if(NOT ${SCALAR_BUILD}) endif() endif() +# CHECK LIBRARIES : GTest + message(STATUS "check for libraries") find_package( GTest REQUIRED ) if(GTest_FOUND) @@ -63,6 +90,8 @@ else() message(FATAL_ERROR "GTest not found") endif() +# CHECK LIBRARIES : Threads + find_package(Threads REQUIRED) if(Threads_FOUND) #include_directories(${Threads_INCLUDE_DIRS}) @@ -70,12 +99,16 @@ else() message(FATAL_ERROR "Threads not found") endif() +# CHECK LIBRARIES : hdf5 + if(DEFINED ENABLE_hdf5) set(ENABLE_hdf5 ${ENABLE_hdf5} CACHE BOOL "enable hdf5") else() set(ENABLE_hdf5 TRUE CACHE BOOL "enable hdf5") endif() +# CHECK LIBRARIES : cereal + if(DEFINED ENABLE_cereal) set(ENABLE_cereal ${ENABLE_cereal} CACHE BOOL "enable hdf5") else() @@ -96,8 +129,12 @@ else() endif() endif() +# DEFINES + add_definitions(-DVERSION="${VERSION}") add_definitions(-DGIT_COMMIT="${GIT_HASH}") add_definitions(-DCXX_FLAGS="${CMAKE_CXX_FLAGS}") +# CONTINUE WITH SUB-DIRECTORIES + add_subdirectory(src) -- 2.45.2 From d66164e87481f5a71d308ebbc41bb2da9aae5d44 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Wed, 3 Jan 2024 11:54:06 +0100 Subject: [PATCH 03/32] mrange: solve lower-dim-reformat issue --- src/include/ranges/mrange.cc.h | 10 +++++++--- src/tests/index_format_test.cc | 14 +++++++++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/include/ranges/mrange.cc.h b/src/include/ranges/mrange.cc.h index b352830..8891653 100644 --- a/src/include/ranges/mrange.cc.h +++ b/src/include/ranges/mrange.cc.h @@ -551,6 +551,12 @@ namespace CNORXZ GMIndex& GMIndex::reformat(const Vector& f, const Vector& s) { + if(f.size() == 1){ + CXZ_ASSERT(s.size() == 1, "f.size() != s.size()"); + CXZ_ASSERT(s[0] == lmax().val(), "got inconsistent size; expeected " + << lmax().val() << ", got " << s[0]); + return *this; + } if constexpr(std::is_same::value){ CXZ_ASSERT(CNORXZ::formatIsTrivial(f,s), "cannot reformat MIndex with format type = None"); @@ -590,9 +596,7 @@ namespace CNORXZ mIPack[i]->reformat(nf,ns); } else { - // TODO: IMPLEMENT!!! - // check trivial format in this partition - CXZ_ERROR("reformating with lower-dimensional formats has not yet been implemented"); + CXZ_ERROR("reformating with lower-dimensional formats is not possible; use sub-indices instead"); } }, NoF {}); mFormat = FormatT(nformat); diff --git a/src/tests/index_format_test.cc b/src/tests/index_format_test.cc index cc2d245..e747d5f 100644 --- a/src/tests/index_format_test.cc +++ b/src/tests/index_format_test.cc @@ -66,6 +66,7 @@ namespace { GMIndex,CIndex,CIndex,CIndex,CIndex> mi(mRange); MIndex mi2(mRange2); + GMIndex,CIndex,GMIndex,CIndex,CIndex,CIndex>,CIndex,CIndex> mi32(mRange); EXPECT_EQ(mi.lmax().val(), mRange->size()); EXPECT_EQ(mi2.lmax().val(), mRange2->size()); auto yi = indexAs( mRange->begin() ); @@ -79,9 +80,20 @@ namespace yi.setFormat( YFormat( Vector{ mi2.format()[CSizeT<0>{}].val(), mi2.format()[CSizeT<1>{}].val(), mi2.format()[CSizeT<2>{}].val(), mi2.format()[CSizeT<4>{}].val() } ) ); VCHECK(toString(yi.deepFormat())); - mi.reformat( yi.deepFormat(), yi.deepMax() ); VCHECK(toString(mi.deepFormat())); VCHECK(toString(mi.deepMax())); + + yi.setFormat( YFormat( Vector{ 13, 143, 1, 15015 } ) ); + VCHECK(toString(yi.deepFormat())); + mi.reformat( yi.deepFormat(), yi.deepMax() ); + VCHECK(toString(mi.deepFormat())); + VCHECK(toString(mi.deepMax())); + + mi32.reformat( Vector { 105,1,2310,1155 }, Vector { 11,105,13,2 } ); + VCHECK(toString(yi.deepFormat())); + mi32.reformat( yi.deepFormat(), yi.deepMax() ); + VCHECK(toString(mi32.deepFormat())); + VCHECK(toString(mi32.deepMax())); } } -- 2.45.2 From 0eb505f41b853176fdc0e8a04b5447e3dc41445e Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Wed, 3 Jan 2024 19:18:20 +0100 Subject: [PATCH 04/32] yrange: solve lower-dim-reformat issue --- src/include/ranges/mrange.cc.h | 8 ++++---- src/lib/ranges/yrange.cc | 12 ++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/include/ranges/mrange.cc.h b/src/include/ranges/mrange.cc.h index 8891653..b9ea9ac 100644 --- a/src/include/ranges/mrange.cc.h +++ b/src/include/ranges/mrange.cc.h @@ -551,21 +551,21 @@ namespace CNORXZ GMIndex& GMIndex::reformat(const Vector& f, const Vector& s) { + // f: input format + // s: input sizes + CXZ_ASSERT(f.size() == s.size(), "input error: f.size() != s.size()"); if(f.size() == 1){ - CXZ_ASSERT(s.size() == 1, "f.size() != s.size()"); CXZ_ASSERT(s[0] == lmax().val(), "got inconsistent size; expeected " << lmax().val() << ", got " << s[0]); return *this; } + if constexpr(std::is_same::value){ CXZ_ASSERT(CNORXZ::formatIsTrivial(f,s), "cannot reformat MIndex with format type = None"); return *this; } else { - CXZ_ASSERT(f.size() == s.size(), "input error: f.size() != s.size()"); - // f: input format - // s: input sizes SizeT j = 0; SizeT j0 = 0; SizeT xi = 1; diff --git a/src/lib/ranges/yrange.cc b/src/lib/ranges/yrange.cc index f93fdbd..c71a5d7 100644 --- a/src/lib/ranges/yrange.cc +++ b/src/lib/ranges/yrange.cc @@ -415,9 +415,15 @@ namespace CNORXZ YIndex& YIndex::reformat(const Vector& f, const Vector& s) { - CXZ_ASSERT(f.size() == s.size(), "input error: f.size() != s.size()"); // f: input format // s: input sizes + CXZ_ASSERT(f.size() == s.size(), "input error: f.size() != s.size()"); + if(f.size() == 1){ + CXZ_ASSERT(s[0] == lmax().val(), "got inconsistent size; expeected " + << lmax().val() << ", got " << s[0]); + return *this; + } + SizeT j = 0; SizeT j0 = 0; SizeT xi = 1; @@ -427,9 +433,7 @@ namespace CNORXZ xi *= mIs[i]->lmax().val(); SizeT xj = s[j]; if(xi < xj) { - // TODO: IMPLEMENT!!! - // check trivial format in this partition - CXZ_ERROR("reformating with lower-dimensional formats has not yet been implemented"); + CXZ_ERROR("reformating with lower-dimensional formats is not possible; use sub-indices instead"); continue; } j0 = j; -- 2.45.2 From 4c71fac091a60d4d9990475054e627bd6a1b882b Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Sat, 13 Jan 2024 18:26:28 +0100 Subject: [PATCH 05/32] more documentation; WIP: mrange doc --- src/include/ranges/eindex.h | 55 ++++++++++++++-- src/include/ranges/index_format.h | 104 ++++++++++++++++++++++++++++-- src/include/ranges/index_mul.h | 48 +++++++++++++- src/include/ranges/index_traits.h | 2 +- src/include/ranges/index_utils.h | 29 --------- src/include/ranges/lindex.h | 46 +++++++++++-- src/include/ranges/mrange.h | 82 ++++++++++++++++++----- 7 files changed, 303 insertions(+), 63 deletions(-) delete mode 100644 src/include/ranges/index_utils.h diff --git a/src/include/ranges/eindex.h b/src/include/ranges/eindex.h index f7c0bc7..f7e2c81 100644 --- a/src/include/ranges/eindex.h +++ b/src/include/ranges/eindex.h @@ -5,7 +5,7 @@ @brief ... - Copyright (c) 2022 Christian Zimmermann. All rights reserved. + Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de **/ @@ -20,7 +20,17 @@ namespace CNORXZ { - + /** ***** + Extension Index + + Elements accessed through this index in a loop + are treated and processed through a single access + allowing the usage of vector extensions + + @tparam MetaT Index meta data type + @tparam S Vector size + @tparam L Static index label + */ template class EIndex : public LIndex,L> { @@ -28,9 +38,15 @@ namespace CNORXZ typedef typename LIndex,L>::IB IB; typedef typename LIndex,L>::RangeType RangeType; - DEFAULT_MEMBERS(EIndex); + DEFAULT_MEMBERS(EIndex); /**< default constructors and assignments */ EIndex(const Sptr,L>>& i); + /** @copydoc IndexInterface::ifor() + + Specialization for EIndex: access all elements + at once, allowing usage of vector extensions + @see EFor + */ template decltype(auto) ifor(const Xpr& xpr, F&& f) const; @@ -38,6 +54,10 @@ namespace CNORXZ Sptr,L>> mLI; }; + /** *** + EIndex is an index + @see is_index + */ template struct is_index> { @@ -53,25 +73,52 @@ namespace CNORXZ { static constexpr bool value = true; }; - + + /** Make index pack from EIndex and second index of arbitrary type + */ template decltype(auto) operator*(const Sptr>& a, const Sptr& b); + /** Create Eindex pointer from LIndex pointer + */ template decltype(auto) eindexPtr(const Sptr,L>>& i); + /** Create Eindex pointer from SIndex pointer + @tparam L Static index label + */ template decltype(auto) eindexPtr(const Sptr>& i); + /** Create Eindex pointer from LIndex pointer + @param l Static index label + */ template decltype(auto) eindexPtr(const Sptr>& i, CSizeT l); + /** Split given index into pack of EIndex and remainder index + @param i Index to be split + @tparam S Vector size + @tparam L1 label of EIndex + @tparam L2 label of remainder index + */ template decltype(auto) eplex(const Sptr& i); + /** Split given index into pack of EIndex and remainder index + @param i Index to be split + @param s Vector size + @param l label of EIndex + */ template decltype(auto) eplex(const Sptr& i, CSizeT s, CSizeT l); + /** Split given index into pack of EIndex and remainder index + @param i Index to be split + @param s Vector size + @param l1 label of EIndex + @param l2 label of remainder index + */ template decltype(auto) eplex(const Sptr& i, CSizeT s, CSizeT l1, CSizeT l2); diff --git a/src/include/ranges/index_format.h b/src/include/ranges/index_format.h index 9f80d32..f67ae74 100644 --- a/src/include/ranges/index_format.h +++ b/src/include/ranges/index_format.h @@ -5,7 +5,7 @@ @brief ... - Copyright (c) 2022 Christian Zimmermann. All rights reserved. + Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de **/ @@ -18,26 +18,51 @@ namespace CNORXZ { + /** **** + Type trait: Check if format has static size. + */ template struct is_static_format { CXZ_CVAL_FALSE; }; - + + /** **** + Multi index format of static size. + Wrapper of standard array of UPos. + @tparam N Format size + */ template class MFormat { public: typedef Arr InputType; - SP_DEFAULT_MEMBERS(constexpr,MFormat); + SP_DEFAULT_MEMBERS(constexpr,MFormat); /**< default constructors and assignments */ + + /** Construct MFormat from standard array. + @param b Input array + */ explicit constexpr MFormat(const Arr& b); + /** Construct MFormat from format of arbitrary type. + The input format size has to match the static size N. + @param f Input format + */ template constexpr MFormat(const FormatT& f); + /** Get underlying array. */ const Arr& all() const; + + /** Get format size. */ constexpr decltype(auto) size() const; + /** Get format element. + @param i CSizeT indicating static element position + */ template constexpr decltype(auto) get(CSizeT i) const; + /** Get format element. + @param i CSizeT indicating static element position + */ template constexpr decltype(auto) operator[](CSizeT i) const; @@ -46,27 +71,58 @@ namespace CNORXZ }; + /** **** + MFormat has static size. + @see is_static_format + */ template struct is_static_format> { CXZ_CVAL_TRUE; }; + /** **** + Multi index format of static size. + Wrapper of standard tuple of position types. + @tparam PosT Position types. + */ template class GMFormat { public: typedef Tuple InputType; - SP_DEFAULT_MEMBERS(constexpr,GMFormat); + SP_DEFAULT_MEMBERS(constexpr,GMFormat); /**< default constructors and assignments */ + + /** Construct from tuple. + @param b Input tuple. + */ explicit constexpr GMFormat(const Tuple& b); + + /** Construct from tuple (move). + @param b Input tuple. + */ explicit constexpr GMFormat(Tuple&& b); + /** Construct MFormat from format of arbitrary type. + The input format size has to match the number of entries + and the input entries have to be compatible with the position types. + @param f Input format + */ template constexpr GMFormat(const FormatT& f); + /** Get underlying tuple. */ const Tuple& all() const; + + /** Get format size. */ constexpr decltype(auto) size() const; + /** Get format element. + @param i CSizeT indicating static element position + */ template constexpr decltype(auto) get(CSizeT i) const; + /** Get format element. + @param i CSizeT indicating static element position + */ template constexpr decltype(auto) operator[](CSizeT i) const; @@ -74,38 +130,76 @@ namespace CNORXZ Tuple mB; }; + /** Create GMFormat from position types. + @param ps Position types. + */ template constexpr decltype(auto) gmformat(const PosT&... ps); + /** **** + GMFormat has static size. + @see is_static_format + */ template struct is_static_format> { CXZ_CVAL_TRUE; }; + /** **** + Multi index format of variable size + Wrapper of standard vector of UPos. + */ class YFormat { public: typedef Vector InputType; - DEFAULT_MEMBERS(YFormat); + DEFAULT_MEMBERS(YFormat); /**< default constructors and assignments */ + + /** Construct from vector. + @param b Input vector. + */ explicit YFormat(const Vector& b); + /** Construct from format of arbitrary type. + @param f Input format. + */ template YFormat(const FormatT& f); + /** Get underlying vector. */ const Vector& all() const; + + /** Get format size. */ SizeT size() const; + /** Get format element. + @param i CSizeT indicating static element position + */ template const UPos& get(CSizeT i) const; + /** Get format element. + @param i CSizeT indicating static element position + */ template const UPos& operator[](CSizeT i) const; + /** Get format element. + @param i E element position + */ const UPos& get(SizeT i) const; + + /** Get format element. + @param i E element position + */ const UPos& operator[](SizeT i) const; private: Vector mB; }; + /** Check if format is trivial, i.e. f[i-1] = f[i]*s[i]. + @param f Vector representing the index format. + @param s Vector representing the sub-index maxima. + */ bool formatIsTrivial(const Vector& f, const Vector& s); } diff --git a/src/include/ranges/index_mul.h b/src/include/ranges/index_mul.h index 5e08cb3..de023d1 100644 --- a/src/include/ranges/index_mul.h +++ b/src/include/ranges/index_mul.h @@ -2,10 +2,11 @@ /** @file include/ranges/index_mul.h - @brief ... + @brief Index multiplication + Indices can be multiplied yielding index packs. - Copyright (c) 2022 Christian Zimmermann. All rights reserved. + Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de **/ @@ -35,30 +36,71 @@ namespace CNORXZ Isq is, Isq js); }; + /** Combine two indices to a static index 2-pack. + @param a First index. + @param b Second index. + @return Static index 2-pack + */ template inline decltype(auto) operator*(const IndexInterface& a, const IndexInterface& b); + /** Extend static index pack on the l.h.s. + @param a Index to be appended. + @param b Index pack to be extended. + @return Extended pack. + */ template inline decltype(auto) operator*(const IndexInterface& a, const SPack& b); + /** Extend static index pack on the r.h.s. + @param a Index pack to be extended. + @param b Index to be appended. + @return Extended pack. + */ template inline decltype(auto) operator*(const SPack& a, const IndexInterface& b); + /** Combine two static index packs + @param a First Index pack. + @param b Second Index pack. + @return New index pack. a is appended on the l.h.s. of b. + */ template inline decltype(auto) operator*(const SPack& a, const SPack& b); + /** Extend dynamic index pack on the l.h.s. + @param a Index to be appended. + @param b Index pack to be extended. + @return Extended pack. + */ template inline decltype(auto) operator*(const IndexInterface& a, const DPack& b); + /** Extend dynamic index pack on the r.h.s. + @param a Index pack to be extended. + @param b Index to be appended. + @return Extended pack. + */ template inline decltype(auto) operator*(const DPack& a, const IndexInterface& b); + /** Combine two dynamic index packs + @param a First Index pack. + @param b Second Index pack. + @return New index pack. a is appended on the l.h.s. of b. + */ inline decltype(auto) operator*(const DPack& a, const DPack& b); - + /** Combine two index pointers to an index 2-pack. + YIndices and DIndices will be combined into a DPack, + otherwise a SPack is returned. + @param a First index. + @param b Second index. + @return Index 2-pack. + */ template decltype(auto) iptrMul(const Sptr& a, const Sptr& b); } diff --git a/src/include/ranges/index_traits.h b/src/include/ranges/index_traits.h index 439337f..601a96e 100644 --- a/src/include/ranges/index_traits.h +++ b/src/include/ranges/index_traits.h @@ -4,7 +4,7 @@ @file include/ranges/index_traits.h @brief index traits - Copyright (c) 2022 Christian Zimmermann. All rights reserved. + Copyright (c) 2023 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de **/ diff --git a/src/include/ranges/index_utils.h b/src/include/ranges/index_utils.h deleted file mode 100644 index e2f3cb4..0000000 --- a/src/include/ranges/index_utils.h +++ /dev/null @@ -1,29 +0,0 @@ -// -*- C++ -*- -/** - - @file include/ranges/index_utils.h - @brief ... - - - Copyright (c) 2022 Christian Zimmermann. All rights reserved. - Mail: chizeta@f3l.de - -**/ - -#ifndef __cxz_index_utils_h__ -#define __cxz_index_utils_h__ - -#include "base/base.h" -#include "index_base.h" - -namespace CNORXZ -{ - // automatically set static format if SIndices are used! - template - constexpr decltype(auto) autoiPtr(const SPack& pack); - - inline SPtr autoiPtr(const DPack& pack); - -} - -#endif diff --git a/src/include/ranges/lindex.h b/src/include/ranges/lindex.h index 283d347..1053909 100644 --- a/src/include/ranges/lindex.h +++ b/src/include/ranges/lindex.h @@ -2,10 +2,12 @@ /** @file include/ranges/lindex.h - @brief ... + @brief Statically labeled index. - - Copyright (c) 2022 Christian Zimmermann. All rights reserved. + Static index labels are usefull to resolve extensions and relations to + other indices at compile time. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de **/ @@ -20,7 +22,11 @@ namespace CNORXZ { - // static label to enforce loop unrolling + /** **** + Statically labeled index. + @tparam Index Underlying index. + @tparam L Static label. + */ template class LIndex : public Index { @@ -28,14 +34,25 @@ namespace CNORXZ typedef typename Index::IB IB; typedef typename Index::RangeType RangeType; - DEFAULT_MEMBERS(LIndex); + DEFAULT_MEMBERS(LIndex); /**< Default constructors and assignments. */ + + /** Construct from index pointer. + @param i Input index. + */ LIndex(const Sptr& i); + /** @copydoc IndexInterface::id() + Specialization: Static id equals L. + */ IndexId id() const; + /** @copydoc IndexInterface::stepSize() + Specialization: stepSize may be static. + */ template decltype(auto) stepSize(const IndexId& id) const; + /** @copydoc IndexInterface::stepSize() */ template decltype(auto) ifor(const Xpr& xpr, F&& f) const; @@ -43,6 +60,10 @@ namespace CNORXZ Sptr mI; }; + /** *** + LIndex is an index + @see is_index + */ template struct is_index> { @@ -59,12 +80,27 @@ namespace CNORXZ static constexpr bool value = index_expression_exists::value; }; + /** Specialize index multiplication for LIndex. + @param a Pointer to first index which is a LIndex. + @param b Pointer to second index of arbitrary type. + @return Resulting index pack. + */ template decltype(auto) operator*(const Sptr>& a, const Sptr& b); + /** Create LIndex from index pointer. + @param i Input index. + @tparam L Static label. + @return Resulting LIndex. + */ template decltype(auto) lindexPtr(const Sptr& i); + /** Create LIndex from index pointer. + @param i Input index. + @param l CSizeT indicating the static label. + @return Resulting LIndex. + */ template decltype(auto) lindexPtr(const Sptr& i, CSizeT l); } diff --git a/src/include/ranges/mrange.h b/src/include/ranges/mrange.h index 2d9b1c0..155fc40 100644 --- a/src/include/ranges/mrange.h +++ b/src/include/ranges/mrange.h @@ -2,10 +2,15 @@ /** @file include/ranges/mrange.h - @brief ... + @brief MRange, GMIndex and MIndex declaration + MRange is a multi-range consisting of of a compile-time fixed number of sub-ranges. - Copyright (c) 2022 Christian Zimmermann. All rights reserved. + GMIndex and MIndex are multi-index consisting of a compile-time fixed number of sub-indices. + The difference between the two index types is that MIndex has a statically trivial format, + while GMIndex can have an arbitrary format (MFormat or GMFormat). + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de **/ @@ -22,8 +27,13 @@ namespace CNORXZ { - // template - // -> Format + IndexTuple + + /** **** + Multi-index with fixed number of sub-indices of arbitrary type + and arbitrary index format. + @tparam FormatT Index format type. + @tparam Indices Sub-index types. + */ template class GMIndex : public IndexInterface, Tuple > @@ -32,7 +42,6 @@ namespace CNORXZ typedef IndexInterface, Tuple> IB; - //typedef Tuple...> IndexPack; typedef Tuple MetaType; typedef MRange RangeType; static constexpr SizeT NI = sizeof...(Indices); @@ -52,36 +61,86 @@ namespace CNORXZ constexpr GMIndex(const RangePtr& range, SizeT lexpos = 0); constexpr GMIndex(const RangePtr& range, const FormatT& format, SizeT lexpos = 0); + /** @copydoc IndexInterface::operator=(SizeT) */ GMIndex& operator=(SizeT pos); + + /** @copydoc IndexInterface::operator++() */ GMIndex& operator++(); + + /** @copydoc IndexInterface::operator--() */ GMIndex& operator--(); + + /** @copydoc IndexInterface::operator+() */ GMIndex operator+(Int n) const; + + /** @copydoc IndexInterface::operator-() */ GMIndex operator-(Int n) const; + + /** @copydoc IndexInterface::operator-(CIndex) */ SizeT operator-(const GMIndex& i) const; + + /** @copydoc IndexInterface::operator+=() */ GMIndex& operator+=(Int n); + + /** @copydoc IndexInterface::operator-=() */ GMIndex& operator-=(Int n); + /** @copydoc IndexInterface::lex() */ SizeT lex() const; + + /** @copydoc IndexInterface::pmax() */ constexpr decltype(auto) pmax() const; + + /** @copydoc IndexInterface::lmax() */ constexpr decltype(auto) lmax() const; + + /** @copydoc IndexInterface::id() */ IndexId<0> id() const; + /** @copydoc IndexInterface::operator*() */ MetaType operator*() const; + /** @copydoc IndexInterface::dim() */ constexpr SizeT dim() const; + + /** @copydoc IndexInterface::range() */ Sptr range() const; + /** @copydoc IndexInterface::stepSize() */ template decltype(auto) stepSize(const IndexId& id) const; + /** @copydoc IndexInterface::stringMeta() */ String stringMeta() const; + + /** @copydoc IndexInterface::meta() */ MetaType meta() const; + + /** @copydoc IndexInterface::at() */ GMIndex& at(const MetaType& metaPos); - decltype(auto) xpr(const Sptr>& _this) const; - + + /** @copydoc IndexInterface::prange() */ + RangePtr prange(const GMIndex& last) const; + + /** @copydoc IndexInterface::deepFormat() */ + auto deepFormat() const; + + /** @copydoc IndexInterface::deepMax() */ + auto deepMax() const; + + /** @copydoc IndexInterface::reformat() */ + GMIndex& reformat(const Vector& f, const Vector& s); + + /** @copydoc IndexInterface::ifor() */ template constexpr decltype(auto) ifor(const Xpr& xpr, F&& f) const; + /** @copydoc IndexInterface::formatIsTrivial() */ + bool formatIsTrivial() const; + + /** @copydoc IndexInterface::xpr() */ + decltype(auto) xpr(const Sptr>& _this) const; + // replace sub-index instances; only use if you know what you are doing! GMIndex& operator()(const Sptr>& mi); GMIndex& operator()(); @@ -89,18 +148,9 @@ namespace CNORXZ const SPack& pack() const; const auto& format() const; const auto& lexFormat() const; - RangePtr prange(const GMIndex& last) const; - auto deepFormat() const; - auto deepMax() const; - - /** @copydoc IndexInterface::reformat() */ - GMIndex& reformat(const Vector& f, const Vector& s); GMIndex& setFormat(const FormatT& bs); - /** @copydoc IndexInterface::formatIsTrivial() */ - bool formatIsTrivial() const; - private: template static constexpr decltype(auto) mkLexFormat(const SPack& ipack, Isq is); -- 2.45.2 From ac352206c71251c3f49baaa55cbfeb77ad34876b Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Sun, 14 Jan 2024 23:57:42 +0100 Subject: [PATCH 06/32] more doc --- README.md | 8 +- src/include/ranges/crange.h | 26 ++--- src/include/ranges/mrange.cc.h | 50 ++++------ src/include/ranges/mrange.h | 172 +++++++++++++++++++++++++++------ 4 files changed, 178 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index 324e643..19c0247 100644 --- a/README.md +++ b/README.md @@ -36,9 +36,9 @@ The tools of the library are accessible within the namespace `CNORXZ`. ### Basics and Library organization -This library consists of several building blocks. For simple usage, the most important building blocks are [ranges](#ranges), [indices](#indices) and [array types](#arrays). +This library consists of several building blocks. For simple usage, the most important building blocks are [ranges](#sec-ranges), [indices](#sec-indices) and [arrays](#sec-array-types). -#### Ranges +#### Ranges {#sec-ranges} Basically, a *range* defines a meta data space. There are several range class types, which are derived from the abstract base class `RangeBase`. Ranges can only be created by the corresponding factory and exclusively exist within a shared pointer; they cannot be copied. Available range class types are: @@ -54,7 +54,7 @@ Basically, a *range* defines a meta data space. There are several range class ty * `YRange` : The same as `MRange` but the number of ranges and their types can be specified at runtime. -#### Indices +#### Indices {#sec-indices} For each range type there is a corresponding index type (`CIndex`, `UIndex`, `SIndex`, `PIndex`, `MIndex`, `YIndex`). They act as const iterators on the ranges and are a crucial component to define operations on containers. In contrast to the ranges, all index types must be known at compile time (static polymorphism, `IndexInterface`). @@ -66,7 +66,7 @@ Apart from range specific indices, there exist also special indices: * `BIndex` : The same as `AIndex`, but not const. -#### Array types +#### Array types {#sec-array-types} Finally, there are the container classes (arrays), which are derived from `CArrayBase` (const) or `ArrayBase` for a given data type `T`. All arrays are defined on a range, their data can be accessed or iterated over using suitable indices. The array-type actually containing data is called `MArray`. Moreover, there exist array-types that do not contain data, but view the data of other arrays or at least parts of the data. These are called `CSlice` (const view) or `Slice`. diff --git a/src/include/ranges/crange.h b/src/include/ranges/crange.h index cfef60b..6eb1fbb 100644 --- a/src/include/ranges/crange.h +++ b/src/include/ranges/crange.h @@ -4,7 +4,7 @@ @file include/ranges/crange.h @brief CRange and CIndex declaration. - Copyright (c) 2022 Christian Zimmermann. All rights reserved. + Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de **/ @@ -34,8 +34,8 @@ namespace CNORXZ DEFAULT_MEMBERS(CIndex); /**< default constructors and assignments */ /** Construct index from range and position. - @param range Range to iterate over - @param pos lexicographic position + @param range Range to iterate over. + @param pos lexicographic position. */ CIndex(const RangePtr& range, SizeT pos = 0); @@ -123,30 +123,30 @@ namespace CNORXZ Sptr mRangePtr; }; - /** make index pack of a CIndex and another index - @tparam type of the second index - @param a pointer to CIndex - @param b pointer to another index + /** make index pack of a CIndex and another index. + @tparam type of the second index. + @param a pointer to CIndex. + @param b pointer to another index. */ template decltype(auto) operator*(const Sptr& a, const Sptr& b); /** **** - specific factory for CRange + Specific factory for CRange. */ class CRangeFactory : public RangeFactoryBase { public: typedef CRange oType; - /** Construct and setup factory - @param size Size of the range to be constructed + /** Construct and setup factory. + @param size Size of the range to be constructed. */ CRangeFactory(SizeT size); - /** Construct and setup factory - @param size Size of the range to be constructed - @param ref Range the range to be constructed is related to + /** Construct and setup factory. + @param size Size of the range to be constructed. + @param ref Range the range to be constructed is related to. */ CRangeFactory(SizeT size, RangePtr ref); diff --git a/src/include/ranges/mrange.cc.h b/src/include/ranges/mrange.cc.h index b9ea9ac..368f8bc 100644 --- a/src/include/ranges/mrange.cc.h +++ b/src/include/ranges/mrange.cc.h @@ -2,10 +2,10 @@ /** @file include/ranges/mrange.cc.h - @brief ... + @brief MRange, GMIndex and MIndex, member definition. - Copyright (c) 2022 Christian Zimmermann. All rights reserved. + Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de **/ @@ -21,9 +21,9 @@ namespace CNORXZ { - /************************ - * GMIndex (private) * - ************************/ + /*=========================+ + | GMIndex (private) | + +=========================*/ template template @@ -147,9 +147,9 @@ namespace CNORXZ } } - /*************** - * GMIndex * - ***************/ + /*===============+ + | GMIndex | + +===============*/ template constexpr GMIndex::GMIndex(const GMIndex& i) : @@ -613,15 +613,6 @@ namespace CNORXZ return *this; } - template - decltype(auto) replaceFormat(const BT1& bs1, const Sptr>& gmi) - { - return iter<0,sizeof...(Indices)> - ( [&](auto i) { return gmi->pack()[CSizeT{}]; }, - [&](const auto&... e) { return std::make_shared> - ( bs1, e... ); } ); - } - template constexpr decltype(auto) mindex(const Sptr&... is) { @@ -652,9 +643,9 @@ namespace CNORXZ return iptrMul(a, b); } - /********************* - * MRangeFactory * - *********************/ + /*=====================+ + | MRangeFactory | + +=====================*/ template MRangeFactory::MRangeFactory(const Tuple...>& rs) : @@ -683,9 +674,9 @@ namespace CNORXZ } } - /************** - * MRange * - **************/ + /*==============+ + | MRange | + +==============*/ template MRange::MRange(const Tuple...>& rs) : @@ -703,7 +694,6 @@ namespace CNORXZ template MArray MRange::sub() const { - // TODO: ZRange (meta and index pos static!) if constexpr(NR == 0) { return MArray(); } @@ -791,9 +781,9 @@ namespace CNORXZ return k; } - /************************ - * MRange (private) * - ************************/ + /*========================+ + | MRange (private) | + +========================*/ template decltype(auto) MRange::mkA() const @@ -802,9 +792,9 @@ namespace CNORXZ [](const auto&... xs) { return Arr { xs... }; } ); } - /**************************** - * non-member functions * - ****************************/ + /*============================+ + | non-member functions | + +============================*/ template RangePtr mrange(const Sptr&... rs) diff --git a/src/include/ranges/mrange.h b/src/include/ranges/mrange.h index 155fc40..b0bda2f 100644 --- a/src/include/ranges/mrange.h +++ b/src/include/ranges/mrange.h @@ -2,7 +2,7 @@ /** @file include/ranges/mrange.h - @brief MRange, GMIndex and MIndex declaration + @brief MRange, GMIndex and MIndex declaration. MRange is a multi-range consisting of of a compile-time fixed number of sub-ranges. @@ -47,18 +47,38 @@ namespace CNORXZ static constexpr SizeT NI = sizeof...(Indices); INDEX_RANDOM_ACCESS_ITERATOR_DEFS(MetaType); + + /** Default constructor. */ constexpr GMIndex() = default; + + /** Move constructor (default). */ constexpr GMIndex(GMIndex&& i) = default; + + /** Move assignment (default). */ constexpr GMIndex& operator=(GMIndex&& i) = default; - // no defaults: + + /** Copy constructor (no default, copy sub-index instances). */ constexpr GMIndex(const GMIndex& i); + + /** Copy assignment (no default, copy sub-index instances). */ constexpr GMIndex& operator=(const GMIndex& i); + /** Construct from index pack. */ constexpr GMIndex(const SPack& pack); + + /** Construct from index pack and format. */ constexpr GMIndex(const FormatT& format, const SPack& pack); + + /** Construct from index pointers. */ constexpr GMIndex(const Sptr&... is); + + /** Construct from index pointers and format. */ constexpr GMIndex(const FormatT& format, const Sptr&... is); + + /** Construct from range. */ constexpr GMIndex(const RangePtr& range, SizeT lexpos = 0); + + /** Construct from range and format. */ constexpr GMIndex(const RangePtr& range, const FormatT& format, SizeT lexpos = 0); /** @copydoc IndexInterface::operator=(SizeT) */ @@ -142,13 +162,24 @@ namespace CNORXZ decltype(auto) xpr(const Sptr>& _this) const; // replace sub-index instances; only use if you know what you are doing! + /** Replace sub-index instances and update index position correspondingly. + @param new index instances. + */ GMIndex& operator()(const Sptr>& mi); + + /** Update index position according to the sub-indices. */ GMIndex& operator()(); + /** Get index pack. */ const SPack& pack() const; + + /** Get index format. */ const auto& format() const; + + /** Get lexicographic index format. */ const auto& lexFormat() const; + /** Assign new index format. */ GMIndex& setFormat(const FormatT& bs); private: @@ -185,53 +216,114 @@ namespace CNORXZ PMaxT mPMax; }; - template - decltype(auto) replaceFormat(const BT1& bs1, const Sptr>& gmi); - - template - struct index_has_const_size> + /** **** + Specialization of index_has_const_size for GMIndex. + @see index_has_const_size + */ + template + struct index_has_const_size> { static constexpr bool value = (index_has_const_size::value and ...); }; - template - struct index_const_size> + /** **** + Specialization of index_const_size for GMIndex. + @see index_const_size + */ + template + struct index_const_size> { static constexpr SizeT value = (index_const_size::value * ...); }; - template - struct index_dim> + /** **** + Specialization of index_dim for GMIndex. + @see index_dim + */ + template + struct index_dim> { static constexpr SizeT value = sizeof...(Indices); }; - template - struct has_sub> + /** **** + Specialization of has_sub for GMIndex. + @see has_sub + */ + template + struct has_sub> { static constexpr bool value = true; }; - template - struct has_static_sub> + /** **** + Specialization of has_static_sub for GMIndex. + @see has_static_sub + */ + template + struct has_static_sub> { static constexpr bool value = true; }; - template - struct index_is_multi> + /** **** + Specialization of index_is_multi for GMIndex. + @see index_is_multi + */ + template + struct index_is_multi> { static constexpr bool value = true; }; + /** *** + MIndex can be used as expression if all its sub-indices can be used as expression + @see index_expression_exists + */ + template + struct index_expression_exists> + { + static constexpr bool value = (index_expression_exists::value and ...); + }; + + /** Create MIndex from index pointers. + @param is Input index pointers. + */ template constexpr decltype(auto) mindex(const Sptr&... is); + /** Create MIndex from index pack. + @param pack Pack of input indices. + */ template constexpr decltype(auto) mindex(const SPack& pack); + /** Create pointer to MIndex from index pack. + @param pack Pack of input indices. + */ template constexpr decltype(auto) mindexPtr(const SPack& pack); + /** Create pointer to GMIndex from index pack and format. + @param bs Index format. + @param pack Pack of input indices. + */ template constexpr decltype(auto) gmindexPtr(const FormatT& bs, const SPack& pack); - + + /** Specialization for index multiplication with GMIndex on the l.h.s. + @param a First index of type GMIndex. + @param b Second index of arbitrary type. + */ template decltype(auto) operator*(const Sptr>& a, const Sptr& b); + /** **** + Specific factory for MRange. + @tparam Ranges Types of the sub-ranges. + */ template class MRangeFactory : public RangeFactoryBase { public: + + /** Construct and setup factory. + @param rs Tuple of sub-ranges. + */ MRangeFactory(const Tuple...>& rs); + + /** Construct and setup factory. + @param rs Tuple of sub-ranges. + @param ref Range the range to be constructed is related to. + */ MRangeFactory(const Tuple...>& rs, const RangePtr& ref); private: @@ -241,7 +333,11 @@ namespace CNORXZ Tuple...> mRs; RangePtr mRef; }; - + + /** **** + Multi Range with compile-time fixed number of sub-ranges + of compile-time fixed type. + */ template class MRange : public RangeInterface> { @@ -261,19 +357,35 @@ namespace CNORXZ virtual const TypeInfo& type() const override final; virtual const TypeInfo& metaType() const override final; virtual RangePtr extend(const RangePtr& r) const override final; - + + /** Get sub-ranges. */ decltype(auto) space() const; + + /** Get sub-range. + @param pos Position of the sub-range. + */ const MetaType get(SizeT pos) const; + + /** Get lexicographic position according to the given meta data value. + @param metaPos Meta data value. + */ SizeT getMeta(const MetaType& metaPos) const; protected: + + /** Dafault constructor */ MRange() = default; + MRange(const MRange& in) = delete; MRange& operator=(const MRange& in) = delete; + + /** Construct from sub-ranges + @param rs Tuple of pointers to sub-ranges. + */ MRange(const Tuple...>& rs); - Tuple...> mRs; - Arr mA; + Tuple...> mRs; /**< Tuple of pointers to sub-ranges. */ + Arr mA; /**< Array of abstract base pointers to sub-ranges. Redundant to mRs */ virtual Vector key() const override final; private: @@ -281,24 +393,22 @@ namespace CNORXZ decltype(auto) mkA() const; }; + /** Create MRange pointer. + @param rs Pointer to sub-ranges. + */ template RangePtr mrange(const Sptr&... rs); + /** **** + Specialization of RangeCast for MRange. + @see RangeCast. + */ template struct RangeCast> { static Sptr> func(const RangePtr& r); }; - /** *** - MIndex can be used as expression if all its sub-indices can be used as expression - @see index_expression_exists - */ - template - struct index_expression_exists> - { - static constexpr bool value = (index_expression_exists::value and ...); - }; } -- 2.45.2 From a5031d80b17466c7fb4d50c193de2e8b32f06d19 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Wed, 17 Jan 2024 22:57:18 +0100 Subject: [PATCH 07/32] prange comments --- src/include/ranges/crange.h | 3 +- src/include/ranges/prange.h | 125 ++++++++++++++++++++++++++++++++---- 2 files changed, 112 insertions(+), 16 deletions(-) diff --git a/src/include/ranges/crange.h b/src/include/ranges/crange.h index 6eb1fbb..4083c68 100644 --- a/src/include/ranges/crange.h +++ b/src/include/ranges/crange.h @@ -123,8 +123,7 @@ namespace CNORXZ Sptr mRangePtr; }; - /** make index pack of a CIndex and another index. - @tparam type of the second index. + /** Make index pack of a CIndex and another index. @param a pointer to CIndex. @param b pointer to another index. */ diff --git a/src/include/ranges/prange.h b/src/include/ranges/prange.h index 562a3e6..619d84b 100644 --- a/src/include/ranges/prange.h +++ b/src/include/ranges/prange.h @@ -2,10 +2,9 @@ /** @file include/ranges/prange.h - @brief ... + @brief PRange, PRangeFactory and PIndex declaration. - - Copyright (c) 2022 Christian Zimmermann. All rights reserved. + Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de **/ @@ -20,7 +19,10 @@ namespace CNORXZ { - + /** **** + Index specific for PRange. + @tparam IndexT Full index type. + */ template class PIndex : public IndexInterface,typename IndexT::MetaType> { @@ -30,51 +32,105 @@ namespace CNORXZ typedef PRange RangeType; typedef typename IndexT::MetaType MetaType; + /** Constructor. + @param range Range to define index on. + @param pos Initial lexicographic position (default = 0). + */ PIndex(const RangePtr& range, SizeT pos = 0); + /** @copydoc IndexInterface::operator=(SizeT) */ PIndex& operator=(SizeT lexpos); + + /** @copydoc IndexInterface::operator++() */ PIndex& operator++(); + + /** @copydoc IndexInterface::operator--() */ PIndex& operator--(); + + /** @copydoc IndexInterface::operator+() */ PIndex operator+(Int n) const; + + /** @copydoc IndexInterface::operator-() */ PIndex operator-(Int n) const; + + /** @copydoc IndexInterface::operator-(PIndex) */ SizeT operator-(const PIndex& i) const; + + /** @copydoc IndexInterface::operator+=() */ PIndex& operator+=(Int n); + + /** @copydoc IndexInterface::operator-=() */ PIndex& operator-=(Int n); + /** @copydoc IndexInterface::lex() */ SizeT lex() const; + + /** @copydoc IndexInterface::pmax() */ UPos pmax() const; + + /** @copydoc IndexInterface::lmax() */ UPos lmax() const; + + /** @copydoc IndexInterface::id() */ IndexId<0> id() const; + /** @copydoc IndexInterface::operator*() */ decltype(auto) operator*() const; + /** @copydoc IndexInterface::dim() */ SizeT dim() const; + + /** @copydoc IndexInterface::range() */ Sptr range() const; + /** @copydoc IndexInterface::stepSize() */ template UPos stepSize(const IndexId& id) const; + + /** @copydoc IndexInterface::stringMeta() */ + String stringMeta() const; + + /** @copydoc IndexInterface::meta() */ + decltype(auto) meta() const; + + /** @copydoc IndexInterface::at() */ + PIndex& at(const MetaType& metaPos); + + /** @copydoc IndexInterface::prange() */ RangePtr prange(const PIndex& last) const; + + /** @copydoc IndexInterface::deepFormat() */ decltype(auto) deepFormat() const; + + /** @copydoc IndexInterface::deepMax() */ decltype(auto) deepMax() const; /** @copydoc IndexInterface::reformat() */ PIndex& reformat(const Vector& f, const Vector& s); - String stringMeta() const; - decltype(auto) meta() const; - PIndex& at(const MetaType& metaPos); - decltype(auto) xpr(const Sptr>& _this) const; - + /** @copydoc IndexInterface::ifor() */ template decltype(auto) ifor(const Xpr& xpr, F&& f) const; - PIndex& operator()(); - PIndex& operator()(const Sptr& i); - const Sptr& orig() const; - /** @copydoc IndexInterface::formatIsTrivial() */ bool formatIsTrivial() const; + /** @copydoc IndexInterface::xpr() */ + decltype(auto) xpr(const Sptr>& _this) const; + + /** Replace instance of index on full range and update current position accordingly. + @param i New index instance. + */ + PIndex& operator()(const Sptr& i); + + /** Update current index position according to the internal index on the full range. */ + PIndex& operator()(); + + /** Get original index. + @return Index corresponding to original range and current position. + */ + const Sptr& orig() const; + private: Sptr mRangePtr; Sptr mOrig; @@ -82,14 +138,31 @@ namespace CNORXZ void mkPos(); }; + /** Make index pack of a PIndex and another index. + @param a pointer to PIndex. + @param b pointer to another index. + */ template decltype(auto) operator*(const Sptr>& a, const Sptr& b); + /** **** + Specific factory for PRange. + */ template class PRangeFactory : public RangeFactoryBase { public: + + /** Construct and setup factory. + @param range Full range (explicit type) the constructed range is part of. + @param _parts Integer vector indicating the parts w.r.t. input range to be covered by the PRange. + */ PRangeFactory(const Sptr& range, const Vector& _parts); + + /** Construct and setup factory. + @param range Full range the constructed range is part of. + @param _parts Integer vector indicating the parts of the full range. + */ PRangeFactory(const RangePtr& range, const Vector& _parts); private: @@ -99,7 +172,14 @@ namespace CNORXZ Sptr mRange; Vector mParts; }; - + + /** **** + Partial Range. + Ranges of these kind represent a part of a given range (full range). + Using a mathematical nomenclature, this would be called a "sub-range". + (The prefix "sub", as well as the letter "S" are, however, already extensively + used in other contexts.) + */ template class PRange : public RangeInterface> { @@ -117,8 +197,20 @@ namespace CNORXZ virtual const TypeInfo& metaType() const override final; virtual RangePtr extend(const RangePtr& r) const override final; + /** Get the full range. + @return Pointer to the full range. + */ Sptr orig() const; + + /** Get the parts. + @return Integer vector indicating the parts contained by the PRange w.r.t. the full range. + */ const Vector& parts() const; + + /** Create a new range of the type of the full range but containing only + the parts covered by the PRange. + @return The created range. + */ RangePtr derive() const; private: @@ -133,6 +225,11 @@ namespace CNORXZ Vector mParts; }; + /** Create a PRange. + Internally calls PRangeFactory. + @param range Range to create a PRange on. + @param parts Integer vector indicating the parts w.r.t. input range to be covered by the PRange. + */ template RangePtr prange(const Sptr& range, const Vector& parts); -- 2.45.2 From 21a11eb4f633b39e52407ff63bed23aa5c80ba5f Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Fri, 19 Jan 2024 23:32:41 +0100 Subject: [PATCH 08/32] ranges: add remaining doxy comments --- src/include/ranges/srange.h | 114 ++++++++++++++++++++++++++-- src/include/ranges/urange.h | 113 +++++++++++++++++++++++---- src/include/ranges/xindex.h | 147 ++++++++++++++++++++++++++++++------ 3 files changed, 326 insertions(+), 48 deletions(-) diff --git a/src/include/ranges/srange.h b/src/include/ranges/srange.h index ea18367..5cf1661 100644 --- a/src/include/ranges/srange.h +++ b/src/include/ranges/srange.h @@ -2,10 +2,9 @@ /** @file include/ranges/srange.h - @brief ... + @brief SRange, SRangeFactory and SIndex declaration. - - Copyright (c) 2022 Christian Zimmermann. All rights reserved. + Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de **/ @@ -20,6 +19,11 @@ namespace CNORXZ { + /** **** + Specific index for SRange. + @tparam MetaT Meta data type. + @tparam S Static size of the range. + */ template class SIndex : public IndexInterface,MetaT> { @@ -29,43 +33,88 @@ namespace CNORXZ typedef MetaT MetaType; INDEX_RANDOM_ACCESS_ITERATOR_DEFS(MetaType); - DEFAULT_MEMBERS(SIndex); + DEFAULT_MEMBERS(SIndex); /**< default constructors and assignments */ + + /** Construct index from range and position. + @param range Range to iterate over. + @param pos lexicographic position. + */ SIndex(const RangePtr& range, SizeT pos = 0); + /** @copydoc IndexInterface::operator=(SizeT) */ SIndex& operator=(SizeT lexpos); + + /** @copydoc IndexInterface::operator++() */ SIndex& operator++(); + + /** @copydoc IndexInterface::operator--() */ SIndex& operator--(); + + /** @copydoc IndexInterface::operator+() */ SIndex operator+(Int n) const; + + /** @copydoc IndexInterface::operator-() */ SIndex operator-(Int n) const; + + /** @copydoc IndexInterface::operator-(SIndex) */ SizeT operator-(const SIndex& i) const; + + /** @copydoc IndexInterface::operator+=() */ SIndex& operator+=(Int n); + + /** @copydoc IndexInterface::operator-=() */ SIndex& operator-=(Int n); + /** @copydoc IndexInterface::lex() */ SizeT lex() const; + + /** @copydoc IndexInterface::pmax() */ SPos pmax() const; + + /** @copydoc IndexInterface::lmax() */ SPos lmax() const; + + /** @copydoc IndexInterface::id() */ IndexId<0> id() const; + /** @copydoc IndexInterface::operator*() */ const MetaT& operator*() const; + /** @copydoc IndexInterface::dim() */ SizeT dim() const; // = 1 + + /** @copydoc IndexInterface::range() */ Sptr range() const; + /** @copydoc IndexInterface::stepSize() */ template UPos stepSize(const IndexId& id) const; + /** @copydoc IndexInterface::stringMeta() */ String stringMeta() const; - const MetaT& meta() const; - SIndex& at(const MetaT& metaPos); - decltype(auto) xpr(const Sptr>& _this) const; + /** @copydoc IndexInterface::meta() */ + const MetaT& meta() const; + + /** @copydoc IndexInterface::at() */ + SIndex& at(const MetaT& metaPos); + + /** @copydoc IndexInterface::prange() */ RangePtr prange(const SIndex& last) const; + + /** @copydoc IndexInterface::deepFormat() */ SizeT deepFormat() const; + + /** @copydoc IndexInterface::deepMax() */ SizeT deepMax() const; /** @copydoc IndexInterface::reformat() */ SIndex& reformat(const Vector& f, const Vector& s); + /** @copydoc IndexInterface::ifor() */ + decltype(auto) xpr(const Sptr>& _this) const; + + /** @copydoc IndexInterface::ifor() */ template decltype(auto) ifor(const Xpr& xpr, F&& f) const; @@ -77,16 +126,43 @@ namespace CNORXZ const MetaT* mMetaPtr; }; + /** Make index pack of a SIndex and another index. + @param a pointer to SIndex. + @param b pointer to another index. + */ template decltype(auto) operator*(const Sptr>& a, const Sptr& b); + /** **** + Specific factory for SRange. + @tparam MetaT Meta data type. + @tparam S Static size of the range. + */ template class SRangeFactory : public RangeFactoryBase { public: + + /** Construct and setup factory. + @param space Meta data array defining the range. + */ SRangeFactory(const Arr& space); + + /** Construct and setup factory. + @param space Meta data array defining the range (move). + */ SRangeFactory(Arr&& space); + + /** Construct and setup factory. + @param space Meta data array defining the range. + @param ref Range the range to be constructed is related to. + */ SRangeFactory(const Arr& space, const RangePtr& ref); + + /** Construct and setup factory. + @param space Meta data array defining the range (move). + @param ref Range the range to be constructed is related to. + */ SRangeFactory(Arr&& space, const RangePtr& ref); private: @@ -97,6 +173,12 @@ namespace CNORXZ RangePtr mRef; }; + /** **** + Static size range. + The same as URange, but the range size is compile-time fixed. + @tparam MetaT Meta data type. + @tparam S Static range size. + */ template class SRange : public RangeInterface> { @@ -114,8 +196,19 @@ namespace CNORXZ virtual const TypeInfo& metaType() const override final; virtual RangePtr extend(const RangePtr& r) const override final; + /** return meta data at given position. + @param pos position, size type + */ const MetaT& get(SizeT pos) const; + + /** Get underlying meta data array. + @return Pointer to first position of meta data array. + */ const MetaT* get() const; + + /** return position for given meta data. + @param metaPos meta data, size type + */ SizeT getMeta(const MetaT& metaPos) const; private: @@ -132,9 +225,16 @@ namespace CNORXZ SERIALIZATION_FUNCTIONS_NOPUB; }; + /** *** + Specialize RangeCast for casts to SRange + @see RangeCast + @tparam MetaT Meta data type. + @tparam S Static range size. + */ template struct RangeCast> { + /** cast the range */ static Sptr> func(const RangePtr& r); }; diff --git a/src/include/ranges/urange.h b/src/include/ranges/urange.h index 8357503..c046081 100644 --- a/src/include/ranges/urange.h +++ b/src/include/ranges/urange.h @@ -2,10 +2,9 @@ /** @file include/ranges/urange.h - @brief ... + @brief URange, URangeFactory and UIndex declaration. - - Copyright (c) 2022 Christian Zimmermann. All rights reserved. + Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de **/ @@ -23,6 +22,10 @@ namespace CNORXZ { + /** **** + Specific index for URange. + @tparam MetaT Meta data type. + */ template class UIndex : public IndexInterface,MetaT> { @@ -33,51 +36,94 @@ namespace CNORXZ typedef MetaT MetaType; INDEX_RANDOM_ACCESS_ITERATOR_DEFS(MetaType); - DEFAULT_MEMBERS(UIndex); + DEFAULT_MEMBERS(UIndex); /**< default constructors and assignments */ + + /** Construct index from range and position. + @param range Range to iterate over. + @param pos lexicographic position. + */ UIndex(const RangePtr& range, SizeT pos = 0); + /** @copydoc IndexInterface::operator=(SizeT) */ UIndex& operator=(SizeT lexpos); + + /** @copydoc IndexInterface::operator++() */ UIndex& operator++(); + + /** @copydoc IndexInterface::operator--() */ UIndex& operator--(); + + /** @copydoc IndexInterface::operator+() */ UIndex operator+(Int n) const; + + /** @copydoc IndexInterface::operator-() */ UIndex operator-(Int n) const; + + /** @copydoc IndexInterface::operator-(UIndex) */ SizeT operator-(const UIndex& i) const; + + /** @copydoc IndexInterface::operator+=() */ UIndex& operator+=(Int n); + + /** @copydoc IndexInterface::operator-=() */ UIndex& operator-=(Int n); + /** @copydoc IndexInterface::lex() */ SizeT lex() const; + + /** @copydoc IndexInterface::pmax() */ UPos pmax() const; + + /** @copydoc IndexInterface::lmax() */ UPos lmax() const; + + /** @copydoc IndexInterface::id() */ IndexId<0> id() const; + /** @copydoc IndexInterface::operator*() */ const MetaT& operator*() const; + /** @copydoc IndexInterface::dim() */ SizeT dim() const; // = 1 + + /** @copydoc IndexInterface::range() */ Sptr range() const; + /** @copydoc IndexInterface::stepSize() */ template decltype(auto) stepSize(const IndexId& id) const; + /** @copydoc IndexInterface::stringMeta() */ String stringMeta() const; - const MetaT& meta() const; - UIndex& at(const MetaT& metaPos); - decltype(auto) xpr(const Sptr>& _this) const; + /** @copydoc IndexInterface::meta() */ + const MetaT& meta() const; + + /** @copydoc IndexInterface::at() */ + UIndex& at(const MetaT& metaPos); + + /** @copydoc IndexInterface::prange() */ RangePtr prange(const UIndex& last) const; + /** @copydoc IndexInterface::deepFormat() */ SizeT deepFormat() const; + /** @copydoc IndexInterface::deepMax() */ SizeT deepMax() const; - + /** @copydoc IndexInterface::reformat() */ UIndex& reformat(const Vector& f, const Vector& s); + /** @copydoc IndexInterface::ifor() */ template decltype(auto) ifor(const Xpr& xpr, F&& f) const; /** @copydoc IndexInterface::formatIsTrivial() */ bool formatIsTrivial() const; + /** @copydoc IndexInterface::xpr() */ + decltype(auto) xpr(const Sptr>& _this) const; + private: Sptr mRangePtr; const MetaT* mMetaPtr; @@ -86,26 +132,39 @@ namespace CNORXZ template void swap(UIndex& a, UIndex& b) { a.swap(b); } - template - decltype(auto) operator*(const Sptr>& a, const Sptr& b); + /** Make index pack of a UIndex and another index. + @param a pointer to UIndex. + @param b pointer to another index. + */ + template + decltype(auto) operator*(const Sptr>& a, const Sptr& b); - template + /** **** + Specific factory for URange. + @tparam MetaT Meta data type. + */ + template class URangeFactory : public RangeFactoryBase { public: - URangeFactory(const Vector& space); - URangeFactory(Vector&& space); - URangeFactory(const Vector& space, const RangePtr& ref); - URangeFactory(Vector&& space, const RangePtr& ref); + URangeFactory(const Vector& space); + URangeFactory(Vector&& space); + URangeFactory(const Vector& space, const RangePtr& ref); + URangeFactory(Vector&& space, const RangePtr& ref); private: URangeFactory() = default; virtual void make() override final; - Vector mSpace; + Vector mSpace; RangePtr mRef; }; + /** **** + Uni-(1-)dimensional range with non-trivial meta data space + i.e. the parameter space can be arbitrary. + @tparam MetaT Meta data type. + */ template class URange : public RangeInterface> { @@ -123,8 +182,21 @@ namespace CNORXZ virtual const TypeInfo& metaType() const override final; virtual RangePtr extend(const RangePtr& r) const override final; + /** Get meta data at given range position. + @param pos Integer indicating requested position. + @return Meta data at given postion. + */ const MetaType& get(SizeT pos) const; + + /** Get meta data array. + @return Pointer to first element of the underlying meta data array. + */ const MetaType* get() const; + + /** Get range position for given meta data. + @param metaPos Meta data. + @return Position of the given meta data if it is contained by the range. + */ SizeT getMeta(const MetaType& metaPos) const; private: @@ -141,12 +213,21 @@ namespace CNORXZ SERIALIZATION_FUNCTIONS_NOPUB; }; + /** *** + Specialize RangeCast for casts to URange + @see RangeCast + */ template struct RangeCast> { + /** cast the range */ static Sptr> func(const RangePtr& r); }; + /** Create an URange, calls URangeFactory. + @param space Meta data space to create an URange on. + @return Created range. + */ template RangePtr urange(const Vector& space); diff --git a/src/include/ranges/xindex.h b/src/include/ranges/xindex.h index bbb9411..e8d3000 100644 --- a/src/include/ranges/xindex.h +++ b/src/include/ranges/xindex.h @@ -2,10 +2,9 @@ /** @file include/ranges/xindex.h - @brief ... + @brief XIndexBase and XIndex template declaration. - - Copyright (c) 2022 Christian Zimmermann. All rights reserved. + Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de **/ @@ -20,66 +19,145 @@ namespace CNORXZ { + /** **** + Abstract index wrapper base. + Can be used for index polymorphism. + Only use if absolutely necessary, indices should always reveal as much as possible + to the compiler! + */ class XIndexBase { public: - //typedef DType MetaType; - - DEFAULT_MEMBERS(XIndexBase); + DEFAULT_MEMBERS(XIndexBase); /**< default constructors and assignments */ + + /** Virtual default destructor */ virtual ~XIndexBase() = default; + + /** Copy this index + @return Pointer to the copy. + */ virtual XIndexPtr copy() const = 0; + + /** Current position. + @return Current position. + */ virtual SizeT pos() const = 0; + /** @copydoc IndexInterface::operator=(SizeT) */ virtual XIndexBase& operator=(SizeT lexpos) = 0; + + /** @copydoc IndexInterface::operator++() */ virtual XIndexBase& operator++() = 0; + + /** @copydoc IndexInterface::operator--() */ virtual XIndexBase& operator--() = 0; + + /** @copydoc IndexInterface::operator+() */ virtual XIndexPtr operator+(Int n) const = 0; + + /** @copydoc IndexInterface::operator-() */ virtual XIndexPtr operator-(Int n) const = 0; + + /** @copydoc IndexInterface::operator-(UIndex) */ virtual SizeT operator-(const XIndexBase& i) const = 0; + + /** @copydoc IndexInterface::operator+=() */ virtual XIndexBase& operator+=(Int n) = 0; + + /** @copydoc IndexInterface::operator-=() */ virtual XIndexBase& operator-=(Int n) = 0; + /** @copydoc IndexInterface::lex() */ virtual SizeT lex() const = 0; + + /** @copydoc IndexInterface::pmax() */ virtual UPos pmax() const = 0; + + /** @copydoc IndexInterface::lmax() */ virtual UPos lmax() const = 0; + + /** @copydoc IndexInterface::id() */ virtual IndexId<0> id() const = 0; + /** @copydoc IndexInterface::operator*() */ virtual DType operator*() const = 0; + /** @copydoc IndexInterface::dim() */ virtual SizeT dim() const = 0; - virtual RangePtr range() const = 0; - virtual UPos stepSize(const IndexId<0>& id) const = 0; - virtual RangePtr prange(const XIndexPtr& last) const = 0; - virtual Vector deepFormat() const = 0; - virtual Vector deepMax() const = 0; - virtual XIndexBase& reformat(const Vector& f, const Vector& s) = 0; + /** @copydoc IndexInterface::range() */ + virtual RangePtr range() const = 0; + + /** @copydoc IndexInterface::stepSize() */ + virtual UPos stepSize(const IndexId<0>& id) const = 0; + + /** @copydoc IndexInterface::stringMeta() */ virtual String stringMeta() const = 0; + + /** @copydoc IndexInterface::meta() */ virtual DType meta() const = 0; + + /** @copydoc IndexInterface::at() */ virtual XIndexBase& at(const DType& meta) = 0; + /** @copydoc IndexInterface::prange() */ + virtual RangePtr prange(const XIndexPtr& last) const = 0; + + /** @copydoc IndexInterface::deepFormat() */ + virtual Vector deepFormat() const = 0; + + /** @copydoc IndexInterface::deepMax() */ + virtual Vector deepMax() const = 0; + + /** @copydoc IndexInterface::reformat() */ + virtual XIndexBase& reformat(const Vector& f, const Vector& s) = 0; + + /** @copydoc IndexInterface::ifor() */ virtual DXpr ifor(const DXpr& xpr, NoF&& f) const = 0; + /** @copydoc IndexInterface::formatIsTrivial() */ virtual bool formatIsTrivial() const = 0; }; - //Sptr& operator++(Sptr& i); - //Sptr& operator--(Sptr& i); - - // MultiIndex Wrapper: + /** **** + Index Wrapper. + @tparam Index Type of index to be wrapped. + @tparam Meta Meta data type of wrapped index. + */ template class XIndex : public XIndexBase { public: - DEFAULT_C(XIndex); - // no default copy/assignment (have to copy objects in shared ptr) + DEFAULT_C(XIndex); /** < default constructor. */ + + /** Copy constructor. + No default: have to copy objects in shared ptr. + */ XIndex(const XIndex& i); + + /** Move constructor. + */ XIndex(XIndex&& i); + + /** Copy assignment. + No default: have to copy objects in shared ptr. + */ XIndex& operator=(const XIndex& i); + + /** Move assignment. + */ XIndex& operator=(XIndex&& i); + + /** Construct. + @param i Pointer to index to be wrapped. + */ XIndex(const IndexPtr& i); + + /** Construct. + @param i Index to be wrapped. + */ XIndex(const IndexInterface& i); virtual XIndexPtr copy() const override final; @@ -104,20 +182,24 @@ namespace CNORXZ virtual SizeT dim() const override final; virtual RangePtr range() const override final; virtual UPos stepSize(const IndexId<0>& id) const override final; + virtual String stringMeta() const override final; + virtual DType meta() const override final; + virtual XIndexBase& at(const DType& meta) override final; virtual RangePtr prange(const XIndexPtr& last) const override final; virtual Vector deepFormat() const override final; virtual Vector deepMax() const override final; virtual XIndex& reformat(const Vector& f, const Vector& s) override final; - - virtual String stringMeta() const override final; - virtual DType meta() const override final; - virtual XIndexBase& at(const DType& meta) override final; - virtual DXpr ifor(const DXpr& xpr, NoF&& f) const override final; - virtual bool formatIsTrivial() const override final; + /** Get underlying index instance. + @return Reference to index. + */ Index& get(); + + /** Get underlying index instance (const). + @return Reference to index. + */ const Index& get() const; private: @@ -125,13 +207,28 @@ namespace CNORXZ }; + /** **** + Specialization: has_sub for XIndexBase. + XIndexBase can have sub-indices. + @see has_sub. + */ template <> struct has_sub { static constexpr bool value = true; }; - + + /** Create XIndex pointer. + @param i Index to be wrapped. + @return Pointer to created index wrapper. + */ template inline XIndexPtr xindexPtr(const Sptr& i); + /** Specialization of xindexPtr(). + If input index type is already a XIndex, the corresponding pointer is just passed. + This is to avoid unwanted chains of index wrappers. + @param i Input index. + @return i. + */ template <> inline XIndexPtr xindexPtr(const Sptr& i); -- 2.45.2 From 91f9a3c957fafefd49937e108344387891c7ef2e Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Sat, 20 Jan 2024 21:33:40 +0100 Subject: [PATCH 09/32] prange: add missing extend() implementation --- src/include/ranges/prange.cc.h | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/include/ranges/prange.cc.h b/src/include/ranges/prange.cc.h index ce687ee..9d2b47e 100644 --- a/src/include/ranges/prange.cc.h +++ b/src/include/ranges/prange.cc.h @@ -338,10 +338,19 @@ namespace CNORXZ template RangePtr PRange::extend(const RangePtr& r) const { - CXZ_ERROR("implement!!!"); - // if r is PRange of same Range, then just add parts - // else derive and add meta of r - return nullptr; + if(r->type() == type()){ + Sptr> rx = std::dynamic_pointer_cast>(r); + if(rx->orig() == orig()){ + Vector p(parts()); + for(auto i: rx->parts()){ + if(find(p.begin(), p.end(), i) != p.end()){ + p.push_back(i); + } + } + return prange(orig(), p); + } + } + return derive()->extend(r); } template @@ -361,8 +370,9 @@ namespace CNORXZ { Vector meta(this->size()); auto i = mRange->begin(); + SizeT j = 0; for(const auto& p: mParts){ - meta = *(i = p); + meta[j++] = *(i = p); } return URangeFactory( meta ).create(); } -- 2.45.2 From 0bfee171e06441bb94c02fe292d5f0f086224341 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Sat, 20 Jan 2024 22:38:15 +0100 Subject: [PATCH 10/32] add POpRoot class + prange: implement xpr() --- src/include/operation/op_types.cc.h | 91 +++++++++++++++++++++-------- src/include/operation/op_types.h | 30 ++++++++++ src/include/ranges/prange.cc.h | 51 ++++++++-------- 3 files changed, 121 insertions(+), 51 deletions(-) diff --git a/src/include/operation/op_types.cc.h b/src/include/operation/op_types.cc.h index ad673a1..1dc514f 100644 --- a/src/include/operation/op_types.cc.h +++ b/src/include/operation/op_types.cc.h @@ -10,9 +10,9 @@ namespace CNORXZ { - /********************** - * COpInterface * - **********************/ + /*====================+ + | COpInterface | + +====================*/ template template @@ -37,9 +37,9 @@ namespace CNORXZ } - /********************* - * OpInterface * - *********************/ + /*===================+ + | OpInterface | + +===================*/ template template @@ -56,9 +56,9 @@ namespace CNORXZ } - /*************** - * COpRoot * - ***************/ + /*=============+ + | COpRoot | + +=============*/ template constexpr COpRoot::COpRoot(const CArrayBase& a, const Sptr& ind) : @@ -123,9 +123,52 @@ namespace CNORXZ return COpRoot(a, ind); } - /**************** - * OpCont * - ****************/ + /*=============+ + | POpRoot | + +=============*/ + + template + constexpr POpRoot::POpRoot(const Sptr& ind, const SizeT* parts, Op&& op) : + mIndex(ind), + mFp(1,parts), + mOp(std::forward(op)) + {} + + template + template + constexpr decltype(auto) POpRoot::operator()(const PosT& pos) const + { + return mOp(mFp(pos)); + } + + template + constexpr decltype(auto) POpRoot::operator()() const + { + return mOp(mFp(SPos<0>())); + } + + template + template + constexpr decltype(auto) POpRoot::rootSteps(const IndexId& id) const + { + return mIndex->stepSize(id); + } + + template + constexpr decltype(auto) POpRoot::data() const + { + return mOp->data(); + } + + template + constexpr decltype(auto) poproot(const Sptr& ind, const SizeT* parts, Op&& op) + { + return POpRoot(ind, parts, std::forward(op)); + } + + /*==============+ + | OpCont | + +==============*/ template constexpr OpCont::OpCont(const Sptr& ind) : @@ -236,9 +279,9 @@ namespace CNORXZ return mC.data(); } - /**************** - * OpRoot * - ****************/ + /*==============+ + | OpRoot | + +==============*/ template constexpr OpRoot::OpRoot(ArrayBase& a, const Sptr& ind) : @@ -327,9 +370,9 @@ namespace CNORXZ return OpRoot(a, ind); } - /******************* - * Operation * - *******************/ + /*=================+ + | Operation | + +=================*/ template constexpr Operation::Operation(F&& f, const Ops&... ops) : @@ -411,9 +454,9 @@ namespace CNORXZ } } - /********************* - * Contraction * - *********************/ + /*===================+ + | Contraction | + +===================*/ template constexpr Contraction::Contraction(CXpr&& cxpr) : @@ -447,9 +490,9 @@ namespace CNORXZ return Contraction( i->ifor( op, f ) ); } - /************************ - * various functions * - ************************/ + /*======================+ + | various functions | + +======================*/ template constexpr decltype(auto) indexOp(const Sptr& i) diff --git a/src/include/operation/op_types.h b/src/include/operation/op_types.h index 2b76f28..ce0fa4a 100644 --- a/src/include/operation/op_types.h +++ b/src/include/operation/op_types.h @@ -97,6 +97,36 @@ namespace CNORXZ template constexpr decltype(auto) coproot(const T* a, const Sptr& ind); + template + class POpRoot : public COpInterface> + { + public: + typedef COpInterface> OI; + + constexpr POpRoot() = default; + + constexpr POpRoot(const Sptr& ind, const SizeT* parts, Op&& op); + + template + constexpr decltype(auto) operator()(const PosT& pos) const; + + constexpr decltype(auto) operator()() const; + + template + constexpr decltype(auto) rootSteps(const IndexId& id) const; + + constexpr decltype(auto) data() const; + + private: + + Sptr mIndex; + FPos mFp; + Op mOp; + }; + + template + constexpr decltype(auto) poproot(const Sptr& ind, const SizeT* parts, Op&& op); + template class OpCont : public OpInterface> { diff --git a/src/include/ranges/prange.cc.h b/src/include/ranges/prange.cc.h index 9d2b47e..96a24e0 100644 --- a/src/include/ranges/prange.cc.h +++ b/src/include/ranges/prange.cc.h @@ -2,10 +2,9 @@ /** @file include/ranges/prange.cc.h - @brief ... + @brief PRange, PRangeFactory and PIndex implementations. - - Copyright (c) 2022 Christian Zimmermann. All rights reserved. + Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de **/ @@ -18,9 +17,9 @@ namespace CNORXZ { - /************** - * PIndex * - **************/ + /*============+ + | PIndex | + +============*/ template PIndex::PIndex(const RangePtr& range, SizeT pos) : @@ -206,9 +205,7 @@ namespace CNORXZ template decltype(auto) PIndex::xpr(const Sptr>& _this) const { - CXZ_ERROR("implement!!!"); - //return poperation( _this, mOrig, mRangePtr->parts(), mOrig->xpr(mOrig) ); - return mOrig->xpr(mOrig); + return poproot( _this, mRangePtr->parts(), mOrig->xpr(mOrig) ); } template @@ -246,9 +243,9 @@ namespace CNORXZ return mOrig; } - /************************ - * PIndex (private) * - ************************/ + /*======================+ + | PIndex (private) | + +======================*/ template void PIndex::mkPos() @@ -264,9 +261,9 @@ namespace CNORXZ CXZ_ERROR("meta position '" << toString(mOrig->meta()) << "' not part of range"); } - /*************************** - * PIndex (non-member) * - ***************************/ + /*=========================+ + | PIndex (non-member) | + +=========================*/ template decltype(auto) operator*(const Sptr>& a, const Sptr& b) @@ -274,9 +271,9 @@ namespace CNORXZ return iptrMul(a, b); } - /********************* - * PRangeFactory * - *********************/ + /*===================+ + | PRangeFactory | + +===================*/ template PRangeFactory::PRangeFactory(const Sptr& range, const Vector& _parts) : @@ -301,9 +298,9 @@ namespace CNORXZ } } - /************** - * PRange * - **************/ + /*============+ + | PRange | + +============*/ template SizeT PRange::size() const @@ -378,9 +375,9 @@ namespace CNORXZ } - /************************ - * PRange (private) * - ************************/ + /*======================+ + | PRange (private) | + +======================*/ template PRange::PRange(const Sptr& range, const Vector& _parts) : @@ -396,9 +393,9 @@ namespace CNORXZ return Vector { mRange->id() }; } - /**************************** - * non-member functions * - ****************************/ + /*==========================+ + | non-member functions | + +==========================*/ template RangePtr prange(const Sptr& range, const Vector& parts) -- 2.45.2 From f77132ddfe7eaeb3b268cab341a25be8bd27ab84 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Mon, 22 Jan 2024 00:03:08 +0100 Subject: [PATCH 11/32] hdf5: fix group mkCont + improve error handling of urange/yrange at() --- src/include/ranges/urange.cc.h | 4 +++- src/lib/ranges/yrange.cc | 7 ++++--- src/opt/hdf5/lib/h5_group.cc | 18 +++++++++++++++--- src/opt/hdf5/lib/h5_table.cc | 5 +++++ src/opt/hdf5/tests/h5_basic_unit_test.cc | 8 ++++++++ 5 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/include/ranges/urange.cc.h b/src/include/ranges/urange.cc.h index 52b5dce..39b4b1b 100644 --- a/src/include/ranges/urange.cc.h +++ b/src/include/ranges/urange.cc.h @@ -299,7 +299,9 @@ namespace CNORXZ { auto b = mSpace.begin(); auto e = mSpace.end(); - return std::lower_bound(b, e, meta, std::less()) - b; + auto i = std::lower_bound(b, e, meta, std::less()); + CXZ_ASSERT(*i == meta, "element with meta data = " << toString(meta) << " not in range"); + return i - b; } template diff --git a/src/lib/ranges/yrange.cc b/src/lib/ranges/yrange.cc index c71a5d7..c1a4d65 100644 --- a/src/lib/ranges/yrange.cc +++ b/src/lib/ranges/yrange.cc @@ -334,12 +334,13 @@ namespace CNORXZ YIndex& YIndex::at(const Vector& meta) { - assert(meta.size() == mIs.size()); - IB::mPos = 0; + CXZ_ASSERT(meta.size() == mIs.size(), "input meta size (" + << meta.size() << ") different from expected size (" + << mIs.size() << ")"); for(SizeT i = 0; i != mIs.size(); ++i){ mIs[i]->at(meta[i]); - IB::mPos += mIs[i]->pos() * mFormat[i].val(); } + mkPos(); return *this; } diff --git a/src/opt/hdf5/lib/h5_group.cc b/src/opt/hdf5/lib/h5_group.cc index e6d3b92..f89b0f4 100644 --- a/src/opt/hdf5/lib/h5_group.cc +++ b/src/opt/hdf5/lib/h5_group.cc @@ -85,6 +85,14 @@ namespace CNORXZ return std::dynamic_pointer_cast( group ); } + Sptr Group::getTable(const String& name) const + { + auto table = this->get(name); + CXZ_ASSERT(table->type() == ContentType::TABLE, + "element '" << name << "' is not of type TABLE"); + return std::dynamic_pointer_cast
( table ); + } + const MArray& Group::get() const { CXZ_ASSERT(this->isOpen(), "tried to get content of closed group"); @@ -118,10 +126,12 @@ namespace CNORXZ return 0; } - static bool isTable(hid_t id) + static bool isTable(hid_t loc_id, const char* name) { + const hid_t id = H5Dopen(loc_id, name, H5P_DEFAULT); if(not H5Aexists(id, "CLASS")){ return false; + H5Dclose(id); } hid_t attrid = H5Aopen(id, "CLASS", H5P_DEFAULT); const hid_t atype = H5Aget_type(attrid); @@ -130,6 +140,7 @@ namespace CNORXZ const herr_t ret = H5Aread(attrid, atype, buff.data()); H5Tclose(atype); H5Aclose(attrid); + H5Dclose(id); if(ret != 0){ return false; } @@ -149,7 +160,8 @@ namespace CNORXZ index(); H5O_info_t oinfo; #if H5_VERS_MINOR > 10 - H5Oget_info(id, &oinfo, H5O_INFO_BASIC); + //H5Oget_info(id, &oinfo, H5O_INFO_BASIC); + H5Oget_info_by_name(id, name, &oinfo, H5O_INFO_BASIC, H5P_DEFAULT); #else H5Oget_info(id, &oinfo); #endif @@ -159,7 +171,7 @@ namespace CNORXZ break; } case H5O_TYPE_DATASET: { - if(isTable(id)){ + if(isTable(id, name)){ *index = std::make_shared
(sname, icd->parent); } else { diff --git a/src/opt/hdf5/lib/h5_table.cc b/src/opt/hdf5/lib/h5_table.cc index d05e3de..f2ef07b 100644 --- a/src/opt/hdf5/lib/h5_table.cc +++ b/src/opt/hdf5/lib/h5_table.cc @@ -14,7 +14,11 @@ namespace CNORXZ hsize_t nrecords = 0; H5TBget_table_info(mParent->id(), mName.c_str(), &nfields, &nrecords); mRecords = CRangeFactory( nrecords ).create(); + Vector> fieldnames(nfields); Vector fieldsptr(nfields); + for(SizeT i = 0; i != nfields; ++i){ + fieldsptr[i] = fieldnames[i].data(); + } Vector offsets(nfields); Vector sizes(nfields); SizeT typesize = 0; @@ -71,6 +75,7 @@ namespace CNORXZ if(mId != 0){ H5Tclose(mType); H5Dclose(mId); + mId = 0; } return *this; } diff --git a/src/opt/hdf5/tests/h5_basic_unit_test.cc b/src/opt/hdf5/tests/h5_basic_unit_test.cc index 172c707..6a3114a 100644 --- a/src/opt/hdf5/tests/h5_basic_unit_test.cc +++ b/src/opt/hdf5/tests/h5_basic_unit_test.cc @@ -101,6 +101,14 @@ namespace h5f.open(); EXPECT_TRUE(h5f.ro()); EXPECT_EQ(h5f.get().size(), 2u); + + EXPECT_THROW(h5f.getGroup("gr0"), std::runtime_error); + auto gr1 = h5f.getGroup("gr1"); + gr1->open(); + auto tab = gr1->getTable("tab1"); + EXPECT_EQ(tab->fields()->size(), 3u); + EXPECT_EQ(tab->records()->size(), 5u); + h5f.close(); } } -- 2.45.2 From d5b42c9c00dc8ffb90f9eea6953d6b8d1fdcf056 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Tue, 23 Jan 2024 02:17:56 +0100 Subject: [PATCH 12/32] hdf5: group iterations --- src/opt/hdf5/include/h5_content_base.h | 7 +++++++ src/opt/hdf5/include/h5_group.cc.h | 17 +++++++++++++++++ src/opt/hdf5/include/h5_group.h | 19 +++++++++++++++++++ src/opt/hdf5/lib/h5_file.cc | 3 ++- src/opt/hdf5/tests/h5_basic_unit_test.cc | 19 +++++++++++++++---- 5 files changed, 60 insertions(+), 5 deletions(-) diff --git a/src/opt/hdf5/include/h5_content_base.h b/src/opt/hdf5/include/h5_content_base.h index dace4d6..d838ae8 100644 --- a/src/opt/hdf5/include/h5_content_base.h +++ b/src/opt/hdf5/include/h5_content_base.h @@ -38,6 +38,13 @@ namespace CNORXZ RangePtr range() const; hid_t id() const; inline bool isOpen() const { return mId != 0; } + + template + ContentBase& addAttribute(const String& name, const T& value); + DType getAttribute(const String& name) const; + bool isAttribute(const String& name) const; + String getAttributesAsString() const; + String getRecursiveAttributesAsString() const; // + all parent's attributes protected: String mName; diff --git a/src/opt/hdf5/include/h5_group.cc.h b/src/opt/hdf5/include/h5_group.cc.h index b1b1b50..033c2d2 100644 --- a/src/opt/hdf5/include/h5_group.cc.h +++ b/src/opt/hdf5/include/h5_group.cc.h @@ -3,6 +3,7 @@ #define __cxz_h5_group_cc_h__ #include "h5_group.h" +#include "xpr/for.h" namespace CNORXZ { @@ -54,6 +55,22 @@ namespace CNORXZ return *this; } + template + decltype(auto) Group::iter(F&& f) const + { + RangePtr gr = *mCont.range()->sub().begin(); + auto gi = std::make_shared>(gr); + return gi->ifor( operation(std::forward(f), mCont(gi)), NoF{} ); + } + + template + decltype(auto) Group::iterRecursive(F&& f) const + { + return iter( [&](const auto& c) { + f(c); + recursion(c, std::forward(f)); + }); + } } } diff --git a/src/opt/hdf5/include/h5_group.h b/src/opt/hdf5/include/h5_group.h index 571c752..28c6616 100644 --- a/src/opt/hdf5/include/h5_group.h +++ b/src/opt/hdf5/include/h5_group.h @@ -41,8 +41,27 @@ namespace CNORXZ template Group& addTable(const String& name, const ArrayBase>& data, const Vector& fnames); + + template + decltype(auto) iter(F&& f) const; + + template + decltype(auto) iterRecursive(F&& f) const; protected: + + template + static void recursion(const C& c, F&& f) + { + if(c->type() == ContentType::GROUP){ + auto cx = std::dynamic_pointer_cast(c); + cx->open(); + if(cx->get().range() != nullptr){ + cx->iterRecursive(std::forward(f))(); + } + } + } + MArray mCont; void mkCont(); diff --git a/src/opt/hdf5/lib/h5_file.cc b/src/opt/hdf5/lib/h5_file.cc index ec1452e..d1ba58c 100644 --- a/src/opt/hdf5/lib/h5_file.cc +++ b/src/opt/hdf5/lib/h5_file.cc @@ -67,7 +67,8 @@ namespace CNORXZ String File::path() const { - return String("/"); + //return String("/"); + return ""; } String File::filename() const diff --git a/src/opt/hdf5/tests/h5_basic_unit_test.cc b/src/opt/hdf5/tests/h5_basic_unit_test.cc index 6a3114a..a0f1ce8 100644 --- a/src/opt/hdf5/tests/h5_basic_unit_test.cc +++ b/src/opt/hdf5/tests/h5_basic_unit_test.cc @@ -39,7 +39,7 @@ namespace Group_Test() { mFileName = testh5file; - mGrps = { "gr1", "gr2" }; + //mGrps = { "gr1", "gr2", "foo", "bar", "moregroups" }; mFs = {"field1","second","real"}; Vector> v ( { {0, -6, 3.141}, @@ -53,7 +53,7 @@ namespace } String mFileName; - Vector mGrps; + //Vector mGrps; Vector mFs; MArray> mTabA; @@ -82,7 +82,15 @@ namespace h5f.open(); h5f.addGroup("gr1"); h5f.addGroup("gr2"); - EXPECT_EQ(h5f.get().size(), 2u); + h5f.addGroup("foo"); + h5f.addGroup("moregroups"); + h5f.addGroup("bar"); + h5f.getGroup("moregroups")->open().addGroup("evenmore"); + h5f.getGroup("moregroups")->getGroup("evenmore")->open().addGroup("we"); + h5f.getGroup("moregroups")->getGroup("evenmore")->addGroup("need"); + h5f.getGroup("moregroups")->getGroup("evenmore")->addGroup("more"); + h5f.getGroup("moregroups")->getGroup("evenmore")->addGroup("groups"); + EXPECT_EQ(h5f.get().size(), 5u); h5f.close(); } @@ -100,14 +108,17 @@ namespace File h5f(mFileName, true); h5f.open(); EXPECT_TRUE(h5f.ro()); - EXPECT_EQ(h5f.get().size(), 2u); + EXPECT_EQ(h5f.get().size(), 5u); EXPECT_THROW(h5f.getGroup("gr0"), std::runtime_error); auto gr1 = h5f.getGroup("gr1"); gr1->open(); auto tab = gr1->getTable("tab1"); + VCHECK(tab->path()); EXPECT_EQ(tab->fields()->size(), 3u); EXPECT_EQ(tab->records()->size(), 5u); + h5f.iter( [](const auto& c) { VCHECK(c->path()); } )(); + h5f.iterRecursive( [](const auto& c) { VCHECK(c->path()); } )(); h5f.close(); } } -- 2.45.2 From 035ae78785adbadfb4ce5238fff3f6c4e96334b9 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Thu, 25 Jan 2024 01:19:26 +0100 Subject: [PATCH 13/32] h5: simple attribute handling (to be tested) --- src/opt/hdf5/include/cnorxz_hdf5.cc.h | 1 + src/opt/hdf5/include/h5_content_base.cc.h | 98 +++++++++++++++++++++++ src/opt/hdf5/include/h5_content_base.h | 8 +- src/opt/hdf5/lib/h5_content_base.cc | 80 ++++++++++++++++++ 4 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 src/opt/hdf5/include/h5_content_base.cc.h diff --git a/src/opt/hdf5/include/cnorxz_hdf5.cc.h b/src/opt/hdf5/include/cnorxz_hdf5.cc.h index bdf3fae..c5cd636 100644 --- a/src/opt/hdf5/include/cnorxz_hdf5.cc.h +++ b/src/opt/hdf5/include/cnorxz_hdf5.cc.h @@ -1,4 +1,5 @@ +#include "h5_content_base.cc.h" #include "h5_type_id.cc.h" #include "h5_group.cc.h" #include "h5_table.cc.h" diff --git a/src/opt/hdf5/include/h5_content_base.cc.h b/src/opt/hdf5/include/h5_content_base.cc.h new file mode 100644 index 0000000..c4bfcd6 --- /dev/null +++ b/src/opt/hdf5/include/h5_content_base.cc.h @@ -0,0 +1,98 @@ + +#ifndef __cxz_h5_content_base_cc_h__ +#define __cxz_h5_content_base_cc_h__ + +#include "h5_content_base.h" + +namespace CNORXZ +{ + namespace hdf5 + { + namespace + { + + template + struct CreateAttribute + { + static inline herr_t write(hid_t id, const String& name, const T& v) + { + CXZ_ERROR("type " << typeid(v).name() << " not supported (name " + << name << ", id = " << id << ")"); + return 0; + } + }; + + template <> + struct CreateAttribute + { + static inline herr_t write(hid_t id, const String& name, const Int& v) + { + const hsize_t dim = 1; + const hid_t type_id = H5Tarray_create(H5T_NATIVE_INT, 1, &dim); + const hid_t space_id = H5Screate(H5S_SCALAR); + const hid_t attr_id = H5Acreate(id, name.c_str(), type_id, space_id, H5P_DEFAULT, + H5P_DEFAULT); + const herr_t err = H5Awrite(attr_id, type_id, &v); + H5Aclose(attr_id); + H5Sclose(space_id); + H5Tclose(type_id); + return err; + } + }; + + template <> + struct CreateAttribute + { + static inline herr_t write(hid_t id, const String& name, const Double& v) + { + const hsize_t dim = 1; + const hid_t type_id = H5Tarray_create(H5T_NATIVE_DOUBLE, 1, &dim); + const hid_t space_id = H5Screate(H5S_SCALAR); + const hid_t attr_id = H5Acreate(id, name.c_str(), type_id, space_id, H5P_DEFAULT, + H5P_DEFAULT); + const herr_t err = H5Awrite(attr_id, type_id, &v); + H5Aclose(attr_id); + H5Sclose(space_id); + H5Tclose(type_id); + return err; + } + }; + + template <> + struct CreateAttribute + { + static inline herr_t write(hid_t id, const String& name, const String& v) + { + const hsize_t len = v.size(); + const hid_t type_id = H5Tcreate(H5T_STRING, len); + const hid_t space_id = H5Screate(H5S_SCALAR); + const hid_t attr_id = H5Acreate(id, name.c_str(), type_id, space_id, H5P_DEFAULT, + H5P_DEFAULT); + const herr_t err = H5Awrite(attr_id, type_id, v.c_str()); + H5Aclose(attr_id); + H5Sclose(space_id); + H5Tclose(type_id); + return err; + } + }; + + + template + inline herr_t writeAttr(hid_t id, const String& name, const T& v, hid_t space_id) + { + return CreateAttribute::write(id, name, v, space_id); + } + } + + template + ContentBase& ContentBase::addAttribute(const String& name, const T& value) + { + const herr_t err = writeAttr(mId, name, value); + CXZ_ASSERT(err >= 0, "error while writing attribute " << name); + return *this; + } + + } +} + +#endif diff --git a/src/opt/hdf5/include/h5_content_base.h b/src/opt/hdf5/include/h5_content_base.h index d838ae8..0c21be4 100644 --- a/src/opt/hdf5/include/h5_content_base.h +++ b/src/opt/hdf5/include/h5_content_base.h @@ -10,6 +10,8 @@ namespace CNORXZ { namespace hdf5 { + // TODO: IO save error handling !!! + enum class ContentType { ATTR = 1, FILE = 2, @@ -42,9 +44,9 @@ namespace CNORXZ template ContentBase& addAttribute(const String& name, const T& value); DType getAttribute(const String& name) const; - bool isAttribute(const String& name) const; - String getAttributesAsString() const; - String getRecursiveAttributesAsString() const; // + all parent's attributes + bool attributeExists(const String& name) const; + Vector getAttributes() const; + Vector getRecursiveAttributes() const; // + all parent's attributes protected: String mName; diff --git a/src/opt/hdf5/lib/h5_content_base.cc b/src/opt/hdf5/lib/h5_content_base.cc index 0726f6c..3418a01 100644 --- a/src/opt/hdf5/lib/h5_content_base.cc +++ b/src/opt/hdf5/lib/h5_content_base.cc @@ -28,5 +28,85 @@ namespace CNORXZ { return mId; } + + DType ContentBase::getAttribute(const String& name) const + { + CXZ_ASSERT(attributeExists(name), "no attribute with name '" << name + << "' in object " << mName); + const hid_t attr = H5Aopen(mId, name.c_str(), H5P_DEFAULT); + CXZ_ASSERT(attr != H5I_INVALID_HID, "error while reading attribute " << name); + const hid_t atype_id = H5Aget_type(attr); + CXZ_ASSERT(atype_id != H5I_INVALID_HID, + "error while determining type of attribute " << name); + + const H5T_class_t tc = H5Tget_class(atype_id); + DType o; + switch(tc){ + case H5T_INTEGER: { + Int v; + H5Aread(attr, atype_id, &v); + o = DType(v); + break; + } + case H5T_FLOAT: { + Double v; + H5Aread(attr, atype_id, &v); + o = DType(v); + break; + } + case H5T_STRING: { + const SizeT asize = H5Tget_size(atype_id); + Vector v(asize); + H5Aread(attr, atype_id, v.data()); + o = DType( String( v.data() ) ); + break; + } + default: + CXZ_ERROR("attribute type id " << atype_id << " not supported"); + } + + H5Tclose(atype_id); + H5Aclose(attr); + return o; + } + + bool ContentBase::attributeExists(const String& name) const + { + CXZ_ASSERT(isOpen(), "tried to get attribute of closed object"); + const htri_t ret = H5Aexists(mId, name.c_str()); + CXZ_ASSERT(ret > 0, "error while reading attribute " << name); + return ret; + } + + Vector ContentBase::getAttributes() const + { + CXZ_ASSERT(isOpen(), "tried to get attribute of closed object"); + struct ItData + { + Vector d; + const ContentBase* _this; + } itdata; + itdata._this = this; + auto itop = [](hid_t loc_id, const char *attr_name, + const H5A_info_t* ainfo, void *op_data) + { + ItData* x = reinterpret_cast(op_data); + x->d.push_back( x->_this->getAttribute( String(attr_name) ) ); + return static_cast(0); + }; + H5Aiterate(id(), H5_INDEX_NAME, H5_ITER_NATIVE, nullptr, + itop, &itdata); + return itdata.d; + } + + Vector ContentBase::getRecursiveAttributes() const + { + Vector out = getAttributes(); + if(mParent){ + Vector par = mParent->getRecursiveAttributes(); + out.insert(out.begin(), par.begin(), par.end()); + } + return out; + } } } -- 2.45.2 From 30a3c0b7565f06cf4998b0bf7f841b50502b7bee Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Sun, 28 Jan 2024 01:57:52 +0100 Subject: [PATCH 14/32] fixes in to_string + h5 attributes --- src/include/base/to_string.cc.h | 46 +++++++++++++++++++---- src/include/base/to_string.h | 14 +++++++ src/opt/hdf5/include/h5_content_base.cc.h | 4 +- src/opt/hdf5/include/h5_content_base.h | 4 +- src/opt/hdf5/include/h5_group.cc.h | 17 +++++++++ src/opt/hdf5/include/h5_group.h | 8 +++- src/opt/hdf5/include/h5_table.h | 3 ++ src/opt/hdf5/lib/h5_content_base.cc | 43 +++++++++++++++++---- src/opt/hdf5/lib/h5_file.cc | 3 ++ src/opt/hdf5/lib/h5_group.cc | 16 ++++---- src/opt/hdf5/tests/h5_basic_unit_test.cc | 41 +++++++++++++++++--- 11 files changed, 165 insertions(+), 34 deletions(-) diff --git a/src/include/base/to_string.cc.h b/src/include/base/to_string.cc.h index 3d31a08..bf1f9d7 100644 --- a/src/include/base/to_string.cc.h +++ b/src/include/base/to_string.cc.h @@ -33,11 +33,16 @@ namespace CNORXZ { std::stringstream ss; ss << "["; - auto it = a.begin(); - for(; it != a.end()-1; ++it){ - ss << toString(*it) << ","; + if(a.size() == 0){ + ss << "]"; + } + else { + auto it = a.begin(); + for(; it != a.end()-1; ++it){ + ss << toString(*it) << ","; + } + ss << toString(*it) << "]"; } - ss << toString(*it) << "]"; return ss.str(); } @@ -46,11 +51,16 @@ namespace CNORXZ { std::stringstream ss; ss << "("; - auto it = a.begin(); - for(; it != a.end()-1; ++it){ - ss << toString(*it) << ","; + if constexpr(N == 0){ + ss << ")"; + } + else { + auto it = a.begin(); + for(; it != a.end()-1; ++it){ + ss << toString(*it) << ","; + } + ss << toString(*it) << ")"; } - ss << toString(*it) << ")"; return ss.str(); } @@ -73,6 +83,26 @@ namespace CNORXZ return String("(") + toString(p.first) + "," + toString(p.second) + ")"; } + template + String ToString>::func(const std::map& p) + { + std::stringstream ss; + ss << "{"; + if(p.size() == 0){ + ss << "}"; + } + else { + auto it = p.begin(); + auto e = p.end(); + e--; + for(; it != e; ++it){ + ss << toString(it->first) << ":" << toString(it->second) << ","; + } + ss << toString(it->first) << ":" << toString(it->second) << "}"; + } + return ss.str(); + } + template String toString(const T& a) { diff --git a/src/include/base/to_string.h b/src/include/base/to_string.h index c64711f..1b00959 100644 --- a/src/include/base/to_string.h +++ b/src/include/base/to_string.h @@ -97,6 +97,20 @@ namespace CNORXZ static String func(const std::pair& t); }; + /** *** + Specialization of ToString for maps + @tparam T key type + @tparam S value type + */ + template + struct ToString> + { + /** cast to string + @param a map to be casted + */ + static String func(const std::map& t); + }; + /** *** Specialization of ToString for DType */ diff --git a/src/opt/hdf5/include/h5_content_base.cc.h b/src/opt/hdf5/include/h5_content_base.cc.h index c4bfcd6..690f82e 100644 --- a/src/opt/hdf5/include/h5_content_base.cc.h +++ b/src/opt/hdf5/include/h5_content_base.cc.h @@ -78,9 +78,9 @@ namespace CNORXZ template - inline herr_t writeAttr(hid_t id, const String& name, const T& v, hid_t space_id) + inline herr_t writeAttr(hid_t id, const String& name, const T& v) { - return CreateAttribute::write(id, name, v, space_id); + return CreateAttribute::write(id, name, v); } } diff --git a/src/opt/hdf5/include/h5_content_base.h b/src/opt/hdf5/include/h5_content_base.h index 0c21be4..bdf346a 100644 --- a/src/opt/hdf5/include/h5_content_base.h +++ b/src/opt/hdf5/include/h5_content_base.h @@ -45,8 +45,8 @@ namespace CNORXZ ContentBase& addAttribute(const String& name, const T& value); DType getAttribute(const String& name) const; bool attributeExists(const String& name) const; - Vector getAttributes() const; - Vector getRecursiveAttributes() const; // + all parent's attributes + std::map getAttributes() const; + std::map getRecursiveAttributes() const; // + all parent's attributes protected: String mName; diff --git a/src/opt/hdf5/include/h5_group.cc.h b/src/opt/hdf5/include/h5_group.cc.h index 033c2d2..a50c64c 100644 --- a/src/opt/hdf5/include/h5_group.cc.h +++ b/src/opt/hdf5/include/h5_group.cc.h @@ -71,6 +71,23 @@ namespace CNORXZ recursion(c, std::forward(f)); }); } + + template + decltype(auto) Group::iter(F&& f) + { + RangePtr gr = *mCont.range()->sub().begin(); + auto gi = std::make_shared>(gr); + return gi->ifor( operation(std::forward(f), mCont(gi)), NoF{} ); + } + + template + decltype(auto) Group::iterRecursive(F&& f) + { + return iter( [&](const auto& c) { + f(c); + recursion(c, std::forward(f)); + }); + } } } diff --git a/src/opt/hdf5/include/h5_group.h b/src/opt/hdf5/include/h5_group.h index 28c6616..afee4b1 100644 --- a/src/opt/hdf5/include/h5_group.h +++ b/src/opt/hdf5/include/h5_group.h @@ -47,7 +47,13 @@ namespace CNORXZ template decltype(auto) iterRecursive(F&& f) const; - + + template + decltype(auto) iter(F&& f); + + template + decltype(auto) iterRecursive(F&& f); + protected: template diff --git a/src/opt/hdf5/include/h5_table.h b/src/opt/hdf5/include/h5_table.h index 603631d..8885230 100644 --- a/src/opt/hdf5/include/h5_table.h +++ b/src/opt/hdf5/include/h5_table.h @@ -50,10 +50,13 @@ namespace CNORXZ STable& appendRecord(const Tuple& t); STable& appendRecord(const MArray>& t); + + MArray> read(); template decltype(auto) iterRecords(F&& f) const; }; + } } diff --git a/src/opt/hdf5/lib/h5_content_base.cc b/src/opt/hdf5/lib/h5_content_base.cc index 3418a01..ce338a6 100644 --- a/src/opt/hdf5/lib/h5_content_base.cc +++ b/src/opt/hdf5/lib/h5_content_base.cc @@ -56,11 +56,37 @@ namespace CNORXZ } case H5T_STRING: { const SizeT asize = H5Tget_size(atype_id); - Vector v(asize); + Vector v(asize+1); H5Aread(attr, atype_id, v.data()); o = DType( String( v.data() ) ); break; } + case H5T_ARRAY: { + const int ndims = H5Tget_array_ndims(atype_id); + CXZ_ASSERT(ndims, "array of dimension " << ndims << " not supported"); + hsize_t dims; + H5Tget_array_dims(atype_id, &dims); + const hid_t att_id = H5Tget_super(atype_id); + const H5T_class_t atc = H5Tget_class(att_id); + switch(atc){ + case H5T_INTEGER: { + Vector v(dims); + H5Aread(attr, atype_id, v.data()); + o = DType(v); + break; + } + case H5T_FLOAT: { + Vector v(dims); + H5Aread(attr, atype_id, v.data()); + o = DType(v); + break; + } + default: + CXZ_ERROR("attribute type id " << atype_id << " not supported"); + } + H5Tclose(att_id); + break; + } default: CXZ_ERROR("attribute type id " << atype_id << " not supported"); } @@ -78,12 +104,12 @@ namespace CNORXZ return ret; } - Vector ContentBase::getAttributes() const + std::map ContentBase::getAttributes() const { CXZ_ASSERT(isOpen(), "tried to get attribute of closed object"); struct ItData { - Vector d; + std::map d; const ContentBase* _this; } itdata; itdata._this = this; @@ -91,7 +117,8 @@ namespace CNORXZ const H5A_info_t* ainfo, void *op_data) { ItData* x = reinterpret_cast(op_data); - x->d.push_back( x->_this->getAttribute( String(attr_name) ) ); + const String n = attr_name; + x->d[n] = x->_this->getAttribute( String(attr_name) ); return static_cast(0); }; H5Aiterate(id(), H5_INDEX_NAME, H5_ITER_NATIVE, nullptr, @@ -99,12 +126,12 @@ namespace CNORXZ return itdata.d; } - Vector ContentBase::getRecursiveAttributes() const + std::map ContentBase::getRecursiveAttributes() const { - Vector out = getAttributes(); + std::map out = getAttributes(); if(mParent){ - Vector par = mParent->getRecursiveAttributes(); - out.insert(out.begin(), par.begin(), par.end()); + std::map par = mParent->getRecursiveAttributes(); + out.insert(par.begin(), par.end()); } return out; } diff --git a/src/opt/hdf5/lib/h5_file.cc b/src/opt/hdf5/lib/h5_file.cc index d1ba58c..ea05180 100644 --- a/src/opt/hdf5/lib/h5_file.cc +++ b/src/opt/hdf5/lib/h5_file.cc @@ -28,6 +28,9 @@ namespace CNORXZ File& File::open() { + if(isOpen()){ + return *this; + } Int ex = this->exists(); const String fn = this->filename(); CXZ_ASSERT( ex != 2, "tried to open non-h5 file '" << fn << "'" ); diff --git a/src/opt/hdf5/lib/h5_group.cc b/src/opt/hdf5/lib/h5_group.cc index f89b0f4..bc4bbbe 100644 --- a/src/opt/hdf5/lib/h5_group.cc +++ b/src/opt/hdf5/lib/h5_group.cc @@ -30,14 +30,16 @@ namespace CNORXZ Group& Group::open() { - if(this->exists()){ - mId = H5Gopen( mParent->id(), mName.c_str(), H5P_DEFAULT ); + if(not isOpen()){ + if(this->exists()){ + mId = H5Gopen( mParent->id(), mName.c_str(), H5P_DEFAULT ); + } + else { + mId = H5Gcreate( mParent->id(), mName.c_str(), + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT ); + } + this->mkCont(); } - else { - mId = H5Gcreate( mParent->id(), mName.c_str(), - H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT ); - } - this->mkCont(); return *this; } diff --git a/src/opt/hdf5/tests/h5_basic_unit_test.cc b/src/opt/hdf5/tests/h5_basic_unit_test.cc index a0f1ce8..1221aaa 100644 --- a/src/opt/hdf5/tests/h5_basic_unit_test.cc +++ b/src/opt/hdf5/tests/h5_basic_unit_test.cc @@ -80,16 +80,27 @@ namespace File h5f(mFileName, false); EXPECT_FALSE(h5f.ro()); h5f.open(); + h5f.addAttribute("fprop", static_cast(3.141)); h5f.addGroup("gr1"); + h5f.getGroup("gr1")->addAttribute("att1", String("text")); h5f.addGroup("gr2"); h5f.addGroup("foo"); - h5f.addGroup("moregroups"); h5f.addGroup("bar"); - h5f.getGroup("moregroups")->open().addGroup("evenmore"); - h5f.getGroup("moregroups")->getGroup("evenmore")->open().addGroup("we"); - h5f.getGroup("moregroups")->getGroup("evenmore")->addGroup("need"); - h5f.getGroup("moregroups")->getGroup("evenmore")->addGroup("more"); - h5f.getGroup("moregroups")->getGroup("evenmore")->addGroup("groups"); + h5f.addGroup("moregroups"); + auto moregroups = h5f.getGroup("moregroups"); + moregroups->open().addGroup("evenmore"); + auto evenmore = moregroups->getGroup("evenmore"); + evenmore->open(); + evenmore->addAttribute("moreatt", static_cast(12)); + evenmore->addGroup("we"); + evenmore->getGroup("we")->addAttribute("wex", static_cast(9)); + evenmore->getGroup("we")->addAttribute("xy", String("xys")); + evenmore->addGroup("need"); + evenmore->getGroup("need")->addAttribute("wex", static_cast(7)); + evenmore->addGroup("more"); + evenmore->getGroup("more")->addAttribute("wex", static_cast(4)); + evenmore->addGroup("groups"); + evenmore->getGroup("groups")->addAttribute("wex", static_cast(2)); EXPECT_EQ(h5f.get().size(), 5u); h5f.close(); } @@ -100,6 +111,7 @@ namespace h5f.open(); h5f.getGroup("gr1")->open().addTable("tab1", mTabA, mFs); h5f.getGroup("gr2")->open().addTable("tab1", mTabA, mFs); + h5f.getGroup("moregroups")->open().getGroup("evenmore")->open().getGroup("need")->open().addTable("tab1", mTabA, mFs); h5f.close(); } @@ -114,13 +126,30 @@ namespace auto gr1 = h5f.getGroup("gr1"); gr1->open(); auto tab = gr1->getTable("tab1"); + DType att = gr1->getAttribute("att1"); + EXPECT_EQ(att.str(), "text"); VCHECK(tab->path()); EXPECT_EQ(tab->fields()->size(), 3u); EXPECT_EQ(tab->records()->size(), 5u); h5f.iter( [](const auto& c) { VCHECK(c->path()); } )(); h5f.iterRecursive( [](const auto& c) { VCHECK(c->path()); } )(); + h5f.iterRecursive( [](const auto& c) { c->open(); VCHECK(toString(c->getRecursiveAttributes())); } )(); + h5f.iterRecursive( [](const auto& c) { + if(c->type() == ContentType::TABLE) { c->open(); VCHECK(toString(c->getRecursiveAttributes())); } + } )(); h5f.close(); } + + TEST_F(Group_Test, ReadTable) + { + File h5f(mFileName, true); + h5f.open(); + auto tab = h5f.getGroup("gr1")->open().getTable("tab1"); + EXPECT_EQ(tab->fields()->size(), 3u); + EXPECT_EQ(tab->records()->size(), 5u); + + + } } // check write to new file -- 2.45.2 From b2637802be68d3e1b6b64644928847d3e73e4380 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Sun, 28 Jan 2024 16:42:11 +0100 Subject: [PATCH 15/32] changes in STable --- src/opt/hdf5/include/h5_group.cc.h | 11 ++-- src/opt/hdf5/include/h5_group.h | 2 +- src/opt/hdf5/include/h5_table.cc.h | 77 ++++++++++++++++-------- src/opt/hdf5/include/h5_table.h | 6 +- src/opt/hdf5/tests/h5_basic_unit_test.cc | 7 ++- 5 files changed, 68 insertions(+), 35 deletions(-) diff --git a/src/opt/hdf5/include/h5_group.cc.h b/src/opt/hdf5/include/h5_group.cc.h index a50c64c..f8f9971 100644 --- a/src/opt/hdf5/include/h5_group.cc.h +++ b/src/opt/hdf5/include/h5_group.cc.h @@ -14,19 +14,20 @@ namespace CNORXZ { auto i = this->getIndexTo(name); auto tab = std::dynamic_pointer_cast
( *i ); + CXZ_ASSERT(tab->type() == ContentType::TABLE, + "element '" << name << "' is not of type TABLE"); if(tab == nullptr){ auto stab = std::dynamic_pointer_cast>(*i); CXZ_ASSERT(stab != nullptr, "wrong format for table '" << name << "'"); return stab; } else { - const RangePtr fields = tab->fields(); (*i)->close(); - *i = std::make_shared>(name, this, fields); - return *i; + auto stab = std::make_shared>(name, this); + *i = stab; + return stab; } } - template Group& Group::addData(const String& name, const ArrayBase& data) @@ -38,7 +39,7 @@ namespace CNORXZ template Group& Group::addTable(const String& name, const ArrayBase>& data, - const Vector& fnames) + const Arr& fnames) { CXZ_ASSERT(this->isOpen(), "tried to extend closed group"); Vector nvec({name}); diff --git a/src/opt/hdf5/include/h5_group.h b/src/opt/hdf5/include/h5_group.h index afee4b1..babb886 100644 --- a/src/opt/hdf5/include/h5_group.h +++ b/src/opt/hdf5/include/h5_group.h @@ -40,7 +40,7 @@ namespace CNORXZ template Group& addTable(const String& name, const ArrayBase>& data, - const Vector& fnames); + const Arr& fnames); template decltype(auto) iter(F&& f) const; diff --git a/src/opt/hdf5/include/h5_table.cc.h b/src/opt/hdf5/include/h5_table.cc.h index 5e4c4d9..2072bd6 100644 --- a/src/opt/hdf5/include/h5_table.cc.h +++ b/src/opt/hdf5/include/h5_table.cc.h @@ -18,36 +18,13 @@ namespace CNORXZ } template - STable::STable(const String& name, const ContentBase* _parent, - const Vector& fnames) : + STable::STable(const String& name, const ContentBase* _parent) : Table(name, _parent) { constexpr SizeT N = sizeof...(Ts); if(mFields == nullptr){ - CXZ_ASSERT(fnames.size() != 0, "field names have to be initialized"); - Vector fields(fnames.size()); - for(SizeT i = 0; i != fields.size(); ++i){ - fields[i].first = i; - fields[i].second = fnames[i]; - } - mFields = URangeFactory(fields).create(); - } - CXZ_ASSERT(mFields->size() == sizeof...(Ts), "expected tuple of size = " << mFields->size() - << ", got: " << sizeof...(Ts)); - Tuple x; - if(mRecords == nullptr) { - mOffsets = MArray( mFields, iter<0,N> - ( [&](auto i) { return getTupleOffset(x, i); }, - [](const auto&... e) { return Vector({e...}); }) ); - mSizes = MArray( mFields, iter<0,N> - ( [&](auto i) { return sizeof(std::get(x)); }, - [](const auto&... e) { return Vector({e...}); }) ); - mTypes = MArray( mFields, iter<0,N> - ( [&](auto i) { return getTypeId(std::get(x)); }, - [](const auto&... e) { return Vector({e...}); }) ); - } - else { + Tuple x; iter<0,N>( [&](auto i) { CXZ_ASSERT ( getTupleOffset(x, i) == mOffsets.data()[i], "wrong offset for field " << i << ": " << getTupleOffset(x, i) @@ -61,6 +38,48 @@ namespace CNORXZ "wrong type for field " << i << ": " << getTypeId(std::get(x)) << " vs " << mTypes.data()[i] ); }, NoF{} ); } + else { + CXZ_ASSERT(mFields->size() == N, "expected tuple of size = " << mFields->size() + << ", got: " << N); + } + } + + template + STable::STable(const String& name, const ContentBase* _parent, + const Arr& fnames) : + Table(name, _parent) + { + initFields(fnames); + } + + template + STable& STable::initFields(const Arr& fnames) + { + constexpr SizeT N = sizeof...(Ts); + CXZ_ASSERT(mFields == nullptr and mRecords == nullptr, + "tried to initialize an existing table"); + + Vector fields(fnames.size()); + for(SizeT i = 0; i != fields.size(); ++i){ + fields[i].first = i; + fields[i].second = fnames[i]; + } + mFields = URangeFactory(fields).create(); + + Tuple x; + mOffsets = MArray + ( mFields, iter<0,N> + ( [&](auto i) { return getTupleOffset(x, i); }, + [](const auto&... e) { return Vector({e...}); }) ); + mSizes = MArray + ( mFields, iter<0,N> + ( [&](auto i) { return sizeof(std::get(x)); }, + [](const auto&... e) { return Vector({e...}); }) ); + mTypes = MArray + ( mFields, iter<0,N> + ( [&](auto i) { return getTypeId(std::get(x)); }, + [](const auto&... e) { return Vector({e...}); }) ); + return *this; } template @@ -75,6 +94,14 @@ namespace CNORXZ } return *this; } + + template + MArray> STable::read() + { + MArray> out(mRecords); + //...!!! + return out; + } } } diff --git a/src/opt/hdf5/include/h5_table.h b/src/opt/hdf5/include/h5_table.h index 8885230..6a9497b 100644 --- a/src/opt/hdf5/include/h5_table.h +++ b/src/opt/hdf5/include/h5_table.h @@ -46,8 +46,12 @@ namespace CNORXZ { public: DEFAULT_MEMBERS(STable); - STable(const String& name, const ContentBase* _parent, const Vector& fnames); + STable(const String& name, const ContentBase* _parent); + STable(const String& name, const ContentBase* _parent, + const Arr& fnames); + STable& initFields(const Arr& fnames); + STable& appendRecord(const Tuple& t); STable& appendRecord(const MArray>& t); diff --git a/src/opt/hdf5/tests/h5_basic_unit_test.cc b/src/opt/hdf5/tests/h5_basic_unit_test.cc index 1221aaa..603ba27 100644 --- a/src/opt/hdf5/tests/h5_basic_unit_test.cc +++ b/src/opt/hdf5/tests/h5_basic_unit_test.cc @@ -55,7 +55,7 @@ namespace String mFileName; //Vector mGrps; - Vector mFs; + Arr mFs; MArray> mTabA; }; @@ -142,12 +142,13 @@ namespace TEST_F(Group_Test, ReadTable) { + typedef Tuple RecType; + File h5f(mFileName, true); h5f.open(); - auto tab = h5f.getGroup("gr1")->open().getTable("tab1"); + auto tab = h5f.getGroup("gr1")->open().getTable("tab1", RecType()); EXPECT_EQ(tab->fields()->size(), 3u); EXPECT_EQ(tab->records()->size(), 5u); - } } -- 2.45.2 From e3ad04e56c82d4c287e384ae29b410938f71703e Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Sun, 28 Jan 2024 22:00:02 +0100 Subject: [PATCH 16/32] add table read functions --- src/opt/hdf5/include/h5_table.cc.h | 27 ++++++------ src/opt/hdf5/include/h5_table.h | 9 +++- src/opt/hdf5/include/h5_type_id.cc.h | 22 +++++++++- src/opt/hdf5/include/h5_type_id.h | 13 ++++++ src/opt/hdf5/lib/h5_table.cc | 56 +++++++++++++++++++++++- src/opt/hdf5/tests/h5_basic_unit_test.cc | 47 +++++++++++++++----- 6 files changed, 144 insertions(+), 30 deletions(-) diff --git a/src/opt/hdf5/include/h5_table.cc.h b/src/opt/hdf5/include/h5_table.cc.h index 2072bd6..1756994 100644 --- a/src/opt/hdf5/include/h5_table.cc.h +++ b/src/opt/hdf5/include/h5_table.cc.h @@ -22,26 +22,25 @@ namespace CNORXZ Table(name, _parent) { constexpr SizeT N = sizeof...(Ts); - if(mFields == nullptr){ + if(mFields != nullptr){ + CXZ_ASSERT(mFields->size() == N, "expected tuple of size = " << mFields->size() + << ", got: " << N); Tuple x; iter<0,N>( [&](auto i) { CXZ_ASSERT ( getTupleOffset(x, i) == mOffsets.data()[i], "wrong offset for field " << i << ": " << getTupleOffset(x, i) << " vs " << mOffsets.data()[i] ); }, NoF{} ); iter<0,N>( [&](auto i) { CXZ_ASSERT - ( sizeof(std::get(x)) == mSizes.data()[i], - "wrong size for field " << i << ": " << sizeof(std::get(x)) + ( sizeof(tget(x)) == mSizes.data()[i], + "wrong size for field " << i << ": " << sizeof(tget(x)) << " vs " << mSizes.data()[i] ); }, NoF{} ); iter<0,N>( [&](auto i) { CXZ_ASSERT - ( getTypeId(std::get(x)) == mTypes.data()[i], - "wrong type for field " << i << ": " << getTypeId(std::get(x)) + ( H5Tget_class(getTypeId(tget(x))) == mTypes.data()[i], + "wrong type for field " << i + << ": " << H5Tget_class(getTypeId(tget(x))) << " vs " << mTypes.data()[i] ); }, NoF{} ); } - else { - CXZ_ASSERT(mFields->size() == N, "expected tuple of size = " << mFields->size() - << ", got: " << N); - } } template @@ -73,11 +72,11 @@ namespace CNORXZ [](const auto&... e) { return Vector({e...}); }) ); mSizes = MArray ( mFields, iter<0,N> - ( [&](auto i) { return sizeof(std::get(x)); }, + ( [&](auto i) { return sizeof(tget(x)); }, [](const auto&... e) { return Vector({e...}); }) ); mTypes = MArray ( mFields, iter<0,N> - ( [&](auto i) { return getTypeId(std::get(x)); }, + ( [&](auto i) { return getTypeId(tget(x)); }, [](const auto&... e) { return Vector({e...}); }) ); return *this; } @@ -96,10 +95,12 @@ namespace CNORXZ } template - MArray> STable::read() + MArray> STable::read() const { + CXZ_ASSERT(isOpen(), "attempt to read table that has not been opened"); MArray> out(mRecords); - //...!!! + H5TBread_table(mParent->id(), mName.c_str(), mTypesize, mOffsets.data(), + mSizes.data(), out.data()); return out; } } diff --git a/src/opt/hdf5/include/h5_table.h b/src/opt/hdf5/include/h5_table.h index 6a9497b..16405b4 100644 --- a/src/opt/hdf5/include/h5_table.h +++ b/src/opt/hdf5/include/h5_table.h @@ -29,6 +29,8 @@ namespace CNORXZ Table& initTable(SizeT n, const void* data, SizeT dsize, SizeT chunk_size); Table& appendRecord(SizeT n, const void* data, SizeT dsize); Table& readRecord(SizeT pos, SizeT n, char* data); + MArray read() const; + const RangePtr& fields() const; const RangePtr& records() const; @@ -39,8 +41,11 @@ namespace CNORXZ MArray mOffsets; MArray mTypes; hid_t mType = 0; + SizeT mTypesize = 0; }; + // caution: the memory ordering is the only thing that counts; + // std::tuple has REVERSE ordering! template class STable : public Table { @@ -51,11 +56,11 @@ namespace CNORXZ const Arr& fnames); STable& initFields(const Arr& fnames); - + STable& appendRecord(const Tuple& t); STable& appendRecord(const MArray>& t); - MArray> read(); + MArray> read() const; template decltype(auto) iterRecords(F&& f) const; diff --git a/src/opt/hdf5/include/h5_type_id.cc.h b/src/opt/hdf5/include/h5_type_id.cc.h index ae646c1..65cdce4 100644 --- a/src/opt/hdf5/include/h5_type_id.cc.h +++ b/src/opt/hdf5/include/h5_type_id.cc.h @@ -8,11 +8,31 @@ namespace CNORXZ { namespace hdf5 { + template + decltype(auto) reverseTuple(const Tuple& t) + { + return iter<0,sizeof...(Ts)> + ( [&](auto i) { return tget(t); }, + [](const auto&... e) { return std::make_tuple(e...); } ); + } + + template + constexpr const auto& tget(const Tuple& t) + { + return std::get(t); + } + + template + constexpr auto& tget(Tuple& t) + { + return std::get(t); + } + template SizeT getTupleOffset(const Tuple& t, CSizeT i) { const PtrId beg = reinterpret_cast(&t); - const PtrId pos = reinterpret_cast(&std::get(t)); + const PtrId pos = reinterpret_cast(&tget(t)); return pos - beg; } diff --git a/src/opt/hdf5/include/h5_type_id.h b/src/opt/hdf5/include/h5_type_id.h index a517dca..b024d82 100644 --- a/src/opt/hdf5/include/h5_type_id.h +++ b/src/opt/hdf5/include/h5_type_id.h @@ -2,10 +2,23 @@ #ifndef __cxz_h5_type_id_h__ #define __cxz_h5_type_id_h__ +#include "base/types.h" + namespace CNORXZ { namespace hdf5 { + template + decltype(auto) reverseTuple(const Tuple& t); + + template + using RTuple = typename std::remove_reference()))>::type; + template + constexpr const auto& tget(const Tuple& t); + + template + constexpr auto& tget(Tuple& t); + template SizeT getTupleOffset(const Tuple& t, CSizeT i); diff --git a/src/opt/hdf5/lib/h5_table.cc b/src/opt/hdf5/lib/h5_table.cc index f2ef07b..21bdef7 100644 --- a/src/opt/hdf5/lib/h5_table.cc +++ b/src/opt/hdf5/lib/h5_table.cc @@ -21,9 +21,8 @@ namespace CNORXZ } Vector offsets(nfields); Vector sizes(nfields); - SizeT typesize = 0; H5TBget_field_info(mParent->id(), mName.c_str(), fieldsptr.data(), sizes.data(), - offsets.data(), &typesize); + offsets.data(), &mTypesize); Vector fields(nfields); for(SizeT i = 0; i != nfields; ++i){ fields[i].first = i; @@ -132,6 +131,59 @@ namespace CNORXZ CXZ_ERROR("not implemented!!!"); return *this; } + + MArray Table::read() const + { + Vector buf(mTypesize*mRecords->size()); + H5TBread_table(mParent->id(), mName.c_str(), mTypesize, mOffsets.data(), + mSizes.data(), buf.data()); + const hid_t dset_id = H5Dopen(mParent->id(), mName.c_str(), H5P_DEFAULT); + const hid_t type_id = H5Dget_type(dset_id); + MArray out(mRecords*mFields); + CIndex fi(mFields); + CIndex ri(mRecords); + const char* b = nullptr; + for(fi = 0; fi.lex() != mFields->size(); ++fi){ + const SizeT off = mOffsets[fi]; + const hid_t tp = H5Tget_member_type( type_id, fi.lex() ); + const hid_t tc = H5Tget_class(tp); + const size_t ts = H5Tget_size(tp); + //const bool sig = H5Tget_sign(tp); + DType x; + switch(tc){ + case H5T_INTEGER: { + if(ts == 4){ + for(ri = 0, b = buf.data(); ri.lex() != mRecords->size(); ++ri, b += mTypesize){ + const Int xi = *reinterpret_cast(b+off); + out[ri*fi] = DType(xi); + } + break; + } + else if(ts == 8){ + for(ri = 0, b = buf.data(); ri.lex() != mRecords->size(); ++ri, b += mTypesize){ + const LInt xi = *reinterpret_cast(b+off); + out[ri*fi] = DType(xi); + } + break; + } + } + case H5T_FLOAT: { + if(ts == 8){ + for(ri = 0, b = buf.data(); ri.lex() != mRecords->size(); ++ri, b += mTypesize){ + const Double xi = *reinterpret_cast(b+off); + out[ri*fi] = DType(xi); + } + break; + } + } + default: + CXZ_ERROR("type " << tp << " not supported"); + } + } + H5Tclose(type_id); + H5Dclose(dset_id); + return out; + } const RangePtr& Table::fields() const { diff --git a/src/opt/hdf5/tests/h5_basic_unit_test.cc b/src/opt/hdf5/tests/h5_basic_unit_test.cc index 603ba27..5c04ef5 100644 --- a/src/opt/hdf5/tests/h5_basic_unit_test.cc +++ b/src/opt/hdf5/tests/h5_basic_unit_test.cc @@ -39,24 +39,32 @@ namespace Group_Test() { mFileName = testh5file; - //mGrps = { "gr1", "gr2", "foo", "bar", "moregroups" }; mFs = {"field1","second","real"}; - Vector> v - ( { {0, -6, 3.141}, - {3, -8, 0.789}, - {34, 4, 10.009}, - {2, -777, -9.77}, - {321, 0, -0.003} + // Tuple has reverse (!) memory ordering: + Vector> v + ( { { 3.141, -6, 0 }, + { 0.789, -8, 3 }, + { 10.009, 4, 34 }, + { -9.77, -777, 2 }, + { -0.003, 0, 321 } } ); RangePtr rs = CRangeFactory(v.size()).create(); - mTabA = MArray>(rs, std::move(v)); + RangePtr fs = CRangeFactory(mFs.size()).create(); + mTabA = MArray>(rs, std::move(v)); + mTabD = MArray(rs*fs); + CIndex i(rs); + CIndex j(fs); + for(i = 0; i.lex() != rs->size(); ++i){ + iter<0,3>( [&](auto jj) + { j = jj; mTabD[i*j] = DType(tget(v[i.lex()])); }, NoF{} ); + } } String mFileName; - //Vector mGrps; Arr mFs; - MArray> mTabA; + MArray> mTabA; + MArray mTabD; }; @@ -142,14 +150,29 @@ namespace TEST_F(Group_Test, ReadTable) { - typedef Tuple RecType; + typedef Tuple RecType; File h5f(mFileName, true); h5f.open(); auto tab = h5f.getGroup("gr1")->open().getTable("tab1", RecType()); EXPECT_EQ(tab->fields()->size(), 3u); EXPECT_EQ(tab->records()->size(), 5u); - + tab->open(); + auto cont = tab->read(); + EXPECT_EQ(cont.size(), mTabA.size()); + CIndex i(mTabA.range()); + for(; i.lex() != i.lmax().val(); ++i){ + EXPECT_EQ( cont[i], mTabA[i] ); + } + auto dtab = h5f.getGroup("gr1")->open().getTable("tab1"); + auto dcont = dtab->read(); + EXPECT_EQ(dcont.range()->dim(), 2u); + EXPECT_EQ(dcont.range()->sub(0)->size(), 5u); + EXPECT_EQ(dcont.range()->sub(1)->size(), 3u); + for(auto ai = dcont.begin(); ai != dcont.end(); ++ai){ + EXPECT_EQ(dcont[ai].str(), mTabD[ai].str()); + } + h5f.close(); } } -- 2.45.2 From f7c00eab8cecfb6ba162faed80adb100aff9d065 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Mon, 29 Jan 2024 23:54:25 +0100 Subject: [PATCH 17/32] improve table read / readRecord --- src/opt/hdf5/include/h5_table.cc.h | 2 +- src/opt/hdf5/include/h5_table.h | 8 +- src/opt/hdf5/lib/h5_table.cc | 140 ++++++++++++++++++----------- 3 files changed, 97 insertions(+), 53 deletions(-) diff --git a/src/opt/hdf5/include/h5_table.cc.h b/src/opt/hdf5/include/h5_table.cc.h index 1756994..542cd4a 100644 --- a/src/opt/hdf5/include/h5_table.cc.h +++ b/src/opt/hdf5/include/h5_table.cc.h @@ -89,7 +89,7 @@ namespace CNORXZ initTable(1, &t, sizeof(t), sizeof(t)); } else { - Table::appendRecord(1, &t, sizeof(t)); + Table::appendRecords(1, &t); } return *this; } diff --git a/src/opt/hdf5/include/h5_table.h b/src/opt/hdf5/include/h5_table.h index 16405b4..e5b547d 100644 --- a/src/opt/hdf5/include/h5_table.h +++ b/src/opt/hdf5/include/h5_table.h @@ -27,8 +27,9 @@ namespace CNORXZ Table& initFieldNames(const Vector& fnames); Table& initTable(SizeT n, const void* data, SizeT dsize, SizeT chunk_size); - Table& appendRecord(SizeT n, const void* data, SizeT dsize); - Table& readRecord(SizeT pos, SizeT n, char* data); + Table& appendRecords(SizeT n, const void* data); + Table& readRecords(SizeT pos, SizeT n, char* data); + MArray readRecord(SizeT pos) const; MArray read() const; const RangePtr& fields() const; @@ -42,6 +43,9 @@ namespace CNORXZ MArray mTypes; hid_t mType = 0; SizeT mTypesize = 0; + MArray> mInterpret; + + void mkTypes(); }; // caution: the memory ordering is the only thing that counts; diff --git a/src/opt/hdf5/lib/h5_table.cc b/src/opt/hdf5/lib/h5_table.cc index 21bdef7..e68754c 100644 --- a/src/opt/hdf5/lib/h5_table.cc +++ b/src/opt/hdf5/lib/h5_table.cc @@ -65,6 +65,7 @@ namespace CNORXZ if(mId == 0){ mId = H5Dopen(mParent->id(), mName.c_str(), H5P_DEFAULT); mType = H5Dget_type(mId); + mkTypes(); } return *this; } @@ -110,78 +111,58 @@ namespace CNORXZ for(auto fi = fr->begin(); fi != fr->end(); ++fi){ fields[fi.lex()] = (*fi).second.c_str(); } + mTypesize = dsize; const herr_t err = H5TBmake_table - (mName.c_str(), mParent->id(), mName.c_str(), mFields->size(), mRecords->size(), dsize, + (mName.c_str(), mParent->id(), mName.c_str(), mFields->size(), mRecords->size(), mTypesize, fields.data(), mOffsets.data(), mTypes.data(), chunk_size, NULL, compress, data); CXZ_ASSERT(err >= 0, "error while initialzing table: error code = " << err); return *this; } - Table& Table::appendRecord(SizeT n, const void* data, SizeT dsize) + Table& Table::appendRecords(SizeT n, const void* data) { - mRecords = mRecords->extend( CRangeFactory(1).create() ); - const herr_t err = H5TBappend_records(mParent->id(), mName.c_str(), n, dsize, + mRecords = mRecords->extend( CRangeFactory(n).create() ); + const herr_t err = H5TBappend_records(mParent->id(), mName.c_str(), n, mTypesize, mOffsets.data(), mSizes.data(), data); CXZ_ASSERT(err >= 0, "error while appending record to table: error code = " << err); return *this; } - Table& Table::readRecord(SizeT pos, SizeT n, char* data) + Table& Table::readRecords(SizeT pos, SizeT n, char* data) { - CXZ_ERROR("not implemented!!!"); + H5TBread_records(mParent->id(), mName.c_str(), pos, n, mTypesize, mOffsets.data(), + mSizes.data(), data); return *this; } + + MArray Table::readRecord(SizeT pos) const + { + Vector buf(mTypesize); + H5TBread_records(mParent->id(), mName.c_str(), pos, 1, mTypesize, mOffsets.data(), + mSizes.data(), buf.data()); + MArray out(mFields); + auto fi = std::make_shared(mFields); + + out(fi) = operation( [&](const SizeT& off, const std::function& f){ + return f( buf.data() + off ); + }, mOffsets(fi), mInterpret(fi) ); + return out; + } + MArray Table::read() const { Vector buf(mTypesize*mRecords->size()); H5TBread_table(mParent->id(), mName.c_str(), mTypesize, mOffsets.data(), mSizes.data(), buf.data()); - const hid_t dset_id = H5Dopen(mParent->id(), mName.c_str(), H5P_DEFAULT); - const hid_t type_id = H5Dget_type(dset_id); MArray out(mRecords*mFields); - CIndex fi(mFields); - CIndex ri(mRecords); - const char* b = nullptr; - for(fi = 0; fi.lex() != mFields->size(); ++fi){ - const SizeT off = mOffsets[fi]; - const hid_t tp = H5Tget_member_type( type_id, fi.lex() ); - const hid_t tc = H5Tget_class(tp); - const size_t ts = H5Tget_size(tp); - //const bool sig = H5Tget_sign(tp); - DType x; - switch(tc){ - case H5T_INTEGER: { - if(ts == 4){ - for(ri = 0, b = buf.data(); ri.lex() != mRecords->size(); ++ri, b += mTypesize){ - const Int xi = *reinterpret_cast(b+off); - out[ri*fi] = DType(xi); - } - break; - } - else if(ts == 8){ - for(ri = 0, b = buf.data(); ri.lex() != mRecords->size(); ++ri, b += mTypesize){ - const LInt xi = *reinterpret_cast(b+off); - out[ri*fi] = DType(xi); - } - break; - } - } - case H5T_FLOAT: { - if(ts == 8){ - for(ri = 0, b = buf.data(); ri.lex() != mRecords->size(); ++ri, b += mTypesize){ - const Double xi = *reinterpret_cast(b+off); - out[ri*fi] = DType(xi); - } - break; - } - } - default: - CXZ_ERROR("type " << tp << " not supported"); - } - } - H5Tclose(type_id); - H5Dclose(dset_id); + auto fi = std::make_shared(mFields); + auto ri = std::make_shared(mRecords); + + out(ri*fi) = operation( [&](const SizeT& off, const std::function& f, const SizeT r){ + return f( buf.data() + r*mTypesize + off ); + }, mOffsets(fi), mInterpret(fi), xpr(ri) ); + return out; } @@ -195,5 +176,64 @@ namespace CNORXZ return mRecords; } + void Table::mkTypes() + { + mTypes = MArray(mFields); + auto i = std::make_shared(mFields); + mTypes(i) = operation( [&](const SizeT n){ return H5Tget_member_type( mType, n ); } , xpr(i) ); + mInterpret = MArray>(mFields); + mInterpret(i) = operation( [](const hid_t tid){ + const hid_t tc = H5Tget_class(tid); + const size_t ts = H5Tget_size(tid); + const bool sig = H5Tget_sign(tid); + std::function of; + switch(tc){ + case H5T_INTEGER:{ + switch(ts){ + case 1:{ // 8-bit + of = sig ? + [](const char* d){ const int8_t x = *reinterpret_cast(d); return DType(static_cast(x)); } : + [](const char* d){ const uint8_t x = *reinterpret_cast(d); return DType(static_cast(x)); }; + break; + } + case 2:{ // 16-bit + of = sig ? + [](const char* d){ const int16_t x = *reinterpret_cast(d); return DType(static_cast(x)); } : + [](const char* d){ const uint16_t x = *reinterpret_cast(d); return DType(static_cast(x)); }; + break; + } + case 4:{ // 32-bit + of = sig ? + [](const char* d){ const int32_t x = *reinterpret_cast(d); return DType(static_cast(x)); } : + [](const char* d){ const uint32_t x = *reinterpret_cast(d); return DType(static_cast(x)); }; + break; + } + case 8:{ // 64-bit + of = sig ? + [](const char* d){ const int64_t x = *reinterpret_cast(d); return DType(static_cast(x)); } : + [](const char* d){ const uint64_t x = *reinterpret_cast(d); return DType(static_cast(x)); }; + break; + } + default: + CXZ_ERROR("got integer of weird size: " << ts); + } + break; + } + case H5T_FLOAT:{ + if(ts == 4) { + of = [](const char* d){ const float x = *reinterpret_cast(d); return DType(static_cast(x)); }; + } + else { + CXZ_ASSERT(ts == 8, "got float of weird size: " << ts); + of = [](const char* d){ const double x = *reinterpret_cast(d); return DType(static_cast(x)); }; + } + break; + } + default: + CXZ_ERROR("got unsupported type"); + }; + return of; + }, mTypes(i) ); + } } } -- 2.45.2 From 39c0a3146f70176e8088bb56c799b854af4ffa23 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Tue, 30 Jan 2024 01:13:58 +0100 Subject: [PATCH 18/32] hdf5: table: iterRecords + test --- src/opt/hdf5/include/h5_group.cc.h | 2 + src/opt/hdf5/include/h5_table.cc.h | 10 +++++ src/opt/hdf5/include/h5_table.h | 3 ++ src/opt/hdf5/tests/h5_basic_unit_test.cc | 53 ++++++++++++++++++++++++ 4 files changed, 68 insertions(+) diff --git a/src/opt/hdf5/include/h5_group.cc.h b/src/opt/hdf5/include/h5_group.cc.h index f8f9971..57070f6 100644 --- a/src/opt/hdf5/include/h5_group.cc.h +++ b/src/opt/hdf5/include/h5_group.cc.h @@ -59,6 +59,7 @@ namespace CNORXZ template decltype(auto) Group::iter(F&& f) const { + CXZ_ASSERT(isOpen(), "try to iterate over closed object"); RangePtr gr = *mCont.range()->sub().begin(); auto gi = std::make_shared>(gr); return gi->ifor( operation(std::forward(f), mCont(gi)), NoF{} ); @@ -76,6 +77,7 @@ namespace CNORXZ template decltype(auto) Group::iter(F&& f) { + CXZ_ASSERT(isOpen(), "try to iterate over closed object"); RangePtr gr = *mCont.range()->sub().begin(); auto gi = std::make_shared>(gr); return gi->ifor( operation(std::forward(f), mCont(gi)), NoF{} ); diff --git a/src/opt/hdf5/include/h5_table.cc.h b/src/opt/hdf5/include/h5_table.cc.h index 542cd4a..5509ffa 100644 --- a/src/opt/hdf5/include/h5_table.cc.h +++ b/src/opt/hdf5/include/h5_table.cc.h @@ -9,6 +9,16 @@ namespace CNORXZ { namespace hdf5 { + template + decltype(auto) Table::iterRecords(F&& f) const + { + auto ri = std::make_shared(mRecords); + return ri->ifor + ( operation( std::forward(f), + operation( [&](const SizeT pos) { return readRecord(pos); } , + xpr(ri) ) ), NoF{} ); + } + template template decltype(auto) STable::iterRecords(F&& f) const diff --git a/src/opt/hdf5/include/h5_table.h b/src/opt/hdf5/include/h5_table.h index e5b547d..c8fa306 100644 --- a/src/opt/hdf5/include/h5_table.h +++ b/src/opt/hdf5/include/h5_table.h @@ -31,6 +31,9 @@ namespace CNORXZ Table& readRecords(SizeT pos, SizeT n, char* data); MArray readRecord(SizeT pos) const; MArray read() const; + + template + decltype(auto) iterRecords(F&& f) const; const RangePtr& fields() const; const RangePtr& records() const; diff --git a/src/opt/hdf5/tests/h5_basic_unit_test.cc b/src/opt/hdf5/tests/h5_basic_unit_test.cc index 5c04ef5..c367ea9 100644 --- a/src/opt/hdf5/tests/h5_basic_unit_test.cc +++ b/src/opt/hdf5/tests/h5_basic_unit_test.cc @@ -174,6 +174,59 @@ namespace } h5f.close(); } + + TEST_F(Group_Test, Read2) + { + File h5f(mFileName, true); + h5f.open(); + Vector paths; + Vector attrs; + + auto checkatt = [](const std::map& m, + const std::map& cs) { + for(const auto& c: cs){ + if(m.count(c.first)){ + if(m.at(c.first).str() != c.second.str()){ + return false; + } + } + else { + return false; + } + } + return true; + }; + std::map constr; + constr["wex"] = DType(Vector{7}); + constr["second"] = DType(static_cast(-777)); + h5f.iterRecursive( [&](const auto& c) { + if(c->type() == ContentType::TABLE) { + c->open(); + auto av = c->getRecursiveAttributes(); + auto cx = std::dynamic_pointer_cast
(c); + SizeT cnt = 0; + cx->iterRecords( [&](const MArray& r) { + auto ax = av; + auto fi = UIndex>(r.range()->sub(0)); + for(; fi.lex() != fi.lmax().val(); ++fi) { ax[fi.meta().second] = r[fi]; } + attrs.push_back(toString(ax)); + if(checkatt(ax,constr)){ + paths.push_back(c->path()+"/@"+toString(cnt)); + } + ++cnt; + } )(); + } + } )(); + for(const auto& x: attrs){ + VCHECK(x); + } + for(const auto& p: paths){ + VCHECK(p); + } + EXPECT_EQ(paths.size(), 1u); + EXPECT_EQ(paths[0], "/moregroups/evenmore/need/tab1/@3"); + h5f.close(); + } } // check write to new file -- 2.45.2 From 84f18e1d5ed9c7c4ef7f1e252c18ee265c6c5f25 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Tue, 30 Jan 2024 19:18:33 +0100 Subject: [PATCH 19/32] hdf5: groups: handle paths to access sub-content objects --- src/opt/hdf5/lib/h5_group.cc | 11 ++++++++++- src/opt/hdf5/tests/h5_basic_unit_test.cc | 6 ++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/opt/hdf5/lib/h5_group.cc b/src/opt/hdf5/lib/h5_group.cc index bc4bbbe..94ebe8d 100644 --- a/src/opt/hdf5/lib/h5_group.cc +++ b/src/opt/hdf5/lib/h5_group.cc @@ -75,7 +75,16 @@ namespace CNORXZ const ContentPtr& Group::get(const String& name) const { - auto i = this->getIndexTo(name); + const String delim = "/"; + const SizeT delimpos = name.find(delim); + const String thisname = name.substr(0, delimpos); + if(delimpos != String::npos and delimpos+1 < name.size()){ + const String next = name.substr(delimpos+1); + auto g = getGroup(thisname); + g->open(); + return g->get(next); + } + auto i = this->getIndexTo(thisname); return *i; } diff --git a/src/opt/hdf5/tests/h5_basic_unit_test.cc b/src/opt/hdf5/tests/h5_basic_unit_test.cc index c367ea9..f6689a3 100644 --- a/src/opt/hdf5/tests/h5_basic_unit_test.cc +++ b/src/opt/hdf5/tests/h5_basic_unit_test.cc @@ -139,6 +139,12 @@ namespace VCHECK(tab->path()); EXPECT_EQ(tab->fields()->size(), 3u); EXPECT_EQ(tab->records()->size(), 5u); + + EXPECT_THROW(h5f.getTable("moregroups/evenmore/need/tab1/a"), std::runtime_error); + auto tab2 = h5f.getTable("moregroups/evenmore/need/tab1/"); + EXPECT_EQ(tab2->fields()->size(), 3u); + EXPECT_EQ(tab2->records()->size(), 5u); + h5f.iter( [](const auto& c) { VCHECK(c->path()); } )(); h5f.iterRecursive( [](const auto& c) { VCHECK(c->path()); } )(); h5f.iterRecursive( [](const auto& c) { c->open(); VCHECK(toString(c->getRecursiveAttributes())); } )(); -- 2.45.2 From df228d545b2a87074d372deae35946dbd6725e1c Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Wed, 31 Jan 2024 00:16:34 +0100 Subject: [PATCH 20/32] WIP: hdf5: Dataset --- src/opt/hdf5/include/h5_dataset.h | 51 +++++++++++++++ src/opt/hdf5/lib/CMakeLists.txt | 1 + src/opt/hdf5/lib/h5_dataset.cc | 103 ++++++++++++++++++++++++++++++ 3 files changed, 155 insertions(+) create mode 100644 src/opt/hdf5/include/h5_dataset.h create mode 100644 src/opt/hdf5/lib/h5_dataset.cc diff --git a/src/opt/hdf5/include/h5_dataset.h b/src/opt/hdf5/include/h5_dataset.h new file mode 100644 index 0000000..13ca30c --- /dev/null +++ b/src/opt/hdf5/include/h5_dataset.h @@ -0,0 +1,51 @@ + +#ifndef __cxz_h5_dataset_h__ +#define __cxz_h5_dataset_h__ + +#include "h5_types.h" +#include "h5_content_base.h" + +namespace CNORXZ +{ + namespace hdf5 + { + class Dataset : public ContentBase + { + public: + DEFAULT_MEMBERS(Dataset); + Dataset(const String& name, const ContentBase* _parent); + ~Dataset(); + + virtual ContentType type() const override final; + virtual bool ro() const override final; + virtual Dataset& open() override final; + virtual Dataset& close() override final; + virtual String path() const override final; + virtual String filename() const override final; + + bool exists() const; + + Dataset& init(const RangePtr& dataRange, hid_t type); + + const RangePtr& dataRange() const; + + protected: + RangePtr mDataRange; + hid_t mType; + hid_t mFilespace; + + }; + + template + class SDataset : public Dataset + { + public: + DEFAULT_MEMBERS(SDataset); + SDataset(const String& name, const ContentBase* _parent); + + MArray read() const; + }; + } +} + +#endif diff --git a/src/opt/hdf5/lib/CMakeLists.txt b/src/opt/hdf5/lib/CMakeLists.txt index 51b08eb..e91ff23 100644 --- a/src/opt/hdf5/lib/CMakeLists.txt +++ b/src/opt/hdf5/lib/CMakeLists.txt @@ -4,6 +4,7 @@ set(libcnorxzhdf5_a_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/h5_file.cc ${CMAKE_CURRENT_SOURCE_DIR}/h5_group.cc ${CMAKE_CURRENT_SOURCE_DIR}/h5_table.cc + ${CMAKE_CURRENT_SOURCE_DIR}/h5_dataset.cc ) add_library(cnorxzhdf5_obj OBJECT diff --git a/src/opt/hdf5/lib/h5_dataset.cc b/src/opt/hdf5/lib/h5_dataset.cc new file mode 100644 index 0000000..e51afe5 --- /dev/null +++ b/src/opt/hdf5/lib/h5_dataset.cc @@ -0,0 +1,103 @@ + +#include "h5_dataset.h" + +namespace CNORXZ +{ + namespace hdf5 + { + Dataset::Dataset(const String& name, const ContentBase* _parent) : + ContentBase(name, _parent) + { + if(exists()){ + open(); + } + } + + Dataset::~Dataset() + { + this->close(); + } + + ContentType Dataset::type() const + { + return ContentType::DSET; + } + + bool Dataset::ro() const + { + return mParent->ro(); + } + + Dataset& Dataset::open() + { + if(mId == 0 and exists()){ + mId = H5Dopen(mParent->id(), mName.c_str(), H5P_DEFAULT); + mType = H5Dget_type(mId); + + mFilespace = H5Dget_space(mId); + SizeT ndims = H5Sget_simple_extent_ndims(mFilespace); + Vector dims(ndims); + H5Sget_simple_extent_dims(mFilespace, dims.data(), nullptr); + + Vector rs(ndims); + for(SizeT i = 0; i != ndims; ++i){ + rs[i] = CRangeFactory(dims[i]).create(); + } + mDataRange = yrange(rs); + + // use when reading!!!: + //hid_t mem_space_id = H5Screate_simple(static_cast(ndims), dims.data(), nullptr); + //hid_t xfer_plist_id = H5Pcreate(H5P_DATASET_XFER); + } + return *this; + } + + Dataset& Dataset::close() + { + if(mId != 0){ + //!!! + H5Sclose(mFilespace); + H5Tclose(mType); + H5Dclose(mId); + mId = 0; + } + return *this; + } + + String Dataset::path() const + { + return mParent->path() + "/" + mName; + } + + String Dataset::filename() const + { + return mParent->filename(); + } + + bool Dataset::exists() const + { + return H5Lexists(mParent->id(), mName.c_str(), H5P_DEFAULT); + } + + Dataset& Dataset::init(const RangePtr& dataRange, hid_t type) + { + // check if type is valid type!!! + const hid_t dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + // TODO: all sub-ranges explicity!!!: + const SizeT ndim = dataRange->dim(); + Vector exts(ndim); + mFilespace = H5Screate_simple(ndim, exts.data(), NULL); + mType = type; + mId = H5Dcreate(mParent->id(), mName.c_str(), mType, mFilespace, + H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + H5Pclose(dcpl_id); + return *this; + } + + const RangePtr& Dataset::dataRange() const + { + return mDataRange; + } + + } +} -- 2.45.2 From e2e233b93186bd6e8fb87bf23bb2cc12ea2209c8 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Wed, 31 Jan 2024 19:09:04 +0100 Subject: [PATCH 21/32] im com --- src/opt/hdf5/include/h5_dataset.cc.h | 18 ++++++++++++++++++ src/opt/hdf5/include/h5_dataset.h | 5 ++++- src/opt/hdf5/lib/h5_dataset.cc | 7 ++++++- 3 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 src/opt/hdf5/include/h5_dataset.cc.h diff --git a/src/opt/hdf5/include/h5_dataset.cc.h b/src/opt/hdf5/include/h5_dataset.cc.h new file mode 100644 index 0000000..77d215c --- /dev/null +++ b/src/opt/hdf5/include/h5_dataset.cc.h @@ -0,0 +1,18 @@ + +#ifndef __cxz_h5_dataset_cc_h__ +#define __cxz_h5_dataset_cc_h__ + +namespace CNORXZ +{ + namespace hdf5 + { + template + Dataset& Dataset::init(const ArrayBase& data) + { + // !!!! + } + + } +} + +#endif diff --git a/src/opt/hdf5/include/h5_dataset.h b/src/opt/hdf5/include/h5_dataset.h index 13ca30c..b464d07 100644 --- a/src/opt/hdf5/include/h5_dataset.h +++ b/src/opt/hdf5/include/h5_dataset.h @@ -26,7 +26,10 @@ namespace CNORXZ bool exists() const; Dataset& init(const RangePtr& dataRange, hid_t type); - + + template + Dataset& init(const ArrayBase& data); + const RangePtr& dataRange() const; protected: diff --git a/src/opt/hdf5/lib/h5_dataset.cc b/src/opt/hdf5/lib/h5_dataset.cc index e51afe5..d8e5d06 100644 --- a/src/opt/hdf5/lib/h5_dataset.cc +++ b/src/opt/hdf5/lib/h5_dataset.cc @@ -81,11 +81,16 @@ namespace CNORXZ Dataset& Dataset::init(const RangePtr& dataRange, hid_t type) { - // check if type is valid type!!! + CXZ_ASSERT(not open(), "tried to initialize dataset that is already extisting"); + const H5T_class_t tc = H5Tget_class(type); + CXZ_ASSERT(tc != H5T_NO_CLASS, "id does not correspond to a data type"); // (did not found anythng better to check if type id is valid)... const hid_t dcpl_id = H5Pcreate(H5P_DATASET_CREATE); // TODO: all sub-ranges explicity!!!: const SizeT ndim = dataRange->dim(); Vector exts(ndim); + for(SizeT i = 0; i != ndim; ++i){ + exts[i] = static_cast( dataRange->sub(i)->size() ); + } mFilespace = H5Screate_simple(ndim, exts.data(), NULL); mType = type; mId = H5Dcreate(mParent->id(), mName.c_str(), mType, mFilespace, -- 2.45.2 From 6f3601daa4ec36925ca0f4aff39168716c7711bc Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Thu, 1 Feb 2024 00:09:39 +0100 Subject: [PATCH 22/32] hdf5: dataset --- src/opt/hdf5/include/cnorxz_hdf5.cc.h | 1 + src/opt/hdf5/include/cnorxz_hdf5.h | 1 + src/opt/hdf5/include/h5_dataset.cc.h | 88 ++++++++++++++++++++++++++- src/opt/hdf5/include/h5_dataset.h | 8 +++ src/opt/hdf5/include/h5_group.cc.h | 42 ++++++++++--- src/opt/hdf5/include/h5_group.h | 12 ++-- src/opt/hdf5/include/h5_types.h | 9 ++- src/opt/hdf5/lib/h5_dataset.cc | 7 +-- src/opt/hdf5/lib/h5_group.cc | 12 +++- 9 files changed, 155 insertions(+), 25 deletions(-) diff --git a/src/opt/hdf5/include/cnorxz_hdf5.cc.h b/src/opt/hdf5/include/cnorxz_hdf5.cc.h index c5cd636..0003c9d 100644 --- a/src/opt/hdf5/include/cnorxz_hdf5.cc.h +++ b/src/opt/hdf5/include/cnorxz_hdf5.cc.h @@ -3,3 +3,4 @@ #include "h5_type_id.cc.h" #include "h5_group.cc.h" #include "h5_table.cc.h" +#include "h5_dataset.cc.h" diff --git a/src/opt/hdf5/include/cnorxz_hdf5.h b/src/opt/hdf5/include/cnorxz_hdf5.h index 5a47a23..b68d3f0 100644 --- a/src/opt/hdf5/include/cnorxz_hdf5.h +++ b/src/opt/hdf5/include/cnorxz_hdf5.h @@ -3,6 +3,7 @@ #include "h5_file.h" #include "h5_group.h" #include "h5_table.h" +#include "h5_dataset.h" #include "h5_type_id.h" #include "cnorxz_hdf5.cc.h" diff --git a/src/opt/hdf5/include/h5_dataset.cc.h b/src/opt/hdf5/include/h5_dataset.cc.h index 77d215c..27b24c1 100644 --- a/src/opt/hdf5/include/h5_dataset.cc.h +++ b/src/opt/hdf5/include/h5_dataset.cc.h @@ -2,6 +2,8 @@ #ifndef __cxz_h5_dataset_cc_h__ #define __cxz_h5_dataset_cc_h__ +#include "h5_dataset.h" + namespace CNORXZ { namespace hdf5 @@ -9,9 +11,91 @@ namespace CNORXZ template Dataset& Dataset::init(const ArrayBase& data) { - // !!!! + const hid_t tid = getTypeId(*data.data()); + init(data.range(), tid); + if(data.begin().formatIsTrivial()){ + Vector dims(mDataRange->dim()); + for(SizeT i = 0; i != dims.size(); ++i){ + dims[i] = mDataRange->sub(i)->size(); + } + const hid_t memspace = H5Screate_simple(dims.size(), dims.data(), NULL); + H5Dwrite(mId, mType, memspace, mFilespace, H5P_DEFAULT, data.data()); + H5Sclose(memspace); + } + else { + CXZ_ERROR("IMPLEMENT!!!"); + } + return *this; } - + + template + SDataset::SDataset(const String& name, const ContentBase* _parent) : + Dataset(name, _parent) + {} + + template + MArray SDataset::read() const + { + Vector dims(mDataRange->dim()); + for(SizeT i = 0; i != dims.size(); ++i){ + dims[i] = mDataRange->sub(i)->size(); + } + const hid_t mem_space_id = H5Screate_simple(static_cast(dims.size()), + dims.data(), nullptr); + const hid_t xfer_plist_id = H5Pcreate(H5P_DATASET_XFER); + MArray out(mDataRange); + const herr_t err = H5Dread(mId, mType, mem_space_id, mFilespace, xfer_plist_id, out.data()); + CXZ_ASSERT(err >= 0, "error while reading dataset '" << mName + << "', errorcode :" << err); + H5Pclose(xfer_plist_id); + H5Sclose(mem_space_id); + return out; + } + + template + template + MArray SDataset::read(const IndexInterface& beg, const IndexInterface& end) const + { + CXZ_ASSERT(beg.dim() == mDataRange->dim(), "got index of inconsistent dimension, got" + << beg.dim() << ", expected " << mDataRange->dim()); + const RangePtr outrange = beg.prange(end); + Vector dims(outrange->dim()); + for(SizeT i = 0; i != dims.size(); ++i){ + dims[i] = outrange->sub(i)->size(); + } + const Vector fpos = mkFPos(beg); + H5Sselect_hyperslab(mFilespace, H5S_SELECT_SET, fpos.data(), NULL, dims.data(), NULL); + const hid_t mem_space_id = H5Screate_simple(static_cast(dims.size()), + dims.data(), nullptr); + const hid_t xfer_plist_id = H5Pcreate(H5P_DATASET_XFER); + MArray out(outrange); + const herr_t err = H5Dread(mId, mType, mem_space_id, mFilespace, xfer_plist_id, out.data()); + CXZ_ASSERT(err >= 0, "error while reading dataset '" << mName + << "', errorcode :" << err); + H5Pclose(xfer_plist_id); + H5Sclose(mem_space_id); + return out; + } + + template + template + Vector SDataset::mkFPos(const IndexInterface& beg) const + { + Vector fpos(beg.dim()); + if constexpr(has_static_sub::value){ + iter<0,index_dim::value> ( [&](auto i) { fpos[i] = beg.pack().get(i).lex(); }, NoF{} ); + } + else if constexpr(has_sub::value){ + for(SizeT i = 0; i != beg.dim(); ++i){ + fpos[i] = beg.pack().get(i).lex(); + } + } + else { + fpos[0] = beg.lex(); + } + return fpos; + } + } } diff --git a/src/opt/hdf5/include/h5_dataset.h b/src/opt/hdf5/include/h5_dataset.h index b464d07..b9b2ed6 100644 --- a/src/opt/hdf5/include/h5_dataset.h +++ b/src/opt/hdf5/include/h5_dataset.h @@ -47,6 +47,14 @@ namespace CNORXZ SDataset(const String& name, const ContentBase* _parent); MArray read() const; + + template + MArray read(const IndexInterface& beg, const IndexInterface& end) const; + + private: + + template + Vector mkFPos(const IndexInterface& beg) const; }; } } diff --git a/src/opt/hdf5/include/h5_group.cc.h b/src/opt/hdf5/include/h5_group.cc.h index 57070f6..9ad13f6 100644 --- a/src/opt/hdf5/include/h5_group.cc.h +++ b/src/opt/hdf5/include/h5_group.cc.h @@ -13,9 +13,9 @@ namespace CNORXZ Sptr> Group::getTable(const String& name, Tuple proto) { auto i = this->getIndexTo(name); - auto tab = std::dynamic_pointer_cast
( *i ); - CXZ_ASSERT(tab->type() == ContentType::TABLE, + CXZ_ASSERT((*i)->type() == ContentType::TABLE, "element '" << name << "' is not of type TABLE"); + auto tab = std::dynamic_pointer_cast
( *i ); if(tab == nullptr){ auto stab = std::dynamic_pointer_cast>(*i); CXZ_ASSERT(stab != nullptr, "wrong format for table '" << name << "'"); @@ -28,12 +28,37 @@ namespace CNORXZ return stab; } } - + template - Group& Group::addData(const String& name, const ArrayBase& data) + Sptr> Group::getDataset(const String& name, T proto) + { + auto i = this->getIndexTo(name); + CXZ_ASSERT((*i)->type() == ContentType::DSET, + "element '" << name << "' is not of type DSET"); + auto dset = std::dynamic_pointer_cast( *i ); + if(dset == nullptr){ + auto sdset = std::dynamic_pointer_cast>(*i); + CXZ_ASSERT(sdset != nullptr, "wrong format for dataset '" << name << "'"); + return sdset; + } + else { + (*i)->close(); + auto sdset = std::make_shared>(name, this); + *i = sdset; + return sdset; + } + } + + template + Group& Group::addDataset(const String& name, const ArrayBase& data) { CXZ_ASSERT(this->isOpen(), "tried to extend closed group"); - CXZ_ERROR("not implemented!!!"); + Vector nvec({name}); + mCont.extend( URangeFactory( nvec ).create() ); + auto ii = getIndexTo(name); + auto dset = std::make_shared>(name, this); + dset->init(data); + *ii = dset; return *this; } @@ -43,11 +68,8 @@ namespace CNORXZ { CXZ_ASSERT(this->isOpen(), "tried to extend closed group"); Vector nvec({name}); - Vector dvec({DType(name)}); - auto extr = URangeFactory( nvec ).create(); - mCont.extend(extr); - auto ii = mCont.begin(); - ii.at(dvec); // 'at' returns YIndex&, so cannot use it inline... + mCont.extend( URangeFactory( nvec ).create() ); + auto ii = getIndexTo(name); auto tab = std::make_shared>(name, this, fnames); for(auto& d: data){ tab->appendRecord(d); diff --git a/src/opt/hdf5/include/h5_group.h b/src/opt/hdf5/include/h5_group.h index babb886..986d04a 100644 --- a/src/opt/hdf5/include/h5_group.h +++ b/src/opt/hdf5/include/h5_group.h @@ -28,20 +28,24 @@ namespace CNORXZ const ContentPtr& get(const String& name) const; Sptr getGroup(const String& name) const; Sptr
getTable(const String& name) const; + Sptr getDataset(const String& name) const; template Sptr> getTable(const String& name, Tuple proto); - - const MArray& get() const; - Group& addGroup(const String& name); template - Group& addData(const String& name, const ArrayBase& data); + Sptr> getDataset(const String& name, T proto); + + const MArray& get() const; + Group& addGroup(const String& name); template Group& addTable(const String& name, const ArrayBase>& data, const Arr& fnames); + template + Group& addDataset(const String& name, const ArrayBase& data); + template decltype(auto) iter(F&& f) const; diff --git a/src/opt/hdf5/include/h5_types.h b/src/opt/hdf5/include/h5_types.h index 744b157..cbb4fa8 100644 --- a/src/opt/hdf5/include/h5_types.h +++ b/src/opt/hdf5/include/h5_types.h @@ -21,7 +21,14 @@ namespace CNORXZ // definition -> h5_table.h template class STable; - + + // definition -> h5_dataset.h + class Dataset; + + // definition -> h5_dataset.h + template + class SDataset; + } } diff --git a/src/opt/hdf5/lib/h5_dataset.cc b/src/opt/hdf5/lib/h5_dataset.cc index d8e5d06..f53a50d 100644 --- a/src/opt/hdf5/lib/h5_dataset.cc +++ b/src/opt/hdf5/lib/h5_dataset.cc @@ -44,10 +44,6 @@ namespace CNORXZ rs[i] = CRangeFactory(dims[i]).create(); } mDataRange = yrange(rs); - - // use when reading!!!: - //hid_t mem_space_id = H5Screate_simple(static_cast(ndims), dims.data(), nullptr); - //hid_t xfer_plist_id = H5Pcreate(H5P_DATASET_XFER); } return *this; } @@ -55,7 +51,6 @@ namespace CNORXZ Dataset& Dataset::close() { if(mId != 0){ - //!!! H5Sclose(mFilespace); H5Tclose(mType); H5Dclose(mId); @@ -81,7 +76,7 @@ namespace CNORXZ Dataset& Dataset::init(const RangePtr& dataRange, hid_t type) { - CXZ_ASSERT(not open(), "tried to initialize dataset that is already extisting"); + CXZ_ASSERT(not isOpen(), "tried to initialize dataset that is already extisting"); const H5T_class_t tc = H5Tget_class(type); CXZ_ASSERT(tc != H5T_NO_CLASS, "id does not correspond to a data type"); // (did not found anythng better to check if type id is valid)... const hid_t dcpl_id = H5Pcreate(H5P_DATASET_CREATE); diff --git a/src/opt/hdf5/lib/h5_group.cc b/src/opt/hdf5/lib/h5_group.cc index 94ebe8d..65b21aa 100644 --- a/src/opt/hdf5/lib/h5_group.cc +++ b/src/opt/hdf5/lib/h5_group.cc @@ -1,6 +1,7 @@ #include "h5_group.h" #include "h5_table.h" +#include "h5_dataset.h" namespace CNORXZ { @@ -104,6 +105,14 @@ namespace CNORXZ return std::dynamic_pointer_cast
( table ); } + Sptr Group::getDataset(const String& name) const + { + auto dset = this->get(name); + CXZ_ASSERT(dset->type() == ContentType::DSET, + "element '" << name << "' is not of type DSET"); + return std::dynamic_pointer_cast( dset ); + } + const MArray& Group::get() const { CXZ_ASSERT(this->isOpen(), "tried to get content of closed group"); @@ -186,8 +195,7 @@ namespace CNORXZ *index = std::make_shared
(sname, icd->parent); } else { - CXZ_ERROR("IMPLEMENT!!!"); - //*index = std::make_shared(sname, icd->parent); + *index = std::make_shared(sname, icd->parent); } break; } -- 2.45.2 From eb3055ddcab6f6ba3f7d22d058ee066eee391007 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Thu, 1 Feb 2024 18:22:08 +0100 Subject: [PATCH 23/32] hdf5: several fixes + yrange: add setSub member function + dindex: add missing operator*(dindex,index) --- src/include/ranges/dindex.cc.h | 8 +++- src/include/ranges/dindex.h | 12 +++-- src/include/ranges/yrange.h | 6 +++ src/lib/ranges/yrange.cc | 13 ++++++ src/opt/hdf5/include/h5_dataset.cc.h | 5 +- src/opt/hdf5/include/h5_type_id.cc.h | 6 +-- src/opt/hdf5/lib/h5_dataset.cc | 1 + src/opt/hdf5/tests/CMakeLists.txt | 6 ++- src/opt/hdf5/tests/h5_basic_unit_test.cc | 59 +++++++++++++++++++++++- 9 files changed, 104 insertions(+), 12 deletions(-) diff --git a/src/include/ranges/dindex.cc.h b/src/include/ranges/dindex.cc.h index 6637298..f752bed 100644 --- a/src/include/ranges/dindex.cc.h +++ b/src/include/ranges/dindex.cc.h @@ -5,7 +5,7 @@ @brief ... - Copyright (c) 2022 Christian Zimmermann. All rights reserved. + Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de **/ @@ -40,6 +40,12 @@ namespace CNORXZ } } + template + decltype(auto) operator*(const Sptr& a, const Sptr& b) + { + return iptrMul(a, b); + } + template I indexAs(const DIndex& i) { diff --git a/src/include/ranges/dindex.h b/src/include/ranges/dindex.h index 041eb50..0408cc2 100644 --- a/src/include/ranges/dindex.h +++ b/src/include/ranges/dindex.h @@ -2,10 +2,9 @@ /** @file include/ranges/dindex.h - @brief ... + @brief DIndex declaration - - Copyright (c) 2022 Christian Zimmermann. All rights reserved. + Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de **/ @@ -154,6 +153,13 @@ namespace CNORXZ XIndexPtr mI; }; + /** Make index pack of a DIndex and another index. + @param a pointer to DIndex. + @param b pointer to another index. + */ + template + decltype(auto) operator*(const Sptr& a, const Sptr& b); + /** Trait-specialization: DIndex can have sub-indices */ diff --git a/src/include/ranges/yrange.h b/src/include/ranges/yrange.h index a5468b9..ea86b6d 100644 --- a/src/include/ranges/yrange.h +++ b/src/include/ranges/yrange.h @@ -186,6 +186,12 @@ namespace CNORXZ */ YIndex& setFormat(const YFormat& bs); + /** Set position of given sub index and update total index position. + @param ind Sub-index number [0,dim()-1]. + @param lex Lexicographic position to be assigned to the index. + */ + YIndex& setSub(SizeT ind, SizeT lex); + private: inline Vector mkFormat() const; inline Vector mkLexFormat() const; diff --git a/src/lib/ranges/yrange.cc b/src/lib/ranges/yrange.cc index c1a4d65..b456995 100644 --- a/src/lib/ranges/yrange.cc +++ b/src/lib/ranges/yrange.cc @@ -494,6 +494,19 @@ namespace CNORXZ return *this; } + YIndex& YIndex::setSub(SizeT ind, SizeT lex) + { + CXZ_ASSERT(ind < dim(), "got index number (" << ind << ") larger than dimension (" + << dim() << ")"); + auto& idx = mIs[ind]; + CXZ_ASSERT(lex < idx->lmax().val(), "tried to set sub-index position " << lex + << ", which is out of scope; maximum position in range is " << idx->lmax().val() ); + (*idx) = lex; + (*this)(); + return *this; + } + + /**************************** * non-member functions * ****************************/ diff --git a/src/opt/hdf5/include/h5_dataset.cc.h b/src/opt/hdf5/include/h5_dataset.cc.h index 27b24c1..6df2e42 100644 --- a/src/opt/hdf5/include/h5_dataset.cc.h +++ b/src/opt/hdf5/include/h5_dataset.cc.h @@ -12,6 +12,7 @@ namespace CNORXZ Dataset& Dataset::init(const ArrayBase& data) { const hid_t tid = getTypeId(*data.data()); + VCHECK(tid); init(data.range(), tid); if(data.begin().formatIsTrivial()){ Vector dims(mDataRange->dim()); @@ -83,11 +84,11 @@ namespace CNORXZ { Vector fpos(beg.dim()); if constexpr(has_static_sub::value){ - iter<0,index_dim::value> ( [&](auto i) { fpos[i] = beg.pack().get(i).lex(); }, NoF{} ); + iter<0,index_dim::value> ( [&](auto i) { fpos[i] = beg.THIS().pack().get(i)->lex(); }, NoF{} ); } else if constexpr(has_sub::value){ for(SizeT i = 0; i != beg.dim(); ++i){ - fpos[i] = beg.pack().get(i).lex(); + fpos[i] = beg.THIS().pack().get(i)->lex(); } } else { diff --git a/src/opt/hdf5/include/h5_type_id.cc.h b/src/opt/hdf5/include/h5_type_id.cc.h index 65cdce4..322ff76 100644 --- a/src/opt/hdf5/include/h5_type_id.cc.h +++ b/src/opt/hdf5/include/h5_type_id.cc.h @@ -48,17 +48,17 @@ namespace CNORXZ inline hid_t TypeId::get() { - return H5T_NATIVE_ULONG; + return H5Tcopy( H5T_NATIVE_ULONG ); } inline hid_t TypeId::get() { - return H5T_NATIVE_INT; + return H5Tcopy( H5T_NATIVE_INT ); } inline hid_t TypeId::get() { - return H5T_NATIVE_DOUBLE; + return H5Tcopy( H5T_NATIVE_DOUBLE ); } template diff --git a/src/opt/hdf5/lib/h5_dataset.cc b/src/opt/hdf5/lib/h5_dataset.cc index f53a50d..e535250 100644 --- a/src/opt/hdf5/lib/h5_dataset.cc +++ b/src/opt/hdf5/lib/h5_dataset.cc @@ -77,6 +77,7 @@ namespace CNORXZ Dataset& Dataset::init(const RangePtr& dataRange, hid_t type) { CXZ_ASSERT(not isOpen(), "tried to initialize dataset that is already extisting"); + mDataRange = dataRange; const H5T_class_t tc = H5Tget_class(type); CXZ_ASSERT(tc != H5T_NO_CLASS, "id does not correspond to a data type"); // (did not found anythng better to check if type id is valid)... const hid_t dcpl_id = H5Pcreate(H5P_DATASET_CREATE); diff --git a/src/opt/hdf5/tests/CMakeLists.txt b/src/opt/hdf5/tests/CMakeLists.txt index 51e299b..ecc318c 100644 --- a/src/opt/hdf5/tests/CMakeLists.txt +++ b/src/opt/hdf5/tests/CMakeLists.txt @@ -1,5 +1,7 @@ +add_definitions(-DTEST_NUMBER_FILE="${CMAKE_SOURCE_DIR}/src/tests/numbers.txt") +include_directories(${CMAKE_SOURCE_DIR}/src/tests) add_executable(h5basic h5_basic_unit_test.cc) -add_dependencies(h5basic cnorxz cnorxzhdf5) -target_link_libraries(h5basic ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${HDF5_LIBS} cnorxz cnorxzhdf5) +add_dependencies(h5basic cnorxz cnorxzhdf5 test_lib) +target_link_libraries(h5basic ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${HDF5_LIBS} cnorxz cnorxzhdf5 test_lib) add_test(NAME h5basic COMMAND h5basic) diff --git a/src/opt/hdf5/tests/h5_basic_unit_test.cc b/src/opt/hdf5/tests/h5_basic_unit_test.cc index f6689a3..7e0cc87 100644 --- a/src/opt/hdf5/tests/h5_basic_unit_test.cc +++ b/src/opt/hdf5/tests/h5_basic_unit_test.cc @@ -7,11 +7,13 @@ #include "gtest/gtest.h" #include "cnorxz_hdf5.h" +#include "test_numbers.h" namespace { using namespace CNORXZ; using namespace CNORXZ::hdf5; + using Test::Numbers; static const String testh5file = "test_file.h5"; @@ -58,6 +60,16 @@ namespace iter<0,3>( [&](auto jj) { j = jj; mTabD[i*j] = DType(tget(v[i.lex()])); }, NoF{} ); } + + const SizeT ddim = 5; + Vector dranges(ddim); + dranges[0] = CRangeFactory(7).create(); + dranges[1] = CRangeFactory(3).create(); + dranges[2] = CRangeFactory(13).create(); + dranges[3] = CRangeFactory(5).create(); + dranges[4] = CRangeFactory(2).create(); + const RangePtr drange = yrange(dranges); + mData = MArray( drange, Numbers::get(0,drange->size()) ); } String mFileName; @@ -65,6 +77,7 @@ namespace Arr mFs; MArray> mTabA; MArray mTabD; + MArray mData; }; @@ -118,10 +131,18 @@ namespace File h5f(mFileName, false); h5f.open(); h5f.getGroup("gr1")->open().addTable("tab1", mTabA, mFs); - h5f.getGroup("gr2")->open().addTable("tab1", mTabA, mFs); + //h5f.getGroup("gr2")->open().addTable("tab1", mTabA, mFs); h5f.getGroup("moregroups")->open().getGroup("evenmore")->open().getGroup("need")->open().addTable("tab1", mTabA, mFs); h5f.close(); } + + TEST_F(Group_Test, CreateDataset) + { + File h5f(mFileName, false); + h5f.open(); + h5f.getGroup("gr2")->open().addDataset("dat1", mData); + h5f.close(); + } TEST_F(Group_Test, Read) { @@ -181,6 +202,42 @@ namespace h5f.close(); } + TEST_F(Group_Test, ReadDataset) + { + File h5f(mFileName, true); + h5f.open(); + auto dset = h5f.getGroup("gr2")->open().getDataset("dat1", Double{}); + auto data = dset->read(); + EXPECT_EQ(data.range()->dim(), 5u); + for(SizeT i = 0; i != 5u; ++i){ + EXPECT_EQ( data.range()->sub(i)->size(), mData.range()->sub(i)->size() ); + } + auto i = std::make_shared(data.range()); + i->ifor( operation( [](Double a, Double b) { EXPECT_EQ(a,b); }, data(i), mData(i) ), NoF{} )(); + h5f.close(); + } + + TEST_F(Group_Test, ReadDatasetPart) + { + File h5f(mFileName, true); + h5f.open(); + auto dset = h5f.getGroup("gr2")->open().getDataset("dat1", Double{}); + YIndex beg(dset->dataRange()); + beg.setSub(0,2); + YIndex end = beg - 1; + end.setSub(0,2); + auto data = dset->read(beg,end); + EXPECT_EQ( data.range()->dim(), 5u ); + EXPECT_EQ( data.range()->sub(0)->size(), 1u ); + for(SizeT i = 1; i != 5u; ++i){ + EXPECT_EQ( data.range()->sub(i)->size(), mData.range()->sub(i)->size() ); + } + auto i = std::make_shared(data.range()); + auto j = std::make_shared( beg.pack().get(0) ); + i->ifor( operation( [](Double a, Double b) { EXPECT_EQ(a,b); }, data(i), mData(j*i) ), NoF{} )(); + h5f.close(); + } + TEST_F(Group_Test, Read2) { File h5f(mFileName, true); -- 2.45.2 From be3f5521e87d542bb353df3478eeaf0d863e8605 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Fri, 2 Feb 2024 00:36:14 +0100 Subject: [PATCH 24/32] WIP: hdf5 documentation --- src/include/ranges/ranges.h | 5 +- src/opt/hdf5/include/cnorxz_hdf5.cc.h | 10 +++ src/opt/hdf5/include/cnorxz_hdf5.h | 10 +++ src/opt/hdf5/include/h5_content_base.h | 117 +++++++++++++++++++++---- src/opt/hdf5/include/h5_file.h | 25 +++++- src/opt/hdf5/include/h5_group.h | 97 +++++++++++++++++++- src/opt/hdf5/include/h5_table.h | 64 +++++++++++++- src/opt/hdf5/lib/h5_content_base.cc | 22 +++-- 8 files changed, 317 insertions(+), 33 deletions(-) diff --git a/src/include/ranges/ranges.h b/src/include/ranges/ranges.h index 5cff132..ac41e2e 100644 --- a/src/include/ranges/ranges.h +++ b/src/include/ranges/ranges.h @@ -2,10 +2,9 @@ /** @file include/ranges/ranges.h - @brief ... + @brief Ranges main header - - Copyright (c) 2022 Christian Zimmermann. All rights reserved. + Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de **/ diff --git a/src/opt/hdf5/include/cnorxz_hdf5.cc.h b/src/opt/hdf5/include/cnorxz_hdf5.cc.h index 0003c9d..0bfac44 100644 --- a/src/opt/hdf5/include/cnorxz_hdf5.cc.h +++ b/src/opt/hdf5/include/cnorxz_hdf5.cc.h @@ -1,3 +1,13 @@ +// -*- C++ -*- +/** + + @file opt/hdf5/include/cnorxz_hdf5.cc.h + @brief CNORXZ HDF5 template sources header + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ #include "h5_content_base.cc.h" #include "h5_type_id.cc.h" diff --git a/src/opt/hdf5/include/cnorxz_hdf5.h b/src/opt/hdf5/include/cnorxz_hdf5.h index b68d3f0..0d93562 100644 --- a/src/opt/hdf5/include/cnorxz_hdf5.h +++ b/src/opt/hdf5/include/cnorxz_hdf5.h @@ -1,3 +1,13 @@ +// -*- C++ -*- +/** + + @file opt/hdf5/include/cnorxz_hdf5.h + @brief CNORXZ HDF5 main header + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ #include "h5_content_base.h" #include "h5_file.h" diff --git a/src/opt/hdf5/include/h5_content_base.h b/src/opt/hdf5/include/h5_content_base.h index bdf346a..7a4dc03 100644 --- a/src/opt/hdf5/include/h5_content_base.h +++ b/src/opt/hdf5/include/h5_content_base.h @@ -1,3 +1,13 @@ +// -*- C++ -*- +/** + + @file opt/hdf5/include/h5_content_base.h + @brief Abstract content base class declaration + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ #ifndef __cxz_h5_content_base_h__ #define __cxz_h5_content_base_h__ @@ -11,50 +21,123 @@ namespace CNORXZ namespace hdf5 { // TODO: IO save error handling !!! - + + /** ***** + Enum indicating the content type. + Used by ContainerBase derivates to indicate the derived type. + */ enum class ContentType { - ATTR = 1, - FILE = 2, - GROUP = 3, - DSET = 4, - TABLE = 5, - VALUE = 6, + NONE = 0, + FILE = 1, + GROUP = 2, + DSET = 3, + TABLE = 4, }; - + + /** **** + Abstract base class for handling hdf5 objects. + */ class ContentBase { public: - DEFAULT_MEMBERS(ContentBase); + DEFAULT_MEMBERS(ContentBase); /**< Default constructors and assignments. */ + + /** Construct the class. + @param _name Content name. + @param _parent Parent content object. Leave null for the root object. + */ ContentBase(const String& _name, const ContentBase* _parent = nullptr); + + /** Virtual default destructor. */ virtual ~ContentBase() = default; + /** Get the content type. + @return Content type. + */ virtual ContentType type() const = 0; + + /** Check if in read-only mode + @return True if read-only else false. + */ virtual bool ro() const = 0; + + /** Open object. + @return Reference to this object. + */ virtual ContentBase& open() = 0; + + /** Close object. + @return Reference to this object. + */ virtual ContentBase& close() = 0; + + /** Get object path. + @return Absolute hdf5 file internal path of this object. + */ virtual String path() const = 0; + + /** Get the file name. + @return Name of the hdf5 file this object is stored in. + */ virtual String filename() const = 0; + /** Get object name. + @return The name of this object. + */ const String& name() const; - const ContentBase* parent() const; - RangePtr range() const; - hid_t id() const; - inline bool isOpen() const { return mId != 0; } + /** Get parent object. + @return Pointer to the parent of this object. + */ + const ContentBase* parent() const; + + /** Get object id. + @return hdf5 id of the h5 object maintained by this object. + */ + hid_t id() const; + + /** Check if object is open, i.e. if there is a valid hdf5 id. + @return True if object is open else false. + */ + bool isOpen() const; + + /** Add attribute to this object. + @tparam T Attribute value type. + @param name Attribute name. + @param value Attribute value. + */ template ContentBase& addAttribute(const String& name, const T& value); + + /** Get an attribute of this object. + @param name Attribute name. + @return The attribute value as DType. + */ DType getAttribute(const String& name) const; + + /** Check if attribute of given name exists in this object. + @param name Attribute name. + @return True if attribute exists else false. + */ bool attributeExists(const String& name) const; + + /** Get all attributes of this object. + @return Std map of key-value pairs. + */ std::map getAttributes() const; + + /** Get all attributes of this object and all its parent objects, recursively. + @return Std map of key-value pairs. + */ std::map getRecursiveAttributes() const; // + all parent's attributes protected: - String mName; - const ContentBase* mParent = nullptr; - RangePtr mRange; - hid_t mId = 0; + String mName; /**< Name of this object. */ + const ContentBase* mParent = nullptr; /**< Pointer to this object's parent. */ + hid_t mId = 0; /**< hdf5 identifier of the hdf5 object handled by this object. */ }; + /** Shortcut for a shared pointer to an abstract content object. */ typedef Sptr ContentPtr; } diff --git a/src/opt/hdf5/include/h5_file.h b/src/opt/hdf5/include/h5_file.h index 79dfaf5..d6157cc 100644 --- a/src/opt/hdf5/include/h5_file.h +++ b/src/opt/hdf5/include/h5_file.h @@ -1,3 +1,13 @@ +// -*- C++ -*- +/** + + @file opt/hdf5/include/h5_file.h + @brief Group declaration. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ #ifndef __cxz_h5_file_h__ #define __cxz_h5_file_h__ @@ -10,14 +20,25 @@ namespace CNORXZ { namespace hdf5 { - // maybe introduce abstraction layer between as base for File and Group + /** **** + Class to handle hdf5 file objects. + Objects of this type usually serve as root object + so they don't have any parent. + */ class File : public Group { public: typedef URange RangeT; - DEFAULT_MEMBERS(File); + DEFAULT_MEMBERS(File); /**< Default constructors and assignments. */ + + /** Construct the class. + @param _name Path to the hdf5 file to be handled. + @param _ro Open in read-only mode if true, otherwise have write access. + */ File(const String& fname, bool _ro = true); + + /** Destructor. Release all involved hdf5 ids. */ ~File(); virtual ContentType type() const override final; diff --git a/src/opt/hdf5/include/h5_group.h b/src/opt/hdf5/include/h5_group.h index 986d04a..6b85f89 100644 --- a/src/opt/hdf5/include/h5_group.h +++ b/src/opt/hdf5/include/h5_group.h @@ -1,3 +1,13 @@ +// -*- C++ -*- +/** + + @file opt/hdf5/include/h5_group.h + @brief Group declaration. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ #ifndef __cxz_h5_group_h__ #define __cxz_h5_group_h__ @@ -9,11 +19,21 @@ namespace CNORXZ { namespace hdf5 { + /** **** + Class to handle hdf5 groups. + */ class Group : public ContentBase { public: - DEFAULT_MEMBERS(Group); + DEFAULT_MEMBERS(Group); /**< Default constructors and assignments. */ + + /** Construct the class. + @param _name Group name. + @param _parent Parent content object. + */ Group(const String& gname, const ContentBase* _parent); + + /** Destructor. Release all involved hdf5 ids. */ ~Group(); virtual ContentType type() const override; @@ -23,43 +43,114 @@ namespace CNORXZ virtual String path() const override; virtual String filename() const override; - virtual Int exists() const; + /** Check if group exists in the parent object. + @return 1 if group exists, 0 if not, -1 if an error occured. + */ + virtual Int exists() const; // TODO: virtual function of base class + /** Get object contained by this group. + @param name Object name. + @return Pointer to the object. + */ const ContentPtr& get(const String& name) const; + + /** Get object contained by this group as group. + Checks if the object is a group. + @param name Group name. + @return Pointer to the group. + */ Sptr getGroup(const String& name) const; + + /** Get object contained by this group as table. + Checks if the object is a table. + @param name Table name. + @return Pointer to the table. + */ Sptr
getTable(const String& name) const; + + /** Get object contained by this group as dataset. + Checks if the object is a table. + @param name Dataset name. + @return Pointer to the dataset. + */ Sptr getDataset(const String& name) const; + /** Get object contained by this group as table for given value type. + Checks if the object is a table. + @tparam Ts Table entry types. + @param name Table name. + @param proto Empty prototype (template argument resolution). + @return Pointer to the table. + */ template Sptr> getTable(const String& name, Tuple proto); + /** Get object contained by this group as dataset for given value type. + Checks if the object is a dataset. + @tparam T Dataset value type. + @param name Dataset name. + @param proto Empty prototype (template argument resolution). + @return Pointer to the dataset. + */ template Sptr> getDataset(const String& name, T proto); + /** Get objects contained by this group. + @returns MArray of object pointers. + */ const MArray& get() const; + + /** Add a new group to this group. + @param name Name of the created group. + */ Group& addGroup(const String& name); + /** Add a new table to this group. + @tparam Ts Table element types. + @param name Name of the created table. + @param data Table data. + @param fnames Table field names. + */ template Group& addTable(const String& name, const ArrayBase>& data, const Arr& fnames); + /** Add a new dataset to this group. + @tparam T Value type. + @param name Name of the created dataset. + @param data Dataset data. + */ template Group& addDataset(const String& name, const ArrayBase& data); + /** Iterate over all group elements (const). + @param F function object to be executed on each group element. + */ template decltype(auto) iter(F&& f) const; + /** Iterate recursively over all group elements (const). + @param F function object to be executed on each group element. + */ template decltype(auto) iterRecursive(F&& f) const; + /** Iterate over all group elements. + @param F function object to be executed on each group element. + */ template decltype(auto) iter(F&& f); + /** Iterate recursively over all group elements. + @param F function object to be executed on each group element. + */ template decltype(auto) iterRecursive(F&& f); protected: + MArray mCont; /**< Group elements. */ + template static void recursion(const C& c, F&& f) { @@ -72,8 +163,6 @@ namespace CNORXZ } } - MArray mCont; - void mkCont(); AIndex getIndexTo(const String& name) const; BIndex getIndexTo(const String& name); diff --git a/src/opt/hdf5/include/h5_table.h b/src/opt/hdf5/include/h5_table.h index c8fa306..6cf82b1 100644 --- a/src/opt/hdf5/include/h5_table.h +++ b/src/opt/hdf5/include/h5_table.h @@ -1,3 +1,13 @@ +// -*- C++ -*- +/** + + @file opt/hdf5/include/h5_table.h + @brief Table declaration. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ #ifndef __cxz_h5_table_h__ #define __cxz_h5_table_h__ @@ -9,13 +19,23 @@ namespace CNORXZ { namespace hdf5 { + /** **** + Class to handle hdf5 tables. + */ class Table : public ContentBase { public: typedef std::pair FieldID; - DEFAULT_MEMBERS(Table); + DEFAULT_MEMBERS(Table); /**< Default constructors and assignments. */ + + /** Construct the class. + @param _name Table name. + @param _parent Parent content object. + */ Table(const String& name, const ContentBase* _parent); + + /** Destructor. Release all involved hdf5 ids. */ ~Table(); virtual ContentType type() const override final; @@ -25,17 +45,57 @@ namespace CNORXZ virtual String path() const override final; virtual String filename() const override final; + /** Ininitialize table field names. + @param fnames Table field names. + */ Table& initFieldNames(const Vector& fnames); + + /** Initialize the table. + @param n Number of records (rows). + @param data Table data. + @param dsize Record type size. + @param chunk_size Chunk size. + */ Table& initTable(SizeT n, const void* data, SizeT dsize, SizeT chunk_size); + + /** Append records to the table. + @param n Number of records to append. + @param data Records data. + */ Table& appendRecords(SizeT n, const void* data); + + /** Read records. + @param pos Number of first record to read. + @param n Number of records to read. + @param data Target pointer. + */ Table& readRecords(SizeT pos, SizeT n, char* data); + + /** Read record. + @param pos Number of the record to be read. + @return DType array with containing the record data. + */ MArray readRecord(SizeT pos) const; + + /** Read table. + @return DType array containing the table data. + */ MArray read() const; + /** Iterate over table records. + @param F function object to be executed on each table record. + */ template decltype(auto) iterRecords(F&& f) const; - + + /** Get fields range. + @return Pointer to the range. + */ const RangePtr& fields() const; + + /** Get records range. + @return Pointer to the range. + */ const RangePtr& records() const; protected: diff --git a/src/opt/hdf5/lib/h5_content_base.cc b/src/opt/hdf5/lib/h5_content_base.cc index ce338a6..8a7a1b3 100644 --- a/src/opt/hdf5/lib/h5_content_base.cc +++ b/src/opt/hdf5/lib/h5_content_base.cc @@ -1,3 +1,15 @@ +// -*- C++ -*- +/** + + @file opt/hdf5/lib/h5_content_base.cc + @brief Content base member function implementation. + + Implementation of all non-virtual member functions of ContentBase. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ #include "h5_content_base.h" @@ -19,16 +31,16 @@ namespace CNORXZ return mParent; } - RangePtr ContentBase::range() const - { - return mRange; - } - hid_t ContentBase::id() const { return mId; } + bool isOpen() const + { + return mId != 0; + } + DType ContentBase::getAttribute(const String& name) const { CXZ_ASSERT(attributeExists(name), "no attribute with name '" << name -- 2.45.2 From 26831adb9483fcbe399542c766b31acb2be6c260 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Fri, 2 Feb 2024 11:55:27 +0100 Subject: [PATCH 25/32] fix compile error --- src/opt/hdf5/lib/h5_content_base.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/opt/hdf5/lib/h5_content_base.cc b/src/opt/hdf5/lib/h5_content_base.cc index 8a7a1b3..acf86d3 100644 --- a/src/opt/hdf5/lib/h5_content_base.cc +++ b/src/opt/hdf5/lib/h5_content_base.cc @@ -36,7 +36,7 @@ namespace CNORXZ return mId; } - bool isOpen() const + bool ContentBase::isOpen() const { return mId != 0; } -- 2.45.2 From 1f7ebae61ba4fc719f1dde66c87266702e00f127 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Fri, 2 Feb 2024 12:13:54 +0100 Subject: [PATCH 26/32] WIP: hdf5 documentation --- src/opt/hdf5/include/h5_table.h | 60 ++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/src/opt/hdf5/include/h5_table.h b/src/opt/hdf5/include/h5_table.h index 6cf82b1..12eb075 100644 --- a/src/opt/hdf5/include/h5_table.h +++ b/src/opt/hdf5/include/h5_table.h @@ -99,36 +99,70 @@ namespace CNORXZ const RangePtr& records() const; protected: - RangePtr mRecords; - RangePtr mFields; // -> FIndex (position -> offset) - MArray mSizes; - MArray mOffsets; - MArray mTypes; - hid_t mType = 0; - SizeT mTypesize = 0; - MArray> mInterpret; + RangePtr mRecords; /**< Records range. */ + RangePtr mFields; /**< Fields range. */ // -> FIndex (position -> offset) + MArray mSizes; /**< Field element type sizes. */ + MArray mOffsets; /**< Field element offsets. */ + MArray mTypes; /**< Field element type ids. */ + hid_t mType = 0; /**< Record type id. */ + SizeT mTypesize = 0; /**< Record type size. */ + MArray> mInterpret; /**< Field element type interpreting functions. */ - void mkTypes(); + void mkTypes(); /**< Type setup function. */ }; - // caution: the memory ordering is the only thing that counts; - // std::tuple has REVERSE ordering! + /** **** + Class to handle hdf5 tables, the record type is known at compile time. + The records are accessed by a std tuple. + Caution: The ordering of the record entries is fixed by their memory location. + The std tuple has a reverse ordering w.r.t. the memory location, i.e. a record + with element types T1-T2-T3 (memory ordering) and field names "T1", "T2", "T3", + requires template argumens . + @tparam Ts Record element types. + */ template class STable : public Table { public: - DEFAULT_MEMBERS(STable); + DEFAULT_MEMBERS(STable); /**< Default constructors and assignments. */ + + /** Construct the class. + @param _name Table name. + @param _parent Parent content object. + */ STable(const String& name, const ContentBase* _parent); + + /** Construct the class. + @param _name Table name. + @param _parent Parent content object. + @param fnames Field names. + */ STable(const String& name, const ContentBase* _parent, const Arr& fnames); + /** Ininitialize and setup table fields. + @param fnames Table field names. + */ STable& initFields(const Arr& fnames); + /** Append record to the table. + @param t Tuple containing the record entries. + */ STable& appendRecord(const Tuple& t); + + /** Append records to the table. + @param t Array of tuples containing the records. + */ STable& appendRecord(const MArray>& t); + /** Read the table. + @return Array of tuples containing the records. + */ MArray> read() const; - + + /** Iterate over all table records. + @param f Function object to be executed on each record. + */ template decltype(auto) iterRecords(F&& f) const; }; -- 2.45.2 From c46d9b1772c02cffb4b4feaf42c3ee449957e8f5 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Fri, 2 Feb 2024 14:51:51 +0100 Subject: [PATCH 27/32] hdf5: virtual function exists() in ContentBase + hdf5 header documentation --- src/include/base/types.h | 20 +++--- src/opt/hdf5/include/h5_content_base.cc.h | 10 +++ src/opt/hdf5/include/h5_content_base.h | 5 ++ src/opt/hdf5/include/h5_dataset.cc.h | 10 +++ src/opt/hdf5/include/h5_dataset.h | 61 ++++++++++++++++-- src/opt/hdf5/include/h5_file.h | 8 ++- src/opt/hdf5/include/h5_group.cc.h | 30 +++++++++ src/opt/hdf5/include/h5_group.h | 6 +- src/opt/hdf5/include/h5_table.cc.h | 10 +++ src/opt/hdf5/include/h5_table.h | 1 + src/opt/hdf5/include/h5_type_id.cc.h | 50 +++++---------- src/opt/hdf5/include/h5_type_id.h | 78 +++++++++++++++-------- src/opt/hdf5/include/h5_types.h | 12 ++++ src/opt/hdf5/lib/h5_file.cc | 24 ++++--- src/opt/hdf5/lib/h5_group.cc | 4 +- src/opt/hdf5/lib/h5_table.cc | 9 ++- src/opt/hdf5/tests/h5_basic_unit_test.cc | 2 +- 17 files changed, 241 insertions(+), 99 deletions(-) diff --git a/src/include/base/types.h b/src/include/base/types.h index 8af8b40..77dbc98 100644 --- a/src/include/base/types.h +++ b/src/include/base/types.h @@ -6,7 +6,7 @@ This file contains the declaration of all library types - Copyright (c) 2022 Christian Zimmermann. All rights reserved. + Copyright (c) 2024 Christian Zimmermann. All rights reserved. Mail: chizeta@f3l.de **/ @@ -28,9 +28,9 @@ namespace CNORXZ { - /********************** - * standard types * - **********************/ + /*====================+ + | standard types | + =====================*/ typedef std::intptr_t PtrId; typedef int32_t Int; @@ -80,9 +80,9 @@ namespace CNORXZ template using CSizeT = std::integral_constant; - /********************* - * library types * - *********************/ + /*===================+ + | library types | + +===================*/ /*** Naming Prefixes: @@ -302,9 +302,9 @@ namespace CNORXZ template class Contraction; - /********************* - * derived types * - *********************/ + /*===================+ + | derived types | + +===================*/ template using Vector = std::vector>; diff --git a/src/opt/hdf5/include/h5_content_base.cc.h b/src/opt/hdf5/include/h5_content_base.cc.h index 690f82e..994c110 100644 --- a/src/opt/hdf5/include/h5_content_base.cc.h +++ b/src/opt/hdf5/include/h5_content_base.cc.h @@ -1,3 +1,13 @@ +// -*- C++ -*- +/** + + @file opt/hdf5/include/h5_content_base.cc.h + @brief Implementation of template member functions of ContentBase + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ #ifndef __cxz_h5_content_base_cc_h__ #define __cxz_h5_content_base_cc_h__ diff --git a/src/opt/hdf5/include/h5_content_base.h b/src/opt/hdf5/include/h5_content_base.h index 7a4dc03..58923a2 100644 --- a/src/opt/hdf5/include/h5_content_base.h +++ b/src/opt/hdf5/include/h5_content_base.h @@ -81,6 +81,11 @@ namespace CNORXZ */ virtual String filename() const = 0; + /** Check if group exists in the parent object. + @return True if object exists, else false. + */ + virtual bool exists() const = 0; + /** Get object name. @return The name of this object. */ diff --git a/src/opt/hdf5/include/h5_dataset.cc.h b/src/opt/hdf5/include/h5_dataset.cc.h index 6df2e42..9c4aaeb 100644 --- a/src/opt/hdf5/include/h5_dataset.cc.h +++ b/src/opt/hdf5/include/h5_dataset.cc.h @@ -1,3 +1,13 @@ +// -*- C++ -*- +/** + + @file opt/hdf5/include/h5_dataset.cc.h + @brief Implementation of template member functions of Dataset and SDataset. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ #ifndef __cxz_h5_dataset_cc_h__ #define __cxz_h5_dataset_cc_h__ diff --git a/src/opt/hdf5/include/h5_dataset.h b/src/opt/hdf5/include/h5_dataset.h index b9b2ed6..ae998d6 100644 --- a/src/opt/hdf5/include/h5_dataset.h +++ b/src/opt/hdf5/include/h5_dataset.h @@ -1,3 +1,13 @@ +// -*- C++ -*- +/** + + @file opt/hdf5/include/h5_dataset.h + @brief Dataset declaration. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ #ifndef __cxz_h5_dataset_h__ #define __cxz_h5_dataset_h__ @@ -9,11 +19,21 @@ namespace CNORXZ { namespace hdf5 { + /** **** + Class to handle hdf5 datasets. + */ class Dataset : public ContentBase { public: - DEFAULT_MEMBERS(Dataset); + DEFAULT_MEMBERS(Dataset); /**< Default constructors and assignments. */ + + /** Construct the class. + @param _name Dataset name. + @param _parent Parent content object. + */ Dataset(const String& name, const ContentBase* _parent); + + /** Destructor. Release all involved hdf5 ids. */ ~Dataset(); virtual ContentType type() const override final; @@ -22,32 +42,59 @@ namespace CNORXZ virtual Dataset& close() override final; virtual String path() const override final; virtual String filename() const override final; + virtual bool exists() const override final; - bool exists() const; - + /** Initalize the dataset. + @param dataRange A potentially multi-dimensional range characterizing the dataset. + @param type Data type id. + */ Dataset& init(const RangePtr& dataRange, hid_t type); + /** Initalize the dataset. + @param data Array containing the dataset. + */ template Dataset& init(const ArrayBase& data); + /** Get the data range. + @return Pointer to the range. + */ const RangePtr& dataRange() const; protected: - RangePtr mDataRange; - hid_t mType; - hid_t mFilespace; + RangePtr mDataRange; /**< The data range. */ + hid_t mType; /**< The data type identifier. */ + hid_t mFilespace; /**< The hdf5 file space identifier. */ }; + /** **** + Class to handle hdf5 datasets, the value type is known at compile time. + @tparam T Dataset value type. + */ template class SDataset : public Dataset { public: - DEFAULT_MEMBERS(SDataset); + DEFAULT_MEMBERS(SDataset); /**< Default constructors and assignments. */ + + /** Construct the class. + @param _name Dataset name. + @param _parent Parent content object. + */ SDataset(const String& name, const ContentBase* _parent); + /** Read the dataset. + @return Array containing the dataset values. + */ MArray read() const; + /** Read a given subset of the dataset. + The subset needs to be hypercubic. + @param beg Index indicating the begin edge of the hypercube. + @param end Index indicating the end edge of the hypercube (inclusive). + @return Array containing the dataset values. + */ template MArray read(const IndexInterface& beg, const IndexInterface& end) const; diff --git a/src/opt/hdf5/include/h5_file.h b/src/opt/hdf5/include/h5_file.h index d6157cc..98a5b91 100644 --- a/src/opt/hdf5/include/h5_file.h +++ b/src/opt/hdf5/include/h5_file.h @@ -47,9 +47,13 @@ namespace CNORXZ virtual File& close() override final; virtual String path() const override final; virtual String filename() const override final; + virtual bool exists() const override final; + + /** Check if handled file is in hdf5 format. + @return True if file is in hdf5 format, else false. + */ + bool ishdf5() const; - virtual Int exists() const override final; - private: bool mRo = true; }; diff --git a/src/opt/hdf5/include/h5_group.cc.h b/src/opt/hdf5/include/h5_group.cc.h index 9ad13f6..e64fb38 100644 --- a/src/opt/hdf5/include/h5_group.cc.h +++ b/src/opt/hdf5/include/h5_group.cc.h @@ -1,3 +1,13 @@ +// -*- C++ -*- +/** + + @file opt/hdf5/include/h5_group.cc.h + @brief Implementation of template member functions of Group. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ #ifndef __cxz_h5_group_cc_h__ #define __cxz_h5_group_cc_h__ @@ -9,6 +19,26 @@ namespace CNORXZ { namespace hdf5 { + template + constexpr const auto& tget(const Tuple& t) + { + return std::get(t); + } + + template + constexpr auto& tget(Tuple& t) + { + return std::get(t); + } + + template + SizeT getTupleOffset(const Tuple& t, CSizeT i) + { + const PtrId beg = reinterpret_cast(&t); + const PtrId pos = reinterpret_cast(&tget(t)); + return pos - beg; + } + template Sptr> Group::getTable(const String& name, Tuple proto) { diff --git a/src/opt/hdf5/include/h5_group.h b/src/opt/hdf5/include/h5_group.h index 6b85f89..41d297a 100644 --- a/src/opt/hdf5/include/h5_group.h +++ b/src/opt/hdf5/include/h5_group.h @@ -42,11 +42,7 @@ namespace CNORXZ virtual Group& close() override; virtual String path() const override; virtual String filename() const override; - - /** Check if group exists in the parent object. - @return 1 if group exists, 0 if not, -1 if an error occured. - */ - virtual Int exists() const; // TODO: virtual function of base class + virtual bool exists() const override; /** Get object contained by this group. @param name Object name. diff --git a/src/opt/hdf5/include/h5_table.cc.h b/src/opt/hdf5/include/h5_table.cc.h index 5509ffa..d46d2c3 100644 --- a/src/opt/hdf5/include/h5_table.cc.h +++ b/src/opt/hdf5/include/h5_table.cc.h @@ -1,3 +1,13 @@ +// -*- C++ -*- +/** + + @file opt/hdf5/include/h5_table.cc.h + @brief Implementation of template member functions of Table and STable. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ #ifndef __cxz_h5_table_cc_h__ #define __cxz_h5_table_cc_h__ diff --git a/src/opt/hdf5/include/h5_table.h b/src/opt/hdf5/include/h5_table.h index 12eb075..79d193f 100644 --- a/src/opt/hdf5/include/h5_table.h +++ b/src/opt/hdf5/include/h5_table.h @@ -44,6 +44,7 @@ namespace CNORXZ virtual Table& close() override final; virtual String path() const override final; virtual String filename() const override final; + virtual bool exists() const override final; /** Ininitialize table field names. @param fnames Table field names. diff --git a/src/opt/hdf5/include/h5_type_id.cc.h b/src/opt/hdf5/include/h5_type_id.cc.h index 322ff76..b7624bd 100644 --- a/src/opt/hdf5/include/h5_type_id.cc.h +++ b/src/opt/hdf5/include/h5_type_id.cc.h @@ -1,3 +1,13 @@ +// -*- C++ -*- +/** + + @file opt/hdf5/include/h5_type_id.h + @brief TypeId template implementation. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ #ifndef __cxz_h5_type_id_cc_h__ #define __cxz_h5_type_id_cc_h__ @@ -8,37 +18,9 @@ namespace CNORXZ { namespace hdf5 { - template - decltype(auto) reverseTuple(const Tuple& t) - { - return iter<0,sizeof...(Ts)> - ( [&](auto i) { return tget(t); }, - [](const auto&... e) { return std::make_tuple(e...); } ); - } - - template - constexpr const auto& tget(const Tuple& t) - { - return std::get(t); - } - - template - constexpr auto& tget(Tuple& t) - { - return std::get(t); - } - - template - SizeT getTupleOffset(const Tuple& t, CSizeT i) - { - const PtrId beg = reinterpret_cast(&t); - const PtrId pos = reinterpret_cast(&tget(t)); - return pos - beg; - } - - /************** - * TypeId * - **************/ + /*============+ + | TypeId | + +============*/ template inline hid_t TypeId::get() @@ -68,9 +50,9 @@ namespace CNORXZ return arrtype; } - /***************** - * getTypeId * - *****************/ + /*===============+ + | getTypeId | + +===============*/ template hid_t getTypeId(T x) diff --git a/src/opt/hdf5/include/h5_type_id.h b/src/opt/hdf5/include/h5_type_id.h index b024d82..896de9f 100644 --- a/src/opt/hdf5/include/h5_type_id.h +++ b/src/opt/hdf5/include/h5_type_id.h @@ -1,3 +1,13 @@ +// -*- C++ -*- +/** + + @file opt/hdf5/include/h5_type_id.h + @brief TypeId template declaration. + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + + **/ #ifndef __cxz_h5_type_id_h__ #define __cxz_h5_type_id_h__ @@ -8,43 +18,57 @@ namespace CNORXZ { namespace hdf5 { - template - decltype(auto) reverseTuple(const Tuple& t); - - template - using RTuple = typename std::remove_reference()))>::type; - template - constexpr const auto& tget(const Tuple& t); - - template - constexpr auto& tget(Tuple& t); - - template - SizeT getTupleOffset(const Tuple& t, CSizeT i); - - /************** - * TypeId * - **************/ - + /** **** + Get a valid hdf5 id for a given type. + Return 0 if not implemented. + @tparam T The type. + */ template - struct TypeId { static inline hid_t get(); }; + struct TypeId + { + /** Get the type id. */ + static inline hid_t get(); + }; + /** Specialization of TypeId for SizeT. */ template <> - struct TypeId { static inline hid_t get(); }; + struct TypeId + { + /** Get the type id for SizeT. */ + static inline hid_t get(); + }; + /** Specialization of TypeId for Int. */ template <> - struct TypeId { static inline hid_t get(); }; + struct TypeId + { + /** Get the type id for Int. */ + static inline hid_t get(); + }; + /** Specialization of TypeId for Double. */ template <> - struct TypeId { static inline hid_t get(); }; + struct TypeId + { + /** Get the type id for Double. */ + static inline hid_t get(); + }; + /** Specialization of TypeId for an array + @tparam T Element type. + @tparam N Array size. + */ template - struct TypeId> { static inline hid_t get(); }; - - /***************** - * getTypeId * - *****************/ + struct TypeId> + { + /** Get the type id for the given array. */ + static inline hid_t get(); + }; + /** Wrapper function for TypeId::get() + @param Variable of given type. + @return A type id of the input variable's type. + */ template hid_t getTypeId(T x); } diff --git a/src/opt/hdf5/include/h5_types.h b/src/opt/hdf5/include/h5_types.h index cbb4fa8..46beb8d 100644 --- a/src/opt/hdf5/include/h5_types.h +++ b/src/opt/hdf5/include/h5_types.h @@ -1,3 +1,15 @@ +// -*- C++ -*- +/** + + @file opt/hdf5/include/types.h + @brief Declaration of hdf5 related library types + + This file contains the declaration of all hdf5 related library types + + Copyright (c) 2024 Christian Zimmermann. All rights reserved. + Mail: chizeta@f3l.de + +**/ #ifndef __h5_types_h__ #define __h5_types_h__ diff --git a/src/opt/hdf5/lib/h5_file.cc b/src/opt/hdf5/lib/h5_file.cc index ea05180..8878062 100644 --- a/src/opt/hdf5/lib/h5_file.cc +++ b/src/opt/hdf5/lib/h5_file.cc @@ -33,7 +33,7 @@ namespace CNORXZ } Int ex = this->exists(); const String fn = this->filename(); - CXZ_ASSERT( ex != 2, "tried to open non-h5 file '" << fn << "'" ); + CXZ_ASSERT( ishdf5(), "tried to open non-h5 file '" << fn << "'" ); if(mRo){ CXZ_ASSERT( ex == 1, "could not open file as read-only: '" << fn << "' does not exist'"); @@ -79,21 +79,25 @@ namespace CNORXZ return name(); } - Int File::exists() const + bool File::exists() const { - Int ex = 0; + bool ex = false; std::ifstream fs(this->filename().c_str(), std::ios_base::binary); if(fs.good()){ - ex = 1; // file exists + ex = true; // file exists } fs.close(); - if(ex != 0){ - if(H5Fis_hdf5(this->filename().c_str()) <= 0){ - ex = 2; // file not in h5 format - } - } return ex; } - + + bool File::ishdf5() const + { + if(exists()){ + if(H5Fis_hdf5(this->filename().c_str()) <= 0){ + return false; + } + } + return true; // a non-existing file can be created in hdf5 format + } } } diff --git a/src/opt/hdf5/lib/h5_group.cc b/src/opt/hdf5/lib/h5_group.cc index 65b21aa..1541d4d 100644 --- a/src/opt/hdf5/lib/h5_group.cc +++ b/src/opt/hdf5/lib/h5_group.cc @@ -69,9 +69,9 @@ namespace CNORXZ return String(); } - Int Group::exists() const + bool Group::exists() const { - return H5Lexists(mParent->id(), mName.c_str(), H5P_DEFAULT) != 0 ? 1 : 0; + return H5Lexists(mParent->id(), mName.c_str(), H5P_DEFAULT) != 0; } const ContentPtr& Group::get(const String& name) const diff --git a/src/opt/hdf5/lib/h5_table.cc b/src/opt/hdf5/lib/h5_table.cc index e68754c..83a264e 100644 --- a/src/opt/hdf5/lib/h5_table.cc +++ b/src/opt/hdf5/lib/h5_table.cc @@ -9,7 +9,7 @@ namespace CNORXZ Table::Table(const String& name, const ContentBase* _parent) : ContentBase(name, _parent) { - if(H5Lexists(mParent->id(), mName.c_str(), H5P_DEFAULT)){ + if(exists()){ hsize_t nfields = 0; hsize_t nrecords = 0; H5TBget_table_info(mParent->id(), mName.c_str(), &nfields, &nrecords); @@ -90,6 +90,13 @@ namespace CNORXZ return mParent->filename(); } + bool Table::exists() const + { + htri_t x = H5Lexists(mParent->id(), mName.c_str(), H5P_DEFAULT); + VCHECK(x); + return x; + } + Table& Table::initFieldNames(const Vector& fnames) { CXZ_ASSERT(mFields == nullptr, "fields already initialized"); diff --git a/src/opt/hdf5/tests/h5_basic_unit_test.cc b/src/opt/hdf5/tests/h5_basic_unit_test.cc index 7e0cc87..69bacce 100644 --- a/src/opt/hdf5/tests/h5_basic_unit_test.cc +++ b/src/opt/hdf5/tests/h5_basic_unit_test.cc @@ -58,7 +58,7 @@ namespace CIndex j(fs); for(i = 0; i.lex() != rs->size(); ++i){ iter<0,3>( [&](auto jj) - { j = jj; mTabD[i*j] = DType(tget(v[i.lex()])); }, NoF{} ); + { j = jj; mTabD[i*j] = DType(std::get<3-jj-1>(v[i.lex()])); }, NoF{} ); } const SizeT ddim = 5; -- 2.45.2 From 27466f6ef15e66eb7e19a56afcd2dc0932ceb776 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Fri, 2 Feb 2024 15:11:03 +0100 Subject: [PATCH 28/32] hdf5: fix doxy errors + remove some unused code --- src/opt/hdf5/include/h5_dataset.h | 4 ++-- src/opt/hdf5/include/h5_file.h | 4 +--- src/opt/hdf5/include/h5_group.h | 25 +++++++++++++++++++------ src/opt/hdf5/include/h5_table.h | 10 +++++----- 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/src/opt/hdf5/include/h5_dataset.h b/src/opt/hdf5/include/h5_dataset.h index ae998d6..671af1b 100644 --- a/src/opt/hdf5/include/h5_dataset.h +++ b/src/opt/hdf5/include/h5_dataset.h @@ -28,7 +28,7 @@ namespace CNORXZ DEFAULT_MEMBERS(Dataset); /**< Default constructors and assignments. */ /** Construct the class. - @param _name Dataset name. + @param name Dataset name. @param _parent Parent content object. */ Dataset(const String& name, const ContentBase* _parent); @@ -79,7 +79,7 @@ namespace CNORXZ DEFAULT_MEMBERS(SDataset); /**< Default constructors and assignments. */ /** Construct the class. - @param _name Dataset name. + @param name Dataset name. @param _parent Parent content object. */ SDataset(const String& name, const ContentBase* _parent); diff --git a/src/opt/hdf5/include/h5_file.h b/src/opt/hdf5/include/h5_file.h index 98a5b91..9ab01b2 100644 --- a/src/opt/hdf5/include/h5_file.h +++ b/src/opt/hdf5/include/h5_file.h @@ -28,12 +28,10 @@ namespace CNORXZ class File : public Group { public: - typedef URange RangeT; - DEFAULT_MEMBERS(File); /**< Default constructors and assignments. */ /** Construct the class. - @param _name Path to the hdf5 file to be handled. + @param fname Path to the hdf5 file to be handled. @param _ro Open in read-only mode if true, otherwise have write access. */ File(const String& fname, bool _ro = true); diff --git a/src/opt/hdf5/include/h5_group.h b/src/opt/hdf5/include/h5_group.h index 41d297a..7fc4629 100644 --- a/src/opt/hdf5/include/h5_group.h +++ b/src/opt/hdf5/include/h5_group.h @@ -28,7 +28,7 @@ namespace CNORXZ DEFAULT_MEMBERS(Group); /**< Default constructors and assignments. */ /** Construct the class. - @param _name Group name. + @param gname Group name. @param _parent Parent content object. */ Group(const String& gname, const ContentBase* _parent); @@ -120,25 +120,25 @@ namespace CNORXZ Group& addDataset(const String& name, const ArrayBase& data); /** Iterate over all group elements (const). - @param F function object to be executed on each group element. + @param f function object to be executed on each group element. */ template decltype(auto) iter(F&& f) const; /** Iterate recursively over all group elements (const). - @param F function object to be executed on each group element. + @param f function object to be executed on each group element. */ template decltype(auto) iterRecursive(F&& f) const; /** Iterate over all group elements. - @param F function object to be executed on each group element. + @param f function object to be executed on each group element. */ template decltype(auto) iter(F&& f); /** Iterate recursively over all group elements. - @param F function object to be executed on each group element. + @param f function object to be executed on each group element. */ template decltype(auto) iterRecursive(F&& f); @@ -147,6 +147,10 @@ namespace CNORXZ MArray mCont; /**< Group elements. */ + /** Recursion helper functon. + @param c Group element. + @param f Function to be executed. + */ template static void recursion(const C& c, F&& f) { @@ -158,9 +162,18 @@ namespace CNORXZ } } } - + + /** Setup group content. */ void mkCont(); + + /** Get index to requested group element (const). + @param name Element name. + */ AIndex getIndexTo(const String& name) const; + + /** Get index to requested group element. + @param name Element name. + */ BIndex getIndexTo(const String& name); }; } diff --git a/src/opt/hdf5/include/h5_table.h b/src/opt/hdf5/include/h5_table.h index 79d193f..cdccc23 100644 --- a/src/opt/hdf5/include/h5_table.h +++ b/src/opt/hdf5/include/h5_table.h @@ -25,12 +25,12 @@ namespace CNORXZ class Table : public ContentBase { public: - typedef std::pair FieldID; + typedef std::pair FieldID; /**< Meta data type of fields range. */ DEFAULT_MEMBERS(Table); /**< Default constructors and assignments. */ /** Construct the class. - @param _name Table name. + @param name Table name. @param _parent Parent content object. */ Table(const String& name, const ContentBase* _parent); @@ -84,7 +84,7 @@ namespace CNORXZ MArray read() const; /** Iterate over table records. - @param F function object to be executed on each table record. + @param f function object to be executed on each table record. */ template decltype(auto) iterRecords(F&& f) const; @@ -128,13 +128,13 @@ namespace CNORXZ DEFAULT_MEMBERS(STable); /**< Default constructors and assignments. */ /** Construct the class. - @param _name Table name. + @param name Table name. @param _parent Parent content object. */ STable(const String& name, const ContentBase* _parent); /** Construct the class. - @param _name Table name. + @param name Table name. @param _parent Parent content object. @param fnames Field names. */ -- 2.45.2 From 190032729a8848b2e09fcc7cb5f8bea186eefbaf Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Fri, 2 Feb 2024 16:27:18 +0100 Subject: [PATCH 29/32] urange: fix bug (seg fault) in getMeta --- src/include/ranges/urange.cc.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/include/ranges/urange.cc.h b/src/include/ranges/urange.cc.h index 39b4b1b..603d096 100644 --- a/src/include/ranges/urange.cc.h +++ b/src/include/ranges/urange.cc.h @@ -300,6 +300,7 @@ namespace CNORXZ auto b = mSpace.begin(); auto e = mSpace.end(); auto i = std::lower_bound(b, e, meta, std::less()); + CXZ_ASSERT(i != e, "element with meta data = " << toString(meta) << " not in range"); // check this first, otherwise the next test may potentially result in a seg fault! CXZ_ASSERT(*i == meta, "element with meta data = " << toString(meta) << " not in range"); return i - b; } -- 2.45.2 From 2023d64a5fd9e43a5ddb4be7173d63c3c36eea79 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Fri, 2 Feb 2024 19:25:09 +0100 Subject: [PATCH 30/32] hdf5: clean up + fixes --- src/opt/hdf5/lib/h5_dataset.cc | 2 +- src/opt/hdf5/lib/h5_file.cc | 1 - src/opt/hdf5/lib/h5_group.cc | 2 +- src/opt/hdf5/lib/h5_table.cc | 4 +--- src/opt/hdf5/tests/h5_basic_unit_test.cc | 1 - 5 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/opt/hdf5/lib/h5_dataset.cc b/src/opt/hdf5/lib/h5_dataset.cc index e535250..1fbfe72 100644 --- a/src/opt/hdf5/lib/h5_dataset.cc +++ b/src/opt/hdf5/lib/h5_dataset.cc @@ -71,7 +71,7 @@ namespace CNORXZ bool Dataset::exists() const { - return H5Lexists(mParent->id(), mName.c_str(), H5P_DEFAULT); + return H5Lexists(mParent->id(), mName.c_str(), H5P_DEFAULT) > 0; } Dataset& Dataset::init(const RangePtr& dataRange, hid_t type) diff --git a/src/opt/hdf5/lib/h5_file.cc b/src/opt/hdf5/lib/h5_file.cc index 8878062..fe42fa5 100644 --- a/src/opt/hdf5/lib/h5_file.cc +++ b/src/opt/hdf5/lib/h5_file.cc @@ -70,7 +70,6 @@ namespace CNORXZ String File::path() const { - //return String("/"); return ""; } diff --git a/src/opt/hdf5/lib/h5_group.cc b/src/opt/hdf5/lib/h5_group.cc index 1541d4d..eecfa1d 100644 --- a/src/opt/hdf5/lib/h5_group.cc +++ b/src/opt/hdf5/lib/h5_group.cc @@ -71,7 +71,7 @@ namespace CNORXZ bool Group::exists() const { - return H5Lexists(mParent->id(), mName.c_str(), H5P_DEFAULT) != 0; + return H5Lexists(mParent->id(), mName.c_str(), H5P_DEFAULT) > 0; } const ContentPtr& Group::get(const String& name) const diff --git a/src/opt/hdf5/lib/h5_table.cc b/src/opt/hdf5/lib/h5_table.cc index 83a264e..268e8b3 100644 --- a/src/opt/hdf5/lib/h5_table.cc +++ b/src/opt/hdf5/lib/h5_table.cc @@ -92,9 +92,7 @@ namespace CNORXZ bool Table::exists() const { - htri_t x = H5Lexists(mParent->id(), mName.c_str(), H5P_DEFAULT); - VCHECK(x); - return x; + return H5Lexists(mParent->id(), mName.c_str(), H5P_DEFAULT) > 0; } Table& Table::initFieldNames(const Vector& fnames) diff --git a/src/opt/hdf5/tests/h5_basic_unit_test.cc b/src/opt/hdf5/tests/h5_basic_unit_test.cc index 69bacce..41399b7 100644 --- a/src/opt/hdf5/tests/h5_basic_unit_test.cc +++ b/src/opt/hdf5/tests/h5_basic_unit_test.cc @@ -131,7 +131,6 @@ namespace File h5f(mFileName, false); h5f.open(); h5f.getGroup("gr1")->open().addTable("tab1", mTabA, mFs); - //h5f.getGroup("gr2")->open().addTable("tab1", mTabA, mFs); h5f.getGroup("moregroups")->open().getGroup("evenmore")->open().getGroup("need")->open().addTable("tab1", mTabA, mFs); h5f.close(); } -- 2.45.2 From dc7137f4004e2697246ebc27db7975844d25e6ba Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Fri, 2 Feb 2024 20:28:22 +0100 Subject: [PATCH 31/32] hdf5: fix oinfo bug occuring for older hdf5 versions --- src/opt/hdf5/lib/h5_group.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/opt/hdf5/lib/h5_group.cc b/src/opt/hdf5/lib/h5_group.cc index eecfa1d..7728b95 100644 --- a/src/opt/hdf5/lib/h5_group.cc +++ b/src/opt/hdf5/lib/h5_group.cc @@ -180,10 +180,9 @@ namespace CNORXZ index(); H5O_info_t oinfo; #if H5_VERS_MINOR > 10 - //H5Oget_info(id, &oinfo, H5O_INFO_BASIC); H5Oget_info_by_name(id, name, &oinfo, H5O_INFO_BASIC, H5P_DEFAULT); #else - H5Oget_info(id, &oinfo); + H5Oget_info_by_name(id, name, &oinfo, H5P_DEFAULT); #endif switch (oinfo.type) { case H5O_TYPE_GROUP: { -- 2.45.2 From 1a26f1407f488e00c189d5b7c3f5b5284a7d7040 Mon Sep 17 00:00:00 2001 From: Christian Zimmermann Date: Fri, 2 Feb 2024 20:31:50 +0100 Subject: [PATCH 32/32] remove deprecated definition in .woodpecker.yml --- .woodpecker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index c80761c..d857ff9 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -7,7 +7,7 @@ steps: - source /opt/rh/gcc-toolset-9/enable - mkdir build-gcc - cd build-gcc - - cmake3 -DRUN_PIPELINE=on -DSCALAR_BUILD=on .. + - cmake3 -DSCALAR_BUILD=on .. - make -j2 - make test - cd .. -- 2.45.2