10 #ifndef UTOPIA_DATAIO_HDFDATASET_HH
11 #define UTOPIA_DATAIO_HDFDATASET_HH
15 #include <unordered_map>
21 #include "../core/type_traits.hh"
67 this->
_log->debug(
"Creating dataset with typesize {} at path {} ...",
72 hid_t group_plist = H5Pcreate(H5P_LINK_CREATE);
73 H5Pset_create_intermediate_group(group_plist, 1);
85 this->
_log->debug(
"Computing chunksizes ...");
91 hid_t plist = H5Pcreate(H5P_DATASET_CREATE);
98 this->
_log->debug(
"Setting given chunksizes ...");
113 "Creating actual dataset and binding it to object class ...");
117 group_plist, plist, H5P_DEFAULT),
122 throw std::runtime_error(
"Invalid dataset id " +
_path +
" " +
134 "Creating actual dataset and binding it to object class ...");
138 group_plist, H5P_DEFAULT, H5P_DEFAULT),
143 throw std::runtime_error(
"Invalid dataset id " +
_path +
" " +
148 this->
_log->debug(
"refcount of dataset after creation {}: {}", _path,
165 this->
_log->debug(
"Writing container data to dataset {}...",
_path);
167 this->
_log->debug(
"Dataset {} 's refcount write begin {}",
_path,
175 if constexpr (std::is_same_v<T, std::vector<value_type_1>> and
176 not Utils::is_container_v<value_type_1> and
177 not Utils::is_string_v<value_type_1>)
179 this->
_log->debug(
"... of simple vectortype");
184 this->
_log->debug(
"... dataset not yet existing, creating it "
185 "for simple vectortype");
186 __create_dataset__<base_type>(0);
191 "... dataset existing, reading out type and writing data");
195 temp_type.
open<base_type>(
"testtype", 0);
197 if (temp_type !=
_type)
199 throw std::runtime_error(
200 "Error, cannot write container data of a "
201 "different type into dataset " +
205 this->
_log->debug(
"Dataset {} 's refcount before write {}",
_path,
215 this->
_log->debug(
"... of nontrivial containertype");
217 std::size_t typesize = 0;
220 if constexpr (Utils::is_container_v<base_type> and
221 Utils::is_array_like_v<base_type>)
230 "... dataset not yet existing, creating it for array type");
231 __create_dataset__<base_type>(typesize);
236 this->
_log->debug(
"... dataset existing, reading out type");
239 temp_type.
open<base_type>(
"testtype", typesize);
241 if (temp_type !=
_type)
243 throw std::runtime_error(
244 "Error, cannot write fixedsize container data of a "
245 "different type into dataset " +
251 "... buffering data into vectortype appropriate for writing");
256 [](
auto &value) -> value_type_1 & {
return value; });
258 this->
_log->debug(
"Dataset {} 's refcount before write {}",
_path,
261 this->
_log->debug(
"... writing data");
278 this->
_log->debug(
"Writing string data to dataset {}...",
_path);
284 const char *buffer =
nullptr;
286 if constexpr (std::is_pointer_v<T>)
289 this->
_log->debug(
"... stringtype is pointer-valued");
290 len = std::strlen(data);
295 this->
_log->debug(
"... stringtype is of not pointer-valued");
297 buffer = data.c_str();
304 "... dataset not yet existing, creating it for stringtypee");
307 __create_dataset__<const char *>(len);
311 this->
_log->debug(
"... dataset existing, reading out type");
314 temp_type.
open<
const char *>(
"testtype", len);
316 if (temp_type !=
_type)
318 throw std::runtime_error(
"Error, cannot write string data of a "
319 "different type into dataset " +
324 this->
_log->debug(
" ... writing data");
343 this->
_log->debug(
"Writing pointer data to dataset {}...",
_path);
351 "... dataset not yet existing, creating it for pointertype");
353 __create_dataset__<basetype>(0);
358 this->
_log->debug(
"... dataset existing, reading out type");
361 temp_type.
open<basetype>(
"testtype", 0);
363 if (temp_type !=
_type)
365 throw std::runtime_error(
366 "Error, cannot write pointer data of a "
367 "different type into dataset " +
371 this->
_log->debug(
" ... writing data");
389 this->
_log->debug(
"Writing scalar data to dataset {}...",
_path);
395 "... dataset not yet existing, creating it for pointertype");
397 __create_dataset__<std::decay_t<T>>(0);
402 this->
_log->debug(
"... dataset existing, reading out type");
405 temp_type.
open<std::decay_t<T>>(
"testtype", 0);
407 if (temp_type !=
_type)
409 throw std::runtime_error(
"Error, cannot write scalar data of a "
410 "different type into dataset " +
415 this->
_log->debug(
" ... writing data");
429 this->
_log->debug(
"Reading container data from dataset {}...",
_path);
439 if constexpr (Utils::is_container_v<value_type_1> ||
440 Utils::is_string_v<value_type_1>)
443 "... reading nested container or container of strings ...");
450 if constexpr (Utils::is_container_v<value_type_2>)
452 throw std::runtime_error(
454 ": Cannot read data into nested containers with depth > 3 "
456 _path +
" into vector containers!");
458 if constexpr (!std::is_same_v<std::vector<value_type_1>, Type>)
460 throw std::runtime_error(
"Dataset" +
_path +
461 ": Can only read data"
462 " into vector containers!");
474 this->
_log->debug(
"... nested type is array-like...");
477 if constexpr (!Utils::is_array_like_v<value_type_1>)
482 throw std::invalid_argument(
484 ": Cannot read into container of non arrays "
485 "when data type in file is fixed array type");
490 H5P_DEFAULT, buffer.data());
494 this->
_log->debug(
"... nested type is string-like...");
496 if constexpr (!Utils::is_string_v<value_type_1>)
498 throw std::invalid_argument(
500 ": Can only read stringdata into string elements");
530 vlentype.
open<std::string>(
"vlentype_temporary", 0ul);
535 "... nested type of variable length type ...");
537 std::vector<char *> temp_buffer(buffer.size());
541 H5P_DEFAULT, &temp_buffer[0]);
553 for (
auto [b, tb] = std::make_tuple(
554 buffer.begin(), temp_buffer.begin());
555 b != buffer.end(); ++b, ++tb)
567 for (
auto &&c : temp_buffer)
578 "... nested type of fixed length type ...");
583 std::string temp_buffer;
585 temp_buffer.resize(buffer.size() * s);
591 H5P_DEFAULT, &temp_buffer[0]);
600 std::size_t buffidx = 0;
601 while (i < temp_buffer.size())
603 buffer[buffidx] = temp_buffer.substr(i, s);
617 "... nested type of variable length array type ... ");
619 std::vector<hvl_t> temp_buffer(buffer.size());
621 herr_t err = H5Dread(
628 this->
_log->debug(
"... transforming the read data to the "
629 "actually desired type ... ");
631 for (std::size_t i = 0; i < buffer.size(); ++i)
633 if constexpr (!Utils::is_array_like_v<value_type_1>)
635 buffer[i].resize(temp_buffer[i].len);
642 it !=
std::end(buffer[i]); ++it, ++j)
644 *it =
static_cast<value_type_2 *
>(temp_buffer[i].p)[j];
648 hid_t tempspace = H5Dget_space(
get_C_id());
651 #if H5_VERSION_GE(1, 12, 0)
667 throw std::runtime_error(
668 "Error when reclaiming memory in " +
_path +
669 " for variable_length datatype");
676 throw std::runtime_error(
678 ": Unknown kind of datatype in dataset when requesting to "
679 "read into container");
686 this->
_log->debug(
"... no nested type to read");
698 this->
_log->debug(
"Reading string data from dataset {}...",
_path);
700 buffer.resize(buffer.size() *
_type.
size());
712 this->
_log->debug(
"Reading pointer data from dataset {}...",
_path);
721 this->
_log->debug(
"Reading scalar data from dataset {}...",
_path);
730 auto log = spdlog::get(
"data_io");
732 log->debug(
"Writing attribute buffer of dataset {}...",
_path);
743 log->debug(
"... currently at attribute {}", path);
751 [&attr](
auto &&arg) {
753 std::forward<std::remove_reference_t<decltype(arg)>>(
814 std::vector<std::pair<std::string, typename HDFType::Variant>>
925 throw std::runtime_error(
927 ": Cannot set capacity after dataset has been created");
931 _rank = capacity.size();
945 throw std::runtime_error(
947 ": Cannot set chunksize after dataset has been created");
951 if (chunksizes.size() !=
_rank and chunksizes.size() != 0)
953 throw std::runtime_error(
955 ": Chunksizes size has to be equal to dataset rank");
975 template <
typename Attrdata>
981 this->
_log->debug(
"Add attribute {} to valid dataset {}",
982 attribute_path,
_path);
990 this->
_log->debug(
"Add atttribute {} to attribute buffer of {} "
991 "because it has not yet been created on disk",
992 attribute_path,
_path);
996 if constexpr (Utils::is_container_v<Attrdata>)
998 if constexpr (not std::is_same_v<
999 std::vector<typename Attrdata::value_type>,
1005 std::vector<typename Attrdata::value_type>(
1012 std::make_pair(attribute_path, data));
1019 std::make_pair(attribute_path, data));
1033 auto log = spdlog::get(
"data_io");
1064 template <HDFCategory cat>
1066 std::vector<hsize_t> capacity = {},
1067 std::vector<hsize_t> chunksizes = {}, hsize_t compress_level = 0)
1070 this->
_log->debug(
"Opening dataset {} within {}", path,
1091 std::vector<hsize_t> capacity = {},
1092 std::vector<hsize_t> chunksizes = {}, hsize_t compress_level = 0)
1095 if (not parent_identifier.
is_valid())
1097 throw std::runtime_error(
"parent id not valid for dataset " + path);
1122 this->
_log->debug(
"... binding existing dataset to object");
1138 hid_t creation_plist = H5Dget_create_plist(
get_C_id());
1139 hid_t layout = H5Pget_layout(creation_plist);
1140 if (layout == H5D_CHUNKED)
1146 throw std::runtime_error(
1147 "Dataset " +
_path +
1148 ": Error in reading out chunksizes while opening.");
1151 H5Pclose(creation_plist);
1159 _capacity.assign(capacity.begin(), capacity.end());
1164 this->
_log->debug(
"... dataset not yet existing, have to wait 'til "
1165 "data becomes available");
1171 if (capacity.size() == 0)
1204 swap(
static_cast<Base &
>(*
this),
static_cast<Base &
>(other));
1226 template <
typename T>
1227 void write(T &&data, [[maybe_unused]] std::vector<hsize_t> shape = {})
1229 this->
_log->debug(
"Writing data to dataset {}",
_path);
1230 this->
_log->debug(
"... current extent {}",
Utils::str(_current_extent));
1247 throw std::runtime_error(
"Rank > 2 not supported");
1284 else if constexpr (std::is_pointer_v<std::decay_t<T>> and
1287 if (shape.size() == 0)
1289 throw std::runtime_error(
1290 "Dataset " +
_path +
1291 ": shape has to be given explicitly when writing "
1337 throw std::runtime_error(
"Dataset " +
_path +
1338 ": Error, dataset cannot be extended "
1339 "because it reached its capacity");
1372 else if constexpr (std::is_pointer_v<std::decay_t<T>> and
1375 if (shape.size() == 0)
1377 throw std::runtime_error(
1378 "Dataset " +
_path +
1379 ": shape has to be given explicitly when writing "
1383 for (std::size_t i = 0; i <
_rank; ++i)
1412 std::vector<hsize_t> counts(
_rank, 0);
1417 counts = {data.size()};
1421 counts = {1, data.size()};
1425 else if constexpr (std::is_pointer_v<std::decay_t<T>> and
1436 for (std::size_t i = 0; i <
_rank; ++i)
1440 throw std::runtime_error(
"Dataset " +
_path +
1441 ": Cannot append data, "
1442 "_new_extent larger than capacity "
1453 throw std::runtime_error(
1454 "Dataset " +
_path +
1455 ": Error when trying to increase extent");
1466 arma::Row<hsize_t>(
_offset) + arma::Row<hsize_t>(counts),
1485 throw std::runtime_error(
"Dataset " +
_path +
1486 ": Error in appending container");
1494 throw std::runtime_error(
"Dataset " +
_path +
1495 ": Error in appending string");
1498 else if constexpr (std::is_pointer_v<std::decay_t<T>> and
1504 throw std::runtime_error(
"Dataset " +
_path +
1505 ": Error in appending pointer");
1513 throw std::runtime_error(
"Dataset " +
_path +
1514 ": Error in appending scalar");
1532 template <
typename Iter,
typename Adaptor>
1535 this->
_log->debug(
"Writing iterator range to dataset {}",
_path);
1542 std::vector<Type> buff(std::distance(
begin,
end));
1544 std::generate(buff.begin(), buff.end(),
1545 [&
begin, &adaptor]() { return adaptor(*(begin++)); });
1573 template <
typename T, std::
size_t d>
1575 std::vector<hsize_t> offset = {})
1577 this->
_log->debug(
"Writing N-dimensional dataset to dataset {}",
_path);
1578 this->
_log->debug(
"... current extent {}",
Utils::str(_current_extent));
1599 if (
_capacity == std::vector<hsize_t>{H5S_UNLIMITED} and
_rank == 1)
1604 for (std::size_t i = 0; i <
_rank; ++i)
1615 for (
auto [i, j] = std::make_tuple(
_rank - d, 0); i <
_rank;
1622 _log->debug(
"Dataset {} does not exist yet, properties were "
1627 _log->debug(
" datashape: {}",
Utils::str(std::vector<std::size_t>(
1628 data.shape(), data.shape() + d)));
1638 throw std::invalid_argument(
1639 "Error, the dimensionality of the dataset, which is " +
1641 ", must be >= the dimensionality of the data to be "
1642 "written, which is " +
1648 _log->debug(
"Dataset {} does exist",
_path);
1649 _log->debug(
"Properties of data to be written");
1651 _log->debug(
" datashape: {}",
1653 data.shape(), data.shape() + d)));
1656 "Properties before change for accomodating new data");
1668 if (offset.size() != 0)
1677 for (std::size_t i = 0; i <
_rank - d; ++i)
1685 for (
auto [i, j] = std::make_tuple(
_rank - d, 0ul); i < d;
1694 throw std::runtime_error(
1695 "Dataset " +
_path +
": Capacity[" +
1698 ", which is too small for a desired new "
1710 throw std::runtime_error(
1711 "Dataset " +
_path +
1712 ": Error when trying to increase extent");
1726 throw std::runtime_error(
1727 "Error in " +
_path +
", capacity " +
1730 " is too small for new extent " +
1735 std::make_tuple(1ul, (d ==
_rank) ? 1ul : 0ul);
1736 i <
_rank && j < d; ++i, ++j)
1744 throw std::runtime_error(
1745 "Error in " +
_path +
1758 throw std::runtime_error(
1759 "Dataset " +
_path +
1760 ": Error when trying to increase extent");
1777 std::vector<hsize_t> counts(
_rank, 1);
1779 for (
auto [i, j] = std::make_tuple(
_rank - d, 0); i <
_rank;
1782 counts[i] = data.shape()[j];
1795 arma::Row<hsize_t>(counts),
1802 "Properties after change for accomodating new data");
1816 if constexpr (std::is_scalar_v<std::decay_t<T>>)
1820 __create_dataset__<std::decay_t<T>>(0);
1825 temp_type.open<std::decay_t<T>>(
"testtype", 0);
1826 if (
_type != temp_type)
1828 throw std::runtime_error(
"Error, cannot write data of a "
1829 "different type into dataset " +
1840 throw std::runtime_error(
1841 "Dataset " +
_path +
1842 ": Error in writing nd-array holding scalar values");
1849 __create_dataset__<std::decay_t<T>>(0);
1854 temp_type.open<std::decay_t<T>>(
"testtype", 0);
1856 if (
_type != temp_type)
1858 throw std::runtime_error(
"Error, cannot write data of a "
1859 "different type into dataset " +
1864 boost::multi_array<const char *, d> buffer(
1865 reinterpret_cast<boost::array<size_t, d>
const &
>(
1871 [](
auto &&
str) { return str.c_str(); });
1880 throw std::runtime_error(
1881 "Dataset " +
_path +
1882 ": Error in writing nd-array holding string values");
1889 hsize_t typesize = Utils::get_size<std::decay_t<T>>::value;
1894 __create_dataset__<std::decay_t<T>>(typesize);
1899 temp_type.open<std::decay_t<T>>(
"testtype", typesize);
1900 if (
_type != temp_type)
1902 throw std::runtime_error(
1903 "Error, cannot write data of a "
1904 "different type into dataset " +
1915 throw std::runtime_error(
1916 "Dataset " +
_path +
1917 ": Error in writing nd-array holding array values");
1925 __create_dataset__<std::decay_t<T>>(0);
1930 temp_type.open<std::decay_t<T>>(
"temp_type", 0);
1931 if (
_type != temp_type)
1933 throw std::runtime_error(
1934 "Error, cannot write data of a "
1935 "different type into dataset " +
1940 if constexpr (std::is_same_v<
1941 std::vector<typename T::value_type>,
1945 boost::multi_array<hvl_t, d> buffer(
1946 reinterpret_cast<boost::array<size_t, d>
const &
>(
1950 data.data(), data.data() + data.num_elements(),
1951 buffer.data(), [](
auto &&v) {
1960 const_cast<Utils::remove_qualifier_t<decltype(
1961 v.data())> *>(v.data())};
1965 herr_t err = H5Dwrite(
1971 throw std::runtime_error(
"Dataset " +
_path +
1972 ": Error in writing nd-array "
1973 "holding vector values");
1983 boost::multi_array<std::vector<typename T::value_type>, d>
1985 reinterpret_cast<boost::array<size_t, d>
const &
>(
1988 boost::multi_array<hvl_t, d> buffer(
1989 reinterpret_cast<boost::array<size_t, d>
const &
>(
1993 data.data(), data.data() + data.num_elements(),
1994 vector_buffer.data(), [](
auto &&v) {
1995 return std::vector<typename T::value_type>(
1996 v.begin(), v.end());
2000 vector_buffer.data() +
2001 vector_buffer.num_elements(),
2002 buffer.data(), [](
auto &&v) {
2003 return hvl_t{v.size(), v.data()};
2007 herr_t err = H5Dwrite(
2013 throw std::runtime_error(
2014 "Dataset " +
_path +
2015 ": Error in writing nd-array holding non-vector "
2016 "container values");
2036 template <
typename Type>
2037 auto read([[maybe_unused]] std::vector<hsize_t> start = {},
2038 [[maybe_unused]] std::vector<hsize_t>
end = {},
2039 [[maybe_unused]] std::vector<hsize_t> stride = {})
2042 "Reading dataset {}, starting at {}, ending at {}, using stride {}",
2047 throw std::runtime_error(
"Dataset " + _path +
2048 ": Dataset id is invalid");
2055 std::vector<hsize_t> readshape;
2058 std::size_t size = 1;
2061 if (start.size() == 0)
2063 readshape = _current_extent;
2068 for (
auto &s : readshape)
2077 if (start.size() != _rank or
end.size() != _rank or
2078 stride.size() != _rank)
2080 throw std::invalid_argument(
2081 "Dataset " + _path +
2082 ": start, end, stride have to be "
2083 "same size as dataset rank, which is " +
2093 std::vector<hsize_t> count(start.size());
2097 for (std::size_t i = 0; i < _rank; ++i)
2099 count[i] = (
end[i] - start[i]) / stride[i];
2102 for (
auto &s : count)
2112 _filespace.
open(*
this);
2113 _memspace.
open(_path +
" memory dataspace", _rank, count, {});
2115 this->_log->debug(
"... selecting slice in filespace for dataset {}",
2124 if constexpr (Utils::is_container_v<Type>)
2127 herr_t err = __read_container__(buffer);
2130 throw std::runtime_error(
"Dataset " + _path +
2131 ": Error reading container type ");
2133 return std::make_tuple(readshape, buffer);
2135 else if constexpr (Utils::is_string_v<Type>)
2142 buffer.resize(size);
2143 herr_t err = __read_stringtype__(buffer);
2146 throw std::runtime_error(
"Dataset " + _path +
2147 ": Error reading string type ");
2150 return std::make_tuple(readshape, buffer);
2152 else if constexpr (std::is_pointer_v<Type> && !Utils::is_string_v<Type>)
2154 std::shared_ptr<Utils::remove_qualifier_t<Type>> buffer(
2155 new Utils::remove_qualifier_t<Type>[size],
2156 std::default_delete<Utils::remove_qualifier_t<Type>[]>());
2158 herr_t err = __read_pointertype__(buffer.get());
2162 std::runtime_error(
"Dataset " + _path +
2163 ": Error reading pointer type ");
2165 return std::make_tuple(readshape, buffer);
2170 herr_t err = __read_scalartype__(buffer);
2173 std::runtime_error(
"Dataset " + _path +
2174 ": Error reading scalar type ");
2176 return std::make_tuple(readshape, buffer);
2230 template <HDFCategory cat>
2232 std::vector<hsize_t> capacity = {},
2233 std::vector<hsize_t> chunksizes = {}, hsize_t compress_level = 0)
2236 open(parent_object, path, capacity, chunksizes, compress_level);
Class for hdf5 attribute, which can be attached to groups and datasets.
Definition: hdfattribute.hh:46
void write(Type attribute_data, std::vector< hsize_t > shape={})
Function for writing data to the attribute.
Definition: hdfattribute.hh:780
static auto buffer(Iter begin, Iter end, Adaptor &&adaptor)
static function for turning an iterator range with arbitrarty datatypes into a vector of data as retu...
Definition: hdfbufferfactory.hh:96
Class representing a HDFDataset, wich reads and writes data and attributes.
Definition: hdfdataset.hh:53
std::vector< std::pair< std::string, typename HDFType::Variant > > _attribute_buffer
A buffer for storing attributes before the dataset exists.
Definition: hdfdataset.hh:815
auto get_capacity()
get the maximum extend of the dataset
Definition: hdfdataset.hh:900
std::vector< hsize_t > _current_extent
the currently occupied size of the dataset in number of elements
Definition: hdfdataset.hh:776
void write(T &&data, [[maybe_unused]] std::vector< hsize_t > shape={})
Writes data of arbitrary type.
Definition: hdfdataset.hh:1227
HDFDataset(const HDFDataset &other)=default
Copy constructor.
auto get_current_extent()
get the current extend of the dataset
Definition: hdfdataset.hh:887
HDFDataspace get_filespace()
Get the file dataspace id.
Definition: hdfdataset.hh:861
void write(Iter begin, Iter end, Adaptor &&adaptor)
Write function for writing iterator ranges [start, end), in accordance with respective stl pattern.
Definition: hdfdataset.hh:1533
HDFDataset & operator=(const HDFDataset &other)=default
Assignment operator.
std::vector< hsize_t > _chunksizes
the chunksizes per dimensions if dataset is extendible or compressed
Definition: hdfdataset.hh:787
herr_t __read_container__(Type &buffer)
Read a cointainer.
Definition: hdfdataset.hh:426
HDFDataspace _filespace
file dataspace identifier
Definition: hdfdataset.hh:827
HDFDataspace get_memspace()
Get the memory dataspace id.
Definition: hdfdataset.hh:854
auto get_attribute_buffer()
Returns the attribute buffer of this dataset.
Definition: hdfdataset.hh:866
auto get_chunksizes()
Get the chunksizes vector.
Definition: hdfdataset.hh:907
herr_t __write_scalartype__(T data)
Writes simple scalars, which are not pointers, containers or strings.
Definition: hdfdataset.hh:387
void set_chunksize(std::vector< hsize_t > chunksizes)
Set the chunksize object.
Definition: hdfdataset.hh:941
auto read([[maybe_unused]] std::vector< hsize_t > start={}, [[maybe_unused]] std::vector< hsize_t > end={}, [[maybe_unused]] std::vector< hsize_t > stride={})
Read (a subset of ) a dataset into a buffer of type 'Type'. Type gives the type of the buffer to read...
Definition: hdfdataset.hh:2037
void __write_attribute_buffer__()
write out the attribute buffer
Definition: hdfdataset.hh:728
void open(const HDFObject< cat > &parent_object, std::string path, std::vector< hsize_t > capacity={}, std::vector< hsize_t > chunksizes={}, hsize_t compress_level=0)
Open the dataset in parent_object with relative path 'path'.
Definition: hdfdataset.hh:1065
HDFDataset & operator=(HDFDataset &&other)=default
Move assignment operator.
herr_t __write_pointertype__(T data)
Writes pointers, shape is like numpy shape arg.
Definition: hdfdataset.hh:341
void add_attribute(std::string attribute_path, Attrdata data)
add attribute to the dataset
Definition: hdfdataset.hh:976
virtual ~HDFDataset()
Destructor.
Definition: hdfdataset.hh:2242
auto __read_stringtype__(Type &buffer)
read attirbute data which contains a single string.
Definition: hdfdataset.hh:696
HDFDataset(HDFDataset &&other)=default
Move constructor.
auto __read_pointertype__(Type buffer)
read pointertype.
Definition: hdfdataset.hh:710
HDFDataset(HDFObject< cat > &parent_object, std::string path, std::vector< hsize_t > capacity={}, std::vector< hsize_t > chunksizes={}, hsize_t compress_level=0)
Construct a new HDFDataset object.
Definition: hdfdataset.hh:2231
auto get_type()
Get the type object.
Definition: hdfdataset.hh:847
HDFIdentifier _parent_identifier
Identifier of the parent object.
Definition: hdfdataset.hh:766
std::size_t get_rank()
get the rank of the dataset, i.e. the dimensionality
Definition: hdfdataset.hh:880
HDFType _type
Type of the data the dataset holds.
Definition: hdfdataset.hh:821
std::vector< hsize_t > _new_extent
buffer for extent update
Definition: hdfdataset.hh:799
void swap(HDFDataset &other)
swap the state of the objects
Definition: hdfdataset.hh:1200
auto get_offset()
Get the offset object.
Definition: hdfdataset.hh:894
hsize_t _rank
number of dimensions of the dataset
Definition: hdfdataset.hh:771
HDFIdentifier get_parent_id()
get a shared_ptr to the parent_object
Definition: hdfdataset.hh:873
HDFDataspace _memspace
memory dataspace identifier
Definition: hdfdataset.hh:833
std::vector< hsize_t > _offset
offset of the data
Definition: hdfdataset.hh:793
auto __read_scalartype__(Type &buffer)
read scalar type, trivial
Definition: hdfdataset.hh:719
std::size_t _compress_level
the level of compression, 0 to 10
Definition: hdfdataset.hh:804
void write_nd(const boost::multi_array< T, d > &data, std::vector< hsize_t > offset={})
Write a boost::multi_array of arbitrary type and dimension to the dataset. The dataset needs to be of...
Definition: hdfdataset.hh:1574
void close()
Close the dataset.
Definition: hdfdataset.hh:1031
auto get_compresslevel()
Get the compress level object.
Definition: hdfdataset.hh:914
void __create_dataset__(std::size_t typesize)
helper function for making a non compressed dataset
Definition: hdfdataset.hh:64
std::vector< hsize_t > _capacity
the maximum number of elements which can be stored in the dataset
Definition: hdfdataset.hh:781
herr_t __write_container__(T &&data)
Writes containers to the dataset.
Definition: hdfdataset.hh:163
void set_capacity(std::vector< hsize_t > capacity)
Set the capacity object, and sets rank of dataset to capacity.size.
Definition: hdfdataset.hh:921
void open(const HDFIdentifier &parent_identifier, std::string path, std::vector< hsize_t > capacity={}, std::vector< hsize_t > chunksizes={}, hsize_t compress_level=0)
Open the dataset in parent_object with relative path 'path'.
Definition: hdfdataset.hh:1090
herr_t __write_stringtype__(T data)
writes stringtypes
Definition: hdfdataset.hh:276
HDFDataset()=default
default consturctor
Class that wraps an HDF5 dataspace and takes care of managing its resources.
Definition: hdfdataspace.hh:37
Wrapper class around an hdf5 identifier, used to manage reference counts of the object this identifie...
Definition: hdfidentifier.hh:29
hid_t get_id() const
Get the HDF5 id held by this object.
Definition: hdfidentifier.hh:53
bool is_valid() const
Check if thi ID refers to a valid object.
Definition: hdfidentifier.hh:77
void set_id(hid_t id)
Set id to the given argument. Only to be used to invalidate objects upon move or similar.
Definition: hdfidentifier.hh:65
Common base class for all HDF5 classes in the DATAIO Module i.e., for all classes that wrap HDF5-C-Li...
Definition: hdfobject.hh:37
auto get_id_object() const
Get the id object.
Definition: hdfobject.hh:99
std::string _path
Name of the object.
Definition: hdfobject.hh:50
auto get_refcount()
Get the reference count of object.
Definition: hdfobject.hh:131
std::string get_path() const
Get the name or path object.
Definition: hdfobject.hh:88
void close()
Close function which takes care of correctly closing the object and managing the reference counter.
Definition: hdfobject.hh:161
virtual bool is_valid() const
Check if the object is still valid.
Definition: hdfobject.hh:143
std::shared_ptr< spdlog::logger > _log
pointer to the logger for dataio
Definition: hdfobject.hh:56
HDFIdentifier _id
Identifier object that binds an instance of this class to an HDF5 object.
Definition: hdfobject.hh:44
hid_t get_C_id() const
Get the C id object.
Definition: hdfobject.hh:120
void bind_to(hid_t id, std::function< herr_t(hid_t) > closing_func, std::string path={})
Open the object and bind it to a HDF5 object identified by 'id' with name 'path'. Object should be cr...
Definition: hdfobject.hh:186
Class which handles the conversion of C-types into hdf5types.
Definition: hdftype.hh:136
void open(T &&object)
Open the HDF5 type associated with an HDFObject, i.e., a dataset or an attribute.
Definition: hdftype.hh:224
void close()
Construct close from the given arguments.
Definition: hdftype.hh:322
auto type_category() const
Get the type category of the held type, i.e., scala, string, varlen,...
Definition: hdftype.hh:199
std::size_t size() const
Size of the type held in bytes.
Definition: hdftype.hh:210
OutputIt transform(const Utopia::ExecPolicy policy, InputIt first1, InputIt last1, OutputIt d_first, UnaryOperation unary_op)
Apply a unary operator to a range and store the result in a new range.
Definition: parallel.hh:368
const Cont calc_chunksize(const hsize_t typesize, const Cont io_extend, Cont max_extend={}, const bool opt_inf_dims=true, const bool larger_high_dims=true, const unsigned int CHUNKSIZE_MAX=1048576, const unsigned int CHUNKSIZE_MIN=8192, const unsigned int CHUNKSIZE_BASE=262144)
Try to guess a good chunksize for a dataset.
Definition: hdfchunking.hh:604
std::string to_string(const Config &node)
Given a config node, returns a string representation of it.
Definition: cfg_utils.hh:110
void swap(WriteTask< BGB, DW, DB, AWG, AWD > &lhs, WriteTask< BGB, DW, DB, AWG, AWD > &rhs)
Swaps the state of lhs and rhs.
Definition: write_task.hh:240
void select_slice(arma::Row< hsize_t > start, arma::Row< hsize_t > end, arma::Row< hsize_t > stride)
Select a slice in the dataspace defined by [start, end, stride] in the manner of numpy....
Definition: hdfdataspace.hh:225
auto path_is_valid(hid_t id, std::string path)
Check if the path given relative to the object identified by 'id' exists and points to a valid hdf5 o...
Definition: hdfutilities.hh:150
void swap(HDFDataset &lhs, HDFDataset &rhs)
Exchange state between lhs and rhs.
Definition: hdfdataset.hh:2253
hsize_t rank()
Get thet dataspace's rank, i.e., number of dimensions.
Definition: hdfdataspace.hh:46
void open()
Open the dataspace - set it to be equivalent to any data that later will be used to write or read.
Definition: hdfdataspace.hh:117
std::pair< arma::Row< hsize_t >, arma::Row< hsize_t > > get_properties()
Get the properties object: size and capacity. @notice The dimensions can be infered from the size of ...
Definition: hdfdataspace.hh:67
This file implements a C++ class which wraps a C HDF5 attribute to a HDF5-object (group or dataset),...
In this file, a class for automatically creating intermediate buffer data structures between the user...
This file provides a class which is responsible for the automatic conversion between C/C++ types and ...
This file provides metafunctions for automatically determining the nature of a C/C++ types at compile...
std::string str(T &&t)
Turn any object for which operator<< exists into a string. Mostly useful for logging data via spdlog ...
Definition: ostream.hh:164
constexpr bool is_container_v
Shorthand for 'is_container::value.
Definition: type_traits.hh:181
typename remove_qualifier< T >::type remove_qualifier_t
Shorthand for 'typename remove_qualifier::value'.
Definition: type_traits.hh:97
constexpr bool is_array_like_v
Shorthand for is_array_like<T>::value.
Definition: type_traits.hh:633
constexpr bool is_string_v
Shorthand for 'is_string<T>::value'.
Definition: type_traits.hh:140
Return the size of a Type T containing other types at compile time. If no object for which an overloa...
Definition: type_traits.hh:438