Utopia::DataIO::_chunk_helpers Namespace Reference

Functions
template<typename Cont , typename Predicate >
std::vector< unsigned short >	find_all_idcs (Cont &vec, Predicate pred)
	Finds all indices of elements in a vector that matches the given predicate.
template<typename Cont = std::vector< hsize_t >>
std::string	to_str (const Cont &vec)
	Helper function to create a string representation of containers.
template<typename Cont , typename Logger >
void	opt_chunks_target (Cont &chunks, double bytes_target, const hsize_t typesize, const unsigned int CHUNKSIZE_MAX, const unsigned int CHUNKSIZE_MIN, const bool larger_high_dims, const Logger &log)
	Optimizes the chunks along all axes to find a good default.
template<typename Cont , typename Logger >
void	opt_chunks_with_max_extend (Cont &chunks, const Cont &max_extend, const hsize_t typesize, const unsigned int CHUNKSIZE_MAX, const bool opt_inf_dims, const bool larger_high_dims, const Logger &log)
	Optimize chunk sizes using max_extend information.

Function Documentation

find_all_idcs()

template<typename Cont , typename Predicate >

std::vector< unsigned short > Utopia::DataIO::_chunk_helpers::find_all_idcs	(	Cont &	vec,
		Predicate	pred
	)

Finds all indices of elements in a vector that matches the given predicate.

Parameters

vec	The object to find the indices in
pred	The predicate to determine the indices to be found

Template Parameters

Cont	The container type
Predicate	The predicate type

{
    // Create the return container
    std::vector< unsigned short > idcs;
 
    // Repeatedly start iterating over the vector until reached the end
    auto iter = vec.begin();
    while ((iter = std::find_if(iter, vec.end(), pred)) != vec.end())
    {
        // Add the value of the iterator to the indices vector
        idcs.push_back(std::distance(vec.begin(), iter));
 
        // Increment iterator to continue with next element
        iter++;
    }
 
    return idcs;
};

opt_chunks_target()

template<typename Cont , typename Logger >

void Utopia::DataIO::_chunk_helpers::opt_chunks_target	(	Cont &	chunks,
		double	bytes_target,
		const hsize_t	typesize,
		const unsigned int	CHUNKSIZE_MAX,
		const unsigned int	CHUNKSIZE_MIN,
		const bool	larger_high_dims,
		const Logger &	log
	)

Optimizes the chunks along all axes to find a good default.

This algorithm is only aware of the current size of the chunks and the target byte size of a chunk. Given that information, it either tries to reduce the extend of chunk dimensions, or enlarge it. To do that, it iterates over all chunk dimensions and either doubles the extend or halves it. Once within 50% of the target byte size, the algorithm stops. Also, it takes care to remain within the bounds of CHUNKSIZE_MAX and CHUNKSIZE_MIN. If a target byte size outside of these bounds is given, it will adjust it accordingly. For a typesize larger than CHUNKSIZE_MAX, this algorithm cannot perform any reasonable actions and will throw an exception; this case should be handled outside of this function!

Parameters

chunks	The current chunk values that are to be optimized
bytes_target	Which byte size to optimize the chunks to
typesize	The byte size of a single entry, needed to calculate the total bytesize of a whole chunk
CHUNKSIZE_MAX	The maximum allowed bytesize of a chunk
CHUNKSIZE_MIN	The minimum allowed bytesize of a chunk
larger_high_dims	If true, dimensions with high indices will be favoured for enlarging chunk extend in that dim
log	The logger object to use

{
    // Helper lambda for calculating bytesize of a chunks configuration
    auto bytes = [&typesize](Cont c) {
        return typesize *
               std::accumulate(c.begin(), c.end(), 1, std::multiplies<>());
    };
 
    // Check the case of typesize larger than CHUNKSIZE_MAX; cannot do anything
    // in that case -> safer to throw an exception.
    if (typesize > CHUNKSIZE_MAX)
    {
        throw std::invalid_argument("Cannot use opt_chunks_target with a "
                                    "typesize larger than CHUNKSIZE_MAX!");
    }
 
    log->debug("Starting optimization towards target size:"
               "  {:7.0f}B  ({:.1f} kiB)",
               bytes_target,
               bytes_target / 1024.);
 
    // Ensure the target chunk size is between CHUNKSIZE_MIN and CHUNKSIZE_MAX
    // in order to not choose too large or too small chunks
    if (bytes_target > CHUNKSIZE_MAX)
    {
        bytes_target = CHUNKSIZE_MAX;
 
        log->debug("Target size too large! New target size:"
                   "  {:7.0f}B  ({:.1f} kiB)",
                   bytes_target,
                   bytes_target / 1024.);
    }
    else if (bytes_target < CHUNKSIZE_MIN)
    {
        bytes_target = CHUNKSIZE_MIN;
 
        log->debug("Target size too small! New target size:"
                   "  {:7.0f}B  ({:.1f} kiB)",
                   bytes_target,
                   bytes_target / 1024.);
    }
 
    // ... and a variable that will store the size (in bytes) of this specific
    // chunk configuration
    std::size_t bytes_chunks;
 
    // Calculate the rank (need it to know iteration -> dim mapping)
    auto rank = chunks.size();
 
    /* Now optimize the chunks for each dimension by repeatedly looping over
     * the vector and dividing the values by two (rounding up).
     *
     * The loop is left when the following condition is fulfilled:
     *   within 50% of target chunk size
     *   AND
     *   within bounds of minimum and maximum chunk size
     *
     * NOTE Limiting the optimization to 42 iterations in order to be on the
     *      safe side. This goes through all entries (rank / 42) times and
     *      halves or doubles the chunk extent.
     */
    for (unsigned short i = 0; i < 42 * rank; i++)
    {
        // With the current values of the chunks, calculate the chunk size
        bytes_chunks = bytes(chunks);
 
        log->debug("Chunks:  {}  ->  {:7d} B  ({:.1f} kiB)",
                   to_str(chunks),
                   bytes_chunks,
                   bytes_chunks / 1024.);
 
        // If close enough to target size, optimization is finished
        if ((std::abs(bytes_chunks - bytes_target) / bytes_target < 0.5) &&
            bytes_chunks <= CHUNKSIZE_MAX && bytes_chunks >= CHUNKSIZE_MIN)
        {
            log->debug("Close enough to target size now.");
            break;
        }
        // else: not yet close enough
 
        // Calculate the dimension this iteration belongs to
        auto dim = i % rank;
 
        // Adjust the chunksize towards the target size
        if (bytes_chunks < bytes_target)
        {
            // Can increase the size of the chunk extend in the current dim
 
            // If high dimensions should be favoured, change the dim to work
            // on such that first the high dimensions are increased in size
            if (larger_high_dims)
            {
                dim = (rank - 1) - dim;
            }
 
            // Multiply by two
            log->debug("Doubling extend of chunk dimension {} ...", dim);
            chunks[dim] = chunks[dim] * 2;
        }
        else
        {
            // Need to decrease the size of the chunk extend in the current dim
 
            // For the larger_high_dims option, smaller dimensions are to be
            // favoured. However, in order to allow a reduction, these need to
            // have a chunk extend that is larger than one; once that is no
            // longer fulfilled, the below if condition will not perform
            // any change of the dim variable.
            if (larger_high_dims && rank > 1 && dim > 0 && chunks[dim - 1] > 1)
            {
                // Stay on low dimensions one step longer
                if (dim > 0)
                {
                    dim--;
                }
 
                // Skip the reduction if this is the last dim
                if (dim == rank - 1)
                {
                    log->debug("Skipping reduction of chunk dimension {}, "
                               "because it is the highest ...",
                               dim);
                    continue;
                }
            }
 
            // Do not continue if halving is not possible
            if (chunks[dim] == 1)
            {
                log->debug("Extend of chunk dimension {} is already 1.", dim);
                continue;
            }
 
            // TODO generalise the above if blocks! The cleanest way would be
            //      to determine which chunk dimensions _can_ be reduced before
            //      determining the dimension that is to be reduced. This would
            //      alleviate the iterations in which the chunk extent is 1
            //      and no halving can take place ...
 
            // Divide the chunk size of the current dim by two
            log->debug("Halving extend of chunk dimension {} ...", dim);
            chunks[dim] = 1 + ((chunks[dim] - 1) / 2); // ceiling!
            // NOTE integer division fun; can do this because all are unsigned
            //      and the chunks entry is always nonzero
        }
    }
 
    return;
}

opt_chunks_with_max_extend()

template<typename Cont , typename Logger >

void Utopia::DataIO::_chunk_helpers::opt_chunks_with_max_extend	(	Cont &	chunks,
		const Cont &	max_extend,
		const hsize_t	typesize,
		const unsigned int	CHUNKSIZE_MAX,
		const bool	opt_inf_dims,
		const bool	larger_high_dims,
		const Logger &	log
	)

Optimize chunk sizes using max_extend information.

This algorithm is aware of the maximum extend of a dataset and can use that information during optimization, aiming to increase the size of the chunks towards CHUNKSIZE_MAX as far as possible without going beyond max_extend. The paradigm here is that the number of chunks needed for read/write operations should be minimized while trying to keep a chunk's byte size below a certain value. The algorithm distinguishes between dimensions that have a finite extend and those that can grow to H5S_UNLIMITED, i.e. "infinite" extend. First, the aim is to try to cover the max_extend in the finite dimensions. It checks if an integer multiple is needed to reach the maximum extend. If, after that, the target CHUNKSIZE_MAX is not yet reached and the opt_inf_dims flag is set, the chunk sizes in the unlimited dimensions are extended as far as possible, assuming that they were chosen unlimited because they will be filled at some point and larger chunk sizes will reduce the number of chunks needed during read/write operations.

Parameters

chunks	The current chunk values that are to be optimized
max_extend	The maximum extend of the dataset
typesize	The byte size of a single entry, needed to calculate the total bytesize of a whole chunk
CHUNKSIZE_MAX	The maximum allowed bytesize of a chunk
opt_inf_dims	Whether to optimize the infinite dimensions or not
larger_high_dims	If true, dimensions with high indices will be favoured for enlarging chunk extend in that dim
log	The logger object to use

Template Parameters

Cont	The container type for the chunks
Logger	The logger type

{
    // Helper lambda for calculating bytesize of a chunks configuration
    auto bytes = [&typesize](Cont c) {
        return typesize *
               std::accumulate(c.begin(), c.end(), 1, std::multiplies<>());
    };
 
    // Check the case of typesize larger than CHUNKSIZE_MAX; cannot do anything
    // in that case -> safer to throw an exception.
    if (typesize > CHUNKSIZE_MAX)
    {
        throw std::invalid_argument(
            "Cannot use opt_chunks_with_max_extend "
            "with a typesize larger than CHUNKSIZE_MAX!");
    }
 
    // .. Parse dims and prepare algorithm ....................................
 
    // Determine the finite dims
    auto dims_fin =
        find_all_idcs(max_extend, [](auto l) { return l != H5S_UNLIMITED; });
    // Ideally, an integer multiple of the chunk size along this dim should
    // be equal to the maximum extend
 
    // Determine the infinite dims
    auto dims_inf =
        find_all_idcs(max_extend, [](auto l) { return l == H5S_UNLIMITED; });
    // As the final extend along these dims is not known, we can not make a
    // good guess for these. Instead, we should use the leverage we have for
    // optimizing the chunk size along the finite dims. The infinite dims will
    // thus, most likely, end up with shorter chunk sizes.
 
    // Declare a container type for storing indices, same as those returned by
    // the find_all_idcs function
    using IdxCont = decltype(dims_fin);
 
    // Create a container with the available dimension indices
    IdxCont dims(chunks.size());
    std::iota(dims.begin(), dims.end(), 0);
 
    // Among the finite dims, determine the dims that can still be filled,
    // i.e. those where the chunk size does not reach the max_extend
    IdxCont dims_fillable;
    for (auto dim : dims_fin)
    {
        if (max_extend[dim] > chunks[dim])
        {
            dims_fillable.push_back(dim);
        }
    }
 
    // Check if to reverse index containers to favour higher dims
    if (larger_high_dims)
    {
        // Reverse all index containers
        std::reverse(dims_fillable.begin(), dims_fillable.end());
        std::reverse(dims_fin.begin(), dims_fin.end());
        std::reverse(dims_inf.begin(), dims_inf.end());
 
        // NOTE do not actually _need_ to reverse the finite dims container,
        // doing it only for consistency.
    }
 
    // .. Optimization of finite (and still fillable) dims ....................
 
    if (!dims_fillable.size())
    {
        log->debug("No finite dimensions available to optimize.");
    }
    else
    {
        log->debug("Optimizing {} finite dimension(s) where max_extend is not "
                   "yet reached ...",
                   dims_fillable.size());
 
        // Loop over the fillable dims indices
        for (auto dim : dims_fillable)
        {
            // Check if there is still potential for optimization
            // NOTE this could be more thorough
            if (bytes(chunks) == CHUNKSIZE_MAX)
            {
                log->debug("Reached maximum chunksize.");
                break;
            }
 
            // Check if the max_extend is an integer multiple of the chunksize
            if (max_extend[dim] % chunks[dim] == 0)
            {
                // Find the divisor
                std::size_t factor = max_extend[dim] / chunks[dim];
 
                // It might fit in completely ...
                if (factor * bytes(chunks) <= CHUNKSIZE_MAX)
                {
                    // It does. Adjust chunks and continue with next dim
                    log->debug("Dimension {} can be filled completely. "
                               "Factor: {}",
                               dim,
                               factor);
                    chunks[dim] = chunks[dim] * factor;
                    continue;
                }
                // else: would not fit in completely
 
                // Starting from the largest possible factor, find the largest
                // integer divisor of the original factor, i.e.: one that will
                // also completely cover the max_extend
                for (std::size_t div = (CHUNKSIZE_MAX / bytes(chunks));
                     div >= 1;
                     div--)
                {
                    // Check if it is an integer divisor
                    if (factor % div == 0)
                    {
                        // Yes! The _new_ factor is now this value
                        factor = div;
                        break;
                    }
                }
                // NOTE Covers the edge case of max. factor == 1: the loop will
                // perform only one iteration and resulting factor will be 1,
                // leading (effectively) to no scaling.
 
                // Scale the chunksize with this factor
                if (factor > 1)
                {
                    log->debug(
                        "Scaling dimension {} with factor {} ...", dim, factor);
 
                    chunks[dim] = chunks[dim] * factor;
                }
            }
            else
            {
                // Not divisible. Check if the max_extend could be reached w/o
                // exceeding the max chunksize
                const double factor = double(max_extend[dim]) / chunks[dim];
 
                if (factor * bytes(chunks) <= CHUNKSIZE_MAX)
                {
                    // Yep. Just extend this dimension to the max_extend, done.
                    log->debug("Dimension {} can be filled completely. "
                               "(difference: {}, factor: {})",
                               dim,
                               max_extend[dim] - chunks[dim],
                               factor);
 
                    chunks[dim] = max_extend[dim];
                }
                else
                {
                    // Cannot further extend.
                    log->debug("Dimension {} cannot be extended to fill "
                               "max_extend without exceeding maximum "
                               "chunksize! "
                               "(difference: {}, factor: {})",
                               dim,
                               max_extend[dim] - chunks[dim],
                               factor);
                }
            }
            // Done with this index
        }
    }
 
    // .. Optimization of infinite dims .......................................
 
    if (!opt_inf_dims)
    {
        log->debug("Optimization of unlimited dimensions is disabled.");
    }
    else if (!dims_inf.size())
    {
        log->debug("No unlimited dimensions available to optimize.");
    }
    else if (bytes(chunks) == CHUNKSIZE_MAX)
    {
        log->debug("Cannot further optimize using unlimited dimensions.");
    }
    else
    {
        log->debug("Optimizing {} unlimited dimension(s) to fill the maximum "
                   "chunk size ...",
                   dims_inf.size());
 
        // Loop over indices of inf. dims
        // NOTE Depending on the chunk sizes, this might only have an effect
        //      on the first index considered ... but that's fine for now.
        for (auto dim : dims_inf)
        {
            // Calculate the factor to make the chunk as big as possible
            const std::size_t factor = CHUNKSIZE_MAX / bytes(chunks); // floors
 
            // If large enough, scale it by that factor
            if (factor > 1)
            {
                log->debug(
                    "Scaling dimension {} with factor {} ...", dim, factor);
 
                chunks[dim] = chunks[dim] * factor;
            }
        }
    }
 
    // Done.
    // Check if everything went fine (only a safeguard ...)
    if (bytes(chunks) > CHUNKSIZE_MAX)
    {
        throw std::runtime_error("Calculated chunks exceed CHUNKSIZE_MAX! "
                                 "This should not have happened!");
    }
 
    return;
}

to_str()

template<typename Cont = std::vector< hsize_t >>

std::string Utopia::DataIO::_chunk_helpers::to_str

(

const Cont &

vec

)

Helper function to create a string representation of containers.

{
    std::stringstream s;
    s << "{ ";
    for (auto& extd : vec)
    {
        if (extd < H5S_UNLIMITED)
        {
            s << extd << " ";
        }
        else
        {
            s << "∞ ";
        }
    }
    s << "}";
    return s.str();
};