Implementation of the HDF5 backend. More...

Collaboration diagram for HDF5 / HH Engine:

Files
file	Handles.cpp
	HDF5 resource handles in C++.

file	Handles.h
	HDF5 resource handles in C++.

file	HH-attributes.h
	HDF5 engine implementation of Attribute.

file	HH-Filters.h
	HDF5 filters.

file	HH-groups.h
	HDF5 group interface.

file	HH-hasattributes.h
	HDF5 engine implementation of Has_Attributes.

file	HH-hasvariables.h
	HDF5 engine implementation of Has_Variables.

file	HH-types.h
	HDF5 engine implementation of ioda::detail::Type_Provider.

file	HH-util.h
	Utility functions for HDF5.

file	HH-variablecreation.h
	HDF5 engine variable creation parameters.

file	HH-variables.h
	HDF5 engine implementation of Variable.

file	HH-attributes.cpp
	HDF5 engine implementation of Attribute.

file	HH-Filters.cpp
	HDF5 filters.

file	HH-groups.cpp
	HDF5 engine implementation of Group.

file	HH-hasattributes.cpp
	HDF5 engine implementation of Has_Attributes.

file	HH-hasvariables.cpp
	HDF5 engine implementation of Has_Variables.

file	HH-types.cpp
	HDF5 engine implementation of Type.

file	HH-util.cpp
	HDF5 utility functions.

file	HH-variablecreation.cpp
	HDF5 engine Variable creation and property lists.

file	HH-variables.cpp
	HDF5 engine implementation of Variable.

Classes
class	ioda::detail::Engines::HH::Handles::HH_hid_t
	A class to wrap HDF5's hid_t resource handles. More...

struct	ioda::detail::Engines::HH::Handles::InvalidHDF5Handle
	Detect invalid HDF5 ids. More...

class	ioda::detail::Engines::HH::HH_Attribute
	This is the implementation of Attributes using HDF5. More...

struct	ioda::detail::Engines::HH::Filters
	Order-obeying filter insertions and replacements. More...

class	ioda::detail::Engines::HH::HH_Group
	This is the implementation of Groups using HDF5. Do not use outside of IODA. More...

class	ioda::detail::Engines::HH::HH_HasAttributes
	This is the implementation of Has_Attributes using HDF5. More...

class	ioda::detail::Engines::HH::DatasetParameterPack
	This encapsulates dataset creation parameters. More...

class	ioda::detail::Engines::HH::HH_Type_Provider
	This is the implementation of Type_Provider using HDF5. Do not use outside of IODA. More...

class	ioda::detail::Engines::HH::HH_Type
	This is the implementation of ioda::Type using HDF5. Do not use outside of IODA. More...

struct	ioda::detail::Engines::HH::ds_list_t
	Duplicate the HDF5 dataset list structure for REFERENCE_LISTs. More...

struct	ioda::detail::Engines::HH::Iterator_find_attr_data_t
	Data to pass to/from iterator classes. More...

struct	ioda::detail::Engines::HH::View_hvl_t< Inner >
	A "view" of hvl_t objects. Adds C++ conveniences to an otherwise troublesome class. More...

struct	ioda::detail::Engines::HH::Vlen_data
	Internal structure to encapsulate resources and prevent leaks. More...

class	ioda::detail::Engines::HH::VariableCreation
	This encapsulates dataset creation parameters. Used for generating HDF5 property lists for variable creation. More...

class	ioda::detail::Engines::HH::HH_Variable
	This is the implementation of Variables using HDF5. More...

Typedefs
typedef hobj_ref_t	ioda::detail::Engines::HH::ref_t

Enumerations
enum class	ioda::detail::Engines::HH::Handles::Handle_Types { ioda::detail::Engines::HH::Handles::ATTRIBUTE , ioda::detail::Engines::HH::Handles::DATASET , ioda::detail::Engines::HH::Handles::DATASPACE , ioda::detail::Engines::HH::Handles::DATATYPE , ioda::detail::Engines::HH::Handles::FILE , ioda::detail::Engines::HH::Handles::GROUP , ioda::detail::Engines::HH::Handles::LINK , ioda::detail::Engines::HH::Handles::PROPERTYLIST , ioda::detail::Engines::HH::Handles::REFERENCE , ioda::detail::Engines::HH::Handles::UNKNOWN }
	Describes what a handle points to. More...

Functions
std::shared_ptr< hid_t >	ioda::detail::Engines::HH::Handles::createStatic (hid_t newh)
	Encapsulate a static hid object in a shared pointer. More...

IODA_HIDDEN std::pair< bool, bool >	ioda::detail::Engines::HH::isFilteravailable (H5Z_filter_t filt)
	Check that the given filter is available for encoding/decoding in the HDF5 pipeline. More...

IODA_HIDDEN bool	ioda::detail::Engines::HH::CanUseSZIP (HH_hid_t dtype)
	Can SZIP encoding be used for a datatype?@detail This depends on the data type and the HDF5 build options. SZIP cannot be applied to compound, array, variable-length, enumerative or user-defined datatypes. More...

IODA_HIDDEN herr_t	ioda::detail::Engines::HH::iterate_find_attr (hid_t loc_id, const char name, void op_data)
	Callback function for H5Aiterate / H5Aiterate2 / H5Aiterate1. More...

IODA_HIDDEN H5_index_t	ioda::detail::Engines::HH::getAttrCreationOrder (hid_t obj, H5O_type_t objType)
	Determine attribute creation order for a dataset. More...

IODA_HIDDEN std::pair< bool, hsize_t >	ioda::detail::Engines::HH::iterativeAttributeSearch (hid_t baseObject, const char *attname, H5_index_t iteration_type)
	Linear search to find an attribute. More...

IODA_HIDDEN HH_Attribute	ioda::detail::Engines::HH::iterativeAttributeSearchAndOpen (hid_t baseObject, H5O_type_t objType, const char *attname)
	Linear search to find and open an attribute, if it exists. More...

IODA_HIDDEN void	ioda::detail::Engines::HH::attr_update_dimension_list (HH_Variable *var, const std::vector< std::vector< ref_t >> &new_dim_list)
	Attribute DIMENSION_LIST update function. More...

IODA_HIDDEN void	ioda::detail::Engines::HH::attr_update_reference_list (HH_Variable *scale, const std::vector< ds_list_t > &ref_var_axis)
	Attribute REFERENCE_LIST update function. More...

	ioda::detail::Engines::HH::View_hvl_t< Inner >::View_hvl_t (hvl_t &obj)

size_t	ioda::detail::Engines::HH::View_hvl_t< Inner >::size () const

void	ioda::detail::Engines::HH::View_hvl_t< Inner >::resize (size_t newlen)

void	ioda::detail::Engines::HH::View_hvl_t< Inner >::clear ()

Inner *	ioda::detail::Engines::HH::View_hvl_t< Inner >::at (size_t i)

Inner *	ioda::detail::Engines::HH::View_hvl_t< Inner >::operator[] (size_t i)

	ioda::detail::Engines::HH::Vlen_data::Vlen_data (size_t sz, HH_hid_t typ, HH_hid_t space)

	ioda::detail::Engines::HH::Vlen_data::~Vlen_data ()

	ioda::detail::Engines::HH::Vlen_data::Vlen_data (const Vlen_data &)=delete

	ioda::detail::Engines::HH::Vlen_data::Vlen_data (Vlen_data &&)=delete

Vlen_data	ioda::detail::Engines::HH::Vlen_data::operator= (const Vlen_data &)=delete

Vlen_data	ioda::detail::Engines::HH::Vlen_data::operator= (Vlen_data &&)=delete

hvl_t &	ioda::detail::Engines::HH::Vlen_data::operator[] (size_t idx)

IODA_HIDDEN std::string	ioda::detail::Engines::HH::getNameFromIdentifier (hid_t obj_id)
	Gets a variable / group / link name from an id. Useful for debugging. More...

HH_hid_t	ioda::detail::Engines::HH::attr_reference_list_type ()

HH_hid_t	ioda::detail::Engines::HH::attr_reference_list_space (hsize_t numrefs)

Variables
hobj_ref_t	ioda::detail::Engines::HH::ds_list_t::ref

unsigned int	ioda::detail::Engines::HH::ds_list_t::dim_idx

std::string	ioda::detail::Engines::HH::Iterator_find_attr_data_t::search_for

hsize_t	ioda::detail::Engines::HH::Iterator_find_attr_data_t::idx = 0

bool	ioda::detail::Engines::HH::Iterator_find_attr_data_t::success = false

hvl_t &	ioda::detail::Engines::HH::View_hvl_t< Inner >::obj

std::unique_ptr< hvl_t[]>	ioda::detail::Engines::HH::Vlen_data::buf

HH_hid_t	ioda::detail::Engines::HH::Vlen_data::typ

HH_hid_t	ioda::detail::Engines::HH::Vlen_data::space

size_t	ioda::detail::Engines::HH::Vlen_data::sz

Detailed Description

Implementation of the HDF5 backend.

Typedef Documentation

◆ ref_t

typedef hobj_ref_t ioda::detail::Engines::HH::ref_t

Definition at line 35 of file HH-util.h.

Enumeration Type Documentation

◆ Handle_Types

enum ioda::detail::Engines::HH::Handles::Handle_Types

strong

Describes what a handle points to.

Deprecated:: To be removed.

Enumerator
ATTRIBUTE
DATASET
DATASPACE
DATATYPE
FILE
GROUP
LINK
PROPERTYLIST
REFERENCE
UNKNOWN

Definition at line 30 of file Handles.h.

Function Documentation

◆ View_hvl_t()

template<class Inner >

ioda::detail::Engines::HH::View_hvl_t< Inner >::View_hvl_t ( hvl_t & obj )

inline

Definition at line 136 of file HH-util.h.

◆ Vlen_data() [1/3]

ioda::detail::Engines::HH::Vlen_data::Vlen_data ( const Vlen_data & )

delete

◆ Vlen_data() [2/3]

ioda::detail::Engines::HH::Vlen_data::Vlen_data	(	size_t	sz,
		HH_hid_t	typ,
		HH_hid_t	space
	)

inline

Definition at line 171 of file HH-util.h.

◆ Vlen_data() [3/3]

ioda::detail::Engines::HH::Vlen_data::Vlen_data ( Vlen_data && )

delete

◆ ~Vlen_data()

ioda::detail::Engines::HH::Vlen_data::~Vlen_data ( )

inline

Definition at line 179 of file HH-util.h.

Here is the call graph for this function:

◆ at()

template<class Inner >

Inner* ioda::detail::Engines::HH::View_hvl_t< Inner >::at ( size_t i )

inline

Definition at line 152 of file HH-util.h.

Here is the call graph for this function:

◆ attr_reference_list_space()

HH_hid_t ioda::detail::Engines::HH::attr_reference_list_space ( hsize_t numrefs )

Definition at line 219 of file HH-util.cpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ attr_reference_list_type()

HH_hid_t ioda::detail::Engines::HH::attr_reference_list_type ( )

Definition at line 205 of file HH-util.cpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ attr_update_dimension_list()

void ioda::detail::Engines::HH::attr_update_dimension_list	(	HH_Variable *	var,
		const std::vector< std::vector< ref_t >> &	new_dim_list
	)

Attribute DIMENSION_LIST update function.

This function exists to update DIMENSION_LISTs without updating the mirrored REFERENCE_LIST entry in the variable's scales. This is done for performance reasons, as attaching dimension scales for hundreds of variables sequentially is very slow.

NOTE: This code does not use the regular atts.open(...) call for performance reasons when we have to repeat this call for hundreds or thousands of variables. We instead do a creation-order-preferred search.

Parameters

var	is the variable of interest.
new_dim_list	is the mapping of dimensions that should be added to the variable.

Definition at line 126 of file HH-util.cpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ attr_update_reference_list()

void ioda::detail::Engines::HH::attr_update_reference_list	(	HH_Variable *	scale,
		const std::vector< ds_list_t > &	ref_var_axis
	)

Attribute REFERENCE_LIST update function.

This function exists to update REFERENCE_LISTs without updating the mirrored DIMENSION_LIST entry. This is done for performance reasons, as attaching dimension scales for hundreds of variables sequentially is very slow.

NOTE: This code does not use the regular atts.open(...) call for performance reasons when we have to repeat this call for hundreds or thousands of variables. We instead do a creation-order-preferred search.

Parameters

scale	is the scale of interest.
ref_var_axis_list	is the mapping of variables-dimension numbers that should be added to the scale's REFERENCE_LIST attribute.

Definition at line 230 of file HH-util.cpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ CanUseSZIP()

bool ioda::detail::Engines::HH::CanUseSZIP ( HH_hid_t dtype )

Can SZIP encoding be used for a datatype?@detail This depends on the data type and the HDF5 build options. SZIP cannot be applied to compound, array, variable-length, enumerative or user-defined datatypes.

Parameters

dtype is the data type.

Returns: true if SZIP can be used for compression, false otherwise.

Definition at line 48 of file HH-Filters.cpp.

Here is the call graph for this function:

◆ clear()

template<class Inner >

void ioda::detail::Engines::HH::View_hvl_t< Inner >::clear ( )

inline

Definition at line 151 of file HH-util.h.

◆ createStatic()

std::shared_ptr<hid_t> ioda::detail::Engines::HH::Handles::createStatic ( hid_t newh )

inline

Encapsulate a static hid object in a shared pointer.

Definition at line 116 of file Handles.h.

◆ getAttrCreationOrder()

H5_index_t ioda::detail::Engines::HH::getAttrCreationOrder	(	hid_t	obj,
		H5O_type_t	objType
	)

Determine attribute creation order for a dataset.

Parameters

obj	is the dataset being queried
objType	is the type of object (Dataset or Group).

Returns: H5_INDEX_CRT_ORDER if creation order is tracked.; H5_INDEX_NAME if creation order is not tracked.

Check if the variable has attribute creation order stored and/or indexed. This is not the default, but it can speed up object list accesses considerably.

Definition at line 55 of file HH-util.cpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ getNameFromIdentifier()

std::string ioda::detail::Engines::HH::getNameFromIdentifier ( hid_t obj_id )

Gets a variable / group / link name from an id. Useful for debugging.

Parameters

obj_id is the object.

Returns: One of the possible object names.

Exceptions

ioda::Exception if obj_id is invalid.

Definition at line 263 of file HH-util.cpp.

Here is the caller graph for this function:

◆ isFilteravailable()

std::pair< bool, bool > ioda::detail::Engines::HH::isFilteravailable ( H5Z_filter_t filt )

Check that the given filter is available for encoding/decoding in the HDF5 pipeline.

Parameters

filt	is the filter in question. Commonly H5Z_FILTER_SZIP.

Returns: A pair of (canEncode, canDecode).

Definition at line 33 of file HH-Filters.cpp.

Here is the caller graph for this function:

◆ iterate_find_attr()

herr_t ioda::detail::Engines::HH::iterate_find_attr	(	hid_t	loc_id,
		const char *	name,
		void *	op_data
	)

Callback function for H5Aiterate / H5Aiterate2 / H5Aiterate1.

Definition at line 37 of file HH-util.cpp.

Here is the caller graph for this function:

◆ iterativeAttributeSearch()

std::pair< bool, hsize_t > ioda::detail::Engines::HH::iterativeAttributeSearch	(	hid_t	baseObject,
		const char *	attname,
		H5_index_t	iteration_type
	)

Linear search to find an attribute.

Parameters

baseObject	is the object that could contain the attribute.
attname	is the name of the attribute.
iteration_type	is the type of iteration for the search. See getAttrCreationOrder.

Returns: A pair of (success_flag, index).

Definition at line 82 of file HH-util.cpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ iterativeAttributeSearchAndOpen()

HH_Attribute ioda::detail::Engines::HH::iterativeAttributeSearchAndOpen	(	hid_t	baseObject,
		H5O_type_t	objType,
		const char *	attname
	)

Linear search to find and open an attribute, if it exists.

Parameters

baseObject	is the object that could contain the attribute.
objType	is the type of object (Dataset or Group).
attname	is the name of the attribute.

Returns: An open handle to the attribute, if it exists.; An invalid handle if the attribute does not exist or upon general failure.

This function is useful because it is faster than the regular attribute open by name routine, which does not take advantage of attribute creation order indexing. Performance is particularly good when there are few attributes attached to the base object.

Definition at line 100 of file HH-util.cpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ operator=() [1/2]

Vlen_data ioda::detail::Engines::HH::Vlen_data::operator= ( const Vlen_data & )

delete

◆ operator=() [2/2]

Vlen_data ioda::detail::Engines::HH::Vlen_data::operator= ( Vlen_data && )

delete

◆ operator[]() [1/2]

template<class Inner >

Inner* ioda::detail::Engines::HH::View_hvl_t< Inner >::operator[] ( size_t i )

inline

Definition at line 157 of file HH-util.h.

◆ operator[]() [2/2]

hvl_t& ioda::detail::Engines::HH::Vlen_data::operator[] ( size_t idx )

inline

Definition at line 197 of file HH-util.h.

◆ resize()

template<class Inner >

void ioda::detail::Engines::HH::View_hvl_t< Inner >::resize ( size_t newlen )

inline

Definition at line 138 of file HH-util.h.

Here is the caller graph for this function:

◆ size()

template<class Inner >

size_t ioda::detail::Engines::HH::View_hvl_t< Inner >::size ( ) const

inline

Definition at line 137 of file HH-util.h.

Here is the caller graph for this function:

Variable Documentation

◆ buf

std::unique_ptr<hvl_t[]> ioda::detail::Engines::HH::Vlen_data::buf

Definition at line 168 of file HH-util.h.

◆ dim_idx

unsigned int ioda::detail::Engines::HH::ds_list_t::dim_idx

Definition at line 44 of file HH-util.h.

◆ idx

hsize_t ioda::detail::Engines::HH::Iterator_find_attr_data_t::idx = 0

Definition at line 50 of file HH-util.h.

◆ obj

template<class Inner >

hvl_t& ioda::detail::Engines::HH::View_hvl_t< Inner >::obj

Definition at line 135 of file HH-util.h.

◆ ref

hobj_ref_t ioda::detail::Engines::HH::ds_list_t::ref

Definition at line 43 of file HH-util.h.

◆ search_for

std::string ioda::detail::Engines::HH::Iterator_find_attr_data_t::search_for

Definition at line 49 of file HH-util.h.

◆ space

HH_hid_t ioda::detail::Engines::HH::Vlen_data::space

Definition at line 169 of file HH-util.h.

◆ success

bool ioda::detail::Engines::HH::Iterator_find_attr_data_t::success = false

Definition at line 51 of file HH-util.h.

◆ sz

size_t ioda::detail::Engines::HH::Vlen_data::sz

Definition at line 170 of file HH-util.h.

◆ typ

HH_hid_t ioda::detail::Engines::HH::Vlen_data::typ

Definition at line 169 of file HH-util.h.

Files

Classes

Typedefs

Enumerations

Functions

Variables

Detailed Description

Typedef Documentation

◆ ref_t

Enumeration Type Documentation

◆ Handle_Types

Function Documentation

◆ View_hvl_t()

◆ Vlen_data() [1/3]

◆ Vlen_data() [2/3]

◆ Vlen_data() [3/3]

◆ ~Vlen_data()

◆ at()

◆ attr_reference_list_space()

◆ attr_reference_list_type()

◆ attr_update_dimension_list()

◆ attr_update_reference_list()

◆ CanUseSZIP()

◆ clear()

◆ createStatic()

◆ getAttrCreationOrder()

◆ getNameFromIdentifier()

◆ isFilteravailable()

◆ iterate_find_attr()

◆ iterativeAttributeSearch()

◆ iterativeAttributeSearchAndOpen()

◆ operator=() [1/2]

◆ operator=() [2/2]

◆ operator[]() [1/2]

◆ operator[]() [2/2]

◆ resize()

◆ size()

Variable Documentation

◆ buf

◆ dim_idx

◆ idx

◆ obj

◆ ref

◆ search_for

◆ space

◆ success

◆ sz

◆ typ