Implementation of IodaIO for netcdf. More...

#include <NetcdfIO.h>

Inheritance diagram for ioda::NetcdfIO:

[legend]

Collaboration diagram for ioda::NetcdfIO:

[legend]

Public Member Functions
	NetcdfIO (const std::string &FileName, const std::string &FileMode, const std::size_t MaxFrameSize)

	~NetcdfIO ()

Public Member Functions inherited from ioda::IodaIO
	IodaIO (const std::string &FileName, const std::string &FileMode, const std::size_t MaxFrameSize)

virtual	~IodaIO ()=0

std::string	fname () const

std::string	fmode () const

std::size_t	nlocs () const

std::size_t	nvars () const

bool	unexpected_data_types () const

bool	excess_dims () const

GroupIter	group_begin ()

GroupIter	group_end ()

std::string	group_name (GroupIter)

VarIter	var_begin (GroupIter)

VarIter	var_end (GroupIter)

std::string	var_name (VarIter)

bool	grp_var_exists (const std::string &, const std::string &)

std::string	var_dtype (VarIter)

std::string	var_dtype (const std::string &, const std::string &)

std::vector< std::size_t >	var_shape (VarIter)

std::vector< std::size_t >	var_shape (const std::string &, const std::string &)

std::vector< std::size_t >	file_shape (VarIter)

std::vector< std::size_t >	file_shape (const std::string &, const std::string &)

std::string	file_name (VarIter)

std::string	file_name (const std::string &, const std::string &)

std::string	file_type (VarIter)

std::string	file_type (const std::string &, const std::string &)

std::size_t	var_id (VarIter)

std::size_t	var_id (const std::string &, const std::string &)

void	grp_var_insert (const std::string &GroupName, const std::string &VarName, const std::string &VarType, const std::vector< std::size_t > &VarShape, const std::string &FileVarName, const std::string &FileType, const std::size_t MaxStringSize=0)

DimIter	dim_begin ()

DimIter	dim_end ()

bool	dim_exists (const std::string &)

std::string	dim_name (DimIter)

int	dim_id (DimIter)

std::size_t	dim_size (DimIter)

std::size_t	dim_id_size (const int &)

std::string	dim_id_name (const int &)

std::size_t	dim_name_size (const std::string &)

int	dim_name_id (const std::string &)

void	dim_insert (const std::string &, const std::size_t)

FrameIter	frame_begin ()
	beginning frame iterator More...

FrameIter	frame_end ()
	ending frame iterator More...

void	frame_initialize ()
	initialize frame access More...

void	frame_finalize ()
	finalize frame access More...

std::size_t	frame_start (FrameIter &)
	start value of current frame More...

std::size_t	frame_size (FrameIter &)
	size value of current frame More...

void	frame_info_init (std::size_t MaxVarSize)
	initialize the frame info container More...

void	frame_info_insert (std::size_t Start, std::size_t Size)
	insert item into the frame info container More...

void	frame_data_init ()
	initialize the frame data container More...

void	frame_read (FrameIter &)
	read from the file into the frame containers More...

void	frame_write (FrameIter &)
	write from the frame containers into the file More...

FrameIntIter	frame_int_begin ()

FrameIntIter	frame_int_end ()

bool	frame_int_has (std::string &GroupName, std::string &VarName)

std::vector< int >	frame_int_get_data (FrameIntIter &iframe)

std::string	frame_int_get_gname (FrameIntIter &iframe)

std::string	frame_int_get_vname (FrameIntIter &iframe)

void	frame_int_get_data (std::string &GroupName, std::string &VarName, std::vector< int > &VarData)

void	frame_int_put_data (std::string &GroupName, std::string &VarName, std::vector< int > &VarData)

FrameFloatIter	frame_float_begin ()

FrameFloatIter	frame_float_end ()

bool	frame_float_has (std::string &GroupName, std::string &VarName)

std::vector< float >	frame_float_get_data (FrameFloatIter &iframe)

std::string	frame_float_get_gname (FrameFloatIter &iframe)

std::string	frame_float_get_vname (FrameFloatIter &iframe)

void	frame_float_get_data (std::string &GroupName, std::string &VarName, std::vector< float > &VarData)

void	frame_float_put_data (std::string &GroupName, std::string &VarName, std::vector< float > &VarData)

FrameStringIter	frame_string_begin ()

FrameStringIter	frame_string_end ()

bool	frame_string_has (std::string &GroupName, std::string &VarName)

std::vector< std::string >	frame_string_get_data (FrameStringIter &iframe)

std::string	frame_string_get_gname (FrameStringIter &iframe)

std::string	frame_string_get_vname (FrameStringIter &iframe)

void	frame_string_get_data (std::string &GroupName, std::string &VarName, std::vector< std::string > &VarData)

void	frame_string_put_data (std::string &GroupName, std::string &VarName, std::vector< std::string > &VarData)

Static Public Member Functions
static const std::string	classname ()
	classname method for object counter More...

Static Public Member Functions inherited from ioda::IodaIO
static void	ExtractGrpVarName (const std::string &Name, std::string &GroupName, std::string &VarName)

Private Member Functions
void	print (std::ostream &os) const
	print method for stream output More...

void	NcReadVar (const std::string &GroupName, const std::string &VarName, const std::vector< std::size_t > &Starts, const std::vector< std::size_t > &Counts, int &FillValue, std::vector< int > &VarData)
	Read data from netcdf file to memory. More...

void	NcReadVar (const std::string &GroupName, const std::string &VarName, const std::vector< std::size_t > &Starts, const std::vector< std::size_t > &Counts, float &FillValue, std::vector< float > &VarData)

void	NcReadVar (const std::string &GroupName, const std::string &VarName, const std::vector< std::size_t > &Starts, const std::vector< std::size_t > &Counts, double &FillValue, std::vector< double > &VarData)

void	NcReadVar (const std::string &GroupName, const std::string &VarName, const std::vector< std::size_t > &Starts, const std::vector< std::size_t > &Counts, char &FillValue, std::vector< std::string > &VarData)

void	NcWriteVar (const std::string &GroupName, const std::string &VarName, const std::vector< std::size_t > &Starts, const std::vector< std::size_t > &Counts, const std::vector< int > &VarData)
	Write data from memory to netcdf file. More...

void	NcWriteVar (const std::string &GroupName, const std::string &VarName, const std::vector< std::size_t > &Starts, const std::vector< std::size_t > &Counts, const std::vector< float > &VarData)

void	NcWriteVar (const std::string &GroupName, const std::string &VarName, const std::vector< std::size_t > &Starts, const std::vector< std::size_t > &Counts, const std::vector< std::string > &VarData)

void	CheckNcCall (int RetCode, std::string &ErrorMsg)
	check results of netcdf call More...

bool	NcAttrExists (const int &AttrOwnerId, const std::string &AttrName)
	Check existence of netcdf attribute. More...

std::string	FormNcVarName (const std::string &GroupName, const std::string &VarName)
	form the netcdf variable name More...

int	GetStringDimBySize (const std::size_t DimSize)
	allocate a dimension for writing a character array More...

void	ReadConvertDateTime (const std::string &GroupName, const std::string &VarName, const std::vector< std::size_t > &Starts, const std::vector< std::size_t > &Counts, std::vector< std::string > &VarData)
	read date and time information from the input netcdf file More...

template<typename DataType >
void	ReplaceFillWithMissing (std::vector< DataType > &VarData, DataType NcFillValue)
	Replace netcdf fill values with JEDI missing values. More...

std::size_t	GetMaxStringSize (const std::vector< std::string > &Strings)
	Get the max string size in a vector of strings. More...

std::vector< int >	GetNcDimIds (const std::string &GroupName, const std::vector< std::size_t > &VarShape)
	Get the netcdf dimension ids. More...

void	DimInsert (const std::string &Name, const std::size_t Size)
	create a dimension in the netcdf file More...

void	InitializeFrame ()

void	FinalizeFrame ()

void	ReadFrame (IodaIO::FrameIter &iframe)
	Read data from the file into the frame containers. More...

void	WriteFrame (IodaIO::FrameIter &iframe)
	Write data from the frame containers into the file. More...

void	GrpVarInsert (const std::string &GroupName, const std::string &VarName, const std::string &VarType, const std::vector< std::size_t > &VarShape, const std::string &FileVarName, const std::string &FileType, const std::size_t MaxStringSize)
	Add entry to the group, variable info container. More...

Private Attributes
int	ncid_
	netcdf file id More...

bool	have_offset_time_
	offset time flag More...

bool	have_date_time_
	date time flag More...

Additional Inherited Members
Public Types inherited from ioda::IodaIO
typedef GroupVarInfoMap::const_iterator	GroupIter
	group-variable map, group iterator More...

typedef VarInfoMap::const_iterator	VarIter
	group-variable map, variable iterator More...

typedef DimInfoMap::const_iterator	DimIter

typedef FrameInfo::const_iterator	FrameIter

typedef FrameDataMap< int >::FrameStoreIter	FrameIntIter

typedef FrameDataMap< float >::FrameStoreIter	FrameFloatIter

typedef FrameDataMap< std::string >::FrameStoreIter	FrameStringIter

Protected Types inherited from ioda::IodaIO
typedef std::map< std::string, VarInfoRec >	VarInfoMap
	variable information map More...

typedef std::map< std::string, VarInfoMap >	GroupVarInfoMap
	group-variable information map More...

typedef std::map< std::string, DimInfoRec >	DimInfoMap
	dimension information map More...

typedef std::vector< FrameInfoRec >	FrameInfo
	frame information map More...

Protected Attributes inherited from ioda::IodaIO
std::string	fname_
	file name More...

std::string	fmode_
	file mode More...

std::size_t	nlocs_
	number of unique locations More...

std::size_t	nvars_
	number of unique variables More...

std::size_t	num_unexpect_dtypes_
	count of unexpected data types More...

std::size_t	num_excess_dims_
	count of too many dimensions More...

GroupVarInfoMap	grp_var_info_
	group-variable information map More...

DimInfoMap	dim_info_
	dimension information map More...

FrameInfo	frame_info_
	frame information vector More...

std::size_t	max_frame_size_
	maximum frame size More...

std::unique_ptr< FrameDataMap< int > >	int_frame_data_
	Containers for file frame. More...

std::unique_ptr< FrameDataMap< float > >	float_frame_data_

std::unique_ptr< FrameDataMap< std::string > >	string_frame_data_

Detailed Description

Implementation of IodaIO for netcdf.

The NetcdfIO class defines the constructor and methods for netcdf file access. These fill in the abstract base class IodaIO methods.

Author: Stephen Herbener (JCSDA)

Definition at line 38 of file NetcdfIO.h.

Constructor & Destructor Documentation

◆ NetcdfIO()

ioda::NetcdfIO::NetcdfIO	(	const std::string &	FileName,
		const std::string &	FileMode,
		const std::size_t	MaxFrameSize
	)

This constructor will open the netcdf file. If opening in read mode, the parameters nlocs and nvars will be set by querying the size of dimensions of the same names in the input file. If opening in write mode, the parameters will be set from the same named arguements to this constructor.

Parameters

[in]	FileName	Path to the netcdf file
[in]	FileMode	"r" for read, "w" for overwrite to an existing file and "W" for create and write to a new file.

Definition at line 49 of file NetcdfIO.cc.

Here is the call graph for this function:

◆ ~NetcdfIO()

ioda::NetcdfIO::~NetcdfIO ( )

Definition at line 219 of file NetcdfIO.cc.

Member Function Documentation

◆ CheckNcCall()

void ioda::NetcdfIO::CheckNcCall	(	int	RetCode,
		std::string &	ErrorMsg
	)

private

check results of netcdf call

This method will check the return code from a netcdf API call. Successful completion of the call is indicated by the return code being equal to NC_NOERR. If the call was not successful, then the error message is written to the OOPS log, and is also sent to the OOPS ABORT call (execution is aborted).

Parameters

[in]	RetCode	Return code from netcdf call
[in]	ErrorMsg	Message for the OOPS error logger

Definition at line 806 of file NetcdfIO.cc.

Here is the caller graph for this function:

◆ classname()

static const std::string ioda::NetcdfIO::classname ( )

inlinestatic

classname method for object counter

This method is supplied for the ObjectCounter base class. It defines a name to identify an object of this class for reporting by OOPS.

Definition at line 48 of file NetcdfIO.h.

◆ DimInsert()

void ioda::NetcdfIO::DimInsert	(	const std::string &	Name,
		const std::size_t	Size
	)

privatevirtual

create a dimension in the netcdf file

This method will create a dimension in the output netcdf file using the name given by DimName and the size given by DimSize. This method also records the dimension name and size for downstream use in the WriteVar methods.

Parameters

[in]	Name	Name of netcdf dimension
[in]	Size	Size of netcdf dimension

Implements ioda::IodaIO.

Definition at line 546 of file NetcdfIO.cc.

Here is the call graph for this function:

◆ FinalizeFrame()

void ioda::NetcdfIO::FinalizeFrame ( )

inlineprivatevirtual

Implements ioda::IodaIO.

Definition at line 111 of file NetcdfIO.h.

◆ FormNcVarName()

std::string ioda::NetcdfIO::FormNcVarName	(	const std::string &	GroupName,
		const std::string &	VarName
	)

private

form the netcdf variable name

This method will construct the name of the variable in the netcdf file from the given GroupName and VarName arguments. The netcdf variable name is "VarName@GroupName".

Parameters

[in]	GroupName	Name of group in ObsSpace database
[in]	VarName	Name of variable in ObsSpace database

Definition at line 844 of file NetcdfIO.cc.

Here is the caller graph for this function:

◆ GetMaxStringSize()

std::size_t ioda::NetcdfIO::GetMaxStringSize ( const std::vector< std::string > & Strings )

private

Get the max string size in a vector of strings.

This method will return the size of the longest string in a vector of strings.

Definition at line 471 of file NetcdfIO.cc.

◆ GetNcDimIds()

std::vector< int > ioda::NetcdfIO::GetNcDimIds	(	const std::string &	GroupName,
		const std::vector< std::size_t > &	VarShape
	)

private

Get the netcdf dimension ids.

This method will determine the netcdf dimension ids associated with the current group name.

Definition at line 487 of file NetcdfIO.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ GetStringDimBySize()

int ioda::NetcdfIO::GetStringDimBySize ( const std::size_t DimSize )

private

allocate a dimension for writing a character array

This method is used for setting up dimensions for a writing a character array in the output netcdf file. A character array is how a vector of strings is represented in netcdf. In order to minimize storage, this method is part of a scheme to always create the smallest character array necessary (the first dimension matches the size of the string vector, the second dimension matches the maximum string size in that vector). First, the existing dimensions that have already been allocated for character arrays are checked and if a match occurs that dimension id is returned. Otherwise, a new dimension of the size DimSize is created in the output netcdf file and that new dimension id is returned. New dimensions are named "nstringN" where N is set to DimSize.

Parameters

[in] DimSize Size of netcdf dimension

Definition at line 869 of file NetcdfIO.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ GrpVarInsert()

void ioda::NetcdfIO::GrpVarInsert	(	const std::string &	GroupName,
		const std::string &	VarName,
		const std::string &	VarType,
		const std::vector< std::size_t > &	VarShape,
		const std::string &	FileVarName,
		const std::string &	FileType,
		const std::size_t	MaxStringSize
	)

privatevirtual

Add entry to the group, variable info container.

Implements ioda::IodaIO.

Definition at line 722 of file NetcdfIO.cc.

Here is the call graph for this function:

◆ InitializeFrame()

void ioda::NetcdfIO::InitializeFrame ( )

inlineprivatevirtual

Implements ioda::IodaIO.

Definition at line 110 of file NetcdfIO.h.

◆ NcAttrExists()

bool ioda::NetcdfIO::NcAttrExists	(	const int &	AttrOwnerId,
		const std::string &	AttrName
	)

private

Check existence of netcdf attribute.

This method will check to see if a netcdf exists in the input netcdf file. This can be used to check for existence of group and variable attributes.

Parameters

[in]	AttrOwnerId	Id number of owner of the attribute
[in]	AttrName	Name attribute

Definition at line 826 of file NetcdfIO.cc.

Here is the caller graph for this function:

◆ NcReadVar() [1/4]

void ioda::NetcdfIO::NcReadVar	(	const std::string &	GroupName,
		const std::string &	VarName,
		const std::vector< std::size_t > &	Starts,
		const std::vector< std::size_t > &	Counts,
		char &	FillValue,
		std::vector< std::string > &	VarData
	)

private

Definition at line 326 of file NetcdfIO.cc.

Here is the call graph for this function:

◆ NcReadVar() [2/4]

void ioda::NetcdfIO::NcReadVar	(	const std::string &	GroupName,
		const std::string &	VarName,
		const std::vector< std::size_t > &	Starts,
		const std::vector< std::size_t > &	Counts,
		double &	FillValue,
		std::vector< double > &	VarData
	)

private

Definition at line 297 of file NetcdfIO.cc.

Here is the call graph for this function:

◆ NcReadVar() [3/4]

void ioda::NetcdfIO::NcReadVar	(	const std::string &	GroupName,
		const std::string &	VarName,
		const std::vector< std::size_t > &	Starts,
		const std::vector< std::size_t > &	Counts,
		float &	FillValue,
		std::vector< float > &	VarData
	)

private

Definition at line 268 of file NetcdfIO.cc.

Here is the call graph for this function:

◆ NcReadVar() [4/4]

void ioda::NetcdfIO::NcReadVar	(	const std::string &	GroupName,
		const std::string &	VarName,
		const std::vector< std::size_t > &	Starts,
		const std::vector< std::size_t > &	Counts,
		int &	FillValue,
		std::vector< int > &	VarData
	)

private

Read data from netcdf file to memory.

The four NcReadVar methods are the same with the exception of the datatype that is being read (integer, float, double, string).

Parameters

[in]	GroupName	Name of ObsSpace group (ObsValue, ObsError, MetaData, etc.)
[in]	VarName	Name of ObsSpace variable
[in]	Starts	Starting indices in file
[in]	Counts	Size of slices to read from file
[out]	FillValue	Netcdf fill value associated with this variable
[out]	VarData	Vector that will receive the file data

Definition at line 239 of file NetcdfIO.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ NcWriteVar() [1/3]

void ioda::NetcdfIO::NcWriteVar	(	const std::string &	GroupName,
		const std::string &	VarName,
		const std::vector< std::size_t > &	Starts,
		const std::vector< std::size_t > &	Counts,
		const std::vector< float > &	VarData
	)

private

Definition at line 428 of file NetcdfIO.cc.

Here is the call graph for this function:

◆ NcWriteVar() [2/3]

void ioda::NetcdfIO::NcWriteVar	(	const std::string &	GroupName,
		const std::string &	VarName,
		const std::vector< std::size_t > &	Starts,
		const std::vector< std::size_t > &	Counts,
		const std::vector< int > &	VarData
	)

private

Write data from memory to netcdf file.

The three NcWriteVar methods are the same with the exception of the datatype that is being written (integer, float, char).

Parameters

[in]	GroupName	Name of ObsSpace group (ObsValue, ObsError, MetaData, etc.)
[in]	VarName	Name of ObsSpace variable
[in]	Starts	Starting indices in file
[in]	Counts	Size of slices to read from file
[in]	VarData	Vector that will be written into the file

Definition at line 406 of file NetcdfIO.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ NcWriteVar() [3/3]

void ioda::NetcdfIO::NcWriteVar	(	const std::string &	GroupName,
		const std::string &	VarName,
		const std::vector< std::size_t > &	Starts,
		const std::vector< std::size_t > &	Counts,
		const std::vector< std::string > &	VarData
	)

private

Definition at line 450 of file NetcdfIO.cc.

Here is the call graph for this function:

◆ print()

void ioda::NetcdfIO::print ( std::ostream & os ) const

private

print method for stream output

This method is supplied for the Printable base class. It defines how to print an object of this class in an output stream.

Definition at line 788 of file NetcdfIO.cc.

◆ ReadConvertDateTime()

void ioda::NetcdfIO::ReadConvertDateTime	(	const std::string &	GroupName,
		const std::string &	VarName,
		const std::vector< std::size_t > &	Starts,
		const std::vector< std::size_t > &	Counts,
		std::vector< std::string > &	VarData
	)

private

read date and time information from the input netcdf file

This method will read date and time information from the input netcdf file and convert that information to absolute date time strings in the ISO 8601 format. The date and time information in the input file is represented as an attribute called "date_time" that contains a reference date and time, and a float variable called "time@MetaData" that contains offset time values relative to the date_time attribute. The date_time attribute is an integer or string in the format, YYYYMMDDHH (year, month, day, hour). The time variable is the offest in units of hours. This is a placeholder function that will be removed once all input files have been converted to store absolute date time information in ISO 8601 strings.

Parameters

[in]	GroupName	Name of group in ObsSpace database
[in]	VarName	Name of variable in ObsSpace database
[out]	VarData	Character array where ISO 8601 date time strings will be placed

Definition at line 910 of file NetcdfIO.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ReadFrame()

void ioda::NetcdfIO::ReadFrame ( IodaIO::FrameIter & iframe )

privatevirtual

Read data from the file into the frame containers.

Implements ioda::IodaIO.

Definition at line 559 of file NetcdfIO.cc.

Here is the call graph for this function:

◆ ReplaceFillWithMissing()

template<typename DataType >

void ioda::NetcdfIO::ReplaceFillWithMissing	(	std::vector< DataType > &	VarData,
		DataType	NcFillValue
	)

private

Replace netcdf fill values with JEDI missing values.

This method replaces elements of VarData that match the netcdf fill value with the corresponding JEDI missing value.

Parameters

[in,out]	VarData	Vector holding data to be converted
[in]	NcFillValue	Netcdf fill value associated with this variable

Definition at line 382 of file NetcdfIO.cc.

◆ WriteFrame()

void ioda::NetcdfIO::WriteFrame ( IodaIO::FrameIter & iframe )

privatevirtual

Write data from the frame containers into the file.

Implements ioda::IodaIO.

Definition at line 673 of file NetcdfIO.cc.

Here is the call graph for this function:

Member Data Documentation

◆ have_date_time_

bool ioda::NetcdfIO::have_date_time_

private

date time flag

This data member is a flag indicating the existence of the date_time variable in the netcdf file.

Definition at line 144 of file NetcdfIO.h.

◆ have_offset_time_

bool ioda::NetcdfIO::have_offset_time_

private

offset time flag

This data member is a flag indicating the existence of the offset time variable in the netcdf file.

Definition at line 136 of file NetcdfIO.h.

◆ ncid_

int ioda::NetcdfIO::ncid_

private

netcdf file id

This data member holds the file id of the open netcdf file. It gives access to the dimensions, attributes and variables in the netcdf file.

Definition at line 128 of file NetcdfIO.h.

The documentation for this class was generated from the following files:

Public Member Functions

Static Public Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ NetcdfIO()

◆ ~NetcdfIO()

Member Function Documentation

◆ CheckNcCall()

◆ classname()

◆ DimInsert()

◆ FinalizeFrame()

◆ FormNcVarName()

◆ GetMaxStringSize()

◆ GetNcDimIds()

◆ GetStringDimBySize()

◆ GrpVarInsert()

◆ InitializeFrame()

◆ NcAttrExists()

◆ NcReadVar() [1/4]

◆ NcReadVar() [2/4]

◆ NcReadVar() [3/4]

◆ NcReadVar() [4/4]

◆ NcWriteVar() [1/3]

◆ NcWriteVar() [2/3]

◆ NcWriteVar() [3/3]

◆ print()

◆ ReadConvertDateTime()

◆ ReadFrame()

◆ ReplaceFillWithMissing()

◆ WriteFrame()

Member Data Documentation

◆ have_date_time_

◆ have_offset_time_

◆ ncid_