Implementation of ObsFrameRead class. More...

#include <ObsFrameRead.h>

Inheritance diagram for ioda::ObsFrameRead:

[legend]

Collaboration diagram for ioda::ObsFrameRead:

[legend]

Public Member Functions
	ObsFrameRead (const ObsSpaceParameters &params)

	~ObsFrameRead ()

std::vector< std::size_t >	index () const override
	return list of indices indicating which locations were selected from ObsIo More...

std::vector< std::size_t >	recnums () const override
	return list of record numbers from ObsIo More...

void	frameInit () override
	initialize for walking through the frames More...

void	frameNext () override
	move to the next frame More...

bool	frameAvailable () override
	true if a frame is available (not past end of frames) More...

Dimensions_t	frameStart () override
	return current frame starting index More...

Dimensions_t	frameCount (const std::string &varName) override
	return current frame count for variable More...

Dimensions_t	adjNlocsFrameStart () const override
	return adjusted nlocs frame start More...

Dimensions_t	adjNlocsFrameCount () const override
	return adjusted nlocs frame count More...

bool	readFrameVar (const std::string &varName, std::vector< int > &varData)
	read a frame variable More...

bool	readFrameVar (const std::string &varName, std::vector< float > &varData)

bool	readFrameVar (const std::string &varName, std::vector< std::string > &varData)

std::shared_ptr< const Distribution >	distribution ()
	return the MPI distribution More...

Public Member Functions inherited from ioda::ObsFrame
	ObsFrame (const ObsSpaceParameters &params)

	~ObsFrame ()

Dimensions_t	ioMaxVarSize () const
	return number of maximum variable size (along first dimension) from ObsIo More...

Dimensions_t	ioNumLocs () const
	return number of locations from ObsIo More...

Dimensions_t	ioNumVars () const
	return number of regular variables from ObsIo More...

Dimensions_t	ioNumDimVars () const
	return number of dimension scale variables from ObsIo More...

Has_Variables &	vars () const
	return variables container from ObsIo More...

Has_Attributes &	atts () const
	return attributes container from ObsIo More...

const VarNameObjectList &	ioVarList () const
	return list of regular variables from ObsIo More...

const VarNameObjectList &	ioDimVarList () const
	return list of dimension scale variables from ObsIo More...

VarDimMap	ioVarDimMap () const
	return map from variables to their attached dimension scales More...

void	ioUpdateVarDimInfo () const
	update variable, dimension info in the ObsIo object More...

bool	ioIsVarDimByNlocs (const std::string &varName) const
	return true if variable is dimensioned by nlocs More...

virtual std::size_t	frameNumLocs () const
	return number of locations More...

virtual std::size_t	frameNumRecs () const
	return number of records More...

Dimensions_t	globalNumLocs () const
	return number of locations that were selected from ObsIo More...

Dimensions_t	globalNumLocsOutsideTimeWindow () const
	return number of locations from obs source that were outside the time window More...

virtual void	frameInit (const VarNameObjectList &varList, const VarNameObjectList &varDimList, const VarDimMap &varDimMap, const Dimensions_t maxVarSize)
	initialize for a write frame object More...

virtual void	frameNext (const VarNameObjectList &varList)
	move to the next frame for a write frame object More...

Selection	createMemSelection (const std::vector< Dimensions_t > &varShape, const Dimensions_t frameCount)
	create selection object for accessing a memory buffer More...

Selection	createEntireFrameSelection (const std::vector< Dimensions_t > &varShape, const Dimensions_t frameCount)
	create selection object for accessing the entire frame variable More...

Selection	createVarSelection (const std::vector< Dimensions_t > &varShape, const Dimensions_t frameStart, const Dimensions_t frameCount)
	create selection object for accessing a frame from a whole variable More...

Static Public Member Functions
static const std::string	classname ()
	classname method for object counter More...

Private Member Functions
void	print (std::ostream &os) const override
	print routine for oops::Printable base class More...

Dimensions_t	basicFrameCount (const Variable &var)
	return current frame count for variable More...

Selection	createIndexedFrameSelection (const std::vector< Dimensions_t > &varShape)
	set up frontend and backend selection objects for the given variable More...

void	genFrameIndexRecNums (std::shared_ptr< Distribution > &dist)
	generate frame indices and corresponding record numbers More...

void	genFrameLocationsAll (std::vector< Dimensions_t > &locIndex, std::vector< Dimensions_t > &frameIndex)
	generate indices for all locations in current frame More...

void	genFrameLocationsTimeWindow (std::vector< Dimensions_t > &locIndex, std::vector< Dimensions_t > &frameIndex)
	generate indices for locations in current frame after filtering out obs outside DA timing window More...

void	genRecordNumbersAll (const std::vector< Dimensions_t > &locIndex, std::vector< Dimensions_t > &records)
	generate record numbers where each location is a unique record (no grouping) More...

void	genRecordNumbersGrouping (const std::vector< std::string > &obsGroupVarList, const std::vector< Dimensions_t > &frameIndex, std::vector< Dimensions_t > &records)
	generate record numbers considering obs grouping More...

void	buildObsGroupingKeys (const std::vector< std::string > &obsGroupVarList, const std::vector< Dimensions_t > &frameIndex, std::vector< std::string > &groupingKeys)
	generate string keys for record number assignment More...

void	applyMpiDistribution (const std::shared_ptr< Distribution > &dist, const std::vector< Dimensions_t > &locIndex, const std::vector< Dimensions_t > &records)
	apply MPI distribution More...

bool	insideTimingWindow (const util::DateTime &ObsDt)

template<typename DataType >
bool	readFrameVarHelper (const std::string &varName, std::vector< DataType > &varData)
	read variable data from frame helper function More...

Private Attributes
std::shared_ptr< Distribution >	dist_
	MPI distribution object. More...

bool	each_process_reads_separate_obs_
	true if obs_io_ produces a different series of observations on each process, false if they are all the same More...

std::string	distname_
	\Brief Distribution Name More...

Dimensions_t	adjusted_nlocs_frame_start_
	current frame start for variable dimensioned along nlocs More...

Dimensions_t	adjusted_nlocs_frame_count_
	current frame count for variable dimensioned along nlocs More...

std::map< std::string, std::size_t >	obs_grouping_
	map for obs grouping via string keys More...

std::vector< std::size_t >	indx_
	indexes of locations to extract from the input obs file More...

std::vector< std::size_t >	recnums_
	record numbers associated with the location indexes More...

std::size_t	next_rec_num_
	next available record number More...

std::size_t	rec_num_increment_
	spacing between record numbers assigned on this process. More...

std::set< std::size_t >	unique_rec_nums_
	unique record numbers More...

std::vector< Dimensions_t >	frame_loc_index_
	location indices for current frame More...

VarDimMap	dims_attached_to_vars_
	map showing association of dim names with each variable name More...

std::map< std::vector< std::string >, Selection >	known_frame_selections_
	cache for frame selection More...

std::map< std::vector< std::string >, Selection >	known_mem_selections_
	cache for memory buffer selection More...

Additional Inherited Members
Protected Member Functions inherited from ioda::ObsFrame
Selection	createObsIoSelection (const std::vector< Dimensions_t > &varShape, const Dimensions_t frameStart, const Dimensions_t frameCount)
	create selection object for accessing an ObsIo variable More...

void	createFrameFromObsGroup (const VarNameObjectList &varList, const VarNameObjectList &dimVarList, const VarDimMap &varDimMap)
	create a frame object based on dimensions and variables from a source ObsGroup More...

Protected Attributes inherited from ioda::ObsFrame
std::shared_ptr< ObsIo >	obs_io_
	ObsIo object. More...

ObsGroup	obs_frame_
	ObsGroup object (temporary storage for a single frame) More...

Dimensions_t	nrecs_
	number of records from source (file or generator) More...

Dimensions_t	nlocs_
	number of locations from source (file or generator) More...

Dimensions_t	gnlocs_
	total number of locations from source (file or generator) that were selected after the timing window filtering More...

Dimensions_t	gnlocs_outside_timewindow_
	number of nlocs from the file (gnlocs) that are outside the time window More...

ObsSpaceParameters	params_
	ObsIo parameter specs. More...

Dimensions_t	max_frame_size_
	maximum frame size More...

Dimensions_t	max_var_size_
	maximum variable size More...

Dimensions_t	frame_start_
	current frame starting index More...

Detailed Description

Implementation of ObsFrameRead class.

This class manages one frame of obs data (subset of locations) when reading data from an ObsIo object. This includes reading the frame, filtering out obs that are outside the DA timing window, generating record numbers, applying obs grouping (optional) and applying the MPI distribution.

Author: Stephen Herbener (JCSDA)

Definition at line 31 of file src/io/ObsFrameRead.h.

Constructor & Destructor Documentation

◆ ObsFrameRead()

ioda::ObsFrameRead::ObsFrameRead ( const ObsSpaceParameters & params )

explicit

Definition at line 20 of file ObsFrameRead.cc.

Here is the call graph for this function:

◆ ~ObsFrameRead()

ioda::ObsFrameRead::~ObsFrameRead ( )

Definition at line 47 of file ObsFrameRead.cc.

Member Function Documentation

◆ adjNlocsFrameCount()

Dimensions_t ioda::ObsFrameRead::adjNlocsFrameCount ( ) const

inlineoverridevirtual

return adjusted nlocs frame count

Reimplemented from ioda::ObsFrame.

Definition at line 75 of file src/io/ObsFrameRead.h.

Here is the caller graph for this function:

◆ adjNlocsFrameStart()

Dimensions_t ioda::ObsFrameRead::adjNlocsFrameStart ( ) const

inlineoverridevirtual

return adjusted nlocs frame start

Reimplemented from ioda::ObsFrame.

Definition at line 72 of file src/io/ObsFrameRead.h.

Here is the caller graph for this function:

◆ applyMpiDistribution()

void ioda::ObsFrameRead::applyMpiDistribution	(	const std::shared_ptr< Distribution > &	dist,
		const std::vector< Dimensions_t > &	locIndex,
		const std::vector< Dimensions_t > &	records
	)

private

apply MPI distribution

Parameters

dist	ioda::Distribution object
records	vector indexed by location containing the record numbers

If "save obs distribution" is set to true in a previous run, global location indices and record numbers have stored in the MetaData/saved_index and MetaData/saved_record_number variables, along with all other variables in separate files.

When the "obsdatain.read obs from separate file" option is set, each process reads a separate input file generated previously, to use the stored index and record_number.

Definition at line 434 of file ObsFrameRead.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ basicFrameCount()

Dimensions_t ioda::ObsFrameRead::basicFrameCount ( const Variable & var )

private

return current frame count for variable

Variables can be of different sizes so it's possible that the frame has moved past the end of some variables but not so for other variables. When the frame is past the end of the given variable, this routine returns a zero to indicate that we're done with this variable.

Parameters

var variable

Definition at line 182 of file ObsFrameRead.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ buildObsGroupingKeys()

void ioda::ObsFrameRead::buildObsGroupingKeys	(	const std::vector< std::string > &	obsGroupVarList,
		const std::vector< Dimensions_t > &	frameIndex,
		std::vector< std::string > &	groupingKeys
	)

private

generate string keys for record number assignment

Parameters

obsGroupVarList	list of variables controlling the grouping function
frameIndex	vector containing frame location indices
groupingKeys	vector of keys for the obs grouping map

Definition at line 361 of file ObsFrameRead.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ classname()

static const std::string ioda::ObsFrameRead::classname ( )

inlinestatic

classname method for object counter

This method is supplied for the ObjectCounter base class. It defines a name to identify an object of this class for reporting by OOPS.

Definition at line 38 of file src/io/ObsFrameRead.h.

◆ createIndexedFrameSelection()

Selection ioda::ObsFrameRead::createIndexedFrameSelection ( const std::vector< Dimensions_t > & varShape )

private

set up frontend and backend selection objects for the given variable

Parameters

varShape dimension sizes for variable being transferred

Definition at line 195 of file ObsFrameRead.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ distribution()

std::shared_ptr<const Distribution> ioda::ObsFrameRead::distribution ( )

inline

return the MPI distribution

Definition at line 94 of file src/io/ObsFrameRead.h.

Here is the caller graph for this function:

◆ frameAvailable()

bool ioda::ObsFrameRead::frameAvailable ( )

overridevirtual

true if a frame is available (not past end of frames)

Implements ioda::ObsFrame.

Definition at line 81 of file ObsFrameRead.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ frameCount()

Dimensions_t ioda::ObsFrameRead::frameCount ( const std::string & varName )

overridevirtual

return current frame count for variable

Variables can be of different sizes so it's possible that the frame has moved past the end of some variables but not so for other variables. When the frame is past the end of the given variable, this routine returns a zero to indicate that we're done with this variable.

Parameters

varName variable name

Implements ioda::ObsFrame.

Definition at line 148 of file ObsFrameRead.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ frameInit()

void ioda::ObsFrameRead::frameInit ( )

overridevirtual

initialize for walking through the frames

Reimplemented from ioda::ObsFrame.

Definition at line 50 of file ObsFrameRead.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ frameNext()

void ioda::ObsFrameRead::frameNext ( )

overridevirtual

move to the next frame

Reimplemented from ioda::ObsFrame.

Definition at line 75 of file ObsFrameRead.cc.

Here is the caller graph for this function:

◆ frameStart()

Dimensions_t ioda::ObsFrameRead::frameStart ( )

overridevirtual

return current frame starting index

Parameters

varName name of variable

Implements ioda::ObsFrame.

Definition at line 143 of file ObsFrameRead.cc.

Here is the caller graph for this function:

◆ genFrameIndexRecNums()

void ioda::ObsFrameRead::genFrameIndexRecNums ( std::shared_ptr< Distribution > & dist )

private

generate frame indices and corresponding record numbers

This method generates a list of indices with their corresponding record numbers, where the indices denote which locations are to be read into this process element.

Definition at line 210 of file ObsFrameRead.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ genFrameLocationsAll()

void ioda::ObsFrameRead::genFrameLocationsAll	(	std::vector< Dimensions_t > &	locIndex,
		std::vector< Dimensions_t > &	frameIndex
	)

private

generate indices for all locations in current frame

Parameters

locIndex	vector of location indices relative to entire obs source
frameIndex	vector of location indices relative to current frame

Definition at line 246 of file ObsFrameRead.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ genFrameLocationsTimeWindow()

void ioda::ObsFrameRead::genFrameLocationsTimeWindow	(	std::vector< Dimensions_t > &	locIndex,
		std::vector< Dimensions_t > &	frameIndex
	)

private

generate indices for locations in current frame after filtering out obs outside DA timing window

Parameters

locIndex	vector of location indices relative to entire obs source
frameIndex	vector of location indices relative to current frame

Definition at line 259 of file ObsFrameRead.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ genRecordNumbersAll()

void ioda::ObsFrameRead::genRecordNumbersAll	(	const std::vector< Dimensions_t > &	locIndex,
		std::vector< Dimensions_t > &	records
	)

private

generate record numbers where each location is a unique record (no grouping)

Parameters

locIndex	vector containing location indices
records	vector indexed by location containing the record numbers

Definition at line 323 of file ObsFrameRead.cc.

Here is the caller graph for this function:

◆ genRecordNumbersGrouping()

void ioda::ObsFrameRead::genRecordNumbersGrouping	(	const std::vector< std::string > &	obsGroupVarList,
		const std::vector< Dimensions_t > &	frameIndex,
		std::vector< Dimensions_t > &	records
	)

private

generate record numbers considering obs grouping

Parameters

obsGroupVarList	list of variables controlling the grouping function
frameIndex	vector containing frame location indices
records	vector indexed by location containing the record numbers

Definition at line 335 of file ObsFrameRead.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ index()

std::vector<std::size_t> ioda::ObsFrameRead::index ( ) const

inlineoverridevirtual

return list of indices indicating which locations were selected from ObsIo

Reimplemented from ioda::ObsFrame.

Definition at line 45 of file src/io/ObsFrameRead.h.

Here is the caller graph for this function:

◆ insideTimingWindow()

bool ioda::ObsFrameRead::insideTimingWindow ( const util::DateTime & ObsDt )

private

return true if observation is inside the DA timing window.

Parameters

obsDt Observation date time object

Definition at line 540 of file ObsFrameRead.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ print()

void ioda::ObsFrameRead::print ( std::ostream & os ) const

overrideprivatevirtual

print routine for oops::Printable base class

Parameters

ostream output stream

Implements ioda::ObsFrame.

Definition at line 177 of file ObsFrameRead.cc.

◆ readFrameVar() [1/3]

bool ioda::ObsFrameRead::readFrameVar	(	const std::string &	varName,
		std::vector< float > &	varData
	)

Definition at line 167 of file ObsFrameRead.cc.

◆ readFrameVar() [2/3]

bool ioda::ObsFrameRead::readFrameVar	(	const std::string &	varName,
		std::vector< int > &	varData
	)

read a frame variable

It's possible for some variables to not be included in the read because the frame has gone past their ending index. Therefore, this function will return true when there exists more data available for the variable in the frame. This function will allocate the proper amount of memory for the output vector varData. The following signatures are for different variable data types.

Parameters

varName	variable name
varData	varible data
varDataSelect	selection information for the selection in memory
frameSelect	selection information for the selection in frame

Definition at line 164 of file ObsFrameRead.cc.

Here is the caller graph for this function:

◆ readFrameVar() [3/3]

bool ioda::ObsFrameRead::readFrameVar	(	const std::string &	varName,
		std::vector< std::string > &	varData
	)

Definition at line 170 of file ObsFrameRead.cc.

◆ readFrameVarHelper()

template<typename DataType >

bool ioda::ObsFrameRead::readFrameVarHelper	(	const std::string &	varName,
		std::vector< DataType > &	varData
	)

inlineprivate

read variable data from frame helper function

Parameters

varName	variable name
varData	varible data

Definition at line 226 of file src/io/ObsFrameRead.h.

Here is the call graph for this function:

◆ recnums()

std::vector<std::size_t> ioda::ObsFrameRead::recnums ( ) const

inlineoverridevirtual

return list of record numbers from ObsIo

Reimplemented from ioda::ObsFrame.

Definition at line 48 of file src/io/ObsFrameRead.h.

Here is the caller graph for this function:

Member Data Documentation

◆ adjusted_nlocs_frame_count_

Dimensions_t ioda::ObsFrameRead::adjusted_nlocs_frame_count_

private

current frame count for variable dimensioned along nlocs

Definition at line 117 of file src/io/ObsFrameRead.h.

◆ adjusted_nlocs_frame_start_

Dimensions_t ioda::ObsFrameRead::adjusted_nlocs_frame_start_

private

current frame start for variable dimensioned along nlocs

This data member is keeping track of the frame start for the contiguous storage where the obs source data will be moved to. Note that the start_ data member is keeping track of the frame start for the obs source itself.

Definition at line 114 of file src/io/ObsFrameRead.h.

◆ dims_attached_to_vars_

VarDimMap ioda::ObsFrameRead::dims_attached_to_vars_

private

map showing association of dim names with each variable name

Definition at line 145 of file src/io/ObsFrameRead.h.

◆ dist_

std::shared_ptr<Distribution> ioda::ObsFrameRead::dist_

private

MPI distribution object.

Definition at line 100 of file src/io/ObsFrameRead.h.

◆ distname_

std::string ioda::ObsFrameRead::distname_

private

\Brief Distribution Name

Definition at line 107 of file src/io/ObsFrameRead.h.

◆ each_process_reads_separate_obs_

bool ioda::ObsFrameRead::each_process_reads_separate_obs_

private

true if obs_io_ produces a different series of observations on each process, false if they are all the same

Definition at line 104 of file src/io/ObsFrameRead.h.

◆ frame_loc_index_

std::vector<Dimensions_t> ioda::ObsFrameRead::frame_loc_index_

private

location indices for current frame

Definition at line 142 of file src/io/ObsFrameRead.h.

◆ indx_

std::vector<std::size_t> ioda::ObsFrameRead::indx_

private

indexes of locations to extract from the input obs file

Definition at line 123 of file src/io/ObsFrameRead.h.

◆ known_frame_selections_

std::map<std::vector<std::string>, Selection> ioda::ObsFrameRead::known_frame_selections_

private

cache for frame selection

Definition at line 148 of file src/io/ObsFrameRead.h.

◆ known_mem_selections_

std::map<std::vector<std::string>, Selection> ioda::ObsFrameRead::known_mem_selections_

private

cache for memory buffer selection

Definition at line 151 of file src/io/ObsFrameRead.h.

◆ next_rec_num_

std::size_t ioda::ObsFrameRead::next_rec_num_

private

next available record number

Definition at line 129 of file src/io/ObsFrameRead.h.

◆ obs_grouping_

std::map<std::string, std::size_t> ioda::ObsFrameRead::obs_grouping_

private

map for obs grouping via string keys

Definition at line 120 of file src/io/ObsFrameRead.h.

◆ rec_num_increment_

std::size_t ioda::ObsFrameRead::rec_num_increment_

private

spacing between record numbers assigned on this process.

Normally 1, but if each process reads observations from a different file, then set to the size of the MPI communicator to ensure record numbers assigned by different processes are distinct.

Definition at line 136 of file src/io/ObsFrameRead.h.

◆ recnums_

std::vector<std::size_t> ioda::ObsFrameRead::recnums_

private

record numbers associated with the location indexes

Definition at line 126 of file src/io/ObsFrameRead.h.

◆ unique_rec_nums_

std::set<std::size_t> ioda::ObsFrameRead::unique_rec_nums_

private

unique record numbers

Definition at line 139 of file src/io/ObsFrameRead.h.

The documentation for this class was generated from the following files:

Public Member Functions

Static Public Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ ObsFrameRead()

◆ ~ObsFrameRead()

Member Function Documentation

◆ adjNlocsFrameCount()

◆ adjNlocsFrameStart()

◆ applyMpiDistribution()

◆ basicFrameCount()

◆ buildObsGroupingKeys()

◆ classname()

◆ createIndexedFrameSelection()

◆ distribution()

◆ frameAvailable()

◆ frameCount()

◆ frameInit()

◆ frameNext()

◆ frameStart()

◆ genFrameIndexRecNums()

◆ genFrameLocationsAll()

◆ genFrameLocationsTimeWindow()

◆ genRecordNumbersAll()

◆ genRecordNumbersGrouping()

◆ index()

◆ insideTimingWindow()

◆ print()

◆ readFrameVar() [1/3]

◆ readFrameVar() [2/3]

◆ readFrameVar() [3/3]

◆ readFrameVarHelper()

◆ recnums()

Member Data Documentation

◆ adjusted_nlocs_frame_count_

◆ adjusted_nlocs_frame_start_

◆ dims_attached_to_vars_

◆ dist_

◆ distname_

◆ each_process_reads_separate_obs_

◆ frame_loc_index_

◆ indx_

◆ known_frame_selections_

◆ known_mem_selections_

◆ next_rec_num_

◆ obs_grouping_

◆ rec_num_increment_

◆ recnums_

◆ unique_rec_nums_