api/odc.cc File Reference

#include <cstdint>
#include <functional>
#include <numeric>
#include <unistd.h>
#include "eckit/exception/Exceptions.h"
#include "eckit/maths/Functions.h"
#include "eckit/runtime/Main.h"
#include "eckit/io/MemoryHandle.h"
#include "eckit/io/FileDescHandle.h"
#include "odc/api/odc.h"
#include "odc/api/Odb.h"

Include dependency graph for api/odc.cc:

Go to the source code of this file.

Classes
struct	odc_reader_t
struct	odc_frame_t
struct	odc_decoder_t
struct	odc_decoder_t::DecodeColumn
struct	odc_encoder_t
struct	odc_encoder_t::EncodeColumn

Namespaces
	anonymous_namespace{odc.cc}

Functions
const char *	odc_error_string (int err)
int	anonymous_namespace{odc.cc}::innerWrapFn (std::function< int()> f)
int	anonymous_namespace{odc.cc}::innerWrapFn (std::function< void()> f)
template<typename FN >
int	anonymous_namespace{odc.cc}::wrapApiFunction (FN f)
int	odc_column_type_count (int *count)
int	odc_column_type_name (int type, const char **type_name)
int	odc_initialise_api ()
int	odc_integer_behaviour (int integerBehaviour)
int	odc_set_failure_handler (odc_failure_handler_t handler, void *context)
int	odc_missing_integer (long *missing_value)
int	odc_missing_double (double *missing_value)
int	odc_set_missing_integer (long missing_integer)
int	odc_set_missing_double (double missing_double)
int	odc_version (const char **version)
int	odc_vcs_version (const char **sha1)
int	odc_open_path (odc_reader_t **reader, const char *filename)
int	odc_open_file_descriptor (odc_reader_t **reader, int fd)
int	odc_open_buffer (odc_reader_t **reader, const void *data, long length)
int	odc_open_stream (odc_reader_t **reader, void *handle, odc_stream_read_t stream_proc)
int	odc_close (const odc_reader_t *reader)
int	odc_new_frame (odc_frame_t **frame, odc_reader_t *reader)
int	odc_free_frame (const odc_frame_t *frame)
int	odc_next_frame (odc_frame_t *frame)
int	odc_next_frame_aggregated (odc_frame_t *frame, long maximum_rows)
int	odc_copy_frame (odc_frame_t *source_frame, odc_frame_t **copy)
int	odc_frame_row_count (const odc_frame_t *frame, long *count)
int	odc_frame_column_count (const odc_frame_t *frame, int *count)
int	odc_frame_column_attributes (const odc_frame_t *frame, int col, const char **name, int *type, int *element_size, int *bitfield_count)
int	odc_frame_bitfield_attributes (const odc_frame_t *frame, int col, int field, const char **name, int *offset, int *size)
int	odc_new_decoder (odc_decoder_t **decoder)
int	odc_free_decoder (const odc_decoder_t *decoder)
int	odc_decoder_defaults_from_frame (odc_decoder_t *decoder, const odc_frame_t *frame)
int	odc_decoder_set_column_major (odc_decoder_t *decoder, bool columnMajor)
int	odc_decoder_set_row_count (odc_decoder_t *decoder, long nrows)
int	odc_decoder_row_count (const odc_decoder_t *decoder, long *nrows)
int	odc_decoder_set_data_array (odc_decoder_t *decoder, void *buffer, long width, long height, bool columnMajor)
int	odc_decoder_data_array (const odc_decoder_t *decoder, const void **data, long *width, long *height, bool *columnMajor)
int	odc_decoder_add_column (odc_decoder_t *decoder, const char *name)
int	odc_decoder_column_count (const odc_decoder_t *decoder, int *count)
int	odc_decoder_column_set_data_size (odc_decoder_t *decoder, int col, int element_size)
int	odc_decoder_column_set_data_array (odc_decoder_t *decoder, int col, int element_size, int stride, void *data)
int	odc_decoder_column_data_array (const odc_decoder_t *decoder, int col, int *element_size, int *stride, const void **data)
static void	fill_in_decoder (odc_decoder_t *decoder, const odc_frame_t *frame)
int	odc_decode_threaded (odc_decoder_t *decoder, const odc_frame_t *frame, long *rows_decoded, int nthreads)
int	odc_decode (odc_decoder_t *decoder, const odc_frame_t *frame, long *rows_decoded)
int	odc_new_encoder (odc_encoder_t **encoder)
int	odc_free_encoder (const odc_encoder_t *encoder)
int	odc_encoder_add_property (odc_encoder_t *encoder, const char *key, const char *value)
int	odc_encoder_set_row_count (odc_encoder_t *encoder, long nrows)
int	odc_encoder_set_rows_per_frame (odc_encoder_t *encoder, long rows_per_frame)
int	odc_encoder_set_data_array (odc_encoder_t *encoder, const void *data, long width, long height, int columnMajorWidth)
int	odc_encoder_add_column (odc_encoder_t *encoder, const char *name, int type)
int	odc_encoder_column_set_data_size (odc_encoder_t *encoder, int col, int element_size)
int	odc_encoder_column_set_data_array (odc_encoder_t *encoder, int col, int element_size, int stride, const void *data)
int	odc_encoder_column_add_bitfield (odc_encoder_t *encoder, int col, const char *name, int nbits)
void	fill_in_encoder (odc_encoder_t *encoder, std::vector< std::unique_ptr< char[]>> &transposedData)
void	odc_encode_to_data_handle (odc_encoder_t *encoder, eckit::DataHandle &dh)
int	odc_encode_to_stream (odc_encoder_t *encoder, void *handle, odc_stream_write_t write_fn, long *bytes_encoded)
int	odc_encode_to_file_descriptor (odc_encoder_t *encoder, int fd, long *bytes_encoded)
int	odc_encode_to_buffer (odc_encoder_t *encoder, void *buffer, long length, long *bytes_encoded)

Variables
static std::string	g_current_error_str
static odc_failure_handler_t	g_failure_handler = nullptr
static void *	g_failure_handler_context = nullptr

Function Documentation

fill_in_decoder()

static void fill_in_decoder ( odc_decoder_t *  decoder, const odc_frame_t *  frame  )

static

Definition at line 578 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

fill_in_encoder()

void fill_in_encoder	(	odc_encoder_t *	encoder,
		std::vector< std::unique_ptr< char[]>> &	transposedData
	)

We are encoding from an externally supplied array

Definition at line 825 of file api/odc.cc.

Here is the caller graph for this function:

odc_close()

int odc_close

(

const odc_reader_t *

reader

)

Closes and destroys the reader. Must be called for every reader created.

Definition at line 332 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_column_type_count()

int odc_column_type_count

(

int *

count

)

Get the number of column types.

Definition at line 163 of file api/odc.cc.

Here is the call graph for this function:

odc_column_type_name()

int odc_column_type_name	(	int	type,
		const char **	type_name
	)

Get the name of the column type identified by the integer

Definition at line 169 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_copy_frame()

int odc_copy_frame	(	odc_frame_t *	source_frame,
		odc_frame_t **	copy
	)

Copies a frame to another newly allocated frame.

Definition at line 381 of file api/odc.cc.

Here is the call graph for this function:

odc_decode()

int odc_decode	(	odc_decoder_t *	decoder,
		const odc_frame_t *	frame,
		long *	rows_decoded
	)

Decode the data described by the frame into the data array(s) configured in the decoder. If no data array has been set above, an array to decode into will be allocated. This can be obtained by using the odc_decoder_data_array function.

Definition at line 714 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_decode_threaded()

int odc_decode_threaded	(	odc_decoder_t *	decoder,
		const odc_frame_t *	frame,
		long *	rows_decoded,
		int	nthreads
	)

Decode the data described by the frame into the data array(s) configured in the decoder. If the frame is logical aggregated over multiple frames in the message, then parallelise the decoding over multiple threads.

Definition at line 643 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_decoder_add_column()

int odc_decoder_add_column	(	odc_decoder_t *	decoder,
		const char *	name
	)

Adds a column to the set for decoding. This is an alternative to odc_decoder_defaults_from_frame

Definition at line 527 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_decoder_column_count()

int odc_decoder_column_count	(	const odc_decoder_t *	decoder,
		int *	count
	)

The number of columns the decoder is configured to decode

Definition at line 536 of file api/odc.cc.

Here is the call graph for this function:

odc_decoder_column_data_array()

int odc_decoder_column_data_array	(	const odc_decoder_t *	decoder,
		int	col,
		int *	element_size,
		int *	stride,
		const void **	data
	)

Gets the buffer and data layout into which the data has been decoded. Only valid after calling odc_decode

Definition at line 566 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_decoder_column_set_data_array()

int odc_decoder_column_set_data_array	(	odc_decoder_t *	decoder,
		int	col,
		int	element_size,
		int	stride,
		void *	data
	)

Sets a specific data array into which the data associated with the column can be decoded This is an alternative to the odc_decoder_set_data_array.

Definition at line 554 of file api/odc.cc.

Here is the call graph for this function:

odc_decoder_column_set_data_size()

int odc_decoder_column_set_data_size	(	odc_decoder_t *	decoder,
		int	col,
		int	element_size
	)

Set the decoded data size for a column in bytes.

Definition at line 544 of file api/odc.cc.

Here is the call graph for this function:

odc_decoder_data_array()

int odc_decoder_data_array	(	const odc_decoder_t *	decoder,
		const void **	data,
		long *	width,
		long *	height,
		bool *	columnMajor
	)

Gets the output data array into which the data may be decoded

Definition at line 508 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_decoder_defaults_from_frame()

int odc_decoder_defaults_from_frame	(	odc_decoder_t *	decoder,
		const odc_frame_t *	frame
	)

Configures a decoder to decode all the data contained in the frame. This is the most typical way to decode data, and is equivalent to adding all columns to the decoder.

Definition at line 453 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_decoder_row_count()

int odc_decoder_row_count	(	const odc_decoder_t *	decoder,
		long *	nrows
	)

Get the number of rows the decoder is configured to decode data into

Definition at line 486 of file api/odc.cc.

Here is the call graph for this function:

odc_decoder_set_column_major()

int odc_decoder_set_column_major	(	odc_decoder_t *	decoder,
		bool	columnMajor
	)

Instruct the decoder to output in column major form

Definition at line 472 of file api/odc.cc.

Here is the call graph for this function:

odc_decoder_set_data_array()

int odc_decoder_set_data_array	(	odc_decoder_t *	decoder,
		void *	data,
		long	width,
		long	height,
		bool	columnMajor
	)

Define the output data array into which the data may be decoded. This is a shortcut to calling odc_decoder_column_set_data_array for every column.

Definition at line 494 of file api/odc.cc.

Here is the call graph for this function:

odc_decoder_set_row_count()

int odc_decoder_set_row_count	(	odc_decoder_t *	decoder,
		long	nrows
	)

If the decoder is to allocate its buffer internally, specify the number of rows to allocate if this is to be different from the number of rows in the frame decoded. This is only really useful if we are re-using a decoder.

Definition at line 479 of file api/odc.cc.

Here is the call graph for this function:

odc_encode_to_buffer()

int odc_encode_to_buffer	(	odc_encoder_t *	encoder,
		void *	buffer,
		long	length,
		long *	bytes_encoded
	)

Encodes ODB2 into a pre-allocated memory buffer. Returns an error if an insufficiently large buffer is supplied

Definition at line 992 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_encode_to_data_handle()

void odc_encode_to_data_handle	(	odc_encoder_t *	encoder,
		eckit::DataHandle &	dh
	)

Definition at line 921 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_encode_to_file_descriptor()

int odc_encode_to_file_descriptor	(	odc_encoder_t *	encoder,
		int	fd,
		long *	bytes_encoded
	)

Encodes ODB2 into an already open file descriptor

Definition at line 982 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_encode_to_stream()

int odc_encode_to_stream	(	odc_encoder_t *	encoder,
		void *	context,
		odc_stream_write_t	write_fn,
		long *	bytes_encoded
	)

Encodes ODB2 in a stream handler

Definition at line 949 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_encoder_add_column()

int odc_encoder_add_column	(	odc_encoder_t *	encoder,
		const char *	name,
		int	type
	)

Todo:

implment the int columnMajorWidth in the above function

Describes a new column in the encoder

Definition at line 778 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_encoder_add_property()

int odc_encoder_add_property	(	odc_encoder_t *	encoder,
		const char *	key,
		const char *	value
	)

Add a property (key-value) to the encoder to encode into the ODB2 file

Definition at line 734 of file api/odc.cc.

Here is the call graph for this function:

odc_encoder_column_add_bitfield()

int odc_encoder_column_add_bitfield	(	odc_encoder_t *	encoder,
		int	col,
		const char *	name,
		int	nbits
	)

Adds a bitfield to a column

Definition at line 807 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_encoder_column_set_data_array()

int odc_encoder_column_set_data_array	(	odc_encoder_t *	encoder,
		int	col,
		int	element_size,
		int	stride,
		const void *	data
	)

Associates a custom data layout and data array with the column. This function is used as an alternative to odc_encoder_set_data_array

Parameters

encoder	the encoder
col	the column (zero indexed)
element_size	the element size in bytes (must be a multiple of 8) [a value of 0 uses default]
stride	the separation in memory between consecutive elements in bytes [0 uses default, i.e. contiguous]
data	a pointer to the first element

Definition at line 795 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_encoder_column_set_data_size()

int odc_encoder_column_set_data_size	(	odc_encoder_t *	encoder,
		int	col,
		int	element_size
	)

Set the source data size for a given column in bytes

Parameters

encoder	the encoder
col	the column (zero indexed)
element_size	the element size in bytes (must be a multiple of 8) [a value of 0 uses default]

Definition at line 786 of file api/odc.cc.

Here is the call graph for this function:

odc_encoder_set_data_array()

int odc_encoder_set_data_array	(	odc_encoder_t *	encoder,
		const void *	data,
		long	width,
		long	height,
		int	columnMajorWidth
	)

Associate a 2D data array to an encoder and describe its shape. Note this does not yet encode the data and the data isn't copied. The client code is responsible for keeping the data in memory until encoded. There can be only one associated data array at a time per encoder, but an encoder can be used to encode a sequence of data arrays with interleaved calls to odc_encode_to_* functions.

Parameters

encoder	the encoder
data	the memory buffer
width	the width of the 2D array in bytes. NOTE it is in BYTES.
height	the height of the 2D array in rows, being greater or equal than the row count.
columnMajorWidth	if zero then 2D array is interpreted as row-major, otherwise its the size in bytes of the column entries, typically 8 bytes.

Definition at line 755 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_encoder_set_row_count()

int odc_encoder_set_row_count	(	odc_encoder_t *	encoder,
		long	nrows
	)

Declare the number of rows you are going to encode

Definition at line 741 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_encoder_set_rows_per_frame()

int odc_encoder_set_rows_per_frame	(	odc_encoder_t *	encoder,
		long	rows_per_frame
	)

Overrides the default number of rows which will be grouped together into frames

Definition at line 748 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_error_string()

const char* odc_error_string

(

int

err

)

Definition at line 93 of file api/odc.cc.

Here is the caller graph for this function:

odc_frame_bitfield_attributes()

int odc_frame_bitfield_attributes	(	const odc_frame_t *	frame,
		int	col,
		int	entry,
		const char **	name,
		int *	offset,
		int *	size
	)

The properties of an entry in a bitfield column. NULL pointers will be understood as a parameter that is not required to be filled in.

Parameters

frame	the frame
col	the column number (0-based)
entry	the entry number in the bitfield (0-based)
name	return the name of the column
offset	return which bit at which the entry in the bitfield begins
size	return the entry size in bits

Definition at line 423 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_frame_column_attributes()

int odc_frame_column_attributes	(	const odc_frame_t *	frame,
		int	col,
		const char **	name,
		int *	type,
		int *	element_size,
		int *	bitfield_count
	)

The properties of a columns in the frame. NULL pointers will be understood as a parameter that is not required to be filled in.

Parameters

frame	the frame
col	the column number (0-based)
name	return the name of the column
type	return the type of the column
element_size	return the element size in bytes (guaranteed to be a multiple of 8)
bitfield_count	return the number of entries associated with the bitfield column (0 if type is not ODC_BITFIELD)

Definition at line 404 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_frame_column_count()

int odc_frame_column_count	(	const odc_frame_t *	frame,
		int *	count
	)

The number of columns in the frame.

Definition at line 396 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_frame_row_count()

int odc_frame_row_count	(	const odc_frame_t *	frame,
		long *	count
	)

The number of rows in the frame.

Definition at line 388 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_free_decoder()

int odc_free_decoder

(

const odc_decoder_t *

decoder

)

Deallocates the decoder

Definition at line 447 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_free_encoder()

int odc_free_encoder

(

const odc_encoder_t *

encoder

)

Deallocates the encoder

Definition at line 728 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_free_frame()

int odc_free_frame

(

const odc_frame_t *

frame

)

Frees the frame object.

Definition at line 352 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_initialise_api()

int odc_initialise_api

(

)

Definition at line 203 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_integer_behaviour()

int odc_integer_behaviour

(

int

integerBehaviour

)

Definition at line 226 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_missing_double()

int odc_missing_double

(

double *

missing_value

)

The value that identifies a missing double (64 bit double) when encoding and decoding via this interface.

Definition at line 250 of file api/odc.cc.

Here is the call graph for this function:

odc_missing_integer()

int odc_missing_integer

(

long *

missing_value

)

The value that identifies a missing integer (64 bit long) when encoding and decoding via this interface.

Definition at line 243 of file api/odc.cc.

Here is the call graph for this function:

odc_new_decoder()

int odc_new_decoder

(

odc_decoder_t **

decoder

)

Creates a decoder for decoding ODB2 format

Definition at line 441 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_new_encoder()

int odc_new_encoder

(

odc_encoder_t **

encoder

)

Creates an encoder for encoding into ODB2 format

Definition at line 722 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_new_frame()

int odc_new_frame	(	odc_frame_t **	frame,
		odc_reader_t *	reader
	)

Creates a frame object, with client responsible to free it.

Definition at line 345 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_next_frame()

int odc_next_frame

(

odc_frame_t *

frame

)

Advance to the next frame. Must be called to access the first frame. Once iteration is complete (no more valid frames), it returns ODC_ITERATION_COMPLETE

Definition at line 359 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_next_frame_aggregated()

int odc_next_frame_aggregated	(	odc_frame_t *	frame,
		long	maximum_rows
	)

Advance to the next logical frame. ODB2 files contain multiple encoded frames. This function merges multiple sequential frames that have the same structure.

Definition at line 370 of file api/odc.cc.

Here is the call graph for this function:

odc_open_buffer()

int odc_open_buffer	(	odc_reader_t **	reader,
		const void *	data,
		long	length
	)

Creates a reader from a memory buffer.

Definition at line 302 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_open_file_descriptor()

int odc_open_file_descriptor	(	odc_reader_t **	reader,
		int	fd
	)

Creates a reader from an already open file descriptor. It will duplicate the file descriptor so the calling code is safe to close the file descriptor.

Definition at line 292 of file api/odc.cc.

Here is the call graph for this function:

odc_open_path()

int odc_open_path	(	odc_reader_t **	reader,
		const char *	filename
	)

Creates a reader and opens the speficied file.

Definition at line 285 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_open_stream()

int odc_open_stream	(	odc_reader_t **	reader,
		void *	context,
		odc_stream_read_t	stream_proc
	)

Creates a reader associated to a stream handler.

Definition at line 309 of file api/odc.cc.

Here is the call graph for this function:

odc_set_failure_handler()

int odc_set_failure_handler	(	odc_failure_handler_t	handler,
		void *	context
	)

Set a function to be called on error in addition to returning an error code. The handler can can access odc_error_string() The handler can also access std::current_exception() if needed (and even rethrow if required).

Definition at line 235 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_set_missing_double()

int odc_set_missing_double

(

double

missing_double

)

Set the value that identifies a missing double (64 bit double) when encoding and decoding via this interface. This differs from the marker used to identify missing values in the encoded ODB message, which depends on the codec used.

Definition at line 263 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_set_missing_integer()

int odc_set_missing_integer

(

long

missing_integer

)

Todo:

In the top CMakelists.txt assert that in this system C long is 64 bit

In the top CMakelists.txt assert that in this system C double is 64 bit

Set the value that identifies a missing integer (64 bit long) when encoding and decoding via this interface. This differs from the marker used to identify missing values in the encoded ODB message, which depends on the codec used.

Definition at line 257 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_vcs_version()

int odc_vcs_version

(

const char **

version

)

Version under VCS system, typically a git sha1. Not useful for computing software dependencies.

Definition at line 275 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

odc_version()

int odc_version

(

const char **

version

)

Version accessors Human readable release version e.g. 1.2.3

Definition at line 269 of file api/odc.cc.

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

g_current_error_str

std::string g_current_error_str

static

Definition at line 89 of file api/odc.cc.

g_failure_handler

odc_failure_handler_t g_failure_handler = nullptr

static

Definition at line 90 of file api/odc.cc.

g_failure_handler_context

void* g_failure_handler_context = nullptr

static

Definition at line 91 of file api/odc.cc.