IODA Bundle
test_encode_odb.cc
Go to the documentation of this file.
1 /*
2  * (C) Copyright 1996-2012 ECMWF.
3  *
4  * This software is licensed under the terms of the Apache Licence Version 2.0
5  * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6  * In applying this licence, ECMWF does not waive the privileges and immunities
7  * granted to it by virtue of its status as an intergovernmental organisation nor
8  * does it submit to any jurisdiction.
9  */
10 
11 #include "eckit/io/Buffer.h"
12 #include "eckit/io/MemoryHandle.h"
13 #include "eckit/testing/Test.h"
14 #include "eckit/exception/Exceptions.h"
15 #include "eckit/eckit_config.h"
16 
17 #include "odc/Writer.h"
18 #include "odc/Reader.h"
19 #include "odc/api/ColumnType.h"
20 
21 using namespace eckit::testing;
22 
23 #if __cplusplus <= 199711L
24 const float float_lowest = -std::numeric_limits<float>::max();
25 const double double_lowest = -std::numeric_limits<double>::max();
26 #else
27 const float float_lowest = std::numeric_limits<float>::lowest();
28 const double double_lowest = std::numeric_limits<double>::lowest();
29 #endif
30 
31 // ------------------------------------------------------------------------------------------------------
32 
33 CASE("Columns are initialised correctly for writing") {
34 
35  eckit::Buffer buf(4096);
36  eckit::MemoryHandle dh(buf);
37 
38  odc::Writer<> oda(dh);
39  odc::Writer<>::iterator writer = oda.begin();
40 
41  // Set up the columns
42 
43  writer->setNumberOfColumns(10);
44 
45  // Now we create the columns
46 
47  writer->setColumn(0, "int", odc::api::INTEGER);
48  writer->setColumn(1, "real", odc::api::REAL);
49  writer->setColumn(2, "str", odc::api::STRING);
50  writer->columns()[2]->dataSizeDoubles(3);
51  writer->setColumn(3, "bitf", odc::api::BITFIELD);
52  writer->setColumn(4, "dbl", odc::api::DOUBLE);
53  writer->setColumn(5, "int2", odc::api::INTEGER);
54  writer->setColumn(6, "real2", odc::api::REAL);
55  writer->setColumn(7, "str2", odc::api::STRING);
56  writer->setColumn(8, "bitf2", odc::api::BITFIELD);
57  writer->setColumn(9, "dbl2", odc::api::DOUBLE);
58 
59  // Check that the columns are correctly created
60 
61  EXPECT(writer->columns()[0]->type() == odc::api::INTEGER);
62  EXPECT(writer->columns()[1]->type() == odc::api::REAL);
63  EXPECT(writer->columns()[2]->type() == odc::api::STRING);
64  EXPECT(writer->columns()[3]->type() == odc::api::BITFIELD);
65  EXPECT(writer->columns()[4]->type() == odc::api::DOUBLE);
66  EXPECT(writer->columns()[5]->type() == odc::api::INTEGER);
67  EXPECT(writer->columns()[6]->type() == odc::api::REAL);
68  EXPECT(writer->columns()[7]->type() == odc::api::STRING);
69  EXPECT(writer->columns()[8]->type() == odc::api::BITFIELD);
70  EXPECT(writer->columns()[9]->type() == odc::api::DOUBLE);
71 
72  // ... with the correct name
73 
74  EXPECT(writer->columns()[0]->name() == "int");
75  EXPECT(writer->columns()[1]->name() == "real");
76  EXPECT(writer->columns()[2]->name() == "str");
77  EXPECT(writer->columns()[3]->name() == "bitf");
78  EXPECT(writer->columns()[4]->name() == "dbl");
79  EXPECT(writer->columns()[5]->name() == "int2");
80  EXPECT(writer->columns()[6]->name() == "real2");
81  EXPECT(writer->columns()[7]->name() == "str2");
82  EXPECT(writer->columns()[8]->name() == "bitf2");
83  EXPECT(writer->columns()[9]->name() == "dbl2");
84 
85  // ... and the correct default codecs
86 
87  EXPECT(writer->columns()[0]->coder().name() == "int32");
88  EXPECT(writer->columns()[1]->coder().name() == "long_real");
89  EXPECT(writer->columns()[2]->coder().name() == "chars");
90  EXPECT(writer->columns()[3]->coder().name() == "int32");
91  EXPECT(writer->columns()[4]->coder().name() == "long_real");
92  EXPECT(writer->columns()[5]->coder().name() == "int32");
93  EXPECT(writer->columns()[6]->coder().name() == "long_real");
94  EXPECT(writer->columns()[7]->coder().name() == "chars");
95  EXPECT(writer->columns()[8]->coder().name() == "int32");
96  EXPECT(writer->columns()[9]->coder().name() == "long_real");
97 
98  // ... and the correct expected data sizes
99 
100  EXPECT(writer->columns()[0]->coder().dataSizeDoubles() == 1);
101  EXPECT(writer->columns()[1]->coder().dataSizeDoubles() == 1);
102  EXPECT(writer->columns()[2]->coder().dataSizeDoubles() == 3);
103  EXPECT(writer->columns()[3]->coder().dataSizeDoubles() == 1);
104  EXPECT(writer->columns()[4]->coder().dataSizeDoubles() == 1);
105  EXPECT(writer->columns()[5]->coder().dataSizeDoubles() == 1);
106  EXPECT(writer->columns()[6]->coder().dataSizeDoubles() == 1);
107  EXPECT(writer->columns()[7]->coder().dataSizeDoubles() == 1);
108  EXPECT(writer->columns()[8]->coder().dataSizeDoubles() == 1);
109  EXPECT(writer->columns()[9]->coder().dataSizeDoubles() == 1);
110 }
111 
112 CASE("If out-of range columns are created, exceptions are thrown") {
113 
114  eckit::Buffer buf(4096);
115  eckit::MemoryHandle dh(buf);
116 
117  odc::Writer<> oda(dh);
118  odc::Writer<>::iterator writer = oda.begin();
119 
120  writer->setNumberOfColumns(10);
121  writer->setColumn(2, "str", odc::api::STRING); // This is fine
122 
123  // If we create columns out of range, it throws exceptions
124 
125  EXPECT_THROWS_AS(writer->setColumn(11, "badnum", odc::api::INTEGER), eckit::AssertionFailed);
126 }
127 
128 CASE("If columns are created with invalid types, exceptions are thrown") {
129 
130  eckit::Buffer buf(4096);
131  eckit::MemoryHandle dh(buf);
132 
133  odc::Writer<> oda(dh);
134  odc::Writer<>::iterator writer = oda.begin();
135 
136  // Set up the columns
137 
138  writer->setNumberOfColumns(10);
139  writer->setColumn(6, "real", odc::api::REAL); // This is fine
140 
141  // We cannot create a column of "IGNORE" type, or any type that is not listed in the enum
142 
143  EXPECT_THROWS_AS(writer->setColumn(0, "ignore", odc::api::IGNORE), eckit::AssertionFailed);
144  EXPECT_THROWS_AS(writer->setColumn(0, "ignore", static_cast<odc::api::ColumnType>(123)), eckit::AssertionFailed);
145 }
146 
147 CASE("Columns names must be unique") {
148 
149  // See issue ODB-372
150 
151  eckit::Buffer buf(4096);
152  eckit::MemoryHandle dh(buf);
153 
154  odc::Writer<> oda(dh);
155  odc::Writer<>::iterator writer = oda.begin();
156 
157  // Set up the columns
158 
159  writer->setNumberOfColumns(10);
160 
161  writer->setColumn(0, "int", odc::api::INTEGER);
162  writer->setColumn(1, "real", odc::api::REAL);
163  writer->setColumn(2, "str", odc::api::STRING);
164  writer->setColumn(3, "bitf", odc::api::BITFIELD);
165  writer->setColumn(4, "dbl", odc::api::DOUBLE);
166 
167  EXPECT_THROWS_AS(writer->setColumn(5, "int", odc::api::INTEGER), eckit::SeriousBug);
168  EXPECT_THROWS_AS(writer->setColumn(6, "real", odc::api::REAL), eckit::SeriousBug);
169  EXPECT_THROWS_AS(writer->setColumn(7, "str", odc::api::STRING), eckit::SeriousBug);
170  EXPECT_THROWS_AS(writer->setColumn(8, "bitf", odc::api::BITFIELD), eckit::SeriousBug);
171  EXPECT_THROWS_AS(writer->setColumn(9, "dbl", odc::api::DOUBLE), eckit::SeriousBug);
172 }
173 
174 
175 CASE("Data is encoded and read back correctly") {
176 
177  const int32_t i1 = 987654321;
178  const int32_t i2 = -1;
179  const int32_t i3 = std::numeric_limits<int32_t>::min();
180  const int32_t i4 = std::numeric_limits<int32_t>::max();
181  const int32_t i5 = 0;
182  const int32_t i6 = -654321;
183 
184  const float f1 = std::numeric_limits<float>::min();
185  const float f2 = std::numeric_limits<float>::max();
186  const float f3 = 0.0;
187  const float f4 = float_lowest;
188  const float f5 = static_cast<float>(654321.123);
189  const float f6 = static_cast<float>(-123456.789e-21);
190 
191  const char* const s1 = "a-string-longstrvvvvlong";
192  const char* const s2 = "string-2";
193  const char* const s3 = "string-3-LLong";
194  const char* const s4 = "string-4";
195  const char* const s5 = "string-5-LLong";
196  const char* const s6 = "string-6";
197 
198  const int32_t b1 = static_cast<int32_t>((uint32_t)std::numeric_limits<uint32_t>::min());
199  const int32_t b2 = static_cast<int32_t>((uint32_t)std::numeric_limits<uint32_t>::max());
200  const int32_t b3 = static_cast<int32_t>((uint32_t)0);
201  const int32_t b4 = static_cast<int32_t>((uint32_t)0xff00ff00);
202  const int32_t b5 = static_cast<int32_t>((uint32_t)0x00ff00ff);
203  const int32_t b6 = static_cast<int32_t>((uint32_t)0xfedcba98);
204 
205  const double d1 = std::numeric_limits<double>::min();
206  const double d2 = std::numeric_limits<double>::max();
207  const double d3 = double_lowest;
208  const double d4 = 0.0;
209  const double d5 = -123456789.0123;
210  const double d6 = 987654321.987e-56;
211 
212  // See issue ODB-372
213 
214  eckit::Buffer buf(4096);
215  eckit::MemoryHandle dhWrite(buf);
216 
217  {
218  odc::Writer<> oda(dhWrite);
219  odc::Writer<>::iterator writer = oda.begin();
220 
221  // Set up the columns
222 
223  writer->setNumberOfColumns(10);
224  writer->setColumn(0, "int", odc::api::INTEGER);
225  writer->setColumn(1, "real", odc::api::REAL);
226  writer->setColumn(2, "str", odc::api::STRING);
227  writer->columns()[2]->dataSizeDoubles(3);
228  writer->setColumn(3, "bitf", odc::api::BITFIELD);
229  writer->setColumn(4, "dbl", odc::api::DOUBLE);
230  writer->setColumn(5, "int2", odc::api::INTEGER);
231  writer->setColumn(6, "real2", odc::api::REAL);
232  writer->setColumn(7, "str2", odc::api::STRING);
233  writer->setColumn(8, "bitf2", odc::api::BITFIELD);
234  writer->setColumn(9, "dbl2", odc::api::DOUBLE);
235  writer->writeHeader();
236 
237  // Test that the data offsets are correct
238 
239  EXPECT(writer->dataOffset(0) == 0);
240  EXPECT(writer->dataOffset(1) == 1);
241  EXPECT(writer->dataOffset(2) == 2);
242  EXPECT(writer->dataOffset(3) == 5);
243  EXPECT(writer->dataOffset(4) == 6);
244  EXPECT(writer->dataOffset(5) == 7);
245  EXPECT(writer->dataOffset(6) == 8);
246  EXPECT(writer->dataOffset(7) == 9);
247  EXPECT(writer->dataOffset(8) == 10);
248  EXPECT(writer->dataOffset(9) == 11);
249 
250  // Append 3 rows of data (in two different ways)
251 
252  (*writer)[0] = i1;
253  (*writer)[1] = static_cast<double>(f1);
254  ::strncpy(reinterpret_cast<char*>(&(*writer)[2]), s1, 24); // strncpy pads with \0
255  (*writer)[3] = b1;
256  (*writer)[4] = d1;
257  (*writer)[5] = i2;
258  (*writer)[6] = static_cast<double>(f2);
259  (*writer)[7] = *reinterpret_cast<const double*>(s2);
260  (*writer)[8] = b2;
261  (*writer)[9] = d2;
262  ++writer;
263 
264  writer->data()[writer->dataOffset(0)] = i3;
265  writer->data()[writer->dataOffset(1)] = static_cast<double>(f3);
266  ::strncpy(reinterpret_cast<char*>(&writer->data()[writer->dataOffset(2)]), s3, 24); // strncpy pads with \0
267  writer->data()[writer->dataOffset(3)] = b3;
268  writer->data()[writer->dataOffset(4)] = d3;
269  writer->data()[writer->dataOffset(5)] = i4;
270  writer->data()[writer->dataOffset(6)] = static_cast<double>(f4);
271  writer->data()[writer->dataOffset(7)] = *reinterpret_cast<const double*>(s4);
272  writer->data()[writer->dataOffset(8)] = b4;
273  writer->data()[writer->dataOffset(9)] = d4;
274  ++writer;
275 
276  writer->data(0) = i5;
277  writer->data(1) = static_cast<double>(f5);
278  ::strncpy(reinterpret_cast<char*>(&writer->data(2)), s5, 24); // strncpy pads with \0
279  writer->data(3) = b5;
280  writer->data(4) = d5;
281  writer->data(5) = i6;
282  writer->data(6) = static_cast<double>(f6);
283  writer->data(7) = *reinterpret_cast<const double*>(s6);
284  writer->data(8) = b6;
285  writer->data(9) = d6;
286  ++writer;
287  }
288 
289  // Read everything back
290 
291  {
292  eckit::MemoryHandle dh(buf.data(), static_cast<size_t>(dhWrite.position()));
293  dh.openForRead();
294  odc::Reader oda(dh);
295  odc::Reader::iterator reader = oda.begin();
296 
297  EXPECT(reader->columns().size() == size_t(10));
298 
299  EXPECT(reader->columns()[0]->dataSizeDoubles() == 1);
300  EXPECT(reader->columns()[1]->dataSizeDoubles() == 1);
301  EXPECT(reader->columns()[2]->dataSizeDoubles() == 3);
302  EXPECT(reader->columns()[3]->dataSizeDoubles() == 1);
303  EXPECT(reader->columns()[4]->dataSizeDoubles() == 1);
304  EXPECT(reader->columns()[5]->dataSizeDoubles() == 1);
305  EXPECT(reader->columns()[6]->dataSizeDoubles() == 1);
306  EXPECT(reader->columns()[7]->dataSizeDoubles() == 1);
307  EXPECT(reader->columns()[8]->dataSizeDoubles() == 1);
308  EXPECT(reader->columns()[9]->dataSizeDoubles() == 1);
309 
310  EXPECT((*reader)[0] == i1);
311  EXPECT((*reader)[1] == static_cast<double>(f1));
312  EXPECT(::strncmp(reinterpret_cast<const char*>(&(*reader)[2]), s1, 24) == 0);
313  EXPECT((*reader)[3] == b1);
314  EXPECT((*reader)[4] == d1);
315  EXPECT((*reader)[5] == i2);
316  EXPECT((*reader)[6] == static_cast<double>(f2));
317  EXPECT((*reader)[7] == *reinterpret_cast<const double*>(s2));
318  EXPECT((*reader)[8] == b2);
319  EXPECT((*reader)[9] == d2);
320  ++reader;
321 
322  EXPECT(reader->data()[reader->dataOffset(0)] == i3);
323  EXPECT(reader->data()[reader->dataOffset(1)] == static_cast<double>(f3));
324  EXPECT(::strncmp(reinterpret_cast<const char*>(&reader->data()[reader->dataOffset(2)]),s3, 24) == 0);
325  EXPECT(reader->data()[reader->dataOffset(3)] == b3);
326  EXPECT(reader->data()[reader->dataOffset(4)] == d3);
327  EXPECT(reader->data()[reader->dataOffset(5)] == i4);
328  EXPECT(reader->data()[reader->dataOffset(6)] == static_cast<double>(f4));
329  EXPECT(reader->data()[reader->dataOffset(7)] == *reinterpret_cast<const double*>(s4));
330  EXPECT(reader->data()[reader->dataOffset(8)] == b4);
331  EXPECT(reader->data()[reader->dataOffset(9)] == d4);
332  ++reader;
333 
334  EXPECT(reader->data(0) == i5);
335  EXPECT(reader->data(1) == static_cast<double>(f5));
336  EXPECT(::strncmp(reinterpret_cast<const char*>(&reader->data(2)), s5, 24) == 0);
337  EXPECT(reader->data(3) == b5);
338  EXPECT(reader->data(4) == d5);
339  EXPECT(reader->data(5) == i6);
340  EXPECT(reader->data(6) == static_cast<double>(f6));
341  EXPECT(reader->data(7) == *reinterpret_cast<const double*>(s6));
342  EXPECT(reader->data(8) == b6);
343  EXPECT(reader->data(9) == d6);
344  }
345 }
346 
347 
348 #if 0 // This test needs to be reassesed -- AssertionFailed is not likely the correct Exception to be thrown
349 
350 CASE("We cannot encode short_real with both possible internal missing values") {
351 
352  const float f1 = std::numeric_limits<float>::min();
353  const float f2 = float_lowest;
354 
355  eckit::Buffer buf(4096);
356 
357  EXPECT_THROWS_AS({
358  eckit::MemoryHandle dh(buf);
359  odc::Writer<> oda(dh);
360  odc::Writer<>::iterator writer = oda.begin();
361 
362  // Set up the columns
363 
364  writer->setNumberOfColumns(1);
365  writer->setColumn(0, "real", odc::api::REAL);
366  writer->writeHeader();
367 
368  // Append 3 rows of data (in two different ways)
369 
370  (*writer)[0] = f1;
371  ++writer;
372 
373  (*writer)[0] = f2;
374  ++writer;
375 
376  }, eckit::AssertionFailed);
377 
378 }
379 
380 #endif
381 
382 CASE("We ASSERT on cases where we try and use an incompletely configured writer") {
383 
384  eckit::Buffer buf(4096);
385 
386  // Illegal to flush an incompletely finished writer
387 
388  eckit::MemoryHandle dh(buf);
389  odc::Writer<> oda(dh);
390  odc::Writer<>::iterator writer = oda.begin();
391 
392  // Set up the columns
393 
394  writer->setNumberOfColumns(2);
395  writer->setColumn(0, "real", odc::api::REAL);
396 
397  // Cannot writeHeader until all the columns are initialised
398  EXPECT_THROWS_AS(writer->writeHeader(), eckit::AssertionFailed);
399 
400  // Cannot write to an uninitialised writer
401  EXPECT_THROWS_AS((*writer)[0] = 1234.56, eckit::AssertionFailed);
402 
403  // Cannot increment an incomplete row
404  EXPECT_THROWS_AS(++writer, eckit::AssertionFailed);
405 }
406 
407 CASE("Data is automatically written after a configurable number of rows") {
408  EXPECT(true);
409 }
410 
411 CASE("Pathological data for integral codecs is correctly encoded") {
412 
413  // The reduced-size integral codecs have special internal values for missingValue.
414  // If we try and encode an integer that happens to collide with that value whilst
415  // missing values are enabled, the codec in use needs to be uprated to the next
416  // biggest size, so things are encoded correctly.
417 
418  eckit::Buffer buf(4096);
419 
420  for (int i = 0; i < 4; i++) {
421 
422  bool withMissing = (i % 2 == 1);
423  bool bits16 = (i > 1);
424 
425 // eckit::Log::info() << "iteration: " << i
426 // << (withMissing ? "T":"F")
427 // << (bits16 ? "T":"F") << std::endl;
428 
429  int32_t i1 = 12345;
430  int32_t i2 = 12345 + (bits16 ? 0xffff : 0xff);
431 
432  {
433  eckit::MemoryHandle dh(buf);
434  odc::Writer<> oda(dh);
435  odc::Writer<>::iterator writer = oda.begin();
436 
437  // Set up the columns
438 
439  writer->setNumberOfColumns(1);
440  writer->setColumn(0, "int", odc::api::INTEGER);
441  writer->writeHeader();
442 
443  // Append 3 rows of data (in two different ways)
444 
445  (*writer)[0] = i1;
446  ++writer;
447 
448  (*writer)[0] = i2;
449  ++writer;
450 
451  if (withMissing) {
452  (*writer)[0] = odc::MDI::integerMDI();
453  ++writer;
454  }
455 
456  // We have not supplied missing values
457  EXPECT(writer->columns()[0]->missingValue() == odc::MDI::integerMDI());
458  EXPECT(writer->columns()[0]->hasMissing() == withMissing);
459  }
460 
461  // Read everything back
462 
463  {
464  eckit::MemoryHandle dh(buf);
465  dh.openForRead();
466  odc::Reader oda(dh);
467  odc::Reader::iterator reader = oda.begin();
468 
469  // We have not supplied missing values
470  EXPECT(reader->columns().size() == size_t(1));
471  EXPECT(reader->columns()[0]->missingValue() == odc::MDI::integerMDI());
472  EXPECT(reader->columns()[0]->hasMissing() == withMissing);
473 
474  // Promotion to int32 occurs to
475 
476  if (withMissing && bits16) {
477  EXPECT(reader->columns()[0]->coder().name() == "int32");
478  } else if (withMissing) {
479  EXPECT(reader->columns()[0]->coder().name() == "int16_missing");
480  } else if (bits16) {
481  EXPECT(reader->columns()[0]->coder().name() == "int16");
482  } else {
483  EXPECT(reader->columns()[0]->coder().name() == "int8");
484  }
485 
486  EXPECT((*reader)[0] == i1);
487  ++reader;
488 
489  EXPECT((*reader)[0] == i2);
490 
491  if (withMissing) {
492  ++reader;
493  EXPECT((*reader)[0] == odc::MDI::integerMDI());
494  }
495  }
496  }
497 
498 }
499 
500 // ------------------------------------------------------------------------------------------------------
501 
502 int main(int argc, char* argv[]) {
503  return run_tests(argc, argv);
504 }
505 
506 
oda
void oda
Definition: ReaderIterator.h:27
odc::MDI::integerMDI
static double integerMDI()
Definition: MDI.h:22
odc::Row_::data
DATA * data()
Definition: IteratorProxy.h:77
odc::Row_::columns
const core::MetaData & columns() const
Definition: IteratorProxy.h:94
odc::Row_::dataOffset
size_t dataOffset(size_t i) const
Definition: IteratorProxy.h:91
02-Attributes.f1
f1
Definition: 02-Attributes.py:119
02-Attributes.d1
d1
Definition: 02-Attributes.py:120
odc::api::BITFIELD
@ BITFIELD
Definition: ColumnType.h:27
proc_gsi_ncdiag.i
i
Definition: proc_gsi_ncdiag.py:155
main
int main(int argc, char *argv[])
Definition: test_encode_odb.cc:502
double_lowest
const double double_lowest
Definition: test_encode_odb.cc:25
CASE
CASE("Columns are initialised correctly for writing")
Definition: test_encode_odb.cc:33
float_lowest
const float float_lowest
Definition: test_encode_odb.cc:24