IODA Bundle
test_codecs_read.cc
Go to the documentation of this file.
1 /*
2  * (C) Copyright 1996-2012 ECMWF.
3  *
4  * This software is licensed under the terms of the Apache Licence Version 2.0
5  * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6  * In applying this licence, ECMWF does not waive the privileges and immunities
7  * granted to it by virtue of its status as an intergovernmental organisation nor
8  * does it submit to any jurisdiction.
9  */
10 
11 #include "eckit/testing/Test.h"
12 #include "eckit/system/SystemInfo.h"
13 #include "eckit/io/DataHandle.h"
14 #include "eckit/eckit_ecbuild_config.h"
15 
16 #include "odc/api/Odb.h"
17 #include "odc/core/Codec.h"
18 #include "odc/codec/Constant.h"
19 #include "odc/codec/Integer.h"
20 #include "odc/codec/IntegerMissing.h"
21 #include "odc/codec/Real.h"
22 #include "odc/codec/String.h"
23 
24 #include <cmath>
25 #include <time.h>
26 #include <stdlib.h>
27 #include <algorithm>
28 
29 using namespace eckit::testing;
30 using namespace odc::core;
31 using namespace odc::codec;
32 
33 namespace {
34  struct TestIntegerDecoding {
35  TestIntegerDecoding() { odc::api::Settings::treatIntegersAsDoubles(false); }
36  ~TestIntegerDecoding() { odc::api::Settings::treatIntegersAsDoubles(true); }
37  };
38 }
39 
40 // ------------------------------------------------------------------------------------------------------
41 
42 // TODO with codecs:
43 //
44 // i) Make them templated on the stream/datahandle directly
45 // ii) Construct them with a specific data handle/stream
46 // iii) Why are we casting data handles via a void* ???
47 
48 // Given the codec-initialising data, add the header on that is used to construct the
49 // codec.
50 
51 size_t prepend_codec_selection_header(std::vector<unsigned char>& data,
52  const std::string& codec_name,
53  bool bigEndian=false) {
54 
55  data.insert(data.begin(), 4, 0);
56  data[bigEndian ? 3 : 0] = static_cast<unsigned char>(codec_name.size());
57 
58  data.insert(data.begin() + 4, codec_name.begin(), codec_name.end());
59 
60  return 4 + codec_name.length();
61 }
62 
63 CASE("Constant values are constant") {
64 
65  // Data in little endian format.
66  // "min" value is used for constants
67 
68  const char* source_data[] = {
69 
70  // Codec header
71  "\x00\x00\x00\x00", // no missing value
72  "\xb7\xe6\x87\xb4\x80\x65\xd2\x41", // min (1234567890.1234567)
73  "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified
74  "\x00\x00\x00\x00\x00\x00\x00\x00", // missing value unspecified
75 
76  };
77 
78  // Loop throumgh endiannesses for the source data
79 
80  for (int i = 0; i < 2; i++) {
81 
82  bool bigEndianSource = (i == 1);
83 
84  std::vector<unsigned char> data;
85 
86  for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) {
87  size_t len = (j == 0) ? 4 : 8;
88  data.insert(data.end(), source_data[j], source_data[j] + len);
89  if (bigEndianSource)
90  std::reverse(data.end()-len, data.end());
91  }
92 
93  // Construct codec directly
94 
95 #ifndef _CRAYC
96  {
97  // Skip name of codec
98  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
99 
100  std::unique_ptr<Codec> c;
101  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
102  c.reset(new CodecConstant<SameByteOrder, double>(odc::api::DOUBLE));
103  } else {
104  c.reset(new CodecConstant<OtherByteOrder, double>(odc::api::DOUBLE));
105  }
106  c->load(ds);
107 
108  EXPECT(ds.position() == eckit::Offset(28));
109  EXPECT(c->dataSizeDoubles() == 1);
110 
111  double tmp;
112  c->decode(&tmp);
113  EXPECT(tmp == 1234567890.1234567);
114  c->decode(&tmp);
115  EXPECT(tmp == 1234567890.1234567);
116  c->decode(&tmp);
117  EXPECT(tmp == 1234567890.1234567);
118  c->decode(&tmp);
119  EXPECT(tmp == 1234567890.1234567);
120 
121  // No further data should have been consumed from the data handle.
122  EXPECT(ds.position() == eckit::Offset(28));
123  }
124 #endif
125 
126  // Construct codec from factory
127 
128  size_t hdrSize = prepend_codec_selection_header(data, "constant", bigEndianSource);
129 
130  {
131  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
132 
133  std::unique_ptr<Codec> c;
134  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
135  c = CodecFactory::instance().load(ds.same(), odc::api::DOUBLE);
136  } else {
137  c = CodecFactory::instance().load(ds.other(), odc::api::DOUBLE);
138  }
139 
140  EXPECT(ds.position() == eckit::Offset(hdrSize + 28));
141  EXPECT(c->dataSizeDoubles() == 1);
142 
143  double tmp;
144  c->decode(&tmp);
145  EXPECT(tmp == 1234567890.1234567);
146  EXPECT(tmp == 1234567890.1234567);
147  EXPECT(tmp == 1234567890.1234567);
148  EXPECT(tmp == 1234567890.1234567);
149 
150  EXPECT(ds.position() == eckit::Offset(hdrSize + 28));
151  }
152  }
153 }
154 
155 CASE("Constant integer values are constant") {
156 
157  // Set to decode to integers rather than doubles
158  TestIntegerDecoding resetter;
159 
160  // Data in little endian format.
161  // "min" value is used for constants
162 
163  const char* source_data[] = {
164 
165  // Codec header
166  "\x00\x00\x00\x00", // no missing value
167  "\x00\x00\x80\xb4\x80\x65\xd2\x41", // min (1234567890.1234567)
168  "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified
169  "\x00\x00\x00\x00\x00\x00\x00\x00", // missing value unspecified
170 
171  };
172 
173  // Loop throumgh endiannesses for the source data
174 
175  for (int i = 0; i < 2; i++) {
176 
177  bool bigEndianSource = (i == 1);
178 
179  std::vector<unsigned char> data;
180 
181  for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) {
182  size_t len = (j == 0) ? 4 : 8;
183  data.insert(data.end(), source_data[j], source_data[j] + len);
184  if (bigEndianSource)
185  std::reverse(data.end()-len, data.end());
186  }
187 
188  // Construct codec directly
189 
190 #ifndef _CRAYC
191  {
192  // Skip name of codec
193  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
194 
195  std::unique_ptr<Codec> c;
196  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
197  c.reset(new CodecConstant<SameByteOrder, int64_t>(odc::api::BITFIELD));
198  } else {
199  c.reset(new CodecConstant<OtherByteOrder, int64_t>(odc::api::INTEGER));
200  }
201  c->load(ds);
202 
203  EXPECT(ds.position() == eckit::Offset(28));
204  EXPECT(c->dataSizeDoubles() == 1);
205 
206  int64_t tmpi;
207  double& tmpd(reinterpret_cast<double&>(tmpi));
208  c->decode(&tmpd);
209  EXPECT(tmpi == 1234567890);
210  c->decode(&tmpd);
211  EXPECT(tmpi == 1234567890);
212  c->decode(&tmpd);
213  EXPECT(tmpi == 1234567890);
214  c->decode(&tmpd);
215  EXPECT(tmpi == 1234567890);
216 
217  // No further data should have been consumed from the data handle.
218  EXPECT(ds.position() == eckit::Offset(28));
219  }
220 #endif
221 
222  // Construct codec from factory
223 
224  size_t hdrSize = prepend_codec_selection_header(data, "constant", bigEndianSource);
225 
226  {
227  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
228 
229  std::unique_ptr<Codec> c;
230  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
231  c = CodecFactory::instance().load(ds.same(), odc::api::INTEGER);
232  } else {
233  c = CodecFactory::instance().load(ds.other(), odc::api::BITFIELD);
234  }
235 
236  EXPECT(ds.position() == eckit::Offset(hdrSize + 28));
237  EXPECT(c->dataSizeDoubles() == 1);
238 
239  int64_t tmpi;
240  double& tmp(reinterpret_cast<double&>(tmpi));
241  c->decode(&tmp);
242  EXPECT(tmpi == 1234567890);
243  EXPECT(tmpi == 1234567890);
244  EXPECT(tmpi == 1234567890);
245  EXPECT(tmpi == 1234567890);
246 
247  EXPECT(ds.position() == eckit::Offset(hdrSize + 28));
248  }
249  }
250 }
251 
252 
253 CASE("constant strings are constant") {
254 
255  // Data in little endian format.
256  // "min" value is used for constants
257 
258  // NOTE that strings are NOT swapped around when things are in the
259  // reverse byte order.
260 
261  const char* source_data[] = {
262 
263  // Codec header
264  "\x00\x00\x00\x00", // no missing value
265  "hi-there", // minimum supplies string
266  "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified
267  "\x00\x00\x00\x00\x00\x00\x00\x00", // missing value unspecified
268  };
269 
270  // Loop through endiannesses for the source data
271 
272  for (int i = 0; i < 2; i++) {
273 
274  bool bigEndianSource = (i == 1);
275 
276  std::vector<unsigned char> data;
277 
278  for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) {
279  size_t len = (j == 0) ? 4 : 8;
280  data.insert(data.end(), source_data[j], source_data[j] + len);
281 
282  // n.b. Don't swap string data around with endianness
283  if (bigEndianSource && j != 1)
284  std::reverse(data.end()-len, data.end());
285  }
286 
287  // Construct codec directly
288 
289 #ifndef _CRAYC
290  {
291  // Skip name of codec
292  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
293 
294  std::unique_ptr<Codec> c;
295  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
296  c.reset(new CodecConstantString<SameByteOrder>(odc::api::STRING));
297  } else {
298  c.reset(new CodecConstantString<OtherByteOrder>(odc::api::STRING));
299  }
300  c->load(ds);
301 
302  EXPECT(ds.position() == eckit::Offset(28));
303  EXPECT(c->dataSizeDoubles() == 1);
304 
305  double val;
306  c->decode(&val);
307  EXPECT(std::string(reinterpret_cast<const char*>(&val), 8) == "hi-there");
308  c->decode(&val);
309  EXPECT(std::string(reinterpret_cast<const char*>(&val), 8) == "hi-there");
310  c->decode(&val);
311  EXPECT(std::string(reinterpret_cast<const char*>(&val), 8) == "hi-there");
312  c->decode(&val);
313  EXPECT(std::string(reinterpret_cast<const char*>(&val), 8) == "hi-there");
314 
315  // No further data should have been consumed from the data handle.
316  EXPECT(ds.position() == eckit::Offset(28));
317  }
318 #endif
319 
320  // Construct codec from factory
321 
322  size_t hdrSize = prepend_codec_selection_header(data, "constant_string", bigEndianSource);
323 
324  {
325  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
326 
327  std::unique_ptr<Codec> c;
328  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
329  c = CodecFactory::instance().load(ds.same(), odc::api::STRING);
330  } else {
331  c = CodecFactory::instance().load(ds.other(), odc::api::STRING);
332  }
333 
334  EXPECT(ds.position() == eckit::Offset(hdrSize + 28));
335  EXPECT(c->dataSizeDoubles() == 1);
336 
337  double val;
338  c->decode(&val);
339  EXPECT(std::string(reinterpret_cast<const char*>(&val), 8) == "hi-there");
340  c->decode(&val);
341  EXPECT(std::string(reinterpret_cast<const char*>(&val), 8) == "hi-there");
342  c->decode(&val);
343  EXPECT(std::string(reinterpret_cast<const char*>(&val), 8) == "hi-there");
344  c->decode(&val);
345  EXPECT(std::string(reinterpret_cast<const char*>(&val), 8) == "hi-there");
346 
347  EXPECT(ds.position() == eckit::Offset(hdrSize + 28));
348  }
349  }
350 }
351 
352 CASE("Constant integer or missing value behaves a bit oddly") {
353 
354  EXPECT(odc::MDI::integerMDI() == 2147483647);
355 
356  // Note that there is absolutely NOTHING that enforces that these are integers...
357  // --> This test tests the generic case, with a double, which is odd
358  // --> TODO: Really we ought to enforce integers for an integer case...
359 
360  // little
361 
362  const char* source_data[] = {
363 
364  // Codec header
365  "\x01\x00\x00\x00", // has missing value
366 // "\x00\x00\x80\x58\x34\x6f\xcd\x41, // min (little-endian: 987654321)
367 // "\x00\x00\x80\x58\x34\x6f\xcd\x41, // max == min
368  "\xad\x69\xfe\x58\x34\x6f\xcd\x41", // minimum value = 987654321.9876
369  "\xad\x69\xfe\x58\x34\x6f\xcd\x41", // maximum value
370  "\x00\x00\xc0\xff\xff\xff\xdf\x41" // missingValue = 2147483647
371  };
372 
373  // Loop through endiannesses for the source data
374 
375  for (int i = 0; i < 2; i++) {
376 
377  bool bigEndianSource = (i == 1);
378 
379  std::vector<unsigned char> data;
380 
381  for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) {
382  size_t len = (j == 0) ? 4 : 8;
383  data.insert(data.end(), source_data[j], source_data[j] + len);
384  if (bigEndianSource)
385  std::reverse(data.end()-len, data.end());
386  }
387 
388  // Insert the sequence of test values
389 
390  data.push_back(0);
391  data.push_back(0xff); // missing
392  for (size_t n = 0; n < 255; n++) {
393  data.push_back(static_cast<unsigned char>(n));
394  }
395  data.push_back(0xff); // missing
396 
397  // Construct codec directly
398 
399 #ifndef _CRAYC
400  {
401  // Skip name of codec
402  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
403 
404  std::unique_ptr<Codec> c;
405  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
406  c.reset(new CodecConstantOrMissing<SameByteOrder, double>(odc::api::DOUBLE));
407  } else {
408  c.reset(new CodecConstantOrMissing<OtherByteOrder, double>(odc::api::INTEGER));
409  }
410  c->load(ds);
411  c->setDataStream(ds);
412 
413  EXPECT(ds.position() == eckit::Offset(28));
414  EXPECT(c->dataSizeDoubles() == 1);
415 
416  double baseValue = 987654321.9876;
417  // double baseValue = 987654321;
418  double decoded;
419  c->decode(&decoded);
420  EXPECT(baseValue == decoded);
421  c->decode(&decoded);
422  EXPECT(decoded == odc::MDI::integerMDI()); // missing
423  for (size_t n = 0; n < 255; n++) {
424  double b = baseValue + n;
425  c->decode(&decoded);
426  EXPECT(b == decoded);
427  }
428  c->decode(&decoded);
429  EXPECT(decoded == odc::MDI::integerMDI()); // missing
430 
431  EXPECT(ds.position() == eckit::Offset(28 + 258));
432  }
433 #endif
434 
435  // Construct codec from factory
436 
437  size_t hdrSize = prepend_codec_selection_header(data, "constant_or_missing", bigEndianSource);
438 
439  {
440  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
441 
442  std::unique_ptr<Codec> c;
443  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
444  c = CodecFactory::instance().load(ds.same(), odc::api::INTEGER);
445  } else {
446  c = CodecFactory::instance().load(ds.other(), odc::api::DOUBLE);
447  }
448  c->setDataStream(ds);
449 
450  EXPECT(ds.position() == eckit::Offset(hdrSize+28));
451  EXPECT(c->dataSizeDoubles() == 1);
452 
453  double baseValue = 987654321.9876;
454  double decoded;
455  c->decode(&decoded);
456  EXPECT(baseValue == decoded);
457  c->decode(&decoded);
458  EXPECT(decoded == odc::MDI::integerMDI()); // missing
459  for (size_t i = 0; i < 255; i++) {
460  double b = baseValue + i;
461  c->decode(&decoded);
462  EXPECT(b == decoded);
463  }
464  c->decode(&decoded);
465  EXPECT(decoded == odc::MDI::integerMDI()); // missing
466 
467  EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + 258));
468  }
469  }
470 }
471 
472 
473 CASE("real constant or missing value is not quite constant") {
474 
475  EXPECT(odc::MDI::realMDI() == -2147483647);
476 
477  // TODO: Really something labelled constant ought to be actually constant...
478  // Do this one big-endian just because.
479 
480  // BIG endian data
481 
482  const char* source_data[] = {
483 
484  // Codec header
485  "\x00\x00\x00\x01", // has missing value
486  "\x41\xcd\x6f\x34\x58\xfe\x69\xad", // min = 987654321.9876 (big-endian)
487  "\x41\xcd\x6f\x34\x58\xfe\x69\xad", // max = 987654321.9876 (big-endian)
488  "\xc1\xdf\xff\xff\xff\xc0\x00\x00" // missingValue = -2147483647
489  };
490 
491  // Loop through endiannesses for the source data
492 
493  for (int i = 0; i < 2; i++) {
494 
495  bool bigEndianSource = (i == 1);
496 
497  std::vector<unsigned char> data;
498 
499  for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) {
500  size_t len = (j == 0) ? 4 : 8;
501  data.insert(data.end(), source_data[j], source_data[j] + len);
502  if (!bigEndianSource)
503  std::reverse(data.end()-len, data.end());
504  }
505 
506  // Insert the sequence of test values
507 
508  data.push_back(0);
509  data.push_back(0xff); // missing
510  for (size_t i = 0; i < 255; i++) {
511  data.push_back(static_cast<unsigned char>(i));
512  }
513  data.push_back(0xff); // missing
514 
515  // Construct codec directly
516 
517 #ifndef _CRAYC
518  {
519  // Skip name of codec
520  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
521 
522  std::unique_ptr<Codec> c;
523  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
524  c.reset(new CodecRealConstantOrMissing<SameByteOrder>(odc::api::DOUBLE));
525  } else {
526  c.reset(new CodecRealConstantOrMissing<OtherByteOrder>(odc::api::DOUBLE));
527  }
528  c->load(ds);
529  c->setDataStream(ds);
530 
531  EXPECT(ds.position() == eckit::Offset(28));
532  EXPECT(c->dataSizeDoubles() == 1);
533 
534  double baseValue = 987654321.9876;
535  // double baseValue = 987654321;
536  double decoded;
537  c->decode(&decoded);
538  EXPECT(baseValue == decoded);
539  c->decode(&decoded);
540  EXPECT(decoded == odc::MDI::realMDI()); // missing
541  for (size_t i = 0; i < 255; i++) {
542  double b = baseValue + i;
543  c->decode(&decoded);
544  EXPECT(b == decoded);
545  }
546  c->decode(&decoded);
547  EXPECT(decoded == odc::MDI::realMDI()); // missing
548 
549  EXPECT(ds.position() == eckit::Offset(28 + 258));
550  }
551 #endif
552 
553  // Construct codec from factory
554 
555  size_t hdrSize = prepend_codec_selection_header(data, "real_constant_or_missing", bigEndianSource);
556 
557  {
558  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
559 
560  std::unique_ptr<Codec> c;
561  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
562  c = CodecFactory::instance().load(ds.same(), odc::api::DOUBLE);
563  } else {
564  c = CodecFactory::instance().load(ds.other(), odc::api::DOUBLE);
565  }
566  c->setDataStream(ds);
567 
568  EXPECT(ds.position() == eckit::Offset(hdrSize + 28));
569  EXPECT(c->dataSizeDoubles() == 1);
570 
571  double baseValue = 987654321.9876;
572  double decoded;
573  c->decode(&decoded);
574  EXPECT(baseValue == decoded);
575  c->decode(&decoded);
576  EXPECT(decoded == odc::MDI::realMDI()); // missing
577  for (size_t i = 0; i < 255; i++) {
578  double b = baseValue + i;
579  c->decode(&decoded);
580  EXPECT(b == decoded);
581  }
582  c->decode(&decoded);
583  EXPECT(decoded == odc::MDI::realMDI()); // missing
584 
585  EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + 258));
586  }
587  }
588 }
589 
590 
591 CASE("Character strings are 8-byte sequences coerced into being treated as doubles") {
592 
593  // n.b. there are no missing values for CodecChars
594 
595  // The values here are unused, and endianness is ignored for chars.
596 
597  const char* source_data[] = {
598 
599  // Codec header
600  "\x00\x00\x00\x00", // 0 = hasMissing
601  "\x00\x00\x00\x00\x00\x00\x00\x00", // min = 987654321.9876 (big-endian)
602  "\x00\x00\x00\x00\x00\x00\x00\x00", // max = 987654321.9876 (big-endian)
603  "\x00\x00\x00\x00\x00\x00\x00\x00", // missingValue = -2147483647
604  "\x00\x00\x00\x00", // Unused 0 value required by chars codec
605 
606  // String data
607  "\0\0\0\0\0\0\0\0",
608  "hi-there",
609  "\0\xff\0\xff\0\xff\0\xff",
610  "a-string",
611  "\xff\xff\xff\xff\xff\xff\xff\xff",
612  };
613 
614  // Loop throumgh endiannesses for the source data
615 
616  for (int i = 0; i < 2; i++) {
617 
618  bool bigEndianSource = (i == 1);
619 
620  std::vector<unsigned char> data;
621 
622  for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) {
623  size_t len = (j == 0 || j == 4) ? 4 : 8;
624  data.insert(data.end(), source_data[j], source_data[j] + len);
625 
626  // n.b. Don't reverse the endianness of the string data.
627  if (bigEndianSource && j < 5)
628  std::reverse(data.end()-len, data.end());
629  }
630 
631  // Construct codec directly
632 
633 #ifndef _CRAYC
634  {
635  // Skip name of codec
636  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
637 
638  std::unique_ptr<Codec> c;
639  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
640  c.reset(new CodecChars<SameByteOrder>(odc::api::STRING));
641  } else {
642  c.reset(new CodecChars<OtherByteOrder>(odc::api::STRING));
643  }
644  c->load(ds);
645  c->setDataStream(ds);
646 
647  EXPECT(ds.position() == eckit::Offset(32));
648  EXPECT(c->dataSizeDoubles() == 1);
649 
650  double val;
651  c->decode(&val);
652  EXPECT(::memcmp(&val, source_data[5], 8) == 0);
653  c->decode(&val);
654  EXPECT(::memcmp(&val, source_data[6], 8) == 0);
655  c->decode(&val);
656  EXPECT(::memcmp(&val, source_data[7], 8) == 0);
657  c->decode(&val);
658  EXPECT(::memcmp(&val, source_data[8], 8) == 0);
659  c->decode(&val);
660  EXPECT(::memcmp(&val, source_data[9], 8) == 0);
661 
662  EXPECT(ds.position() == eckit::Offset(32 + (8 * 5)));
663  }
664 #endif
665 
666  // Construct codec from factory
667 
668  size_t hdrSize = prepend_codec_selection_header(data, "chars", bigEndianSource);
669 
670  {
671  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
672 
673  std::unique_ptr<Codec> c;
674  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
675  c = CodecFactory::instance().load(ds.same(), odc::api::STRING);
676  } else {
677  c = CodecFactory::instance().load(ds.other(), odc::api::STRING);
678  }
679  c->setDataStream(ds);
680 
681  EXPECT(ds.position() == eckit::Offset(hdrSize + 32));
682  EXPECT(c->dataSizeDoubles() == 1);
683 
684  double val;
685  c->decode(&val);
686  EXPECT(::memcmp(&val, source_data[5], 8) == 0);
687  c->decode(&val);
688  EXPECT(::memcmp(&val, source_data[6], 8) == 0);
689  c->decode(&val);
690  EXPECT(::memcmp(&val, source_data[7], 8) == 0);
691  c->decode(&val);
692  EXPECT(::memcmp(&val, source_data[8], 8) == 0);
693  c->decode(&val);
694  EXPECT(::memcmp(&val, source_data[9], 8) == 0);
695 
696  EXPECT(ds.position() == eckit::Offset(hdrSize + 32 + (8 * 5)));
697  }
698  }
699 }
700 
701 
702 CASE("long floating point values can include the missing data value") {
703 
704  const char* source_data[] = {
705 
706  // Codec header
707  "\x00\x00\x00\x00", // no missing value
708  "\x00\x00\x00\x00\x00\x00\x00\x00", // minimum unspecified
709  "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified
710  "\x00\x00\x00\x00\x00\x00\x00\x00", // missing value unspecified
711 
712  // data to encode
713  "\x00\x00\x00\x00\x00\x00\x00\x00", // 0.0
714  "\x53\xa4\x0c\x54\x34\x6f\x9d\x41", // 123456789.0123456
715  "\x9b\xe6\x57\xb7\x80\x65\x02\xc2", // -9876543210.9876
716  "\x00\x00\x00\x00\x00\x00\xf0\x7f", // +inf
717  "\x00\x00\x00\x00\x00\x00\xf0\xff", // -inf
718  "\x7f\xf7\xff\xff\xff\xff\xff\xff", // NaN (signalling)
719  "\x7f\xff\xff\xff\xff\xff\xff\xff", // NaN (quiet)
720  "\x00\x00\xc0\xff\xff\xff\xdf\xc1" // -2147483647 (otherwise the missing value)
721  };
722 
723  // Loop through endiannesses for the source data
724 
725  for (int i = 0; i < 2; i++) {
726 
727  bool bigEndianSource = (i == 1);
728 
729  std::vector<unsigned char> data;
730 
731  for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) {
732  size_t len = (j == 0) ? 4 : 8;
733  data.insert(data.end(), source_data[j], source_data[j] + len);
734  if (bigEndianSource)
735  std::reverse(data.end()-len, data.end());
736  }
737 
738  // Construct codec directly
739 
740 #ifndef _CRAYC
741  {
742  // Skip name of codec
743  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
744 
745  std::unique_ptr<Codec> c;
746  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
747  c.reset(new CodecLongReal<SameByteOrder>(odc::api::DOUBLE));
748  } else {
749  c.reset(new CodecLongReal<OtherByteOrder>(odc::api::DOUBLE));
750  }
751  c->load(ds);
752  c->setDataStream(ds);
753 
754  EXPECT(ds.position() == eckit::Offset(28));
755  EXPECT(c->dataSizeDoubles() == 1);
756 
757  double val;
758  c->decode(&val);
759  EXPECT(val == 0);
760  c->decode(&val);
761  EXPECT(val == 123456789.0123456);
762  c->decode(&val);
763  EXPECT(val == -9876543210.9876);
764  c->decode(&val);
765  EXPECT(std::isinf(val));
766  EXPECT(val > 0);
767  c->decode(&val);
768  EXPECT(std::isinf(val));
769  EXPECT(val < 0);
770  c->decode(&val);
771  EXPECT(std::isnan(val));
772  c->decode(&val);
773  EXPECT(std::isnan(val));
774  c->decode(&val);
775  EXPECT(val == -2147483647);
776 
777  EXPECT(ds.position() == eckit::Offset(28 + (8 * 8)));
778  }
779 #endif
780 
781  // Construct codec from factory
782 
783  size_t hdrSize = prepend_codec_selection_header(data, "long_real", bigEndianSource);
784 
785  {
786  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
787 
788  std::unique_ptr<Codec> c;
789  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
790  c = CodecFactory::instance().load(ds.same(), odc::api::DOUBLE);
791  } else {
792  c = CodecFactory::instance().load(ds.other(), odc::api::DOUBLE);
793  }
794  c->setDataStream(ds);
795 
796  EXPECT(ds.position() == eckit::Offset(hdrSize + 28));
797  EXPECT(c->dataSizeDoubles() == 1);
798 
799  double val;
800  c->decode(&val);
801  EXPECT(val == 0);
802  c->decode(&val);
803  EXPECT(val == 123456789.0123456);
804  c->decode(&val);
805  EXPECT(val == -9876543210.9876);
806  c->decode(&val);
807  EXPECT(std::isinf(val));
808  EXPECT(val > 0);
809  c->decode(&val);
810  EXPECT(std::isinf(val));
811  EXPECT(val < 0);
812  c->decode(&val);
813  EXPECT(std::isnan(val));
814  c->decode(&val);
815  EXPECT(std::isnan(val));
816  c->decode(&val);
817  EXPECT(val == -2147483647);
818 
819  EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + (8 * 8)));
820  }
821  }
822 }
823 
824 
825 CASE("short floating point values can include the missing data value") {
826 
827  // Use a curious, custom missingValue to show it is being used.
828 
829  const char* source_data[] = {
830 
831  // Codec header
832  "\x00\x00\x00\x00", // no missing value
833  "\x00\x00\x00\x00\x00\x00\x00\x00", // minimum unspecified
834  "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified
835  "\x04\x4f\xab\xa0\xe4\x4e\x91\x26", // missing value = 6.54565456545599971850917315786e-123
836 
837  // data to encode
838  "\x00\x00\x00\x00", // 0.0
839  "\x12\xbf\x1f\x49", // 654321.123
840  "\x00\x00\x80\x00", // Smallest available, missing value for short_real (1.17549435082229e-38)
841  "\xff\xff\x7f\xff", // Lowest available, missing value for short_real2 (-3.40282346638529e+38)
842  "\x00\x00\x80\x7f", // +inf
843  "\x00\x00\x80\xff", // -inf
844  "\xff\xff\xbf\x7f", // NaN (signalling)
845  "\xff\xff\xff\x7f", // NaN (quiet)
846  };
847 
848  // Loop through endiannesses for the source data
849 
850  for (int i = 0; i < 4; i++) {
851 
852  bool bigEndianSource = (i % 2 == 0);
853  bool secondCodec = (i > 1);
854 
855  std::vector<unsigned char> data;
856 
857  for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) {
858  size_t len = (j == 0 || j > 3) ? 4 : 8;
859  data.insert(data.end(), source_data[j], source_data[j] + len);
860  if (bigEndianSource)
861  std::reverse(data.end()-len, data.end());
862  }
863 
864  // Construct codec directly
865 
866 #ifndef _CRAYC
867  {
868  // Skip name of codec
869  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
870 
871  std::unique_ptr<Codec> c;
872  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
873  if (secondCodec) {
874  c.reset(new CodecShortReal2<SameByteOrder>(odc::api::REAL));
875  } else {
876  c.reset(new CodecShortReal<SameByteOrder>(odc::api::DOUBLE));
877  }
878  } else {
879  if (secondCodec) {
880  c.reset(new CodecShortReal2<OtherByteOrder>(odc::api::DOUBLE));
881  } else {
882  c.reset(new CodecShortReal<OtherByteOrder>(odc::api::REAL));
883  }
884  }
885  c->load(ds);
886  c->setDataStream(ds);
887 
888  EXPECT(ds.position() == eckit::Offset(28));
889  EXPECT(c->dataSizeDoubles() == 1);
890 
891  // n.b. == comparisons for floats as we are testing BIT reproducability of decoding
892  double val;
893  c->decode(&val);
894  EXPECT(val == 0);
895  c->decode(&val);
896  EXPECT(val == float(654321.123));
897  // Each of the two codecs has a different internal missing value. Check it is correctly recognised.
898  if (secondCodec) {
899  c->decode(&val);
900  EXPECT(val == float(1.17549435082229e-38));
901  c->decode(&val);
902  EXPECT(val == 6.54565456545599971850917315786e-123);
903  } else {
904  c->decode(&val);
905  EXPECT(val == 6.54565456545599971850917315786e-123);
906  c->decode(&val);
907  EXPECT(val == float(-3.40282346638529e+38));
908  }
909  c->decode(&val);
910  EXPECT(std::isinf(val));
911  EXPECT(val > 0);
912  c->decode(&val);
913  EXPECT(std::isinf(val));
914  EXPECT(val < 0);
915  c->decode(&val);
916  EXPECT(std::isnan(val));
917  c->decode(&val);
918  EXPECT(std::isnan(val));
919 
920  EXPECT(ds.position() == eckit::Offset(28 + (8 * 4)));
921  }
922 #endif
923 
924  // Construct codec from factory
925 
926  size_t hdrSize = prepend_codec_selection_header(data, secondCodec ? "short_real2" : "short_real", bigEndianSource);
927 
928  {
929  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
930 
931  std::unique_ptr<Codec> c;
932  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
933  c = CodecFactory::instance().load(ds.same(), odc::api::DOUBLE);
934  } else {
935  c = CodecFactory::instance().load(ds.other(), odc::api::REAL);
936  }
937  c->setDataStream(ds);
938 
939  EXPECT(ds.position() == eckit::Offset(hdrSize + 28));
940  EXPECT(c->dataSizeDoubles() == 1);
941 
942  double val;
943  c->decode(&val);
944  EXPECT(val == 0);
945  c->decode(&val);
946  EXPECT(val == float(654321.123));
947  // Each of the two codecs has a different internal missing value. Check it is correctly recognised.
948  if (secondCodec) {
949  c->decode(&val);
950  EXPECT(val == float(1.17549435082229e-38));
951  c->decode(&val);
952  EXPECT(val == 6.54565456545599971850917315786e-123);
953  } else {
954  c->decode(&val);
955  EXPECT(val == 6.54565456545599971850917315786e-123);
956  c->decode(&val);
957  EXPECT(val == float(-3.40282346638529e+38));
958  }
959  c->decode(&val);
960  EXPECT(std::isinf(val));
961  EXPECT(val > 0);
962  c->decode(&val);
963  EXPECT(std::isinf(val));
964  EXPECT(val < 0);
965  c->decode(&val);
966  EXPECT(std::isnan(val));
967  c->decode(&val);
968  EXPECT(std::isnan(val));
969 
970  EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + (8 * 4)));
971  }
972  }
973 }
974 
975 CASE("32bit integers can be decoded direct to integers") {
976 
977  // Set to decode to integers rather than doubles
978  TestIntegerDecoding resetter;
979 
980  // Use a curious, custom missingValue to show it is being used.
981 
982  const char* source_data[] = {
983 
984  // Codec header
985  "\x00\x00\x00\x00", // no missing value
986  "\x00\x00\x00\x00\x00\x00\x00\x00", // minimum unspecified
987  "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified
988  "\x00\x00\x00\x00\x00\x00\x00\x00", // missing value unspecified
989 
990  // data to encode
991  "\x00\x00\x00\x00", // 0.0
992  "\xff\xff\xff\xff", // -1
993  "\xff\xff\xff\x7f", // 2147483647 == largest
994  "\x00\x00\x00\x80", // -2147483648 == smallest
995  "\x96\x28\x9c\xff" // -6543210
996  };
997 
998  // Loop through endiannesses for the source data
999 
1000  for (int i = 0; i < 2; i++) {
1001 
1002  bool bigEndianSource = (i == 1);
1003 
1004  std::vector<unsigned char> data;
1005 
1006  for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) {
1007  size_t len = (j == 0 || j > 3) ? 4 : 8;
1008  data.insert(data.end(), source_data[j], source_data[j] + len);
1009  if (bigEndianSource)
1010  std::reverse(data.end()-len, data.end());
1011  }
1012 
1013  // Construct codec directly
1014 
1015 #ifndef _CRAYC
1016  {
1017  // Skip name of codec
1018  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
1019 
1020  std::unique_ptr<Codec> c;
1021  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
1022  c.reset(new CodecInt32<SameByteOrder, int64_t>(odc::api::INTEGER));
1023  } else {
1024  c.reset(new CodecInt32<OtherByteOrder, int64_t>(odc::api::INTEGER));
1025  }
1026  c->load(ds);
1027  c->setDataStream(ds);
1028 
1029  EXPECT(ds.position() == eckit::Offset(28));
1030  EXPECT(c->dataSizeDoubles() == 1);
1031 
1032  double val;
1033  int64_t& intVal = reinterpret_cast<int64_t&>(val);
1034  c->decode(&val);
1035  EXPECT(intVal == 0);
1036  c->decode(&val);
1037  EXPECT(intVal == -1);
1038  c->decode(&val);
1039  EXPECT(intVal == 2147483647);
1040  c->decode(&val);
1041  EXPECT(intVal == -2147483648);
1042  c->decode(&val);
1043  EXPECT(intVal == -6543210);
1044 
1045  EXPECT(ds.position() == eckit::Offset(28 + (5 * 4)));
1046  }
1047 #endif
1048 
1049  // Construct codec from factory
1050 
1051  size_t hdrSize = prepend_codec_selection_header(data, "int32", bigEndianSource);
1052 
1053  {
1054  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
1055 
1056  std::unique_ptr<Codec> c;
1057  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
1058  c = CodecFactory::instance().load(ds.same(), odc::api::INTEGER);
1059  } else {
1060  c = CodecFactory::instance().load(ds.other(), odc::api::INTEGER);
1061  }
1062  c->setDataStream(ds);
1063 
1064  EXPECT(ds.position() == eckit::Offset(hdrSize + 28));
1065  EXPECT(c->dataSizeDoubles() == 1);
1066 
1067  double val;
1068  int64_t& intVal = reinterpret_cast<int64_t&>(val);
1069  c->decode(&val);
1070  EXPECT(intVal == 0);
1071  c->decode(&val);
1072  EXPECT(intVal == -1);
1073  c->decode(&val);
1074  EXPECT(intVal == 2147483647);
1075  c->decode(&val);
1076  EXPECT(intVal == -2147483648);
1077  c->decode(&val);
1078  EXPECT(intVal == -6543210);
1079 
1080  EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + (5 * 4)));
1081  }
1082  }
1083 }
1084 
1085 
1086 CASE("32bit integers are as-is") {
1087 
1088  // Use a curious, custom missingValue to show it is being used.
1089 
1090  const char* source_data[] = {
1091 
1092  // Codec header
1093  "\x00\x00\x00\x00", // no missing value
1094  "\x00\x00\x00\x00\x00\x00\x00\x00", // minimum unspecified
1095  "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified
1096  "\x00\x00\x00\x00\x00\x00\x00\x00", // missing value unspecified
1097 
1098  // data to encode
1099  "\x00\x00\x00\x00", // 0.0
1100  "\xff\xff\xff\xff", // -1
1101  "\xff\xff\xff\x7f", // 2147483647 == largest
1102  "\x00\x00\x00\x80", // -2147483648 == smallest
1103  "\x96\x28\x9c\xff" // -6543210
1104  };
1105 
1106  // Loop through endiannesses for the source data
1107 
1108  for (int i = 0; i < 2; i++) {
1109 
1110  bool bigEndianSource = (i == 1);
1111 
1112  std::vector<unsigned char> data;
1113 
1114  for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) {
1115  size_t len = (j == 0 || j > 3) ? 4 : 8;
1116  data.insert(data.end(), source_data[j], source_data[j] + len);
1117  if (bigEndianSource)
1118  std::reverse(data.end()-len, data.end());
1119  }
1120 
1121  // Construct codec directly
1122 
1123 #ifndef _CRAYC
1124  {
1125  // Skip name of codec
1126  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
1127 
1128  std::unique_ptr<Codec> c;
1129  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
1130  c.reset(new CodecInt32<SameByteOrder, double>(odc::api::INTEGER));
1131  } else {
1132  c.reset(new CodecInt32<OtherByteOrder, double>(odc::api::INTEGER));
1133  }
1134  c->load(ds);
1135  c->setDataStream(ds);
1136 
1137  EXPECT(ds.position() == eckit::Offset(28));
1138  EXPECT(c->dataSizeDoubles() == 1);
1139 
1140  double val;
1141  c->decode(&val);
1142  EXPECT(val == 0);
1143  c->decode(&val);
1144  EXPECT(val == -1);
1145  c->decode(&val);
1146  EXPECT(val == 2147483647);
1147  c->decode(&val);
1148  EXPECT(val == -2147483648);
1149  c->decode(&val);
1150  EXPECT(val == -6543210);
1151 
1152  EXPECT(ds.position() == eckit::Offset(28 + (5 * 4)));
1153  }
1154 #endif
1155 
1156  // Construct codec from factory
1157 
1158  size_t hdrSize = prepend_codec_selection_header(data, "int32", bigEndianSource);
1159 
1160  {
1161  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
1162 
1163  std::unique_ptr<Codec> c;
1164  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
1165  c = CodecFactory::instance().load(ds.same(), odc::api::INTEGER);
1166  } else {
1167  c = CodecFactory::instance().load(ds.other(), odc::api::INTEGER);
1168  }
1169  c->setDataStream(ds);
1170 
1171  EXPECT(ds.position() == eckit::Offset(hdrSize + 28));
1172  EXPECT(c->dataSizeDoubles() == 1);
1173 
1174  double val;
1175  c->decode(&val);
1176  EXPECT(val == 0);
1177  c->decode(&val);
1178  EXPECT(val == -1);
1179  c->decode(&val);
1180  EXPECT(val == 2147483647);
1181  c->decode(&val);
1182  EXPECT(val == -2147483648);
1183  c->decode(&val);
1184  EXPECT(val == -6543210);
1185 
1186  EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + (5 * 4)));
1187  }
1188  }
1189 }
1190 
1191 
1192 CASE("16bit integers are stored with an offset. This need not (strictly) be integral!!") {
1193 
1194  // n.b. we use a non-standard, non-integral minimum to demonstrate the offset behaviour.
1195 
1196  // Use a curious, custom missingValue to show it is being used.
1197 
1198  const char* source_data[] = {
1199 
1200  // Codec header
1201  "\x00\x00\x00\x00", // no missing value
1202  "\xcd\xcc\xcc\xcc\xcc\xdc\x5e\xc0", // minimum = -123.45
1203  "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified
1204  "\x04\x4f\xab\xa0\xe4\x4e\x91\x26", // missing value = 6.54565456545599971850917315786e-123
1205 
1206  // data to encode
1207  "\x00\x00", // 0.0
1208  "\xff\xff", // 65535 and the missing value
1209  "\xff\x7f", // 32767 (no negatives)
1210  "\x00\x80", // 32768 (no negatives)
1211  "\x39\x30" // 12345
1212  };
1213 
1214  // Loop through endiannesses for the source data
1215 
1216  for (int i = 0; i < 4; i++) {
1217 
1218  bool bigEndianSource = (i % 2 == 0);
1219 
1220  bool withMissing = (i > 1);
1221 
1222  std::vector<unsigned char> data;
1223 
1224  for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) {
1225  size_t len = (j == 0) ? 4 : (j > 3) ? 2 : 8;
1226  data.insert(data.end(), source_data[j], source_data[j] + len);
1227  if (bigEndianSource)
1228  std::reverse(data.end()-len, data.end());
1229  }
1230 
1231  // Construct codec directly
1232 
1233 #ifndef _CRAYC
1234  {
1235  // Skip name of codec
1236  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
1237 
1238  std::unique_ptr<Codec> c;
1239  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
1240  if (withMissing) {
1241  c.reset(new CodecInt16Missing<SameByteOrder, double>(odc::api::INTEGER));
1242  } else {
1243  c.reset(new CodecInt16<SameByteOrder, double>(odc::api::INTEGER));
1244  }
1245  } else {
1246  if (withMissing) {
1247  c.reset(new CodecInt16Missing<OtherByteOrder, double>(odc::api::INTEGER));
1248  } else {
1249  c.reset(new CodecInt16<OtherByteOrder, double>(odc::api::INTEGER));
1250  }
1251  }
1252  c->load(ds);
1253  c->setDataStream(ds);
1254 
1255  EXPECT(ds.position() == eckit::Offset(28));
1256  EXPECT(c->dataSizeDoubles() == 1);
1257 
1258  double val;
1259  c->decode(&val);
1260  EXPECT(val == (double(-123.45) + 0));
1261  c->decode(&val);
1262  if (withMissing) {
1263  EXPECT(val == 6.54565456545599971850917315786e-123);
1264  } else {
1265  EXPECT(val == (double(-123.45) + 65535));
1266  }
1267  c->decode(&val);
1268  EXPECT(val == (double(-123.45) + 32767));
1269  c->decode(&val);
1270  EXPECT(val == (double(-123.45) + 32768));
1271  c->decode(&val);
1272  EXPECT(val == (double(-123.45) + 12345));
1273 
1274  EXPECT(ds.position() == eckit::Offset(28 + (5 * 2)));
1275  }
1276 #endif
1277 
1278  // Construct codec from factory
1279 
1280  size_t hdrSize = prepend_codec_selection_header(data, withMissing ? "int16_missing" : "int16", bigEndianSource);
1281 
1282  {
1283  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
1284 
1285  std::unique_ptr<Codec> c;
1286  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
1287  c = CodecFactory::instance().load(ds.same(), odc::api::INTEGER);
1288  } else {
1289  c = CodecFactory::instance().load(ds.other(), odc::api::INTEGER);
1290  }
1291  c->setDataStream(ds);
1292 
1293  EXPECT(ds.position() == eckit::Offset(hdrSize + 28));
1294  EXPECT(c->dataSizeDoubles() == 1);
1295 
1296  double val;
1297  c->decode(&val);
1298  EXPECT(val == (double(-123.45) + 0));
1299  c->decode(&val);
1300  if (withMissing) {
1301  EXPECT(val == 6.54565456545599971850917315786e-123);
1302  } else {
1303  EXPECT(val == (double(-123.45) + 65535));
1304  }
1305  c->decode(&val);
1306  EXPECT(val == (double(-123.45) + 32767));
1307  c->decode(&val);
1308  EXPECT(val == (double(-123.45) + 32768));
1309  c->decode(&val);
1310  EXPECT(val == (double(-123.45) + 12345));
1311 
1312  EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + (5 * 2)));
1313  }
1314  }
1315 }
1316 
1317 
1318 CASE("16bit integers are stored with an offset and can be decoded to integers") {
1319 
1320  // Set to decode to integers rather than doubles
1321  TestIntegerDecoding resetter;
1322 
1323  // n.b. we use a non-standard, non-integral minimum to demonstrate the offset behaviour.
1324 
1325  // Use a curious, custom missingValue to show it is being used.
1326 
1327  const char* source_data[] = {
1328 
1329  // Codec header
1330  "\x00\x00\x00\x00", // no missing value
1331  "\x00\x00\x00\x00\x00\xc0\x5e\xc0", // minimum = -123
1332  "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified
1333  "\x04\x4f\xab\xa0\xe4\x4e\x91\x26", // missing value = 6.54565456545599971850917315786e-123
1334 
1335  // data to encode
1336  "\x00\x00", // 0.0
1337  "\xff\xff", // 65535 and the missing value
1338  "\xff\x7f", // 32767 (no negatives)
1339  "\x00\x80", // 32768 (no negatives)
1340  "\x39\x30" // 12345
1341  };
1342 
1343  // Loop through endiannesses for the source data
1344 
1345  for (int i = 0; i < 4; i++) {
1346 
1347  bool bigEndianSource = (i % 2 == 0);
1348 
1349  bool withMissing = (i > 1);
1350 
1351  std::vector<unsigned char> data;
1352 
1353  for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) {
1354  size_t len = (j == 0) ? 4 : (j > 3) ? 2 : 8;
1355  data.insert(data.end(), source_data[j], source_data[j] + len);
1356  if (bigEndianSource)
1357  std::reverse(data.end()-len, data.end());
1358  }
1359 
1360  // Construct codec directly
1361 
1362 #ifndef _CRAYC
1363  {
1364  // Skip name of codec
1365  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
1366 
1367  std::unique_ptr<Codec> c;
1368  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
1369  if (withMissing) {
1370  c.reset(new CodecInt16Missing<SameByteOrder, int64_t>(odc::api::INTEGER));
1371  } else {
1372  c.reset(new CodecInt16<SameByteOrder, int64_t>(odc::api::INTEGER));
1373  }
1374  } else {
1375  if (withMissing) {
1376  c.reset(new CodecInt16Missing<OtherByteOrder, int64_t>(odc::api::INTEGER));
1377  } else {
1378  c.reset(new CodecInt16<OtherByteOrder, int64_t>(odc::api::INTEGER));
1379  }
1380  }
1381  c->load(ds);
1382  c->setDataStream(ds);
1383 
1384  EXPECT(ds.position() == eckit::Offset(28));
1385  EXPECT(c->dataSizeDoubles() == 1);
1386 
1387  double val;
1388  int64_t& intVal(reinterpret_cast<int64_t&>(val));
1389  c->decode(&val);
1390  EXPECT(intVal == -123 + 0);
1391  c->decode(&val);
1392  if (withMissing) {
1393  // missing Value returned unchanged. TODO: change this behaviour...
1394 // EXPECT(val == 6.54565456545599971850917315786e-123);
1395  } else {
1396  EXPECT(intVal == -123 + 65535);
1397  }
1398  c->decode(&val);
1399  EXPECT(intVal == -123 + 32767);
1400  c->decode(&val);
1401  EXPECT(intVal == -123 + 32768);
1402  c->decode(&val);
1403  EXPECT(intVal == -123 + 12345);
1404 
1405  EXPECT(ds.position() == eckit::Offset(28 + (5 * 2)));
1406  }
1407 #endif
1408 
1409  // Construct codec from factory
1410 
1411  size_t hdrSize = prepend_codec_selection_header(data, withMissing ? "int16_missing" : "int16", bigEndianSource);
1412 
1413  {
1414  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
1415 
1416  std::unique_ptr<Codec> c;
1417  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
1418  c = CodecFactory::instance().load(ds.same(), odc::api::INTEGER);
1419  } else {
1420  c = CodecFactory::instance().load(ds.other(), odc::api::INTEGER);
1421  }
1422  c->setDataStream(ds);
1423 
1424  EXPECT(ds.position() == eckit::Offset(hdrSize + 28));
1425  EXPECT(c->dataSizeDoubles() == 1);
1426 
1427  double val;
1428  int64_t& intVal(reinterpret_cast<int64_t&>(val));
1429  c->decode(&val);
1430  EXPECT(intVal == -123 + 0);
1431  c->decode(&val);
1432  if (withMissing) {
1433 // EXPECT(val == 6.54565456545599971850917315786e-123);
1434  } else {
1435  EXPECT(intVal == -123 + 65535);
1436  }
1437  c->decode(&val);
1438  EXPECT(intVal == -123 + 32767);
1439  c->decode(&val);
1440  EXPECT(intVal == -123 + 32768);
1441  c->decode(&val);
1442  EXPECT(intVal == -123 + 12345);
1443 
1444  EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + (5 * 2)));
1445  }
1446  }
1447 }
1448 
1449 
1450 CASE("8bit integers are stored with an offset. This need not (strictly) be integral!!") {
1451 
1452  // n.b. we use a non-standard, non-integral minimum to demonstrate the offset behaviour.
1453 
1454  // Use a curious, custom missingValue to show it is being used.
1455 
1456  const char* source_data[] = {
1457 
1458  // Codec header
1459  "\x00\x00\x00\x00", // no missing value
1460  "\x00\x00\x00\x00\x80\x88\xb3\xc0", // minimum = -5000.5
1461  "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified
1462  "\x04\x4f\xab\xa0\xe4\x4e\x91\x26", // missing value = 6.54565456545599971850917315786e-123
1463  };
1464 
1465  // Loop through endiannesses for the source data
1466 
1467  for (int i = 0; i < 4; i++) {
1468 
1469  bool bigEndianSource = (i % 2 == 0);
1470 
1471  bool withMissing = (i > 1);
1472 
1473  std::vector<unsigned char> data;
1474 
1475  for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) {
1476  size_t len = (j == 0) ? 4 : 8;
1477  data.insert(data.end(), source_data[j], source_data[j] + len);
1478  if (bigEndianSource)
1479  std::reverse(data.end()-len, data.end());
1480  }
1481 
1482  // Add all of the data values
1483 
1484  for (int n = 0; n < 256; n++) {
1485  data.push_back(static_cast<unsigned char>(n));
1486  }
1487 
1488  // Construct codec directly
1489 
1490 #ifndef _CRAYC
1491  {
1492  // Skip name of codec
1493  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
1494 
1495  std::unique_ptr<Codec> c;
1496  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
1497  if (withMissing) {
1498  c.reset(new CodecInt8Missing<SameByteOrder, double>(odc::api::INTEGER));
1499  } else {
1500  c.reset(new CodecInt8<SameByteOrder, double>(odc::api::INTEGER));
1501  }
1502  } else {
1503  if (withMissing) {
1504  c.reset(new CodecInt8Missing<OtherByteOrder, double>(odc::api::INTEGER));
1505  } else {
1506  c.reset(new CodecInt8<OtherByteOrder, double>(odc::api::INTEGER));
1507  }
1508  }
1509  c->load(ds);
1510  c->setDataStream(ds);
1511 
1512  EXPECT(ds.position() == eckit::Offset(28));
1513  EXPECT(c->dataSizeDoubles() == 1);
1514 
1515  double val;
1516  for (int n = 0; n < 255; n++) {
1517  c->decode(&val);
1518  EXPECT(val == (double(-5000.5) + n));
1519  }
1520 
1521  c->decode(&val);
1522  EXPECT(val == (withMissing ? 6.54565456545599971850917315786e-123 : (-5000.5 + 255)));
1523 
1524  EXPECT(ds.position() == eckit::Offset(28 + 256));
1525  }
1526 #endif
1527 
1528  // Construct codec from factory
1529 
1530  size_t hdrSize = prepend_codec_selection_header(data, withMissing ? "int8_missing" : "int8", bigEndianSource);
1531 
1532  {
1533  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
1534 
1535  std::unique_ptr<Codec> c;
1536  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
1537  c = CodecFactory::instance().load(ds.same(), odc::api::INTEGER);
1538  } else {
1539  c = CodecFactory::instance().load(ds.other(), odc::api::INTEGER);
1540  }
1541  c->setDataStream(ds);
1542 
1543  EXPECT(ds.position() == eckit::Offset(hdrSize + 28));
1544  EXPECT(c->dataSizeDoubles() == 1);
1545 
1546  double val;
1547  for (int n = 0; n < 255; n++) {
1548  c->decode(&val);
1549  EXPECT(val == (double(-5000.5) + n));
1550  }
1551 
1552  c->decode(&val);
1553  EXPECT(val == (withMissing ? 6.54565456545599971850917315786e-123 : (-5000.5 + 255)));
1554 
1555  EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + 256));
1556  }
1557  }
1558 }
1559 
1560 
1561 CASE("8bit integers are stored with an offset and can be decoded to integers") {
1562 
1563  // Set to decode to integers rather than doubles
1564  TestIntegerDecoding resetter;
1565 
1566  // n.b. we use a non-standard, non-integral minimum to demonstrate the offset behaviour.
1567 
1568  // Use a curious, custom missingValue to show it is being used.
1569 
1570  const char* source_data[] = {
1571 
1572  // Codec header
1573  "\x00\x00\x00\x00", // no missing value
1574  "\x00\x00\x00\x00\x00\x88\xb3\xc0", // minimum = -5000
1575  "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified
1576  "\x04\x4f\xab\xa0\xe4\x4e\x91\x26", // missing value = 6.54565456545599971850917315786e-123
1577  };
1578 
1579  // Loop through endiannesses for the source data
1580 
1581  for (int i = 0; i < 4; i++) {
1582 
1583  bool bigEndianSource = (i % 2 == 0);
1584 
1585  bool withMissing = (i > 1);
1586 
1587  std::vector<unsigned char> data;
1588 
1589  for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) {
1590  size_t len = (j == 0) ? 4 : 8;
1591  data.insert(data.end(), source_data[j], source_data[j] + len);
1592  if (bigEndianSource)
1593  std::reverse(data.end()-len, data.end());
1594  }
1595 
1596  // Add all of the data values
1597 
1598  for (int n = 0; n < 256; n++) {
1599  data.push_back(static_cast<unsigned char>(n));
1600  }
1601 
1602  // Construct codec directly
1603 
1604 #ifndef _CRAYC
1605  {
1606  // Skip name of codec
1607  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
1608 
1609  std::unique_ptr<Codec> c;
1610  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
1611  if (withMissing) {
1612  c.reset(new CodecInt8Missing<SameByteOrder, int64_t>(odc::api::INTEGER));
1613  } else {
1614  c.reset(new CodecInt8<SameByteOrder, int64_t>(odc::api::INTEGER));
1615  }
1616  } else {
1617  if (withMissing) {
1618  c.reset(new CodecInt8Missing<OtherByteOrder, int64_t>(odc::api::INTEGER));
1619  } else {
1620  c.reset(new CodecInt8<OtherByteOrder, int64_t>(odc::api::INTEGER));
1621  }
1622  }
1623  c->load(ds);
1624  c->setDataStream(ds);
1625 
1626  EXPECT(ds.position() == eckit::Offset(28));
1627  EXPECT(c->dataSizeDoubles() == 1);
1628 
1629  double val;
1630  int64_t& intVal = reinterpret_cast<int64_t&>(val);
1631  for (int n = 0; n < 255; n++) {
1632  c->decode(&val);
1633  EXPECT(intVal == (-5000 + n));
1634  }
1635 
1636  c->decode(&val);
1637  // TODO: Missing
1638 // EXPECT(val == (withMissing ? 6.54565456545599971850917315786e-123 : (-5000.5 + 255)));
1639 
1640  EXPECT(ds.position() == eckit::Offset(28 + 256));
1641  }
1642 #endif
1643 
1644  // Construct codec from factory
1645 
1646  size_t hdrSize = prepend_codec_selection_header(data, withMissing ? "int8_missing" : "int8", bigEndianSource);
1647 
1648  {
1649  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
1650 
1651  std::unique_ptr<Codec> c;
1652  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
1653  c = CodecFactory::instance().load(ds.same(), odc::api::INTEGER);
1654  } else {
1655  c = CodecFactory::instance().load(ds.other(), odc::api::INTEGER);
1656  }
1657  c->setDataStream(ds);
1658 
1659  EXPECT(ds.position() == eckit::Offset(hdrSize + 28));
1660  EXPECT(c->dataSizeDoubles() == 1);
1661 
1662  double val;
1663  int64_t& intVal = reinterpret_cast<int64_t&>(val);
1664  for (int n = 0; n < 255; n++) {
1665  c->decode(&val);
1666  EXPECT(intVal == (-5000 + n));
1667  }
1668 
1669  c->decode(&val);
1670  // TODO: Missing
1671 // EXPECT(val == (withMissing ? 6.54565456545599971850917315786e-123 : (-5000.5 + 255)));
1672 
1673  EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + 256));
1674  }
1675  }
1676 }
1677 
1678 CASE("Character strings can be stored in a flat list, and indexed") {
1679 
1680  // n.b. no missing values
1681 
1682  const char* source_data[] = {
1683 
1684  // Codec header
1685  "\x00\x00\x00\x00", // 0 = hasMissing
1686  "\x00\x00\x00\x00\x00\x00\x00\x00", // min unspecified
1687  "\x00\x00\x00\x00\x00\x00\x00\x00", // max unspecified
1688  "\x00\x00\x00\x00\x00\x00\x00\x00", // missingValue unspecified
1689 
1690  // How many strings are there in the table?
1691  "\x06\x00\x00\x00",
1692 
1693  // String data (prepended with lengths)
1694  // length, data, "cnt (discarded)", index
1695 
1696  "\x02\x00\x00\x00", "ab", "\x00\x00\x00\x00", "\x03\x00\x00\x00", // This string is too short
1697  "\x06\x00\x00\x00", "ghijkl", "\x00\x00\x00\x00", "\x04\x00\x00\x00",
1698  "\x08\x00\x00\x00", "mnopqrst", "\x00\x00\x00\x00", "\x05\x00\x00\x00", // 8-byte length
1699  "\x08\x00\x00\x00", "uvwxyzab", "\x00\x00\x00\x00", "\x01\x00\x00\x00", // too long
1700  "\x08\x00\x00\x00", "ghijklmn", "\x00\x00\x00\x00", "\x00\x00\x00\x00",
1701  "\x08\x00\x00\x00", "opqrstuv", "\x00\x00\x00\x00", "\x02\x00\x00\x00"
1702  };
1703 
1704  // Loop throumgh endiannesses for the source data
1705 
1706  for (int i = 0; i < 4; i++) {
1707 
1708  bool bigEndianSource = (i % 2 == 0);
1709 
1710  bool bits16 = (i > 1);
1711 
1712  std::vector<unsigned char> data;
1713 
1714  for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) {
1715  size_t len =
1716  (j < 5) ? ((j == 0 || j == 4) ? 4 : 8)
1717  : ((j+2) % 4 == 0 ? ::strlen(source_data[j]) : 4);
1718  data.insert(data.end(), source_data[j], source_data[j] + len);
1719 
1720  // n.b. Don't reverse the endianness of the string data.
1721  if (bigEndianSource && !((j > 5) && ((j+2) % 4 == 0)))
1722  std::reverse(data.end()-len, data.end());
1723  }
1724 
1725  // Which strings do we wish to decode (look at them in reverse. nb refers to index column)
1726 
1727  for (int n = 5; n >= 0; n--) {
1728  if (bits16 && bigEndianSource)
1729  data.push_back(0);
1730  data.push_back(static_cast<unsigned char>(n));
1731  if (bits16 && !bigEndianSource)
1732  data.push_back(0);
1733  }
1734 
1735  // Construct codec directly
1736 
1737 #ifndef _CRAYC
1738  {
1739  // Skip name of codec
1740  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
1741 
1742  std::unique_ptr<Codec> c;
1743  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
1744  if (bits16) {
1745  c.reset(new CodecInt16String<SameByteOrder>(odc::api::STRING));
1746  } else {
1747  c.reset(new CodecInt8String<SameByteOrder>(odc::api::STRING));
1748  }
1749  } else {
1750  if (bits16) {
1751  c.reset(new CodecInt16String<OtherByteOrder>(odc::api::STRING));
1752  } else {
1753  c.reset(new CodecInt8String<OtherByteOrder>(odc::api::STRING));
1754  }
1755  }
1756  c->load(ds);
1757  c->setDataStream(ds);
1758 
1759  EXPECT(ds.position() == eckit::Offset(144));
1760  EXPECT(c->dataSizeDoubles() == 1);
1761 
1762  double val;
1763  c->decode(&val);
1764  EXPECT(std::string(reinterpret_cast<const char*>(&val), 8) == "mnopqrst");
1765  c->decode(&val);
1766  EXPECT(std::string(reinterpret_cast<const char*>(&val), 6) == "ghijkl"); // silently works for shorter strings
1767  c->decode(&val);
1768  EXPECT(std::string(reinterpret_cast<const char*>(&val), 2) == "ab"); // silently works for shorter strings
1769  c->decode(&val);
1770  EXPECT(std::string(reinterpret_cast<const char*>(&val), 8) == "opqrstuv");
1771  c->decode(&val);
1772  EXPECT(std::string(reinterpret_cast<const char*>(&val), 8) == "uvwxyzab"); // gets truncated to 8
1773  c->decode(&val);
1774  EXPECT(std::string(reinterpret_cast<const char*>(&val), 8) == "ghijklmn");
1775 
1776  EXPECT(ds.position() == eckit::Offset(144 + (6 * (bits16 ? 2 : 1))));
1777  }
1778 #endif
1779 
1780  // Construct codec from factory
1781 
1782  size_t hdrSize = prepend_codec_selection_header(data, bits16 ? "int16_string" : "int8_string", bigEndianSource);
1783 
1784  {
1785  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
1786 
1787  std::unique_ptr<Codec> c;
1788  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
1789  c = CodecFactory::instance().load(ds.same(), odc::api::STRING);
1790  } else {
1791  c = CodecFactory::instance().load(ds.other(), odc::api::STRING);
1792  }
1793  c->setDataStream(ds);
1794 
1795  EXPECT(ds.position() == eckit::Offset(hdrSize + 144));
1796  EXPECT(c->dataSizeDoubles() == 1);
1797 
1798  double val;
1799  c->decode(&val);
1800  EXPECT(std::string(reinterpret_cast<const char*>(&val), 8) == "mnopqrst");
1801  c->decode(&val);
1802  EXPECT(std::string(reinterpret_cast<const char*>(&val), 6) == "ghijkl"); // silently works for shorter strings
1803  c->decode(&val);
1804  EXPECT(std::string(reinterpret_cast<const char*>(&val), 2) == "ab"); // silently works for shorter strings
1805  c->decode(&val);
1806  EXPECT(std::string(reinterpret_cast<const char*>(&val), 8) == "opqrstuv");
1807  c->decode(&val);
1808  EXPECT(std::string(reinterpret_cast<const char*>(&val), 8) == "uvwxyzab"); // gets truncated to 8
1809  c->decode(&val);
1810  EXPECT(std::string(reinterpret_cast<const char*>(&val), 8) == "ghijklmn");
1811 
1812  EXPECT(ds.position() == eckit::Offset(hdrSize + 144 + (6 * (bits16 ? 2 : 1))));
1813  }
1814  }
1815 }
1816 
1817 
1818 CASE("Character strings can be stored in a flat list, and indexed, and be longer than 8 bytes") {
1819 
1820  // n.b. no missing values
1821 
1822  const char* source_data[] = {
1823 
1824  // Codec header
1825  "\x00\x00\x00\x00", // 0 = hasMissing
1826  "\x00\x00\x00\x00\x00\x00\x00\x00", // min unspecified
1827  "\x00\x00\x00\x00\x00\x00\x00\x00", // max unspecified
1828  "\x00\x00\x00\x00\x00\x00\x00\x00", // missingValue unspecified
1829 
1830  // How many strings are there in the table?
1831  "\x06\x00\x00\x00",
1832 
1833  // String data (prepended with lengths)
1834  // length, data, "cnt (discarded)", index
1835 
1836  "\x02\x00\x00\x00", "ab", "\x00\x00\x00\x00", "\x03\x00\x00\x00", // This string is too short
1837  "\x06\x00\x00\x00", "ghijkl", "\x00\x00\x00\x00", "\x04\x00\x00\x00",
1838  "\x08\x00\x00\x00", "mnopqrst", "\x00\x00\x00\x00", "\x05\x00\x00\x00", // 8-byte length
1839  "\x0c\x00\x00\x00", "uvwxyzabcdef", "\x00\x00\x00\x00", "\x01\x00\x00\x00", // 12-byte
1840  "\x10\x00\x00\x00", "ghijklmnopqrstuv", "\x00\x00\x00\x00", "\x00\x00\x00\x00", // 16-byte
1841  "\x08\x00\x00\x00", "opqrstuv", "\x00\x00\x00\x00", "\x02\x00\x00\x00"
1842  };
1843 
1844  // Loop throumgh endiannesses for the source data
1845 
1846  for (int i = 0; i < 4; i++) {
1847 
1848  bool bigEndianSource = (i % 2 == 0);
1849 
1850  bool bits16 = (i > 1);
1851 
1852  std::vector<unsigned char> data;
1853 
1854  for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) {
1855  size_t len =
1856  (j < 5) ? ((j == 0 || j == 4) ? 4 : 8)
1857  : ((j+2) % 4 == 0 ? ::strlen(source_data[j]) : 4);
1858  data.insert(data.end(), source_data[j], source_data[j] + len);
1859 
1860  // n.b. Don't reverse the endianness of the string data.
1861  if (bigEndianSource && !((j > 5) && ((j+2) % 4 == 0)))
1862  std::reverse(data.end()-len, data.end());
1863  }
1864 
1865  // Which strings do we wish to decode (look at them in reverse. nb refers to index column)
1866 
1867  for (int n = 5; n >= 0; n--) {
1868  if (bits16 && bigEndianSource)
1869  data.push_back(0);
1870  data.push_back(static_cast<unsigned char>(n));
1871  if (bits16 && !bigEndianSource)
1872  data.push_back(0);
1873  }
1874 
1875  // Construct codec directly
1876 
1877 #ifndef _CRAYC
1878  {
1879  // Skip name of codec
1880  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
1881 
1882  std::unique_ptr<Codec> c;
1883  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
1884  if (bits16) {
1885  c.reset(new CodecInt16String<SameByteOrder>(odc::api::STRING));
1886  } else {
1887  c.reset(new CodecInt8String<SameByteOrder>(odc::api::STRING));
1888  }
1889  } else {
1890  if (bits16) {
1891  c.reset(new CodecInt16String<OtherByteOrder>(odc::api::STRING));
1892  } else {
1893  c.reset(new CodecInt8String<OtherByteOrder>(odc::api::STRING));
1894  }
1895  }
1896  c->load(ds);
1897  c->setDataStream(ds);
1898 
1899  EXPECT(ds.position() == eckit::Offset(156));
1900 
1901  // n.b. This is different. 16 bytes possible!!!
1902  EXPECT(c->dataSizeDoubles() == 2);
1903 
1904  double val[2];
1905  const char* val_c = reinterpret_cast<const char*>(val);
1906  c->decode(val);
1907  EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "mnopqrst");
1908  c->decode(val);
1909  EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "ghijkl");
1910  c->decode(val);
1911  EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "ab");
1912  c->decode(val);
1913  EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "opqrstuv");
1914  c->decode(val);
1915  EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "uvwxyzabcdef");
1916  c->decode(val);
1917  EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "ghijklmnopqrstuv");
1918 
1919  EXPECT(ds.position() == eckit::Offset(156 + (6 * (bits16 ? 2 : 1))));
1920  }
1921 #endif
1922 
1923  // Construct codec from factory
1924 
1925  size_t hdrSize = prepend_codec_selection_header(data, bits16 ? "int16_string" : "int8_string", bigEndianSource);
1926 
1927  {
1928  GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size());
1929 
1930  std::unique_ptr<Codec> c;
1931  if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) {
1932  c = CodecFactory::instance().load(ds.same(), odc::api::STRING);
1933  } else {
1934  c = CodecFactory::instance().load(ds.other(), odc::api::STRING);
1935  }
1936  c->setDataStream(ds);
1937 
1938  EXPECT(ds.position() == eckit::Offset(hdrSize + 156));
1939 
1940  // n.b. This is different. 16 bytes possible!!!
1941  EXPECT(c->dataSizeDoubles() == 2);
1942 
1943  double val[2];
1944  const char* val_c = reinterpret_cast<const char*>(val);
1945  c->decode(val);
1946  EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "mnopqrst");
1947  c->decode(val);
1948  EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "ghijkl");
1949  c->decode(val);
1950  EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "ab");
1951  c->decode(val);
1952  EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "opqrstuv");
1953  c->decode(val);
1954  EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "uvwxyzabcdef");
1955  c->decode(val);
1956  EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "ghijklmnopqrstuv");
1957 
1958  EXPECT(ds.position() == eckit::Offset(hdrSize + 156 + (6 * (bits16 ? 2 : 1))));
1959  }
1960  }
1961 }
1962 
1963 // ------------------------------------------------------------------------------------------------------
1964 
1965 int main(int argc, char* argv[]) {
1966 
1967  return run_tests(argc, argv);
1968 }
odc::MDI::realMDI
static double realMDI()
Definition: MDI.h:21
odc::MDI::integerMDI
static double integerMDI()
Definition: MDI.h:22
odc::api::Settings::treatIntegersAsDoubles
static void treatIntegersAsDoubles(bool flag)
Definition: Odb.cc:343
odc::core::GeneralDataStream::same
DataStream< SameByteOrder > & same()
Definition: DataStream.h:129
odc::core::GeneralDataStream::position
eckit::Offset position() const
Definition: DataStream.h:132
odc::core::GeneralDataStream::other
DataStream< OtherByteOrder > & other()
Definition: DataStream.h:130
odc::api::BITFIELD
@ BITFIELD
Definition: ColumnType.h:27
oops_run_time_test.val
val
Definition: oops_run_time_test.py:28
plotting-05-ObsGroup.c
c
Definition: plotting-05-ObsGroup.py:41
proc_gsi_ncdiag.i
i
Definition: proc_gsi_ncdiag.py:155
anonymous_namespace{test_codecs_read.cc}::TestIntegerDecoding
Definition: test_codecs_read.cc:34
anonymous_namespace{test_codecs_read.cc}::TestIntegerDecoding::TestIntegerDecoding
TestIntegerDecoding()
Definition: test_codecs_read.cc:35
anonymous_namespace{test_codecs_read.cc}::TestIntegerDecoding::~TestIntegerDecoding
~TestIntegerDecoding()
Definition: test_codecs_read.cc:36
main
int main(int argc, char *argv[])
Definition: test_codecs_read.cc:1965
prepend_codec_selection_header
size_t prepend_codec_selection_header(std::vector< unsigned char > &data, const std::string &codec_name, bool bigEndian=false)
Definition: test_codecs_read.cc:51
CASE
CASE("Constant values are constant")
Definition: test_codecs_read.cc:63