IODA Bundle
test_decode_odb.cc
Go to the documentation of this file.
1 /*
2  * (C) Copyright 1996-2012 ECMWF.
3  *
4  * This software is licensed under the terms of the Apache Licence Version 2.0
5  * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6  * In applying this licence, ECMWF does not waive the privileges and immunities
7  * granted to it by virtue of its status as an intergovernmental organisation nor
8  * does it submit to any jurisdiction.
9  */
10 
11 #include "eckit/config/Resource.h"
12 #include "eckit/testing/Test.h"
13 #include "eckit/types/FloatCompare.h"
14 #include "eckit/value/Value.h"
15 
16 #include "odc/Reader.h"
17 
18 // Some of the math.h/cmath functions are not clean when switching to C++11
19 #if __cplusplus <= 199711L
20 #include <math.h>
21 #else
22 #include <cmath>
23 #define fabs(x) std::fabs((x))
24 #define modf(x,y) std::modf((x),(y))
25 #endif
26 
27 using namespace eckit::testing;
28 using eckit::types::is_approximately_equal;
29 
30 
31 // ------------------------------------------------------------------------------------------------------
32 
33 struct CellData {
34  template <typename T>
35  CellData(const std::string n, const T& v, bool m=false) : name(n), value(v), missing(m) {}
36 
37  std::string name;
38  eckit::Value value;
39  bool missing;
40 };
41 
42 
43 // Checker picks a couple of rows in 2000010106.odb to test.
44 // Can be extended to check anything depending on initialisation
45 // TODO: Build a random ODB, and test it.
46 
47 class ODBChecker {
48 
49 public: // types
50 
51  typedef std::map<size_t, std::vector<CellData> > RowStore;
52 
53 public: // methods
54 
55  ODBChecker() {
56 
57  std::vector<CellData> rowData;
58 
59  rowData.push_back(CellData("expver@desc", std::string("0018 ")));
60  rowData.push_back(CellData("andate@desc", 20000101));
61  rowData.push_back(CellData("antime@desc", 60000));
62  rowData.push_back(CellData("seqno@hdr", 66969));
63  rowData.push_back(CellData("obstype@hdr", 2));
64  rowData.push_back(CellData("obschar@hdr", 67132561));
65  rowData.push_back(CellData("subtype@hdr", 145));
66  rowData.push_back(CellData("date@hdr", 20000101));
67  rowData.push_back(CellData("time@hdr", 32200));
68  rowData.push_back(CellData("rdbflag@hdr", 0));
69  rowData.push_back(CellData("status@hdr", 4));
70  rowData.push_back(CellData("event1@hdr", 512));
71  rowData.push_back(CellData("blacklist@hdr", 0));
72  rowData.push_back(CellData("sortbox@hdr", 2147483647, true));
73  rowData.push_back(CellData("sitedep@hdr", 0));
74  rowData.push_back(CellData("statid@hdr", std::string("MR413SRA")));
75  rowData.push_back(CellData("ident@hdr", 0));
76  rowData.push_back(CellData("lat@hdr", 0.831300));
77  rowData.push_back(CellData("lon@hdr", -2.057394));
78  rowData.push_back(CellData("stalt@hdr", -2147483647.000000, true));
79  rowData.push_back(CellData("modoro@hdr", 717.562744));
80  rowData.push_back(CellData("trlat@hdr", 0.831300));
81  rowData.push_back(CellData("trlon@hdr", 4.225791));
82  rowData.push_back(CellData("instspec@hdr", 3095));
83  rowData.push_back(CellData("event2@hdr", 0));
84  rowData.push_back(CellData("anemoht@hdr", 0.000000));
85  rowData.push_back(CellData("baroht@hdr", 0.000000));
86  rowData.push_back(CellData("sensor@hdr", 0));
87  rowData.push_back(CellData("numlev@hdr", 1));
88  rowData.push_back(CellData("varno_presence@hdr", 12));
89  rowData.push_back(CellData("varno@body", 3));
90  rowData.push_back(CellData("vertco_type@body", 1));
91  rowData.push_back(CellData("rdbflag@body", 0));
92  rowData.push_back(CellData("anflag@body", 0));
93  rowData.push_back(CellData("status@body", 4));
94  rowData.push_back(CellData("event1@body", 33554432));
95  rowData.push_back(CellData("blacklist@body", 0));
96  rowData.push_back(CellData("entryno@body", 1));
97  rowData.push_back(CellData("press@body", 23840.000000));
98  rowData.push_back(CellData("press_rl@body", -2147483647.000000, true));
99  rowData.push_back(CellData("obsvalue@body", 34.739117));
100  rowData.push_back(CellData("aux1@body", 35.000000));
101  rowData.push_back(CellData("event2@body", 0));
102  rowData.push_back(CellData("ppcode@body", 0));
103  rowData.push_back(CellData("level@body", 0));
104  rowData.push_back(CellData("biascorr@body", 0.000000));
105  rowData.push_back(CellData("final_obs_error@errstat", 2.920646));
106  rowData.push_back(CellData("obs_error@errstat", 2.920646));
107  rowData.push_back(CellData("repres_error@errstat", -2147483647.000000, true));
108  rowData.push_back(CellData("pers_error@errstat", -2147483647.000000, true));
109  rowData.push_back(CellData("fg_error@errstat", 3.002484));
110 
111  data_[0] = rowData;
112 
113  rowData.clear();
114  rowData.push_back(CellData("expver@desc", std::string("0018 ")));
115  rowData.push_back(CellData("andate@desc", 20000101));
116  rowData.push_back(CellData("antime@desc", 60000));
117  rowData.push_back(CellData("seqno@hdr", 6020684));
118  rowData.push_back(CellData("obstype@hdr", 7));
119  rowData.push_back(CellData("obschar@hdr", 135265490));
120  rowData.push_back(CellData("subtype@hdr", 54));
121  rowData.push_back(CellData("date@hdr", 20000101));
122  rowData.push_back(CellData("time@hdr", 54533));
123  rowData.push_back(CellData("rdbflag@hdr", 0));
124  rowData.push_back(CellData("status@hdr", 44));
125  rowData.push_back(CellData("event1@hdr", 2));
126  rowData.push_back(CellData("blacklist@hdr", 16777223));
127  rowData.push_back(CellData("sortbox@hdr", 2147483647, true));
128  rowData.push_back(CellData("sitedep@hdr", 2147483647, true));
129  rowData.push_back(CellData("statid@hdr", std::string(" 203")));
130  rowData.push_back(CellData("ident@hdr", 203));
131  rowData.push_back(CellData("lat@hdr", -0.933479));
132  rowData.push_back(CellData("lon@hdr", -2.107894));
133  rowData.push_back(CellData("stalt@hdr", 870000.000000));
134  rowData.push_back(CellData("modoro@hdr", 0.963609));
135  rowData.push_back(CellData("trlat@hdr", -0.933479));
136  rowData.push_back(CellData("trlon@hdr", 4.175291));
137  rowData.push_back(CellData("instspec@hdr", 9215));
138  rowData.push_back(CellData("event2@hdr", 0));
139  rowData.push_back(CellData("anemoht@hdr", 0.000000));
140  rowData.push_back(CellData("baroht@hdr", 0.000000));
141  rowData.push_back(CellData("sensor@hdr", 0));
142  rowData.push_back(CellData("numlev@hdr", 0));
143  rowData.push_back(CellData("varno_presence@hdr", 1032));
144  rowData.push_back(CellData("varno@body", 119));
145  rowData.push_back(CellData("vertco_type@body", 3));
146  rowData.push_back(CellData("rdbflag@body", 0));
147  rowData.push_back(CellData("anflag@body", 48));
148  rowData.push_back(CellData("status@body", 44));
149  rowData.push_back(CellData("event1@body", 512));
150  rowData.push_back(CellData("blacklist@body", 0));
151  rowData.push_back(CellData("entryno@body", 6));
152  rowData.push_back(CellData("press@body", 6.000000));
153  rowData.push_back(CellData("press_rl@body", -2147483647.000000, true));
154  rowData.push_back(CellData("obsvalue@body", 243.490005));
155  rowData.push_back(CellData("aux1@body", -2147483647.000000, true));
156  rowData.push_back(CellData("event2@body", 0));
157  rowData.push_back(CellData("ppcode@body", 0));
158  rowData.push_back(CellData("level@body", 0));
159  rowData.push_back(CellData("biascorr@body", 0.493023));
160  rowData.push_back(CellData("final_obs_error@errstat", 0.600000));
161  rowData.push_back(CellData("obs_error@errstat", 0.600000));
162  rowData.push_back(CellData("repres_error@errstat", -2147483647.000000, true));
163  rowData.push_back(CellData("pers_error@errstat", 0.958788));
164  rowData.push_back(CellData("fg_error@errstat", 0.269232));
165 
166  data_[371426] = rowData;
167  }
168 
169  ~ODBChecker() {}
170 
171  void checkRow(size_t num, const odc::Reader::iterator& row) {
172 
173  if (data_.find(num) != data_.end()) {
174  std::vector<CellData>& reference(data_[num]);
175  EXPECT(reference.size() == row->columns().size());
176 
177  for (size_t i = 0; i < reference.size(); i++) {
178  EXPECT(row->columns()[i]->name() == reference[i].name);
179 
180  // Data is always returned in an array of (8-byte) doubles. Actual data is of types
181  // of size <= 8 byte. Needs some casting to access.
182 
183  if (reference[i].value.isString()) {
184  std::string s(reinterpret_cast<const char*>(&row->data()[i]), 8);
185  EXPECT(reference[i].value == s);
186  } else if (reference[i].value.isNumber()) {
187  double intpart;
188  EXPECT(modf(row->data()[i], &intpart) == 0.0);
189  EXPECT(static_cast<long long>(reference[i].value) == static_cast<long long>(intpart));
190  } else if (reference[i].value.isDouble()) {
191  EXPECT(is_approximately_equal(static_cast<double>(reference[i].value), row->data()[i],
192  fabs(1.0e-5 * static_cast<double>(reference[i].value))));
193  } else {
194  // We don't want unknown data types slipping in here!!!
195  EXPECT(false);
196  }
197 
198  // Check that the missing values are reported correctly
199  EXPECT(reference[i].missing == (row->data()[i] == row->columns()[i]->missingValue()));
200  }
201  }
202  }
203 
204  size_t highestRow() const {
205  size_t biggestRow = 0;
206  for (RowStore::const_iterator it = data_.begin(); it != data_.end(); ++it) {
207  biggestRow = (it->first > biggestRow) ? it->first : biggestRow;
208  }
209  return biggestRow;
210  }
211 
212 private: // members
213 
214  RowStore data_;
215 };
216 
217 
218 // ------------------------------------------------------------------------------------------------------
219 
220 eckit::Resource<eckit::PathName> testDataPath("$TEST_DATA_DIRECTORY", "..");
221 
222 CASE("The correct number of rows are decoded") {
223 
224  eckit::PathName filename = testDataPath / "2000010106.odb";
225 
226  odc::Reader in(filename);
227  odc::Reader::iterator it = in.begin();
228 
229  // Each table in the ODB will contain a maximum of 10000 rows!
230  EXPECT(it->columns().rowsNumber() == 10000);
231 
232  size_t count = 0;
233  for (; it != in.end(); ++it) {
234 
235  if (count < 3320000) {
236  EXPECT(it->columns().rowsNumber() == 10000);
237  } else {
238  EXPECT(it->columns().rowsNumber() == 1753);
239  }
240  count++;
241  }
242 
243  // All of the lines correctly decoded
244  EXPECT(count == 3321753);
245 }
246 
247 
248 
249 CASE("The correct data is present in a selection of random rows") {
250 
251  eckit::PathName filename = testDataPath / "2000010106.odb";
252 
253  odc::Reader in(filename);
254  odc::Reader::iterator it = in.begin();
255 
256  ODBChecker checker;
257 
258  size_t biggestRow = checker.highestRow();
259  size_t count = 0;
260 
261  for (; it != in.end() && count <= biggestRow; ++it) {
262  checker.checkRow(count, it);
263  count++;
264  }
265 }
266 
267 // TODO: Test missing values
268 // TODO: Test reading a randomly created ODB.
269 
270 // ------------------------------------------------------------------------------------------------------
271 
272 int main(int argc, char* argv[]) {
273  return run_tests(argc, argv);
274 }
275 
count
static void count(void *counter, const double *data, size_t n)
Definition: UnitTests.cc:531
ODBChecker::highestRow
size_t highestRow() const
Definition: test_decode_odb.cc:204
ODBChecker::data_
RowStore data_
Definition: test_decode_odb.cc:214
ODBChecker::checkRow
void checkRow(size_t num, const odc::Reader::iterator &row)
Definition: test_decode_odb.cc:171
ODBChecker::RowStore
std::map< size_t, std::vector< CellData > > RowStore
Definition: test_decode_odb.cc:51
odc::Reader::end
const iterator end() const
Definition: Reader.cc:81
odc::Reader::begin
iterator begin()
Definition: Reader.cc:74
odc::Row_::data
DATA * data()
Definition: IteratorProxy.h:77
odc::Row_::columns
const core::MetaData & columns() const
Definition: IteratorProxy.h:94
odc::core::MetaData::rowsNumber
unsigned long long rowsNumber() const
Definition: MetaData.h:39
03-VariablesIntro.name
name
Definition: 03-VariablesIntro.py:74
anonymous_namespace{IodaDescription.cpp}::ConfKeys::Global::Value
const char * Value
Definition: IodaDescription.cpp:46
proc_gsi_ncdiag.i
i
Definition: proc_gsi_ncdiag.py:155
CellData::value
eckit::Value value
Definition: test_decode_odb.cc:38
CellData::CellData
CellData(const std::string n, const T &v, bool m=false)
Definition: test_decode_odb.cc:35
CellData::name
std::string name
Definition: test_decode_odb.cc:37
main
int main(int argc, char *argv[])
Definition: test_decode_odb.cc:272
testDataPath
eckit::Resource< eckit::PathName > testDataPath("$TEST_DATA_DIRECTORY", "..")
CASE
CASE("The correct number of rows are decoded")
Definition: test_decode_odb.cc:222