IODA Bundle
test_concatenated_odbs.cc
Go to the documentation of this file.
1 /*
2  * (C) Copyright 1996-2012 ECMWF.
3  *
4  * This software is licensed under the terms of the Apache Licence Version 2.0
5  * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6  * In applying this licence, ECMWF does not waive the privileges and immunities
7  * granted to it by virtue of its status as an intergovernmental organisation nor
8  * does it submit to any jurisdiction.
9  */
10 
11 #include "eckit/testing/Test.h"
12 #include "eckit/io/FileHandle.h"
13 #include "eckit/io/MemoryHandle.h"
14 #include "eckit/io/MultiHandle.h"
15 
16 #include "TemporaryFiles.h"
17 
18 #include "odc/Comparator.h"
19 #include "odc/core/Exceptions.h"
20 #include "odc/Reader.h"
21 #include "odc/Writer.h"
22 
23 #include <stdint.h>
24 
25 using namespace eckit::testing;
26 
27 
28 // ------------------------------------------------------------------------------------------------------
29 
30 CASE("ODBs concatenated in a file are valid (columns change)") {
31 
32  SETUP("Write multiple ODBs, then concatenate them together") {
33 
34  // Construct four ODBs, with differing characteristics
35 
36  class TemporaryODB1 : public TemporaryFile {
37  public:
38  TemporaryODB1() {
39  odc::Writer<> oda(path());
40  odc::Writer<>::iterator writer = oda.begin();
41 
42  writer->setNumberOfColumns(3);
43  writer->setColumn(0, "x", odc::api::REAL);
44  writer->setColumn(1, "y", odc::api::REAL);
45  writer->setColumn(2, "z", odc::api::INTEGER);
46  writer->writeHeader();
47 
48  for (size_t i = 1; i <= 2; i++) {
49  (*writer)[0] = i; // col 0
50  (*writer)[1] = i; // col 1
51  (*writer)[2] = i; // col 2
52  ++writer;
53  }
54  }
55  };
56 
57  // Change column names
58 
59  class TemporaryODB2 : public TemporaryFile {
60  public:
61  TemporaryODB2() {
62  odc::Writer<> oda(path());
63  odc::Writer<>::iterator writer = oda.begin();
64 
65  writer->setNumberOfColumns(3);
66  writer->setColumn(0, "x", odc::api::REAL);
67  writer->setColumn(1, "y", odc::api::INTEGER);
68  writer->setColumn(2, "v", odc::api::REAL);
69  writer->writeHeader();
70 
71  for (size_t i = 1; i <= 2; i++) {
72  (*writer)[0] = i * 10; // col 0
73  (*writer)[1] = i * 100; // col 1
74  (*writer)[2] = i * 1000; // col 2
75  ++writer;
76  }
77  }
78  };
79 
80  // Increase number of columns
81 
82  class TemporaryODB3 : public TemporaryFile {
83  public:
84  TemporaryODB3() {
85  odc::Writer<> oda(path());
86  odc::Writer<>::iterator writer = oda.begin();
87 
88  writer->setNumberOfColumns(4);
89  writer->setColumn(0, "x", odc::api::INTEGER);
90  writer->setColumn(1, "v", odc::api::REAL);
91  writer->setColumn(2, "y", odc::api::REAL);
92  writer->setColumn(3, "z", odc::api::REAL);
93  writer->writeHeader();
94 
95  for (size_t i = 1; i <= 2; i++) {
96  (*writer)[0] = i * 10; // col 0
97  (*writer)[1] = i * 1000; // col 1
98  (*writer)[2] = i * 100; // col 2
99  (*writer)[3] = 13; // col 3
100  ++writer;
101  }
102  }
103  };
104 
105  // Decrease number of columns
106 
107  class TemporaryODB4 : public TemporaryFile {
108  public:
109  TemporaryODB4() {
110  odc::Writer<> oda(path());
111  odc::Writer<>::iterator writer = oda.begin();
112 
113  writer->setNumberOfColumns(2);
114  writer->setColumn(0, "x", odc::api::REAL);
115  writer->setColumn(1, "v", odc::api::REAL);
116  writer->writeHeader();
117 
118  for (size_t i = 1; i <= 2; i++) {
119  (*writer)[0] = i * 5; // col 0
120  (*writer)[1] = i * 7; // col 1
121  ++writer;
122  }
123  }
124  };
125 
126  TemporaryODB1 tmpODB1;
127  TemporaryODB2 tmpODB2;
128  TemporaryODB3 tmpODB3;
129  TemporaryODB4 tmpODB4;
130 
131  // Directly concatenate files (append them to the first one)
132 
133  TemporaryFile combinedFile;
134 
135  {
136  std::vector<eckit::DataHandle*> readHandles;
137  readHandles.push_back(new eckit::FileHandle(tmpODB1.path()));
138  readHandles.push_back(new eckit::FileHandle(tmpODB2.path()));
139  readHandles.push_back(new eckit::FileHandle(tmpODB3.path()));
140  readHandles.push_back(new eckit::FileHandle(tmpODB4.path()));
141 
142  eckit::MultiHandle aggregateHandle(readHandles);
143  eckit::FileHandle out_handle(combinedFile.path());
144  aggregateHandle.openForRead();
145  aggregateHandle.saveInto(out_handle);
146  }
147 
148 
149  SECTION("The data in the concatenated files is correct") {
150 
151  odc::Reader in(combinedFile.path());
152  odc::Reader::iterator it = in.begin();
153 
154  EXPECT(it->columns().size() == 3);
155  EXPECT(it->columns()[0]->name() == "x");
156  EXPECT(it->columns()[1]->name() == "y");
157  EXPECT(it->columns()[2]->name() == "z");
158  EXPECT(it->columns().rowsNumber() == 2);
159 
160  for (size_t i = 1; i < 3; i++) {
161  for (size_t j = 0; j < 3; j++) { EXPECT((*it)[j] == i); }
162  ++it;
163  }
164 
165  // Check that we have changed the name of a column
166 
167  EXPECT(it->columns().size() == 3);
168  EXPECT(it->columns()[0]->name() == "x");
169  EXPECT(it->columns()[1]->name() == "y");
170  EXPECT(it->columns()[2]->name() == "v");
171  EXPECT(!it->columns().hasColumn("z"));
172  EXPECT(it->columns().rowsNumber() == 2);
173 
174  for (size_t i = 1; i < 3; i++) {
175  EXPECT((*it)[0] == 10 * i);
176  EXPECT((*it)[1] == 100 * i);
177  EXPECT((*it)[2] == 1000 * i);
178  ++it;
179  }
180 
181  // Now we have 4 columns
182 
183  EXPECT(it->columns().size() == 4);
184  EXPECT(it->columns()[0]->name() == "x");
185  EXPECT(it->columns()[1]->name() == "v");
186  EXPECT(it->columns()[2]->name() == "y");
187  EXPECT(it->columns()[3]->name() == "z");
188  EXPECT(it->columns().rowsNumber() == 2);
189 
190  for (size_t i = 1; i < 3; i++) {
191  EXPECT((*it)[0] == 10 * i);
192  EXPECT((*it)[1] == 1000 * i);
193  EXPECT((*it)[2] == 100 * i);
194  ++it;
195  }
196 
197  // And back to 2 columns
198 
199  EXPECT(it->columns().size() == 2);
200  EXPECT(it->columns()[0]->name() == "x");
201  EXPECT(it->columns()[1]->name() == "v");
202  EXPECT(it->columns().rowsNumber() == 2);
203 
204  for (size_t i = 1; i < 3; i++) {
205  EXPECT((*it)[0] == 5 * i);
206  EXPECT((*it)[1] == 7 * i);
207  ++it;
208  }
209  }
210 
211  SECTION("A copy of the concatenated file is identical") {
212 
213  odc::Reader in(combinedFile.path());
214  odc::Reader::iterator it = in.begin();
215  odc::Reader::iterator end = in.end();
216 
217  TemporaryFile copyFile;
218  odc::Writer<> out(copyFile.path());
219  odc::Writer<>::iterator o = out.begin();
220  o->pass1(it, end);
221 
222  // Check that ODB-API thinks the files are the same
223  EXPECT_NO_THROW(odc::Comparator().compare(combinedFile.path(), copyFile.path()));
224  }
225  }
226 }
227 
228 
229 CASE("If corrupt data follows a valid ODB this should not be treated as a new ODB") {
230 
231  // See ODB-376
232 
233  // Construct a valid ODB in a buffer, followed by some invalid data
234 
235  eckit::Buffer buf(4096);
236 
237  eckit::MemoryHandle writeDH(buf);
238 
239  {
240  odc::Writer<> oda(writeDH);
241  odc::Writer<>::iterator writer = oda.begin();
242 
243  writer->setNumberOfColumns(3);
244  writer->setColumn(0, "x", odc::api::REAL);
245  writer->setColumn(1, "y", odc::api::REAL);
246  writer->setColumn(2, "z", odc::api::INTEGER);
247  writer->writeHeader();
248 
249  for (size_t i = 1; i <= 2; i++) {
250  (*writer)[0] = i; // col 0
251  (*writer)[1] = i; // col 1
252  (*writer)[2] = i; // col 2
253  ++writer;
254  }
255  }
256 
257  // And write some invalid data on the end of the buffer
258 
259  const uint32_t invalid_data = 0xBAADF00D;
260  writeDH.write(&invalid_data, sizeof(invalid_data));
261 
262  // Now read the data. We should get the data back, and then an error...
263 
264  eckit::MemoryHandle readDH(buf);
265  readDH.openForRead();
266 
267  odc::Reader in(readDH);
268  odc::Reader::iterator it = in.begin();
269 
270  EXPECT(static_cast<long>(it->data()[0]) == 1);
271  EXPECT(static_cast<long>(it->data()[1]) == 1);
272  EXPECT(static_cast<long>(it->data()[2]) == 1);
273  ++it;
274 
275  EXPECT(static_cast<long>(it->data()[0]) == 2);
276  EXPECT(static_cast<long>(it->data()[1]) == 2);
277  EXPECT(static_cast<long>(it->data()[2]) == 2);
278 
279  // Where we would expect an EOF, or a new table, we now have corrupt data. This increment should
280  // NOT succeed, but should complain vociferously!!!
281 
282  EXPECT_THROWS_AS(++it, odc::core::ODBInvalid);
283 }
284 
285 CASE("If a corrupted ODB (with no row data following the header) then report an error") {
286 
287  // See ODB-376
288 
289  // Construct a valid ODB in a buffer, followed by some invalid data
290 
291  eckit::Buffer buf(4096);
292 
293  eckit::MemoryHandle writeDH(buf);
294 
295  {
296  odc::Writer<> oda(writeDH);
297  odc::Writer<>::iterator writer = oda.begin();
298 
299  writer->setNumberOfColumns(3);
300  writer->setColumn(0, "x", odc::api::REAL);
301  writer->setColumn(1, "y", odc::api::REAL);
302  writer->setColumn(2, "z", odc::api::INTEGER);
303  writer->writeHeader();
304 
305  for (size_t i = 1; i <= 2; i++) {
306  (*writer)[0] = i; // col 0
307  (*writer)[1] = i; // col 1
308  (*writer)[2] = i; // col 2
309  ++writer;
310  }
311  }
312 
313  // The header size is 320 bytes. Copy the data from the start...
314 
315 // writeDH.write(buf.data(), 322);
316  writeDH.write(buf.data(), 322 - 80); // -80 == we no longer encode (empty) flags
317 
318  // Now read the data. We should get the data back, and then an error...
319 
320  eckit::MemoryHandle readDH(buf.data(), static_cast<size_t>(writeDH.position()));
321  readDH.openForRead();
322 
323  odc::Reader in(readDH);
324  odc::Reader::iterator it = in.begin();
325 
326  EXPECT(static_cast<long>(it->data()[0]) == 1);
327  EXPECT(static_cast<long>(it->data()[1]) == 1);
328  EXPECT(static_cast<long>(it->data()[2]) == 1);
329  ++it;
330 
331  EXPECT(static_cast<long>(it->data()[0]) == 2);
332  EXPECT(static_cast<long>(it->data()[1]) == 2);
333  EXPECT(static_cast<long>(it->data()[2]) == 2);
334 
335  // Where we would expect an EOF, or a new table, we now have corrupt data. This increment should
336  // NOT succeed, but should complain vociferously!!!
337 
338  EXPECT_THROWS_AS(++it, eckit::SeriousBug);
339 }
340 
341 
342 // ------------------------------------------------------------------------------------------------------
343 
344 int main(int argc, char* argv[]) {
345  return run_tests(argc, argv);
346 }
347 
oda
void oda
Definition: ReaderIterator.h:27
TemporaryFile::path
const eckit::PathName & path() const
Definition: TemporaryFiles.h:38
odc::Reader::end
const iterator end() const
Definition: Reader.cc:81
odc::Reader::begin
iterator begin()
Definition: Reader.cc:74
odc::Row_::pass1
unsigned long pass1(T b, const T e)
Definition: IteratorProxy.h:114
odc::Row_::data
DATA * data()
Definition: IteratorProxy.h:77
odc::Row_::columns
const core::MetaData & columns() const
Definition: IteratorProxy.h:94
odc::Writer::begin
iterator begin(bool openDataHandle=true)
Definition: Writer.cc:92
odc::core::MetaData::hasColumn
bool hasColumn(const std::string &) const
Definition: MetaData.cc:87
odc::core::MetaData::rowsNumber
unsigned long long rowsNumber() const
Definition: MetaData.h:39
proc_gsi_ncdiag.i
i
Definition: proc_gsi_ncdiag.py:155
main
int main(int argc, char *argv[])
Definition: test_concatenated_odbs.cc:344
CASE
CASE("ODBs concatenated in a file are valid (columns change)")
Definition: test_concatenated_odbs.cc:30