OOPS
test/mpi/mpi.h
Go to the documentation of this file.
1 /*
2  * (C) Copyright 2020 Met Office UK
3  *
4  * This software is licensed under the terms of the Apache Licence Version 2.0
5  * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6  */
7 
8 #ifndef TEST_MPI_MPI_H_
9 #define TEST_MPI_MPI_H_
10 
11 #include <Eigen/Dense>
12 
13 #include <string>
14 #include <vector>
15 
16 #include "eckit/config/LocalConfiguration.h"
17 #include "eckit/mpi/Comm.h"
18 #include "eckit/testing/Test.h"
19 
20 #include "oops/../test/TestEnvironment.h"
21 #include "oops/mpi/mpi.h"
22 #include "oops/runs/Test.h"
23 #include "oops/util/DateTime.h"
24 #include "oops/util/Expect.h"
25 #include "oops/util/parameters/Parameters.h"
26 #include "oops/util/parameters/RequiredParameter.h"
27 
28 namespace eckit
29 {
30  // Don't use the contracted output for these types: the current implementation works only
31  // with integer types.
32  template <> struct VectorPrintSelector<util::DateTime> { typedef VectorPrintSimple selector; };
33 } // namespace eckit
34 
35 namespace test {
36 
37 class TestParameters : public oops::Parameters {
38  OOPS_CONCRETE_PARAMETERS(TestParameters, Parameters)
39  public:
40  oops::RequiredParameter<std::vector<util::DateTime>> values{"values", this};
41 };
42 
43 // -----------------------------------------------------------------------------------------------
44 CASE("mpi/mpi/defaultCommunicators") {
45  const eckit::mpi::Comm & world = oops::mpi::world();
46  size_t worldsize = world.size();
47  EXPECT_EQUAL(worldsize, 4);
48 
49  const eckit::mpi::Comm & talk_to_myself = oops::mpi::myself();
50  size_t myownsize = talk_to_myself.size();
51  EXPECT_EQUAL(myownsize, 1);
52 }
53 // -----------------------------------------------------------------------------------------------
54 CASE("mpi/mpi/allGathervUsingSerialize") {
55  const eckit::Configuration &conf = TestEnvironment::config();
56  const eckit::mpi::Comm &comm = oops::mpi::world();
57 
58  TestParameters localParams;
59  const size_t rank = comm.rank();
60  localParams.deserialize(conf.getSubConfiguration("local" + std::to_string(rank)));
61  const std::vector<util::DateTime> &localValues = localParams.values;
62 
63  TestParameters globalParams;
64  globalParams.deserialize(conf.getSubConfiguration("global"));
65  const std::vector<util::DateTime> &expectedGlobalValues = globalParams.values;
66 
67  size_t numGlobalValues;
68  comm.allReduce(localValues.size(), numGlobalValues, eckit::mpi::Operation::SUM);
69 
70  std::vector<util::DateTime> globalValues(numGlobalValues);
71  oops::mpi::allGathervUsingSerialize(comm, localValues.begin(), localValues.end(),
72  globalValues.begin());
73  EXPECT_EQUAL(globalValues, expectedGlobalValues);
74 }
75 // -----------------------------------------------------------------------------------------------
76 CASE("mpi/mpi/SendReceive") {
77  const eckit::Configuration &conf = TestEnvironment::config();
78  const eckit::mpi::Comm &comm = oops::mpi::world();
79  const size_t rank = comm.rank();
80  int source = (rank + 3) % comm.size();
81  int destination = (rank + 1) % comm.size();
82  int tag_send = destination;
83  int tag_recv = rank;
84 
85  util::DateTime sendValue(conf.getString("send"+ std::to_string(rank)));
86  util::DateTime expectedValue(conf.getString("expected"+ std::to_string(rank)));
87  util::DateTime receivedValue;
88 
89  if (rank < 3) {
90  oops::mpi::send(comm, sendValue, destination, tag_send);
91  oops::mpi::receive(comm, receivedValue, source, tag_recv);
92  } else {
93  oops::mpi::receive(comm, receivedValue, source, tag_recv);
94  oops::mpi::send(comm, sendValue, destination, tag_send);
95  }
96  EXPECT_EQUAL(receivedValue, expectedValue);
97 }
98 // -----------------------------------------------------------------------------------------------
99 CASE("mpi/mpi/gatherSerializable") {
100  const eckit::Configuration &conf = TestEnvironment::config();
101  const eckit::mpi::Comm &comm = oops::mpi::world();
102 
103  TestParameters localParams;
104  const size_t rank = comm.rank();
105  localParams.deserialize(conf.getSubConfiguration("local" + std::to_string(rank)));
106  const std::vector<util::DateTime> &localValues = localParams.values;
107 
108  TestParameters globalParams;
109  globalParams.deserialize(conf.getSubConfiguration("global"));
110  const std::vector<util::DateTime> &expectedGlobalValues = globalParams.values;
111 
112  size_t numGlobalValues;
113  comm.allReduce(localValues.size(), numGlobalValues, eckit::mpi::Operation::SUM);
114 
115  std::vector<util::DateTime> globalValues(numGlobalValues);
116 
117  util::DateTime zeroDate("0001-01-01T00:00:00Z");
118  for (int ii = 0; ii < numGlobalValues; ++ii) {
119  globalValues[ii] = zeroDate;
120  }
121 
122  std::vector<util::DateTime> zeroValues = globalValues;
123 
124  int root_gather = conf.getInt("root for gathering", 0);
125 
126  oops::mpi::gather(comm, localValues, globalValues, root_gather);
127  if (rank == root_gather) {
128  EXPECT_EQUAL(globalValues, expectedGlobalValues);
129  } else {
130  EXPECT_EQUAL(globalValues, zeroValues);
131  }
132 }
133 // -----------------------------------------------------------------------------------------------
134 CASE("mpi/mpi/gatherDouble") {
135  const eckit::Configuration &conf = TestEnvironment::config();
136  const eckit::mpi::Comm &comm = oops::mpi::world();
137  const size_t rank = comm.rank();
138 
139  std::vector<double> localDouble;
140  conf.get("localDouble" + std::to_string(rank), localDouble);
141 
142  std::vector<double> globalDoubleExpected;
143  conf.get("globalDouble", globalDoubleExpected);
144 
145  size_t numGlobalDouble;
146  comm.allReduce(localDouble.size(), numGlobalDouble, eckit::mpi::Operation::SUM);
147 
148  std::vector<double> globalDouble(numGlobalDouble, 0.0);
149  std::vector<double> zerosDouble = globalDouble;
150 
151  int root_gather = conf.getInt("root for gathering", 0);
152 
153  oops::mpi::gather(comm, localDouble, globalDouble, root_gather);
154 
155  if (rank == root_gather) {
156  EXPECT_EQUAL(globalDouble, globalDoubleExpected);
157  } else {
158  EXPECT_EQUAL(globalDouble, zerosDouble);
159  }
160 }
161 // -----------------------------------------------------------------------------------------------
162 CASE("mpi/mpi/allGatherEigen") {
163  const eckit::mpi::Comm &comm = oops::mpi::world();
164  const size_t rank = comm.rank();
165  Eigen::VectorXd localEigen = rank * Eigen::VectorXd::Ones(5);
166  std::vector<Eigen::VectorXd> globalEigen = {Eigen::VectorXd::Zero(5), Eigen::VectorXd::Zero(5),
167  Eigen::VectorXd::Zero(5), Eigen::VectorXd::Zero(5)};
168  std::vector<Eigen::VectorXd> expectedEigen = {0*Eigen::VectorXd::Ones(5),
169  1*Eigen::VectorXd::Ones(5),
170  2*Eigen::VectorXd::Ones(5),
171  3*Eigen::VectorXd::Ones(5)};
172  oops::mpi::allGather(comm, localEigen, globalEigen);
173  EXPECT_EQUAL(expectedEigen[0], globalEigen[0]);
174  EXPECT_EQUAL(expectedEigen[1], globalEigen[1]);
175  EXPECT_EQUAL(expectedEigen[2], globalEigen[2]);
176  EXPECT_EQUAL(expectedEigen[3], globalEigen[3]);
177 }
178 // -----------------------------------------------------------------------------------------------
179 
180 class Mpi : public oops::Test {
181  private:
182  std::string testid() const override {return "test::mpi::mpi";}
183 
184  void register_tests() const override {}
185  void clear() const override {}
186 };
187 
188 } // namespace test
189 
190 #endif // TEST_MPI_MPI_H_
mpi.h
test::CASE
CASE("test_linearmodelparameterswrapper_valid_name")
Definition: LinearModelFactory.cc:22
test::TestParameters::values
oops::RequiredParameter< std::vector< util::DateTime > > values
Definition: test/mpi/mpi.h:40
test
Definition: LinearModelFactory.cc:20
oops::mpi::allGather
void allGather(const eckit::mpi::Comm &comm, const Eigen::VectorXd &sendbuf, std::vector< Eigen::VectorXd > &recvbuf)
Definition: oops/mpi/mpi.cc:58
oops::mpi::myself
const eckit::mpi::Comm & myself()
Default communicator with each MPI task by itself.
Definition: oops/mpi/mpi.cc:28
test::Mpi::testid
std::string testid() const override
Definition: test/mpi/mpi.h:182
oops::mpi::allGathervUsingSerialize
void allGathervUsingSerialize(const eckit::mpi::Comm &comm, CIter first, CIter last, Iter recvbuf)
A wrapper around the MPI all gather operation for serializable types.
Definition: oops/mpi/mpi.h:107
test::Mpi::register_tests
void register_tests() const override
Definition: test/mpi/mpi.h:184
eckit
Definition: FieldL95.h:22
oops::mpi::send
void send(const eckit::mpi::Comm &comm, const SERIALIZABLE &sendobj, const int dest, const int tag)
Extend eckit Comm for Serializable oops objects.
Definition: oops/mpi/mpi.h:36
Test.h
test::TestEnvironment::config
static const eckit::Configuration & config()
Definition: TestEnvironment.h:40
test::Mpi::clear
void clear() const override
Definition: test/mpi/mpi.h:185
oops::mpi::gather
void gather(const eckit::mpi::Comm &comm, const std::vector< double > &send, std::vector< double > &recv, const size_t root)
Definition: oops/mpi/mpi.cc:34
TestEnvironment.h
test::TestParameters
Definition: test/mpi/mpi.h:37
test::Mpi
Definition: test/mpi/mpi.h:180
oops::Test
Definition: Test.h:39
oops::mpi::world
const eckit::mpi::Comm & world()
Default communicator with all MPI tasks (ie MPI_COMM_WORLD)
Definition: oops/mpi/mpi.cc:22
util
Definition: ObservationL95.h:32
oops::mpi::receive
void receive(const eckit::mpi::Comm &comm, SERIALIZABLE &recvobj, const int source, const int tag)
Definition: oops/mpi/mpi.h:46