15 #include "eckit/config/LocalConfiguration.h"
16 #include "oops/util/IntSetParser.h"
17 #include "oops/util/Logger.h"
18 #include "oops/util/missingValues.h"
19 #include "oops/util/PartialDateTime.h"
28 std::vector<eckit::LocalConfiguration> masks;
29 config.get(
"where", masks);
32 for (
size_t jm = 0; jm < masks.size(); ++jm) {
33 eckit::LocalConfiguration varconf(masks[jm],
"variable");
41 const float & vmin,
const float & vmax,
42 std::vector<bool> & mask) {
43 const float not_set_value = util::missingValue(not_set_value);
44 const size_t n = data.size();
46 if (vmin != not_set_value || vmax != not_set_value) {
47 for (
size_t jj = 0; jj < n; ++jj) {
48 if (vmin != not_set_value && data[jj] < vmin) mask[jj] =
false;
49 if (vmax != not_set_value && data[jj] > vmax) mask[jj] =
false;
57 const std::string & vmin,
const std::string & vmax,
58 std::vector<bool> & mask) {
59 const std::string not_set_value =
"0000-00-00T00:00:00Z";
61 if (vmin != not_set_value || vmax != not_set_value) {
62 util::PartialDateTime pdt_vmin(vmin), pdt_vmax(vmax);
64 for (
size_t jj = 0; jj < data.size(); ++jj) {
65 if (vmin != not_set_value && pdt_vmin > data[jj]) mask[jj] =
false;
66 if (vmax != not_set_value && pdt_vmax < data[jj]) mask[jj] =
false;
74 std::vector<bool> & mask) {
76 const size_t n = data.size();
77 for (
size_t jj = 0; jj < n; ++jj) {
78 if (data[jj] ==
missing) mask[jj] =
false;
85 std::vector<bool> & mask) {
87 const size_t n = data.size();
88 for (
size_t jj = 0; jj < n; ++jj) {
89 if (data[jj] !=
missing) mask[jj] =
false;
96 const std::set<T> & whitelist,
97 std::vector<bool> & mask) {
98 for (
size_t jj = 0; jj < data.size(); ++jj) {
106 const std::set<T> & blacklist,
107 std::vector<bool> & mask) {
109 for (
size_t jj = 0; jj < data.size(); ++jj) {
116 const std::set<std::string> & blacklist,
117 std::vector<bool> & mask) {
118 for (
size_t jj = 0; jj < data.size(); ++jj) {
126 const float not_set_value = util::missingValue(not_set_value);
127 const float vmin = mask.getFloat(
"minvalue", not_set_value);
128 const float vmax = mask.getFloat(
"maxvalue", not_set_value);
130 if (vmin != not_set_value || vmax != not_set_value) {
131 std::vector<float> data;
132 filterdata.
get(varname, data);
139 const std::string not_set_value(
"0000-00-00T00:00:00Z");
140 const std::string vmin = mask.getString(
"minvalue", not_set_value);
141 const std::string vmax = mask.getString(
"maxvalue", not_set_value);
144 if (vmin != not_set_value || vmax != not_set_value) {
145 std::vector<util::DateTime> data;
146 filterdata.
get(varname, data);
153 const std::set<int> & flags,
154 std::vector<bool> & mask) {
155 std::bitset<32> flags_bs;
156 for (
const int &elem : flags) {
159 for (
size_t jj = 0; jj < data.size(); ++jj) {
160 if ((data[jj] & flags_bs.to_ulong()) != 0) mask[jj] =
false;
165 void isInString(std::vector<bool> & where, eckit::LocalConfiguration
const & mask,
167 std::vector<std::string> data;
168 std::vector<std::string> whitelistvec = mask.getStringVector(
"is_in");
169 std::set<std::string> whitelist(whitelistvec.begin(), whitelistvec.end());
170 filterdata.
get(varname, data);
175 void isInInteger(std::vector<bool> & where, eckit::LocalConfiguration
const & mask,
177 std::vector<int> data;
178 std::set<int> whitelist = oops::parseIntSet(mask.getString(
"is_in"));
179 filterdata.
get(varname, data);
184 void isNotInString(std::vector<bool> & where, eckit::LocalConfiguration
const & mask,
186 std::vector<std::string> data;
187 std::vector<std::string> blacklistvec = mask.getStringVector(
"is_not_in");
188 std::set<std::string> blacklist(blacklistvec.begin(), blacklistvec.end());
189 filterdata.
get(varname, data);
194 void isNotInInteger(std::vector<bool> & where, eckit::LocalConfiguration
const & mask,
196 std::vector<int> data;
197 filterdata.
get(varname, data);
198 std::set<int> blacklist = oops::parseIntSet(mask.getString(
"is_not_in"));
205 const size_t nlocs = filterdata.
nlocs();
208 std::vector<bool> where(nlocs,
true);
210 std::vector<eckit::LocalConfiguration> masks;
211 config.get(
"where", masks);
213 for (
size_t jm = 0; jm < masks.size(); ++jm) {
214 eckit::LocalConfiguration varconf(masks[jm],
"variable");
216 for (
size_t jvar = 0; jvar < var.
size(); ++jvar) {
217 if (var.
group() !=
"VarMetaData") {
219 ioda::ObsDtype dtype = filterdata.
dtype(varname);
221 if (dtype == ioda::ObsDtype::DateTime) {
228 if (masks[jm].has(
"is_defined")) {
229 if (filterdata.
has(varname)) {
230 std::vector<float> data;
231 filterdata.
get(varname, data);
234 std::fill(where.begin(), where.end(),
false);
239 if (masks[jm].has(
"is_not_defined")) {
240 std::vector<float> data;
241 filterdata.
get(varname, data);
246 if (masks[jm].has(
"is_in")) {
247 if (dtype == ioda::ObsDtype::String) {
248 isInString(where, masks[jm], filterdata, varname);
249 }
else if (dtype == ioda::ObsDtype::Integer) {
250 isInInteger(where, masks[jm], filterdata, varname);
252 throw eckit::UserError(
253 "Only integer and string variables may be used for processWhere 'is_in'",
259 if (masks[jm].has(
"is_not_in")) {
260 if (dtype == ioda::ObsDtype::String) {
262 }
else if (dtype == ioda::ObsDtype::Integer) {
265 throw eckit::UserError(
266 "Only integer and string variables may be used for processWhere 'is_not_in'",
272 if (masks[jm].has(
"any_bit_set_of")) {
273 if (dtype == ioda::ObsDtype::Integer) {
274 std::vector<int> data;
275 std::set<int> flags = oops::parseIntSet(masks[jm].getString(
"any_bit_set_of"));
276 filterdata.
get(varname, data);
279 throw eckit::UserError(
280 "Only integer variables may be used for processWhere 'any_bit_set_of'",
289 for (
size_t jj = 0; jj < nlocs; ++jj) {
290 if (where[jj] ==
false) ++ii;
293 oops::Log::debug() <<
"processWhere: selected " << ii <<
" obs." << std::endl;