bes Updated for version 3.20.10
GridAggregationBase.cc
1
2// This file is part of the "NcML Module" project, a BES module designed
3// to allow NcML files to be used to be used as a wrapper to add
4// AIS to existing datasets of any format.
5//
6// Copyright (c) 2010 OPeNDAP, Inc.
7// Author: Michael Johnson <m.johnson@opendap.org>
8//
9// For more information, please also see the main website: http://opendap.org/
10//
11// This library is free software; you can redistribute it and/or
12// modify it under the terms of the GNU Lesser General Public
13// License as published by the Free Software Foundation; either
14// version 2.1 of the License, or (at your option) any later version.
15//
16// This library is distributed in the hope that it will be useful,
17// but WITHOUT ANY WARRANTY; without even the implied warranty of
18// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19// Lesser General Public License for more details.
20//
21// You should have received a copy of the GNU Lesser General Public
22// License along with this library; if not, write to the Free Software
23// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24//
25// Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26//
27// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29
30
31#include <libdap/Array.h> // libdap
32#include <libdap/D4Group.h>
33#include <libdap/Constructor.h>
34#include <libdap/D4Maps.h>
35#include <libdap/InternalErr.h>
36
37#include "BESStopWatch.h"
38
39#include "AggregationUtil.h" // agg_util
40#include "GridAggregationBase.h" // agg_util
41
42#include "NCMLDebug.h"
43
44using libdap::Array;
45using libdap::BaseType;
46using libdap::Grid;
47
48using libdap::D4Group;
49using libdap::Constructor;
50using libdap::InternalErr;
51using libdap::D4Maps;
52using libdap::D4Map;
53
54// Local debug flags
55static const string DEBUG_CHANNEL("agg_util");
56static const bool PRINT_CONSTRAINTS(false);
57
58// Timeouts are now handled in/by the BES framework in BESInterface.
59// jhrg 12/29/15
60#undef USE_LOCAL_TIMEOUT_SCHEME
61
62namespace agg_util {
63GridAggregationBase::GridAggregationBase(const libdap::Grid& proto, const AMDList& memberDatasets,
64 const DDSLoader& loaderProto) :
65 Grid(proto), _loader(loaderProto.getDHI()), _pSubGridProto(cloneSubGridProto(proto)), _memberDatasets(
66 memberDatasets)
67{
68}
69
70GridAggregationBase::GridAggregationBase(const string& name, const AMDList& memberDatasets,
71 const DDSLoader& loaderProto) :
72 Grid(name), _loader(loaderProto.getDHI()), _pSubGridProto(0), _memberDatasets(memberDatasets)
73{
74}
75
76GridAggregationBase::GridAggregationBase(const GridAggregationBase& proto) :
77 Grid(proto), _loader(proto._loader.getDHI()), _pSubGridProto(0) // init below
78 , _memberDatasets()
79{
80 duplicate(proto);
81}
82
83/* virtual */
84GridAggregationBase::~GridAggregationBase()
85{
86 cleanup();
87}
88
89GridAggregationBase&
90GridAggregationBase::operator=(const GridAggregationBase& rhs)
91{
92 if (this != &rhs) {
93 cleanup();
94 Grid::operator=(rhs);
95 duplicate(rhs);
96 }
97 return *this;
98}
99
100
101void
102GridAggregationBase::transform_to_dap4(D4Group *root, Constructor *container)
103{
104 Grid::transform_to_dap4(root,container);
105
106#if 0 // I removed this method because I think the parent class implementation should work correctly.
107 BaseType *btp = array_var()->transform_to_dap4(root, container);
108 Array *coverage = static_cast<Array*>(btp);
109 if (!coverage) throw InternalErr(__FILE__, __LINE__, "Expected an Array while transforming a Grid (coverage)");
110
111 coverage->set_parent(container);
112
113 // Next find the maps; add them to the coverage and to the container,
114 // the latter only on the condition that they are not already there.
115
116 for (Map_iter i = map_begin(), e = map_end(); i != e; ++i) {
117 btp = (*i)->transform_to_dap4(root, container);
118 Array *map = static_cast<Array*>(btp);
119 if (!map) throw InternalErr(__FILE__, __LINE__, "Expected an Array while transforming a Grid (map)");
120
121 // map must be non-null (Grids cannot contain Grids in DAP2)
122 if (map) {
123 // Only add the map/array if it not already present; given the scoping rules
124 // for DAP2 and the assumption the DDS is valid, testing for the same name
125 // is good enough.
126 if (!root->var(map->name())) {
127 map->set_parent(container);
128 container->add_var_nocopy(map); // this adds the array to the container
129 }
130 D4Map *dap4_map = new D4Map(map->name(), map, coverage); // bind the 'map' to the coverage
131 coverage->maps()->add_map(dap4_map); // bind the coverage to the map
132 }
133 else {
134 throw InternalErr(__FILE__, __LINE__,
135 "transform_to_dap4() returned a null value where there can be no Grid.");
136 }
137 }
138
139 container->add_var_nocopy(coverage);
140#endif
141}
142
143void GridAggregationBase::setShapeFrom(const libdap::Grid& constProtoSubGrid, bool addMaps)
144{
145 // calls used are semantically const, but not syntactically.
146 Grid& protoSubGrid = const_cast<Grid&>(constProtoSubGrid);
147
148 // Save a clone of the template for read() to use.
149 // We always use these maps...
150 _pSubGridProto = auto_ptr<Grid>(cloneSubGridProto(protoSubGrid));
151
152 // Pass in the data array and maps from the proto by hand.
153 Array* pDataArrayTemplate = protoSubGrid.get_array();
154 VALID_PTR(pDataArrayTemplate);
155 set_array(static_cast<Array*>(pDataArrayTemplate->ptr_duplicate()));
156
157 // Now the maps in order if asked
158 if (addMaps) {
159 Grid::Map_iter endIt = protoSubGrid.map_end();
160 for (Grid::Map_iter it = protoSubGrid.map_begin(); it != endIt; ++it) {
161 // have to case, the iter is for some reason BaseType*
162 Array* pMap = dynamic_cast<Array*>(*it);
163 VALID_PTR(pMap);
164 add_map(pMap, true); // add as a copy
165 }
166 }
167}
168
169/* virtual */
170const AMDList&
172{
173 return _memberDatasets;
174}
175
176/* virtual */
178{
179 BESDEBUG_FUNC(DEBUG_CHANNEL, "Function entered..." << endl);
180
181 if (read_p()) {
182 BESDEBUG_FUNC(DEBUG_CHANNEL, "read_p() set, early exit!");
183 return true;
184 }
185
186 if (PRINT_CONSTRAINTS) {
187 printConstraints(*(get_array()));
188 }
189
190 // Call the subclass hook methods to do this work properly
192
193 // Now make the read call on the data array.
194 // The aggregation subclass will do the right thing.
195 Array* pAggArray = get_array();
196 VALID_PTR(pAggArray);
197
198 // Only do this portion if the array part is supposed to serialize!
199 if (pAggArray->send_p() || pAggArray->is_in_selection()) {
200 pAggArray->read();
201 }
202
203 // Set the cache bit.
204 set_read_p(true);
205 return true;
206}
207
208#define PIPELINING 1
209
230bool
231GridAggregationBase::serialize(libdap::ConstraintEvaluator &eval, libdap::DDS &dds, libdap::Marshaller &m,
232 bool ce_eval)
233{
234 BESStopWatch sw;
235 if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("GridAggregationBase::serialize", "");
236
237 bool status = false;
238
239 if (!read_p()) {
240 if (PRINT_CONSTRAINTS) {
241 printConstraints(*(get_array()));
242 }
243#if USE_LOCAL_TIMEOUT_SCHEME
244 dds.timeout_on();
245#endif
246 // Call the subclass hook methods to do this work properly
247 // *** Replace Map code readAndAggregateConstrainedMapsHook();
248
249 // Transfers constraints to the proto grid and reads it
251
252 // Make the call to serialize the data array.
253 // The aggregation subclass will do the right thing.
254 Array* pAggArray = get_array();
255 VALID_PTR(pAggArray);
256
257 // Only do this portion if the array part is supposed to serialize!
258 if (pAggArray->send_p() || pAggArray->is_in_selection()) {
259#if PIPELINING
260 pAggArray->serialize(eval, dds, m, ce_eval);
261#else
262 pAggArray->read();
263#endif
264 }
265
266 // Get the read-in, constrained maps from the proto grid and serialize them.
267 // *** Replace copyProtoMapsIntoThisGrid(getAggregationDimension());
268
269 Grid* pSubGridTemplate = getSubGridTemplate();
270 VALID_PTR(pSubGridTemplate);
271
272 Map_iter mapIt;
273 Map_iter mapEndIt = map_end();
274 for (mapIt = map_begin(); mapIt != mapEndIt; ++mapIt) {
275 Array* pOutMap = static_cast<Array*>(*mapIt);
276 VALID_PTR(pOutMap);
277
278 // If it isn't getting dumped, then don't bother with it
279 if (!(pOutMap->send_p() || pOutMap->is_in_selection())) {
280 continue;
281 }
282
283 // We don't want to touch the aggregation dimension since it's
284 // handled specially.
285 if (pOutMap->name() == getAggregationDimension().name) {
286 if (PRINT_CONSTRAINTS) {
287 BESDEBUG_FUNC(DEBUG_CHANNEL,
288 "About to call read() on the map for the new outer dimension name=" << getAggregationDimension().name << " It's constraints are:" << endl);
289 printConstraints(*pOutMap);
290 }
291
292 // Make sure it's read with these constraints.
293#if PIPELINING
294 pOutMap->serialize(eval, dds, m, ce_eval);
295#else
296 pOutMap->read();
297#endif
298 continue;
299 }
300
301 // Otherwise, find the map in the protogrid and copy it's data into this.
302 Array* pProtoGridMap = const_cast<Array*>(AggregationUtil::findMapByName(*pSubGridTemplate, pOutMap->name()));
303 NCML_ASSERT_MSG(pProtoGridMap, "Couldn't find map in prototype grid for map name=" + pOutMap->name());
304 BESDEBUG_FUNC(DEBUG_CHANNEL,
305 "About to call read() on prototype map vector name=" << pOutMap->name() << " and calling transfer constraints..." << endl);
306
307 // Make sure the protogrid maps were properly read
308 NCML_ASSERT_MSG(pProtoGridMap->read_p(), "Expected the prototype map to have been read but it wasn't.");
309
310 // Make sure the lengths match to be sure we're not gonna blow memory up
311 NCML_ASSERT_MSG(pOutMap->length() == pProtoGridMap->length(),
312 "Expected the prototype and output maps to have same length() after transfer of constraints, but they were not so we can't copy the data!");
313
314 // The dimensions will have been set up correctly now so length() is correct...
315 // We assume the pProtoGridMap matches at this point as well.
316 // So we can use this call to copy from one vector to the other
317 // so we don't use temp storage in between
318#if PIPELINING
319 pProtoGridMap->serialize(eval, dds, m, ce_eval);
320#else
321 pOutMap->reserve_value_capacity(); // reserves mem for length
322 pOutMap->set_value_slice_from_row_major_vector(*pProtoGridMap, 0);
323#endif
324 pOutMap->set_read_p(true);
325 }
326
327 // *** End replaced Map code
328#if USE_LOCAL_TIMEOUT_SCHEME
329 dds.timeout_off();
330#endif
331 // Set the cache bit.
332 set_read_p(true);
333
334#if PIPELINING
335 status = true;
336#else
337 status = libdap::Grid::serialize(eval, dds, m, ce_eval);
338#endif
339 }
340 else {
341 status = libdap::Grid::serialize(eval, dds, m, ce_eval);
342 }
343
344 return status;
345}
346
349
350Grid*
352{
353 return _pSubGridProto.get();
354}
355
356void GridAggregationBase::duplicate(const GridAggregationBase& rhs)
357{
358 _loader = DDSLoader(rhs._loader.getDHI());
359
360 std::auto_ptr<Grid> pGridTemplateClone(
361 ((rhs._pSubGridProto.get()) ? (static_cast<Grid*>(rhs._pSubGridProto->ptr_duplicate())) : (0)));
362 _pSubGridProto = pGridTemplateClone;
363
364 _memberDatasets = rhs._memberDatasets;
365}
366
367void GridAggregationBase::cleanup()
368{
369 _loader.cleanup();
370
371 _memberDatasets.clear();
372 _memberDatasets.resize(0);
373}
374
375/* virtual */
377{
378 // Transfers constraints to the proto grid and reads it
380
381 // Copy the read-in, constrained maps from the proto grid
382 // into our output maps.
384}
385
386/* static */
387libdap::Grid*
388GridAggregationBase::cloneSubGridProto(const libdap::Grid& proto)
389{
390 return static_cast<Grid*>(const_cast<Grid&>(proto).ptr_duplicate());
391}
392
393void GridAggregationBase::printConstraints(const Array& fromArray)
394{
395 ostringstream oss;
396 AggregationUtil::printConstraints(oss, fromArray);
397 BESDEBUG("ncml:2", "Constraints for Grid: " << name() << ": " << oss.str() << endl);
398}
399
401{
402 Grid* pSubGridTemplate = getSubGridTemplate();
403 VALID_PTR(pSubGridTemplate);
404
405 // Call the specialized subclass constraint transfer method
406 transferConstraintsToSubGridHook(pSubGridTemplate);
407
408 // Pass it the values for the aggregated grid...
409 pSubGridTemplate->set_send_p(send_p());
410 pSubGridTemplate->set_in_selection(is_in_selection());
411
412 // Those settings will be used by read.
413 pSubGridTemplate->read();
414
415 // For some reason, some handlers only set read_p for the parts, not the whole!!
416 pSubGridTemplate->set_read_p(true);
417}
418
420{
421 Grid* pSubGridTemplate = getSubGridTemplate();
422 VALID_PTR(pSubGridTemplate);
423
424 Map_iter mapIt;
425 Map_iter mapEndIt = map_end();
426 for (mapIt = map_begin(); mapIt != mapEndIt; ++mapIt) {
427 Array* pOutMap = static_cast<Array*>(*mapIt);
428 VALID_PTR(pOutMap);
429
430 // If it isn't getting dumped, then don't bother with it
431 if (!(pOutMap->send_p() || pOutMap->is_in_selection())) {
432 continue;
433 }
434
435 // We don't want to touch the aggregation dimension since it's
436 // handled specially.
437 if (pOutMap->name() == aggDim.name) {
438 if (PRINT_CONSTRAINTS) {
439 BESDEBUG_FUNC(DEBUG_CHANNEL,
440 "About to call read() on the map for the new outer dimension name=" << aggDim.name << " It's constraints are:" << endl);
441 printConstraints(*pOutMap);
442 }
443
444 // Make sure it's read with these constraints.
445 pOutMap->read();
446 continue;
447 }
448
449 // Otherwise, find the map in the protogrid and copy it's data into this.
450 Array* pProtoGridMap = const_cast<Array*>(AggregationUtil::findMapByName(*pSubGridTemplate, pOutMap->name()));
451 NCML_ASSERT_MSG(pProtoGridMap, "Couldn't find map in prototype grid for map name=" + pOutMap->name());
452 BESDEBUG_FUNC(DEBUG_CHANNEL,
453 "About to call read() on prototype map vector name=" << pOutMap->name() << " and calling transfer constraints..." << endl);
454
455 // Make sure the protogrid maps were properly read
456 NCML_ASSERT_MSG(pProtoGridMap->read_p(), "Expected the prototype map to have been read but it wasn't.");
457
458 // Make sure the lengths match to be sure we're not gonna blow memory up
459 NCML_ASSERT_MSG(pOutMap->length() == pProtoGridMap->length(),
460 "Expected the prototype and output maps to have same length() "
461 "after transfer of constraints, but they were not so we can't "
462 "copy the data!");
463
464 // The dimensions will have been set up correctly now so length() is correct...
465 // We assume the pProtoGridMap matches at this point as well.
466 // So we can use this call to copy from one vector to the other
467 // so we don't use temp storage in between
468 pOutMap->reserve_value_capacity(); // reserves mem for length
469 pOutMap->set_value_slice_from_row_major_vector(*pProtoGridMap, 0);
470 pOutMap->set_read_p(true);
471 }
472}
473
474/* virtual */
476{
477 THROW_NCML_INTERNAL_ERROR("Impl me!");
478}
479
480}
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:168
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
static const libdap::Array * findMapByName(const libdap::Grid &inGrid, const std::string &findName)
static void printConstraints(std::ostream &os, const libdap::Array &fromArray)
void cleanup()
restore dhi to clean state
Definition: DDSLoader.cc:256
BESDataHandlerInterface & getDHI() const
Definition: DDSLoader.h:120
virtual const Dimension & getAggregationDimension() const =0
virtual bool serialize(libdap::ConstraintEvaluator &eval, libdap::DDS &dds, libdap::Marshaller &m, bool ce_eval)
virtual void readAndAggregateConstrainedMapsHook()
void copyProtoMapsIntoThisGrid(const Dimension &aggDim)
void setShapeFrom(const libdap::Grid &protoSubGrid, bool addMaps)
virtual const AMDList & getDatasetList() const
virtual void transferConstraintsToSubGridHook(Grid *pSubGrid)
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...