31#include <libdap/Ancillary.h>
32#include <libdap/DMR.h>
33#include <libdap/D4Group.h>
34#include <libdap/DAS.h>
36#include <libdap/InternalErr.h>
37#include <libdap/mime_util.h>
39#include <BESResponseHandler.h>
40#include <BESResponseNames.h>
41#include <BESDapNames.h>
42#include <BESDataNames.h>
43#include <BESDASResponse.h>
44#include <BESDDSResponse.h>
45#include <BESDataDDSResponse.h>
46#include <BESVersionInfo.h>
47#include <BESContainer.h>
48#include <ObjMemCache.h>
50#include <BESDMRResponse.h>
52#include <BESConstraintFuncs.h>
53#include <BESServiceRegistry.h>
56#include <TheBESKeys.h>
58#include <BESDapError.h>
59#include <BESInternalFatalError.h>
61#include <BESStopWatch.h>
64#define PUGIXML_NO_XPATH
65#define PUGIXML_HEADER_ONLY
69#include "DmrppNames.h"
71#include "DmrppTypeFactory.h"
73#include "DmrppRequestHandler.h"
74#include "CurlHandlePool.h"
76#include "CredentialsManager.h"
82#define MODULE_NAME "dmrpp_module"
84#define MODULE_VERSION "unset"
87#define prolog std::string("DmrppRequestHandler::").append(__func__).append("() - ")
89#define USE_DMZ_TO_MANAGE_XML 1
97shared_ptr<DMZ> DmrppRequestHandler::dmz(
nullptr);
101CurlHandlePool *DmrppRequestHandler::curl_handle_pool = 0;
103bool DmrppRequestHandler::d_use_transfer_threads =
true;
104unsigned int DmrppRequestHandler::d_max_transfer_threads = 8;
106bool DmrppRequestHandler::d_use_compute_threads =
true;
107unsigned int DmrppRequestHandler::d_max_compute_threads = 8;
110unsigned long long DmrppRequestHandler::d_contiguous_concurrent_threshold = DMRPP_DEFAULT_CONTIGUOUS_CONCURRENT_THRESHOLD;
115bool DmrppRequestHandler::d_require_chunks =
false;
118bool DmrppRequestHandler::d_emulate_original_filter_order_behavior =
false;
120static void read_key_value(
const std::string &key_name,
bool &key_value)
122 bool key_found =
false;
127 key_value = (value ==
"true" || value ==
"yes");
131static void read_key_value(
const std::string &key_name,
unsigned int &key_value)
133 bool key_found =
false;
137 istringstream iss(value);
141static void read_key_value(
const std::string &key_name,
unsigned long long &key_value)
143 bool key_found =
false;
147 istringstream iss(value);
169 read_key_value(DMRPP_USE_TRANSFER_THREADS_KEY, d_use_transfer_threads);
170 read_key_value(DMRPP_MAX_TRANSFER_THREADS_KEY, d_max_transfer_threads);
171 msg << prolog <<
"Concurrent Transfer Threads: ";
172 if(DmrppRequestHandler::d_use_transfer_threads){
173 msg <<
"Enabled. max_transfer_threads: " << DmrppRequestHandler::d_max_transfer_threads << endl;
176 msg <<
"Disabled." << endl;
179 INFO_LOG(msg.str() );
180 msg.str(std::string());
182 read_key_value(DMRPP_USE_COMPUTE_THREADS_KEY, d_use_compute_threads);
183 read_key_value(DMRPP_MAX_COMPUTE_THREADS_KEY, d_max_compute_threads);
184 msg << prolog <<
"Concurrent Compute Threads: ";
185 if(DmrppRequestHandler::d_use_compute_threads){
186 msg <<
"Enabled. max_compute_threads: " << DmrppRequestHandler::d_max_compute_threads << endl;
189 msg <<
"Disabled." << endl;
192 INFO_LOG(msg.str() );
193 msg.str(std::string());
196 read_key_value(DMRPP_CONTIGUOUS_CONCURRENT_THRESHOLD_KEY, d_contiguous_concurrent_threshold);
197 msg << prolog <<
"Contiguous Concurrency Threshold: " << d_contiguous_concurrent_threshold <<
" bytes." << endl;
198 INFO_LOG(msg.str() );
200#if !HAVE_CURL_MULTI_API
201 if (DmrppRequestHandler::d_use_transfer_threads)
202 ERROR_LOG(
"The DMR++ handler is configured to use parallel transfers, but the libcurl Multi API is not present, defaulting to serial transfers");
207 if (!curl_handle_pool)
215 curl_global_init(CURL_GLOBAL_DEFAULT);
218DmrppRequestHandler::~DmrppRequestHandler()
220 delete curl_handle_pool;
221 curl_global_cleanup();
230handle_exception(
const string &file,
int line)
237 catch (InternalErr &e) {
238 throw BESDapError(e.get_error_message(),
true, e.get_error_code(), file, line);
241 throw BESDapError(e.get_error_message(),
false, e.get_error_code(), file, line);
243 catch (std::exception &e) {
251void DmrppRequestHandler::build_dmr_from_file(
BESContainer *container, DMR* dmr)
253 string data_pathname = container->
access();
255 dmr->set_filename(data_pathname);
256 dmr->set_name(name_path(data_pathname));
258#if USE_DMZ_TO_MANAGE_XML
259 dmz = shared_ptr<DMZ>(
new DMZ);
262 DmrppTypeFactory BaseFactory(dmz);
263 dmr->set_factory(&BaseFactory);
265 dmz->parse_xml_doc(data_pathname);
266 dmz->build_thin_dmr(dmr);
268 dmz->load_all_attributes(dmr);
270 DmrppTypeFactory BaseFactory;
271 dmr->set_factory(&BaseFactory);
273 DmrppParserSax2 parser;
274 ifstream in(data_pathname.c_str(), ios::in);
275 parser.intern(in, dmr);
295 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
299 if (!bdmr)
throw BESInternalError(
"Cast error, expected a BESDMRResponse object.", __FILE__, __LINE__);
302 build_dmr_from_file(dhi.
container, bdmr->get_dmr());
307 handle_exception(__FILE__, __LINE__);
310 BESDEBUG(MODULE, prolog <<
"END" << endl);
325 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
329 if (!bdmr)
throw BESInternalError(
"Cast error, expected a BESDMRResponse object.", __FILE__, __LINE__);
332 build_dmr_from_file(dhi.
container, bdmr->get_dmr());
337 bdmr->set_dap4_constraint(dhi);
338 bdmr->set_dap4_function(dhi);
341 handle_exception(__FILE__, __LINE__);
344 BESDEBUG(MODULE, prolog <<
"END" << endl);
359 DDS *dds = bdds->get_dds();
360 if (!container_name_str.empty()) dds->container_name(container_name_str);
364 DDS *cached_dds_ptr = 0;
365 if (dds_cache && (cached_dds_ptr =
static_cast<DDS*
>(dds_cache->
get(accessed)))) {
366 BESDEBUG(MODULE, prolog <<
"DDS Cached hit for : " << accessed << endl);
367 *dds = *cached_dds_ptr;
371 build_dmr_from_file(dhi.
container, &dmr);
381 dds_cache->
add(
new DDS(*dds), accessed);
394 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
398 if (!bdds)
throw BESInternalError(
"Cast error, expected a BESDataDDSResponse object.", __FILE__, __LINE__);
401 get_dds_from_dmr_or_cache<BESDataDDSResponse>(dhi, bdds);
406 handle_exception(__FILE__, __LINE__);
409 BESDEBUG(MODULE, prolog <<
"END" << endl);
422 BESDEBUG(MODULE, prolog <<
"BEGIN" << endl);
426 if (!bdds)
throw BESInternalError(
"Cast error, expected a BESDDSResponse object.", __FILE__, __LINE__);
429 get_dds_from_dmr_or_cache<BESDDSResponse>(dhi, bdds);
435 handle_exception(__FILE__, __LINE__);
438 BESDEBUG(MODULE, prolog <<
"END" << endl);
453 if (!bdas)
throw BESInternalError(
"Cast error, expected a BESDASResponse object.", __FILE__, __LINE__);
458 DAS *das = bdas->get_das();
459 if (!container_name_str.empty()) das->container_name(container_name_str);
463 DAS *cached_das_ptr = 0;
464 if (das_cache && (cached_das_ptr =
static_cast<DAS*
>(das_cache->
get(accessed)))) {
466 *das = *cached_das_ptr;
470 build_dmr_from_file(dhi.
container, &dmr);
477 unique_ptr<DDS> dds(dmr.getDDS());
481 Ancillary::read_ancillary_das(*das, accessed);
487 das_cache->
add(
new DAS(*das), accessed);
494 handle_exception(__FILE__, __LINE__);
497 BESDEBUG(MODULE, prolog <<
"END" << endl);
507 info->add_module(MODULE_NAME, MODULE_VERSION);
518 map<string, string> attrs;
519 attrs[
"name"] = MODULE_NAME;
520 attrs[
"version"] = MODULE_VERSION;
521 list<string> services;
523 if (services.size() > 0) {
525 attrs[
"handles"] = handles;
527 info->begin_tag(
"module", &attrs);
528 info->end_tag(
"module");
535 strm << BESIndent::LMarg <<
"DmrppRequestHandler::dump - (" << (
void *)
this <<
")" << endl;
538 BESIndent::UnIndent();
A container is something that holds data. E.G., a netcdf file or a database entry.
std::string get_symbolic_name() const
retrieve the symbolic name for this container
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
Holds a DDS object within the BES.
virtual void clear_container()
clear the container in the DAP response object
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
bool get_explicit_containers() const
Should containers be explicitly represented in the DD* responses?
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
BESContainer * container
pointer to current container in this interface
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Abstract exception class for the BES with basic string message.
informational response object
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
Represents a specific data type request handler.
virtual bool add_method(const std::string &name, p_request_handler_method method)
add a handler method to the request handler that knows how to fill in a specific response object
virtual void dump(std::ostream &strm) const
dumps information about this object
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
static std::string lowercase(const std::string &s)
static std::string implode(const std::list< std::string > &values, char delim)
static CredentialsManager * theCM()
Returns the singleton instance of the CrednetialsManager.
An in-memory cache for DapObj (DAS, DDS, ...) objects.
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
static TheBESKeys * TheKeys()
static bool dap_build_dds(BESDataHandlerInterface &dhi)
void dump(std::ostream &strm) const override
dumps information about this object
static bool dap_build_dap2data(BESDataHandlerInterface &dhi)
static bool dap_build_dmr(BESDataHandlerInterface &dhi)
static bool dap_build_das(BESDataHandlerInterface &dhi)
static bool dap_build_dap4data(BESDataHandlerInterface &dhi)
Build a DAP4 data response. Adds timing to dap_build_dmr()
DmrppRequestHandler(const std::string &name)