2 * \file HTTPClient.cxx - simple HTTP client engine for SimHear
5 // Written by James Turner
7 // Copyright (C) 2013 James Turner <zakalawe@mac.com>
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Library General Public
11 // License as published by the Free Software Foundation; either
12 // version 2 of the License, or (at your option) any later version.
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Library General Public License for more details.
19 // You should have received a copy of the GNU General Public License
20 // along with this program; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
25 #include "HTTPClient.hxx"
26 #include "HTTPFileRequest.hxx"
30 #include <cstdlib> // rand()
36 #include <boost/foreach.hpp>
37 #include <boost/algorithm/string/case_conv.hpp>
39 #include <simgear/simgear_config.h>
41 #include <curl/multi.h>
43 #include <simgear/io/sg_netChat.hxx>
45 #include <simgear/misc/strutils.hxx>
46 #include <simgear/compiler.h>
47 #include <simgear/debug/logstream.hxx>
48 #include <simgear/timing/timestamp.hxx>
49 #include <simgear/structure/exception.hxx>
51 #if defined( HAVE_VERSION_H ) && HAVE_VERSION_H
54 # if !defined(SIMGEAR_VERSION)
55 # define SIMGEAR_VERSION "simgear-development"
65 extern const int DEFAULT_HTTP_PORT = 80;
66 const char* CONTENT_TYPE_URL_ENCODED = "application/x-www-form-urlencoded";
69 typedef std::multimap<std::string, Connection*> ConnectionDict;
70 typedef std::list<Request_ptr> RequestList;
72 class Client::ClientPrivate
77 void createCurlMulti()
79 curlMulti = curl_multi_init();
80 // see https://curl.haxx.se/libcurl/c/CURLMOPT_PIPELINING.html
81 // we request HTTP 1.1 pipelining
82 curl_multi_setopt(curlMulti, CURLMOPT_PIPELINING, 1 /* aka CURLPIPE_HTTP1 */);
83 #if (LIBCURL_VERSION_MINOR >= 30)
84 curl_multi_setopt(curlMulti, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long) maxConnections);
85 curl_multi_setopt(curlMulti, CURLMOPT_MAX_PIPELINE_LENGTH,
86 (long) maxPipelineDepth);
87 curl_multi_setopt(curlMulti, CURLMOPT_MAX_HOST_CONNECTIONS,
88 (long) maxHostConnections);
92 typedef std::map<Request_ptr, CURL*> RequestCurlMap;
93 RequestCurlMap requests;
95 std::string userAgent;
98 std::string proxyAuth;
99 unsigned int maxConnections;
100 unsigned int maxHostConnections;
101 unsigned int maxPipelineDepth;
103 RequestList pendingRequests;
105 SGTimeStamp timeTransferSample;
106 unsigned int bytesTransferred;
107 unsigned int lastTransferRate;
108 uint64_t totalBytesDownloaded;
115 d->maxConnections = 4;
116 d->maxHostConnections = 4;
117 d->bytesTransferred = 0;
118 d->lastTransferRate = 0;
119 d->timeTransferSample.stamp();
120 d->totalBytesDownloaded = 0;
121 d->maxPipelineDepth = 5;
122 setUserAgent("SimGear-" SG_STRINGIZE(SIMGEAR_VERSION));
124 static bool didInitCurlGlobal = false;
125 if (!didInitCurlGlobal) {
126 curl_global_init(CURL_GLOBAL_ALL);
127 didInitCurlGlobal = true;
130 d->createCurlMulti();
135 curl_multi_cleanup(d->curlMulti);
138 void Client::setMaxConnections(unsigned int maxCon)
140 d->maxConnections = maxCon;
141 #if (LIBCURL_VERSION_MINOR >= 30)
142 curl_multi_setopt(d->curlMulti, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long) maxCon);
146 void Client::setMaxHostConnections(unsigned int maxHostCon)
148 d->maxHostConnections = maxHostCon;
149 #if (LIBCURL_VERSION_MINOR >= 30)
150 curl_multi_setopt(d->curlMulti, CURLMOPT_MAX_HOST_CONNECTIONS, (long) maxHostCon);
154 void Client::setMaxPipelineDepth(unsigned int depth)
156 d->maxPipelineDepth = depth;
157 #if (LIBCURL_VERSION_MINOR >= 30)
158 curl_multi_setopt(d->curlMulti, CURLMOPT_MAX_PIPELINE_LENGTH, (long) depth);
162 void Client::update(int waitTimeout)
164 int remainingActive, messagesInQueue;
165 curl_multi_perform(d->curlMulti, &remainingActive);
168 while ((msg = curl_multi_info_read(d->curlMulti, &messagesInQueue))) {
169 if (msg->msg == CURLMSG_DONE) {
171 CURL *e = msg->easy_handle;
172 curl_easy_getinfo(e, CURLINFO_PRIVATE, &rawReq);
174 // ensure request stays valid for the moment
175 // eg if responseComplete cancels us
176 Request_ptr req(rawReq);
179 curl_easy_getinfo(e, CURLINFO_RESPONSE_CODE, &responseCode);
181 // remove from the requests map now,
182 // in case the callbacks perform a cancel. We'll use
183 // the absence from the request dict in cancel to avoid
185 ClientPrivate::RequestCurlMap::iterator it = d->requests.find(req);
186 assert(it != d->requests.end());
187 assert(it->second == e);
188 d->requests.erase(it);
190 if (msg->data.result == 0) {
191 req->responseComplete();
193 SG_LOG(SG_IO, SG_WARN, "CURL Result:" << msg->data.result << " " << curl_easy_strerror(msg->data.result));
194 req->setFailure(msg->data.result, curl_easy_strerror(msg->data.result));
197 curl_multi_remove_handle(d->curlMulti, e);
198 curl_easy_cleanup(e);
200 // should never happen since CURLMSG_DONE is the only code
202 SG_LOG(SG_IO, SG_ALERT, "unknown CurlMSG:" << msg->msg);
204 } // of curl message processing loop
205 SGTimeStamp::sleepForMSec(waitTimeout);
208 void Client::makeRequest(const Request_ptr& r)
210 if( r->isComplete() )
213 if( r->url().find("://") == std::string::npos ) {
214 r->setFailure(EINVAL, "malformed URL");
220 ClientPrivate::RequestCurlMap::iterator rit = d->requests.find(r);
221 assert(rit == d->requests.end());
223 CURL* curlRequest = curl_easy_init();
224 curl_easy_setopt(curlRequest, CURLOPT_URL, r->url().c_str());
226 d->requests[r] = curlRequest;
228 curl_easy_setopt(curlRequest, CURLOPT_PRIVATE, r.get());
229 // disable built-in libCurl progress feedback
230 curl_easy_setopt(curlRequest, CURLOPT_NOPROGRESS, 1);
232 curl_easy_setopt(curlRequest, CURLOPT_WRITEFUNCTION, requestWriteCallback);
233 curl_easy_setopt(curlRequest, CURLOPT_WRITEDATA, r.get());
234 curl_easy_setopt(curlRequest, CURLOPT_HEADERFUNCTION, requestHeaderCallback);
235 curl_easy_setopt(curlRequest, CURLOPT_HEADERDATA, r.get());
237 curl_easy_setopt(curlRequest, CURLOPT_USERAGENT, d->userAgent.c_str());
238 curl_easy_setopt(curlRequest, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
240 if (!d->proxy.empty()) {
241 curl_easy_setopt(curlRequest, CURLOPT_PROXY, d->proxy.c_str());
242 curl_easy_setopt(curlRequest, CURLOPT_PROXYPORT, d->proxyPort);
244 if (!d->proxyAuth.empty()) {
245 curl_easy_setopt(curlRequest, CURLOPT_PROXYAUTH, CURLAUTH_BASIC);
246 curl_easy_setopt(curlRequest, CURLOPT_PROXYUSERPWD, d->proxyAuth.c_str());
250 std::string method = boost::to_lower_copy(r->method());
251 if (method == "get") {
252 curl_easy_setopt(curlRequest, CURLOPT_HTTPGET, 1);
253 } else if (method == "put") {
254 curl_easy_setopt(curlRequest, CURLOPT_PUT, 1);
255 curl_easy_setopt(curlRequest, CURLOPT_UPLOAD, 1);
256 } else if (method == "post") {
257 // see http://curl.haxx.se/libcurl/c/CURLOPT_POST.html
258 curl_easy_setopt(curlRequest, CURLOPT_HTTPPOST, 1);
260 std::string q = r->query().substr(1);
261 curl_easy_setopt(curlRequest, CURLOPT_COPYPOSTFIELDS, q.c_str());
263 // reset URL to exclude query pieces
264 std::string urlWithoutQuery = r->url();
265 std::string::size_type queryPos = urlWithoutQuery.find('?');
266 urlWithoutQuery.resize(queryPos);
267 curl_easy_setopt(curlRequest, CURLOPT_URL, urlWithoutQuery.c_str());
269 curl_easy_setopt(curlRequest, CURLOPT_CUSTOMREQUEST, r->method().c_str());
272 struct curl_slist* headerList = NULL;
273 if (r->hasBodyData() && (method != "post")) {
274 curl_easy_setopt(curlRequest, CURLOPT_UPLOAD, 1);
275 curl_easy_setopt(curlRequest, CURLOPT_INFILESIZE, r->bodyLength());
276 curl_easy_setopt(curlRequest, CURLOPT_READFUNCTION, requestReadCallback);
277 curl_easy_setopt(curlRequest, CURLOPT_READDATA, r.get());
278 std::string h = "Content-Type:" + r->bodyType();
279 headerList = curl_slist_append(headerList, h.c_str());
282 StringMap::const_iterator it;
283 for (it = r->requestHeaders().begin(); it != r->requestHeaders().end(); ++it) {
284 std::string h = it->first + ": " + it->second;
285 headerList = curl_slist_append(headerList, h.c_str());
288 if (headerList != NULL) {
289 curl_easy_setopt(curlRequest, CURLOPT_HTTPHEADER, headerList);
292 curl_multi_add_handle(d->curlMulti, curlRequest);
294 // this seems premature, but we don't have a callback from Curl we could
295 // use to trigger when the requst is actually sent.
299 void Client::cancelRequest(const Request_ptr &r, std::string reason)
301 ClientPrivate::RequestCurlMap::iterator it = d->requests.find(r);
302 if(it == d->requests.end()) {
303 // already being removed, presumably inside ::update()
304 // nothing more to do
308 CURLMcode err = curl_multi_remove_handle(d->curlMulti, it->second);
309 assert(err == CURLM_OK);
311 // clear the request pointer form the curl-easy object
312 curl_easy_setopt(it->second, CURLOPT_PRIVATE, 0);
314 curl_easy_cleanup(it->second);
315 d->requests.erase(it);
317 r->setFailure(-1, reason);
320 //------------------------------------------------------------------------------
321 FileRequestRef Client::save( const std::string& url,
322 const std::string& filename )
324 FileRequestRef req = new FileRequest(url, filename);
329 //------------------------------------------------------------------------------
330 MemoryRequestRef Client::load(const std::string& url)
332 MemoryRequestRef req = new MemoryRequest(url);
337 void Client::requestFinished(Connection* con)
342 void Client::setUserAgent(const std::string& ua)
347 const std::string& Client::userAgent() const
352 const std::string& Client::proxyHost() const
357 const std::string& Client::proxyAuth() const
362 void Client::setProxy( const std::string& proxy,
364 const std::string& auth )
371 bool Client::hasActiveRequests() const
373 return !d->requests.empty();
376 void Client::receivedBytes(unsigned int count)
378 d->bytesTransferred += count;
379 d->totalBytesDownloaded += count;
382 unsigned int Client::transferRateBytesPerSec() const
384 unsigned int e = d->timeTransferSample.elapsedMSec();
386 // too long a window, ignore
387 d->timeTransferSample.stamp();
388 d->bytesTransferred = 0;
389 d->lastTransferRate = 0;
393 if (e < 100) { // avoid really narrow windows
394 return d->lastTransferRate;
397 unsigned int ratio = (d->bytesTransferred * 1000) / e;
398 // run a low-pass filter
399 unsigned int smoothed = ((400 - e) * d->lastTransferRate) + (e * ratio);
402 d->timeTransferSample.stamp();
403 d->bytesTransferred = 0;
404 d->lastTransferRate = smoothed;
408 uint64_t Client::totalBytesDownloaded() const
410 return d->totalBytesDownloaded;
413 size_t Client::requestWriteCallback(char *ptr, size_t size, size_t nmemb, void *userdata)
415 size_t byteSize = size * nmemb;
416 Request* req = static_cast<Request*>(userdata);
417 req->processBodyBytes(ptr, byteSize);
419 Client* cl = req->http();
421 cl->receivedBytes(byteSize);
427 size_t Client::requestReadCallback(char *ptr, size_t size, size_t nmemb, void *userdata)
429 size_t maxBytes = size * nmemb;
430 Request* req = static_cast<Request*>(userdata);
431 size_t actualBytes = req->getBodyData(ptr, 0, maxBytes);
435 size_t Client::requestHeaderCallback(char *rawBuffer, size_t size, size_t nitems, void *userdata)
437 size_t byteSize = size * nitems;
438 Request* req = static_cast<Request*>(userdata);
439 std::string h = strutils::simplify(std::string(rawBuffer, byteSize));
441 if (req->readyState() == HTTP::Request::OPENED) {
442 req->responseStart(h);
447 // got a 100-continue reponse; restart
448 if (req->responseCode() == 100) {
449 req->setReadyState(HTTP::Request::OPENED);
453 req->responseHeadersComplete();
457 if (req->responseCode() == 100) {
458 return byteSize; // skip headers associated with 100-continue status
461 size_t colonPos = h.find(':');
462 if (colonPos == std::string::npos) {
463 SG_LOG(SG_IO, SG_WARN, "malformed HTTP response header:" << h);
467 std::string key = strutils::simplify(h.substr(0, colonPos));
468 std::string lkey = boost::to_lower_copy(key);
469 std::string value = strutils::strip(h.substr(colonPos + 1));
471 req->responseHeader(lkey, value);
475 void Client::debugDumpRequests()
477 SG_LOG(SG_IO, SG_INFO, "== HTTP request dump");
478 ClientPrivate::RequestCurlMap::iterator it = d->requests.begin();
479 for (; it != d->requests.end(); ++it) {
480 SG_LOG(SG_IO, SG_INFO, "\t" << it->first->url());
482 SG_LOG(SG_IO, SG_INFO, "==");
485 void Client::clearAllConnections()
487 curl_multi_cleanup(d->curlMulti);
488 d->createCurlMulti();
491 } // of namespace HTTP
493 } // of namespace simgear