simgear/io/HTTPClient.cxx

   1 /**
   2  * \file HTTPClient.cxx - simple HTTP client engine for SimHear
   3  */
   4
   5 // Written by James Turner
   6 //
   7 // Copyright (C) 2013  James Turner  <zakalawe@mac.com>
   8 //
   9 // This library is free software; you can redistribute it and/or
  10 // modify it under the terms of the GNU Library General Public
  11 // License as published by the Free Software Foundation; either
  12 // version 2 of the License, or (at your option) any later version.
  13 //
  14 // This library is distributed in the hope that it will be useful,
  15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17 // Library General Public License for more details.
  18 //
  19 // You should have received a copy of the GNU General Public License
  20 // along with this program; if not, write to the Free Software
  21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  22 //
  23
  24
  25 #include "HTTPClient.hxx"
  26 #include "HTTPFileRequest.hxx"
  27
  28 #include <sstream>
  29 #include <cassert>
  30 #include <cstdlib> // rand()
  31 #include <list>
  32 #include <errno.h>
  33 #include <map>
  34 #include <stdexcept>
  35
  36 #include <boost/foreach.hpp>
  37 #include <boost/algorithm/string/case_conv.hpp>
  38
  39 #include <simgear/simgear_config.h>
  40
  41 #include <curl/multi.h>
  42
  43 #include <simgear/io/sg_netChat.hxx>
  44
  45 #include <simgear/misc/strutils.hxx>
  46 #include <simgear/compiler.h>
  47 #include <simgear/debug/logstream.hxx>
  48 #include <simgear/timing/timestamp.hxx>
  49 #include <simgear/structure/exception.hxx>
  50
  51 #if defined( HAVE_VERSION_H ) && HAVE_VERSION_H
  52 #include "version.h"
  53 #else
  54 #  if !defined(SIMGEAR_VERSION)
  55 #    define SIMGEAR_VERSION "simgear-development"
  56 #  endif
  57 #endif
  58
  59 namespace simgear
  60 {
  61
  62 namespace HTTP
  63 {
  64
  65 extern const int DEFAULT_HTTP_PORT = 80;
  66 const char* CONTENT_TYPE_URL_ENCODED = "application/x-www-form-urlencoded";
  67
  68 class Connection;
  69 typedef std::multimap<std::string, Connection*> ConnectionDict;
  70 typedef std::list<Request_ptr> RequestList;
  71
  72 class Client::ClientPrivate
  73 {
  74 public:
  75     CURLM* curlMulti;
  76
  77     void createCurlMulti()
  78     {
  79         curlMulti = curl_multi_init();
  80         // see https://curl.haxx.se/libcurl/c/CURLMOPT_PIPELINING.html
  81         // we request HTTP 1.1 pipelining
  82         curl_multi_setopt(curlMulti, CURLMOPT_PIPELINING, 1 /* aka CURLPIPE_HTTP1 */);
  83 #if (LIBCURL_VERSION_MINOR >= 30)
  84         curl_multi_setopt(curlMulti, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long) maxConnections);
  85 #endif
  86         curl_multi_setopt(curlMulti, CURLMOPT_MAX_PIPELINE_LENGTH,
  87                           (long) maxPipelineDepth);
  88         curl_multi_setopt(curlMulti, CURLMOPT_MAX_HOST_CONNECTIONS,
  89                           (long) maxHostConnections);
  90
  91
  92     }
  93
  94     typedef std::map<Request_ptr, CURL*> RequestCurlMap;
  95     RequestCurlMap requests;
  96
  97     std::string userAgent;
  98     std::string proxy;
  99     int proxyPort;
 100     std::string proxyAuth;
 101     unsigned int maxConnections;
 102     unsigned int maxHostConnections;
 103     unsigned int maxPipelineDepth;
 104
 105     RequestList pendingRequests;
 106
 107     SGTimeStamp timeTransferSample;
 108     unsigned int bytesTransferred;
 109     unsigned int lastTransferRate;
 110     uint64_t totalBytesDownloaded;
 111 };
 112
 113 Client::Client() :
 114     d(new ClientPrivate)
 115 {
 116     d->proxyPort = 0;
 117     d->maxConnections = 4;
 118     d->maxHostConnections = 4;
 119     d->bytesTransferred = 0;
 120     d->lastTransferRate = 0;
 121     d->timeTransferSample.stamp();
 122     d->totalBytesDownloaded = 0;
 123     d->maxPipelineDepth = 5;
 124     setUserAgent("SimGear-" SG_STRINGIZE(SIMGEAR_VERSION));
 125
 126     static bool didInitCurlGlobal = false;
 127     if (!didInitCurlGlobal) {
 128       curl_global_init(CURL_GLOBAL_ALL);
 129       didInitCurlGlobal = true;
 130     }
 131
 132     d->createCurlMulti();
 133 }
 134
 135 Client::~Client()
 136 {
 137   curl_multi_cleanup(d->curlMulti);
 138 }
 139
 140 void Client::setMaxConnections(unsigned int maxCon)
 141 {
 142     d->maxConnections = maxCon;
 143 #if (LIBCURL_VERSION_MINOR >= 30)
 144     curl_multi_setopt(d->curlMulti, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long) maxCon);
 145 #endif
 146 }
 147
 148 void Client::setMaxHostConnections(unsigned int maxHostCon)
 149 {
 150     d->maxHostConnections = maxHostCon;
 151     curl_multi_setopt(d->curlMulti, CURLMOPT_MAX_HOST_CONNECTIONS, (long) maxHostCon);
 152 }
 153
 154 void Client::setMaxPipelineDepth(unsigned int depth)
 155 {
 156     d->maxPipelineDepth = depth;
 157     curl_multi_setopt(d->curlMulti, CURLMOPT_MAX_PIPELINE_LENGTH, (long) depth);
 158 }
 159
 160 void Client::update(int waitTimeout)
 161 {
 162     int remainingActive, messagesInQueue;
 163     curl_multi_perform(d->curlMulti, &remainingActive);
 164
 165     CURLMsg* msg;
 166     while ((msg = curl_multi_info_read(d->curlMulti, &messagesInQueue))) {
 167       if (msg->msg == CURLMSG_DONE) {
 168         Request* rawReq = 0;
 169         CURL *e = msg->easy_handle;
 170         curl_easy_getinfo(e, CURLINFO_PRIVATE, &rawReq);
 171
 172         // ensure request stays valid for the moment
 173         // eg if responseComplete cancels us
 174         Request_ptr req(rawReq);
 175
 176         long responseCode;
 177         curl_easy_getinfo(e, CURLINFO_RESPONSE_CODE, &responseCode);
 178
 179           // remove from the requests map now,
 180           // in case the callbacks perform a cancel. We'll use
 181           // the absence from the request dict in cancel to avoid
 182           // a double remove
 183           ClientPrivate::RequestCurlMap::iterator it = d->requests.find(req);
 184           assert(it != d->requests.end());
 185           assert(it->second == e);
 186           d->requests.erase(it);
 187
 188         if (msg->data.result == 0) {
 189           req->responseComplete();
 190         } else {
 191           SG_LOG(SG_IO, SG_WARN, "CURL Result:" << msg->data.result << " " << curl_easy_strerror(msg->data.result));
 192           req->setFailure(msg->data.result, curl_easy_strerror(msg->data.result));
 193         }
 194
 195         curl_multi_remove_handle(d->curlMulti, e);
 196         curl_easy_cleanup(e);
 197       } else {
 198           // should never happen since CURLMSG_DONE is the only code
 199           // defined!
 200           SG_LOG(SG_IO, SG_ALERT, "unknown CurlMSG:" << msg->msg);
 201       }
 202     } // of curl message processing loop
 203     SGTimeStamp::sleepForMSec(waitTimeout);
 204 }
 205
 206 void Client::makeRequest(const Request_ptr& r)
 207 {
 208     if( r->isComplete() )
 209       return;
 210
 211     if( r->url().find("://") == std::string::npos ) {
 212         r->setFailure(EINVAL, "malformed URL");
 213         return;
 214     }
 215
 216     r->_client = this;
 217
 218     ClientPrivate::RequestCurlMap::iterator rit = d->requests.find(r);
 219     assert(rit == d->requests.end());
 220
 221     CURL* curlRequest = curl_easy_init();
 222     curl_easy_setopt(curlRequest, CURLOPT_URL, r->url().c_str());
 223
 224     d->requests[r] = curlRequest;
 225
 226     curl_easy_setopt(curlRequest, CURLOPT_PRIVATE, r.get());
 227     // disable built-in libCurl progress feedback
 228     curl_easy_setopt(curlRequest, CURLOPT_NOPROGRESS, 1);
 229
 230     curl_easy_setopt(curlRequest, CURLOPT_WRITEFUNCTION, requestWriteCallback);
 231     curl_easy_setopt(curlRequest, CURLOPT_WRITEDATA, r.get());
 232     curl_easy_setopt(curlRequest, CURLOPT_HEADERFUNCTION, requestHeaderCallback);
 233     curl_easy_setopt(curlRequest, CURLOPT_HEADERDATA, r.get());
 234
 235     curl_easy_setopt(curlRequest, CURLOPT_USERAGENT, d->userAgent.c_str());
 236     curl_easy_setopt(curlRequest, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
 237
 238     if (!d->proxy.empty()) {
 239       curl_easy_setopt(curlRequest, CURLOPT_PROXY, d->proxy.c_str());
 240       curl_easy_setopt(curlRequest, CURLOPT_PROXYPORT, d->proxyPort);
 241
 242       if (!d->proxyAuth.empty()) {
 243         curl_easy_setopt(curlRequest, CURLOPT_PROXYAUTH, CURLAUTH_BASIC);
 244         curl_easy_setopt(curlRequest, CURLOPT_PROXYUSERPWD, d->proxyAuth.c_str());
 245       }
 246     }
 247
 248     std::string method = boost::to_lower_copy(r->method());
 249     if (method == "get") {
 250       curl_easy_setopt(curlRequest, CURLOPT_HTTPGET, 1);
 251     } else if (method == "put") {
 252       curl_easy_setopt(curlRequest, CURLOPT_PUT, 1);
 253       curl_easy_setopt(curlRequest, CURLOPT_UPLOAD, 1);
 254     } else if (method == "post") {
 255       // see http://curl.haxx.se/libcurl/c/CURLOPT_POST.html
 256       curl_easy_setopt(curlRequest, CURLOPT_HTTPPOST, 1);
 257
 258       std::string q = r->query().substr(1);
 259       curl_easy_setopt(curlRequest, CURLOPT_COPYPOSTFIELDS, q.c_str());
 260
 261       // reset URL to exclude query pieces
 262       std::string urlWithoutQuery = r->url();
 263       std::string::size_type queryPos = urlWithoutQuery.find('?');
 264       urlWithoutQuery.resize(queryPos);
 265       curl_easy_setopt(curlRequest, CURLOPT_URL, urlWithoutQuery.c_str());
 266     } else {
 267       curl_easy_setopt(curlRequest, CURLOPT_CUSTOMREQUEST, r->method().c_str());
 268     }
 269
 270     struct curl_slist* headerList = NULL;
 271     if (r->hasBodyData() && (method != "post")) {
 272       curl_easy_setopt(curlRequest, CURLOPT_UPLOAD, 1);
 273       curl_easy_setopt(curlRequest, CURLOPT_INFILESIZE, r->bodyLength());
 274       curl_easy_setopt(curlRequest, CURLOPT_READFUNCTION, requestReadCallback);
 275       curl_easy_setopt(curlRequest, CURLOPT_READDATA, r.get());
 276       std::string h = "Content-Type:" + r->bodyType();
 277       headerList = curl_slist_append(headerList, h.c_str());
 278     }
 279
 280     StringMap::const_iterator it;
 281     for (it = r->requestHeaders().begin(); it != r->requestHeaders().end(); ++it) {
 282       std::string h = it->first + ": " + it->second;
 283       headerList = curl_slist_append(headerList, h.c_str());
 284     }
 285
 286     if (headerList != NULL) {
 287       curl_easy_setopt(curlRequest, CURLOPT_HTTPHEADER, headerList);
 288     }
 289
 290     curl_multi_add_handle(d->curlMulti, curlRequest);
 291
 292 // this seems premature, but we don't have a callback from Curl we could
 293 // use to trigger when the requst is actually sent.
 294     r->requestStart();
 295 }
 296
 297 void Client::cancelRequest(const Request_ptr &r, std::string reason)
 298 {
 299     ClientPrivate::RequestCurlMap::iterator it = d->requests.find(r);
 300     if(it == d->requests.end()) {
 301         // already being removed, presumably inside ::update()
 302         // nothing more to do
 303         return;
 304     }
 305
 306     CURLMcode err = curl_multi_remove_handle(d->curlMulti, it->second);
 307     assert(err == CURLM_OK);
 308
 309     // clear the request pointer form the curl-easy object
 310     curl_easy_setopt(it->second, CURLOPT_PRIVATE, 0);
 311
 312     curl_easy_cleanup(it->second);
 313     d->requests.erase(it);
 314
 315     r->setFailure(-1, reason);
 316 }
 317
 318 //------------------------------------------------------------------------------
 319 FileRequestRef Client::save( const std::string& url,
 320                              const std::string& filename )
 321 {
 322   FileRequestRef req = new FileRequest(url, filename);
 323   makeRequest(req);
 324   return req;
 325 }
 326
 327 //------------------------------------------------------------------------------
 328 MemoryRequestRef Client::load(const std::string& url)
 329 {
 330   MemoryRequestRef req = new MemoryRequest(url);
 331   makeRequest(req);
 332   return req;
 333 }
 334
 335 void Client::requestFinished(Connection* con)
 336 {
 337
 338 }
 339
 340 void Client::setUserAgent(const std::string& ua)
 341 {
 342     d->userAgent = ua;
 343 }
 344
 345 const std::string& Client::userAgent() const
 346 {
 347     return d->userAgent;
 348 }
 349
 350 const std::string& Client::proxyHost() const
 351 {
 352     return d->proxy;
 353 }
 354
 355 const std::string& Client::proxyAuth() const
 356 {
 357     return d->proxyAuth;
 358 }
 359
 360 void Client::setProxy( const std::string& proxy,
 361                        int port,
 362                        const std::string& auth )
 363 {
 364     d->proxy = proxy;
 365     d->proxyPort = port;
 366     d->proxyAuth = auth;
 367 }
 368
 369 bool Client::hasActiveRequests() const
 370 {
 371     return !d->requests.empty();
 372 }
 373
 374 void Client::receivedBytes(unsigned int count)
 375 {
 376     d->bytesTransferred += count;
 377     d->totalBytesDownloaded += count;
 378 }
 379
 380 unsigned int Client::transferRateBytesPerSec() const
 381 {
 382     unsigned int e = d->timeTransferSample.elapsedMSec();
 383     if (e > 400) {
 384         // too long a window, ignore
 385         d->timeTransferSample.stamp();
 386         d->bytesTransferred = 0;
 387         d->lastTransferRate = 0;
 388         return 0;
 389     }
 390
 391     if (e < 100) { // avoid really narrow windows
 392         return d->lastTransferRate;
 393     }
 394
 395     unsigned int ratio = (d->bytesTransferred * 1000) / e;
 396     // run a low-pass filter
 397     unsigned int smoothed = ((400 - e) * d->lastTransferRate) + (e * ratio);
 398     smoothed /= 400;
 399
 400     d->timeTransferSample.stamp();
 401     d->bytesTransferred = 0;
 402     d->lastTransferRate = smoothed;
 403     return smoothed;
 404 }
 405
 406 uint64_t Client::totalBytesDownloaded() const
 407 {
 408     return d->totalBytesDownloaded;
 409 }
 410
 411 size_t Client::requestWriteCallback(char *ptr, size_t size, size_t nmemb, void *userdata)
 412 {
 413   size_t byteSize = size * nmemb;
 414   Request* req = static_cast<Request*>(userdata);
 415   req->processBodyBytes(ptr, byteSize);
 416
 417   Client* cl = req->http();
 418   if (cl) {
 419     cl->receivedBytes(byteSize);
 420   }
 421
 422   return byteSize;
 423 }
 424
 425 size_t Client::requestReadCallback(char *ptr, size_t size, size_t nmemb, void *userdata)
 426 {
 427   size_t maxBytes = size * nmemb;
 428   Request* req = static_cast<Request*>(userdata);
 429   size_t actualBytes = req->getBodyData(ptr, 0, maxBytes);
 430   return actualBytes;
 431 }
 432
 433 size_t Client::requestHeaderCallback(char *rawBuffer, size_t size, size_t nitems, void *userdata)
 434 {
 435   size_t byteSize = size * nitems;
 436   Request* req = static_cast<Request*>(userdata);
 437   std::string h = strutils::simplify(std::string(rawBuffer, byteSize));
 438
 439   if (req->readyState() == HTTP::Request::OPENED) {
 440     req->responseStart(h);
 441     return byteSize;
 442   }
 443
 444   if (h.empty()) {
 445       // got a 100-continue reponse; restart
 446       if (req->responseCode() == 100) {
 447           req->setReadyState(HTTP::Request::OPENED);
 448           return byteSize;
 449       }
 450
 451     req->responseHeadersComplete();
 452     return byteSize;
 453   }
 454
 455   if (req->responseCode() == 100) {
 456       return byteSize; // skip headers associated with 100-continue status
 457   }
 458
 459   size_t colonPos = h.find(':');
 460   if (colonPos == std::string::npos) {
 461       SG_LOG(SG_IO, SG_WARN, "malformed HTTP response header:" << h);
 462       return byteSize;
 463   }
 464
 465   std::string key = strutils::simplify(h.substr(0, colonPos));
 466   std::string lkey = boost::to_lower_copy(key);
 467   std::string value = strutils::strip(h.substr(colonPos + 1));
 468
 469   req->responseHeader(lkey, value);
 470   return byteSize;
 471 }
 472
 473 void Client::debugDumpRequests()
 474 {
 475     SG_LOG(SG_IO, SG_INFO, "== HTTP request dump");
 476     ClientPrivate::RequestCurlMap::iterator it = d->requests.begin();
 477     for (; it != d->requests.end(); ++it) {
 478         SG_LOG(SG_IO, SG_INFO, "\t" << it->first->url());
 479     }
 480     SG_LOG(SG_IO, SG_INFO, "==");
 481 }
 482
 483 void Client::clearAllConnections()
 484 {
 485     curl_multi_cleanup(d->curlMulti);
 486     d->createCurlMulti();
 487 }
 488
 489 } // of namespace HTTP
 490
 491 } // of namespace simgear