simgear/io/HTTPClient.cxx

   1 /**
   2  * \file HTTPClient.cxx - simple HTTP client engine for SimHear
   3  */
   4
   5 // Written by James Turner
   6 //
   7 // Copyright (C) 2013  James Turner  <zakalawe@mac.com>
   8 //
   9 // This library is free software; you can redistribute it and/or
  10 // modify it under the terms of the GNU Library General Public
  11 // License as published by the Free Software Foundation; either
  12 // version 2 of the License, or (at your option) any later version.
  13 //
  14 // This library is distributed in the hope that it will be useful,
  15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17 // Library General Public License for more details.
  18 //
  19 // You should have received a copy of the GNU General Public License
  20 // along with this program; if not, write to the Free Software
  21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  22 //
  23
  24
  25 #include "HTTPClient.hxx"
  26 #include "HTTPFileRequest.hxx"
  27
  28 #include <sstream>
  29 #include <cassert>
  30 #include <cstdlib> // rand()
  31 #include <list>
  32 #include <errno.h>
  33 #include <map>
  34 #include <stdexcept>
  35
  36 #include <boost/foreach.hpp>
  37 #include <boost/algorithm/string/case_conv.hpp>
  38
  39 #include <simgear/simgear_config.h>
  40
  41 #include <curl/multi.h>
  42
  43 #include <simgear/io/sg_netChat.hxx>
  44
  45 #include <simgear/misc/strutils.hxx>
  46 #include <simgear/compiler.h>
  47 #include <simgear/debug/logstream.hxx>
  48 #include <simgear/timing/timestamp.hxx>
  49 #include <simgear/structure/exception.hxx>
  50
  51 #if defined( HAVE_VERSION_H ) && HAVE_VERSION_H
  52 #include "version.h"
  53 #else
  54 #  if !defined(SIMGEAR_VERSION)
  55 #    define SIMGEAR_VERSION "simgear-development"
  56 #  endif
  57 #endif
  58
  59 namespace simgear
  60 {
  61
  62 namespace HTTP
  63 {
  64
  65 extern const int DEFAULT_HTTP_PORT = 80;
  66 const char* CONTENT_TYPE_URL_ENCODED = "application/x-www-form-urlencoded";
  67
  68 class Connection;
  69 typedef std::multimap<std::string, Connection*> ConnectionDict;
  70 typedef std::list<Request_ptr> RequestList;
  71
  72 class Client::ClientPrivate
  73 {
  74 public:
  75     CURLM* curlMulti;
  76
  77     void createCurlMulti()
  78     {
  79         curlMulti = curl_multi_init();
  80         // see https://curl.haxx.se/libcurl/c/CURLMOPT_PIPELINING.html
  81         // we request HTTP 1.1 pipelining
  82         curl_multi_setopt(curlMulti, CURLMOPT_PIPELINING, 1 /* aka CURLPIPE_HTTP1 */);
  83 #if (LIBCURL_VERSION_MINOR >= 30)
  84         curl_multi_setopt(curlMulti, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long) maxConnections);
  85         curl_multi_setopt(curlMulti, CURLMOPT_MAX_PIPELINE_LENGTH,
  86                           (long) maxPipelineDepth);
  87         curl_multi_setopt(curlMulti, CURLMOPT_MAX_HOST_CONNECTIONS,
  88                           (long) maxHostConnections);
  89 #endif
  90     }
  91
  92     typedef std::map<Request_ptr, CURL*> RequestCurlMap;
  93     RequestCurlMap requests;
  94
  95     std::string userAgent;
  96     std::string proxy;
  97     int proxyPort;
  98     std::string proxyAuth;
  99     unsigned int maxConnections;
 100     unsigned int maxHostConnections;
 101     unsigned int maxPipelineDepth;
 102
 103     RequestList pendingRequests;
 104
 105     SGTimeStamp timeTransferSample;
 106     unsigned int bytesTransferred;
 107     unsigned int lastTransferRate;
 108     uint64_t totalBytesDownloaded;
 109 };
 110
 111 Client::Client() :
 112     d(new ClientPrivate)
 113 {
 114     d->proxyPort = 0;
 115     d->maxConnections = 4;
 116     d->maxHostConnections = 4;
 117     d->bytesTransferred = 0;
 118     d->lastTransferRate = 0;
 119     d->timeTransferSample.stamp();
 120     d->totalBytesDownloaded = 0;
 121     d->maxPipelineDepth = 5;
 122     setUserAgent("SimGear-" SG_STRINGIZE(SIMGEAR_VERSION));
 123
 124     static bool didInitCurlGlobal = false;
 125     if (!didInitCurlGlobal) {
 126       curl_global_init(CURL_GLOBAL_ALL);
 127       didInitCurlGlobal = true;
 128     }
 129
 130     d->createCurlMulti();
 131 }
 132
 133 Client::~Client()
 134 {
 135   curl_multi_cleanup(d->curlMulti);
 136 }
 137
 138 void Client::setMaxConnections(unsigned int maxCon)
 139 {
 140     d->maxConnections = maxCon;
 141 #if (LIBCURL_VERSION_MINOR >= 30)
 142     curl_multi_setopt(d->curlMulti, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long) maxCon);
 143 #endif
 144 }
 145
 146 void Client::setMaxHostConnections(unsigned int maxHostCon)
 147 {
 148     d->maxHostConnections = maxHostCon;
 149 #if (LIBCURL_VERSION_MINOR >= 30)
 150     curl_multi_setopt(d->curlMulti, CURLMOPT_MAX_HOST_CONNECTIONS, (long) maxHostCon);
 151 #endif
 152 }
 153
 154 void Client::setMaxPipelineDepth(unsigned int depth)
 155 {
 156     d->maxPipelineDepth = depth;
 157 #if (LIBCURL_VERSION_MINOR >= 30)
 158     curl_multi_setopt(d->curlMulti, CURLMOPT_MAX_PIPELINE_LENGTH, (long) depth);
 159 #endif
 160 }
 161
 162 void Client::update(int waitTimeout)
 163 {
 164     if (d->requests.empty()) {
 165         // curl_multi_wait returns immediately if there's no requests active,
 166         // but that can cause high CPU usage for us.
 167         SGTimeStamp::sleepForMSec(waitTimeout);
 168         return;
 169     }
 170
 171     int remainingActive, messagesInQueue, numFds;
 172     curl_multi_wait(d->curlMulti, NULL, 0, waitTimeout, &numFds);
 173     curl_multi_perform(d->curlMulti, &remainingActive);
 174
 175     CURLMsg* msg;
 176     while ((msg = curl_multi_info_read(d->curlMulti, &messagesInQueue))) {
 177       if (msg->msg == CURLMSG_DONE) {
 178         Request* rawReq = 0;
 179         CURL *e = msg->easy_handle;
 180         curl_easy_getinfo(e, CURLINFO_PRIVATE, &rawReq);
 181
 182         // ensure request stays valid for the moment
 183         // eg if responseComplete cancels us
 184         Request_ptr req(rawReq);
 185
 186         long responseCode;
 187         curl_easy_getinfo(e, CURLINFO_RESPONSE_CODE, &responseCode);
 188
 189           // remove from the requests map now,
 190           // in case the callbacks perform a cancel. We'll use
 191           // the absence from the request dict in cancel to avoid
 192           // a double remove
 193           ClientPrivate::RequestCurlMap::iterator it = d->requests.find(req);
 194           assert(it != d->requests.end());
 195           assert(it->second == e);
 196           d->requests.erase(it);
 197
 198         if (msg->data.result == 0) {
 199           req->responseComplete();
 200         } else {
 201           SG_LOG(SG_IO, SG_WARN, "CURL Result:" << msg->data.result << " " << curl_easy_strerror(msg->data.result));
 202           req->setFailure(msg->data.result, curl_easy_strerror(msg->data.result));
 203         }
 204
 205         curl_multi_remove_handle(d->curlMulti, e);
 206         curl_easy_cleanup(e);
 207       } else {
 208           // should never happen since CURLMSG_DONE is the only code
 209           // defined!
 210           SG_LOG(SG_IO, SG_ALERT, "unknown CurlMSG:" << msg->msg);
 211       }
 212     } // of curl message processing loop
 213 }
 214
 215 void Client::makeRequest(const Request_ptr& r)
 216 {
 217     if( r->isComplete() )
 218       return;
 219
 220     if( r->url().find("://") == std::string::npos ) {
 221         r->setFailure(EINVAL, "malformed URL");
 222         return;
 223     }
 224
 225     r->_client = this;
 226
 227     ClientPrivate::RequestCurlMap::iterator rit = d->requests.find(r);
 228     assert(rit == d->requests.end());
 229
 230     CURL* curlRequest = curl_easy_init();
 231     curl_easy_setopt(curlRequest, CURLOPT_URL, r->url().c_str());
 232
 233     d->requests[r] = curlRequest;
 234
 235     curl_easy_setopt(curlRequest, CURLOPT_PRIVATE, r.get());
 236     // disable built-in libCurl progress feedback
 237     curl_easy_setopt(curlRequest, CURLOPT_NOPROGRESS, 1);
 238
 239     curl_easy_setopt(curlRequest, CURLOPT_WRITEFUNCTION, requestWriteCallback);
 240     curl_easy_setopt(curlRequest, CURLOPT_WRITEDATA, r.get());
 241     curl_easy_setopt(curlRequest, CURLOPT_HEADERFUNCTION, requestHeaderCallback);
 242     curl_easy_setopt(curlRequest, CURLOPT_HEADERDATA, r.get());
 243
 244     curl_easy_setopt(curlRequest, CURLOPT_USERAGENT, d->userAgent.c_str());
 245     curl_easy_setopt(curlRequest, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
 246
 247     if (!d->proxy.empty()) {
 248       curl_easy_setopt(curlRequest, CURLOPT_PROXY, d->proxy.c_str());
 249       curl_easy_setopt(curlRequest, CURLOPT_PROXYPORT, d->proxyPort);
 250
 251       if (!d->proxyAuth.empty()) {
 252         curl_easy_setopt(curlRequest, CURLOPT_PROXYAUTH, CURLAUTH_BASIC);
 253         curl_easy_setopt(curlRequest, CURLOPT_PROXYUSERPWD, d->proxyAuth.c_str());
 254       }
 255     }
 256
 257     std::string method = boost::to_lower_copy(r->method());
 258     if (method == "get") {
 259       curl_easy_setopt(curlRequest, CURLOPT_HTTPGET, 1);
 260     } else if (method == "put") {
 261       curl_easy_setopt(curlRequest, CURLOPT_PUT, 1);
 262       curl_easy_setopt(curlRequest, CURLOPT_UPLOAD, 1);
 263     } else if (method == "post") {
 264       // see http://curl.haxx.se/libcurl/c/CURLOPT_POST.html
 265       curl_easy_setopt(curlRequest, CURLOPT_HTTPPOST, 1);
 266
 267       std::string q = r->query().substr(1);
 268       curl_easy_setopt(curlRequest, CURLOPT_COPYPOSTFIELDS, q.c_str());
 269
 270       // reset URL to exclude query pieces
 271       std::string urlWithoutQuery = r->url();
 272       std::string::size_type queryPos = urlWithoutQuery.find('?');
 273       urlWithoutQuery.resize(queryPos);
 274       curl_easy_setopt(curlRequest, CURLOPT_URL, urlWithoutQuery.c_str());
 275     } else {
 276       curl_easy_setopt(curlRequest, CURLOPT_CUSTOMREQUEST, r->method().c_str());
 277     }
 278
 279     struct curl_slist* headerList = NULL;
 280     if (r->hasBodyData() && (method != "post")) {
 281       curl_easy_setopt(curlRequest, CURLOPT_UPLOAD, 1);
 282       curl_easy_setopt(curlRequest, CURLOPT_INFILESIZE, r->bodyLength());
 283       curl_easy_setopt(curlRequest, CURLOPT_READFUNCTION, requestReadCallback);
 284       curl_easy_setopt(curlRequest, CURLOPT_READDATA, r.get());
 285       std::string h = "Content-Type:" + r->bodyType();
 286       headerList = curl_slist_append(headerList, h.c_str());
 287     }
 288
 289     StringMap::const_iterator it;
 290     for (it = r->requestHeaders().begin(); it != r->requestHeaders().end(); ++it) {
 291       std::string h = it->first + ": " + it->second;
 292       headerList = curl_slist_append(headerList, h.c_str());
 293     }
 294
 295     if (headerList != NULL) {
 296       curl_easy_setopt(curlRequest, CURLOPT_HTTPHEADER, headerList);
 297     }
 298
 299     curl_multi_add_handle(d->curlMulti, curlRequest);
 300
 301 // this seems premature, but we don't have a callback from Curl we could
 302 // use to trigger when the requst is actually sent.
 303     r->requestStart();
 304 }
 305
 306 void Client::cancelRequest(const Request_ptr &r, std::string reason)
 307 {
 308     ClientPrivate::RequestCurlMap::iterator it = d->requests.find(r);
 309     if(it == d->requests.end()) {
 310         // already being removed, presumably inside ::update()
 311         // nothing more to do
 312         return;
 313     }
 314
 315     CURLMcode err = curl_multi_remove_handle(d->curlMulti, it->second);
 316     if (err != CURLM_OK) {
 317       SG_LOG(SG_IO, SG_WARN, "curl_multi_remove_handle failed:" << err);
 318     }
 319
 320     // clear the request pointer form the curl-easy object
 321     curl_easy_setopt(it->second, CURLOPT_PRIVATE, 0);
 322
 323     curl_easy_cleanup(it->second);
 324     d->requests.erase(it);
 325
 326     r->setFailure(-1, reason);
 327 }
 328
 329 //------------------------------------------------------------------------------
 330 FileRequestRef Client::save( const std::string& url,
 331                              const std::string& filename )
 332 {
 333   FileRequestRef req = new FileRequest(url, filename);
 334   makeRequest(req);
 335   return req;
 336 }
 337
 338 //------------------------------------------------------------------------------
 339 MemoryRequestRef Client::load(const std::string& url)
 340 {
 341   MemoryRequestRef req = new MemoryRequest(url);
 342   makeRequest(req);
 343   return req;
 344 }
 345
 346 void Client::requestFinished(Connection* con)
 347 {
 348
 349 }
 350
 351 void Client::setUserAgent(const std::string& ua)
 352 {
 353     d->userAgent = ua;
 354 }
 355
 356 const std::string& Client::userAgent() const
 357 {
 358     return d->userAgent;
 359 }
 360
 361 const std::string& Client::proxyHost() const
 362 {
 363     return d->proxy;
 364 }
 365
 366 const std::string& Client::proxyAuth() const
 367 {
 368     return d->proxyAuth;
 369 }
 370
 371 void Client::setProxy( const std::string& proxy,
 372                        int port,
 373                        const std::string& auth )
 374 {
 375     d->proxy = proxy;
 376     d->proxyPort = port;
 377     d->proxyAuth = auth;
 378 }
 379
 380 bool Client::hasActiveRequests() const
 381 {
 382     return !d->requests.empty();
 383 }
 384
 385 void Client::receivedBytes(unsigned int count)
 386 {
 387     d->bytesTransferred += count;
 388     d->totalBytesDownloaded += count;
 389 }
 390
 391 unsigned int Client::transferRateBytesPerSec() const
 392 {
 393     unsigned int e = d->timeTransferSample.elapsedMSec();
 394     if (e > 400) {
 395         // too long a window, ignore
 396         d->timeTransferSample.stamp();
 397         d->bytesTransferred = 0;
 398         d->lastTransferRate = 0;
 399         return 0;
 400     }
 401
 402     if (e < 100) { // avoid really narrow windows
 403         return d->lastTransferRate;
 404     }
 405
 406     unsigned int ratio = (d->bytesTransferred * 1000) / e;
 407     // run a low-pass filter
 408     unsigned int smoothed = ((400 - e) * d->lastTransferRate) + (e * ratio);
 409     smoothed /= 400;
 410
 411     d->timeTransferSample.stamp();
 412     d->bytesTransferred = 0;
 413     d->lastTransferRate = smoothed;
 414     return smoothed;
 415 }
 416
 417 uint64_t Client::totalBytesDownloaded() const
 418 {
 419     return d->totalBytesDownloaded;
 420 }
 421
 422 size_t Client::requestWriteCallback(char *ptr, size_t size, size_t nmemb, void *userdata)
 423 {
 424   size_t byteSize = size * nmemb;
 425   Request* req = static_cast<Request*>(userdata);
 426   req->processBodyBytes(ptr, byteSize);
 427
 428   Client* cl = req->http();
 429   if (cl) {
 430     cl->receivedBytes(byteSize);
 431   }
 432
 433   return byteSize;
 434 }
 435
 436 size_t Client::requestReadCallback(char *ptr, size_t size, size_t nmemb, void *userdata)
 437 {
 438   size_t maxBytes = size * nmemb;
 439   Request* req = static_cast<Request*>(userdata);
 440   size_t actualBytes = req->getBodyData(ptr, 0, maxBytes);
 441   return actualBytes;
 442 }
 443
 444 size_t Client::requestHeaderCallback(char *rawBuffer, size_t size, size_t nitems, void *userdata)
 445 {
 446   size_t byteSize = size * nitems;
 447   Request* req = static_cast<Request*>(userdata);
 448   std::string h = strutils::simplify(std::string(rawBuffer, byteSize));
 449
 450   if (req->readyState() == HTTP::Request::OPENED) {
 451     req->responseStart(h);
 452     return byteSize;
 453   }
 454
 455   if (h.empty()) {
 456       // got a 100-continue reponse; restart
 457       if (req->responseCode() == 100) {
 458           req->setReadyState(HTTP::Request::OPENED);
 459           return byteSize;
 460       }
 461
 462     req->responseHeadersComplete();
 463     return byteSize;
 464   }
 465
 466   if (req->responseCode() == 100) {
 467       return byteSize; // skip headers associated with 100-continue status
 468   }
 469
 470   size_t colonPos = h.find(':');
 471   if (colonPos == std::string::npos) {
 472       SG_LOG(SG_IO, SG_WARN, "malformed HTTP response header:" << h);
 473       return byteSize;
 474   }
 475
 476   std::string key = strutils::simplify(h.substr(0, colonPos));
 477   std::string lkey = boost::to_lower_copy(key);
 478   std::string value = strutils::strip(h.substr(colonPos + 1));
 479
 480   req->responseHeader(lkey, value);
 481   return byteSize;
 482 }
 483
 484 void Client::debugDumpRequests()
 485 {
 486     SG_LOG(SG_IO, SG_INFO, "== HTTP request dump");
 487     ClientPrivate::RequestCurlMap::iterator it = d->requests.begin();
 488     for (; it != d->requests.end(); ++it) {
 489         SG_LOG(SG_IO, SG_INFO, "\t" << it->first->url());
 490     }
 491     SG_LOG(SG_IO, SG_INFO, "==");
 492 }
 493
 494 void Client::clearAllConnections()
 495 {
 496     curl_multi_cleanup(d->curlMulti);
 497     d->createCurlMulti();
 498 }
 499
 500 } // of namespace HTTP
 501
 502 } // of namespace simgear