simgear/io/HTTPClient.cxx

   1 /**
   2  * \file HTTPClient.cxx - simple HTTP client engine for SimHear
   3  */
   4
   5 // Written by James Turner
   6 //
   7 // Copyright (C) 2013  James Turner  <zakalawe@mac.com>
   8 //
   9 // This library is free software; you can redistribute it and/or
  10 // modify it under the terms of the GNU Library General Public
  11 // License as published by the Free Software Foundation; either
  12 // version 2 of the License, or (at your option) any later version.
  13 //
  14 // This library is distributed in the hope that it will be useful,
  15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17 // Library General Public License for more details.
  18 //
  19 // You should have received a copy of the GNU General Public License
  20 // along with this program; if not, write to the Free Software
  21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  22 //
  23
  24
  25 #include "HTTPClient.hxx"
  26 #include "HTTPFileRequest.hxx"
  27
  28 #include <sstream>
  29 #include <cassert>
  30 #include <cstdlib> // rand()
  31 #include <list>
  32 #include <errno.h>
  33 #include <map>
  34 #include <stdexcept>
  35
  36 #include <boost/foreach.hpp>
  37 #include <boost/algorithm/string/case_conv.hpp>
  38
  39 #include <simgear/simgear_config.h>
  40
  41 #include <curl/multi.h>
  42
  43 #include <simgear/io/sg_netChat.hxx>
  44
  45 #include <simgear/misc/strutils.hxx>
  46 #include <simgear/compiler.h>
  47 #include <simgear/debug/logstream.hxx>
  48 #include <simgear/timing/timestamp.hxx>
  49 #include <simgear/structure/exception.hxx>
  50
  51 #if defined( HAVE_VERSION_H ) && HAVE_VERSION_H
  52 #include "version.h"
  53 #else
  54 #  if !defined(SIMGEAR_VERSION)
  55 #    define SIMGEAR_VERSION "simgear-development"
  56 #  endif
  57 #endif
  58
  59 namespace simgear
  60 {
  61
  62 namespace HTTP
  63 {
  64
  65 extern const int DEFAULT_HTTP_PORT = 80;
  66 const char* CONTENT_TYPE_URL_ENCODED = "application/x-www-form-urlencoded";
  67
  68 class Connection;
  69 typedef std::multimap<std::string, Connection*> ConnectionDict;
  70 typedef std::list<Request_ptr> RequestList;
  71
  72 class Client::ClientPrivate
  73 {
  74 public:
  75     CURLM* curlMulti;
  76
  77     void createCurlMulti()
  78     {
  79         curlMulti = curl_multi_init();
  80         // see https://curl.haxx.se/libcurl/c/CURLMOPT_PIPELINING.html
  81         // we request HTTP 1.1 pipelining
  82         curl_multi_setopt(curlMulti, CURLMOPT_PIPELINING, 1 /* aka CURLPIPE_HTTP1 */);
  83 #if (LIBCURL_VERSION_MINOR >= 30)
  84         curl_multi_setopt(curlMulti, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long) maxConnections);
  85         curl_multi_setopt(curlMulti, CURLMOPT_MAX_PIPELINE_LENGTH,
  86                           (long) maxPipelineDepth);
  87         curl_multi_setopt(curlMulti, CURLMOPT_MAX_HOST_CONNECTIONS,
  88                           (long) maxHostConnections);
  89 #endif
  90     }
  91
  92     typedef std::map<Request_ptr, CURL*> RequestCurlMap;
  93     RequestCurlMap requests;
  94
  95     std::string userAgent;
  96     std::string proxy;
  97     int proxyPort;
  98     std::string proxyAuth;
  99     unsigned int maxConnections;
 100     unsigned int maxHostConnections;
 101     unsigned int maxPipelineDepth;
 102
 103     RequestList pendingRequests;
 104
 105     SGTimeStamp timeTransferSample;
 106     unsigned int bytesTransferred;
 107     unsigned int lastTransferRate;
 108     uint64_t totalBytesDownloaded;
 109 };
 110
 111 Client::Client() :
 112     d(new ClientPrivate)
 113 {
 114     d->proxyPort = 0;
 115     d->maxConnections = 4;
 116     d->maxHostConnections = 4;
 117     d->bytesTransferred = 0;
 118     d->lastTransferRate = 0;
 119     d->timeTransferSample.stamp();
 120     d->totalBytesDownloaded = 0;
 121     d->maxPipelineDepth = 5;
 122     setUserAgent("SimGear-" SG_STRINGIZE(SIMGEAR_VERSION));
 123
 124     static bool didInitCurlGlobal = false;
 125     if (!didInitCurlGlobal) {
 126       curl_global_init(CURL_GLOBAL_ALL);
 127       didInitCurlGlobal = true;
 128     }
 129
 130     d->createCurlMulti();
 131 }
 132
 133 Client::~Client()
 134 {
 135   curl_multi_cleanup(d->curlMulti);
 136 }
 137
 138 void Client::setMaxConnections(unsigned int maxCon)
 139 {
 140     d->maxConnections = maxCon;
 141 #if (LIBCURL_VERSION_MINOR >= 30)
 142     curl_multi_setopt(d->curlMulti, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long) maxCon);
 143 #endif
 144 }
 145
 146 void Client::setMaxHostConnections(unsigned int maxHostCon)
 147 {
 148     d->maxHostConnections = maxHostCon;
 149 #if (LIBCURL_VERSION_MINOR >= 30)
 150     curl_multi_setopt(d->curlMulti, CURLMOPT_MAX_HOST_CONNECTIONS, (long) maxHostCon);
 151 #endif
 152 }
 153
 154 void Client::setMaxPipelineDepth(unsigned int depth)
 155 {
 156     d->maxPipelineDepth = depth;
 157 #if (LIBCURL_VERSION_MINOR >= 30)
 158     curl_multi_setopt(d->curlMulti, CURLMOPT_MAX_PIPELINE_LENGTH, (long) depth);
 159 #endif
 160 }
 161
 162 void Client::update(int waitTimeout)
 163 {
 164     int remainingActive, messagesInQueue;
 165     curl_multi_perform(d->curlMulti, &remainingActive);
 166
 167     CURLMsg* msg;
 168     while ((msg = curl_multi_info_read(d->curlMulti, &messagesInQueue))) {
 169       if (msg->msg == CURLMSG_DONE) {
 170         Request* rawReq = 0;
 171         CURL *e = msg->easy_handle;
 172         curl_easy_getinfo(e, CURLINFO_PRIVATE, &rawReq);
 173
 174         // ensure request stays valid for the moment
 175         // eg if responseComplete cancels us
 176         Request_ptr req(rawReq);
 177
 178         long responseCode;
 179         curl_easy_getinfo(e, CURLINFO_RESPONSE_CODE, &responseCode);
 180
 181           // remove from the requests map now,
 182           // in case the callbacks perform a cancel. We'll use
 183           // the absence from the request dict in cancel to avoid
 184           // a double remove
 185           ClientPrivate::RequestCurlMap::iterator it = d->requests.find(req);
 186           assert(it != d->requests.end());
 187           assert(it->second == e);
 188           d->requests.erase(it);
 189
 190         if (msg->data.result == 0) {
 191           req->responseComplete();
 192         } else {
 193           SG_LOG(SG_IO, SG_WARN, "CURL Result:" << msg->data.result << " " << curl_easy_strerror(msg->data.result));
 194           req->setFailure(msg->data.result, curl_easy_strerror(msg->data.result));
 195         }
 196
 197         curl_multi_remove_handle(d->curlMulti, e);
 198         curl_easy_cleanup(e);
 199       } else {
 200           // should never happen since CURLMSG_DONE is the only code
 201           // defined!
 202           SG_LOG(SG_IO, SG_ALERT, "unknown CurlMSG:" << msg->msg);
 203       }
 204     } // of curl message processing loop
 205     SGTimeStamp::sleepForMSec(waitTimeout);
 206 }
 207
 208 void Client::makeRequest(const Request_ptr& r)
 209 {
 210     if( r->isComplete() )
 211       return;
 212
 213     if( r->url().find("://") == std::string::npos ) {
 214         r->setFailure(EINVAL, "malformed URL");
 215         return;
 216     }
 217
 218     r->_client = this;
 219
 220     ClientPrivate::RequestCurlMap::iterator rit = d->requests.find(r);
 221     assert(rit == d->requests.end());
 222
 223     CURL* curlRequest = curl_easy_init();
 224     curl_easy_setopt(curlRequest, CURLOPT_URL, r->url().c_str());
 225
 226     d->requests[r] = curlRequest;
 227
 228     curl_easy_setopt(curlRequest, CURLOPT_PRIVATE, r.get());
 229     // disable built-in libCurl progress feedback
 230     curl_easy_setopt(curlRequest, CURLOPT_NOPROGRESS, 1);
 231
 232     curl_easy_setopt(curlRequest, CURLOPT_WRITEFUNCTION, requestWriteCallback);
 233     curl_easy_setopt(curlRequest, CURLOPT_WRITEDATA, r.get());
 234     curl_easy_setopt(curlRequest, CURLOPT_HEADERFUNCTION, requestHeaderCallback);
 235     curl_easy_setopt(curlRequest, CURLOPT_HEADERDATA, r.get());
 236
 237     curl_easy_setopt(curlRequest, CURLOPT_USERAGENT, d->userAgent.c_str());
 238     curl_easy_setopt(curlRequest, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
 239
 240     if (!d->proxy.empty()) {
 241       curl_easy_setopt(curlRequest, CURLOPT_PROXY, d->proxy.c_str());
 242       curl_easy_setopt(curlRequest, CURLOPT_PROXYPORT, d->proxyPort);
 243
 244       if (!d->proxyAuth.empty()) {
 245         curl_easy_setopt(curlRequest, CURLOPT_PROXYAUTH, CURLAUTH_BASIC);
 246         curl_easy_setopt(curlRequest, CURLOPT_PROXYUSERPWD, d->proxyAuth.c_str());
 247       }
 248     }
 249
 250     std::string method = boost::to_lower_copy(r->method());
 251     if (method == "get") {
 252       curl_easy_setopt(curlRequest, CURLOPT_HTTPGET, 1);
 253     } else if (method == "put") {
 254       curl_easy_setopt(curlRequest, CURLOPT_PUT, 1);
 255       curl_easy_setopt(curlRequest, CURLOPT_UPLOAD, 1);
 256     } else if (method == "post") {
 257       // see http://curl.haxx.se/libcurl/c/CURLOPT_POST.html
 258       curl_easy_setopt(curlRequest, CURLOPT_HTTPPOST, 1);
 259
 260       std::string q = r->query().substr(1);
 261       curl_easy_setopt(curlRequest, CURLOPT_COPYPOSTFIELDS, q.c_str());
 262
 263       // reset URL to exclude query pieces
 264       std::string urlWithoutQuery = r->url();
 265       std::string::size_type queryPos = urlWithoutQuery.find('?');
 266       urlWithoutQuery.resize(queryPos);
 267       curl_easy_setopt(curlRequest, CURLOPT_URL, urlWithoutQuery.c_str());
 268     } else {
 269       curl_easy_setopt(curlRequest, CURLOPT_CUSTOMREQUEST, r->method().c_str());
 270     }
 271
 272     struct curl_slist* headerList = NULL;
 273     if (r->hasBodyData() && (method != "post")) {
 274       curl_easy_setopt(curlRequest, CURLOPT_UPLOAD, 1);
 275       curl_easy_setopt(curlRequest, CURLOPT_INFILESIZE, r->bodyLength());
 276       curl_easy_setopt(curlRequest, CURLOPT_READFUNCTION, requestReadCallback);
 277       curl_easy_setopt(curlRequest, CURLOPT_READDATA, r.get());
 278       std::string h = "Content-Type:" + r->bodyType();
 279       headerList = curl_slist_append(headerList, h.c_str());
 280     }
 281
 282     StringMap::const_iterator it;
 283     for (it = r->requestHeaders().begin(); it != r->requestHeaders().end(); ++it) {
 284       std::string h = it->first + ": " + it->second;
 285       headerList = curl_slist_append(headerList, h.c_str());
 286     }
 287
 288     if (headerList != NULL) {
 289       curl_easy_setopt(curlRequest, CURLOPT_HTTPHEADER, headerList);
 290     }
 291
 292     curl_multi_add_handle(d->curlMulti, curlRequest);
 293
 294 // this seems premature, but we don't have a callback from Curl we could
 295 // use to trigger when the requst is actually sent.
 296     r->requestStart();
 297 }
 298
 299 void Client::cancelRequest(const Request_ptr &r, std::string reason)
 300 {
 301     ClientPrivate::RequestCurlMap::iterator it = d->requests.find(r);
 302     if(it == d->requests.end()) {
 303         // already being removed, presumably inside ::update()
 304         // nothing more to do
 305         return;
 306     }
 307
 308     CURLMcode err = curl_multi_remove_handle(d->curlMulti, it->second);
 309     assert(err == CURLM_OK);
 310
 311     // clear the request pointer form the curl-easy object
 312     curl_easy_setopt(it->second, CURLOPT_PRIVATE, 0);
 313
 314     curl_easy_cleanup(it->second);
 315     d->requests.erase(it);
 316
 317     r->setFailure(-1, reason);
 318 }
 319
 320 //------------------------------------------------------------------------------
 321 FileRequestRef Client::save( const std::string& url,
 322                              const std::string& filename )
 323 {
 324   FileRequestRef req = new FileRequest(url, filename);
 325   makeRequest(req);
 326   return req;
 327 }
 328
 329 //------------------------------------------------------------------------------
 330 MemoryRequestRef Client::load(const std::string& url)
 331 {
 332   MemoryRequestRef req = new MemoryRequest(url);
 333   makeRequest(req);
 334   return req;
 335 }
 336
 337 void Client::requestFinished(Connection* con)
 338 {
 339
 340 }
 341
 342 void Client::setUserAgent(const std::string& ua)
 343 {
 344     d->userAgent = ua;
 345 }
 346
 347 const std::string& Client::userAgent() const
 348 {
 349     return d->userAgent;
 350 }
 351
 352 const std::string& Client::proxyHost() const
 353 {
 354     return d->proxy;
 355 }
 356
 357 const std::string& Client::proxyAuth() const
 358 {
 359     return d->proxyAuth;
 360 }
 361
 362 void Client::setProxy( const std::string& proxy,
 363                        int port,
 364                        const std::string& auth )
 365 {
 366     d->proxy = proxy;
 367     d->proxyPort = port;
 368     d->proxyAuth = auth;
 369 }
 370
 371 bool Client::hasActiveRequests() const
 372 {
 373     return !d->requests.empty();
 374 }
 375
 376 void Client::receivedBytes(unsigned int count)
 377 {
 378     d->bytesTransferred += count;
 379     d->totalBytesDownloaded += count;
 380 }
 381
 382 unsigned int Client::transferRateBytesPerSec() const
 383 {
 384     unsigned int e = d->timeTransferSample.elapsedMSec();
 385     if (e > 400) {
 386         // too long a window, ignore
 387         d->timeTransferSample.stamp();
 388         d->bytesTransferred = 0;
 389         d->lastTransferRate = 0;
 390         return 0;
 391     }
 392
 393     if (e < 100) { // avoid really narrow windows
 394         return d->lastTransferRate;
 395     }
 396
 397     unsigned int ratio = (d->bytesTransferred * 1000) / e;
 398     // run a low-pass filter
 399     unsigned int smoothed = ((400 - e) * d->lastTransferRate) + (e * ratio);
 400     smoothed /= 400;
 401
 402     d->timeTransferSample.stamp();
 403     d->bytesTransferred = 0;
 404     d->lastTransferRate = smoothed;
 405     return smoothed;
 406 }
 407
 408 uint64_t Client::totalBytesDownloaded() const
 409 {
 410     return d->totalBytesDownloaded;
 411 }
 412
 413 size_t Client::requestWriteCallback(char *ptr, size_t size, size_t nmemb, void *userdata)
 414 {
 415   size_t byteSize = size * nmemb;
 416   Request* req = static_cast<Request*>(userdata);
 417   req->processBodyBytes(ptr, byteSize);
 418
 419   Client* cl = req->http();
 420   if (cl) {
 421     cl->receivedBytes(byteSize);
 422   }
 423
 424   return byteSize;
 425 }
 426
 427 size_t Client::requestReadCallback(char *ptr, size_t size, size_t nmemb, void *userdata)
 428 {
 429   size_t maxBytes = size * nmemb;
 430   Request* req = static_cast<Request*>(userdata);
 431   size_t actualBytes = req->getBodyData(ptr, 0, maxBytes);
 432   return actualBytes;
 433 }
 434
 435 size_t Client::requestHeaderCallback(char *rawBuffer, size_t size, size_t nitems, void *userdata)
 436 {
 437   size_t byteSize = size * nitems;
 438   Request* req = static_cast<Request*>(userdata);
 439   std::string h = strutils::simplify(std::string(rawBuffer, byteSize));
 440
 441   if (req->readyState() == HTTP::Request::OPENED) {
 442     req->responseStart(h);
 443     return byteSize;
 444   }
 445
 446   if (h.empty()) {
 447       // got a 100-continue reponse; restart
 448       if (req->responseCode() == 100) {
 449           req->setReadyState(HTTP::Request::OPENED);
 450           return byteSize;
 451       }
 452
 453     req->responseHeadersComplete();
 454     return byteSize;
 455   }
 456
 457   if (req->responseCode() == 100) {
 458       return byteSize; // skip headers associated with 100-continue status
 459   }
 460
 461   size_t colonPos = h.find(':');
 462   if (colonPos == std::string::npos) {
 463       SG_LOG(SG_IO, SG_WARN, "malformed HTTP response header:" << h);
 464       return byteSize;
 465   }
 466
 467   std::string key = strutils::simplify(h.substr(0, colonPos));
 468   std::string lkey = boost::to_lower_copy(key);
 469   std::string value = strutils::strip(h.substr(colonPos + 1));
 470
 471   req->responseHeader(lkey, value);
 472   return byteSize;
 473 }
 474
 475 void Client::debugDumpRequests()
 476 {
 477     SG_LOG(SG_IO, SG_INFO, "== HTTP request dump");
 478     ClientPrivate::RequestCurlMap::iterator it = d->requests.begin();
 479     for (; it != d->requests.end(); ++it) {
 480         SG_LOG(SG_IO, SG_INFO, "\t" << it->first->url());
 481     }
 482     SG_LOG(SG_IO, SG_INFO, "==");
 483 }
 484
 485 void Client::clearAllConnections()
 486 {
 487     curl_multi_cleanup(d->curlMulti);
 488     d->createCurlMulti();
 489 }
 490
 491 } // of namespace HTTP
 492
 493 } // of namespace simgear