simgear/io/HTTPClient.cxx

   1 /**
   2  * \file HTTPClient.cxx - simple HTTP client engine for SimHear
   3  */
   4
   5 // Written by James Turner
   6 //
   7 // Copyright (C) 2013  James Turner  <zakalawe@mac.com>
   8 //
   9 // This library is free software; you can redistribute it and/or
  10 // modify it under the terms of the GNU Library General Public
  11 // License as published by the Free Software Foundation; either
  12 // version 2 of the License, or (at your option) any later version.
  13 //
  14 // This library is distributed in the hope that it will be useful,
  15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17 // Library General Public License for more details.
  18 //
  19 // You should have received a copy of the GNU General Public License
  20 // along with this program; if not, write to the Free Software
  21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  22 //
  23
  24 #include "HTTPClient.hxx"
  25 #include "HTTPFileRequest.hxx"
  26
  27 #include <sstream>
  28 #include <cassert>
  29 #include <cstdlib> // rand()
  30 #include <list>
  31 #include <errno.h>
  32 #include <map>
  33 #include <stdexcept>
  34
  35 #include <boost/foreach.hpp>
  36 #include <boost/algorithm/string/case_conv.hpp>
  37
  38 #include <simgear/io/sg_netChat.hxx>
  39 #include <simgear/io/HTTPContentDecode.hxx>
  40 #include <simgear/misc/strutils.hxx>
  41 #include <simgear/compiler.h>
  42 #include <simgear/debug/logstream.hxx>
  43 #include <simgear/timing/timestamp.hxx>
  44 #include <simgear/structure/exception.hxx>
  45
  46 #if defined( HAVE_VERSION_H ) && HAVE_VERSION_H
  47 #include "version.h"
  48 #else
  49 #  if !defined(SIMGEAR_VERSION)
  50 #    define SIMGEAR_VERSION "simgear-development"
  51 #  endif
  52 #endif
  53
  54 namespace simgear
  55 {
  56
  57 namespace HTTP
  58 {
  59
  60 extern const int DEFAULT_HTTP_PORT = 80;
  61 const char* CONTENT_TYPE_URL_ENCODED = "application/x-www-form-urlencoded";
  62 const unsigned int MAX_INFLIGHT_REQUESTS = 32;
  63
  64 class Connection;
  65 typedef std::multimap<std::string, Connection*> ConnectionDict;
  66 typedef std::list<Request_ptr> RequestList;
  67
  68 class Client::ClientPrivate
  69 {
  70 public:
  71     std::string userAgent;
  72     std::string proxy;
  73     int proxyPort;
  74     std::string proxyAuth;
  75     NetChannelPoller poller;
  76     unsigned int maxConnections;
  77
  78     RequestList pendingRequests;
  79
  80 // connections by host (potentially more than one)
  81     ConnectionDict connections;
  82
  83     SGTimeStamp timeTransferSample;
  84     unsigned int bytesTransferred;
  85     unsigned int lastTransferRate;
  86     uint64_t totalBytesDownloaded;
  87 };
  88
  89 class Connection : public NetChat
  90 {
  91 public:
  92     Connection(Client* pr) :
  93         client(pr),
  94         state(STATE_CLOSED),
  95         port(DEFAULT_HTTP_PORT)
  96     {
  97     }
  98
  99     virtual ~Connection()
 100     {
 101     }
 102
 103     virtual void handleBufferRead (NetBuffer& buffer)
 104     {
 105       if( !activeRequest || !activeRequest->isComplete() )
 106         return NetChat::handleBufferRead(buffer);
 107
 108       // Request should be aborted (signaled by setting its state to complete).
 109
 110       // force the state to GETTING_BODY, to simplify logic in
 111       // responseComplete and handleClose
 112       state = STATE_GETTING_BODY;
 113       responseComplete();
 114     }
 115
 116     void setServer(const std::string& h, short p)
 117     {
 118         host = h;
 119         port = p;
 120     }
 121
 122     // socket-level errors
 123     virtual void handleError(int error)
 124     {
 125         if (error == ENOENT) {
 126         // name lookup failure
 127             // we won't have an active request yet, so the logic below won't
 128             // fire to actually call setFailure. Let's fail all of the requests
 129             BOOST_FOREACH(Request_ptr req, sentRequests) {
 130                 req->setFailure(error, "hostname lookup failure");
 131             }
 132
 133             BOOST_FOREACH(Request_ptr req, queuedRequests) {
 134                 req->setFailure(error, "hostname lookup failure");
 135             }
 136
 137         // name lookup failure, abandon all requests on this connection
 138             sentRequests.clear();
 139             queuedRequests.clear();
 140         }
 141
 142         NetChat::handleError(error);
 143         if (activeRequest) {
 144             SG_LOG(SG_IO, SG_INFO, "HTTP socket error");
 145             activeRequest->setFailure(error, "socket error");
 146             activeRequest = NULL;
 147             _contentDecoder.reset();
 148         }
 149
 150         state = STATE_SOCKET_ERROR;
 151     }
 152
 153     virtual void handleClose()
 154     {
 155         NetChat::handleClose();
 156
 157     // closing of the connection from the server side when getting the body,
 158         bool canCloseState = (state == STATE_GETTING_BODY);
 159         if (canCloseState && activeRequest) {
 160         // force state here, so responseComplete can avoid closing the
 161         // socket again
 162             state =  STATE_CLOSED;
 163             responseComplete();
 164         } else {
 165             if (activeRequest) {
 166                 activeRequest->setFailure(500, "server closed connection");
 167                 // remove the failed request from sentRequests, so it does
 168                 // not get restored
 169                 RequestList::iterator it = std::find(sentRequests.begin(),
 170                     sentRequests.end(), activeRequest);
 171                 if (it != sentRequests.end()) {
 172                     sentRequests.erase(it);
 173                 }
 174                 activeRequest = NULL;
 175                 _contentDecoder.reset();
 176             }
 177
 178             state = STATE_CLOSED;
 179         }
 180
 181       if (sentRequests.empty()) {
 182         return;
 183       }
 184
 185     // restore sent requests to the queue, so they will be re-sent
 186     // when the connection opens again
 187       queuedRequests.insert(queuedRequests.begin(),
 188                               sentRequests.begin(), sentRequests.end());
 189       sentRequests.clear();
 190     }
 191
 192     void handleTimeout()
 193     {
 194         NetChat::handleError(ETIMEDOUT);
 195         if (activeRequest) {
 196             SG_LOG(SG_IO, SG_DEBUG, "HTTP socket timeout");
 197             activeRequest->setFailure(ETIMEDOUT, "socket timeout");
 198             activeRequest = NULL;
 199             _contentDecoder.reset();
 200         }
 201
 202         state = STATE_SOCKET_ERROR;
 203     }
 204
 205     void queueRequest(const Request_ptr& r)
 206     {
 207         queuedRequests.push_back(r);
 208         tryStartNextRequest();
 209     }
 210
 211     void beginResponse()
 212     {
 213         assert(!sentRequests.empty());
 214         assert(state == STATE_WAITING_FOR_RESPONSE);
 215
 216         activeRequest = sentRequests.front();
 217         try {
 218             activeRequest->responseStart(buffer);
 219         } catch (sg_exception& e) {
 220             handleError(EIO);
 221         }
 222
 223       state = STATE_GETTING_HEADERS;
 224       buffer.clear();
 225       if (activeRequest->responseCode() == 204) {
 226         noMessageBody = true;
 227       } else if (activeRequest->method() == "HEAD") {
 228         noMessageBody = true;
 229       } else {
 230         noMessageBody = false;
 231       }
 232
 233       bodyTransferSize = -1;
 234       chunkedTransfer = false;
 235       _contentDecoder.reset();
 236     }
 237
 238     void tryStartNextRequest()
 239     {
 240       while( !queuedRequests.empty()
 241           && queuedRequests.front()->isComplete() )
 242         queuedRequests.pop_front();
 243
 244       if (queuedRequests.empty()) {
 245         idleTime.stamp();
 246         return;
 247       }
 248
 249       if (sentRequests.size() > MAX_INFLIGHT_REQUESTS) {
 250         return;
 251       }
 252
 253       if (state == STATE_CLOSED) {
 254           if (!connectToHost()) {
 255               return;
 256           }
 257
 258           setTerminator("\r\n");
 259           state = STATE_IDLE;
 260       }
 261
 262       Request_ptr r = queuedRequests.front();
 263       r->requestStart();
 264
 265       std::stringstream headerData;
 266       std::string path = r->path();
 267       assert(!path.empty());
 268       std::string query = r->query();
 269       std::string bodyData;
 270
 271       if (!client->proxyHost().empty()) {
 272           path = r->scheme() + "://" + r->host() + r->path();
 273       }
 274
 275       if (r->bodyType() == CONTENT_TYPE_URL_ENCODED) {
 276           headerData << r->method() << " " << path << " HTTP/1.1\r\n";
 277           bodyData = query.substr(1); // URL-encode, drop the leading '?'
 278           headerData << "Content-Type:" << CONTENT_TYPE_URL_ENCODED << "\r\n";
 279           headerData << "Content-Length:" << bodyData.size() << "\r\n";
 280       } else {
 281           headerData << r->method() << " " << path << query << " HTTP/1.1\r\n";
 282           if( r->hasBodyData() )
 283           {
 284             headerData << "Content-Length:" << r->bodyLength() << "\r\n";
 285             headerData << "Content-Type:" << r->bodyType() << "\r\n";
 286           }
 287       }
 288
 289       headerData << "Host: " << r->hostAndPort() << "\r\n";
 290       headerData << "User-Agent:" << client->userAgent() << "\r\n";
 291       headerData << "Accept-Encoding: deflate, gzip\r\n";
 292       if (!client->proxyAuth().empty()) {
 293           headerData << "Proxy-Authorization: " << client->proxyAuth() << "\r\n";
 294       }
 295
 296       BOOST_FOREACH(const StringMap::value_type& h, r->requestHeaders()) {
 297           headerData << h.first << ": " << h.second << "\r\n";
 298       }
 299
 300       headerData << "\r\n"; // final CRLF to terminate the headers
 301       if (!bodyData.empty()) {
 302           headerData << bodyData;
 303       }
 304
 305       bool ok = push(headerData.str().c_str());
 306       if (!ok) {
 307           SG_LOG(SG_IO, SG_WARN, "HTTPClient: over-stuffed the socket");
 308           // we've over-stuffed the socket, give up for now, let things
 309           // drain down before trying to start any more requests.
 310           return;
 311       }
 312
 313       if( r->hasBodyData() )
 314         for(size_t body_bytes_sent = 0; body_bytes_sent < r->bodyLength();)
 315         {
 316           char buf[4096];
 317           size_t len = r->getBodyData(buf, body_bytes_sent, 4096);
 318           if( len )
 319           {
 320             if( !bufferSend(buf, len) )
 321             {
 322               SG_LOG(SG_IO,
 323                      SG_WARN,
 324                      "overflow the HTTP::Connection output buffer");
 325               state = STATE_SOCKET_ERROR;
 326               return;
 327             }
 328             body_bytes_sent += len;
 329           }
 330           else
 331           {
 332             SG_LOG(SG_IO,
 333                    SG_WARN,
 334                    "HTTP asynchronous request body generation is unsupported");
 335             break;
 336           }
 337         }
 338
 339       //   SG_LOG(SG_IO, SG_INFO, "did start request:" << r->url() <<
 340       //       "\n\t @ " << reinterpret_cast<void*>(r.ptr()) <<
 341       //      "\n\t on connection " << this);
 342       // successfully sent, remove from queue, and maybe send the next
 343       queuedRequests.pop_front();
 344       sentRequests.push_back(r);
 345       state = STATE_WAITING_FOR_RESPONSE;
 346
 347       // pipelining, let's maybe send the next request right away
 348       tryStartNextRequest();
 349     }
 350
 351     virtual void collectIncomingData(const char* s, int n)
 352     {
 353         idleTime.stamp();
 354         client->receivedBytes(static_cast<unsigned int>(n));
 355
 356         if(   (state == STATE_GETTING_BODY)
 357            || (state == STATE_GETTING_CHUNKED_BYTES) )
 358           _contentDecoder.receivedBytes(s, n);
 359         else
 360           buffer.append(s, n);
 361     }
 362
 363     virtual void foundTerminator(void)
 364     {
 365         idleTime.stamp();
 366         switch (state) {
 367         case STATE_WAITING_FOR_RESPONSE:
 368             beginResponse();
 369             break;
 370
 371         case STATE_GETTING_HEADERS:
 372             processHeader();
 373             buffer.clear();
 374             break;
 375
 376         case STATE_GETTING_BODY:
 377             responseComplete();
 378             break;
 379
 380         case STATE_GETTING_CHUNKED:
 381             processChunkHeader();
 382             break;
 383
 384         case STATE_GETTING_CHUNKED_BYTES:
 385             setTerminator("\r\n");
 386             state = STATE_GETTING_CHUNKED;
 387             buffer.clear();
 388             break;
 389
 390
 391         case STATE_GETTING_TRAILER:
 392             processTrailer();
 393             buffer.clear();
 394             break;
 395
 396         case STATE_IDLE:
 397             SG_LOG(SG_IO, SG_WARN, "HTTP got data in IDLE state, bad server?");
 398
 399         default:
 400             break;
 401         }
 402     }
 403
 404     bool hasIdleTimeout() const
 405     {
 406         if (state != STATE_IDLE) {
 407             return false;
 408         }
 409
 410         assert(sentRequests.empty());
 411         return idleTime.elapsedMSec() > 1000 * 10; // ten seconds
 412     }
 413
 414     bool hasErrorTimeout() const
 415     {
 416       if (state == STATE_IDLE) {
 417         return false;
 418       }
 419
 420       return idleTime.elapsedMSec() > (1000 * 30); // 30 seconds
 421     }
 422
 423     bool hasError() const
 424     {
 425         return (state == STATE_SOCKET_ERROR);
 426     }
 427
 428     bool shouldStartNext() const
 429     {
 430       return !queuedRequests.empty() && (sentRequests.size() < MAX_INFLIGHT_REQUESTS);
 431     }
 432
 433     bool isActive() const
 434     {
 435         return !queuedRequests.empty() || !sentRequests.empty();
 436     }
 437 private:
 438     bool connectToHost()
 439     {
 440         SG_LOG(SG_IO, SG_DEBUG, "HTTP connecting to " << host << ":" << port);
 441
 442         if (!open()) {
 443             SG_LOG(SG_ALL, SG_WARN, "HTTP::Connection: connectToHost: open() failed");
 444             return false;
 445         }
 446
 447         if (connect(host.c_str(), port) != 0) {
 448             return false;
 449         }
 450
 451         return true;
 452     }
 453
 454
 455     void processHeader()
 456     {
 457         std::string h = strutils::simplify(buffer);
 458         if (h.empty()) { // blank line terminates headers
 459             headersComplete();
 460             return;
 461         }
 462
 463         int colonPos = buffer.find(':');
 464         if (colonPos < 0) {
 465             SG_LOG(SG_IO, SG_WARN, "malformed HTTP response header:" << h);
 466             return;
 467         }
 468
 469         std::string key = strutils::simplify(buffer.substr(0, colonPos));
 470         std::string lkey = boost::to_lower_copy(key);
 471         std::string value = strutils::strip(buffer.substr(colonPos + 1));
 472
 473         // only consider these if getting headers (as opposed to trailers
 474         // of a chunked transfer)
 475         if (state == STATE_GETTING_HEADERS) {
 476             if (lkey == "content-length") {
 477
 478                 int sz = strutils::to_int(value);
 479                 if (bodyTransferSize <= 0) {
 480                     bodyTransferSize = sz;
 481                 }
 482                 activeRequest->setResponseLength(sz);
 483             } else if (lkey == "transfer-length") {
 484                 bodyTransferSize = strutils::to_int(value);
 485             } else if (lkey == "transfer-encoding") {
 486                 processTransferEncoding(value);
 487             } else if (lkey == "content-encoding") {
 488                 _contentDecoder.setEncoding(value);
 489             }
 490         }
 491
 492         activeRequest->responseHeader(lkey, value);
 493     }
 494
 495     void processTransferEncoding(const std::string& te)
 496     {
 497         if (te == "chunked") {
 498             chunkedTransfer = true;
 499         } else {
 500             SG_LOG(SG_IO, SG_WARN, "unsupported transfer encoding:" << te);
 501             // failure
 502         }
 503     }
 504
 505     void processChunkHeader()
 506     {
 507         if (buffer.empty()) {
 508             // blank line after chunk data
 509             return;
 510         }
 511
 512         int chunkSize = 0;
 513         int semiPos = buffer.find(';');
 514         if (semiPos >= 0) {
 515             // extensions ignored for the moment
 516             chunkSize = strutils::to_int(buffer.substr(0, semiPos), 16);
 517         } else {
 518             chunkSize = strutils::to_int(buffer, 16);
 519         }
 520
 521         buffer.clear();
 522         if (chunkSize == 0) {  //  trailer start
 523             state = STATE_GETTING_TRAILER;
 524             return;
 525         }
 526
 527         state = STATE_GETTING_CHUNKED_BYTES;
 528         setByteCount(chunkSize);
 529     }
 530
 531     void processTrailer()
 532     {
 533         if (buffer.empty()) {
 534             // end of trailers
 535             responseComplete();
 536             return;
 537         }
 538
 539     // process as a normal header
 540         processHeader();
 541     }
 542
 543     void headersComplete()
 544     {
 545         activeRequest->responseHeadersComplete();
 546         _contentDecoder.initWithRequest(activeRequest);
 547
 548         if (chunkedTransfer) {
 549             state = STATE_GETTING_CHUNKED;
 550         } else if (noMessageBody || (bodyTransferSize == 0)) {
 551             // force the state to GETTING_BODY, to simplify logic in
 552             // responseComplete and handleClose
 553             state = STATE_GETTING_BODY;
 554             responseComplete();
 555         } else {
 556             setByteCount(bodyTransferSize); // may be -1, that's fine
 557             state = STATE_GETTING_BODY;
 558         }
 559     }
 560
 561     void responseComplete()
 562     {
 563         Request_ptr completedRequest = activeRequest;
 564         _contentDecoder.finish();
 565
 566         assert(sentRequests.front() == activeRequest);
 567         sentRequests.pop_front();
 568         bool doClose = activeRequest->closeAfterComplete();
 569         activeRequest = NULL;
 570
 571         if ((state == STATE_GETTING_BODY) || (state == STATE_GETTING_TRAILER)) {
 572             if (doClose) {
 573           // this will bring us into handleClose() above, which updates
 574           // state to STATE_CLOSED
 575               close();
 576
 577           // if we have additional requests waiting, try to start them now
 578               tryStartNextRequest();
 579             }
 580         }
 581
 582         if (state != STATE_CLOSED)  {
 583             state = sentRequests.empty() ? STATE_IDLE : STATE_WAITING_FOR_RESPONSE;
 584         }
 585
 586     // notify request after we change state, so this connection is idle
 587     // if completion triggers other requests (which is likely)
 588         //   SG_LOG(SG_IO, SG_INFO, "*** responseComplete:" << activeRequest->url());
 589         completedRequest->responseComplete();
 590         client->requestFinished(this);
 591
 592         setTerminator("\r\n");
 593     }
 594
 595     enum ConnectionState {
 596         STATE_IDLE = 0,
 597         STATE_WAITING_FOR_RESPONSE,
 598         STATE_GETTING_HEADERS,
 599         STATE_GETTING_BODY,
 600         STATE_GETTING_CHUNKED,
 601         STATE_GETTING_CHUNKED_BYTES,
 602         STATE_GETTING_TRAILER,
 603         STATE_SOCKET_ERROR,
 604         STATE_CLOSED             ///< connection should be closed now
 605     };
 606
 607     Client* client;
 608     Request_ptr activeRequest;
 609     ConnectionState state;
 610     std::string host;
 611     short port;
 612     std::string buffer;
 613     int bodyTransferSize;
 614     SGTimeStamp idleTime;
 615     bool chunkedTransfer;
 616     bool noMessageBody;
 617
 618     RequestList queuedRequests;
 619     RequestList sentRequests;
 620
 621     ContentDecoder _contentDecoder;
 622 };
 623
 624 Client::Client() :
 625     d(new ClientPrivate)
 626 {
 627     d->proxyPort = 0;
 628     d->maxConnections = 4;
 629     d->bytesTransferred = 0;
 630     d->lastTransferRate = 0;
 631     d->timeTransferSample.stamp();
 632     d->totalBytesDownloaded = 0;
 633
 634     setUserAgent("SimGear-" SG_STRINGIZE(SIMGEAR_VERSION));
 635 }
 636
 637 Client::~Client()
 638 {
 639 }
 640
 641 void Client::setMaxConnections(unsigned int maxCon)
 642 {
 643     if (maxCon < 1) {
 644         throw sg_range_exception("illegal HTTP::Client::setMaxConnections value");
 645     }
 646
 647     d->maxConnections = maxCon;
 648 }
 649
 650 void Client::update(int waitTimeout)
 651 {
 652     if (!d->poller.hasChannels() && (waitTimeout > 0)) {
 653         SGTimeStamp::sleepForMSec(waitTimeout);
 654     } else {
 655         d->poller.poll(waitTimeout);
 656     }
 657
 658     bool waitingRequests = !d->pendingRequests.empty();
 659     ConnectionDict::iterator it = d->connections.begin();
 660     for (; it != d->connections.end(); ) {
 661         Connection* con = it->second;
 662         if (con->hasIdleTimeout() ||
 663             con->hasError() ||
 664             con->hasErrorTimeout() ||
 665             (!con->isActive() && waitingRequests))
 666         {
 667             if (con->hasErrorTimeout()) {
 668                 // tell the connection we're timing it out
 669                 con->handleTimeout();
 670             }
 671
 672         // connection has been idle for a while, clean it up
 673         // (or if we have requests waiting for a different host,
 674         // or an error condition
 675             ConnectionDict::iterator del = it++;
 676             delete del->second;
 677             d->connections.erase(del);
 678         } else {
 679             if (it->second->shouldStartNext()) {
 680                 it->second->tryStartNextRequest();
 681             }
 682             ++it;
 683         }
 684     } // of connection iteration
 685
 686     if (waitingRequests && (d->connections.size() < d->maxConnections)) {
 687         RequestList waiting(d->pendingRequests);
 688         d->pendingRequests.clear();
 689
 690         // re-submit all waiting requests in order; this takes care of
 691         // finding multiple pending items targetted to the same (new)
 692         // connection
 693         BOOST_FOREACH(Request_ptr req, waiting) {
 694             makeRequest(req);
 695         }
 696     }
 697 }
 698
 699 void Client::makeRequest(const Request_ptr& r)
 700 {
 701     if( r->isComplete() )
 702       return;
 703
 704     if( r->url().find("://") == std::string::npos ) {
 705         r->setFailure(EINVAL, "malformed URL");
 706         return;
 707     }
 708
 709     if( r->url().find("http://") != 0 ) {
 710         r->setFailure(EINVAL, "only HTTP protocol is supported");
 711         return;
 712     }
 713
 714     std::string host = r->host();
 715     int port = r->port();
 716     if (!d->proxy.empty()) {
 717         host = d->proxy;
 718         port = d->proxyPort;
 719     }
 720
 721     Connection* con = NULL;
 722     std::stringstream ss;
 723     ss << host << "-" << port;
 724     std::string connectionId = ss.str();
 725     bool havePending = !d->pendingRequests.empty();
 726     bool atConnectionsLimit = d->connections.size() >= d->maxConnections;
 727     ConnectionDict::iterator consEnd = d->connections.end();
 728
 729     // assign request to an existing Connection.
 730     // various options exist here, examined in order
 731     ConnectionDict::iterator it = d->connections.find(connectionId);
 732     if (atConnectionsLimit && (it == consEnd)) {
 733         // maximum number of connections active, queue this request
 734         // when a connection goes inactive, we'll start this one
 735         d->pendingRequests.push_back(r);
 736         return;
 737     }
 738
 739     // scan for an idle Connection to the same host (likely if we're
 740     // retrieving multiple resources from the same host in quick succession)
 741     // if we have pending requests (waiting for a free Connection), then
 742     // force new requests on this id to always use the first Connection
 743     // (instead of the random selection below). This ensures that when
 744     // there's pressure on the number of connections to keep alive, one
 745     // host can't DoS every other.
 746     int count = 0;
 747     for (; (it != consEnd) && (it->first == connectionId); ++it, ++count) {
 748         if (havePending || !it->second->isActive()) {
 749             con = it->second;
 750             break;
 751         }
 752     }
 753
 754     if (!con && atConnectionsLimit) {
 755         // all current connections are busy (active), and we don't
 756         // have free connections to allocate, so let's assign to
 757         // an existing one randomly. Ideally we'd used whichever one will
 758         // complete first but we don't have that info.
 759         int index = rand() % count;
 760         for (it = d->connections.find(connectionId); index > 0; --index) { ; }
 761         con = it->second;
 762     }
 763
 764     // allocate a new connection object
 765     if (!con) {
 766         con = new Connection(this);
 767         con->setServer(host, port);
 768         d->poller.addChannel(con);
 769         d->connections.insert(d->connections.end(),
 770             ConnectionDict::value_type(connectionId, con));
 771     }
 772
 773     con->queueRequest(r);
 774 }
 775
 776 //------------------------------------------------------------------------------
 777 FileRequestRef Client::save( const std::string& url,
 778                              const std::string& filename )
 779 {
 780   FileRequestRef req = new FileRequest(url, filename);
 781   makeRequest(req);
 782   return req;
 783 }
 784
 785 //------------------------------------------------------------------------------
 786 MemoryRequestRef Client::load(const std::string& url)
 787 {
 788   MemoryRequestRef req = new MemoryRequest(url);
 789   makeRequest(req);
 790   return req;
 791 }
 792
 793 void Client::requestFinished(Connection* con)
 794 {
 795
 796 }
 797
 798 void Client::setUserAgent(const std::string& ua)
 799 {
 800     d->userAgent = ua;
 801 }
 802
 803 const std::string& Client::userAgent() const
 804 {
 805     return d->userAgent;
 806 }
 807
 808 const std::string& Client::proxyHost() const
 809 {
 810     return d->proxy;
 811 }
 812
 813 const std::string& Client::proxyAuth() const
 814 {
 815     return d->proxyAuth;
 816 }
 817
 818 void Client::setProxy( const std::string& proxy,
 819                        int port,
 820                        const std::string& auth )
 821 {
 822     d->proxy = proxy;
 823     d->proxyPort = port;
 824     d->proxyAuth = auth;
 825 }
 826
 827 bool Client::hasActiveRequests() const
 828 {
 829     ConnectionDict::const_iterator it = d->connections.begin();
 830     for (; it != d->connections.end(); ++it) {
 831         if (it->second->isActive()) return true;
 832     }
 833
 834     return false;
 835 }
 836
 837 void Client::receivedBytes(unsigned int count)
 838 {
 839     d->bytesTransferred += count;
 840     d->totalBytesDownloaded += count;
 841 }
 842
 843 unsigned int Client::transferRateBytesPerSec() const
 844 {
 845     unsigned int e = d->timeTransferSample.elapsedMSec();
 846     if (e > 400) {
 847         // too long a window, ignore
 848         d->timeTransferSample.stamp();
 849         d->bytesTransferred = 0;
 850         d->lastTransferRate = 0;
 851         return 0;
 852     }
 853
 854     if (e < 100) { // avoid really narrow windows
 855         return d->lastTransferRate;
 856     }
 857
 858     unsigned int ratio = (d->bytesTransferred * 1000) / e;
 859     // run a low-pass filter
 860     unsigned int smoothed = ((400 - e) * d->lastTransferRate) + (e * ratio);
 861     smoothed /= 400;
 862
 863     d->timeTransferSample.stamp();
 864     d->bytesTransferred = 0;
 865     d->lastTransferRate = smoothed;
 866     return smoothed;
 867 }
 868
 869 uint64_t Client::totalBytesDownloaded() const
 870 {
 871     return d->totalBytesDownloaded;
 872 }
 873
 874 } // of namespace HTTP
 875
 876 } // of namespace simgear