simgear/io/HTTPClient.cxx

   1 #include "HTTPClient.hxx"
   2
   3 #include <sstream>
   4 #include <cassert>
   5 #include <list>
   6
   7 #include <boost/foreach.hpp>
   8 #include <boost/algorithm/string/case_conv.hpp>
   9
  10 #include <simgear/io/sg_netChat.hxx>
  11 #include <simgear/misc/strutils.hxx>
  12 #include <simgear/compiler.h>
  13 #include <simgear/debug/logstream.hxx>
  14 #include <simgear/timing/timestamp.hxx>
  15
  16 #if defined( HAVE_VERSION_H ) && HAVE_VERSION_H
  17 #include "version.h"
  18 #else
  19 #  if !defined(SIMGEAR_VERSION)
  20 #    define SIMGEAR_VERSION "simgear-development"
  21 #  endif
  22 #endif
  23
  24 using std::string;
  25 using std::stringstream;
  26 using std::vector;
  27
  28 namespace simgear
  29 {
  30
  31 namespace HTTP
  32 {
  33
  34 extern const int DEFAULT_HTTP_PORT = 80;
  35 const char* CONTENT_TYPE_URL_ENCODED = "application/x-www-form-urlencoded";
  36
  37 class Connection : public NetChat
  38 {
  39 public:
  40     Connection(Client* pr) :
  41         client(pr),
  42         state(STATE_CLOSED),
  43         port(DEFAULT_HTTP_PORT)
  44     {
  45
  46     }
  47
  48     void setServer(const string& h, short p)
  49     {
  50         host = h;
  51         port = p;
  52     }
  53
  54     // socket-level errors
  55     virtual void handleError(int error)
  56     {
  57         NetChat::handleError(error);
  58         if (activeRequest) {
  59             SG_LOG(SG_IO, SG_INFO, "HTTP socket error");
  60             activeRequest->setFailure(error, "socket error");
  61             activeRequest = NULL;
  62         }
  63
  64         state = STATE_SOCKET_ERROR;
  65     }
  66
  67     virtual void handleClose()
  68     {
  69         NetChat::handleClose();
  70
  71         if ((state == STATE_GETTING_BODY) && activeRequest) {
  72         // force state here, so responseComplete can avoid closing the
  73         // socket again
  74             state =  STATE_CLOSED;
  75             responseComplete();
  76         } else {
  77             state = STATE_CLOSED;
  78         }
  79     }
  80
  81     void queueRequest(const Request_ptr& r)
  82     {
  83         if (!activeRequest) {
  84             startRequest(r);
  85         } else {
  86             queuedRequests.push_back(r);
  87         }
  88     }
  89
  90     void startRequest(const Request_ptr& r)
  91     {
  92         if (state == STATE_CLOSED) {
  93             if (!connectToHost()) {
  94                 return;
  95             }
  96
  97             state = STATE_IDLE;
  98         }
  99
 100         activeRequest = r;
 101         state = STATE_SENT_REQUEST;
 102         bodyTransferSize = -1;
 103         chunkedTransfer = false;
 104         noMessageBody = (r->method() == "HEAD");
 105         setTerminator("\r\n");
 106
 107         stringstream headerData;
 108         string path = r->path();
 109         string query = r->query();
 110         string bodyData;
 111
 112         if (!client->proxyHost().empty()) {
 113             path = r->scheme() + "://" + r->host() + r->path();
 114         }
 115
 116         if (r->method() == "POST") {
 117             headerData << r->method() << " " << path << " HTTP/1.1\r\n";
 118             bodyData = query.substr(1); // URL-encode, drop the leading '?'
 119             headerData << "Content-Type:" << CONTENT_TYPE_URL_ENCODED << "\r\n";
 120             headerData << "Content-Length:" << bodyData.size() << "\r\n";
 121         } else {
 122             headerData << r->method() << " " << path << query << " HTTP/1.1\r\n";
 123         }
 124
 125         headerData << "Host: " << r->hostAndPort() << "\r\n";
 126         headerData << "User-Agent:" << client->userAgent() << "\r\n";
 127         if (!client->proxyAuth().empty()) {
 128             headerData << "Proxy-Authorization: " << client->proxyAuth() << "\r\n";
 129         }
 130
 131         BOOST_FOREACH(string h, r->requestHeaders()) {
 132             headerData << h << ": " << r->header(h) << "\r\n";
 133         }
 134
 135         headerData << "\r\n"; // final CRLF to terminate the headers
 136         if (!bodyData.empty()) {
 137             headerData << bodyData;
 138         }
 139
 140         bool ok = push(headerData.str().c_str());
 141         if (!ok) {
 142             SG_LOG(SG_IO, SG_WARN, "HTTP writing to socket failed");
 143             state = STATE_SOCKET_ERROR;
 144             return;
 145         }
 146     }
 147
 148     virtual void collectIncomingData(const char* s, int n)
 149     {
 150         if ((state == STATE_GETTING_BODY) || (state == STATE_GETTING_CHUNKED_BYTES)) {
 151             activeRequest->processBodyBytes(s, n);
 152         } else {
 153             buffer += string(s, n);
 154         }
 155     }
 156
 157     virtual void foundTerminator(void)
 158     {
 159         switch (state) {
 160         case STATE_SENT_REQUEST:
 161             activeRequest->responseStart(buffer);
 162             state = STATE_GETTING_HEADERS;
 163             buffer.clear();
 164             if (activeRequest->responseCode() == 204) {
 165                 noMessageBody = true;
 166             }
 167
 168             break;
 169
 170         case STATE_GETTING_HEADERS:
 171             processHeader();
 172             buffer.clear();
 173             break;
 174
 175         case STATE_GETTING_BODY:
 176             responseComplete();
 177             break;
 178
 179         case STATE_GETTING_CHUNKED:
 180             processChunkHeader();
 181             break;
 182
 183         case STATE_GETTING_CHUNKED_BYTES:
 184             setTerminator("\r\n");
 185             state = STATE_GETTING_CHUNKED;
 186             break;
 187
 188
 189         case STATE_GETTING_TRAILER:
 190             processTrailer();
 191             buffer.clear();
 192             break;
 193
 194         default:
 195             break;
 196         }
 197     }
 198
 199     bool hasIdleTimeout() const
 200     {
 201         if (state != STATE_IDLE) {
 202             return false;
 203         }
 204
 205         return idleTime.elapsedMSec() > 1000 * 10; // ten seconds
 206     }
 207
 208     bool hasError() const
 209     {
 210         return (state == STATE_SOCKET_ERROR);
 211     }
 212
 213     bool shouldStartNext() const
 214     {
 215         return !activeRequest && !queuedRequests.empty() &&
 216             ((state == STATE_CLOSED) || (state == STATE_IDLE));
 217     }
 218
 219     void startNext()
 220     {
 221         Request_ptr next = queuedRequests.front();
 222         queuedRequests.pop_front();
 223         startRequest(next);
 224     }
 225 private:
 226     bool connectToHost()
 227     {
 228         SG_LOG(SG_IO, SG_INFO, "HTTP connecting to " << host << ":" << port);
 229
 230         if (!open()) {
 231             SG_LOG(SG_ALL, SG_WARN, "HTTP::Connection: connectToHost: open() failed");
 232             return false;
 233         }
 234
 235         if (connect(host.c_str(), port) != 0) {
 236             return false;
 237         }
 238
 239         return true;
 240     }
 241
 242
 243     void processHeader()
 244     {
 245         string h = strutils::simplify(buffer);
 246         if (h.empty()) { // blank line terminates headers
 247             headersComplete();
 248
 249             if (chunkedTransfer) {
 250                 state = STATE_GETTING_CHUNKED;
 251             } else if (noMessageBody || (bodyTransferSize == 0)) {
 252                 // force the state to GETTING_BODY, to simplify logic in
 253                 // responseComplete and handleClose
 254                 state = STATE_GETTING_BODY;
 255                 responseComplete();
 256             } else {
 257                 setByteCount(bodyTransferSize); // may be -1, that's fine
 258                 state = STATE_GETTING_BODY;
 259             }
 260
 261             return;
 262         }
 263
 264         int colonPos = buffer.find(':');
 265         if (colonPos < 0) {
 266             SG_LOG(SG_IO, SG_WARN, "malformed HTTP response header:" << h);
 267             return;
 268         }
 269
 270         string key = strutils::simplify(buffer.substr(0, colonPos));
 271         string lkey = boost::to_lower_copy(key);
 272         string value = strutils::strip(buffer.substr(colonPos + 1));
 273
 274         // only consider these if getting headers (as opposed to trailers
 275         // of a chunked transfer)
 276         if (state == STATE_GETTING_HEADERS) {
 277             if (lkey == "content-length") {
 278                 int sz = strutils::to_int(value);
 279                 if (bodyTransferSize <= 0) {
 280                     bodyTransferSize = sz;
 281                 }
 282                 activeRequest->setResponseLength(sz);
 283             } else if (lkey == "transfer-length") {
 284                 bodyTransferSize = strutils::to_int(value);
 285             } else if (lkey == "transfer-encoding") {
 286                 processTransferEncoding(value);
 287             }
 288         }
 289
 290         activeRequest->responseHeader(lkey, value);
 291     }
 292
 293     void processTransferEncoding(const string& te)
 294     {
 295         if (te == "chunked") {
 296             chunkedTransfer = true;
 297         } else {
 298             SG_LOG(SG_IO, SG_WARN, "unsupported transfer encoding:" << te);
 299             // failure
 300         }
 301     }
 302
 303     void processChunkHeader()
 304     {
 305         if (buffer.empty()) {
 306             // blank line after chunk data
 307             return;
 308         }
 309
 310         int chunkSize = 0;
 311         int semiPos = buffer.find(';');
 312         if (semiPos >= 0) {
 313             // extensions ignored for the moment
 314             chunkSize = strutils::to_int(buffer.substr(0, semiPos), 16);
 315         } else {
 316             chunkSize = strutils::to_int(buffer, 16);
 317         }
 318
 319         buffer.clear();
 320         if (chunkSize == 0) {  //  trailer start
 321             state = STATE_GETTING_TRAILER;
 322             return;
 323         }
 324
 325         state = STATE_GETTING_CHUNKED_BYTES;
 326         setByteCount(chunkSize);
 327     }
 328
 329     void processTrailer()
 330     {
 331         if (buffer.empty()) {
 332             // end of trailers
 333             responseComplete();
 334             return;
 335         }
 336
 337     // process as a normal header
 338         processHeader();
 339     }
 340
 341     void headersComplete()
 342     {
 343         activeRequest->responseHeadersComplete();
 344     }
 345
 346     void responseComplete()
 347     {
 348         activeRequest->responseComplete();
 349         client->requestFinished(this);
 350
 351         bool doClose = activeRequest->closeAfterComplete();
 352         activeRequest = NULL;
 353         if (state == STATE_GETTING_BODY) {
 354             state = STATE_IDLE;
 355             if (doClose) {
 356             // this will bring us into handleClose() above, which updates
 357             // state to STATE_CLOSED
 358                 close();
 359             }
 360         }
 361
 362         setTerminator("\r\n");
 363
 364     // if we have more requests, and we're idle, can start the next
 365     // request immediately. Note we cannot do this if we're in STATE_CLOSED,
 366     // since NetChannel::close cleans up state after calling handleClose;
 367     // instead we pick up waiting requests in update()
 368         if (!queuedRequests.empty() && (state == STATE_IDLE)) {
 369             startNext();
 370         } else {
 371             idleTime.stamp();
 372         }
 373     }
 374
 375     enum ConnectionState {
 376         STATE_IDLE = 0,
 377         STATE_SENT_REQUEST,
 378         STATE_GETTING_HEADERS,
 379         STATE_GETTING_BODY,
 380         STATE_GETTING_CHUNKED,
 381         STATE_GETTING_CHUNKED_BYTES,
 382         STATE_GETTING_TRAILER,
 383         STATE_SOCKET_ERROR,
 384         STATE_CLOSED             ///< connection should be closed now
 385     };
 386
 387     Client* client;
 388     Request_ptr activeRequest;
 389     ConnectionState state;
 390     string host;
 391     short port;
 392     std::string buffer;
 393     int bodyTransferSize;
 394     SGTimeStamp idleTime;
 395     bool chunkedTransfer;
 396     bool noMessageBody;
 397
 398     std::list<Request_ptr> queuedRequests;
 399 };
 400
 401 Client::Client()
 402 {
 403     setUserAgent("SimGear-" SG_STRINGIZE(SIMGEAR_VERSION));
 404 }
 405
 406 void Client::update()
 407 {
 408     NetChannel::poll();
 409
 410     ConnectionDict::iterator it = _connections.begin();
 411     for (; it != _connections.end(); ) {
 412         if (it->second->hasIdleTimeout() || it->second->hasError()) {
 413         // connection has been idle for a while, clean it up
 414         // (or has an error condition, again clean it up)
 415             SG_LOG(SG_IO, SG_INFO, "cleaning up " << it->second);
 416             ConnectionDict::iterator del = it++;
 417             delete del->second;
 418             _connections.erase(del);
 419         } else {
 420             if (it->second->shouldStartNext()) {
 421                 it->second->startNext();
 422             }
 423
 424             ++it;
 425         }
 426     } // of connecion iteration
 427 }
 428
 429 void Client::makeRequest(const Request_ptr& r)
 430 {
 431     string host = r->host();
 432     int port = r->port();
 433     if (!_proxy.empty()) {
 434         host = _proxy;
 435         port = _proxyPort;
 436     }
 437
 438     stringstream ss;
 439     ss << host << "-" << port;
 440     string connectionId = ss.str();
 441
 442     if (_connections.find(connectionId) == _connections.end()) {
 443         Connection* con = new Connection(this);
 444         con->setServer(host, port);
 445         _connections[connectionId] = con;
 446     }
 447
 448     _connections[connectionId]->queueRequest(r);
 449 }
 450
 451 void Client::requestFinished(Connection* con)
 452 {
 453
 454 }
 455
 456 void Client::setUserAgent(const string& ua)
 457 {
 458     _userAgent = ua;
 459 }
 460
 461 void Client::setProxy(const string& proxy, int port, const string& auth)
 462 {
 463     _proxy = proxy;
 464     _proxyPort = port;
 465     _proxyAuth = auth;
 466 }
 467
 468 } // of namespace HTTP
 469
 470 } // of namespace simgear