From 7525fd5e3ee741ea323df5811647faa5c5763a4b Mon Sep 17 00:00:00 2001 From: James Turner Date: Sun, 7 Aug 2011 10:49:13 +0100 Subject: [PATCH] Update HTTP code to support HTTP/1.0 responses, eg metarproxy --- simgear/io/HTTPClient.cxx | 107 +++++++++++++++++++++++++------------ simgear/io/HTTPClient.hxx | 1 + simgear/io/HTTPRequest.cxx | 26 +++++++-- simgear/io/HTTPRequest.hxx | 16 ++++++ simgear/io/httpget.cxx | 10 ++-- simgear/io/test_HTTP.cxx | 69 ++++++++++++++++++++---- 6 files changed, 178 insertions(+), 51 deletions(-) diff --git a/simgear/io/HTTPClient.cxx b/simgear/io/HTTPClient.cxx index 3ea5f02a..ddcae484 100644 --- a/simgear/io/HTTPClient.cxx +++ b/simgear/io/HTTPClient.cxx @@ -38,30 +38,24 @@ class Connection : public NetChat public: Connection(Client* pr) : client(pr), - state(STATE_IDLE) + state(STATE_CLOSED), + port(DEFAULT_HTTP_PORT) { setTerminator("\r\n"); } - bool connectToHost(const string& host, short port) + void setServer(const string& h, short p) { - if (!open()) { - SG_LOG(SG_ALL, SG_WARN, "HTTP::Connection: connectToHost: open() failed"); - return false; - } - - if (connect(host.c_str(), port) != 0) { - return false; - } - - return true; + host = h; + port = p; } // socket-level errors virtual void handleError(int error) - { + { NetChat::handleError(error); if (activeRequest) { + SG_LOG(SG_IO, SG_INFO, "HTTP socket error"); activeRequest->setFailure(error, "socket error"); activeRequest = NULL; } @@ -69,6 +63,19 @@ public: state = STATE_SOCKET_ERROR; } + virtual void handleClose() + { + if ((state == STATE_GETTING_BODY) && activeRequest) { + // force state here, so responseComplete can avoid closing the + // socket again + state = STATE_CLOSED; + responseComplete(); + } + + state = STATE_CLOSED; + NetChat::handleClose(); + } + void queueRequest(const Request_ptr& r) { if (!activeRequest) { @@ -80,9 +87,17 @@ public: void startRequest(const Request_ptr& r) { + if (state == STATE_CLOSED) { + if (!connectToHost()) { + return; + } + + state = STATE_IDLE; + } + activeRequest = r; state = STATE_IDLE; - bodyTransferSize = 0; + bodyTransferSize = -1; chunkedTransfer = false; stringstream headerData; @@ -106,7 +121,12 @@ public: // TODO - add request body support for PUT, etc operations - push(headerData.str().c_str()); + bool ok = push(headerData.str().c_str()); + if (!ok) { + SG_LOG(SG_IO, SG_WARN, "HTTP writing to socket failed"); + state = STATE_SOCKET_ERROR; + return; + } } virtual void collectIncomingData(const char* s, int n) @@ -169,6 +189,23 @@ public: return (state == STATE_SOCKET_ERROR); } private: + bool connectToHost() + { + SG_LOG(SG_IO, SG_INFO, "HTTP connecting to " << host << ":" << port); + + if (!open()) { + SG_LOG(SG_ALL, SG_WARN, "HTTP::Connection: connectToHost: open() failed"); + return false; + } + + if (connect(host.c_str(), port) != 0) { + return false; + } + + return true; + } + + void processHeader() { string h = strutils::simplify(buffer); @@ -177,12 +214,11 @@ private: if (chunkedTransfer) { state = STATE_GETTING_CHUNKED; - } else if (bodyTransferSize > 0) { - state = STATE_GETTING_BODY; - setByteCount(bodyTransferSize); } else { - responseComplete(); + setByteCount(bodyTransferSize); // may be -1, that's fine + state = STATE_GETTING_BODY; } + return; } @@ -272,9 +308,18 @@ private: { activeRequest->responseComplete(); client->requestFinished(this); + + bool doClose = activeRequest->closeAfterComplete(); activeRequest = NULL; + if (state == STATE_GETTING_BODY) { + state = STATE_IDLE; + if (doClose) { + // this will bring us into handleClose() above, which updates + // state to STATE_CLOSED + close(); + } + } - state = STATE_IDLE; setTerminator("\r\n"); if (!queuedRequests.empty()) { @@ -293,12 +338,15 @@ private: STATE_GETTING_CHUNKED, STATE_GETTING_CHUNKED_BYTES, STATE_GETTING_TRAILER, - STATE_SOCKET_ERROR + STATE_SOCKET_ERROR, + STATE_CLOSED ///< connection should be closed now }; Client* client; Request_ptr activeRequest; ConnectionState state; + string host; + short port; std::string buffer; int bodyTransferSize; SGTimeStamp idleTime; @@ -315,11 +363,13 @@ Client::Client() void Client::update() { NetChannel::poll(); - + ConnectionDict::iterator it = _connections.begin(); for (; it != _connections.end(); ) { if (it->second->hasIdleTimeout() || it->second->hasError()) { // connection has been idle for a while, clean it up + // (or has an error condition, again clean it up) + SG_LOG(SG_IO, SG_INFO, "cleaning up " << it->second); ConnectionDict::iterator del = it++; delete del->second; _connections.erase(del); @@ -344,18 +394,7 @@ void Client::makeRequest(const Request_ptr& r) if (_connections.find(connectionId) == _connections.end()) { Connection* con = new Connection(this); - bool ok = con->connectToHost(host, port); - if (!ok) { - // since NetChannel connect is non-blocking, this failure - // path is unlikely, but still checked for. - SG_LOG(SG_IO, SG_WARN, "unable to connect to host:" - << host << " (port:" << port << ")"); - delete con; - - r->setFailure(-1, "unable to connect to host"); - return; - } - + con->setServer(host, port); _connections[connectionId] = con; } diff --git a/simgear/io/HTTPClient.hxx b/simgear/io/HTTPClient.hxx index 67065935..7cc3771f 100644 --- a/simgear/io/HTTPClient.hxx +++ b/simgear/io/HTTPClient.hxx @@ -37,6 +37,7 @@ private: void requestFinished(Connection* con); friend class Connection; + friend class Request; std::string _userAgent; std::string _proxy; diff --git a/simgear/io/HTTPRequest.cxx b/simgear/io/HTTPRequest.cxx index 84a19c99..4b7a11bf 100644 --- a/simgear/io/HTTPRequest.cxx +++ b/simgear/io/HTTPRequest.cxx @@ -18,9 +18,11 @@ extern const int DEFAULT_HTTP_PORT; Request::Request(const string& url, const string method) : _method(method), _url(url), + _responseVersion(HTTP_VERSION_UNKNOWN), _responseStatus(0), _responseLength(0), - _receivedBodyBytes(0) + _receivedBodyBytes(0), + _willClose(false) { } @@ -52,17 +54,21 @@ void Request::responseStart(const string& r) string_list parts = strutils::split(r, NULL, maxSplit); if (parts.size() != 3) { SG_LOG(SG_IO, SG_WARN, "HTTP::Request: malformed response start:" << r); - _responseStatus = 400; - _responseReason = "bad HTTP response header"; + setFailure(400, "malformed HTTP response header"); return; } + _responseVersion = decodeVersion(parts[0]); _responseStatus = strutils::to_int(parts[1]); _responseReason = parts[2]; } void Request::responseHeader(const string& key, const string& value) { + if (key == "connection") { + _willClose = (value.find("close") >= 0); + } + _responseHeaders[key] = value; } @@ -185,6 +191,20 @@ void Request::failed() // no-op in base class } +Request::HTTPVersion Request::decodeVersion(const string& v) +{ + if (v == "HTTP/1.1") return HTTP_1_1; + if (v == "HTTP/1.0") return HTTP_1_0; + if (strutils::starts_with(v, "HTTP/0.")) return HTTP_0_x; + return HTTP_VERSION_UNKNOWN; +} + +bool Request::closeAfterComplete() const +{ +// for non HTTP/1.1 connections, assume server closes + return _willClose || (_responseVersion != HTTP_1_1); +} + } // of namespace HTTP } // of namespace simgear diff --git a/simgear/io/HTTPRequest.hxx b/simgear/io/HTTPRequest.hxx index 1c7dc083..0e346419 100644 --- a/simgear/io/HTTPRequest.hxx +++ b/simgear/io/HTTPRequest.hxx @@ -50,6 +50,20 @@ public: */ unsigned int responseBytesReceived() const { return _receivedBodyBytes; } + + enum HTTPVersion { + HTTP_VERSION_UNKNOWN = 0, + HTTP_0_x, // 0.9 or similar + HTTP_1_0, + HTTP_1_1 + }; + + HTTPVersion responseVersion() const + { return _responseVersion; } + + static HTTPVersion decodeVersion(const std::string& v); + + bool closeAfterComplete() const; protected: Request(const std::string& url, const std::string method = "GET"); @@ -68,10 +82,12 @@ private: std::string _method; std::string _url; + HTTPVersion _responseVersion; int _responseStatus; std::string _responseReason; unsigned int _responseLength; unsigned int _receivedBodyBytes; + bool _willClose; typedef std::map HeaderDict; HeaderDict _responseHeaders; diff --git a/simgear/io/httpget.cxx b/simgear/io/httpget.cxx index c52117a9..a8e22606 100644 --- a/simgear/io/httpget.cxx +++ b/simgear/io/httpget.cxx @@ -5,12 +5,14 @@ #include // for STDOUT_FILENO #include #include +#include #include #include #include #include #include +#include using namespace simgear; using std::cout; @@ -126,12 +128,13 @@ int main(int argc, char* argv[]) if (colonPos >= 0) { proxyHost = proxy.substr(0, colonPos); proxyPort = strutils::to_int(proxy.substr(colonPos + 1)); - cout << proxyHost << " " << proxyPort << endl; } cl.setProxy(proxyHost, proxyPort, proxyAuth); } + signal(SIGPIPE, SIG_IGN); + if (!outFile) { outFile = new SGFile(STDOUT_FILENO); } @@ -150,9 +153,10 @@ int main(int argc, char* argv[]) cl.makeRequest(req); while (!req->complete()) { - NetChannel::poll(100); + cl.update(); + sleepForMSec(100); } - + if (req->responseCode() != 200) { cerr << "got response:" << req->responseCode() << endl; cerr << "\treason:" << req->responseReason() << endl; diff --git a/simgear/io/test_HTTP.cxx b/simgear/io/test_HTTP.cxx index aa813824..d452ec1e 100644 --- a/simgear/io/test_HTTP.cxx +++ b/simgear/io/test_HTTP.cxx @@ -147,7 +147,7 @@ public: if (path == "/test1") { string contentStr(BODY1); stringstream d; - d << "HTTP1.1 " << 200 << " " << reasonForCode(200) << "\r\n"; + d << "HTTP/1.1 " << 200 << " " << reasonForCode(200) << "\r\n"; d << "Content-Length:" << contentStr.size() << "\r\n"; d << "\r\n"; // final CRLF to terminate the headers d << contentStr; @@ -156,7 +156,7 @@ public: sendBody2(); } else if (path == "/testchunked") { stringstream d; - d << "HTTP1.1 " << 200 << " " << reasonForCode(200) << "\r\n"; + d << "HTTP/1.1 " << 200 << " " << reasonForCode(200) << "\r\n"; d << "Transfer-Encoding:chunked\r\n"; d << "\r\n"; d << "8\r\n"; // first chunk @@ -172,47 +172,70 @@ public: } else if (path == "http://www.google.com/test2") { // proxy test if (requestHeaders["host"] != "www.google.com") { - sendErrorResponse(400); + sendErrorResponse(400, true, "bad destination"); } if (requestHeaders["proxy-authorization"] != string()) { - sendErrorResponse(401); // shouldn't supply auth + sendErrorResponse(401, false, "bad auth"); // shouldn't supply auth } sendBody2(); } else if (path == "http://www.google.com/test3") { // proxy test if (requestHeaders["host"] != "www.google.com") { - sendErrorResponse(400); + sendErrorResponse(400, true, "bad destination"); } if (requestHeaders["proxy-authorization"] != "ABCDEF") { - sendErrorResponse(401); // forbidden + sendErrorResponse(401, false, "bad auth"); // forbidden } sendBody2(); + } else if (path == "/test_1_0") { + string contentStr(BODY1); + stringstream d; + d << "HTTP/1.0 " << 200 << " " << reasonForCode(200) << "\r\n"; + d << "\r\n"; // final CRLF to terminate the headers + d << contentStr; + push(d.str().c_str()); + closeWhenDone(); + } else if (path == "/test_close") { + string contentStr(BODY1); + stringstream d; + d << "HTTP/1.1 " << 200 << " " << reasonForCode(200) << "\r\n"; + d << "Connection: close\r\n"; + d << "\r\n"; // final CRLF to terminate the headers + d << contentStr; + push(d.str().c_str()); + closeWhenDone(); } else { - sendErrorResponse(404); + sendErrorResponse(404, true, ""); } } void sendBody2() { stringstream d; - d << "HTTP1.1 " << 200 << " " << reasonForCode(200) << "\r\n"; + d << "HTTP/1.1 " << 200 << " " << reasonForCode(200) << "\r\n"; d << "Content-Length:" << body2Size << "\r\n"; d << "\r\n"; // final CRLF to terminate the headers push(d.str().c_str()); bufferSend(body2, body2Size); } - void sendErrorResponse(int code) + void sendErrorResponse(int code, bool close, string content) { cerr << "sending error " << code << " for " << path << endl; stringstream headerData; - headerData << "HTTP1.1 " << code << " " << reasonForCode(code) << "\r\n"; + headerData << "HTTP/1.1 " << code << " " << reasonForCode(code) << "\r\n"; + headerData << "Content-Length:" << content.size() << "\r\n"; headerData << "\r\n"; // final CRLF to terminate the headers push(headerData.str().c_str()); + push(content.c_str()); + + if (close) { + closeWhenDone(); + } } string reasonForCode(int code) @@ -252,7 +275,7 @@ public: { simgear::IPAddress addr ; int handle = accept ( &addr ) ; - cout << "did accept from " << addr.getHost() << ":" << addr.getPort() << endl; + //cout << "did accept from " << addr.getHost() << ":" << addr.getPort() << endl; TestServerChannel* chan = new TestServerChannel(); chan->setHandle(handle); } @@ -359,6 +382,30 @@ int main(int argc, char* argv[]) COMPARE(tr->responseLength(), 0); } + cout << "done1" << endl; +// test HTTP/1.0 + { + TestRequest* tr = new TestRequest("http://localhost:2000/test_1_0"); + HTTP::Request_ptr own(tr); + cl.makeRequest(tr); + waitForComplete(tr); + COMPARE(tr->responseCode(), 200); + COMPARE(tr->responseLength(), strlen(BODY1)); + COMPARE(tr->bodyData, string(BODY1)); + } + + cout << "done2" << endl; +// test HTTP/1.1 Connection::close + { + TestRequest* tr = new TestRequest("http://localhost:2000/test_close"); + HTTP::Request_ptr own(tr); + cl.makeRequest(tr); + waitForComplete(tr); + COMPARE(tr->responseCode(), 200); + COMPARE(tr->responseLength(), strlen(BODY1)); + COMPARE(tr->bodyData, string(BODY1)); + } + cout << "done3" << endl; // test connectToHost failure /* { -- 2.39.5