From f93fead8f277403bd1759bc20d995e8935c53bc9 Mon Sep 17 00:00:00 2001 From: Thomas Geymayer Date: Sun, 27 Oct 2013 18:40:14 +0100 Subject: [PATCH] io: refactor and improve HTTP modules. - refactor code used multiple times spread over sg/fg into one single location. - allow aborting requests. - Provide two common request types: * FileRequest: Save response into file * MemoryRequest: Keep resonse in memory (std::string) - extend HTTP::Client interface: * urlretrieve: Save url to file (shortcut for making a FileRequest) * urlload: Get respons into memory (shortcut for making a MemoryRequest) --- simgear/io/CMakeLists.txt | 4 + simgear/io/HTTPClient.cxx | 159 ++++++++------ simgear/io/HTTPClient.hxx | 22 +- simgear/io/HTTPFileRequest.cxx | 82 ++++++++ simgear/io/HTTPFileRequest.hxx | 56 +++++ simgear/io/HTTPMemoryRequest.cxx | 55 +++++ simgear/io/HTTPMemoryRequest.hxx | 58 ++++++ simgear/io/HTTPRequest.cxx | 347 +++++++++++++++++++++---------- simgear/io/HTTPRequest.hxx | 156 +++++++++++--- simgear/io/SVNRepository.cxx | 104 +++------ simgear/io/httpget.cxx | 29 +-- simgear/io/sg_netChat.hxx | 2 + simgear/io/test_HTTP.cxx | 44 +--- simgear/package/Catalog.cxx | 7 +- simgear/package/Install.cxx | 2 +- 15 files changed, 781 insertions(+), 346 deletions(-) create mode 100644 simgear/io/HTTPFileRequest.cxx create mode 100644 simgear/io/HTTPFileRequest.hxx create mode 100644 simgear/io/HTTPMemoryRequest.cxx create mode 100644 simgear/io/HTTPMemoryRequest.hxx diff --git a/simgear/io/CMakeLists.txt b/simgear/io/CMakeLists.txt index 10346725..b72a0040 100644 --- a/simgear/io/CMakeLists.txt +++ b/simgear/io/CMakeLists.txt @@ -15,6 +15,8 @@ set(HEADERS sg_socket.hxx sg_socket_udp.hxx HTTPClient.hxx + HTTPFileRequest.hxx + HTTPMemoryRequest.hxx HTTPRequest.hxx HTTPContentDecode.hxx DAVMultiStatus.hxx @@ -36,6 +38,8 @@ set(SOURCES sg_socket.cxx sg_socket_udp.cxx HTTPClient.cxx + HTTPFileRequest.cxx + HTTPMemoryRequest.cxx HTTPRequest.cxx HTTPContentDecode.cxx DAVMultiStatus.cxx diff --git a/simgear/io/HTTPClient.cxx b/simgear/io/HTTPClient.cxx index 1686fb21..03f13428 100644 --- a/simgear/io/HTTPClient.cxx +++ b/simgear/io/HTTPClient.cxx @@ -22,12 +22,12 @@ // #include "HTTPClient.hxx" +#include "HTTPFileRequest.hxx" #include #include #include // rand() #include -#include #include #include #include @@ -51,10 +51,6 @@ # endif #endif -using std::string; -using std::stringstream; -using std::vector; - namespace simgear { @@ -103,8 +99,21 @@ public: virtual ~Connection() { } + + virtual void handleBufferRead (NetBuffer& buffer) + { + if( !activeRequest || !activeRequest->isComplete() ) + return NetChat::handleBufferRead(buffer); + + // Request should be aborted (signaled by setting its state to complete). + + // force the state to GETTING_BODY, to simplify logic in + // responseComplete and handleClose + state = STATE_GETTING_BODY; + responseComplete(); + } - void setServer(const string& h, short p) + void setServer(const std::string& h, short p) { host = h; port = p; @@ -224,6 +233,10 @@ public: void tryStartNextRequest() { + while( !queuedRequests.empty() + && queuedRequests.front()->isComplete() ) + queuedRequests.pop_front(); + if (queuedRequests.empty()) { idleTime.stamp(); return; @@ -244,28 +257,28 @@ public: Request_ptr r = queuedRequests.front(); r->requestStart(); - requestBodyBytesToSend = r->requestBodyLength(); - - stringstream headerData; - string path = r->path(); + + std::stringstream headerData; + std::string path = r->path(); assert(!path.empty()); - string query = r->query(); - string bodyData; + std::string query = r->query(); + std::string bodyData; if (!client->proxyHost().empty()) { path = r->scheme() + "://" + r->host() + r->path(); } - if (r->requestBodyType() == CONTENT_TYPE_URL_ENCODED) { + if (r->bodyType() == CONTENT_TYPE_URL_ENCODED) { headerData << r->method() << " " << path << " HTTP/1.1\r\n"; bodyData = query.substr(1); // URL-encode, drop the leading '?' headerData << "Content-Type:" << CONTENT_TYPE_URL_ENCODED << "\r\n"; headerData << "Content-Length:" << bodyData.size() << "\r\n"; } else { headerData << r->method() << " " << path << query << " HTTP/1.1\r\n"; - if (requestBodyBytesToSend >= 0) { - headerData << "Content-Length:" << requestBodyBytesToSend << "\r\n"; - headerData << "Content-Type:" << r->requestBodyType() << "\r\n"; + if( r->hasBodyData() ) + { + headerData << "Content-Length:" << r->bodyLength() << "\r\n"; + headerData << "Content-Type:" << r->bodyType() << "\r\n"; } } @@ -276,8 +289,8 @@ public: headerData << "Proxy-Authorization: " << client->proxyAuth() << "\r\n"; } - BOOST_FOREACH(string h, r->requestHeaders()) { - headerData << h << ": " << r->header(h) << "\r\n"; + BOOST_FOREACH(const StringMap::value_type& h, r->requestHeaders()) { + headerData << h.first << ": " << h.second << "\r\n"; } headerData << "\r\n"; // final CRLF to terminate the headers @@ -292,33 +305,42 @@ public: // drain down before trying to start any more requests. return; } - - while (requestBodyBytesToSend > 0) { - char buf[4096]; - int len = r->getBodyData(buf, 4096); - if (len > 0) { - requestBodyBytesToSend -= len; - if (!bufferSend(buf, len)) { - SG_LOG(SG_IO, SG_WARN, "overflow the HTTP::Connection output buffer"); - state = STATE_SOCKET_ERROR; - return; + + if( r->hasBodyData() ) + for(size_t body_bytes_sent = 0; body_bytes_sent < r->bodyLength();) + { + char buf[4096]; + size_t len = r->getBodyData(buf, body_bytes_sent, 4096); + if( len ) + { + if( !bufferSend(buf, len) ) + { + SG_LOG(SG_IO, + SG_WARN, + "overflow the HTTP::Connection output buffer"); + state = STATE_SOCKET_ERROR; + return; + } + body_bytes_sent += len; + } + else + { + SG_LOG(SG_IO, + SG_WARN, + "HTTP asynchronous request body generation is unsupported"); + break; } - // SG_LOG(SG_IO, SG_INFO, "sent body:\n" << string(buf, len) << "\n%%%%%%%%%"); - } else { - SG_LOG(SG_IO, SG_WARN, "HTTP asynchronous request body generation is unsupported"); - break; } - } - // SG_LOG(SG_IO, SG_INFO, "did start request:" << r->url() << - // "\n\t @ " << reinterpret_cast(r.ptr()) << - // "\n\t on connection " << this); - // successfully sent, remove from queue, and maybe send the next + // SG_LOG(SG_IO, SG_INFO, "did start request:" << r->url() << + // "\n\t @ " << reinterpret_cast(r.ptr()) << + // "\n\t on connection " << this); + // successfully sent, remove from queue, and maybe send the next queuedRequests.pop_front(); sentRequests.push_back(r); - state = STATE_WAITING_FOR_RESPONSE; + state = STATE_WAITING_FOR_RESPONSE; - // pipelining, let's maybe send the next request right away + // pipelining, let's maybe send the next request right away tryStartNextRequest(); } @@ -326,12 +348,12 @@ public: { idleTime.stamp(); client->receivedBytes(static_cast(n)); - - if ((state == STATE_GETTING_BODY) || (state == STATE_GETTING_CHUNKED_BYTES)) { - _contentDecoder.receivedBytes(s, n); - } else { - buffer += string(s, n); - } + + if( (state == STATE_GETTING_BODY) + || (state == STATE_GETTING_CHUNKED_BYTES) ) + _contentDecoder.receivedBytes(s, n); + else + buffer.append(s, n); } virtual void foundTerminator(void) @@ -428,7 +450,7 @@ private: void processHeader() { - string h = strutils::simplify(buffer); + std::string h = strutils::simplify(buffer); if (h.empty()) { // blank line terminates headers headersComplete(); return; @@ -440,9 +462,9 @@ private: return; } - string key = strutils::simplify(buffer.substr(0, colonPos)); - string lkey = boost::to_lower_copy(key); - string value = strutils::strip(buffer.substr(colonPos + 1)); + std::string key = strutils::simplify(buffer.substr(0, colonPos)); + std::string lkey = boost::to_lower_copy(key); + std::string value = strutils::strip(buffer.substr(colonPos + 1)); // only consider these if getting headers (as opposed to trailers // of a chunked transfer) @@ -466,7 +488,7 @@ private: activeRequest->responseHeader(lkey, value); } - void processTransferEncoding(const string& te) + void processTransferEncoding(const std::string& te) { if (te == "chunked") { chunkedTransfer = true; @@ -534,7 +556,7 @@ private: void responseComplete() { - Request_ptr completedRequest = activeRequest; + Request_ptr completedRequest = activeRequest; _contentDecoder.finish(); assert(sentRequests.front() == activeRequest); @@ -581,14 +603,13 @@ private: Client* client; Request_ptr activeRequest; ConnectionState state; - string host; + std::string host; short port; std::string buffer; int bodyTransferSize; SGTimeStamp idleTime; bool chunkedTransfer; bool noMessageBody; - int requestBodyBytesToSend; RequestList queuedRequests; RequestList sentRequests; @@ -669,6 +690,9 @@ void Client::update(int waitTimeout) void Client::makeRequest(const Request_ptr& r) { + if( r->isComplete() ) + return; + if( r->url().find("://") == std::string::npos ) { r->setFailure(EINVAL, "malformed URL"); return; @@ -679,7 +703,7 @@ void Client::makeRequest(const Request_ptr& r) return; } - string host = r->host(); + std::string host = r->host(); int port = r->port(); if (!d->proxy.empty()) { host = d->proxy; @@ -687,9 +711,9 @@ void Client::makeRequest(const Request_ptr& r) } Connection* con = NULL; - stringstream ss; + std::stringstream ss; ss << host << "-" << port; - string connectionId = ss.str(); + std::string connectionId = ss.str(); bool havePending = !d->pendingRequests.empty(); bool atConnectionsLimit = d->connections.size() >= d->maxConnections; ConnectionDict::iterator consEnd = d->connections.end(); @@ -741,12 +765,29 @@ void Client::makeRequest(const Request_ptr& r) con->queueRequest(r); } +//------------------------------------------------------------------------------ +FileRequestRef Client::urlretrieve( const std::string& url, + const std::string& filename ) +{ + FileRequestRef req = new FileRequest(url, filename); + makeRequest(req); + return req; +} + +//------------------------------------------------------------------------------ +MemoryRequestRef Client::urlload(const std::string& url) +{ + MemoryRequestRef req = new MemoryRequest(url); + makeRequest(req); + return req; +} + void Client::requestFinished(Connection* con) { } -void Client::setUserAgent(const string& ua) +void Client::setUserAgent(const std::string& ua) { d->userAgent = ua; } @@ -766,7 +807,9 @@ const std::string& Client::proxyAuth() const return d->proxyAuth; } -void Client::setProxy(const string& proxy, int port, const string& auth) +void Client::setProxy( const std::string& proxy, + int port, + const std::string& auth ) { d->proxy = proxy; d->proxyPort = port; diff --git a/simgear/io/HTTPClient.hxx b/simgear/io/HTTPClient.hxx index 0831d8ae..0ed210e5 100644 --- a/simgear/io/HTTPClient.hxx +++ b/simgear/io/HTTPClient.hxx @@ -27,7 +27,8 @@ #include // for std::auto_ptr #include // for uint_64t -#include +#include +#include namespace simgear { @@ -47,7 +48,24 @@ public: void update(int waitTimeout = 0); void makeRequest(const Request_ptr& r); - + + /** + * Download a resource and save it to a file. + * + * @param url The resource to download + * @param filename Path to the target file + * @param data Data for POST request + */ + FileRequestRef urlretrieve( const std::string& url, + const std::string& filename ); + + /** + * Request a resource and keep it in memory. + * + * @param url The resource to download + */ + MemoryRequestRef urlload(const std::string& url); + void setUserAgent(const std::string& ua); void setProxy(const std::string& proxy, int port, const std::string& auth = ""); diff --git a/simgear/io/HTTPFileRequest.cxx b/simgear/io/HTTPFileRequest.cxx new file mode 100644 index 00000000..e5cc71d6 --- /dev/null +++ b/simgear/io/HTTPFileRequest.cxx @@ -0,0 +1,82 @@ +// HTTP request writing response to a file. +// +// Copyright (C) 2013 Thomas Geymayer +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Library General Public +// License as published by the Free Software Foundation; either +// version 2 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU Library General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + +#include "HTTPFileRequest.hxx" +#include + +namespace simgear +{ +namespace HTTP +{ + + //---------------------------------------------------------------------------- + FileRequest::FileRequest(const std::string& url, const std::string& path): + Request(url, "GET"), + _filename(path) + { + + } + + //---------------------------------------------------------------------------- + void FileRequest::responseHeadersComplete() + { + Request::responseHeadersComplete(); + + if( !_filename.empty() ) + // TODO validate path? (would require to expose fgValidatePath somehow to + // simgear) + _file.open(_filename.c_str(), std::ios::binary | std::ios::trunc); + + if( !_file ) + { + SG_LOG + ( + SG_IO, + SG_WARN, + "HTTP::FileRequest: failed to open file '" << _filename << "'" + ); + + abort("Failed to open file."); + } + } + + //---------------------------------------------------------------------------- + void FileRequest::gotBodyData(const char* s, int n) + { + if( !_file ) + { + SG_LOG + ( + SG_IO, + SG_WARN, + "HTTP::FileRequest: error writing to '" << _filename << "'" + ); + return; + } + + _file.write(s, n); + } + + //---------------------------------------------------------------------------- + void FileRequest::onAlways() + { + _file.close(); + } + +} // namespace HTTP +} // namespace simgear diff --git a/simgear/io/HTTPFileRequest.hxx b/simgear/io/HTTPFileRequest.hxx new file mode 100644 index 00000000..8adc9003 --- /dev/null +++ b/simgear/io/HTTPFileRequest.hxx @@ -0,0 +1,56 @@ +///@file HTTP request writing response to a file. +// +// Copyright (C) 2013 Thomas Geymayer +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Library General Public +// License as published by the Free Software Foundation; either +// version 2 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU Library General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + +#ifndef SG_HTTP_FILEREQUEST_HXX_ +#define SG_HTTP_FILEREQUEST_HXX_ + +#include "HTTPRequest.hxx" +#include + +namespace simgear +{ +namespace HTTP +{ + + class FileRequest: + public Request + { + public: + + /** + * + * @param url Adress to download from + * @param path Path to file for saving response + */ + FileRequest(const std::string& url, const std::string& path); + + protected: + std::string _filename; + std::ofstream _file; + + virtual void responseHeadersComplete(); + virtual void gotBodyData(const char* s, int n); + virtual void onAlways(); + }; + + typedef SGSharedPtr FileRequestRef; + +} // namespace HTTP +} // namespace simgear + +#endif /* SG_HTTP_FILEREQUEST_HXX_ */ diff --git a/simgear/io/HTTPMemoryRequest.cxx b/simgear/io/HTTPMemoryRequest.cxx new file mode 100644 index 00000000..3ce32b29 --- /dev/null +++ b/simgear/io/HTTPMemoryRequest.cxx @@ -0,0 +1,55 @@ +// HTTP request keeping response in memory. +// +// Copyright (C) 2013 Thomas Geymayer +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Library General Public +// License as published by the Free Software Foundation; either +// version 2 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU Library General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + +#include "HTTPMemoryRequest.hxx" + +namespace simgear +{ +namespace HTTP +{ + + //---------------------------------------------------------------------------- + MemoryRequest::MemoryRequest(const std::string& url): + Request(url, "GET") + { + + } + + //---------------------------------------------------------------------------- + const std::string& MemoryRequest::responseBody() const + { + return _response; + } + + //---------------------------------------------------------------------------- + void MemoryRequest::responseHeadersComplete() + { + Request::responseHeadersComplete(); + + if( responseLength() ) + _response.reserve( responseLength() ); + } + + //---------------------------------------------------------------------------- + void MemoryRequest::gotBodyData(const char* s, int n) + { + _response.append(s, n); + } + +} // namespace HTTP +} // namespace simgear diff --git a/simgear/io/HTTPMemoryRequest.hxx b/simgear/io/HTTPMemoryRequest.hxx new file mode 100644 index 00000000..bea45d0f --- /dev/null +++ b/simgear/io/HTTPMemoryRequest.hxx @@ -0,0 +1,58 @@ +///@file HTTP request keeping response in memory. +// +// Copyright (C) 2013 Thomas Geymayer +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Library General Public +// License as published by the Free Software Foundation; either +// version 2 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Library General Public License for more details. +// +// You should have received a copy of the GNU Library General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + +#ifndef SG_HTTP_MEMORYREQUEST_HXX_ +#define SG_HTTP_MEMORYREQUEST_HXX_ + +#include "HTTPRequest.hxx" +#include + +namespace simgear +{ +namespace HTTP +{ + + class MemoryRequest: + public Request + { + public: + + /** + * + * @param url Adress to download from + */ + MemoryRequest(const std::string& url); + + /** + * Body contents of server response. + */ + const std::string& responseBody() const; + + protected: + std::string _response; + + virtual void responseHeadersComplete(); + virtual void gotBodyData(const char* s, int n); + }; + + typedef SGSharedPtr MemoryRequestRef; + +} // namespace HTTP +} // namespace simgear + +#endif /* SG_HTTP_MEMORYREQUEST_HXX_ */ diff --git a/simgear/io/HTTPRequest.cxx b/simgear/io/HTTPRequest.cxx index 299411dc..6b2a8290 100644 --- a/simgear/io/HTTPRequest.cxx +++ b/simgear/io/HTTPRequest.cxx @@ -1,59 +1,116 @@ #include "HTTPRequest.hxx" -#include #include #include - -using std::string; -using std::map; +#include +#include namespace simgear { - namespace HTTP { extern const int DEFAULT_HTTP_PORT; -Request::Request(const string& url, const string method) : - _method(method), - _url(url), - _responseVersion(HTTP_VERSION_UNKNOWN), - _responseStatus(0), - _responseLength(0), - _receivedBodyBytes(0), - _willClose(false) +//------------------------------------------------------------------------------ +Request::Request(const std::string& url, const std::string method): + _method(method), + _url(url), + _responseVersion(HTTP_VERSION_UNKNOWN), + _responseStatus(0), + _responseLength(0), + _receivedBodyBytes(0), + _ready_state(UNSENT), + _willClose(false) { - + } +//------------------------------------------------------------------------------ Request::~Request() { - + } -void Request::setUrl(const string& url) +//------------------------------------------------------------------------------ +Request* Request::done(const Callback& cb) { - _url = url; + if( _ready_state == DONE ) + cb(this); + else + _cb_done = cb; + + return this; } -string_list Request::requestHeaders() const +//------------------------------------------------------------------------------ +Request* Request::fail(const Callback& cb) { - string_list r; - return r; + if( _ready_state == FAILED ) + cb(this); + else + _cb_fail = cb; + + return this; +} + +//------------------------------------------------------------------------------ +Request* Request::always(const Callback& cb) +{ + if( isComplete() ) + cb(this); + else + _cb_always = cb; + + return this; +} + +//------------------------------------------------------------------------------ +void Request::setBodyData( const std::string& data, + const std::string& type ) +{ + _request_data = data; + _request_media_type = type; + + if( !data.empty() && _method == "GET" ) + _method = "POST"; +} + +//---------------------------------------------------------------------------- +void Request::setBodyData(const SGPropertyNode* data) +{ + if( !data ) + setBodyData(""); + + std::stringstream buf; + writeProperties(buf, data, true); + + setBodyData(buf.str(), "application/xml"); } -string Request::header(const std::string& name) const +//------------------------------------------------------------------------------ +void Request::setUrl(const std::string& url) { - return string(); + _url = url; } +//------------------------------------------------------------------------------ void Request::requestStart() { - + setReadyState(OPENED); +} + +//------------------------------------------------------------------------------ +Request::HTTPVersion decodeHTTPVersion(const std::string& v) +{ + if( v == "HTTP/1.1" ) return Request::HTTP_1_1; + if( v == "HTTP/1.0" ) return Request::HTTP_1_0; + if( strutils::starts_with(v, "HTTP/0.") ) return Request::HTTP_0_x; + return Request::HTTP_VERSION_UNKNOWN; } -void Request::responseStart(const string& r) +//------------------------------------------------------------------------------ +void Request::responseStart(const std::string& r) { const int maxSplit = 2; // HTTP/1.1 nnn reason-string string_list parts = strutils::split(r, NULL, maxSplit); @@ -63,42 +120,72 @@ void Request::responseStart(const string& r) return; } - _responseVersion = decodeVersion(parts[0]); + _responseVersion = decodeHTTPVersion(parts[0]); _responseStatus = strutils::to_int(parts[1]); _responseReason = parts[2]; } -void Request::responseHeader(const string& key, const string& value) +//------------------------------------------------------------------------------ +void Request::responseHeader(const std::string& key, const std::string& value) { - if (key == "connection") { - _willClose = (value.find("close") != string::npos); - } - - _responseHeaders[key] = value; + if( key == "connection" ) + _willClose = (value.find("close") != std::string::npos); + + _responseHeaders[key] = value; } +//------------------------------------------------------------------------------ void Request::responseHeadersComplete() { - // no op + setReadyState(HEADERS_RECEIVED); } -void Request::processBodyBytes(const char* s, int n) +//------------------------------------------------------------------------------ +void Request::responseComplete() { - _receivedBodyBytes += n; - gotBodyData(s, n); + if( !isComplete() ) + setReadyState(DONE); } +//------------------------------------------------------------------------------ void Request::gotBodyData(const char* s, int n) +{ + setReadyState(LOADING); +} + +//------------------------------------------------------------------------------ +void Request::onDone() { } -void Request::responseComplete() +//------------------------------------------------------------------------------ +void Request::onFail() { - + SG_LOG + ( + SG_IO, + SG_INFO, + "request failed:" << url() << " : " + << responseCode() << "/" << responseReason() + ); } - -string Request::scheme() const + +//------------------------------------------------------------------------------ +void Request::onAlways() +{ + +} + +//------------------------------------------------------------------------------ +void Request::processBodyBytes(const char* s, int n) +{ + _receivedBodyBytes += n; + gotBodyData(s, n); +} + +//------------------------------------------------------------------------------ +std::string Request::scheme() const { int firstColon = url().find(":"); if (firstColon > 0) { @@ -107,10 +194,11 @@ string Request::scheme() const return ""; // couldn't parse scheme } - -string Request::path() const + +//------------------------------------------------------------------------------ +std::string Request::path() const { - string u(url()); + std::string u(url()); int schemeEnd = u.find("://"); if (schemeEnd < 0) { return ""; // couldn't parse scheme @@ -132,10 +220,10 @@ string Request::path() const return u.substr(hostEnd, query - hostEnd); } - -string Request::query() const +//------------------------------------------------------------------------------ +std::string Request::query() const { - string u(url()); + std::string u(url()); int query = u.find('?'); if (query < 0) { return ""; //no query string found @@ -144,104 +232,153 @@ string Request::query() const return u.substr(query); //includes question mark } - - -string Request::host() const +//------------------------------------------------------------------------------ +std::string Request::host() const { - string hp(hostAndPort()); - int colonPos = hp.find(':'); - if (colonPos >= 0) { - return hp.substr(0, colonPos); // trim off the colon and port - } else { - return hp; // no port specifier - } + std::string hp(hostAndPort()); + int colonPos = hp.find(':'); + if (colonPos >= 0) { + return hp.substr(0, colonPos); // trim off the colon and port + } else { + return hp; // no port specifier + } } +//------------------------------------------------------------------------------ unsigned short Request::port() const { - string hp(hostAndPort()); - int colonPos = hp.find(':'); - if (colonPos >= 0) { - return (unsigned short) strutils::to_int(hp.substr(colonPos + 1)); - } else { - return DEFAULT_HTTP_PORT; - } + std::string hp(hostAndPort()); + int colonPos = hp.find(':'); + if (colonPos >= 0) { + return (unsigned short) strutils::to_int(hp.substr(colonPos + 1)); + } else { + return DEFAULT_HTTP_PORT; + } } -string Request::hostAndPort() const +//------------------------------------------------------------------------------ +std::string Request::hostAndPort() const { - string u(url()); - int schemeEnd = u.find("://"); - if (schemeEnd < 0) { - return ""; // couldn't parse scheme - } - - int hostEnd = u.find('/', schemeEnd + 3); - if (hostEnd < 0) { // all remainder of URL is host - return u.substr(schemeEnd + 3); - } - - return u.substr(schemeEnd + 3, hostEnd - (schemeEnd + 3)); + std::string u(url()); + int schemeEnd = u.find("://"); + if (schemeEnd < 0) { + return ""; // couldn't parse scheme + } + + int hostEnd = u.find('/', schemeEnd + 3); + if (hostEnd < 0) { // all remainder of URL is host + return u.substr(schemeEnd + 3); + } + + return u.substr(schemeEnd + 3, hostEnd - (schemeEnd + 3)); } +//------------------------------------------------------------------------------ void Request::setResponseLength(unsigned int l) { - _responseLength = l; + _responseLength = l; } +//------------------------------------------------------------------------------ unsigned int Request::responseLength() const { -// if the server didn't supply a content length, use the number -// of bytes we actually received (so far) - if ((_responseLength == 0) && (_receivedBodyBytes > 0)) { - return _receivedBodyBytes; - } - - return _responseLength; + // if the server didn't supply a content length, use the number + // of bytes we actually received (so far) + if( (_responseLength == 0) && (_receivedBodyBytes > 0) ) + return _receivedBodyBytes; + + return _responseLength; } +//------------------------------------------------------------------------------ void Request::setFailure(int code, const std::string& reason) { - _responseStatus = code; - _responseReason = reason; - failed(); + _responseStatus = code; + _responseReason = reason; + setReadyState(FAILED); } -void Request::failed() +//------------------------------------------------------------------------------ +void Request::setReadyState(ReadyState state) { - SG_LOG(SG_IO, SG_INFO, "request failed:" << url() << " : " - << responseCode() << "/" << responseReason()); + _ready_state = state; + if( state == DONE ) + { + if( _cb_done ) + _cb_done(this); + onDone(); + } + else if( state == FAILED ) + { + if( _cb_fail ) + _cb_fail(this); + onFail(); + } + else + return; + + if( _cb_always ) + _cb_always(this); + onAlways(); } -Request::HTTPVersion Request::decodeVersion(const string& v) +//------------------------------------------------------------------------------ +void Request::abort() { - if (v == "HTTP/1.1") return HTTP_1_1; - if (v == "HTTP/1.0") return HTTP_1_0; - if (strutils::starts_with(v, "HTTP/0.")) return HTTP_0_x; - return HTTP_VERSION_UNKNOWN; + abort("Request aborted."); } +//---------------------------------------------------------------------------- +void Request::abort(const std::string& reason) +{ + if( isComplete() ) + return; + + setFailure(-1, reason); + _willClose = true; +} + +//------------------------------------------------------------------------------ bool Request::closeAfterComplete() const { -// for non HTTP/1.1 connections, assume server closes - return _willClose || (_responseVersion != HTTP_1_1); + // for non HTTP/1.1 connections, assume server closes + return _willClose || (_responseVersion != HTTP_1_1); } - -int Request::requestBodyLength() const + +//------------------------------------------------------------------------------ +bool Request::isComplete() const { - return -1; + return _ready_state == DONE || _ready_state == FAILED; } -std::string Request::requestBodyType() const +//------------------------------------------------------------------------------ +bool Request::hasBodyData() const { - return "text/plain"; + return !_request_media_type.empty(); } - -int Request::getBodyData(char*, int maxCount) const + +//------------------------------------------------------------------------------ +std::string Request::bodyType() const { - return 0; + return _request_media_type; } -} // of namespace HTTP +//------------------------------------------------------------------------------ +size_t Request::bodyLength() const +{ + return _request_data.length(); +} +//------------------------------------------------------------------------------ +size_t Request::getBodyData(char* s, size_t offset, size_t max_count) const +{ + size_t bytes_available = _request_data.size() - offset; + size_t bytes_to_read = std::min(bytes_available, max_count); + + memcpy(s, _request_data.data() + offset, bytes_to_read); + + return bytes_to_read; +} + +} // of namespace HTTP } // of namespace simgear diff --git a/simgear/io/HTTPRequest.hxx b/simgear/io/HTTPRequest.hxx index f4939d32..89a953bc 100644 --- a/simgear/io/HTTPRequest.hxx +++ b/simgear/io/HTTPRequest.hxx @@ -3,21 +3,85 @@ #include +#include #include #include #include +#include + +class SGPropertyNode; + namespace simgear { - namespace HTTP { -class Request : public SGReferenced +class Request: + public SGReferenced { public: + typedef boost::function Callback; + + enum ReadyState + { + UNSENT = 0, + OPENED, + HEADERS_RECEIVED, + LOADING, + DONE, + FAILED + }; + virtual ~Request(); - + + /** + * + */ + StringMap& requestHeaders() { return _request_headers; } + const StringMap& requestHeaders() const { return _request_headers; } + std::string& requestHeader(const std::string& key) + { return _request_headers[key]; } + const std::string requestHeader(const std::string& key) const + { return _request_headers.get(key); } + + /** + * Set the handler to be called when the request successfully completes. + * + * @note If the request is already complete, the handler is called + * immediately. + */ + Request* done(const Callback& cb); + + /** + * Set the handler to be called when the request completes or aborts with an + * error. + * + * @note If the request has already failed, the handler is called + * immediately. + */ + Request* fail(const Callback& cb); + + /** + * Set the handler to be called when the request either successfully + * completes or fails. + * + * @note If the request is already complete or has already failed, the + * handler is called immediately. + */ + Request* always(const Callback& cb); + + /** + * Set the data for the body of the request. The request is automatically + * send using the POST method. + * + * @param data Body data + * @param type Media Type (aka MIME) of the body data + */ + void setBodyData( const std::string& data, + const std::string& type = "text/plain" ); + void setBodyData( const SGPropertyNode* data ); + virtual void setUrl(const std::string& url); virtual std::string method() const @@ -32,8 +96,8 @@ public: virtual unsigned short port() const; virtual std::string query() const; - virtual string_list requestHeaders() const; - virtual std::string header(const std::string& name) const; + StringMap const& responseHeaders() const + { return _responseHeaders; } virtual int responseCode() const { return _responseStatus; } @@ -41,26 +105,30 @@ public: virtual std::string responseReason() const { return _responseReason; } - void setResponseLength(unsigned int l); + void setResponseLength(unsigned int l); virtual unsigned int responseLength() const; /** - * Query the size of the request body. -1 (the default value) means no - * request body + * Check if request contains body data. + */ + virtual bool hasBodyData() const; + + /** + * Retrieve the request body content type. + */ + virtual std::string bodyType() const; + + /** + * Retrieve the size of the request body. */ - virtual int requestBodyLength() const; + virtual size_t bodyLength() const; /** * Retrieve the body data bytes. Will be passed the maximum body bytes * to return in the buffer, and must return the actual number * of bytes written. */ - virtual int getBodyData(char* s, int count) const; - - /** - * retrieve the request body content type. Default is text/plain - */ - virtual std::string requestBodyType() const; + virtual size_t getBodyData(char* s, size_t offset, size_t max_count) const; /** * running total of body bytes received so far. Can be used @@ -80,9 +148,21 @@ public: HTTPVersion responseVersion() const { return _responseVersion; } - static HTTPVersion decodeVersion(const std::string& v); - + ReadyState readyState() const { return _ready_state; } + + /** + * Request aborting this request. + */ + void abort(); + + /** + * Request aborting this request and specify the reported reaseon for it. + */ + void abort(const std::string& reason); + bool closeAfterComplete() const; + bool isComplete() const; + protected: Request(const std::string& url, const std::string method = "GET"); @@ -91,34 +171,48 @@ protected: virtual void responseHeader(const std::string& key, const std::string& value); virtual void responseHeadersComplete(); virtual void responseComplete(); - virtual void failed(); virtual void gotBodyData(const char* s, int n); + + virtual void onDone(); + virtual void onFail(); + virtual void onAlways(); + private: friend class Client; friend class Connection; friend class ContentDecoder; + Request(const Request&); // = delete; + Request& operator=(const Request&); // = delete; + void processBodyBytes(const char* s, int n); void setFailure(int code, const std::string& reason); + void setReadyState(ReadyState state); - std::string _method; - std::string _url; - HTTPVersion _responseVersion; - int _responseStatus; - std::string _responseReason; - unsigned int _responseLength; - unsigned int _receivedBodyBytes; - bool _willClose; - - typedef std::map HeaderDict; - HeaderDict _responseHeaders; + std::string _method; + std::string _url; + StringMap _request_headers; + std::string _request_data; + std::string _request_media_type; + + HTTPVersion _responseVersion; + int _responseStatus; + std::string _responseReason; + StringMap _responseHeaders; + unsigned int _responseLength; + unsigned int _receivedBodyBytes; + + Callback _cb_done, + _cb_fail, + _cb_always; + + ReadyState _ready_state; + bool _willClose; }; typedef SGSharedPtr Request_ptr; } // of namespace HTTP - } // of namespace simgear #endif // of SG_HTTP_REQUEST_HXX - diff --git a/simgear/io/SVNRepository.cxx b/simgear/io/SVNRepository.cxx index b74df28c..a896b3e9 100644 --- a/simgear/io/SVNRepository.cxx +++ b/simgear/io/SVNRepository.cxx @@ -119,40 +119,9 @@ namespace { // anonmouse Request(repo->baseUrl, "PROPFIND"), _repo(repo) { - } - - virtual string_list requestHeaders() const - { - string_list r; - r.push_back("Depth"); - return r; - } - - virtual string header(const string& name) const - { - if (name == "Depth") { - return "0"; - } - - return string(); - } - - virtual string requestBodyType() const - { - return "application/xml; charset=\"utf-8\""; - } - - virtual int requestBodyLength() const - { - return strlen(PROPFIND_REQUEST_BODY); - } - - virtual int getBodyData(char* buf, int count) const - { - int bodyLen = strlen(PROPFIND_REQUEST_BODY); - assert(count >= bodyLen); - memcpy(buf, PROPFIND_REQUEST_BODY, bodyLen); - return bodyLen; + requestHeader("Depth") = "0"; + setBodyData( PROPFIND_REQUEST_BODY, + "application/xml; charset=\"utf-8\"" ); } protected: @@ -169,7 +138,7 @@ namespace { // anonmouse } } - virtual void responseComplete() + virtual void onDone() { if (responseCode() == 207) { _davStatus.finishParse(); @@ -189,9 +158,9 @@ namespace { // anonmouse _davStatus.parseXML(s, n); } - virtual void failed() + virtual void onFail() { - HTTP::Request::failed(); + HTTP::Request::onFail(); _repo->propFindFailed(this, SVNRepository::SVN_ERROR_SOCKET); } @@ -200,64 +169,43 @@ namespace { // anonmouse DAVMultiStatus _davStatus; }; -class UpdateReportRequest : public HTTP::Request +class UpdateReportRequest: + public HTTP::Request { public: UpdateReportRequest(SVNRepoPrivate* repo, const std::string& aVersionName, bool startEmpty) : HTTP::Request("", "REPORT"), - _requestSent(0), _parser(repo->p), _repo(repo), _failed(false) { setUrl(repo->vccUrl); - - _request = + std::string request = "\n" "\n" "" + repo->baseUrl + "\n" - "unknown\n"; - - _request += "\n"; - - if (!startEmpty) { - string_list entries; - _repo->rootCollection->mergeUpdateReportDetails(0, entries); - BOOST_FOREACH(string e, entries) { - _request += e + "\n"; - } - } + "unknown\n" + "\n"; - _request += ""; - } - - virtual string requestBodyType() const - { - return "application/xml; charset=\"utf-8\""; - } + if( !startEmpty ) + { + string_list entries; + _repo->rootCollection->mergeUpdateReportDetails(0, entries); + BOOST_FOREACH(string e, entries) + { + request += e + "\n"; + } + } - virtual int requestBodyLength() const - { - return _request.size(); - } + request += ""; - virtual int getBodyData(char* buf, int count) const - { - int len = std::min(count, requestBodyLength() - _requestSent); - memcpy(buf, _request.c_str() + _requestSent, len); - _requestSent += len; - return len; + setBodyData(request, "application/xml; charset=\"utf-8\""); } protected: - virtual void responseHeadersComplete() - { - - } - - virtual void responseComplete() + virtual void onDone() { if (_failed) { return; @@ -300,14 +248,12 @@ protected: } } - virtual void failed() + virtual void onFail() { - HTTP::Request::failed(); + HTTP::Request::onFail(); _repo->updateFailed(this, SVNRepository::SVN_ERROR_SOCKET); } private: - string _request; - mutable int _requestSent; SVNReportParser _parser; SVNRepoPrivate* _repo; bool _failed; diff --git a/simgear/io/httpget.cxx b/simgear/io/httpget.cxx index 3b0506c5..876ba14f 100644 --- a/simgear/io/httpget.cxx +++ b/simgear/io/httpget.cxx @@ -48,35 +48,11 @@ public: } string key = h.substr(0, colonPos); - _headers[key] = h.substr(colonPos + 1); + requestHeader(key) = h.substr(colonPos + 1); } - virtual string_list requestHeaders() const - { - string_list r; - std::map::const_iterator it; - for (it = _headers.begin(); it != _headers.end(); ++it) { - r.push_back(it->first); - } - - return r; - } - - virtual string header(const string& name) const - { - std::map::const_iterator it = _headers.find(name); - if (it == _headers.end()) { - return string(); - } - - return it->second; - } protected: - virtual void responseHeadersComplete() - { - } - - virtual void responseComplete() + virtual void onDone() { _complete = true; } @@ -88,7 +64,6 @@ protected: private: bool _complete; SGFile* _file; - std::map _headers; }; int main(int argc, char* argv[]) diff --git a/simgear/io/sg_netChat.hxx b/simgear/io/sg_netChat.hxx index 6b227f89..3a5c08a6 100644 --- a/simgear/io/sg_netChat.hxx +++ b/simgear/io/sg_netChat.hxx @@ -70,6 +70,8 @@ class NetChat : public NetBufferChannel { std::string terminator; int bytesToCollect; + +protected: virtual void handleBufferRead (NetBuffer& buffer) ; public: diff --git a/simgear/io/test_HTTP.cxx b/simgear/io/test_HTTP.cxx index 442e9ffd..1849434d 100644 --- a/simgear/io/test_HTTP.cxx +++ b/simgear/io/test_HTTP.cxx @@ -53,48 +53,23 @@ public: bool complete; bool failed; string bodyData; - string bodyContentType; - + TestRequest(const std::string& url, const std::string method = "GET") : HTTP::Request(url, method), complete(false) { - bodyContentType = "text/plain"; + } - std::map sendHeaders; std::map headers; protected: - string_list requestHeaders() const - { - string_list r; - std::map::const_iterator it; - for (it = sendHeaders.begin(); it != sendHeaders.end(); ++it) { - r.push_back(it->first); - } - return r; - } - string header(const string& name) const - { - std::map::const_iterator it = sendHeaders.find(name); - if (it == sendHeaders.end()) { - return string(); - } - - return it->second; - } - - virtual void responseHeadersComplete() - { - } - - virtual void responseComplete() + virtual void onDone() { complete = true; } - virtual void failure() + virtual void onFail() { failed = true; } @@ -105,11 +80,6 @@ protected: bodyData += string(s, n); } - virtual std::string requestBodyType() const - { - return bodyContentType; - } - virtual void responseHeader(const string& header, const string& value) { headers[header] = value; @@ -535,8 +505,8 @@ int main(int argc, char* argv[]) { TestRequest* tr = new TestRequest("http://localhost:2000/test_headers"); HTTP::Request_ptr own(tr); - tr->sendHeaders["X-Foo"] = "Bar"; - tr->sendHeaders["X-AnotherHeader"] = "A longer value"; + tr->requestHeader("X-Foo") = "Bar"; + tr->requestHeader("X-AnotherHeader") = "A longer value"; cl.makeRequest(tr); waitForComplete(&cl, tr); @@ -721,7 +691,7 @@ int main(int argc, char* argv[]) { cout << "POST" << endl; TestRequest* tr = new TestRequest("http://localhost:2000/test_post?foo=abc&bar=1234&username=johndoe", "POST"); - tr->bodyContentType = "application/x-www-form-urlencoded"; + tr->setBodyData("", "application/x-www-form-urlencoded"); HTTP::Request_ptr own(tr); cl.makeRequest(tr); diff --git a/simgear/package/Catalog.cxx b/simgear/package/Catalog.cxx index eede6066..160d2fbf 100644 --- a/simgear/package/Catalog.cxx +++ b/simgear/package/Catalog.cxx @@ -50,17 +50,12 @@ public: } protected: - virtual void responseHeadersComplete() - { - - } - virtual void gotBodyData(const char* s, int n) { m_buffer += std::string(s, n); } - virtual void responseComplete() + virtual void onDone() { if (responseCode() != 200) { SG_LOG(SG_GENERAL, SG_ALERT, "catalog download failure:" << m_owner->url()); diff --git a/simgear/package/Install.cxx b/simgear/package/Install.cxx index df0b24b0..d513786c 100644 --- a/simgear/package/Install.cxx +++ b/simgear/package/Install.cxx @@ -81,7 +81,7 @@ protected: m_owner->installProgress(m_buffer.size(), responseLength()); } - virtual void responseComplete() + virtual void onDone() { if (responseCode() != 200) { SG_LOG(SG_GENERAL, SG_ALERT, "download failure"); -- 2.39.5