X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=simgear%2Fio%2FHTTPRepository.cxx;h=dc4dc6eae999d9cf4fbdd74f04bfa4687fa36bb4;hb=974cb3b3d31bb687b0dda2e37960394ee5f1b6e5;hp=477232eacbe55bb6c7f12c0d1503722c9557105c;hpb=4eb272f4f080bc447b364ed5f452ff9dc9bf74ef;p=simgear.git

diff --git a/simgear/io/HTTPRepository.cxx b/simgear/io/HTTPRepository.cxx
index 477232ea..dc4dc6ea 100644
--- a/simgear/io/HTTPRepository.cxx
+++ b/simgear/io/HTTPRepository.cxx
@@ -21,7 +21,6 @@
 #include <simgear_config.h>
 
 #include <iostream>
-#include <cstring>
 #include <cassert>
 #include <algorithm>
 #include <sstream>
@@ -40,56 +39,10 @@
 #include <simgear/io/sg_file.hxx>
 #include <simgear/misc/sgstream.hxx>
 #include <simgear/structure/exception.hxx>
+#include <simgear/timing/timestamp.hxx>
 
 #include <simgear/misc/sg_hash.hxx>
 
-#if defined(SG_WINDOWS)
-
-/*
- * public domain strtok_r() by Charlie Gordon
- *
- *   from comp.lang.c  9/14/2007
- *
- *      http://groups.google.com/group/comp.lang.c/msg/2ab1ecbb86646684
- *
- *     (Declaration that it's public domain):
- *      http://groups.google.com/group/comp.lang.c/msg/7c7b39328fefab9c
- */
-
-char* strtok_r(
-    char *str,
-    const char *delim,
-    char **nextp)
-{
-    char *ret;
-
-    if (str == NULL)
-    {
-        str = *nextp;
-    }
-
-    str += strspn(str, delim);
-
-    if (*str == '\0')
-    {
-        return NULL;
-    }
-
-    ret = str;
-
-    str += strcspn(str, delim);
-
-    if (*str)
-    {
-        *str++ = '\0';
-    }
-
-    *nextp = str;
-
-    return ret;
-}
-#endif
-
 namespace simgear
 {
 
@@ -104,13 +57,20 @@ namespace simgear
         {
         }
 
-        virtual void cancel()
+        virtual void cancel();
+
+        size_t contentSize() const
         {
-            _directory = 0;
-            abort("Repository cancelled request");
+            return _contentSize;
+        }
+
+        void setContentSize(size_t sz)
+        {
+            _contentSize = sz;
         }
     protected:
         HTTPDirectory* _directory;
+        size_t _contentSize;
     };
 
     typedef SGSharedPtr<HTTPRepoGetRequest> RepoRequestPtr;
@@ -129,20 +89,23 @@ public:
 
     typedef std::vector<HashCacheEntry> HashCache;
     HashCache hashes;
+    bool hashCacheDirty;
 
     struct Failure
     {
         SGPath path;
-        AbstractRepository::ResultCode error;
+        HTTPRepository::ResultCode error;
     };
 
     typedef std::vector<Failure> FailureList;
     FailureList failures;
 
     HTTPRepoPrivate(HTTPRepository* parent) :
-    p(parent),
-    isUpdating(false),
-    status(AbstractRepository::REPO_NO_ERROR)
+        hashCacheDirty(false),
+        p(parent),
+        isUpdating(false),
+        status(HTTPRepository::REPO_NO_ERROR),
+        totalDownloaded(0)
     { ; }
 
     ~HTTPRepoPrivate();
@@ -152,11 +115,14 @@ public:
     std::string baseUrl;
     SGPath basePath;
     bool isUpdating;
-    AbstractRepository::ResultCode status;
+    HTTPRepository::ResultCode status;
     HTTPDirectory* rootDir;
+    size_t totalDownloaded;
 
-    HTTP::Request_ptr updateFile(HTTPDirectory* dir, const std::string& name);
-    HTTP::Request_ptr updateDir(HTTPDirectory* dir, const std::string& hash);
+    HTTP::Request_ptr updateFile(HTTPDirectory* dir, const std::string& name,
+                                 size_t sz);
+    HTTP::Request_ptr updateDir(HTTPDirectory* dir, const std::string& hash,
+                                size_t sz);
 
     std::string hashForPath(const SGPath& p);
     void updatedFileContents(const SGPath& p, const std::string& newHash);
@@ -164,13 +130,15 @@ public:
     std::string computeHashForPath(const SGPath& p);
     void writeHashCache();
 
-    void failedToGetRootIndex(AbstractRepository::ResultCode st);
+    void failedToGetRootIndex(HTTPRepository::ResultCode st);
     void failedToUpdateChild(const SGPath& relativePath,
-                             AbstractRepository::ResultCode fileStatus);
+                             HTTPRepository::ResultCode fileStatus);
 
     typedef std::vector<RepoRequestPtr> RequestVector;
-    RequestVector requests;
+    RequestVector queuedRequests,
+        activeRequests;
 
+    void makeRequest(RepoRequestPtr req);
     void finishedRequest(const RepoRequestPtr& req);
 
     HTTPDirectory* getOrCreateDirectory(const std::string& path);
@@ -191,10 +159,10 @@ class HTTPDirectory
             DirectoryType
         };
 
-        ChildInfo(Type ty, const char* nameData, const char* hashData) :
+        ChildInfo(Type ty, const std::string & nameData, const std::string & hashData) :
             type(ty),
             name(nameData),
-            hash(hashData ? hashData : ""),
+            hash(hashData),
             sizeInBytes(0)
         {
         }
@@ -206,9 +174,9 @@ class HTTPDirectory
             sizeInBytes(other.sizeInBytes)
         { }
 
-        void setSize(const char* sizeData)
+        void setSize(const std::string & sizeData)
         {
-            sizeInBytes = ::strtol(sizeData, NULL, 10);
+            sizeInBytes = ::strtol(sizeData.c_str(), NULL, 10);
         }
 
         bool operator<(const ChildInfo& other) const
@@ -235,9 +203,9 @@ public:
       if (p.exists()) {
           try {
               // already exists on disk
-              bool ok = parseDirIndex(children);
+              parseDirIndex(children);
               std::sort(children.begin(), children.end());
-          } catch (sg_exception& e) {
+          } catch (sg_exception& ) {
               // parsing cache failed
               children.clear();
           }
@@ -269,7 +237,7 @@ public:
         std::sort(children.begin(), children.end());
     }
 
-    void failedToUpdate(AbstractRepository::ResultCode status)
+    void failedToUpdate(HTTPRepository::ResultCode status)
     {
         if (_relativePath.isNull()) {
             // root dir failed
@@ -289,28 +257,32 @@ public:
         PathList fsChildren = d.children(0);
         PathList::const_iterator it = fsChildren.begin();
 
+
         for (; it != fsChildren.end(); ++it) {
             ChildInfo info(it->isDir() ? ChildInfo::DirectoryType : ChildInfo::FileType,
-                           it->file().c_str(), NULL);
+                           it->file(), "");
             std::string hash = hashForChild(info);
 
             ChildInfoList::iterator c = findIndexChild(it->file());
             if (c == children.end()) {
+                SG_LOG(SG_TERRASYNC, SG_DEBUG, "is orphan '" << it->file() << "'" );
                 orphans.push_back(it->file());
             } else if (c->hash != hash) {
+                SG_LOG(SG_TERRASYNC, SG_DEBUG, "hash mismatch'" << it->file() );
                 // file exists, but hash mismatch, schedule update
                 if (!hash.empty()) {
-                    //SG_LOG(SG_TERRASYNC, SG_INFO, "file exists but hash is wrong for:" << c->name);
-                    //SG_LOG(SG_TERRASYNC, SG_INFO, "on disk:" << hash << " vs in info:" << c->hash);
+                    SG_LOG(SG_TERRASYNC, SG_DEBUG, "file exists but hash is wrong for:" << it->file() );
+                    SG_LOG(SG_TERRASYNC, SG_DEBUG, "on disk:" << hash << " vs in info:" << c->hash);
                 }
 
-                toBeUpdated.push_back(c->name);
+                toBeUpdated.push_back(it->file() );
             } else {
                 // file exists and hash is valid. If it's a directory,
                 // perform a recursive check.
+                SG_LOG(SG_TERRASYNC, SG_DEBUG, "file exists hash is good:" << it->file() );
                 if (c->type == ChildInfo::DirectoryType) {
                     SGPath p(relativePath());
-                    p.append(c->name);
+                    p.append(it->file());
                     HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str());
                     childDir->updateChildrenBasedOnHash();
                 }
@@ -318,10 +290,8 @@ public:
 
             // remove existing file system children from the index list,
             // so we can detect new children
-            string_list::iterator it = std::find(indexNames.begin(), indexNames.end(), c->name);
-            if (it != indexNames.end()) {
-                indexNames.erase(it);
-            }
+            // https://en.wikibooks.org/wiki/More_C%2B%2B_Idioms/Erase-Remove
+            indexNames.erase(std::remove(indexNames.begin(), indexNames.end(), it->file()), indexNames.end());
         } // of real children iteration
 
         // all remaining names in indexChilden are new children
@@ -360,13 +330,14 @@ public:
                 continue;
             }
 
+            SG_LOG(SG_TERRASYNC,SG_DEBUG, "scheduling update for " << *it );
             if (cit->type == ChildInfo::FileType) {
-                _repository->updateFile(this, *it);
+                _repository->updateFile(this, *it, cit->sizeInBytes);
             } else {
                 SGPath p(relativePath());
                 p.append(*it);
                 HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str());
-                _repository->updateDir(childDir, cit->hash);
+                _repository->updateDir(childDir, cit->hash, cit->sizeInBytes);
             }
         }
     }
@@ -383,7 +354,7 @@ public:
         return _relativePath;
     }
 
-    void didUpdateFile(const std::string& file, const std::string& hash)
+    void didUpdateFile(const std::string& file, const std::string& hash, size_t sz)
     {
         // check hash matches what we expected
         ChildInfoList::iterator it = findIndexChild(file);
@@ -394,16 +365,17 @@ public:
             fpath.append(file);
 
             if (it->hash != hash) {
-                _repository->failedToUpdateChild(_relativePath, AbstractRepository::REPO_ERROR_CHECKSUM);
+                _repository->failedToUpdateChild(_relativePath, HTTPRepository::REPO_ERROR_CHECKSUM);
             } else {
                 _repository->updatedFileContents(fpath, hash);
+                _repository->totalDownloaded += sz;
                 //SG_LOG(SG_TERRASYNC, SG_INFO, "did update:" << fpath);
             } // of hash matches
         } // of found in child list
     }
 
     void didFailToUpdateFile(const std::string& file,
-                             AbstractRepository::ResultCode status)
+                             HTTPRepository::ResultCode status)
     {
         SGPath fpath(_relativePath);
         fpath.append(file);
@@ -439,42 +411,48 @@ private:
             throw sg_io_exception("cannot open dirIndex file", p);
         }
 
-        char lineBuffer[512];
-        char* lastToken;
-
         while (!indexStream.eof() ) {
-            indexStream.getline(lineBuffer, 512);
-            lastToken = 0;
-            char* typeData = ::strtok_r(lineBuffer, ":", &lastToken);
-            if (!typeData) {
-                continue; // skip blank line
+            std::string line;
+            std::getline( indexStream, line );
+            line = simgear::strutils::strip(line);
+
+            // skip blank line or comment beginning with '#'
+            if( line.empty() || line[0] == '#' )
+                continue;
+
+            string_list tokens = simgear::strutils::split( line, ":" );
+
+            std::string typeData = tokens[0];
+
+            if( typeData == "version" ) {
+                if( tokens.size() < 2 ) {
+                    SG_LOG(SG_TERRASYNC, SG_WARN, "malformed .dirindex file: missing version number in line '" << line << "'" );
+                    break;
+                }
+                if( tokens[1] != "1" ) {
+                    SG_LOG(SG_TERRASYNC, SG_WARN, "invalid .dirindex file: wrong version number '" << tokens[1] << "' (expected 1)" );
+                    break;
+                }
+                continue; // version is good, continue
             }
 
-            if (!typeData) {
-                // malformed entry
-                throw sg_io_exception("Malformed dir index file", p);
+            if( typeData == "path" ) {
+                continue; // ignore path, next line
             }
 
-            if (!strcmp(typeData, "version")) {
-                continue;
-            } else if (!strcmp(typeData, "path")) {
+            if( tokens.size() < 3 ) {
+                SG_LOG(SG_TERRASYNC, SG_WARN, "malformed .dirindex file: not enough tokens in line '" << line << "' (ignoring line)" );
                 continue;
             }
 
-            char* nameData = ::strtok_r(NULL, ":", &lastToken);
-            char* hashData = ::strtok_r(NULL, ":", &lastToken);
-            char* sizeData = ::strtok_r(NULL, ":", &lastToken);
-
-            if (typeData[0] == 'f') {
-                children.push_back(ChildInfo(ChildInfo::FileType, nameData, hashData));
-            } else if (typeData[0] == 'd') {
-                children.push_back(ChildInfo(ChildInfo::DirectoryType, nameData, hashData));
-            } else {
-                throw sg_io_exception("Malformed line code in dir index file", p);
+            if (typeData != "f" && typeData != "d" ) {
+                SG_LOG(SG_TERRASYNC, SG_WARN, "malformed .dirindex file: invalid type in line '" << line << "', expected 'd' or 'f', (ignoring line)" );
+                continue;
             }
+            children.push_back(ChildInfo(typeData == "f" ? ChildInfo::FileType : ChildInfo::DirectoryType, tokens[1], tokens[2]));
 
-            if (sizeData) {
-                children.back().setSize(sizeData);
+            if (tokens.size() > 3) {
+                children.back().setSize(tokens[3]);
             }
         }
 
@@ -526,6 +504,7 @@ HTTPRepository::HTTPRepository(const SGPath& base, HTTP::Client *cl) :
     _d->http = cl;
     _d->basePath = base;
     _d->rootDir = new HTTPDirectory(_d.get(), "");
+    _d->parseHashCache();
 }
 
 HTTPRepository::~HTTPRepository()
@@ -561,7 +540,7 @@ void HTTPRepository::update()
     _d->status = REPO_NO_ERROR;
     _d->isUpdating = true;
     _d->failures.clear();
-    _d->updateDir(_d->rootDir, std::string());
+    _d->updateDir(_d->rootDir, std::string(), 0);
 }
 
 bool HTTPRepository::isDoingSync() const
@@ -573,7 +552,35 @@ bool HTTPRepository::isDoingSync() const
     return _d->isUpdating;
 }
 
-AbstractRepository::ResultCode
+size_t HTTPRepository::bytesToDownload() const
+{
+    size_t result = 0;
+
+    HTTPRepoPrivate::RequestVector::const_iterator r;
+    for (r = _d->queuedRequests.begin(); r != _d->queuedRequests.end(); ++r) {
+        result += (*r)->contentSize();
+    }
+
+    for (r = _d->activeRequests.begin(); r != _d->activeRequests.end(); ++r) {
+        result += (*r)->contentSize() - (*r)->responseBytesReceived();
+    }
+
+    return result;
+}
+
+size_t HTTPRepository::bytesDownloaded() const
+{
+    size_t result = _d->totalDownloaded;
+
+    HTTPRepoPrivate::RequestVector::const_iterator r;
+    for (r = _d->activeRequests.begin(); r != _d->activeRequests.end(); ++r) {
+        result += (*r)->responseBytesReceived();
+    }
+
+    return result;
+}
+
+HTTPRepository::ResultCode
 HTTPRepository::failure() const
 {
     if ((_d->status == REPO_NO_ERROR) && !_d->failures.empty()) {
@@ -583,6 +590,12 @@ HTTPRepository::failure() const
     return _d->status;
 }
 
+    void HTTPRepoGetRequest::cancel()
+    {
+        _directory->repository()->http->cancelRequest(this, "Reposiotry cancelled");
+        _directory = 0;
+    }
+
     class FileGetRequest : public HTTPRepoGetRequest
     {
     public:
@@ -599,10 +612,10 @@ HTTPRepository::failure() const
         virtual void gotBodyData(const char* s, int n)
         {
             if (!file.get()) {
-                file.reset(new SGFile(pathInRepo.str()));
+                file.reset(new SGBinaryFile(pathInRepo.str()));
                 if (!file->open(SG_IO_OUT)) {
                   SG_LOG(SG_TERRASYNC, SG_WARN, "unable to create file " << pathInRepo);
-                  abort("Unable to create output file");
+                  _directory->repository()->http->cancelRequest(this, "Unable to create output file");
                 }
 
                 sha1_init(&hashContext);
@@ -617,12 +630,14 @@ HTTPRepository::failure() const
             file->close();
             if (responseCode() == 200) {
                 std::string hash = strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
-                _directory->didUpdateFile(fileName, hash);
-                //SG_LOG(SG_TERRASYNC, SG_INFO, "got file " << fileName << " in " << _directory->absolutePath());
+                _directory->didUpdateFile(fileName, hash, contentSize());
+                SG_LOG(SG_TERRASYNC, SG_DEBUG, "got file " << fileName << " in " << _directory->absolutePath());
             } else if (responseCode() == 404) {
-                _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_FILE_NOT_FOUND);
+                SG_LOG(SG_TERRASYNC, SG_WARN, "terrasync file not found on server: " << fileName << " for " << _directory->absolutePath());
+                _directory->didFailToUpdateFile(fileName, HTTPRepository::REPO_ERROR_FILE_NOT_FOUND);
             } else {
-                _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_HTTP);
+                SG_LOG(SG_TERRASYNC, SG_WARN, "terrasync file download error on server: " << fileName << " for " << _directory->absolutePath() << ": " << responseCode() );
+                _directory->didFailToUpdateFile(fileName, HTTPRepository::REPO_ERROR_HTTP);
             }
 
             _directory->repository()->finishedRequest(this);
@@ -631,9 +646,12 @@ HTTPRepository::failure() const
         virtual void onFail()
         {
             file.reset();
-            pathInRepo.remove();
+            if (pathInRepo.exists()) {
+                pathInRepo.remove();
+            }
+            
             if (_directory) {
-                _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_SOCKET);
+                _directory->didFailToUpdateFile(fileName, HTTPRepository::REPO_ERROR_SOCKET);
                 _directory->repository()->finishedRequest(this);
             }
         }
@@ -646,7 +664,7 @@ HTTPRepository::failure() const
         std::string fileName; // if empty, we're getting the directory itself
         SGPath pathInRepo;
         simgear::sha1nfo hashContext;
-        std::auto_ptr<SGFile> file;
+        std::auto_ptr<SGBinaryFile> file;
     };
 
     class DirGetRequest : public HTTPRepoGetRequest
@@ -683,7 +701,7 @@ HTTPRepository::failure() const
             if (responseCode() == 200) {
                 std::string hash = strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
                 if (!_targetHash.empty() && (hash != _targetHash)) {
-                    _directory->failedToUpdate(AbstractRepository::REPO_ERROR_CHECKSUM);
+                    _directory->failedToUpdate(HTTPRepository::REPO_ERROR_CHECKSUM);
                     _directory->repository()->finishedRequest(this);
                     return;
                 }
@@ -711,17 +729,22 @@ HTTPRepository::failure() const
                     //SG_LOG(SG_TERRASYNC, SG_INFO, "updated dir index " << _directory->absolutePath());
                 }
 
+                _directory->repository()->totalDownloaded += contentSize();
+
                 try {
                     // either way we've confirmed the index is valid so update
                     // children now
+                    SGTimeStamp st;
+                    st.stamp();
                     _directory->updateChildrenBasedOnHash();
-                } catch (sg_exception& e) {
-                    _directory->failedToUpdate(AbstractRepository::REPO_ERROR_IO);
+                    SG_LOG(SG_TERRASYNC, SG_INFO, "after update of:" << _directory->absolutePath() << " child update took:" << st.elapsedMSec());
+                } catch (sg_exception& ) {
+                    _directory->failedToUpdate(HTTPRepository::REPO_ERROR_IO);
                 }
             } else if (responseCode() == 404) {
-                _directory->failedToUpdate(AbstractRepository::REPO_ERROR_FILE_NOT_FOUND);
+                _directory->failedToUpdate(HTTPRepository::REPO_ERROR_FILE_NOT_FOUND);
             } else {
-                _directory->failedToUpdate(AbstractRepository::REPO_ERROR_HTTP);
+                _directory->failedToUpdate(HTTPRepository::REPO_ERROR_HTTP);
             }
 
             _directory->repository()->finishedRequest(this);
@@ -730,7 +753,7 @@ HTTPRepository::failure() const
         virtual void onFail()
         {
             if (_directory) {
-                _directory->failedToUpdate(AbstractRepository::REPO_ERROR_SOCKET);
+                _directory->failedToUpdate(HTTPRepository::REPO_ERROR_SOCKET);
                 _directory->repository()->finishedRequest(this);
             }
         }
@@ -761,24 +784,24 @@ HTTPRepository::failure() const
         }
 
         RequestVector::iterator r;
-        for (r=requests.begin(); r != requests.end(); ++r) {
+        for (r=activeRequests.begin(); r != activeRequests.end(); ++r) {
             (*r)->cancel();
         }
     }
 
-    HTTP::Request_ptr HTTPRepoPrivate::updateFile(HTTPDirectory* dir, const std::string& name)
+    HTTP::Request_ptr HTTPRepoPrivate::updateFile(HTTPDirectory* dir, const std::string& name, size_t sz)
     {
         RepoRequestPtr r(new FileGetRequest(dir, name));
-        requests.push_back(r);
-        http->makeRequest(r);
+        r->setContentSize(sz);
+        makeRequest(r);
         return r;
     }
 
-    HTTP::Request_ptr HTTPRepoPrivate::updateDir(HTTPDirectory* dir, const std::string& hash)
+    HTTP::Request_ptr HTTPRepoPrivate::updateDir(HTTPDirectory* dir, const std::string& hash, size_t sz)
     {
         RepoRequestPtr r(new DirGetRequest(dir, hash));
-        requests.push_back(r);
-        http->makeRequest(r);
+        r->setContentSize(sz);
+        makeRequest(r);
         return r;
     }
 
@@ -820,7 +843,7 @@ HTTPRepository::failure() const
         sha1_init(&info);
         char* buf = static_cast<char*>(malloc(1024 * 1024));
         size_t readLen;
-        SGFile f(p.str());
+        SGBinaryFile f(p.str());
         if (!f.open(SG_IO_IN)) {
             throw sg_io_exception("Couldn't open file for compute hash", p);
         }
@@ -840,6 +863,7 @@ HTTPRepository::failure() const
         HashCache::iterator it = std::find_if(hashes.begin(), hashes.end(), HashEntryWithPath(p.str()));
         if (it != hashes.end()) {
             hashes.erase(it);
+            hashCacheDirty = true;
         }
 
         if (newHash.empty()) {
@@ -858,11 +882,15 @@ HTTPRepository::failure() const
         entry.lengthBytes = p2.sizeInBytes();
         hashes.push_back(entry);
 
-        writeHashCache();
+        hashCacheDirty = true;
     }
 
     void HTTPRepoPrivate::writeHashCache()
     {
+        if (!hashCacheDirty) {
+            return;
+        }
+
         SGPath cachePath = basePath;
         cachePath.append(".hashes");
 
@@ -873,6 +901,7 @@ HTTPRepository::failure() const
             << it->lengthBytes << ":" << it->hashHex << "\n";
         }
         stream.close();
+        hashCacheDirty = false;
     }
 
     void HTTPRepoPrivate::parseHashCache()
@@ -885,25 +914,34 @@ HTTPRepository::failure() const
         }
 
         std::ifstream stream(cachePath.c_str(), std::ios::in);
-        char buf[2048];
-        char* lastToken;
 
         while (!stream.eof()) {
-            stream.getline(buf, 2048);
-            lastToken = 0;
-            char* nameData = ::strtok_r(buf, ":", &lastToken);
-            char* timeData = ::strtok_r(NULL, ":", &lastToken);
-            char* sizeData = ::strtok_r(NULL, ":", &lastToken);
-            char* hashData = ::strtok_r(NULL, ":", &lastToken);
-            if (!nameData || !timeData || !sizeData || !hashData) {
+            std::string line;
+            std::getline(stream,line);
+            line = simgear::strutils::strip(line);
+            if( line.empty() || line[0] == '#' )
+                continue;
+
+            string_list tokens = simgear::strutils::split( line, ":" );
+            if( tokens.size() < 4 ) {
+                SG_LOG(SG_TERRASYNC, SG_WARN, "invalid entry in '" << cachePath.str() << "': '" << line << "' (ignoring line)");
+                continue;
+            }
+            const std::string nameData = simgear::strutils::strip(tokens[0]);
+            const std::string timeData = simgear::strutils::strip(tokens[1]);
+            const std::string sizeData = simgear::strutils::strip(tokens[2]);
+            const std::string hashData = simgear::strutils::strip(tokens[3]);
+
+            if (nameData.empty() || timeData.empty() || sizeData.empty() || hashData.empty() ) {
+                SG_LOG(SG_TERRASYNC, SG_WARN, "invalid entry in '" << cachePath.str() << "': '" << line << "' (ignoring line)");
                 continue;
             }
 
             HashCacheEntry entry;
             entry.filePath = nameData;
             entry.hashHex = hashData;
-            entry.modTime = strtol(timeData, NULL, 10);
-            entry.lengthBytes = strtol(sizeData, NULL, 10);
+            entry.modTime = strtol(timeData.c_str(), NULL, 10);
+            entry.lengthBytes = strtol(sizeData.c_str(), NULL, 10);
             hashes.push_back(entry);
         }
     }
@@ -951,26 +989,46 @@ HTTPRepository::failure() const
         return false;
     }
 
+    void HTTPRepoPrivate::makeRequest(RepoRequestPtr req)
+    {
+        if (activeRequests.size() > 4) {
+            queuedRequests.push_back(req);
+        } else {
+            activeRequests.push_back(req);
+            http->makeRequest(req);
+        }
+    }
+
     void HTTPRepoPrivate::finishedRequest(const RepoRequestPtr& req)
     {
-        RequestVector::iterator it = std::find(requests.begin(), requests.end(), req);
-        if (it == requests.end()) {
+        RequestVector::iterator it = std::find(activeRequests.begin(), activeRequests.end(), req);
+        if (it == activeRequests.end()) {
             throw sg_exception("lost request somehow", req->url());
         }
-        requests.erase(it);
-        if (requests.empty()) {
+        activeRequests.erase(it);
+
+        if (!queuedRequests.empty()) {
+            RepoRequestPtr rr = queuedRequests.front();
+            queuedRequests.erase(queuedRequests.begin());
+            activeRequests.push_back(rr);
+            http->makeRequest(rr);
+        }
+
+        writeHashCache();
+
+        if (activeRequests.empty() && queuedRequests.empty()) {
             isUpdating = false;
         }
     }
 
-    void HTTPRepoPrivate::failedToGetRootIndex(AbstractRepository::ResultCode st)
+    void HTTPRepoPrivate::failedToGetRootIndex(HTTPRepository::ResultCode st)
     {
         SG_LOG(SG_TERRASYNC, SG_WARN, "Failed to get root of repo:" << baseUrl);
         status = st;
     }
 
     void HTTPRepoPrivate::failedToUpdateChild(const SGPath& relativePath,
-                                              AbstractRepository::ResultCode fileStatus)
+                                              HTTPRepository::ResultCode fileStatus)
     {
         Failure f;
         f.path = relativePath;