1 // HTTPRepository.cxx -- plain HTTP TerraSync remote client
3 // Copyright (C) 20126 James Turner <zakalawe@mac.com>
5 // This program is free software; you can redistribute it and/or
6 // modify it under the terms of the GNU General Public License as
7 // published by the Free Software Foundation; either version 2 of the
8 // License, or (at your option) any later version.
10 // This program is distributed in the hope that it will be useful, but
11 // WITHOUT ANY WARRANTY; without even the implied warranty of
12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 // General Public License for more details.
15 // You should have received a copy of the GNU General Public License
16 // along with this program; if not, write to the Free Software
17 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 #include "HTTPRepository.hxx"
21 #include <simgear_config.h>
35 #include "simgear/debug/logstream.hxx"
36 #include "simgear/misc/strutils.hxx"
37 #include <simgear/misc/sg_dir.hxx>
38 #include <simgear/io/HTTPClient.hxx>
39 #include <simgear/io/sg_file.hxx>
40 #include <simgear/misc/sgstream.hxx>
41 #include <simgear/structure/exception.hxx>
42 #include <simgear/timing/timestamp.hxx>
44 #include <simgear/misc/sg_hash.hxx>
51 class HTTPRepoGetRequest : public HTTP::Request
54 HTTPRepoGetRequest(HTTPDirectory* d, const std::string& u) :
60 size_t contentSize() const
65 void setContentSize(size_t sz)
70 HTTPDirectory* _directory;
74 typedef SGSharedPtr<HTTPRepoGetRequest> RepoRequestPtr;
88 typedef std::vector<HashCacheEntry> HashCache;
95 HTTPRepository::ResultCode error;
98 typedef std::vector<Failure> FailureList;
101 HTTPRepoPrivate(HTTPRepository* parent) :
102 hashCacheDirty(false),
105 updateEverything(false),
106 status(HTTPRepository::REPO_NO_ERROR),
112 HTTPRepository* p; // link back to outer
117 bool updateEverything;
118 string_list updatePaths;
119 HTTPRepository::ResultCode status;
120 HTTPDirectory* rootDir;
121 size_t totalDownloaded;
123 void updateWaiting();
125 HTTP::Request_ptr updateFile(HTTPDirectory* dir, const std::string& name,
127 HTTP::Request_ptr updateDir(HTTPDirectory* dir, const std::string& hash,
130 std::string hashForPath(const SGPath& p);
131 void updatedFileContents(const SGPath& p, const std::string& newHash);
132 void parseHashCache();
133 std::string computeHashForPath(const SGPath& p);
134 void writeHashCache();
136 void failedToGetRootIndex(HTTPRepository::ResultCode st);
137 void failedToUpdateChild(const SGPath& relativePath,
138 HTTPRepository::ResultCode fileStatus);
140 typedef std::vector<RepoRequestPtr> RequestVector;
141 RequestVector queuedRequests,
144 void makeRequest(RepoRequestPtr req);
145 void finishedRequest(const RepoRequestPtr& req);
147 HTTPDirectory* getOrCreateDirectory(const std::string& path);
148 bool deleteDirectory(const std::string& path);
150 typedef std::vector<HTTPDirectory*> DirectoryVector;
151 DirectoryVector directories;
165 ChildInfo(Type ty, const std::string & nameData, const std::string & hashData) :
173 ChildInfo(const ChildInfo& other) :
177 sizeInBytes(other.sizeInBytes)
180 void setSize(const std::string & sizeData)
182 sizeInBytes = ::strtol(sizeData.c_str(), NULL, 10);
185 bool operator<(const ChildInfo& other) const
187 return name < other.name;
191 std::string name, hash;
195 typedef std::vector<ChildInfo> ChildInfoList;
196 ChildInfoList children;
200 HTTPDirectory(HTTPRepoPrivate* repo, const std::string& path) :
207 SGPath p(absolutePath());
210 // already exists on disk
211 parseDirIndex(children);
212 std::sort(children.begin(), children.end());
213 } catch (sg_exception& ) {
214 // parsing cache failed
220 HTTPRepoPrivate* repository() const
225 std::string url() const
227 if (_relativePath.str().empty()) {
228 return _repository->baseUrl;
231 return _repository->baseUrl + "/" + _relativePath.str();
234 void dirIndexUpdated(const std::string& hash)
236 SGPath fpath(absolutePath());
237 fpath.append(".dirindex");
238 _repository->updatedFileContents(fpath, hash);
243 parseDirIndex(children);
244 std::sort(children.begin(), children.end());
247 void failedToUpdate(HTTPRepository::ResultCode status)
249 _state = UpdateFailed;
250 if (_relativePath.isNull()) {
252 _repository->failedToGetRootIndex(status);
254 _repository->failedToUpdateChild(_relativePath, status);
258 void updateChildrenBasedOnHash()
260 // if we got here for a dir which is still updating or excluded
261 // from updates, just bail out right now.
262 if (_state != Updated) {
266 string_list indexNames = indexChildren(),
267 toBeUpdated, orphans;
268 simgear::Dir d(absolutePath());
269 PathList fsChildren = d.children(0);
270 PathList::const_iterator it = fsChildren.begin();
273 for (; it != fsChildren.end(); ++it) {
274 ChildInfo info(it->isDir() ? ChildInfo::DirectoryType : ChildInfo::FileType,
276 std::string hash = hashForChild(info);
278 ChildInfoList::iterator c = findIndexChild(it->file());
279 if (c == children.end()) {
280 SG_LOG(SG_TERRASYNC, SG_DEBUG, "is orphan '" << it->file() << "'" );
281 orphans.push_back(it->file());
282 } else if (c->hash != hash) {
283 SG_LOG(SG_TERRASYNC, SG_DEBUG, "hash mismatch'" << it->file() );
284 // file exists, but hash mismatch, schedule update
286 SG_LOG(SG_TERRASYNC, SG_DEBUG, "file exists but hash is wrong for:" << it->file() );
287 SG_LOG(SG_TERRASYNC, SG_DEBUG, "on disk:" << hash << " vs in info:" << c->hash);
290 toBeUpdated.push_back(it->file() );
292 // file exists and hash is valid. If it's a directory,
293 // perform a recursive check.
294 SG_LOG(SG_TERRASYNC, SG_DEBUG, "file exists hash is good:" << it->file() );
295 if (c->type == ChildInfo::DirectoryType) {
296 SGPath p(relativePath());
297 p.append(it->file());
298 HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str());
299 if (childDir->_state == NotUpdated) {
300 childDir->_state = Updated;
302 childDir->updateChildrenBasedOnHash();
306 // remove existing file system children from the index list,
307 // so we can detect new children
308 // https://en.wikibooks.org/wiki/More_C%2B%2B_Idioms/Erase-Remove
309 indexNames.erase(std::remove(indexNames.begin(), indexNames.end(), it->file()), indexNames.end());
310 } // of real children iteration
312 // all remaining names in indexChilden are new children
313 toBeUpdated.insert(toBeUpdated.end(), indexNames.begin(), indexNames.end());
315 removeOrphans(orphans);
316 scheduleUpdates(toBeUpdated);
319 void markAsUpToDate()
324 void markAsUpdating()
326 assert(_state == NotUpdated);
327 _state = HTTPDirectory::UpdateInProgress;
332 // assert because this should only get invoked on newly created
333 // directory objects which are inside the sub-tree(s) to be updated
334 assert(_state == DoNotUpdate);
338 void markSubtreeAsNeedingUpdate()
340 if (_state == Updated) {
341 _state = NotUpdated; // reset back to not-updated
344 ChildInfoList::iterator cit;
345 for (cit = children.begin(); cit != children.end(); ++cit) {
346 if (cit->type == ChildInfo::DirectoryType) {
347 SGPath p(relativePath());
349 HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str());
350 childDir->markSubtreeAsNeedingUpdate();
352 } // of child iteration
355 void markSubtreeAsEnabled()
357 if (_state == DoNotUpdate) {
361 ChildInfoList::iterator cit;
362 for (cit = children.begin(); cit != children.end(); ++cit) {
363 if (cit->type == ChildInfo::DirectoryType) {
364 SGPath p(relativePath());
366 HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str());
367 childDir->markSubtreeAsEnabled();
369 } // of child iteration
373 void markAncestorChainAsEnabled()
375 if (_state == DoNotUpdate) {
379 if (_relativePath.isNull()) {
383 std::string prPath = _relativePath.dir();
384 if (prPath.empty()) {
385 _repository->rootDir->markAncestorChainAsEnabled();
387 HTTPDirectory* prDir = _repository->getOrCreateDirectory(prPath);
388 prDir->markAncestorChainAsEnabled();
392 void updateIfWaiting(const std::string& hash, size_t sz)
394 if (_state == NotUpdated) {
395 _repository->updateDir(this, hash, sz);
399 if ((_state == DoNotUpdate) || (_state == UpdateInProgress)) {
403 ChildInfoList::iterator cit;
404 for (cit = children.begin(); cit != children.end(); ++cit) {
405 if (cit->type == ChildInfo::DirectoryType) {
406 SGPath p(relativePath());
408 HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str());
409 childDir->updateIfWaiting(cit->hash, cit->sizeInBytes);
411 } // of child iteration
414 void removeOrphans(const string_list& orphans)
416 string_list::const_iterator it;
417 for (it = orphans.begin(); it != orphans.end(); ++it) {
422 string_list indexChildren() const
425 r.reserve(children.size());
426 ChildInfoList::const_iterator it;
427 for (it=children.begin(); it != children.end(); ++it) {
428 r.push_back(it->name);
433 void scheduleUpdates(const string_list& names)
435 string_list::const_iterator it;
436 for (it = names.begin(); it != names.end(); ++it) {
437 ChildInfoList::iterator cit = findIndexChild(*it);
438 if (cit == children.end()) {
439 SG_LOG(SG_TERRASYNC, SG_WARN, "scheduleUpdate, unknown child:" << *it);
443 SG_LOG(SG_TERRASYNC,SG_DEBUG, "scheduling update for " << *it );
444 if (cit->type == ChildInfo::FileType) {
445 _repository->updateFile(this, *it, cit->sizeInBytes);
447 SGPath p(relativePath());
449 HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str());
450 if (childDir->_state == DoNotUpdate) {
451 SG_LOG(SG_TERRASYNC, SG_WARN, "scheduleUpdate, child:" << *it << " is marked do not update so skipping");
455 _repository->updateDir(childDir, cit->hash, cit->sizeInBytes);
460 SGPath absolutePath() const
462 SGPath r(_repository->basePath);
463 r.append(_relativePath.str());
467 SGPath relativePath() const
469 return _relativePath;
472 void didUpdateFile(const std::string& file, const std::string& hash, size_t sz)
474 // check hash matches what we expected
475 ChildInfoList::iterator it = findIndexChild(file);
476 if (it == children.end()) {
477 SG_LOG(SG_TERRASYNC, SG_WARN, "updated file but not found in dir:" << _relativePath << " " << file);
479 SGPath fpath(absolutePath());
482 if (it->hash != hash) {
483 // we don't erase the file on a hash mismatch, becuase if we're syncing during the
484 // middle of a server-side update, the downloaded file may actually become valid.
485 _repository->failedToUpdateChild(_relativePath, HTTPRepository::REPO_ERROR_CHECKSUM);
487 _repository->updatedFileContents(fpath, hash);
488 _repository->totalDownloaded += sz;
490 } // of found in child list
493 void didFailToUpdateFile(const std::string& file,
494 HTTPRepository::ResultCode status)
496 SGPath fpath(_relativePath);
498 _repository->failedToUpdateChild(fpath, status);
504 ChildWithName(const std::string& n) : name(n) {}
507 bool operator()(const ChildInfo& info) const
508 { return info.name == name; }
511 ChildInfoList::iterator findIndexChild(const std::string& name)
513 return std::find_if(children.begin(), children.end(), ChildWithName(name));
516 bool parseDirIndex(ChildInfoList& children)
518 SGPath p(absolutePath());
519 p.append(".dirindex");
524 std::ifstream indexStream( p.c_str(), std::ios::in );
526 if ( !indexStream.is_open() ) {
527 throw sg_io_exception("cannot open dirIndex file", p);
530 while (!indexStream.eof() ) {
532 std::getline( indexStream, line );
533 line = simgear::strutils::strip(line);
535 // skip blank line or comment beginning with '#'
536 if( line.empty() || line[0] == '#' )
539 string_list tokens = simgear::strutils::split( line, ":" );
541 std::string typeData = tokens[0];
543 if( typeData == "version" ) {
544 if( tokens.size() < 2 ) {
545 SG_LOG(SG_TERRASYNC, SG_WARN, "malformed .dirindex file: missing version number in line '" << line << "'" );
548 if( tokens[1] != "1" ) {
549 SG_LOG(SG_TERRASYNC, SG_WARN, "invalid .dirindex file: wrong version number '" << tokens[1] << "' (expected 1)" );
552 continue; // version is good, continue
555 if( typeData == "path" ) {
556 continue; // ignore path, next line
559 if( tokens.size() < 3 ) {
560 SG_LOG(SG_TERRASYNC, SG_WARN, "malformed .dirindex file: not enough tokens in line '" << line << "' (ignoring line)" );
564 if (typeData != "f" && typeData != "d" ) {
565 SG_LOG(SG_TERRASYNC, SG_WARN, "malformed .dirindex file: invalid type in line '" << line << "', expected 'd' or 'f', (ignoring line)" );
568 children.push_back(ChildInfo(typeData == "f" ? ChildInfo::FileType : ChildInfo::DirectoryType, tokens[1], tokens[2]));
570 if (tokens.size() > 3) {
571 children.back().setSize(tokens[3]);
578 void removeChild(const std::string& name)
580 SGPath p(absolutePath());
584 SGPath fpath(_relativePath);
588 ok = _repository->deleteDirectory(fpath.str());
590 // remove the hash cache entry
591 _repository->updatedFileContents(p, std::string());
596 SG_LOG(SG_TERRASYNC, SG_WARN, "removal failed for:" << p);
597 throw sg_io_exception("Failed to remove existing file/dir:", p);
601 std::string hashForChild(const ChildInfo& child) const
603 SGPath p(absolutePath());
604 p.append(child.name);
605 if (child.type == ChildInfo::DirectoryType) {
606 p.append(".dirindex");
608 return _repository->hashForPath(p);
611 HTTPRepoPrivate* _repository;
612 SGPath _relativePath; // in URL and file-system space
626 HTTPRepository::HTTPRepository(const SGPath& base, HTTP::Client *cl) :
627 _d(new HTTPRepoPrivate(this))
631 _d->rootDir = new HTTPDirectory(_d.get(), "");
632 _d->parseHashCache();
635 HTTPRepository::~HTTPRepository()
639 void HTTPRepository::setBaseUrl(const std::string &url)
644 std::string HTTPRepository::baseUrl() const
649 HTTP::Client* HTTPRepository::http() const
654 SGPath HTTPRepository::fsBase() const
659 void HTTPRepository::update()
661 _d->rootDir->markSubtreeAsNeedingUpdate();
665 void HTTPRepository::setEntireRepositoryMode()
667 if (!_d->updateEverything) {
668 // this is a one-way decision
669 _d->updateEverything = true;
672 // probably overkill but not expensive so let's check everything
673 // we have in case someone did something funky and switched from partial
674 // to 'whole repo' updating.
675 _d->rootDir->markSubtreeAsEnabled();
679 void HTTPRepository::addSubpath(const std::string& relPath)
681 if (_d->updateEverything) {
682 SG_LOG(SG_TERRASYNC, SG_WARN, "called HTTPRepository::addSubpath but updating everything");
686 _d->updatePaths.push_back(relPath);
688 HTTPDirectory* dir = _d->getOrCreateDirectory(relPath);
689 dir->markSubtreeAsEnabled();
690 dir->markAncestorChainAsEnabled();
695 bool HTTPRepository::isDoingSync() const
697 if (_d->status != REPO_NO_ERROR) {
701 return _d->isUpdating;
704 size_t HTTPRepository::bytesToDownload() const
708 HTTPRepoPrivate::RequestVector::const_iterator r;
709 for (r = _d->queuedRequests.begin(); r != _d->queuedRequests.end(); ++r) {
710 result += (*r)->contentSize();
713 for (r = _d->activeRequests.begin(); r != _d->activeRequests.end(); ++r) {
714 result += (*r)->contentSize() - (*r)->responseBytesReceived();
720 size_t HTTPRepository::bytesDownloaded() const
722 size_t result = _d->totalDownloaded;
724 HTTPRepoPrivate::RequestVector::const_iterator r;
725 for (r = _d->activeRequests.begin(); r != _d->activeRequests.end(); ++r) {
726 result += (*r)->responseBytesReceived();
732 HTTPRepository::ResultCode
733 HTTPRepository::failure() const
735 if ((_d->status == REPO_NO_ERROR) && !_d->failures.empty()) {
736 return REPO_PARTIAL_UPDATE;
742 class FileGetRequest : public HTTPRepoGetRequest
745 FileGetRequest(HTTPDirectory* d, const std::string& file) :
746 HTTPRepoGetRequest(d, makeUrl(d, file)),
749 pathInRepo = _directory->absolutePath();
750 pathInRepo.append(fileName);
754 virtual void gotBodyData(const char* s, int n)
757 file.reset(new SGBinaryFile(pathInRepo.str()));
758 if (!file->open(SG_IO_OUT)) {
759 SG_LOG(SG_TERRASYNC, SG_WARN, "unable to create file " << pathInRepo);
760 _directory->repository()->http->cancelRequest(this, "Unable to create output file");
763 sha1_init(&hashContext);
766 sha1_write(&hashContext, s, n);
770 virtual void onDone()
774 if (responseCode() == 200) {
775 std::string hash = strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
776 _directory->didUpdateFile(fileName, hash, contentSize());
777 SG_LOG(SG_TERRASYNC, SG_DEBUG, "got file " << fileName << " in " << _directory->absolutePath());
778 } else if (responseCode() == 404) {
779 SG_LOG(SG_TERRASYNC, SG_WARN, "terrasync file not found on server: " << fileName << " for " << _directory->absolutePath());
780 _directory->didFailToUpdateFile(fileName, HTTPRepository::REPO_ERROR_FILE_NOT_FOUND);
782 SG_LOG(SG_TERRASYNC, SG_WARN, "terrasync file download error on server: " << fileName << " for " << _directory->absolutePath() << ": " << responseCode() );
783 _directory->didFailToUpdateFile(fileName, HTTPRepository::REPO_ERROR_HTTP);
786 _directory->repository()->finishedRequest(this);
789 virtual void onFail()
792 if (pathInRepo.exists()) {
797 _directory->didFailToUpdateFile(fileName, HTTPRepository::REPO_ERROR_SOCKET);
798 _directory->repository()->finishedRequest(this);
802 static std::string makeUrl(HTTPDirectory* d, const std::string& file)
804 return d->url() + "/" + file;
807 std::string fileName; // if empty, we're getting the directory itself
809 simgear::sha1nfo hashContext;
810 std::auto_ptr<SGBinaryFile> file;
813 class DirGetRequest : public HTTPRepoGetRequest
816 DirGetRequest(HTTPDirectory* d, const std::string& targetHash) :
817 HTTPRepoGetRequest(d, makeUrl(d)),
819 _targetHash(targetHash)
821 sha1_init(&hashContext);
829 bool isRootDir() const
835 virtual void gotBodyData(const char* s, int n)
837 body += std::string(s, n);
838 sha1_write(&hashContext, s, n);
841 virtual void onDone()
843 if (responseCode() == 200) {
844 std::string hash = strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
845 if (!_targetHash.empty() && (hash != _targetHash)) {
846 _directory->failedToUpdate(HTTPRepository::REPO_ERROR_CHECKSUM);
847 _directory->repository()->finishedRequest(this);
851 std::string curHash = _directory->repository()->hashForPath(path());
852 if (hash != curHash) {
853 simgear::Dir d(_directory->absolutePath());
855 if (!d.create(0700)) {
856 throw sg_io_exception("Unable to create directory", d.path());
860 // dir index data has changed, so write to disk and update
861 // the hash accordingly
862 std::ofstream of(pathInRepo().c_str(), std::ios::trunc | std::ios::out);
864 throw sg_io_exception("Failed to open directory index file for writing", pathInRepo().c_str());
867 of.write(body.data(), body.size());
869 _directory->dirIndexUpdated(hash);
871 _directory->markAsUpToDate();
874 _directory->repository()->totalDownloaded += contentSize();
877 // either way we've confirmed the index is valid so update
881 _directory->updateChildrenBasedOnHash();
882 SG_LOG(SG_TERRASYNC, SG_INFO, "after update of:" << _directory->absolutePath() << " child update took:" << st.elapsedMSec());
883 } catch (sg_exception& ) {
884 _directory->failedToUpdate(HTTPRepository::REPO_ERROR_IO);
886 } else if (responseCode() == 404) {
887 _directory->failedToUpdate(HTTPRepository::REPO_ERROR_FILE_NOT_FOUND);
889 _directory->failedToUpdate(HTTPRepository::REPO_ERROR_HTTP);
892 _directory->repository()->finishedRequest(this);
895 virtual void onFail()
898 _directory->failedToUpdate(HTTPRepository::REPO_ERROR_SOCKET);
899 _directory->repository()->finishedRequest(this);
903 static std::string makeUrl(HTTPDirectory* d)
905 return d->url() + "/.dirindex";
908 SGPath pathInRepo() const
910 SGPath p(_directory->absolutePath());
911 p.append(".dirindex");
915 simgear::sha1nfo hashContext;
917 bool _isRootDir; ///< is this the repository root?
918 std::string _targetHash;
921 HTTPRepoPrivate::~HTTPRepoPrivate()
923 // take a copy since cancelRequest will fail and hence remove
924 // remove activeRequests, invalidating any iterator to it.
925 RequestVector copyOfActive(activeRequests);
926 RequestVector::iterator rq;
927 for (rq = copyOfActive.begin(); rq != copyOfActive.end(); ++rq) {
928 http->cancelRequest(*rq, "Repository object deleted");
931 DirectoryVector::iterator it;
932 for (it=directories.begin(); it != directories.end(); ++it) {
937 HTTP::Request_ptr HTTPRepoPrivate::updateFile(HTTPDirectory* dir, const std::string& name, size_t sz)
939 RepoRequestPtr r(new FileGetRequest(dir, name));
940 r->setContentSize(sz);
945 HTTP::Request_ptr HTTPRepoPrivate::updateDir(HTTPDirectory* dir, const std::string& hash, size_t sz)
947 dir->markAsUpdating();
948 RepoRequestPtr r(new DirGetRequest(dir, hash));
949 r->setContentSize(sz);
955 class HashEntryWithPath
958 HashEntryWithPath(const std::string& p) : path(p) {}
959 bool operator()(const HTTPRepoPrivate::HashCacheEntry& entry) const
960 { return entry.filePath == path; }
965 std::string HTTPRepoPrivate::hashForPath(const SGPath& p)
967 HashCache::iterator it = std::find_if(hashes.begin(), hashes.end(), HashEntryWithPath(p.str()));
968 if (it != hashes.end()) {
969 // ensure data on disk hasn't changed.
970 // we could also use the file type here if we were paranoid
971 if ((p.sizeInBytes() == it->lengthBytes) && (p.modTime() == it->modTime)) {
975 // entry in the cache, but it's stale so remove and fall through
979 std::string hash = computeHashForPath(p);
980 updatedFileContents(p, hash);
984 std::string HTTPRepoPrivate::computeHashForPath(const SGPath& p)
987 return std::string();
990 char* buf = static_cast<char*>(malloc(1024 * 1024));
992 SGBinaryFile f(p.str());
993 if (!f.open(SG_IO_IN)) {
994 throw sg_io_exception("Couldn't open file for compute hash", p);
996 while ((readLen = f.read(buf, 1024 * 1024)) > 0) {
997 sha1_write(&info, buf, readLen);
1002 std::string hashBytes((char*) sha1_result(&info), HASH_LENGTH);
1003 return strutils::encodeHex(hashBytes);
1006 void HTTPRepoPrivate::updatedFileContents(const SGPath& p, const std::string& newHash)
1008 // remove the existing entry
1009 HashCache::iterator it = std::find_if(hashes.begin(), hashes.end(), HashEntryWithPath(p.str()));
1010 if (it != hashes.end()) {
1012 hashCacheDirty = true;
1015 if (newHash.empty()) {
1016 return; // we're done
1019 // use a cloned SGPath and reset its caching to force one stat() call
1021 p2.set_cached(false);
1022 p2.set_cached(true);
1024 HashCacheEntry entry;
1025 entry.filePath = p.str();
1026 entry.hashHex = newHash;
1027 entry.modTime = p2.modTime();
1028 entry.lengthBytes = p2.sizeInBytes();
1029 hashes.push_back(entry);
1031 hashCacheDirty = true;
1034 void HTTPRepoPrivate::writeHashCache()
1036 if (!hashCacheDirty) {
1040 SGPath cachePath = basePath;
1041 cachePath.append(".hashes");
1043 std::ofstream stream(cachePath.c_str(),std::ios::out | std::ios::trunc);
1044 HashCache::const_iterator it;
1045 for (it = hashes.begin(); it != hashes.end(); ++it) {
1046 stream << it->filePath << ":" << it->modTime << ":"
1047 << it->lengthBytes << ":" << it->hashHex << "\n";
1050 hashCacheDirty = false;
1053 void HTTPRepoPrivate::parseHashCache()
1056 SGPath cachePath = basePath;
1057 cachePath.append(".hashes");
1058 if (!cachePath.exists()) {
1062 std::ifstream stream(cachePath.c_str(), std::ios::in);
1064 while (!stream.eof()) {
1066 std::getline(stream,line);
1067 line = simgear::strutils::strip(line);
1068 if( line.empty() || line[0] == '#' )
1071 string_list tokens = simgear::strutils::split( line, ":" );
1072 if( tokens.size() < 4 ) {
1073 SG_LOG(SG_TERRASYNC, SG_WARN, "invalid entry in '" << cachePath.str() << "': '" << line << "' (ignoring line)");
1076 const std::string nameData = simgear::strutils::strip(tokens[0]);
1077 const std::string timeData = simgear::strutils::strip(tokens[1]);
1078 const std::string sizeData = simgear::strutils::strip(tokens[2]);
1079 const std::string hashData = simgear::strutils::strip(tokens[3]);
1081 if (nameData.empty() || timeData.empty() || sizeData.empty() || hashData.empty() ) {
1082 SG_LOG(SG_TERRASYNC, SG_WARN, "invalid entry in '" << cachePath.str() << "': '" << line << "' (ignoring line)");
1086 HashCacheEntry entry;
1087 entry.filePath = nameData;
1088 entry.hashHex = hashData;
1089 entry.modTime = strtol(timeData.c_str(), NULL, 10);
1090 entry.lengthBytes = strtol(sizeData.c_str(), NULL, 10);
1091 hashes.push_back(entry);
1095 class DirectoryWithPath
1098 DirectoryWithPath(const std::string& p) : path(p) {}
1099 bool operator()(const HTTPDirectory* entry) const
1100 { return entry->relativePath().str() == path; }
1105 HTTPDirectory* HTTPRepoPrivate::getOrCreateDirectory(const std::string& path)
1107 DirectoryWithPath p(path);
1108 DirectoryVector::iterator it = std::find_if(directories.begin(), directories.end(), p);
1109 if (it != directories.end()) {
1113 HTTPDirectory* d = new HTTPDirectory(this, path);
1114 directories.push_back(d);
1115 if (updateEverything) {
1118 string_list::const_iterator s;
1119 bool shouldUpdate = false;
1121 for (s = updatePaths.begin(); s != updatePaths.end(); ++s) {
1122 size_t minLen = std::min(path.size(), s->size());
1123 if (s->compare(0, minLen, path, 0, minLen) == 0) {
1124 shouldUpdate = true;
1127 } // of paths iteration
1137 bool HTTPRepoPrivate::deleteDirectory(const std::string& path)
1139 DirectoryWithPath p(path);
1140 DirectoryVector::iterator it = std::find_if(directories.begin(), directories.end(), p);
1141 if (it != directories.end()) {
1142 HTTPDirectory* d = *it;
1143 directories.erase(it);
1144 Dir dir(d->absolutePath());
1145 bool result = dir.remove(true);
1148 // update the hash cache too
1149 updatedFileContents(d->absolutePath(), std::string());
1157 void HTTPRepoPrivate::makeRequest(RepoRequestPtr req)
1159 if (activeRequests.size() > 4) {
1160 queuedRequests.push_back(req);
1162 activeRequests.push_back(req);
1163 http->makeRequest(req);
1167 void HTTPRepoPrivate::finishedRequest(const RepoRequestPtr& req)
1169 RequestVector::iterator it = std::find(activeRequests.begin(), activeRequests.end(), req);
1170 // in some cases, for example a checksum failure, we clear the active
1171 // and queued request vectors, so the ::find above can fail
1172 if (it != activeRequests.end()) {
1173 activeRequests.erase(it);
1176 if (!queuedRequests.empty()) {
1177 RepoRequestPtr rr = queuedRequests.front();
1178 queuedRequests.erase(queuedRequests.begin());
1179 activeRequests.push_back(rr);
1180 http->makeRequest(rr);
1185 if (activeRequests.empty() && queuedRequests.empty()) {
1190 void HTTPRepoPrivate::failedToGetRootIndex(HTTPRepository::ResultCode st)
1192 SG_LOG(SG_TERRASYNC, SG_WARN, "Failed to get root of repo:" << baseUrl);
1196 void HTTPRepoPrivate::failedToUpdateChild(const SGPath& relativePath,
1197 HTTPRepository::ResultCode fileStatus)
1199 if (fileStatus == HTTPRepository::REPO_ERROR_CHECKSUM) {
1200 // stop updating, and mark repository as failed, becuase this
1201 // usually indicates we need to start a fresh update from the
1203 // (we could issue a retry here, but we leave that to higher layers)
1204 status = fileStatus;
1206 queuedRequests.clear();
1208 RequestVector copyOfActive(activeRequests);
1209 RequestVector::iterator rq;
1210 for (rq = copyOfActive.begin(); rq != copyOfActive.end(); ++rq) {
1211 //SG_LOG(SG_TERRASYNC, SG_DEBUG, "cancelling request for:" << (*rq)->url());
1212 http->cancelRequest(*rq, "Repository updated failed");
1216 SG_LOG(SG_TERRASYNC, SG_WARN, "failed to update repository:" << baseUrl
1217 << ", possibly modified during sync");
1221 f.path = relativePath;
1222 f.error = fileStatus;
1223 failures.push_back(f);
1225 SG_LOG(SG_TERRASYNC, SG_WARN, "failed to update entry:" << relativePath << " code:" << fileStatus);
1228 void HTTPRepoPrivate::updateWaiting()
1231 status = HTTPRepository::REPO_NO_ERROR;
1236 // find to-be-updated sub-trees and kick them off
1237 rootDir->updateIfWaiting(std::string(), 0);
1239 // maybe there was nothing to do
1240 if (activeRequests.empty()) {
1241 status = HTTPRepository::REPO_NO_ERROR;
1246 } // of namespace simgear