From: James Turner Date: Mon, 24 Jun 2013 06:46:27 +0000 (+0100) Subject: Persistent SVN update cache. X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=b9bd2734eb47ae4325c0e8ddbc8fbc3292294271;p=simgear.git Persistent SVN update cache. Make the already present cache of updated URLs persistent, with a definable lifetime, currently 24 hours. This ensures terrasync-ed resources are checked less often (max once per day) instead of each FGFS launch, which should greatly cut down requests to the backend. --- diff --git a/simgear/scene/tsync/terrasync.cxx b/simgear/scene/tsync/terrasync.cxx index 1007c958..7144b15a 100644 --- a/simgear/scene/tsync/terrasync.cxx +++ b/simgear/scene/tsync/terrasync.cxx @@ -55,12 +55,14 @@ #include #include "terrasync.hxx" + #include #include #include #include #include #include +#include #ifdef SG_SVN_CLIENT # include @@ -172,7 +174,10 @@ public: void setAllowedErrorCount(int errors) {_allowed_errors = errors;} #if defined(HAVE_SVN_CLIENT_H) || defined(SG_SVN_CLIENT) - void setUseBuiltin(bool built_in) { _use_built_in = built_in;} + + void setCachePath(const SGPath& p) {_persistentCachePath = p;} + void setCacheHits(unsigned int hits) {_cache_hits = hits;} + void setUseBuiltin(bool built_in) { _use_built_in = built_in;} #endif volatile bool _active; @@ -184,13 +189,18 @@ public: volatile int _success_count; volatile int _consecutive_errors; volatile int _allowed_errors; - + volatile int _cache_hits; private: virtual void run(); bool syncTree(const char* dir, bool& isNewDirectory); bool syncTreeExternal(const char* dir); - - + + bool isPathCached(const WaitingTile& next) const; + void syncPath(const WaitingTile& next); + + void initCompletedTilesPersistentCache(); + void writeCompletedTilesPersistentCache() const; + #if defined(SG_SVN_CLIENT) bool syncTreeInternal(const char* dir); bool _use_built_in; @@ -219,6 +229,7 @@ private: string _svn_command; string _rsync_server; string _local_dir; + SGPath _persistentCachePath; }; #ifdef HAVE_SVN_CLIENT_H @@ -238,6 +249,7 @@ SGTerraSync::SvnThread::SvnThread() : _success_count(0), _consecutive_errors(0), _allowed_errors(6), + _cache_hits(0), #if defined(HAVE_SVN_CLIENT_H) || defined(SG_SVN_CLIENT) _use_built_in(true), #endif @@ -564,49 +576,23 @@ bool SGTerraSync::SvnThread::syncTreeExternal(const char* dir) void SGTerraSync::SvnThread::run() { _active = true; + + initCompletedTilesPersistentCache(); + while (!_stop) { WaitingTile next = waitingTiles.pop_front(); if (_stop) break; - CompletedTiles::iterator ii = - _completedTiles.find( next._dir ); - time_t now = time(0); - if ((ii == _completedTiles.end())|| - (ii->second < now )) - { - bool isNewDirectory = false; - - _busy = true; - if (!syncTree(next._dir.c_str(),isNewDirectory)) - { - _consecutive_errors++; - _fail_count++; - _completedTiles[ next._dir ] = now + UpdateInterval::FailedAttempt; - } - else - { - _consecutive_errors = 0; - _success_count++; - SG_LOG(SG_TERRAIN,SG_INFO, - "Successfully synchronized directory '" << next._dir << "'"); - if (next._refreshScenery) - { - // updated a tile - _updated_tile_count++; - if (isNewDirectory) - { - // for now only report new directories to refresh display - // (i.e. only when ocean needs to be replaced with actual data) - _freshTiles.push_back(next); - _is_dirty = true; - } - } - _completedTiles[ next._dir ] = now + UpdateInterval::SuccessfulAttempt; - } - _busy = false; + if (isPathCached(next)) { + _cache_hits++; + SG_LOG(SG_TERRAIN, SG_DEBUG, + "Cache hit for: '" << next._dir << "'"); + continue; } + + syncPath(next); if ((_allowed_errors >= 0)&& (_consecutive_errors >= _allowed_errors)) @@ -621,6 +607,61 @@ void SGTerraSync::SvnThread::run() _is_dirty = true; } +bool SGTerraSync::SvnThread::isPathCached(const WaitingTile& next) const +{ + CompletedTiles::const_iterator ii = _completedTiles.find( next._dir ); + if (ii == _completedTiles.end()) { + return false; + } + + // check if the path still physically exists + SGPath p(_local_dir); + p.append(next._dir); + if (!p.exists()) { + return false; + } + + time_t now = time(0); + return (ii->second > now); +} + +void SGTerraSync::SvnThread::syncPath(const WaitingTile& next) +{ + bool isNewDirectory = false; + time_t now = time(0); + + _busy = true; + if (!syncTree(next._dir.c_str(),isNewDirectory)) + { + _consecutive_errors++; + _fail_count++; + _completedTiles[ next._dir ] = now + UpdateInterval::FailedAttempt; + } + else + { + _consecutive_errors = 0; + _success_count++; + SG_LOG(SG_TERRAIN,SG_INFO, + "Successfully synchronized directory '" << next._dir << "'"); + if (next._refreshScenery) + { + // updated a tile + _updated_tile_count++; + if (isNewDirectory) + { + // for now only report new directories to refresh display + // (i.e. only when ocean needs to be replaced with actual data) + _freshTiles.push_back(next); + _is_dirty = true; + } + } + + _completedTiles[ next._dir ] = now + UpdateInterval::SuccessfulAttempt; + writeCompletedTilesPersistentCache(); + } + _busy = false; +} + #if defined(HAVE_SVN_CLIENT_H) // Configure our subversion session int SGTerraSync::SvnThread::svnClientSetup(void) @@ -722,6 +763,52 @@ int SGTerraSync::SvnThread::svnClientSetup(void) } #endif // of defined(HAVE_SVN_CLIENT_H) +void SGTerraSync::SvnThread::initCompletedTilesPersistentCache() +{ + if (!_persistentCachePath.exists()) { + return; + } + + SGPropertyNode_ptr cacheRoot(new SGPropertyNode); + time_t now = time(0); + + readProperties(_persistentCachePath.str(), cacheRoot); + for (unsigned int i=0; inChildren(); ++i) { + SGPropertyNode* entry = cacheRoot->getChild(i); + string tileName = entry->getStringValue("path"); + time_t stamp = entry->getIntValue("stamp"); + if (stamp < now) { + continue; + } + + _completedTiles[tileName] = stamp; + } +} + +void SGTerraSync::SvnThread::writeCompletedTilesPersistentCache() const +{ + // cache is disabled + if (_persistentCachePath.isNull()) { + return; + } + + std::ofstream f(_persistentCachePath.c_str(), std::ios::trunc); + if (!f.is_open()) { + return; + } + + SGPropertyNode_ptr cacheRoot(new SGPropertyNode); + CompletedTiles::const_iterator it = _completedTiles.begin(); + for (; it != _completedTiles.end(); ++it) { + SGPropertyNode* entry = cacheRoot->addChild("entry"); + entry->setStringValue("path", it->first); + entry->setIntValue("stamp", it->second); + } + + writeProperties(f, cacheRoot, true /* write_all */); + f.close(); +} + /////////////////////////////////////////////////////////////////////////////// // SGTerraSync //////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////// @@ -771,7 +858,9 @@ void SGTerraSync::reinit() _svnThread->setRsyncServer(_terraRoot->getStringValue("rsync-server","")); _svnThread->setLocalDir(_terraRoot->getStringValue("scenery-dir","")); _svnThread->setAllowedErrorCount(_terraRoot->getIntValue("max-errors",5)); - + _svnThread->setCachePath(SGPath(_terraRoot->getStringValue("cache-path",""))); + _svnThread->setCacheHits(_terraRoot->getIntValue("cache-hit", 0)); + #if defined(HAVE_SVN_CLIENT_H) || defined(SG_SVN_CLIENT) _svnThread->setUseBuiltin(_terraRoot->getBoolValue("use-built-in-svn",true)); #else @@ -805,6 +894,8 @@ void SGTerraSync::bind() _tiedProperties.Tie( _terraRoot->getNode("update-count", true), (int*) &_svnThread->_success_count ); _tiedProperties.Tie( _terraRoot->getNode("error-count", true), (int*) &_svnThread->_fail_count ); _tiedProperties.Tie( _terraRoot->getNode("tile-count", true), (int*) &_svnThread->_updated_tile_count ); + _tiedProperties.Tie( _terraRoot->getNode("cache-hits", true), (int*) &_svnThread->_cache_hits ); + _terraRoot->getNode("busy", true)->setAttribute(SGPropertyNode::WRITE,false); _terraRoot->getNode("active", true)->setAttribute(SGPropertyNode::WRITE,false); _terraRoot->getNode("update-count", true)->setAttribute(SGPropertyNode::WRITE,false); @@ -1033,3 +1124,4 @@ bool SGTerraSync::schedulePosition(int lat, int lon) return Ok; } + diff --git a/simgear/scene/tsync/terrasync.hxx b/simgear/scene/tsync/terrasync.hxx index cdffd723..0ba4bed6 100644 --- a/simgear/scene/tsync/terrasync.hxx +++ b/simgear/scene/tsync/terrasync.hxx @@ -26,6 +26,7 @@ #include class SGPath; +class SGBucket; namespace simgear { @@ -49,6 +50,9 @@ public: bool isIdle(); bool schedulePosition(int lat, int lon); + + bool scheduleTile(const SGBucket& bucket); + void setTileRefreshCb(SGTerraSyncCallback refreshCb, void* userData = NULL); /// retrive the associated log object, for displaying log @@ -70,6 +74,8 @@ private: SGPropertyNode_ptr _terraRoot; SGPropertyNode_ptr _refreshDisplay; SGPropertyNode_ptr _stalledNode; + SGPropertyNode_ptr _cacheHits; + SGTerraSyncCallback _refreshCb; void* _userCbData; simgear::TiedPropertyList _tiedProperties;