using namespace simgear;
using namespace std;
-const char* rsync_cmd =
+const char* rsync_cmd =
"rsync --verbose --archive --delete --perms --owner --group";
const char* svn_options =
static const double FailedAttempt = 10*60;
}
-typedef map<string,time_t> CompletedTiles;
+typedef map<string,time_t> TileAgeCache;
///////////////////////////////////////////////////////////////////////////////
// helper functions ///////////////////////////////////////////////////////////
SharedModels,
AIData
};
-
+
enum Status
{
Invalid = 0,
Waiting,
Cached, ///< using already cached result
Updated,
- NotFound,
+ NotFound,
Failed
};
-
+
SyncItem() :
_dir(),
_type(Stop),
_status(Invalid)
{
}
-
+
SyncItem(string dir, Type ty) :
_dir(dir),
_type(ty),
_status(Waiting)
{}
-
+
string _dir;
Type _type;
Status _status;
isNewDirectory(false),
busy(false)
{}
-
+
SyncItem currentItem;
bool isNewDirectory;
std::queue<SyncItem> queue;
std::auto_ptr<SVNRepository> repository;
SGTimeStamp stamp;
bool busy; ///< is the slot working or idle
-
+
unsigned int nextWarnTimeout;
};
return SYNC_SLOT_SHARED_DATA;
case SyncItem::AIData:
return SYNC_SLOT_AI_DATA;
-
+
default:
return SYNC_SLOT_SHARED_DATA;
}
}
+///////////////////////////////////////////////////////////////////////////////
+// Base server query
+///////////////////////////////////////////////////////////////////////////////
+
+class ServerSelectQuery : public HTTP::Request
+{
+public:
+ ServerSelectQuery() :
+ HTTP::Request("http://scenery.flightgear.org/svn-server", "GET")
+ {
+ }
+
+ std::string svnUrl() const
+ {
+ std::string s = simgear::strutils::strip(m_url);
+ if (s.at(s.length() - 1) == '/') {
+ s = s.substr(0, s.length() - 1);
+ }
+
+ return s;
+ }
+protected:
+ virtual void gotBodyData(const char* s, int n)
+ {
+ m_url.append(std::string(s, n));
+ }
+
+ virtual void onFail()
+ {
+ SG_LOG(SG_TERRASYNC, SG_ALERT, "Failed to query TerraSync SVN server");
+ HTTP::Request::onFail();
+ }
+
+private:
+ std::string m_url;
+};
+
///////////////////////////////////////////////////////////////////////////////
// SGTerraSync::SvnThread /////////////////////////////////////////////////////
void setSvnServer(string server) { _svn_server = stripPath(server);}
void setSvnDataServer(string server) { _svn_data_server = stripPath(server);}
-
+
void setExtSvnUtility(string svn_util) { _svn_command = simgear::strutils::strip(svn_util);}
void setRsyncServer(string server) { _rsync_server = simgear::strutils::strip(server);}
void setLocalDir(string dir) { _local_dir = stripPath(dir);}
volatile int _transfer_rate;
// kbytes, not bytes, because bytes might overflow 2^31
volatile int _total_kb_downloaded;
-
+
private:
virtual void run();
-
+
// external model run and helpers
void runExternal();
void syncPathExternal(const SyncItem& next);
void updateSyncSlot(SyncSlot& slot);
// commond helpers between both internal and external models
-
- bool isPathCached(const SyncItem& next) const;
+
+ SyncItem::Status isPathCached(const SyncItem& next) const;
void initCompletedTilesPersistentCache();
void writeCompletedTilesPersistentCache() const;
void updated(SyncItem item, bool isNewDirectory);
void fail(SyncItem failedItem);
-
+ void notFound(SyncItem notFoundItem);
+
bool _use_built_in;
HTTP::Client _http;
SyncSlot _syncSlots[NUM_SYNC_SLOTS];
volatile bool _is_dirty;
volatile bool _stop;
SGBlockingDeque <SyncItem> waitingTiles;
- CompletedTiles _completedTiles;
+
+ TileAgeCache _completedTiles;
+ TileAgeCache _notFoundItems;
+
SGBlockingDeque <SyncItem> _freshTiles;
bool _use_svn;
string _svn_server;
if (_local_dir=="")
{
- SG_LOG(SG_TERRAIN,SG_ALERT,
+ SG_LOG(SG_TERRASYNC,SG_ALERT,
"Cannot start scenery download. Local cache directory is undefined.");
_fail_count++;
_stalled = true;
SGPath path(_local_dir);
if (!path.exists())
{
- SG_LOG(SG_TERRAIN,SG_ALERT,
+ SG_LOG(SG_TERRASYNC,SG_ALERT,
"Cannot start scenery download. Directory '" << _local_dir <<
"' does not exist. Set correct directory path or create directory folder.");
_fail_count++;
path.append("version");
if (path.exists())
{
- SG_LOG(SG_TERRAIN,SG_ALERT,
+ SG_LOG(SG_TERRASYNC,SG_ALERT,
"Cannot start scenery download. Directory '" << _local_dir <<
"' contains the base package. Use a separate directory.");
_fail_count++;
_use_svn |= _use_built_in;
- if ((_use_svn)&&(_svn_server==""))
- {
- SG_LOG(SG_TERRAIN,SG_ALERT,
- "Cannot start scenery download. Subversion scenery server is undefined.");
- _fail_count++;
- _stalled = true;
- return false;
- }
+
if ((!_use_svn)&&(_rsync_server==""))
{
- SG_LOG(SG_TERRAIN,SG_ALERT,
+ SG_LOG(SG_TERRASYNC,SG_ALERT,
"Cannot start scenery download. Rsync scenery server is undefined.");
_fail_count++;
_stalled = true;
// not really an alert - but we want to (always) see this message, so user is
// aware we're downloading scenery (and using bandwidth).
- SG_LOG(SG_TERRAIN,SG_ALERT,
+ SG_LOG(SG_TERRASYNC,SG_ALERT,
"Starting automatic scenery download/synchronization. "
<< status
<< "Directory: '" << _local_dir << "'.");
// windows command line parsing is just lovely...
// to allow white spaces, the system call needs this:
// ""C:\Program Files\something.exe" somearg "some other arg""
- // Note: whitespace strings quoted by a pair of "" _and_ the
+ // Note: whitespace strings quoted by a pair of "" _and_ the
// entire string needs to be wrapped by "" too.
// The svn url needs forward slashes (/) as a path separator while
// the local path needs windows-native backslash as a path separator.
#else
command = buf.str();
#endif
- SG_LOG(SG_TERRAIN,SG_DEBUG, "sync command '" << command << "'");
+ SG_LOG(SG_TERRASYNC,SG_DEBUG, "sync command '" << command << "'");
#ifdef SG_WINDOWS
// tbd: does Windows support "popen"?
if (rc)
{
- SG_LOG(SG_TERRAIN,SG_ALERT,
+ SG_LOG(SG_TERRASYNC,SG_ALERT,
"Failed to synchronize directory '" << dir << "', " <<
"error code= " << rc);
return false;
{
_active = true;
initCompletedTilesPersistentCache();
-
+
+ {
+ if (_svn_server.empty()) {
+ SG_LOG(SG_TERRASYNC,SG_INFO, "Querying closest TerraSync server");
+ ServerSelectQuery* ssq = new ServerSelectQuery;
+ HTTP::Request_ptr req = ssq;
+ _http.makeRequest(req);
+ while (!req->isComplete()) {
+ _http.update(20);
+ }
+
+ if (req->readyState() == HTTP::Request::DONE) {
+ _svn_server = ssq->svnUrl();
+ SG_LOG(SG_TERRASYNC,SG_INFO, "Closest TerraSync server:" << _svn_server);
+ } else {
+ SG_LOG(SG_TERRASYNC,SG_WARN, "Failed to query closest TerraSync server");
+ }
+ } else {
+ SG_LOG(SG_TERRASYNC,SG_INFO, "Explicit: TerraSync server:" << _svn_server);
+ }
+
+ if (_svn_server.empty()) {
+ // default value
+ _svn_server = "http://foxtrot.mgras.net:8080/terrascenery/trunk/data/Scenery";
+ }
+ }
+
if (_use_built_in) {
runInternal();
} else {
runExternal();
}
-
+
_active = false;
_running = false;
_is_dirty = true;
if (_stop)
break;
- if (isPathCached(next)) {
+ SyncItem::Status cacheStatus = isPathCached(next);
+ if (cacheStatus != SyncItem::Invalid) {
_cache_hits++;
- SG_LOG(SG_TERRAIN, SG_DEBUG,
+ SG_LOG(SG_TERRASYNC, SG_DEBUG,
"Cache hit for: '" << next._dir << "'");
- next._status = SyncItem::Cached;
+ next._status = cacheStatus;
_freshTiles.push_back(next);
_is_dirty = true;
continue;
}
-
+
syncPathExternal(next);
if ((_allowed_errors >= 0)&&
SGPath path( _local_dir );
path.append( next._dir );
bool isNewDirectory = !path.exists();
-
+
try {
if (isNewDirectory) {
int rc = path.create_dir( 0755 );
if (rc) {
- SG_LOG(SG_TERRAIN,SG_ALERT,
+ SG_LOG(SG_TERRASYNC,SG_ALERT,
"Cannot create directory '" << path << "', return code = " << rc );
throw sg_exception("Cannot create directory for terrasync", path.str());
}
}
-
+
if (!runExternalSyncCommand(next._dir.c_str())) {
throw sg_exception("Running external sync command failed");
}
_busy = false;
return;
}
-
+
updated(next, isNewDirectory);
_busy = false;
}
{
if (slot.repository.get()) {
if (slot.repository->isDoingSync()) {
-#if 0
- if (slot.stamp.elapsedMSec() > slot.nextWarnTimeout) {
- SG_LOG(SG_TERRAIN, SG_INFO, "sync taking a long time:" << slot.currentItem._dir << " taken " << slot.stamp.elapsedMSec());
- SG_LOG(SG_TERRAIN, SG_INFO, "HTTP status:" << _http.hasActiveRequests());
+#if 1
+ if (slot.stamp.elapsedMSec() > (int)slot.nextWarnTimeout) {
+ SG_LOG(SG_TERRASYNC, SG_INFO, "sync taking a long time:" << slot.currentItem._dir << " taken " << slot.stamp.elapsedMSec());
+ SG_LOG(SG_TERRASYNC, SG_INFO, "HTTP request count:" << _http.hasActiveRequests());
slot.nextWarnTimeout += 10000;
}
#endif
return; // easy, still working
}
-
+
// check result
SVNRepository::ResultCode res = slot.repository->failure();
if (res == SVNRepository::SVN_ERROR_NOT_FOUND) {
- // this is fine, but maybe we should use a different return code
- // in the future to higher layers can distinguish this case
- slot.currentItem._status = SyncItem::NotFound;
- _freshTiles.push_back(slot.currentItem);
- _is_dirty = true;
+ notFound(slot.currentItem);
} else if (res != SVNRepository::SVN_NO_ERROR) {
fail(slot.currentItem);
} else {
updated(slot.currentItem, slot.isNewDirectory);
- SG_LOG(SG_TERRAIN, SG_DEBUG, "sync of " << slot.repository->baseUrl() << " finished ("
+ SG_LOG(SG_TERRASYNC, SG_DEBUG, "sync of " << slot.repository->baseUrl() << " finished ("
<< slot.stamp.elapsedMSec() << " msec");
}
slot.busy = false;
slot.repository.reset();
}
-
+
// init and start sync of the next repository
if (!slot.queue.empty()) {
slot.currentItem = slot.queue.front();
slot.queue.pop();
-
+
SGPath path(_local_dir);
path.append(slot.currentItem._dir);
slot.isNewDirectory = !path.exists();
if (slot.isNewDirectory) {
int rc = path.create_dir( 0755 );
if (rc) {
- SG_LOG(SG_TERRAIN,SG_ALERT,
+ SG_LOG(SG_TERRASYNC,SG_ALERT,
"Cannot create directory '" << path << "', return code = " << rc );
fail(slot.currentItem);
return;
}
} // of creating directory step
-
+
string serverUrl(_svn_server);
if (slot.currentItem._type == SyncItem::AIData) {
serverUrl = _svn_data_server;
}
-
+
slot.repository.reset(new SVNRepository(path, &_http));
slot.repository->setBaseUrl(serverUrl + "/" + slot.currentItem._dir);
slot.repository->update();
-
+
slot.nextWarnTimeout = 20000;
slot.stamp.stamp();
slot.busy = true;
- SG_LOG(SG_TERRAIN, SG_DEBUG, "sync of " << slot.repository->baseUrl() << " started, queue size is " << slot.queue.size());
+ SG_LOG(SG_TERRASYNC, SG_INFO, "sync of " << slot.repository->baseUrl() << " started, queue size is " << slot.queue.size());
}
}
_transfer_rate = _http.transferRateBytesPerSec();
// convert from bytes to kbytes
_total_kb_downloaded = static_cast<int>(_http.totalBytesDownloaded() / 1024);
-
+
if (_stop)
break;
-
+
// drain the waiting tiles queue into the sync slot queues.
while (!waitingTiles.empty()) {
SyncItem next = waitingTiles.pop_front();
- if (isPathCached(next)) {
+ SyncItem::Status cacheStatus = isPathCached(next);
+ if (cacheStatus != SyncItem::Invalid) {
_cache_hits++;
- SG_LOG(SG_TERRAIN, SG_DEBUG, "TerraSync Cache hit for: '" << next._dir << "'");
- next._status = SyncItem::Cached;
+ SG_LOG(SG_TERRASYNC, SG_DEBUG, "\nTerraSync Cache hit for: '" << next._dir << "'");
+ next._status = cacheStatus;
_freshTiles.push_back(next);
_is_dirty = true;
continue;
unsigned int slot = syncSlotForType(next._type);
_syncSlots[slot].queue.push(next);
}
-
+
bool anySlotBusy = false;
// update each sync slot in turn
for (unsigned int slot=0; slot < NUM_SYNC_SLOTS; ++slot) {
} // of thread running loop
}
-bool SGTerraSync::SvnThread::isPathCached(const SyncItem& next) const
+SyncItem::Status SGTerraSync::SvnThread::isPathCached(const SyncItem& next) const
{
- CompletedTiles::const_iterator ii = _completedTiles.find( next._dir );
+ TileAgeCache::const_iterator ii = _completedTiles.find( next._dir );
if (ii == _completedTiles.end()) {
- return false;
+ ii = _notFoundItems.find( next._dir );
+ // Invalid means 'not cached', otherwise we want to return to
+ // higher levels the cache status
+ return (ii == _notFoundItems.end()) ? SyncItem::Invalid : SyncItem::NotFound;
}
-
- // check if the path still physically exists
+
+ // check if the path still physically exists. This is needed to
+ // cope with the user manipulating our cache dir
SGPath p(_local_dir);
p.append(next._dir);
if (!p.exists()) {
- return false;
+ return SyncItem::Invalid;
}
-
+
time_t now = time(0);
- return (ii->second > now);
+ return (ii->second > now) ? SyncItem::Cached : SyncItem::Invalid;
}
void SGTerraSync::SvnThread::fail(SyncItem failedItem)
_fail_count++;
failedItem._status = SyncItem::Failed;
_freshTiles.push_back(failedItem);
+ SG_LOG(SG_TERRASYNC,SG_INFO,
+ "Failed to sync'" << failedItem._dir << "'");
_completedTiles[ failedItem._dir ] = now + UpdateInterval::FailedAttempt;
_is_dirty = true;
}
+void SGTerraSync::SvnThread::notFound(SyncItem item)
+{
+ // treat not found as authorative, so use the same cache expiry
+ // as succesful download. Important for MP models and similar so
+ // we don't spam the server with lookups for models that don't
+ // exist
+
+ time_t now = time(0);
+ item._status = SyncItem::NotFound;
+ _freshTiles.push_back(item);
+ _is_dirty = true;
+ _notFoundItems[ item._dir ] = now + UpdateInterval::SuccessfulAttempt;
+ writeCompletedTilesPersistentCache();
+}
+
void SGTerraSync::SvnThread::updated(SyncItem item, bool isNewDirectory)
{
time_t now = time(0);
_consecutive_errors = 0;
_success_count++;
- SG_LOG(SG_TERRAIN,SG_INFO,
+ SG_LOG(SG_TERRASYNC,SG_INFO,
"Successfully synchronized directory '" << item._dir << "'");
-
+
item._status = SyncItem::Updated;
if (item._type == SyncItem::Tile) {
_updated_tile_count++;
if (!_persistentCachePath.exists()) {
return;
}
-
+
SGPropertyNode_ptr cacheRoot(new SGPropertyNode);
time_t now = time(0);
-
+
try {
readProperties(_persistentCachePath.str(), cacheRoot);
} catch (sg_exception& e) {
- SG_LOG(SG_TERRAIN, SG_INFO, "corrupted persistent cache, discarding");
+ SG_LOG(SG_TERRASYNC, SG_INFO, "corrupted persistent cache, discarding");
return;
}
-
+
for (int i=0; i<cacheRoot->nChildren(); ++i) {
SGPropertyNode* entry = cacheRoot->getChild(i);
+ bool isNotFound = (strcmp(entry->getName(), "not-found") == 0);
string tileName = entry->getStringValue("path");
time_t stamp = entry->getIntValue("stamp");
if (stamp < now) {
continue;
}
-
- _completedTiles[tileName] = stamp;
+
+ if (isNotFound) {
+ _completedTiles[tileName] = stamp;
+ } else {
+ _notFoundItems[tileName] = stamp;
+ }
}
}
if (_persistentCachePath.isNull()) {
return;
}
-
+
std::ofstream f(_persistentCachePath.c_str(), std::ios::trunc);
if (!f.is_open()) {
return;
}
-
+
SGPropertyNode_ptr cacheRoot(new SGPropertyNode);
- CompletedTiles::const_iterator it = _completedTiles.begin();
+ TileAgeCache::const_iterator it = _completedTiles.begin();
for (; it != _completedTiles.end(); ++it) {
SGPropertyNode* entry = cacheRoot->addChild("entry");
entry->setStringValue("path", it->first);
entry->setIntValue("stamp", it->second);
}
-
+
+ it = _notFoundItems.begin();
+ for (; it != _notFoundItems.end(); ++it) {
+ SGPropertyNode* entry = cacheRoot->addChild("not-found");
+ entry->setStringValue("path", it->first);
+ entry->setIntValue("stamp", it->second);
+ }
+
writeProperties(f, cacheRoot, true /* write_all */);
f.close();
}
_inited(false)
{
_svnThread = new SvnThread();
- _log = new BufferedLogCallback(SG_TERRAIN, SG_INFO);
+ _log = new BufferedLogCallback(SG_TERRASYNC, SG_INFO);
_log->truncateAt(255);
-
+
sglog().addCallback(_log);
}
if (_inited) {
return;
}
-
+
_inited = true;
-
+
assert(_terraRoot);
_terraRoot->setBoolValue("built-in-svn-available",svn_built_in_available);
-
+
reinit();
}
{
return;
}
-
+
_svnThread->stop();
if (_terraRoot->getBoolValue("enabled",false))
if (_bound) {
return;
}
-
+
_bound = true;
_tiedProperties.Tie( _terraRoot->getNode("busy", true), (bool*) &_svnThread->_busy );
_tiedProperties.Tie( _terraRoot->getNode("active", true), (bool*) &_svnThread->_active );
_tiedProperties.Tie( _terraRoot->getNode("tile-count", true), (int*) &_svnThread->_updated_tile_count );
_tiedProperties.Tie( _terraRoot->getNode("cache-hits", true), (int*) &_svnThread->_cache_hits );
_tiedProperties.Tie( _terraRoot->getNode("transfer-rate-bytes-sec", true), (int*) &_svnThread->_transfer_rate );
-
+
// use kbytes here because propety doesn't support 64-bit and we might conceivably
// download more than 2G in a single session
_tiedProperties.Tie( _terraRoot->getNode("downloaded-kbytes", true), (int*) &_svnThread->_total_kb_downloaded );
-
+
_terraRoot->getNode("busy", true)->setAttribute(SGPropertyNode::WRITE,false);
_terraRoot->getNode("active", true)->setAttribute(SGPropertyNode::WRITE,false);
_terraRoot->getNode("update-count", true)->setAttribute(SGPropertyNode::WRITE,false);
_tiedProperties.Untie();
_bound = false;
_inited = false;
-
+
_terraRoot.clear();
_stalledNode.clear();
_cacheHits.clear();
{
if (_svnThread->_stalled)
{
- SG_LOG(SG_TERRAIN,SG_ALERT,
+ SG_LOG(SG_TERRASYNC,SG_ALERT,
"Automatic scenery download/synchronization stalled. Too many errors.");
}
else
{
// not really an alert - just always show this message
- SG_LOG(SG_TERRAIN,SG_ALERT,
+ SG_LOG(SG_TERRASYNC,SG_ALERT,
"Automatic scenery download/synchronization has stopped.");
}
_stalledNode->setBoolValue(_svnThread->_stalled);
while (_svnThread->hasNewTiles())
{
SyncItem next = _svnThread->getNewTile();
-
+
if ((next._type == SyncItem::Tile) || (next._type == SyncItem::AIData)) {
_activeTileDirs.erase(next._dir);
}
_svnThread->request( w );
}
}
-
+
SyncItem w("Models", SyncItem::SharedModels);
_svnThread->request( w );
}
if (_activeTileDirs.find(dir) != _activeTileDirs.end()) {
continue;
}
-
+
_activeTileDirs.insert(dir);
SyncItem w(dir, SyncItem::Tile);
_svnThread->request( w );
if (!_svnThread->_running) {
return false;
}
-
+
const char* terrainobjects[3] = { "Terrain/", "Objects/", 0 };
for (const char** tree = &terrainobjects[0]; *tree; tree++) {
string s = *tree + sceneryDir;
if (_activeTileDirs.find(dataDir) != _activeTileDirs.end()) {
return;
}
-
+
_activeTileDirs.insert(dataDir);
SyncItem w(dataDir, SyncItem::AIData);
_svnThread->request( w );
if (!_svnThread->_running) {
return false;
}
-
+
return (_activeTileDirs.find(dataDir) != _activeTileDirs.end());
}
void SGTerraSync::reposition()
{
// stub, remove
-}
\ No newline at end of file
+}
projects / simgear.git / blobdiff