1 // terrasync.cxx -- scenery fetcher
3 // Started by Curtis Olson, November 2002.
5 // Copyright (C) 2002 Curtis L. Olson - http://www.flightgear.org/~curt
6 // Copyright (C) 2008 Alexander R. Perry <alex.perry@ieee.org>
7 // Copyright (C) 2011 Thorsten Brehm <brehmt@gmail.com>
9 // This program is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU General Public License as
11 // published by the Free Software Foundation; either version 2 of the
12 // License, or (at your option) any later version.
14 // This program is distributed in the hope that it will be useful, but
15 // WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // General Public License for more details.
19 // You should have received a copy of the GNU General Public License
20 // along with this program; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
26 # include <simgear_config.h>
29 #include <simgear/compiler.h>
38 #elif defined(_MSC_VER)
40 # ifndef HAVE_SVN_CLIENT_H
46 #include <stdlib.h> // atoi() atof() abs() system()
47 #include <signal.h> // signal()
54 #include <simgear/compiler.h>
56 #include "terrasync.hxx"
57 #include <simgear/bucket/newbucket.hxx>
58 #include <simgear/misc/sg_path.hxx>
59 #include <simgear/threads/SGQueue.hxx>
60 #include <simgear/scene/tgdb/TileCache.hxx>
61 #include <simgear/misc/sg_dir.hxx>
62 #include <OpenThreads/Thread>
64 #ifdef HAVE_SVN_CLIENT_H
65 # ifdef HAVE_LIBSVN_CLIENT_1
66 # include <svn_version.h>
67 # include <svn_auth.h>
68 # include <svn_client.h>
69 # include <svn_cmdline.h>
70 # include <svn_pools.h>
72 # undef HAVE_SVN_CLIENT_H
76 #ifdef HAVE_SVN_CLIENT_H
77 static const svn_version_checklist_t mysvn_checklist[] = {
78 { "svn_subr", svn_subr_version },
79 { "svn_client", svn_client_version },
82 static const bool svn_built_in_available = true;
84 static const bool svn_built_in_available = false;
87 using namespace simgear;
89 const char* rsync_cmd =
90 "rsync --verbose --archive --delete --perms --owner --group";
95 typedef map<string,time_t> CompletedTiles;
97 ///////////////////////////////////////////////////////////////////////////////
98 // WaitingTile ////////////////////////////////////////////////////////////////
99 ///////////////////////////////////////////////////////////////////////////////
103 WaitingTile(string dir,bool refresh) :
104 _dir(dir), _refreshScenery(refresh) {}
106 bool _refreshScenery;
109 ///////////////////////////////////////////////////////////////////////////////
110 // SGTerraSync::SvnThread /////////////////////////////////////////////////////
111 ///////////////////////////////////////////////////////////////////////////////
112 class SGTerraSync::SvnThread : public OpenThreads::Thread
116 virtual ~SvnThread( ) { stop(); }
121 bool isIdle() {return waitingTiles.empty();}
122 void request(const WaitingTile& dir) {waitingTiles.push_front(dir);}
123 bool isDirty() { bool r = _is_dirty;_is_dirty = false;return r;}
124 bool hasNewTiles() { return !_freshTiles.empty();}
125 WaitingTile getNewTile() { return _freshTiles.pop_front();}
127 void setSvnServer(string server) { _svn_server = server;}
128 void setRsyncServer(string server) { _rsync_server = server;}
129 void setLocalDir(string dir) { _local_dir = dir;}
130 string getLocalDir() { return _local_dir;}
131 void setUseSvn(bool use_svn) { _use_svn = use_svn;}
133 #ifdef HAVE_SVN_CLIENT_H
134 void setUseBuiltin(bool built_in) { _use_built_in = built_in;}
137 volatile bool _active;
138 volatile bool _running;
140 volatile bool _stalled;
141 volatile int _fail_count;
142 volatile int _updated_tile_count;
143 volatile int _success_count;
144 volatile int _consecutive_errors;
148 bool syncTree(const char* dir);
149 bool syncTreeExternal(const char* dir);
151 #ifdef HAVE_SVN_CLIENT_H
152 static int svnClientSetup(void);
153 bool syncTreeInternal(const char* dir);
157 // Things we need for doing subversion checkout - often
158 static apr_pool_t *_svn_pool;
159 static svn_client_ctx_t *_svn_ctx;
160 static svn_opt_revision_t *_svn_rev;
161 static svn_opt_revision_t *_svn_rev_peg;
164 volatile bool _is_dirty;
166 SGBlockingDeque <WaitingTile> waitingTiles;
167 CompletedTiles _completedTiles;
168 SGBlockingDeque <WaitingTile> _freshTiles;
171 string _rsync_server;
175 #ifdef HAVE_SVN_CLIENT_H
176 apr_pool_t* SGTerraSync::SvnThread::_svn_pool = NULL;
177 svn_client_ctx_t* SGTerraSync::SvnThread::_svn_ctx = NULL;
178 svn_opt_revision_t* SGTerraSync::SvnThread::_svn_rev = NULL;
179 svn_opt_revision_t* SGTerraSync::SvnThread::_svn_rev_peg = NULL;
182 SGTerraSync::SvnThread::SvnThread() :
188 _updated_tile_count(0),
190 _consecutive_errors(0),
191 #ifdef HAVE_SVN_CLIENT_H
198 #ifdef HAVE_SVN_CLIENT_H
199 int errCode = SGTerraSync::SvnThread::svnClientSetup();
200 if (errCode != EXIT_SUCCESS)
202 SG_LOG(SG_TERRAIN,SG_ALERT,
203 "Failed to initialize built-in SVN client, error = " << errCode);
208 void SGTerraSync::SvnThread::stop()
210 // drop any pending requests
211 waitingTiles.clear();
216 // set stop flag and wake up the thread with an empty request
218 WaitingTile w("",false);
224 bool SGTerraSync::SvnThread::start()
231 SG_LOG(SG_TERRAIN,SG_ALERT,
232 "Cannot start scenery download. No local cache directory defined.");
238 SGPath path(_local_dir);
239 path.append("version");
242 SG_LOG(SG_TERRAIN,SG_ALERT,
243 "Cannot start scenery download. Directory '" << _local_dir <<
244 "' contains the base package. Use a separate directory.");
249 #ifdef HAVE_SVN_CLIENT_H
250 _use_svn |= _use_built_in;
253 if ((_use_svn)&&(_svn_server==""))
255 SG_LOG(SG_TERRAIN,SG_ALERT,
256 "Cannot start scenery download. Subversion scenery server is undefined.");
262 if ((!_use_svn)&&(_rsync_server==""))
264 SG_LOG(SG_TERRAIN,SG_ALERT,
265 "Cannot start scenery download. Rsync scenery server is undefined.");
272 _updated_tile_count = 0;
274 _consecutive_errors = 0;
279 // not really an alert - but we want to (always) see this message, so user is
280 // aware we're downloading scenery (and using bandwidth).
281 SG_LOG(SG_TERRAIN,SG_ALERT,
282 "Starting automatic scenery download/synchronization. Directory: '" << _local_dir << "'");
284 OpenThreads::Thread::start();
288 // sync one directory tree
289 bool SGTerraSync::SvnThread::syncTree(const char* dir)
292 SGPath path( _local_dir );
295 rc = path.create_dir( 0755 );
298 SG_LOG(SG_TERRAIN,SG_ALERT,
299 "Cannot create directory '" << dir << "', return code = " << rc );
303 #ifdef HAVE_SVN_CLIENT_H
305 return syncTreeInternal(dir);
309 return syncTreeExternal(dir);
314 #ifdef HAVE_SVN_CLIENT_H
315 bool SGTerraSync::SvnThread::syncTreeInternal(const char* dir)
317 SG_LOG(SG_TERRAIN,SG_DEBUG, "Synchronizing scenery directory " << dir);
320 SG_LOG(SG_TERRAIN,SG_ALERT,
321 "Built-in SVN client failed to initialize.");
326 char dest_base_dir[512];
327 snprintf( command, 512,
328 "%s/%s", _svn_server.c_str(), dir);
329 snprintf( dest_base_dir, 512,
330 "%s/%s", _local_dir.c_str(), dir);
331 svn_error_t *err = NULL;
333 apr_pool_t *subpool = svn_pool_create(_svn_pool);
336 #if (SVN_VER_MINOR >= 5)
337 err = svn_client_checkout3(NULL,
343 0, // ignore-externals = false
344 0, // allow unver obstructions = false
349 err = svn_client_checkout2(NULL,
354 1, // recurse=true - same as svn_depth_infinity for checkout3 above
355 0, // ignore externals = false
360 bool ReturnValue = true;
363 // Report errors from the checkout attempt
364 SG_LOG(SG_TERRAIN,SG_ALERT,
365 "Failed to synchronize directory '" << dir << "', " <<
367 svn_error_clear(err);
369 err = svn_client_cleanup(dest_base_dir,
373 SG_LOG(SG_TERRAIN,SG_ALERT,
374 "SVN repository cleanup successful for '" << dir << "'.");
379 SG_LOG(SG_TERRAIN,SG_DEBUG, "Done with scenery directory " << dir);
381 svn_pool_destroy(subpool);
386 bool SGTerraSync::SvnThread::syncTreeExternal(const char* dir)
392 snprintf( command, 512,
393 "%s %s/%s %s/%s", svn_cmd,
394 _svn_server.c_str(), dir,
395 _local_dir.c_str(), dir );
397 snprintf( command, 512,
398 "%s %s/%s/ %s/%s/", rsync_cmd,
399 _rsync_server.c_str(), dir,
400 _local_dir.c_str(), dir );
402 SG_LOG(SG_TERRAIN,SG_DEBUG, "sync command '" << command << "'");
403 rc = system( command );
406 SG_LOG(SG_TERRAIN,SG_ALERT,
407 "Failed to synchronize directory '" << dir << "', " <<
408 "error code= " << rc);
414 void SGTerraSync::SvnThread::run()
419 WaitingTile next = waitingTiles.pop_front();
423 CompletedTiles::iterator ii =
424 _completedTiles.find( next._dir );
425 time_t now = time(0);
426 if ((ii == _completedTiles.end())||
427 ((ii->second + 60*60*24) < now ))
430 if (!syncTree(next._dir.c_str()))
432 _consecutive_errors++;
437 _consecutive_errors = 0;
439 SG_LOG(SG_TERRAIN,SG_INFO,
440 "Successfully synchronized directory '" << next._dir << "'");
441 if (next._refreshScenery)
444 _updated_tile_count++;
445 _freshTiles.push_back(next);
450 _completedTiles[ next._dir ] = now;
453 if (_consecutive_errors >= 5)
465 #ifdef HAVE_SVN_CLIENT_H
466 // Configure our subversion session
467 int SGTerraSync::SvnThread::svnClientSetup(void)
469 // Are we already prepared?
470 if (_svn_pool) return EXIT_SUCCESS;
471 // No, so initialize svn internals generally
474 // there is a segfault when providing an error stream.
475 // Apparently, calling setvbuf with a nul buffer is
476 // not supported under msvc 7.1 ( code inside svn_cmdline_init )
477 if (svn_cmdline_init("terrasync", 0) != EXIT_SUCCESS)
480 if (svn_cmdline_init("terrasync", stderr) != EXIT_SUCCESS)
483 /* Oh no! svn_cmdline_init configures the locale - affecting numeric output
484 * formats (i.e. sprintf("%f", ...)). fgfs relies on "C" locale in many places
485 * (including assumptions on required sprintf buffer sizes). Things go horribly
486 * wrong when the locale is changed to anything else but "C". Might be enough to
487 * revert LC_NUMERIC locale - but we'll do a complete revert for now...*/
488 setlocale(LC_ALL,"C");
491 apr_pool_create(&pool, NULL);
492 svn_error_t *err = NULL;
493 SVN_VERSION_DEFINE(_svn_version);
494 err = svn_ver_check_list(&_svn_version, mysvn_checklist);
496 return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
497 err = svn_ra_initialize(pool);
499 return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
500 char *config_dir = NULL;
501 err = svn_config_ensure(config_dir, pool);
503 return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
504 err = svn_client_create_context(&_svn_ctx, pool);
506 return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
507 err = svn_config_get_config(&(_svn_ctx->config),
510 return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
512 cfg = ( svn_config_t*) apr_hash_get(
514 SVN_CONFIG_CATEGORY_CONFIG,
515 APR_HASH_KEY_STRING);
517 return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
519 svn_auth_baton_t *ab=NULL;
521 #if (SVN_VER_MINOR >= 6)
522 err = svn_cmdline_create_auth_baton (&ab,
523 TRUE, NULL, NULL, config_dir, TRUE, FALSE, cfg,
524 _svn_ctx->cancel_func, _svn_ctx->cancel_baton, pool);
526 err = svn_cmdline_setup_auth_baton(&ab,
527 TRUE, NULL, NULL, config_dir, TRUE, cfg,
528 _svn_ctx->cancel_func, _svn_ctx->cancel_baton, pool);
532 return svn_cmdline_handle_exit_error(err, pool, "fgfs: ");
534 _svn_ctx->auth_baton = ab;
535 #if (SVN_VER_MINOR >= 5)
536 _svn_ctx->conflict_func = NULL;
537 _svn_ctx->conflict_baton = NULL;
540 // Now our magic revisions
541 _svn_rev = (svn_opt_revision_t*) apr_palloc(pool,
542 sizeof(svn_opt_revision_t));
545 _svn_rev_peg = (svn_opt_revision_t*) apr_palloc(pool,
546 sizeof(svn_opt_revision_t));
549 _svn_rev->kind = svn_opt_revision_head;
550 _svn_rev_peg->kind = svn_opt_revision_unspecified;
551 // Success if we got this far
557 ///////////////////////////////////////////////////////////////////////////////
558 // SGTerraSync ////////////////////////////////////////////////////////////////
559 ///////////////////////////////////////////////////////////////////////////////
560 SGTerraSync::SGTerraSync(SGPropertyNode_ptr root) :
564 _terraRoot(root->getNode("/sim/terrasync",true)),
567 _svnThread = new SvnThread();
570 SGTerraSync::~SGTerraSync()
572 _tiedProperties.Untie();
577 void SGTerraSync::init()
579 _refresh_display = _terraRoot->getNode("refresh-display",true);
580 _terraRoot->getNode("built-in-svn-available",true)->setBoolValue(svn_built_in_available);
584 void SGTerraSync::reinit()
586 // do not reinit when enabled and we're already up and running
587 if ((_terraRoot->getNode("enabled",true)->getBoolValue())&&
588 (_svnThread->_active && _svnThread->_running))
593 if (_terraRoot->getNode("enabled",true)->getBoolValue())
595 _svnThread->setSvnServer(_terraRoot->getNode("svn-server",true)->getStringValue());
596 _svnThread->setRsyncServer(_terraRoot->getNode("rsync-server",true)->getStringValue());
597 _svnThread->setLocalDir(_terraRoot->getNode("scenery-dir",true)->getStringValue());
599 #ifdef HAVE_SVN_CLIENT_H
600 _svnThread->setUseBuiltin(_terraRoot->getNode("use-built-in-svn",true)->getBoolValue());
602 _terraRoot->getNode("use-built-in-svn",true)->setBoolValue(false);
604 _svnThread->setUseSvn(_terraRoot->getNode("use-svn",true)->getBoolValue());
606 if (_svnThread->start())
607 syncAirportsModels();
610 _stalled_node->setBoolValue(_svnThread->_stalled);
615 void SGTerraSync::bind()
617 _tiedProperties.Tie( _terraRoot->getNode("busy", true), (bool*) &_svnThread->_busy );
618 _tiedProperties.Tie( _terraRoot->getNode("active", true), (bool*) &_svnThread->_active );
619 _tiedProperties.Tie( _terraRoot->getNode("update-count", true), (int*) &_svnThread->_success_count );
620 _tiedProperties.Tie( _terraRoot->getNode("error-count", true), (int*) &_svnThread->_fail_count );
621 _tiedProperties.Tie( _terraRoot->getNode("tile-count", true), (int*) &_svnThread->_updated_tile_count );
622 _terraRoot->getNode("busy", true)->setAttribute(SGPropertyNode::WRITE,false);
623 _terraRoot->getNode("active", true)->setAttribute(SGPropertyNode::WRITE,false);
624 _terraRoot->getNode("update-count", true)->setAttribute(SGPropertyNode::WRITE,false);
625 _terraRoot->getNode("error-count", true)->setAttribute(SGPropertyNode::WRITE,false);
626 _terraRoot->getNode("tile-count", true)->setAttribute(SGPropertyNode::WRITE,false);
627 // stalled is used as a signal handler (to connect listeners triggering GUI pop-ups)
628 _stalled_node = _terraRoot->getNode("stalled", true);
629 _stalled_node->setBoolValue(_svnThread->_stalled);
630 _stalled_node->setAttribute(SGPropertyNode::PRESERVE,true);
633 void SGTerraSync::unbind()
636 _tiedProperties.Untie();
639 void SGTerraSync::update(double)
641 static SGBucket bucket;
642 if (_svnThread->isDirty())
644 if (!_svnThread->_active)
646 if (_svnThread->_stalled)
648 SG_LOG(SG_TERRAIN,SG_ALERT,
649 "Automatic scenery download/synchronization stalled. Too many errors.");
653 // not really an alert - just always show this message
654 SG_LOG(SG_TERRAIN,SG_ALERT,
655 "Automatic scenery download/synchronization has stopped.");
657 _stalled_node->setBoolValue(_svnThread->_stalled);
660 if (!_refresh_display->getBoolValue())
663 while (_svnThread->hasNewTiles())
665 WaitingTile next = _svnThread->getNewTile();
666 if (next._refreshScenery)
668 refreshScenery(_svnThread->getLocalDir(),next._dir);
674 void SGTerraSync::refreshScenery(SGPath path,const string& relativeDir)
676 // find tiles to be refreshed
679 path.append(relativeDir);
682 simgear::Dir dir(path);
683 //TODO need to be smarter here. only update tiles which actually
684 // changed recently. May also be possible to use information from the
685 // built-in SVN client directly (instead of checking directory contents).
686 PathList tileList = dir.children(simgear::Dir::TYPE_FILE, ".stg");
687 for (unsigned int i=0; i<tileList.size(); ++i)
689 // reload scenery tile
690 long index = atoi(tileList[i].file().c_str());
691 _tile_cache->refresh_tile(index);
697 bool SGTerraSync::isIdle() {return _svnThread->isIdle();}
699 void SGTerraSync::setTileCache(TileCache* tile_cache)
701 _tile_cache = tile_cache;
704 void SGTerraSync::syncAirportsModels()
707 for ( synced_other = 'K'; synced_other <= 'Z'; synced_other++ )
710 snprintf( dir, 512, "Airports/%c", synced_other );
711 WaitingTile w(dir,false);
712 _svnThread->request( w );
714 for ( synced_other = 'A'; synced_other <= 'J'; synced_other++ )
717 snprintf( dir, 512, "Airports/%c", synced_other );
718 WaitingTile w(dir,false);
719 _svnThread->request( w );
721 WaitingTile w("Models",false);
722 _svnThread->request( w );
726 void SGTerraSync::syncArea( int lat, int lon )
728 if ( lat < -90 || lat > 90 || lon < -180 || lon > 180 )
731 int baselat, baselon;
734 int base = (int)(lat / 10);
735 if ( lat == base * 10 ) {
738 baselat = (base - 1) * 10;
742 baselat = (int)(lat / 10) * 10;
746 int base = (int)(lon / 10);
747 if ( lon == base * 10 ) {
750 baselon = (base - 1) * 10;
754 baselon = (int)(lon / 10) * 10;
758 const char* terrainobjects[3] = { "Terrain", "Objects", 0 };
760 for (const char** tree = &terrainobjects[0]; *tree; tree++)
763 snprintf( dir, 512, "%s/%c%03d%c%02d/%c%03d%c%02d",
765 EW, abs(baselon), NS, abs(baselat),
766 EW, abs(lon), NS, abs(lat) );
767 WaitingTile w(dir,refresh);
768 _svnThread->request( w );
774 void SGTerraSync::syncAreas( int lat, int lon, int lat_dir, int lon_dir )
776 if ( lat_dir == 0 && lon_dir == 0 ) {
777 // do surrounding 8 1x1 degree areas.
778 for ( int i = lat - 1; i <= lat + 1; ++i ) {
779 for ( int j = lon - 1; j <= lon + 1; ++j ) {
780 if ( i != lat || j != lon ) {
786 if ( lat_dir != 0 ) {
787 syncArea( lat + lat_dir, lon - 1 );
788 syncArea( lat + lat_dir, lon + 1 );
789 syncArea( lat + lat_dir, lon );
791 if ( lon_dir != 0 ) {
792 syncArea( lat - 1, lon + lon_dir );
793 syncArea( lat + 1, lon + lon_dir );
794 syncArea( lat, lon + lon_dir );
798 // do current 1x1 degree area first
799 syncArea( lat, lon );
803 bool SGTerraSync::schedulePosition(int lat, int lon)
805 // Ignore messages where the location does not change
806 if ( lat != last_lat || lon != last_lon )
808 SG_LOG(SG_TERRAIN,SG_DEBUG, "Requesting scenery update for position " <<
810 int lat_dir, lon_dir, dist;
811 if ( last_lat == NOWHERE || last_lon == NOWHERE )
813 lat_dir = lon_dir = 0;
816 dist = lat - last_lat;
819 lat_dir = dist / abs(dist);
825 dist = lon - last_lon;
828 lon_dir = dist / abs(dist);
835 SG_LOG(SG_TERRAIN,SG_DEBUG, "Scenery update for " <<
836 "lat = " << lat << ", lon = " << lon <<
837 ", lat_dir = " << lat_dir << ", " <<
838 "lon_dir = " << lon_dir);
840 syncAreas( lat, lon, lat_dir, lon_dir );