From: Torsten Dreyer Date: Wed, 11 May 2016 12:59:36 +0000 (+0200) Subject: Move terrasync.py to /scripts/python X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=9999241fb3fd1e1d3ee4e3c386812e48895c6f51;p=flightgear.git Move terrasync.py to /scripts/python --- diff --git a/scripts/python/terrasync.py b/scripts/python/terrasync.py new file mode 100755 index 000000000..a26d67ee6 --- /dev/null +++ b/scripts/python/terrasync.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (C) 2016 Torsten Dreyer +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# terrasync.py - synchronize terrascenery data to your local disk +# needs dnspython (pip install dnspython) +# +# this is a terrible stub, please improve + +import os +import hashlib +import urllib.request + +dirindex = ".dirindex" +DIRINDEXVERSION = 1 + +URL="automatic" +TARGET="." + +######################################################################## + +def fn_hash_of_file(fname): + hash = hashlib.sha1() + try: + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash.update(chunk) + except: + pass + + return hash.hexdigest() + +######################################################################## +def do_download_file( _url, _path, _localfile, _hash ): + if os.path.exists( _localfile ): + h = fn_hash_of_file(_localfile) + if h == _hash: + print("hash match for ", _localfile) + return + + r = urllib.request.urlopen( _url + _path ) + f = open(_localfile, 'wb') + f.write( r.read() ) + f.close() + print("downloaded ", _localfile, " from ", _url + _path ) + +######################################################################## +def do_terrasync( _url, _path, _localdir ): + url = _url + _path + print("syncing ",url, " to ", _localdir ) + + if not os.path.exists( _localdir ): + os.makedirs( _localdir ) + + try: + for line in urllib.request.urlopen(url + "/.dirindex").readlines(): + tokens = line.decode("utf-8").rstrip().split(':') + if( len(tokens) == 0 ): + continue + + # TODO: check version number, should be equal to DIRINDEXVERSION + # otherwise complain and terminate + if( tokens[0] == "version" ): + continue + + if( tokens[0] == "path" ): + continue + + if( tokens[0] == "d" ): + do_terrasync( url, "/" + tokens[1], os.path.join(_localdir,tokens[1] ) ) + + if( tokens[0] == "f" ): + do_download_file( url, "/" + tokens[1], os.path.join(_localdir,tokens[1]), tokens[2] ) + + except urllib.error.HTTPError as err: + if err.code == 404 and _path == "": + # HACK: only the master on SF provides .dirindex for root, fake it if it's missing + print("Using static root hack.") + if not _url.startswith("http://flightgear.sourceforge.net/scenery") and _path == "": + for _sub in ("Models", "Terrain", "Objects", "Airports" ): + do_terrasync( _url, "/" + _sub, os.path.join(_localdir,_sub) ) + return + + else: + raise + #TODO: cleanup orphan files + +######################################################################## + +import getopt, sys, random, re + +try: + opts, args = getopt.getopt(sys.argv[1:], "u:t:", [ "url=", "target=" ]) + +except getopt.GetoptError: + print("terrasync.py [--url=http://some.server.org/scenery] [--target=/some/path]") + sys.exit(2) + +for opt, arg in opts: + if opt in( "-u", "--url"): + URL = arg + + elif opt in ( "-t", "--target"): + TARGET= arg + +# automatic URL lookup from DNS NAPTR +# - lookup terrasync.flightgear.org, type=NAPTR, service="ws20", flags="U" +# - sort by order,preference ascending +# - pick entries with lowest order and preference +# - randomly pick one of those +# - use regexp fields URL +if URL == "automatic": + import dns.resolver + dnsResolver = dns.resolver.Resolver() + + order = -1 + preference = -1 + + # find lowes preference/order for service 'ws20' and flags 'U' + dnsAnswer = dnsResolver.query("terrasync.flightgear.org", "NAPTR" ) + for naptr in dnsAnswer: + if naptr.service != b'ws20' or naptr.flags != b'U': + continue + + if order == -1 or naptr.order < order: + order = naptr.order + preference = naptr.preference + + if order == naptr.order: + if naptr.preference < preference: + preference = naptr.preference + + + # grab candidats + candidates = [] + for naptr in dnsAnswer: + if naptr.service != b'ws20' or naptr.flags != b'U' or naptr.preference != preference or naptr.order != order: + continue + + candidates.append( naptr.regexp.decode('utf-8') ) + + if not candidates: + print("sorry, no terrascenery URLs found. You may specify one with --url=http://some.url.org/foo") + sys.exit(3) + + _url = random.choice(candidates) + _subst = _url.split(_url[0]) # split string, first character is separator regexreplacement + URL = re.sub(_subst[1], _subst[2], "" ) # apply regex substitude on empty string + +print( "terrasyncing from ", URL, "to ", TARGET ) +do_terrasync( URL, "", TARGET ) + +######################################################################## diff --git a/utils/TerraSync/terrasync.py b/utils/TerraSync/terrasync.py deleted file mode 100755 index a26d67ee6..000000000 --- a/utils/TerraSync/terrasync.py +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright (C) 2016 Torsten Dreyer -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# terrasync.py - synchronize terrascenery data to your local disk -# needs dnspython (pip install dnspython) -# -# this is a terrible stub, please improve - -import os -import hashlib -import urllib.request - -dirindex = ".dirindex" -DIRINDEXVERSION = 1 - -URL="automatic" -TARGET="." - -######################################################################## - -def fn_hash_of_file(fname): - hash = hashlib.sha1() - try: - with open(fname, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hash.update(chunk) - except: - pass - - return hash.hexdigest() - -######################################################################## -def do_download_file( _url, _path, _localfile, _hash ): - if os.path.exists( _localfile ): - h = fn_hash_of_file(_localfile) - if h == _hash: - print("hash match for ", _localfile) - return - - r = urllib.request.urlopen( _url + _path ) - f = open(_localfile, 'wb') - f.write( r.read() ) - f.close() - print("downloaded ", _localfile, " from ", _url + _path ) - -######################################################################## -def do_terrasync( _url, _path, _localdir ): - url = _url + _path - print("syncing ",url, " to ", _localdir ) - - if not os.path.exists( _localdir ): - os.makedirs( _localdir ) - - try: - for line in urllib.request.urlopen(url + "/.dirindex").readlines(): - tokens = line.decode("utf-8").rstrip().split(':') - if( len(tokens) == 0 ): - continue - - # TODO: check version number, should be equal to DIRINDEXVERSION - # otherwise complain and terminate - if( tokens[0] == "version" ): - continue - - if( tokens[0] == "path" ): - continue - - if( tokens[0] == "d" ): - do_terrasync( url, "/" + tokens[1], os.path.join(_localdir,tokens[1] ) ) - - if( tokens[0] == "f" ): - do_download_file( url, "/" + tokens[1], os.path.join(_localdir,tokens[1]), tokens[2] ) - - except urllib.error.HTTPError as err: - if err.code == 404 and _path == "": - # HACK: only the master on SF provides .dirindex for root, fake it if it's missing - print("Using static root hack.") - if not _url.startswith("http://flightgear.sourceforge.net/scenery") and _path == "": - for _sub in ("Models", "Terrain", "Objects", "Airports" ): - do_terrasync( _url, "/" + _sub, os.path.join(_localdir,_sub) ) - return - - else: - raise - #TODO: cleanup orphan files - -######################################################################## - -import getopt, sys, random, re - -try: - opts, args = getopt.getopt(sys.argv[1:], "u:t:", [ "url=", "target=" ]) - -except getopt.GetoptError: - print("terrasync.py [--url=http://some.server.org/scenery] [--target=/some/path]") - sys.exit(2) - -for opt, arg in opts: - if opt in( "-u", "--url"): - URL = arg - - elif opt in ( "-t", "--target"): - TARGET= arg - -# automatic URL lookup from DNS NAPTR -# - lookup terrasync.flightgear.org, type=NAPTR, service="ws20", flags="U" -# - sort by order,preference ascending -# - pick entries with lowest order and preference -# - randomly pick one of those -# - use regexp fields URL -if URL == "automatic": - import dns.resolver - dnsResolver = dns.resolver.Resolver() - - order = -1 - preference = -1 - - # find lowes preference/order for service 'ws20' and flags 'U' - dnsAnswer = dnsResolver.query("terrasync.flightgear.org", "NAPTR" ) - for naptr in dnsAnswer: - if naptr.service != b'ws20' or naptr.flags != b'U': - continue - - if order == -1 or naptr.order < order: - order = naptr.order - preference = naptr.preference - - if order == naptr.order: - if naptr.preference < preference: - preference = naptr.preference - - - # grab candidats - candidates = [] - for naptr in dnsAnswer: - if naptr.service != b'ws20' or naptr.flags != b'U' or naptr.preference != preference or naptr.order != order: - continue - - candidates.append( naptr.regexp.decode('utf-8') ) - - if not candidates: - print("sorry, no terrascenery URLs found. You may specify one with --url=http://some.url.org/foo") - sys.exit(3) - - _url = random.choice(candidates) - _subst = _url.split(_url[0]) # split string, first character is separator regexreplacement - URL = re.sub(_subst[1], _subst[2], "" ) # apply regex substitude on empty string - -print( "terrasyncing from ", URL, "to ", TARGET ) -do_terrasync( URL, "", TARGET ) - -########################################################################