X-Git-Url: https://git.mxchange.org/?p=quix0rs-apt-p2p.git;a=blobdiff_plain;f=test.py;h=ac7892ad7cbae233e00b8a94bdd79ac82b4bf4ff;hp=150b5c2dbf5d4aec085c3880f0c42c5e7f6d7c8e;hb=21f3067f2e5f694c696835f5eceab0eba5c3d479;hpb=19dab12a49b05e17b3fc9e6fc246c31f157b4f48 diff --git a/test.py b/test.py index 150b5c2..ac7892a 100755 --- a/test.py +++ b/test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python """Automated tests of the apt-p2p functionality. @@ -29,7 +29,8 @@ the apt-p2p program. @var CWD: the working directory the script was run from @type apt_conf_template: C{string} @var apt_conf_template: the template to use for the apt.conf file - +@type apt_p2p_conf_template: C{string} +@var apt_p2p_conf_template: the template to use for the apt-p2p.conf file """ from time import sleep, time @@ -47,7 +48,6 @@ tests = {'1': ('Start a single bootstrap and downloader, test updating and downl (1, ['install', 'ada-reference-manual']), (1, ['install', 'aspectj-doc']), (1, ['install', 'fop-doc']), - (1, ['install', 'jswat-doc']), (1, ['install', 'asis-doc']), (1, ['install', 'bison-doc']), (1, ['install', 'crash-whitepaper']), @@ -64,13 +64,11 @@ tests = {'1': ('Start a single bootstrap and downloader, test updating and downl (1, ['install', 'aap-doc']), (1, ['install', 'ada-reference-manual']), (1, ['install', 'fop-doc']), - (1, ['install', 'jswat-doc']), (1, ['install', 'bison-doc']), (1, ['install', 'crash-whitepaper']), (2, ['install', 'aap-doc']), (2, ['install', 'ada-reference-manual']), (2, ['install', 'fop-doc']), - (2, ['install', 'jswat-doc']), (2, ['install', 'bison-doc']), (2, ['install', 'crash-whitepaper']), ]), @@ -264,6 +262,105 @@ tests = {'1': ('Start a single bootstrap and downloader, test updating and downl (6, ['install', 'kde-icons-oxygen']), ]), + 'a': ('Test pipelining and caching, can also interrupt or restart to test resuming.', + {1: {'clean': False}}, + {1: {'clean': False}, + 2: {'clean': False}}, + [(1, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + (1, ['update']), + (1, ['update']), + (1, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + (1, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + (2, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + (2, ['update']), + (2, ['update']), + (2, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + (2, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + ]), + + 'b': ('Start 2 downloaders and test source downloads.', + {1: {}}, + {1: {'types': ['deb-src']}, + 2: {'types': ['deb-src']}}, + [(1, ['update']), + (2, ['update']), + (1, ['source', 'aboot-base']), + (2, ['source', 'aboot-base']), + (1, ['source', 'aap-doc']), + (1, ['source', 'ada-reference-manual']), + (1, ['source', 'fop-doc']), + (1, ['source', 'bison-doc']), + (1, ['source', 'crash-whitepaper']), + (2, ['source', 'aap-doc']), + (2, ['source', 'ada-reference-manual']), + (2, ['source', 'fop-doc']), + (2, ['source', 'bison-doc']), + (2, ['source', 'crash-whitepaper']), + ]), + + 'c': ('Test downloading from peers and just a mirror.', + {1: {}}, + {1: {}, + 2: {}}, + [(1, ['update']), + (1, ['install', 'aboot-base', 'ada-reference-manual', + 'fop-doc', 'bison-doc', 'crash-whitepaper', + 'apt-howto-common', 'aptitude-doc-en', 'asr-manpages', + 'alcovebook-sgml-doc', 'airstrike-common', + ]), + (2, ['update']), + (2, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + ]), + } assert 'all' not in tests @@ -370,7 +467,7 @@ UNLOAD_PACKAGES_CACHE = 5m # Refresh the DHT keys after this much time has passed. # This should be a time slightly less than the DHT's KEY_EXPIRE value. -KEY_REFRESH = 57m +KEY_REFRESH = 2.5h # The user name to try and run as (leave blank to run as current user) USERNAME = @@ -394,10 +491,10 @@ BOOTSTRAP_NODE = %(BOOTSTRAP_NODE)s CHECKPOINT_INTERVAL = 5m # concurrent xmlrpc calls per find node/value request! -CONCURRENT_REQS = 4 +CONCURRENT_REQS = 8 # how many hosts to post to -STORE_REDUNDANCY = 3 +STORE_REDUNDANCY = 6 # How many values to attempt to retrieve from the DHT. # Setting this to 0 will try and get all values (which could take a while if @@ -416,8 +513,24 @@ MIN_PING_INTERVAL = 15m # refresh buckets that haven't been touched in this long BUCKET_STALENESS = 1h +# Whether it's OK to add nodes to the routing table that use an IP +# address from a known local/private range. +# If not specified here, the LOCAL_OK value in the DEFAULT section will be used. +LOCAL_OK = yes + # expire entries older than this -KEY_EXPIRE = 1h +KEY_EXPIRE = 3h + +# Timeout KRPC requests to nodes after this time. +KRPC_TIMEOUT = 9s + +# KRPC requests are resent using exponential backoff starting with this delay. +# The request will first be resent after the delay set here. +# The request will be resent again after twice the delay set here. etc. +# e.g. if TIMEOUT is 9 sec., and INITIAL_DELAY is 2 sec., then requests will +# be resent at times 0, 2 (2 sec. later), and 6 (4 sec. later), and then will +# timeout at 9. +KRPC_INITIAL_DELAY = 2s # whether to spew info about the requests/responses in the protocol SPEW = yes @@ -489,6 +602,8 @@ def start(cmd, args, work_dir = None): """ new_cmd = [cmd] + args + if work_dir: + os.chdir(work_dir) pid = os.spawnvp(os.P_NOWAIT, new_cmd[0], new_cmd) return pid @@ -550,11 +665,15 @@ def apt_get(num_down, cmd): """ + downloader_dir = down_dir(num_down) + rmrf(join([downloader_dir, 'var', 'cache', 'apt', 'archives'])) + makedirs([downloader_dir, 'var', 'cache', 'apt', 'archives', 'partial']) + print '*************** apt-get (' + str(num_down) + ') ' + ' '.join(cmd) + ' ****************' apt_conf = join([down_dir(num_down), 'etc', 'apt', 'apt.conf']) dpkg_status = join([down_dir(num_down), 'var', 'lib', 'dpkg', 'status']) args = ['-d', '-c', apt_conf, '-o', 'Dir::state::status='+dpkg_status] + cmd - pid = start('apt-get', args) + pid = start('apt-get', args, downloader_dir) return pid def bootstrap_address(num_boot): @@ -594,7 +713,7 @@ def boot_dir(num_boot): return os.path.join(CWD,'bootstrap' + str(num_boot)) def start_downloader(bootstrap_addresses, num_down, options = {}, - mirror = 'ftp.us.debian.org/debian', + types = ['deb'], mirror = 'ftp.us.debian.org/debian', suites = 'main contrib non-free', clean = True): """Initialize a new downloader process. @@ -609,6 +728,9 @@ def start_downloader(bootstrap_addresses, num_down, options = {}, @param options: the dictionary of string formatting values for creating the apt-p2p configuration file (see L{apt_p2p_conf_template} above). (optional, defaults to only using the default arguments) + @type types: C{list} of C{string} + @param types: the type of sources.list line to add + (optional, defaults to only 'deb') @type mirror: C{string} @param mirror: the Debian mirror to use (optional, defaults to 'ftp.us.debian.org/debian') @@ -635,21 +757,25 @@ def start_downloader(bootstrap_addresses, num_down, options = {}, except: pass - # Create the directory structure needed by apt - makedirs([downloader_dir, 'etc', 'apt', 'apt.conf.d']) - makedirs([downloader_dir, 'var', 'lib', 'apt', 'lists', 'partial']) - makedirs([downloader_dir, 'var', 'lib', 'dpkg']) - makedirs([downloader_dir, 'var', 'cache', 'apt', 'archives', 'partial']) - touch([downloader_dir, 'var', 'lib', 'apt', 'lists', 'lock']) - touch([downloader_dir, 'var', 'lib', 'dpkg', 'lock']) - touch([downloader_dir, 'var', 'lib', 'dpkg', 'status']) - touch([downloader_dir, 'var', 'cache', 'apt', 'archives', 'lock']) - + # Create the directory structure needed by apt + makedirs([downloader_dir, 'etc', 'apt', 'apt.conf.d']) + makedirs([downloader_dir, 'var', 'lib', 'apt', 'lists', 'partial']) + makedirs([downloader_dir, 'var', 'lib', 'dpkg']) + rmrf(join([downloader_dir, 'var', 'cache', 'apt', 'archives'])) + makedirs([downloader_dir, 'var', 'cache', 'apt', 'archives', 'partial']) + touch([downloader_dir, 'var', 'lib', 'apt', 'lists', 'lock']) + touch([downloader_dir, 'var', 'lib', 'dpkg', 'lock']) + touch([downloader_dir, 'var', 'lib', 'dpkg', 'status']) + touch([downloader_dir, 'var', 'cache', 'apt', 'archives', 'lock']) + + if not exists(join([downloader_dir, 'etc', 'apt', 'sources.list'])): # Create apt's config files f = open(join([downloader_dir, 'etc', 'apt', 'sources.list']), 'w') - f.write('deb http://localhost:1%02d77/%s/ unstable %s\n' % (num_down, mirror, suites)) + for type in types: + f.write('%s http://localhost:1%02d77/%s/ unstable %s\n' % (type, num_down, mirror, suites)) f.close() + if not exists(join([downloader_dir, 'etc', 'apt', 'apt.conf'])): f = open(join([downloader_dir, 'etc', 'apt', 'apt.conf']), 'w') f.write('Dir "' + downloader_dir + '"') f.write(apt_conf_template)