X-Git-Url: https://git.mxchange.org/?p=quix0rs-apt-p2p.git;a=blobdiff_plain;f=test.py;h=b94e01249d2356664fed8168a8e3c6e1ea1493d4;hp=044aeff3ed0080cc3dcfb4eb7b68050014fb59c5;hb=05422476cb06c6ccd2def7709a251e618e1eafb3;hpb=1abfc43747d252f5d0bf11f119e0daabecc4f3a6 diff --git a/test.py b/test.py old mode 100644 new mode 100755 index 044aeff..b94e012 --- a/test.py +++ b/test.py @@ -1,10 +1,984 @@ -import unittest +#!/usr/bin/env python -import hash, node, messages -import listener, dispatcher -import ktable, transactions, khashmir +"""Automated tests of the apt-p2p functionality. -import bencode, btemplate +This script runs several automatic tests of some of the functionality in +the apt-p2p program. -tests = unittest.defaultTestLoader.loadTestsFromNames(['hash', 'node', 'bencode', 'btemplate', 'listener', 'messages', 'dispatcher', 'transactions', 'ktable']) -result = unittest.TextTestRunner().run(tests) +@type tests: C{dictionary} +@var tests: all of the tests that can be run. + The keys are the test names (strings) which are used on the command-line + to identify the tests (can not be 'all' or 'help'). The values are tuples + with four elements: a description of the test (C{string}), the bootstrap + nodes to start (C{dictionary}), the downloaders to start (C{dictionary}, + and the apt-get commands to run (C{list}). + + The bootstrap nodes keys are integers, which must be in the range 1-9. + The values are the dictionary of keyword options to pass to the function + that starts the bootstrap node (see L{start_bootstrap} below). + + The downloaders keys are also integers in the range 1-99. The values are + the dictionary of keyword options to pass to the function + that starts the downloader node (see L{start_downloader} below). + + The apt-get commands' list elements are tuples with 2 elements: the + downloader to run the command on, and the list of command-line + arguments to specify to the apt-get program. + +@type CWD: C{string} +@var CWD: the working directory the script was run from +@type apt_conf_template: C{string} +@var apt_conf_template: the template to use for the apt.conf file + +""" + +from time import sleep, time +import sys, os, signal +from traceback import print_exc +from os.path import exists + +tests = {'1': ('Start a single bootstrap and downloader, test updating and downloading ' + + 'using HTTP only.', + {1: {}}, + {1: {}}, + [(1, ['update']), + (1, ['install', 'aboot-base']), + (1, ['install', 'aap-doc']), + (1, ['install', 'ada-reference-manual']), + (1, ['install', 'aspectj-doc']), + (1, ['install', 'fop-doc']), + (1, ['install', 'asis-doc']), + (1, ['install', 'bison-doc']), + (1, ['install', 'crash-whitepaper']), + ]), + + '2': ('Start a single bootstrap and 2 downloaders to test downloading from a peer.', + {1: {}}, + {1: {}, + 2: {}}, + [(1, ['update']), + (2, ['update']), + (1, ['install', 'aboot-base']), + (2, ['install', 'aboot-base']), + (1, ['install', 'aap-doc']), + (1, ['install', 'ada-reference-manual']), + (1, ['install', 'fop-doc']), + (1, ['install', 'bison-doc']), + (1, ['install', 'crash-whitepaper']), + (2, ['install', 'aap-doc']), + (2, ['install', 'ada-reference-manual']), + (2, ['install', 'fop-doc']), + (2, ['install', 'bison-doc']), + (2, ['install', 'crash-whitepaper']), + ]), + + '3': ('Start a single bootstrap and 6 downloaders, to test downloading' + + ' speeds from each other.', + {1: {}}, + {1: {}, + 2: {}, + 3: {}, + 4: {}, + 5: {}, + 6: {}}, + [(1, ['update']), + (1, ['install', 'aboot-base']), + (1, ['install', 'ada-reference-manual']), + (1, ['install', 'fop-doc']), + (1, ['install', 'crash-whitepaper']), + (2, ['update']), + (2, ['install', 'aboot-base']), + (2, ['install', 'ada-reference-manual']), + (2, ['install', 'fop-doc']), + (2, ['install', 'crash-whitepaper']), + (3, ['update']), + (3, ['install', 'aboot-base']), + (3, ['install', 'ada-reference-manual']), + (3, ['install', 'fop-doc']), + (3, ['install', 'crash-whitepaper']), + (4, ['update']), + (4, ['install', 'aboot-base']), + (4, ['install', 'ada-reference-manual']), + (4, ['install', 'fop-doc']), + (4, ['install', 'crash-whitepaper']), + (5, ['update']), + (5, ['install', 'aboot-base']), + (5, ['install', 'ada-reference-manual']), + (5, ['install', 'fop-doc']), + (5, ['install', 'crash-whitepaper']), + (6, ['update']), + (6, ['install', 'aboot-base']), + (6, ['install', 'ada-reference-manual']), + (6, ['install', 'fop-doc']), + (6, ['install', 'crash-whitepaper']), + ]), + + '4': ('Start a single bootstrap and 1 downloader, requesting the same' + + ' packages multiple times to test caching.', + {1: {}}, + {1: {}}, + [(1, ['update']), + (1, ['install', 'aboot-base']), + (1, ['install', 'ada-reference-manual']), + (1, ['install', 'fop-doc']), + (1, ['install', 'crash-whitepaper']), + (1, ['update']), + (1, ['install', 'aboot-base']), + (1, ['install', 'ada-reference-manual']), + (1, ['install', 'fop-doc']), + (1, ['install', 'crash-whitepaper']), + (1, ['update']), + (1, ['install', 'aboot-base']), + (1, ['install', 'ada-reference-manual']), + (1, ['install', 'fop-doc']), + (1, ['install', 'crash-whitepaper']), + ]), + + '5': ('Start a single bootstrap and 6 downloaders, update all to test' + + ' that they can all see each other.', + {1: {}}, + {1: ([], {'suites': 'contrib non-free'}), + 2: ([], {'suites': 'contrib non-free'}), + 3: ([], {'suites': 'contrib non-free'}), + 4: ([], {'suites': 'contrib non-free'}), + 5: ([], {'suites': 'contrib non-free'}), + 6: ([], {'suites': 'contrib non-free'})}, + [(1, ['update']), + (2, ['update']), + (3, ['update']), + (4, ['update']), + (5, ['update']), + (6, ['update']), + ]), + + '6': ('Test caching with multiple apt-get updates.', + {1: {}}, + {1: {}}, + [(1, ['update']), + (1, ['update']), + (1, ['update']), + (1, ['update']), + ]), + + '7': ('Test pipelining of multiple simultaneous downloads.', + {1: {}}, + {1: {}}, + [(1, ['update']), + (1, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + ]), + + '8': ('Test pipelining of multiple simultaneous downloads with many peers.', + {1: {}}, + {1: {}, + 2: {}, + 3: {}, + 4: {}, + 5: {}, + 6: {}}, + [(1, ['update']), + (1, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + (2, ['update']), + (2, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + (3, ['update']), + (3, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + (4, ['update']), + (4, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + (5, ['update']), + (5, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + (6, ['update']), + (6, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + ]), + + '9': ('Start a single bootstrap and 6 downloaders and test downloading' + + ' a very large file.', + {1: {}}, + {1: {}, + 2: {}, + 3: {}, + 4: {}, + 5: {}, + 6: {}}, + [(1, ['update']), + (1, ['install', 'kde-icons-oxygen']), + (2, ['update']), + (2, ['install', 'kde-icons-oxygen']), + (3, ['update']), + (3, ['install', 'kde-icons-oxygen']), + (4, ['update']), + (4, ['install', 'kde-icons-oxygen']), + (5, ['update']), + (5, ['install', 'kde-icons-oxygen']), + (6, ['update']), + (6, ['install', 'kde-icons-oxygen']), + ]), + + 'a': ('Test pipelining and caching, can also interrupt or restart to test resuming.', + {1: {'clean': False}}, + {1: {'clean': False}, + 2: {'clean': False}}, + [(1, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + (1, ['update']), + (1, ['update']), + (1, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + (1, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + (2, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + (2, ['update']), + (2, ['update']), + (2, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + (2, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + ]), + + 'b': ('Start 2 downloaders and test source downloads.', + {1: {}}, + {1: {'types': ['deb-src']}, + 2: {'types': ['deb-src']}}, + [(1, ['update']), + (2, ['update']), + (1, ['source', 'aboot-base']), + (2, ['source', 'aboot-base']), + (1, ['source', 'aap-doc']), + (1, ['source', 'ada-reference-manual']), + (1, ['source', 'fop-doc']), + (1, ['source', 'bison-doc']), + (1, ['source', 'crash-whitepaper']), + (2, ['source', 'aap-doc']), + (2, ['source', 'ada-reference-manual']), + (2, ['source', 'fop-doc']), + (2, ['source', 'bison-doc']), + (2, ['source', 'crash-whitepaper']), + ]), + + 'c': ('Test downloading from peers and just a mirror.', + {1: {}}, + {1: {}, + 2: {}}, + [(1, ['update']), + (1, ['install', 'aboot-base', 'ada-reference-manual', + 'fop-doc', 'bison-doc', 'crash-whitepaper', + 'apt-howto-common', 'aptitude-doc-en', 'asr-manpages', + 'alcovebook-sgml-doc', 'airstrike-common', + ]), + (2, ['update']), + (2, ['install', 'aboot-base', 'aap-doc', 'ada-reference-manual', + 'aspectj-doc', 'fop-doc', 'asis-doc', + 'bison-doc', 'crash-whitepaper', + 'bash-doc', 'apt-howto-common', 'autotools-dev', + 'aptitude-doc-en', 'asr-manpages', + 'atomix-data', 'alcovebook-sgml-doc', + 'afbackup-common', 'airstrike-common', + ]), + ]), + + } + +assert 'all' not in tests +assert 'help' not in tests + +CWD = os.getcwd() +apt_conf_template = """ +{ + // Location of the state dir + State "var/lib/apt/" + { + Lists "lists/"; + xstatus "xstatus"; + userstatus "status.user"; + cdroms "cdroms.list"; + }; + + // Location of the cache dir + Cache "var/cache/apt/" { + Archives "archives/"; + srcpkgcache "srcpkgcache.bin"; + pkgcache "pkgcache.bin"; + }; + + // Config files + Etc "etc/apt/" { + SourceList "sources.list"; + Main "apt.conf"; + Preferences "preferences"; + Parts "apt.conf.d/"; + }; + + // Locations of binaries + Bin { + methods "/usr/lib/apt/methods/"; + gzip "/bin/gzip"; + gpg "/usr/bin/gpgv"; + dpkg "/usr/bin/dpkg --simulate"; + dpkg-source "/usr/bin/dpkg-source"; + dpkg-buildpackage "/usr/bin/dpkg-buildpackage"; + apt-get "/usr/bin/apt-get"; + apt-cache "/usr/bin/apt-cache"; + }; +}; + +/* Options you can set to see some debugging text They correspond to names + of classes in the source code */ +Debug +{ + pkgProblemResolver "false"; + pkgDepCache::AutoInstall "false"; // what packages apt install to satify dependencies + pkgAcquire "false"; + pkgAcquire::Worker "false"; + pkgAcquire::Auth "false"; + pkgDPkgPM "false"; + pkgDPkgProgressReporting "false"; + pkgOrderList "false"; + BuildDeps "false"; + + pkgInitialize "false"; // This one will dump the configuration space + NoLocking "false"; + Acquire::Ftp "false"; // Show ftp command traffic + Acquire::Http "false"; // Show http command traffic + Acquire::gpgv "false"; // Show the gpgv traffic + aptcdrom "false"; // Show found package files + IdentCdrom "false"; + +} +""" +apt_p2p_conf_template = """ +[DEFAULT] + +# Port to listen on for all requests (TCP and UDP) +PORT = %(PORT)s + +# The rate to limit sending data to peers to, in KBytes/sec. +# Set this to 0 to not limit the upload bandwidth. +UPLOAD_LIMIT = 100 + +# The minimum number of peers before the mirror is not used. +# If there are fewer peers than this for a file, the mirror will also be +# used to speed up the download. Set to 0 to never use the mirror if +# there are peers. +MIN_DOWNLOAD_PEERS = 3 + +# Directory to store the downloaded files in +CACHE_DIR = %(CACHE_DIR)s + +# Other directories containing packages to share with others +# WARNING: all files in these directories will be hashed and available +# for everybody to download +# OTHER_DIRS = + +# Whether it's OK to use an IP addres from a known local/private range +LOCAL_OK = yes + +# Whether a remote peer can access the statistics page +REMOTE_STATS = yes + +# Unload the packages cache after an interval of inactivity this long. +# The packages cache uses a lot of memory, and only takes a few seconds +# to reload when a new request arrives. +UNLOAD_PACKAGES_CACHE = 5m + +# Refresh the DHT keys after this much time has passed. +# This should be a time slightly less than the DHT's KEY_EXPIRE value. +KEY_REFRESH = 57m + +# The user name to try and run as (leave blank to run as current user) +USERNAME = + +# Which DHT implementation to use. +# It must be possile to do "from .DHT import DHT" to get a class that +# implements the IDHT interface. +DHT = apt_p2p_Khashmir + +# Whether to only run the DHT (for providing only a bootstrap node) +DHT-ONLY = %(DHT-ONLY)s + +[apt_p2p_Khashmir] +# bootstrap nodes to contact to join the DHT +BOOTSTRAP = %(BOOTSTRAP)s + +# whether this node is a bootstrap node +BOOTSTRAP_NODE = %(BOOTSTRAP_NODE)s + +# checkpoint every this many seconds +CHECKPOINT_INTERVAL = 5m + +# concurrent xmlrpc calls per find node/value request! +CONCURRENT_REQS = 4 + +# how many hosts to post to +STORE_REDUNDANCY = 3 + +# How many values to attempt to retrieve from the DHT. +# Setting this to 0 will try and get all values (which could take a while if +# a lot of nodes have values). Setting it negative will try to get that +# number of results from only the closest STORE_REDUNDANCY nodes to the hash. +# The default is a large negative number so all values from the closest +# STORE_REDUNDANCY nodes will be retrieved. +RETRIEVE_VALUES = -10000 + +# how many times in a row a node can fail to respond before it's booted from the routing table +MAX_FAILURES = 3 + +# never ping a node more often than this +MIN_PING_INTERVAL = 15m + +# refresh buckets that haven't been touched in this long +BUCKET_STALENESS = 1h + +# expire entries older than this +KEY_EXPIRE = 1h + +# Timeout KRPC requests to nodes after this time. +KRPC_TIMEOUT = 14s + +# KRPC requests are resent using exponential backoff starting with this delay. +# The request will first be resent after the delay set here. +# The request will be resent again after twice the delay set here. etc. +# e.g. if TIMEOUT is 14 sec., and INITIAL_DELAY is 2 sec., then requests will +# be resent at times 0, 2 (2 sec. later), and 6 (4 sec. later), and then will +# timeout at 14. +KRPC_INITIAL_DELAY = 2s + +# whether to spew info about the requests/responses in the protocol +SPEW = yes +""" + +def rmrf(top): + """Remove all the files and directories below a top-level one. + + @type top: C{string} + @param top: the top-level directory to start at + + """ + + for root, dirs, files in os.walk(top, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) + +def join(dir): + """Join together a list of directories into a path string. + + @type dir: C{list} of C{string} + @param dir: the path to join together + @rtype: C{string} + @return: the joined together path + + """ + + joined = '' + for i in dir: + joined = os.path.join(joined, i) + return joined + +def makedirs(dir): + """Create all the directories to make a path. + + @type dir: C{list} of C{string} + @param dir: the path to create + + """ + if not os.path.exists(join(dir)): + os.makedirs(join(dir)) + +def touch(path): + """Create an empty file. + + @type path: C{list} of C{string} + @param path: the path to create + + """ + + f = open(join(path), 'w') + f.close() + +def start(cmd, args, work_dir = None): + """Fork and start a background process running. + + @type cmd: C{string} + @param cmd: the name of the command to run + @type args: C{list} of C{string} + @param args: the argument to pass to the command + @type work_dir: C{string} + @param work_dir: the directory to change to to execute the child process in + (optional, defaults to the current directory) + @rtype: C{int} + @return: the PID of the forked process + + """ + + new_cmd = [cmd] + args + if work_dir: + os.chdir(work_dir) + pid = os.spawnvp(os.P_NOWAIT, new_cmd[0], new_cmd) + return pid + +def stop(pid): + """Stop a forked background process that is running. + + @type pid: C{int} + @param pid: the PID of the process to stop + @rtype: C{int} + @return: the return status code from the child + + """ + + # First try a keyboard interrupt + os.kill(pid, signal.SIGINT) + for i in xrange(5): + sleep(1) + (r_pid, r_value) = os.waitpid(pid, os.WNOHANG) + if r_pid: + return r_value + + # Try a keyboard interrupt again, just in case + os.kill(pid, signal.SIGINT) + for i in xrange(5): + sleep(1) + (r_pid, r_value) = os.waitpid(pid, os.WNOHANG) + if r_pid: + return r_value + + # Try a terminate + os.kill(pid, signal.SIGTERM) + for i in xrange(5): + sleep(1) + (r_pid, r_value) = os.waitpid(pid, os.WNOHANG) + if r_pid: + return r_value + + # Finally a kill, don't return until killed + os.kill(pid, signal.SIGKILL) + while not r_pid: + sleep(1) + (r_pid, r_value) = os.waitpid(pid, os.WNOHANG) + + return r_value + +def apt_get(num_down, cmd): + """Start an apt-get process in the background. + + The default argument specified to the apt-get invocation are + 'apt-get -d -q -c '. Any additional arguments (including + the apt-get action to use) should be specified. + + @type num_down: C{int} + @param num_down: the number of the downloader to use + @type cmd: C{list} of C{string} + @param cmd: the arguments to pass to the apt-get process + @rtype: C{int} + @return: the PID of the background process + + """ + + downloader_dir = down_dir(num_down) + rmrf(join([downloader_dir, 'var', 'cache', 'apt', 'archives'])) + makedirs([downloader_dir, 'var', 'cache', 'apt', 'archives', 'partial']) + + print '*************** apt-get (' + str(num_down) + ') ' + ' '.join(cmd) + ' ****************' + apt_conf = join([down_dir(num_down), 'etc', 'apt', 'apt.conf']) + dpkg_status = join([down_dir(num_down), 'var', 'lib', 'dpkg', 'status']) + args = ['-d', '-c', apt_conf, '-o', 'Dir::state::status='+dpkg_status] + cmd + pid = start('apt-get', args, downloader_dir) + return pid + +def bootstrap_address(num_boot): + """Determine the bootstrap address to use for a node. + + @type num_boot: C{int} + @param num_boot: the number of the bootstrap node + @rtype: C{string} + @return: the bootstrap address to use + + """ + + return 'localhost:1' + str(num_boot) + '969' + +def down_dir(num_down): + """Determine the working directory to use for a downloader. + + @type num_down: C{int} + @param num_down: the number of the downloader + @rtype: C{string} + @return: the downloader's directory + + """ + + return os.path.join(CWD,'downloader' + str(num_down)) + +def boot_dir(num_boot): + """Determine the working directory to use for a bootstrap node. + + @type num_boot: C{int} + @param num_boot: the number of the bootstrap node + @rtype: C{string} + @return: the bootstrap node's directory + + """ + + return os.path.join(CWD,'bootstrap' + str(num_boot)) + +def start_downloader(bootstrap_addresses, num_down, options = {}, + types = ['deb'], mirror = 'ftp.us.debian.org/debian', + suites = 'main contrib non-free', clean = True): + """Initialize a new downloader process. + + The default arguments specified to the downloader invocation are + the configuration directory, apt port, minport, maxport and the + maximum upload rate. + Any additional arguments needed should be specified by L{options}. + + @type num_down: C{int} + @param num_down: the number of the downloader to use + @type options: C{dictionary} + @param options: the dictionary of string formatting values for creating + the apt-p2p configuration file (see L{apt_p2p_conf_template} above). + (optional, defaults to only using the default arguments) + @type types: C{list} of C{string} + @param types: the type of sources.list line to add + (optional, defaults to only 'deb') + @type mirror: C{string} + @param mirror: the Debian mirror to use + (optional, defaults to 'ftp.us.debian.org/debian') + @type suites: C{string} + @param suites: space separated list of suites to download + (optional, defaults to 'main contrib non-free') + @type clean: C{boolean} + @param clean: whether to remove any previous downloader files + (optional, defaults to removing them) + @rtype: C{int} + @return: the PID of the downloader process + + """ + + assert num_down < 100 + + print '************************** Starting Downloader ' + str(num_down) + ' **************************' + + downloader_dir = down_dir(num_down) + + if clean: + try: + rmrf(downloader_dir) + except: + pass + + # Create the directory structure needed by apt + makedirs([downloader_dir, 'etc', 'apt', 'apt.conf.d']) + makedirs([downloader_dir, 'var', 'lib', 'apt', 'lists', 'partial']) + makedirs([downloader_dir, 'var', 'lib', 'dpkg']) + rmrf(join([downloader_dir, 'var', 'cache', 'apt', 'archives'])) + makedirs([downloader_dir, 'var', 'cache', 'apt', 'archives', 'partial']) + touch([downloader_dir, 'var', 'lib', 'apt', 'lists', 'lock']) + touch([downloader_dir, 'var', 'lib', 'dpkg', 'lock']) + touch([downloader_dir, 'var', 'lib', 'dpkg', 'status']) + touch([downloader_dir, 'var', 'cache', 'apt', 'archives', 'lock']) + + if not exists(join([downloader_dir, 'etc', 'apt', 'sources.list'])): + # Create apt's config files + f = open(join([downloader_dir, 'etc', 'apt', 'sources.list']), 'w') + for type in types: + f.write('%s http://localhost:1%02d77/%s/ unstable %s\n' % (type, num_down, mirror, suites)) + f.close() + + if not exists(join([downloader_dir, 'etc', 'apt', 'apt.conf'])): + f = open(join([downloader_dir, 'etc', 'apt', 'apt.conf']), 'w') + f.write('Dir "' + downloader_dir + '"') + f.write(apt_conf_template) + f.close() + + defaults = {'PORT': '1%02d77' % num_down, + 'CACHE_DIR': downloader_dir, + 'DHT-ONLY': 'no', + 'BOOTSTRAP': bootstrap_addresses, + 'BOOTSTRAP_NODE': 'no'} + + for k in options: + defaults[k] = options[k] + f = open(join([downloader_dir, 'apt-p2p.conf']), 'w') + f.write(apt_p2p_conf_template % defaults) + f.close() + + pid = start('python', [join([sys.path[0], 'apt-p2p.py']), + '--config-file=' + join([downloader_dir, 'apt-p2p.conf']), + '--log-file=' + join([downloader_dir, 'apt-p2p.log']),], + downloader_dir) + return pid + +def start_bootstrap(bootstrap_addresses, num_boot, options = [], clean = True): + """Initialize a new bootstrap node process. + + The default arguments specified to the apt-p2p invocation are + the state file and port to use. Any additional arguments needed + should be specified by L{options}. + + @type num_boot: C{int} + @param num_boot: the number of the bootstrap node to use + @type options: C{list} of C{string} + @param options: the arguments to pass to the bootstrap node + (optional, defaults to only using the default arguments) + @type clean: C{boolean} + @param clean: whether to remove any previous bootstrap node files + (optional, defaults to removing them) + @rtype: C{int} + @return: the PID of the downloader process + + """ + + assert num_boot < 10 + + print '************************** Starting Bootstrap ' + str(num_boot) + ' **************************' + + bootstrap_dir = boot_dir(num_boot) + + if clean: + try: + rmrf(bootstrap_dir) + except: + pass + + makedirs([bootstrap_dir]) + + defaults = {'PORT': '1%d969' % num_boot, + 'CACHE_DIR': bootstrap_dir, + 'DHT-ONLY': 'yes', + 'BOOTSTRAP': bootstrap_addresses, + 'BOOTSTRAP_NODE': 'yes'} + + for k in options: + defaults[k] = options[k] + f = open(join([bootstrap_dir, 'apt-p2p.conf']), 'w') + f.write(apt_p2p_conf_template % defaults) + f.close() + + pid = start('python', [join([sys.path[0], 'apt-p2p.py']), + '--config-file=' + join([bootstrap_dir, 'apt-p2p.conf']), + '--log-file=' + join([bootstrap_dir, 'apt-p2p.log']),], + bootstrap_dir) + + return pid + +def run_test(bootstraps, downloaders, apt_get_queue): + """Run a single test. + + @type bootstraps: C{dictionary} of {C{int}: C{list} of C{string}} + @param bootstraps: the bootstrap nodes to start, keys are the bootstrap numbers and + values are the list of options to invoke the bootstrap node with + @type downloaders: C{dictionary} of {C{int}: (C{int}, C{list} of C{string})} + @param downloaders: the downloaders to start, keys are the downloader numbers and + values are the list of options to invoke the downloader with + @type apt_get_queue: C{list} of (C{int}, C{list} of C{string}) + @param apt_get_queue: the apt-get downloader to use and commands to execute + @rtype: C{list} of (C{float}, C{int}) + @return: the execution time and returned status code for each element of apt_get_queue + + """ + + running_bootstraps = {} + running_downloaders = {} + running_apt_get = {} + apt_get_results = [] + + try: + boot_keys = bootstraps.keys() + boot_keys.sort() + bootstrap_addresses = bootstrap_address(boot_keys[0]) + for i in xrange(1, len(boot_keys)): + bootstrap_addresses += '\n ' + bootstrap_address(boot_keys[i]) + + for k, v in bootstraps.items(): + running_bootstraps[k] = start_bootstrap(bootstrap_addresses, k, **v) + + sleep(5) + + for k, v in downloaders.items(): + running_downloaders[k] = start_downloader(bootstrap_addresses, k, **v) + + sleep(5) + + for (num_down, cmd) in apt_get_queue: + running_apt_get[num_down] = apt_get(num_down, cmd) + start_time = time() + (pid, r_value) = os.waitpid(running_apt_get[num_down], 0) + elapsed = time() - start_time + del running_apt_get[num_down] + r_value = r_value / 256 + apt_get_results.append((elapsed, r_value)) + + if r_value == 0: + print '********** apt-get completed successfully in ' + str(elapsed) + ' sec. *****************' + else: + print '********** apt-get finished with status ' + str(r_value) + ' in ' + str(elapsed) + ' sec. ************' + + sleep(5) + + except: + print '************************** Exception occurred **************************' + print_exc() + print '************************** will attempt to shut down *******************' + + print '*********************** shutting down the apt-gets *******************' + for k, v in running_apt_get.items(): + try: + print 'apt-get', k, stop(v) + except: + print '************************** Exception occurred **************************' + print_exc() + + sleep(5) + + print '*********************** shutting down the downloaders *******************' + for k, v in running_downloaders.items(): + try: + print 'downloader', k, stop(v) + except: + print '************************** Exception occurred **************************' + print_exc() + + sleep(5) + + print '************************** shutting down the bootstraps *******************' + for k, v in running_bootstraps.items(): + try: + print 'bootstrap', k, stop(v) + except: + print '************************** Exception occurred **************************' + print_exc() + + print '************************** Test Results *******************' + i = -1 + for (num_down, cmd) in apt_get_queue: + i += 1 + s = str(num_down) + ': "apt-get ' + ' '.join(cmd) + '" ' + if len(apt_get_results) > i: + (elapsed, r_value) = apt_get_results[i] + s += 'took ' + str(elapsed) + ' secs (' + str(r_value) + ')' + else: + s += 'did not complete' + print s + + return apt_get_results + +def get_usage(): + """Get the usage information to display to the user. + + @rtype: C{string} + @return: the usage information to display + + """ + + s = 'Usage: ' + sys.argv[0] + ' (all||help)\n\n' + s += ' all - run all the tests\n' + s += ' help - display this usage information\n' + s += ' - run the test (see list below for valid tests)\n\n' + + t = tests.items() + t.sort() + for k, v in t: + s += 'test "' + str(k) + '" - ' + v[0] + '\n' + + return s + +if __name__ == '__main__': + if len(sys.argv) != 2: + print get_usage() + elif sys.argv[1] == 'all': + for k, v in tests.items(): + run_test(v[1], v[2], v[3]) + elif sys.argv[1] in tests: + v = tests[sys.argv[1]] + run_test(v[1], v[2], v[3]) + elif sys.argv[1] == 'help': + print get_usage() + else: + print 'Unknown test to run:', sys.argv[1], '\n' + print get_usage() + \ No newline at end of file