2 # Copyright (C) 2002 Manuel Estrada Sainz <ranty@debian.org>
4 # This library is free software; you can redistribute it and/or
5 # modify it under the terms of version 2.1 of the GNU Lesser General Public
6 # License as published by the Free Software Foundation.
8 # This library is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 # Lesser General Public License for more details.
13 # You should have received a copy of the GNU Lesser General Public
14 # License along with this library; if not, write to the Free Software
15 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 import apt_pkg, apt_inst, sys, os, stat
18 from os.path import dirname, basename
19 import re, shelve, shutil, fcntl
20 from twisted.internet import process
21 from twisted.python import log
23 from twisted.trial import unittest
28 class AptDpkgInfo(UserDict.UserDict):
30 Gets control fields from a .deb file.
32 And then behaves like a regular python dictionary.
34 See AptPackages.get_mirror_path
37 def __init__(self, filename):
38 UserDict.UserDict.__init__(self)
40 filehandle = open(filename);
42 self.control = apt_inst.debExtractControl(filehandle)
44 # Make sure that file is always closed.
47 log.msg("Had problems reading: %s"%(filename))
49 for line in self.control.split('\n'):
50 if line.find(': ') != -1:
51 key, value = line.split(': ', 1)
52 self.data[key] = value
54 class PackageFileList(UserDict.DictMixin):
56 Manages a list of package files belonging to a backend
58 def __init__(self, backendName, cache_dir):
59 self.cache_dir = cache_dir
60 self.packagedb_dir = cache_dir+'/'+ aptpkg_dir + \
61 '/backends/' + backendName
62 if not os.path.exists(self.packagedb_dir):
63 os.makedirs(self.packagedb_dir)
68 if self.packages is None:
69 self.packages = shelve.open(self.packagedb_dir+'/packages.db')
72 if self.packages is not None:
75 def update_file(self, filename, cache_path, file_path):
77 Called from apt_proxy.py when files get updated so we can update our
78 fake lists/ directory and sources.list.
80 if filename=="Packages" or filename=="Release" or filename=="Sources":
81 log.msg("Registering package file: "+cache_path)
82 self.packages[cache_path] = file_path
86 def check_files(self):
88 Check all files in the database to make sure it exists.
90 files = self.packages.keys()
91 #print self.packages.keys()
93 if not os.path.exists(self.packages[f]):
94 log.msg("File in packages database has been deleted: "+f)
97 def __getitem__(self, key): return self.packages[key]
98 def __setitem__(self, key, item): self.packages[key] = item
99 def __delitem__(self, key): del self.packages[key]
100 def keys(self): return self.packages.keys()
104 Uses AptPackagesServer to answer queries about packages.
106 Makes a fake configuration for python-apt for each backend.
108 DEFAULT_APT_CONFIG = {
110 #'APT::Architecture' : 'amd64', # TODO: Fix this, see bug #436011 and #285360
111 #'APT::Default-Release' : 'unstable',
114 'Dir::State' : 'apt/', # var/lib/apt/
115 'Dir::State::Lists': 'lists/', # lists/
116 #'Dir::State::cdroms' : 'cdroms.list',
117 'Dir::State::userstatus' : 'status.user',
118 'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
119 'Dir::Cache' : '.apt/cache/', # var/cache/apt/
120 #'Dir::Cache::archives' : 'archives/',
121 'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
122 'Dir::Cache::pkgcache' : 'pkgcache.bin',
123 'Dir::Etc' : 'apt/etc/', # etc/apt/
124 'Dir::Etc::sourcelist' : 'sources.list',
125 'Dir::Etc::vendorlist' : 'vendors.list',
126 'Dir::Etc::vendorparts' : 'vendors.list.d',
127 #'Dir::Etc::main' : 'apt.conf',
128 #'Dir::Etc::parts' : 'apt.conf.d',
129 #'Dir::Etc::preferences' : 'preferences',
131 #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
132 'Dir::Bin::dpkg' : '/usr/bin/dpkg',
134 #'DPkg::Pre-Install-Pkgs' : '',
136 #'DPkg::Tools::Options' : '',
137 #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
138 #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
139 #'DPkg::Post-Invoke' : '',
141 essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
143 essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
145 def __init__(self, backendName, cache_dir):
147 Construct new packages manager
148 backend: Name of backend associated with this packages file
149 cache_dir: cache directory from config file
151 self.backendName = backendName
152 self.cache_dir = cache_dir
153 self.apt_config = copy.deepcopy(self.DEFAULT_APT_CONFIG)
155 self.status_dir = (cache_dir+'/'+ aptpkg_dir
156 +'/backends/'+backendName)
157 for dir in self.essential_dirs:
158 path = self.status_dir+'/'+dir
159 if not os.path.exists(path):
161 for file in self.essential_files:
162 path = self.status_dir+'/'+file
163 if not os.path.exists(path):
168 self.apt_config['Dir'] = self.status_dir
169 self.apt_config['Dir::State::status'] = self.status_dir + '/apt/dpkg/status'
170 #os.system('find '+self.status_dir+' -ls ')
171 #print "status:"+self.apt_config['Dir::State::status']
172 self.packages = PackageFileList(backendName, cache_dir)
173 self.indexrecords = {}
175 #print "Loaded aptPackages [%s] %s " % (self.backendName, self.cache_dir)
179 #print "start aptPackages [%s] %s " % (self.backendName, self.cache_dir)
180 self.packages.close()
181 #print "Deleted aptPackages [%s] %s " % (self.backendName, self.cache_dir)
183 def addRelease(self, cache_path, file_path):
185 Dirty hack until python-apt supports apt-pkg/indexrecords.h
188 self.indexrecords[cache_path] = {}
190 read_packages = False
191 f = open(file_path, 'r')
197 read_packages = False
199 # Read the various headers from the file
200 h, v = line.split(":", 1)
201 if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
205 # Bad header line, just ignore it
206 log.msg("WARNING: Ignoring badly formatted Release line: %s" % line)
208 # Skip to the next line
211 # Read file names from the multiple hash sections of the file
214 self.indexrecords[cache_path].setdefault(p[2], {})[hash_type] = (p[0], p[1])
218 def file_updated(self, filename, cache_path, file_path):
220 A file in the backend has changed. If this affects us, unload our apt database
222 if filename == "Release":
223 self.addRelease(cache_path, file_path)
224 if self.packages.update_file(filename, cache_path, file_path):
227 def __save_stdout(self):
228 self.real_stdout_fd = os.dup(1)
231 def __restore_stdout(self):
232 os.dup2(self.real_stdout_fd, 1)
233 os.close(self.real_stdout_fd)
234 del self.real_stdout_fd
238 Regenerates the fake configuration and load the packages server.
240 if self.loaded: return True
242 #print "Load:", self.status_dir
243 shutil.rmtree(self.status_dir+'/apt/lists/')
244 os.makedirs(self.status_dir+'/apt/lists/partial')
245 sources_filename = self.status_dir+'/'+'apt/etc/sources.list'
246 sources = open(sources_filename, 'w')
248 self.packages.check_files()
249 for f in self.packages:
250 # we should probably clear old entries from self.packages and
251 # take into account the recorded mtime as optimization
252 filepath = self.packages[f]
253 fake_uri='http://apt-dht/'+f
254 if f.endswith('Sources'):
255 source_line='deb-src '+dirname(fake_uri)+'/ /'
257 source_line='deb '+dirname(fake_uri)+'/ /'
258 listpath=(self.status_dir+'/apt/lists/'
259 +apt_pkg.URItoFileName(fake_uri))
260 sources.write(source_line+'\n')
261 log.msg("Sources line: " + source_line)
262 sources_count = sources_count + 1
265 #we should empty the directory instead
269 os.symlink(self.packages[f], listpath)
272 if sources_count == 0:
273 log.msg("No Packages files available for %s backend"%(self.backendName))
276 log.msg("Loading Packages database for "+self.status_dir)
277 #apt_pkg.Config = apt_pkg.newConfiguration(); #-- this causes unit tests to fail!
278 for key, value in self.apt_config.items():
279 apt_pkg.Config[key] = value
280 # print "apt_pkg config:"
281 # for I in apt_pkg.Config.keys():
282 # print "%s \"%s\";"%(I,apt_pkg.Config[I]);
284 # apt_pkg prints progress messages to stdout, disable
287 self.cache = apt_pkg.GetCache()
289 self.__restore_stdout()
291 self.records = apt_pkg.GetPkgRecords(self.cache)
292 self.srcrecords = apt_pkg.GetPkgSrcRecords()
293 #for p in self.cache.Packages:
295 #log.debug("%s packages found" % (len(self.cache)),'apt_pkg')
300 "Tries to make the packages server quit."
309 self.packages.close()
311 def findHash(self, path):
312 for release in self.indexrecords:
313 if path.startswith(release[:-7]):
314 for indexFile in self.indexrecords[release]:
315 if release[:-7] + indexFile == path:
316 return self.indexrecords[release][indexFile]['SHA1']
321 package = path.split('/')[-1].split('_')[0]
324 for version in self.cache[package].VersionList:
326 for verFile in version.FileList:
327 if self.records.Lookup(verFile):
328 if self.records.FileName == path:
329 return (self.records.SHA1Hash, size)
333 self.srcrecords.Restart()
334 if self.srcrecords.Lookup(package):
335 for f in self.srcrecords.Files:
342 def get_mirror_path(self, name, version):
343 "Find the path for version 'version' of package 'name'"
344 if not self.load(): return None
346 for pack_vers in self.cache[name].VersionList:
347 if(pack_vers.VerStr == version):
348 file, index = pack_vers.FileList[0]
349 self.records.Lookup((file,index))
350 path = self.records.FileName
351 if len(path)>2 and path[0:2] == './':
352 path = path[2:] # Remove any leading './'
360 def get_mirror_versions(self, package_name):
362 Find the available versions of the package name given
363 @type package_name: string
364 @param package_name: package name to search for e.g. ;apt'
365 @return: A list of mirror versions available
369 if not self.load(): return vers
371 for pack_vers in self.cache[package_name].VersionList:
372 vers.append(pack_vers.VerStr)
378 def cleanup(factory):
379 for backend in factory.backends.values():
380 backend.get_packages_db().cleanup()
382 def get_mirror_path(factory, file):
384 Look for the path of 'file' in all backends.
386 info = AptDpkgInfo(file)
388 for backend in factory.backends.values():
389 path = backend.get_packages_db().get_mirror_path(info['Package'],
392 paths.append('/'+backend.base+'/'+path)
395 def get_mirror_versions(factory, package):
397 Look for the available version of a package in all backends, given
398 an existing package name
401 for backend in factory.backends.values():
402 vers = backend.get_packages_db().get_mirror_versions(package)
404 path = backend.get_packages_db().get_mirror_path(package, ver)
405 all_vers.append((ver, "%s/%s"%(backend.base,path)))
408 def closest_match(info, others):
410 return apt_pkg.VersionCompare(a[0], b[0])
413 version = info['Version']
415 for ver,path in others:
422 match = others[-1][1]
424 dirname=re.sub(r'/[^/]*$', '', match)
425 version=re.sub(r'^[^:]*:', '', info['Version'])
426 if dirname.find('/pool/') != -1:
427 return "/%s/%s_%s_%s.deb"%(dirname, info['Package'],
428 version, info['Architecture'])
430 return "/%s/%s_%s.deb"%(dirname, info['Package'], version)
432 def import_directory(factory, dir, recursive=0):
434 Import all files in a given directory into the cache
435 This is used by apt-proxy-import to import new files
440 if not os.path.exists(dir):
441 log.err('Directory ' + dir + ' does not exist')
445 log.msg("Importing packages from directory tree: " + dir)
446 for root, dirs, files in os.walk(dir):
448 imported_count += import_file(factory, root, file)
450 log.msg("Importing packages from directory: " + dir)
451 for file in os.listdir(dir):
452 mode = os.stat(dir + '/' + file)[stat.ST_MODE]
453 if not stat.S_ISDIR(mode):
454 imported_count += import_file(factory, dir, file)
456 for backend in factory.backends.values():
457 backend.get_packages_db().unload()
459 log.msg("Imported %s files" % (imported_count))
460 return imported_count
462 def import_file(factory, dir, file):
464 Import a .deb or .udeb into cache from given filename
466 if file[-4:]!='.deb' and file[-5:]!='.udeb':
467 log.msg("Ignoring (unknown file type):"+ file)
470 log.msg("considering: " + dir + '/' + file)
472 paths = get_mirror_path(factory, dir+'/'+file)
474 log.msg(file + ' skipped - wrong format or corrupted')
478 log.msg("WARNING: multiple ocurrences")
479 log.msg(str(paths), 'import')
480 cache_path = paths[0]
482 log.msg("Not found, trying to guess")
483 info = AptDpkgInfo(dir+'/'+file)
484 cache_path = closest_match(info,
485 get_mirror_versions(factory, info['Package']))
487 log.msg("MIRROR_PATH:"+ cache_path)
488 src_path = dir+'/'+file
489 dest_path = factory.config.cache_dir+cache_path
491 if not os.path.exists(dest_path):
492 log.msg("IMPORTING:" + src_path)
493 dest_path = re.sub(r'/\./', '/', dest_path)
494 if not os.path.exists(dirname(dest_path)):
495 os.makedirs(dirname(dest_path))
496 f = open(dest_path, 'w')
497 fcntl.lockf(f.fileno(), fcntl.LOCK_EX)
499 shutil.copy2(src_path, dest_path)
501 if hasattr(factory, 'access_times'):
502 atime = os.stat(src_path)[stat.ST_ATIME]
503 factory.access_times[cache_path] = atime
504 log.msg(file + ' imported')
507 log.msg(file + ' skipped - already in cache')
511 log.msg(file + ' skipped - no suitable backend found')
514 class TestAptPackages(unittest.TestCase):
515 """Unit tests for the AptPackages cache."""
524 self.client = AptPackages('whatever', '/tmp')
526 self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Packages$" | tail -n 1').read().rstrip('\n')
527 self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Sources$" | tail -n 1').read().rstrip('\n')
528 for f in os.walk('/var/lib/apt/lists').next()[2]:
529 if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
533 self.client.file_updated('Release',
534 self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/'),
535 '/var/lib/apt/lists/' + self.releaseFile)
536 self.client.file_updated('Packages',
537 self.packagesFile[self.packagesFile.find('_debian_')+1:].replace('_','/'),
538 '/var/lib/apt/lists/' + self.packagesFile)
539 self.client.file_updated('Sources',
540 self.sourcesFile[self.sourcesFile.find('_debian_')+1:].replace('_','/'),
541 '/var/lib/apt/lists/' + self.sourcesFile)
545 def test_pkg_hash(self):
546 self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
548 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
549 '/var/lib/apt/lists/' + self.packagesFile +
550 ' | grep -E "^SHA1:" | head -n 1' +
551 ' | cut -d\ -f 2').read().rstrip('\n')
553 self.failUnless(self.client.records.SHA1Hash == pkg_hash,
554 "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
556 def test_src_hash(self):
557 self.client.srcrecords.Lookup('dpkg')
559 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
560 '/var/lib/apt/lists/' + self.sourcesFile +
561 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
562 ' | cut -d\ -f 2').read().split('\n')[:-1]
564 for f in self.client.srcrecords.Files:
565 self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
567 def test_index_hash(self):
568 indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
570 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
571 '/var/lib/apt/lists/' + self.releaseFile +
572 ' | grep -E " main/binary-i386/Packages.bz2$"'
573 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
575 self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
577 def test_findHash(self):
578 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
579 '/var/lib/apt/lists/' + self.releaseFile +
580 ' | grep -E " main/binary-i386/Packages.bz2$"'
581 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
582 idx_path = self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')[:-7] + 'main/binary-i386/Packages.bz2'
584 found_hash = self.client.findHash(idx_path)
585 self.failUnless(found_hash[0] == idx_hash,
586 "Hashes don't match: %s != %s" % (found_hash[0], idx_hash))
588 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
589 '/var/lib/apt/lists/' + self.packagesFile +
590 ' | grep -E "^SHA1:" | head -n 1' +
591 ' | cut -d\ -f 2').read().rstrip('\n')
592 pkg_path = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
593 '/var/lib/apt/lists/' + self.packagesFile +
594 ' | grep -E "^Filename:" | head -n 1' +
595 ' | cut -d\ -f 2').read().rstrip('\n')
597 found_hash = self.client.findHash(pkg_path)
598 self.failUnless(found_hash[0] == pkg_hash,
599 "Hashes don't match: %s != %s" % (found_hash[0], pkg_hash))
601 src_dir = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
602 '/var/lib/apt/lists/' + self.sourcesFile +
603 ' | grep -E "^Directory:" | head -n 1' +
604 ' | cut -d\ -f 2').read().rstrip('\n')
605 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
606 '/var/lib/apt/lists/' + self.sourcesFile +
607 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
608 ' | cut -d\ -f 2').read().split('\n')[:-1]
609 src_paths = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
610 '/var/lib/apt/lists/' + self.sourcesFile +
611 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
612 ' | cut -d\ -f 4').read().split('\n')[:-1]
614 for i in range(len(src_hashes)):
615 found_hash = self.client.findHash(src_dir + '/' + src_paths[i])
616 self.failUnless(found_hash[0] == src_hashes[i],
617 "%s hashes don't match: %s != %s" %
618 (src_dir + '/' + src_paths[i], found_hash[0], src_hashes[i]))
621 for p in self.pending_calls:
624 self.pending_calls = []
625 self.client.cleanup()