2 # Copyright (C) 2002 Manuel Estrada Sainz <ranty@debian.org>
4 # This library is free software; you can redistribute it and/or
5 # modify it under the terms of version 2.1 of the GNU Lesser General Public
6 # License as published by the Free Software Foundation.
8 # This library is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 # Lesser General Public License for more details.
13 # You should have received a copy of the GNU Lesser General Public
14 # License along with this library; if not, write to the Free Software
15 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 import apt_pkg, apt_inst, sys, os, stat
18 from os.path import dirname, basename
19 import re, shelve, shutil, fcntl
20 from twisted.internet import process
21 from twisted.python import log
23 from twisted.trial import unittest
28 class AptDpkgInfo(UserDict.UserDict):
30 Gets control fields from a .deb file.
32 And then behaves like a regular python dictionary.
34 See AptPackages.get_mirror_path
37 def __init__(self, filename):
38 UserDict.UserDict.__init__(self)
40 filehandle = open(filename);
42 self.control = apt_inst.debExtractControl(filehandle)
44 # Make sure that file is always closed.
47 log.msg("Had problems reading: %s"%(filename))
49 for line in self.control.split('\n'):
50 if line.find(': ') != -1:
51 key, value = line.split(': ', 1)
52 self.data[key] = value
54 class PackageFileList(UserDict.DictMixin):
56 Manages a list of package files belonging to a backend
58 def __init__(self, backendName, cache_dir):
59 self.cache_dir = cache_dir
60 self.packagedb_dir = cache_dir+'/'+ aptpkg_dir + \
61 '/backends/' + backendName
62 if not os.path.exists(self.packagedb_dir):
63 os.makedirs(self.packagedb_dir)
68 if self.packages is None:
69 self.packages = shelve.open(self.packagedb_dir+'/packages.db')
72 if self.packages is not None:
75 def update_file(self, filename, cache_path, file_path):
77 Called from apt_proxy.py when files get updated so we can update our
78 fake lists/ directory and sources.list.
80 if filename=="Packages" or filename=="Release" or filename=="Sources":
81 log.msg("Registering package file: "+cache_path)
82 self.packages[cache_path] = file_path
86 def check_files(self):
88 Check all files in the database to make sure it exists.
90 files = self.packages.keys()
91 #print self.packages.keys()
93 if not os.path.exists(self.packages[f]):
94 log.msg("File in packages database has been deleted: "+f)
97 def __getitem__(self, key): return self.packages[key]
98 def __setitem__(self, key, item): self.packages[key] = item
99 def __delitem__(self, key): del self.packages[key]
100 def keys(self): return self.packages.keys()
104 Uses AptPackagesServer to answer queries about packages.
106 Makes a fake configuration for python-apt for each backend.
108 DEFAULT_APT_CONFIG = {
110 #'APT::Architecture' : 'amd64', # TODO: Fix this, see bug #436011 and #285360
111 #'APT::Default-Release' : 'unstable',
114 'Dir::State' : 'apt/', # var/lib/apt/
115 'Dir::State::Lists': 'lists/', # lists/
116 #'Dir::State::cdroms' : 'cdroms.list',
117 'Dir::State::userstatus' : 'status.user',
118 'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
119 'Dir::Cache' : '.apt/cache/', # var/cache/apt/
120 #'Dir::Cache::archives' : 'archives/',
121 'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
122 'Dir::Cache::pkgcache' : 'pkgcache.bin',
123 'Dir::Etc' : 'apt/etc/', # etc/apt/
124 'Dir::Etc::sourcelist' : 'sources.list',
125 'Dir::Etc::vendorlist' : 'vendors.list',
126 'Dir::Etc::vendorparts' : 'vendors.list.d',
127 #'Dir::Etc::main' : 'apt.conf',
128 #'Dir::Etc::parts' : 'apt.conf.d',
129 #'Dir::Etc::preferences' : 'preferences',
131 #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
132 'Dir::Bin::dpkg' : '/usr/bin/dpkg',
134 #'DPkg::Pre-Install-Pkgs' : '',
136 #'DPkg::Tools::Options' : '',
137 #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
138 #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
139 #'DPkg::Post-Invoke' : '',
141 essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
143 essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
145 def __init__(self, backendName, cache_dir):
147 Construct new packages manager
148 backend: Name of backend associated with this packages file
149 cache_dir: cache directory from config file
151 self.backendName = backendName
152 self.cache_dir = cache_dir
153 self.apt_config = copy.deepcopy(self.DEFAULT_APT_CONFIG)
155 self.status_dir = (cache_dir+'/'+ aptpkg_dir
156 +'/backends/'+backendName)
157 for dir in self.essential_dirs:
158 path = self.status_dir+'/'+dir
159 if not os.path.exists(path):
161 for file in self.essential_files:
162 path = self.status_dir+'/'+file
163 if not os.path.exists(path):
168 self.apt_config['Dir'] = self.status_dir
169 self.apt_config['Dir::State::status'] = self.status_dir + '/apt/dpkg/status'
170 #os.system('find '+self.status_dir+' -ls ')
171 #print "status:"+self.apt_config['Dir::State::status']
172 self.packages = PackageFileList(backendName, cache_dir)
173 self.indexrecords = {}
175 #print "Loaded aptPackages [%s] %s " % (self.backendName, self.cache_dir)
179 #print "start aptPackages [%s] %s " % (self.backendName, self.cache_dir)
180 self.packages.close()
181 #print "Deleted aptPackages [%s] %s " % (self.backendName, self.cache_dir)
183 def addRelease(self, cache_path, file_path):
185 Dirty hack until python-apt supports apt-pkg/indexrecords.h
188 self.indexrecords[cache_path] = {}
190 read_packages = False
191 f = open(file_path, 'r')
197 read_packages = False
199 # Read the various headers from the file
200 h, v = line.split(":", 1)
201 if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
205 # Bad header line, just ignore it
206 log.msg("WARNING: Ignoring badly formatted Release line: %s" % line)
208 # Skip to the next line
211 # Read file names from the multiple hash sections of the file
214 self.indexrecords[cache_path].setdefault(p[2], {})[hash_type] = (p[0], p[1])
218 def file_updated(self, filename, cache_path, file_path):
220 A file in the backend has changed. If this affects us, unload our apt database
222 if filename == "Release":
223 self.addRelease(cache_path, file_path)
224 if self.packages.update_file(filename, cache_path, file_path):
227 def __save_stdout(self):
228 self.real_stdout_fd = os.dup(1)
231 def __restore_stdout(self):
232 os.dup2(self.real_stdout_fd, 1)
233 os.close(self.real_stdout_fd)
234 del self.real_stdout_fd
238 Regenerates the fake configuration and load the packages server.
240 if self.loaded: return True
242 #print "Load:", self.status_dir
243 shutil.rmtree(self.status_dir+'/apt/lists/')
244 os.makedirs(self.status_dir+'/apt/lists/partial')
245 sources_filename = self.status_dir+'/'+'apt/etc/sources.list'
246 sources = open(sources_filename, 'w')
248 self.packages.check_files()
249 for f in self.packages:
250 # we should probably clear old entries from self.packages and
251 # take into account the recorded mtime as optimization
252 filepath = self.packages[f]
253 fake_uri='http://apt-dht/'+f
254 if f.endswith('Sources'):
255 source_line='deb-src '+dirname(fake_uri)+'/ /'
257 source_line='deb '+dirname(fake_uri)+'/ /'
258 listpath=(self.status_dir+'/apt/lists/'
259 +apt_pkg.URItoFileName(fake_uri))
260 sources.write(source_line+'\n')
261 log.msg("Sources line: " + source_line)
262 sources_count = sources_count + 1
265 #we should empty the directory instead
269 os.symlink(self.packages[f], listpath)
272 if sources_count == 0:
273 log.msg("No Packages files available for %s backend"%(self.backendName))
276 log.msg("Loading Packages database for "+self.status_dir)
277 #apt_pkg.Config = apt_pkg.newConfiguration(); #-- this causes unit tests to fail!
278 for key, value in self.apt_config.items():
279 apt_pkg.Config[key] = value
280 # print "apt_pkg config:"
281 # for I in apt_pkg.Config.keys():
282 # print "%s \"%s\";"%(I,apt_pkg.Config[I]);
284 # apt_pkg prints progress messages to stdout, disable
287 self.cache = apt_pkg.GetCache()
289 self.__restore_stdout()
291 self.records = apt_pkg.GetPkgRecords(self.cache)
292 self.srcrecords = apt_pkg.GetPkgSrcRecords()
293 #for p in self.cache.Packages:
295 #log.debug("%s packages found" % (len(self.cache)),'apt_pkg')
300 "Tries to make the packages server quit."
309 self.packages.close()
311 def get_mirror_path(self, name, version):
312 "Find the path for version 'version' of package 'name'"
313 if not self.load(): return None
315 for pack_vers in self.cache[name].VersionList:
316 if(pack_vers.VerStr == version):
317 file, index = pack_vers.FileList[0]
318 self.records.Lookup((file,index))
319 path = self.records.FileName
320 if len(path)>2 and path[0:2] == './':
321 path = path[2:] # Remove any leading './'
329 def get_mirror_versions(self, package_name):
331 Find the available versions of the package name given
332 @type package_name: string
333 @param package_name: package name to search for e.g. ;apt'
334 @return: A list of mirror versions available
338 if not self.load(): return vers
340 for pack_vers in self.cache[package_name].VersionList:
341 vers.append(pack_vers.VerStr)
347 def cleanup(factory):
348 for backend in factory.backends.values():
349 backend.get_packages_db().cleanup()
351 def get_mirror_path(factory, file):
353 Look for the path of 'file' in all backends.
355 info = AptDpkgInfo(file)
357 for backend in factory.backends.values():
358 path = backend.get_packages_db().get_mirror_path(info['Package'],
361 paths.append('/'+backend.base+'/'+path)
364 def get_mirror_versions(factory, package):
366 Look for the available version of a package in all backends, given
367 an existing package name
370 for backend in factory.backends.values():
371 vers = backend.get_packages_db().get_mirror_versions(package)
373 path = backend.get_packages_db().get_mirror_path(package, ver)
374 all_vers.append((ver, "%s/%s"%(backend.base,path)))
377 def closest_match(info, others):
379 return apt_pkg.VersionCompare(a[0], b[0])
382 version = info['Version']
384 for ver,path in others:
391 match = others[-1][1]
393 dirname=re.sub(r'/[^/]*$', '', match)
394 version=re.sub(r'^[^:]*:', '', info['Version'])
395 if dirname.find('/pool/') != -1:
396 return "/%s/%s_%s_%s.deb"%(dirname, info['Package'],
397 version, info['Architecture'])
399 return "/%s/%s_%s.deb"%(dirname, info['Package'], version)
401 def import_directory(factory, dir, recursive=0):
403 Import all files in a given directory into the cache
404 This is used by apt-proxy-import to import new files
409 if not os.path.exists(dir):
410 log.err('Directory ' + dir + ' does not exist')
414 log.msg("Importing packages from directory tree: " + dir)
415 for root, dirs, files in os.walk(dir):
417 imported_count += import_file(factory, root, file)
419 log.msg("Importing packages from directory: " + dir)
420 for file in os.listdir(dir):
421 mode = os.stat(dir + '/' + file)[stat.ST_MODE]
422 if not stat.S_ISDIR(mode):
423 imported_count += import_file(factory, dir, file)
425 for backend in factory.backends.values():
426 backend.get_packages_db().unload()
428 log.msg("Imported %s files" % (imported_count))
429 return imported_count
431 def import_file(factory, dir, file):
433 Import a .deb or .udeb into cache from given filename
435 if file[-4:]!='.deb' and file[-5:]!='.udeb':
436 log.msg("Ignoring (unknown file type):"+ file)
439 log.msg("considering: " + dir + '/' + file)
441 paths = get_mirror_path(factory, dir+'/'+file)
443 log.msg(file + ' skipped - wrong format or corrupted')
447 log.msg("WARNING: multiple ocurrences")
448 log.msg(str(paths), 'import')
449 cache_path = paths[0]
451 log.msg("Not found, trying to guess")
452 info = AptDpkgInfo(dir+'/'+file)
453 cache_path = closest_match(info,
454 get_mirror_versions(factory, info['Package']))
456 log.msg("MIRROR_PATH:"+ cache_path)
457 src_path = dir+'/'+file
458 dest_path = factory.config.cache_dir+cache_path
460 if not os.path.exists(dest_path):
461 log.msg("IMPORTING:" + src_path)
462 dest_path = re.sub(r'/\./', '/', dest_path)
463 if not os.path.exists(dirname(dest_path)):
464 os.makedirs(dirname(dest_path))
465 f = open(dest_path, 'w')
466 fcntl.lockf(f.fileno(), fcntl.LOCK_EX)
468 shutil.copy2(src_path, dest_path)
470 if hasattr(factory, 'access_times'):
471 atime = os.stat(src_path)[stat.ST_ATIME]
472 factory.access_times[cache_path] = atime
473 log.msg(file + ' imported')
476 log.msg(file + ' skipped - already in cache')
480 log.msg(file + ' skipped - no suitable backend found')
483 class TestAptPackages(unittest.TestCase):
484 """Unit tests for the AptPackages cache."""
493 self.client = AptPackages('whatever', '/tmp')
495 self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Packages$" | tail -n 1').read().rstrip('\n')
496 self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Sources$" | tail -n 1').read().rstrip('\n')
497 for f in os.walk('/var/lib/apt/lists').next()[2]:
498 if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
502 self.client.file_updated('Release',
503 self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/'),
504 '/var/lib/apt/lists/' + self.releaseFile)
505 self.client.file_updated('Packages',
506 self.packagesFile[self.packagesFile.find('_debian_')+1:].replace('_','/'),
507 '/var/lib/apt/lists/' + self.packagesFile)
508 self.client.file_updated('Sources',
509 self.sourcesFile[self.sourcesFile.find('_debian_')+1:].replace('_','/'),
510 '/var/lib/apt/lists/' + self.sourcesFile)
514 def test_pkg_hash(self):
515 self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
517 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
518 '/var/lib/apt/lists/' + self.packagesFile +
519 ' | grep -E "^SHA1:" | head -n 1' +
520 ' | cut -d\ -f 2').read().rstrip('\n')
522 self.failUnless(self.client.records.SHA1Hash == pkg_hash,
523 "Hashes don't match: %s != %s" % (self.client.records.SHA1Hash, pkg_hash))
525 def test_src_hash(self):
526 self.client.srcrecords.Lookup('dpkg')
528 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
529 '/var/lib/apt/lists/' + self.sourcesFile +
530 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
531 ' | cut -d\ -f 2').read().split('\n')[:-1]
533 for f in self.client.srcrecords.Files:
534 self.failUnless(f[0] in src_hashes, "Couldn't find %s in: %r" % (f[0], src_hashes))
536 def test_index_hash(self):
537 indexhash = self.client.indexrecords[self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/')]['main/binary-i386/Packages.bz2']['SHA1'][0]
539 idx_hash = os.popen('grep -A 3000 -E "^SHA1:" ' +
540 '/var/lib/apt/lists/' + self.releaseFile +
541 ' | grep -E " main/binary-i386/Packages.bz2$"'
542 ' | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
544 self.failUnless(indexhash == idx_hash, "Hashes don't match: %s != %s" % (indexhash, idx_hash))
547 for p in self.pending_calls:
550 self.pending_calls = []
551 self.client.cleanup()