2 # Copyright (C) 2002 Manuel Estrada Sainz <ranty@debian.org>
4 # This library is free software; you can redistribute it and/or
5 # modify it under the terms of version 2.1 of the GNU Lesser General Public
6 # License as published by the Free Software Foundation.
8 # This library is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 # Lesser General Public License for more details.
13 # You should have received a copy of the GNU Lesser General Public
14 # License along with this library; if not, write to the Free Software
15 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 import apt_pkg, apt_inst, sys, os, stat
18 from os.path import dirname, basename
19 import re, shelve, shutil, fcntl
20 from twisted.internet import process
21 from twisted.python import log
23 from twisted.trial import unittest
28 class AptDpkgInfo(UserDict.UserDict):
30 Gets control fields from a .deb file.
32 And then behaves like a regular python dictionary.
34 See AptPackages.get_mirror_path
37 def __init__(self, filename):
38 UserDict.UserDict.__init__(self)
40 filehandle = open(filename);
42 self.control = apt_inst.debExtractControl(filehandle)
44 # Make sure that file is always closed.
47 log.msg("Had problems reading: %s"%(filename))
49 for line in self.control.split('\n'):
50 if line.find(': ') != -1:
51 key, value = line.split(': ', 1)
52 self.data[key] = value
54 class PackageFileList(UserDict.DictMixin):
56 Manages a list of package files belonging to a backend
58 def __init__(self, backendName, cache_dir):
59 self.cache_dir = cache_dir
60 self.packagedb_dir = cache_dir+'/'+ aptpkg_dir + \
61 '/backends/' + backendName
62 if not os.path.exists(self.packagedb_dir):
63 os.makedirs(self.packagedb_dir)
68 if self.packages is None:
69 self.packages = shelve.open(self.packagedb_dir+'/packages.db')
72 if self.packages is not None:
75 def update_file(self, filename, cache_path, file_path):
77 Called from apt_proxy.py when files get updated so we can update our
78 fake lists/ directory and sources.list.
80 if filename=="Packages" or filename=="Release" or filename=="Sources":
81 log.msg("Registering package file: "+cache_path)
82 self.packages[cache_path] = file_path
86 def check_files(self):
88 Check all files in the database to make sure it exists.
90 files = self.packages.keys()
91 #print self.packages.keys()
93 if not os.path.exists(self.packages[f]):
94 log.msg("File in packages database has been deleted: "+f)
97 def __getitem__(self, key): return self.packages[key]
98 def __setitem__(self, key, item): self.packages[key] = item
99 def __delitem__(self, key): del self.packages[key]
100 def keys(self): return self.packages.keys()
104 Uses AptPackagesServer to answer queries about packages.
106 Makes a fake configuration for python-apt for each backend.
108 DEFAULT_APT_CONFIG = {
110 #'APT::Architecture' : 'amd64', # TODO: Fix this, see bug #436011 and #285360
111 #'APT::Default-Release' : 'unstable',
114 'Dir::State' : 'apt/', # var/lib/apt/
115 'Dir::State::Lists': 'lists/', # lists/
116 #'Dir::State::cdroms' : 'cdroms.list',
117 'Dir::State::userstatus' : 'status.user',
118 'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
119 'Dir::Cache' : '.apt/cache/', # var/cache/apt/
120 #'Dir::Cache::archives' : 'archives/',
121 'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
122 'Dir::Cache::pkgcache' : 'pkgcache.bin',
123 'Dir::Etc' : 'apt/etc/', # etc/apt/
124 'Dir::Etc::sourcelist' : 'sources.list',
125 'Dir::Etc::vendorlist' : 'vendors.list',
126 'Dir::Etc::vendorparts' : 'vendors.list.d',
127 #'Dir::Etc::main' : 'apt.conf',
128 #'Dir::Etc::parts' : 'apt.conf.d',
129 #'Dir::Etc::preferences' : 'preferences',
131 #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
132 'Dir::Bin::dpkg' : '/usr/bin/dpkg',
134 #'DPkg::Pre-Install-Pkgs' : '',
136 #'DPkg::Tools::Options' : '',
137 #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
138 #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
139 #'DPkg::Post-Invoke' : '',
141 essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
143 essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
145 def __init__(self, backendName, cache_dir):
147 Construct new packages manager
148 backend: Name of backend associated with this packages file
149 cache_dir: cache directory from config file
151 self.backendName = backendName
152 self.cache_dir = cache_dir
153 self.apt_config = copy.deepcopy(self.DEFAULT_APT_CONFIG)
155 self.status_dir = (cache_dir+'/'+ aptpkg_dir
156 +'/backends/'+backendName)
157 for dir in self.essential_dirs:
158 path = self.status_dir+'/'+dir
159 if not os.path.exists(path):
161 for file in self.essential_files:
162 path = self.status_dir+'/'+file
163 if not os.path.exists(path):
168 self.apt_config['Dir'] = self.status_dir
169 self.apt_config['Dir::State::status'] = self.status_dir + '/apt/dpkg/status'
170 #os.system('find '+self.status_dir+' -ls ')
171 #print "status:"+self.apt_config['Dir::State::status']
172 self.packages = PackageFileList(backendName, cache_dir)
174 #print "Loaded aptPackages [%s] %s " % (self.backendName, self.cache_dir)
178 #print "start aptPackages [%s] %s " % (self.backendName, self.cache_dir)
179 self.packages.close()
180 #print "Deleted aptPackages [%s] %s " % (self.backendName, self.cache_dir)
181 def file_updated(self, filename, cache_path, file_path):
183 A file in the backend has changed. If this affects us, unload our apt database
185 if self.packages.update_file(filename, cache_path, file_path):
188 def __save_stdout(self):
189 self.real_stdout_fd = os.dup(1)
192 def __restore_stdout(self):
193 os.dup2(self.real_stdout_fd, 1)
194 os.close(self.real_stdout_fd)
195 del self.real_stdout_fd
199 Regenerates the fake configuration and load the packages server.
201 if self.loaded: return True
203 #print "Load:", self.status_dir
204 shutil.rmtree(self.status_dir+'/apt/lists/')
205 os.makedirs(self.status_dir+'/apt/lists/partial')
206 sources_filename = self.status_dir+'/'+'apt/etc/sources.list'
207 sources = open(sources_filename, 'w')
209 self.packages.check_files()
210 for f in self.packages:
211 # we should probably clear old entries from self.packages and
212 # take into account the recorded mtime as optimization
213 filepath = self.packages[f]
214 fake_uri='http://apt-dht/'+f
215 if f.endswith('Sources'):
216 source_line='deb-src '+dirname(fake_uri)+'/ /'
218 source_line='deb '+dirname(fake_uri)+'/ /'
219 listpath=(self.status_dir+'/apt/lists/'
220 +apt_pkg.URItoFileName(fake_uri))
221 sources.write(source_line+'\n')
222 log.msg("Sources line: " + source_line)
223 sources_count = sources_count + 1
226 #we should empty the directory instead
230 os.symlink(self.packages[f], listpath)
233 if sources_count == 0:
234 log.msg("No Packages files available for %s backend"%(self.backendName))
237 log.msg("Loading Packages database for "+self.status_dir)
238 #apt_pkg.Config = apt_pkg.newConfiguration(); #-- this causes unit tests to fail!
239 for key, value in self.apt_config.items():
240 apt_pkg.Config[key] = value
241 # print "apt_pkg config:"
242 # for I in apt_pkg.Config.keys():
243 # print "%s \"%s\";"%(I,apt_pkg.Config[I]);
245 # apt_pkg prints progress messages to stdout, disable
248 self.cache = apt_pkg.GetCache()
250 self.__restore_stdout()
252 self.records = apt_pkg.GetPkgRecords(self.cache)
253 self.srcrecords = apt_pkg.GetPkgSrcRecords()
254 #for p in self.cache.Packages:
256 #log.debug("%s packages found" % (len(self.cache)),'apt_pkg')
261 "Tries to make the packages server quit."
270 self.packages.close()
272 def get_mirror_path(self, name, version):
273 "Find the path for version 'version' of package 'name'"
274 if not self.load(): return None
276 for pack_vers in self.cache[name].VersionList:
277 if(pack_vers.VerStr == version):
278 file, index = pack_vers.FileList[0]
279 self.records.Lookup((file,index))
280 path = self.records.FileName
281 if len(path)>2 and path[0:2] == './':
282 path = path[2:] # Remove any leading './'
290 def get_mirror_versions(self, package_name):
292 Find the available versions of the package name given
293 @type package_name: string
294 @param package_name: package name to search for e.g. ;apt'
295 @return: A list of mirror versions available
299 if not self.load(): return vers
301 for pack_vers in self.cache[package_name].VersionList:
302 vers.append(pack_vers.VerStr)
308 def cleanup(factory):
309 for backend in factory.backends.values():
310 backend.get_packages_db().cleanup()
312 def get_mirror_path(factory, file):
314 Look for the path of 'file' in all backends.
316 info = AptDpkgInfo(file)
318 for backend in factory.backends.values():
319 path = backend.get_packages_db().get_mirror_path(info['Package'],
322 paths.append('/'+backend.base+'/'+path)
325 def get_mirror_versions(factory, package):
327 Look for the available version of a package in all backends, given
328 an existing package name
331 for backend in factory.backends.values():
332 vers = backend.get_packages_db().get_mirror_versions(package)
334 path = backend.get_packages_db().get_mirror_path(package, ver)
335 all_vers.append((ver, "%s/%s"%(backend.base,path)))
338 def closest_match(info, others):
340 return apt_pkg.VersionCompare(a[0], b[0])
343 version = info['Version']
345 for ver,path in others:
352 match = others[-1][1]
354 dirname=re.sub(r'/[^/]*$', '', match)
355 version=re.sub(r'^[^:]*:', '', info['Version'])
356 if dirname.find('/pool/') != -1:
357 return "/%s/%s_%s_%s.deb"%(dirname, info['Package'],
358 version, info['Architecture'])
360 return "/%s/%s_%s.deb"%(dirname, info['Package'], version)
362 def import_directory(factory, dir, recursive=0):
364 Import all files in a given directory into the cache
365 This is used by apt-proxy-import to import new files
370 if not os.path.exists(dir):
371 log.err('Directory ' + dir + ' does not exist')
375 log.msg("Importing packages from directory tree: " + dir)
376 for root, dirs, files in os.walk(dir):
378 imported_count += import_file(factory, root, file)
380 log.msg("Importing packages from directory: " + dir)
381 for file in os.listdir(dir):
382 mode = os.stat(dir + '/' + file)[stat.ST_MODE]
383 if not stat.S_ISDIR(mode):
384 imported_count += import_file(factory, dir, file)
386 for backend in factory.backends.values():
387 backend.get_packages_db().unload()
389 log.msg("Imported %s files" % (imported_count))
390 return imported_count
392 def import_file(factory, dir, file):
394 Import a .deb or .udeb into cache from given filename
396 if file[-4:]!='.deb' and file[-5:]!='.udeb':
397 log.msg("Ignoring (unknown file type):"+ file)
400 log.msg("considering: " + dir + '/' + file)
402 paths = get_mirror_path(factory, dir+'/'+file)
404 log.msg(file + ' skipped - wrong format or corrupted')
408 log.msg("WARNING: multiple ocurrences")
409 log.msg(str(paths), 'import')
410 cache_path = paths[0]
412 log.msg("Not found, trying to guess")
413 info = AptDpkgInfo(dir+'/'+file)
414 cache_path = closest_match(info,
415 get_mirror_versions(factory, info['Package']))
417 log.msg("MIRROR_PATH:"+ cache_path)
418 src_path = dir+'/'+file
419 dest_path = factory.config.cache_dir+cache_path
421 if not os.path.exists(dest_path):
422 log.msg("IMPORTING:" + src_path)
423 dest_path = re.sub(r'/\./', '/', dest_path)
424 if not os.path.exists(dirname(dest_path)):
425 os.makedirs(dirname(dest_path))
426 f = open(dest_path, 'w')
427 fcntl.lockf(f.fileno(), fcntl.LOCK_EX)
429 shutil.copy2(src_path, dest_path)
431 if hasattr(factory, 'access_times'):
432 atime = os.stat(src_path)[stat.ST_ATIME]
433 factory.access_times[cache_path] = atime
434 log.msg(file + ' imported')
437 log.msg(file + ' skipped - already in cache')
441 log.msg(file + ' skipped - no suitable backend found')
444 class TestAptPackages(unittest.TestCase):
445 """Unit tests for the AptPackages cache."""
454 self.client = AptPackages('whatever', '/tmp')
456 self.packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Packages$" | tail -n 1').read().rstrip('\n')
457 self.sourcesFile = os.popen('ls -Sr /var/lib/apt/lists/ | grep -E "Sources$" | tail -n 1').read().rstrip('\n')
458 for f in os.walk('/var/lib/apt/lists').next()[2]:
459 if f[-7:] == "Release" and self.packagesFile.startswith(f[:-7]):
463 self.client.file_updated('Release',
464 self.releaseFile[self.releaseFile.find('_debian_')+1:].replace('_','/'),
465 '/var/lib/apt/lists/' + self.releaseFile)
466 self.client.file_updated('Packages',
467 self.packagesFile[self.packagesFile.find('_debian_')+1:].replace('_','/'),
468 '/var/lib/apt/lists/' + self.packagesFile)
469 self.client.file_updated('Sources',
470 self.sourcesFile[self.sourcesFile.find('_debian_')+1:].replace('_','/'),
471 '/var/lib/apt/lists/' + self.sourcesFile)
475 def test_pkg_hash(self):
476 self.client.records.Lookup(self.client.cache['dpkg'].VersionList[0].FileList[0])
478 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' +
479 '/var/lib/apt/lists/' + self.packagesFile +
480 ' | grep -E "^SHA1:" | head -n 1' +
481 ' | cut -d\ -f 2').read().rstrip('\n')
483 self.failUnless(self.client.records.SHA1Hash == pkg_hash)
485 def test_src_hash(self):
486 self.client.srcrecords.Lookup('dpkg')
488 src_hashes = os.popen('grep -A 20 -E "^Package: dpkg$" ' +
489 '/var/lib/apt/lists/' + self.sourcesFile +
490 ' | grep -A 4 -E "^Files:" | grep -E "^ " ' +
491 ' | cut -d\ -f 2').read().split('\n')[:-1]
493 for f in self.client.srcrecords.Files:
494 self.failUnless(f[0] in src_hashes)
497 for p in self.pending_calls:
500 self.pending_calls = []
501 self.client.cleanup()