2 # Copyright (C) 2002 Manuel Estrada Sainz <ranty@debian.org>
4 # This library is free software; you can redistribute it and/or
5 # modify it under the terms of version 2.1 of the GNU Lesser General Public
6 # License as published by the Free Software Foundation.
8 # This library is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 # Lesser General Public License for more details.
13 # You should have received a copy of the GNU Lesser General Public
14 # License along with this library; if not, write to the Free Software
15 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 import apt_pkg, apt_inst, sys, os, stat
18 from os.path import dirname, basename
19 import re, shelve, shutil, fcntl
20 from twisted.internet import process
21 from twisted.python import log
23 from twisted.trial import unittest
28 class AptDpkgInfo(UserDict.UserDict):
30 Gets control fields from a .deb file.
32 And then behaves like a regular python dictionary.
34 See AptPackages.get_mirror_path
37 def __init__(self, filename):
38 UserDict.UserDict.__init__(self)
40 filehandle = open(filename);
42 self.control = apt_inst.debExtractControl(filehandle)
44 # Make sure that file is always closed.
47 log.msg("Had problems reading: %s"%(filename))
49 for line in self.control.split('\n'):
50 if line.find(': ') != -1:
51 key, value = line.split(': ', 1)
52 self.data[key] = value
54 class PackageFileList(UserDict.DictMixin):
56 Manages a list of package files belonging to a backend
58 def __init__(self, backendName, cache_dir):
59 self.cache_dir = cache_dir
60 self.packagedb_dir = cache_dir+'/'+ aptpkg_dir + \
61 '/backends/' + backendName
62 if not os.path.exists(self.packagedb_dir):
63 os.makedirs(self.packagedb_dir)
68 if self.packages is None:
69 self.packages = shelve.open(self.packagedb_dir+'/packages.db')
72 if self.packages is not None:
75 def update_file(self, filename, cache_path, file_path):
77 Called from apt_proxy.py when files get updated so we can update our
78 fake lists/ directory and sources.list.
80 if filename=="Packages" or filename=="Release":
81 log.msg("Registering package file: "+cache_path)
82 self.packages[cache_path] = file_path
86 def check_files(self):
88 Check all files in the database to make sure it exists.
90 files = self.packages.keys()
91 #print self.packages.keys()
93 if not os.path.exists(self.packages[f]):
94 log.msg("File in packages database has been deleted: "+f)
97 def __getitem__(self, key): return self.packages[key]
98 def __setitem__(self, key, item): self.packages[key] = item
99 def __delitem__(self, key): del self.packages[key]
100 def keys(self): return self.packages.keys()
104 Uses AptPackagesServer to answer queries about packages.
106 Makes a fake configuration for python-apt for each backend.
108 DEFAULT_APT_CONFIG = {
110 #'APT::Architecture' : 'amd64', # TODO: Fix this, see bug #436011 and #285360
111 #'APT::Default-Release' : 'unstable',
114 'Dir::State' : 'apt/', # var/lib/apt/
115 'Dir::State::Lists': 'lists/', # lists/
116 #'Dir::State::cdroms' : 'cdroms.list',
117 'Dir::State::userstatus' : 'status.user',
118 'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
119 'Dir::Cache' : '.apt/cache/', # var/cache/apt/
120 #'Dir::Cache::archives' : 'archives/',
121 'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
122 'Dir::Cache::pkgcache' : 'pkgcache.bin',
123 'Dir::Etc' : 'apt/etc/', # etc/apt/
124 'Dir::Etc::sourcelist' : 'sources.list',
125 'Dir::Etc::vendorlist' : 'vendors.list',
126 'Dir::Etc::vendorparts' : 'vendors.list.d',
127 #'Dir::Etc::main' : 'apt.conf',
128 #'Dir::Etc::parts' : 'apt.conf.d',
129 #'Dir::Etc::preferences' : 'preferences',
131 #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
132 'Dir::Bin::dpkg' : '/usr/bin/dpkg',
134 #'DPkg::Pre-Install-Pkgs' : '',
136 #'DPkg::Tools::Options' : '',
137 #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
138 #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
139 #'DPkg::Post-Invoke' : '',
141 essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
143 essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
145 def __init__(self, backendName, cache_dir):
147 Construct new packages manager
148 backend: Name of backend associated with this packages file
149 cache_dir: cache directory from config file
151 self.backendName = backendName
152 self.cache_dir = cache_dir
153 self.apt_config = copy.deepcopy(self.DEFAULT_APT_CONFIG)
155 self.status_dir = (cache_dir+'/'+ aptpkg_dir
156 +'/backends/'+backendName)
157 for dir in self.essential_dirs:
158 path = self.status_dir+'/'+dir
159 if not os.path.exists(path):
161 for file in self.essential_files:
162 path = self.status_dir+'/'+file
163 if not os.path.exists(path):
168 self.apt_config['Dir'] = self.status_dir
169 self.apt_config['Dir::State::status'] = self.status_dir + '/apt/dpkg/status'
170 #os.system('find '+self.status_dir+' -ls ')
171 #print "status:"+self.apt_config['Dir::State::status']
172 self.packages = PackageFileList(backendName, cache_dir)
174 #print "Loaded aptPackages [%s] %s " % (self.backendName, self.cache_dir)
178 #print "start aptPackages [%s] %s " % (self.backendName, self.cache_dir)
179 self.packages.close()
180 #print "Deleted aptPackages [%s] %s " % (self.backendName, self.cache_dir)
181 def file_updated(self, filename, cache_path, file_path):
183 A file in the backend has changed. If this affects us, unload our apt database
185 if self.packages.update_file(filename, cache_path, file_path):
188 def __save_stdout(self):
189 self.real_stdout_fd = os.dup(1)
192 def __restore_stdout(self):
193 os.dup2(self.real_stdout_fd, 1)
194 os.close(self.real_stdout_fd)
195 del self.real_stdout_fd
199 Regenerates the fake configuration and load the packages server.
201 if self.loaded: return True
203 #print "Load:", self.status_dir
204 shutil.rmtree(self.status_dir+'/apt/lists/')
205 os.makedirs(self.status_dir+'/apt/lists/partial')
206 sources_filename = self.status_dir+'/'+'apt/etc/sources.list'
207 sources = open(sources_filename, 'w')
209 self.packages.check_files()
210 for f in self.packages:
211 # we should probably clear old entries from self.packages and
212 # take into account the recorded mtime as optimization
213 filepath = self.packages[f]
214 fake_uri='http://apt-dht/'+f
215 source_line='deb '+dirname(fake_uri)+'/ /'
216 listpath=(self.status_dir+'/apt/lists/'
217 +apt_pkg.URItoFileName(fake_uri))
218 sources.write(source_line+'\n')
219 log.msg("Sources line: " + source_line)
220 sources_count = sources_count + 1
223 #we should empty the directory instead
227 os.symlink(self.packages[f], listpath)
230 if sources_count == 0:
231 log.msg("No Packages files available for %s backend"%(self.backendName))
234 log.msg("Loading Packages database for "+self.status_dir)
235 #apt_pkg.Config = apt_pkg.newConfiguration(); #-- this causes unit tests to fail!
236 for key, value in self.apt_config.items():
237 apt_pkg.Config[key] = value
238 # print "apt_pkg config:"
239 # for I in apt_pkg.Config.keys():
240 # print "%s \"%s\";"%(I,apt_pkg.Config[I]);
242 # apt_pkg prints progress messages to stdout, disable
245 self.cache = apt_pkg.GetCache()
247 self.__restore_stdout()
249 self.records = apt_pkg.GetPkgRecords(self.cache)
250 #for p in self.cache.Packages:
252 #log.debug("%s packages found" % (len(self.cache)),'apt_pkg')
257 "Tries to make the packages server quit."
266 def get_mirror_path(self, name, version):
267 "Find the path for version 'version' of package 'name'"
268 if not self.load(): return None
270 for pack_vers in self.cache[name].VersionList:
271 if(pack_vers.VerStr == version):
272 file, index = pack_vers.FileList[0]
273 self.records.Lookup((file,index))
274 path = self.records.FileName
275 if len(path)>2 and path[0:2] == './':
276 path = path[2:] # Remove any leading './'
284 def get_mirror_versions(self, package_name):
286 Find the available versions of the package name given
287 @type package_name: string
288 @param package_name: package name to search for e.g. ;apt'
289 @return: A list of mirror versions available
293 if not self.load(): return vers
295 for pack_vers in self.cache[package_name].VersionList:
296 vers.append(pack_vers.VerStr)
302 def cleanup(factory):
303 for backend in factory.backends.values():
304 backend.get_packages_db().cleanup()
306 def get_mirror_path(factory, file):
308 Look for the path of 'file' in all backends.
310 info = AptDpkgInfo(file)
312 for backend in factory.backends.values():
313 path = backend.get_packages_db().get_mirror_path(info['Package'],
316 paths.append('/'+backend.base+'/'+path)
319 def get_mirror_versions(factory, package):
321 Look for the available version of a package in all backends, given
322 an existing package name
325 for backend in factory.backends.values():
326 vers = backend.get_packages_db().get_mirror_versions(package)
328 path = backend.get_packages_db().get_mirror_path(package, ver)
329 all_vers.append((ver, "%s/%s"%(backend.base,path)))
332 def closest_match(info, others):
334 return apt_pkg.VersionCompare(a[0], b[0])
337 version = info['Version']
339 for ver,path in others:
346 match = others[-1][1]
348 dirname=re.sub(r'/[^/]*$', '', match)
349 version=re.sub(r'^[^:]*:', '', info['Version'])
350 if dirname.find('/pool/') != -1:
351 return "/%s/%s_%s_%s.deb"%(dirname, info['Package'],
352 version, info['Architecture'])
354 return "/%s/%s_%s.deb"%(dirname, info['Package'], version)
356 def import_directory(factory, dir, recursive=0):
358 Import all files in a given directory into the cache
359 This is used by apt-proxy-import to import new files
364 if not os.path.exists(dir):
365 log.err('Directory ' + dir + ' does not exist')
369 log.msg("Importing packages from directory tree: " + dir)
370 for root, dirs, files in os.walk(dir):
372 imported_count += import_file(factory, root, file)
374 log.msg("Importing packages from directory: " + dir)
375 for file in os.listdir(dir):
376 mode = os.stat(dir + '/' + file)[stat.ST_MODE]
377 if not stat.S_ISDIR(mode):
378 imported_count += import_file(factory, dir, file)
380 for backend in factory.backends.values():
381 backend.get_packages_db().unload()
383 log.msg("Imported %s files" % (imported_count))
384 return imported_count
386 def import_file(factory, dir, file):
388 Import a .deb or .udeb into cache from given filename
390 if file[-4:]!='.deb' and file[-5:]!='.udeb':
391 log.msg("Ignoring (unknown file type):"+ file)
394 log.msg("considering: " + dir + '/' + file)
396 paths = get_mirror_path(factory, dir+'/'+file)
398 log.msg(file + ' skipped - wrong format or corrupted')
402 log.msg("WARNING: multiple ocurrences")
403 log.msg(str(paths), 'import')
404 cache_path = paths[0]
406 log.msg("Not found, trying to guess")
407 info = AptDpkgInfo(dir+'/'+file)
408 cache_path = closest_match(info,
409 get_mirror_versions(factory, info['Package']))
411 log.msg("MIRROR_PATH:"+ cache_path)
412 src_path = dir+'/'+file
413 dest_path = factory.config.cache_dir+cache_path
415 if not os.path.exists(dest_path):
416 log.msg("IMPORTING:" + src_path)
417 dest_path = re.sub(r'/\./', '/', dest_path)
418 if not os.path.exists(dirname(dest_path)):
419 os.makedirs(dirname(dest_path))
420 f = open(dest_path, 'w')
421 fcntl.lockf(f.fileno(), fcntl.LOCK_EX)
423 shutil.copy2(src_path, dest_path)
425 if hasattr(factory, 'access_times'):
426 atime = os.stat(src_path)[stat.ST_ATIME]
427 factory.access_times[cache_path] = atime
428 log.msg(file + ' imported')
431 log.msg(file + ' skipped - already in cache')
435 log.msg(file + ' skipped - no suitable backend found')
438 class TestAptPackages(unittest.TestCase):
439 """Unit tests for the AptPackages cache."""
444 a = AptPackages('whatever', '/tmp')
446 packagesFile = os.popen('ls -Sr /var/lib/apt/lists/ | tail -n 1').read().rstrip('\n')
447 for f in os.walk('/var/lib/apt/lists').next()[2]:
448 if f[-7:] == "Release" and packagesFile.startswith(f[:-7]):
452 a.file_updated('Release', releaseFile[releaseFile.find('_debian_')+1:].replace('_','/'), '/var/lib/apt/lists/' + releaseFile)
453 a.file_updated('Packages', packagesFile[packagesFile.find('_debian_')+1:].replace('_','/'), '/var/lib/apt/lists/' + packagesFile)
457 a.records.Lookup(a.cache['dpkg'].VersionList[0].FileList[0])
459 pkg_hash = os.popen('grep -A 30 -E "^Package: dpkg$" ' + '/var/lib/apt/lists/' + packagesFile + ' | grep -E "^SHA1:" | head -n 1 | cut -d\ -f 2').read().rstrip('\n')
461 self.failUnless(a.records.SHA1Hash == pkg_hash)
464 for p in self.pending_calls:
467 self.pending_calls = []