2 # Copyright (C) 2002 Manuel Estrada Sainz <ranty@debian.org>
4 # This library is free software; you can redistribute it and/or
5 # modify it under the terms of version 2.1 of the GNU Lesser General Public
6 # License as published by the Free Software Foundation.
8 # This library is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 # Lesser General Public License for more details.
13 # You should have received a copy of the GNU Lesser General Public
14 # License along with this library; if not, write to the Free Software
15 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 import apt_pkg, apt_inst, sys, os, stat
18 from os.path import dirname, basename
19 import re, shelve, shutil, fcntl
20 from twisted.internet import process
26 class AptDpkgInfo(UserDict.UserDict):
28 Gets control fields from a .deb file.
30 And then behaves like a regular python dictionary.
32 See AptPackages.get_mirror_path
35 def __init__(self, filename):
36 UserDict.UserDict.__init__(self)
38 filehandle = open(filename);
40 self.control = apt_inst.debExtractControl(filehandle)
42 # Make sure that file is always closed.
45 log.debug("Had problems reading: %s"%(filename), 'AptDpkgInfo')
47 for line in self.control.split('\n'):
48 if line.find(': ') != -1:
49 key, value = line.split(': ', 1)
50 self.data[key] = value
52 class PackageFileList:
54 Manages a list of package files belonging to a backend
56 def __init__(self, backendName, cache_dir):
57 self.cache_dir = cache_dir
58 self.packagedb_dir = cache_dir+'/'+ aptpkg_dir + \
59 '/backends/' + backendName
60 if not os.path.exists(self.packagedb_dir):
61 os.makedirs(self.packagedb_dir)
66 if self.packages is None:
67 self.packages = shelve.open(self.packagedb_dir+'/packages.db')
69 if self.packages is not None:
72 def update_file(self, entry):
74 Called from apt_proxy.py when files get updated so we can update our
75 fake lists/ directory and sources.list.
77 @param entry CacheEntry for cached file
79 if entry.filename=="Packages" or entry.filename=="Release":
80 log.msg("Registering package file: "+entry.cache_path, 'apt_pkg', 4)
81 stat_result = os.stat(entry.file_path)
82 self.packages[entry.cache_path] = stat_result
86 Get list of files in database. Each file will be checked that it exists
88 files = self.packages.keys()
89 #print self.packages.keys()
91 if not os.path.exists(self.cache_dir + os.sep + f):
92 log.debug("File in packages database has been deleted: "+f, 'apt_pkg')
93 del files[files.index(f)]
99 Uses AptPackagesServer to answer queries about packages.
101 Makes a fake configuration for python-apt for each backend.
103 DEFAULT_APT_CONFIG = {
105 'APT::Architecture' : 'i386', # TODO: Fix this, see bug #436011 and #285360
106 #'APT::Default-Release' : 'unstable',
109 'Dir::State' : 'apt/', # var/lib/apt/
110 'Dir::State::Lists': 'lists/', # lists/
111 #'Dir::State::cdroms' : 'cdroms.list',
112 'Dir::State::userstatus' : 'status.user',
113 'Dir::State::status': 'dpkg/status', # '/var/lib/dpkg/status'
114 'Dir::Cache' : '.apt/cache/', # var/cache/apt/
115 #'Dir::Cache::archives' : 'archives/',
116 'Dir::Cache::srcpkgcache' : 'srcpkgcache.bin',
117 'Dir::Cache::pkgcache' : 'pkgcache.bin',
118 'Dir::Etc' : 'apt/etc/', # etc/apt/
119 'Dir::Etc::sourcelist' : 'sources.list',
120 'Dir::Etc::vendorlist' : 'vendors.list',
121 'Dir::Etc::vendorparts' : 'vendors.list.d',
122 #'Dir::Etc::main' : 'apt.conf',
123 #'Dir::Etc::parts' : 'apt.conf.d',
124 #'Dir::Etc::preferences' : 'preferences',
126 #'Dir::Bin::methods' : '', #'/usr/lib/apt/methods'
127 'Dir::Bin::dpkg' : '/usr/bin/dpkg',
129 #'DPkg::Pre-Install-Pkgs' : '',
131 #'DPkg::Tools::Options' : '',
132 #'DPkg::Tools::Options::/usr/bin/apt-listchanges' : '',
133 #'DPkg::Tools::Options::/usr/bin/apt-listchanges::Version' : '2',
134 #'DPkg::Post-Invoke' : '',
136 essential_dirs = ('apt', 'apt/cache', 'apt/dpkg', 'apt/etc', 'apt/lists',
138 essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
140 def __init__(self, backendName, cache_dir):
142 Construct new packages manager
143 backend: Name of backend associated with this packages file
144 cache_dir: cache directory from config file
146 self.backendName = backendName
147 self.cache_dir = cache_dir
148 self.apt_config = copy.deepcopy(self.DEFAULT_APT_CONFIG)
150 self.status_dir = (cache_dir+'/'+ aptpkg_dir
151 +'/backends/'+backendName)
152 for dir in self.essential_dirs:
153 path = self.status_dir+'/'+dir
154 if not os.path.exists(path):
156 for file in self.essential_files:
157 path = self.status_dir+'/'+file
158 if not os.path.exists(path):
163 self.apt_config['Dir'] = self.status_dir
164 self.apt_config['Dir::State::status'] = self.status_dir + '/apt/dpkg/status'
165 #os.system('find '+self.status_dir+' -ls ')
166 #print "status:"+self.apt_config['Dir::State::status']
167 self.packages = PackageFileList(backendName, cache_dir)
169 #print "Loaded aptPackages [%s] %s " % (self.backendName, self.cache_dir)
173 #print "start aptPackages [%s] %s " % (self.backendName, self.cache_dir)
174 self.packages.close()
175 #print "Deleted aptPackages [%s] %s " % (self.backendName, self.cache_dir)
176 def file_updated(self, entry):
178 A file in the backend has changed. If this affects us, unload our apt database
180 if self.packages.update_file(entry):
183 def __save_stdout(self):
184 self.real_stdout_fd = os.dup(1)
187 def __restore_stdout(self):
188 os.dup2(self.real_stdout_fd, 1)
189 os.close(self.real_stdout_fd)
190 del self.real_stdout_fd
194 Regenerates the fake configuration and load the packages server.
196 if self.loaded: return True
198 #print "Load:", self.status_dir
199 shutil.rmtree(self.status_dir+'/apt/lists/')
200 os.makedirs(self.status_dir+'/apt/lists/partial')
201 sources_filename = self.status_dir+'/'+'apt/etc/sources.list'
202 sources = open(sources_filename, 'w')
204 for file in self.packages.get_files():
205 # we should probably clear old entries from self.packages and
206 # take into account the recorded mtime as optimization
207 filepath = self.cache_dir + file
208 fake_uri='http://apt-dht/'+file
209 source_line='deb '+dirname(fake_uri)+'/ /'
210 listpath=(self.status_dir+'/apt/lists/'
211 +apt_pkg.URItoFileName(fake_uri))
212 sources.write(source_line+'\n')
213 log.debug("Sources line: " + source_line, 'apt_pkg')
214 sources_count = sources_count + 1
217 #we should empty the directory instead
221 os.symlink('../../../../../'+file, listpath)
224 if sources_count == 0:
225 log.msg("No Packages files available for %s backend"%(self.backendName), 'apt_pkg')
228 log.msg("Loading Packages database for "+self.status_dir,'apt_pkg')
229 #apt_pkg.Config = apt_pkg.newConfiguration(); #-- this causes unit tests to fail!
230 for key, value in self.apt_config.items():
231 apt_pkg.Config[key] = value
232 # print "apt_pkg config:"
233 # for I in apt_pkg.Config.keys():
234 # print "%s \"%s\";"%(I,apt_pkg.Config[I]);
236 if log.isEnabled('apt'):
237 self.cache = apt_pkg.GetCache()
239 # apt_pkg prints progress messages to stdout, disable
242 self.cache = apt_pkg.GetCache()
244 self.__restore_stdout()
246 self.records = apt_pkg.GetPkgRecords(self.cache)
247 #for p in self.cache.Packages:
249 #log.debug("%s packages found" % (len(self.cache)),'apt_pkg')
254 "Tries to make the packages server quit."
263 def get_mirror_path(self, name, version):
264 "Find the path for version 'version' of package 'name'"
265 if not self.load(): return None
267 for pack_vers in self.cache[name].VersionList:
268 if(pack_vers.VerStr == version):
269 file, index = pack_vers.FileList[0]
270 self.records.Lookup((file,index))
271 path = self.records.FileName
272 if len(path)>2 and path[0:2] == './':
273 path = path[2:] # Remove any leading './'
281 def get_mirror_versions(self, package_name):
283 Find the available versions of the package name given
284 @type package_name: string
285 @param package_name: package name to search for e.g. ;apt'
286 @return: A list of mirror versions available
290 if not self.load(): return vers
292 for pack_vers in self.cache[package_name].VersionList:
293 vers.append(pack_vers.VerStr)
299 def cleanup(factory):
300 for backend in factory.backends.values():
301 backend.get_packages_db().cleanup()
303 def get_mirror_path(factory, file):
305 Look for the path of 'file' in all backends.
307 info = AptDpkgInfo(file)
309 for backend in factory.backends.values():
310 path = backend.get_packages_db().get_mirror_path(info['Package'],
313 paths.append('/'+backend.base+'/'+path)
316 def get_mirror_versions(factory, package):
318 Look for the available version of a package in all backends, given
319 an existing package name
322 for backend in factory.backends.values():
323 vers = backend.get_packages_db().get_mirror_versions(package)
325 path = backend.get_packages_db().get_mirror_path(package, ver)
326 all_vers.append((ver, "%s/%s"%(backend.base,path)))
329 def closest_match(info, others):
331 return apt_pkg.VersionCompare(a[0], b[0])
334 version = info['Version']
336 for ver,path in others:
343 match = others[-1][1]
345 dirname=re.sub(r'/[^/]*$', '', match)
346 version=re.sub(r'^[^:]*:', '', info['Version'])
347 if dirname.find('/pool/') != -1:
348 return "/%s/%s_%s_%s.deb"%(dirname, info['Package'],
349 version, info['Architecture'])
351 return "/%s/%s_%s.deb"%(dirname, info['Package'], version)
353 def import_directory(factory, dir, recursive=0):
355 Import all files in a given directory into the cache
356 This is used by apt-proxy-import to import new files
361 if not os.path.exists(dir):
362 log.err('Directory ' + dir + ' does not exist', 'import')
366 log.msg("Importing packages from directory tree: " + dir, 'import',3)
367 for root, dirs, files in os.walk(dir):
369 imported_count += import_file(factory, root, file)
371 log.debug("Importing packages from directory: " + dir, 'import',3)
372 for file in os.listdir(dir):
373 mode = os.stat(dir + '/' + file)[stat.ST_MODE]
374 if not stat.S_ISDIR(mode):
375 imported_count += import_file(factory, dir, file)
377 for backend in factory.backends.values():
378 backend.get_packages_db().unload()
380 log.msg("Imported %s files" % (imported_count))
381 return imported_count
383 def import_file(factory, dir, file):
385 Import a .deb or .udeb into cache from given filename
387 if file[-4:]!='.deb' and file[-5:]!='.udeb':
388 log.msg("Ignoring (unknown file type):"+ file, 'import')
391 log.debug("considering: " + dir + '/' + file, 'import')
393 paths = get_mirror_path(factory, dir+'/'+file)
395 log.msg(file + ' skipped - wrong format or corrupted', 'import')
399 log.debug("WARNING: multiple ocurrences", 'import')
400 log.debug(str(paths), 'import')
401 cache_path = paths[0]
403 log.debug("Not found, trying to guess", 'import')
404 info = AptDpkgInfo(dir+'/'+file)
405 cache_path = closest_match(info,
406 get_mirror_versions(factory, info['Package']))
408 log.debug("MIRROR_PATH:"+ cache_path, 'import')
409 src_path = dir+'/'+file
410 dest_path = factory.config.cache_dir+cache_path
412 if not os.path.exists(dest_path):
413 log.debug("IMPORTING:" + src_path, 'import')
414 dest_path = re.sub(r'/\./', '/', dest_path)
415 if not os.path.exists(dirname(dest_path)):
416 os.makedirs(dirname(dest_path))
417 f = open(dest_path, 'w')
418 fcntl.lockf(f.fileno(), fcntl.LOCK_EX)
420 shutil.copy2(src_path, dest_path)
422 if hasattr(factory, 'access_times'):
423 atime = os.stat(src_path)[stat.ST_ATIME]
424 factory.access_times[cache_path] = atime
425 log.msg(file + ' imported', 'import')
428 log.msg(file + ' skipped - already in cache', 'import')
432 log.msg(file + ' skipped - no suitable backend found', 'import')
435 def test(factory, file):
436 "Just for testing purposes, this should probably go to hell soon."
437 for backend in factory.backends:
438 backend.get_packages_db().load()
440 info = AptDpkgInfo(file)
441 path = get_mirror_path(factory, file)
443 print "\t%s:%s"%(info['Version'], path)
445 vers = get_mirror_versions(factory, info['Package'])
446 print "Other Versions:"
448 print "\t%s:%s"%(ver)
450 print "\t%s:%s"%(info['Version'], closest_match(info, vers))
452 if __name__ == '__main__':
453 from apt_proxy_conf import factoryConfig
455 def debug(self, msg):
457 factory = DummyFactory()
458 factoryConfig(factory)
460 '/home/ranty/work/apt-proxy/related/tools/galeon_1.2.5-1_i386.deb')
462 '/storage/apt-proxy/debian/dists/potato/main/binary-i386/base/'
463 +'libstdc++2.10_2.95.2-13.deb')