# Copyright (C) 2006 Novell Inc. All rights reserved. # This program is free software; it may be used, copied, modified # and distributed under the terms of the GNU General Public Licence, # either version 2, or (at your option) any later version. import sys, os import urllib2 from urlgrabber.grabber import URLGrabber, URLGrabError from urlgrabber.mirror import MirrorGroup from util import packagequery try: from meter import TextMeter except: TextMeter = None def join_url(self, base_url, rel_url): """to override _join_url of MirrorGroup, because we want to pass full URLs instead of base URL where relative_url is added later... IOW, we make MirrorGroup ignore relative_url""" return base_url class Fetcher: def __init__(self, cachedir = '/tmp', api_host_options = {}, urllist = [], http_debug = False, cookiejar = None, offline = False): __version__ = '0.1' __user_agent__ = 'osbuild/%s' % __version__ # set up progress bar callback if sys.stdout.isatty() and TextMeter: self.progress_obj = TextMeter(fo=sys.stdout) else: self.progress_obj = None self.cachedir = cachedir self.urllist = urllist self.http_debug = http_debug self.offline = offline passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm() for host in api_host_options.keys(): passmgr.add_password(None, host, api_host_options[host]['user'], api_host_options[host]['pass']) openers = (urllib2.HTTPBasicAuthHandler(passmgr), ) if cookiejar: openers += (urllib2.HTTPCookieProcessor(cookiejar), ) self.gr = URLGrabber(user_agent=__user_agent__, keepalive=1, opener = urllib2.build_opener(*openers), progress_obj=self.progress_obj, failure_callback=(self.failureReport,(),{}), ) def failureReport(self, errobj): """failure output for failovers from urlgrabber""" #log(0, '%s: %s' % (errobj.url, str(errobj.exception))) #log(0, 'Trying other mirror.') print 'Trying openSUSE Build Service server for %s (%s), since it is not on %s.' \ % (self.curpac, self.curpac.project, errobj.url.split('/')[2]) raise errobj.exception def fetch(self, pac, prefix=''): # for use by the failure callback self.curpac = pac if self.offline: return True MirrorGroup._join_url = join_url mg = MirrorGroup(self.gr, pac.urllist) if self.http_debug: print print 'URLs to try for package \'%s\':' % pac print '\n'.join(pac.urllist) print try: # it returns the filename ret = mg.urlgrab(pac.filename, filename = pac.fullpartname, text = '%s(%s) %s' %(prefix, pac.project, pac.filename)) except URLGrabError, e: print print >>sys.stderr, 'Error:', e.strerror print >>sys.stderr, 'Failed to retrieve %s from the following locations (in order):' % pac.filename print >>sys.stderr, '\n'.join(pac.urllist) sys.exit(1) pkgq = packagequery.PackageQuery.query(pac.fullpartname) arch = pkgq.arch() # SOURCERPM = 1044 if pkgq.filename_suffix == 'rpm' and not pkgq.getTag(1044): # NOSOURCE = 1051, NOPATCH = 1052 if pkgq.getTag(1051) or pkgq.getTag(1052): arch = "nosrc" else: arch = "src" if pkgq.release(): canonname = '%s-%s-%s.%s.%s' % (pkgq.name(), pkgq.version(), pkgq.release(), arch, pkgq.filename_suffix) else: canonname = '%s-%s.%s.%s' % (pkgq.name(), pkgq.version(), arch, pkgq.filename_suffix) pac.filename = canonname pac.fullfilename = os.path.join(pac.localdir, canonname) os.rename(pac.fullpartname, pac.fullfilename) def dirSetup(self, pac): dir = os.path.join(self.cachedir, pac.localdir) if not os.path.exists(dir): try: os.makedirs(dir, mode=0755) except OSError, e: print >>sys.stderr, 'packagecachedir is not writable for you?' print >>sys.stderr, e sys.exit(1) def run(self, buildinfo): all = 0 cached = 0 for i in buildinfo.deps: i.makeurls(self.cachedir, self.urllist) all += 1 if os.path.exists(i.fullfilename): cached += 1 miss = 0 if all: miss = 100.0*(all-cached)/all print "%.1f%% cache miss. %d/%d dependencies cached.\n" % (miss,cached,all) needed = all-cached done = 1 for i in buildinfo.deps: i.makeurls(self.cachedir, self.urllist) if not os.path.exists(i.fullfilename): self.dirSetup(i) try: # if there isn't a progress bar, there is no output at all if not self.progress_obj: print '%d/%d (%s) %s' % (done, needed, i.project, i.filename) self.fetch(i, "%d/%d " % (done,needed)) except KeyboardInterrupt: print 'Cancelled by user (ctrl-c)' print 'Exiting.' if os.path.exists(i.fullpartname): print 'Cleaning up incomplete file', i.fullpartname os.unlink(i.fullpartname) sys.exit(0) done += 1 def verify_pacs(pac_list): """Take a list of rpm filenames and run rpm -K on them. In case of failure, exit. Check all packages in one go, since this takes only 6 seconds on my Athlon 700 instead of 20 when calling 'rpm -K' for each of them. """ import subprocess if not pac_list: return # don't care about the return value because we check the # output anyway, and rpm always writes to stdout. # save locale first (we rely on English rpm output here) saved_LC_ALL = os.environ.get('LC_ALL') os.environ['LC_ALL'] = 'en_EN' o = subprocess.Popen(['rpm', '-K'] + pac_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True).stdout # restore locale if saved_LC_ALL: os.environ['LC_ALL'] = saved_LC_ALL else: os.environ.pop('LC_ALL') for line in o.readlines(): if not 'OK' in line: print print >>sys.stderr, 'The following package could not be verified:' print >>sys.stderr, line sys.exit(1) if 'NOT OK' in line: print print >>sys.stderr, 'The following package could not be verified:' print >>sys.stderr, line if 'MISSING KEYS' in line: missing_key = line.split('#')[-1].split(')')[0] print >>sys.stderr, """ - If the key is missing, install it first. For example, do the following: gpg --keyserver pgp.mit.edu --recv-keys %(name)s gpg --armor --export %(name)s > %(dir)s/keyfile-%(name)s and, as root: rpm --import %(dir)s/keyfile-%(name)s Then, just start the build again. - If you do not trust the packages, you should configure osc build for XEN or KVM - You may use --no-verify to skip the verification (which is a risk for your system). """ % {'name': missing_key, 'dir': os.path.expanduser('~')} else: print >>sys.stderr, """ - If the signature is wrong, you may try deleting the package manually and re-run this program, so it is fetched again. """ sys.exit(1)