diff --git a/NEWS b/NEWS index 62dc486a..db7149c3 100644 --- a/NEWS +++ b/NEWS @@ -21,6 +21,7 @@ - added osc mv command which can rename file and leave them under version control - added new commands "dependson" and "whatdependson" to find out which packages get triggered before checkin/request accept. + - add new "osc build --linksource" option, speeds up esp. image building a lot - Incompatible changes: * osc se now prints Project Package, instead of Package Project for easier copy&paste. diff --git a/osc/build.py b/osc/build.py index e609484b..e9757fdc 100644 --- a/osc/build.py +++ b/osc/build.py @@ -73,7 +73,6 @@ class Buildinfo: raise urllib2.URLError('invalid protocol for the apiurl: \'%s\'' % apiurl) self.buildtype = buildtype - self.apiurl = apiurl # are we building .rpm or .deb? # XXX: shouldn't we deliver the type via the buildinfo? @@ -162,6 +161,7 @@ class Pac: self.filename = '%(name)s-%(version)s-%(release)s.%(arch)s.%(pacsuffix)s' % self.mp else: self.filename = '%(name)s-%(version)s.%(arch)s.%(pacsuffix)s' % self.mp + self.partname = '%s.part' % self.filename self.mp['filename'] = self.filename if self.mp['repopackage'] == '_repository': @@ -184,6 +184,7 @@ class Pac: # that the filename is suitable as identifier) self.localdir = '%s/%s/%s/%s' % (cachedir, self.project, self.repository, self.arch) self.fullfilename = os.path.join(self.localdir, self.filename) + self.fullpartname = os.path.join(self.localdir, self.partname) self.url_local = 'file://%s/' % self.fullfilename # first, add the local URL @@ -312,6 +313,8 @@ def main(opts, argv): if opts.ccache: buildargs.append('--ccache') xp.append('ccache') + if opts.linksources: + buildargs.append('--linksources') if opts.baselibs: buildargs.append('--baselibs') if opts.debuginfo: @@ -501,6 +504,7 @@ def main(opts, argv): # OBS 1.5 and before has no downloadurl defined in buildinfo if bi.downloadurl: urllist.append(bi.downloadurl + '/%(extproject)s/%(extrepository)s/%(arch)s/%(filename)s') + urllist.append( '%(apiurl)s/build/%(project)s/%(repository)s/%(repoarch)s/%(repopackage)s/%(repofilename)s' ) fetcher = Fetcher(cachedir = config['packagecachedir'], urllist = urllist, @@ -538,7 +542,10 @@ def main(opts, argv): if not os.path.exists(os.path.join(pradir)): os.makedirs(os.path.join(pradir)) if not os.path.exists(tffn): - os.symlink(sffn, tffn) + if opts.linksources: + os.link(sffn, tffn) + else: + os.symlink(sffn, tffn) if bi.pacsuffix == 'rpm': if config['build-type'] == "xen" or config['build-type'] == "kvm": diff --git a/osc/commandline.py b/osc/commandline.py index ba50c491..208f9ae8 100755 --- a/osc/commandline.py +++ b/osc/commandline.py @@ -2995,6 +2995,8 @@ Please submit there instead, or use --nodevelproject to force direct submission. 'in the spec file will invalidate this option.') @cmdln.option('--local-package', action='store_true', help='build a package which does not exist on the server') + @cmdln.option('--linksources', action='store_true', + help='use hard links instead of a deep copied source') @cmdln.option('--alternative-project', metavar='PROJECT', help='specify the build target project') @cmdln.option('-d', '--debuginfo', action='store_true', diff --git a/osc/fetch.py b/osc/fetch.py index a1b8b02b..fe3c86fa 100644 --- a/osc/fetch.py +++ b/osc/fetch.py @@ -7,9 +7,7 @@ import sys, os import urllib2 from urlgrabber.grabber import URLGrabber, URLGrabError from urlgrabber.mirror import MirrorGroup -from core import makeurl -from util import packagequery, cpio -import tempfile +from util import packagequery try: from meter import TextMeter except: @@ -40,7 +38,6 @@ class Fetcher: self.urllist = urllist self.http_debug = http_debug self.offline = offline - self.cpio = {} passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm() for host in api_host_options.keys(): @@ -82,31 +79,18 @@ class Fetcher: print '\n'.join(pac.urllist) print - (fd, tmpfile) = tempfile.mkstemp(prefix='osc_build') try: - try: - # it returns the filename - ret = mg.urlgrab(pac.filename, - filename = tmpfile, - text = '%s(%s) %s' %(prefix, pac.project, pac.filename)) - self.move_package(tmpfile, pac.localdir, pac) - except URLGrabError, e: - if e.errno == 256: - self.cpio.setdefault(pac.project, {})[pac.name] = pac - return - print - print >>sys.stderr, 'Error:', e.strerror - print >>sys.stderr, 'Failed to retrieve %s from the following locations (in order):' % pac.filename - print >>sys.stderr, '\n'.join(pac.urllist) - sys.exit(1) - finally: - os.close(fd) - if os.path.exists(tmpfile): - os.unlink(tmpfile) + # it returns the filename + ret = mg.urlgrab(pac.filename, + filename = pac.fullpartname, + text = '%s(%s) %s' %(prefix, pac.project, pac.filename)) + except URLGrabError, e: + print >>sys.stderr, 'Error:', e.strerror + print >>sys.stderr, 'Failed to retrieve %s from the following locations (in order):' % pac.filename + print >>sys.stderr, '\n'.join(pac.urllist) + sys.exit(1) - def move_package(self, tmpfile, destdir, pac_obj = None): - import shutil - pkgq = packagequery.PackageQuery.query(tmpfile) + pkgq = packagequery.PackageQuery.query(pac.fullpartname) arch = pkgq.arch() # SOURCERPM = 1044 if pkgq.filename_suffix == 'rpm' and not pkgq.getTag(1044): @@ -119,11 +103,10 @@ class Fetcher: canonname = '%s-%s-%s.%s.%s' % (pkgq.name(), pkgq.version(), pkgq.release(), arch, pkgq.filename_suffix) else: canonname = '%s-%s.%s.%s' % (pkgq.name(), pkgq.version(), arch, pkgq.filename_suffix) - fullfilename = os.path.join(destdir, canonname) - if pac_obj is not None: - pac_obj.filename = canonname - pac_obj.fullfilename = fullfilename - shutil.move(tmpfile, fullfilename) + pac.filename = canonname + pac.fullfilename = os.path.join(pac.localdir, canonname) + + os.rename(pac.fullpartname, pac.fullfilename) def dirSetup(self, pac): dir = os.path.join(self.cachedir, pac.localdir) @@ -137,7 +120,6 @@ class Fetcher: def run(self, buildinfo): - from urllib import quote_plus cached = 0 all = len(buildinfo.deps) for i in buildinfo.deps: @@ -166,34 +148,13 @@ class Fetcher: except KeyboardInterrupt: print 'Cancelled by user (ctrl-c)' print 'Exiting.' + if os.path.exists(i.fullpartname): + print 'Cleaning up incomplete file', i.fullpartname + os.unlink(i.fullpartname) sys.exit(0) done += 1 - for project, pkgs in self.cpio.iteritems(): - repo = pkgs.values()[0].repository - query = [ 'binary=%s' % quote_plus(i) for i in pkgs.keys() ] - query.append('view=cpio') - try: - (fd, tmparchive) = tempfile.mkstemp(prefix='osc_build_cpio') - (fd, tmpfile) = tempfile.mkstemp(prefix='osc_build') - url = makeurl(buildinfo.apiurl, - ['public/build', project, repo, buildinfo.buildarch, '_repository'], - query=query) - self.gr.urlgrab(url, filename = tmparchive, text = 'fetching cpio for \'%s\'' % project) - archive = cpio.CpioRead(tmparchive) - archive.read() - for hdr in archive: - if hdr.filename == '.errors': - import oscerr - archive.copyin_file(hdr.filename) - raise oscerr.APIError('CPIO archive is incomplete (see .errors file)') - pac = pkgs[hdr.filename.rsplit('.', 1)[0]] - archive.copyin_file(hdr.filename, os.path.dirname(tmpfile), os.path.basename(tmpfile)) - self.move_package(tmpfile, pac.localdir, pac) - finally: - if os.path.exists(tmparchive): - os.unlink(tmparchive) - if os.path.exists(tmpfile): - os.unlink(tmpfile) + + def verify_pacs(pac_list): """Take a list of rpm filenames and run rpm -K on them.