- fixed #477690 ("osc fetching binaries really slow"). "osc build" does only one request (per project) to the API and fetches all binaries in a cpio archive (in case the binaries weren't found on d.o.o) instead of doing one request for each package.

- fetch.py: get rid of the "partname concept" for fetching binaries and use tempfiles instead (the former concept can lead to unexpected errors in some cases)

osc/build.py

@@ -73,6 +73,7 @@ class Buildinfo:
             raise urllib2.URLError('invalid protocol for the apiurl: \'%s\'' % apiurl)
 
         self.buildtype = buildtype
+        self.apiurl = apiurl
 
         # are we building .rpm or .deb?
         # XXX: shouldn't we deliver the type via the buildinfo?
@@ -160,7 +161,6 @@ class Pac:
             self.filename = '%(name)s-%(version)s-%(release)s.%(arch)s.%(pacsuffix)s' % self.mp
         else:
             self.filename = '%(name)s-%(version)s.%(arch)s.%(pacsuffix)s' % self.mp
-        self.partname = '%s.part' % self.filename
 
         self.mp['filename'] = self.filename
         if self.mp['repopackage'] == '_repository':
@@ -183,7 +183,6 @@ class Pac:
         # that the filename is suitable as identifier)
         self.localdir = '%s/%s/%s/%s' % (cachedir, self.project, self.repository, self.arch)
         self.fullfilename = os.path.join(self.localdir, self.filename)
-        self.fullpartname = os.path.join(self.localdir, self.partname)
         self.url_local = 'file://%s/' % self.fullfilename
 
         # first, add the local URL
@@ -457,7 +456,6 @@ def main(opts, argv):
     # OBS 1.5 and before has no downloadurl defined in buildinfo
     if bi.downloadurl:
         urllist.append(bi.downloadurl + '/%(extproject)s/%(extrepository)s/%(arch)s/%(filename)s')
-    urllist.append( '%(apiurl)s/build/%(project)s/%(repository)s/%(repoarch)s/%(repopackage)s/%(repofilename)s' )
 
     fetcher = Fetcher(cachedir = config['packagecachedir'],
                       urllist = urllist,

osc/fetch.py

@@ -7,7 +7,9 @@ import sys, os
 import urllib2
 from urlgrabber.grabber import URLGrabber, URLGrabError
 from urlgrabber.mirror import MirrorGroup
-from util import packagequery
+from core import makeurl
+from util import packagequery, cpio
+import tempfile
 try:
     from meter import TextMeter
 except:
@@ -38,6 +40,7 @@ class Fetcher:
         self.urllist = urllist
         self.http_debug = http_debug
         self.offline = offline
+        self.cpio = {}
 
         passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
         for host in api_host_options.keys():
@@ -79,21 +82,29 @@ class Fetcher:
             print '\n'.join(pac.urllist)
             print
 
+        (fd, tmpfile) = tempfile.mkstemp(prefix='osc_build')
         try:
             # it returns the filename
             ret = mg.urlgrab(pac.filename,
-                             filename = pac.fullpartname,
+                             filename = tmpfile,
                              text = '%s(%s) %s' %(prefix, pac.project, pac.filename))
-
+            self.move_package(tmpfile, pac.localdir, pac)
         except URLGrabError, e:
+            if e.errno == 256:
+                self.cpio.setdefault(pac.project, {})[pac.name] = pac
+                return
             print
             print >>sys.stderr, 'Error:', e.strerror
             print >>sys.stderr, 'Failed to retrieve %s from the following locations (in order):' % pac.filename
             print >>sys.stderr, '\n'.join(pac.urllist)
-
             sys.exit(1)
+        finally:
+            if os.path.exists(tmpfile):
+                os.unlink(tmpfile)
 
-        pkgq = packagequery.PackageQuery.query(pac.fullpartname)
+    def move_package(self, tmpfile, destdir, pac_obj = None):
+        import shutil
+        pkgq = packagequery.PackageQuery.query(tmpfile)
         arch = pkgq.arch()
         # SOURCERPM = 1044
         if pkgq.filename_suffix == 'rpm' and not pkgq.getTag(1044):
@@ -106,10 +117,11 @@ class Fetcher:
             canonname = '%s-%s-%s.%s.%s' % (pkgq.name(), pkgq.version(), pkgq.release(), arch, pkgq.filename_suffix)
         else:
             canonname = '%s-%s.%s.%s' % (pkgq.name(), pkgq.version(), arch, pkgq.filename_suffix)
-        pac.filename = canonname
-        pac.fullfilename = os.path.join(pac.localdir, canonname)
-
-        os.rename(pac.fullpartname, pac.fullfilename)
+        fullfilename = os.path.join(destdir, canonname)
+        if pac_obj is not None:
+            pac_obj.filename = canonname
+            pac_obj.fullfilename = fullfilename
+        shutil.move(tmpfile, fullfilename)
 
     def dirSetup(self, pac):
         dir = os.path.join(self.cachedir, pac.localdir)
@@ -123,6 +135,7 @@ class Fetcher:
 
 
     def run(self, buildinfo):
+        from urllib import quote_plus
         cached = 0
         all = len(buildinfo.deps)
         for i in buildinfo.deps:
@@ -147,13 +160,34 @@ class Fetcher:
                 except KeyboardInterrupt:
                     print 'Cancelled by user (ctrl-c)'
                     print 'Exiting.'
-                    if os.path.exists(i.fullpartname):
-                        print 'Cleaning up incomplete file', i.fullpartname
-                        os.unlink(i.fullpartname)
                     sys.exit(0)
             done += 1
-
-
+        for project, pkgs in self.cpio.iteritems():
+            repo = pkgs.values()[0].repository
+            query = [ 'binary=%s' % quote_plus(i) for i in pkgs.keys() ]
+            query.append('view=cpio')
+            try:
+                (fd, tmparchive) = tempfile.mkstemp(prefix='osc_build_cpio')
+                (fd, tmpfile) = tempfile.mkstemp(prefix='osc_build')
+                url = makeurl(buildinfo.apiurl,
+                              ['public/build', project, repo, buildinfo.buildarch, '_repository'],
+                              query=query)
+                self.gr.urlgrab(url, filename = tmparchive, text = 'fetching cpio for \'%s\'' % project)
+                archive = cpio.CpioRead(tmparchive)
+                archive.read()
+                for hdr in archive:
+                    if hdr.filename == '.errors':
+                        import oscerr
+                        archive.copyin_file(hdr.filename)
+                        raise oscerr.APIError('CPIO archive is incomplete (see .errors file)')
+                    pac = pkgs[hdr.filename.rsplit('.', 1)[0]]
+                    archive.copyin_file(hdr.filename, os.path.dirname(tmpfile), os.path.basename(tmpfile))
+                    self.move_package(tmpfile, pac.localdir, pac)
+            finally:
+                if os.path.exists(tmparchive):
+                    os.unlink(tmparchive)
+                if os.path.exists(tmpfile):
+                    os.unlink(tmpfile)
 
 def verify_pacs(pac_list):
     """Take a list of rpm filenames and run rpm -K on them.