* Revert fetch-cpio support for now, too many complains due to broken

product building atm.
* add --link-source option for "osc build" to speed up product building

NEWS

@@ -21,6 +21,7 @@
  - added osc mv command which can rename file and leave them under version control
  - added new commands "dependson" and "whatdependson" to find out which packages get
    triggered before checkin/request accept.
+ - add new "osc build --linksource" option, speeds up esp. image building a lot
  - Incompatible changes:
    * osc se now prints Project Package, instead of Package Project
      for easier copy&paste.

osc/build.py

@@ -73,7 +73,6 @@ class Buildinfo:
             raise urllib2.URLError('invalid protocol for the apiurl: \'%s\'' % apiurl)
 
         self.buildtype = buildtype
-        self.apiurl = apiurl
 
         # are we building .rpm or .deb?
         # XXX: shouldn't we deliver the type via the buildinfo?
@@ -162,6 +161,7 @@ class Pac:
             self.filename = '%(name)s-%(version)s-%(release)s.%(arch)s.%(pacsuffix)s' % self.mp
         else:
             self.filename = '%(name)s-%(version)s.%(arch)s.%(pacsuffix)s' % self.mp
+        self.partname = '%s.part' % self.filename
 
         self.mp['filename'] = self.filename
         if self.mp['repopackage'] == '_repository':
@@ -184,6 +184,7 @@ class Pac:
         # that the filename is suitable as identifier)
         self.localdir = '%s/%s/%s/%s' % (cachedir, self.project, self.repository, self.arch)
         self.fullfilename = os.path.join(self.localdir, self.filename)
+        self.fullpartname = os.path.join(self.localdir, self.partname)
         self.url_local = 'file://%s/' % self.fullfilename
 
         # first, add the local URL
@@ -312,6 +313,8 @@ def main(opts, argv):
     if opts.ccache:
         buildargs.append('--ccache')
         xp.append('ccache')
+    if opts.linksources:
+        buildargs.append('--linksources')
     if opts.baselibs:
         buildargs.append('--baselibs')
     if opts.debuginfo:
@@ -501,6 +504,7 @@ def main(opts, argv):
     # OBS 1.5 and before has no downloadurl defined in buildinfo
     if bi.downloadurl:
         urllist.append(bi.downloadurl + '/%(extproject)s/%(extrepository)s/%(arch)s/%(filename)s')
+    urllist.append( '%(apiurl)s/build/%(project)s/%(repository)s/%(repoarch)s/%(repopackage)s/%(repofilename)s' )
 
     fetcher = Fetcher(cachedir = config['packagecachedir'],
                       urllist = urllist,
@@ -538,7 +542,10 @@ def main(opts, argv):
             if not os.path.exists(os.path.join(pradir)):
                 os.makedirs(os.path.join(pradir))
             if not os.path.exists(tffn):
-                os.symlink(sffn, tffn)
+                if opts.linksources:
+                   os.link(sffn, tffn)
+                else:
+                   os.symlink(sffn, tffn)
 
     if bi.pacsuffix == 'rpm':
         if config['build-type'] == "xen" or config['build-type'] == "kvm":

osc/commandline.py

@@ -2995,6 +2995,8 @@ Please submit there instead, or use --nodevelproject to force direct submission.
                   'in the spec file will invalidate this option.')
     @cmdln.option('--local-package', action='store_true',
                   help='build a package which does not exist on the server')
+    @cmdln.option('--linksources', action='store_true',
+                  help='use hard links instead of a deep copied source')
     @cmdln.option('--alternative-project', metavar='PROJECT',
                   help='specify the build target project')
     @cmdln.option('-d', '--debuginfo', action='store_true',

osc/fetch.py

@@ -7,9 +7,7 @@ import sys, os
 import urllib2
 from urlgrabber.grabber import URLGrabber, URLGrabError
 from urlgrabber.mirror import MirrorGroup
-from core import makeurl
-from util import packagequery, cpio
-import tempfile
+from util import packagequery
 try:
     from meter import TextMeter
 except:
@@ -40,7 +38,6 @@ class Fetcher:
         self.urllist = urllist
         self.http_debug = http_debug
         self.offline = offline
-        self.cpio = {}
 
         passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
         for host in api_host_options.keys():
@@ -82,31 +79,18 @@ class Fetcher:
             print '\n'.join(pac.urllist)
             print
 
-        (fd, tmpfile) = tempfile.mkstemp(prefix='osc_build')
         try:
-            try:
-                # it returns the filename
-                ret = mg.urlgrab(pac.filename,
-                                 filename = tmpfile,
-                                 text = '%s(%s) %s' %(prefix, pac.project, pac.filename))
-                self.move_package(tmpfile, pac.localdir, pac)
-            except URLGrabError, e:
-                if e.errno == 256:
-                    self.cpio.setdefault(pac.project, {})[pac.name] = pac
-                    return
-                print
-                print >>sys.stderr, 'Error:', e.strerror
-                print >>sys.stderr, 'Failed to retrieve %s from the following locations (in order):' % pac.filename
-                print >>sys.stderr, '\n'.join(pac.urllist)
-                sys.exit(1)
-        finally:
-            os.close(fd)
-            if os.path.exists(tmpfile):
-                os.unlink(tmpfile)
+            # it returns the filename
+            ret = mg.urlgrab(pac.filename,
+                             filename = pac.fullpartname,
+                             text = '%s(%s) %s' %(prefix, pac.project, pac.filename))
+        except URLGrabError, e:
+            print >>sys.stderr, 'Error:', e.strerror
+            print >>sys.stderr, 'Failed to retrieve %s from the following locations (in order):' % pac.filename
+            print >>sys.stderr, '\n'.join(pac.urllist)
+            sys.exit(1)
 
-    def move_package(self, tmpfile, destdir, pac_obj = None):
-        import shutil
-        pkgq = packagequery.PackageQuery.query(tmpfile)
+        pkgq = packagequery.PackageQuery.query(pac.fullpartname)
         arch = pkgq.arch()
         # SOURCERPM = 1044
         if pkgq.filename_suffix == 'rpm' and not pkgq.getTag(1044):
@@ -119,11 +103,10 @@ class Fetcher:
             canonname = '%s-%s-%s.%s.%s' % (pkgq.name(), pkgq.version(), pkgq.release(), arch, pkgq.filename_suffix)
         else:
             canonname = '%s-%s.%s.%s' % (pkgq.name(), pkgq.version(), arch, pkgq.filename_suffix)
-        fullfilename = os.path.join(destdir, canonname)
-        if pac_obj is not None:
-            pac_obj.filename = canonname
-            pac_obj.fullfilename = fullfilename
-        shutil.move(tmpfile, fullfilename)
+        pac.filename = canonname
+        pac.fullfilename = os.path.join(pac.localdir, canonname)
+
+        os.rename(pac.fullpartname, pac.fullfilename)
 
     def dirSetup(self, pac):
         dir = os.path.join(self.cachedir, pac.localdir)
@@ -137,7 +120,6 @@ class Fetcher:
 
 
     def run(self, buildinfo):
-        from urllib import quote_plus
         cached = 0
         all = len(buildinfo.deps)
         for i in buildinfo.deps:
@@ -166,34 +148,13 @@ class Fetcher:
                 except KeyboardInterrupt:
                     print 'Cancelled by user (ctrl-c)'
                     print 'Exiting.'
+                    if os.path.exists(i.fullpartname):
+                        print 'Cleaning up incomplete file', i.fullpartname
+                        os.unlink(i.fullpartname)
                     sys.exit(0)
             done += 1
-        for project, pkgs in self.cpio.iteritems():
-            repo = pkgs.values()[0].repository
-            query = [ 'binary=%s' % quote_plus(i) for i in pkgs.keys() ]
-            query.append('view=cpio')
-            try:
-                (fd, tmparchive) = tempfile.mkstemp(prefix='osc_build_cpio')
-                (fd, tmpfile) = tempfile.mkstemp(prefix='osc_build')
-                url = makeurl(buildinfo.apiurl,
-                              ['public/build', project, repo, buildinfo.buildarch, '_repository'],
-                              query=query)
-                self.gr.urlgrab(url, filename = tmparchive, text = 'fetching cpio for \'%s\'' % project)
-                archive = cpio.CpioRead(tmparchive)
-                archive.read()
-                for hdr in archive:
-                    if hdr.filename == '.errors':
-                        import oscerr
-                        archive.copyin_file(hdr.filename)
-                        raise oscerr.APIError('CPIO archive is incomplete (see .errors file)')
-                    pac = pkgs[hdr.filename.rsplit('.', 1)[0]]
-                    archive.copyin_file(hdr.filename, os.path.dirname(tmpfile), os.path.basename(tmpfile))
-                    self.move_package(tmpfile, pac.localdir, pac)
-            finally:
-                if os.path.exists(tmparchive):
-                    os.unlink(tmparchive)
-                if os.path.exists(tmpfile):
-                    os.unlink(tmpfile)
+
+
 
 def verify_pacs(pac_list):
     """Take a list of rpm filenames and run rpm -K on them.