- fixed #590606 ("osc/fetch.py does not support authenticated URLs")

bye bye urlgrabber.grabber.URLGrabber: now we're using our own "grabber" object
(OscFileGrabber()) because this way we can reuse the urllib2 handlers (the latest
python-urlgrabber version uses python-curl instead of urllib2...).

osc/build.py

@@ -195,7 +195,7 @@ class Pac:
         # that the filename is suitable as identifier)
         self.localdir = '%s/%s/%s/%s' % (cachedir, self.project, self.repository, self.arch)
         self.fullfilename = os.path.join(self.localdir, self.filename)
-        self.url_local = 'file://%s/' % self.fullfilename
+        self.url_local = 'file://%s' % self.fullfilename
 
         # first, add the local URL
         self.urllist.append(self.url_local)

osc/core.py

@@ -1909,11 +1909,14 @@ def http_request(method, url, headers={}, data=None, file=None, timeout=100):
         data = ''
 
     req = urllib2.Request(url)
-
+    api_host_options = {}
-    api_host_options=conf.get_apiurl_api_host_options(url)
+    try:
-
+        api_host_options = conf.get_apiurl_api_host_options(url)
         for header, value in api_host_options['http_headers']:
             req.add_header(header, value)
+    except:
+        # "external" request (url is no apiurl)
+        pass
 
     req.get_method = lambda: method
 
@@ -1954,12 +1957,12 @@ def http_request(method, url, headers={}, data=None, file=None, timeout=100):
 
     old_timeout = socket.getdefaulttimeout()
     # XXX: dirty hack as timeout doesn't work with python-m2crypto
-    if old_timeout != timeout and not api_host_options['sslcertck']:
+    if old_timeout != timeout and not api_host_options.get('sslcertck'):
         socket.setdefaulttimeout(timeout)
     try:
         fd = urllib2.urlopen(req, data=data)
     finally:
-        if old_timeout != timeout and not api_host_options['sslcertck']:
+        if old_timeout != timeout and not api_host_options.get('sslcertck'):
             socket.setdefaulttimeout(old_timeout)
         if hasattr(conf.cookiejar, 'save'):
             conf.cookiejar.save(ignore_discard=True)
@@ -3680,7 +3683,7 @@ def get_prj_results(apiurl, prj, hide_legend=False, csv=False, status_filter=Non
     return r
 
 
-def streamfile(url, http_meth = http_GET, bufsize=8192, data=None, progress_obj=None):
+def streamfile(url, http_meth = http_GET, bufsize=8192, data=None, progress_obj=None, text=None):
     """
     performs http_meth on url and read bufsize bytes from the response
     until EOF is reached. After each read bufsize bytes are yielded to the
@@ -3690,7 +3693,7 @@ def streamfile(url, http_meth = http_GET, bufsize=8192, data=None, progress_obj=
     if progress_obj:
         import urlparse
         basename = os.path.basename(urlparse.urlsplit(url)[2])
-        progress_obj.start(basename=basename, size=int(f.info().get('Content-Length', -1)))
+        progress_obj.start(basename=basename, text=text, size=int(f.info().get('Content-Length', -1)))
     data = f.read(bufsize)
     read = len(data)
     while len(data):

osc/fetch.py

@@ -5,9 +5,9 @@
 
 import sys, os
 import urllib2
-from urlgrabber.grabber import URLGrabber, URLGrabError
+from urlgrabber.grabber import URLGrabError
 from urlgrabber.mirror import MirrorGroup
-from core import makeurl
+from core import makeurl, streamfile
 from util import packagequery, cpio
 import conf
 import tempfile
@@ -23,22 +23,41 @@ def join_url(self, base_url, rel_url):
     IOW, we make MirrorGroup ignore relative_url"""
     return base_url
 
+class OscFileGrabber:
+    def __init__(self, progress_obj = None):
+        self.progress_obj = progress_obj
+
+    def urlgrab(self, url, filename, text = None, **kwargs):
+        if url.startswith('file://'):
+            file = url.replace('file://', '', 1)
+            if os.path.isfile(file):
+                return file
+            else:
+                raise URLGrabError(2, 'Local file \'%s\' does not exist' % file)
+        f = open(filename, 'wb')
+        try:
+            try:
+                for i in streamfile(url, progress_obj=self.progress_obj, text=text):
+                    f.write(i)
+            except urllib2.HTTPError, e:
+                exc = URLGrabError(14, str(e))
+                exc.url = url
+                exc.exception = e
+                exc.code = e.code
+                raise exc
+        finally:
+            f.close()
+        return filename
 
 class Fetcher:
     def __init__(self, cachedir = '/tmp', api_host_options = {}, urllist = [], http_debug = False,
                  cookiejar = None, offline = False, enable_cpio = False):
-
-        __version__ = '0.1'
-        __user_agent__ = 'osbuild/%s' % __version__
-
         # set up progress bar callback
         if sys.stdout.isatty() and TextMeter:
             self.progress_obj = TextMeter(fo=sys.stdout)
         else:
             self.progress_obj = None
 
-
-        self.nopac = False
         self.cachedir = cachedir
         self.urllist = urllist
         self.http_debug = http_debug
@@ -52,24 +71,15 @@ class Fetcher:
         openers = (urllib2.HTTPBasicAuthHandler(passmgr), )
         if cookiejar:
             openers += (urllib2.HTTPCookieProcessor(cookiejar), )
-        self.gr = URLGrabber(user_agent=__user_agent__,
+        self.gr = OscFileGrabber(progress_obj=self.progress_obj)
-                            keepalive=1,
-                            opener = urllib2.build_opener(*openers),
-                            progress_obj=self.progress_obj,
-                            failure_callback=(self.failureReport,(),{}),
-                            )
-
 
     def failureReport(self, errobj):
         """failure output for failovers from urlgrabber"""
-
+        if errobj.url.startswith('file://'):
-        #log(0, '%s: %s' % (errobj.url, str(errobj.exception)))
+            return {}
-        #log(0, 'Trying other mirror.')
-        if not self.nopac:
         print 'Trying openSUSE Build Service server for %s (%s), not found at %s.' \
               % (self.curpac, self.curpac.project, errobj.url.split('/')[2])
-        raise errobj.exception
+        return {}
-
 
     def fetch(self, pac, prefix=''):
         # for use by the failure callback
@@ -79,7 +89,7 @@ class Fetcher:
             return True
 
         MirrorGroup._join_url = join_url
-        mg = MirrorGroup(self.gr, pac.urllist)
+        mg = MirrorGroup(self.gr, pac.urllist, failure_callback=(self.failureReport,(),{}))
 
         if self.http_debug:
             print
@@ -90,8 +100,7 @@ class Fetcher:
         (fd, tmpfile) = tempfile.mkstemp(prefix='osc_build')
         try:
             try:
-                # it returns the filename
+                mg.urlgrab(pac.filename,
-                ret = mg.urlgrab(pac.filename,
                            filename = tmpfile,
                            text = '%s(%s) %s' %(prefix, pac.project, pac.filename))
                 self.move_package(tmpfile, pac.localdir, pac)
@@ -137,7 +146,6 @@ class Fetcher:
                 print >>sys.stderr, e
                 sys.exit(1)
 
-
     def run(self, buildinfo):
         from urllib import quote_plus
         cached = 0
@@ -197,7 +205,6 @@ class Fetcher:
                 if os.path.exists(tmpfile):
                     os.unlink(tmpfile)
 
-        self.nopac = True
         prjs = buildinfo.projects.keys()
         for i in prjs:
             dest = "%s/%s" % (self.cachedir, i)
@@ -232,8 +239,6 @@ class Fetcher:
                     if len(l) > 1 and l[1] and not l[0] in buildinfo.projects:
                         prjs.append(l[0])
 
-        self.nopac = False
-
 def verify_pacs_old(pac_list):
     """Take a list of rpm filenames and run rpm -K on them.