mirrors/github.com_openSUSE_osc
1
0
Fork 0
mirror of https://github.com/openSUSE/osc.git synced 2024-11-09 22:36:14 +01:00
Code

replace urlgrabber to enable python3 compatibility

Browse Source 
- new module grabber.py
  * OscMirrorGroup to keep urlgrabber.mirrorgroup behavior
  * OscFileGrabber moved here

- meter.py
  * reworked to use progressbar module instead of progressbar
    of urlgrabber and simplified the module.

- babysitter.py
  * removed URLGrabErrorr

- build.py
  * removed adding of url_local to urllist. (not needed anymore)
  * removed URLGrabError

- commandline.py
  * switched from urlgrabber.urlgrab to OscFileGrabber().urlgrab

- core.py
  * reworked progressbar behavior

- fetch.py
  * removed join_url (not needed anymore)
  * moved OscFileGrabber to grabber.py
  * removed failureReport (not needed anymore)
This commit is contained in:
lethliel 2018-02-02 15:47:17 +01:00
parent cbd10644a2
commit 65b053abb3
9 changed files with 115 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README
View File

@ -26,10 +26,6 @@ Alternatively, you can directly use osc-wrapper.py from the source dir
The program needs the cElementTree python module installed. On SUSE, the
respective package is called python-elementtree (before 10.2: python-xml).
For local building, you will need python-urlgrabber in addition. Those are
standard package on SUSE Linux since a while. If your version is too old, you
can find python-elementtree and python-urlgrabber here:
http://download.opensuse.org/repositories/devel:/languages:/python/

2
osc/__init__.py
View File

@ -1,3 +1,3 @@
__all__ = ['babysitter', 'core', 'commandline', 'oscerr', 'othermethods', 'build', 'fetch', 'meter']
__all__ = ['babysitter', 'core', 'commandline', 'oscerr', 'build', 'fetch', 'meter', 'grabber']
# vim: sw=4 et

3
osc/babysitter.py
View File

@ -11,7 +11,6 @@ import pdb
import sys
import signal
import traceback
from urlgrabber.grabber import URLGrabError
from osc import oscerr
from .oscsslexcp import NoSecureSSLError
@ -132,8 +131,6 @@ def run(prg, argv=None):
print(e, file=sys.stderr)
except URLError as e:
print('Failed to reach a server:\n', e.reason, file=sys.stderr)
except URLGrabError as e:
print('Failed to grab %s: %s' % (e.url, e.strerror), file=sys.stderr)
except IOError as e:
# ignore broken pipe
if e.errno != errno.EPIPE:

16
osc/build.py
View File

@ -256,18 +256,8 @@ class Pac:
def makeurls(self, cachedir, urllist):
self.urllist = []
# build up local URL
# by using the urlgrabber with local urls, we basically build up a cache.
# the cache has no validation, since the package servers don't support etags,
# or if-modified-since, so the caching is simply name-based (on the assumption
# that the filename is suitable as identifier)
self.localdir = '%s/%s/%s/%s' % (cachedir, self.project, self.repository, self.arch)
self.fullfilename = os.path.join(self.localdir, self.canonname)
self.url_local = 'file://%s' % self.fullfilename
# first, add the local URL
self.urllist.append(self.url_local)
# remote URLs
for url in urllist:
@ -319,14 +309,14 @@ def get_preinstall_image(apiurl, arch, cache_dir, img_info):
print(e, file=sys.stderr)
sys.exit(1)
if sys.stdout.isatty() and TextMeter:
progress_obj = TextMeter(fo=sys.stdout)
progress_obj = TextMeter()
else:
progress_obj = None
gr = OscFileGrabber(progress_obj=progress_obj)
try:
gr.urlgrab(url, filename=ifile_path_part, text='fetching image')
except URLGrabError as e:
print("Failed to download! ecode:%i errno:%i" % (e.code, e.errno))
except HTTPError as e:
print("Failed to download! ecode:%i reason:%i" % (e.code, e.reason))
return ('', '', [])
# download ok, rename partial file to final file name
os.rename(ifile_path_part, ifile_path)

9
osc/commandline.py
View File

@ -167,7 +167,7 @@ class Osc(cmdln.Cmdln):
self.download_progress = None
if conf.config.get('show_download_progress', False):
from .meter import TextMeter
self.download_progress = TextMeter(hide_finished=True)
self.download_progress = TextMeter()
def get_cmd_help(self, cmdname):
@ -7709,9 +7709,12 @@ Please submit there instead, or use --nodevelproject to force direct submission.
sys.exit(1)
if '://' in srpm:
if srpm.endswith('/'):
print('%s is not a valid link. It must not end with /' % srpm)
sys.exit(1)
print('trying to fetch', srpm)
import urlgrabber
urlgrabber.urlgrab(srpm)
from .grabber import OscFileGrabber
OscFileGrabber().urlgrab(srpm)
srpm = os.path.basename(srpm)
srpm = os.path.abspath(srpm)

9
osc/core.py
View File

@ -5992,8 +5992,11 @@ def streamfile(url, http_meth = http_GET, bufsize=8192, data=None, progress_obj=
cl = int(cl)
if progress_obj:
basename = os.path.basename(urlsplit(url)[2])
progress_obj.start(basename=basename, text=text, size=cl)
if not text:
basename = os.path.basename(urlsplit(url)[2])
else:
basename = text
progress_obj.start(basename, cl)
if bufsize == "line":
bufsize = 8192
@ -6012,7 +6015,7 @@ def streamfile(url, http_meth = http_GET, bufsize=8192, data=None, progress_obj=
yield data
if progress_obj:
progress_obj.end(read)
progress_obj.end()
f.close()
if not cl is None and read != cl:

105
osc/fetch.py
View File

@ -15,64 +15,26 @@ except ImportError:
from urllib import quote_plus
from urllib2 import HTTPBasicAuthHandler, HTTPCookieProcessor, HTTPPasswordMgrWithDefaultRealm, HTTPError
from urlgrabber.grabber import URLGrabber, URLGrabError
from urlgrabber.mirror import MirrorGroup
from .core import makeurl, streamfile, dgst
from .grabber import OscFileGrabber, OscMirrorGroup
from .util import packagequery, cpio
from . import conf
from . import oscerr
import tempfile
import re
try:
from .meter import TextMeter
except:
except ImportError:
TextMeter = None
def join_url(self, base_url, rel_url):
"""to override _join_url of MirrorGroup, because we want to
pass full URLs instead of base URL where relative_url is added later...
IOW, we make MirrorGroup ignore relative_url
"""
return base_url
class OscFileGrabber(URLGrabber):
def __init__(self, progress_obj=None):
# we cannot use super because we still have to support
# older urlgrabber versions where URLGrabber is an old-style class
URLGrabber.__init__(self)
self.progress_obj = progress_obj
def urlgrab(self, url, filename, text=None, **kwargs):
if url.startswith('file://'):
f = url.replace('file://', '', 1)
if os.path.isfile(f):
return f
else:
raise URLGrabError(2, 'Local file \'%s\' does not exist' % f)
with file(filename, 'wb') as f:
try:
for i in streamfile(url, progress_obj=self.progress_obj,
text=text):
f.write(i)
except HTTPError as e:
exc = URLGrabError(14, str(e))
exc.url = url
exc.exception = e
exc.code = e.code
raise exc
except IOError as e:
raise URLGrabError(4, str(e))
return filename
class Fetcher:
def __init__(self, cachedir='/tmp', api_host_options={}, urllist=[],
http_debug=False, cookiejar=None, offline=False, enable_cpio=True):
# set up progress bar callback
if sys.stdout.isatty() and TextMeter:
self.progress_obj = TextMeter(fo=sys.stdout)
self.progress_obj = TextMeter()
else:
self.progress_obj = None
@ -92,14 +54,6 @@ class Fetcher:
openers += (HTTPCookieProcessor(cookiejar), )
self.gr = OscFileGrabber(progress_obj=self.progress_obj)
def failureReport(self, errobj):
"""failure output for failovers from urlgrabber"""
if errobj.url.startswith('file://'):
return {}
print('%s/%s: attempting download from api, since not found at %s'
% (self.curpac.project, self.curpac, errobj.url.split('/')[2]))
return {}
def __add_cpio(self, pac):
prpap = '%s/%s/%s/%s' % (pac.project, pac.repository, pac.repoarch, pac.repopackage)
self.cpio.setdefault(prpap, {})[pac.repofilename] = pac
@ -156,8 +110,8 @@ class Fetcher:
raise oscerr.APIError('failed to fetch file \'%s\': '
'missing in CPIO archive' %
pac.repofilename)
except URLGrabError as e:
if e.errno != 14 or e.code != 414:
except HTTPError as e:
if e.code != 414:
raise
# query str was too large
keys = list(pkgs.keys())
@ -181,8 +135,7 @@ class Fetcher:
# for use by the failure callback
self.curpac = pac
MirrorGroup._join_url = join_url
mg = MirrorGroup(self.gr, pac.urllist, failure_callback=(self.failureReport, (), {}))
mg = OscMirrorGroup(self.gr, pac.urllist)
if self.http_debug:
print('\nURLs to try for package \'%s\':' % pac, file=sys.stderr)
@ -192,19 +145,22 @@ class Fetcher:
try:
with tempfile.NamedTemporaryFile(prefix='osc_build',
delete=False) as tmpfile:
mg.urlgrab(pac.filename, filename=tmpfile.name,
mg_stat = mg.urlgrab(pac.filename, filename=tmpfile.name,
text='%s(%s) %s' % (prefix, pac.project, pac.filename))
self.move_package(tmpfile.name, pac.localdir, pac)
except URLGrabError as e:
if self.enable_cpio and e.errno == 256:
self.__add_cpio(pac)
return
print()
print('Error:', e.strerror, file=sys.stderr)
print('Failed to retrieve %s from the following locations '
'(in order):' % pac.filename, file=sys.stderr)
print('\n'.join(pac.urllist), file=sys.stderr)
sys.exit(1)
if mg_stat:
self.move_package(tmpfile.name, pac.localdir, pac)
if not mg_stat:
if self.enable_cpio:
print('%s/%s: attempting download from api, since not found'
% (pac.project, pac.name))
self.__add_cpio(pac)
return
print()
print('Error: Failed to retrieve %s from the following locations '
'(in order):' % pac.filename, file=sys.stderr)
print('\n'.join(pac.urllist), file=sys.stderr)
sys.exit(1)
finally:
if os.path.exists(tmpfile.name):
os.unlink(tmpfile.name)
@ -285,12 +241,12 @@ class Fetcher:
continue
try:
# if there isn't a progress bar, there is no output at all
prefix = ''
if not self.progress_obj:
print('%d/%d (%s) %s' % (done, needed, i.project, i.filename))
self.fetch(i)
if self.progress_obj:
print(" %d/%d\r" % (done, needed), end=' ')
sys.stdout.flush()
else:
prefix = '[%d/%d] ' % (done, needed)
self.fetch(i, prefix=prefix)
except KeyboardInterrupt:
print('Cancelled by user (ctrl-c)')
@ -308,10 +264,11 @@ class Fetcher:
dest += '/_pubkey'
url = makeurl(buildinfo.apiurl, ['source', i, '_pubkey'])
try_parent = False
try:
if self.offline and not os.path.exists(dest):
# may need to try parent
raise URLGrabError(2)
try_parent = True
elif not self.offline:
OscFileGrabber().urlgrab(url, dest)
# not that many keys usually
@ -324,12 +281,14 @@ class Fetcher:
if os.path.exists(dest):
os.unlink(dest)
sys.exit(0)
except URLGrabError as e:
except HTTPError as e:
# Not found is okay, let's go to the next project
if e.errno == 14 and e.code != 404:
if e.code != 404:
print("Invalid answer from server", e, file=sys.stderr)
sys.exit(1)
try_parent = True
if try_parent:
if self.http_debug:
print("can't fetch key for %s: %s" % (i, e.strerror), file=sys.stderr)
print("url: %s" % url, file=sys.stderr)

49
osc/grabber.py Normal file
View File

@ -0,0 +1,49 @@
# Copyright (C) 2018 SUSE Linux. All rights reserved.
# This program is free software; it may be used, copied, modified
# and distributed under the terms of the GNU General Public Licence,
# either version 2, or (at your option) any later version.
import sys
import os.path
from .core import streamfile
try:
from urllib.request import HTTPError
from urllib.parse import urlparse
from urllib.parse import unquote
except ImportError:
from urllib2 import HTTPError
from urlparse import urlparse
from urllib import unquote
class OscFileGrabber(object):
def __init__(self, progress_obj=None):
self.progress_obj = progress_obj
def urlgrab(self, url, filename=None, text=None):
if filename is None:
parts = urlparse(url)
filename = os.path.basename(unquote(parts[2]))
with open(filename, 'wb') as f:
for i in streamfile(url, progress_obj=self.progress_obj,
text=text):
f.write(i)
class OscMirrorGroup(object):
def __init__(self, grabber, mirrors):
self._grabber = grabber
self._mirrors = mirrors
def urlgrab(self, url, filename=None, text=None):
tries = 0
for mirror in self._mirrors:
try:
self._grabber.urlgrab(mirror, filename, text)
return True
except HTTPError as e:
print('Error %s' % e.code)
tries += 1
return False

112
osc/meter.py
View File

@ -1,103 +1,27 @@
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330,
# Boston, MA 02111-1307 USA
# Copyright (C) 2018 SUSE Linux. All rights reserved.
# This program is free software; it may be used, copied, modified
# and distributed under the terms of the GNU General Public Licence,
# either version 2, or (at your option) any later version.
# this is basically a copy of python-urlgrabber's TextMeter class,
# with support added for dynamical sizing according to screen size.
# it uses getScreenWidth() scrapped from smart.
# 2007-04-24, poeml
from __future__ import print_function
from urlgrabber.progress import BaseMeter, format_time, format_number
import sys, os
def getScreenWidth():
import termios, struct, fcntl
s = struct.pack('HHHH', 0, 0, 0, 0)
try:
x = fcntl.ioctl(1, termios.TIOCGWINSZ, s)
except IOError:
return 80
return struct.unpack('HHHH', x)[1]
import progressbar as pb
class TextMeter(BaseMeter):
def __init__(self, fo=sys.stderr, hide_finished=False):
BaseMeter.__init__(self)
self.fo = fo
self.hide_finished = hide_finished
try:
width = int(os.environ['COLUMNS'])
except (KeyError, ValueError):
width = getScreenWidth()
class TextMeter(object):
#self.unsized_templ = '\r%-60.60s %5sB %s '
self.unsized_templ = '\r%%-%s.%ss %%5sB %%s ' % (width *2/5, width*3/5)
#self.sized_templ = '\r%-45.45s %3i%% |%-15.15s| %5sB %8s '
self.bar_length = width/5
self.sized_templ = '\r%%-%s.%ss %%3i%%%% |%%-%s.%ss| %%5sB %%8s ' % (width*4/10, width*4/10, self.bar_length, self.bar_length)
def _do_start(self, *args, **kwargs):
BaseMeter._do_start(self, *args, **kwargs)
self._do_update(0)
def _do_update(self, amount_read, now=None):
etime = self.re.elapsed_time()
fetime = format_time(etime)
fread = format_number(amount_read)
#self.size = None
if self.text is not None:
text = self.text
def start(self, basename, size=None):
if size is None:
widgets = [basename + ': ', pb.AnimatedMarker(), ' ', pb.Timer()]
self.bar = pb.ProgressBar(widgets=widgets, maxval=pb.UnknownLength)
else:
text = self.basename
if self.size is None:
out = self.unsized_templ % \
(text, fread, fetime)
else:
rtime = self.re.remaining_time()
frtime = format_time(rtime)
frac = self.re.fraction_read()
bar = '='*int(self.bar_length * frac)
widgets = [basename + ': ', pb.Percentage(), pb.Bar(), ' ',
pb.ETA()]
self.bar = pb.ProgressBar(widgets=widgets, maxval=size)
self.bar.start()
out = self.sized_templ % \
(text, frac*100, bar, fread, frtime) + 'ETA '
def update(self, amount_read):
self.bar.update(amount_read)
self.fo.write(out)
self.fo.flush()
def _do_end(self, amount_read, now=None):
total_time = format_time(self.re.elapsed_time())
total_size = format_number(amount_read)
if self.text is not None:
text = self.text
else:
text = self.basename
if self.size is None:
out = self.unsized_templ % \
(text, total_size, total_time)
else:
bar = '=' * self.bar_length
out = self.sized_templ % \
(text, 100, bar, total_size, total_time) + ' '
if self.hide_finished:
self.fo.write('\r'+ ' '*len(out) + '\r')
else:
self.fo.write(out + '\n')
self.fo.flush()
def end(self):
self.bar.finish()
# vim: sw=4 et