mirrors/github.com_openSUSE_osc
1
0
Fork 0
mirror of https://github.com/openSUSE/osc.git synced 2025-02-24 11:12:14 +01:00
Code

- Fetcher.__fetch_cpio: split cpio request into smaller requests if the server returns code 414

Browse Source
This commit is contained in:
Marcus Huewe 2012-10-10 17:03:49 +02:00
parent 15b27b7ff1
commit 161b9ca274
1 changed files with 56 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
osc/fetch.py
View File

@ -5,6 +5,8 @@
import sys, os import sys, os
import urllib2 import urllib2
from urllib import quote_plus
from urlgrabber.grabber import URLGrabError from urlgrabber.grabber import URLGrabError
from urlgrabber.mirror import MirrorGroup from urlgrabber.mirror import MirrorGroup
from core import makeurl, streamfile from core import makeurl, streamfile
@ -89,11 +91,10 @@ class Fetcher:
prpap = '%s/%s/%s/%s' % (pac.project, pac.repository, pac.repoarch, pac.repopackage) prpap = '%s/%s/%s/%s' % (pac.project, pac.repository, pac.repoarch, pac.repopackage)
self.cpio.setdefault(prpap, {})[pac.repofilename] = pac self.cpio.setdefault(prpap, {})[pac.repofilename] = pac
def __fetch_cpio(self, apiurl): def __download_cpio_archive(self, apiurl, project, repo, arch, package, **pkgs):
from urllib import quote_plus if not pkgs:
for prpap, pkgs in self.cpio.iteritems(): return
project, repo, arch, package = prpap.split('/', 3) query = ['binary=%s' % quote_plus(i) for i in pkgs]
query = ['binary=%s' % quote_plus(i) for i in pkgs.keys()]
query.append('view=cpio') query.append('view=cpio')
tmparchive = tmpfile = None tmparchive = tmpfile = None
try: try:
@ -125,12 +126,29 @@ class Fetcher:
if not os.path.isfile(pac.fullfilename): if not os.path.isfile(pac.fullfilename):
raise oscerr.APIError('failed to fetch file \'%s\': ' \ raise oscerr.APIError('failed to fetch file \'%s\': ' \
'does not exist in CPIO archive' % pac.repofilename) 'does not exist in CPIO archive' % pac.repofilename)
except URLGrabError, e:
if e.errno != 14 or e.code != 414:
raise
# query str was too large
keys = pkgs.keys()
if len(keys) == 1:
raise oscerr.APIError('unable to fetch cpio archive: server always returns code 414')
n = len(pkgs) / 2
new_pkgs = dict([(k, pkgs[k]) for k in keys[:n]])
self.__download_cpio_archive(apiurl, project, repo, arch, package, **new_pkgs)
new_pkgs = dict([(k, pkgs[k]) for k in keys[n:]])
self.__download_cpio_archive(apiurl, project, repo, arch, package, **new_pkgs)
finally: finally:
if not tmparchive is None and os.path.exists(tmparchive): if not tmparchive is None and os.path.exists(tmparchive):
os.unlink(tmparchive) os.unlink(tmparchive)
if not tmpfile is None and os.path.exists(tmpfile): if not tmpfile is None and os.path.exists(tmpfile):
os.unlink(tmpfile) os.unlink(tmpfile)
def __fetch_cpio(self, apiurl):
for prpap, pkgs in self.cpio.iteritems():
project, repo, arch, package = prpap.split('/', 3)
self.__download_cpio_archive(apiurl, project, repo, arch, package, **pkgs)
def fetch(self, pac, prefix=''): def fetch(self, pac, prefix=''):
# for use by the failure callback # for use by the failure callback
self.curpac = pac self.curpac = pac