yudaike/openSUSE-release-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import urllib2 python agnostic

Browse Source
This commit is contained in:
Stephan Kulow 2018-11-16 08:32:25 +01:00
parent ac7b288e74
commit ea9061ccf5
22 changed files with 160 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
ReviewBot.py
View File

@ -28,7 +28,12 @@ except ImportError:
from osc import conf
import osc.core
import urllib2
try:
from urllib.error import HTTPError, URLError
except ImportError:
# python 2.x
from urllib2 import HTTPError, URLError
from itertools import count
class PackageLookup(object):
@ -57,7 +62,7 @@ class PackageLookup(object):
try:
return osc.core.http_GET(osc.core.makeurl(self.apiurl,
['source', prj, '00Meta', 'lookup.yml']))
except urllib2.HTTPError as e:
except HTTPError as e:
# in case the project doesn't exist yet (like sle update)
if e.code != 404:
raise e
@ -458,7 +463,7 @@ class ReviewBot(object):
url = osc.core.makeurl(apiurl, ('source', project, package), query=query)
try:
return ET.parse(osc.core.http_GET(url)).getroot()
except (urllib2.HTTPError, urllib2.URLError):
except (HTTPError, URLError):
return None
def get_originproject(self, project, package, rev=None):
@ -493,7 +498,7 @@ class ReviewBot(object):
url = osc.core.makeurl(self.apiurl, ('source', src_project, src_package), query=query)
try:
root = ET.parse(osc.core.http_GET(url)).getroot()
except urllib2.HTTPError:
except HTTPError:
return (None, None)
if root is not None:
@ -521,7 +526,7 @@ class ReviewBot(object):
return True
if self.review_group and self._has_open_review_by(root, 'by_group', self.review_group):
return True
except urllib2.HTTPError as e:
except HTTPError as e:
print('ERROR in URL %s [%s]' % (url, e))
return False
@ -670,7 +675,7 @@ class ReviewBot(object):
self.logger.debug("checking %s in %s"%(package, project))
try:
si = osc.core.show_package_meta(self.apiurl, project, package)
except (urllib2.HTTPError, urllib2.URLError):
except (HTTPError, URLError):
si = None
if si is None:
self.logger.debug("new package")
@ -686,7 +691,7 @@ class ReviewBot(object):
u = osc.core.makeurl(self.apiurl, [ 'source', project, package, '_history' ], { 'limit': history_limit })
try:
r = osc.core.http_GET(u)
except urllib2.HTTPError as e:
except HTTPError as e:
self.logger.debug("package has no history!?")
return None

9
ToolBase.py
View File

@ -8,7 +8,12 @@ import logging
import signal
import sys
import time
import urllib2
try:
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError
import osc.conf
import osc.core
@ -48,7 +53,7 @@ class ToolBase(object):
def retried_GET(self, url):
try:
return http_GET(url)
except urllib2.HTTPError as e:
except HTTPError as e:
if 500 <= e.code <= 599:
print 'Retrying {}'.format(url)
time.sleep(1)

18
abichecker/abichecker.py
View File

@ -24,7 +24,12 @@ import osc.conf
import osc.core
from osc.util.cpio import CpioRead
import urllib2
try:
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError
import rpm
from collections import namedtuple
from osclib.pkgcache import PkgCache
@ -783,7 +788,7 @@ class ABIChecker(ReviewBot.ReviewBot):
[ 'view=cpioheaders' ])
try:
r = osc.core.http_GET(u)
except urllib2.HTTPError as e:
except HTTPError as e:
raise FetchError('failed to fetch header information: %s'%e)
tmpfile = NamedTemporaryFile(prefix="cpio-", delete=False)
for chunk in r:
@ -813,7 +818,7 @@ class ABIChecker(ReviewBot.ReviewBot):
url = osc.core.makeurl(self.apiurl, ('build', prj, repo, arch, pkg))
try:
root = ET.parse(osc.core.http_GET(url)).getroot()
except urllib2.HTTPError:
except HTTPError:
return None
return dict([(node.attrib['filename'], node.attrib['mtime']) for node in root.findall('binary')])
@ -828,7 +833,7 @@ class ABIChecker(ReviewBot.ReviewBot):
'srcmd5' : rev }
url = osc.core.makeurl(self.apiurl, ('build', src_project, '_result'), query)
return ET.parse(osc.core.http_GET(url)).getroot()
except urllib2.HTTPError as e:
except HTTPError as e:
if e.code != 404:
self.logger.error('ERROR in URL %s [%s]' % (url, e))
raise
@ -855,7 +860,7 @@ class ABIChecker(ReviewBot.ReviewBot):
url = osc.core.makeurl(self.apiurl, ('source', project, '_meta'))
try:
root = ET.parse(osc.core.http_GET(url)).getroot()
except urllib2.HTTPError:
except HTTPError:
return None
repos = set()
@ -912,7 +917,7 @@ class ABIChecker(ReviewBot.ReviewBot):
url = osc.core.makeurl(self.apiurl, ('source', src_project, '_meta'))
try:
root = ET.parse(osc.core.http_GET(url)).getroot()
except urllib2.HTTPError:
except HTTPError:
return None
# set of source repo name, target repo name, arch
@ -1087,4 +1092,3 @@ class CommandLineInterface(ReviewBot.CommandLineInterface):
if __name__ == "__main__":
app = CommandLineInterface()
sys.exit( app.main() )

15
biarchtool.py
View File

@ -4,7 +4,11 @@ from xml.etree import cElementTree as ET
import sys
import cmdln
import logging
import urllib2
try:
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError
import osc.core
import ToolBase
@ -158,7 +162,7 @@ class BiArchTool(ToolBase.ToolBase):
try:
x = ET.fromstring(self.cached_GET(self.makeurl(['source', self.project, pkg, '_history'], {'rev': '1'})))
# catch deleted packages
except urllib2.HTTPError as e:
except HTTPError as e:
if e.code == 404:
continue
raise e
@ -202,7 +206,7 @@ class BiArchTool(ToolBase.ToolBase):
self.http_PUT(pkgmetaurl, data=ET.tostring(pkgmeta))
if self.caching:
self._invalidate__cached_GET(pkgmetaurl)
except urllib2.HTTPError as e:
except HTTPError as e:
logger.error('failed to update %s: %s', pkg, e)
def add_explicit_disable(self, wipebinaries=False):
@ -242,7 +246,7 @@ class BiArchTool(ToolBase.ToolBase):
'cmd' : 'wipe',
'arch': self.arch,
'package' : pkg }))
except urllib2.HTTPError as e:
except HTTPError as e:
logger.error('failed to update %s: %s', pkg, e)
@ -326,7 +330,7 @@ class BiArchTool(ToolBase.ToolBase):
'cmd' : 'wipe',
'arch': self.arch,
'package' : pkg }))
except urllib2.HTTPError as e:
except HTTPError as e:
logger.error('failed to update %s: %s', pkg, e)
class CommandLineInterface(ToolBase.CommandLineInterface):
@ -399,4 +403,3 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
if __name__ == "__main__":
app = CommandLineInterface()
sys.exit( app.main() )

9
check_maintenance_incidents.py
View File

@ -13,7 +13,11 @@ except ImportError:
import osc.conf
import osc.core
import urllib2
try:
from urllib.error import HTTPError, URLError
except ImportError:
# python 2.x
from urllib2 import HTTPError, URLError
import yaml
from osclib.memoize import memoize
@ -56,7 +60,7 @@ class MaintenanceChecker(ReviewBot.ReviewBot):
url = osc.core.makeurl(apiurl, ('source', project, '00Meta', 'lookup.yml'))
try:
return yaml.safe_load(osc.core.http_GET(url))
except (urllib2.HTTPError, urllib2.URLError):
except (HTTPError, URLError):
return None
# check if pkgname was submitted by the correct maintainer. If not, set
@ -165,4 +169,3 @@ if __name__ == "__main__":
app = ReviewBot.CommandLineInterface()
app.clazz = MaintenanceChecker
sys.exit( app.main() )

13
check_source.py
View File

@ -17,7 +17,12 @@ from osclib.conf import Config
from osclib.core import devel_project_get
from osclib.core import devel_project_fallback
from osclib.core import group_members
import urllib2
try:
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError
import ReviewBot
from osclib.conf import str2bool
@ -124,7 +129,7 @@ class CheckSource(ReviewBot.ReviewBot):
shutil.rmtree(os.path.join(target_package, '.osc'))
os.rename(target_package, '_old')
old_info = self.package_source_parse(target_project, target_package)
except urllib2.HTTPError:
except HTTPError:
self.logger.error('failed to checkout %s/%s' % (target_project, target_package))
CheckSource.checkout_package(self.apiurl, source_project, source_package, revision=source_revision,
@ -218,7 +223,7 @@ class CheckSource(ReviewBot.ReviewBot):
try:
xml = ET.parse(osc.core.http_GET(url)).getroot()
except urllib2.HTTPError as e:
except HTTPError as e:
self.logger.error('ERROR in URL %s [%s]' % (url, e))
return ret
@ -261,7 +266,7 @@ class CheckSource(ReviewBot.ReviewBot):
try:
result = osc.core.show_project_sourceinfo(self.apiurl, action.tgt_project, True, (action.tgt_package))
root = ET.fromstring(result)
except urllib2.HTTPError:
except HTTPError:
return None
# Decline the delete request if there is another delete/submit request against the same package

10
check_source_in_factory.py
View File

@ -13,7 +13,12 @@ except ImportError:
import osc.conf
import osc.core
import urllib2
try:
from urllib.error import HTTPError, URLError
except ImportError:
# python 2.x
from urllib2 import HTTPError, URLError
import yaml
import ReviewBot
@ -89,7 +94,7 @@ class FactorySourceChecker(ReviewBot.ReviewBot):
sr = srprefix
break
requests = osc.core.get_request_list(apiurl, project, package, None, ['new', 'review'], 'submit')
except (urllib2.HTTPError, urllib2.URLError):
except (HTTPError, URLError):
self.logger.error("caught exception while checking %s/%s", project, package)
return None
@ -169,4 +174,3 @@ class CommandLineInterface(ReviewBot.CommandLineInterface):
if __name__ == "__main__":
app = CommandLineInterface()
sys.exit( app.main() )

9
compare_pkglist.py
View File

@ -3,7 +3,12 @@
import argparse
import logging
import sys
import urllib2
try:
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError
import re
from xml.etree import cElementTree as ET
@ -53,7 +58,7 @@ class CompareList(object):
url = makeurl(self.apiurl, ['source', project, '_meta'])
try:
http_GET(url)
except urllib2.HTTPError:
except HTTPError:
return False
return True

18
fcc_submitter.py
View File

@ -3,7 +3,13 @@
import argparse
import logging
import sys
import urllib2
try:
from urllib.error import HTTPError, URLError
except ImportError:
# python 2.x
from urllib2 import HTTPError, URLError
import random
import re
from xml.etree import cElementTree as ET
@ -115,7 +121,7 @@ class FccFreezer(object):
l = ET.tostring(flink)
try:
http_PUT(url, data=l)
except urllib2.HTTPError as e:
except HTTPError as e:
raise e
class FccSubmitter(object):
@ -159,7 +165,7 @@ class FccSubmitter(object):
def get_link(self, project, package):
try:
link = http_GET(makeurl(self.apiurl, ['source', project, package, '_link'])).read()
except (urllib2.HTTPError, urllib2.URLError):
except (HTTPError, URLError):
return None
return ET.fromstring(link)
@ -201,7 +207,7 @@ class FccSubmitter(object):
try:
logging.debug("Gathering package_meta %s/%s" % (tgt_project, tgt_package))
osc.core.show_package_meta(self.apiurl, tgt_project, tgt_package)
except (urllib2.HTTPError, urllib2.URLError):
except (HTTPError, URLError):
return True
return False
@ -222,7 +228,7 @@ class FccSubmitter(object):
def check_multiple_specfiles(self, project, package):
try:
url = makeurl(self.apiurl, ['source', project, package], { 'expand': '1' } )
except urllib2.HTTPError as e:
except HTTPError as e:
if e.code == 404:
return None
raise e
@ -283,7 +289,7 @@ class FccSubmitter(object):
url = makeurl(self.apiurl, ['source', project, package, '{}?expand=1'.format('fcc_skip_pkgs')])
try:
return http_GET(url).read()
except urllib2.HTTPError:
except HTTPError:
return ''
def crawl(self):

11
leaper.py
View File

@ -16,7 +16,13 @@ except ImportError:
import osc.core
from osclib.conf import Config
from osclib.core import devel_project_get
import urllib2
try:
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError
import yaml
import ReviewBot
from check_source_in_factory import FactorySourceChecker
@ -64,7 +70,7 @@ class Leaper(ReviewBot.ReviewBot):
root = ET.parse(osc.core.http_GET(osc.core.makeurl(self.apiurl, ['source', project],
query=query))).getroot()
packages = [i.get('name') for i in root.findall('entry')]
except urllib2.HTTPError as e:
except HTTPError as e:
# in case the project doesn't exist yet (like sle update)
if e.code != 404:
raise e
@ -583,4 +589,3 @@ class CommandLineInterface(ReviewBot.CommandLineInterface):
if __name__ == "__main__":
app = CommandLineInterface()
sys.exit( app.main() )

9
legal-auto.py
View File

@ -12,7 +12,12 @@ import cmdln
import requests as REQ
import json
import time
import urllib2
try:
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError
try:
from xml.etree import cElementTree as ET
@ -53,7 +58,7 @@ class LegalAuto(ReviewBot.ReviewBot):
def retried_GET(self, url):
try:
return http_GET(url)
except urllib2.HTTPError, e:
except HTTPError, e:
if 500 <= e.code <= 599:
print 'Retrying {}'.format(url)
time.sleep(1)

21
manager_42.py
View File

@ -12,7 +12,13 @@ import osc.conf
import osc.core
from osc.core import get_commitlog
from osc.core import get_request_list
import urllib2
try:
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError
import subprocess
import time
import yaml
@ -86,7 +92,7 @@ class Manager42(object):
self.lookup = {}
try:
self.lookup = yaml.safe_load(self._load_lookup_file(project))
except urllib2.HTTPError as e:
except HTTPError as e:
if e.code != 404:
raise
@ -118,7 +124,7 @@ class Manager42(object):
def retried_GET(self, url):
try:
return http_GET(url)
except urllib2.HTTPError as e:
except HTTPError as e:
if 500 <= e.code <= 599:
logger.warn('Retrying {}'.format(url))
time.sleep(1)
@ -135,7 +141,7 @@ class Manager42(object):
query=query)))
packages = [i.get('name') for i in root.findall('entry')]
except urllib2.HTTPError as e:
except HTTPError as e:
if e.code == 404:
logger.error("{}: {}".format(project, e))
packages = []
@ -158,7 +164,7 @@ class Manager42(object):
for package in sorted(packages):
try:
self.check_one_package(package)
except urllib2.HTTPError as e:
except HTTPError as e:
logger.error("Failed to check {}: {}".format(package, e))
pass
@ -225,7 +231,7 @@ class Manager42(object):
query['deleted'] = 1
return self.cached_GET(makeurl(self.apiurl,
['source', project, package, '_history'], query))
except urllib2.HTTPError as e:
except HTTPError as e:
if e.code == 404:
return None
raise
@ -360,7 +366,7 @@ class Manager42(object):
try:
link = self.cached_GET(makeurl(self.apiurl,
['source', project, package, '_link']))
except urllib2.HTTPError:
except HTTPError:
return None
return ET.fromstring(link)
@ -430,4 +436,3 @@ if __name__ == '__main__':
http_DELETE = dryrun('DELETE')
sys.exit(main(args))

6
obs_clone.py
View File

@ -10,7 +10,13 @@ from osc.core import http_PUT
from osc.core import makeurl
from osc.core import show_upstream_rev
from osclib.core import project_pseudometa_package
try:
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError
import argparse
import osc.conf
import sys

2
osclib/cache.py
View File

@ -11,7 +11,7 @@ import sys
try:
from urllib.parse import unquote
from urllib.parse import urlsplit, SplitResult
from urllib.request import URLError, HTTPError
from urllib.error import URLError, HTTPError
from io import StringIO
except ImportError:
# python 2.x

9
osclib/cycle.py
View File

@ -1,4 +1,9 @@
import urllib2
try:
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError
from xml.etree import cElementTree as ET
from osc.core import http_GET
@ -140,7 +145,7 @@ class CycleDetector(object):
# print('Generating _builddepinfo for (%s, %s, %s)' % (project, repository, arch))
url = makeurl(self.apiurl, ['build/%s/%s/%s/_builddepinfo' % (project, repository, arch)])
root = http_GET(url).read()
except urllib2.HTTPError as e:
except HTTPError as e:
print('ERROR in URL %s [%s]' % (url, e))
return root

2
osclib/prio_command.py
View File

@ -4,7 +4,7 @@ import json
import osc
try:
from urllib.request import HTTPError
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError

2
osclib/repair_command.py
View File

@ -3,7 +3,7 @@ from __future__ import print_function
import re
try:
from urllib.request import HTTPError
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError

9
suppkg_rebuild.py
View File

@ -3,7 +3,12 @@
import argparse
import logging
import sys
import urllib2
try:
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError
import re
import yaml
from xml.etree import cElementTree as ET
@ -80,7 +85,7 @@ class StagingHelper(object):
url = makeurl(self.apiurl, ['source', project, pkg], query=query)
try:
root = ET.parse(http_GET(url)).getroot()
except urllib2.HTTPError as e:
except HTTPError as e:
if e.code == 404:
continue
raise

9
tests/obs.py
View File

@ -6,7 +6,12 @@ import re
import string
import time
import urllib
import urllib2
try:
from urllib.parse import unquote
except ImportError:
# python 2.x
from urllib import unquote
import urlparse
import xml.etree.cElementTree as ET
@ -852,7 +857,7 @@ class OBS(object):
@GET('/search/request')
def search_request(self, request, uri, headers):
"""Return a search result for /search/request."""
query = urllib2.unquote(urlparse.urlparse(uri).query)
query = unquote(urlparse.urlparse(uri).query)
assert query in (
"match=state/@name='review'+and+review[@by_group='factory-staging'+and+@state='new']+and+(target[@project='openSUSE:Factory']+or+target[@project='openSUSE:Factory:NonFree'])",
"match=state/@name='review'+and+review[@by_user='factory-repo-checker'+and+@state='new']+and+(target[@project='openSUSE:Factory']+or+target[@project='openSUSE:Factory:NonFree'])"

10
totest-manager.py
View File

@ -17,7 +17,13 @@ import json
import os
import re
import sys
import urllib2
try:
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError
import logging
import signal
import time
@ -101,7 +107,7 @@ class ToTestBase(object):
url = self.api.makeurl(['build', project, self.product_repo, self.product_arch, product])
try:
f = self.api.retried_GET(url)
except urllib2.HTTPError:
except HTTPError:
return []
ret = []

13
update_crawler.py
View File

@ -4,7 +4,13 @@ import argparse
import itertools
import logging
import sys
import urllib2
try:
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError
import time
from xml.etree import cElementTree as ET
@ -79,7 +85,7 @@ class UpdateCrawler(object):
def retried_GET(self, url):
try:
return http_GET(url)
except urllib2.HTTPError as e:
except HTTPError as e:
if 500 <= e.code <= 599:
print 'Retrying {}'.format(url)
time.sleep(1)
@ -191,7 +197,7 @@ class UpdateCrawler(object):
)))
if root.get('project') is None and root.get('cicount'):
return True
except urllib2.HTTPError as err:
except HTTPError as err:
# if there is no link, it can't be a link
if err.code == 404:
return False
@ -370,4 +376,3 @@ if __name__ == '__main__':
http_DELETE = dryrun('DELETE')
sys.exit(main(args))