Harmonize imports on using lxml for XML parsing

This commit is contained in:
Stephan Kulow 2022-02-18 10:16:17 +01:00
parent 3f8763ae56
commit e52dc6ce2b
37 changed files with 71 additions and 191 deletions

View File

@ -1,11 +1,9 @@
#!/usr/bin/python3
from pprint import pprint
import os
import sys
import re
import logging
from optparse import OptionParser
import cmdln
from collections import namedtuple
from collections import OrderedDict
@ -26,11 +24,7 @@ import signal
import datetime
import time
import yaml
try:
from xml.etree import cElementTree as ET
except ImportError:
import cElementTree as ET
from lxml import etree as ET
from osc import conf
import osc.core

View File

@ -1,16 +1,14 @@
#!/usr/bin/python3
from xml.etree import cElementTree as ET
from lxml import etree as ET
import cmdln
import datetime
import itertools
import logging
import signal
import sys
import time
from urllib.error import HTTPError, URLError
from urllib.parse import quote_plus
import osc.conf
import osc.core

View File

@ -1 +0,0 @@
../ReviewBot.py

View File

@ -1,7 +1,6 @@
#!/usr/bin/python3
from optparse import OptionParser
from pprint import pformat, pprint
from pprint import pformat
from stat import S_ISREG, S_ISLNK
from tempfile import NamedTemporaryFile
import cmdln
@ -15,7 +14,7 @@ import time
import abichecker_dbmodel as DB
import sqlalchemy.orm.exc
from xml.etree import cElementTree as ET
from lxml import etree as ET
import osc.conf
import osc.core

View File

@ -1 +0,0 @@
../osclib

View File

@ -1,11 +1,10 @@
#!/usr/bin/python3
from xml.etree import cElementTree as ET
from lxml import etree as ET
import sys
import cmdln
import logging
from urllib.error import HTTPError
import osc.core
import ToolBase

View File

@ -5,11 +5,10 @@ import time
import osc
import osc.core
import osc.conf
import xml.etree.ElementTree as ET
from lxml import etree as ET
import logging
import argparse
import sys
from collections import namedtuple
from osclib.util import mail_send_with_details
import email.utils

View File

@ -1,17 +1,6 @@
#!/usr/bin/python3
from pprint import pprint
import os
import re
import sys
import logging
from optparse import OptionParser
import cmdln
try:
from xml.etree import cElementTree as ET
except ImportError:
import cElementTree as ET
import osc.conf
import osc.core

View File

@ -6,12 +6,7 @@ import shutil
import subprocess
import sys
try:
from xml.etree import cElementTree as ET
except ImportError:
import cElementTree as ET
from lxml import etree as ETL
from lxml import etree as ET
import osc.conf
import osc.core
@ -346,7 +341,7 @@ class CheckSource(ReviewBot.ReviewBot):
)
if not self.required_maintainer: return True
meta = ETL.fromstringlist(show_project_meta(self.apiurl, source_project))
meta = ET.fromstringlist(show_project_meta(self.apiurl, source_project))
maintainers = meta.xpath('//person[@role="maintainer"]/@userid')
maintainers += ['group:' + g for g in meta.xpath('//group[@role="maintainer"]/@groupid')]

View File

@ -1,23 +1,10 @@
#!/usr/bin/python3
from pprint import pprint
import os
import sys
import re
import logging
from optparse import OptionParser
import cmdln
try:
from xml.etree import cElementTree as ET
except ImportError:
import cElementTree as ET
from lxml import etree as ET
import osc.conf
import osc.core
from urllib.error import HTTPError, URLError
import yaml
import ReviewBot

View File

@ -1,16 +1,10 @@
#!/usr/bin/python3
import sys
import re
import osc.conf
import osc.core
try:
from xml.etree import cElementTree as ET
except ImportError:
import cElementTree as ET
from lxml import etree as ET
from urllib.error import HTTPError, URLError
import ReviewBot

View File

@ -3,20 +3,12 @@
import argparse
import logging
import sys
try:
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError
import re
from xml.etree import cElementTree as ET
from urllib.error import HTTPError
from lxml import etree as ET
import osc.conf
import osc.core
from osc import oscerr
OPENSUSE = 'openSUSE:Leap:15.2'
SLE = 'SUSE:SLE-15-SP2:GA'

View File

@ -5,7 +5,6 @@ import io
import os
import sys
import logging
from optparse import OptionParser
import rpm
import pickle
import cmdln
@ -18,11 +17,6 @@ data_version = 3
changelog_max_lines = 100 # maximum number of changelog lines per package
try:
from xml.etree import cElementTree as ET
except ImportError:
import cElementTree as ET
class ChangeLogger(cmdln.Cmdln):
def __init__(self, *args, **kwargs):
cmdln.Cmdln.__init__(self, args, kwargs)

View File

@ -5,24 +5,17 @@ import logging
import sys
import time
try:
from urllib.error import HTTPError, URLError
except ImportError:
# python 2.x
from urllib2 import HTTPError, URLError
from urllib.error import HTTPError, URLError
import random
import re
from xml.etree import cElementTree as ET
from lxml import etree as ET
import osc.conf
import osc.core
from osclib.core import devel_project_get
from osclib.core import project_pseudometa_package
from osc import oscerr
from osclib.memoize import memoize
OPENSUSE = 'openSUSE:Leap:15.2'
OPENSUSE_PREVERSION = 'openSUSE:Leap:15.1'
OPENSUSE_RELEASED_VERSION = ['openSUSE:Leap:15.0', 'openSUSE:Leap:15.1']

View File

@ -23,7 +23,7 @@
import argparse
import os
import sys
from xml.etree import cElementTree as ET
from lxml import etree as ET
import osc.core
from osclib.core import target_archs

View File

@ -3,17 +3,16 @@
import argparse
import bugzilla
import dateutil.parser
from datetime import timedelta, datetime
from datetime import datetime
from dateutil.tz import tzlocal
import os
from random import shuffle
import requests.exceptions
import subprocess
import sys
import tempfile
from xmlrpclib import Fault
import yaml
from xml.etree import cElementTree as ET
from lxml import etree as ET
import osc.conf
import osc.core

View File

@ -1,12 +1,7 @@
from dateutil.parser import parse as date_parse
from metrics import timestamp
import requests
try:
from urllib.parse import urljoin
except ImportError:
# python 2.x
from urlparse import urljoin
from urllib.parse import urljoin
import yaml

View File

@ -14,7 +14,7 @@ import ReviewBot
from oqamaint.suse import SUSEUpdate
from xml.etree import cElementTree as ET
from lxml import etree as ET
import json
QA_UNKNOWN = 0

View File

@ -4,11 +4,7 @@ import re
from collections import namedtuple
import osc.core
from oqamaint.update import Update
try:
from xml.etree import cElementTree as ET
except ImportError:
from xml.etree import ElementTree as ET
from lxml import etree as ET
Package = namedtuple('Package', ('name', 'version', 'release'))

View File

@ -5,7 +5,7 @@ import requests
from gzip import decompress
from xml.etree import cElementTree as ET
from lxml import etree as ET
import osc.core
from osclib.memoize import memoize

View File

@ -1,6 +1,5 @@
import osc.core
from osc.core import get_dependson
from xml.etree import cElementTree as ET
from lxml import etree as ET
from osc import cmdln
@cmdln.option('-p', '--project', metavar='PROJECT', dest='project', default='openSUSE:Factory')

View File

@ -1,22 +1,17 @@
import re
import time
from urllib.error import HTTPError
import warnings
from xml.etree import cElementTree as ET
from lxml import etree as ET
from osc.core import change_request_state, show_package_meta, wipebinaries
from osc.core import http_GET, http_PUT, http_DELETE, http_POST
from osc.core import delete_package, search, meta_get_packagelist
from osc.core import Request
from osc.core import change_request_state
from osc.core import http_GET, http_POST
from osc.core import delete_package, meta_get_packagelist
from osc import conf
from osc.util.helper import decode_it
from osclib.core import attribute_value_save
from osclib.core import attribute_value_load
from osclib.core import source_file_load
from osclib.core import source_file_save
from osclib.request_finder import RequestFinder
from datetime import date

View File

@ -1,18 +1,11 @@
import json
from urllib.error import HTTPError
from colorama import Fore
from osc import oscerr
from osc.core import get_request
from osc.core import show_package_meta
from osc import conf
from osclib.select_command import SelectCommand
from osclib.supersede_command import SupersedeCommand
from osclib.request_finder import RequestFinder
from osclib.request_splitter import RequestSplitter
from xml.etree import cElementTree as ET
class AdiCommand:
def __init__(self, api):

View File

@ -7,7 +7,6 @@ import sys
from urllib.parse import unquote
from urllib.parse import urlsplit, SplitResult
from urllib.error import URLError, HTTPError
from io import BytesIO
from osc import conf
@ -16,7 +15,7 @@ from osclib.cache_manager import CacheManager
from osclib.conf import str2bool
from osclib.util import rmtree_nfs_safe
from time import time
from xml.etree import cElementTree as ET
from lxml import etree as ET
def http_request(method, url, headers={}, data=None, file=None):
"""

View File

@ -1,15 +1,10 @@
from xml.etree import cElementTree as ET
from lxml import etree as ET
from osc.core import makeurl
from osc.core import http_GET
from osclib.core import fileinfo_ext_all
from osclib.core import builddepinfo
try:
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError
from urllib.error import HTTPError
class CleanupRings(object):
def __init__(self, api):

View File

@ -1,7 +1,7 @@
from datetime import datetime
from dateutil.parser import parse as date_parse
import re
from xml.etree import cElementTree as ET
from lxml import etree as ET
from osc.core import http_DELETE
from osc.core import http_GET

View File

@ -1,12 +1,10 @@
from collections import namedtuple
from collections import OrderedDict
from datetime import datetime
from datetime import timezone
from dateutil.parser import parse as date_parse
import re
import socket
from xml.etree import cElementTree as ET
from lxml import etree as ETL
from lxml import etree as ET
from urllib.error import HTTPError
from osc.core import create_submit_request
@ -30,7 +28,6 @@ from osc.util.helper import decode_it
from osc import conf
from osclib.conf import Config
from osclib.memoize import memoize
import subprocess
import traceback
BINARY_REGEX = r'(?:.*::)?(?P<filename>(?P<name>.*)-(?P<version>[^-]+)-(?P<release>[^-]+)\.(?P<arch>[^-\.]+))'
@ -41,7 +38,7 @@ REQUEST_STATES_MINUS_ACCEPTED = ['new', 'review', 'declined', 'revoked', 'supers
@memoize(session=True)
def group_members(apiurl, group, maintainers=False):
url = makeurl(apiurl, ['group', group])
root = ETL.parse(http_GET(url)).getroot()
root = ET.parse(http_GET(url)).getroot()
if maintainers:
return root.xpath('maintainer/@userid')
@ -56,6 +53,11 @@ def groups_members(apiurl, groups):
return members
# osc uses xml.etree while we rely on lxml
def convert_from_osc_et(xml):
from xml.etree import ElementTree as oscET
return ET.fromstring(oscET.tostring(xml))
@memoize(session=True)
def owner_fallback(apiurl, project, package):
root = owner(apiurl, package, project=project)
@ -63,12 +65,12 @@ def owner_fallback(apiurl, project, package):
if not entry or project.startswith(entry.get('project')):
# Fallback to global (ex Factory) maintainer.
root = owner(apiurl, package)
return root
return convert_from_osc_et(root)
@memoize(session=True)
def maintainers_get(apiurl, project, package=None):
if package is None:
meta = ETL.fromstringlist(show_project_meta(apiurl, project))
meta = ET.fromstringlist(show_project_meta(apiurl, project))
maintainers = meta.xpath('//person[@role="maintainer"]/@userid')
groups = meta.xpath('//group[@role="maintainer"]/@groupid')
@ -76,9 +78,7 @@ def maintainers_get(apiurl, project, package=None):
return maintainers
# Ugly reparse, but real xpath makes the rest much cleaner.
root = owner_fallback(apiurl, project, package)
root = ETL.fromstringlist(ET.tostringlist(root))
maintainers = root.xpath('//person[@role="maintainer"]/@name')
groups = root.xpath('//group[@role="maintainer"]/@name')
@ -92,7 +92,7 @@ def package_role_expand(apiurl, project, package, role='maintainer', inherit=Tru
All users with a certain role on a package, including those who have the role directly assigned
and those who are part of a group with that role.
"""
meta = ETL.fromstringlist(show_package_meta(apiurl, project, package))
meta = ET.fromstringlist(show_package_meta(apiurl, project, package))
users = meta_role_expand(apiurl, meta, role)
if inherit:
@ -106,7 +106,7 @@ def project_role_expand(apiurl, project, role='maintainer'):
All users with a certain role on a project, including those who have the role directly assigned
and those who are part of a group with that role.
"""
meta = ETL.fromstringlist(show_project_meta(apiurl, project))
meta = ET.fromstringlist(show_project_meta(apiurl, project))
return meta_role_expand(apiurl, meta, role)
def meta_role_expand(apiurl, meta, role='maintainer'):
@ -129,7 +129,7 @@ def package_list(apiurl, project):
@memoize(session=True)
def target_archs(apiurl, project, repository='standard'):
meta = ETL.fromstringlist(show_project_meta(apiurl, project))
meta = ET.fromstringlist(show_project_meta(apiurl, project))
return meta.xpath('repository[@name="{}"]/arch/text()'.format(repository))
@memoize(session=True)
@ -260,7 +260,7 @@ def project_list_prefix(apiurl, prefix):
"""Get a list of project with the same prefix."""
query = {'match': 'starts-with(@name, "{}")'.format(prefix)}
url = makeurl(apiurl, ['search', 'project', 'id'], query)
root = ETL.parse(http_GET(url)).getroot()
root = ET.parse(http_GET(url)).getroot()
return root.xpath('project/@name')
def project_locked(apiurl, project):
@ -291,7 +291,7 @@ def builddepinfo(apiurl, project, repo, arch, order = False):
if order:
query['view'] = 'order'
url = makeurl(apiurl, ['build', project, repo, arch, '_builddepinfo'], query)
return ETL.parse(http_GET(url)).getroot()
return ET.parse(http_GET(url)).getroot()
def entity_email(apiurl, key, entity_type='person', include_name=False):
url = makeurl(apiurl, [entity_type, key])
@ -363,7 +363,7 @@ def package_list_kind_filtered(apiurl, project, kinds_allowed=['source']):
'nofilename': '1',
}
url = makeurl(apiurl, ['source', project], query)
root = ETL.parse(http_GET(url)).getroot()
root = ET.parse(http_GET(url)).getroot()
for package in root.xpath('sourceinfo/@package'):
kind = package_kind(apiurl, project, package)
@ -377,7 +377,7 @@ def attribute_value_load(apiurl, project, name, namespace='OSRT', package=None):
url = makeurl(apiurl, path)
try:
root = ETL.parse(http_GET(url)).getroot()
root = ET.parse(http_GET(url)).getroot()
except HTTPError as e:
if e.code == 404:
return None
@ -445,7 +445,7 @@ def repository_path_search(apiurl, project, search_project, search_repository):
queue = []
# Initialize breadth first search queue with repositories from top project.
root = ETL.fromstringlist(show_project_meta(apiurl, project))
root = ET.fromstringlist(show_project_meta(apiurl, project))
for repository in root.xpath('repository[path[@project and @repository]]/@name'):
queue.append((repository, project, repository))
@ -454,7 +454,7 @@ def repository_path_search(apiurl, project, search_project, search_repository):
for repository_top, project, repository in queue:
if root.get('name') != project:
# Repositories for a single project are in a row so cache parsing.
root = ETL.fromstringlist(show_project_meta(apiurl, project))
root = ET.fromstringlist(show_project_meta(apiurl, project))
paths = root.findall('repository[@name="{}"]/path'.format(repository))
for path in paths:
@ -515,7 +515,7 @@ def repository_published(apiurl, project, repository, archs=[]):
archs = list(archs)
archs.append('i586')
root = ETL.fromstringlist(show_results_meta(
root = ET.fromstringlist(show_results_meta(
apiurl, project, multibuild=True, repository=[repository], arch=archs))
return not len(root.xpath('result[@state!="published" and @state!="unpublished"]'))
@ -533,7 +533,7 @@ def project_meta_revision(apiurl, project):
def package_source_changed(apiurl, project, package):
url = makeurl(apiurl, ['source', project, package, '_history'], {'limit': 1})
root = ETL.parse(http_GET(url)).getroot()
root = ET.parse(http_GET(url)).getroot()
return datetime.fromtimestamp(int(root.find('revision/time').text), timezone.utc).replace(tzinfo=None)
def package_source_age(apiurl, project, package):
@ -562,7 +562,7 @@ def package_kind(apiurl, project, package):
try:
url = makeurl(apiurl, ['source', project, package, '_meta'])
root = ETL.parse(http_GET(url)).getroot()
root = ET.parse(http_GET(url)).getroot()
except HTTPError as e:
if e.code == 404:
return None
@ -590,7 +590,7 @@ def entity_source_link(apiurl, project, package=None):
else:
parts = ['source', project, '_meta']
url = makeurl(apiurl, parts)
root = ETL.parse(http_GET(url)).getroot()
root = ET.parse(http_GET(url)).getroot()
except HTTPError as e:
if e.code == 404:
return None
@ -631,7 +631,7 @@ def package_source_hash(apiurl, project, package, revision=None):
try:
url = makeurl(apiurl, ['source', project, package], query)
root = ETL.parse(http_GET(url)).getroot()
root = ET.parse(http_GET(url)).getroot()
except HTTPError as e:
if e.code == 400 or e.code == 404:
# 400: revision not found, 404: package not found.
@ -649,7 +649,7 @@ def package_source_hash(apiurl, project, package, revision=None):
def package_source_hash_history(apiurl, project, package, limit=5, include_project_link=False):
try:
# get_commitlog() reverses the order so newest revisions are first.
root = ETL.fromstringlist(
root = ET.fromstringlist(
get_commitlog(apiurl, project, package, None, format='xml'))
except HTTPError as e:
if e.code == 404:
@ -692,7 +692,7 @@ def package_source_hash_history(apiurl, project, package, limit=5, include_proje
def package_version(apiurl, project, package):
try:
url = makeurl(apiurl, ['source', project, package, '_history'], {'limit': 1})
root = ETL.parse(http_GET(url)).getroot()
root = ET.parse(http_GET(url)).getroot()
except HTTPError as e:
if e.code == 404:
return False
@ -801,7 +801,7 @@ def issue_trackers(apiurl):
def issue_tracker_by_url(apiurl, tracker_url):
url = makeurl(apiurl, ['issue_trackers'])
root = ETL.parse(http_GET(url)).getroot()
root = ET.parse(http_GET(url)).getroot()
if not tracker_url.endswith('/'):
# All trackers are formatted with trailing slash.
tracker_url += '/'
@ -854,7 +854,7 @@ def duplicated_binaries_in_repo(apiurl, project, repository):
def search(apiurl, path, xpath, query={}):
query['match'] = xpath
url = makeurl(apiurl, ['search', path], query)
return ETL.parse(http_GET(url)).getroot()
return ET.parse(http_GET(url)).getroot()
def action_is_patchinfo(action):
return (action.type == 'maintenance_incident' and (
@ -1180,4 +1180,4 @@ def message_suffix(action, message=None):
def request_state_change(apiurl, request_id, state):
query = { 'newstate': state, 'cmd': 'changestate' }
url = makeurl(apiurl, ['request', request_id], query)
return ETL.parse(http_POST(url)).getroot().get('code')
return ET.parse(http_POST(url)).getroot().get('code')

View File

@ -1,8 +1,5 @@
from osclib.core import source_file_load
from osclib.core import source_file_save
import time
import re
from xml.etree import cElementTree as ET
from lxml import etree as ET
MAX_FROZEN_AGE = 6.5

View File

@ -1,20 +1,14 @@
from __future__ import print_function
from datetime import datetime
import time
import warnings
from xml.etree import cElementTree as ET
from lxml import etree as ET
from osc import conf
from osc.core import makeurl
from osc.core import http_GET
from osc.core import http_POST
try:
from urllib.error import HTTPError
except ImportError:
# python 2.x
from urllib2 import HTTPError
from urllib.error import HTTPError
class OBSLock(object):
"""Implement a distributed lock using a shared OBS resource."""

View File

@ -6,7 +6,7 @@ import subprocess
import tempfile
import glob
from fnmatch import fnmatch
from lxml import etree as ETL
from lxml import etree as ET
from osc.core import http_GET
import yaml
@ -171,7 +171,7 @@ def installcheck(directories, arch, whitelist, ignore_conflicts):
def mirrorRepomd(cachedir, url):
# Use repomd.xml to get the location of primary.xml.gz
repoindex = ETL.fromstring(requests.get('{}/repodata/repomd.xml'.format(url)).content)
repoindex = ET.fromstring(requests.get('{}/repodata/repomd.xml'.format(url)).content)
primarypath = repoindex.xpath("string(./repo:data[@type='primary']/repo:location/@href)",
namespaces={'repo': 'http://linux.duke.edu/metadata/repo'})
if not primarypath.endswith(".xml.gz"):
@ -195,7 +195,7 @@ def mirror(apiurl, project, repository, arch):
if not os.path.exists(directory):
os.makedirs(directory)
meta = ETL.parse(http_GET('{}/public/source/{}/_meta'.format(apiurl, project))).getroot()
meta = ET.parse(http_GET('{}/public/source/{}/_meta'.format(apiurl, project))).getroot()
repotag = meta.xpath("/project/repository[@name='{}']".format(repository))[0]
if arch not in repotag.xpath("./arch/text()"):
# Arch not in this project, skip mirroring

View File

@ -1,4 +1,4 @@
from xml.etree import cElementTree as ET
from lxml import etree as ET
from osc import oscerr
from osc.core import http_GET

View File

@ -2,11 +2,10 @@
from configparser import ConfigParser
from xdg.BaseDirectory import load_first_config
from xml.etree import cElementTree as ET
from lxml import etree as ET
import sys
import cmdln
import logging
import os
import osc.core

View File

@ -6,14 +6,12 @@ import sys
from urllib.error import HTTPError
import re
import yaml
from xml.etree import cElementTree as ET
from lxml import etree as ET
from collections import defaultdict
import osc.conf
import osc.core
from osc import oscerr
from osc.util.helper import decode_list
from osclib.conf import Config
from osclib.stagingapi import StagingAPI

View File

@ -8,11 +8,7 @@ import httpretty
import osc
from . import OBSLocal
try:
from urllib.parse import urlparse, parse_qs
except ImportError:
# python 2.x
from urlparse import urlparse, parse_qs
from urllib.parse import urlparse, parse_qs
import sys
import re

View File

@ -2,15 +2,9 @@ import os
import unittest
import logging
import httpretty
import osc
import re
from . import OBSLocal
try:
from urllib.parse import urlparse, parse_qs
except ImportError:
# python 2.x
from urlparse import urlparse, parse_qs
from urllib.parse import urlparse, parse_qs
from check_maintenance_incidents import MaintenanceChecker

View File

@ -13,7 +13,7 @@ import logging
import re
import yaml
from enum import IntEnum
from xml.etree import cElementTree as ET
from lxml import etree as ET
from osclib.stagingapi import StagingAPI
from urllib.error import HTTPError
from ttm.totest import ToTest

View File

@ -9,7 +9,7 @@
# Distribute under GPLv2 or GPLv3
import re
from xml.etree import cElementTree as ET
from lxml import etree as ET
from ttm.manager import ToTestManager, NotFoundException, QAResult