326a087168
With the python shebang remaining as Python 2, switch the scripts to using Python 3. This also allows us to clean up some imports.
228 lines
9.2 KiB
Python
Executable File
228 lines
9.2 KiB
Python
Executable File
#!/usr/bin/python3
|
|
|
|
from copy import deepcopy
|
|
from lxml import etree as ET
|
|
from osc.core import copy_pac as copy_package
|
|
from osc.core import get_commitlog
|
|
from osc.core import http_GET
|
|
from osc.core import http_POST
|
|
from osc.core import http_PUT
|
|
from osc.core import makeurl
|
|
from osc.core import show_upstream_rev
|
|
from osclib.core import project_pseudometa_package
|
|
from urllib.error import HTTPError
|
|
|
|
import argparse
|
|
import osc.conf
|
|
import sys
|
|
|
|
|
|
def project_fence(project):
|
|
if ((project.startswith('openSUSE:') and project_fence.project.startswith('openSUSE:')) and
|
|
not project.startswith(project_fence.project)):
|
|
# Exclude other openSUSE:* projects while cloning a specifc one.
|
|
return False
|
|
if project.startswith('openSUSE:Factory:ARM'):
|
|
# Troublesome.
|
|
return False
|
|
# Perhaps use devel project list as filter, but for now quick exclude.
|
|
if project.startswith('SUSE:') or project.startswith('Ubuntu:'):
|
|
return False
|
|
|
|
return True
|
|
|
|
def entity_clone(apiurl_source, apiurl_target, path, sanitize=None, clone=None, after=None):
|
|
if not hasattr(entity_clone, 'cloned'):
|
|
entity_clone.cloned = []
|
|
|
|
if path[0] == 'source' and not project_fence(path[1]):
|
|
# Skip projects outside of fence by marking as cloned.
|
|
if path not in entity_clone.cloned:
|
|
entity_clone.cloned.append(path)
|
|
|
|
if path in entity_clone.cloned:
|
|
print('skip {}'.format('/'.join(path)))
|
|
return
|
|
|
|
print('clone {}'.format('/'.join(path)))
|
|
entity_clone.cloned.append(path)
|
|
|
|
url = makeurl(apiurl_source, path)
|
|
entity = ET.parse(http_GET(url)).getroot()
|
|
|
|
if sanitize:
|
|
sanitize(entity)
|
|
if clone:
|
|
clone(apiurl_source, apiurl_target, entity)
|
|
|
|
url = makeurl(apiurl_target, path)
|
|
http_PUT(url, data=ET.tostring(entity))
|
|
|
|
if after:
|
|
after(apiurl_source, apiurl_target, entity)
|
|
|
|
def users_clone(apiurl_source, apiurl_target, entity):
|
|
for person in entity.findall('person'):
|
|
path = ['person', person.get('userid')]
|
|
entity_clone(apiurl_source, apiurl_target, path, person_sanitize, after=person_clone_after)
|
|
|
|
for group in entity.findall('group'):
|
|
path = ['group', group.get('groupid')]
|
|
entity_clone(apiurl_source, apiurl_target, path, clone=group_clone)
|
|
|
|
def project_references_remove(project):
|
|
# Remove links that reference other projects.
|
|
for link in project.xpath('link[@project]'):
|
|
link.getparent().remove(link)
|
|
|
|
# Remove repositories that reference other projects.
|
|
for repository in project.xpath('repository[releasetarget or path]'):
|
|
repository.getparent().remove(repository)
|
|
|
|
# clone(Factory)
|
|
# - stripped
|
|
# - after
|
|
# - clone(Factory:ToTest)
|
|
# - stripped
|
|
# - after
|
|
# - clone(Factory)...skip
|
|
# - write real
|
|
# - write real
|
|
def project_clone(apiurl_source, apiurl_target, project):
|
|
users_clone(apiurl_source, apiurl_target, project)
|
|
project_workaround(project)
|
|
|
|
# Write stripped version that does not include repos with path references.
|
|
url = makeurl(apiurl_target, ['source', project.get('name'), '_meta'])
|
|
stripped = deepcopy(project)
|
|
project_references_remove(stripped)
|
|
http_PUT(url, data=ET.tostring(stripped))
|
|
|
|
for link in project.xpath('link[@project]'):
|
|
if not project_fence(link.get('project')):
|
|
project.remove(link)
|
|
break
|
|
|
|
# Valid reference to project and thus should be cloned.
|
|
path = ['source', link.get('project'), '_meta']
|
|
entity_clone(apiurl_source, apiurl_target, path, clone=project_clone)
|
|
|
|
# Clone projects referenced in repository paths.
|
|
for repository in project.findall('repository'):
|
|
for target in repository.xpath('./path') + repository.xpath('./releasetarget'):
|
|
if not project_fence(target.get('project')):
|
|
project.remove(repository)
|
|
break
|
|
|
|
# Valid reference to project and thus should be cloned.
|
|
path = ['source', target.get('project'), '_meta']
|
|
entity_clone(apiurl_source, apiurl_target, path, clone=project_clone)
|
|
|
|
def project_workaround(project):
|
|
if project.get('name') == 'openSUSE:Factory':
|
|
# See #1335 for details about temporary workaround in revision 429, but
|
|
# suffice it to say that over-complicates clone with multiple loops and
|
|
# may be introduced from time to time when Factory repo is hosed.
|
|
scariness = project.xpath('repository[@name="standard"]/path[contains(@project, ":0-Bootstrap")]')
|
|
if len(scariness):
|
|
scariness[0].getparent().remove(scariness[0])
|
|
|
|
def package_clone(apiurl_source, apiurl_target, package):
|
|
# Clone project that contains the package.
|
|
path = ['source', package.get('project'), '_meta']
|
|
entity_clone(apiurl_source, apiurl_target, path, clone=project_clone)
|
|
|
|
# Clone the dependencies of package.
|
|
users_clone(apiurl_source, apiurl_target, package)
|
|
|
|
# Clone devel project referenced by package.
|
|
devel = package.find('devel')
|
|
if devel is not None:
|
|
path = ['source', devel.get('project'), devel.get('package'), '_meta']
|
|
entity_clone(apiurl_source, apiurl_target, path, clone=package_clone, after=package_clone_after)
|
|
|
|
def package_clone_after(apiurl_source, apiurl_target, package):
|
|
copy_package(apiurl_source, package.get('project'), package.get('name'),
|
|
apiurl_target, package.get('project'), package.get('name'),
|
|
# TODO Would be preferable to preserve links, but need to
|
|
# recreat them since they do not match with copied package.
|
|
expand=True,
|
|
# TODO Can copy server-side if inner-connect is setup, but not
|
|
# clear how to trigger the equivalent of save in admin UI.
|
|
client_side_copy=True)
|
|
|
|
def person_sanitize(person):
|
|
person.find('email').text = person.find('email').text.split('@')[0] + '@example.com'
|
|
watchlist = person.find('watchlist')
|
|
if watchlist:
|
|
person.remove(watchlist)
|
|
|
|
def person_clone_after(apiurl_source, apiurl_target, person):
|
|
url = makeurl(apiurl_target, ['person', person.find('login').text], {'cmd': 'change_password'})
|
|
http_POST(url, data='opensuse')
|
|
|
|
def group_clone(apiurl_source, apiurl_target, group):
|
|
for person in group.findall('maintainer') + group.findall('person/person'):
|
|
path = ['person', person.get('userid')]
|
|
entity_clone(apiurl_source, apiurl_target, path, person_sanitize, after=person_clone_after)
|
|
|
|
def clone_do(apiurl_source, apiurl_target, project):
|
|
print('clone {} from {} to {}'.format(project, apiurl_source, apiurl_target))
|
|
|
|
try:
|
|
# TODO Decide how to choose what to clone via args.
|
|
|
|
# Rather than handle the self-referencing craziness with a proper solver
|
|
# the leaf can simple be used to start the chain and works as desired.
|
|
# Disable this when running clone repeatedly during developing as the
|
|
# projects cannot be cleanly re-created without more work.
|
|
entity_clone(apiurl_source, apiurl_target, ['source', project + ':Rings:1-MinimalX', '_meta'],
|
|
clone=project_clone)
|
|
|
|
pseudometa_project, pseudometa_package = project_pseudometa_package(apiurl_source, project)
|
|
entity_clone(apiurl_source, apiurl_target, ['source', pseudometa_project, pseudometa_package, '_meta'],
|
|
clone=package_clone, after=package_clone_after)
|
|
|
|
entity_clone(apiurl_source, apiurl_target, ['source', project, 'drush', '_meta'],
|
|
clone=package_clone, after=package_clone_after)
|
|
|
|
entity_clone(apiurl_source, apiurl_target, ['group', 'opensuse-review-team'],
|
|
clone=group_clone)
|
|
except HTTPError as e:
|
|
# Print full output for any errors since message can be cryptic.
|
|
print(e.read())
|
|
return 1
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(description='Clone projects and dependencies between OBS instances.')
|
|
parser.set_defaults(func=clone_do)
|
|
|
|
parser.add_argument('-S', '--apiurl-source', metavar='URL', help='source API URL')
|
|
parser.add_argument('-T', '--apiurl-target', metavar='URL', help='target API URL')
|
|
parser.add_argument('-c', '--cache', action='store_true', help='cache source queries for 24 hours')
|
|
parser.add_argument('-d', '--debug', action='store_true', help='print info useful for debuging')
|
|
parser.add_argument('-p', '--project', default='openSUSE:Factory', help='project from which to clone')
|
|
|
|
args = parser.parse_args()
|
|
|
|
osc.conf.get_config(override_apiurl=args.apiurl_target)
|
|
apiurl_target = osc.conf.config['apiurl']
|
|
osc.conf.get_config(override_apiurl=args.apiurl_source)
|
|
apiurl_source = osc.conf.config['apiurl']
|
|
|
|
if apiurl_target == apiurl_source:
|
|
print('target APIURL must not be the same as source APIURL')
|
|
sys.exit(1)
|
|
|
|
if args.cache:
|
|
from osclib.cache import Cache
|
|
Cache.PATTERNS = {}
|
|
# Prevent caching source information from local clone.
|
|
Cache.PATTERNS['/source/[^/]+/[^/]+/[^/]+?rev'] = 0
|
|
Cache.PATTERNS['.*'] = Cache.TTL_LONG * 2
|
|
Cache.init('clone')
|
|
|
|
osc.conf.config['debug'] = args.debug
|
|
project_fence.project = args.project
|
|
sys.exit(args.func(apiurl_source, apiurl_target, args.project))
|