diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..c979f7fb --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.open-build-service/ diff --git a/.flake8 b/.flake8 index cfe6ec8c..ea7bc083 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,4 @@ [flake8] -exclude = abichecker, openqa, openqa-maintenance.py +exclude = .open-build-service/, abichecker, openqa, openqa-maintenance.py max-line-length = 100 ignore = E501,F401,E302,E228,E128,E251,E201,E202,E203,E305,F841,E265,E261,E266,E712,E401,E126,E502,E222,E241,E711,E226,E125,E123,W293,W391,E731,E101,E227,E713,E225,E124,E221,E127,E701,E714,W503,E129,E303,E741,E722 diff --git a/.gitignore b/.gitignore index 6207308e..b7552d2f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ .coverage .coveralls.yml .docker-tmp/ +.open-build-service/ diff --git a/.travis.yml b/.travis.yml index 5d8ad413..a7080731 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,16 +49,26 @@ matrix: - flake8 - env: TEST_SUITE=nosetests sudo: required + services: + - docker language: python python: 2.7 before_install: # provides xmllint used by test_bootstrap_copy (tests.freeze_tests.TestFreeze) - sudo apt-get install libxml2-utils install: - # needed to install osc from git in requirements.txt - - pip install pycurl urlgrabber + # urlgrabber needed to install osc from git in requirements.txt + # m2crypto for osc to be runable as used in docker-compose-obs + - pip install pycurl urlgrabber m2crypto - pip install -r requirements.txt - pip install python-coveralls + before_script: + # travis-ci/travis-ci#7008: stop services to make room for OBS setup + - sudo service mysql stop + - sudo service memcached stop + - ./dist/ci/docker-compose-obs + # Needs python prefix to use the correct interpretor. + - python ./obs_clone.py --cache --debug --apiurl-target local script: - nosetests --with-coverage --cover-package=. --cover-inclusive after_success: @@ -73,10 +83,13 @@ deploy: cache: directories: + # obs_clone.py + - ~/.cache/osc-plugin-factory-clone + # distribution jobs `osc build` - .docker-tmp pip: true # 7 days timeout: 604800 before_cache: - sudo rm -rf .docker-tmp/build-root - - sudo chmod o+rw .docker-tmp + - sudo chmod -R o+rw .docker-tmp diff --git a/devel-project.py b/devel-project.py index 55dc4bf7..bbb74129 100755 --- a/devel-project.py +++ b/devel-project.py @@ -41,7 +41,8 @@ def devel_projects_get(apiurl, project): devel_projects[devel.attrib['project']] = True # Ensure self does not end up in list. - del devel_projects[project] + if project in devel_projects: + del devel_projects[project] return sorted(devel_projects) diff --git a/dist/ci/docker-compose-obs b/dist/ci/docker-compose-obs new file mode 100755 index 00000000..79f8b6c5 --- /dev/null +++ b/dist/ci/docker-compose-obs @@ -0,0 +1,48 @@ +#!/bin/bash + +set -e + +OBS_DIRECTORY=${OBS_DIRECTORY:-.open-build-service} + +if [ ! -d "$OBS_DIRECTORY" ] ; then + git clone --depth 1 https://github.com/openSUSE/open-build-service.git "$OBS_DIRECTORY" + + if [ ! -f ~/.oscrc ] ; then + # openSUSE/open-build-service#3946: would elevate the need, but rejected. + eval $(./dist/ci/osc-credentials-throwaway) + OBS_USER="$user" OBS_PASS="$pass" ./dist/ci/osc-init + fi + +cat <> ~/.oscrc + +[http://0.0.0.0:3000] +user = Admin +pass = opensuse +aliases = local +EOF +fi + +cd "$OBS_DIRECTORY" + +git pull origin master + +# https://github.com/openSUSE/open-build-service/blob/master/CONTRIBUTING.md +git submodule init +git submodule update + +rake docker:build + +# See `docker-compose logs` instead of changing output redirection. +docker-compose up -d --remove-orphans + +osc='osc' +if ! which osc &> /dev/null ; then + # Workaround for pip install that occurs on travis. + osc='osc-wrapper.py' +fi + +until $osc -q -A local api /about 2> /dev/null ; do + echo "waiting for OBS to be responsive..." + ((c++)) && ((c==60)) && docker-compose logs && exit 1 + sleep 1 +done diff --git a/dist/package/openSUSE-release-tools.spec b/dist/package/openSUSE-release-tools.spec index 5c4aa2a4..3b909663 100644 --- a/dist/package/openSUSE-release-tools.spec +++ b/dist/package/openSUSE-release-tools.spec @@ -251,7 +251,8 @@ make %{?_smp_mflags} %check %if 0%{?is_opensuse} -make check +# TODO openSUSE/osc-plugin-factory#1221: decide how to handle integration tests +# make check %endif %install diff --git a/obs_clone.py b/obs_clone.py new file mode 100755 index 00000000..5012d0fb --- /dev/null +++ b/obs_clone.py @@ -0,0 +1,190 @@ +#!/usr/bin/python + +from copy import deepcopy +from lxml import etree as ET +from osc.core import copy_pac as copy_package +from osc.core import get_commitlog +from osc.core import http_GET +from osc.core import http_POST +from osc.core import http_PUT +from osc.core import makeurl +from osc.core import show_upstream_rev +from urllib2 import HTTPError +import argparse +import osc.conf +import sys + + +def project_fence(project): + if ((project.startswith('openSUSE:') and project_fence.project.startswith('openSUSE:')) and + not project.startswith(project_fence.project)): + # Exclude other openSUSE:* projects while cloning a specifc one. + return False + if project.startswith('openSUSE:Factory:ARM'): + # Troublesome. + return False + # Perhaps use devel project list as filter, but for now quick exclude. + if project.startswith('SUSE:') or project.startswith('Ubuntu:'): + return False + + return True + +def entity_clone(apiurl_source, apiurl_target, path, sanitize=None, clone=None, after=None): + if not hasattr(entity_clone, 'cloned'): + entity_clone.cloned = [] + + if path[0] == 'source' and not project_fence(path[1]): + # Skip projects outside of fence by marking as cloned. + if path not in entity_clone.cloned: + entity_clone.cloned.append(path) + + if path in entity_clone.cloned: + print('skip {}'.format('/'.join(path))) + return + + print('clone {}'.format('/'.join(path))) + entity_clone.cloned.append(path) + + url = makeurl(apiurl_source, path) + entity = ET.parse(http_GET(url)).getroot() + + if sanitize: + sanitize(entity) + if clone: + clone(apiurl_source, apiurl_target, entity) + + url = makeurl(apiurl_target, path) + http_PUT(url, data=ET.tostring(entity)) + + if after: + after(apiurl_source, apiurl_target, entity) + +def users_clone(apiurl_source, apiurl_target, entity): + for person in entity.findall('person'): + path = ['person', person.get('userid')] + entity_clone(apiurl_source, apiurl_target, path, person_sanitize, after=person_clone_after) + + for group in entity.findall('group'): + path = ['group', group.get('groupid')] + entity_clone(apiurl_source, apiurl_target, path, clone=group_clone) + +def project_repositories_remove(project): + # Remove repositories that reference other projects. + for repository in project.xpath('repository[releasetarget or path]'): + repository.getparent().remove(repository) + +# clone(Factory) +# - stripped +# - after +# - clone(Factory:ToTest) +# - stripped +# - after +# - clone(Factory)...skip +# - write real +# - write real +def project_clone(apiurl_source, apiurl_target, project): + users_clone(apiurl_source, apiurl_target, project) + + # Write stripped version that does not include repos with path references. + url = makeurl(apiurl_target, ['source', project.get('name'), '_meta']) + stripped = deepcopy(project) + project_repositories_remove(stripped) + http_PUT(url, data=ET.tostring(stripped)) + + # Clone projects referenced in repository paths. + for repository in project.findall('repository'): + for target in repository.xpath('./path') + repository.xpath('./releasetarget'): + if not project_fence(target.get('project')): + project.remove(repository) + break + + # Valid reference to project and thus should be cloned. + path = ['source', target.get('project'), '_meta'] + entity_clone(apiurl_source, apiurl_target, path, clone=project_clone) + +def package_clone(apiurl_source, apiurl_target, package): + # Clone project that contains the package. + path = ['source', package.get('project'), '_meta'] + entity_clone(apiurl_source, apiurl_target, path, clone=project_clone) + + # Clone the dependencies of package. + users_clone(apiurl_source, apiurl_target, package) + + # Clone devel project referenced by package. + devel = package.find('devel') + if devel is not None: + path = ['source', devel.get('project'), devel.get('package'), '_meta'] + entity_clone(apiurl_source, apiurl_target, path, clone=package_clone, after=package_clone_after) + +def package_clone_after(apiurl_source, apiurl_target, package): + copy_package(apiurl_source, package.get('project'), package.get('name'), + apiurl_target, package.get('project'), package.get('name'), + # TODO Would be preferable to preserve links, but need to + # recreat them since they do not match with copied package. + expand=True, + # TODO Can copy server-side if inner-connect is setup, but not + # clear how to trigger the equivalent of save in admin UI. + client_side_copy=True) + +def person_sanitize(person): + person.find('email').text = person.find('email').text.split('@')[0] + '@example.com' + +def person_clone_after(apiurl_source, apiurl_target, person): + url = makeurl(apiurl_target, ['person', person.find('login').text], {'cmd': 'change_password'}) + http_POST(url, data='opensuse') + +def group_clone(apiurl_source, apiurl_target, group): + for person in group.findall('maintainer') + group.findall('person/person'): + path = ['person', person.get('userid')] + entity_clone(apiurl_source, apiurl_target, path, person_sanitize, after=person_clone_after) + +def clone_do(apiurl_source, apiurl_target, project): + print('clone {} from {} to {}'.format(project, apiurl_source, apiurl_target)) + + try: + # TODO Decide how to choose what to clone via args. + entity_clone(apiurl_source, apiurl_target, ['source', project + ':Staging', 'dashboard', '_meta'], + clone=package_clone, after=package_clone_after) + + entity_clone(apiurl_source, apiurl_target, ['source', project, 'drush', '_meta'], + clone=package_clone, after=package_clone_after) + + entity_clone(apiurl_source, apiurl_target, ['group', 'opensuse-review-team'], + clone=group_clone) + except HTTPError as e: + # Print full output for any errors since message can be cryptic. + print(e.read()) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Clone projects and dependencies between OBS instances.') + parser.set_defaults(func=clone_do) + + parser.add_argument('-S', '--apiurl-source', metavar='URL', help='source API URL') + parser.add_argument('-T', '--apiurl-target', metavar='URL', help='target API URL') + parser.add_argument('-c', '--cache', action='store_true', help='cache source queries for 24 hours') + parser.add_argument('-d', '--debug', action='store_true', help='print info useful for debuging') + parser.add_argument('-p', '--project', default='openSUSE:Factory', help='project from which to clone') + + args = parser.parse_args() + + osc.conf.get_config(override_apiurl=args.apiurl_target) + apiurl_target = osc.conf.config['apiurl'] + osc.conf.get_config(override_apiurl=args.apiurl_source) + apiurl_source = osc.conf.config['apiurl'] + + if apiurl_target == apiurl_source: + print('target APIURL must not be the same as source APIURL') + sys.exit(1) + + if args.cache: + from osclib.cache import Cache + Cache.CACHE_DIR = Cache.CACHE_DIR + '-clone' + Cache.PATTERNS = {} + # Prevent caching source information from local clone. + Cache.PATTERNS['/source/[^/]+/[^/]+/[^/]+?rev'] = 0 + Cache.PATTERNS['.*'] = Cache.TTL_LONG * 2 + Cache.init() + + osc.conf.config['debug'] = args.debug + project_fence.project = args.project + sys.exit(args.func(apiurl_source, apiurl_target, args.project)) diff --git a/osclib/cache.py b/osclib/cache.py index b6b1171e..59838cfd 100644 --- a/osclib/cache.py +++ b/osclib/cache.py @@ -167,6 +167,9 @@ class Cache(object): match, project = Cache.match(url) if match: path = Cache.path(url, project, include_file=True, makedirs=True) + ttl = Cache.PATTERNS[match] + if ttl == 0: + return data # Since urlopen does not return a seekable stream it cannot be reset # after writing to cache. As such a wrapper must be used. This could diff --git a/tests/OBSLocal.py b/tests/OBSLocal.py new file mode 100644 index 00000000..63f602c0 --- /dev/null +++ b/tests/OBSLocal.py @@ -0,0 +1,96 @@ +import os +from lxml import etree as ET +from osc import conf +from osc.core import get_request +from osc.core import http_GET +from osc.core import makeurl +import subprocess +import unittest + +OSCRC = os.path.expanduser('~/.oscrc-test') +APIURL = 'local-test' + +class OBSLocalTestCase(unittest.TestCase): + script = None + script_apiurl = True + script_debug = True + script_debug_osc = True + + @classmethod + def setUpClass(cls): + # TODO #1214: Workaround for tests/obs.py's lack of cleanup. + import httpretty + httpretty.disable() + + def setUp(self): + self.oscrc('Admin') + conf.get_config(override_conffile=OSCRC, + override_no_keyring=True, + override_no_gnome_keyring=True) + self.apiurl = conf.config['apiurl'] + self.assertOBS() + + def assertOBS(self): + url = makeurl(self.apiurl, ['about']) + root = ET.parse(http_GET(url)).getroot() + self.assertEqual(root.tag, 'about') + + @staticmethod + def oscrc(userid): + with open(OSCRC, 'w+') as f: + f.write('\n'.join([ + '[general]', + 'apiurl = http://0.0.0.0:3000', + '[http://0.0.0.0:3000]', + 'user = {}'.format(userid), + 'pass = opensuse', + 'email = {}@example.com'.format(userid), + 'aliases = {}'.format(APIURL), + '', + ])) + + def osc_user(self, userid): + conf.config['api_host_options'][self.apiurl]['user'] = userid + self.oscrc(userid) + + def execute_script(self, args): + if self.script: + args.insert(0, self.script) + if self.script_debug: + args.insert(1, '--debug') + if self.script_debug_osc: + args.insert(1, '--osc-debug') + args.insert(0, '-p') + args.insert(0, 'run') + args.insert(0, 'coverage') + + self.execute(args) + + def execute_osc(self, args): + # The wrapper allows this to work properly when osc installed via pip. + args.insert(0, 'osc-wrapper.py') + self.execute(args) + + def execute(self, args): + print('$ ' + ' '.join(args)) # Print command for debugging. + try: + env = os.environ + env['OSC_CONFIG'] = OSCRC + self.output = subprocess.check_output(args, stderr=subprocess.STDOUT, env=env) + except subprocess.CalledProcessError as e: + print(e.output) + raise e + print(self.output) # For debugging assertion failures. + + def assertOutput(self, string): + self.assertTrue(string in self.output, '[MISSING] ' + string) + + def assertReview(self, rid, **kwargs): + request = get_request(self.apiurl, rid) + for review in request.reviews: + for key, value in kwargs.items(): + if hasattr(review, key) and getattr(review, key) == value[0]: + self.assertEqual(review.state, value[1], '{}={} not {}'.format(key, value[0], value[1])) + return + + self.fail('{} not found'.format(kwargs)) diff --git a/tests/devel_project_tests.py b/tests/devel_project_tests.py new file mode 100644 index 00000000..b9ed2043 --- /dev/null +++ b/tests/devel_project_tests.py @@ -0,0 +1,21 @@ +from OBSLocal import OBSLocalTestCase +import unittest + + +class TestDevelProject(OBSLocalTestCase): + script = './devel-project.py' + script_debug_osc = False + + def test_list(self): + self.osc_user('staging-bot') + self.execute_script(['list', '--write']) + self.assertOutput('server:php:applications') + # TODO Assert --write worked and in file. + + def test_reviews(self): + self.osc_user('staging-bot') + self.execute_script(['reviews']) + + def test_requests(self): + self.osc_user('staging-bot') + self.execute_script(['requests'])