Merge pull request #1207 from jberry-suse/obs-docker-compose-test-rebase

Provide local OBS setup, base data creation, test harness, and initial set of tests.

.dockerignore

@@ -0,0 +1 @@
+.open-build-service/

.flake8

@@ -1,4 +1,4 @@
 [flake8]
-exclude = abichecker, openqa, openqa-maintenance.py
+exclude = .open-build-service/, abichecker, openqa, openqa-maintenance.py
 max-line-length = 100
 ignore = E501,F401,E302,E228,E128,E251,E201,E202,E203,E305,F841,E265,E261,E266,E712,E401,E126,E502,E222,E241,E711,E226,E125,E123,W293,W391,E731,E101,E227,E713,E225,E124,E221,E127,E701,E714,W503,E129,E303,E741,E722

.gitignore

@@ -4,3 +4,4 @@
 .coverage
 .coveralls.yml
 .docker-tmp/
+.open-build-service/

.travis.yml

@@ -49,16 +49,26 @@ matrix:
         - flake8
     - env: TEST_SUITE=nosetests
       sudo: required
+      services:
+        - docker
       language: python
       python: 2.7
       before_install:
         # provides xmllint used by test_bootstrap_copy (tests.freeze_tests.TestFreeze)
         - sudo apt-get install libxml2-utils
       install:
-        # needed to install osc from git in requirements.txt
-        - pip install pycurl urlgrabber
+        # urlgrabber needed to install osc from git in requirements.txt
+        # m2crypto for osc to be runable as used in docker-compose-obs
+        - pip install pycurl urlgrabber m2crypto
         - pip install -r requirements.txt
         - pip install python-coveralls
+      before_script:
+        # travis-ci/travis-ci#7008: stop services to make room for OBS setup
+        - sudo service mysql stop
+        - sudo service memcached stop
+        - ./dist/ci/docker-compose-obs
+        # Needs python prefix to use the correct interpretor.
+        - python ./obs_clone.py --cache --debug --apiurl-target local
       script:
         - nosetests --with-coverage --cover-package=. --cover-inclusive
       after_success:
@@ -73,10 +83,13 @@ deploy:
 
 cache:
   directories:
+    # obs_clone.py
+    - ~/.cache/osc-plugin-factory-clone
+    # distribution jobs `osc build`
     - .docker-tmp
   pip: true
   # 7 days
   timeout: 604800
 before_cache:
   - sudo rm -rf .docker-tmp/build-root
-  - sudo chmod o+rw .docker-tmp
+  - sudo chmod -R o+rw .docker-tmp

devel-project.py

@@ -41,7 +41,8 @@ def devel_projects_get(apiurl, project):
         devel_projects[devel.attrib['project']] = True
 
     # Ensure self does not end up in list.
-    del devel_projects[project]
+    if project in devel_projects:
+        del devel_projects[project]
 
     return sorted(devel_projects)
 

dist/ci/docker-compose-obs

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -e
+
+OBS_DIRECTORY=${OBS_DIRECTORY:-.open-build-service}
+
+if [ ! -d "$OBS_DIRECTORY" ] ; then
+  git clone --depth 1 https://github.com/openSUSE/open-build-service.git "$OBS_DIRECTORY"
+
+  if [ ! -f ~/.oscrc ] ; then
+    # openSUSE/open-build-service#3946: would elevate the need, but rejected.
+    eval $(./dist/ci/osc-credentials-throwaway)
+    OBS_USER="$user" OBS_PASS="$pass" ./dist/ci/osc-init
+  fi
+
+cat <<EOF >> ~/.oscrc
+
+[http://0.0.0.0:3000]
+user = Admin
+pass = opensuse
+aliases = local
+EOF
+fi
+
+cd "$OBS_DIRECTORY"
+
+git pull origin master
+
+# https://github.com/openSUSE/open-build-service/blob/master/CONTRIBUTING.md
+git submodule init
+git submodule update
+
+rake docker:build
+
+# See `docker-compose logs` instead of changing output redirection.
+docker-compose up -d --remove-orphans
+
+osc='osc'
+if ! which osc &> /dev/null ; then
+  # Workaround for pip install that occurs on travis.
+  osc='osc-wrapper.py'
+fi
+
+until $osc -q -A local api /about 2> /dev/null ; do
+  echo "waiting for OBS to be responsive..."
+  ((c++)) && ((c==60)) && docker-compose logs && exit 1
+  sleep 1
+done

dist/package/openSUSE-release-tools.spec

@@ -251,7 +251,8 @@ make %{?_smp_mflags}
 
 %check
 %if 0%{?is_opensuse}
-make check
+# TODO openSUSE/osc-plugin-factory#1221: decide how to handle integration tests
+# make check
 %endif
 
 %install

obs_clone.py

@@ -0,0 +1,190 @@
+#!/usr/bin/python
+
+from copy import deepcopy
+from lxml import etree as ET
+from osc.core import copy_pac as copy_package
+from osc.core import get_commitlog
+from osc.core import http_GET
+from osc.core import http_POST
+from osc.core import http_PUT
+from osc.core import makeurl
+from osc.core import show_upstream_rev
+from urllib2 import HTTPError
+import argparse
+import osc.conf
+import sys
+
+
+def project_fence(project):
+    if ((project.startswith('openSUSE:') and project_fence.project.startswith('openSUSE:')) and
+        not project.startswith(project_fence.project)):
+        # Exclude other openSUSE:* projects while cloning a specifc one.
+        return False
+    if project.startswith('openSUSE:Factory:ARM'):
+        # Troublesome.
+        return False
+    # Perhaps use devel project list as filter, but for now quick exclude.
+    if project.startswith('SUSE:') or project.startswith('Ubuntu:'):
+        return False
+
+    return True
+
+def entity_clone(apiurl_source, apiurl_target, path, sanitize=None, clone=None, after=None):
+    if not hasattr(entity_clone, 'cloned'):
+        entity_clone.cloned = []
+
+    if path[0] == 'source' and not project_fence(path[1]):
+        # Skip projects outside of fence by marking as cloned.
+        if path not in entity_clone.cloned:
+            entity_clone.cloned.append(path)
+
+    if path in entity_clone.cloned:
+        print('skip {}'.format('/'.join(path)))
+        return
+
+    print('clone {}'.format('/'.join(path)))
+    entity_clone.cloned.append(path)
+
+    url = makeurl(apiurl_source, path)
+    entity = ET.parse(http_GET(url)).getroot()
+
+    if sanitize:
+        sanitize(entity)
+    if clone:
+        clone(apiurl_source, apiurl_target, entity)
+
+    url = makeurl(apiurl_target, path)
+    http_PUT(url, data=ET.tostring(entity))
+
+    if after:
+        after(apiurl_source, apiurl_target, entity)
+
+def users_clone(apiurl_source, apiurl_target, entity):
+    for person in entity.findall('person'):
+        path = ['person', person.get('userid')]
+        entity_clone(apiurl_source, apiurl_target, path, person_sanitize, after=person_clone_after)
+
+    for group in entity.findall('group'):
+        path = ['group', group.get('groupid')]
+        entity_clone(apiurl_source, apiurl_target, path, clone=group_clone)
+
+def project_repositories_remove(project):
+    # Remove repositories that reference other projects.
+    for repository in project.xpath('repository[releasetarget or path]'):
+        repository.getparent().remove(repository)
+
+# clone(Factory)
+# - stripped
+# - after
+#   - clone(Factory:ToTest)
+#     - stripped
+#     - after
+#       - clone(Factory)...skip
+#       - write real
+#   - write real
+def project_clone(apiurl_source, apiurl_target, project):
+    users_clone(apiurl_source, apiurl_target, project)
+
+    # Write stripped version that does not include repos with path references.
+    url = makeurl(apiurl_target, ['source', project.get('name'), '_meta'])
+    stripped = deepcopy(project)
+    project_repositories_remove(stripped)
+    http_PUT(url, data=ET.tostring(stripped))
+
+    # Clone projects referenced in repository paths.
+    for repository in project.findall('repository'):
+        for target in repository.xpath('./path') + repository.xpath('./releasetarget'):
+            if not project_fence(target.get('project')):
+                project.remove(repository)
+                break
+
+            # Valid reference to project and thus should be cloned.
+            path = ['source', target.get('project'), '_meta']
+            entity_clone(apiurl_source, apiurl_target, path, clone=project_clone)
+
+def package_clone(apiurl_source, apiurl_target, package):
+    # Clone project that contains the package.
+    path = ['source', package.get('project'), '_meta']
+    entity_clone(apiurl_source, apiurl_target, path, clone=project_clone)
+
+    # Clone the dependencies of package.
+    users_clone(apiurl_source, apiurl_target, package)
+
+    # Clone devel project referenced by package.
+    devel = package.find('devel')
+    if devel is not None:
+        path = ['source', devel.get('project'), devel.get('package'), '_meta']
+        entity_clone(apiurl_source, apiurl_target, path, clone=package_clone, after=package_clone_after)
+
+def package_clone_after(apiurl_source, apiurl_target, package):
+    copy_package(apiurl_source, package.get('project'), package.get('name'),
+                 apiurl_target, package.get('project'), package.get('name'),
+                 # TODO Would be preferable to preserve links, but need to
+                 # recreat them since they do not match with copied package.
+                 expand=True,
+                 # TODO Can copy server-side if inner-connect is setup, but not
+                 # clear how to trigger the equivalent of save in admin UI.
+                 client_side_copy=True)
+
+def person_sanitize(person):
+    person.find('email').text = person.find('email').text.split('@')[0] + '@example.com'
+
+def person_clone_after(apiurl_source, apiurl_target, person):
+    url = makeurl(apiurl_target, ['person', person.find('login').text], {'cmd': 'change_password'})
+    http_POST(url, data='opensuse')
+
+def group_clone(apiurl_source, apiurl_target, group):
+    for person in group.findall('maintainer') + group.findall('person/person'):
+        path = ['person', person.get('userid')]
+        entity_clone(apiurl_source, apiurl_target, path, person_sanitize, after=person_clone_after)
+
+def clone_do(apiurl_source, apiurl_target, project):
+    print('clone {} from {} to {}'.format(project, apiurl_source, apiurl_target))
+
+    try:
+        # TODO Decide how to choose what to clone via args.
+        entity_clone(apiurl_source, apiurl_target, ['source', project + ':Staging', 'dashboard', '_meta'],
+                     clone=package_clone, after=package_clone_after)
+
+        entity_clone(apiurl_source, apiurl_target, ['source', project, 'drush', '_meta'],
+                     clone=package_clone, after=package_clone_after)
+
+        entity_clone(apiurl_source, apiurl_target, ['group', 'opensuse-review-team'],
+                     clone=group_clone)
+    except HTTPError as e:
+        # Print full output for any errors since message can be cryptic.
+        print(e.read())
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Clone projects and dependencies between OBS instances.')
+    parser.set_defaults(func=clone_do)
+
+    parser.add_argument('-S', '--apiurl-source', metavar='URL', help='source API URL')
+    parser.add_argument('-T', '--apiurl-target', metavar='URL', help='target API URL')
+    parser.add_argument('-c', '--cache', action='store_true', help='cache source queries for 24 hours')
+    parser.add_argument('-d', '--debug', action='store_true', help='print info useful for debuging')
+    parser.add_argument('-p', '--project', default='openSUSE:Factory', help='project from which to clone')
+
+    args = parser.parse_args()
+
+    osc.conf.get_config(override_apiurl=args.apiurl_target)
+    apiurl_target = osc.conf.config['apiurl']
+    osc.conf.get_config(override_apiurl=args.apiurl_source)
+    apiurl_source = osc.conf.config['apiurl']
+
+    if apiurl_target == apiurl_source:
+        print('target APIURL must not be the same as source APIURL')
+        sys.exit(1)
+
+    if args.cache:
+        from osclib.cache import Cache
+        Cache.CACHE_DIR = Cache.CACHE_DIR + '-clone'
+        Cache.PATTERNS = {}
+        # Prevent caching source information from local clone.
+        Cache.PATTERNS['/source/[^/]+/[^/]+/[^/]+?rev'] = 0
+        Cache.PATTERNS['.*'] = Cache.TTL_LONG * 2
+        Cache.init()
+
+    osc.conf.config['debug'] = args.debug
+    project_fence.project = args.project
+    sys.exit(args.func(apiurl_source, apiurl_target, args.project))

osclib/cache.py

@@ -167,6 +167,9 @@ class Cache(object):
         match, project = Cache.match(url)
         if match:
             path = Cache.path(url, project, include_file=True, makedirs=True)
+            ttl = Cache.PATTERNS[match]
+            if ttl == 0:
+                return data
 
             # Since urlopen does not return a seekable stream it cannot be reset
             # after writing to cache. As such a wrapper must be used. This could

tests/OBSLocal.py

@@ -0,0 +1,96 @@
+import os
+from lxml import etree as ET
+from osc import conf
+from osc.core import get_request
+from osc.core import http_GET
+from osc.core import makeurl
+import subprocess
+import unittest
+
+OSCRC = os.path.expanduser('~/.oscrc-test')
+APIURL = 'local-test'
+
+class OBSLocalTestCase(unittest.TestCase):
+    script = None
+    script_apiurl = True
+    script_debug = True
+    script_debug_osc = True
+
+    @classmethod
+    def setUpClass(cls):
+        # TODO #1214: Workaround for tests/obs.py's lack of cleanup.
+        import httpretty
+        httpretty.disable()
+
+    def setUp(self):
+        self.oscrc('Admin')
+        conf.get_config(override_conffile=OSCRC,
+                        override_no_keyring=True,
+                        override_no_gnome_keyring=True)
+        self.apiurl = conf.config['apiurl']
+        self.assertOBS()
+
+    def assertOBS(self):
+        url = makeurl(self.apiurl, ['about'])
+        root = ET.parse(http_GET(url)).getroot()
+        self.assertEqual(root.tag, 'about')
+
+    @staticmethod
+    def oscrc(userid):
+        with open(OSCRC, 'w+') as f:
+            f.write('\n'.join([
+                '[general]',
+                'apiurl = http://0.0.0.0:3000',
+                '[http://0.0.0.0:3000]',
+                'user = {}'.format(userid),
+                'pass = opensuse',
+                'email = {}@example.com'.format(userid),
+                'aliases = {}'.format(APIURL),
+                '',
+            ]))
+
+    def osc_user(self, userid):
+        conf.config['api_host_options'][self.apiurl]['user'] = userid
+        self.oscrc(userid)
+
+    def execute_script(self, args):
+        if self.script:
+            args.insert(0, self.script)
+        if self.script_debug:
+            args.insert(1, '--debug')
+        if self.script_debug_osc:
+            args.insert(1, '--osc-debug')
+        args.insert(0, '-p')
+        args.insert(0, 'run')
+        args.insert(0, 'coverage')
+
+        self.execute(args)
+
+    def execute_osc(self, args):
+        # The wrapper allows this to work properly when osc installed via pip.
+        args.insert(0, 'osc-wrapper.py')
+        self.execute(args)
+
+    def execute(self, args):
+        print('$ ' + ' '.join(args)) # Print command for debugging.
+        try:
+            env = os.environ
+            env['OSC_CONFIG'] = OSCRC
+            self.output = subprocess.check_output(args, stderr=subprocess.STDOUT, env=env)
+        except subprocess.CalledProcessError as e:
+            print(e.output)
+            raise e
+        print(self.output) # For debugging assertion failures.
+
+    def assertOutput(self, string):
+        self.assertTrue(string in self.output, '[MISSING] ' + string)
+
+    def assertReview(self, rid, **kwargs):
+        request = get_request(self.apiurl, rid)
+        for review in request.reviews:
+            for key, value in kwargs.items():
+                if hasattr(review, key) and getattr(review, key) == value[0]:
+                    self.assertEqual(review.state, value[1], '{}={} not {}'.format(key, value[0], value[1]))
+                    return
+
+        self.fail('{} not found'.format(kwargs))

tests/devel_project_tests.py

@@ -0,0 +1,21 @@
+from OBSLocal import OBSLocalTestCase
+import unittest
+
+
+class TestDevelProject(OBSLocalTestCase):
+    script = './devel-project.py'
+    script_debug_osc = False
+
+    def test_list(self):
+        self.osc_user('staging-bot')
+        self.execute_script(['list', '--write'])
+        self.assertOutput('server:php:applications')
+        # TODO Assert --write worked and in file.
+
+    def test_reviews(self):
+        self.osc_user('staging-bot')
+        self.execute_script(['reviews'])
+
+    def test_requests(self):
+        self.osc_user('staging-bot')
+        self.execute_script(['requests'])