openSUSE-release-tools/manager_42.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright (c) 2015-2017 SUSE Linux GmbH
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import argparse
import itertools
import logging
import sys
from xml.etree import cElementTree as ET

import osc.conf
import osc.core
import urllib2
import time
import yaml
from collections import namedtuple

from osclib.memoize import memoize

logger = logging.getLogger()

makeurl = osc.core.makeurl
http_GET = osc.core.http_GET
http_DELETE = osc.core.http_DELETE
http_PUT = osc.core.http_PUT
http_POST = osc.core.http_POST

class Manager42(object):

    config_defaults = {
        'ignored_packages' : [
            '00Meta',
            '00aggregates',
            '000product',
            '000package-groups',
            '000release-packages',
            ],
        'project_preference_order' : [],
        'drop_if_vanished_from' : [],
        'from_prj' : 'openSUSE:Leap:42.3',
        'factory' : 'openSUSE:Factory',
        }

    def __init__(self, caching = True, configfh = None):
        self.caching = caching
        self.apiurl = osc.conf.config['apiurl']
        self.config = self._load_config(configfh)
        self.force = False

        self.parse_lookup(self.config.from_prj)
        self.fill_package_meta()
        self.packages = dict()
        for project in [self.config.from_prj] + self.config.project_preference_order:
            self._fill_package_list(project)

    # FIXME: add to ToolBase and rebase Manager42 on that
    def _load_config(self, handle = None):
        d = self.__class__.config_defaults
        y = yaml.safe_load(handle) if handle is not None else {}
        return namedtuple('BotConfig', sorted(d.keys()))(*[ y.get(p, d[p]) for p in sorted(d.keys()) ])

    def latest_packages(self):
        data = self.cached_GET(makeurl(self.apiurl,
                                       ['project', 'latest_commits',
                                           self.config.from_prj]))
        lc = ET.fromstring(data)
        packages = set()
        for entry in lc.findall('{http://www.w3.org/2005/Atom}entry'):
            title = entry.find('{http://www.w3.org/2005/Atom}title').text
            if title.startswith('In '):
                packages.add(title[3:].split(' ')[0])
        return sorted(packages)

    def all_packages(self):
        return self.packages[self.config.from_prj]

    def parse_lookup(self, project):
        self.lookup_changes = 0
        self.lookup = {}
        try:
            self.lookup = yaml.safe_load(self._load_lookup_file(project))
        except urllib2.HTTPError as e:
            if e.code != 404:
                raise

    def _load_lookup_file(self, prj):
        return self.cached_GET(makeurl(self.apiurl,
                                       ['source', prj, '00Meta', 'lookup.yml']))

    def _put_lookup_file(self, prj, data):
        return http_PUT(makeurl(self.apiurl,
                                ['source', prj, '00Meta', 'lookup.yml']), data=data)

    def store_lookup(self):
        if self.lookup_changes == 0:
            logger.info('no change to lookup.yml')
            return
        data = yaml.dump(self.lookup, default_flow_style=False, explicit_start=True)
        self._put_lookup_file(self.config.from_prj, data)
        self.lookup_changes = 0

    @memoize()
    def _cached_GET(self, url):
        return self.retried_GET(url).read()

    def cached_GET(self, url):
        if self.caching:
            return self._cached_GET(url)
        return self.retried_GET(url).read()

    def retried_GET(self, url):
        try:
            return http_GET(url)
        except urllib2.HTTPError as e:
            if 500 <= e.code <= 599:
                logger.warn('Retrying {}'.format(url))
                time.sleep(1)
                return self.retried_GET(url)
            raise e

    def get_source_packages(self, project, expand=False):
        """Return the list of packages in a project."""
        query = {'expand': 1} if expand else {}
        try:
            root = ET.fromstring(
                self.cached_GET(makeurl(self.apiurl,
                                 ['source', project],
                                 query=query)))
            packages = [i.get('name') for i in root.findall('entry')]

        except urllib2.HTTPError as e:
            if e.code == 404:
                logger.error("{}: {}".format(project, e))
                packages = []
            else:
                raise

        return packages

    def _get_source_package(self, project, package, revision):
        opts = { 'view': 'info' }
        if revision:
            opts['rev'] = revision
        return self.cached_GET(makeurl(self.apiurl,
                                ['source', project, package], opts))


    def crawl(self, packages):
        """Main method of the class that runs the crawler."""

        for package in sorted(packages):
            try:
                self.check_one_package(package)
            except urllib2.HTTPError as e:
                logger.error("Failed to check {}: {}".format(package, e))
                pass

            # avoid loosing too much work
            if self.lookup_changes > 50:
                self.store_lookup()

        if self.lookup_changes:
            self.store_lookup()

    def get_inconsistent(self):
        known = set(self.lookup.keys())
        stale = known - set(self.packages[self.config.from_prj])
        unknown = set(self.packages[self.config.from_prj]) - known

        if (stale):
            logger.info("stale packages: %s", ', '.join(stale))
        if (unknown):
            logger.info("unknown packages: %s", ', '.join(unknown))

        return (stale|unknown)

    def get_package_history(self, project, package, deleted = False):
        try:
            query = {}
            if deleted:
                query['deleted'] = 1
            return self.cached_GET(makeurl(self.apiurl,
                                   ['source', project, package, '_history'], query))
        except urllib2.HTTPError as e:
            if e.code == 404:
                return None
            raise


    def _is_ignored(self, project, package):
        if package in self.config.ignored_packages:
            logger.debug("%s in ignore list", package)
            return True
        return False

    def _fill_package_list(self, project):
        if project not in self.packages:
            self.packages[project] = [ p for p in self.get_source_packages(project) if not self._is_ignored(project, p) ]

    def check_source_in_project(self, project, package, verifymd5, deleted=False):

        self._fill_package_list(project)

        if not deleted and not package in self.packages[project]:
            return None, None

        his = self.get_package_history(project, package, deleted)
        if his is None:
            return None, None

        his = ET.fromstring(his)
        historyrevs = dict()
        revs = list()
        for rev in his.findall('revision'):
            historyrevs[rev.find('srcmd5').text] = rev.get('rev')
            revs.append(rev.find('srcmd5').text)
        revs.reverse()
        for i in range(min(len(revs), 5)): # check last commits
            srcmd5=revs.pop(0)
            root = self.cached_GET(makeurl(self.apiurl,
                                    ['source', project, package], { 'rev': srcmd5, 'view': 'info'}))
            root = ET.fromstring(root)
            if root.get('verifymd5') == verifymd5:
                return srcmd5, historyrevs[srcmd5]
        return None, None


    # check if we can find the srcmd5 in any of our underlay
    # projects
    def check_one_package(self, package):
        lproject = self.lookup.get(package, None)
        if not package in self.packages[self.config.from_prj]:
            logger.info("{} vanished".format(package))
            if self.lookup.get(package):
                del self.lookup[package]
                self.lookup_changes += 1
            return

        root = ET.fromstring(self._get_source_package(self.config.from_prj, package, None))
        linked = root.find('linked')
        if not linked is None and linked.get('package') != package:
            lstring = 'subpackage of {}'.format(linked.get('package'))
            if lstring != lproject:
                logger.warn("{} links to {} (was {})".format(package, linked.get('package'), lproject))
                self.lookup[package] = lstring
                self.lookup_changes += 1
            else:
                logger.debug("{} correctly marked as subpackage of {}".format(package, linked.get('package')))
            return

        pm = self.package_metas[package]
        devel = pm.find('devel')
        if devel is not None or (lproject is not None and lproject.startswith('Devel;')):
            develprj = None
            develpkg = None
            if devel is None:
                (dummy, develprj, develpkg) = lproject.split(';')
                logger.warn('{} lacks devel project setting {}/{}'.format(package, develprj, develpkg))
            else:
                develprj = devel.get('project')
                develpkg = devel.get('package')
            srcmd5, rev = self.check_source_in_project(develprj, develpkg,
                                                       root.get('verifymd5'))
            if srcmd5:
                lstring = 'Devel;{};{}'.format(develprj, develpkg)
                if not package in self.lookup or lstring != self.lookup[package]:
                    logger.debug("{} from devel {}/{} (was {})".format(package, develprj, develpkg, lproject))
                    self.lookup[package] = lstring
                    self.lookup_changes += 1
                else:
                    logger.debug("{} lookup from {}/{} is correct".format(package, develprj, develpkg))
                return

        elif lproject and lproject != 'FORK' and not lproject.startswith('subpackage '):
            srcmd5, rev = self.check_source_in_project(lproject, package, root.get('verifymd5'))
            if srcmd5:
                logger.debug("{} lookup from {} is correct".format(package, lproject))
                # if it's from Factory we check if the package can be found elsewhere meanwhile
                if not self.force and lproject != self.config.factory:
                    return
            elif lproject == self.config.factory and not package in self.packages[lproject]:
                his = self.get_package_history(lproject, package, deleted=True)
                if his:
                    logger.debug("{} got dropped from {}".format(package, lproject))

        logger.debug("check where %s came from", package)
        foundit = False
        for project in self.config.project_preference_order:
            srcmd5, rev = self.check_source_in_project(project, package, root.get('verifymd5'))
            if srcmd5:
                if project != lproject:
                    if project.endswith(':SLE-workarounds'):
                        logger.info('{} is from {} but should come from {}'.format(package, project, lproject))
                    else:
                        logger.info('{} -> {} (was {})'.format(package, project, lproject))
                        self.lookup[package] = project
                        self.lookup_changes += 1
                else:
                    logger.debug('{} still coming from {}'.format(package, project))
                foundit = True
                break

        if not foundit:
            if lproject == 'FORK':
                logger.debug("{}: lookup is correctly marked as fork".format(package))
            elif lproject in self.config.drop_if_vanished_from:
                logger.info('{} dropped from {}'.format(package, lproject))
            else:
                logger.info('{} is a fork (was {})'.format(package, lproject))
                self.lookup[package] = 'FORK'
                self.lookup_changes += 1

    def get_link(self, project, package):
        try:
            link = self.cached_GET(makeurl(self.apiurl,
                                    ['source', project, package, '_link']))
        except urllib2.HTTPError:
            return None
        return ET.fromstring(link)

    def fill_package_meta(self):
        self.package_metas = dict()
        url = makeurl(self.apiurl, ['search', 'package'],
                "match=[@project='%s']" % self.config.from_prj)
        root = ET.fromstring(self.cached_GET(url))
        for p in root.findall('package'):
            name = p.attrib['name']
            self.package_metas[name] = p


def main(args):
    # Configure OSC
    osc.conf.get_config(override_apiurl=args.apiurl)
    osc.conf.config['debug'] = args.debug

    uc = Manager42(caching = args.cache_requests, configfh = args.config )
    given_packages = set(args.packages)
    if args.all:
        given_packages = set(uc.all_packages())
    elif not given_packages:
        given_packages = set(uc.latest_packages())

    if args.check_inconsistent:
        given_packages |= uc.get_inconsistent()

    if args.force:
        uc.force = True
    uc.crawl(given_packages)

if __name__ == '__main__':
    description = 'maintain 00Meta/lookup.yml'
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument('-A', '--apiurl', metavar='URL', help='API URL')
    parser.add_argument('-d', '--debug', action='store_true',
                        help='print info useful for debuging')
    parser.add_argument('-a', '--all', action='store_true',
                        help='check all packages')
    parser.add_argument('-c', '--config', dest='config', metavar='FILE',
                        type=argparse.FileType('r'), required = True,
                        help='read config file FILE')
    parser.add_argument('-n', '--dry', action='store_true',
                        help='dry run, no POST, PUT, DELETE')
    parser.add_argument('--force', action='store_true',
                        help='don\'t take previous lookup information into consideration')
    parser.add_argument('--cache-requests', action='store_true', default=False,
                        help='cache GET requests. Not recommended for daily use.')
    parser.add_argument('--check-inconsistent', action='store_true', default=False,
                        help='also check insonsistent lookup entries')
    parser.add_argument("packages", nargs='*', help="packages to check")

    args = parser.parse_args()

    # Set logging configuration
    logging.basicConfig(level=logging.DEBUG if args.debug
                        else logging.INFO,
                        format='%(asctime)s - %(module)s:%(lineno)d - %(levelname)s - %(message)s')

    if args.dry:
        def dryrun(t, *args, **kwargs):
            return lambda *args, **kwargs: logger.debug("dryrun %s %s %s", t, args, str(kwargs)[:200])

        http_POST = dryrun('POST')
        http_PUT = dryrun('PUT')
        http_DELETE = dryrun('DELETE')

    sys.exit(main(args))