openSUSE-release-tools/fcc_submitter.py

#!/usr/bin/python

import argparse
import logging
import sys

try:
    from urllib.error import HTTPError, URLError
except ImportError:
    # python 2.x
    from urllib2 import HTTPError, URLError

import random
import re
from xml.etree import cElementTree as ET

import osc.conf
import osc.core
from osclib.core import devel_project_get
from osclib.core import project_pseudometa_package

from osc import oscerr
from osclib.memoize import memoize

OPENSUSE = 'openSUSE:Leap:15.1'
OPENSUSE_PREVERSION = 'openSUSE:Leap:15.0'
FCC = '{}:Staging:FactoryCandidates'.format(OPENSUSE)

makeurl = osc.core.makeurl
http_GET = osc.core.http_GET
http_POST = osc.core.http_POST
http_PUT = osc.core.http_PUT

class FccFreezer(object):
    def __init__(self):
        self.factory = 'openSUSE:Factory'
        self.apiurl = osc.conf.config['apiurl']
        self.debug = osc.conf.config['debug']

    def list_packages(self, project):
        url = makeurl(self.apiurl, ['source', project])
        pkglist = []

        root = ET.parse(http_GET(url)).getroot()
        xmllines = root.findall("./entry")
        for pkg in xmllines:
            pkglist.append(pkg.attrib['name'])

        return set(pkglist)

    def check_one_source(self, flink, si, pkglist, pkglist_prever):
        """
        Insert package information to the temporary frozenlinks.
        Return package name if the package can not fit the condition
        add to the frozenlinks, can be the ignored package.
        """
        package = si.get('package')
        logging.debug("Processing %s" % (package))

        # If the package is an internal one (e.g _product)
        if package.startswith('_') or package.startswith('Test-DVD') or package.startswith('000'):
            return None

        # filter out multibuild package
        for originpackage in si.findall('originpackage'):
            if ':' in package and package.split(':')[0] == originpackage.text:
                return package

        for linked in si.findall('linked'):
            if linked.get('project') == self.factory:
                if linked.get('package') in pkglist or linked.get('package') in pkglist_prever:
                    return package
                url = makeurl(self.apiurl, ['source', self.factory, package], {'view': 'info', 'nofilename': '1'})
                # print(package, linked.get('package'), linked.get('project'))
                f = http_GET(url)
                proot = ET.parse(f).getroot()
                lsrcmd5 = proot.get('lsrcmd5')
                if lsrcmd5 is None:
                    raise Exception("{}/{} is not a link but we expected one".format(self.factory, package))
                ET.SubElement(flink, 'package', {'name': package, 'srcmd5': lsrcmd5, 'vrev': si.get('vrev')})
                return None

        if package in pkglist or package in pkglist_prever:
            return package

        if package in ['rpmlint-mini-AGGR']:
            # we should not freeze aggregates
            return None

        ET.SubElement(flink, 'package', {'name': package, 'srcmd5': si.get('srcmd5'), 'vrev': si.get('vrev')})
        return None

    def receive_sources(self, flink):
        ignored_sources = []
        pkglist = self.list_packages(OPENSUSE)
        # we also don't want the package is exist in the previous version
        pkglist_prever = self.list_packages(OPENSUSE_PREVERSION)

        url = makeurl(self.apiurl, ['source', self.factory], {'view': 'info', 'nofilename': '1'})
        f = http_GET(url)
        root = ET.parse(f).getroot()

        for si in root.findall('sourceinfo'):
            package = self.check_one_source(flink, si, pkglist, pkglist_prever)
            if package is not None:
                ignored_sources.append(str(package))
        return ignored_sources

    def freeze(self):
        """Main method"""
        flink = ET.Element('frozenlinks')

        fl = ET.SubElement(flink, 'frozenlink', {'project': self.factory})
        ignored_sources = self.receive_sources(fl)
        if self.debug:
            logging.debug("Dump ignored source")
            for source in ignored_sources:
                logging.debug("Ignored source: %s" % source)

        url = makeurl(self.apiurl, ['source', FCC, '_project', '_frozenlinks'], {'meta': '1'})
        l = ET.tostring(flink)
        try:
            http_PUT(url, data=l)
        except HTTPError as e:
            raise e

class FccSubmitter(object):
    def __init__(self, from_prj, to_prj, submit_limit):
        self.from_prj = from_prj
        self.to_prj = to_prj
        self.factory = 'openSUSE:Factory'
        self.submit_limit = int(submit_limit)
        self.apiurl = osc.conf.config['apiurl']
        self.debug = osc.conf.config['debug']
        self.sle_base_prjs = [
                'SUSE:SLE-15-SP1:GA',
                'SUSE:SLE-15:GA',
                'SUSE:SLE-12-SP4:Update',
                'SUSE:SLE-12-SP4:GA',
                'SUSE:SLE-12-SP3:Update',
                'SUSE:SLE-12-SP3:GA',
                'SUSE:SLE-12-SP2:Update',
                'SUSE:SLE-12-SP2:GA',
                'SUSE:SLE-12-SP1:Update',
                'SUSE:SLE-12-SP1:GA',
                'SUSE:SLE-12:Update',
                'SUSE:SLE-12:GA',
                ]
        # the skip list against devel project
        self.skip_devel_project_list = [
                'mobile:synchronization:FACTORY'
                ]
        # put the except packages from skip_devel_project_list, use regex in this list, eg. "^golang-x-(\w+)", "^nodejs$"
        self.except_pkgs_list = []

    def get_source_packages(self, project, expand=False):
        """Return the list of packages in a project."""
        query = {'expand': 1} if expand else {}
        root = ET.parse(http_GET(makeurl(self.apiurl, ['source', project],
                                 query=query))).getroot()
        packages = [i.get('name') for i in root.findall('entry')]

        return packages

    def get_request_list(self, package):
        return osc.core.get_request_list(self.apiurl, self.to_prj, package, req_state=('new', 'review', 'declined', 'revoked'))

    def get_link(self, project, package):
        try:
            link = http_GET(makeurl(self.apiurl, ['source', project, package, '_link'])).read()
        except (HTTPError, URLError):
            return None
        return ET.fromstring(link)

    def get_build_succeeded_packages(self, project):
        """Get the build succeeded packages from `from_prj` project.
        """

        f = osc.core.show_prj_results_meta(self.apiurl, project)
        root = ET.fromstringlist(f)
        #print ET.dump(root)

        failed_multibuild_pacs = []
        pacs = []
        for node in root.findall('result'):
            if node.get('repository') == 'standard' and node.get('arch') == 'x86_64':
                for pacnode in node.findall('status'):
                    if ':' in pacnode.get('package'):
                        mainpac = pacnode.get('package').split(':')[0]
                        if pacnode.get('code') not in ['succeeded', 'excluded']:
                            failed_multibuild_pacs.append(pacnode.get('package'))
                            if mainpac not in failed_multibuild_pacs:
                                failed_multibuild_pacs.append(mainpac)
                            if mainpac in pacs:
                                pacs.remove(mainpac)
                        else:
                            if mainpac in failed_multibuild_pacs:
                                failed_multibuild_pacs.append(pacnode.get('package'))
                            elif mainpac not in pacs:
                                pacs.append(mainpac)
                        continue
                    if pacnode.get('code') == 'succeeded':
                        pacs.append(pacnode.get('package'))
            else:
                logging.error("Can not find standard/x86_64 results")

        return pacs

    def is_new_package(self, tgt_project, tgt_package):
        try:
            logging.debug("Gathering package_meta %s/%s" % (tgt_project, tgt_package))
            osc.core.show_package_meta(self.apiurl, tgt_project, tgt_package)
        except (HTTPError, URLError):
            return True
        return False

    def create_submitrequest(self, package):
        """Create a submit request using the osc.commandline.Osc class."""
        src_project = self.factory # submit from Factory only
        dst_project = self.to_prj

        msg = 'Automatic request from %s by F-C-C Submitter. Please review this change and decline it if Leap do not need it.' % src_project
        res = osc.core.create_submit_request(self.apiurl,
                                             src_project,
                                             package,
                                             dst_project,
                                             package,
                                             message=msg)
        return res

    def check_multiple_specfiles(self, project, package):
        try:
            url = makeurl(self.apiurl, ['source', project, package], { 'expand': '1' } )
        except HTTPError as e:
            if e.code == 404:
                return None
            raise e
        root = ET.fromstring(http_GET(url).read())
        data = {}
        linkinfo = root.find('linkinfo')
        if linkinfo is not None:
            data['linkinfo'] = linkinfo.attrib['package']
        else:
            data['linkinfo'] = None

        files = [ entry.get('name').replace('.spec', '') for entry in root.findall('entry') if entry.get('name').endswith('.spec') ]
        data['specs'] = files

        if len(files) > 1:
            return data
        else:
            return False

    def is_sle_base_pkgs(self, package):
        link = self.get_link(self.to_prj, package)
        if link is None or link.get('project') not in self.sle_base_prjs:
            logging.debug("%s not from SLE base"%package)
            return False
        return True

    def list_pkgs(self):
        """List build succeeded packages"""
        succeeded_packages = []
        succeeded_packages = self.get_build_succeeded_packages(self.from_prj)
        if not len(succeeded_packages) > 0:
            logging.info('No build succeeded package in %s'%self.from_prj)
            return

        print 'Build succeeded packages:'
        print '-------------------------------------'
        for pkg in succeeded_packages:
            print pkg

        print '-------------------------------------'
        print "Found {} build succeded packages".format(len(succeeded_packages))

    def get_deleted_packages(self, project):
        query = 'states=accepted&types=delete&project={}&view=collection'
        query = query.format(project)
        url = makeurl(self.apiurl, ['request'], query)
        f = http_GET(url)
        root = ET.parse(f).getroot()

        pkgs = []
        for sr in root.findall('request'):
            tgt_package = sr.find('action').find('target').get('package')
            pkgs.append(tgt_package)

        return pkgs

    def load_skip_pkgs_list(self, project, package):
        url = makeurl(self.apiurl, ['source', project, package, '{}?expand=1'.format('fcc_skip_pkgs')])
        try:
            return http_GET(url).read()
        except HTTPError:
            return ''

    def crawl(self):
        """Main method"""
        succeeded_packages = []
        succeeded_packages = self.get_build_succeeded_packages(self.from_prj)
        if not len(succeeded_packages) > 0:
            logging.info('No build succeeded package in %s'%self.from_prj)
            return

        # randomize the list
        random.shuffle(succeeded_packages)
        # get souce packages from target
        target_packages = self.get_source_packages(self.to_prj)
        deleted_packages = self.get_deleted_packages(self.to_prj)
        if self.to_prj.startswith("openSUSE:"):
            deleted_packages = deleted_packages + self.get_deleted_packages(OPENSUSE_PREVERSION)

        pseudometa_project, pseudometa_package = project_pseudometa_package(self.apiurl, 'openSUSE:Factory')
        skip_pkgs_list = self.load_skip_pkgs_list(pseudometa_project, pseudometa_package).splitlines()

        ms_packages = [] # collect multi specs packages

        for i in range(0, min(int(self.submit_limit), len(succeeded_packages))):
            package = succeeded_packages[i]
            submit_ok = True

            if package in deleted_packages:
                logging.info('%s has been dropped from %s, ignore it!'%(package, self.to_prj))
                submit_ok = False

            if self.is_sle_base_pkgs(package) is True:
                logging.info('%s origin from SLE base, skip for now!'%package)
                submit_ok = False

            # make sure it is new package
            new_pkg = self.is_new_package(self.to_prj, package)
            if new_pkg is not True:
                logging.info('%s is not a new package, do not submit.' % package)
                submit_ok = False

            multi_specs = self.check_multiple_specfiles(self.factory, package)
            if multi_specs is None:
                logging.info('%s does not exist in %s'%(package, 'openSUSE:Factory'))
                submit_ok = False

            if multi_specs:
                if multi_specs['linkinfo']:
                    logging.info('%s in %s is sub-package of %s, skip it!'%(package, 'openSUSE:Factory', multi_specs['linkinfo']))
                    ms_packages.append(package)
                    submit_ok = False

                for spec in multi_specs['specs']:
                    if spec not in succeeded_packages:
                        logging.info('%s is sub-pacakge of %s but build failed, skip it!'%(spec, package))
                        submit_ok = False

            if not submit_ok:
                continue

            # make sure the package non-exist in target yet ie. expand=False
            if package not in target_packages:
                # make sure there is no request against same package
                request = self.get_request_list(package)
                if request:
                    logging.debug("There is a request to %s / %s already or it has been declined/revoked, skip!"%(package, self.to_prj))
                else:
                    logging.info("%d - Preparing submit %s to %s"%(i, package, self.to_prj))
                    # get devel project
                    devel_prj, devel_pkg = devel_project_get(self.apiurl, self.factory, package)
                    # check devel project does not in the skip list
                    if devel_prj in self.skip_devel_project_list:
                        # check the except packages list
                        match = None
                        for elem in self.except_pkgs_list:
                            m = re.search(elem, package)
                            if m is not None:
                                match = True

                        if match is not True:
                            logging.info('%s/%s is in the skip list, do not submit.' % (devel_prj, package))
                            continue
                        else:
                            pass

                    # check package does not in the skip list
                    match = None
                    for elem in skip_pkgs_list:
                        m = re.search(elem, package)
                        if m is not None:
                            match = True

                    if match is True:
                        logging.info('%s is in the skip list, do not submit.' % package)
                        continue
                    else:
                        pass

                    res = self.create_submitrequest(package)
                    if res and res is not None:
                        logging.info('Created request %s for %s' % (res, package))
                    else:
                        logging.error('Error occurred when creating submit request')
            else:
                logging.debug('%s is exist in %s, skip!'%(package, self.to_prj))

        # dump multi specs packages
        print("Multi-specfile packages:")
        if ms_packages:
            for pkg in ms_packages:
                print pkg
        else:
            print 'None'



def main(args):
    # Configure OSC
    osc.conf.get_config(override_apiurl=args.apiurl)
    osc.conf.config['debug'] = args.debug

    if args.freeze:
        print "freezing {}".format(FCC)
        freezer = FccFreezer()
        freezer.freeze()
    else:
        uc = FccSubmitter(args.from_prj, args.to_prj, args.submit_limit)
        if args.list_packages:
            uc.list_pkgs()
        else:
            uc.crawl()

if __name__ == '__main__':
    description = 'Create SR from FactoryCandidates to '\
                  'openSUSE Leap project for new build succeded packages.'
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument('-A', '--apiurl', metavar='URL', help='API URL')
    parser.add_argument('-d', '--debug', action='store_true',
                        help='print info useful for debuging')
    parser.add_argument('-f', '--from', dest='from_prj', metavar='PROJECT',
                        help='project where to check (default: %s)' % FCC,
                        default=FCC)
    parser.add_argument('-t', '--to', dest='to_prj', metavar='PROJECT',
                        help='project where to submit the packages (default: %s)' % OPENSUSE,
                        default=OPENSUSE)
    parser.add_argument('-r', '--freeze', dest='freeze', action='store_true', help='rebase FCC project')
    parser.add_argument('-s', '--list', dest='list_packages', action='store_true', help='list build succeeded packages')
    parser.add_argument('-l', '--limit', dest='submit_limit', metavar='NUMBERS', help='limit numbers packages to submit (default: 100)', default=100)

    args = parser.parse_args()

    # Set logging configuration
    logging.basicConfig(level=logging.DEBUG if args.debug
                        else logging.INFO)

    sys.exit(main(args))