openSUSE-release-tools/osclib/stagingapi.py

# -*- coding: utf-8 -*-
#
# (C) 2014 mhrusecky@suse.cz, openSUSE.org
# (C) 2014 tchvatal@suse.cz, openSUSE.org
# Distribute under GPLv2 or GPLv3

import json
import logging
import re
import urllib2
import time
from xml.etree import cElementTree as ET

import yaml

from osc import oscerr
from osc.core import change_review_state
from osc.core import delete_package
from osc.core import get_request
from osc.core import make_meta_url
from osc.core import makeurl
from osc.core import metafile
from osc.core import http_GET
from osc.core import http_POST
from osc.core import http_PUT

from osclib.comments import CommentAPI


class StagingAPI(object):
    """
    Class containing various api calls to work with staging projects.
    """

    def __init__(self, apiurl):
        """
        Initialize instance variables
        """

        self.apiurl = apiurl
        self.rings = (
            'openSUSE:Factory:Rings:0-Bootstrap',
            'openSUSE:Factory:Rings:1-MinimalX',
            'openSUSE:Factory:Rings:2-TestDVD'
        )
        self.ring_packages = self._generate_ring_packages()
        self.packages_staged = self._get_staged_requests()

    def makeurl(self, l, query=None):
        """
        Wrapper around osc's makeurl passing our apiurl
        :return url made for l and query
        """
        query = [] if not query else query
        return makeurl(self.apiurl, l, query)

    def retried_GET(self, url):
        try:
            return http_GET(url)
        except urllib2.HTTPError, e:
            if e.code / 100 == 5:
                print 'Retrying {}'.format(url)
                return self.retried_GET(url)
            raise e

    def retried_POST(self, url):
        try:
            return http_POST(url)
        except urllib2.HTTPError, e:
            if e.code == 504:
                print 'Timeout on {}'.format(url)
                return '<status code="timeout"/>'
            if e.code / 100 == 5:
                print 'Retrying {}'.format(url)
                return self.retried_POST(url)
            raise e

    def retried_PUT(self, url, data):
        try:
            return http_PUT(url, data=data)
        except urllib2.HTTPError, e:
            if e.code / 100 == 5:
                print 'Retrying {}'.format(url)
                return self.retried_PUT(url, data)
            raise e

    def _generate_ring_packages(self):
        """
        Generate dictionary with names of the rings
        :return dictionary with ring names
        """

        ret = {}

        for prj in self.rings:
            url = self.makeurl(['source', prj])
            root = http_GET(url)
            for entry in ET.parse(root).getroot().findall('entry'):
                pkg = entry.attrib['name']
                if pkg in ret and pkg != 'Test-DVD-x86_64':
                    msg = '{} is defined in two projects ({} and {})'
                    raise Exception(msg.format(pkg, ret[pkg], prj))
                ret[pkg] = prj
        return ret

    def _get_staged_requests(self):
        """
        Get all requests that are already staged
        :return dict of staged requests with their project and srid
        """

        packages_staged = dict()
        for prj in self.get_staging_projects():
            meta = self.get_prj_pseudometa(prj)
            for req in meta['requests']:
                packages_staged[req['package']] = {'prj': prj, 'rq_id': req['id']}

        return packages_staged

    def get_package_information(self, project, pkgname):
        """
        Get the revision packagename and source project to copy from
        based on content provided
        :param project: the project we are having the package in
        :param pkgname: name of the package we want to identify
        :return dict ( project, package, revision, md5sum )
        """

        package_info = {}

        url = self.makeurl(['source', project, pkgname])
        content = http_GET(url)
        root = ET.parse(content).getroot()
        package_info['dir_srcmd5'] = root.attrib['srcmd5']

        linkinfo = root.find('linkinfo')
        package_info['srcmd5'] = linkinfo.attrib['srcmd5']
        package_info['rev'] = linkinfo.attrib.get('rev', None)
        package_info['project'] = linkinfo.attrib['project']
        package_info['package'] = linkinfo.attrib['package']

        return package_info

    def move_between_project(self, source_project, req_id,
                             destination_project):
        """
        Move selected package from one staging to another
        :param source_project: Source project
        :param request: request to move
        :param destination_project: Destination project
        """

        # Get the relevant information about source
        meta = self.get_prj_pseudometa(source_project)
        found = False
        for req in meta['requests']:
            if int(req['id']) == int(req_id):
                found = True
                break
        if not found:
            return None

        # Copy the package
        self.rq_to_prj(req_id, destination_project)
        # Delete the old one
        self.rm_from_prj(source_project, request_id=req_id,
                         msg='Moved to {}'.format(destination_project))

        # Build disable the old project if empty
        self.build_switch_staging_project(source_project)

        return True

    def get_staging_projects(self):
        """
        Get all current running staging projects
        :return list of known staging projects
        """

        projects = []

        query = "id?match=starts-with(@name,'openSUSE:Factory:Staging:')"
        url = self.makeurl(['search', 'project', query])
        projxml = http_GET(url)
        root = ET.parse(projxml).getroot()
        for val in root.findall('project'):
            projects.append(val.get('name'))
        return projects

    def do_change_review_state(self, request_id, newstate, message=None,
                               by_group=None, by_user=None, by_project=None):
        """
        Change review state of the staging request
        :param request_id: id of the request
        :param newstate: state of the new request
        :param message: message for the review
        :param by_group, by_user, by_project: review type
        """

        message = '' if not message else message

        req = get_request(self.apiurl, str(request_id))
        if not req:
            raise oscerr.WrongArgs('Request {} not found'.format(request_id))

        for review in req.reviews:
            if review.by_group == by_group and \
               review.by_user == by_user and \
               review.by_project == by_project and \
               review.state == 'new':

                # call osc's function
                return change_review_state(self.apiurl, str(request_id),
                                           newstate,
                                           message=message,
                                           by_group=by_group,
                                           by_user=by_user,
                                           by_project=by_project)

        return False

    def accept_non_ring_request(self, request):
        """
        Accept review of requests that are not yet in any ring so we
        don't delay their testing.
        :param request: request to check
        """

        # Consolidate all data from request
        request_id = int(request.get('id'))
        action = request.findall('action')
        if not action:
            msg = 'Request {} has no action'.format(request_id)
            raise oscerr.WrongArgs(msg)
        # we care only about first action
        action = action[0]

        # Where are we targeting the package
        target_project = action.find('target').get('project')
        target_package = action.find('target').get('package')

        # If the values are empty it is no error
        if not target_project or not target_package:
            msg = 'no target/package in request {}, action {}; '
            msg = msg.format(request_id, action)
            logging.info(msg)

        # Verify the package ring
        ring = self.ring_packages.get(target_package, None)
        if not ring:
            # accept the request here
            message = 'No need for staging, not in tested ring projects.'
            self.do_change_review_state(request_id, 'accepted', message=message,
                                        by_group='factory-staging')

    def supseded_request(self, request):
        """
        Returns a staging info for a request or None
        :param request - a Request instance
        :return dict with 'prj' and 'rq_id' of the old request
        """
        # Consolidate all data from request
        request_id = int(request.get('id'))
        action = request.findall('action')
        if not action:
            msg = 'Request {} has no action'.format(request_id)
            raise oscerr.WrongArgs(msg)
        # we care only about first action
        action = action[0]

        # Where are we targeting the package
        target_project = action.find('target').get('project')
        target_package = action.find('target').get('package')

        # If the values are empty it is no error
        if not target_project or not target_package:
            msg = 'no target/package in request {}, action {}; '
            msg = msg.format(request_id, action)
            logging.info(msg)

        # If the package is currently tracked then we do the replacement
        stage_info = self.packages_staged.get(target_package, {'prj': '', 'rq_id': 0})
        if stage_info['rq_id'] != 0 and int(stage_info['rq_id']) != request_id:
            return stage_info
        return None

    def update_superseded_request(self, request):
        """
        Replace superseded requests that are already in some
        staging prj
        :param request: request we are checking if it is fine
        """

        stage_info = self.supseded_request(request)
        request_id = int(request.get('id'))

        if stage_info:
            # Remove the old request
            self.rm_from_prj(stage_info['prj'],
                             request_id=stage_info['rq_id'],
                             msg='Replaced by newer request',
                             review='declined')
            # Add the new one that should be replacing it
            self.rq_to_prj(request_id, stage_info['prj'])

    def get_open_requests(self):
        """
        Get all requests with open review for staging project
        that are not yet included in any staging project
        :return list of pending open review requests
        """

        requests = []

        # xpath query, using the -m, -r, -s options
        where = "@by_group='factory-staging'+and+@state='new'"

        query = "match=state/@name='review'+and+review[{}]".format(where)
        url = self.makeurl(['search', 'request'], query)
        f = http_GET(url)
        root = ET.parse(f).getroot()

        for rq in root.findall('request'):
            requests.append(rq)
        return requests

    def dispatch_open_requests(self):
        """
        Verify all requests and dispatch them to staging projects or
        approve them

        """

        # get all current pending requests
        requests = self.get_open_requests()
        # check if we can reduce it down by accepting some
        for rq in requests:
            self.accept_non_ring_request(rq)
            self.update_superseded_request(rq)

    def get_prj_pseudometa(self, project):
        """
        Gets project data from YAML in project description
        :param project: project to read data from
        :return structured object with metadata
        """

        url = make_meta_url('prj', project, self.apiurl)
        f = http_GET(url)
        root = ET.parse(f).getroot()
        description = root.find('description')
        # If YAML parsing fails, load default
        # FIXME: Better handling of errors
        # * broken description
        # * directly linked packages
        # * removed linked packages
        try:
            data = yaml.load(description.text)
            data['requests']
        except:
            data = yaml.load('requests: []')
        return data

    def set_prj_pseudometa(self, project, meta):
        """
        Sets project description to the YAML of the provided object
        :param project: project to save into
        :param meta: data to save
        """

        # Get current metadata
        url = make_meta_url('prj', project, self.apiurl)
        root = ET.parse(http_GET(url)).getroot()
        # Find description
        description = root.find('description')
        # Order the requests and replace it with yaml
        meta['requests'] = sorted(meta['requests'], key=lambda x: x['id'])
        description.text = yaml.dump(meta)
        # Find title
        title = root.find('title')
        # Put something nice into title as well
        new_title = []
        for request in meta['requests']:
            new_title.append(request['package'])
        nt = ', '.join(sorted(new_title))
        title.text = nt[:240]
        # Write XML back
        url = make_meta_url('prj', project, self.apiurl, force=True)
        f = metafile(url, ET.tostring(root))
        http_PUT(f.url, file=f.filename)

    def _add_rq_to_prj_pseudometa(self, project, request_id, package):
        """
        Records request as part of the project within metadata
        :param project: project to record into
        :param request_id: request id to record
        :param package: package the request is about
        """

        data = self.get_prj_pseudometa(project)
        append = True
        for request in data['requests']:
            if request['package'] == package:
                # Only update if needed (to save calls to get_request)
                if request['id'] != request_id or not request.get('author'):
                    request['id'] = request_id
                    request['author'] = get_request(self.apiurl, str(request_id)).get_creator()
                append = False
        if append:
            author = get_request(self.apiurl, str(request_id)).get_creator()
            data['requests'].append({'id': request_id, 'package': package, 'author': author})
        self.set_prj_pseudometa(project, data)

    def get_request_id_for_package(self, project, package):
        """
        Query the request id from meta
        :param project: project the package is in
        :param package: package we want to query for
        """
        data = self.get_prj_pseudometa(project)
        for x in data['requests']:
            if x['package'] == package:
                return int(x['id'])
        return None

    def get_package_for_request_id(self, project, request_id):
        """
        Query the request id from meta
        :param project: project the package is in
        :param package: package we want to query for
        """
        data = self.get_prj_pseudometa(project)
        request_id = int(request_id)
        for x in data['requests']:
            if x['id'] == request_id:
                return x['package']
        return None

    def _remove_package_from_prj_pseudometa(self, project, package):
        """
        Delete request from the project pseudometa
        :param project: project to remove from
        :param package: package we want to remove from meta
        """

        data = self.get_prj_pseudometa(project)
        data['requests'] = filter(lambda x: x['package'] != package, data['requests'])
        self.set_prj_pseudometa(project, data)

    def rm_from_prj(self, project, package=None, request_id=None,
                    msg=None, review='accepted'):
        """
        Delete request from the project
        :param project: project to remove from
        :param request_id: request we want to remove
        :param msg: message for the log
        :param review: review state for the review, defautl accepted
        """

        if not request_id:
            request_id = self.get_request_id_for_package(project, package)
        if not package:
            package = self.get_package_for_request_id(project, request_id)
        if not package or not request_id:
            return

        self._remove_package_from_prj_pseudometa(project, package)
        subprj = self.map_ring_package_to_subject(project, package)
        delete_package(self.apiurl, subprj, package, force=True, msg=msg)
        self.set_review(request_id, project, state=review, msg=msg)

    def create_package_container(self, project, package, disable_build=False):
        """
        Creates a package container without any fields in project/package
        :param project: project to create it
        :param package: package name
        :param disable_build: should the package be created with build
                              flag disabled
        """
        dst_meta = '<package name="{}"><title/><description/></package>'
        dst_meta = dst_meta.format(package)
        if disable_build:
            root = ET.fromstring(dst_meta)
            elm = ET.SubElement(root, 'build')
            ET.SubElement(elm, 'disable')
            dst_meta = ET.tostring(root)

        url = self.makeurl(['source', project, package, '_meta'])
        http_PUT(url, data=dst_meta)

    def check_one_request(self, request, project):
        """
        Check if a staging request is ready to be approved. Reviews
        for the project are ignored, other open reviews will block the
        acceptance
        :param project: staging project
        :param request_id: request id to check

        """

        url = self.makeurl(['request', str(request)])
        root = ET.parse(http_GET(url)).getroot()

        # relevant info for printing
        package = str(root.find('action').find('target').attrib['package'])

        state = root.find('state').get('name')
        if state in ['declined', 'superseded', 'revoked']:
            return '{}: {}'.format(package, state)

        # instead of just printing the state of the whole request find
        # out who is remaining on the review and print it out,
        # otherwise print out that it is ready for approval and
        # waiting on others from GR to be accepted
        review_state = root.findall('review')
        failing_groups = []
        for i in review_state:
            if i.attrib['state'] == 'accepted':
                continue
            if i.get('by_project', None) == project:
                continue
            for attrib in ['by_group', 'by_user', 'by_project', 'by_package']:
                value = i.get(attrib, None)
                if value:
                    failing_groups.append(value)

        if not failing_groups:
            return None
        else:
            state = 'missing reviews: ' + ', '.join(failing_groups)
            return '{}: {}'.format(package, state)

    def check_ring_packages(self, project, requests):
        """
        Checks if packages from requests are in some ring or not
        :param project: project to check
        :param requests: list of requests to verify
        :return True (has ring packages) / False (has no ring packages)
        """

        for request in requests:
            pkg = self.get_package_for_request_id(project, request)
            if pkg in self.ring_packages:
                return True

        return False

    def check_project_status(self, project, verbose=False):
        """
        Checks a staging project for acceptance. Checks all open
        requests for open reviews and build status
        :param project: project to check
        :return true (ok)/false (empty prj) or list of strings with
                informations)
        """

        # Report
        report = list()

        # First ensure we dispatched the open requests so we do not
        # pass projects with update/superseded requests
        for request in self.get_open_requests():
            stage_info = self.supseded_request(request)
            if stage_info and stage_info['prj'] == project:
                return ['Request {} is superseded'.format(stage_info['rq_id'])]

        # all requests with open review
        requests = self.list_requests_in_prj(project)
        open_requests = set(requests)

        # all tracked requests - some of them might be declined, so we
        # don't see them above
        meta = self.get_prj_pseudometa(project)
        for req in meta['requests']:
            req = req['id']
            if req in open_requests:
                open_requests.remove(req)
            if req not in requests:
                requests.append(req)
        if open_requests:
            return ['Request(s) {} are not tracked but are open for the prj'.format(','.join(map(str, open_requests)))]

        # If we find no requests in staging then it is empty so we ignore it
        if not requests:
            return False

        # Check if the requests are acceptable and bail out on
        # first failure unless verbose as it is slow
        for request in requests:
            ret = self.check_one_request(request, project)
            if ret:
                report.append(ret)
                if not verbose:
                    break

        # Check the build/openQA only if we have some ring packages
        if self.check_ring_packages(project, requests):
            # Check the buildstatus
            buildstatus = self.gather_build_status(project)
            if buildstatus:
                # here no append as we are adding list to list
                report += self.generate_build_status_details(buildstatus, verbose)
            # Check the openqa state
            ret = self.find_openqa_state(project)
            if ret:
                report.append(ret)

        if report:
            return report
        elif not self.project_exists(project + ":DVD"):
            # The only case we are green
            return True

        # now check the same for the subprj
        project = project + ":DVD"
        buildstatus = self.gather_build_status(project)

        if buildstatus:
            report += self.generate_build_status_details(buildstatus, verbose)

        # Check the openqa state
        ret = self.find_openqa_state(project)
        if ret:
            report.append(ret)

        if report:
            return report

        return True

    def days_since_last_freeze(self, project):
        """
        Checks the last update for the frozen links
        :param project: project to check
        :return age in days(float) of the last update
        """
        url = self.makeurl(['source', project, '_project'], {'meta': '1'})
        root = ET.parse(http_GET(url)).getroot()
        for entry in root.findall('entry'):
            if entry.get('name') == '_frozenlinks':
                return (time.time() - float(entry.get('mtime')))/3600/24
        return 100000  # quite some!

    def find_openqa_jobs(self, project):
        url = self.makeurl(['build', project, 'images', 'x86_64', 'Test-DVD-x86_64'])
        root = ET.parse(http_GET(url)).getroot()

        filename = None
        for binary in root.findall('binary'):
            filename = binary.get('filename', '')
            if filename.endswith('.iso'):
                break

        if not filename:
            return None

        jobtemplate = '-Staging'
        if project.endswith(':DVD'):
            jobtemplate = '-Staging2'
            project = project[:-4]

        jobname = 'openSUSE-Staging:'
        jobname += project.split(':')[-1] + jobtemplate
        result = re.match('Test-Build([^-]+)-Media.iso', filename)
        jobname += '-DVD-x86_64-Build' + result.group(1) + "-Media.iso"

        try:
            url = "https://openqa.opensuse.org/api/v1/jobs?iso={}".format(jobname)
            f = urllib2.urlopen(url)
        except urllib2.HTTPError:
            return None

        jobs = json.load(f)['jobs']

        bestjobs = {}
        for job in jobs:
            if job['result'] != 'incomplete' and not job['clone_id']:
                if job['test'] == 'miniuefi':
                    continue
                if job['name'] not in bestjobs or bestjobs[job['name']]['result'] != 'passed':
                    bestjobs[job['name']] = job

        return bestjobs

    def find_openqa_state(self, project):
        """
        Checks the openqa state of the project
        :param project: project to check
        :return None or list with issue informations
        """

        jobs = self.find_openqa_jobs(project)

        if not jobs:
            return 'No openQA result yet'

        for job in jobs.values():
            check = self.check_if_job_is_ok(job)
            if check:
                return check

        return None

    def check_if_job_is_ok(self, job):
        url = 'https://openqa.opensuse.org/tests/{}/file/results.json'.format(job['id'])
        try:
            f = urllib2.urlopen(url)
        except urllib2.HTTPError:
            return "Can't open {}".format(url)

        try:
            openqa = json.load(f)
        except ValueError:
            return "Can't decode {}".format(url)

        overall = openqa.get('overall', 'inprogress')
        if job['test'] == 'miniuefi':
            return None  # ignore
        # pprint.pprint(openqa)
        # pprint.pprint(job)
        if overall != 'ok':
            return "openQA's overall status is {} for https://openqa.opensuse.org/tests/{}".format(overall, job['id'])

        for module in openqa['testmodules']:
            # zypper_in fails at the moment - urgent fix needed
            if module['result'] == 'ok':
                continue
            if module['name'] in []:
                continue
            return '{} test failed: https://openqa.opensuse.org/tests/{}'.format(module['name'], job['id'])
        return None

    def gather_build_status(self, project):
        """
        Checks whether everything is built in project
        :param project: project to check
        """
        # Get build results
        url = self.makeurl(['build', project, '_result?code=failed&code=broken&code=unresolvable'])
        root = ET.parse(http_GET(url)).getroot()

        # Check them
        broken = []
        working = []
        # Iterate through repositories
        for results in root.findall('result'):
            building = False
            if results.get('state') not in ('published', 'unpublished') or results.get('dirty') == 'true':
                working.append({
                    'path': '{}/{}'.format(results.get('repository'),
                                           results.get('arch')),
                    'state': results.get('state')
                })
                building = True
            # Iterate through packages
            for node in results:
                # Find broken
                result = node.get('code')
                if result in ('broken', 'failed') or (result == 'unresolvable' and not building):
                    broken.append({
                        'pkg': node.get('package'),
                        'state': result,
                        'path': '{}/{}'.format(results.get('repository'),
                                               results.get('arch'))
                    })

        # Print the results
        if not working and not broken:
            return None
        else:
            return [project, working, broken]

    def generate_build_status_details(self, details, verbose=False):
        """
        Generate list of strings for the buildstatus detail report.
        :param details: buildstatus informations about project
        :return list of strings for printing
        """

        retval = list()
        if not isinstance(details, list):
            return retval
        project, working, broken = details

        if working:
            retval.append('At least following repositories is still building:')
            for i in working:
                retval.append('    {}: {}'.format(i['path'], i['state']))
                if not verbose:
                    break
        if broken:
            retval.append('Following packages are broken:')
            for i in broken:
                retval.append('    {} ({}): {}'.format(i['pkg'], i['path'],
                                                       i['state']))

        return retval

    def rq_to_prj(self, request_id, project):
        """
        Links request to project - delete or submit
        :param request_id: request to link
        :param project: project to link into
        """
        # read info from sr
        tar_pkg = None

        req = get_request(self.apiurl, str(request_id))
        if not req:
            raise oscerr.WrongArgs('Request {} not found'.format(request_id))

        act = req.get_actions('submit')
        if act:
            tar_pkg = self.submit_to_prj(act[0], project)

        act = req.get_actions('delete')
        if act:
            tar_pkg = self.delete_to_prj(act[0], project)

        if not tar_pkg:
            msg = 'Request {} is not a submit or delete request'
            msg = msg.format(request_id)
            raise oscerr.WrongArgs(msg)

        # register the package name
        self._add_rq_to_prj_pseudometa(project, int(request_id), tar_pkg)

        # add review
        self.add_review(request_id, project)

        # now remove the staging checker
        self.do_change_review_state(request_id, 'accepted',
                                    by_group='factory-staging',
                                    message='Picked {}'.format(project))
        return True

    def map_ring_package_to_subject(self, project, pkg):
        """
        Returns the subproject (if any) to use for the pkg depending on the ring
        the package is in
        :param project the staging prj
        :param pkg the package to add
        """
        # it's actually a pretty stupid algorithm, but it might become more complex later

        if self.ring_packages.get(pkg) == 'openSUSE:Factory:Rings:2-TestDVD':
            return project + ":DVD"

        return project

    def delete_to_prj(self, act, project):
        """
        Hides Package in project
        :param act: action for delete request
        :param project: project to hide in
        """

        tar_pkg = act.tgt_package
        project = self.map_ring_package_to_subject(project, tar_pkg)

        # create build disabled package
        self.create_package_container(project, tar_pkg, disable_build=True)
        # now trigger wipebinaries to emulate a delete
        url = self.makeurl(['build', project],
                           {'cmd': 'wipe', 'package': tar_pkg})
        http_POST(url)

        return tar_pkg

    def submit_to_prj(self, act, project):
        """
        Links sources from request to project
        :param act: action for submit request
        :param project: project to link into
        """

        src_prj = act.src_project
        src_rev = act.src_rev
        src_pkg = act.src_package
        tar_pkg = act.tgt_package

        disable_build = False
        if not self.ring_packages.get(tar_pkg):
            disable_build = True
        else:
            project = self.map_ring_package_to_subject(project, tar_pkg)

        self.create_package_container(project, tar_pkg,
                                      disable_build=disable_build)

        # if it's disabled anyway, it doesn't make a difference if we link or not
        if disable_build:
            return tar_pkg

        # expand the revision to a md5
        url = self.makeurl(['source', src_prj, src_pkg],
                           {'rev': src_rev, 'expand': 1})
        f = http_GET(url)
        root = ET.parse(f).getroot()
        src_rev = root.attrib['srcmd5']
        src_vrev = root.attrib.get('vrev')

        # link stuff - not using linkpac because linkpac copies meta
        # from source
        root = ET.Element('link', package=src_pkg, project=src_prj,
                          rev=src_rev)
        if src_vrev:
            root.attrib['vrev'] = src_vrev
        url = self.makeurl(['source', project, tar_pkg, '_link'])
        http_PUT(url, data=ET.tostring(root))
        return tar_pkg

    def prj_from_letter(self, letter):
        if ':' in letter:  # not a letter
            return letter
        return 'openSUSE:Factory:Staging:%s' % letter

    def list_requests_in_prj(self, project):
        where = "@by_project='%s'+and+@state='new'" % project

        url = self.makeurl(['search', 'request', 'id'],
                           "match=state/@name='review'+and+review[%s]" % where)
        f = http_GET(url)
        root = ET.parse(f).getroot()
        list = []
        for rq in root.findall('request'):
            list.append(int(rq.get('id')))

        return list

    def add_review(self, request_id, by_project=None, by_group=None, msg=None):
        """
        Adds review by project to the request
        :param request_id: request to add review to
        :param project: project to assign review to
        """
        req = get_request(self.apiurl, str(request_id))
        if not req:
            raise oscerr.WrongArgs('Request {} not found'.format(request_id))
        for i in req.reviews:
            if by_project and i.by_project == by_project and i.state == 'new':
                return
            if by_group and i.by_group == by_group and i.state == 'new':
                return

        # don't try to change reviews if the request is dead
        if req.state.name not in ('new', 'review'):
            return

        query = {}
        if by_project:
            query['by_project'] = by_project
            if not msg:
                msg = 'Being evaluated by staging project "{}"'
                msg = msg.format(by_project)
        if by_group:
            query['by_group'] = by_group
            if not msg:
                msg = 'Being evaluated by group "{}"'.format(by_group)
        if not query:
            raise oscerr.WrongArgs('We need a group or a project')
        query['cmd'] = 'addreview'
        url = self.makeurl(['request', str(request_id)], query)
        http_POST(url, data=msg)

    def set_review(self, request_id, project, state='accepted', msg=None):
        """
        Sets review for request done by project
        :param request_id: request to change review for
        :param project: project to do the review
        """
        req = get_request(self.apiurl, str(request_id))
        if not req:
            raise oscerr.WrongArgs('Request {} not found'.format(request_id))
        # don't try to change reviews if the request is dead
        if req.state.name not in ('new', 'review'):
            return
        cont = False
        for i in req.reviews:
            if i.by_project == project and i.state == 'new':
                cont = True
        if not cont:
            return
        if not msg:
            msg = 'Reviewed by staging project "{}" with result: "{}"'
            msg = msg.format(project, state)
        self.do_change_review_state(request_id, state, by_project=project,
                                    message=msg)

    def switch_flag_in_prj(self, project, flag='build', state='disable', repository=None, arch=None):
        url = self.makeurl(['source', project, '_meta'])
        prjmeta = ET.parse(http_GET(url)).getroot()

        flagxml = prjmeta.find(flag)
        if not flagxml:  # appending is fine
            flagxml = ET.SubElement(prjmeta, flag)

        foundone = False
        for build in flagxml:
            if build.get('repository', None) == repository and build.get('arch', None) == arch:
                build.tag = state
                foundone = True

        # need to add a global one
        if not foundone:
            query = {}
            if arch:
                query['arch'] = arch
            if repository:
                query['repository'] = repository
            ET.SubElement(flagxml, state, query)

        http_PUT(url, data=ET.tostring(prjmeta))

    def build_switch_prj(self, project, state):
        """
        Switch build state of project to desired state
        :param project: project to switch state for
        :param state: desired state for build
        """
        self.switch_flag_in_prj(project, flag='build', state=state, repository=None, arch=None)

    def prj_frozen_enough(self, project):
        """
        Check if we can and should refreeze the prj"
        :param project the project to check
        :returns True if we can select into it
        """

        data = self.get_prj_pseudometa(project)
        if data['requests']:
            return True  # already has content

        # young enough
        if self.days_since_last_freeze(project) < 6.5:
            return True

        return False

    def build_switch_staging_project(self, target_project):
        """
        Verify what packages are in project and switch the build
        accordingly.
        :param target_project: project we validate and switch
        """
        meta = self.get_prj_pseudometa(target_project)
        staged_requests = list()
        for request in meta['requests']:
            staged_requests.append(request['id'])
        target_flag = 'disable'
        if self.check_ring_packages(target_project, staged_requests):
            target_flag = 'enable'
        self.build_switch_prj(target_project, target_flag)

        if self.project_exists(target_project + ":DVD"):
            self.build_switch_prj(target_project + ":DVD", target_flag)

    def project_exists(self, project):
        """
        Return true if the given project exists
        :param project: project name to check
        """
        url = self.makeurl(['source', project, '_meta'])
        try:
            http_GET(url)
        except urllib2.HTTPError:
            return False
        return True

    def update_status_comments(self, project, command):
        """
        Refresh the status comments, used for notification purposes, based on
        the current list of requests. To ensure that all involved users
        (and nobody else) get notified, old status comments are deleted and
        a new one is created.
        :param project: project name
        :param command: name of the command to include in the message
        """

        # TODO: we need to discuss the best way to keep track of status
        # comments. Right now they are marked with an initial markdown
        # comment. Maybe a cleaner approach would be to store something
        # like 'last_status_comment_id' in the pseudometa. But the current
        # OBS API for adding comments doesn't return the id of the created
        # comment.

        comment_api = CommentAPI(self.apiurl)

        comments = comment_api.get_comments(project_name=project)
        for comment in comments.values():
            # TODO: update the comment removing the user mentions instead of
            # deleting the whole comment. But there is currently not call in
            # OBS API to update a comment
            if comment['comment'].startswith('<!--- osc staging'):
                comment_api.delete(comment['id'])
                break  # There can be only one! (if we keep deleting them)

        meta = self.get_prj_pseudometa(project)
        lines = ['<!--- osc staging %s --->' % command]
        lines.append('The list of requests tracked in %s has changed:\n' % project)
        for req in meta['requests']:
            author = req.get('autor', None)
            if not author:
                # Old style metadata
                author = get_request(self.apiurl, str(req['id'])).get_creator()
            lines.append('  * Request#%s for package %s submitted by [AT]%s' % (req['id'], req['package'], author))
        msg = '\n'.join(lines)
        comment_api.add_comment(project_name=project, comment=msg)