Merge pull request #3 from aplanas/master

Some basic refactorization. Add outliers detector from Base:build.

osc-check_repo.py

@@ -3,31 +3,52 @@
 # Distribute under GPLv2 or GPLv3
 #
 # Copy this script to ~/.osc-plugins/ or /var/lib/osc-plugins .
-# Then try to run 'osc checker --help' to see the usage.
+# Then try to run 'osc check_repo --help' to see the usage.
 
-import socket
 import os
-import traceback
+import re
 import subprocess
+import shutil
+from urllib import quote_plus
+import urllib2
 
-def _check_repo_change_review_state(self, opts, id, newstate, message='', supersed=None):
+from xml.etree import cElementTree as ET
+
+from osc import oscerr
+from osc.core import (get_binary_file,
+                      get_buildinfo,
+                      http_GET,
+                      http_POST,
+                      makeurl,
+                      Request)
+
+
+def _check_repo_change_review_state(self, opts, id_, newstate, message='', supersed=None):
     """ taken from osc/osc/core.py, improved:
         - verbose option added,
         - empty by_user=& removed.
         - numeric id can be int().
     """
-    query = {'cmd': 'changereviewstate', 'newstate': newstate }
+    query = {
-    query['by_user']= 'factory-repo-checker'
+        'cmd': 'changereviewstate',
-    if supersed: query['superseded_by'] = supersed
+        'newstate': newstate,
-#    if message: query['comment'] = message
+        'by_user': 'factory-repo-checker',
-    u = makeurl(opts.apiurl, ['request', str(id)], query=query)
+    }
+    if supersed:
+        query['superseded_by'] = supersed
+    # if message:
+    #     query['comment'] = message
+
+    code = 404
+    u = makeurl(opts.apiurl, ['request', str(id_)], query=query)
     try:
         f = http_POST(u, data=message)
         root = ET.parse(f).getroot()
-      return root.attrib['code']
+        code = root.attrib['code']
-    except urllib2.HTTPError:
+    except urllib2.HTTPError, e:
-      print "error", u
+        print 'ERROR in URL %s [%s]'%(u, e)
-      return 404
+    return code
+
 
 def _check_repo_find_submit_request(self, opts, project, package):
     xpath = "(action/target/@project='%s' and action/target/@package='%s' and action/@type='submit' and (state/@name='new' or state/@name='review' or state/@name='accepted'))" % (project, package)
@@ -44,6 +65,7 @@ def _check_repo_find_submit_request(self, opts, project, package):
         return int(r.reqid)
     return None
 
+        
 def _check_repo_fetch_group(self, opts, group):
     if opts.groups.get(group): return
     u = makeurl(opts.apiurl, ['request', str(group)])
@@ -56,6 +78,7 @@ def _check_repo_fetch_group(self, opts, group):
         opts.grouped[id] = group
     opts.groups[group] = a
 
+
 def _check_repo_avoid_wrong_friends(self, prj, repo, arch, pkg, opts):
     try:
         url = makeurl(opts.apiurl, ["build", prj, repo, arch, pkg])
@@ -68,11 +91,12 @@ def _check_repo_avoid_wrong_friends(self, prj, repo, arch, pkg, opts):
         return False
     return True
 
+
 def _check_repo_one_request(self, rq, opts):
 
     class CheckRepoPackage:
         def __repr__(self):
-            return "[%d:%s/%s]" % (int(self.request),self.sproject, self.spackage)
+            return '[%d:%s/%s]' % (int(self.request), self.sproject, self.spackage)
 
 	def __init__(self):
 	    self.updated = False
@@ -81,7 +105,6 @@ def _check_repo_one_request(self, rq, opts):
 
 
     id = int(rq.get('id'))
-    approved_actions = 0
     actions = rq.findall('action')
     if len(actions) != 1:
        msg = "only one action per request is supported - create a group instead: https://github.com/SUSE/hackweek/wiki/Improved-Factory-devel-project-submission-workflow"
@@ -181,6 +204,7 @@ def _check_repo_one_request(self, rq, opts):
         sp.rev = root.attrib['srcmd5']
     return packs
 
+
 def _check_repo_buildsuccess(self, p, opts):
     try:
         url = makeurl(opts.apiurl, ['build', p.sproject, "_result?lastsuccess&package=%s&pathproject=%s&srcmd5=%s" % (quote_plus(p.spackage), quote_plus(p.tproject), p.rev)])
@@ -197,7 +221,6 @@ def _check_repo_buildsuccess(self, p, opts):
     alldisabled = True
     foundbuilding = None
     foundfailed = None
-    found64 = True
 
     tocheckrepos = []
     for repo in root.findall('repository'):
@@ -283,6 +306,7 @@ def _check_repo_buildsuccess(self, p, opts):
 
     return True
 
+
 def _check_repo_repo_list(self, prj, repo, arch, pkg, opts):
     url = makeurl(opts.apiurl, ['build', prj, repo, arch, pkg])
     files = []
@@ -304,15 +328,17 @@ def _check_repo_repo_list(self, prj, repo, arch, pkg, opts):
             if result.group(4) == 'src':
                 continue
             files.append((fn, pname, result.group(4)))
-    except urllib2.HTTPError, err:
+    except urllib2.HTTPError:
-        pass
+        print "error", url
     return files
 
+
 def _check_repo_get_binary(self, apiurl, prj, repo, arch, package, file, target):
     if os.path.exists(target):
         return
     get_binary_file(apiurl, prj, repo, arch, file, package = package, target_filename = target)
 
+
 def _check_repo_download(self, p, destdir, opts):
     if p.build_excluded:
         return [], []
@@ -356,6 +382,32 @@ def _check_repo_download(self, p, destdir, opts):
         toignore.append(fn[1])
     return toignore, downloads
 
+
+def _get_build_deps(self, prj, repo, arch, pkg, opts):
+    xml = get_buildinfo(opts.apiurl, prj, pkg, repo, arch)
+    root = ET.fromstring(xml)
+    return [e.attrib['name'] for e in root.findall('bdep')]
+
+
+def _get_base_build_bin(self, opts):
+    """Get Base:build pagacke list"""
+    binaries = {}
+    for arch in ('x86_64', 'i586'):
+        url = makeurl(opts.apiurl, ['/build/Base:build/standard/%s/_repository'%arch,])
+        f = http_GET(url)
+        root = ET.parse(f).getroot()
+        binaries[arch] = set([e.attrib['filename'][:-4] for e in root.findall('binary')])
+    return binaries
+
+
+def _get_base_build_src(self, opts):
+    """Get Base:build pagacke list"""
+    url = makeurl(opts.apiurl, ['/source/Base:build',])
+    f = http_GET(url)
+    root = ET.parse(f).getroot()
+    return set([e.attrib['name'] for e in root.findall('entry')])
+
+
 def _check_repo_group(self, id, reqs, opts):
     print "\ncheck group", reqs
     for p in reqs:
@@ -395,6 +447,20 @@ def _check_repo_group(self, id, reqs, opts):
         downloads.extend(d)
         toignore.extend(i)
 
+    # Get all the Base:build packages (source and binary)
+    base_build_bin = self._get_base_build_bin(opts)
+    base_build_src = self._get_base_build_src(opts)
+    for p in reqs:
+        # Be sure that if the package is in Base:build, all the
+        # dependecies are also in Base:build
+        if p.spackage in base_build_src:
+            # TODO - Check all the arch for this package
+            for arch in ('x86_64', 'i586'):
+                build_deps = set(self._get_build_deps(p.sproject, p.goodrepo, arch, p.spackage, opts))
+                outliers = build_deps - base_build_bin[arch]
+                if outliers:
+                    print 'Outliers (%s)'%arch, outliers
+
     for p in reqs:
         smissing = []
         for package in p.missings:
@@ -451,60 +517,56 @@ def _check_repo_group(self, id, reqs, opts):
 	p.updated = True
     shutil.rmtree(destdir)
 
-def _check_repo_fetch_request(self, id, opts):
+
-  url = makeurl(opts.apiurl, ['request', str(id)])
+def _check_repo_fetch_request(self, id_, opts):
+    url = makeurl(opts.apiurl, ['request', str(id_)])
     f = http_GET(url)
     xml = ET.parse(f)
     root = xml.getroot()
     return self._check_repo_one_request(root, opts)
 
+
 def do_check_repo(self, subcmd, opts, *args):
     """${cmd_name}: checker review of submit requests.
 
     Usage:
-      osc checker [OPT] [list] [FILTER|PACKAGE_SRC]
+      osc check_repo [OPT] [list] [FILTER|PACKAGE_SRC]
            Shows pending review requests and their current state.
 
     ${cmd_option_list}
     """
 
     if len(args) == 0:
-        raise oscerr.WrongArgs("Please give a subcommand to 'osc checker' or try 'osc help checker'")
+        raise oscerr.WrongArgs("Please give a subcommand to 'osc check_repo' or try 'osc help check_repo'")
 
     opts.mode = ''
     opts.groups = {}
     opts.grouped = {}
     opts.verbose = False
 
-    from pprint import pprint
-
     opts.apiurl = self.get_api_url()
 
-    tmphome = None
-
     if args[0] == 'skip':
-        for id in args[1:]:
+        for id_ in args[1:]:
-            self._check_repo_change_review_state(opts, id, 'accepted', message="skip review")
+            self._check_repo_change_review_state(opts, id_, 'accepted', message='skip review')
         return
-    ids = {}
+
-    for a in args:
+    ids = [arg for arg in args if arg.isdigit()]
-        if (re.match('\d+', a)):
-            ids[a] = 1
 
     packs = []
-    if (not len(ids)):
+    if not ids:
         # xpath query, using the -m, -r, -s options
         where = "@by_user='factory-repo-checker'+and+@state='new'"
-
+        url = makeurl(opts.apiurl, ['search', 'request'], 
-        url = makeurl(opts.apiurl, ['search','request'], "match=state/@name='review'+and+review["+where+"]")
+                      "match=state/@name='review'+and+review["+where+"]")
         f = http_GET(url)
         root = ET.parse(f).getroot()
         for rq in root.findall('request'):
             packs.extend(self._check_repo_one_request(rq, opts))
     else:
         # we have a list, use them.
-        for id in ids.keys():
+        for id_ in ids:
-	    packs.extend(self._check_repo_fetch_request(id, opts))
+	    packs.extend(self._check_repo_fetch_request(id_, opts))
 
     groups = {}
     for p in packs:
@@ -512,11 +574,5 @@ def do_check_repo(self, subcmd, opts, *args):
         a.append(p)
         groups[p.group] = a
 
-    for id, reqs in groups.items():
+    for id_, reqs in groups.items():
-        self._check_repo_group(id, reqs, opts)
+        self._check_repo_group(id_, reqs, opts)
-
-#Local Variables:
-#mode: python
-#py-indent-offset: 4
-#tab-width: 8
-#End: