Move more fuctions to checkrepo.py

osc-check_repo.py

@@ -20,164 +20,30 @@
 from collections import defaultdict
 from collections import namedtuple
 import os
-import re
 import shutil
 import subprocess
 import tempfile
-from urllib import quote_plus
-import urllib2
 import sys
-from xml.etree import cElementTree as ET
 
 from osc import oscerr
 from osc import cmdln
 
-from osc.core import get_binary_file
-from osc.core import get_buildinfo
-from osc.core import http_GET
-from osc.core import makeurl
-from osc.core import Request
 
 # Expand sys.path to search modules inside the pluging directory
 _plugin_dir = os.path.expanduser('~/.osc-plugins')
 sys.path.append(_plugin_dir)
-from osclib.checkrepo import CheckRepo, DOWNLOADS
+from osclib.checkrepo import CheckRepo
 from osclib.cycle import CycleDetector
 from osclib.memoize import CACHEDIR
 
 
-def _check_repo_find_submit_request(self, opts, project, package):
-    xpath = "(action/target/@project='%s' and "\
-            "action/target/@package='%s' and "\
-            "action/@type='submit' and "\
-            "(state/@name='new' or state/@name='review' or "\
-            "state/@name='accepted'))" % (project, package)
-    try:
-        url = makeurl(opts.apiurl, ['search', 'request'], 'match=%s' % quote_plus(xpath))
-        f = http_GET(url)
-        collection = ET.parse(f).getroot()
-    except urllib2.HTTPError, e:
-        print('ERROR in URL %s [%s]' % (url, e))
-        return None
-    for root in collection.findall('request'):
-        r = Request()
-        r.read(root)
-        return int(r.reqid)
-    return None
-
-
-def _check_repo_repo_list(self, prj, repo, arch, pkg, opts):
-    url = makeurl(opts.apiurl, ['build', prj, repo, arch, pkg])
-    files = []
-    try:
-        binaries = ET.parse(http_GET(url)).getroot()
-        for bin_ in binaries.findall('binary'):
-            fn = bin_.attrib['filename']
-            mt = int(bin_.attrib['mtime'])
-            result = re.match(r'(.*)-([^-]*)-([^-]*)\.([^-\.]+)\.rpm', fn)
-            if not result:
-                if fn == 'rpmlint.log':
-                    files.append((fn, '', '', mt))
-                continue
-            pname = result.group(1)
-            if pname.endswith('-debuginfo') or pname.endswith('-debuginfo-32bit'):
-                continue
-            if pname.endswith('-debugsource'):
-                continue
-            if result.group(4) == 'src':
-                continue
-            files.append((fn, pname, result.group(4), mt))
-    except urllib2.HTTPError:
-        pass
-        # print " - WARNING: Can't found list of packages (RPM) for %s in %s (%s, %s)" % (pkg, prj, repo, arch)
-    return files
-
-
-def _check_repo_get_binary(self, apiurl, prj, repo, arch, package, file, target, mtime):
-    if os.path.exists(target):
-        # we need to check the mtime too as the file might get updated
-        cur = os.path.getmtime(target)
-        if cur > mtime:
-            return
-
-    get_binary_file(apiurl, prj, repo, arch, file, package=package, target_filename=target)
-
-
-def _download(self, request, todownload, opts):
-    """Download the packages refereced in a request."""
-    last_disturl = None
-    last_disturldir = None
-
-    # We need to order the files to download.  First RPM packages (to
-    # set disturl), after that the rest.
-
-    todownload_rpm = [rpm for rpm in todownload if rpm[3].endswith('.rpm')]
-    todownload_rest = [rpm for rpm in todownload if not rpm[3].endswith('.rpm')]
-
-    for _project, _repo, arch, fn, mt in todownload_rpm:
-        repodir = os.path.join(DOWNLOADS, request.src_package, _project, _repo)
-        if not os.path.exists(repodir):
-            os.makedirs(repodir)
-        t = os.path.join(repodir, fn)
-        self._check_repo_get_binary(opts.apiurl, _project, _repo,
-                                    arch, request.src_package, fn, t, mt)
-
-        # Organize the files into DISTURL directories.
-        disturl = self.checkrepo._md5_disturl(self.checkrepo._disturl(t))
-        disturldir = os.path.join(repodir, disturl)
-        last_disturl, last_disturldir = disturl, disturldir
-        file_in_disturl = os.path.join(disturldir, fn)
-        if not os.path.exists(disturldir):
-            os.makedirs(disturldir)
-        try:
-            os.symlink(t, file_in_disturl)
-        except:
-            pass
-            # print 'Found previous link.'
-
-        request.downloads[(_project, _repo, disturl)].append(file_in_disturl)
-
-    for _project, _repo, arch, fn, mt in todownload_rest:
-        repodir = os.path.join(DOWNLOADS, request.src_package, _project, _repo)
-        if not os.path.exists(repodir):
-            os.makedirs(repodir)
-        t = os.path.join(repodir, fn)
-        self._check_repo_get_binary(opts.apiurl, _project, _repo,
-                                    arch, request.src_package, fn, t, mt)
-
-        file_in_disturl = os.path.join(last_disturldir, fn)
-        if last_disturldir:
-            try:
-                os.symlink(t, file_in_disturl)
-            except:
-                pass
-                # print 'Found previous link.'
-        else:
-            print "I don't know where to put", fn
-
-        request.downloads[(_project, _repo, last_disturl)].append(file_in_disturl)
-
-
-def _check_repo_toignore(self, request, opts):
-    toignore = set()
-    for fn in self._check_repo_repo_list(request.tgt_project, 'standard', 'x86_64', request.tgt_package, opts):
-        if fn[1]:
-            toignore.add(fn[1])
-
-    # now fetch -32bit pack list
-    for fn in self._check_repo_repo_list(request.tgt_project, 'standard', 'i586', request.tgt_package, opts):
-        if fn[1] and fn[2] == 'x86_64':
-            toignore.add(fn[1])
-    return toignore
-
-
 def _check_repo_download(self, request, opts):
     request.downloads = defaultdict(list)
 
     if request.is_cached:
         request.downloads = self.checkrepo._get_downloads_from_local(request)
         # print ' - Found cached version for', request.str_compact()
-        return self._check_repo_toignore(request, opts)
+        return self.checkrepo._toignore(request)
 
     if request.build_excluded:
         return set()
@@ -188,49 +54,37 @@ def _check_repo_download(self, request, opts):
         repo = goodrepo[1]
 
         # we can assume x86_64 is there
-        todownload = [ToDownload(request.src_project, repo, 'x86_64', fn[0], fn[3])
+        todownload = [ToDownload(request.src_project, repo, 'x86_64',
-                      for fn in self._check_repo_repo_list(request.src_project,
+                                 fn[0], fn[3]) for fn in
-                                                           repo,
+                      self.checkrepo.get_package_list_from_repository(
-                                                           'x86_64',
+                          request.src_project, repo, 'x86_64',
-                                                           request.src_package,
+                          request.src_package)]
-                                                           opts)]
 
-        self._download(request, todownload, opts)
+        self.checkrepo.download(request, todownload)
         if request.error:
             return set()
 
     if 'openSUSE:Factory:Staging:' in str(request.group):
-        todownload = [
+        todownload = [ToDownload(request.group, 'standard', 'x86_64',
-            ToDownload(request.group, 'standard', 'x86_64', fn[0], fn[3])
+                                 fn[0], fn[3]) for fn in
-            for fn in self._check_repo_repo_list(request.group,
+                      self.checkrepo.get_package_list_from_repository(
-                                                 'standard',
+                          request.group, 'standard', 'x86_64',
-                                                 'x86_64',
+                          request.src_package)]
-                                                 request.src_package,
-                                                 opts)]
 
-        self._download(request, todownload, opts)
+        self.checkrepo.download(request, todownload)
         if request.error:
             return set()
 
-        todownload = [
+        todownload = [ToDownload(request.group + ':DVD', 'standard',
-            ToDownload(request.group + ':DVD', 'standard', 'x86_64', fn[0], fn[3])
+                                 'x86_64', fn[0], fn[3]) for fn in
-            for fn in self._check_repo_repo_list(request.group + ':DVD',
+                      self.checkrepo.get_package_list_from_repository(
-                                                 'standard',
+                          request.group + ':DVD', 'standard',
-                                                 'x86_64',
+                          'x86_64', request.src_package)]
-                                                 request.src_package,
-                                                 opts)]
 
-        self._download(request, todownload, opts)
+        self.checkrepo.download(request, todownload)
         if request.error:
             return set()
-    return self._check_repo_toignore(request, opts)
+    return self.checkrepo._toignore(request)
-
-
-def _get_buildinfo(self, opts, prj, repo, arch, pkg):
-    """Get the build info for a package"""
-    xml = get_buildinfo(opts.apiurl, prj, pkg, repo, arch)
-    root = ET.fromstring(xml)
-    return [e.attrib['name'] for e in root.findall('bdep')]
 
 
 # Used in _check_repo_group only to cache error messages
@@ -278,7 +132,7 @@ def _check_repo_group(self, id_, requests, opts):
         i = set()
         if rq.action_type == 'delete':
             # for delete requests we only care for toignore
-            i = self._check_repo_toignore(rq, opts)
+            i = self.checkrepo._toignore(rq)
         else:
             # we need to call it to fetch the good repos to download
             # but the return value is of no interest right now.
@@ -310,7 +164,7 @@ def _check_repo_group(self, id_, requests, opts):
                     alreadyin = True
             if alreadyin:
                 continue
-            request = self._check_repo_find_submit_request(opts, rq.tgt_project, package)
+            request = self.checkrepo.find_request_id(rq.tgt_project, package)
             if request:
                 greqs = opts.groups.get(rq.group, [])
                 if request in greqs:

osclib/checkrepo.py

@@ -16,11 +16,13 @@
 
 from collections import defaultdict
 import os
+import re
 import subprocess
 from urllib import quote_plus
 import urllib2
 from xml.etree import cElementTree as ET
 
+from osc.core import get_binary_file
 from osc.core import http_GET
 from osc.core import http_POST
 from osc.core import makeurl
@@ -171,6 +173,32 @@ class CheckRepo(object):
             print('ERROR in URL %s [%s]' % (url, e))
         return requests
 
+    def find_request_id(self, project, package):
+        """Return a request id that is in new, review of accepted state for a
+        specific project/package.
+
+        """
+        xpath = "(action/target/@project='%s' and "\
+                "action/target/@package='%s' and "\
+                "action/@type='submit' and "\
+                "(state/@name='new' or state/@name='review' or "\
+                "state/@name='accepted'))" % (project, package)
+        query = {
+            'match': quote_plus(xpath)
+        }
+
+        request_id = None
+        try:
+            url = makeurl(self.apiurl, ('search', 'request'), query=query)
+            collection = ET.parse(http_GET(url)).getroot()
+            for root in collection.findall('request'):
+                r = Request()
+                r.read(root)
+                request_id = int(r.reqid)
+        except urllib2.HTTPError, e:
+            print('ERROR in URL %s [%s]' % (url, e))
+        return request_id
+
     def _build(self, project, repository, arch, package):
         """Return the build XML document from OBS."""
         xml = ''
@@ -384,7 +412,7 @@ class CheckRepo(object):
                 else:
                     msg = '%s is no longer the submitted version, please resubmit HEAD' % spec
                 print '[DECLINED] CHECK MANUALLY', msg
-                # self.checkrepo.change_review_state(id_, 'declined', message=msg)
+                # self.change_review_state(id_, 'declined', message=msg)
                 rq.updated = True
 
             sp = Request(request_id=rq.request_id,
@@ -436,6 +464,87 @@ class CheckRepo(object):
             return False
         return True
 
+    def get_binary_file(self, project, repository, arch, package, filename, target, mtime):
+        """Get a binary file from OBS."""
+        if os.path.exists(target):
+            # we need to check the mtime too as the file might get updated
+            cur = os.path.getmtime(target)
+            if cur > mtime:
+                return
+
+        get_binary_file(self.apiurl, project, repository, arch,
+                        filename, package=package,
+                        target_filename=target)
+
+    def download(self, request, todownload):
+        """Download the packages refereced in a request."""
+        last_disturl = None
+        last_disturldir = None
+
+        # We need to order the files to download.  First RPM packages (to
+        # set disturl), after that the rest.
+
+        todownload_rpm = [rpm for rpm in todownload if rpm[3].endswith('.rpm')]
+        todownload_rest = [rpm for rpm in todownload if not rpm[3].endswith('.rpm')]
+
+        for _project, _repo, arch, fn, mt in todownload_rpm:
+            repodir = os.path.join(DOWNLOADS, request.src_package, _project, _repo)
+            if not os.path.exists(repodir):
+                os.makedirs(repodir)
+            t = os.path.join(repodir, fn)
+            self.get_binary_file(_project, _repo, arch,
+                                 request.src_package, fn, t, mt)
+
+            # Organize the files into DISTURL directories.
+            disturl = self._md5_disturl(self._disturl(t))
+            disturldir = os.path.join(repodir, disturl)
+            last_disturl, last_disturldir = disturl, disturldir
+            file_in_disturl = os.path.join(disturldir, fn)
+            if not os.path.exists(disturldir):
+                os.makedirs(disturldir)
+            try:
+                os.symlink(t, file_in_disturl)
+            except:
+                pass
+                # print 'Found previous link.'
+
+            request.downloads[(_project, _repo, disturl)].append(file_in_disturl)
+
+        for _project, _repo, arch, fn, mt in todownload_rest:
+            repodir = os.path.join(DOWNLOADS, request.src_package, _project, _repo)
+            if not os.path.exists(repodir):
+                os.makedirs(repodir)
+            t = os.path.join(repodir, fn)
+            self.get_binary_file(_project, _repo, arch,
+                                 request.src_package, fn, t, mt)
+
+            file_in_disturl = os.path.join(last_disturldir, fn)
+            if last_disturldir:
+                try:
+                    os.symlink(t, file_in_disturl)
+                except:
+                    pass
+                    # print 'Found previous link.'
+            else:
+                print "I don't know where to put", fn
+
+            request.downloads[(_project, _repo, last_disturl)].append(file_in_disturl)
+
+    def _toignore(self, request):
+        """Return the list of files to ignore during the checkrepo."""
+        toignore = set()
+        for fn in self.get_package_list_from_repository(
+                request.tgt_project, 'standard', 'x86_64', request.tgt_package):
+            if fn[1]:
+                toignore.add(fn[1])
+
+        # now fetch -32bit pack list
+        for fn in self.get_package_list_from_repository(
+                request.tgt_project, 'standard', 'i586', request.tgt_package):
+            if fn[1] and fn[2] == 'x86_64':
+                toignore.add(fn[1])
+        return toignore
+
     def _disturl(self, filename):
         """Get the DISTURL from a RPM file."""
         pid = subprocess.Popen(
@@ -672,3 +781,33 @@ class CheckRepo(object):
             return False
 
         return True
+
+    def get_package_list_from_repository(self, project, repository, arch, package):
+        url = makeurl(self.apiurl, ('build', project, repository, arch, package))
+        files = []
+        try:
+            binaries = ET.parse(http_GET(url)).getroot()
+            for binary in binaries.findall('binary'):
+                filename = binary.attrib['filename']
+                mtime = int(binary.attrib['mtime'])
+
+                result = re.match(r'(.*)-([^-]*)-([^-]*)\.([^-\.]+)\.rpm', filename)
+                if not result:
+                    if filename == 'rpmlint.log':
+                        files.append((filename, '', '', mtime))
+                    continue
+
+                pname = result.group(1)
+                if pname.endswith('-debuginfo') or pname.endswith('-debuginfo-32bit'):
+                    continue
+                if pname.endswith('-debugsource'):
+                    continue
+                if result.group(4) == 'src':
+                    continue
+
+                files.append((filename, pname, result.group(4), mtime))
+        except urllib2.HTTPError:
+            pass
+            # print " - WARNING: Can't found list of packages (RPM) for %s in %s (%s, %s)" % (
+            #     package, project, repository, arch)
+        return files