Merge pull request #1189 from jberry-suse/staging-config

osclib: provide ConfigCommand and expose via osc-staging (utilize in repo_checker)

openqa-comments.py

@@ -231,6 +231,5 @@ if __name__ == '__main__':
     if args.staging:
         openQA.report(api.prj_from_letter(args.staging))
     else:
-        for staging in api.get_staging_projects():
-            if not staging.endswith(':DVD'):
-                openQA.report(staging)
+        for staging in api.get_staging_projects(include_dvd=False):
+            openQA.report(staging)

osc-staging.py

@@ -38,6 +38,7 @@ from osclib.check_command import CheckCommand
 from osclib.check_duplicate_binaries_command import CheckDuplicateBinariesCommand
 from osclib.cleanup_rings import CleanupRings
 from osclib.conf import Config
+from osclib.config_command import ConfigCommand
 from osclib.freeze_command import FreezeCommand
 from osclib.ignore_command import IgnoreCommand
 from osclib.unignore_command import UnignoreCommand
@@ -136,6 +137,8 @@ def clean_args(args):
 @cmdln.option('--strategy', help='apply a specific strategy')
 @cmdln.option('--no-color', action='store_true', help='strip colors from output (or add staging.color = 0 to the .oscrc general section')
 @cmdln.option('--save', action='store_true', help='save the result to the dashboard container')
+@cmdln.option('--append', action='store_true', help='append to existing value')
+@cmdln.option('--clear', action='store_true', help='clear value')
 def do_staging(self, subcmd, opts, *args):
     """${cmd_name}: Commands to work with staging projects
 
@@ -161,6 +164,34 @@ def do_staging(self, subcmd, opts, *args):
 
     "check_duplicate_binaries" list binaries provided by multiple packages
 
+    "config" will modify or view staging specific configuration
+
+        Target project level configuration that applies to all stagings can be
+        found in the $PROJECT:Staging/dashboard container in file "config". Both
+        configuration locations follow the .oscrc format (space separated list).
+
+        config
+            Print all staging configuration.
+        config key
+            Print the value of key for stagings.
+        conf key value...
+            Set the value of key for stagings.
+        config --clear
+            Clear all staging configuration.
+        config --clear key
+            Clear (unset) a single key from staging configuration
+        config --append key value...
+            Append value to existing value or set if no existing value.
+
+        All of the above may be restricted to a set of stagings.
+
+        The staging configuration is automatically cleared anytime staging
+        psuedometa is cleared (accept, or unstage all requests).
+
+        The keys that may be set in staging configuration are:
+
+        - repo_checker-binary-whitelist[-arch]: appended to target project list
+
     "cleanup_rings" will try to cleanup rings content and print
         out problems
 
@@ -288,6 +319,7 @@ def do_staging(self, subcmd, opts, *args):
         osc staging adi [--move] [--by-develproject] [--split] [REQUEST...]
         osc staging check [--old] [STAGING...]
         osc staging check_duplicate_binaries
+        osc staging config [--append] [--clear] [STAGING...] [key] [value]
         osc staging cleanup_rings
         osc staging freeze [--no-bootstrap] STAGING...
         osc staging frozenage [STAGING...]
@@ -320,6 +352,7 @@ def do_staging(self, subcmd, opts, *args):
         'accept',
         'adi',
         'check',
+        'config',
         'frozenage',
         'unignore',
         'select',
@@ -393,6 +426,25 @@ def do_staging(self, subcmd, opts, *args):
                     print()
         elif cmd == 'check_duplicate_binaries':
             CheckDuplicateBinariesCommand(api).perform(opts.save)
+        elif cmd == 'config':
+            projects = set()
+            key = value = None
+            stagings = api.get_staging_projects_short(None) + \
+                       api.get_staging_projects(include_dvd=False)
+            for arg in args[1:]:
+                if arg in stagings:
+                    projects.add(api.prj_from_short(arg))
+                elif key is None:
+                    key = arg
+                elif value is None:
+                    value = arg
+                else:
+                    value += ' ' + arg
+
+            if not len(projects):
+                projects = api.get_staging_projects(include_dvd=False)
+
+            ConfigCommand(api).perform(projects, key, value, opts.append, opts.clear)
         elif cmd == 'freeze':
             for prj in args[1:]:
                 prj = api.prj_from_short(prj)
@@ -454,7 +506,7 @@ def do_staging(self, subcmd, opts, *args):
         elif cmd == 'select':
             # Include list of all stagings in short-hand and by full name.
             existing_stagings = api.get_staging_projects_short(None)
-            existing_stagings += [p for p in api.get_staging_projects() if not p.endswith(':DVD')]
+            existing_stagings += api.get_staging_projects(include_dvd=False)
             stagings = []
             requests = []
             for arg in args[1:]:

osclib/config_command.py

@@ -0,0 +1,28 @@
+class ConfigCommand(object):
+    def __init__(self, api):
+        self.api = api
+
+    def perform(self, projects, key=None, value=None, append=False, clear=False):
+        project_max_format = '{:<' + str(len(max(projects, key=len))) + '} {}'
+        for project in projects:
+            meta = self.api.get_prj_pseudometa(project)
+            meta.setdefault('config', {})
+
+            if clear:
+                if key:
+                    meta['config'].pop(key, None)
+                else:
+                    meta.pop('config', None)
+                self.api.set_prj_pseudometa(project, meta)
+            elif value:
+                value_project = value
+                if append:
+                    value_project = ' '.join([meta['config'].get(key, ''), value_project.strip()])
+                meta['config'][key] = value_project.strip()
+                self.api.set_prj_pseudometa(project, meta)
+
+            keys = [key] if key else meta.get('config', {}).keys()
+            for key_print in keys:
+                print('{} = {}'.format(
+                    project_max_format.format(project, key_print) if len(projects) > 1 else key_print,
+                    meta['config'].get(key_print)))

osclib/freeze_command.py

@@ -14,6 +14,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
+from osclib.config_command import ConfigCommand
 import time
 import re
 from xml.etree import cElementTree as ET
@@ -119,6 +120,9 @@ class FreezeCommand(object):
     def perform(self, prj, copy_bootstrap=True):
         self.prj = prj
 
+        # Depending on what eventually lives in config this may need to change.
+        ConfigCommand(self.api).perform([prj], clear=True)
+
         if self.api.is_adi_project(prj):
             src_prj = self.api.find_devel_project_from_adi_frozenlinks(self.prj)
             if src_prj is None:

osclib/stagingapi.py

@@ -325,7 +325,7 @@ class StagingAPI(object):
 
         return True
 
-    def get_staging_projects(self):
+    def get_staging_projects(self, include_dvd=True):
         """
         Get all current running staging projects
         :return list of known staging projects
@@ -338,7 +338,10 @@ class StagingAPI(object):
         projxml = http_GET(url)
         root = ET.parse(projxml).getroot()
         for val in root.findall('project'):
-            projects.append(val.get('name'))
+            project = val.get('name')
+            if not include_dvd and project.endswith(':DVD'):
+                continue
+            projects.append(project)
         return projects
 
     def extract_staging_short(self, p):
@@ -361,9 +364,8 @@ class StagingAPI(object):
                     and None for both.
         """
         projects = []
-        for project in self.get_staging_projects():
-            if project.endswith(':DVD') or \
-               (adi is not None and self.is_adi_project(project) != adi):
+        for project in self.get_staging_projects(include_dvd=False):
+            if adi is not None and self.is_adi_project(project) != adi:
                 continue
             short = self.extract_staging_short(project)
             if adi is False and len(short) > 1:
@@ -736,7 +738,10 @@ class StagingAPI(object):
         description = root.find('description')
         # Order the requests and replace it with yaml
         meta['requests'] = sorted(meta.get('requests', []), key=lambda x: x['id'])
-        description.text = yaml.dump(meta)
+        yaml_new = yaml.dump(meta)
+        if yaml_new == description.text:
+            return
+        description.text = yaml_new
         # Find title
         title = root.find('title')
         # Put something nice into title as well

repo_checker.py

@@ -170,7 +170,7 @@ class RepoChecker(ReviewBot.ReviewBot):
             # Only bother if staging can match arch, but layered first.
             directories.insert(0, self.mirror(project, arch))
 
-            whitelist = self.binary_whitelist(project, arch)
+            whitelist = self.binary_whitelist(project, arch, group)
 
             # Perform checks on group.
             results = {
@@ -240,11 +240,14 @@ class RepoChecker(ReviewBot.ReviewBot):
             if binary.package in packages:
                 yield binary.name
 
-    def binary_whitelist(self, project, arch):
+    def binary_whitelist(self, project, arch, group):
+        additions = self.staging_api(project).get_prj_pseudometa(group).get('config', {})
         prefix = 'repo_checker-binary-whitelist'
         whitelist = set()
         for key in [prefix, '-'.join([prefix, arch])]:
             whitelist.update(self.staging_config[project].get(key, '').split(' '))
+            whitelist.update(additions.get(key, '').split(' '))
+        whitelist = filter(None, whitelist)
         return whitelist
 
     def install_check(self, project, directories, arch, ignore=[], whitelist=[], parse=False):