pkglistgen: Optionally include suggested packages too

pkglistgen.py

@@ -70,7 +70,11 @@ class Group(object):
         self.develpkgs = []
         self.silents = set()
         self.ignored = set()
-        self.recommended = set()
+        # special feature for SLE. Patterns are marked for expansion
+        # of recommended packages, all others aren't. Only works
+        # with recommends on actual package names, not virtual
+        # provides.
+        self.expand_recommended = set()
 
         pkglist.groups[self.safe_name] = self
 
@@ -100,7 +104,7 @@ class Group(object):
                 elif rel == 'silent':
                     self.silents.add(name)
                 elif rel == 'recommended':
-                    self.recommended.add(name)
+                    self.expand_recommended.add(name)
                 else:
                     arch = rel
 
@@ -116,7 +120,7 @@ class Group(object):
 
         self.locked.update(group.locked)
         self.silents.update(group.silents)
-        self.recommended.update(group.recommended)
+        self.expand_recommended.update(group.expand_recommended)
 
     # do not repeat packages
     def ignore(self, without):
@@ -135,7 +139,7 @@ class Group(object):
             self.ignore(g)
         self.ignored.add(without)
 
-    def solve(self, ignore_recommended=False):
+    def solve(self, ignore_recommended=False, include_suggested = False):
         """ base: list of base groups or None """
 
         if self.solved:
@@ -147,27 +151,30 @@ class Group(object):
 
         self.srcpkgs = set()
         self.recommends = dict()
+        self.suggested = dict()
         for arch in self.architectures:
             pool = self.pkglist._prepare_pool(arch)
             # pool.set_debuglevel(10)
+            suggested = []
 
-            tosolv = self.packages[arch]
+            # packages resulting from explicit recommended expansion
-            while tosolv:
+            extra = []
-                n, group = tosolv.pop(0)
+
+            def solve_one_package(n, group):
                 jobs = list(self.pkglist.lockjobs[arch])
                 sel = pool.select(str(n), solv.Selection.SELECTION_NAME)
                 if sel.isempty():
                     logger.debug('{}.{}: package {} not found'.format(self.name, arch, n))
                     self.not_found.setdefault(n, set()).add(arch)
-                    continue
+                    return
                 else:
-                    if n in self.recommended:
+                    if n in self.expand_recommended:
                         for s in sel.solvables():
                             for dep in s.lookup_deparray(solv.SOLVABLE_RECOMMENDS):
                                 # only add recommends that exist as packages
                                 rec = pool.select(dep.str(), solv.Selection.SELECTION_NAME)
                                 if not rec.isempty():
-                                    tosolv.append([dep.str(), group + ":recommended:" + n])
+                                    extra.append([dep.str(), group + ":recommended:" + n])
 
                     jobs += sel.jobs(solv.Job.SOLVER_INSTALL)
 
@@ -200,23 +207,24 @@ class Group(object):
                         else:
                             logger.debug(msg)
                         self.unresolvable[arch][n] = str(problem)
-                    continue
+                    return
 
-                trans = solver.transaction()
+                if hasattr(solver, 'get_recommended'):
-                if trans.isempty():
-                    logger.error('%s.%s: nothing to do', self.name, arch)
-                    continue
-
-                if 'get_recommended' in dir(solver):
                     for s in solver.get_recommended():
                         if s.name in locked:
                             continue
                         self.recommends.setdefault(s.name, group + ':' + n)
                     for s in solver.get_suggested():
-                        self.recommends.setdefault(s.name, group + ':' + n)
+                        suggested.append([s.name, group + ':suggested:' + n])
+                        self.suggested.setdefault(s.name, group + ':' + n)
                 else:
                     logger.warn('newer libsolv needed for recommends!')
 
+                trans = solver.transaction()
+                if trans.isempty():
+                    logger.error('%s.%s: nothing to do', self.name, arch)
+                    return
+
                 for s in trans.newsolvables():
                     solved[arch].setdefault(s.name, group + ':' + n)
                     reason, rule = solver.describe_decision(s)
@@ -229,6 +237,18 @@ class Group(object):
                         src = s.lookup_str(solv.SOLVABLE_SOURCENAME)
                     self.srcpkgs.add(src)
 
+            for n, group in self.packages[arch]:
+                solve_one_package(n, group)
+
+            if include_suggested:
+                seen = set()
+                while suggested:
+                    n, group = suggested.pop()
+                    if n in seen:
+                        continue
+                    seen.add(n)
+                    solve_one_package(n, group)
+
         common = None
         # compute common packages across all architectures
         for arch in self.architectures:
@@ -292,17 +312,19 @@ class Group(object):
             if not already_present:
                 self.develpkgs.append(p)
 
-    def filter_duplicated_recommends(self, modules):
+    def _filter_already_selected(self, modules, pkgdict):
-        recommends = self.recommends
         # erase our own - so we don't filter our own
-        self.recommends = dict()
+        for p in pkgdict.keys():
-        for p in recommends:
             already_present = False
             for m in modules:
                 for arch in ['*'] + self.architectures:
                     already_present = already_present or (p in m.solved_packages[arch])
-            if not already_present:
+            if already_present:
-                self.recommends[p] = recommends[p]
+                del pkgdict[p]
+
+    def filter_already_selected(self, modules):
+        self._filter_already_selected(modules, self.recommends)
+        self._filter_already_selected(modules, self.suggested)
 
     def toxml(self, arch, ignore_broken = False):
         packages = self.solved_packages[arch]
@@ -356,11 +378,18 @@ class Group(object):
             c = ET.Comment("\nDevelopment packages:\n  - " + "\n  - ".join(sorted(self.develpkgs)) + "\n")
             root.append(c)
         if arch == '*' and self.recommends:
-            comment = "\nRecommended and suggested packages:\n"
+            comment = "\nRecommended packages:\n"
             for p in sorted(self.recommends.keys()):
                 comment += "  - {} # {}\n".format(p, self.recommends[p])
             c = ET.Comment(comment)
             root.append(c)
+        if arch == '*' and self.suggested:
+            comment = "\nSuggested packages:\n"
+            for p in sorted(self.suggested.keys()):
+                comment += "  - {} # {}\n".format(p, self.suggested[p])
+            c = ET.Comment(comment)
+            root.append(c)
+
 
         return root
 
@@ -389,6 +418,7 @@ class PkgListGen(ToolBase.ToolBase):
         self.lockjobs = dict()
         self.ignore_broken = False
         self.ignore_recommended = False
+        self.include_suggested = False
         self.unwanted = set()
         self.output = None
 
@@ -483,7 +513,7 @@ class PkgListGen(ToolBase.ToolBase):
         g = self.groups[groupname]
         for i in includes:
             g.inherit(self.groups[i])
-        g.solve(self.ignore_recommended)
+        g.solve(self.ignore_recommended, self.include_suggested)
         for e in excludes:
             g.ignore(self.groups[e])
 
@@ -680,6 +710,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
 
     @cmdln.option('--ignore-unresolvable', action='store_true', help='ignore unresolvable and missing packges')
     @cmdln.option('--ignore-recommended', action='store_true', help='do not include recommended packages automatically')
+    @cmdln.option('--include-suggested', action='store_true', help='include suggested packges also')
     def do_solve(self, subcmd, opts):
         """${cmd_name}: Solve groups
 
@@ -696,6 +727,10 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
             self.tool.ignore_broken = True
         if opts.ignore_recommended:
             self.tool.ignore_recommended = True
+        if opts.include_suggested:
+            if opts.ignore_recommended:
+                raise cmdln.CmdlnUserError("--ignore-recommended and --include-suggested don't work together")
+            self.tool.include_suggested = True
 
         modules = []
         # the yml parser makes an array out of everything, so
@@ -713,7 +748,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
         for module in modules:
             module.check_dups(modules)
             module.collect_devel_packages(modules)
-            module.filter_duplicated_recommends(modules)
+            module.filter_already_selected(modules)
 
         self.tool._collect_unsorted_packages(modules)
         self.tool._write_all_groups()

pkglistgen.sh

@@ -17,6 +17,14 @@ releases="000release-packages"
 
 cachedir=${XDG_CACHE_HOME:-~/.cache}/opensuse-packagelists/$api/$project
 todo=("$product" "$groups")
+solveargs=()
+
+if [ -n "$IGNORE_RECOMMENDED" ]; then
+	solveargs+=('--ignore-recommended')
+fi
+if [ -n "$INCLUDE_SUGGESTED" ]; then
+	solveargs+=('--include-suggested')
+fi
 
 _osc=`type -p osc`
 osc()
@@ -83,8 +91,10 @@ rm -f supportstatus.txt groups.yml package-groups.changes
 for i in *.spec.in; do
   mv -v $i "${i%.in}"
 done
-${self%.sh}.py -i "$cachedir/$groups" -r $repos -o . -a x86_64 update
+if ! ${self%.sh}.py -i "$cachedir/$groups" -r $repos -o . -a x86_64 update; then
-${self%.sh}.py -i "$cachedir/$groups" -r $repos -o . -a x86_64 solve
+	echo "no change in packages"
+fi
+${self%.sh}.py -i "$cachedir/$groups" -r $repos -o . -a x86_64 solve "${solveargs[@]}"
 for i in $delete_products; do
 	rm -vf -- "$i"
 done