35eeb2af2b
15.3 introduced something unexpected: aarch64, ppc64le and s390x are part of the repos - and all architectures that aren't native to openSUSE:Factory are considered to bear dropped packages, e.g. *all of them* On top of that the setdefault using the same hash caused packages to come and go for repos, creating unstable output
764 lines
32 KiB
Python
764 lines
32 KiB
Python
import ToolBase
|
|
import glob
|
|
import logging
|
|
import os
|
|
import re
|
|
import solv
|
|
import shutil
|
|
import subprocess
|
|
import yaml
|
|
import sys
|
|
import tempfile
|
|
|
|
from lxml import etree as ET
|
|
|
|
from osc.core import checkout_package
|
|
|
|
from osc.core import http_GET
|
|
from osc.core import HTTPError
|
|
from osc.core import show_results_meta
|
|
from osc.core import Package
|
|
from osc.core import undelete_package
|
|
from osclib.core import attribute_value_load
|
|
from osclib.core import target_archs
|
|
from osclib.conf import str2bool
|
|
from osclib.core import repository_path_expand
|
|
from osclib.core import repository_arch_state
|
|
from osclib.cache_manager import CacheManager
|
|
|
|
from urllib.parse import urlparse
|
|
|
|
from pkglistgen import file_utils
|
|
from pkglistgen.group import Group
|
|
|
|
SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))
|
|
|
|
PRODUCT_SERVICE = '/usr/lib/obs/service/create_single_product'
|
|
|
|
# share header cache with repochecker
|
|
CACHEDIR = CacheManager.directory('repository-meta')
|
|
|
|
class PkgListGen(ToolBase.ToolBase):
|
|
|
|
def __init__(self):
|
|
ToolBase.ToolBase.__init__(self)
|
|
self.logger = logging.getLogger(__name__)
|
|
self.reset()
|
|
|
|
def reset(self):
|
|
# package -> supportatus
|
|
self.packages = dict()
|
|
self.groups = dict()
|
|
self._supportstatus = None
|
|
self.input_dir = '.'
|
|
self.output_dir = '.'
|
|
self.lockjobs = dict()
|
|
self.ignore_broken = False
|
|
self.unwanted = set()
|
|
self.output = None
|
|
self.locales = set()
|
|
self.did_update = False
|
|
self.filtered_architectures = None
|
|
self.dry_run = False
|
|
self.all_architectures = None
|
|
|
|
def filter_architectures(self, architectures):
|
|
self.filtered_architectures = sorted(list(set(architectures) & set(self.all_architectures)))
|
|
|
|
def _load_supportstatus(self):
|
|
# XXX
|
|
fn = os.path.join(self.input_dir, 'supportstatus.txt')
|
|
self._supportstatus = dict()
|
|
if os.path.exists(fn):
|
|
with open(fn, 'r') as fh:
|
|
for l in fh:
|
|
# pkg, status
|
|
a = l.rstrip().split(' ')
|
|
if len(a) > 1:
|
|
self._supportstatus[a[0]] = a[1]
|
|
|
|
def supportstatus(self, package):
|
|
if self._supportstatus is None:
|
|
self._load_supportstatus()
|
|
|
|
return self._supportstatus.get(package)
|
|
|
|
def _load_group_file(self, fn):
|
|
output = None
|
|
unwanted = None
|
|
with open(fn, 'r') as fh:
|
|
self.logger.debug('reading %s', fn)
|
|
for groupname, group in yaml.safe_load(fh).items():
|
|
if groupname == 'OUTPUT':
|
|
output = group
|
|
continue
|
|
if groupname == 'UNWANTED':
|
|
unwanted = set(group)
|
|
continue
|
|
g = Group(groupname, self)
|
|
g.parse_yml(group)
|
|
return output, unwanted
|
|
|
|
def load_all_groups(self):
|
|
for fn in glob.glob(os.path.join(self.input_dir, 'group*.yml')):
|
|
o, u = self._load_group_file(fn)
|
|
if o:
|
|
if self.output is not None:
|
|
raise Exception('OUTPUT defined multiple times')
|
|
self.output = o
|
|
if u:
|
|
self.unwanted |= u
|
|
|
|
# required to generate release spec files (only)
|
|
def write_group_stubs(self):
|
|
archs = ['*'] + self.all_architectures
|
|
for name in self.groups:
|
|
group = self.groups[name]
|
|
group.solved_packages = dict()
|
|
fn = '{}.group'.format(group.name)
|
|
with open(os.path.join(self.output_dir, fn), 'w') as fh:
|
|
for arch in archs:
|
|
x = group.toxml(arch, group.ignore_broken, None)
|
|
x = ET.tostring(x, pretty_print=True, encoding='unicode')
|
|
fh.write(x)
|
|
|
|
def write_all_groups(self):
|
|
self._check_supplements()
|
|
summary = dict()
|
|
archs = ['*'] + self.all_architectures
|
|
for name in self.groups:
|
|
group = self.groups[name]
|
|
if not group.solved:
|
|
continue
|
|
summary[name] = group.summary()
|
|
fn = '{}.group'.format(group.name)
|
|
with open(os.path.join(self.output_dir, fn), 'w') as fh:
|
|
comment = group.comment
|
|
for arch in archs:
|
|
x = group.toxml(arch, group.ignore_broken, comment)
|
|
# only comment first time
|
|
comment = None
|
|
x = ET.tostring(x, pretty_print=True, encoding='unicode')
|
|
x = re.sub(r'\s*<!-- reason:', ' <!-- reason:', x)
|
|
fh.write(x)
|
|
return summary
|
|
|
|
def solve_module(self, groupname, includes, excludes, use_recommends):
|
|
g = self.groups[groupname]
|
|
importants = set()
|
|
for i in includes:
|
|
name = i
|
|
if isinstance(i, dict):
|
|
name = list(i)[0]
|
|
if i[name] != 'support':
|
|
importants.add(name)
|
|
else:
|
|
importants.add(name)
|
|
g.inherit(self.groups[name])
|
|
g.solve(use_recommends)
|
|
for e in excludes:
|
|
g.ignore(self.groups[e])
|
|
for i in importants:
|
|
group = self.groups[i]
|
|
for arch in group.packages:
|
|
if not arch in g.solved_packages:
|
|
continue
|
|
for package in group.packages[arch]:
|
|
if package[0] in g.solved_packages[arch]:
|
|
continue
|
|
if not package[0] in g.solved_packages['*']:
|
|
self.logger.error(f'Missing {package[0]} in {groupname} for {arch}')
|
|
|
|
def expand_repos(self, project, repo='standard'):
|
|
return repository_path_expand(self.apiurl, project, repo)
|
|
|
|
def _check_supplements(self):
|
|
tocheck = set()
|
|
tocheck_locales = set()
|
|
for arch in self.filtered_architectures:
|
|
pool = self.prepare_pool(arch, True)
|
|
sel = pool.Selection()
|
|
for s in pool.solvables_iter():
|
|
sel.add_raw(solv.Job.SOLVER_SOLVABLE, s.id)
|
|
|
|
for s in sel.solvables():
|
|
for dep in s.lookup_deparray(solv.SOLVABLE_SUPPLEMENTS):
|
|
for d in dep.str().split(' '):
|
|
if d.startswith('namespace:modalias') or d.startswith('namespace:filesystem'):
|
|
tocheck.add(s.name)
|
|
|
|
for l in self.locales:
|
|
i = pool.str2id('locale({})'.format(l))
|
|
for s in pool.whatprovides(i):
|
|
tocheck_locales.add(s.name)
|
|
|
|
all_grouped = set()
|
|
for g in self.groups.values():
|
|
if g.solved:
|
|
for arch in g.solved_packages.keys():
|
|
if g.solved_packages[arch]:
|
|
all_grouped.update(g.solved_packages[arch])
|
|
|
|
for p in tocheck - all_grouped:
|
|
self.logger.warning('package %s has supplements but is not grouped', p)
|
|
|
|
for p in tocheck_locales - all_grouped:
|
|
self.logger.warning('package %s provides supported locale but is not grouped', p)
|
|
|
|
def prepare_pool(self, arch, ignore_conflicts):
|
|
pool = solv.Pool()
|
|
# the i586 DVD is really a i686 one
|
|
if arch == 'i586':
|
|
pool.setarch('i686')
|
|
else:
|
|
pool.setarch(arch)
|
|
|
|
self.lockjobs[arch] = []
|
|
solvables = set()
|
|
|
|
for project, reponame in self.repos:
|
|
repo = pool.add_repo(project)
|
|
s = os.path.join(CACHEDIR, 'repo-{}-{}-{}.solv'.format(project, reponame, arch))
|
|
r = repo.add_solv(s)
|
|
if not r:
|
|
if not self.did_update:
|
|
raise Exception(
|
|
'failed to add repo {}/{}/{}. Need to run update first?'.format(project, reponame, arch))
|
|
continue
|
|
for solvable in repo.solvables_iter():
|
|
if ignore_conflicts:
|
|
solvable.unset(solv.SOLVABLE_CONFLICTS)
|
|
solvable.unset(solv.SOLVABLE_OBSOLETES)
|
|
# only take the first solvable in the repo chain
|
|
if solvable.name in solvables:
|
|
self.lockjobs[arch].append(pool.Job(solv.Job.SOLVER_SOLVABLE | solv.Job.SOLVER_LOCK, solvable.id))
|
|
solvables.add(solvable.name)
|
|
|
|
pool.addfileprovides()
|
|
pool.createwhatprovides()
|
|
|
|
for l in self.locales:
|
|
pool.set_namespaceproviders(solv.NAMESPACE_LANGUAGE, pool.Dep(l), True)
|
|
|
|
return pool
|
|
|
|
# parse file and merge all groups
|
|
def _parse_unneeded(self, filename):
|
|
filename = os.path.join(self.input_dir, filename)
|
|
if not os.path.isfile(filename):
|
|
return set()
|
|
fh = open(filename, 'r')
|
|
self.logger.debug('reading %s', filename)
|
|
result = set()
|
|
for groupname, group in yaml.safe_load(fh).items():
|
|
result.update(group)
|
|
return result
|
|
|
|
# the unsorted group is special and will contain all the rest for
|
|
# the FTP tree. We filter it with unneeded though to create a
|
|
# unsorted.yml file for release manager review
|
|
def _collect_unsorted_packages(self, modules, unsorted):
|
|
uneeded_regexps = [re.compile(r)
|
|
for r in self._parse_unneeded('unneeded.yml')]
|
|
|
|
packages = dict()
|
|
if unsorted:
|
|
unsorted.solved_packages = dict()
|
|
unsorted.solved_packages['*'] = dict()
|
|
|
|
for arch in self.filtered_architectures:
|
|
pool = self.prepare_pool(arch, False)
|
|
pool.Selection()
|
|
archpacks = [s.name for s in pool.solvables_iter()]
|
|
|
|
# copy
|
|
filtered = list(archpacks)
|
|
for r in uneeded_regexps:
|
|
filtered = [p for p in filtered if not r.match(p)]
|
|
|
|
# convert to set
|
|
filtered = set(filtered) - self.unwanted
|
|
for g in modules:
|
|
if unsorted and g == unsorted:
|
|
continue
|
|
for a in ('*', arch):
|
|
filtered -= set(g.solved_packages[a])
|
|
for package in filtered:
|
|
packages.setdefault(package, []).append(arch)
|
|
|
|
if unsorted:
|
|
archpacks = set(archpacks)
|
|
unsorted.solved_packages[arch] = dict()
|
|
for g in modules:
|
|
archpacks -= set(g.solved_packages[arch])
|
|
archpacks -= set(g.solved_packages['*'])
|
|
unsorted.solved_packages[arch] = dict()
|
|
for p in archpacks:
|
|
unsorted.solved_packages[arch][p] = None
|
|
|
|
if unsorted:
|
|
common = None
|
|
for arch in self.filtered_architectures:
|
|
if common is None:
|
|
common = set(unsorted.solved_packages[arch])
|
|
continue
|
|
common &= set(unsorted.solved_packages[arch])
|
|
for p in common:
|
|
unsorted.solved_packages['*'][p] = None
|
|
for arch in self.filtered_architectures:
|
|
del unsorted.solved_packages[arch][p]
|
|
|
|
with open(os.path.join(self.output_dir, 'unsorted.yml'), 'w') as fh:
|
|
fh.write('unsorted:\n')
|
|
for p in sorted(packages):
|
|
fh.write(' - ')
|
|
fh.write(p)
|
|
if len(packages[p]) != len(self.filtered_architectures):
|
|
fh.write(': [')
|
|
fh.write(','.join(sorted(packages[p])))
|
|
fh.write(']')
|
|
reason = self._find_reason(p, modules)
|
|
if reason:
|
|
fh.write(' # ' + reason)
|
|
fh.write(' \n')
|
|
|
|
# give a hint if the package is related to a group
|
|
def _find_reason(self, package, modules):
|
|
# go through the modules multiple times to find the "best"
|
|
for g in modules:
|
|
if package in g.recommends:
|
|
return 'recommended by ' + g.recommends[package]
|
|
for g in modules:
|
|
if package in g.suggested:
|
|
return 'suggested by ' + g.suggested[package]
|
|
for g in modules:
|
|
if package in g.develpkgs:
|
|
return 'devel package of ' + g.develpkgs[package]
|
|
return None
|
|
|
|
def update_repos(self, architectures):
|
|
# only there to parse the repos
|
|
bs_mirrorfull = os.path.join(SCRIPT_PATH, '..', 'bs_mirrorfull')
|
|
global_update = False
|
|
|
|
for project, repo in self.repos:
|
|
for arch in architectures:
|
|
# TODO: refactor to common function with repo_checker.py
|
|
d = os.path.join(CACHEDIR, project, repo, arch)
|
|
if not os.path.exists(d):
|
|
os.makedirs(d)
|
|
|
|
# Fetch state before mirroring in-case it changes during download.
|
|
state = repository_arch_state(self.apiurl, project, repo, arch)
|
|
|
|
if state is None:
|
|
# Repo might not have this architecture
|
|
continue
|
|
|
|
# Would be preferable to include hash in name, but cumbersome to handle without
|
|
# reworking a fair bit since the state needs to be tracked.
|
|
solv_file = os.path.join(CACHEDIR, 'repo-{}-{}-{}.solv'.format(project, repo, arch))
|
|
solv_file_hash = '{}::{}'.format(solv_file, state)
|
|
if os.path.exists(solv_file) and os.path.exists(solv_file_hash):
|
|
# Solve file exists and hash unchanged, skip updating solv.
|
|
self.logger.debug('skipping solv generation for {} due to matching state {}'.format(
|
|
'/'.join([project, repo, arch]), state))
|
|
continue
|
|
|
|
# Either hash changed or new, so remove any old hash files.
|
|
file_utils.unlink_list(None, glob.glob(solv_file + '::*'))
|
|
global_update = True
|
|
|
|
self.logger.debug('updating %s', d)
|
|
args = [bs_mirrorfull]
|
|
args.append('--nodebug')
|
|
args.append('{}/public/build/{}/{}/{}'.format(self.apiurl, project, repo, arch))
|
|
args.append(d)
|
|
with subprocess.Popen(args, stdout=subprocess.PIPE) as p:
|
|
for line in p.stdout:
|
|
self.logger.info(line.decode('utf-8').rstrip())
|
|
if p.wait() != 0:
|
|
raise Exception("Mirroring repository failed")
|
|
|
|
files = [os.path.join(d, f)
|
|
for f in os.listdir(d) if f.endswith('.rpm')]
|
|
fh = open(solv_file, 'w')
|
|
p = subprocess.Popen(
|
|
['rpms2solv', '-m', '-', '-0'], stdin=subprocess.PIPE, stdout=fh)
|
|
p.communicate(bytes('\0'.join(files), 'utf-8'))
|
|
fh.close()
|
|
if p.wait() != 0:
|
|
raise Exception("rpm2solv failed")
|
|
|
|
# Create hash file now that solv creation is complete.
|
|
open(solv_file_hash, 'a').close()
|
|
self.did_update = True
|
|
|
|
return global_update
|
|
|
|
def create_weakremovers(self, target, target_config, directory, output):
|
|
drops = dict()
|
|
dropped_repos = dict()
|
|
|
|
root = yaml.safe_load(open(os.path.join(directory, 'config.yml')))
|
|
for item in root:
|
|
key = list(item)[0]
|
|
opts = item[key]
|
|
# cast 15.1 to string :)
|
|
key = str(key)
|
|
|
|
oldrepos = set(glob.glob(os.path.join(directory, '{}_*.packages.xz'.format(key))))
|
|
oldrepos |= set(glob.glob(os.path.join(directory, '{}.packages.xz'.format(key))))
|
|
for oldrepo in sorted(oldrepos):
|
|
pool = solv.Pool()
|
|
pool.setarch()
|
|
|
|
# we need some progress in the debug output - or gocd gets nervous
|
|
self.logger.debug('checking {}'.format(oldrepo))
|
|
oldsysrepo = pool.add_repo(oldrepo)
|
|
defvendorid = oldsysrepo.meta.lookup_id(solv.SUSETAGS_DEFAULTVENDOR)
|
|
f = tempfile.TemporaryFile()
|
|
# FIXME: port to lzma module with python3
|
|
st = subprocess.call(['xz', '-cd', oldrepo], stdout=f.fileno())
|
|
os.lseek(f.fileno(), 0, os.SEEK_SET)
|
|
oldsysrepo.add_susetags(solv.xfopen_fd(None, f.fileno()), defvendorid, None, solv.Repo.REPO_NO_INTERNALIZE | solv.Repo.SUSETAGS_RECORD_SHARES)
|
|
|
|
for arch in self.all_architectures:
|
|
for project, repo in self.repos:
|
|
fn = os.path.join(CACHEDIR, 'repo-{}-{}-{}.solv'.format(project, repo, arch))
|
|
r = pool.add_repo('/'.join([project, repo]))
|
|
r.add_solv(fn)
|
|
|
|
pool.createwhatprovides()
|
|
|
|
accepted_archs = set(self.all_architectures)
|
|
accepted_archs.add('noarch')
|
|
|
|
for s in oldsysrepo.solvables_iter():
|
|
if not s.arch in accepted_archs:
|
|
continue
|
|
|
|
oldarch = s.arch
|
|
if oldarch == 'i686':
|
|
oldarch = 'i586'
|
|
|
|
haveit = False
|
|
for s2 in pool.whatprovides(s.nameid):
|
|
if s2.repo == oldsysrepo or s.nameid != s2.nameid:
|
|
continue
|
|
newarch = s2.arch
|
|
if newarch == 'i686':
|
|
newarch = 'i586'
|
|
if oldarch != newarch and newarch != 'noarch' and oldarch != 'noarch':
|
|
continue
|
|
haveit = True
|
|
break
|
|
if haveit:
|
|
continue
|
|
|
|
# check for already obsoleted packages
|
|
nevr = pool.rel2id(s.nameid, s.evrid, solv.REL_EQ)
|
|
for s2 in pool.whatmatchesdep(solv.SOLVABLE_OBSOLETES, nevr):
|
|
if s2.repo == oldsysrepo:
|
|
continue
|
|
haveit = True
|
|
break
|
|
if haveit:
|
|
continue
|
|
if s.name not in drops:
|
|
drops[s.name] = {'repo': key, 'archs': set()}
|
|
if oldarch == 'noarch':
|
|
drops[s.name]['archs'] |= set(self.all_architectures)
|
|
else:
|
|
drops[s.name]['archs'].add(oldarch)
|
|
dropped_repos[key] = 1
|
|
|
|
del pool
|
|
|
|
for repo in sorted(dropped_repos):
|
|
repo_output = False
|
|
exclusives = dict()
|
|
for name in sorted(drops):
|
|
if drops[name]['repo'] != repo:
|
|
continue
|
|
if drops[name]['archs'] == set(self.all_architectures):
|
|
if not repo_output:
|
|
print('#', repo, file=output)
|
|
repo_output = True
|
|
print('Provides: weakremover({})'.format(name), file=output)
|
|
else:
|
|
jarch = ' '.join(sorted(drops[name]['archs']))
|
|
exclusives.setdefault(jarch, []).append(name)
|
|
|
|
for arch in sorted(exclusives):
|
|
if not repo_output:
|
|
print('#', repo, file=output)
|
|
repo_output = True
|
|
print('%ifarch {}'.format(arch), file=output)
|
|
for name in sorted(exclusives[arch]):
|
|
print('Provides: weakremover({})'.format(name), file=output)
|
|
print('%endif', file=output)
|
|
output.flush()
|
|
|
|
def solve_project(self, ignore_unresolvable=False, ignore_recommended=False, locale=None, locales_from=None):
|
|
self.load_all_groups()
|
|
if not self.output:
|
|
self.logger.error('OUTPUT not defined')
|
|
return
|
|
|
|
if ignore_unresolvable:
|
|
self.ignore_broken = True
|
|
global_use_recommends = not ignore_recommended
|
|
if locale:
|
|
self.locales |= set(locale.split(' '))
|
|
if locales_from:
|
|
with open(os.path.join(self.input_dir, locales_from), 'r') as fh:
|
|
root = ET.parse(fh).getroot()
|
|
self.locales |= set([lang.text for lang in root.findall('.//linguas/language')])
|
|
|
|
modules = []
|
|
# the yml parser makes an array out of everything, so
|
|
# we loop a bit more than what we support
|
|
for group in self.output:
|
|
groupname = list(group)[0]
|
|
settings = group[groupname]
|
|
if not settings: # e.g. unsorted
|
|
settings = {}
|
|
includes = settings.get('includes', [])
|
|
excludes = settings.get('excludes', [])
|
|
use_recommends = settings.get('recommends', global_use_recommends)
|
|
self.solve_module(groupname, includes, excludes, use_recommends)
|
|
g = self.groups[groupname]
|
|
# the default is a little double negated but Factory has ignore_broken
|
|
# as default and we only disable it for single groups (for now)
|
|
g.ignore_broken = not settings.get('require_all', not self.ignore_broken)
|
|
g.conflicts = settings.get('conflicts', [])
|
|
g.default_support_status = settings.get('default-support', 'unsupported')
|
|
modules.append(g)
|
|
|
|
# not defined for openSUSE
|
|
overlap = self.groups.get('overlap')
|
|
for module in modules:
|
|
module.check_dups(modules, overlap)
|
|
module.collect_devel_packages()
|
|
module.filter_already_selected(modules)
|
|
|
|
if overlap:
|
|
ignores = [x.name for x in overlap.ignored]
|
|
self.solve_module(overlap.name, [], ignores, use_recommends=False)
|
|
overlapped = set(overlap.solved_packages['*'])
|
|
for arch in self.filtered_architectures:
|
|
overlapped |= set(overlap.solved_packages[arch])
|
|
for module in modules:
|
|
if module.name == 'overlap' or module in overlap.ignored:
|
|
continue
|
|
for arch in ['*'] + self.filtered_architectures:
|
|
for p in overlapped:
|
|
module.solved_packages[arch].pop(p, None)
|
|
|
|
self._collect_unsorted_packages(modules, self.groups.get('unsorted'))
|
|
return self.write_all_groups()
|
|
|
|
def strip_medium_from_staging(self, path):
|
|
# staging projects don't need source and debug medium - and the glibc source
|
|
# rpm conflicts between standard and bootstrap_copy repository causing the
|
|
# product builder to fail
|
|
medium = re.compile('name="(DEBUG|SOURCE)MEDIUM"')
|
|
for name in glob.glob(os.path.join(path, '*.kiwi')):
|
|
lines = open(name).readlines()
|
|
lines = [l for l in lines if not medium.search(l)]
|
|
open(name, 'w').writelines(lines)
|
|
|
|
def build_stub(self, destination, extension):
|
|
with open(os.path.join(destination, '.'.join(['stub', extension])), 'w+') as f:
|
|
f.write('# prevent building single {} files twice\n'.format(extension))
|
|
f.write('Name: stub\n')
|
|
f.write('Version: 0.0\n')
|
|
|
|
def commit_package(self, path):
|
|
if self.dry_run:
|
|
package = Package(path)
|
|
for i in package.get_diff():
|
|
print(''.join(i))
|
|
else:
|
|
# No proper API function to perform the same operation.
|
|
print(subprocess.check_output(
|
|
' '.join(['cd', path, '&&', 'osc', 'addremove']), shell=True, encoding='utf-8'))
|
|
package = Package(path)
|
|
package.commit(msg='Automatic update', skip_local_service_run=True)
|
|
|
|
def replace_product_version(self, product_file, product_version):
|
|
product_version = '<version>{}</version>'.format(product_version)
|
|
lines = open(product_file).readlines()
|
|
new_lines = []
|
|
for line in lines:
|
|
new_lines.append(line.replace('<version></version>', product_version))
|
|
open(product_file, 'w').write(''.join(new_lines))
|
|
|
|
def update_and_solve_target(self, api, target_project, target_config, main_repo,
|
|
project, scope, force, no_checkout,
|
|
only_release_packages, stop_after_solve):
|
|
self.all_architectures = target_config.get('pkglistgen-archs').split(' ')
|
|
self.repos = self.expand_repos(project, main_repo)
|
|
print('[{}] {}/{}: update and solve'.format(scope, project, main_repo))
|
|
|
|
group = target_config.get('pkglistgen-group', '000package-groups')
|
|
product = target_config.get('pkglistgen-product', '000product')
|
|
release = target_config.get('pkglistgen-release', '000release-packages')
|
|
oldrepos = target_config.get('pkglistgen-repos', '000update-repos')
|
|
|
|
url = api.makeurl(['source', project])
|
|
packages = ET.parse(http_GET(url)).getroot()
|
|
if packages.find('entry[@name="{}"]'.format(product)) is None:
|
|
if not self.dry_run:
|
|
undelete_package(api.apiurl, project, product, 'revive')
|
|
# TODO disable build.
|
|
print('{} undeleted, skip dvd until next cycle'.format(product))
|
|
return
|
|
elif not force:
|
|
root = ET.fromstringlist(show_results_meta(api.apiurl, project, product,
|
|
repository=[main_repo], multibuild=True))
|
|
if len(root.xpath('result[@state="building"]')) or len(root.xpath('result[@state="dirty"]')):
|
|
print('{}/{} build in progress'.format(project, product))
|
|
return
|
|
|
|
drop_list = api.item_exists(project, oldrepos)
|
|
checkout_list = [group, product, release]
|
|
if drop_list and not only_release_packages:
|
|
checkout_list.append(oldrepos)
|
|
|
|
if packages.find('entry[@name="{}"]'.format(release)) is None:
|
|
if not self.dry_run:
|
|
undelete_package(api.apiurl, project, release, 'revive')
|
|
print('{} undeleted, skip dvd until next cycle'.format(release))
|
|
return
|
|
|
|
# Cache dir specific to hostname and project.
|
|
host = urlparse(api.apiurl).hostname
|
|
cache_dir = CacheManager.directory('pkglistgen', host, project)
|
|
|
|
if not no_checkout:
|
|
if os.path.exists(cache_dir):
|
|
shutil.rmtree(cache_dir)
|
|
os.makedirs(cache_dir)
|
|
|
|
group_dir = os.path.join(cache_dir, group)
|
|
product_dir = os.path.join(cache_dir, product)
|
|
release_dir = os.path.join(cache_dir, release)
|
|
oldrepos_dir = os.path.join(cache_dir, oldrepos)
|
|
|
|
for package in checkout_list:
|
|
if no_checkout:
|
|
print('Skipping checkout of {}/{}'.format(project, package))
|
|
continue
|
|
checkout_package(api.apiurl, project, package, expand_link=True,
|
|
prj_dir=cache_dir, outdir=os.path.join(cache_dir, package))
|
|
|
|
file_utils.unlink_all_except(release_dir, ['weakremovers.inc'])
|
|
if not only_release_packages:
|
|
file_utils.unlink_all_except(product_dir)
|
|
file_utils.copy_directory_contents(group_dir, product_dir,
|
|
['supportstatus.txt', 'groups.yml',
|
|
'reference-unsorted.yml', 'reference-summary.yml',
|
|
'package-groups.changes'])
|
|
file_utils.change_extension(product_dir, '.spec.in', '.spec')
|
|
file_utils.change_extension(product_dir, '.product.in', '.product')
|
|
|
|
self.input_dir = group_dir
|
|
self.output_dir = product_dir
|
|
|
|
print('-> do_update')
|
|
# make sure we only calculcate existant architectures
|
|
self.filter_architectures(target_archs(api.apiurl, project, main_repo))
|
|
self.update_repos(self.filtered_architectures)
|
|
|
|
if only_release_packages:
|
|
self.load_all_groups()
|
|
self.write_group_stubs()
|
|
else:
|
|
summary = self.solve_project(ignore_unresolvable=str2bool(target_config.get('pkglistgen-ignore-unresolvable')),
|
|
ignore_recommended=str2bool(target_config.get('pkglistgen-ignore-recommended')),
|
|
locale = target_config.get('pkglistgen-locale'),
|
|
locales_from = target_config.get('pkglistgen-locales-from'))
|
|
|
|
if stop_after_solve:
|
|
return
|
|
|
|
if drop_list:
|
|
weakremovers_file = os.path.join(release_dir, 'weakremovers.inc')
|
|
self.create_weakremovers(project, target_config, oldrepos_dir, output=open(weakremovers_file, 'w'))
|
|
|
|
delete_products = target_config.get('pkglistgen-delete-products', '').split(' ')
|
|
file_utils.unlink_list(product_dir, delete_products)
|
|
|
|
print('-> product service')
|
|
product_version = attribute_value_load(api.apiurl, project, 'ProductVersion')
|
|
if not product_version:
|
|
# for stagings the product version doesn't matter (I hope)
|
|
product_version = '1'
|
|
for product_file in glob.glob(os.path.join(product_dir, '*.product')):
|
|
self.replace_product_version(product_file, product_version)
|
|
print(subprocess.check_output(
|
|
[PRODUCT_SERVICE, product_file, product_dir, project], encoding='utf-8'))
|
|
|
|
for delete_kiwi in target_config.get('pkglistgen-delete-kiwis-{}'.format(scope), '').split(' '):
|
|
delete_kiwis = glob.glob(os.path.join(product_dir, delete_kiwi))
|
|
file_utils.unlink_list(product_dir, delete_kiwis)
|
|
if scope == 'staging':
|
|
self.strip_medium_from_staging(product_dir)
|
|
|
|
spec_files = glob.glob(os.path.join(product_dir, '*.spec'))
|
|
file_utils.move_list(spec_files, release_dir)
|
|
inc_files = glob.glob(os.path.join(group_dir, '*.inc'))
|
|
# filter special inc file
|
|
inc_files = filter(lambda file: file.endswith('weakremovers.inc'), inc_files)
|
|
file_utils.move_list(inc_files, release_dir)
|
|
|
|
# do not overwrite weakremovers.inc if it exists
|
|
# we will commit there afterwards if needed
|
|
if os.path.exists(os.path.join(group_dir, 'weakremovers.inc')) and \
|
|
not os.path.exists(os.path.join(release_dir, 'weakremovers.inc')):
|
|
file_utils.move_list([os.path.join(group_dir, 'weakremovers.inc')], release_dir)
|
|
|
|
file_utils.multibuild_from_glob(release_dir, '*.spec')
|
|
self.build_stub(release_dir, 'spec')
|
|
self.commit_package(release_dir)
|
|
|
|
if only_release_packages:
|
|
return
|
|
|
|
file_utils.multibuild_from_glob(product_dir, '*.kiwi')
|
|
self.build_stub(product_dir, 'kiwi')
|
|
self.commit_package(product_dir)
|
|
|
|
error_output = b''
|
|
reference_summary = os.path.join(group_dir, 'reference-summary.yml')
|
|
if os.path.isfile(reference_summary):
|
|
summary_file = os.path.join(product_dir, 'summary.yml')
|
|
with open(summary_file, 'w') as f:
|
|
f.write('# Summary of packages in groups')
|
|
for group in sorted(summary):
|
|
# the unsorted group should appear filtered by
|
|
# unneeded.yml - so we need the content of unsorted.yml
|
|
# not unsorted.group (this grew a little unnaturally)
|
|
if group == 'unsorted':
|
|
continue
|
|
f.write('\n' + group + ':\n')
|
|
for package in sorted(summary[group]):
|
|
f.write(' - ' + package + '\n')
|
|
|
|
try:
|
|
error_output += subprocess.check_output(['diff', '-u', reference_summary, summary_file])
|
|
except subprocess.CalledProcessError as e:
|
|
error_output += e.output
|
|
reference_unsorted = os.path.join(group_dir, 'reference-unsorted.yml')
|
|
unsorted_file = os.path.join(product_dir, 'unsorted.yml')
|
|
try:
|
|
error_output += subprocess.check_output(['diff', '-u', reference_unsorted, unsorted_file])
|
|
except subprocess.CalledProcessError as e:
|
|
error_output += e.output
|
|
|
|
if len(error_output) > 0:
|
|
self.logger.error('Difference in yml:\n' + error_output.decode('utf-8'))
|
|
return True
|