770 lines
32 KiB
Python
770 lines
32 KiB
Python
import ToolBase
|
|
import glob
|
|
import logging
|
|
import os
|
|
import re
|
|
import solv
|
|
import shutil
|
|
import subprocess
|
|
import yaml
|
|
import sys
|
|
import tempfile
|
|
|
|
from lxml import etree as ET
|
|
|
|
from osc.core import checkout_package
|
|
|
|
from osc.core import http_GET
|
|
from osc.core import HTTPError
|
|
from osc.core import show_results_meta
|
|
from osc.core import Package
|
|
from osc.core import undelete_package
|
|
from osclib.core import attribute_value_load
|
|
from osclib.core import target_archs
|
|
from osclib.conf import str2bool
|
|
from osclib.core import repository_path_expand
|
|
from osclib.core import repository_arch_state
|
|
from osclib.cache_manager import CacheManager
|
|
|
|
from urllib.parse import urlparse
|
|
|
|
from pkglistgen import file_utils
|
|
from pkglistgen.group import Group
|
|
|
|
SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))
|
|
|
|
PRODUCT_SERVICE = '/usr/lib/obs/service/create_single_product'
|
|
|
|
# share header cache with repochecker
|
|
CACHEDIR = CacheManager.directory('repository-meta')
|
|
|
|
class PkgListGen(ToolBase.ToolBase):
|
|
|
|
def __init__(self):
|
|
ToolBase.ToolBase.__init__(self)
|
|
self.logger = logging.getLogger(__name__)
|
|
self.reset()
|
|
|
|
def reset(self):
|
|
# package -> supportatus
|
|
self.packages = dict()
|
|
self.groups = dict()
|
|
self._supportstatus = None
|
|
self.input_dir = '.'
|
|
self.output_dir = '.'
|
|
self.lockjobs = dict()
|
|
self.ignore_broken = False
|
|
self.unwanted = set()
|
|
self.output = None
|
|
self.locales = set()
|
|
self.filtered_architectures = None
|
|
self.dry_run = False
|
|
self.all_architectures = None
|
|
|
|
def filter_architectures(self, architectures):
|
|
self.filtered_architectures = sorted(list(set(architectures) & set(self.all_architectures)))
|
|
|
|
def _load_supportstatus(self):
|
|
# XXX
|
|
fn = os.path.join(self.input_dir, 'supportstatus.txt')
|
|
self._supportstatus = dict()
|
|
if os.path.exists(fn):
|
|
with open(fn, 'r') as fh:
|
|
for line in fh:
|
|
# pkg, status
|
|
fields = line.rstrip().split(' ')
|
|
if len(fields) > 1:
|
|
self._supportstatus[fields[0]] = fields[1]
|
|
|
|
def supportstatus(self, package):
|
|
if self._supportstatus is None:
|
|
self._load_supportstatus()
|
|
|
|
return self._supportstatus.get(package)
|
|
|
|
def _load_group_file(self, fn):
|
|
output = None
|
|
unwanted = None
|
|
with open(fn, 'r') as fh:
|
|
self.logger.debug('reading %s', fn)
|
|
for groupname, group in yaml.safe_load(fh).items():
|
|
if groupname == 'OUTPUT':
|
|
output = group
|
|
continue
|
|
if groupname == 'UNWANTED':
|
|
unwanted = set(group)
|
|
continue
|
|
g = Group(groupname, self)
|
|
g.parse_yml(group)
|
|
return output, unwanted
|
|
|
|
def load_all_groups(self):
|
|
for fn in glob.glob(os.path.join(self.input_dir, 'group*.yml')):
|
|
o, u = self._load_group_file(fn)
|
|
if o:
|
|
if self.output is not None:
|
|
raise Exception('OUTPUT defined multiple times')
|
|
self.output = o
|
|
if u:
|
|
self.unwanted |= u
|
|
|
|
# required to generate release spec files (only)
|
|
def write_group_stubs(self):
|
|
archs = ['*'] + self.all_architectures
|
|
for name in self.groups:
|
|
group = self.groups[name]
|
|
group.solved_packages = dict()
|
|
fn = '{}.group'.format(group.name)
|
|
with open(os.path.join(self.output_dir, fn), 'w') as fh:
|
|
for arch in archs:
|
|
x = group.toxml(arch, group.ignore_broken, None)
|
|
x = ET.tostring(x, pretty_print=True, encoding='unicode')
|
|
fh.write(x)
|
|
|
|
def write_all_groups(self):
|
|
self._check_supplements()
|
|
summary = dict()
|
|
archs = ['*'] + self.all_architectures
|
|
for name in self.groups:
|
|
group = self.groups[name]
|
|
if not group.solved:
|
|
continue
|
|
summary[name] = group.summary()
|
|
fn = '{}.group'.format(group.name)
|
|
with open(os.path.join(self.output_dir, fn), 'w') as fh:
|
|
comment = group.comment
|
|
for arch in archs:
|
|
x = group.toxml(arch, group.ignore_broken, comment)
|
|
# only comment first time
|
|
comment = None
|
|
x = ET.tostring(x, pretty_print=True, encoding='unicode')
|
|
x = re.sub(r'\s*<!-- reason:', ' <!-- reason:', x)
|
|
fh.write(x)
|
|
return summary
|
|
|
|
def solve_module(self, groupname, includes, excludes, use_recommends):
|
|
g = self.groups[groupname]
|
|
importants = set()
|
|
for i in includes:
|
|
name = i
|
|
if isinstance(i, dict):
|
|
name = list(i)[0]
|
|
if i[name] != 'support':
|
|
importants.add(name)
|
|
else:
|
|
importants.add(name)
|
|
g.inherit(self.groups[name])
|
|
g.solve(use_recommends)
|
|
for e in excludes:
|
|
g.ignore(self.groups[e])
|
|
for i in importants:
|
|
group = self.groups[i]
|
|
for arch in group.packages:
|
|
if arch not in g.solved_packages:
|
|
continue
|
|
for package in group.packages[arch]:
|
|
if package[0] in g.solved_packages[arch]:
|
|
continue
|
|
if package[0] not in g.solved_packages['*']:
|
|
self.logger.error(f'Missing {package[0]} in {groupname} for {arch}')
|
|
|
|
def expand_repos(self, project, repo='standard'):
|
|
return repository_path_expand(self.apiurl, project, repo)
|
|
|
|
def _check_supplements(self):
|
|
tocheck = set()
|
|
tocheck_locales = set()
|
|
for arch in self.filtered_architectures:
|
|
pool = self.prepare_pool(arch, True)
|
|
sel = pool.Selection()
|
|
for s in pool.solvables_iter():
|
|
sel.add_raw(solv.Job.SOLVER_SOLVABLE, s.id)
|
|
|
|
for s in sel.solvables():
|
|
for dep in s.lookup_deparray(solv.SOLVABLE_SUPPLEMENTS):
|
|
for d in dep.str().split(' '):
|
|
if d.startswith('namespace:modalias') or d.startswith('namespace:filesystem'):
|
|
tocheck.add(s.name)
|
|
|
|
for locale in self.locales:
|
|
id = pool.str2id('locale({})'.format(locale))
|
|
for s in pool.whatprovides(id):
|
|
tocheck_locales.add(s.name)
|
|
|
|
all_grouped = set()
|
|
for g in self.groups.values():
|
|
if g.solved:
|
|
for arch in g.solved_packages.keys():
|
|
if g.solved_packages[arch]:
|
|
all_grouped.update(g.solved_packages[arch])
|
|
|
|
for p in tocheck - all_grouped:
|
|
self.logger.warning('package %s has supplements but is not grouped', p)
|
|
|
|
for p in tocheck_locales - all_grouped:
|
|
self.logger.warning('package %s provides supported locale but is not grouped', p)
|
|
|
|
def prepare_pool(self, arch, ignore_conflicts):
|
|
pool = solv.Pool()
|
|
# the i586 DVD is really a i686 one
|
|
if arch == 'i586':
|
|
pool.setarch('i686')
|
|
else:
|
|
pool.setarch(arch)
|
|
|
|
self.lockjobs[arch] = []
|
|
solvables = set()
|
|
|
|
for project, reponame in self.repos:
|
|
repo = pool.add_repo(project)
|
|
# check back the repo state to avoid suprises
|
|
state = repository_arch_state(self.apiurl, project, reponame, arch)
|
|
if state is None:
|
|
self.logger.debug(f'Skipping {project}/{reponame}/{arch}')
|
|
continue
|
|
s = f'repo-{project}-{reponame}-{arch}-{state}.solv'
|
|
if not repo.add_solv(s):
|
|
raise Exception('failed to add repo {}/{}/{}'.format(project, reponame, arch))
|
|
for solvable in repo.solvables_iter():
|
|
if ignore_conflicts:
|
|
solvable.unset(solv.SOLVABLE_CONFLICTS)
|
|
solvable.unset(solv.SOLVABLE_OBSOLETES)
|
|
# only take the first solvable in the repo chain
|
|
if not self.use_newest_version and solvable.name in solvables:
|
|
self.lockjobs[arch].append(pool.Job(solv.Job.SOLVER_SOLVABLE | solv.Job.SOLVER_LOCK, solvable.id))
|
|
solvables.add(solvable.name)
|
|
|
|
pool.addfileprovides()
|
|
pool.createwhatprovides()
|
|
|
|
for locale in self.locales:
|
|
pool.set_namespaceproviders(solv.NAMESPACE_LANGUAGE, pool.Dep(locale), True)
|
|
|
|
return pool
|
|
|
|
# parse file and merge all groups
|
|
def _parse_unneeded(self, filename):
|
|
filename = os.path.join(self.input_dir, filename)
|
|
if not os.path.isfile(filename):
|
|
return set()
|
|
fh = open(filename, 'r')
|
|
self.logger.debug('reading %s', filename)
|
|
result = set()
|
|
for group in yaml.safe_load(fh).values():
|
|
result.update(group)
|
|
return result
|
|
|
|
# the unsorted group is special and will contain all the rest for
|
|
# the FTP tree. We filter it with unneeded though to create a
|
|
# unsorted.yml file for release manager review
|
|
def _collect_unsorted_packages(self, modules, unsorted):
|
|
uneeded_regexps = [re.compile(r)
|
|
for r in self._parse_unneeded('unneeded.yml')]
|
|
|
|
packages = dict()
|
|
if unsorted:
|
|
unsorted.solved_packages = dict()
|
|
unsorted.solved_packages['*'] = dict()
|
|
|
|
for arch in self.filtered_architectures:
|
|
pool = self.prepare_pool(arch, False)
|
|
pool.Selection()
|
|
archpacks = [s.name for s in pool.solvables_iter()]
|
|
|
|
# copy
|
|
filtered = list(archpacks)
|
|
for r in uneeded_regexps:
|
|
filtered = [p for p in filtered if not r.match(p)]
|
|
|
|
# convert to set
|
|
filtered = set(filtered) - self.unwanted
|
|
for g in modules:
|
|
if unsorted and g == unsorted:
|
|
continue
|
|
for a in ('*', arch):
|
|
filtered -= set(g.solved_packages[a])
|
|
for package in filtered:
|
|
packages.setdefault(package, []).append(arch)
|
|
|
|
if unsorted:
|
|
archpacks = set(archpacks)
|
|
unsorted.solved_packages[arch] = dict()
|
|
for g in modules:
|
|
archpacks -= set(g.solved_packages[arch])
|
|
archpacks -= set(g.solved_packages['*'])
|
|
unsorted.solved_packages[arch] = dict()
|
|
for p in archpacks:
|
|
unsorted.solved_packages[arch][p] = None
|
|
|
|
if unsorted:
|
|
common = None
|
|
for arch in self.filtered_architectures:
|
|
if common is None:
|
|
common = set(unsorted.solved_packages[arch])
|
|
continue
|
|
common &= set(unsorted.solved_packages[arch])
|
|
for p in common:
|
|
unsorted.solved_packages['*'][p] = None
|
|
for arch in self.filtered_architectures:
|
|
del unsorted.solved_packages[arch][p]
|
|
|
|
with open(os.path.join(self.output_dir, 'unsorted.yml'), 'w') as fh:
|
|
fh.write('unsorted:\n')
|
|
for p in sorted(packages):
|
|
fh.write(' - ')
|
|
fh.write(p)
|
|
if len(packages[p]) != len(self.filtered_architectures):
|
|
fh.write(': [')
|
|
fh.write(','.join(sorted(packages[p])))
|
|
fh.write(']')
|
|
reason = self._find_reason(p, modules)
|
|
if reason:
|
|
fh.write(' # ' + reason)
|
|
fh.write(' \n')
|
|
|
|
# give a hint if the package is related to a group
|
|
def _find_reason(self, package, modules):
|
|
# go through the modules multiple times to find the "best"
|
|
for g in modules:
|
|
if package in g.recommends:
|
|
return 'recommended by ' + g.recommends[package]
|
|
for g in modules:
|
|
if package in g.suggested:
|
|
return 'suggested by ' + g.suggested[package]
|
|
for g in modules:
|
|
if package in g.develpkgs:
|
|
return 'devel package of ' + g.develpkgs[package]
|
|
return None
|
|
|
|
def update_one_repo(self, project, repo, arch, solv_file, solv_file_hash):
|
|
# Either hash changed or new, so remove any old hash files.
|
|
file_utils.unlink_list(None, glob.glob(solv_file + '::*'))
|
|
|
|
d = os.path.join(CACHEDIR, project, repo, arch)
|
|
if not os.path.exists(d):
|
|
os.makedirs(d)
|
|
|
|
self.logger.debug('updating %s', d)
|
|
|
|
# only there to parse the repos
|
|
bs_mirrorfull = os.path.join(SCRIPT_PATH, '..', 'bs_mirrorfull')
|
|
|
|
args = [bs_mirrorfull]
|
|
args.append('--nodebug')
|
|
args.append('{}/public/build/{}/{}/{}'.format(self.apiurl, project, repo, arch))
|
|
args.append(d)
|
|
with subprocess.Popen(args, stdout=subprocess.PIPE) as p:
|
|
for line in p.stdout:
|
|
self.logger.info(line.decode('utf-8').rstrip())
|
|
if p.wait() != 0:
|
|
raise Exception("Mirroring repository failed")
|
|
|
|
files = [os.path.join(d, f)
|
|
for f in os.listdir(d) if f.endswith('.rpm')]
|
|
suffix = f'.{os.getpid()}.tmp'
|
|
fh = open(solv_file + suffix, 'w')
|
|
p = subprocess.Popen(
|
|
['rpms2solv', '-m', '-', '-0'], stdin=subprocess.PIPE, stdout=fh)
|
|
p.communicate(bytes('\0'.join(files), 'utf-8'))
|
|
fh.close()
|
|
if p.wait() != 0:
|
|
raise Exception("rpm2solv failed")
|
|
os.rename(solv_file + suffix, solv_file)
|
|
|
|
# Create hash file now that solv creation is complete.
|
|
open(solv_file_hash, 'a').close()
|
|
|
|
def update_repos(self, architectures):
|
|
for project, repo in self.repos:
|
|
for arch in architectures:
|
|
# Fetch state before mirroring in-case it changes during download.
|
|
state = repository_arch_state(self.apiurl, project, repo, arch)
|
|
if state is None:
|
|
# Repo might not have this architecture
|
|
continue
|
|
|
|
repo_solv_name = 'repo-{}-{}-{}.solv'.format(project, repo, arch)
|
|
# Would be preferable to include hash in name, but cumbersome to handle without
|
|
# reworking a fair bit since the state needs to be tracked.
|
|
solv_file = os.path.join(CACHEDIR, repo_solv_name)
|
|
solv_file_hash = '{}::{}'.format(solv_file, state)
|
|
if os.path.exists(solv_file) and os.path.exists(solv_file_hash):
|
|
# Solve file exists and hash unchanged, skip updating solv.
|
|
self.logger.debug('skipping solv generation for {} due to matching state {}'.format(
|
|
'/'.join([project, repo, arch]), state))
|
|
else:
|
|
self.update_one_repo(project, repo, arch, solv_file, solv_file_hash)
|
|
shutil.copy(solv_file, f'./repo-{project}-{repo}-{arch}-{state}.solv')
|
|
|
|
def create_weakremovers(self, target, target_config, directory, output):
|
|
drops = dict()
|
|
dropped_repos = dict()
|
|
|
|
root = yaml.safe_load(open(os.path.join(directory, 'config.yml')))
|
|
for item in root:
|
|
key = list(item)[0]
|
|
opts = item[key]
|
|
# cast 15.1 to string :)
|
|
key = str(key)
|
|
|
|
oldrepos = set(glob.glob(os.path.join(directory, '{}_*.packages.xz'.format(key))))
|
|
oldrepos |= set(glob.glob(os.path.join(directory, '{}.packages.xz'.format(key))))
|
|
for oldrepo in sorted(oldrepos):
|
|
pool = solv.Pool()
|
|
pool.setarch()
|
|
|
|
# we need some progress in the debug output - or gocd gets nervous
|
|
self.logger.debug('checking {}'.format(oldrepo))
|
|
oldsysrepo = pool.add_repo(oldrepo)
|
|
defvendorid = oldsysrepo.meta.lookup_id(solv.SUSETAGS_DEFAULTVENDOR)
|
|
f = tempfile.TemporaryFile()
|
|
# FIXME: port to lzma module with python3
|
|
st = subprocess.call(['xz', '-cd', oldrepo], stdout=f.fileno())
|
|
os.lseek(f.fileno(), 0, os.SEEK_SET)
|
|
oldsysrepo.add_susetags(solv.xfopen_fd(None, f.fileno()), defvendorid, None, solv.Repo.REPO_NO_INTERNALIZE | solv.Repo.SUSETAGS_RECORD_SHARES)
|
|
|
|
for arch in self.all_architectures:
|
|
for project, repo in self.repos:
|
|
# check back the repo state to avoid suprises
|
|
state = repository_arch_state(self.apiurl, project, repo, arch)
|
|
if state is None:
|
|
self.logger.debug(f'Skipping {project}/{repo}/{arch}')
|
|
fn = f'repo-{project}-{repo}-{arch}-{state}.solv'
|
|
r = pool.add_repo('/'.join([project, repo]))
|
|
if not r.add_solv(fn):
|
|
raise Exception('failed to add repo {}/{}/{}.'.format(project, repo, arch))
|
|
|
|
pool.createwhatprovides()
|
|
|
|
accepted_archs = set(self.all_architectures)
|
|
accepted_archs.add('noarch')
|
|
|
|
for s in oldsysrepo.solvables_iter():
|
|
oldarch = s.arch
|
|
if oldarch == 'i686':
|
|
oldarch = 'i586'
|
|
|
|
if oldarch not in accepted_archs:
|
|
continue
|
|
|
|
haveit = False
|
|
for s2 in pool.whatprovides(s.nameid):
|
|
if s2.repo == oldsysrepo or s.nameid != s2.nameid:
|
|
continue
|
|
newarch = s2.arch
|
|
if newarch == 'i686':
|
|
newarch = 'i586'
|
|
if oldarch != newarch and newarch != 'noarch' and oldarch != 'noarch':
|
|
continue
|
|
haveit = True
|
|
break
|
|
if haveit:
|
|
continue
|
|
|
|
# check for already obsoleted packages
|
|
nevr = pool.rel2id(s.nameid, s.evrid, solv.REL_EQ)
|
|
for s2 in pool.whatmatchesdep(solv.SOLVABLE_OBSOLETES, nevr):
|
|
if s2.repo == oldsysrepo:
|
|
continue
|
|
haveit = True
|
|
break
|
|
if haveit:
|
|
continue
|
|
if s.name not in drops:
|
|
drops[s.name] = {'repo': key, 'archs': set()}
|
|
if oldarch == 'noarch':
|
|
drops[s.name]['archs'] |= set(self.all_architectures)
|
|
else:
|
|
drops[s.name]['archs'].add(oldarch)
|
|
dropped_repos[key] = 1
|
|
|
|
del pool
|
|
|
|
for repo in sorted(dropped_repos):
|
|
repo_output = False
|
|
exclusives = dict()
|
|
for name in sorted(drops):
|
|
if drops[name]['repo'] != repo:
|
|
continue
|
|
if drops[name]['archs'] == set(self.all_architectures):
|
|
if not repo_output:
|
|
print('#', repo, file=output)
|
|
repo_output = True
|
|
print('Provides: weakremover({})'.format(name), file=output)
|
|
else:
|
|
jarch = ' '.join(sorted(drops[name]['archs']))
|
|
exclusives.setdefault(jarch, []).append(name)
|
|
|
|
for arch in sorted(exclusives):
|
|
if not repo_output:
|
|
print('#', repo, file=output)
|
|
repo_output = True
|
|
print('%ifarch {}'.format(arch), file=output)
|
|
for name in sorted(exclusives[arch]):
|
|
print('Provides: weakremover({})'.format(name), file=output)
|
|
print('%endif', file=output)
|
|
output.flush()
|
|
|
|
def solve_project(self, ignore_unresolvable=False, ignore_recommended=False, locale=None, locales_from=None):
|
|
self.load_all_groups()
|
|
if not self.output:
|
|
self.logger.error('OUTPUT not defined')
|
|
return
|
|
|
|
if ignore_unresolvable:
|
|
self.ignore_broken = True
|
|
global_use_recommends = not ignore_recommended
|
|
if locale:
|
|
self.locales |= set(locale.split(' '))
|
|
if locales_from:
|
|
with open(os.path.join(self.input_dir, locales_from), 'r') as fh:
|
|
root = ET.parse(fh).getroot()
|
|
self.locales |= set([lang.text for lang in root.findall('.//linguas/language')])
|
|
|
|
modules = []
|
|
# the yml parser makes an array out of everything, so
|
|
# we loop a bit more than what we support
|
|
for group in self.output:
|
|
groupname = list(group)[0]
|
|
settings = group[groupname]
|
|
if not settings: # e.g. unsorted
|
|
settings = {}
|
|
includes = settings.get('includes', [])
|
|
excludes = settings.get('excludes', [])
|
|
use_recommends = settings.get('recommends', global_use_recommends)
|
|
self.solve_module(groupname, includes, excludes, use_recommends)
|
|
g = self.groups[groupname]
|
|
# the default is a little double negated but Factory has ignore_broken
|
|
# as default and we only disable it for single groups (for now)
|
|
g.ignore_broken = not settings.get('require_all', not self.ignore_broken)
|
|
g.conflicts = settings.get('conflicts', [])
|
|
g.default_support_status = settings.get('default-support', 'unsupported')
|
|
modules.append(g)
|
|
|
|
# not defined for openSUSE
|
|
overlap = self.groups.get('overlap')
|
|
for module in modules:
|
|
module.check_dups(modules, overlap)
|
|
module.collect_devel_packages()
|
|
module.filter_already_selected(modules)
|
|
|
|
if overlap:
|
|
ignores = [x.name for x in overlap.ignored]
|
|
self.solve_module(overlap.name, [], ignores, use_recommends=False)
|
|
overlapped = set(overlap.solved_packages['*'])
|
|
for arch in self.filtered_architectures:
|
|
overlapped |= set(overlap.solved_packages[arch])
|
|
for module in modules:
|
|
if module.name == 'overlap' or module in overlap.ignored:
|
|
continue
|
|
for arch in ['*'] + self.filtered_architectures:
|
|
for p in overlapped:
|
|
module.solved_packages[arch].pop(p, None)
|
|
|
|
self._collect_unsorted_packages(modules, self.groups.get('unsorted'))
|
|
return self.write_all_groups()
|
|
|
|
def strip_medium_from_staging(self, path):
|
|
# staging projects don't need source and debug medium - and the glibc source
|
|
# rpm conflicts between standard and bootstrap_copy repository causing the
|
|
# product builder to fail
|
|
medium = re.compile('name="(DEBUG|SOURCE)MEDIUM"')
|
|
for name in glob.glob(os.path.join(path, '*.kiwi')):
|
|
lines = open(name).readlines()
|
|
lines = [x for x in lines if not medium.search(x)]
|
|
open(name, 'w').writelines(lines)
|
|
|
|
def build_stub(self, destination, extension):
|
|
with open(os.path.join(destination, '.'.join(['stub', extension])), 'w+') as f:
|
|
f.write('# prevent building single {} files twice\n'.format(extension))
|
|
f.write('Name: stub\n')
|
|
f.write('Version: 0.0\n')
|
|
|
|
def commit_package(self, path):
|
|
if self.dry_run:
|
|
package = Package(path)
|
|
for i in package.get_diff():
|
|
print(''.join(i))
|
|
else:
|
|
# No proper API function to perform the same operation.
|
|
print(subprocess.check_output(
|
|
' '.join(['cd', path, '&&', 'osc', 'addremove']), shell=True, encoding='utf-8'))
|
|
package = Package(path)
|
|
package.commit(msg='Automatic update', skip_local_service_run=True)
|
|
|
|
def replace_product_version(self, product_file, product_version):
|
|
product_version = '<version>{}</version>'.format(product_version)
|
|
lines = open(product_file).readlines()
|
|
new_lines = []
|
|
for line in lines:
|
|
new_lines.append(line.replace('<version></version>', product_version))
|
|
open(product_file, 'w').write(''.join(new_lines))
|
|
|
|
def update_and_solve_target(self, api, target_project, target_config, main_repo,
|
|
project, scope, force, no_checkout,
|
|
only_release_packages, stop_after_solve):
|
|
self.all_architectures = target_config.get('pkglistgen-archs').split(' ')
|
|
self.use_newest_version = str2bool(target_config.get('pkglistgen-use-newest-version', 'False'))
|
|
self.repos = self.expand_repos(project, main_repo)
|
|
print('[{}] {}/{}: update and solve'.format(scope, project, main_repo))
|
|
|
|
group = target_config.get('pkglistgen-group', '000package-groups')
|
|
product = target_config.get('pkglistgen-product', '000product')
|
|
release = target_config.get('pkglistgen-release', '000release-packages')
|
|
oldrepos = target_config.get('pkglistgen-repos', '000update-repos')
|
|
|
|
url = api.makeurl(['source', project])
|
|
packages = ET.parse(http_GET(url)).getroot()
|
|
if packages.find('entry[@name="{}"]'.format(product)) is None:
|
|
if not self.dry_run:
|
|
undelete_package(api.apiurl, project, product, 'revive')
|
|
# TODO disable build.
|
|
print('{} undeleted, skip dvd until next cycle'.format(product))
|
|
return
|
|
elif not force:
|
|
root = ET.fromstringlist(show_results_meta(api.apiurl, project, product,
|
|
repository=[main_repo], multibuild=True))
|
|
if len(root.xpath('result[@state="building"]')) or len(root.xpath('result[@state="dirty"]')):
|
|
print('{}/{} build in progress'.format(project, product))
|
|
return
|
|
|
|
drop_list = api.item_exists(project, oldrepos)
|
|
checkout_list = [group, product, release]
|
|
if drop_list and not only_release_packages:
|
|
checkout_list.append(oldrepos)
|
|
|
|
if packages.find('entry[@name="{}"]'.format(release)) is None:
|
|
if not self.dry_run:
|
|
undelete_package(api.apiurl, project, release, 'revive')
|
|
print('{} undeleted, skip dvd until next cycle'.format(release))
|
|
return
|
|
|
|
# Cache dir specific to hostname and project.
|
|
host = urlparse(api.apiurl).hostname
|
|
cache_dir = CacheManager.directory('pkglistgen', host, project)
|
|
|
|
if not no_checkout:
|
|
if os.path.exists(cache_dir):
|
|
shutil.rmtree(cache_dir)
|
|
os.makedirs(cache_dir)
|
|
|
|
group_dir = os.path.join(cache_dir, group)
|
|
product_dir = os.path.join(cache_dir, product)
|
|
release_dir = os.path.join(cache_dir, release)
|
|
oldrepos_dir = os.path.join(cache_dir, oldrepos)
|
|
|
|
for package in checkout_list:
|
|
if no_checkout:
|
|
print('Skipping checkout of {}/{}'.format(project, package))
|
|
continue
|
|
checkout_package(api.apiurl, project, package, expand_link=True,
|
|
prj_dir=cache_dir, outdir=os.path.join(cache_dir, package))
|
|
|
|
file_utils.unlink_all_except(release_dir, ['weakremovers.inc'])
|
|
if not only_release_packages:
|
|
file_utils.unlink_all_except(product_dir)
|
|
file_utils.copy_directory_contents(group_dir, product_dir,
|
|
['supportstatus.txt', 'groups.yml',
|
|
'reference-unsorted.yml', 'reference-summary.yml',
|
|
'package-groups.changes'])
|
|
file_utils.change_extension(product_dir, '.spec.in', '.spec')
|
|
file_utils.change_extension(product_dir, '.product.in', '.product')
|
|
|
|
self.input_dir = group_dir
|
|
self.output_dir = product_dir
|
|
|
|
print('-> do_update')
|
|
# make sure we only calculcate existant architectures
|
|
self.filter_architectures(target_archs(api.apiurl, project, main_repo))
|
|
self.update_repos(self.filtered_architectures)
|
|
|
|
if only_release_packages:
|
|
self.load_all_groups()
|
|
self.write_group_stubs()
|
|
else:
|
|
summary = self.solve_project(ignore_unresolvable=str2bool(target_config.get('pkglistgen-ignore-unresolvable')),
|
|
ignore_recommended=str2bool(target_config.get('pkglistgen-ignore-recommended')),
|
|
locale = target_config.get('pkglistgen-locale'),
|
|
locales_from = target_config.get('pkglistgen-locales-from'))
|
|
|
|
if stop_after_solve:
|
|
return
|
|
|
|
if drop_list and not only_release_packages:
|
|
weakremovers_file = os.path.join(release_dir, 'weakremovers.inc')
|
|
self.create_weakremovers(project, target_config, oldrepos_dir, output=open(weakremovers_file, 'w'))
|
|
|
|
delete_products = target_config.get('pkglistgen-delete-products', '').split(' ')
|
|
file_utils.unlink_list(product_dir, delete_products)
|
|
|
|
print('-> product service')
|
|
product_version = attribute_value_load(api.apiurl, project, 'ProductVersion')
|
|
if not product_version:
|
|
# for stagings the product version doesn't matter (I hope)
|
|
product_version = '1'
|
|
for product_file in glob.glob(os.path.join(product_dir, '*.product')):
|
|
self.replace_product_version(product_file, product_version)
|
|
print(subprocess.check_output(
|
|
[PRODUCT_SERVICE, product_file, product_dir, project], encoding='utf-8'))
|
|
|
|
for delete_kiwi in target_config.get('pkglistgen-delete-kiwis-{}'.format(scope), '').split(' '):
|
|
delete_kiwis = glob.glob(os.path.join(product_dir, delete_kiwi))
|
|
file_utils.unlink_list(product_dir, delete_kiwis)
|
|
if scope == 'staging':
|
|
self.strip_medium_from_staging(product_dir)
|
|
|
|
spec_files = glob.glob(os.path.join(product_dir, '*.spec'))
|
|
file_utils.move_list(spec_files, release_dir)
|
|
inc_files = glob.glob(os.path.join(group_dir, '*.inc'))
|
|
# filter special inc file
|
|
inc_files = filter(lambda file: file.endswith('weakremovers.inc'), inc_files)
|
|
file_utils.move_list(inc_files, release_dir)
|
|
|
|
# do not overwrite weakremovers.inc if it exists
|
|
# we will commit there afterwards if needed
|
|
if os.path.exists(os.path.join(group_dir, 'weakremovers.inc')) and \
|
|
not os.path.exists(os.path.join(release_dir, 'weakremovers.inc')):
|
|
file_utils.move_list([os.path.join(group_dir, 'weakremovers.inc')], release_dir)
|
|
|
|
file_utils.multibuild_from_glob(release_dir, '*.spec')
|
|
self.build_stub(release_dir, 'spec')
|
|
self.commit_package(release_dir)
|
|
|
|
if only_release_packages:
|
|
return
|
|
|
|
file_utils.multibuild_from_glob(product_dir, '*.kiwi')
|
|
self.build_stub(product_dir, 'kiwi')
|
|
self.commit_package(product_dir)
|
|
|
|
error_output = b''
|
|
reference_summary = os.path.join(group_dir, 'reference-summary.yml')
|
|
if os.path.isfile(reference_summary):
|
|
summary_file = os.path.join(product_dir, 'summary.yml')
|
|
with open(summary_file, 'w') as f:
|
|
f.write('# Summary of packages in groups')
|
|
for group in sorted(summary):
|
|
# the unsorted group should appear filtered by
|
|
# unneeded.yml - so we need the content of unsorted.yml
|
|
# not unsorted.group (this grew a little unnaturally)
|
|
if group == 'unsorted':
|
|
continue
|
|
f.write('\n' + group + ':\n')
|
|
for package in sorted(summary[group]):
|
|
f.write(' - ' + package + '\n')
|
|
|
|
try:
|
|
error_output += subprocess.check_output(['diff', '-u', reference_summary, summary_file])
|
|
except subprocess.CalledProcessError as e:
|
|
error_output += e.output
|
|
reference_unsorted = os.path.join(group_dir, 'reference-unsorted.yml')
|
|
unsorted_file = os.path.join(product_dir, 'unsorted.yml')
|
|
try:
|
|
error_output += subprocess.check_output(['diff', '-u', reference_unsorted, unsorted_file])
|
|
except subprocess.CalledProcessError as e:
|
|
error_output += e.output
|
|
|
|
if len(error_output) > 0:
|
|
self.logger.error('Difference in yml:\n' + error_output.decode('utf-8'))
|
|
return True
|