918 lines
39 KiB
Python
918 lines
39 KiB
Python
import ToolBase
|
|
import glob
|
|
import logging
|
|
import os
|
|
import re
|
|
import solv
|
|
import shutil
|
|
import subprocess
|
|
import yaml
|
|
|
|
from datetime import datetime, timezone
|
|
|
|
from typing import Any, Mapping, Optional
|
|
|
|
from lxml import etree as ET
|
|
|
|
from osc.core import checkout_package
|
|
|
|
from osc.core import http_GET
|
|
from osc.core import show_results_meta
|
|
from osc.core import Package
|
|
from osc.core import undelete_package
|
|
from osclib.core import attribute_value_load
|
|
from osclib.core import target_archs
|
|
from osclib.conf import str2bool
|
|
from osclib.core import repository_path_expand
|
|
from osclib.core import repository_arch_state
|
|
from osclib.cache_manager import CacheManager
|
|
from osclib.pkglistgen_comments import PkglistComments
|
|
from osclib.repomirror import RepoMirror
|
|
|
|
from urllib.parse import urlparse
|
|
|
|
from pkglistgen import file_utils
|
|
from pkglistgen.group import Group
|
|
|
|
SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))
|
|
|
|
PRODUCT_SERVICE = '/usr/lib/obs/service/create_single_product'
|
|
PRODUCT_COMPOSE_SEPERATOR_LINE = '# The following is generated by openSUSE-release-tools'
|
|
|
|
# share header cache with repochecker
|
|
CACHEDIR = CacheManager.directory('repository-meta')
|
|
|
|
|
|
class MismatchedRepoException(Exception):
|
|
"""raised on repos that restarted building"""
|
|
|
|
|
|
class PkgListGen(ToolBase.ToolBase):
|
|
|
|
def __init__(self):
|
|
ToolBase.ToolBase.__init__(self)
|
|
self.logger = logging.getLogger(__name__)
|
|
self.comment = PkglistComments(self.apiurl)
|
|
self.reset()
|
|
|
|
def reset(self):
|
|
# package -> supportatus
|
|
self.packages = dict()
|
|
self.groups = dict()
|
|
self._supportstatus = None
|
|
self.input_dir = '.'
|
|
self.output_dir = '.'
|
|
self.lockjobs = dict()
|
|
self.ignore_broken = False
|
|
self.productcompose_dir = None
|
|
self.skip_productcompose = False
|
|
self.unwanted = set()
|
|
self.output = None
|
|
self.locales = set()
|
|
self.filtered_architectures = None
|
|
self.dry_run = False
|
|
self.all_architectures = None
|
|
|
|
def filter_architectures(self, architectures):
|
|
self.filtered_architectures = sorted(list(set(architectures) & set(self.all_architectures)))
|
|
|
|
def _load_supportstatus(self):
|
|
# XXX
|
|
fn = os.path.join(self.input_dir, 'supportstatus.txt')
|
|
self._supportstatus = dict()
|
|
if os.path.exists(fn):
|
|
with open(fn, 'r') as fh:
|
|
for line in fh:
|
|
# pkg, status
|
|
fields = line.rstrip().split(' ')
|
|
if len(fields) > 1:
|
|
self._supportstatus[fields[0]] = fields[1]
|
|
|
|
def supportstatus(self, package):
|
|
if self._supportstatus is None:
|
|
self._load_supportstatus()
|
|
|
|
return self._supportstatus.get(package)
|
|
|
|
def _load_group_file(self, fn):
|
|
output = None
|
|
unwanted = None
|
|
with open(fn, 'r') as fh:
|
|
self.logger.debug('reading %s', fn)
|
|
for groupname, group in yaml.safe_load(fh).items():
|
|
if groupname == 'OUTPUT':
|
|
output = group
|
|
continue
|
|
if groupname == 'UNWANTED':
|
|
unwanted = set(group)
|
|
continue
|
|
g = Group(groupname, self)
|
|
g.parse_yml(group)
|
|
return output, unwanted
|
|
|
|
def group_input_files(self):
|
|
return glob.glob(os.path.join(self.input_dir, 'group*.yml'))
|
|
|
|
def load_all_groups(self):
|
|
for fn in self.group_input_files():
|
|
o, u = self._load_group_file(fn)
|
|
if o:
|
|
if self.output is not None:
|
|
raise Exception('OUTPUT defined multiple times')
|
|
self.output = o
|
|
if u:
|
|
self.unwanted |= u
|
|
|
|
# required to generate release spec files (only)
|
|
def write_group_stubs(self):
|
|
archs = ['*'] + self.all_architectures
|
|
for name in self.groups:
|
|
group = self.groups[name]
|
|
group.solved_packages = dict()
|
|
fn = '{}.group'.format(group.name)
|
|
with open(os.path.join(self.output_dir, fn), 'w') as fh:
|
|
for arch in archs:
|
|
x = group.toxml(arch, group.ignore_broken, None)
|
|
x = ET.tostring(x, pretty_print=True, encoding='unicode')
|
|
fh.write(x)
|
|
|
|
def write_productcompose(self):
|
|
self._check_supplements()
|
|
summary = dict()
|
|
archs = ['*'] + self.all_architectures
|
|
# a single file covering all builds via multibuild flavors
|
|
with open(os.path.join(self.productcompose_dir, 'default.productcompose'), 'a') as opfh:
|
|
opfh.write(PRODUCT_COMPOSE_SEPERATOR_LINE + "\n")
|
|
for name in self.groups:
|
|
group = self.groups[name]
|
|
if not group.solved:
|
|
continue
|
|
opfh.write(f"# {group.comment}\n")
|
|
for arch in archs:
|
|
opfh.write(f"- name: {name}")
|
|
if arch != '*':
|
|
opfh.write(f"_{arch}")
|
|
opfh.write('\n')
|
|
if group.default_support_status and group.default_support_status != 'unsupported':
|
|
opfh.write(f" supportstatus: {group.default_support_status}\n")
|
|
opfh.write(' flavors:\n')
|
|
for f in group.flavors:
|
|
opfh.write(f" - {f}\n")
|
|
if arch == '*':
|
|
for included_arch in self.all_architectures:
|
|
opfh.write(' - ' + f + '_' + included_arch + '\n')
|
|
else:
|
|
opfh.write(' - ' + f + '_' + arch + '\n')
|
|
opfh.write(' architectures:\n')
|
|
opfh.write(' - ' + arch + '\n')
|
|
opfh.write(' packages:\n')
|
|
opfh.write(group.tocompose(' ', arch, group.ignore_broken))
|
|
# write main group including all sets
|
|
opfh.write("- name: main\n add:\n")
|
|
for name in self.groups:
|
|
group = self.groups[name]
|
|
if not group.solved:
|
|
continue
|
|
for f in group.flavors:
|
|
opfh.write(' - ' + f + '\n')
|
|
for included_arch in self.all_architectures:
|
|
opfh.write(' - ' + f + '_' + included_arch + '\n')
|
|
|
|
return summary
|
|
|
|
def write_all_groups(self):
|
|
self._check_supplements()
|
|
summary = dict()
|
|
archs = ['*'] + self.all_architectures
|
|
for name in self.groups:
|
|
group = self.groups[name]
|
|
if not group.solved:
|
|
continue
|
|
summary[name] = group.summary()
|
|
fn = '{}.group'.format(group.name)
|
|
with open(os.path.join(self.output_dir, fn), 'w') as fh:
|
|
comment = group.comment
|
|
for arch in archs:
|
|
x = group.toxml(arch, group.ignore_broken, comment)
|
|
# only comment first time
|
|
comment = None
|
|
x = ET.tostring(x, pretty_print=True, encoding='unicode')
|
|
x = re.sub(r'\s*<!-- reason:', ' <!-- reason:', x)
|
|
fh.write(x)
|
|
|
|
return summary
|
|
|
|
def solve_module(self, groupname, includes, excludes, use_recommends):
|
|
g = self.groups[groupname]
|
|
importants = set()
|
|
for i in includes:
|
|
name = i
|
|
if isinstance(i, dict):
|
|
name = list(i)[0]
|
|
if i[name] != 'support':
|
|
importants.add(name)
|
|
else:
|
|
importants.add(name)
|
|
g.inherit(self.groups[name])
|
|
g.solve(use_recommends)
|
|
for e in excludes:
|
|
g.ignore(self.groups[e])
|
|
for i in importants:
|
|
group = self.groups[i]
|
|
for arch in group.packages:
|
|
if arch not in g.solved_packages:
|
|
continue
|
|
for package in group.packages[arch]:
|
|
if package[0] in g.solved_packages[arch]:
|
|
continue
|
|
if package[0] not in g.solved_packages['*']:
|
|
self.logger.error(f'Missing {package[0]} in {groupname} for {arch}')
|
|
|
|
def expand_repos(self, project: str, repo='standard'):
|
|
return repository_path_expand(self.apiurl, project, repo)
|
|
|
|
def _check_supplements(self):
|
|
tocheck = set()
|
|
tocheck_locales = set()
|
|
for arch in self.filtered_architectures:
|
|
pool = self.prepare_pool(arch, True)
|
|
sel = pool.Selection()
|
|
for s in pool.solvables_iter():
|
|
sel.add_raw(solv.Job.SOLVER_SOLVABLE, s.id)
|
|
|
|
for s in sel.solvables():
|
|
for dep in s.lookup_deparray(solv.SOLVABLE_SUPPLEMENTS):
|
|
for d in dep.str().split(' '):
|
|
if d.startswith('namespace:modalias') or d.startswith('namespace:filesystem'):
|
|
tocheck.add(s.name)
|
|
|
|
for locale in self.locales:
|
|
id = pool.str2id('locale({})'.format(locale))
|
|
for s in pool.whatprovides(id):
|
|
tocheck_locales.add(s.name)
|
|
|
|
all_grouped = set()
|
|
for g in self.groups.values():
|
|
if g.solved:
|
|
for arch in g.solved_packages.keys():
|
|
if g.solved_packages[arch]:
|
|
all_grouped.update(g.solved_packages[arch])
|
|
|
|
for p in tocheck - all_grouped:
|
|
self.logger.warning('package %s has supplements but is not grouped', p)
|
|
|
|
for p in tocheck_locales - all_grouped:
|
|
self.logger.warning('package %s provides supported locale but is not grouped', p)
|
|
|
|
def prepare_pool(self, arch, ignore_conflicts):
|
|
pool = solv.Pool()
|
|
# the i586 DVD is really a i686 one
|
|
if arch == 'i586':
|
|
pool.setarch('i686')
|
|
else:
|
|
pool.setarch(arch)
|
|
|
|
self.lockjobs[arch] = []
|
|
solvables = set()
|
|
|
|
for project, reponame in self.repos:
|
|
repo = pool.add_repo(project)
|
|
# check back the repo state to avoid suprises
|
|
state = repository_arch_state(self.apiurl, project, reponame, arch)
|
|
if state is None:
|
|
continue
|
|
s = f'repo-{project}-{reponame}-{arch}-{state}.solv'
|
|
if not repo.add_solv(s):
|
|
raise MismatchedRepoException('failed to add repo {}/{}/{}'.format(project, reponame, arch))
|
|
for solvable in repo.solvables_iter():
|
|
if ignore_conflicts:
|
|
solvable.unset(solv.SOLVABLE_CONFLICTS)
|
|
solvable.unset(solv.SOLVABLE_OBSOLETES)
|
|
# only take the first solvable in the repo chain
|
|
if not self.use_newest_version and solvable.name in solvables:
|
|
self.lockjobs[arch].append(pool.Job(solv.Job.SOLVER_SOLVABLE | solv.Job.SOLVER_LOCK, solvable.id))
|
|
solvables.add(solvable.name)
|
|
|
|
pool.addfileprovides()
|
|
pool.createwhatprovides()
|
|
|
|
for locale in self.locales:
|
|
pool.set_namespaceproviders(solv.NAMESPACE_LANGUAGE, pool.Dep(locale), True)
|
|
|
|
return pool
|
|
|
|
# parse file and merge all groups
|
|
def _parse_unneeded(self, filename):
|
|
filename = os.path.join(self.input_dir, filename)
|
|
if not os.path.isfile(filename):
|
|
return set()
|
|
fh = open(filename, 'r')
|
|
self.logger.debug('reading %s', filename)
|
|
result = set()
|
|
for group in yaml.safe_load(fh).values():
|
|
result.update(group)
|
|
return result
|
|
|
|
# the unsorted group is special and will contain all the rest for
|
|
# the FTP tree. We filter it with unneeded though to create a
|
|
# unsorted.yml file for release manager review
|
|
def _collect_unsorted_packages(self, modules, unsorted):
|
|
unneeded_regexps = [re.compile(r'\A' + r + r'\Z')
|
|
for r in self._parse_unneeded('unneeded.yml')]
|
|
|
|
packages = dict()
|
|
if unsorted:
|
|
unsorted.solved_packages = dict()
|
|
unsorted.solved_packages['*'] = dict()
|
|
|
|
for arch in self.filtered_architectures:
|
|
pool = self.prepare_pool(arch, False)
|
|
pool.Selection()
|
|
archpacks = [s.name for s in pool.solvables_iter()]
|
|
|
|
# copy
|
|
filtered = list(archpacks)
|
|
for r in unneeded_regexps:
|
|
filtered = [p for p in filtered if not r.match(p)]
|
|
|
|
# convert to set
|
|
filtered = set(filtered) - self.unwanted
|
|
for g in modules:
|
|
if unsorted and g == unsorted:
|
|
continue
|
|
for a in ('*', arch):
|
|
filtered -= set(g.solved_packages[a])
|
|
for package in filtered:
|
|
packages.setdefault(package, []).append(arch)
|
|
|
|
if unsorted:
|
|
archpacks = set(archpacks)
|
|
unsorted.solved_packages[arch] = dict()
|
|
for g in modules:
|
|
archpacks -= set(g.solved_packages[arch])
|
|
archpacks -= set(g.solved_packages['*'])
|
|
unsorted.solved_packages[arch] = dict()
|
|
for p in archpacks:
|
|
unsorted.solved_packages[arch][p] = None
|
|
|
|
if unsorted:
|
|
common = None
|
|
for arch in self.filtered_architectures:
|
|
if common is None:
|
|
common = set(unsorted.solved_packages[arch])
|
|
continue
|
|
common &= set(unsorted.solved_packages[arch])
|
|
for p in common:
|
|
unsorted.solved_packages['*'][p] = None
|
|
for arch in self.filtered_architectures:
|
|
del unsorted.solved_packages[arch][p]
|
|
|
|
with open(os.path.join(self.output_dir, 'unsorted.yml'), 'w') as fh:
|
|
fh.write('unsorted:\n')
|
|
for p in sorted(packages):
|
|
fh.write(' - ')
|
|
fh.write(p)
|
|
if len(packages[p]) != len(self.filtered_architectures):
|
|
fh.write(': [')
|
|
fh.write(','.join(sorted(packages[p])))
|
|
fh.write(']')
|
|
reason = self._find_reason(p, modules)
|
|
if reason:
|
|
fh.write(' # ' + reason)
|
|
fh.write(' \n')
|
|
|
|
# give a hint if the package is related to a group
|
|
def _find_reason(self, package, modules):
|
|
# go through the modules multiple times to find the "best"
|
|
for g in modules:
|
|
if package in g.recommends:
|
|
return 'recommended by ' + g.recommends[package]
|
|
for g in modules:
|
|
if package in g.suggested:
|
|
return 'suggested by ' + g.suggested[package]
|
|
for g in modules:
|
|
if package in g.develpkgs:
|
|
return 'devel package of ' + g.develpkgs[package]
|
|
return None
|
|
|
|
def update_one_repo(self, project, repo, arch, solv_file, solv_file_hash):
|
|
# Either hash changed or new, so remove any old hash files.
|
|
file_utils.unlink_list(None, glob.glob(solv_file + '::*'))
|
|
|
|
d = os.path.join(CACHEDIR, project, repo, arch)
|
|
if not os.path.exists(d):
|
|
os.makedirs(d)
|
|
|
|
self.logger.debug('updating %s', d)
|
|
|
|
rm = RepoMirror(self.apiurl)
|
|
rm.mirror(d, project, repo, arch)
|
|
|
|
files = [os.path.join(d, f)
|
|
for f in os.listdir(d) if f.endswith('.rpm')]
|
|
suffix = f'.{os.getpid()}.tmp'
|
|
fh = open(solv_file + suffix, 'w')
|
|
p = subprocess.Popen(
|
|
['rpms2solv', '-m', '-', '-0'], stdin=subprocess.PIPE, stdout=fh)
|
|
p.communicate(bytes('\0'.join(files), 'utf-8'))
|
|
fh.close()
|
|
if p.wait() != 0:
|
|
raise Exception("rpm2solv failed")
|
|
os.rename(solv_file + suffix, solv_file)
|
|
|
|
# Create hash file now that solv creation is complete.
|
|
open(solv_file_hash, 'a').close()
|
|
|
|
def update_repos(self, architectures):
|
|
for project, repo in self.repos:
|
|
for arch in architectures:
|
|
# Fetch state before mirroring in-case it changes during download.
|
|
state = repository_arch_state(self.apiurl, project, repo, arch)
|
|
if state is None:
|
|
# Repo might not have this architecture
|
|
continue
|
|
|
|
repo_solv_name = 'repo-{}-{}-{}.solv'.format(project, repo, arch)
|
|
# Would be preferable to include hash in name, but cumbersome to handle without
|
|
# reworking a fair bit since the state needs to be tracked.
|
|
solv_file = os.path.join(CACHEDIR, repo_solv_name)
|
|
solv_file_hash = '{}::{}'.format(solv_file, state)
|
|
if os.path.exists(solv_file) and os.path.exists(solv_file_hash):
|
|
# Solve file exists and hash unchanged, skip updating solv.
|
|
self.logger.debug('skipping solv generation for {} due to matching state {}'.format(
|
|
'/'.join([project, repo, arch]), state))
|
|
else:
|
|
self.update_one_repo(project, repo, arch, solv_file, solv_file_hash)
|
|
shutil.copy(solv_file, f'./repo-{project}-{repo}-{arch}-{state}.solv')
|
|
|
|
def create_weakremovers(self, target, target_config, directory, output):
|
|
drops = dict()
|
|
dropped_repos = dict()
|
|
|
|
root = yaml.safe_load(open(os.path.join(directory, 'config.yml')))
|
|
for item in root:
|
|
key = list(item)[0]
|
|
# cast 15.1 to string :)
|
|
key = str(key)
|
|
|
|
oldrepos = set()
|
|
for suffix in ['xz', 'zst']:
|
|
oldrepos |= set(glob.glob(os.path.join(directory, f"{key}_*.packages.{suffix}")))
|
|
oldrepos |= set(glob.glob(os.path.join(directory, f"{key}.packages.{suffix}")))
|
|
for oldrepo in sorted(oldrepos):
|
|
pool = solv.Pool()
|
|
pool.setarch()
|
|
|
|
# we need some progress in the debug output - or gocd gets nervous
|
|
self.logger.debug('checking {}'.format(oldrepo))
|
|
oldsysrepo = file_utils.add_susetags(pool, oldrepo)
|
|
|
|
for arch in self.all_architectures:
|
|
for project, repo in self.repos:
|
|
# check back the repo state to avoid suprises
|
|
state = repository_arch_state(self.apiurl, project, repo, arch)
|
|
if state is None:
|
|
self.logger.debug(f'Skipping {project}/{repo}/{arch}')
|
|
fn = f'repo-{project}-{repo}-{arch}-{state}.solv'
|
|
r = pool.add_repo('/'.join([project, repo]))
|
|
if not r.add_solv(fn):
|
|
raise MismatchedRepoException('failed to add repo {}/{}/{}.'.format(project, repo, arch))
|
|
|
|
pool.createwhatprovides()
|
|
|
|
accepted_archs = set(self.all_architectures)
|
|
accepted_archs.add('noarch')
|
|
|
|
for s in oldsysrepo.solvables_iter():
|
|
oldarch = s.arch
|
|
if oldarch == 'i686':
|
|
oldarch = 'i586'
|
|
|
|
if oldarch not in accepted_archs:
|
|
continue
|
|
|
|
haveit = False
|
|
for s2 in pool.whatprovides(s.nameid):
|
|
if s2.repo == oldsysrepo or s.nameid != s2.nameid:
|
|
continue
|
|
newarch = s2.arch
|
|
if newarch == 'i686':
|
|
newarch = 'i586'
|
|
if oldarch != newarch and newarch != 'noarch' and oldarch != 'noarch':
|
|
continue
|
|
haveit = True
|
|
break
|
|
if haveit:
|
|
continue
|
|
|
|
# check for already obsoleted packages
|
|
nevr = pool.rel2id(s.nameid, s.evrid, solv.REL_EQ)
|
|
for s2 in pool.whatmatchesdep(solv.SOLVABLE_OBSOLETES, nevr):
|
|
if s2.repo == oldsysrepo:
|
|
continue
|
|
haveit = True
|
|
break
|
|
if haveit:
|
|
continue
|
|
if s.name not in drops:
|
|
drops[s.name] = {'repo': key, 'archs': set()}
|
|
if oldarch == 'noarch':
|
|
drops[s.name]['archs'] |= set(self.all_architectures)
|
|
else:
|
|
drops[s.name]['archs'].add(oldarch)
|
|
dropped_repos[key] = 1
|
|
|
|
del pool
|
|
|
|
for repo in sorted(dropped_repos):
|
|
repo_output = False
|
|
exclusives = dict()
|
|
for name in sorted(drops):
|
|
if drops[name]['repo'] != repo:
|
|
continue
|
|
if drops[name]['archs'] == set(self.all_architectures):
|
|
if not repo_output:
|
|
print('#', repo, file=output)
|
|
repo_output = True
|
|
print('Provides: weakremover({})'.format(name), file=output)
|
|
else:
|
|
jarch = ' '.join(sorted(drops[name]['archs']))
|
|
exclusives.setdefault(jarch, []).append(name)
|
|
|
|
for arch in sorted(exclusives):
|
|
if not repo_output:
|
|
print('#', repo, file=output)
|
|
repo_output = True
|
|
print('%ifarch {}'.format(arch), file=output)
|
|
for name in sorted(exclusives[arch]):
|
|
print('Provides: weakremover({})'.format(name), file=output)
|
|
print('%endif', file=output)
|
|
output.flush()
|
|
|
|
def solve_project(
|
|
self,
|
|
ignore_unresolvable=False,
|
|
ignore_recommended=False,
|
|
locale: Optional[str] = None,
|
|
locales_from: Optional[str] = None
|
|
):
|
|
self.load_all_groups()
|
|
if not self.output:
|
|
self.logger.error('OUTPUT not defined')
|
|
return
|
|
|
|
if ignore_unresolvable:
|
|
self.ignore_broken = True
|
|
global_use_recommends = not ignore_recommended
|
|
if locale:
|
|
self.locales |= set(locale.split(' '))
|
|
if locales_from:
|
|
with open(os.path.join(self.input_dir, locales_from), 'r') as fh:
|
|
root = ET.parse(fh).getroot()
|
|
self.locales |= set([lang.text for lang in root.findall('.//linguas/language')])
|
|
|
|
modules = []
|
|
# the yml parser makes an array out of everything, so
|
|
# we loop a bit more than what we support
|
|
for group in self.output:
|
|
groupname = list(group)[0]
|
|
settings = group[groupname]
|
|
if not settings: # e.g. unsorted
|
|
settings = {}
|
|
includes = settings.get('includes', [])
|
|
excludes = settings.get('excludes', [])
|
|
use_recommends = settings.get('recommends', global_use_recommends)
|
|
self.solve_module(groupname, includes, excludes, use_recommends)
|
|
g = self.groups[groupname]
|
|
g.flavors = [groupname] # main flavor of productcompose _multibuild flavor
|
|
# the default is a little double negated but Factory has ignore_broken
|
|
# as default and we only disable it for single groups (for now)
|
|
g.ignore_broken = not settings.get('require_all', not self.ignore_broken)
|
|
g.conflicts = settings.get('conflicts', [])
|
|
g.default_support_status = settings.get('default-support', 'unsupported')
|
|
modules.append(g)
|
|
|
|
# not defined for openSUSE
|
|
overlap = self.groups.get('overlap')
|
|
for module in modules:
|
|
module.check_dups(modules, overlap)
|
|
module.collect_devel_packages()
|
|
module.filter_already_selected(modules)
|
|
|
|
if overlap:
|
|
ignores = [x.name for x in overlap.ignored]
|
|
self.solve_module(overlap.name, [], ignores, use_recommends=False)
|
|
overlapped = set(overlap.solved_packages['*'])
|
|
for arch in self.filtered_architectures:
|
|
overlapped |= set(overlap.solved_packages[arch])
|
|
for module in modules:
|
|
if module.name == 'overlap' or module in overlap.ignored:
|
|
continue
|
|
for arch in ['*'] + self.filtered_architectures:
|
|
for p in overlapped:
|
|
module.solved_packages[arch].pop(p, None)
|
|
|
|
self._collect_unsorted_packages(modules, self.groups.get('unsorted'))
|
|
if not self.skip_productcompose:
|
|
self.write_productcompose()
|
|
|
|
def strip_medium_from_staging(self, path):
|
|
# staging projects don't need source and debug medium - and the glibc source
|
|
# rpm conflicts between standard and bootstrap_copy repository causing the
|
|
# product builder to fail
|
|
medium = re.compile('name="(DEBUG|SOURCE)MEDIUM"')
|
|
for name in glob.glob(os.path.join(path, '*.kiwi')):
|
|
lines = open(name).readlines()
|
|
lines = [x for x in lines if not medium.search(x)]
|
|
open(name, 'w').writelines(lines)
|
|
|
|
def build_stub(self, destination, extension):
|
|
with open(os.path.join(destination, '.'.join(['stub', extension])), 'w+') as f:
|
|
f.write('# prevent building single {} files twice\n'.format(extension))
|
|
f.write('Name: stub\n')
|
|
f.write('Version: 0.0\n')
|
|
|
|
def commit_package(self, path):
|
|
if self.dry_run:
|
|
package = Package(path)
|
|
for i in package.get_diff():
|
|
logging.info(''.join([d.decode('ascii') for d in i]))
|
|
else:
|
|
# No proper API function to perform the same operation.
|
|
logging.debug(subprocess.check_output(
|
|
['osc', 'addremove'], cwd=path, encoding='utf-8'))
|
|
package = Package(path)
|
|
package.commit(msg='Automatic update', skip_local_service_run=True)
|
|
|
|
def replace_product_version(self, product_file, product_version):
|
|
product_version = '<version>{}</version>'.format(product_version)
|
|
lines = open(product_file).readlines()
|
|
new_lines = []
|
|
for line in lines:
|
|
new_lines.append(line.replace('<version></version>', product_version))
|
|
open(product_file, 'w').write(''.join(new_lines))
|
|
|
|
def update_and_solve_target(
|
|
self,
|
|
api,
|
|
target_project: str,
|
|
target_config: Mapping[str, Any],
|
|
main_repo: str,
|
|
git_url: str,
|
|
project: str,
|
|
scope: str,
|
|
force: bool,
|
|
no_checkout: bool,
|
|
only_release_packages: bool,
|
|
stop_after_solve: bool,
|
|
custom_cache_tag
|
|
):
|
|
self.all_architectures = target_config.get('pkglistgen-archs').split(' ')
|
|
self.use_newest_version = str2bool(target_config.get('pkglistgen-use-newest-version', 'False'))
|
|
self.repos = self.expand_repos(project, main_repo)
|
|
logging.debug('[{}] {}/{}: update and solve'.format(scope, project, main_repo))
|
|
|
|
group = target_config.get('pkglistgen-group', '000package-groups')
|
|
product = target_config.get('pkglistgen-product', '000product')
|
|
productcompose = target_config.get('pkglistgen-product', '000productcompose')
|
|
release = target_config.get('pkglistgen-release', '000release-packages')
|
|
oldrepos = target_config.get('pkglistgen-repos', '000update-repos')
|
|
|
|
# Cache dir specific to hostname and project.
|
|
host = urlparse(api.apiurl).hostname
|
|
prefix_dir = 'pkglistgen'
|
|
if custom_cache_tag:
|
|
prefix_dir += '-' + custom_cache_tag
|
|
cache_dir = CacheManager.directory(prefix_dir, host, project)
|
|
|
|
drop_list = []
|
|
checkout_list = [group, product, release]
|
|
if not force:
|
|
root = ET.fromstringlist(show_results_meta(api.apiurl, project, product,
|
|
repository=[main_repo], multibuild=True))
|
|
if len(root.xpath('result[@state="building"]')) or len(root.xpath('result[@state="dirty"]')):
|
|
logging.info('{}/{} build in progress'.format(project, product))
|
|
return
|
|
if git_url:
|
|
if os.path.exists(cache_dir + "/.git"):
|
|
# reset and update existing clone
|
|
logging.debug(subprocess.check_output(
|
|
['git', 'reset', '--hard'], cwd=cache_dir, encoding='utf-8'))
|
|
logging.debug(subprocess.check_output(
|
|
['git', 'pull', '--rebase'], cwd=cache_dir, encoding='utf-8'))
|
|
else:
|
|
if os.path.exists(cache_dir):
|
|
shutil.rmtree(cache_dir)
|
|
logging.debug(subprocess.check_output(
|
|
['git', 'clone', '--recurse-submodules', git_url, cache_dir], encoding='utf-8'))
|
|
if os.path.exists(cache_dir + '/' + '000update-repos'):
|
|
logging.error('No support for 000update-repos in git projects atm')
|
|
return
|
|
else:
|
|
url = api.makeurl(['source', project])
|
|
packages = ET.parse(http_GET(url)).getroot()
|
|
if packages.find('entry[@name="{}"]'.format(product)) is None:
|
|
if not self.dry_run:
|
|
undelete_package(api.apiurl, project, product, 'revive')
|
|
# TODO disable build.
|
|
logging.info('{} undeleted, skip dvd until next cycle'.format(product))
|
|
return
|
|
|
|
drop_list = api.item_exists(project, oldrepos)
|
|
if drop_list and not only_release_packages:
|
|
checkout_list.append(oldrepos)
|
|
|
|
if packages.find('entry[@name="{}"]'.format(release)) is None:
|
|
if not self.dry_run:
|
|
undelete_package(api.apiurl, project, release, 'revive')
|
|
logging.info('{} undeleted, skip dvd until next cycle'.format(release))
|
|
return
|
|
|
|
if not no_checkout:
|
|
if os.path.exists(cache_dir):
|
|
shutil.rmtree(cache_dir)
|
|
os.makedirs(cache_dir)
|
|
|
|
group_dir = os.path.join(cache_dir, group)
|
|
product_dir = os.path.join(cache_dir, product)
|
|
self.productcompose_dir = os.path.join(cache_dir, productcompose)
|
|
release_dir = os.path.join(cache_dir, release)
|
|
oldrepos_dir = os.path.join(cache_dir, oldrepos)
|
|
|
|
self.input_dir = group_dir
|
|
self.output_dir = product_dir
|
|
|
|
if not no_checkout and not git_url:
|
|
logging.debug('Skipping checkout of {}'.format(project))
|
|
for package in checkout_list:
|
|
checkout_package(api.apiurl, project, package, expand_link=True,
|
|
prj_dir=cache_dir, outdir=os.path.join(cache_dir, package))
|
|
|
|
if not os.path.isdir(product_dir):
|
|
# otherwise just unset product_dir to skip all product-builder actions
|
|
product_dir = None
|
|
self.output_dir = self.productcompose_dir
|
|
|
|
if not only_release_packages:
|
|
file_utils.unlink_all_except(product_dir)
|
|
|
|
ignore_list = ['supportstatus.txt', 'summary-staging.txt', 'package-groups.changes',
|
|
'default.productcompose.in']
|
|
ignore_list += self.group_input_files()
|
|
|
|
# old product-builder
|
|
if product_dir:
|
|
file_utils.copy_directory_contents(group_dir, product_dir, ignore_list)
|
|
file_utils.change_extension(product_dir, '.spec.in', '.spec')
|
|
file_utils.change_extension(product_dir, '.product.in', '.product')
|
|
file_utils.unlink_all_except(release_dir, ['weakremovers.inc', '*.changes'])
|
|
|
|
# new product-composer
|
|
fn = os.path.join(group_dir, 'default.productcompose.in')
|
|
if os.path.isfile(fn):
|
|
if not os.path.isdir(self.productcompose_dir):
|
|
raise Exception('default.productcompose.in exists, but output directory is missing in git!')
|
|
lines = open(fn).readlines()
|
|
new_lines = []
|
|
for line in lines:
|
|
if line.startswith(PRODUCT_COMPOSE_SEPERATOR_LINE):
|
|
break
|
|
new_lines.append(line)
|
|
open(os.path.join(self.productcompose_dir, 'default.productcompose'), 'w').write(''.join(new_lines))
|
|
if git_url:
|
|
logging.debug(subprocess.check_output(
|
|
['git', 'add', self.productcompose_dir, f"{self.productcompose_dir}/default.productcompose"],
|
|
cwd=cache_dir, encoding='utf-8'))
|
|
if os.path.isfile(os.path.join(group_dir, 'supportstatus.txt')):
|
|
shutil.copy(os.path.join(group_dir, 'supportstatus.txt'), self.productcompose_dir)
|
|
self.skip_productcompose = False
|
|
else:
|
|
# No template file, so we don't create one either
|
|
self.skip_productcompose = True
|
|
|
|
logging.debug('-> do_update')
|
|
# make sure we only calculcate existant architectures
|
|
self.filter_architectures(target_archs(api.apiurl, project, main_repo))
|
|
self.update_repos(self.filtered_architectures)
|
|
|
|
if only_release_packages:
|
|
self.load_all_groups()
|
|
self.write_group_stubs()
|
|
else:
|
|
summary = self.solve_project(
|
|
ignore_unresolvable=str2bool(target_config.get('pkglistgen-ignore-unresolvable')),
|
|
ignore_recommended=str2bool(target_config.get('pkglistgen-ignore-recommended')),
|
|
locale=target_config.get('pkglistgen-locale'),
|
|
locales_from=target_config.get('pkglistgen-locales-from')
|
|
)
|
|
if product_dir:
|
|
self.write_all_groups()
|
|
|
|
if stop_after_solve:
|
|
return
|
|
|
|
if drop_list and not only_release_packages:
|
|
weakremovers_file = os.path.join(release_dir, 'weakremovers.inc')
|
|
try:
|
|
self.create_weakremovers(project, target_config, oldrepos_dir, output=open(weakremovers_file, 'w'))
|
|
except MismatchedRepoException:
|
|
logging.error("Failed to create weakremovers.inc due to mismatch in repos - project most likey started building again.")
|
|
return
|
|
|
|
if product_dir:
|
|
delete_products = target_config.get('pkglistgen-delete-products', '').split(' ')
|
|
file_utils.unlink_list(product_dir, delete_products)
|
|
|
|
logging.debug('-> product service')
|
|
product_version = attribute_value_load(api.apiurl, project, 'ProductVersion')
|
|
if not product_version:
|
|
# for stagings the product version doesn't matter (I hope)
|
|
product_version = '1'
|
|
for product_file in glob.glob(os.path.join(product_dir, '*.product')):
|
|
self.replace_product_version(product_file, product_version)
|
|
logging.debug(subprocess.check_output(
|
|
[PRODUCT_SERVICE, product_file, product_dir, project], encoding='utf-8'))
|
|
|
|
for delete_kiwi in target_config.get('pkglistgen-delete-kiwis-{}'.format(scope), '').split(' '):
|
|
delete_kiwis = glob.glob(os.path.join(product_dir, delete_kiwi))
|
|
file_utils.unlink_list(product_dir, delete_kiwis)
|
|
if scope == 'staging':
|
|
self.strip_medium_from_staging(product_dir)
|
|
|
|
spec_files = glob.glob(os.path.join(product_dir, '*.spec'))
|
|
file_utils.move_list(spec_files, release_dir)
|
|
inc_files = glob.glob(os.path.join(group_dir, '*.inc'))
|
|
# filter special inc file
|
|
inc_files = filter(lambda file: file.endswith('weakremovers.inc'), inc_files)
|
|
file_utils.move_list(inc_files, release_dir)
|
|
|
|
# do not overwrite weakremovers.inc if it exists
|
|
# we will commit there afterwards if needed
|
|
if os.path.exists(os.path.join(group_dir, 'weakremovers.inc')) and \
|
|
not os.path.exists(os.path.join(release_dir, 'weakremovers.inc')):
|
|
file_utils.move_list([os.path.join(group_dir, 'weakremovers.inc')], release_dir)
|
|
|
|
file_utils.multibuild_from_glob(release_dir, '*.spec')
|
|
self.build_stub(release_dir, 'spec')
|
|
|
|
if git_url:
|
|
logging.debug(subprocess.check_output(
|
|
'git add *.spec', cwd=release_dir, shell=True, encoding='utf-8'))
|
|
elif product_dir:
|
|
todo_spec_files = []
|
|
package = Package(release_dir)
|
|
if package.get_status(False, ' '):
|
|
todo_spec_files = glob.glob(os.path.join(release_dir, '*.spec'))
|
|
for spec_file in todo_spec_files:
|
|
changes_file = os.path.splitext(spec_file)[0] + '.changes'
|
|
with open(changes_file, 'w', encoding="utf-8") as f:
|
|
date = datetime.now(timezone.utc)
|
|
date = date.strftime("%a %b %d %H:%M:%S %Z %Y")
|
|
f.write(
|
|
"-------------------------------------------------------------------\n"
|
|
+ date + " - openSUSE <packaging@lists.opensuse.org>\n\n"
|
|
"- automatically generated by openSUSE-release-tools/pkglistgen\n\n"
|
|
)
|
|
|
|
self.commit_package(release_dir)
|
|
|
|
if only_release_packages:
|
|
return
|
|
|
|
if product_dir:
|
|
file_utils.multibuild_from_glob(product_dir, '*.kiwi')
|
|
self.build_stub(product_dir, 'kiwi')
|
|
|
|
reference_summary = os.path.join(group_dir, f'summary-{scope}.txt')
|
|
if os.path.isfile(reference_summary):
|
|
summary_file = os.path.join(self.output_dir, f'summary-{scope}.txt')
|
|
output = []
|
|
for group in summary:
|
|
for package in sorted(summary[group]):
|
|
output.append(f'{package}:{group}')
|
|
|
|
with open(summary_file, 'w') as f:
|
|
for line in sorted(output):
|
|
f.write(line + '\n')
|
|
|
|
if git_url:
|
|
if product_dir and os.path.isdir(product_dir):
|
|
logging.debug(subprocess.check_output(
|
|
['git', 'add', product_dir],
|
|
cwd=cache_dir, encoding='utf-8'))
|
|
if os.path.isdir(self.productcompose_dir):
|
|
logging.debug(subprocess.check_output(
|
|
['git', 'add', self.productcompose_dir],
|
|
cwd=cache_dir, encoding='utf-8'))
|
|
|
|
logging.debug(subprocess.check_output(
|
|
['git', 'commit', '-m', 'Update by pkglistgen of openSUSE-release-tool'], cwd=cache_dir, encoding='utf-8'))
|
|
if not self.dry_run:
|
|
logging.debug(subprocess.check_output(
|
|
['git', 'push'], cwd=cache_dir))
|
|
elif not self.dry_run:
|
|
self.commit_package(product_dir)
|
|
|
|
if os.path.isfile(reference_summary):
|
|
return self.comment.handle_package_diff(project, reference_summary, summary_file)
|