From 0ec9983825ebd43d47ddd18334a63746c47e2f24 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Wed, 9 Jan 2019 12:41:10 +0100 Subject: [PATCH] pkglistgen: Move droplist generation aside We will move this to a seperate step after the migration --- pkglistgen/cli.py | 32 +---- pkglistgen/file_utils.py | 5 +- pkglistgen/solv_utils.py | 2 +- pkglistgen/tool.py | 47 +------ pkglistgen/update_repo_handler.py | 208 ++++++++++++++++++++++++++++++ 5 files changed, 222 insertions(+), 72 deletions(-) create mode 100644 pkglistgen/update_repo_handler.py diff --git a/pkglistgen/cli.py b/pkglistgen/cli.py index cead099c..27ed79fc 100755 --- a/pkglistgen/cli.py +++ b/pkglistgen/cli.py @@ -14,6 +14,7 @@ from osclib.conf import Config from osclib.stagingapi import StagingAPI from pkglistgen import solv_utils from pkglistgen.tool import PkgListGen +from pkglistgen.update_repo_handler import update_project class CommandLineInterface(ToolBase.CommandLineInterface): SCOPES = ['all', 'target', 'rings', 'staging'] @@ -41,38 +42,15 @@ class CommandLineInterface(ToolBase.CommandLineInterface): """ return self.tool.create_sle_weakremovers(target, prjs) - @cmdln.option('-o', '--output-dir', dest='output_dir', metavar='DIR', help='output directory', default='.') - def do_create_droplist(self, subcmd, opts, *oldsolv): - """${cmd_name}: generate list of obsolete packages + def do_handle_update_repos(self, subcmd, opts, project): + """${cmd_name}: Update 00update-repos - The globally specified repositories are taken as the current - package set. All solv files specified on the command line - are old versions of those repos. - - The command outputs all package names that are no longer - contained in or provided by the current repos. + Reads config.yml from 00update-repos and will create required solv files ${cmd_usage} ${cmd_option_list} """ - return self.tool.create_droplist(self.options.output_dir, oldsolv) - - @cmdln.option('-o', '--output-dir', dest='output_dir', metavar='DIR', help='output directory', default='.') - @cmdln.option('--overwrite', action='store_true', help='overwrite if output file exists') - def do_dump_solv(self, subcmd, opts, baseurl): - """${cmd_name}: fetch repomd and dump solv - - Dumps solv from published repository. Use solve to generate from - pre-published repository. - - If an output directory is specified, a file named according - to the build is created there. Otherwise the solv file is - dumped to stdout. - - ${cmd_usage} - ${cmd_option_list} - """ - return solv_utils.dump_solv(baseurl=baseurl, output_dir=self.options.output_dir, overwrite=opts.overwrite) + return update_project(conf.config['apiurl'], project) @cmdln.option('-f', '--force', action='store_true', help='continue even if build is in progress') @cmdln.option('-p', '--project', help='target project') diff --git a/pkglistgen/file_utils.py b/pkglistgen/file_utils.py index ed043043..7213f810 100644 --- a/pkglistgen/file_utils.py +++ b/pkglistgen/file_utils.py @@ -5,6 +5,10 @@ import shutil from lxml import etree as ET +def copy_list(file_list, destination): + for name in file_list: + shutil.copy(name, os.path.join(destination, os.path.basename(name))) + def move_list(file_list, destination): for name in file_list: os.rename(name, os.path.join(destination, os.path.basename(name))) @@ -49,4 +53,3 @@ def unlink_list(path, names): if os.path.isfile(name_path): os.unlink(name_path) - diff --git a/pkglistgen/solv_utils.py b/pkglistgen/solv_utils.py index 2f88502b..e21131f1 100644 --- a/pkglistgen/solv_utils.py +++ b/pkglistgen/solv_utils.py @@ -216,7 +216,7 @@ def solv_cache_update(apiurl, cache_dir_solv, target_project, family_last, famil return prior -def update_merge(self, nonfree, repos, architectures): +def update_merge(nonfree, repos, architectures): """Merge free and nonfree solv files or copy free to merged""" for project, repo in repos: for arch in architectures: diff --git a/pkglistgen/tool.py b/pkglistgen/tool.py index 79020c83..85ba6235 100644 --- a/pkglistgen/tool.py +++ b/pkglistgen/tool.py @@ -438,11 +438,11 @@ class PkgListGen(ToolBase.ToolBase): print('Provides: weakremover({})'.format(name)) print('%endif') - + # TODO: no longer used, needs to be migrated def create_droplist(self, output_dir, oldsolv): drops = dict() - for arch in self.architectures: + for arch in self.filtered_architectures: for old in oldsolv: @@ -495,11 +495,6 @@ class PkgListGen(ToolBase.ToolBase): print(' %s' % p, file=ofh) def solve_project(self, ignore_unresolvable=False, ignore_recommended=False, locale=None, locales_from=None): - """ - Generates solv from pre-published repository contained in local cache. - Use dump_solv to extract solv from published repository. - """ - self.load_all_groups() if not self.output: self.logger.error('OUTPUT not defined') @@ -654,21 +649,6 @@ class PkgListGen(ToolBase.ToolBase): self.filter_architectures(target_archs(api.apiurl, project, main_repo)) self.update_repos(self.filtered_architectures) - nonfree = target_config.get('nonfree') - if nonfree and drop_list: - print('-> do_update nonfree') - - # Switch to nonfree repo (ugly, but that's how the code was setup). - repos_ = self.repos - self.repos = self.expand_repos(nonfree, main_repo) - self.update_repos(self.filtered_architectures) - - # Switch repo back to main target project. - self.repos = repos_ - - print('-> update_merge') - solv_utils.update_merge(nonfree if drop_list else False, self.repos, self.architectures) - if only_release_packages: self.load_all_groups() self.write_group_stubs() @@ -681,25 +661,6 @@ class PkgListGen(ToolBase.ToolBase): if stop_after_solve: return - if drop_list: - # Ensure solv files from all releases in product family are updated. - print('-> solv_cache_update') - cache_dir_solv = CacheManager.directory('pkglistgen', 'solv') - family_last = target_config.get('pkglistgen-product-family-last') - family_include = target_config.get('pkglistgen-product-family-include') - solv_prior = solv_utils.solv_cache_update(api.apiurl, cache_dir_solv, target_project, family_last, family_include) - - # Include pre-final release solv files for target project. These - # files will only exist from previous runs. - cache_dir_solv_current = os.path.join(cache_dir_solv, target_project) - solv_prior.update(glob.glob(os.path.join(cache_dir_solv_current, '*.merged.solv'))) - for solv_file in solv_prior: - self.logger.debug(solv_file.replace(cache_dir_solv, '')) - - print('-> do_create_droplist') - # Reset to product after solv_cache_update(). - self.create_droplist(product_dir, *solv_prior) - delete_products = target_config.get('pkglistgen-delete-products', '').split(' ') file_utils.unlink_list(product_dir, delete_products) @@ -715,9 +676,9 @@ class PkgListGen(ToolBase.ToolBase): self.strip_medium_from_staging(product_dir) spec_files = glob.glob(os.path.join(product_dir, '*.spec')) - file_utils.move_list(spec_files, release_dir) + file_utils.copy_list(spec_files, release_dir) inc_files = glob.glob(os.path.join(group_dir, '*.inc')) - file_utils.move_list(inc_files, release_dir) + file_utils.copy_list(inc_files, release_dir) file_utils.multibuild_from_glob(release_dir, '*.spec') self.build_stub(release_dir, 'spec') diff --git a/pkglistgen/update_repo_handler.py b/pkglistgen/update_repo_handler.py new file mode 100644 index 00000000..ecc25e0e --- /dev/null +++ b/pkglistgen/update_repo_handler.py @@ -0,0 +1,208 @@ +from __future__ import print_function + +import filecmp +import glob +import gzip +import hashlib +import io +import logging +import os.path +import re +import random +import string +import subprocess +import sys +import shutil +import tempfile + +from lxml import etree as ET + +from osc import conf +import osc.core +from osclib.util import project_list_family +from osclib.util import project_list_family_prior +from osclib.conf import Config +from osclib.cache_manager import CacheManager + +import requests + +import solv + +import yaml + +# share header cache with repochecker +CACHEDIR = CacheManager.directory('repository-meta') + +try: + from urllib.parse import urljoin +except ImportError: + # python 2.x + from urlparse import urljoin + +logger = logging.getLogger() + +def dump_solv_build(baseurl): + """Determine repo format and build string from remote repository.""" + + if not baseurl.endswith('/'): + baseurl += '/' + + buildre = re.compile('.*-Build(.*)') + url = urljoin(baseurl, 'media.1/media') + with requests.get(url) as media: + for i, line in enumerate(media.iter_lines()): + if i != 1: + continue + build = buildre.match(line) + if build: + return build.group(1) + + url = urljoin(baseurl, 'media.1/build') + with requests.get(url) as build: + name = build.content.strip() + build = buildre.match(name) + if build: + return build.group(1) + + url = urljoin(baseurl, 'repodata/repomd.xml') + with requests.get(url) as media: + root = ET.parse(url) + rev = root.find('.//{http://linux.duke.edu/metadata/repo}revision') + if rev is not None: + return rev.text + + raise Exception(baseurl + 'includes no build number') + +def dump_solv(baseurl, output_dir): + name = None + ofh = sys.stdout + if output_dir: + build = dump_solv_build(baseurl) + name = os.path.join(output_dir, '{}.solv'.format(build)) + + pool = solv.Pool() + pool.setarch() + + repo = pool.add_repo(''.join(random.choice(string.letters) for _ in range(5))) + url = urljoin(baseurl, 'repodata/repomd.xml') + repomd = requests.get(url) + ns = {'r': 'http://linux.duke.edu/metadata/repo'} + root = ET.fromstring(repomd.content) + print(url, root) + primary_element = root.find('.//r:data[@type="primary"]', ns) + location = primary_element.find('r:location', ns).get('href') + sha256_expected = primary_element.find('r:checksum[@type="sha256"]', ns).text + + path_prefix = 'TODO' + f = tempfile.TemporaryFile() + f.write(repomd.content) + f.flush() + os.lseek(f.fileno(), 0, os.SEEK_SET) + repo.add_repomdxml(f, 0) + url = urljoin(baseurl, path_prefix + location) + with requests.get(url, stream=True) as primary: + sha256 = hashlib.sha256(primary.content).hexdigest() + if sha256 != sha256_expected: + raise Exception('checksums do not match {} != {}'.format(sha256, sha256_expected)) + + content = gzip.GzipFile(fileobj=io.BytesIO(primary.content)) + os.lseek(f.fileno(), 0, os.SEEK_SET) + f.write(content.read()) + f.flush() + os.lseek(f.fileno(), 0, os.SEEK_SET) + repo.add_rpmmd(f, None, 0) + repo.create_stubs() + + ofh = open(name + '.new', 'w') + repo.write(ofh) + + if name is not None: + # Only update file if overwrite or different. + ofh.flush() # Ensure entirely written before comparing. + os.rename(name + '.new', name) + return name + +def solv_cache_update(apiurl, cache_dir_solv, target_project, family_last, family_include): + """Dump solv files (do_dump_solv) for all products in family.""" + prior = set() + + project_family = project_list_family_prior( + apiurl, target_project, include_self=True, last=family_last) + if family_include: + # Include projects from a different family if desired. + project_family.extend(project_list_family(apiurl, family_include)) + + for project in project_family: + Config(apiurl, project) + project_config = conf.config[project] + + baseurl = project_config.get('download-baseurl') + if not baseurl: + baseurl = project_config.get('download-baseurl-' + project.replace(':', '-')) + baseurl_update = project_config.get('download-baseurl-update') + print(project, baseurl, baseurl_update) + continue + + if not baseurl: + logger.warning('no baseurl configured for {}'.format(project)) + continue + + urls = [urljoin(baseurl, 'repo/oss/')] + if baseurl_update: + urls.append(urljoin(baseurl_update, 'oss/')) + if project_config.get('nonfree'): + urls.append(urljoin(baseurl, 'repo/non-oss/')) + if baseurl_update: + urls.append(urljoin(baseurl_update, 'non-oss/')) + + names = [] + for url in urls: + project_display = project + if 'update' in url: + project_display += ':Update' + print('-> dump_solv for {}/{}'.format( + project_display, os.path.basename(os.path.normpath(url)))) + logger.debug(url) + + output_dir = os.path.join(cache_dir_solv, project) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + solv_name = dump_solv(baseurl=url, output_dir=output_dir, overwrite=False) + if solv_name: + names.append(solv_name) + + if not len(names): + logger.warning('no solv files were dumped for {}'.format(project)) + continue + + print(prior) + return prior + + +def update_merge(nonfree, repos, architectures): + """Merge free and nonfree solv files or copy free to merged""" + for project, repo in repos: + for arch in architectures: + solv_file = os.path.join( + CACHEDIR, 'repo-{}-{}-{}.solv'.format(project, repo, arch)) + solv_file_merged = os.path.join( + CACHEDIR, 'repo-{}-{}-{}.merged.solv'.format(project, repo, arch)) + + if not nonfree: + shutil.copyfile(solv_file, solv_file_merged) + continue + + solv_file_nonfree = os.path.join( + CACHEDIR, 'repo-{}-{}-{}.solv'.format(nonfree, repo, arch)) + +def fetch_item(key, opts): + ret = dump_solv(opts['url'], '/tmp') + print(key, opts, ret) + +def update_project(apiurl, project): + url = osc.core.makeurl(apiurl, ['source', project, '00update-repos', 'config.yml']) + root = yaml.safe_load(osc.core.http_GET(url)) + for item in root: + key = item.keys()[0] + fetch_item(key, item[key])