yudaike/openSUSE-release-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pkglistgen: Move droplist generation aside

Browse Source 
We will move this to a seperate step after the migration
This commit is contained in:
Stephan Kulow 2019-01-09 12:41:10 +01:00
parent 902627c14c
commit 0ec9983825
5 changed files with 222 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
pkglistgen/cli.py
View File

@ -14,6 +14,7 @@ from osclib.conf import Config
from osclib.stagingapi import StagingAPI
from pkglistgen import solv_utils
from pkglistgen.tool import PkgListGen
from pkglistgen.update_repo_handler import update_project
class CommandLineInterface(ToolBase.CommandLineInterface):
SCOPES = ['all', 'target', 'rings', 'staging']
@ -41,38 +42,15 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
"""
return self.tool.create_sle_weakremovers(target, prjs)
@cmdln.option('-o', '--output-dir', dest='output_dir', metavar='DIR', help='output directory', default='.')
def do_create_droplist(self, subcmd, opts, *oldsolv):
"""${cmd_name}: generate list of obsolete packages
def do_handle_update_repos(self, subcmd, opts, project):
"""${cmd_name}: Update 00update-repos
The globally specified repositories are taken as the current
package set. All solv files specified on the command line
are old versions of those repos.
The command outputs all package names that are no longer
contained in or provided by the current repos.
Reads config.yml from 00update-repos and will create required solv files
${cmd_usage}
${cmd_option_list}
"""
return self.tool.create_droplist(self.options.output_dir, oldsolv)
@cmdln.option('-o', '--output-dir', dest='output_dir', metavar='DIR', help='output directory', default='.')
@cmdln.option('--overwrite', action='store_true', help='overwrite if output file exists')
def do_dump_solv(self, subcmd, opts, baseurl):
"""${cmd_name}: fetch repomd and dump solv
Dumps solv from published repository. Use solve to generate from
pre-published repository.
If an output directory is specified, a file named according
to the build is created there. Otherwise the solv file is
dumped to stdout.
${cmd_usage}
${cmd_option_list}
"""
return solv_utils.dump_solv(baseurl=baseurl, output_dir=self.options.output_dir, overwrite=opts.overwrite)
return update_project(conf.config['apiurl'], project)
@cmdln.option('-f', '--force', action='store_true', help='continue even if build is in progress')
@cmdln.option('-p', '--project', help='target project')

5
pkglistgen/file_utils.py
View File

@ -5,6 +5,10 @@ import shutil
from lxml import etree as ET
def copy_list(file_list, destination):
for name in file_list:
shutil.copy(name, os.path.join(destination, os.path.basename(name)))
def move_list(file_list, destination):
for name in file_list:
os.rename(name, os.path.join(destination, os.path.basename(name)))
@ -49,4 +53,3 @@ def unlink_list(path, names):
if os.path.isfile(name_path):
os.unlink(name_path)

2
pkglistgen/solv_utils.py
View File

@ -216,7 +216,7 @@ def solv_cache_update(apiurl, cache_dir_solv, target_project, family_last, famil
return prior
def update_merge(self, nonfree, repos, architectures):
def update_merge(nonfree, repos, architectures):
"""Merge free and nonfree solv files or copy free to merged"""
for project, repo in repos:
for arch in architectures:

47
pkglistgen/tool.py
View File

@ -438,11 +438,11 @@ class PkgListGen(ToolBase.ToolBase):
print('Provides: weakremover({})'.format(name))
print('%endif')
# TODO: no longer used, needs to be migrated
def create_droplist(self, output_dir, oldsolv):
drops = dict()
for arch in self.architectures:
for arch in self.filtered_architectures:
for old in oldsolv:
@ -495,11 +495,6 @@ class PkgListGen(ToolBase.ToolBase):
print(' <obsoletepackage>%s</obsoletepackage>' % p, file=ofh)
def solve_project(self, ignore_unresolvable=False, ignore_recommended=False, locale=None, locales_from=None):
"""
Generates solv from pre-published repository contained in local cache.
Use dump_solv to extract solv from published repository.
"""
self.load_all_groups()
if not self.output:
self.logger.error('OUTPUT not defined')
@ -654,21 +649,6 @@ class PkgListGen(ToolBase.ToolBase):
self.filter_architectures(target_archs(api.apiurl, project, main_repo))
self.update_repos(self.filtered_architectures)
nonfree = target_config.get('nonfree')
if nonfree and drop_list:
print('-> do_update nonfree')
# Switch to nonfree repo (ugly, but that's how the code was setup).
repos_ = self.repos
self.repos = self.expand_repos(nonfree, main_repo)
self.update_repos(self.filtered_architectures)
# Switch repo back to main target project.
self.repos = repos_
print('-> update_merge')
solv_utils.update_merge(nonfree if drop_list else False, self.repos, self.architectures)
if only_release_packages:
self.load_all_groups()
self.write_group_stubs()
@ -681,25 +661,6 @@ class PkgListGen(ToolBase.ToolBase):
if stop_after_solve:
return
if drop_list:
# Ensure solv files from all releases in product family are updated.
print('-> solv_cache_update')
cache_dir_solv = CacheManager.directory('pkglistgen', 'solv')
family_last = target_config.get('pkglistgen-product-family-last')
family_include = target_config.get('pkglistgen-product-family-include')
solv_prior = solv_utils.solv_cache_update(api.apiurl, cache_dir_solv, target_project, family_last, family_include)
# Include pre-final release solv files for target project. These
# files will only exist from previous runs.
cache_dir_solv_current = os.path.join(cache_dir_solv, target_project)
solv_prior.update(glob.glob(os.path.join(cache_dir_solv_current, '*.merged.solv')))
for solv_file in solv_prior:
self.logger.debug(solv_file.replace(cache_dir_solv, ''))
print('-> do_create_droplist')
# Reset to product after solv_cache_update().
self.create_droplist(product_dir, *solv_prior)
delete_products = target_config.get('pkglistgen-delete-products', '').split(' ')
file_utils.unlink_list(product_dir, delete_products)
@ -715,9 +676,9 @@ class PkgListGen(ToolBase.ToolBase):
self.strip_medium_from_staging(product_dir)
spec_files = glob.glob(os.path.join(product_dir, '*.spec'))
file_utils.move_list(spec_files, release_dir)
file_utils.copy_list(spec_files, release_dir)
inc_files = glob.glob(os.path.join(group_dir, '*.inc'))
file_utils.move_list(inc_files, release_dir)
file_utils.copy_list(inc_files, release_dir)
file_utils.multibuild_from_glob(release_dir, '*.spec')
self.build_stub(release_dir, 'spec')

208
pkglistgen/update_repo_handler.py Normal file
View File

@ -0,0 +1,208 @@
from __future__ import print_function
import filecmp
import glob
import gzip
import hashlib
import io
import logging
import os.path
import re
import random
import string
import subprocess
import sys
import shutil
import tempfile
from lxml import etree as ET
from osc import conf
import osc.core
from osclib.util import project_list_family
from osclib.util import project_list_family_prior
from osclib.conf import Config
from osclib.cache_manager import CacheManager
import requests
import solv
import yaml
# share header cache with repochecker
CACHEDIR = CacheManager.directory('repository-meta')
try:
from urllib.parse import urljoin
except ImportError:
# python 2.x
from urlparse import urljoin
logger = logging.getLogger()
def dump_solv_build(baseurl):
"""Determine repo format and build string from remote repository."""
if not baseurl.endswith('/'):
baseurl += '/'
buildre = re.compile('.*-Build(.*)')
url = urljoin(baseurl, 'media.1/media')
with requests.get(url) as media:
for i, line in enumerate(media.iter_lines()):
if i != 1:
continue
build = buildre.match(line)
if build:
return build.group(1)
url = urljoin(baseurl, 'media.1/build')
with requests.get(url) as build:
name = build.content.strip()
build = buildre.match(name)
if build:
return build.group(1)
url = urljoin(baseurl, 'repodata/repomd.xml')
with requests.get(url) as media:
root = ET.parse(url)
rev = root.find('.//{http://linux.duke.edu/metadata/repo}revision')
if rev is not None:
return rev.text
raise Exception(baseurl + 'includes no build number')
def dump_solv(baseurl, output_dir):
name = None
ofh = sys.stdout
if output_dir:
build = dump_solv_build(baseurl)
name = os.path.join(output_dir, '{}.solv'.format(build))
pool = solv.Pool()
pool.setarch()
repo = pool.add_repo(''.join(random.choice(string.letters) for _ in range(5)))
url = urljoin(baseurl, 'repodata/repomd.xml')
repomd = requests.get(url)
ns = {'r': 'http://linux.duke.edu/metadata/repo'}
root = ET.fromstring(repomd.content)
print(url, root)
primary_element = root.find('.//r:data[@type="primary"]', ns)
location = primary_element.find('r:location', ns).get('href')
sha256_expected = primary_element.find('r:checksum[@type="sha256"]', ns).text
path_prefix = 'TODO'
f = tempfile.TemporaryFile()
f.write(repomd.content)
f.flush()
os.lseek(f.fileno(), 0, os.SEEK_SET)
repo.add_repomdxml(f, 0)
url = urljoin(baseurl, path_prefix + location)
with requests.get(url, stream=True) as primary:
sha256 = hashlib.sha256(primary.content).hexdigest()
if sha256 != sha256_expected:
raise Exception('checksums do not match {} != {}'.format(sha256, sha256_expected))
content = gzip.GzipFile(fileobj=io.BytesIO(primary.content))
os.lseek(f.fileno(), 0, os.SEEK_SET)
f.write(content.read())
f.flush()
os.lseek(f.fileno(), 0, os.SEEK_SET)
repo.add_rpmmd(f, None, 0)
repo.create_stubs()
ofh = open(name + '.new', 'w')
repo.write(ofh)
if name is not None:
# Only update file if overwrite or different.
ofh.flush() # Ensure entirely written before comparing.
os.rename(name + '.new', name)
return name
def solv_cache_update(apiurl, cache_dir_solv, target_project, family_last, family_include):
"""Dump solv files (do_dump_solv) for all products in family."""
prior = set()
project_family = project_list_family_prior(
apiurl, target_project, include_self=True, last=family_last)
if family_include:
# Include projects from a different family if desired.
project_family.extend(project_list_family(apiurl, family_include))
for project in project_family:
Config(apiurl, project)
project_config = conf.config[project]
baseurl = project_config.get('download-baseurl')
if not baseurl:
baseurl = project_config.get('download-baseurl-' + project.replace(':', '-'))
baseurl_update = project_config.get('download-baseurl-update')
print(project, baseurl, baseurl_update)
continue
if not baseurl:
logger.warning('no baseurl configured for {}'.format(project))
continue
urls = [urljoin(baseurl, 'repo/oss/')]
if baseurl_update:
urls.append(urljoin(baseurl_update, 'oss/'))
if project_config.get('nonfree'):
urls.append(urljoin(baseurl, 'repo/non-oss/'))
if baseurl_update:
urls.append(urljoin(baseurl_update, 'non-oss/'))
names = []
for url in urls:
project_display = project
if 'update' in url:
project_display += ':Update'
print('-> dump_solv for {}/{}'.format(
project_display, os.path.basename(os.path.normpath(url))))
logger.debug(url)
output_dir = os.path.join(cache_dir_solv, project)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
solv_name = dump_solv(baseurl=url, output_dir=output_dir, overwrite=False)
if solv_name:
names.append(solv_name)
if not len(names):
logger.warning('no solv files were dumped for {}'.format(project))
continue
print(prior)
return prior
def update_merge(nonfree, repos, architectures):
"""Merge free and nonfree solv files or copy free to merged"""
for project, repo in repos:
for arch in architectures:
solv_file = os.path.join(
CACHEDIR, 'repo-{}-{}-{}.solv'.format(project, repo, arch))
solv_file_merged = os.path.join(
CACHEDIR, 'repo-{}-{}-{}.merged.solv'.format(project, repo, arch))
if not nonfree:
shutil.copyfile(solv_file, solv_file_merged)
continue
solv_file_nonfree = os.path.join(
CACHEDIR, 'repo-{}-{}-{}.solv'.format(nonfree, repo, arch))
def fetch_item(key, opts):
ret = dump_solv(opts['url'], '/tmp')
print(key, opts, ret)
def update_project(apiurl, project):
url = osc.core.makeurl(apiurl, ['source', project, '00update-repos', 'config.yml'])
root = yaml.safe_load(osc.core.http_GET(url))
for item in root:
key = item.keys()[0]
fetch_item(key, item[key])