From c4862cd75a8be379de5268d8c0aa51a061dba314 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Sat, 24 Nov 2018 19:49:10 +0100 Subject: [PATCH 01/11] pkglistgen: Add --staging option for easier scoping To run on gitlab worker, it's easier to just use the project name as argument, as this is also what we pass to other jobs --- pkglistgen.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkglistgen.py b/pkglistgen.py index 4b81e054..f286469d 100755 --- a/pkglistgen.py +++ b/pkglistgen.py @@ -1154,6 +1154,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface): @cmdln.option('-s', '--scope', action='append', default=['all'], help='scope on which to operate ({}, staging:$letter)'.format(', '.join(SCOPES))) @cmdln.option('--no-checkout', action='store_true', help='reuse checkout in cache') @cmdln.option('--stop-after-solve', action='store_true', help='only create group files') + @cmdln.option('--staging', help='Only solve that one staging') def do_update_and_solve(self, subcmd, opts): """${cmd_name}: update and solve for given scope @@ -1163,6 +1164,11 @@ class CommandLineInterface(ToolBase.CommandLineInterface): self.error_occured = False + if opts.staging: + match = re.match('(.*):Staging:(.*)', opts.staging) + opts.scope = ['staging:' + match.group(2)] + opts.project = match.group(1) + if not opts.project: raise ValueError('project is required') opts.staging_project = None From a989b7c8ce0fc0a648fde60ba1156681dc947e9a Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Sun, 25 Nov 2018 18:47:33 +0100 Subject: [PATCH 02/11] Set OBS_NAME ourselves on api.suse.de (the default is correct for api.opensuse.org) --- pkglistgen.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkglistgen.py b/pkglistgen.py index f286469d..21e0e944 100755 --- a/pkglistgen.py +++ b/pkglistgen.py @@ -1197,6 +1197,10 @@ class CommandLineInterface(ToolBase.CommandLineInterface): config = Config(apiurl, target_project) api = StagingAPI(apiurl, target_project) + if apiurl.find('suse.de') > 0: + # used by product converter + os.environ['OBS_NAME'] = 'build.suse.de' + target_config = conf.config[target_project] if opts.scope == 'ports': archs_key = 'pkglistgen-archs-ports' From ac7c2b50729f9ea0630557b61305763889f4f3ca Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Mon, 26 Nov 2018 14:28:52 +0100 Subject: [PATCH 03/11] Remove skip_release option This was used for ring2 support - which we don't do anymore --- pkglistgen.py | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/pkglistgen.py b/pkglistgen.py index 21e0e944..a0bbc808 100755 --- a/pkglistgen.py +++ b/pkglistgen.py @@ -1258,7 +1258,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface): self.error_occured = True def update_and_solve_target(self, api, target_project, target_config, main_repo, opts, - skip_release=False, drop_list=False): + drop_list=False): print('[{}] {}/{}: update and solve'.format(opts.scope, opts.project, main_repo)) group = target_config.get('pkglistgen-group', '000package-groups') @@ -1286,15 +1286,13 @@ class CommandLineInterface(ToolBase.CommandLineInterface): print('{}/{} build in progress'.format(opts.project, product)) return - checkout_list = [group, product] - if not skip_release: - checkout_list.append(release) + checkout_list = [group, product, release] - if packages.find('entry[@name="{}"]'.format(release)) is None: - if not self.options.dry: - undelete_package(api.apiurl, opts.project, release, 'revive') - print('{} undeleted, skip dvd until next cycle'.format(release)) - return + if packages.find('entry[@name="{}"]'.format(release)) is None: + if not self.options.dry: + undelete_package(api.apiurl, opts.project, release, 'revive') + print('{} undeleted, skip dvd until next cycle'.format(release)) + return # Cache dir specific to hostname and project. host = urlparse(api.apiurl).hostname @@ -1315,8 +1313,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface): continue checkout_package(api.apiurl, opts.project, package, expand_link=True, prj_dir=cache_dir) - if not skip_release: - self.unlink_all_except(release_dir) + self.unlink_all_except(release_dir) self.unlink_all_except(product_dir) self.copy_directory_contents(group_dir, product_dir, ['supportstatus.txt', 'groups.yml', 'package-groups.changes']) @@ -1393,21 +1390,17 @@ class CommandLineInterface(ToolBase.CommandLineInterface): self.strip_medium_from_staging(product_dir) spec_files = glob.glob(os.path.join(product_dir, '*.spec')) - if skip_release: - self.tool.unlink_list(None, spec_files) - else: - self.move_list(spec_files, release_dir) - inc_files = glob.glob(os.path.join(group_dir, '*.inc')) - self.move_list(inc_files, release_dir) + self.move_list(spec_files, release_dir) + inc_files = glob.glob(os.path.join(group_dir, '*.inc')) + self.move_list(inc_files, release_dir) self.multibuild_from_glob(product_dir, '*.kiwi') self.build_stub(product_dir, 'kiwi') self.commit_package(product_dir) - if not skip_release: - self.multibuild_from_glob(release_dir, '*.spec') - self.build_stub(release_dir, 'spec') - self.commit_package(release_dir) + self.multibuild_from_glob(release_dir, '*.spec') + self.build_stub(release_dir, 'spec') + self.commit_package(release_dir) if api.item_exists(opts.project, '000product-summary'): summary_str = "# Summary of packages in groups" From 5b04108f81147ffda58728bfafc4b10ffa0af351 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Mon, 26 Nov 2018 14:26:40 +0100 Subject: [PATCH 04/11] Add option to only generate release packages We want to do this before the packages build --- pkglistgen.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pkglistgen.py b/pkglistgen.py index a0bbc808..1be522b0 100755 --- a/pkglistgen.py +++ b/pkglistgen.py @@ -1155,6 +1155,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface): @cmdln.option('--no-checkout', action='store_true', help='reuse checkout in cache') @cmdln.option('--stop-after-solve', action='store_true', help='only create group files') @cmdln.option('--staging', help='Only solve that one staging') + @cmdln.option('--only-release-packages', action='store_true', help='Generate 000release-packages only') def do_update_and_solve(self, subcmd, opts): """${cmd_name}: update and solve for given scope @@ -1314,7 +1315,8 @@ class CommandLineInterface(ToolBase.CommandLineInterface): checkout_package(api.apiurl, opts.project, package, expand_link=True, prj_dir=cache_dir) self.unlink_all_except(release_dir) - self.unlink_all_except(product_dir) + if not opts.only_release_packages: + self.unlink_all_except(product_dir) self.copy_directory_contents(group_dir, product_dir, ['supportstatus.txt', 'groups.yml', 'package-groups.changes']) self.change_extension(product_dir, '.spec.in', '.spec') @@ -1350,7 +1352,8 @@ class CommandLineInterface(ToolBase.CommandLineInterface): opts.include_suggested = str2bool(target_config.get('pkglistgen-include-suggested')) opts.locale = target_config.get('pkglistgen-local') opts.locales_from = target_config.get('pkglistgen-locales-from') - summary = self.do_solve('solve', opts) + if not opts.only_release_packages: + summary = self.do_solve('solve', opts) if opts.stop_after_solve: return @@ -1394,14 +1397,17 @@ class CommandLineInterface(ToolBase.CommandLineInterface): inc_files = glob.glob(os.path.join(group_dir, '*.inc')) self.move_list(inc_files, release_dir) - self.multibuild_from_glob(product_dir, '*.kiwi') - self.build_stub(product_dir, 'kiwi') - self.commit_package(product_dir) - self.multibuild_from_glob(release_dir, '*.spec') self.build_stub(release_dir, 'spec') self.commit_package(release_dir) + if opts.only_release_packages: + return + + self.multibuild_from_glob(product_dir, '*.kiwi') + self.build_stub(product_dir, 'kiwi') + self.commit_package(product_dir) + if api.item_exists(opts.project, '000product-summary'): summary_str = "# Summary of packages in groups" for group in sorted(summary.keys()): From 9273bd42a370bf011438d501348ebac5c6ab7790 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Wed, 28 Nov 2018 07:53:38 +0100 Subject: [PATCH 05/11] Delete kiwi by glob openSUSE only has 2 to remove, but SLE has a whole list of e.g. s390 --- pkglistgen.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkglistgen.py b/pkglistgen.py index 1be522b0..83a51b42 100755 --- a/pkglistgen.py +++ b/pkglistgen.py @@ -1387,8 +1387,9 @@ class CommandLineInterface(ToolBase.CommandLineInterface): print(subprocess.check_output( [PRODUCT_SERVICE, product_file, product_dir, opts.project])) - delete_kiwis = target_config.get('pkglistgen-delete-kiwis-{}'.format(opts.scope), '').split(' ') - self.tool.unlink_list(product_dir, delete_kiwis) + for delete_kiwi in target_config.get('pkglistgen-delete-kiwis-{}'.format(opts.scope), '').split(' '): + delete_kiwis = glob.glob(os.path.join(product_dir, delete_kiwi)) + self.tool.unlink_list(product_dir, delete_kiwis) if opts.scope == 'staging': self.strip_medium_from_staging(product_dir) From c2e68c2f9a26d94fffde842895f84177cce22610 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Wed, 28 Nov 2018 08:14:41 +0100 Subject: [PATCH 06/11] Fix deleting of files The Package class would already look at the files, so the addremove was too late --- pkglistgen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkglistgen.py b/pkglistgen.py index 83a51b42..36f65155 100755 --- a/pkglistgen.py +++ b/pkglistgen.py @@ -1540,14 +1540,15 @@ class CommandLineInterface(ToolBase.CommandLineInterface): f.close() def commit_package(self, path): - package = Package(path) if self.options.dry: + package = Package(path) for i in package.get_diff(): print(''.join(i)) else: # No proper API function to perform the same operation. print(subprocess.check_output( ' '.join(['cd', path, '&&', 'osc', 'addremove']), shell=True)) + package = Package(path) package.commit(msg='Automatic update', skip_local_service_run=True) From 1eee0c68de812811237fa3e8e3d03bb3186278d0 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Mon, 3 Dec 2018 15:42:38 +0100 Subject: [PATCH 07/11] Remove 'ports' scope - powerpc uses target project --- pkglistgen.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/pkglistgen.py b/pkglistgen.py index 36f65155..388cc23a 100755 --- a/pkglistgen.py +++ b/pkglistgen.py @@ -1203,9 +1203,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface): os.environ['OBS_NAME'] = 'build.suse.de' target_config = conf.config[target_project] - if opts.scope == 'ports': - archs_key = 'pkglistgen-archs-ports' - elif opts.scope == 'arm': + if opts.scope == 'arm': archs_key = 'pkglistgen-archs-arm' else: archs_key = 'pkglistgen-archs' @@ -1217,25 +1215,15 @@ class CommandLineInterface(ToolBase.CommandLineInterface): if opts.scope == 'target': self.repos = self.tool.expand_repos(target_project, main_repo) self.update_and_solve_target_wrapper(api, target_project, target_config, main_repo, opts, drop_list=True) - return self.error_occured elif opts.scope == 'arm': main_repo = 'ports' opts.project += ':ARM' self.repos = self.tool.expand_repos(opts.project, main_repo) self.update_and_solve_target_wrapper(api, target_project, target_config, main_repo, opts, drop_list=True) - return self.error_occured - elif opts.scope == 'ports': - # TODO Continue supporting #1297, but should be abstracted. - main_repo = 'ports' - opts.project += ':Ports' - self.repos = self.tool.expand_repos(opts.project, main_repo) - self.update_and_solve_target_wrapper(api, target_project, target_config, main_repo, opts, drop_list=True) - return self.error_occured elif opts.scope == 'rings': opts.project = api.rings[1] self.repos = self.tool.expand_repos(api.rings[1], main_repo) self.update_and_solve_target_wrapper(api, target_project, target_config, main_repo, opts) - return self.error_occured elif opts.scope == 'staging': letters = api.get_staging_projects_short() for letter in letters: @@ -1244,7 +1232,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface): opts.project = api.prj_from_short(letter) self.repos = self.tool.expand_repos(opts.project, main_repo) self.update_and_solve_target_wrapper(api, target_project, target_config, main_repo, opts) - return self.error_occured + return self.error_occured def update_and_solve_target_wrapper(self, *args, **kwargs): try: @@ -1330,7 +1318,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface): self.tool.update_repos(opts) nonfree = target_config.get('nonfree') - if opts.scope not in ('arm', 'ports') and nonfree and drop_list: + if opts.scope != 'arm' and nonfree and drop_list: print('-> do_update nonfree') # Switch to nonfree repo (ugly, but that's how the code was setup). From c31c25953041087d2bcd6847d1e87e7463e44afc Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Mon, 3 Dec 2018 19:33:54 +0100 Subject: [PATCH 08/11] Also remove 'arm' scope --- osclib/conf.py | 2 -- pkglistgen.py | 14 +++----------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/osclib/conf.py b/osclib/conf.py index 765bdca2..c8878f27 100644 --- a/osclib/conf.py +++ b/osclib/conf.py @@ -98,8 +98,6 @@ DEFAULT = { # - F-C-C submitter requests (maxlin_factory) 'unselect-cleanup-whitelist': 'leaper maxlin_factory', 'pkglistgen-archs': 'x86_64', - 'pkglistgen-archs-arm': 'aarch64', - 'pkglistgen-archs-ports': 'aarch64 ppc64le', 'pkglistgen-locales-from': 'openSUSE.product', 'pkglistgen-include-suggested': 'False', 'pkglistgen-delete-kiwis-rings': 'openSUSE-ftp-ftp-x86_64.kiwi openSUSE-cd-mini-x86_64.kiwi', diff --git a/pkglistgen.py b/pkglistgen.py index 388cc23a..00e38cc0 100755 --- a/pkglistgen.py +++ b/pkglistgen.py @@ -747,7 +747,7 @@ class PkgListGen(ToolBase.ToolBase): class CommandLineInterface(ToolBase.CommandLineInterface): - SCOPES = ['all', 'target', 'rings', 'staging', 'arm'] + SCOPES = ['all', 'target', 'rings', 'staging'] def __init__(self, *args, **kwargs): ToolBase.CommandLineInterface.__init__(self, args, kwargs) @@ -1203,10 +1203,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface): os.environ['OBS_NAME'] = 'build.suse.de' target_config = conf.config[target_project] - if opts.scope == 'arm': - archs_key = 'pkglistgen-archs-arm' - else: - archs_key = 'pkglistgen-archs' + archs_key = 'pkglistgen-archs' if archs_key in target_config: self.options.architectures = target_config.get(archs_key).split(' ') @@ -1215,11 +1212,6 @@ class CommandLineInterface(ToolBase.CommandLineInterface): if opts.scope == 'target': self.repos = self.tool.expand_repos(target_project, main_repo) self.update_and_solve_target_wrapper(api, target_project, target_config, main_repo, opts, drop_list=True) - elif opts.scope == 'arm': - main_repo = 'ports' - opts.project += ':ARM' - self.repos = self.tool.expand_repos(opts.project, main_repo) - self.update_and_solve_target_wrapper(api, target_project, target_config, main_repo, opts, drop_list=True) elif opts.scope == 'rings': opts.project = api.rings[1] self.repos = self.tool.expand_repos(api.rings[1], main_repo) @@ -1318,7 +1310,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface): self.tool.update_repos(opts) nonfree = target_config.get('nonfree') - if opts.scope != 'arm' and nonfree and drop_list: + if nonfree and drop_list: print('-> do_update nonfree') # Switch to nonfree repo (ugly, but that's how the code was setup). From ee72a3efb08f63102d96584874fba0df32faa138 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Tue, 4 Dec 2018 04:40:28 +0100 Subject: [PATCH 09/11] No longer default to all scopes but make it configurable By default only generate the pkglists for the target project, only for leap we configure rings and stagings as additional scopes to look at --- osclib/conf.py | 1 + pkglistgen.py | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/osclib/conf.py b/osclib/conf.py index c8878f27..01c8fbcb 100644 --- a/osclib/conf.py +++ b/osclib/conf.py @@ -98,6 +98,7 @@ DEFAULT = { # - F-C-C submitter requests (maxlin_factory) 'unselect-cleanup-whitelist': 'leaper maxlin_factory', 'pkglistgen-archs': 'x86_64', + 'pkglistgen-scopes': 'target rings staging', 'pkglistgen-locales-from': 'openSUSE.product', 'pkglistgen-include-suggested': 'False', 'pkglistgen-delete-kiwis-rings': 'openSUSE-ftp-ftp-x86_64.kiwi openSUSE-cd-mini-x86_64.kiwi', diff --git a/pkglistgen.py b/pkglistgen.py index 00e38cc0..e0126c63 100755 --- a/pkglistgen.py +++ b/pkglistgen.py @@ -1174,9 +1174,17 @@ class CommandLineInterface(ToolBase.CommandLineInterface): raise ValueError('project is required') opts.staging_project = None + apiurl = conf.config['apiurl'] + config = Config(apiurl, opts.project) + target_config = conf.config[opts.project] + + if apiurl.find('suse.de') > 0: + # used by product converter + os.environ['OBS_NAME'] = 'build.suse.de' + # special case for all if opts.scope == ['all']: - opts.scope = self.SCOPES[1:] + opts.scope = target_config.get('pkglistgen-scopes', 'target').split(' ') for scope in opts.scope: if scope.startswith('staging:'): @@ -1186,23 +1194,17 @@ class CommandLineInterface(ToolBase.CommandLineInterface): if scope not in self.SCOPES: raise ValueError('scope "{}" must be one of: {}'.format(scope, ', '.join(self.SCOPES))) opts.scope = scope - self.real_update_and_solve(copy.deepcopy(opts)) + self.real_update_and_solve(target_config, copy.deepcopy(opts)) return self.error_occured # note: scope is a value here - while it's an array above - def real_update_and_solve(self, opts): + def real_update_and_solve(self, target_config, opts): # Store target project as opts.project will contain subprojects. target_project = opts.project apiurl = conf.config['apiurl'] - config = Config(apiurl, target_project) api = StagingAPI(apiurl, target_project) - if apiurl.find('suse.de') > 0: - # used by product converter - os.environ['OBS_NAME'] = 'build.suse.de' - - target_config = conf.config[target_project] archs_key = 'pkglistgen-archs' if archs_key in target_config: From 5da698c8aff6530c8cb1044867b3ddae770d0b23 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Tue, 4 Dec 2018 07:03:11 +0100 Subject: [PATCH 10/11] Do not generate drop list for leap:15.1:arm at the moment I moved the baseurl into the remote config so that Dirk can enable it when the repo was published --- osclib/conf.py | 1 - pkglistgen.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/osclib/conf.py b/osclib/conf.py index 01c8fbcb..8dc6b0ba 100644 --- a/osclib/conf.py +++ b/osclib/conf.py @@ -113,7 +113,6 @@ DEFAULT = { 'openqa': 'https://openqa.opensuse.org', 'main-repo': 'ports', 'pseudometa_package': 'openSUSE:%(project)s:ARM:Staging/dashboard', - 'download-baseurl': 'http://download.opensuse.org/ports/aarch64/distribution/leap/%(version)s/', 'mail-list': 'opensuse-arm@opensuse.org', 'mail-maintainer': 'Dirk Mueller ', 'mail-noreply': 'noreply@opensuse.org', diff --git a/pkglistgen.py b/pkglistgen.py index e0126c63..34b759cb 100755 --- a/pkglistgen.py +++ b/pkglistgen.py @@ -1071,7 +1071,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface): if name is not None and '-Build' in name: return name, 'build' - raise Exception('media.1/{media,build} includes no build number') + raise Exception(baseurl + '/media.1/{media,build} includes no build number') @cmdln.option('--ignore-unresolvable', action='store_true', help='ignore unresolvable and missing packges') @cmdln.option('--ignore-recommended', action='store_true', help='do not include recommended packages automatically') From 53595a24bffe9c8d92b7c44bd3675059c6d6b328 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Tue, 4 Dec 2018 07:41:13 +0100 Subject: [PATCH 11/11] droplist: Ignore Leap:15.1 for Leap:15.1:ARM The heuristic calculating the product family doesn't really work for ports, so give the tool some hints in the config. And as 15.1 doesn't have a baseurl yet, we need to give 15.0 one specifically. --- osclib/conf.py | 2 ++ osclib/util.py | 3 +++ pkglistgen.py | 2 ++ 3 files changed, 7 insertions(+) diff --git a/osclib/conf.py b/osclib/conf.py index 8dc6b0ba..bdec733d 100644 --- a/osclib/conf.py +++ b/osclib/conf.py @@ -113,6 +113,8 @@ DEFAULT = { 'openqa': 'https://openqa.opensuse.org', 'main-repo': 'ports', 'pseudometa_package': 'openSUSE:%(project)s:ARM:Staging/dashboard', + 'pkglistgen-product-family-include': 'openSUSE:Leap:15.0:ARM', + 'download-baseurl-openSUSE-Leap-15.0-ARM': 'http://download.opensuse.org/ports/aarch64/distribution/leap/15.0/', 'mail-list': 'opensuse-arm@opensuse.org', 'mail-maintainer': 'Dirk Mueller ', 'mail-noreply': 'noreply@opensuse.org', diff --git a/osclib/util.py b/osclib/util.py index 06c4da69..b8543f33 100644 --- a/osclib/util.py +++ b/osclib/util.py @@ -11,6 +11,9 @@ def project_list_family(apiurl, project): if project == 'openSUSE:Factory': return [project] + if project.endswith(':ARM') or project.endswith(':PowerPC'): + return [project] + count_original = project.count(':') if project.startswith('SUSE:SLE'): project = ':'.join(project.split(':')[:2]) diff --git a/pkglistgen.py b/pkglistgen.py index 34b759cb..c6913bc1 100755 --- a/pkglistgen.py +++ b/pkglistgen.py @@ -1424,6 +1424,8 @@ class CommandLineInterface(ToolBase.CommandLineInterface): project_config = conf.config[project] baseurl = project_config.get('download-baseurl') + if not baseurl: + baseurl = project_config.get('download-baseurl-' + project.replace(':', '-')) baseurl_update = project_config.get('download-baseurl-update') if not baseurl: logger.warning('no baseurl configured for {}'.format(project))