From bb23be210be039d56ab74001db0db146239b06c6 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Mon, 15 Apr 2019 17:20:42 +0200 Subject: [PATCH 1/5] For consistency reasons always link the sub packages we're selecting So far we skipped links between packages in the same ring because we've frozen a _link pointing from one to the other, but this does not work with default OBS and creates quite some problems also in adi projects where it's trying to be smart. Now simplify this a lot by simply linking it in any case --- osclib/stagingapi.py | 66 +++++++++++++------------------------------- 1 file changed, 19 insertions(+), 47 deletions(-) diff --git a/osclib/stagingapi.py b/osclib/stagingapi.py index 873fd4c5..19a61f76 100644 --- a/osclib/stagingapi.py +++ b/osclib/stagingapi.py @@ -820,14 +820,10 @@ class StagingAPI(object): if self._supersede: self.is_package_disabled(project, package, store=True) - for sub_prj, sub_pkg in self.get_sub_packages(package, project): - sub_prj = project + for sub_pkg in self.get_sub_packages(package, project): if self._supersede: - self.is_package_disabled(sub_prj, sub_pkg, store=True) - # Skip inner-project links for letter staging - if not self.is_adi_project(project) and sub_prj == project: - continue - delete_package(self.apiurl, sub_prj, sub_pkg, force=True, msg=msg) + self.is_package_disabled(project, sub_pkg, store=True) + delete_package(self.apiurl, project, sub_pkg, force=True, msg=msg) # Delete the main package in the last delete_package(self.apiurl, project, package, force=True, msg=msg) @@ -1085,35 +1081,16 @@ class StagingAPI(object): """ ret = [] - # Started the logic. Note that, return empty tuple in case selecting - # non-ring package to a letter staging. - if self.is_adi_project(project): - if not self.item_exists(project, package): - return ret - # For adi package, do not trust the layout in the devel project, we - # must to guarantee the sub-pacakges are created according to the - # specfiles of main package. Therefore, main package must be - # created before through get_sub_packages(). - filelist = self.get_filelist_for_package(pkgname=package, project=project, expand='1', extension='spec') - mainspec = "{}{}".format(package, '.spec') - if mainspec in filelist: - filelist.remove(mainspec) - for spec in filelist: - ret.append((project, spec[:-5])) - elif self.ring_packages.get(package): - project = self.ring_packages.get(package) - - url = self.makeurl(['source', project, package], - {'cmd': 'showlinked'}) - - # showlinked is a POST for rather bizzare reasons - f = http_POST(url) - root = ET.parse(f).getroot() - - for pkg in root.findall('package'): - # ensure sub-package is valid in rings - if pkg.get('project') in self.rings and pkg.get('name') != package: - ret.append((pkg.get('project'), pkg.get('name'))) + # Do not trust the layout in the devel project, must to + # guarantee the sub-pacakges are created according to the + # specfiles of main package. Therefore, main package must be + # created before through get_sub_packages(). + filelist = self.get_filelist_for_package(pkgname=package, project=project, expand='1', extension='spec') + mainspec = "{}{}".format(package, '.spec') + if mainspec in filelist: + filelist.remove(mainspec) + for spec in filelist: + ret.append(spec[:-5]) return ret @@ -1139,13 +1116,12 @@ class StagingAPI(object): tar_pkg = act.tgt_package self.create_and_wipe_package(project, tar_pkg) - for sub_prj, sub_pkg in self.get_sub_packages(tar_pkg, project): - sub_prj = project - self.create_and_wipe_package(sub_prj, sub_pkg) + for sub_pkg in self.get_sub_packages(tar_pkg, project): + self.create_and_wipe_package(project, sub_pkg) # create a link so unselect can find it root = ET.Element('link', package=tar_pkg, project=project) - url = self.makeurl(['source', sub_prj, sub_pkg, '_link']) + url = self.makeurl(['source', project, sub_pkg, '_link']) http_PUT(url, data=ET.tostring(root)) return tar_pkg @@ -1188,15 +1164,11 @@ class StagingAPI(object): self.apiurl, src_prj, src_pkg, '{}.spec'.format(src_pkg), src_rev)): baselibs = True - for sub_prj, sub_pkg in self.get_sub_packages(tar_pkg, project): - sub_prj = project - # Skip inner-project links for letter staging - if not self.is_adi_project(project) and sub_prj == project: - continue - self.create_package_container(sub_prj, sub_pkg) + for sub_pkg in self.get_sub_packages(tar_pkg, project): + self.create_package_container(project, sub_pkg) root = ET.Element('link', package=tar_pkg, project=project) - url = self.makeurl(['source', sub_prj, sub_pkg, '_link']) + url = self.makeurl(['source', project, sub_pkg, '_link']) http_PUT(url, data=ET.tostring(root)) if baselibs is False and 'baselibs.conf' in str(source_file_load( From 2765089077a976a0bbee1f423ef9c914c72908b1 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Mon, 15 Apr 2019 17:25:05 +0200 Subject: [PATCH 2/5] freeze: Remove code handling Test-DVD We no longer have products using that --- osclib/freeze_command.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/osclib/freeze_command.py b/osclib/freeze_command.py index 6fb2cfa3..a0988a28 100644 --- a/osclib/freeze_command.py +++ b/osclib/freeze_command.py @@ -144,26 +144,9 @@ class FreezeCommand(object): time.sleep(1) self.build_switch_bootstrap_copy('disable') - # Update the version information found in the Test-DVD package, to match openSUSE-release - if self.api.item_exists(prj, "openSUSE-release"): - version = self.api.package_version(prj, 'openSUSE-release') - for arch in ['x86_64', 'ppc64le']: - self.update_product_version(prj, 'Test-DVD-' + arch, arch, version) - # Set the original build status for the project self.api.build_switch_prj(prj, build_status) - def update_product_version(self, project, product, arch, version): - if not self.api.item_exists(project, product): - return None - - kiwifile = source_file_load(self.api.apiurl, project, product, 'PRODUCT-'+arch+'.kiwi') - - tmpkiwifile = re.sub(r'.*', '%s' % version, kiwifile) - newkiwifile = re.sub(r'.*', '%s' % version, tmpkiwifile) - - source_file_save(self.api.apiurl, project, product, 'PRODUCT-' + arch + '.kiwi', newkiwifile) - def prj_meta_for_bootstrap_copy(self, prj): root = ET.Element('project', {'name': prj}) ET.SubElement(root, 'title') From 00a0bdc982af505432bed5e1721118dfc94120ee Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Mon, 15 Apr 2019 17:34:44 +0200 Subject: [PATCH 3/5] freeze: Stop ignoring inter project links We link all packages by their srcmd5 - and only fix the links on selecting new packages --- osclib/freeze_command.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/osclib/freeze_command.py b/osclib/freeze_command.py index a0988a28..93c03f9f 100644 --- a/osclib/freeze_command.py +++ b/osclib/freeze_command.py @@ -225,23 +225,6 @@ class FreezeCommand(object): if si.find('originproject') != None: return None - # we have to check if its a link within the staging project - # in this case we need to keep the link as is, and not freezing - # the target. Otherwise putting kernel-source into staging prj - # won't get updated kernel-default (and many other cases) - for linked in si.findall('linked'): - if linked.get('project') in self.projectlinks: - # take the unexpanded md5 from Factory / 13.2 link - url = self.api.makeurl(['source', self.api.project, package], - {'view': 'info', 'nofilename': '1'}) - # print(package, linked.get('package'), linked.get('project')) - f = self.api.retried_GET(url) - proot = ET.parse(f).getroot() - lsrcmd5 = proot.get('lsrcmd5') - if lsrcmd5 is None: - raise Exception("{}/{} is not a link but we expected one".format(self.api.project, package)) - ET.SubElement(flink, 'package', {'name': package, 'srcmd5': lsrcmd5, 'vrev': si.get('vrev')}) - return package if package in ['rpmlint-mini-AGGR']: return package # we should not freeze aggregates ET.SubElement(flink, 'package', {'name': package, 'srcmd5': si.get('srcmd5'), 'vrev': si.get('vrev')}) From aa70305aa2da8aa5ac0ad87a9fad0d866405b05d Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Tue, 16 Apr 2019 11:57:16 +0200 Subject: [PATCH 4/5] Fix delete requests with multiple spec files First gather the links and then wipe - or it will always be empty --- osclib/stagingapi.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/osclib/stagingapi.py b/osclib/stagingapi.py index 19a61f76..d8b2c300 100644 --- a/osclib/stagingapi.py +++ b/osclib/stagingapi.py @@ -1114,9 +1114,11 @@ class StagingAPI(object): """ tar_pkg = act.tgt_package + # need to get the subpackages before we wipe it + sub_packages = self.get_sub_packages(tar_pkg, project) self.create_and_wipe_package(project, tar_pkg) - for sub_pkg in self.get_sub_packages(tar_pkg, project): + for sub_pkg in sub_packages: self.create_and_wipe_package(project, sub_pkg) # create a link so unselect can find it From 925032e0f92e7a429b21459142bad988400ee1e7 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Mon, 6 May 2019 22:38:57 +0200 Subject: [PATCH 5/5] Redo select tests to test package with multiple spec files --- osclib/core.py | 1 - tests/OBSLocal.py | 15 +++++---- tests/select_tests.py | 71 +++++++++++++++++++++++++++-------------- tests/unselect_tests.py | 16 +--------- 4 files changed, 57 insertions(+), 46 deletions(-) diff --git a/osclib/core.py b/osclib/core.py index a51b85a2..4c508d3c 100644 --- a/osclib/core.py +++ b/osclib/core.py @@ -80,7 +80,6 @@ def maintainers_get(apiurl, project, package=None): return maintainers -@memoize(session=True) def package_list(apiurl, project): url = makeurl(apiurl, ['source', project], { 'expand': 1 }) root = ET.parse(http_GET(url)).getroot() diff --git a/tests/OBSLocal.py b/tests/OBSLocal.py index 71f68420..dd136df3 100644 --- a/tests/OBSLocal.py +++ b/tests/OBSLocal.py @@ -267,14 +267,17 @@ class StagingWorkflow(object): project_links=project_links) return self.projects[name] - def create_submit_request(self, project, package, text=None): - project = self.create_project(project) - package = Package(name=package, project=project) - package.create_commit(text) + def submit_package(self, package=None): request = Request(source_package=package, target_project=self.project) self.requests.append(request) return request + def create_submit_request(self, project, package, text=None): + project = self.create_project(project) + package = Package(name=package, project=project) + package.create_commit(text=text) + return self.submit_package(package) + def create_staging(self, suffix, freeze=False, rings=None): project_links = [] if rings == 0: @@ -404,8 +407,8 @@ class Package(object): pass self.project = None - def create_commit(self, text=None): - url = osc.core.makeurl(APIURL, ['source', self.project.name, self.name, 'README']) + def create_commit(self, text=None, filename='README'): + url = osc.core.makeurl(APIURL, ['source', self.project.name, self.name, filename]) if not text: text = ''.join([random.choice(string.ascii_letters) for i in range(40)]) osc.core.http_PUT(url, data=text) diff --git a/tests/select_tests.py b/tests/select_tests.py index 6bbfe68a..e2b0b82b 100644 --- a/tests/select_tests.py +++ b/tests/select_tests.py @@ -6,8 +6,11 @@ from osclib.cache import Cache from osclib.cache_manager import CacheManager from osclib.comments import CommentAPI from osclib.conf import Config +from osclib.core import package_list from osclib.select_command import SelectCommand +from osclib.unselect_command import UnselectCommand from osclib.stagingapi import StagingAPI +from osclib.memoize import memoize_session_reset import logging from mock import MagicMock @@ -15,28 +18,33 @@ from . import OBSLocal class TestSelect(unittest.TestCase): + def setUp(self): + self.wf = OBSLocal.StagingWorkflow() + + def tearDown(self): + del self.wf + super(TestSelect, self).tearDown() + def test_old_frozen(self): - wf = OBSLocal.StagingWorkflow() - wf.api.prj_frozen_enough = MagicMock(return_value=False) + self.wf.api.prj_frozen_enough = MagicMock(return_value=False) # check it won't allow selecting - staging = wf.create_staging('Old') - self.assertEqual(False, SelectCommand(wf.api, staging.name).perform(['gcc'])) + staging = self.wf.create_staging('Old') + self.assertEqual(False, SelectCommand(self.wf.api, staging.name).perform(['gcc'])) def test_select_comments(self): - wf = OBSLocal.StagingWorkflow() - wf.setup_rings() + self.wf.setup_rings() - staging_b = wf.create_staging('B', freeze=True) + staging_b = self.wf.create_staging('B', freeze=True) - c_api = CommentAPI(wf.api.apiurl) + c_api = CommentAPI(self.wf.api.apiurl) comments = c_api.get_comments(project_name=staging_b.name) - r1 = wf.create_submit_request('devel:wine', 'wine') - r2 = wf.create_submit_request('devel:gcc', 'gcc') + r1 = self.wf.create_submit_request('devel:wine', 'wine') + r2 = self.wf.create_submit_request('devel:gcc', 'gcc') # First select - self.assertEqual(True, SelectCommand(wf.api, staging_b.name).perform(['gcc', 'wine'])) + self.assertEqual(True, SelectCommand(self.wf.api, staging_b.name).perform(['gcc', 'wine'])) first_select_comments = c_api.get_comments(project_name=staging_b.name) last_id = sorted(first_select_comments.keys())[-1] first_select_comment = first_select_comments[last_id] @@ -47,8 +55,8 @@ class TestSelect(unittest.TestCase): self.assertTrue(expected in first_select_comment['comment']) # Second select - r3 = wf.create_submit_request('devel:gcc', 'gcc8') - self.assertEqual(True, SelectCommand(wf.api, staging_b.name).perform(['gcc8'])) + r3 = self.wf.create_submit_request('devel:gcc', 'gcc8') + self.assertEqual(True, SelectCommand(self.wf.api, staging_b.name).perform(['gcc8'])) second_select_comments = c_api.get_comments(project_name=staging_b.name) last_id = sorted(second_select_comments.keys())[-1] second_select_comment = second_select_comments[last_id] @@ -60,22 +68,37 @@ class TestSelect(unittest.TestCase): self.assertTrue('added request#{} for package gcc8 submitted by Admin'.format(r3.reqid) in second_select_comment['comment']) def test_no_matches(self): - wf = OBSLocal.StagingWorkflow() - - staging = wf.create_staging('N', freeze=True) + staging = self.wf.create_staging('N', freeze=True) # search for requests with self.assertRaises(oscerr.WrongArgs) as cm: - SelectCommand(wf.api, staging.name).perform(['bash']) + SelectCommand(self.wf.api, staging.name).perform(['bash']) self.assertEqual(str(cm.exception), "No SR# found for: bash") def test_selected(self): - wf = OBSLocal.StagingWorkflow() + self.wf.setup_rings() + staging = self.wf.create_staging('S', freeze=True) + self.wf.create_submit_request('devel:wine', 'wine') - wf.setup_rings() - staging = wf.create_staging('S', freeze=True) - - request = wf.create_submit_request('devel:wine', 'wine') - - ret = SelectCommand(wf.api, staging.name).perform(['wine']) + ret = SelectCommand(self.wf.api, staging.name).perform(['wine']) self.assertEqual(True, ret) + + def test_select_multiple_spec(self): + self.wf.setup_rings() + staging = self.wf.create_staging('A', freeze=True) + + project = self.wf.create_project('devel:gcc') + package = OBSLocal.Package(name='gcc8', project=project) + package.create_commit(filename='gcc8.spec') + package.create_commit(filename='gcc8-tests.spec') + self.wf.submit_package(package) + + ret = SelectCommand(self.wf.api, staging.name).perform(['gcc8']) + self.assertEqual(True, ret) + + self.assertEqual(package_list(self.wf.apiurl, staging.name), ['gcc8', 'gcc8-tests']) + uc = UnselectCommand(self.wf.api) + self.assertIsNone(uc.perform(['gcc8'])) + + # no stale links + self.assertEqual([], package_list(self.wf.apiurl, staging.name)) diff --git a/tests/unselect_tests.py b/tests/unselect_tests.py index bca93448..8ee0dc10 100644 --- a/tests/unselect_tests.py +++ b/tests/unselect_tests.py @@ -1,9 +1,7 @@ import unittest from osclib.conf import Config from osclib.stagingapi import StagingAPI -from osclib.select_command import SelectCommand from osclib.unselect_command import UnselectCommand -from osclib.core import package_list_without_links from . import OBSLocal class TestUnselect(OBSLocal.TestCase): @@ -15,16 +13,4 @@ class TestUnselect(OBSLocal.TestCase): obsolete = wf.api.project_status_requests('obsolete', UnselectCommand.filter_obsolete) self.assertSequenceEqual([], obsolete) - def test_free_staging(self): - wf = OBSLocal.StagingWorkflow() - wf.setup_rings() - - staging_a = wf.create_staging('A', freeze=True) - winerq = wf.create_submit_request('devel:wine', 'wine') - self.assertEqual(True, SelectCommand(wf.api, staging_a.name).perform(['wine'])) - self.assertEqual(['wine'], package_list_without_links(wf.apiurl, staging_a.name)) - - uc = UnselectCommand(wf.api) - self.assertIsNone(uc.perform(['wine'])) - - self.assertEqual([], package_list_without_links(wf.apiurl, staging_a.name)) + # most testing for unselect happens in select_tests