diff --git a/_lastrevision b/_lastrevision index 2975e7d..0aa4287 100644 --- a/_lastrevision +++ b/_lastrevision @@ -1 +1 @@ -6fdab5ecba96a2695eadd798abc2dcfe6ff2b99b \ No newline at end of file +b5b0d996824665d5e87b375b74a2bd22ff73cbb0 \ No newline at end of file diff --git a/accumulated-changes-from-yomi-167.patch b/accumulated-changes-from-yomi-167.patch new file mode 100644 index 0000000..f33669b --- /dev/null +++ b/accumulated-changes-from-yomi-167.patch @@ -0,0 +1,186 @@ +From 46a60d81604eaf6f9fc3712e02d1066e959c96e3 Mon Sep 17 00:00:00 2001 +From: Alberto Planas +Date: Tue, 22 Oct 2019 11:02:33 +0200 +Subject: [PATCH] Accumulated changes from Yomi (#167) + +* core.py: ignore wrong product_name files + +Some firmwares (like some NUC machines), do not provide valid +/sys/class/dmi/id/product_name strings. In those cases an +UnicodeDecodeError exception happens. + +This patch ignore this kind of issue during the grains creation. + +(cherry picked from commit 2d57d2a6063488ad9329a083219e3826e945aa2d) + +* zypperpkg: understand product type + +(cherry picked from commit b865491b74679140f7a71c5ba50d482db47b600f) +--- + salt/grains/core.py | 4 +++ + salt/modules/zypperpkg.py | 30 +++++++++++++------ + tests/unit/grains/test_core.py | 45 ++++++++++++++++++++++++++++ + tests/unit/modules/test_zypperpkg.py | 26 ++++++++++++++++ + 4 files changed, 96 insertions(+), 9 deletions(-) + +diff --git a/salt/grains/core.py b/salt/grains/core.py +index fa188a6ff7..fdabe484a8 100644 +--- a/salt/grains/core.py ++++ b/salt/grains/core.py +@@ -986,6 +986,10 @@ def _virtual(osdata): + grains['virtual'] = 'gce' + elif 'BHYVE' in output: + grains['virtual'] = 'bhyve' ++ except UnicodeDecodeError: ++ # Some firmwares provide non-valid 'product_name' ++ # files, ignore them ++ pass + except IOError: + pass + elif osdata['kernel'] == 'FreeBSD': +diff --git a/salt/modules/zypperpkg.py b/salt/modules/zypperpkg.py +index da1953b2a5..a87041aa70 100644 +--- a/salt/modules/zypperpkg.py ++++ b/salt/modules/zypperpkg.py +@@ -861,23 +861,35 @@ def list_pkgs(versions_as_list=False, root=None, includes=None, **kwargs): + _ret[pkgname] = sorted(ret[pkgname], key=lambda d: d['version']) + + for include in includes: ++ if include == 'product': ++ products = list_products(all=False, root=root) ++ for product in products: ++ extended_name = '{}:{}'.format(include, product['name']) ++ _ret[extended_name] = [{ ++ 'epoch': product['epoch'], ++ 'version': product['version'], ++ 'release': product['release'], ++ 'arch': product['arch'], ++ 'install_date': None, ++ 'install_date_time_t': None, ++ }] + if include in ('pattern', 'patch'): + if include == 'pattern': +- pkgs = list_installed_patterns(root=root) ++ elements = list_installed_patterns(root=root) + elif include == 'patch': +- pkgs = list_installed_patches(root=root) ++ elements = list_installed_patches(root=root) + else: +- pkgs = [] +- for pkg in pkgs: +- pkg_extended_name = '{}:{}'.format(include, pkg) +- info = info_available(pkg_extended_name, ++ elements = [] ++ for element in elements: ++ extended_name = '{}:{}'.format(include, element) ++ info = info_available(extended_name, + refresh=False, + root=root) +- _ret[pkg_extended_name] = [{ ++ _ret[extended_name] = [{ + 'epoch': None, +- 'version': info[pkg]['version'], ++ 'version': info[element]['version'], + 'release': None, +- 'arch': info[pkg]['arch'], ++ 'arch': info[element]['arch'], + 'install_date': None, + 'install_date_time_t': None, + }] +diff --git a/tests/unit/grains/test_core.py b/tests/unit/grains/test_core.py +index 889fb90074..aa04a7a7ac 100644 +--- a/tests/unit/grains/test_core.py ++++ b/tests/unit/grains/test_core.py +@@ -1117,6 +1117,51 @@ class CoreGrainsTestCase(TestCase, LoaderModuleMockMixin): + 'uuid': '' + }) + ++ @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') ++ def test_kernelparams_return(self): ++ expectations = [ ++ ('BOOT_IMAGE=/vmlinuz-3.10.0-693.2.2.el7.x86_64', ++ {'kernelparams': [('BOOT_IMAGE', '/vmlinuz-3.10.0-693.2.2.el7.x86_64')]}), ++ ('root=/dev/mapper/centos_daemon-root', ++ {'kernelparams': [('root', '/dev/mapper/centos_daemon-root')]}), ++ ('rhgb quiet ro', ++ {'kernelparams': [('rhgb', None), ('quiet', None), ('ro', None)]}), ++ ('param="value1"', ++ {'kernelparams': [('param', 'value1')]}), ++ ('param="value1 value2 value3"', ++ {'kernelparams': [('param', 'value1 value2 value3')]}), ++ ('param="value1 value2 value3" LANG="pl" ro', ++ {'kernelparams': [('param', 'value1 value2 value3'), ('LANG', 'pl'), ('ro', None)]}), ++ ('ipv6.disable=1', ++ {'kernelparams': [('ipv6.disable', '1')]}), ++ ('param="value1:value2:value3"', ++ {'kernelparams': [('param', 'value1:value2:value3')]}), ++ ('param="value1,value2,value3"', ++ {'kernelparams': [('param', 'value1,value2,value3')]}), ++ ('param="value1" param="value2" param="value3"', ++ {'kernelparams': [('param', 'value1'), ('param', 'value2'), ('param', 'value3')]}), ++ ] ++ ++ for cmdline, expectation in expectations: ++ with patch('salt.utils.files.fopen', mock_open(read_data=cmdline)): ++ self.assertEqual(core.kernelparams(), expectation) ++ ++ @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') ++ @patch('os.path.exists') ++ @patch('salt.utils.platform.is_proxy') ++ def test__hw_data_linux_empty(self, is_proxy, exists): ++ is_proxy.return_value = False ++ exists.return_value = True ++ with patch('salt.utils.files.fopen', mock_open(read_data='')): ++ self.assertEqual(core._hw_data({'kernel': 'Linux'}), { ++ 'biosreleasedate': '', ++ 'biosversion': '', ++ 'manufacturer': '', ++ 'productname': '', ++ 'serialnumber': '', ++ 'uuid': '' ++ }) ++ + @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') + @skipIf(six.PY2, 'UnicodeDecodeError is throw in Python 3') + @patch('os.path.exists') +diff --git a/tests/unit/modules/test_zypperpkg.py b/tests/unit/modules/test_zypperpkg.py +index 695d982ca6..7617113401 100644 +--- a/tests/unit/modules/test_zypperpkg.py ++++ b/tests/unit/modules/test_zypperpkg.py +@@ -943,6 +943,32 @@ Repository 'DUMMY' not found by its alias, number, or URI. + with self.assertRaisesRegex(CommandExecutionError, '^Advisory id "SUSE-PATCH-XXX" not found$'): + zypper.install(advisory_ids=['SUSE-PATCH-XXX']) + ++ @patch('salt.modules.zypperpkg._systemd_scope', ++ MagicMock(return_value=False)) ++ @patch('salt.modules.zypperpkg.list_products', ++ MagicMock(return_value={'openSUSE': {'installed': False, 'summary': 'test'}})) ++ @patch('salt.modules.zypperpkg.list_pkgs', MagicMock(side_effect=[{"product:openSUSE": "15.2"}, ++ {"product:openSUSE": "15.3"}])) ++ def test_install_product_ok(self): ++ ''' ++ Test successfully product installation. ++ ''' ++ with patch.dict(zypper.__salt__, ++ { ++ 'pkg_resource.parse_targets': MagicMock( ++ return_value=(['product:openSUSE'], None)) ++ }): ++ with patch('salt.modules.zypperpkg.__zypper__.noraise.call', MagicMock()) as zypper_mock: ++ ret = zypper.install('product:openSUSE', includes=['product']) ++ zypper_mock.assert_called_once_with( ++ '--no-refresh', ++ 'install', ++ '--auto-agree-with-licenses', ++ '--name', ++ 'product:openSUSE' ++ ) ++ self.assertDictEqual(ret, {"product:openSUSE": {"old": "15.2", "new": "15.3"}}) ++ + def test_remove_purge(self): + ''' + Test package removal +-- +2.23.0 + + diff --git a/add-missing-fun-for-returns-from-wfunc-executions.patch b/add-missing-fun-for-returns-from-wfunc-executions.patch new file mode 100644 index 0000000..ee0faba --- /dev/null +++ b/add-missing-fun-for-returns-from-wfunc-executions.patch @@ -0,0 +1,36 @@ +From fa957bcb842a29a340a980a03cd8e54b06e7e21b Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Wed, 9 Oct 2019 13:03:33 +0100 +Subject: [PATCH] Add missing 'fun' for returns from wfunc executions + +--- + salt/client/ssh/__init__.py | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py +index 1453430e73..0df918d634 100644 +--- a/salt/client/ssh/__init__.py ++++ b/salt/client/ssh/__init__.py +@@ -682,6 +682,8 @@ class SSH(object): + data = {'return': data} + if 'id' not in data: + data['id'] = id_ ++ if 'fun' not in data: ++ data['fun'] = fun + data['jid'] = jid # make the jid in the payload the same as the jid in the tag + self.event.fire_event( + data, +@@ -797,6 +799,8 @@ class SSH(object): + data = {'return': data} + if 'id' not in data: + data['id'] = id_ ++ if 'fun' not in data: ++ data['fun'] = fun + data['jid'] = jid # make the jid in the payload the same as the jid in the tag + self.event.fire_event( + data, +-- +2.22.0 + + diff --git a/adds-the-possibility-to-also-use-downloadonly-in-kwa.patch b/adds-the-possibility-to-also-use-downloadonly-in-kwa.patch new file mode 100644 index 0000000..32b38b7 --- /dev/null +++ b/adds-the-possibility-to-also-use-downloadonly-in-kwa.patch @@ -0,0 +1,88 @@ +From f9e7ace2f7c56a7fb4df60a048131dbd6887340b Mon Sep 17 00:00:00 2001 +From: Jochen Breuer +Date: Fri, 27 Sep 2019 11:33:47 +0200 +Subject: [PATCH] Adds the possibility to also use downloadonly in kwargs + +The download_only parameter in the apt module is not in line with +the yum and zypper modules. Both of them use downloadonly without +the underline. + +With this change apt now additionally supports the downloadonly +parameter. + +Fixes #54790 +--- + salt/modules/aptpkg.py | 7 ++++--- + tests/unit/modules/test_aptpkg.py | 30 ++++++++++++++++++++++++++++++ + 2 files changed, 34 insertions(+), 3 deletions(-) + +diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py +index a11bb51c16..1a60255a1d 100644 +--- a/salt/modules/aptpkg.py ++++ b/salt/modules/aptpkg.py +@@ -1054,8 +1054,9 @@ def upgrade(refresh=True, dist_upgrade=False, **kwargs): + Skip refreshing the package database if refresh has already occurred within + seconds + +- download_only +- Only download the packages, don't unpack or install them ++ download_only (or downloadonly) ++ Only download the packages, don't unpack or install them. Use ++ downloadonly to be in line with yum and zypper module. + + .. versionadded:: 2018.3.0 + +@@ -1086,7 +1087,7 @@ def upgrade(refresh=True, dist_upgrade=False, **kwargs): + cmd.append('--force-yes') + if kwargs.get('skip_verify', False): + cmd.append('--allow-unauthenticated') +- if kwargs.get('download_only', False): ++ if kwargs.get('download_only', False) or kwargs.get('downloadonly', False): + cmd.append('--download-only') + + cmd.append('dist-upgrade' if dist_upgrade else 'upgrade') +diff --git a/tests/unit/modules/test_aptpkg.py b/tests/unit/modules/test_aptpkg.py +index 85360da181..d3fac5902a 100644 +--- a/tests/unit/modules/test_aptpkg.py ++++ b/tests/unit/modules/test_aptpkg.py +@@ -393,6 +393,36 @@ class AptPkgTestCase(TestCase, LoaderModuleMockMixin): + with patch.multiple(aptpkg, **patch_kwargs): + self.assertEqual(aptpkg.upgrade(), dict()) + ++ def test_upgrade_downloadonly(self): ++ ''' ++ Tests the download-only options for upgrade. ++ ''' ++ with patch('salt.utils.pkg.clear_rtag', MagicMock()): ++ with patch('salt.modules.aptpkg.list_pkgs', ++ MagicMock(return_value=UNINSTALL)): ++ mock_cmd = MagicMock(return_value={ ++ 'retcode': 0, ++ 'stdout': UPGRADE ++ }) ++ patch_kwargs = { ++ '__salt__': { ++ 'config.get': MagicMock(return_value=True), ++ 'cmd.run_all': mock_cmd ++ }, ++ } ++ with patch.multiple(aptpkg, **patch_kwargs): ++ aptpkg.upgrade() ++ args_matching = [True for args in patch_kwargs['__salt__']['cmd.run_all'].call_args.args if "--download-only" in args] ++ self.assertFalse(any(args_matching)) ++ ++ aptpkg.upgrade(downloadonly=True) ++ args_matching = [True for args in patch_kwargs['__salt__']['cmd.run_all'].call_args.args if "--download-only" in args] ++ self.assertTrue(any(args_matching)) ++ ++ aptpkg.upgrade(download_only=True) ++ args_matching = [True for args in patch_kwargs['__salt__']['cmd.run_all'].call_args.args if "--download-only" in args] ++ self.assertTrue(any(args_matching)) ++ + def test_show(self): + ''' + Test that the pkg.show function properly parses apt-cache show output. +-- +2.16.4 + + diff --git a/fix-a-wrong-rebase-in-test_core.py-180.patch b/fix-a-wrong-rebase-in-test_core.py-180.patch new file mode 100644 index 0000000..9bb019e --- /dev/null +++ b/fix-a-wrong-rebase-in-test_core.py-180.patch @@ -0,0 +1,105 @@ +From 329f90fcde205237545cd623f55f0f6c228bf893 Mon Sep 17 00:00:00 2001 +From: Alberto Planas +Date: Fri, 25 Oct 2019 15:43:16 +0200 +Subject: [PATCH] Fix a wrong rebase in test_core.py (#180) + +* core: ignore wrong product_name files + +Some firmwares (like some NUC machines), do not provide valid +/sys/class/dmi/id/product_name strings. In those cases an +UnicodeDecodeError exception happens. + +This patch ignore this kind of issue during the grains creation. + +(cherry picked from commit 27b001bd5408359aa5dd219bfd900095ed592fe8) + +* core: remove duplicate dead code + +(cherry picked from commit bd0213bae00b737b24795bec3c030ebfe476e0d8) +--- + salt/grains/core.py | 4 +-- + tests/unit/grains/test_core.py | 45 ---------------------------------- + 2 files changed, 2 insertions(+), 47 deletions(-) + +diff --git a/salt/grains/core.py b/salt/grains/core.py +index fdabe484a8..bf54c54553 100644 +--- a/salt/grains/core.py ++++ b/salt/grains/core.py +@@ -989,7 +989,7 @@ def _virtual(osdata): + except UnicodeDecodeError: + # Some firmwares provide non-valid 'product_name' + # files, ignore them +- pass ++ log.debug('The content in /sys/devices/virtual/dmi/id/product_name is not valid') + except IOError: + pass + elif osdata['kernel'] == 'FreeBSD': +@@ -2490,7 +2490,7 @@ def _hw_data(osdata): + except UnicodeDecodeError: + # Some firmwares provide non-valid 'product_name' + # files, ignore them +- pass ++ log.debug('The content in /sys/devices/virtual/dmi/id/product_name is not valid') + except (IOError, OSError) as err: + # PermissionError is new to Python 3, but corresponds to the EACESS and + # EPERM error numbers. Use those instead here for PY2 compatibility. +diff --git a/tests/unit/grains/test_core.py b/tests/unit/grains/test_core.py +index aa04a7a7ac..889fb90074 100644 +--- a/tests/unit/grains/test_core.py ++++ b/tests/unit/grains/test_core.py +@@ -1117,51 +1117,6 @@ class CoreGrainsTestCase(TestCase, LoaderModuleMockMixin): + 'uuid': '' + }) + +- @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') +- def test_kernelparams_return(self): +- expectations = [ +- ('BOOT_IMAGE=/vmlinuz-3.10.0-693.2.2.el7.x86_64', +- {'kernelparams': [('BOOT_IMAGE', '/vmlinuz-3.10.0-693.2.2.el7.x86_64')]}), +- ('root=/dev/mapper/centos_daemon-root', +- {'kernelparams': [('root', '/dev/mapper/centos_daemon-root')]}), +- ('rhgb quiet ro', +- {'kernelparams': [('rhgb', None), ('quiet', None), ('ro', None)]}), +- ('param="value1"', +- {'kernelparams': [('param', 'value1')]}), +- ('param="value1 value2 value3"', +- {'kernelparams': [('param', 'value1 value2 value3')]}), +- ('param="value1 value2 value3" LANG="pl" ro', +- {'kernelparams': [('param', 'value1 value2 value3'), ('LANG', 'pl'), ('ro', None)]}), +- ('ipv6.disable=1', +- {'kernelparams': [('ipv6.disable', '1')]}), +- ('param="value1:value2:value3"', +- {'kernelparams': [('param', 'value1:value2:value3')]}), +- ('param="value1,value2,value3"', +- {'kernelparams': [('param', 'value1,value2,value3')]}), +- ('param="value1" param="value2" param="value3"', +- {'kernelparams': [('param', 'value1'), ('param', 'value2'), ('param', 'value3')]}), +- ] +- +- for cmdline, expectation in expectations: +- with patch('salt.utils.files.fopen', mock_open(read_data=cmdline)): +- self.assertEqual(core.kernelparams(), expectation) +- +- @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') +- @patch('os.path.exists') +- @patch('salt.utils.platform.is_proxy') +- def test__hw_data_linux_empty(self, is_proxy, exists): +- is_proxy.return_value = False +- exists.return_value = True +- with patch('salt.utils.files.fopen', mock_open(read_data='')): +- self.assertEqual(core._hw_data({'kernel': 'Linux'}), { +- 'biosreleasedate': '', +- 'biosversion': '', +- 'manufacturer': '', +- 'productname': '', +- 'serialnumber': '', +- 'uuid': '' +- }) +- + @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') + @skipIf(six.PY2, 'UnicodeDecodeError is throw in Python 3') + @patch('os.path.exists') +-- +2.23.0 + + diff --git a/fix-failing-unit-tests-for-batch-async.patch b/fix-failing-unit-tests-for-batch-async.patch new file mode 100644 index 0000000..c4b335f --- /dev/null +++ b/fix-failing-unit-tests-for-batch-async.patch @@ -0,0 +1,185 @@ +From 8378bb24a5a53973e8dba7658b8b3465d967329f Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Fri, 4 Oct 2019 15:00:50 +0100 +Subject: [PATCH] Fix failing unit tests for batch async + +--- + salt/cli/batch_async.py | 2 +- + tests/unit/cli/test_batch_async.py | 57 +++++++++++++++++------------- + 2 files changed, 34 insertions(+), 25 deletions(-) + +diff --git a/salt/cli/batch_async.py b/salt/cli/batch_async.py +index f9e736f804..6d0dca1da5 100644 +--- a/salt/cli/batch_async.py ++++ b/salt/cli/batch_async.py +@@ -88,7 +88,7 @@ class BatchAsync(object): + io_loop=ioloop, + keep_loop=True) + self.scheduled = False +- self.patterns = {} ++ self.patterns = set() + + def __set_event_handler(self): + ping_return_pattern = 'salt/job/{0}/ret/*'.format(self.ping_jid) +diff --git a/tests/unit/cli/test_batch_async.py b/tests/unit/cli/test_batch_async.py +index 441f9c58b9..12dfe543bc 100644 +--- a/tests/unit/cli/test_batch_async.py ++++ b/tests/unit/cli/test_batch_async.py +@@ -68,8 +68,8 @@ class AsyncBatchTestCase(AsyncTestCase, TestCase): + ret = self.batch.start() + # assert start_batch is called later with batch_presence_ping_timeout as param + self.assertEqual( +- self.batch.event.io_loop.call_later.call_args[0], +- (self.batch.batch_presence_ping_timeout, self.batch.start_batch)) ++ self.batch.event.io_loop.spawn_callback.call_args[0], ++ (self.batch.start_batch,)) + # assert test.ping called + self.assertEqual( + self.batch.local.run_job_async.call_args[0], +@@ -88,8 +88,8 @@ class AsyncBatchTestCase(AsyncTestCase, TestCase): + ret = self.batch.start() + # assert start_batch is called later with gather_job_timeout as param + self.assertEqual( +- self.batch.event.io_loop.call_later.call_args[0], +- (self.batch.opts['gather_job_timeout'], self.batch.start_batch)) ++ self.batch.event.io_loop.spawn_callback.call_args[0], ++ (self.batch.start_batch,)) + + def test_batch_fire_start_event(self): + self.batch.minions = set(['foo', 'bar']) +@@ -113,12 +113,11 @@ class AsyncBatchTestCase(AsyncTestCase, TestCase): + def test_start_batch_calls_next(self): + self.batch.run_next = MagicMock(return_value=MagicMock()) + self.batch.event = MagicMock() +- future = tornado.gen.Future() +- future.set_result(None) +- self.batch.run_next = MagicMock(return_value=future) + self.batch.start_batch() + self.assertEqual(self.batch.initialized, True) +- self.assertEqual(len(self.batch.run_next.mock_calls), 1) ++ self.assertEqual( ++ self.batch.event.io_loop.spawn_callback.call_args[0], ++ (self.batch.run_next,)) + + def test_batch_fire_done_event(self): + self.batch.targeted_minions = {'foo', 'baz', 'bar'} +@@ -154,14 +153,14 @@ class AsyncBatchTestCase(AsyncTestCase, TestCase): + future = tornado.gen.Future() + future.set_result({'minions': ['foo', 'bar']}) + self.batch.local.run_job_async.return_value = future +- ret = self.batch.run_next().result() ++ self.batch.run_next() + self.assertEqual( + self.batch.local.run_job_async.call_args[0], + ({'foo', 'bar'}, 'my.fun', [], 'list') + ) + self.assertEqual( +- self.batch.event.io_loop.call_later.call_args[0], +- (self.batch.opts['timeout'], self.batch.find_job, {'foo', 'bar'}) ++ self.batch.event.io_loop.spawn_callback.call_args[0], ++ (self.batch.find_job, {'foo', 'bar'}) + ) + self.assertEqual(self.batch.active, {'bar', 'foo'}) + +@@ -252,13 +251,14 @@ class AsyncBatchTestCase(AsyncTestCase, TestCase): + self.assertEqual(self.batch.active, set()) + self.assertEqual(self.batch.done_minions, {'foo'}) + self.assertEqual( +- self.batch.event.io_loop.call_later.call_args[0], +- (self.batch.batch_delay, self.batch.run_next)) ++ self.batch.event.io_loop.spawn_callback.call_args[0], ++ (self.batch.schedule_next,)) + + def test_batch__event_handler_find_job_return(self): + self.batch.event = MagicMock( +- unpack=MagicMock(return_value=('salt/job/1236/ret/foo', {'id': 'foo'}))) ++ unpack=MagicMock(return_value=('salt/job/1236/ret/foo', {'id': 'foo', 'return': 'deadbeaf'}))) + self.batch.start() ++ self.batch.patterns.add(('salt/job/1236/ret/*', 'find_job_return')) + self.batch._BatchAsync__event_handler(MagicMock()) + self.assertEqual(self.batch.find_job_returned, {'foo'}) + +@@ -275,10 +275,13 @@ class AsyncBatchTestCase(AsyncTestCase, TestCase): + future = tornado.gen.Future() + future.set_result({}) + self.batch.local.run_job_async.return_value = future ++ self.batch.minions = set(['foo', 'bar']) ++ self.batch.jid_gen = MagicMock(return_value="1234") ++ tornado.gen.sleep = MagicMock(return_value=future) + self.batch.find_job({'foo', 'bar'}) + self.assertEqual( +- self.batch.event.io_loop.call_later.call_args[0], +- (self.batch.opts['gather_job_timeout'], self.batch.check_find_job, {'foo', 'bar'}) ++ self.batch.event.io_loop.spawn_callback.call_args[0], ++ (self.batch.check_find_job, {'foo', 'bar'}, "1234") + ) + + @tornado.testing.gen_test +@@ -288,17 +291,21 @@ class AsyncBatchTestCase(AsyncTestCase, TestCase): + future = tornado.gen.Future() + future.set_result({}) + self.batch.local.run_job_async.return_value = future ++ self.batch.minions = set(['foo', 'bar']) ++ self.batch.jid_gen = MagicMock(return_value="1234") ++ tornado.gen.sleep = MagicMock(return_value=future) + self.batch.find_job({'foo', 'bar'}) + self.assertEqual( +- self.batch.event.io_loop.call_later.call_args[0], +- (self.batch.opts['gather_job_timeout'], self.batch.check_find_job, {'foo'}) ++ self.batch.event.io_loop.spawn_callback.call_args[0], ++ (self.batch.check_find_job, {'foo'}, "1234") + ) + + def test_batch_check_find_job_did_not_return(self): + self.batch.event = MagicMock() + self.batch.active = {'foo'} + self.batch.find_job_returned = set() +- self.batch.check_find_job({'foo'}) ++ self.batch.patterns = { ('salt/job/1234/ret/*', 'find_job_return') } ++ self.batch.check_find_job({'foo'}, jid="1234") + self.assertEqual(self.batch.find_job_returned, set()) + self.assertEqual(self.batch.active, set()) + self.assertEqual(len(self.batch.event.io_loop.add_callback.mock_calls), 0) +@@ -306,9 +313,10 @@ class AsyncBatchTestCase(AsyncTestCase, TestCase): + def test_batch_check_find_job_did_return(self): + self.batch.event = MagicMock() + self.batch.find_job_returned = {'foo'} +- self.batch.check_find_job({'foo'}) ++ self.batch.patterns = { ('salt/job/1234/ret/*', 'find_job_return') } ++ self.batch.check_find_job({'foo'}, jid="1234") + self.assertEqual( +- self.batch.event.io_loop.add_callback.call_args[0], ++ self.batch.event.io_loop.spawn_callback.call_args[0], + (self.batch.find_job, {'foo'}) + ) + +@@ -329,7 +337,8 @@ class AsyncBatchTestCase(AsyncTestCase, TestCase): + # both not yet done but only 'foo' responded to find_job + not_done = {'foo', 'bar'} + +- self.batch.check_find_job(not_done) ++ self.batch.patterns = { ('salt/job/1234/ret/*', 'find_job_return') } ++ self.batch.check_find_job(not_done, jid="1234") + + # assert 'bar' removed from active + self.assertEqual(self.batch.active, {'foo'}) +@@ -339,7 +348,7 @@ class AsyncBatchTestCase(AsyncTestCase, TestCase): + + # assert 'find_job' schedueled again only for 'foo' + self.assertEqual( +- self.batch.event.io_loop.add_callback.call_args[0], ++ self.batch.event.io_loop.spawn_callback.call_args[0], + (self.batch.find_job, {'foo'}) + ) + +@@ -347,4 +356,4 @@ class AsyncBatchTestCase(AsyncTestCase, TestCase): + self.batch.event = MagicMock() + self.batch.scheduled = True + self.batch.schedule_next() +- self.assertEqual(len(self.batch.event.io_loop.call_later.mock_calls), 0) ++ self.assertEqual(len(self.batch.event.io_loop.spawn_callback.mock_calls), 0) +-- +2.22.0 + + diff --git a/fix-for-older-mock-module.patch b/fix-for-older-mock-module.patch new file mode 100644 index 0000000..01d5f73 --- /dev/null +++ b/fix-for-older-mock-module.patch @@ -0,0 +1,37 @@ +From 7e4c53ab89927b6b700603a74131da318c93b957 Mon Sep 17 00:00:00 2001 +From: Jochen Breuer +Date: Fri, 25 Oct 2019 16:18:58 +0200 +Subject: [PATCH] Fix for older mock module + +Seems like args is not working with older mock modules. +--- + tests/unit/modules/test_aptpkg.py | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/tests/unit/modules/test_aptpkg.py b/tests/unit/modules/test_aptpkg.py +index d3fac5902a..bc6b610d86 100644 +--- a/tests/unit/modules/test_aptpkg.py ++++ b/tests/unit/modules/test_aptpkg.py +@@ -412,15 +412,15 @@ class AptPkgTestCase(TestCase, LoaderModuleMockMixin): + } + with patch.multiple(aptpkg, **patch_kwargs): + aptpkg.upgrade() +- args_matching = [True for args in patch_kwargs['__salt__']['cmd.run_all'].call_args.args if "--download-only" in args] ++ args_matching = [True for args in patch_kwargs['__salt__']['cmd.run_all'].call_args[0] if "--download-only" in args] + self.assertFalse(any(args_matching)) + + aptpkg.upgrade(downloadonly=True) +- args_matching = [True for args in patch_kwargs['__salt__']['cmd.run_all'].call_args.args if "--download-only" in args] ++ args_matching = [True for args in patch_kwargs['__salt__']['cmd.run_all'].call_args[0] if "--download-only" in args] + self.assertTrue(any(args_matching)) + + aptpkg.upgrade(download_only=True) +- args_matching = [True for args in patch_kwargs['__salt__']['cmd.run_all'].call_args.args if "--download-only" in args] ++ args_matching = [True for args in patch_kwargs['__salt__']['cmd.run_all'].call_args[0] if "--download-only" in args] + self.assertTrue(any(args_matching)) + + def test_show(self): +-- +2.16.4 + + diff --git a/prevent-systemd-run-description-issue-when-running-a.patch b/prevent-systemd-run-description-issue-when-running-a.patch new file mode 100644 index 0000000..9f14025 --- /dev/null +++ b/prevent-systemd-run-description-issue-when-running-a.patch @@ -0,0 +1,42 @@ +From 44a91c2ce6df78d93ce0ef659dedb0e41b1c2e04 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Mon, 30 Sep 2019 12:06:08 +0100 +Subject: [PATCH] Prevent systemd-run description issue when running + aptpkg (bsc#1152366) + +--- + salt/modules/aptpkg.py | 2 +- + tests/unit/modules/test_aptpkg.py | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py +index d49a48310e..a11bb51c16 100644 +--- a/salt/modules/aptpkg.py ++++ b/salt/modules/aptpkg.py +@@ -165,7 +165,7 @@ def _call_apt(args, scope=True, **kwargs): + ''' + cmd = [] + if scope and salt.utils.systemd.has_scope(__context__) and __salt__['config.get']('systemd.scope', True): +- cmd.extend(['systemd-run', '--scope', '--description "{0}"'.format(__name__)]) ++ cmd.extend(['systemd-run', '--scope', '--description', '"{0}"'.format(__name__)]) + cmd.extend(args) + + params = {'output_loglevel': 'trace', +diff --git a/tests/unit/modules/test_aptpkg.py b/tests/unit/modules/test_aptpkg.py +index 06f3a9f6aa..85360da181 100644 +--- a/tests/unit/modules/test_aptpkg.py ++++ b/tests/unit/modules/test_aptpkg.py +@@ -544,7 +544,7 @@ class AptUtilsTestCase(TestCase, LoaderModuleMockMixin): + with patch.dict(aptpkg.__salt__, {'cmd.run_all': MagicMock(), 'config.get': MagicMock(return_value=True)}): + aptpkg._call_apt(['apt-get', 'purge', 'vim']) # pylint: disable=W0106 + aptpkg.__salt__['cmd.run_all'].assert_called_once_with( +- ['systemd-run', '--scope', '--description "salt.modules.aptpkg"', 'apt-get', 'purge', 'vim'], env={}, ++ ['systemd-run', '--scope', '--description', '"salt.modules.aptpkg"', 'apt-get', 'purge', 'vim'], env={}, + output_loglevel='trace', python_shell=False) + + def test_call_apt_with_kwargs(self): +-- +2.22.0 + + diff --git a/salt.changes b/salt.changes index 2c9018b..1906484 100644 --- a/salt.changes +++ b/salt.changes @@ -1,3 +1,83 @@ +------------------------------------------------------------------- +Fri Oct 25 14:39:23 UTC 2019 - Jochen Breuer + +- Fix for aptpkg test with older mock modules + +- Added: + * fix-for-older-mock-module.patch + +------------------------------------------------------------------- +Fri Oct 25 13:52:01 UTC 2019 - Pablo Suárez Hernández + +- Remove wrong tests for core grain and improve debug logging +- Use rich RPM deps to get a compatible version of tornado into the + buildroot. + +- Added: + * fix-a-wrong-rebase-in-test_core.py-180.patch + +------------------------------------------------------------------- +Tue Oct 22 09:29:19 UTC 2019 - Pablo Suárez Hernández + +- core.py: ignore wrong product_name files +- zypperpkg: understand product type + +- Added: + * accumulated-changes-from-yomi-167.patch + +------------------------------------------------------------------- +Mon Oct 21 15:10:37 UTC 2019 - Jochen Breuer + +- Enable usage of downloadonly parameter for apt module + +- Added: + * adds-the-possibility-to-also-use-downloadonly-in-kwa.patch + +------------------------------------------------------------------- +Wed Oct 9 12:40:33 UTC 2019 - Pablo Suárez Hernández + +- Add missing 'fun' on events coming from salt-ssh wfunc executions (bsc#1151947) + +- Added: + * add-missing-fun-for-returns-from-wfunc-executions.patch + +------------------------------------------------------------------- +Fri Oct 4 14:35:10 UTC 2019 - Pablo Suárez Hernández + +- Fix failing unit tests for batch async + +- Added: + * fix-failing-unit-tests-for-batch-async.patch + +------------------------------------------------------------------- +Thu Oct 3 14:50:41 UTC 2019 - Pablo Suárez Hernández + +- Fix memory consumption problem on BatchAsync (bsc#1137642) + +- Added: + * use-current-ioloop-for-the-localclient-instance-of-b.patch + +------------------------------------------------------------------- +Tue Oct 1 13:17:58 UTC 2019 - Pablo Suárez Hernández + +- Fix dependencies for RHEL 8 + +------------------------------------------------------------------- +Mon Sep 30 11:15:32 UTC 2019 - Pablo Suárez Hernández + +- Prevent systemd-run description issue when running aptpkg (bsc#1152366) + +- Added: + * prevent-systemd-run-description-issue-when-running-a.patch + +------------------------------------------------------------------- +Thu Sep 26 15:17:59 UTC 2019 - Pablo Suárez Hernández + +- Take checksums arg into account for postgres.datadir_init (bsc#1151650) + +- Added: + * take-checksums-arg-into-account-for-postgres.datadir.patch + ------------------------------------------------------------------- Thu Sep 26 10:23:39 UTC 2019 - Pablo Suárez Hernández diff --git a/salt.spec b/salt.spec index 7935a33..741fb71 100644 --- a/salt.spec +++ b/salt.spec @@ -14,6 +14,7 @@ # Please submit bugfixes or comments via http://bugs.opensuse.org/ # +%global debug_package %{nil} %if 0%{?suse_version} >= 1320 # SLE15 @@ -243,8 +244,27 @@ Patch82: virt.volume_infos-silence-libvirt-error-message-175.patch Patch83: fix-virt.full_info-176.patch # PATCH-FIX_OPENSUSE: https://github.com/openSUSE/salt/commit/002543df392f65d95dbc127dc058ac897f2035ed Patch84: improve-batch_async-to-release-consumed-memory-bsc-1.patch +# PATCH_FIX_UPSTREAM: https://github.com/saltstack/salt/pull/54770 +Patch85: take-checksums-arg-into-account-for-postgres.datadir.patch +# PATCH_FIX_UPSTREAM: https://github.com/saltstack/salt/pull/54077 +# PATCH_FIX_OPENSUSE: https://github.com/openSUSE/salt/commit/44a91c2ce6df78d93ce0ef659dedb0e41b1c2e04 +Patch86: prevent-systemd-run-description-issue-when-running-a.patch +# PATCH_FIX_OPENSUSE: https://github.com/openSUSE/salt/commit/55d8a777d6a9b19c959e14a4060e5579e92cd106 +Patch87: use-current-ioloop-for-the-localclient-instance-of-b.patch +# PATCH_FIX_OPENSUSE: https://github.com/openSUSE/salt/commit/8378bb24a5a53973e8dba7658b8b3465d967329f +Patch88: fix-failing-unit-tests-for-batch-async.patch +# PATCH_FIX_UPSTREAM: https://github.com/saltstack/salt/pull/54935 +Patch89: add-missing-fun-for-returns-from-wfunc-executions.patch +# PATCH_FIX_OPENSUSE: https://github.com/openSUSE/salt/pull/179 +Patch90: adds-the-possibility-to-also-use-downloadonly-in-kwa.patch +# PATCH_FIX_UPSTREAM: https://github.com/saltstack/salt/pull/53326 +# PATCH_FIX_UPSTREAM: https://github.com/saltstack/salt/pull/54954 +Patch91: accumulated-changes-from-yomi-167.patch +# PATCH_FIX_OPENSUSE: https://github.com/openSUSE/salt/pull/180 +Patch92: fix-a-wrong-rebase-in-test_core.py-180.patch +# PATCH_FIX_OPENSUSE: https://github.com/openSUSE/salt/pull/181 +Patch93: fix-for-older-mock-module.patch -# BuildRoot: %{_tmppath}/%{name}-%{version}-build BuildRoot: %{_tmppath}/%{name}-%{version}-build BuildRequires: logrotate %if 0%{?suse_version} > 1020 @@ -340,9 +360,15 @@ BuildRequires: python-futures >= 2.0 BuildRequires: python-msgpack-python > 0.3 BuildRequires: python-psutil BuildRequires: python-requests >= 1.0.0 +%if 0%{?suse_version} >= 1500 || 0%{?rhel} >= 8 +# We can't cope with tornado 5.x and newer (boo#1101780); this is only relevant for SLE >= 15 and TW +# where tornado exists in multiple versions +BuildRequires: (python-tornado >= 4.2.1 with python-tornado < 5) +%else BuildRequires: python-tornado >= 4.2.1 # We can't cope with tornado 5.x and newer (boo#1101780) BuildConflicts: python3-tornado >= 5 +%endif # requirements/zeromq.txt BuildRequires: python-pycrypto >= 2.6.1 @@ -424,21 +450,31 @@ BuildRequires: python3-devel # requirements/base.txt %if 0%{?rhel} BuildRequires: python3-jinja2 +BuildRequires: python3-markupsafe +BuildRequires: python3-msgpack > 0.3 +BuildRequires: python3-zmq >= 2.2.0 +BuildRequires: python3-m2crypto %else BuildRequires: python3-Jinja2 -%endif BuildRequires: python3-MarkupSafe -BuildRequires: python3-PyYAML BuildRequires: python3-msgpack-python > 0.3 +BuildRequires: python3-pyzmq >= 2.2.0 +BuildRequires: python3-pycrypto >= 2.6.1 +%endif +BuildRequires: python3-PyYAML BuildRequires: python3-psutil BuildRequires: python3-requests >= 1.0.0 +%if 0%{?suse_version} >= 1500 || 0%{?rhel} >= 8 +# We can't cope with tornado 5.x and newer (boo#1101780); this is only relevant for SLE >= 15 and TW, +# where tornado exists in multiple versions +BuildRequires: (python3-tornado >= 4.2.1 with python3-tornado < 5) +%else BuildRequires: python3-tornado >= 4.2.1 # We can't cope with tornado 5.x and newer (boo#1101780) BuildConflicts: python3-tornado >= 5 +%endif # requirements/zeromq.txt -BuildRequires: python3-pycrypto >= 2.6.1 -BuildRequires: python3-pyzmq >= 2.2.0 %if %{with test} # requirements/dev_python27.txt BuildRequires: python3-boto >= 2.32.1 @@ -461,15 +497,24 @@ Requires: python3-certifi %if 0%{?rhel} Requires: python3-jinja2 Requires: yum +Requires: python3-markupsafe +Requires: python3-msgpack > 0.3 +Requires: python3-m2crypto +Requires: python3-zmq >= 2.2.0 +%if 0%{?rhel} == 8 +Requires: dnf +%endif %if 0%{?rhel} == 6 Requires: yum-plugin-security %endif %else Requires: python3-Jinja2 -%endif Requires: python3-MarkupSafe -Requires: python3-PyYAML Requires: python3-msgpack-python > 0.3 +Requires: python3-pycrypto >= 2.6.1 +Requires: python3-pyzmq >= 2.2.0 +%endif +Requires: python3-PyYAML Requires: python3-psutil Requires: python3-requests >= 1.0.0 Requires: python3-tornado >= 4.2.1 @@ -486,8 +531,6 @@ Suggests: python3-timelib Suggests: python3-gnupg # requirements/zeromq.txt %endif -Requires: python3-pycrypto >= 2.6.1 -Requires: python3-pyzmq >= 2.2.0 # %if 0%{?suse_version} # python-xml is part of python-base in all rhel versions @@ -805,6 +848,15 @@ cp %{S:5} ./.travis.yml %patch82 -p1 %patch83 -p1 %patch84 -p1 +%patch85 -p1 +%patch86 -p1 +%patch87 -p1 +%patch88 -p1 +%patch89 -p1 +%patch90 -p1 +%patch91 -p1 +%patch92 -p1 +%patch93 -p1 %build %if 0%{?build_py2} @@ -1311,10 +1363,12 @@ fi %endif %endif +%if 0%{?build_py2} %posttrans -n python2-salt # force re-generate a new thin.tgz rm -f %{_localstatedir}/cache/salt/master/thin/version rm -f %{_localstatedir}/cache/salt/minion/thin/version +%endif %if 0%{?build_py3} %posttrans -n python3-salt @@ -1345,10 +1399,8 @@ rm -f %{_localstatedir}/cache/salt/minion/thin/version %config(noreplace) %attr(0640, root, salt) %{_sysconfdir}/salt/cloud.providers %dir %attr(0750, root, salt) %{_localstatedir}/cache/salt/cloud %if 0%{?default_py3} -%{python3_sitelib}/salt/cloud/deploy/bootstrap-salt.sh %attr(755,root,root)%{python3_sitelib}/salt/cloud/deploy/bootstrap-salt.sh %else -%{python_sitelib}/salt/cloud/deploy/bootstrap-salt.sh %attr(755,root,root)%{python_sitelib}/salt/cloud/deploy/bootstrap-salt.sh %endif %{_mandir}/man1/salt-cloud.1.* @@ -1378,7 +1430,6 @@ rm -f %{_localstatedir}/cache/salt/minion/thin/version %dir %attr(0750, root, root) %{_sysconfdir}/salt/minion.d/ %dir %attr(0750, root, root) %{_sysconfdir}/salt/pki/minion/ %dir %attr(0750, root, root) %{_localstatedir}/cache/salt/minion/ -#%dir %ghost %attr(0750, root, salt) %{_localstatedir}/run/salt/minion %{_sbindir}/rcsalt-minion # Install plugin only on SUSE machines @@ -1450,7 +1501,6 @@ rm -f %{_localstatedir}/cache/salt/minion/thin/version %dir %attr(0750, salt, salt) %{_localstatedir}/cache/salt/master/roots/ %dir %attr(0750, salt, salt) %{_localstatedir}/cache/salt/master/syndics/ %dir %attr(0750, salt, salt) %{_localstatedir}/cache/salt/master/tokens/ -#%dir %ghost %attr(0750, salt, salt) %{_localstatedir}/run/salt/master/ %files %defattr(-,root,root,-) @@ -1473,7 +1523,6 @@ rm -f %{_localstatedir}/cache/salt/minion/thin/version %dir %attr(0750, root, salt) %{_sysconfdir}/salt/pki %dir %attr(0750, salt, salt) %{_localstatedir}/log/salt %dir %attr(0750, root, salt) %{_localstatedir}/cache/salt -#%dir %ghost %attr(0750, root, salt) %{_localstatedir}/run/salt %dir %attr(0750, root, salt) /srv/spm %if %{with systemd} /usr/lib/tmpfiles.d/salt.conf diff --git a/take-checksums-arg-into-account-for-postgres.datadir.patch b/take-checksums-arg-into-account-for-postgres.datadir.patch new file mode 100644 index 0000000..79d6cd6 --- /dev/null +++ b/take-checksums-arg-into-account-for-postgres.datadir.patch @@ -0,0 +1,41 @@ +From 7ed3e99a4979a13c7142ed5ba73c09a282e03147 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Thu, 26 Sep 2019 15:57:58 +0100 +Subject: [PATCH] Take checksums arg into account for + postgres.datadir_init (bsc#1151650) + +Update unit test for postgres.datadir_init +--- + salt/modules/postgres.py | 1 + + tests/unit/modules/test_postgres.py | 1 + + 2 files changed, 2 insertions(+) + +diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py +index b6f7cbe5d4..f0d1b034b9 100644 +--- a/salt/modules/postgres.py ++++ b/salt/modules/postgres.py +@@ -3151,6 +3151,7 @@ def datadir_init(name, + password=password, + encoding=encoding, + locale=locale, ++ checksums=checksums, + runas=runas) + return ret['retcode'] == 0 + +diff --git a/tests/unit/modules/test_postgres.py b/tests/unit/modules/test_postgres.py +index 03fb7fddfd..6f10fcf2e0 100644 +--- a/tests/unit/modules/test_postgres.py ++++ b/tests/unit/modules/test_postgres.py +@@ -1467,6 +1467,7 @@ class PostgresTestCase(TestCase, LoaderModuleMockMixin): + locale=None, + password='test', + runas='postgres', ++ checksums=False, + user='postgres', + ) + self.assertTrue(ret) +-- +2.22.0 + + diff --git a/use-current-ioloop-for-the-localclient-instance-of-b.patch b/use-current-ioloop-for-the-localclient-instance-of-b.patch new file mode 100644 index 0000000..945ff8b --- /dev/null +++ b/use-current-ioloop-for-the-localclient-instance-of-b.patch @@ -0,0 +1,36 @@ +From 55d8a777d6a9b19c959e14a4060e5579e92cd106 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Thu, 3 Oct 2019 15:19:02 +0100 +Subject: [PATCH] Use current IOLoop for the LocalClient instance of + BatchAsync (bsc#1137642) + +--- + salt/cli/batch_async.py | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/salt/cli/batch_async.py b/salt/cli/batch_async.py +index 2bb50459c8..f9e736f804 100644 +--- a/salt/cli/batch_async.py ++++ b/salt/cli/batch_async.py +@@ -52,7 +52,7 @@ class BatchAsync(object): + ''' + def __init__(self, parent_opts, jid_gen, clear_load): + ioloop = tornado.ioloop.IOLoop.current() +- self.local = salt.client.get_local_client(parent_opts['conf_file']) ++ self.local = salt.client.get_local_client(parent_opts['conf_file'], io_loop=ioloop) + if 'gather_job_timeout' in clear_load['kwargs']: + clear_load['gather_job_timeout'] = clear_load['kwargs'].pop('gather_job_timeout') + else: +@@ -266,6 +266,7 @@ class BatchAsync(object): + yield + + def __del__(self): ++ self.local = None + self.event = None + self.ioloop = None + gc.collect() +-- +2.22.0 + +