From 04ffd31d2e5b93d714216c55f5db91d05a23bd0df2db54fa7a1a0088278af72d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= Date: Tue, 7 May 2019 15:40:32 +0000 Subject: [PATCH 1/2] osc copypac from project:systemsmanagement:saltstack:testing package:salt revision:265 OBS-URL: https://build.opensuse.org/package/show/systemsmanagement:saltstack/salt?expand=0&rev=139 --- _lastrevision | 2 +- ...n-there-are-ipv6-established-connect.patch | 90 +++++++++++++++++++ salt.changes | 9 ++ salt.spec | 3 + 4 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 do-not-crash-when-there-are-ipv6-established-connect.patch diff --git a/_lastrevision b/_lastrevision index 43041da..1d70143 100644 --- a/_lastrevision +++ b/_lastrevision @@ -1 +1 @@ -8d79ae9a816ab27810786c5a4a60021af08ec366 \ No newline at end of file +9cbb16539a0cce2bd2d423ca440ea8b142cc13ea \ No newline at end of file diff --git a/do-not-crash-when-there-are-ipv6-established-connect.patch b/do-not-crash-when-there-are-ipv6-established-connect.patch new file mode 100644 index 0000000..721530d --- /dev/null +++ b/do-not-crash-when-there-are-ipv6-established-connect.patch @@ -0,0 +1,90 @@ +From f185eabfb4b529157cf7464b32beebeb8b944310 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Tue, 7 May 2019 15:33:51 +0100 +Subject: [PATCH] Do not crash when there are IPv6 established + connections (bsc#1130784) + +Add unit test for '_netlink_tool_remote_on' +--- + salt/utils/network.py | 9 +++++---- + tests/unit/utils/test_network.py | 16 ++++++++++++++++ + 2 files changed, 21 insertions(+), 4 deletions(-) + +diff --git a/salt/utils/network.py b/salt/utils/network.py +index c72d2aec41..3f0522b9a5 100644 +--- a/salt/utils/network.py ++++ b/salt/utils/network.py +@@ -1457,7 +1457,7 @@ def _parse_tcp_line(line): + + def _netlink_tool_remote_on(port, which_end): + ''' +- Returns set of ipv4 host addresses of remote established connections ++ Returns set of IPv4/IPv6 host addresses of remote established connections + on local or remote tcp port. + + Parses output of shell 'ss' to get connections +@@ -1467,6 +1467,7 @@ def _netlink_tool_remote_on(port, which_end): + LISTEN 0 511 *:80 *:* + LISTEN 0 128 *:22 *:* + ESTAB 0 0 127.0.0.1:56726 127.0.0.1:4505 ++ ESTAB 0 0 [::ffff:127.0.0.1]:41323 [::ffff:127.0.0.1]:4505 + ''' + remotes = set() + valid = False +@@ -1486,14 +1487,14 @@ def _netlink_tool_remote_on(port, which_end): + elif 'ESTAB' not in line: + continue + chunks = line.split() +- local_host, local_port = chunks[3].split(':', 1) +- remote_host, remote_port = chunks[4].split(':', 1) ++ local_host, local_port = chunks[3].rsplit(':', 1) ++ remote_host, remote_port = chunks[4].rsplit(':', 1) + + if which_end == 'remote_port' and int(remote_port) != port: + continue + if which_end == 'local_port' and int(local_port) != port: + continue +- remotes.add(remote_host) ++ remotes.add(remote_host.strip("[]")) + + if valid is False: + remotes = None +diff --git a/tests/unit/utils/test_network.py b/tests/unit/utils/test_network.py +index ca627777a7..ecf7d7c45b 100644 +--- a/tests/unit/utils/test_network.py ++++ b/tests/unit/utils/test_network.py +@@ -120,6 +120,14 @@ USER COMMAND PID FD PROTO LOCAL ADDRESS FOREIGN ADDRESS + salt-master python2.781106 35 tcp4 127.0.0.1:61115 127.0.0.1:4506 + ''' + ++LINUX_NETLINK_SS_OUTPUT = '''\ ++State Recv-Q Send-Q Local Address:Port Peer Address:Port ++TIME-WAIT 0 0 [::1]:8009 [::1]:40368 ++LISTEN 0 128 127.0.0.1:5903 0.0.0.0:* ++ESTAB 0 0 [::ffff:127.0.0.1]:4506 [::ffff:127.0.0.1]:32315 ++ESTAB 0 0 192.168.122.1:4506 192.168.122.177:24545 ++''' ++ + IPV4_SUBNETS = {True: ('10.10.0.0/24',), + False: ('10.10.0.0', '10.10.0.0/33', 'FOO', 9, '0.9.800.1000/24')} + IPV6_SUBNETS = {True: ('::1/128',), +@@ -453,6 +461,14 @@ class NetworkTestCase(TestCase): + remotes = network._freebsd_remotes_on('4506', 'remote') + self.assertEqual(remotes, set(['127.0.0.1'])) + ++ def test_netlink_tool_remote_on(self): ++ with patch('salt.utils.platform.is_sunos', lambda: False): ++ with patch('salt.utils.platform.is_linux', lambda: True): ++ with patch('subprocess.check_output', ++ return_value=LINUX_NETLINK_SS_OUTPUT): ++ remotes = network._netlink_tool_remote_on('4506', 'local') ++ self.assertEqual(remotes, set(['192.168.122.177', '::ffff:127.0.0.1'])) ++ + def test_generate_minion_id_distinct(self): + ''' + Test if minion IDs are distinct in the pool. +-- +2.17.1 + + diff --git a/salt.changes b/salt.changes index 66cb04c..db4764e 100644 --- a/salt.changes +++ b/salt.changes @@ -1,3 +1,12 @@ +------------------------------------------------------------------- +Tue May 7 15:37:39 UTC 2019 - psuarezhernandez@suse.com + +- Do not make Salt CLI to crash when there are IPv6 established + connections (bsc#1130784) + +- Added: + * do-not-crash-when-there-are-ipv6-established-connect.patch + ------------------------------------------------------------------- Fri May 3 09:42:06 UTC 2019 - mdinca diff --git a/salt.spec b/salt.spec index e4f8046..bd6107a 100644 --- a/salt.spec +++ b/salt.spec @@ -169,6 +169,8 @@ Patch50: do-not-report-patches-as-installed-when-not-all-the-.patch Patch51: use-threadpool-from-multiprocessing.pool-to-avoid-le.patch # PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/52519 (partial porting) Patch52: fix-syndic-start-issue.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/52888 +Patch53: do-not-crash-when-there-are-ipv6-established-connect.patch # BuildRoot: %{_tmppath}/%{name}-%{version}-build BuildRoot: %{_tmppath}/%{name}-%{version}-build @@ -679,6 +681,7 @@ cp %{S:5} ./.travis.yml %patch50 -p1 %patch51 -p1 %patch52 -p1 +%patch53 -p1 %build %if 0%{?build_py2} From a19f8488589e1e23a5f67eb65d18d022e50dad02fae459a4c10995e59c9bdf0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mihai=20Dinc=C4=83?= Date: Wed, 8 May 2019 14:50:44 +0000 Subject: [PATCH 2/2] osc copypac from project:systemsmanagement:saltstack:testing package:salt revision:266 OBS-URL: https://build.opensuse.org/package/show/systemsmanagement:saltstack/salt?expand=0&rev=140 --- _lastrevision | 2 +- fix-async-batch-multiple-done-events.patch | 138 +++++++++++++++++++++ salt.changes | 8 ++ salt.spec | 4 + 4 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 fix-async-batch-multiple-done-events.patch diff --git a/_lastrevision b/_lastrevision index 1d70143..7b5d4c0 100644 --- a/_lastrevision +++ b/_lastrevision @@ -1 +1 @@ -9cbb16539a0cce2bd2d423ca440ea8b142cc13ea \ No newline at end of file +8fe9649055af571bfa44483318fa5a8476035001 \ No newline at end of file diff --git a/fix-async-batch-multiple-done-events.patch b/fix-async-batch-multiple-done-events.patch new file mode 100644 index 0000000..f399bff --- /dev/null +++ b/fix-async-batch-multiple-done-events.patch @@ -0,0 +1,138 @@ +From 2dcee9c2773f588cc5ca040b1d22c1e8036dcbf7 Mon Sep 17 00:00:00 2001 +From: Mihai Dinca +Date: Tue, 7 May 2019 12:24:35 +0200 +Subject: [PATCH] Fix async-batch multiple done events + +--- + salt/cli/batch_async.py | 17 ++++++++++++----- + tests/unit/cli/test_batch_async.py | 20 +++++++++++++------- + 2 files changed, 25 insertions(+), 12 deletions(-) + +diff --git a/salt/cli/batch_async.py b/salt/cli/batch_async.py +index 9c20b2fc6e..8c8f481e34 100644 +--- a/salt/cli/batch_async.py ++++ b/salt/cli/batch_async.py +@@ -84,6 +84,7 @@ class BatchAsync(object): + listen=True, + io_loop=ioloop, + keep_loop=True) ++ self.scheduled = False + + def __set_event_handler(self): + ping_return_pattern = 'salt/job/{0}/ret/*'.format(self.ping_jid) +@@ -116,8 +117,7 @@ class BatchAsync(object): + if minion in self.active: + self.active.remove(minion) + self.done_minions.add(minion) +- # call later so that we maybe gather more returns +- self.event.io_loop.call_later(self.batch_delay, self.schedule_next) ++ self.schedule_next() + + def _get_next(self): + to_run = self.minions.difference( +@@ -137,7 +137,7 @@ class BatchAsync(object): + self.active = self.active.difference(self.timedout_minions) + running = batch_minions.difference(self.done_minions).difference(self.timedout_minions) + if timedout_minions: +- self.event.io_loop.call_later(self.batch_delay, self.schedule_next) ++ self.schedule_next() + if running: + self.event.io_loop.add_callback(self.find_job, running) + +@@ -189,7 +189,7 @@ class BatchAsync(object): + "metadata": self.metadata + } + self.event.fire_event(data, "salt/batch/{0}/start".format(self.batch_jid)) +- yield self.schedule_next() ++ yield self.run_next() + + def end_batch(self): + left = self.minions.symmetric_difference(self.done_minions.union(self.timedout_minions)) +@@ -204,8 +204,14 @@ class BatchAsync(object): + self.event.fire_event(data, "salt/batch/{0}/done".format(self.batch_jid)) + self.event.remove_event_handler(self.__event_handler) + +- @tornado.gen.coroutine + def schedule_next(self): ++ if not self.scheduled: ++ self.scheduled = True ++ # call later so that we maybe gather more returns ++ self.event.io_loop.call_later(self.batch_delay, self.run_next) ++ ++ @tornado.gen.coroutine ++ def run_next(self): + next_batch = self._get_next() + if next_batch: + self.active = self.active.union(next_batch) +@@ -225,3 +231,4 @@ class BatchAsync(object): + self.active = self.active.difference(next_batch) + else: + self.end_batch() ++ self.scheduled = False +diff --git a/tests/unit/cli/test_batch_async.py b/tests/unit/cli/test_batch_async.py +index d519157d92..441f9c58b9 100644 +--- a/tests/unit/cli/test_batch_async.py ++++ b/tests/unit/cli/test_batch_async.py +@@ -111,14 +111,14 @@ class AsyncBatchTestCase(AsyncTestCase, TestCase): + + @tornado.testing.gen_test + def test_start_batch_calls_next(self): +- self.batch.schedule_next = MagicMock(return_value=MagicMock()) ++ self.batch.run_next = MagicMock(return_value=MagicMock()) + self.batch.event = MagicMock() + future = tornado.gen.Future() + future.set_result(None) +- self.batch.schedule_next = MagicMock(return_value=future) ++ self.batch.run_next = MagicMock(return_value=future) + self.batch.start_batch() + self.assertEqual(self.batch.initialized, True) +- self.assertEqual(len(self.batch.schedule_next.mock_calls), 1) ++ self.assertEqual(len(self.batch.run_next.mock_calls), 1) + + def test_batch_fire_done_event(self): + self.batch.targeted_minions = {'foo', 'baz', 'bar'} +@@ -154,7 +154,7 @@ class AsyncBatchTestCase(AsyncTestCase, TestCase): + future = tornado.gen.Future() + future.set_result({'minions': ['foo', 'bar']}) + self.batch.local.run_job_async.return_value = future +- ret = self.batch.schedule_next().result() ++ ret = self.batch.run_next().result() + self.assertEqual( + self.batch.local.run_job_async.call_args[0], + ({'foo', 'bar'}, 'my.fun', [], 'list') +@@ -253,7 +253,7 @@ class AsyncBatchTestCase(AsyncTestCase, TestCase): + self.assertEqual(self.batch.done_minions, {'foo'}) + self.assertEqual( + self.batch.event.io_loop.call_later.call_args[0], +- (self.batch.batch_delay, self.batch.schedule_next)) ++ (self.batch.batch_delay, self.batch.run_next)) + + def test_batch__event_handler_find_job_return(self): + self.batch.event = MagicMock( +@@ -263,10 +263,10 @@ class AsyncBatchTestCase(AsyncTestCase, TestCase): + self.assertEqual(self.batch.find_job_returned, {'foo'}) + + @tornado.testing.gen_test +- def test_batch_schedule_next_end_batch_when_no_next(self): ++ def test_batch_run_next_end_batch_when_no_next(self): + self.batch.end_batch = MagicMock() + self.batch._get_next = MagicMock(return_value={}) +- self.batch.schedule_next() ++ self.batch.run_next() + self.assertEqual(len(self.batch.end_batch.mock_calls), 1) + + @tornado.testing.gen_test +@@ -342,3 +342,9 @@ class AsyncBatchTestCase(AsyncTestCase, TestCase): + self.batch.event.io_loop.add_callback.call_args[0], + (self.batch.find_job, {'foo'}) + ) ++ ++ def test_only_on_run_next_is_scheduled(self): ++ self.batch.event = MagicMock() ++ self.batch.scheduled = True ++ self.batch.schedule_next() ++ self.assertEqual(len(self.batch.event.io_loop.call_later.mock_calls), 0) +-- +2.21.0 + + diff --git a/salt.changes b/salt.changes index db4764e..270d245 100644 --- a/salt.changes +++ b/salt.changes @@ -1,3 +1,11 @@ +------------------------------------------------------------------- +Wed May 8 08:48:49 UTC 2019 - Mihai Dincă + +- Fix async-batch to fire a single done event + +- Added: + * fix-async-batch-multiple-done-events.patch + ------------------------------------------------------------------- Tue May 7 15:37:39 UTC 2019 - psuarezhernandez@suse.com diff --git a/salt.spec b/salt.spec index bd6107a..2e96e7b 100644 --- a/salt.spec +++ b/salt.spec @@ -171,6 +171,9 @@ Patch51: use-threadpool-from-multiprocessing.pool-to-avoid-le.patch Patch52: fix-syndic-start-issue.patch # PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/52888 Patch53: do-not-crash-when-there-are-ipv6-established-connect.patch +# PATCH-FIX_OPENSUSE: https://github.com/openSUSE/salt/pull/144 +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/52855 +Patch54: fix-async-batch-multiple-done-events.patch # BuildRoot: %{_tmppath}/%{name}-%{version}-build BuildRoot: %{_tmppath}/%{name}-%{version}-build @@ -682,6 +685,7 @@ cp %{S:5} ./.travis.yml %patch51 -p1 %patch52 -p1 %patch53 -p1 +%patch54 -p1 %build %if 0%{?build_py2}