pool/salt
SHA256
2
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Accepting request 1114819 from home:vizhestkov:branches:systemsmanagement:saltstack

Browse Source 
- Fix optimization_order opt to prevent testsuite fails
- Imporve salt.utils.json.find_json to avoid fails
  with transactional salt salt-ssh managed clients (bsc#1213293)
- Use salt-call from salt bundle with transactional_update
- Only call native_str on curl_debug message in tornado when needed
- Implement the calling for batch async from the salt CLI
- Added:
  * implement-the-calling-for-batch-async-from-the-salt-.patch
  * imporve-salt.utils.json.find_json-bsc-1213293.patch
  * only-call-native_str-on-curl_debug-message-in-tornad.patch
  * use-salt-call-from-salt-bundle-with-transactional_up.patch
  * fix-optimization_order-opt-to-prevent-test-fails.patch

OBS-URL: https://build.opensuse.org/request/show/1114819
OBS-URL: https://build.opensuse.org/package/show/systemsmanagement:saltstack/salt?expand=0&rev=218
This commit is contained in:
Pablo Suárez Hernández 2023-10-02 14:14:56 +00:00 committed by Git OBS Bridge
parent a4148647f7
commit 40425ec176
8 changed files with 573 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
_lastrevision
View File

@ -1 +1 @@
d9e734d2a130c3a4f5e14f8e2e00be98052a41f9 283e739d11222a69bc566092b1b9d0d43ae59a52

62
fix-optimization_order-opt-to-prevent-test-fails.patch Normal file
View File

@ -0,0 +1,62 @@
From aaf593d17f51a517e0adb6e9ec1c0d768ab5f855 Mon Sep 17 00:00:00 2001
From: Victor Zhestkov <vzhestkov@suse.com>
Date: Mon, 2 Oct 2023 14:24:27 +0200
Subject: [PATCH] Fix optimization_order opt to prevent test fails
---
tests/pytests/unit/grains/test_core.py | 4 ++--
tests/pytests/unit/loader/test_loader.py | 2 +-
tests/pytests/unit/test_config.py | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/tests/pytests/unit/grains/test_core.py b/tests/pytests/unit/grains/test_core.py
index 993c723950..36545287b9 100644
--- a/tests/pytests/unit/grains/test_core.py
+++ b/tests/pytests/unit/grains/test_core.py
@@ -156,7 +156,7 @@ def test_network_grains_secondary_ip(tmp_path):
opts = {
"cachedir": str(cache_dir),
"extension_modules": str(extmods),
- "optimization_order": [0],
+ "optimization_order": [0, 1, 2],
}
with patch("salt.utils.network.interfaces", side_effect=[data]):
grains = salt.loader.grain_funcs(opts)
@@ -243,7 +243,7 @@ def test_network_grains_cache(tmp_path):
opts = {
"cachedir": str(cache_dir),
"extension_modules": str(extmods),
- "optimization_order": [0],
+ "optimization_order": [0, 1, 2],
}
with patch(
"salt.utils.network.interfaces", side_effect=[call_1, call_2]
diff --git a/tests/pytests/unit/loader/test_loader.py b/tests/pytests/unit/loader/test_loader.py
index f4a4b51a58..86348749db 100644
--- a/tests/pytests/unit/loader/test_loader.py
+++ b/tests/pytests/unit/loader/test_loader.py
@@ -57,7 +57,7 @@ def test_raw_mod_functions():
"Ensure functions loaded by raw_mod are LoaderFunc instances"
opts = {
"extension_modules": "",
- "optimization_order": [0],
+ "optimization_order": [0, 1, 2],
}
ret = salt.loader.raw_mod(opts, "grains", "get")
for k, v in ret.items():
diff --git a/tests/pytests/unit/test_config.py b/tests/pytests/unit/test_config.py
index cb343cb75e..76d5605360 100644
--- a/tests/pytests/unit/test_config.py
+++ b/tests/pytests/unit/test_config.py
@@ -16,7 +16,7 @@ def test_call_id_function(tmp_path):
"cachedir": str(cache_dir),
"extension_modules": str(extmods),
"grains": {"osfinger": "meh"},
- "optimization_order": [0],
+ "optimization_order": [0, 1, 2],
}
ret = salt.config.call_id_function(opts)
assert ret == "meh"
--
2.42.0

145
implement-the-calling-for-batch-async-from-the-salt-.patch Normal file
View File

@ -0,0 +1,145 @@
From 7ab208fd2d23eaa582cdbba912d4538d8c87e5f4 Mon Sep 17 00:00:00 2001
From: Victor Zhestkov <vzhestkov@suse.com>
Date: Mon, 2 Oct 2023 13:24:15 +0200
Subject: [PATCH] Implement the calling for batch async from the salt
CLI
* Implement calling batch async with salt CLI
* Add the test for calling batch async with salt CLI
---
salt/cli/salt.py | 53 ++++++++++++++++++++++++++++-
tests/pytests/unit/cli/test_salt.py | 50 +++++++++++++++++++++++++++
2 files changed, 102 insertions(+), 1 deletion(-)
create mode 100644 tests/pytests/unit/cli/test_salt.py
diff --git a/salt/cli/salt.py b/salt/cli/salt.py
index f90057f668..e19cfa5ce6 100644
--- a/salt/cli/salt.py
+++ b/salt/cli/salt.py
@@ -47,7 +47,12 @@ class SaltCMD(salt.utils.parsers.SaltCMDOptionParser):
self.exit(2, "{}\n".format(exc))
return
- if self.options.batch or self.options.static:
+ if self.options.batch and self.config["async"]:
+ # _run_batch_async() will just return the jid and exit
+ # Execution will not continue past this point
+ # in batch async mode. Batch async is handled by the master.
+ self._run_batch_async()
+ elif self.options.batch or self.options.static:
# _run_batch() will handle all output and
# exit with the appropriate error condition
# Execution will not continue past this point
@@ -296,6 +301,52 @@ class SaltCMD(salt.utils.parsers.SaltCMDOptionParser):
retcode = job_retcode
sys.exit(retcode)
+ def _run_batch_async(self):
+ kwargs = {
+ "tgt": self.config["tgt"],
+ "fun": self.config["fun"],
+ "arg": self.config["arg"],
+ "timeout": self.options.timeout,
+ "show_timeout": self.options.show_timeout,
+ "show_jid": self.options.show_jid,
+ "batch": self.config["batch"],
+ }
+ tgt = kwargs.pop("tgt", "")
+ fun = kwargs.pop("fun", "")
+
+ if self.config.get("eauth", ""):
+ kwargs.update(
+ {
+ "eauth": self.config["eauth"],
+ }
+ )
+ for opt in ("username", "password"):
+ if opt in self.config:
+ kwargs[opt] = self.config[opt]
+
+ try:
+ ret = self.local_client.run_job(tgt, fun, **kwargs)
+ except (
+ AuthenticationError,
+ AuthorizationError,
+ SaltInvocationError,
+ EauthAuthenticationError,
+ SaltClientError,
+ ) as exc:
+ ret = str(exc)
+ self.exit(2, "ERROR: {}\n".format(exc))
+ if "jid" in ret and "error" not in ret:
+ salt.utils.stringutils.print_cli(
+ "Executed command with job ID: {}".format(ret["jid"])
+ )
+ else:
+ self._output_ret(ret, self.config.get("output", "nested"))
+
+ if "error" in ret:
+ sys.exit(1)
+
+ sys.exit(0)
+
def _print_errors_summary(self, errors):
if errors:
salt.utils.stringutils.print_cli("\n")
diff --git a/tests/pytests/unit/cli/test_salt.py b/tests/pytests/unit/cli/test_salt.py
new file mode 100644
index 0000000000..d9f4b5b097
--- /dev/null
+++ b/tests/pytests/unit/cli/test_salt.py
@@ -0,0 +1,50 @@
+import pytest
+
+from tests.support.mock import MagicMock, patch
+
+
+def test_saltcmd_batch_async_call():
+ """
+ Test calling batch async with salt CLI
+ """
+ import salt.cli.salt
+
+ local_client = MagicMock()
+ local_client.run_job = MagicMock(return_value={"jid": 123456})
+ with pytest.raises(SystemExit) as exit_info, patch(
+ "sys.argv",
+ [
+ "salt",
+ "--batch=10",
+ "--async",
+ "*",
+ "test.arg",
+ "arg1",
+ "arg2",
+ "kwarg1=val1",
+ ],
+ ), patch("salt.cli.salt.SaltCMD.process_config_dir", MagicMock), patch(
+ "salt.output.display_output", MagicMock()
+ ), patch(
+ "salt.client.get_local_client", return_value=local_client
+ ), patch(
+ "salt.utils.stringutils.print_cli", MagicMock()
+ ) as print_cli:
+ salt_cmd = salt.cli.salt.SaltCMD()
+ salt_cmd.config = {
+ "async": True,
+ "batch": 10,
+ "tgt": "*",
+ "fun": "test.arg",
+ "arg": ["arg1", "arg2", {"__kwarg__": True, "kwarg1": "val1"}],
+ }
+ salt_cmd._mixin_after_parsed_funcs = []
+ salt_cmd.run()
+
+ local_client.run_job.assert_called_once()
+ assert local_client.run_job.mock_calls[0].args[0] == "*"
+ assert local_client.run_job.mock_calls[0].args[1] == "test.arg"
+ assert local_client.run_job.mock_calls[0].kwargs["arg"] == ["arg1", "arg2", {"__kwarg__": True, "kwarg1": "val1"}]
+ assert local_client.run_job.mock_calls[0].kwargs["batch"] == 10
+ print_cli.assert_called_once_with("Executed command with job ID: 123456")
+ assert exit_info.value.code == 0
--
2.42.0

204
imporve-salt.utils.json.find_json-bsc-1213293.patch Normal file
View File

@ -0,0 +1,204 @@
From d5a36df8db87be578b6bf258ff2dd5bf9b234e79 Mon Sep 17 00:00:00 2001
From: Victor Zhestkov <vzhestkov@suse.com>
Date: Mon, 2 Oct 2023 13:26:20 +0200
Subject: [PATCH] Imporve salt.utils.json.find_json (bsc#1213293)
* Improve salt.utils.json.find_json
* Move tests of find_json to pytest
---
salt/utils/json.py | 39 +++++++-
tests/pytests/unit/utils/test_json.py | 122 ++++++++++++++++++++++++++
2 files changed, 158 insertions(+), 3 deletions(-)
create mode 100644 tests/pytests/unit/utils/test_json.py
diff --git a/salt/utils/json.py b/salt/utils/json.py
index 33cdbf401d..0845b64694 100644
--- a/salt/utils/json.py
+++ b/salt/utils/json.py
@@ -32,18 +32,51 @@ def find_json(raw):
"""
ret = {}
lines = __split(raw)
+ lengths = list(map(len, lines))
+ starts = []
+ ends = []
+
+ # Search for possible starts end ends of the json fragments
for ind, _ in enumerate(lines):
+ line = lines[ind].lstrip()
+ if line == "{" or line == "[":
+ starts.append((ind, line))
+ if line == "}" or line == "]":
+ ends.append((ind, line))
+
+ # List all the possible pairs of starts and ends,
+ # and fill the length of each block to sort by size after
+ starts_ends = []
+ for start, start_br in starts:
+ for end, end_br in reversed(ends):
+ if end > start and (
+ (start_br == "{" and end_br == "}")
+ or (start_br == "[" and end_br == "]")
+ ):
+ starts_ends.append((start, end, sum(lengths[start : end + 1])))
+
+ # Iterate through all the possible pairs starting from the largest
+ starts_ends.sort(key=lambda x: (x[2], x[1] - x[0], x[0]), reverse=True)
+ for start, end, _ in starts_ends:
+ working = "\n".join(lines[start : end + 1])
try:
- working = "\n".join(lines[ind:])
- except UnicodeDecodeError:
- working = "\n".join(salt.utils.data.decode(lines[ind:]))
+ ret = json.loads(working)
+ except ValueError:
+ continue
+ if ret:
+ return ret
+ # Fall back to old implementation for backward compatibility
+ # excpecting json after the text
+ for ind, _ in enumerate(lines):
+ working = "\n".join(lines[ind:])
try:
ret = json.loads(working)
except ValueError:
continue
if ret:
return ret
+
if not ret:
# Not json, raise an error
raise ValueError
diff --git a/tests/pytests/unit/utils/test_json.py b/tests/pytests/unit/utils/test_json.py
new file mode 100644
index 0000000000..72b1023003
--- /dev/null
+++ b/tests/pytests/unit/utils/test_json.py
@@ -0,0 +1,122 @@
+"""
+Tests for salt.utils.json
+"""
+
+import textwrap
+
+import pytest
+
+import salt.utils.json
+
+
+def test_find_json():
+ some_junk_text = textwrap.dedent(
+ """
+ Just some junk text
+ with multiline
+ """
+ )
+ some_warning_message = textwrap.dedent(
+ """
+ [WARNING] Test warning message
+ """
+ )
+ test_small_json = textwrap.dedent(
+ """
+ {
+ "local": true
+ }
+ """
+ )
+ test_sample_json = """
+ {
+ "glossary": {
+ "title": "example glossary",
+ "GlossDiv": {
+ "title": "S",
+ "GlossList": {
+ "GlossEntry": {
+ "ID": "SGML",
+ "SortAs": "SGML",
+ "GlossTerm": "Standard Generalized Markup Language",
+ "Acronym": "SGML",
+ "Abbrev": "ISO 8879:1986",
+ "GlossDef": {
+ "para": "A meta-markup language, used to create markup languages such as DocBook.",
+ "GlossSeeAlso": ["GML", "XML"]
+ },
+ "GlossSee": "markup"
+ }
+ }
+ }
+ }
+ }
+ """
+ expected_ret = {
+ "glossary": {
+ "GlossDiv": {
+ "GlossList": {
+ "GlossEntry": {
+ "GlossDef": {
+ "GlossSeeAlso": ["GML", "XML"],
+ "para": (
+ "A meta-markup language, used to create markup"
+ " languages such as DocBook."
+ ),
+ },
+ "GlossSee": "markup",
+ "Acronym": "SGML",
+ "GlossTerm": "Standard Generalized Markup Language",
+ "SortAs": "SGML",
+ "Abbrev": "ISO 8879:1986",
+ "ID": "SGML",
+ }
+ },
+ "title": "S",
+ },
+ "title": "example glossary",
+ }
+ }
+
+ # First test the valid JSON
+ ret = salt.utils.json.find_json(test_sample_json)
+ assert ret == expected_ret
+
+ # Now pre-pend some garbage and re-test
+ garbage_prepend_json = f"{some_junk_text}{test_sample_json}"
+ ret = salt.utils.json.find_json(garbage_prepend_json)
+ assert ret == expected_ret
+
+ # Now post-pend some garbage and re-test
+ garbage_postpend_json = f"{test_sample_json}{some_junk_text}"
+ ret = salt.utils.json.find_json(garbage_postpend_json)
+ assert ret == expected_ret
+
+ # Now pre-pend some warning and re-test
+ warning_prepend_json = f"{some_warning_message}{test_sample_json}"
+ ret = salt.utils.json.find_json(warning_prepend_json)
+ assert ret == expected_ret
+
+ # Now post-pend some warning and re-test
+ warning_postpend_json = f"{test_sample_json}{some_warning_message}"
+ ret = salt.utils.json.find_json(warning_postpend_json)
+ assert ret == expected_ret
+
+ # Now put around some garbage and re-test
+ garbage_around_json = f"{some_junk_text}{test_sample_json}{some_junk_text}"
+ ret = salt.utils.json.find_json(garbage_around_json)
+ assert ret == expected_ret
+
+ # Now pre-pend small json and re-test
+ small_json_pre_json = f"{test_small_json}{test_sample_json}"
+ ret = salt.utils.json.find_json(small_json_pre_json)
+ assert ret == expected_ret
+
+ # Now post-pend small json and re-test
+ small_json_post_json = f"{test_sample_json}{test_small_json}"
+ ret = salt.utils.json.find_json(small_json_post_json)
+ assert ret == expected_ret
+
+ # Test to see if a ValueError is raised if no JSON is passed in
+ with pytest.raises(ValueError):
+ ret = salt.utils.json.find_json(some_junk_text)
--
2.42.0

31
only-call-native_str-on-curl_debug-message-in-tornad.patch Normal file
View File

@ -0,0 +1,31 @@
From b76b74bd9640adf3b6798e4de4b89aaa7af62c9f Mon Sep 17 00:00:00 2001
From: Victor Zhestkov <vzhestkov@suse.com>
Date: Mon, 2 Oct 2023 13:24:43 +0200
Subject: [PATCH] Only call native_str on curl_debug message in tornado
when needed
Co-authored-by: Ben Darnell <ben@bendarnell.com>
---
salt/ext/tornado/curl_httpclient.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/salt/ext/tornado/curl_httpclient.py b/salt/ext/tornado/curl_httpclient.py
index 8652343cf7..9e4133fd13 100644
--- a/salt/ext/tornado/curl_httpclient.py
+++ b/salt/ext/tornado/curl_httpclient.py
@@ -494,10 +494,11 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
def _curl_debug(self, debug_type, debug_msg):
debug_types = ('I', '<', '>', '<', '>')
- debug_msg = native_str(debug_msg)
if debug_type == 0:
+ debug_msg = native_str(debug_msg)
curl_log.debug('%s', debug_msg.strip())
elif debug_type in (1, 2):
+ debug_msg = native_str(debug_msg)
for line in debug_msg.splitlines():
curl_log.debug('%s %s', debug_types[debug_type], line)
elif debug_type == 4:
--
2.42.0

17
salt.changes
View File

@ -1,3 +1,20 @@
-------------------------------------------------------------------
Mon Oct 2 12:49:41 UTC 2023 - Victor Zhestkov <vzhestkov@suse.com>
- Fix optimization_order opt to prevent testsuite fails
- Imporve salt.utils.json.find_json to avoid fails
with transactional salt salt-ssh managed clients (bsc#1213293)
- Use salt-call from salt bundle with transactional_update
- Only call native_str on curl_debug message in tornado when needed
- Implement the calling for batch async from the salt CLI
- Added:
* implement-the-calling-for-batch-async-from-the-salt-.patch
* imporve-salt.utils.json.find_json-bsc-1213293.patch
* only-call-native_str-on-curl_debug-message-in-tornad.patch
* use-salt-call-from-salt-bundle-with-transactional_up.patch
* fix-optimization_order-opt-to-prevent-test-fails.patch
------------------------------------------------------------------- -------------------------------------------------------------------
Mon Oct 2 09:50:00 UTC 2023 - Pablo Suárez Hernández <pablo.suarezhernandez@suse.com> Mon Oct 2 09:50:00 UTC 2023 - Pablo Suárez Hernández <pablo.suarezhernandez@suse.com>

10
salt.spec
View File

@ -306,6 +306,16 @@ Patch79: revert-usage-of-long-running-req-channel-bsc-1213960.patch
Patch80: write-salt-version-before-building-when-using-with-s.patch Patch80: write-salt-version-before-building-when-using-with-s.patch
# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65036 # PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65036
Patch81: fix-calculation-of-sls-context-vars-when-trailing-do.patch Patch81: fix-calculation-of-sls-context-vars-when-trailing-do.patch
# PATCH-FIX_OPENSUSE: https://github.com/openSUSE/salt/pull/594
Patch82: implement-the-calling-for-batch-async-from-the-salt-.patch
# PATCH-FIX_UPSTREAM: https://github.com/tornadoweb/tornado/pull/2277
Patch83: only-call-native_str-on-curl_debug-message-in-tornad.patch
# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65204
Patch84: use-salt-call-from-salt-bundle-with-transactional_up.patch
# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65181
Patch85: imporve-salt.utils.json.find_json-bsc-1213293.patch
# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65266
Patch86: fix-optimization_order-opt-to-prevent-test-fails.patch
### IMPORTANT: The line below is used as a snippet marker. Do not touch it. ### IMPORTANT: The line below is used as a snippet marker. Do not touch it.
### SALT PATCHES LIST END ### SALT PATCHES LIST END

103
use-salt-call-from-salt-bundle-with-transactional_up.patch Normal file
View File

@ -0,0 +1,103 @@
From 0459d3f711eb9898f56a97d0bf0eb66fd1421a56 Mon Sep 17 00:00:00 2001
From: Victor Zhestkov <vzhestkov@suse.com>
Date: Mon, 2 Oct 2023 13:25:52 +0200
Subject: [PATCH] Use salt-call from salt bundle with
transactional_update
* Use salt-call from the bundle with transactional_update
* Add test checking which salt-call is selected by executable
---
salt/modules/transactional_update.py | 13 +++++-
.../unit/modules/test_transactional_update.py | 44 +++++++++++++++++++
2 files changed, 56 insertions(+), 1 deletion(-)
diff --git a/salt/modules/transactional_update.py b/salt/modules/transactional_update.py
index 658ebccc6b..d6915475f5 100644
--- a/salt/modules/transactional_update.py
+++ b/salt/modules/transactional_update.py
@@ -276,6 +276,9 @@ transaction.
"""
import logging
+import os.path
+import pathlib
+import sys
import salt.client.ssh.state
import salt.client.ssh.wrapper.state
@@ -941,10 +944,18 @@ def call(function, *args, **kwargs):
activate_transaction = kwargs.pop("activate_transaction", False)
try:
+ # Set default salt-call command
+ salt_call_cmd = "salt-call"
+ python_exec_dir = os.path.dirname(sys.executable)
+ if "venv-salt-minion" in pathlib.Path(python_exec_dir).parts:
+ # If the module is executed with the Salt Bundle,
+ # use salt-call from the Salt Bundle
+ salt_call_cmd = os.path.join(python_exec_dir, "salt-call")
+
safe_kwargs = salt.utils.args.clean_kwargs(**kwargs)
salt_argv = (
[
- "salt-call",
+ salt_call_cmd,
"--out",
"json",
"-l",
diff --git a/tests/pytests/unit/modules/test_transactional_update.py b/tests/pytests/unit/modules/test_transactional_update.py
index 5d9294c49b..dbd72fd74b 100644
--- a/tests/pytests/unit/modules/test_transactional_update.py
+++ b/tests/pytests/unit/modules/test_transactional_update.py
@@ -670,3 +670,47 @@ def test_single_queue_true():
"salt.modules.transactional_update.call", MagicMock(return_value="result")
):
assert tu.single("pkg.installed", name="emacs", queue=True) == "result"
+
+
+@pytest.mark.parametrize(
+ "executable,salt_call_cmd",
+ [
+ ("/usr/bin/python3", "salt-call"),
+ (
+ "/usr/lib/venv-salt-minion/bin/python",
+ "/usr/lib/venv-salt-minion/bin/salt-call",
+ ),
+ ],
+)
+def test_call_which_salt_call_selected_with_executable(executable, salt_call_cmd):
+ """Test transactional_update.chroot which salt-call used"""
+ utils_mock = {
+ "json.find_json": MagicMock(return_value={"return": "result"}),
+ }
+ salt_mock = {
+ "cmd.run_all": MagicMock(return_value={"retcode": 0, "stdout": ""}),
+ }
+ with patch("sys.executable", executable), patch.dict(
+ tu.__utils__, utils_mock
+ ), patch.dict(tu.__salt__, salt_mock):
+ assert tu.call("test.ping") == "result"
+
+ salt_mock["cmd.run_all"].assert_called_with(
+ [
+ "transactional-update",
+ "--non-interactive",
+ "--drop-if-no-change",
+ "--no-selfupdate",
+ "--continue",
+ "--quiet",
+ "run",
+ salt_call_cmd,
+ "--out",
+ "json",
+ "-l",
+ "quiet",
+ "--no-return-event",
+ "--",
+ "test.ping",
+ ]
+ )
--
2.42.0