pool/salt
SHA256
2
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Accepting request 1085018 from home:agraul:branches:systemsmanagement:saltstack

Browse Source 
remove leftover patches

OBS-URL: https://build.opensuse.org/request/show/1085018
OBS-URL: https://build.opensuse.org/package/show/systemsmanagement:saltstack/salt?expand=0&rev=211
This commit is contained in:
Alexander Graul 2023-05-05 09:46:40 +00:00 committed by Git OBS Bridge
parent 2686359b2c
commit ab9c251387
22 changed files with 0 additions and 4286 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

224
add-amazon-ec2-detection-for-virtual-grains-bsc-1195.patch
View File

@ -1,224 +0,0 @@
From 1434a128559df8183c032af722dc3d187bda148a Mon Sep 17 00:00:00 2001
From: Victor Zhestkov <Victor.Zhestkov@suse.com>
Date: Thu, 1 Sep 2022 14:46:24 +0300
Subject: [PATCH] Add Amazon EC2 detection for virtual grains
(bsc#1195624)
* Add ignore_retcode to quiet run functions
* Implement Amazon EC2 detection for virtual grains
* Add test for virtual grain detection of Amazon EC2
* Also detect the product of Amazon EC2 instance
* Add changelog entry
---
changelog/62539.added | 1 +
salt/grains/core.py | 18 ++++
salt/modules/cmdmod.py | 4 +
tests/pytests/unit/grains/test_core.py | 117 +++++++++++++++++++++++++
4 files changed, 140 insertions(+)
create mode 100644 changelog/62539.added
diff --git a/changelog/62539.added b/changelog/62539.added
new file mode 100644
index 0000000000..5f402d61c2
--- /dev/null
+++ b/changelog/62539.added
@@ -0,0 +1 @@
+Implementation of Amazon EC2 instance detection and setting `virtual_subtype` grain accordingly including the product if possible to identify.
diff --git a/salt/grains/core.py b/salt/grains/core.py
index 23d8b8ea42..047c33ffd3 100644
--- a/salt/grains/core.py
+++ b/salt/grains/core.py
@@ -1171,6 +1171,24 @@ def _virtual(osdata):
if grains.get("virtual_subtype") and grains["virtual"] == "physical":
grains["virtual"] = "virtual"
+ # Try to detect if the instance is running on Amazon EC2
+ if grains["virtual"] in ("qemu", "kvm", "xen"):
+ dmidecode = salt.utils.path.which("dmidecode")
+ if dmidecode:
+ ret = __salt__["cmd.run_all"](
+ [dmidecode, "-t", "system"], ignore_retcode=True
+ )
+ output = ret["stdout"]
+ if "Manufacturer: Amazon EC2" in output:
+ grains["virtual_subtype"] = "Amazon EC2"
+ product = re.match(
+ r".*Product Name: ([^\r\n]*).*", output, flags=re.DOTALL
+ )
+ if product:
+ grains["virtual_subtype"] = "Amazon EC2 ({})".format(product[1])
+ elif re.match(r".*Version: [^\r\n]+\.amazon.*", output, flags=re.DOTALL):
+ grains["virtual_subtype"] = "Amazon EC2"
+
for command in failed_commands:
log.info(
"Although '%s' was found in path, the current user "
diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py
index a26220718a..07b6e100d2 100644
--- a/salt/modules/cmdmod.py
+++ b/salt/modules/cmdmod.py
@@ -932,6 +932,7 @@ def _run_quiet(
success_retcodes=None,
success_stdout=None,
success_stderr=None,
+ ignore_retcode=None,
):
"""
Helper for running commands quietly for minion startup
@@ -958,6 +959,7 @@ def _run_quiet(
success_retcodes=success_retcodes,
success_stdout=success_stdout,
success_stderr=success_stderr,
+ ignore_retcode=ignore_retcode,
)["stdout"]
@@ -980,6 +982,7 @@ def _run_all_quiet(
success_retcodes=None,
success_stdout=None,
success_stderr=None,
+ ignore_retcode=None,
):
"""
@@ -1012,6 +1015,7 @@ def _run_all_quiet(
success_retcodes=success_retcodes,
success_stdout=success_stdout,
success_stderr=success_stderr,
+ ignore_retcode=ignore_retcode,
)
diff --git a/tests/pytests/unit/grains/test_core.py b/tests/pytests/unit/grains/test_core.py
index 5c43dbdb09..7c4ea1f17f 100644
--- a/tests/pytests/unit/grains/test_core.py
+++ b/tests/pytests/unit/grains/test_core.py
@@ -2823,3 +2823,120 @@ def test_get_server_id():
with patch.dict(core.__opts__, {"id": "otherid"}):
assert core.get_server_id() != expected
+
+
+@pytest.mark.skip_unless_on_linux
+def test_virtual_set_virtual_ec2():
+ osdata = {}
+
+ (
+ osdata["kernel"],
+ osdata["nodename"],
+ osdata["kernelrelease"],
+ osdata["kernelversion"],
+ osdata["cpuarch"],
+ _,
+ ) = platform.uname()
+
+ which_mock = MagicMock(
+ side_effect=[
+ # Check with virt-what
+ "/usr/sbin/virt-what",
+ "/usr/sbin/virt-what",
+ None,
+ "/usr/sbin/dmidecode",
+ # Check with systemd-detect-virt
+ None,
+ "/usr/bin/systemd-detect-virt",
+ None,
+ "/usr/sbin/dmidecode",
+ # Check with systemd-detect-virt when no dmidecode available
+ None,
+ "/usr/bin/systemd-detect-virt",
+ None,
+ None,
+ ]
+ )
+ cmd_run_all_mock = MagicMock(
+ side_effect=[
+ # Check with virt-what
+ {"retcode": 0, "stderr": "", "stdout": "xen"},
+ {
+ "retcode": 0,
+ "stderr": "",
+ "stdout": "\n".join(
+ [
+ "dmidecode 3.2",
+ "Getting SMBIOS data from sysfs.",
+ "SMBIOS 2.7 present.",
+ "",
+ "Handle 0x0100, DMI type 1, 27 bytes",
+ "System Information",
+ " Manufacturer: Xen",
+ " Product Name: HVM domU",
+ " Version: 4.11.amazon",
+ " Serial Number: 12345678-abcd-4321-dcba-0123456789ab",
+ " UUID: 01234567-dcba-1234-abcd-abcdef012345",
+ " Wake-up Type: Power Switch",
+ " SKU Number: Not Specified",
+ " Family: Not Specified",
+ "",
+ "Handle 0x2000, DMI type 32, 11 bytes",
+ "System Boot Information",
+ " Status: No errors detected",
+ ]
+ ),
+ },
+ # Check with systemd-detect-virt
+ {"retcode": 0, "stderr": "", "stdout": "kvm"},
+ {
+ "retcode": 0,
+ "stderr": "",
+ "stdout": "\n".join(
+ [
+ "dmidecode 3.2",
+ "Getting SMBIOS data from sysfs.",
+ "SMBIOS 2.7 present.",
+ "",
+ "Handle 0x0001, DMI type 1, 27 bytes",
+ "System Information",
+ " Manufacturer: Amazon EC2",
+ " Product Name: m5.large",
+ " Version: Not Specified",
+ " Serial Number: 01234567-dcba-1234-abcd-abcdef012345",
+ " UUID: 12345678-abcd-4321-dcba-0123456789ab",
+ " Wake-up Type: Power Switch",
+ " SKU Number: Not Specified",
+ " Family: Not Specified",
+ ]
+ ),
+ },
+ # Check with systemd-detect-virt when no dmidecode available
+ {"retcode": 0, "stderr": "", "stdout": "kvm"},
+ ]
+ )
+
+ with patch("salt.utils.path.which", which_mock), patch.dict(
+ core.__salt__,
+ {
+ "cmd.run": salt.modules.cmdmod.run,
+ "cmd.run_all": cmd_run_all_mock,
+ "cmd.retcode": salt.modules.cmdmod.retcode,
+ "smbios.get": salt.modules.smbios.get,
+ },
+ ):
+
+ virtual_grains = core._virtual(osdata.copy())
+
+ assert virtual_grains["virtual"] == "xen"
+ assert virtual_grains["virtual_subtype"] == "Amazon EC2"
+
+ virtual_grains = core._virtual(osdata.copy())
+
+ assert virtual_grains["virtual"] == "kvm"
+ assert virtual_grains["virtual_subtype"] == "Amazon EC2 (m5.large)"
+
+ virtual_grains = core._virtual(osdata.copy())
+
+ assert virtual_grains["virtual"] == "kvm"
+ assert "virtual_subtype" not in virtual_grains
--
2.37.3

1056
add-support-for-name-pkgs-and-diff_attr-parameters-t.patch
View File

File diff suppressed because it is too large Load Diff

124
align-amazon-ec2-nitro-grains-with-upstream-pr-bsc-1.patch
View File

@ -1,124 +0,0 @@
From d1e9af256fa67cd792ce11e6e9c1e24a1fe2054f Mon Sep 17 00:00:00 2001
From: Victor Zhestkov <Victor.Zhestkov@suse.com>
Date: Fri, 28 Oct 2022 13:19:46 +0300
Subject: [PATCH] Align Amazon EC2 (Nitro) grains with upstream PR
(bsc#1203685)
* Set virtual to Nitro for Amazon EC2 kvm instances
* Add few mocks to prevent false failing
possible in some specific environments
* Add one more possible test case returning Nitro
---
salt/grains/core.py | 8 +++++++-
tests/pytests/unit/grains/test_core.py | 27 +++++++++++++++++++++++++-
2 files changed, 33 insertions(+), 2 deletions(-)
diff --git a/salt/grains/core.py b/salt/grains/core.py
index 76f3767ddf..f359c07432 100644
--- a/salt/grains/core.py
+++ b/salt/grains/core.py
@@ -860,6 +860,10 @@ def _virtual(osdata):
grains["virtual"] = "container"
grains["virtual_subtype"] = "LXC"
break
+ elif "amazon" in output:
+ grains["virtual"] = "Nitro"
+ grains["virtual_subtype"] = "Amazon EC2"
+ break
elif command == "virt-what":
for line in output.splitlines():
if line in ("kvm", "qemu", "uml", "xen"):
@@ -1174,7 +1178,7 @@ def _virtual(osdata):
grains["virtual"] = "virtual"
# Try to detect if the instance is running on Amazon EC2
- if grains["virtual"] in ("qemu", "kvm", "xen"):
+ if grains["virtual"] in ("qemu", "kvm", "xen", "amazon"):
dmidecode = salt.utils.path.which("dmidecode")
if dmidecode:
ret = __salt__["cmd.run_all"](
@@ -1182,6 +1186,8 @@ def _virtual(osdata):
)
output = ret["stdout"]
if "Manufacturer: Amazon EC2" in output:
+ if grains["virtual"] != "xen":
+ grains["virtual"] = "Nitro"
grains["virtual_subtype"] = "Amazon EC2"
product = re.match(
r".*Product Name: ([^\r\n]*).*", output, flags=re.DOTALL
diff --git a/tests/pytests/unit/grains/test_core.py b/tests/pytests/unit/grains/test_core.py
index c06cdb2db0..6f3bef69f2 100644
--- a/tests/pytests/unit/grains/test_core.py
+++ b/tests/pytests/unit/grains/test_core.py
@@ -2888,6 +2888,11 @@ def test_virtual_set_virtual_ec2():
"/usr/bin/systemd-detect-virt",
None,
None,
+ # Check with systemd-detect-virt returning amazon and no dmidecode available
+ None,
+ "/usr/bin/systemd-detect-virt",
+ None,
+ None,
]
)
cmd_run_all_mock = MagicMock(
@@ -2946,9 +2951,22 @@ def test_virtual_set_virtual_ec2():
},
# Check with systemd-detect-virt when no dmidecode available
{"retcode": 0, "stderr": "", "stdout": "kvm"},
+ # Check with systemd-detect-virt returning amazon and no dmidecode available
+ {"retcode": 0, "stderr": "", "stdout": "amazon"},
]
)
+ def _mock_is_file(filename):
+ if filename in (
+ "/proc/1/cgroup",
+ "/proc/cpuinfo",
+ "/sys/devices/virtual/dmi/id/product_name",
+ "/proc/xen/xsd_kva",
+ "/proc/xen/capabilities",
+ ):
+ return False
+ return True
+
with patch("salt.utils.path.which", which_mock), patch.dict(
core.__salt__,
{
@@ -2957,6 +2975,8 @@ def test_virtual_set_virtual_ec2():
"cmd.retcode": salt.modules.cmdmod.retcode,
"smbios.get": salt.modules.smbios.get,
},
+ ), patch("os.path.isfile", _mock_is_file), patch(
+ "os.path.isdir", return_value=False
):
virtual_grains = core._virtual(osdata.copy())
@@ -2966,7 +2986,7 @@ def test_virtual_set_virtual_ec2():
virtual_grains = core._virtual(osdata.copy())
- assert virtual_grains["virtual"] == "kvm"
+ assert virtual_grains["virtual"] == "Nitro"
assert virtual_grains["virtual_subtype"] == "Amazon EC2 (m5.large)"
virtual_grains = core._virtual(osdata.copy())
@@ -2974,6 +2994,11 @@ def test_virtual_set_virtual_ec2():
assert virtual_grains["virtual"] == "kvm"
assert "virtual_subtype" not in virtual_grains
+ virtual_grains = core._virtual(osdata.copy())
+
+ assert virtual_grains["virtual"] == "Nitro"
+ assert virtual_grains["virtual_subtype"] == "Amazon EC2"
+
@pytest.mark.skip_on_windows
def test_linux_proc_files_with_non_utf8_chars():
--
2.37.3

151
allow-entrypoint-compatibility-for-importlib-metadat.patch
View File

@ -1,151 +0,0 @@
From 6dc653b0cf8e6e043e13bea7009ded604ceb7b71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?=
<psuarezhernandez@suse.com>
Date: Thu, 12 Jan 2023 15:43:56 +0000
Subject: [PATCH] Allow entrypoint compatibility for
importlib-metadata>=5.0.0 (#572)
add tests and make sure the compat code is in an else :)
changelog
switch to try/except
Co-authored-by: MKLeb <calebb@vmware.com>
---
changelog/62854.fixed | 1 +
salt/utils/entrypoints.py | 15 +++--
.../pytests/functional/loader/test_loader.py | 67 +++++++++++++++++--
3 files changed, 74 insertions(+), 9 deletions(-)
create mode 100644 changelog/62854.fixed
diff --git a/changelog/62854.fixed b/changelog/62854.fixed
new file mode 100644
index 0000000000..13e6df4fe3
--- /dev/null
+++ b/changelog/62854.fixed
@@ -0,0 +1 @@
+Use select instead of iterating over entrypoints as a dictionary for importlib_metadata>=5.0.0
diff --git a/salt/utils/entrypoints.py b/salt/utils/entrypoints.py
index 3effa0b494..9452878ade 100644
--- a/salt/utils/entrypoints.py
+++ b/salt/utils/entrypoints.py
@@ -38,13 +38,20 @@ def iter_entry_points(group, name=None):
entry_points_listing = []
entry_points = importlib_metadata.entry_points()
- for entry_point_group, entry_points_list in entry_points.items():
- if entry_point_group != group:
- continue
- for entry_point in entry_points_list:
+ try:
+ for entry_point in entry_points.select(group=group):
if name is not None and entry_point.name != name:
continue
entry_points_listing.append(entry_point)
+ except AttributeError:
+ # importlib-metadata<5.0.0
+ for entry_point_group, entry_points_list in entry_points.items():
+ if entry_point_group != group:
+ continue
+ for entry_point in entry_points_list:
+ if name is not None and entry_point.name != name:
+ continue
+ entry_points_listing.append(entry_point)
return entry_points_listing
diff --git a/tests/pytests/functional/loader/test_loader.py b/tests/pytests/functional/loader/test_loader.py
index 6dfd97b0e6..a13d90d5eb 100644
--- a/tests/pytests/functional/loader/test_loader.py
+++ b/tests/pytests/functional/loader/test_loader.py
@@ -1,5 +1,4 @@
import json
-import sys
import pytest
import salt.utils.versions
@@ -143,10 +142,6 @@ def test_utils_loader_does_not_load_extensions(
assert "foobar.echo" not in loader_functions
-@pytest.mark.skipif(
- sys.version_info < (3, 6),
- reason="importlib-metadata>=3.3.0 does not exist for Py3.5",
-)
def test_extension_discovery_without_reload_with_importlib_metadata_installed(
venv, salt_extension, salt_minion_factory
):
@@ -209,6 +204,68 @@ def test_extension_discovery_without_reload_with_importlib_metadata_installed(
assert "foobar.echo2" in loader_functions
+def test_extension_discovery_without_reload_with_importlib_metadata_5_installed(
+ venv, salt_extension, salt_minion_factory
+):
+ # Install our extension into the virtualenv
+ installed_packages = venv.get_installed_packages()
+ assert salt_extension.name not in installed_packages
+ venv.install("importlib-metadata>=3.3.0")
+ code = """
+ import sys
+ import json
+ import subprocess
+ import salt._logging
+ import salt.loader
+
+ extension_path = "{}"
+
+ minion_config = json.loads(sys.stdin.read())
+ salt._logging.set_logging_options_dict(minion_config)
+ salt._logging.setup_logging()
+ loader = salt.loader.minion_mods(minion_config)
+
+ if "foobar.echo1" in loader:
+ sys.exit(1)
+
+ # Install the extension
+ proc = subprocess.run(
+ [sys.executable, "-m", "pip", "install", extension_path],
+ check=False,
+ shell=False,
+ stdout=subprocess.PIPE,
+ )
+ if proc.returncode != 0:
+ sys.exit(2)
+
+ loader = salt.loader.minion_mods(minion_config)
+ if "foobar.echo1" not in loader:
+ sys.exit(3)
+
+ print(json.dumps(list(loader)))
+ """.format(
+ salt_extension.srcdir
+ )
+ ret = venv.run_code(
+ code, input=json.dumps(salt_minion_factory.config.copy()), check=False
+ )
+ # Exitcode 1 - Extension was already installed
+ # Exitcode 2 - Failed to install the extension
+ # Exitcode 3 - Extension was not found within the same python process after being installed
+ assert ret.returncode == 0
+ installed_packages = venv.get_installed_packages()
+ assert salt_extension.name in installed_packages
+
+ loader_functions = json.loads(ret.stdout)
+
+ # A non existing module should not appear in the loader
+ assert "monty.python" not in loader_functions
+
+ # But our extension's modules should appear on the loader
+ assert "foobar.echo1" in loader_functions
+ assert "foobar.echo2" in loader_functions
+
+
def test_extension_discovery_without_reload_with_bundled_importlib_metadata(
venv, salt_extension, salt_minion_factory
):
--
2.37.3

51
clarify-pkg.installed-pkg_verify-documentation.patch
View File

@ -1,51 +0,0 @@
From 5ed2295489fc13e48b981c323c846bde927cb800 Mon Sep 17 00:00:00 2001
From: Alexander Graul <agraul@suse.com>
Date: Fri, 21 Oct 2022 14:39:21 +0200
Subject: [PATCH] Clarify pkg.installed pkg_verify documentation
There have been misunderstandings what the pkg_verify parameter does and
bug reports that it does not work, based on the wrong assumption that
this parameter changes the installation of new packages. The docstring
also stated that it was only provided by `yum`, but `zypper` also
provides this feature (actually it is `rpm` itself in both cases that
does the verification check)
Related issue: https://github.com/saltstack/salt/issues/44878
(cherry picked from commit 2ed5f3c29d3b4313d904b7c081e5a29bf5e309c7)
---
salt/states/pkg.py | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/salt/states/pkg.py b/salt/states/pkg.py
index cda966a1e8..13532521d5 100644
--- a/salt/states/pkg.py
+++ b/salt/states/pkg.py
@@ -1277,14 +1277,15 @@ def installed(
.. versionadded:: 2014.7.0
- For requested packages that are already installed and would not be
- targeted for upgrade or downgrade, use pkg.verify to determine if any
- of the files installed by the package have been altered. If files have
- been altered, the reinstall option of pkg.install is used to force a
- reinstall. Types to ignore can be passed to pkg.verify. Additionally,
- ``verify_options`` can be used to modify further the behavior of
- pkg.verify. See examples below. Currently, this option is supported
- for the following pkg providers: :mod:`yumpkg <salt.modules.yumpkg>`.
+ Use pkg.verify to check if already installed packages require
+ reinstallion. Requested packages that are already installed and not
+ targeted for up- or downgrade are verified with pkg.verify to determine
+ if any file installed by the package have been modified or if package
+ dependencies are not fulfilled. ``ignore_types`` and ``verify_options``
+ can be passed to pkg.verify. See examples below. Currently, this option
+ is supported for the following pkg providers:
+ :mod:`yum <salt.modules.yumpkg>`,
+ :mod:`zypperpkg <salt.modules.zypperpkg>`.
Examples:
--
2.37.3

28
detect-module.run-syntax.patch
View File

@ -1,28 +0,0 @@
From dd147ab110e71ea0f1091923c9230ade01f226d4 Mon Sep 17 00:00:00 2001
From: Victor Zhestkov <Victor.Zhestkov@suse.com>
Date: Fri, 28 Oct 2022 13:19:23 +0300
Subject: [PATCH] Detect module.run syntax
* Detect module run syntax version
* Update module.run docs and add changelog
* Add test for module.run without any args
Co-authored-by: Daniel A. Wozniak <dwozniak@saltstack.com>
---
changelog/58763.fixed | 1 +
1 file changed, 1 insertion(+)
create mode 100644 changelog/58763.fixed
diff --git a/changelog/58763.fixed b/changelog/58763.fixed
new file mode 100644
index 0000000000..53ee8304c0
--- /dev/null
+++ b/changelog/58763.fixed
@@ -0,0 +1 @@
+Detect new and legacy styles of calling module.run and support them both.
--
2.37.3

50
fix-salt.states.file.managed-for-follow_symlinks-tru.patch
View File

@ -1,50 +0,0 @@
From e328d2029c93153c519e10e9596c635f6f3febcf Mon Sep 17 00:00:00 2001
From: Petr Pavlu <31453820+petrpavlu@users.noreply.github.com>
Date: Fri, 8 Jul 2022 10:11:52 +0200
Subject: [PATCH] Fix salt.states.file.managed() for follow_symlinks=True
and test=True (bsc#1199372) (#535)
When managing file /etc/test as follows:
> file /etc/test:
> file.managed:
> - name: /etc/test
> - source: salt://config/test
> - mode: 644
> - follow_symlinks: True
and with /etc/test being a symlink to a different file, an invocation of
"salt-call '*' state.apply test=True" can report that the file should be
updated even when a subsequent run of the same command without the test
parameter makes no changes.
The problem is that the test code path doesn't take correctly into
account the follow_symlinks=True setting and ends up comparing
permissions of the symlink instead of its target file.
The patch addresses the problem by extending functions
salt.modules.file.check_managed(), check_managed_changes() and
check_file_meta() to have the follow_symlinks parameter which gets
propagated to the salt.modules.file.stats() call and by updating
salt.states.file.managed() to forward the same parameter to
salt.modules.file.check_managed_changes().
Fixes #62066.
[Cherry-picked from upstream commit
95bfbe31a2dc54723af3f1783d40de152760fe1a.]
---
changelog/62066.fixed | 1 +
1 file changed, 1 insertion(+)
create mode 100644 changelog/62066.fixed
diff --git a/changelog/62066.fixed b/changelog/62066.fixed
new file mode 100644
index 0000000000..68216a03c1
--- /dev/null
+++ b/changelog/62066.fixed
@@ -0,0 +1 @@
+Fixed salt.states.file.managed() for follow_symlinks=True and test=True
--
2.37.3

183
fix-state.apply-in-test-mode-with-file-state-module-.patch
View File

@ -1,183 +0,0 @@
From 58317cda7a347581b495ab7fd71ce75f0740d8d6 Mon Sep 17 00:00:00 2001
From: Victor Zhestkov <Victor.Zhestkov@suse.com>
Date: Thu, 1 Sep 2022 14:44:26 +0300
Subject: [PATCH] Fix state.apply in test mode with file state module on
user/group checking (bsc#1202167)
* Do not fail on checking user/group in test mode
* fixes saltstack/salt#61846 reporting of errors in test mode
Co-authored-by: nicholasmhughes <nicholasmhughes@gmail.com>
* Add tests for _check_user usage
Co-authored-by: nicholasmhughes <nicholasmhughes@gmail.com>
---
changelog/61846.fixed | 1 +
salt/states/file.py | 5 +++
tests/pytests/unit/states/file/test_copy.py | 35 ++++++++++++++++
.../unit/states/file/test_filestate.py | 42 +++++++++++++++++++
.../pytests/unit/states/file/test_managed.py | 31 ++++++++++++++
5 files changed, 114 insertions(+)
create mode 100644 changelog/61846.fixed
diff --git a/changelog/61846.fixed b/changelog/61846.fixed
new file mode 100644
index 0000000000..c4024efe9f
--- /dev/null
+++ b/changelog/61846.fixed
@@ -0,0 +1 @@
+Fix the reporting of errors for file.directory in test mode
diff --git a/salt/states/file.py b/salt/states/file.py
index 1083bb46d6..5cb58f5454 100644
--- a/salt/states/file.py
+++ b/salt/states/file.py
@@ -379,6 +379,11 @@ def _check_user(user, group):
gid = __salt__["file.group_to_gid"](group)
if gid == "":
err += "Group {} is not available".format(group)
+ if err and __opts__["test"]:
+ # Write the warning with error message, but prevent failing,
+ # in case of applying the state in test mode.
+ log.warning(err)
+ return ""
return err
diff --git a/tests/pytests/unit/states/file/test_copy.py b/tests/pytests/unit/states/file/test_copy.py
index ce7161f02d..a11adf5ae0 100644
--- a/tests/pytests/unit/states/file/test_copy.py
+++ b/tests/pytests/unit/states/file/test_copy.py
@@ -205,3 +205,38 @@ def test_copy(tmp_path):
)
res = filestate.copy_(name, source, group=group, preserve=False)
assert res == ret
+
+
+def test_copy_test_mode_user_group_not_present():
+ """
+ Test file copy in test mode with no user or group existing
+ """
+ source = "/tmp/src_copy_no_user_group_test_mode"
+ filename = "/tmp/copy_no_user_group_test_mode"
+ with patch.dict(
+ filestate.__salt__,
+ {
+ "file.group_to_gid": MagicMock(side_effect=["1234", "", ""]),
+ "file.user_to_uid": MagicMock(side_effect=["", "4321", ""]),
+ "file.get_mode": MagicMock(return_value="0644"),
+ },
+ ), patch.dict(filestate.__opts__, {"test": True}), patch.object(
+ os.path, "exists", return_value=True
+ ):
+ ret = filestate.copy_(
+ source, filename, group="nonexistinggroup", user="nonexistinguser"
+ )
+ assert ret["result"] is not False
+ assert "is not available" not in ret["comment"]
+
+ ret = filestate.copy_(
+ source, filename, group="nonexistinggroup", user="nonexistinguser"
+ )
+ assert ret["result"] is not False
+ assert "is not available" not in ret["comment"]
+
+ ret = filestate.copy_(
+ source, filename, group="nonexistinggroup", user="nonexistinguser"
+ )
+ assert ret["result"] is not False
+ assert "is not available" not in ret["comment"]
diff --git a/tests/pytests/unit/states/file/test_filestate.py b/tests/pytests/unit/states/file/test_filestate.py
index 2f9f369fb2..c373cb3449 100644
--- a/tests/pytests/unit/states/file/test_filestate.py
+++ b/tests/pytests/unit/states/file/test_filestate.py
@@ -577,3 +577,45 @@ def test_mod_run_check_cmd():
assert filestate.mod_run_check_cmd(cmd, filename) == ret
assert filestate.mod_run_check_cmd(cmd, filename)
+
+
+def test_recurse_test_mode_user_group_not_present():
+ """
+ Test file recurse in test mode with no user or group existing
+ """
+ filename = "/tmp/recurse_no_user_group_test_mode"
+ source = "salt://tmp/src_recurse_no_user_group_test_mode"
+ mock_l = MagicMock(return_value=[])
+ mock_emt = MagicMock(return_value=["tmp/src_recurse_no_user_group_test_mode"])
+ with patch.dict(
+ filestate.__salt__,
+ {
+ "file.group_to_gid": MagicMock(side_effect=["1234", "", ""]),
+ "file.user_to_uid": MagicMock(side_effect=["", "4321", ""]),
+ "file.get_mode": MagicMock(return_value="0644"),
+ "file.source_list": MagicMock(return_value=[source, ""]),
+ "cp.list_master_dirs": mock_emt,
+ "cp.list_master": mock_l,
+ },
+ ), patch.dict(filestate.__opts__, {"test": True}), patch.object(
+ os.path, "exists", return_value=True
+ ), patch.object(
+ os.path, "isdir", return_value=True
+ ):
+ ret = filestate.recurse(
+ filename, source, group="nonexistinggroup", user="nonexistinguser"
+ )
+ assert ret["result"] is not False
+ assert "is not available" not in ret["comment"]
+
+ ret = filestate.recurse(
+ filename, source, group="nonexistinggroup", user="nonexistinguser"
+ )
+ assert ret["result"] is not False
+ assert "is not available" not in ret["comment"]
+
+ ret = filestate.recurse(
+ filename, source, group="nonexistinggroup", user="nonexistinguser"
+ )
+ assert ret["result"] is not False
+ assert "is not available" not in ret["comment"]
diff --git a/tests/pytests/unit/states/file/test_managed.py b/tests/pytests/unit/states/file/test_managed.py
index 9d9fb17717..0b341e09a9 100644
--- a/tests/pytests/unit/states/file/test_managed.py
+++ b/tests/pytests/unit/states/file/test_managed.py
@@ -373,3 +373,34 @@ def test_managed():
filestate.managed(name, user=user, group=group)
== ret
)
+
+
+def test_managed_test_mode_user_group_not_present():
+ """
+ Test file managed in test mode with no user or group existing
+ """
+ filename = "/tmp/managed_no_user_group_test_mode"
+ with patch.dict(
+ filestate.__salt__,
+ {
+ "file.group_to_gid": MagicMock(side_effect=["1234", "", ""]),
+ "file.user_to_uid": MagicMock(side_effect=["", "4321", ""]),
+ },
+ ), patch.dict(filestate.__opts__, {"test": True}):
+ ret = filestate.managed(
+ filename, group="nonexistinggroup", user="nonexistinguser"
+ )
+ assert ret["result"] is not False
+ assert "is not available" not in ret["comment"]
+
+ ret = filestate.managed(
+ filename, group="nonexistinggroup", user="nonexistinguser"
+ )
+ assert ret["result"] is not False
+ assert "is not available" not in ret["comment"]
+
+ ret = filestate.managed(
+ filename, group="nonexistinggroup", user="nonexistinguser"
+ )
+ assert ret["result"] is not False
+ assert "is not available" not in ret["comment"]
--
2.37.3

37
fix-test_ipc-unit-tests.patch
View File

@ -1,37 +0,0 @@
From 4cc528dadfbffdeb90df41bbd848d0c2c7efec78 Mon Sep 17 00:00:00 2001
From: Alexander Graul <agraul@suse.com>
Date: Tue, 12 Jul 2022 14:02:58 +0200
Subject: [PATCH] Fix test_ipc unit tests
---
tests/unit/transport/test_ipc.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/tests/unit/transport/test_ipc.py b/tests/unit/transport/test_ipc.py
index 4a0a7c29e2..af001d9650 100644
--- a/tests/unit/transport/test_ipc.py
+++ b/tests/unit/transport/test_ipc.py
@@ -105,8 +105,8 @@ class IPCMessagePubSubCase(salt.ext.tornado.testing.AsyncTestCase):
self.stop()
# Now let both waiting data at once
- client1.read_async(handler)
- client2.read_async(handler)
+ client1.read_async()
+ client2.read_async()
self.pub_channel.publish("TEST")
self.wait()
self.assertEqual(len(call_cnt), 2)
@@ -148,7 +148,7 @@ class IPCMessagePubSubCase(salt.ext.tornado.testing.AsyncTestCase):
pass
try:
- ret1 = yield client1.read_async(handler)
+ ret1 = yield client1.read_async()
self.wait()
except StreamClosedError as ex:
assert False, "StreamClosedError was raised inside the Future"
--
2.37.3

103
fixes-pkg.version_cmp-on-openeuler-systems-and-a-few.patch
View File

@ -1,103 +0,0 @@
From 62b3e3491f283a5b5ac243e1f5904ad17e0353bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Osuna?=
<17827278+raulillo82@users.noreply.github.com>
Date: Tue, 31 Jan 2023 13:13:32 +0100
Subject: [PATCH] Fixes pkg.version_cmp on openEuler systems and a few
other OS flavors (#576)
* Fix bug/issue #540
* Fix bug/issue #540
* Add tests for __virtual__ function
* Minor changes to align with upstream
---
changelog/540.fixed | 1 +
salt/modules/rpm_lowpkg.py | 12 +++++--
tests/pytests/unit/modules/test_rpm_lowpkg.py | 31 +++++++++++++++++++
3 files changed, 42 insertions(+), 2 deletions(-)
create mode 100644 changelog/540.fixed
diff --git a/changelog/540.fixed b/changelog/540.fixed
new file mode 100644
index 0000000000..50cb42bf40
--- /dev/null
+++ b/changelog/540.fixed
@@ -0,0 +1 @@
+Fix pkg.version_cmp on openEuler and a few other os flavors.
diff --git a/salt/modules/rpm_lowpkg.py b/salt/modules/rpm_lowpkg.py
index c8e984c021..01cd575bc7 100644
--- a/salt/modules/rpm_lowpkg.py
+++ b/salt/modules/rpm_lowpkg.py
@@ -62,14 +62,22 @@ def __virtual__():
" grains.",
)
- enabled = ("amazon", "xcp", "xenserver", "virtuozzolinux")
+ enabled = (
+ "amazon",
+ "xcp",
+ "xenserver",
+ "virtuozzolinux",
+ "virtuozzo",
+ "issabel pbx",
+ "openeuler",
+ )
if os_family in ["redhat", "suse"] or os_grain in enabled:
return __virtualname__
return (
False,
"The rpm execution module failed to load: only available on redhat/suse type"
- " systems or amazon, xcp or xenserver.",
+ " systems or amazon, xcp, xenserver, virtuozzolinux, virtuozzo, issabel pbx or openeuler.",
)
diff --git a/tests/pytests/unit/modules/test_rpm_lowpkg.py b/tests/pytests/unit/modules/test_rpm_lowpkg.py
index f19afa854e..e07f71eb61 100644
--- a/tests/pytests/unit/modules/test_rpm_lowpkg.py
+++ b/tests/pytests/unit/modules/test_rpm_lowpkg.py
@@ -35,6 +35,37 @@ def _called_with_root(mock):
def configure_loader_modules():
return {rpm: {"rpm": MagicMock(return_value=MagicMock)}}
+def test___virtual___openeuler():
+ patch_which = patch("salt.utils.path.which", return_value=True)
+ with patch.dict(
+ rpm.__grains__, {"os": "openEuler", "os_family": "openEuler"}
+ ), patch_which:
+ assert rpm.__virtual__() == "lowpkg"
+
+
+def test___virtual___issabel_pbx():
+ patch_which = patch("salt.utils.path.which", return_value=True)
+ with patch.dict(
+ rpm.__grains__, {"os": "Issabel Pbx", "os_family": "IssabeL PBX"}
+ ), patch_which:
+ assert rpm.__virtual__() == "lowpkg"
+
+
+def test___virtual___virtuozzo():
+ patch_which = patch("salt.utils.path.which", return_value=True)
+ with patch.dict(
+ rpm.__grains__, {"os": "virtuozzo", "os_family": "VirtuoZZO"}
+ ), patch_which:
+ assert rpm.__virtual__() == "lowpkg"
+
+
+def test___virtual___with_no_rpm():
+ patch_which = patch("salt.utils.path.which", return_value=False)
+ ret = rpm.__virtual__()
+ assert isinstance(ret, tuple)
+ assert ret[0] is False
+
+
# 'list_pkgs' function tests: 2
--
2.37.3

105
fopen-workaround-bad-buffering-for-binary-mode-563.patch
View File

@ -1,105 +0,0 @@
From 7d5b1d2178d0573f137b9481ded85419a36998ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?=
<psuarezhernandez@suse.com>
Date: Thu, 6 Oct 2022 10:55:50 +0100
Subject: [PATCH] fopen: Workaround bad buffering for binary mode (#563)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
A lot of code assumes Python 2.x behavior for buffering, in which 1 is a
special value meaning line buffered.
Python 3 makes this value unusable, so fallback to the default buffering
size, and report these calls to be fixed.
Fixes: https://github.com/saltstack/salt/issues/57584
Do not drop buffering from kwargs to avoid errors
Add unit test around linebuffering in binary mode
Add changelog file
Co-authored-by: Pablo Suárez Hernández <psuarezhernandez@suse.com>
Co-authored-by: Ismael Luceno <iluceno@suse.de>
---
changelog/62817.fixed | 1 +
salt/utils/files.py | 8 ++++++++
tests/pytests/unit/utils/test_files.py | 13 ++++++++++++-
3 files changed, 21 insertions(+), 1 deletion(-)
create mode 100644 changelog/62817.fixed
diff --git a/changelog/62817.fixed b/changelog/62817.fixed
new file mode 100644
index 0000000000..ff335f2916
--- /dev/null
+++ b/changelog/62817.fixed
@@ -0,0 +1 @@
+Prevent annoying RuntimeWarning message about line buffering (buffering=1) not being supported in binary mode
diff --git a/salt/utils/files.py b/salt/utils/files.py
index 1cf636a753..3c57cce713 100644
--- a/salt/utils/files.py
+++ b/salt/utils/files.py
@@ -6,6 +6,7 @@ Functions for working with files
import codecs
import contextlib
import errno
+import io
import logging
import os
import re
@@ -382,6 +383,13 @@ def fopen(*args, **kwargs):
if not binary and not kwargs.get("newline", None):
kwargs["newline"] = ""
+ # Workaround callers with bad buffering setting for binary files
+ if kwargs.get("buffering") == 1 and "b" in kwargs.get("mode", ""):
+ log.debug(
+ "Line buffering (buffering=1) isn't supported in binary mode, the default buffer size will be used"
+ )
+ kwargs["buffering"] = io.DEFAULT_BUFFER_SIZE
+
f_handle = open(*args, **kwargs) # pylint: disable=resource-leakage
if is_fcntl_available():
diff --git a/tests/pytests/unit/utils/test_files.py b/tests/pytests/unit/utils/test_files.py
index fd88167b16..bd18bc5750 100644
--- a/tests/pytests/unit/utils/test_files.py
+++ b/tests/pytests/unit/utils/test_files.py
@@ -4,11 +4,12 @@ Unit Tests for functions located in salt/utils/files.py
import copy
+import io
import os
import pytest
import salt.utils.files
-from tests.support.mock import patch
+from tests.support.mock import MagicMock, patch
def test_safe_rm():
@@ -74,6 +75,16 @@ def test_fopen_with_disallowed_fds():
)
+def test_fopen_binary_line_buffering(tmp_path):
+ tmp_file = os.path.join(tmp_path, "foobar")
+ with patch("builtins.open") as open_mock, patch(
+ "salt.utils.files.is_fcntl_available", MagicMock(return_value=False)
+ ):
+ salt.utils.files.fopen(os.path.join(tmp_path, "foobar"), mode="b", buffering=1)
+ assert open_mock.called
+ assert open_mock.call_args[1]["buffering"] == io.DEFAULT_BUFFER_SIZE
+
+
def _create_temp_structure(temp_directory, structure):
for folder, files in structure.items():
current_directory = os.path.join(temp_directory, folder)
--
2.37.3

56
ignore-erros-on-reading-license-files-with-dpkg_lowp.patch
View File

@ -1,56 +0,0 @@
From 7cac5f67eb0d586314f9e7c987b8a620e28eeac3 Mon Sep 17 00:00:00 2001
From: Victor Zhestkov <vzhestkov@suse.com>
Date: Mon, 27 Jun 2022 17:55:49 +0300
Subject: [PATCH] Ignore erros on reading license files with dpkg_lowpkg
(bsc#1197288)
* Ignore erros on reading license files with dpkg_lowpkg (bsc#1197288)
* Add test for license reading with dpkg_lowpkg
---
salt/modules/dpkg_lowpkg.py | 2 +-
tests/pytests/unit/modules/test_dpkg_lowpkg.py | 18 ++++++++++++++++++
2 files changed, 19 insertions(+), 1 deletion(-)
create mode 100644 tests/pytests/unit/modules/test_dpkg_lowpkg.py
diff --git a/salt/modules/dpkg_lowpkg.py b/salt/modules/dpkg_lowpkg.py
index afbd619490..2c25b1fb2a 100644
--- a/salt/modules/dpkg_lowpkg.py
+++ b/salt/modules/dpkg_lowpkg.py
@@ -361,7 +361,7 @@ def _get_pkg_license(pkg):
licenses = set()
cpr = "/usr/share/doc/{}/copyright".format(pkg)
if os.path.exists(cpr):
- with salt.utils.files.fopen(cpr) as fp_:
+ with salt.utils.files.fopen(cpr, errors="ignore") as fp_:
for line in salt.utils.stringutils.to_unicode(fp_.read()).split(os.linesep):
if line.startswith("License:"):
licenses.add(line.split(":", 1)[1].strip())
diff --git a/tests/pytests/unit/modules/test_dpkg_lowpkg.py b/tests/pytests/unit/modules/test_dpkg_lowpkg.py
new file mode 100644
index 0000000000..1a89660c02
--- /dev/null
+++ b/tests/pytests/unit/modules/test_dpkg_lowpkg.py
@@ -0,0 +1,18 @@
+import os
+
+import salt.modules.dpkg_lowpkg as dpkg
+from tests.support.mock import MagicMock, mock_open, patch
+
+
+def test_get_pkg_license():
+ """
+ Test _get_pkg_license for ignore errors on reading license from copyright files
+ """
+ license_read_mock = mock_open(read_data="")
+ with patch.object(os.path, "exists", MagicMock(return_value=True)), patch(
+ "salt.utils.files.fopen", license_read_mock
+ ):
+ dpkg._get_pkg_license("bash")
+
+ assert license_read_mock.calls[0].args[0] == "/usr/share/doc/bash/copyright"
+ assert license_read_mock.calls[0].kwargs["errors"] == "ignore"
--
2.37.3

250
ignore-extend-declarations-from-excluded-sls-files.patch
View File

@ -1,250 +0,0 @@
From e4aff9ca68ce142c87ec875846d8916b9df8e6c5 Mon Sep 17 00:00:00 2001
From: Alexander Graul <agraul@suse.com>
Date: Fri, 21 Oct 2022 14:39:52 +0200
Subject: [PATCH] Ignore extend declarations from excluded sls files
* Use both test sls files
(cherry picked from commit 3cb5f5a14ff68d0bd809a4adba7d820534d0f7c7)
* Test that excluded sls files can't extend others
(cherry picked from commit e91c1a608b3c016b2c30bf324e969cd097ddf776)
* Ignore extend declarations from excluded sls files
sls files that are excluded should not affect other sls files by
extending their states. Exclude statements are processed very late in
the state processing pipeline to ensure they are not overridden. By that
time, extend declarations are already processed.
Luckily, it's not necessary to change much, during the extend
declarations processing it is easy to check if the sls file that
contains a given extend declaration is excluded.
(cherry picked from commit 856b23c45dd3be78d8879a0b0c4aa6356afea3cf)
---
changelog/62082.fixed | 1 +
salt/state.py | 19 +++
.../unit/state/test_state_highstate.py | 152 +++++++++++++++++-
3 files changed, 171 insertions(+), 1 deletion(-)
create mode 100644 changelog/62082.fixed
diff --git a/changelog/62082.fixed b/changelog/62082.fixed
new file mode 100644
index 0000000000..02e5f5ff40
--- /dev/null
+++ b/changelog/62082.fixed
@@ -0,0 +1 @@
+Ignore extend declarations in sls files that are excluded.
diff --git a/salt/state.py b/salt/state.py
index 316dcdec63..f5579fbb69 100644
--- a/salt/state.py
+++ b/salt/state.py
@@ -1680,6 +1680,25 @@ class State:
else:
name = ids[0][0]
+ sls_excludes = []
+ # excluded sls are plain list items or dicts with an "sls" key
+ for exclude in high.get("__exclude__", []):
+ if isinstance(exclude, str):
+ sls_excludes.append(exclude)
+ elif exclude.get("sls"):
+ sls_excludes.append(exclude["sls"])
+
+ if body.get("__sls__") in sls_excludes:
+ log.debug(
+ "Cannot extend ID '%s' in '%s:%s' because '%s:%s' is excluded.",
+ name,
+ body.get("__env__", "base"),
+ body.get("__sls__", "base"),
+ body.get("__env__", "base"),
+ body.get("__sls__", "base"),
+ )
+ continue
+
for state, run in body.items():
if state.startswith("__"):
continue
diff --git a/tests/pytests/unit/state/test_state_highstate.py b/tests/pytests/unit/state/test_state_highstate.py
index 059f83fd9f..7c72cc8e09 100644
--- a/tests/pytests/unit/state/test_state_highstate.py
+++ b/tests/pytests/unit/state/test_state_highstate.py
@@ -3,9 +3,11 @@
"""
import logging
+import textwrap
import pytest # pylint: disable=unused-import
import salt.state
+from salt.utils.odict import OrderedDict
log = logging.getLogger(__name__)
@@ -180,7 +182,7 @@ def test_find_sls_ids_with_exclude(highstate, state_tree_dir):
with pytest.helpers.temp_file(
"slsfile1.sls", slsfile1, sls_dir
), pytest.helpers.temp_file(
- "slsfile1.sls", slsfile1, sls_dir
+ "slsfile2.sls", slsfile2, sls_dir
), pytest.helpers.temp_file(
"stateB.sls", stateB, sls_dir
), pytest.helpers.temp_file(
@@ -196,3 +198,151 @@ def test_find_sls_ids_with_exclude(highstate, state_tree_dir):
high, _ = highstate.render_highstate(matches)
ret = salt.state.find_sls_ids("issue-47182.stateA.newer", high)
assert ret == [("somestuff", "cmd")]
+
+
+def test_dont_extend_in_excluded_sls_file(highstate, state_tree_dir):
+ """
+ See https://github.com/saltstack/salt/issues/62082#issuecomment-1245461333
+ """
+ top_sls = textwrap.dedent(
+ """\
+ base:
+ '*':
+ - test1
+ - exclude
+ """
+ )
+ exclude_sls = textwrap.dedent(
+ """\
+ exclude:
+ - sls: test2
+ """
+ )
+ test1_sls = textwrap.dedent(
+ """\
+ include:
+ - test2
+
+ test1:
+ cmd.run:
+ - name: echo test1
+ """
+ )
+ test2_sls = textwrap.dedent(
+ """\
+ extend:
+ test1:
+ cmd.run:
+ - name: echo "override test1 in test2"
+
+ test2-id:
+ cmd.run:
+ - name: echo test2
+ """
+ )
+ sls_dir = str(state_tree_dir)
+ with pytest.helpers.temp_file(
+ "top.sls", top_sls, sls_dir
+ ), pytest.helpers.temp_file(
+ "test1.sls", test1_sls, sls_dir
+ ), pytest.helpers.temp_file(
+ "test2.sls", test2_sls, sls_dir
+ ), pytest.helpers.temp_file(
+ "exclude.sls", exclude_sls, sls_dir
+ ):
+ # manually compile the high data, error checking is not needed in this
+ # test case.
+ top = highstate.get_top()
+ matches = highstate.top_matches(top)
+ high, _ = highstate.render_highstate(matches)
+
+ # high is mutated by call_high and the different "pipeline steps"
+ assert high == OrderedDict(
+ [
+ (
+ "__extend__",
+ [
+ {
+ "test1": OrderedDict(
+ [
+ ("__sls__", "test2"),
+ ("__env__", "base"),
+ (
+ "cmd",
+ [
+ OrderedDict(
+ [
+ (
+ "name",
+ 'echo "override test1 in test2"',
+ )
+ ]
+ ),
+ "run",
+ ],
+ ),
+ ]
+ )
+ }
+ ],
+ ),
+ (
+ "test1",
+ OrderedDict(
+ [
+ (
+ "cmd",
+ [
+ OrderedDict([("name", "echo test1")]),
+ "run",
+ {"order": 10001},
+ ],
+ ),
+ ("__sls__", "test1"),
+ ("__env__", "base"),
+ ]
+ ),
+ ),
+ (
+ "test2-id",
+ OrderedDict(
+ [
+ (
+ "cmd",
+ [
+ OrderedDict([("name", "echo test2")]),
+ "run",
+ {"order": 10000},
+ ],
+ ),
+ ("__sls__", "test2"),
+ ("__env__", "base"),
+ ]
+ ),
+ ),
+ ("__exclude__", [OrderedDict([("sls", "test2")])]),
+ ]
+ )
+ highstate.state.call_high(high)
+ # assert that the extend declaration was not applied
+ assert high == OrderedDict(
+ [
+ (
+ "test1",
+ OrderedDict(
+ [
+ (
+ "cmd",
+ [
+ OrderedDict([("name", "echo test1")]),
+ "run",
+ {"order": 10001},
+ ],
+ ),
+ ("__sls__", "test1"),
+ ("__env__", "base"),
+ ]
+ ),
+ )
+ ]
+ )
--
2.37.3

214
ignore-non-utf8-characters-while-reading-files-with-.patch
View File

@ -1,214 +0,0 @@
From 3f1b1180ba34e9ab3a4453248c733f11aa193f1b Mon Sep 17 00:00:00 2001
From: Victor Zhestkov <Victor.Zhestkov@suse.com>
Date: Wed, 14 Sep 2022 14:57:29 +0300
Subject: [PATCH] Ignore non utf8 characters while reading files with
core grains module (bsc#1202165)
* Ignore UnicodeDecodeError on reading files with core grains
* Add tests for non utf8 chars in cmdline
* Blacken modified lines
* Fix the tests
* Add changelog entry
* Change ignore to surrogateescape for kernelparameters
* Turn static test files to dynamic
---
changelog/62633.fixed | 1 +
salt/grains/core.py | 12 ++-
tests/pytests/unit/grains/test_core.py | 118 +++++++++++++++++++++++++
3 files changed, 128 insertions(+), 3 deletions(-)
create mode 100644 changelog/62633.fixed
diff --git a/changelog/62633.fixed b/changelog/62633.fixed
new file mode 100644
index 0000000000..1ab74f9122
--- /dev/null
+++ b/changelog/62633.fixed
@@ -0,0 +1 @@
+Prevent possible tracebacks in core grains module by ignoring non utf8 characters in /proc/1/environ, /proc/1/cmdline, /proc/cmdline
diff --git a/salt/grains/core.py b/salt/grains/core.py
index 047c33ffd3..76f3767ddf 100644
--- a/salt/grains/core.py
+++ b/salt/grains/core.py
@@ -1089,7 +1089,9 @@ def _virtual(osdata):
if ("virtual_subtype" not in grains) or (grains["virtual_subtype"] != "LXC"):
if os.path.isfile("/proc/1/environ"):
try:
- with salt.utils.files.fopen("/proc/1/environ", "r") as fhr:
+ with salt.utils.files.fopen(
+ "/proc/1/environ", "r", errors="ignore"
+ ) as fhr:
fhr_contents = fhr.read()
if "container=lxc" in fhr_contents:
grains["virtual"] = "container"
@@ -1909,7 +1911,9 @@ def os_data():
grains["init"] = "systemd"
except OSError:
try:
- with salt.utils.files.fopen("/proc/1/cmdline") as fhr:
+ with salt.utils.files.fopen(
+ "/proc/1/cmdline", "r", errors="ignore"
+ ) as fhr:
init_cmdline = fhr.read().replace("\x00", " ").split()
except OSError:
pass
@@ -3154,7 +3158,9 @@ def kernelparams():
return {}
else:
try:
- with salt.utils.files.fopen("/proc/cmdline", "r") as fhr:
+ with salt.utils.files.fopen(
+ "/proc/cmdline", "r", errors="surrogateescape"
+ ) as fhr:
cmdline = fhr.read()
grains = {"kernelparams": []}
for data in [
diff --git a/tests/pytests/unit/grains/test_core.py b/tests/pytests/unit/grains/test_core.py
index 7c4ea1f17f..c06cdb2db0 100644
--- a/tests/pytests/unit/grains/test_core.py
+++ b/tests/pytests/unit/grains/test_core.py
@@ -11,6 +11,7 @@ import os
import pathlib
import platform
import socket
+import tempfile
import textwrap
from collections import namedtuple
@@ -2738,6 +2739,38 @@ def test_kernelparams_return_linux(cmdline, expectation):
assert core.kernelparams() == expectation
+@pytest.mark.skip_unless_on_linux
+def test_kernelparams_return_linux_non_utf8():
+ _salt_utils_files_fopen = salt.utils.files.fopen
+
+ expected = {
+ "kernelparams": [
+ ("TEST_KEY1", "VAL1"),
+ ("TEST_KEY2", "VAL2"),
+ ("BOOTABLE_FLAG", "\udc80"),
+ ("TEST_KEY_NOVAL", None),
+ ("TEST_KEY3", "3"),
+ ]
+ }
+
+ with tempfile.TemporaryDirectory() as tempdir:
+
+ def _open_mock(file_name, *args, **kwargs):
+ return _salt_utils_files_fopen(
+ os.path.join(tempdir, "cmdline"), *args, **kwargs
+ )
+
+ with salt.utils.files.fopen(
+ os.path.join(tempdir, "cmdline"),
+ "wb",
+ ) as cmdline_fh, patch("salt.utils.files.fopen", _open_mock):
+ cmdline_fh.write(
+ b'TEST_KEY1=VAL1 TEST_KEY2=VAL2 BOOTABLE_FLAG="\x80" TEST_KEY_NOVAL TEST_KEY3=3\n'
+ )
+ cmdline_fh.close()
+ assert core.kernelparams() == expected
+
+
def test_linux_gpus():
"""
Test GPU detection on Linux systems
@@ -2940,3 +2973,88 @@ def test_virtual_set_virtual_ec2():
assert virtual_grains["virtual"] == "kvm"
assert "virtual_subtype" not in virtual_grains
+
+
+@pytest.mark.skip_on_windows
+def test_linux_proc_files_with_non_utf8_chars():
+ _salt_utils_files_fopen = salt.utils.files.fopen
+
+ empty_mock = MagicMock(return_value={})
+
+ with tempfile.TemporaryDirectory() as tempdir:
+
+ def _mock_open(filename, *args, **kwargs):
+ return _salt_utils_files_fopen(
+ os.path.join(tempdir, "cmdline-1"), *args, **kwargs
+ )
+
+ with salt.utils.files.fopen(
+ os.path.join(tempdir, "cmdline-1"),
+ "wb",
+ ) as cmdline_fh, patch("os.path.isfile", return_value=False), patch(
+ "salt.utils.files.fopen", _mock_open
+ ), patch.dict(
+ core.__salt__,
+ {
+ "cmd.retcode": salt.modules.cmdmod.retcode,
+ "cmd.run": MagicMock(return_value=""),
+ },
+ ), patch.object(
+ core, "_linux_bin_exists", return_value=False
+ ), patch.object(
+ core, "_parse_lsb_release", return_value=empty_mock
+ ), patch.object(
+ core, "_parse_os_release", return_value=empty_mock
+ ), patch.object(
+ core, "_hw_data", return_value=empty_mock
+ ), patch.object(
+ core, "_virtual", return_value=empty_mock
+ ), patch.object(
+ core, "_bsd_cpudata", return_value=empty_mock
+ ), patch.object(
+ os, "stat", side_effect=OSError()
+ ):
+ cmdline_fh.write(
+ b"/usr/lib/systemd/systemd\x00--switched-root\x00--system\x00--deserialize\x0028\x80\x00"
+ )
+ cmdline_fh.close()
+ os_grains = core.os_data()
+ assert os_grains != {}
+
+
+@pytest.mark.skip_on_windows
+def test_virtual_linux_proc_files_with_non_utf8_chars():
+ _salt_utils_files_fopen = salt.utils.files.fopen
+
+ def _is_file_mock(filename):
+ if filename == "/proc/1/environ":
+ return True
+ return False
+
+ with tempfile.TemporaryDirectory() as tempdir:
+
+ def _mock_open(filename, *args, **kwargs):
+ return _salt_utils_files_fopen(
+ os.path.join(tempdir, "environ"), *args, **kwargs
+ )
+
+ with salt.utils.files.fopen(
+ os.path.join(tempdir, "environ"),
+ "wb",
+ ) as environ_fh, patch("os.path.isfile", _is_file_mock), patch(
+ "salt.utils.files.fopen", _mock_open
+ ), patch.object(
+ salt.utils.path, "which", MagicMock(return_value=None)
+ ), patch.dict(
+ core.__salt__,
+ {
+ "cmd.run_all": MagicMock(
+ return_value={"retcode": 1, "stderr": "", "stdout": ""}
+ ),
+ "cmd.run": MagicMock(return_value=""),
+ },
+ ):
+ environ_fh.write(b"KEY1=VAL1 KEY2=VAL2\x80 KEY2=VAL2")
+ environ_fh.close()
+ virt_grains = core._virtual({"kernel": "Linux"})
+ assert virt_grains == {"virtual": "physical"}
--
2.37.3

63
include-stdout-in-error-message-for-zypperpkg-559.patch
View File

@ -1,63 +0,0 @@
From f9fe9ea009915478ea8f7896dff2c281e68b5d36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yeray=20Guti=C3=A9rrez=20Cedr=C3=A9s?=
<yeray.gutierrez@suse.com>
Date: Fri, 14 Oct 2022 08:41:40 +0100
Subject: [PATCH] Include stdout in error message for zypperpkg (#559)
---
salt/modules/zypperpkg.py | 5 +++++
tests/unit/modules/test_zypperpkg.py | 17 ++++++++++++++++-
2 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/salt/modules/zypperpkg.py b/salt/modules/zypperpkg.py
index c787d4009d..5d745c432d 100644
--- a/salt/modules/zypperpkg.py
+++ b/salt/modules/zypperpkg.py
@@ -339,6 +339,11 @@ class _Zypper:
and self.__call_result["stderr"].strip()
or ""
)
+ msg += (
+ self.__call_result["stdout"]
+ and self.__call_result["stdout"].strip()
+ or ""
+ )
if msg:
_error_msg.append(msg)
else:
diff --git a/tests/unit/modules/test_zypperpkg.py b/tests/unit/modules/test_zypperpkg.py
index 37d555844c..bcd001cd85 100644
--- a/tests/unit/modules/test_zypperpkg.py
+++ b/tests/unit/modules/test_zypperpkg.py
@@ -207,11 +207,26 @@ class ZypperTestCase(TestCase, LoaderModuleMockMixin):
):
zypper.__zypper__.xml.call("crashme")
+ output_to_user_stdout = "Output to user to stdout"
+ output_to_user_stderr = "Output to user to stderr"
+ sniffer = RunSniffer(
+ stdout=output_to_user_stdout, stderr=output_to_user_stderr, retcode=1
+ )
+ with patch.dict(
+ "salt.modules.zypperpkg.__salt__", {"cmd.run_all": sniffer}
+ ), patch.object(zypper.__zypper__, "_is_rpm_lock", return_value=False):
with self.assertRaisesRegex(
- CommandExecutionError, "^Zypper command failure: Check Zypper's logs.$"
+ CommandExecutionError,
+ "^Zypper command failure: {}$".format(
+ output_to_user_stderr + output_to_user_stdout
+ ),
):
zypper.__zypper__.call("crashme again")
+ sniffer = RunSniffer(retcode=1)
+ with patch.dict(
+ "salt.modules.zypperpkg.__salt__", {"cmd.run_all": sniffer}
+ ), patch.object(zypper.__zypper__, "_is_rpm_lock", return_value=False):
zypper.__zypper__.noraise.call("stay quiet")
self.assertEqual(zypper.__zypper__.error_msg, "Check Zypper's logs.")
--
2.37.3

414
make-pass-renderer-configurable-other-fixes-532.patch
View File

@ -1,414 +0,0 @@
From 030e2cb20af09673d5f38d68bcb257c6c839a2f3 Mon Sep 17 00:00:00 2001
From: Daniel Mach <daniel.mach@gmail.com>
Date: Thu, 6 Oct 2022 11:58:23 +0200
Subject: [PATCH] Make pass renderer configurable & other fixes (#532)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* pass: Use a secure way of handling pass arguments
The original code would fail on pass paths with spaces,
because they would be split into multiple arguments.
* pass: Strip only trailing newline characters from the secret
* pass: Do not modify $HOME env globally
Just set $HOME for calling the pass binary
to avoid affecting anything outside the pass renderer.
* pass: Use pass executable path from _get_pass_exec()
* Make the pass renderer more configurable
1. Allow us to make the pass renderer fail during pillar rendering
when a secret corresponding with a pass path cannot be fetched.
For this we add a master config variable pass_strict_fetch.
2. Allow to have prefix for variables that should be processed
with the pass renderer.
For this we add a master config variable pass_variable_prefix.
3. Allow us to configure pass' GNUPGHOME and PASSWORD_STORE_DIR
environmental variables.
For this we add master config variables pass_gnupghome and pass_dir.
* Add tests for the pass renderer
* pass: Handle FileNotFoundError when pass binary is not available
Co-authored-by: Marcus Rückert <darix@nordisch.org>
---
changelog/62120.added | 4 +
changelog/62120.fixed | 4 +
salt/config/__init__.py | 12 ++
salt/renderers/pass.py | 104 ++++++++++++--
tests/pytests/unit/renderers/test_pass.py | 164 ++++++++++++++++++++++
5 files changed, 274 insertions(+), 14 deletions(-)
create mode 100644 changelog/62120.added
create mode 100644 changelog/62120.fixed
create mode 100644 tests/pytests/unit/renderers/test_pass.py
diff --git a/changelog/62120.added b/changelog/62120.added
new file mode 100644
index 0000000000..4303d124f0
--- /dev/null
+++ b/changelog/62120.added
@@ -0,0 +1,4 @@
+Config option pass_variable_prefix allows to distinguish variables that contain paths to pass secrets.
+Config option pass_strict_fetch allows to error out when a secret cannot be fetched from pass.
+Config option pass_dir allows setting the PASSWORD_STORE_DIR env for pass.
+Config option pass_gnupghome allows setting the $GNUPGHOME env for pass.
diff --git a/changelog/62120.fixed b/changelog/62120.fixed
new file mode 100644
index 0000000000..22a9711383
--- /dev/null
+++ b/changelog/62120.fixed
@@ -0,0 +1,4 @@
+Pass executable path from _get_path_exec() is used when calling the program.
+The $HOME env is no longer modified globally.
+Only trailing newlines are stripped from the fetched secret.
+Pass process arguments are handled in a secure way.
diff --git a/salt/config/__init__.py b/salt/config/__init__.py
index 7cdee12c4d..0cc0deb874 100644
--- a/salt/config/__init__.py
+++ b/salt/config/__init__.py
@@ -967,6 +967,14 @@ VALID_OPTS = immutabletypes.freeze(
# Use Adler32 hashing algorithm for server_id (default False until Sodium, "adler32" after)
# Possible values are: False, adler32, crc32
"server_id_use_crc": (bool, str),
+ # pass renderer: Fetch secrets only for the template variables matching the prefix
+ "pass_variable_prefix": str,
+ # pass renderer: Whether to error out when unable to fetch a secret
+ "pass_strict_fetch": bool,
+ # pass renderer: Set GNUPGHOME env for Pass
+ "pass_gnupghome": str,
+ # pass renderer: Set PASSWORD_STORE_DIR env for Pass
+ "pass_dir": str,
}
)
@@ -1608,6 +1616,10 @@ DEFAULT_MASTER_OPTS = immutabletypes.freeze(
"fips_mode": False,
"detect_remote_minions": False,
"remote_minions_port": 22,
+ "pass_variable_prefix": "",
+ "pass_strict_fetch": False,
+ "pass_gnupghome": "",
+ "pass_dir": "",
}
)
diff --git a/salt/renderers/pass.py b/salt/renderers/pass.py
index 71b1021b96..ba0f152c23 100644
--- a/salt/renderers/pass.py
+++ b/salt/renderers/pass.py
@@ -45,6 +45,34 @@ Install pass binary
pass:
pkg.installed
+
+Salt master configuration options
+
+.. code-block:: yaml
+
+ # If the prefix is *not* set (default behavior), all template variables are
+ # considered for fetching secrets from Pass. Those that cannot be resolved
+ # to a secret are passed through.
+ #
+ # If the prefix is set, only the template variables with matching prefix are
+ # considered for fetching the secrets, other variables are passed through.
+ #
+ # For ease of use it is recommended to set the following options as well:
+ # renderer: 'jinja|yaml|pass'
+ # pass_strict_fetch: true
+ #
+ pass_variable_prefix: 'pass:'
+
+ # If set to 'true', error out when unable to fetch a secret for a template variable.
+ pass_strict_fetch: true
+
+ # Set GNUPGHOME env for Pass.
+ # Defaults to: ~/.gnupg
+ pass_gnupghome: <path>
+
+ # Set PASSWORD_STORE_DIR env for Pass.
+ # Defaults to: ~/.password-store
+ pass_dir: <path>
"""
@@ -54,7 +82,7 @@ from os.path import expanduser
from subprocess import PIPE, Popen
import salt.utils.path
-from salt.exceptions import SaltRenderError
+from salt.exceptions import SaltConfigurationError, SaltRenderError
log = logging.getLogger(__name__)
@@ -75,18 +103,71 @@ def _fetch_secret(pass_path):
Fetch secret from pass based on pass_path. If there is
any error, return back the original pass_path value
"""
- cmd = "pass show {}".format(pass_path.strip())
- log.debug("Fetching secret: %s", cmd)
+ pass_exec = _get_pass_exec()
+
+ # Make a backup in case we want to return the original value without stripped whitespaces
+ original_pass_path = pass_path
+
+ # Remove the optional prefix from pass path
+ pass_prefix = __opts__["pass_variable_prefix"]
+ if pass_prefix:
+ # If we do not see our prefix we do not want to process this variable
+ # and we return the unmodified pass path
+ if not pass_path.startswith(pass_prefix):
+ return pass_path
+
+ # strip the prefix from the start of the string
+ pass_path = pass_path[len(pass_prefix) :]
+
+ # The pass_strict_fetch option must be used with pass_variable_prefix
+ pass_strict_fetch = __opts__["pass_strict_fetch"]
+ if pass_strict_fetch and not pass_prefix:
+ msg = "The 'pass_strict_fetch' option requires 'pass_variable_prefix' option enabled"
+ raise SaltConfigurationError(msg)
+
+ # Remove whitespaces from the pass_path
+ pass_path = pass_path.strip()
- proc = Popen(cmd.split(" "), stdout=PIPE, stderr=PIPE)
- pass_data, pass_error = proc.communicate()
+ cmd = [pass_exec, "show", pass_path]
+ log.debug("Fetching secret: %s", " ".join(cmd))
+
+ # Make sure environment variable HOME is set, since Pass looks for the
+ # password-store under ~/.password-store.
+ env = os.environ.copy()
+ env["HOME"] = expanduser("~")
+
+ pass_dir = __opts__["pass_dir"]
+ if pass_dir:
+ env["PASSWORD_STORE_DIR"] = pass_dir
+
+ pass_gnupghome = __opts__["pass_gnupghome"]
+ if pass_gnupghome:
+ env["GNUPGHOME"] = pass_gnupghome
+
+ try:
+ proc = Popen(cmd, stdout=PIPE, stderr=PIPE, env=env)
+ pass_data, pass_error = proc.communicate()
+ pass_returncode = proc.returncode
+ except OSError as e:
+ pass_data, pass_error = "", str(e)
+ pass_returncode = 1
# The version of pass used during development sent output to
# stdout instead of stderr even though its returncode was non zero.
- if proc.returncode or not pass_data:
- log.warning("Could not fetch secret: %s %s", pass_data, pass_error)
- pass_data = pass_path
- return pass_data.strip()
+ if pass_returncode or not pass_data:
+ try:
+ pass_error = pass_error.decode("utf-8")
+ except (AttributeError, ValueError):
+ pass
+ msg = "Could not fetch secret '{}' from the password store: {}".format(
+ pass_path, pass_error
+ )
+ if pass_strict_fetch:
+ raise SaltRenderError(msg)
+ else:
+ log.warning(msg)
+ return original_pass_path
+ return pass_data.rstrip("\r\n")
def _decrypt_object(obj):
@@ -108,9 +189,4 @@ def render(pass_info, saltenv="base", sls="", argline="", **kwargs):
"""
Fetch secret from pass based on pass_path
"""
- _get_pass_exec()
-
- # Make sure environment variable HOME is set, since Pass looks for the
- # password-store under ~/.password-store.
- os.environ["HOME"] = expanduser("~")
return _decrypt_object(pass_info)
diff --git a/tests/pytests/unit/renderers/test_pass.py b/tests/pytests/unit/renderers/test_pass.py
new file mode 100644
index 0000000000..74e822c7ec
--- /dev/null
+++ b/tests/pytests/unit/renderers/test_pass.py
@@ -0,0 +1,164 @@
+import importlib
+
+import pytest
+
+import salt.config
+import salt.exceptions
+from tests.support.mock import MagicMock, patch
+
+# "pass" is a reserved keyword, we need to import it differently
+pass_ = importlib.import_module("salt.renderers.pass")
+
+
+@pytest.fixture
+def configure_loader_modules():
+ return {
+ pass_: {
+ "__opts__": salt.config.DEFAULT_MASTER_OPTS.copy(),
+ "_get_pass_exec": MagicMock(return_value="/usr/bin/pass"),
+ }
+ }
+
+
+# The default behavior is that if fetching a secret from pass fails,
+# the value is passed through. Even the trailing newlines are preserved.
+def test_passthrough():
+ pass_path = "secret\n"
+ expected = pass_path
+ result = pass_.render(pass_path)
+
+ assert result == expected
+
+
+# Fetch a secret in the strict mode.
+def test_strict_fetch():
+ config = {
+ "pass_variable_prefix": "pass:",
+ "pass_strict_fetch": True,
+ }
+
+ popen_mock = MagicMock(spec=pass_.Popen)
+ popen_mock.return_value.communicate.return_value = ("password123456\n", "")
+ popen_mock.return_value.returncode = 0
+
+ mocks = {
+ "Popen": popen_mock,
+ }
+
+ pass_path = "pass:secret"
+ expected = "password123456"
+ with patch.dict(pass_.__opts__, config), patch.dict(pass_.__dict__, mocks):
+ result = pass_.render(pass_path)
+
+ assert result == expected
+
+
+# Fail to fetch a secret in the strict mode.
+def test_strict_fetch_fail():
+ config = {
+ "pass_variable_prefix": "pass:",
+ "pass_strict_fetch": True,
+ }
+
+ popen_mock = MagicMock(spec=pass_.Popen)
+ popen_mock.return_value.communicate.return_value = ("", "Secret not found")
+ popen_mock.return_value.returncode = 1
+
+ mocks = {
+ "Popen": popen_mock,
+ }
+
+ pass_path = "pass:secret"
+ with patch.dict(pass_.__opts__, config), patch.dict(pass_.__dict__, mocks):
+ with pytest.raises(salt.exceptions.SaltRenderError):
+ pass_.render(pass_path)
+
+
+# Passthrough a value that doesn't have a pass prefix.
+def test_strict_fetch_passthrough():
+ config = {
+ "pass_variable_prefix": "pass:",
+ "pass_strict_fetch": True,
+ }
+
+ pass_path = "variable-without-pass-prefix\n"
+ expected = pass_path
+ with patch.dict(pass_.__opts__, config):
+ result = pass_.render(pass_path)
+
+ assert result == expected
+
+
+# Fetch a secret in the strict mode. The pass path contains spaces.
+def test_strict_fetch_pass_path_with_spaces():
+ config = {
+ "pass_variable_prefix": "pass:",
+ "pass_strict_fetch": True,
+ }
+
+ popen_mock = MagicMock(spec=pass_.Popen)
+ popen_mock.return_value.communicate.return_value = ("password123456\n", "")
+ popen_mock.return_value.returncode = 0
+
+ mocks = {
+ "Popen": popen_mock,
+ }
+
+ pass_path = "pass:se cr et"
+ with patch.dict(pass_.__opts__, config), patch.dict(pass_.__dict__, mocks):
+ pass_.render(pass_path)
+
+ call_args, call_kwargs = popen_mock.call_args_list[0]
+ assert call_args[0] == ["/usr/bin/pass", "show", "se cr et"]
+
+
+# Fetch a secret in the strict mode. The secret contains leading and trailing whitespaces.
+def test_strict_fetch_secret_with_whitespaces():
+ config = {
+ "pass_variable_prefix": "pass:",
+ "pass_strict_fetch": True,
+ }
+
+ popen_mock = MagicMock(spec=pass_.Popen)
+ popen_mock.return_value.communicate.return_value = (" \tpassword123456\t \r\n", "")
+ popen_mock.return_value.returncode = 0
+
+ mocks = {
+ "Popen": popen_mock,
+ }
+
+ pass_path = "pass:secret"
+ expected = " \tpassword123456\t " # only the trailing newlines get striped
+ with patch.dict(pass_.__opts__, config), patch.dict(pass_.__dict__, mocks):
+ result = pass_.render(pass_path)
+
+ assert result == expected
+
+
+# Test setting env variables based on config values:
+# - pass_gnupghome -> GNUPGHOME
+# - pass_dir -> PASSWORD_STORE_DIR
+def test_env():
+ config = {
+ "pass_variable_prefix": "pass:",
+ "pass_strict_fetch": True,
+ "pass_gnupghome": "/path/to/gnupghome",
+ "pass_dir": "/path/to/secretstore",
+ }
+
+ popen_mock = MagicMock(spec=pass_.Popen)
+ popen_mock.return_value.communicate.return_value = ("password123456\n", "")
+ popen_mock.return_value.returncode = 0
+
+ mocks = {
+ "Popen": popen_mock,
+ }
+
+ pass_path = "pass:secret"
+ expected = " \tpassword123456\t " # only the trailing newlines get striped
+ with patch.dict(pass_.__opts__, config), patch.dict(pass_.__dict__, mocks):
+ result = pass_.render(pass_path)
+
+ call_args, call_kwargs = popen_mock.call_args_list[0]
+ assert call_kwargs["env"]["GNUPGHOME"] == config["pass_gnupghome"]
+ assert call_kwargs["env"]["PASSWORD_STORE_DIR"] == config["pass_dir"]
--
2.37.3

30
make-sure-saltcacheloader-use-correct-fileclient-519.patch
View File

@ -1,30 +0,0 @@
From 03ce925098fb96ad2f2f4b7d4c151ef63aede75f Mon Sep 17 00:00:00 2001
From: Witek Bedyk <wbedyk@suse.com>
Date: Thu, 19 May 2022 12:52:12 +0200
Subject: [PATCH] Make sure SaltCacheLoader use correct fileclient (#519)
Backported from https://github.com/saltstack/salt/pull/61895
Signed-off-by: Witek Bedyk <witold.bedyk@suse.com>
---
salt/state.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/salt/state.py b/salt/state.py
index db228228a7..316dcdec63 100644
--- a/salt/state.py
+++ b/salt/state.py
@@ -4170,6 +4170,9 @@ class BaseHighState:
)
else:
try:
+ # Make sure SaltCacheLoader use correct fileclient
+ if context is None:
+ context = {"fileclient": self.client}
state = compile_template(
fn_,
self.state.rend,
--
2.37.3

296
normalize-package-names-once-with-pkg.installed-remo.patch
View File

@ -1,296 +0,0 @@
From f2dc43cf1db3fee41e328c68545ccac2576021ca Mon Sep 17 00:00:00 2001
From: Victor Zhestkov <vzhestkov@suse.com>
Date: Mon, 27 Jun 2022 18:01:21 +0300
Subject: [PATCH] Normalize package names once with pkg.installed/removed
using yum (bsc#1195895)
* Normalize the package names only once on install/remove
* Add test for checking pkg.installed/removed with only normalisation
* Fix split_arch conditions
* Fix test_pkg
---
salt/modules/yumpkg.py | 18 ++-
salt/states/pkg.py | 3 +-
tests/pytests/unit/states/test_pkg.py | 177 +++++++++++++++++++++++++-
3 files changed, 192 insertions(+), 6 deletions(-)
diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py
index 46f0b1f613..f52e084346 100644
--- a/salt/modules/yumpkg.py
+++ b/salt/modules/yumpkg.py
@@ -1460,7 +1460,12 @@ def install(
try:
pkg_params, pkg_type = __salt__["pkg_resource.parse_targets"](
- name, pkgs, sources, saltenv=saltenv, normalize=normalize, **kwargs
+ name,
+ pkgs,
+ sources,
+ saltenv=saltenv,
+ normalize=normalize and kwargs.get("split_arch", True),
+ **kwargs
)
except MinionError as exc:
raise CommandExecutionError(exc)
@@ -1612,7 +1617,10 @@ def install(
except ValueError:
pass
else:
- if archpart in salt.utils.pkg.rpm.ARCHES:
+ if archpart in salt.utils.pkg.rpm.ARCHES and (
+ archpart != __grains__["osarch"]
+ or kwargs.get("split_arch", True)
+ ):
arch = "." + archpart
pkgname = namepart
@@ -2143,11 +2151,13 @@ def remove(name=None, pkgs=None, **kwargs): # pylint: disable=W0613
arch = ""
pkgname = target
try:
- namepart, archpart = target.rsplit(".", 1)
+ namepart, archpart = pkgname.rsplit(".", 1)
except ValueError:
pass
else:
- if archpart in salt.utils.pkg.rpm.ARCHES:
+ if archpart in salt.utils.pkg.rpm.ARCHES and (
+ archpart != __grains__["osarch"] or kwargs.get("split_arch", True)
+ ):
arch = "." + archpart
pkgname = namepart
# Since we don't always have the arch info, epoch information has to parsed out. But
diff --git a/salt/states/pkg.py b/salt/states/pkg.py
index ef4e062145..cda966a1e8 100644
--- a/salt/states/pkg.py
+++ b/salt/states/pkg.py
@@ -1873,6 +1873,7 @@ def installed(
normalize=normalize,
update_holds=update_holds,
ignore_epoch=ignore_epoch,
+ split_arch=False,
**kwargs
)
except CommandExecutionError as exc:
@@ -2940,7 +2941,7 @@ def _uninstall(
}
changes = __salt__["pkg.{}".format(action)](
- name, pkgs=pkgs, version=version, **kwargs
+ name, pkgs=pkgs, version=version, split_arch=False, **kwargs
)
new = __salt__["pkg.list_pkgs"](versions_as_list=True, **kwargs)
failed = []
diff --git a/tests/pytests/unit/states/test_pkg.py b/tests/pytests/unit/states/test_pkg.py
index cba8201bda..ecb841e8ec 100644
--- a/tests/pytests/unit/states/test_pkg.py
+++ b/tests/pytests/unit/states/test_pkg.py
@@ -2,6 +2,8 @@ import logging
import pytest
import salt.modules.beacons as beaconmod
+import salt.modules.pkg_resource as pkg_resource
+import salt.modules.yumpkg as yumpkg
import salt.states.beacon as beaconstate
import salt.states.pkg as pkg
import salt.utils.state as state_utils
@@ -17,7 +19,7 @@ def configure_loader_modules():
pkg: {
"__env__": "base",
"__salt__": {},
- "__grains__": {"os": "CentOS"},
+ "__grains__": {"os": "CentOS", "os_family": "RedHat"},
"__opts__": {"test": False, "cachedir": ""},
"__instance_id__": "",
"__low__": {},
@@ -25,6 +27,15 @@ def configure_loader_modules():
},
beaconstate: {"__salt__": {}, "__opts__": {}},
beaconmod: {"__salt__": {}, "__opts__": {}},
+ pkg_resource: {
+ "__salt__": {},
+ "__grains__": {"os": "CentOS", "os_family": "RedHat"},
+ },
+ yumpkg: {
+ "__salt__": {},
+ "__grains__": {"osarch": "x86_64", "osmajorrelease": 7},
+ "__opts__": {},
+ },
}
@@ -726,3 +737,167 @@ def test_held_unheld(package_manager):
hold_mock.assert_not_called()
unhold_mock.assert_any_call(name="held-test", pkgs=["baz"])
unhold_mock.assert_any_call(name="held-test", pkgs=["bar"])
+
+
+def test_installed_with_single_normalize():
+ """
+ Test pkg.installed with preventing multiple package name normalisation
+ """
+
+ list_no_weird_installed = {
+ "pkga": "1.0.1",
+ "pkgb": "1.0.2",
+ "pkgc": "1.0.3",
+ }
+ list_no_weird_installed_ver_list = {
+ "pkga": ["1.0.1"],
+ "pkgb": ["1.0.2"],
+ "pkgc": ["1.0.3"],
+ }
+ list_with_weird_installed = {
+ "pkga": "1.0.1",
+ "pkgb": "1.0.2",
+ "pkgc": "1.0.3",
+ "weird-name-1.2.3-1234.5.6.test7tst.x86_64": "20220214-2.1",
+ }
+ list_with_weird_installed_ver_list = {
+ "pkga": ["1.0.1"],
+ "pkgb": ["1.0.2"],
+ "pkgc": ["1.0.3"],
+ "weird-name-1.2.3-1234.5.6.test7tst.x86_64": ["20220214-2.1"],
+ }
+ list_pkgs = MagicMock(
+ side_effect=[
+ # For the package with version specified
+ list_no_weird_installed_ver_list,
+ {},
+ list_no_weird_installed,
+ list_no_weird_installed_ver_list,
+ list_with_weird_installed,
+ list_with_weird_installed_ver_list,
+ # For the package with no version specified
+ list_no_weird_installed_ver_list,
+ {},
+ list_no_weird_installed,
+ list_no_weird_installed_ver_list,
+ list_with_weird_installed,
+ list_with_weird_installed_ver_list,
+ ]
+ )
+
+ salt_dict = {
+ "pkg.install": yumpkg.install,
+ "pkg.list_pkgs": list_pkgs,
+ "pkg.normalize_name": yumpkg.normalize_name,
+ "pkg_resource.version_clean": pkg_resource.version_clean,
+ "pkg_resource.parse_targets": pkg_resource.parse_targets,
+ }
+
+ with patch("salt.modules.yumpkg.list_pkgs", list_pkgs), patch(
+ "salt.modules.yumpkg.version_cmp", MagicMock(return_value=0)
+ ), patch(
+ "salt.modules.yumpkg._call_yum", MagicMock(return_value={"retcode": 0})
+ ) as call_yum_mock, patch.dict(
+ pkg.__salt__, salt_dict
+ ), patch.dict(
+ pkg_resource.__salt__, salt_dict
+ ), patch.dict(
+ yumpkg.__salt__, salt_dict
+ ), patch.dict(
+ yumpkg.__grains__, {"os": "CentOS", "osarch": "x86_64", "osmajorrelease": 7}
+ ), patch.object(
+ yumpkg, "list_holds", MagicMock()
+ ):
+
+ expected = {
+ "weird-name-1.2.3-1234.5.6.test7tst.x86_64": {
+ "old": "",
+ "new": "20220214-2.1",
+ }
+ }
+ ret = pkg.installed(
+ "test_install",
+ pkgs=[{"weird-name-1.2.3-1234.5.6.test7tst.x86_64.noarch": "20220214-2.1"}],
+ )
+ call_yum_mock.assert_called_once()
+ assert (
+ call_yum_mock.mock_calls[0].args[0][2]
+ == "weird-name-1.2.3-1234.5.6.test7tst.x86_64-20220214-2.1"
+ )
+ assert ret["result"]
+ assert ret["changes"] == expected
+
+
+def test_removed_with_single_normalize():
+ """
+ Test pkg.removed with preventing multiple package name normalisation
+ """
+
+ list_no_weird_installed = {
+ "pkga": "1.0.1",
+ "pkgb": "1.0.2",
+ "pkgc": "1.0.3",
+ }
+ list_no_weird_installed_ver_list = {
+ "pkga": ["1.0.1"],
+ "pkgb": ["1.0.2"],
+ "pkgc": ["1.0.3"],
+ }
+ list_with_weird_installed = {
+ "pkga": "1.0.1",
+ "pkgb": "1.0.2",
+ "pkgc": "1.0.3",
+ "weird-name-1.2.3-1234.5.6.test7tst.x86_64": "20220214-2.1",
+ }
+ list_with_weird_installed_ver_list = {
+ "pkga": ["1.0.1"],
+ "pkgb": ["1.0.2"],
+ "pkgc": ["1.0.3"],
+ "weird-name-1.2.3-1234.5.6.test7tst.x86_64": ["20220214-2.1"],
+ }
+ list_pkgs = MagicMock(
+ side_effect=[
+ list_with_weird_installed_ver_list,
+ list_with_weird_installed,
+ list_no_weird_installed,
+ list_no_weird_installed_ver_list,
+ ]
+ )
+
+ salt_dict = {
+ "pkg.remove": yumpkg.remove,
+ "pkg.list_pkgs": list_pkgs,
+ "pkg.normalize_name": yumpkg.normalize_name,
+ "pkg_resource.parse_targets": pkg_resource.parse_targets,
+ "pkg_resource.version_clean": pkg_resource.version_clean,
+ }
+
+ with patch("salt.modules.yumpkg.list_pkgs", list_pkgs), patch(
+ "salt.modules.yumpkg.version_cmp", MagicMock(return_value=0)
+ ), patch(
+ "salt.modules.yumpkg._call_yum", MagicMock(return_value={"retcode": 0})
+ ) as call_yum_mock, patch.dict(
+ pkg.__salt__, salt_dict
+ ), patch.dict(
+ pkg_resource.__salt__, salt_dict
+ ), patch.dict(
+ yumpkg.__salt__, salt_dict
+ ):
+
+ expected = {
+ "weird-name-1.2.3-1234.5.6.test7tst.x86_64": {
+ "old": "20220214-2.1",
+ "new": "",
+ }
+ }
+ ret = pkg.removed(
+ "test_remove",
+ pkgs=[{"weird-name-1.2.3-1234.5.6.test7tst.x86_64.noarch": "20220214-2.1"}],
+ )
+ call_yum_mock.assert_called_once()
+ assert (
+ call_yum_mock.mock_calls[0].args[0][2]
+ == "weird-name-1.2.3-1234.5.6.test7tst.x86_64-20220214-2.1"
+ )
+ assert ret["result"]
+ assert ret["changes"] == expected
--
2.37.3

297
retry-if-rpm-lock-is-temporarily-unavailable-547.patch
View File

@ -1,297 +0,0 @@
From 4a9ec335e7da2f0e3314580e43075bb69fe90c38 Mon Sep 17 00:00:00 2001
From: Witek Bedyk <witold.bedyk@suse.com>
Date: Mon, 29 Aug 2022 14:16:00 +0200
Subject: [PATCH] Retry if RPM lock is temporarily unavailable (#547)
* Retry if RPM lock is temporarily unavailable
Backported from saltstack/salt#62204
Signed-off-by: Witek Bedyk <witold.bedyk@suse.com>
* Sync formating fixes from upstream
Signed-off-by: Witek Bedyk <witold.bedyk@suse.com>
---
changelog/62204.fixed | 1 +
salt/modules/zypperpkg.py | 117 +++++++++++++++++----------
tests/unit/modules/test_zypperpkg.py | 45 ++++++++++-
3 files changed, 115 insertions(+), 48 deletions(-)
create mode 100644 changelog/62204.fixed
diff --git a/changelog/62204.fixed b/changelog/62204.fixed
new file mode 100644
index 0000000000..59f1914593
--- /dev/null
+++ b/changelog/62204.fixed
@@ -0,0 +1 @@
+Fixed Zypper module failing on RPM lock file being temporarily unavailable.
diff --git a/salt/modules/zypperpkg.py b/salt/modules/zypperpkg.py
index 2c36e2968a..c787d4009d 100644
--- a/salt/modules/zypperpkg.py
+++ b/salt/modules/zypperpkg.py
@@ -14,6 +14,7 @@ Package support for openSUSE via the zypper package manager
import configparser
import datetime
+import errno
import fnmatch
import logging
import os
@@ -39,6 +40,9 @@ from salt.exceptions import CommandExecutionError, MinionError, SaltInvocationEr
# pylint: disable=import-error,redefined-builtin,no-name-in-module
from salt.utils.versions import LooseVersion
+if salt.utils.files.is_fcntl_available():
+ import fcntl
+
log = logging.getLogger(__name__)
HAS_ZYPP = False
@@ -106,6 +110,7 @@ class _Zypper:
XML_DIRECTIVES = ["-x", "--xmlout"]
# ZYPPER_LOCK is not affected by --root
ZYPPER_LOCK = "/var/run/zypp.pid"
+ RPM_LOCK = "/var/lib/rpm/.rpm.lock"
TAG_RELEASED = "zypper/released"
TAG_BLOCKED = "zypper/blocked"
@@ -276,7 +281,7 @@ class _Zypper:
and self.exit_code not in self.WARNING_EXIT_CODES
)
- def _is_lock(self):
+ def _is_zypper_lock(self):
"""
Is this is a lock error code?
@@ -284,6 +289,23 @@ class _Zypper:
"""
return self.exit_code == self.LOCK_EXIT_CODE
+ def _is_rpm_lock(self):
+ """
+ Is this an RPM lock error?
+ """
+ if salt.utils.files.is_fcntl_available():
+ if self.exit_code > 0 and os.path.exists(self.RPM_LOCK):
+ with salt.utils.files.fopen(self.RPM_LOCK, mode="w+") as rfh:
+ try:
+ fcntl.lockf(rfh, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except OSError as err:
+ if err.errno == errno.EAGAIN:
+ return True
+ else:
+ fcntl.lockf(rfh, fcntl.LOCK_UN)
+
+ return False
+
def _is_xml_mode(self):
"""
Is Zypper's output is in XML format?
@@ -306,7 +328,7 @@ class _Zypper:
raise CommandExecutionError("No output result from Zypper?")
self.exit_code = self.__call_result["retcode"]
- if self._is_lock():
+ if self._is_zypper_lock() or self._is_rpm_lock():
return False
if self._is_error():
@@ -387,48 +409,11 @@ class _Zypper:
if self._check_result():
break
- if os.path.exists(self.ZYPPER_LOCK):
- try:
- with salt.utils.files.fopen(self.ZYPPER_LOCK) as rfh:
- data = __salt__["ps.proc_info"](
- int(rfh.readline()),
- attrs=["pid", "name", "cmdline", "create_time"],
- )
- data["cmdline"] = " ".join(data["cmdline"])
- data["info"] = "Blocking process created at {}.".format(
- datetime.datetime.utcfromtimestamp(
- data["create_time"]
- ).isoformat()
- )
- data["success"] = True
- except Exception as err: # pylint: disable=broad-except
- data = {
- "info": (
- "Unable to retrieve information about blocking process: {}".format(
- err.message
- )
- ),
- "success": False,
- }
- else:
- data = {
- "info": "Zypper is locked, but no Zypper lock has been found.",
- "success": False,
- }
-
- if not data["success"]:
- log.debug("Unable to collect data about blocking process.")
- else:
- log.debug("Collected data about blocking process.")
-
- __salt__["event.fire_master"](data, self.TAG_BLOCKED)
- log.debug(
- "Fired a Zypper blocked event to the master with the data: %s", data
- )
- log.debug("Waiting 5 seconds for Zypper gets released...")
- time.sleep(5)
- if not was_blocked:
- was_blocked = True
+ if self._is_zypper_lock():
+ self._handle_zypper_lock_file()
+ if self._is_rpm_lock():
+ self._handle_rpm_lock_file()
+ was_blocked = True
if was_blocked:
__salt__["event.fire_master"](
@@ -451,6 +436,50 @@ class _Zypper:
or self.__call_result["stdout"]
)
+ def _handle_zypper_lock_file(self):
+ if os.path.exists(self.ZYPPER_LOCK):
+ try:
+ with salt.utils.files.fopen(self.ZYPPER_LOCK) as rfh:
+ data = __salt__["ps.proc_info"](
+ int(rfh.readline()),
+ attrs=["pid", "name", "cmdline", "create_time"],
+ )
+ data["cmdline"] = " ".join(data["cmdline"])
+ data["info"] = "Blocking process created at {}.".format(
+ datetime.datetime.utcfromtimestamp(
+ data["create_time"]
+ ).isoformat()
+ )
+ data["success"] = True
+ except Exception as err: # pylint: disable=broad-except
+ data = {
+ "info": (
+ "Unable to retrieve information about "
+ "blocking process: {}".format(err)
+ ),
+ "success": False,
+ }
+ else:
+ data = {
+ "info": "Zypper is locked, but no Zypper lock has been found.",
+ "success": False,
+ }
+ if not data["success"]:
+ log.debug("Unable to collect data about blocking process.")
+ else:
+ log.debug("Collected data about blocking process.")
+ __salt__["event.fire_master"](data, self.TAG_BLOCKED)
+ log.debug("Fired a Zypper blocked event to the master with the data: %s", data)
+ log.debug("Waiting 5 seconds for Zypper gets released...")
+ time.sleep(5)
+
+ def _handle_rpm_lock_file(self):
+ data = {"info": "RPM is temporarily locked.", "success": True}
+ __salt__["event.fire_master"](data, self.TAG_BLOCKED)
+ log.debug("Fired an RPM blocked event to the master with the data: %s", data)
+ log.debug("Waiting 5 seconds for RPM to get released...")
+ time.sleep(5)
+
__zypper__ = _Zypper()
diff --git a/tests/unit/modules/test_zypperpkg.py b/tests/unit/modules/test_zypperpkg.py
index 3f1560a385..37d555844c 100644
--- a/tests/unit/modules/test_zypperpkg.py
+++ b/tests/unit/modules/test_zypperpkg.py
@@ -4,6 +4,7 @@
import configparser
+import errno
import io
import os
from xml.dom import minidom
@@ -97,7 +98,7 @@ class ZypperTestCase(TestCase, LoaderModuleMockMixin):
}
with patch.dict(
zypper.__salt__, {"cmd.run_all": MagicMock(return_value=ref_out)}
- ):
+ ), patch.object(zypper.__zypper__, "_is_rpm_lock", return_value=False):
upgrades = zypper.list_upgrades(refresh=False)
self.assertEqual(len(upgrades), 3)
for pkg, version in {
@@ -198,7 +199,9 @@ class ZypperTestCase(TestCase, LoaderModuleMockMixin):
' type="error">Booya!</message></stream>'
)
sniffer = RunSniffer(stdout=stdout_xml_snippet, retcode=1)
- with patch.dict("salt.modules.zypperpkg.__salt__", {"cmd.run_all": sniffer}):
+ with patch.dict(
+ "salt.modules.zypperpkg.__salt__", {"cmd.run_all": sniffer}
+ ), patch.object(zypper.__zypper__, "_is_rpm_lock", return_value=False):
with self.assertRaisesRegex(
CommandExecutionError, "^Zypper command failure: Booya!$"
):
@@ -232,7 +235,7 @@ class ZypperTestCase(TestCase, LoaderModuleMockMixin):
with patch.dict(
"salt.modules.zypperpkg.__salt__",
{"cmd.run_all": MagicMock(return_value=ref_out)},
- ):
+ ), patch.object(zypper.__zypper__, "_is_rpm_lock", return_value=False):
with self.assertRaisesRegex(
CommandExecutionError,
"^Zypper command failure: Some handled zypper internal error{}Another"
@@ -245,7 +248,7 @@ class ZypperTestCase(TestCase, LoaderModuleMockMixin):
with patch.dict(
"salt.modules.zypperpkg.__salt__",
{"cmd.run_all": MagicMock(return_value=ref_out)},
- ):
+ ), patch.object(zypper.__zypper__, "_is_rpm_lock", return_value=False):
with self.assertRaisesRegex(
CommandExecutionError, "^Zypper command failure: Check Zypper's logs.$"
):
@@ -2064,3 +2067,37 @@ pattern() = package-c"""
python_shell=False,
env={"ZYPP_READONLY_HACK": "1"},
)
+
+ def test_is_rpm_lock_no_error(self):
+ with patch.object(os.path, "exists", return_value=True):
+ self.assertFalse(zypper.__zypper__._is_rpm_lock())
+
+ def test_rpm_lock_does_not_exist(self):
+ if salt.utils.files.is_fcntl_available():
+ zypper.__zypper__.exit_code = 1
+ with patch.object(
+ os.path, "exists", return_value=False
+ ) as mock_path_exists:
+ self.assertFalse(zypper.__zypper__._is_rpm_lock())
+ mock_path_exists.assert_called_with(zypper.__zypper__.RPM_LOCK)
+ zypper.__zypper__._reset()
+
+ def test_rpm_lock_acquirable(self):
+ if salt.utils.files.is_fcntl_available():
+ zypper.__zypper__.exit_code = 1
+ with patch.object(os.path, "exists", return_value=True), patch(
+ "fcntl.lockf", side_effect=OSError(errno.EAGAIN, "")
+ ) as lockf_mock, patch("salt.utils.files.fopen", mock_open()):
+ self.assertTrue(zypper.__zypper__._is_rpm_lock())
+ lockf_mock.assert_called()
+ zypper.__zypper__._reset()
+
+ def test_rpm_lock_not_acquirable(self):
+ if salt.utils.files.is_fcntl_available():
+ zypper.__zypper__.exit_code = 1
+ with patch.object(os.path, "exists", return_value=True), patch(
+ "fcntl.lockf"
+ ) as lockf_mock, patch("salt.utils.files.fopen", mock_open()):
+ self.assertFalse(zypper.__zypper__._is_rpm_lock())
+ self.assertEqual(lockf_mock.call_count, 2)
+ zypper.__zypper__._reset()
--
2.37.3

87
set-default-target-for-pip-from-venv_pip_target-envi.patch
View File

@ -1,87 +0,0 @@
From d561491c48ee30472e0d4699ba389648ef0d863a Mon Sep 17 00:00:00 2001
From: Victor Zhestkov <vzhestkov@suse.com>
Date: Mon, 27 Jun 2022 18:02:31 +0300
Subject: [PATCH] Set default target for pip from VENV_PIP_TARGET
environment variable
* Use VENV_PIP_TARGET as a target for pkg.install
if set and no target specified on the call
* Add test for VENV_PIP_TARGET environment variable
* Changelog entry
---
changelog/62089.changed | 1 +
salt/modules/pip.py | 6 +++++
tests/pytests/unit/modules/test_pip.py | 31 ++++++++++++++++++++++++++
3 files changed, 38 insertions(+)
create mode 100644 changelog/62089.changed
diff --git a/changelog/62089.changed b/changelog/62089.changed
new file mode 100644
index 0000000000..09feb2e922
--- /dev/null
+++ b/changelog/62089.changed
@@ -0,0 +1 @@
+Use VENV_PIP_TARGET environment variable as a default target for pip if present.
diff --git a/salt/modules/pip.py b/salt/modules/pip.py
index da26416662..9410024fd5 100644
--- a/salt/modules/pip.py
+++ b/salt/modules/pip.py
@@ -858,6 +858,12 @@ def install(
if build:
cmd.extend(["--build", build])
+ # Use VENV_PIP_TARGET environment variable value as target
+ # if set and no target specified on the function call
+ target_env = os.environ.get("VENV_PIP_TARGET", None)
+ if target is None and target_env is not None:
+ target = target_env
+
if target:
cmd.extend(["--target", target])
diff --git a/tests/pytests/unit/modules/test_pip.py b/tests/pytests/unit/modules/test_pip.py
index 405ec6c82e..ae9005d806 100644
--- a/tests/pytests/unit/modules/test_pip.py
+++ b/tests/pytests/unit/modules/test_pip.py
@@ -1773,3 +1773,34 @@ def test_when_version_is_called_with_a_user_it_should_be_passed_to_undelying_run
cwd=None,
python_shell=False,
)
+
+
+def test_install_target_from_VENV_PIP_TARGET_in_resulting_command():
+ pkg = "pep8"
+ target = "/tmp/foo"
+ target_env = "/tmp/bar"
+ mock = MagicMock(return_value={"retcode": 0, "stdout": ""})
+ environment = os.environ.copy()
+ environment["VENV_PIP_TARGET"] = target_env
+ with patch.dict(pip.__salt__, {"cmd.run_all": mock}), patch.object(
+ os, "environ", environment
+ ):
+ pip.install(pkg)
+ expected = [sys.executable, "-m", "pip", "install", "--target", target_env, pkg]
+ mock.assert_called_with(
+ expected,
+ saltenv="base",
+ runas=None,
+ use_vt=False,
+ python_shell=False,
+ )
+ mock.reset_mock()
+ pip.install(pkg, target=target)
+ expected = [sys.executable, "-m", "pip", "install", "--target", target, pkg]
+ mock.assert_called_with(
+ expected,
+ saltenv="base",
+ runas=None,
+ use_vt=False,
+ python_shell=False,
+ )
--
2.37.3

362
state.apply-don-t-check-for-cached-pillar-errors.patch
View File

@ -1,362 +0,0 @@
From cba6455bd0480bfb80c466a2b34a702a9afb5bd5 Mon Sep 17 00:00:00 2001
From: Alexander Graul <agraul@suse.com>
Date: Tue, 25 Jan 2022 17:20:55 +0100
Subject: [PATCH] state.apply: don't check for cached pillar errors
state.apply request new pillar data from the server. This done to always
have the most up-to-date pillar to work with. Previously, checking for
pillar errors looked at both the new pillar and the in-memory pillar.
The latter might contain pillar rendering errors even if the former does
not.
For this reason, only the new pillar should be checked, not both.
---
changelog/52354.fixed | 1 +
changelog/57180.fixed | 1 +
changelog/59339.fixed | 1 +
salt/modules/state.py | 17 ++-
.../modules/state/test_state_pillar_errors.py | 131 ++++++++++++++++++
.../pytests/unit/modules/state/test_state.py | 115 +++++----------
6 files changed, 177 insertions(+), 89 deletions(-)
create mode 100644 changelog/52354.fixed
create mode 100644 changelog/57180.fixed
create mode 100644 changelog/59339.fixed
create mode 100644 tests/pytests/integration/modules/state/test_state_pillar_errors.py
diff --git a/changelog/52354.fixed b/changelog/52354.fixed
new file mode 100644
index 0000000000..af885d77fa
--- /dev/null
+++ b/changelog/52354.fixed
@@ -0,0 +1 @@
+Don't check for cached pillar errors on state.apply
diff --git a/changelog/57180.fixed b/changelog/57180.fixed
new file mode 100644
index 0000000000..af885d77fa
--- /dev/null
+++ b/changelog/57180.fixed
@@ -0,0 +1 @@
+Don't check for cached pillar errors on state.apply
diff --git a/changelog/59339.fixed b/changelog/59339.fixed
new file mode 100644
index 0000000000..af885d77fa
--- /dev/null
+++ b/changelog/59339.fixed
@@ -0,0 +1 @@
+Don't check for cached pillar errors on state.apply
diff --git a/salt/modules/state.py b/salt/modules/state.py
index f214291328..c0feabe842 100644
--- a/salt/modules/state.py
+++ b/salt/modules/state.py
@@ -106,18 +106,17 @@ def _set_retcode(ret, highstate=None):
def _get_pillar_errors(kwargs, pillar=None):
"""
- Checks all pillars (external and internal) for errors.
- Return an error message, if anywhere or None.
+ Check pillar for errors.
+
+ If a pillar is passed, it will be checked. Otherwise, the in-memory pillar
+ will checked instead. Passing kwargs['force'] = True short cuts the check
+ and always returns None, indicating no errors.
:param kwargs: dictionary of options
- :param pillar: external pillar
- :return: None or an error message
+ :param pillar: pillar
+ :return: None or a list of error messages
"""
- return (
- None
- if kwargs.get("force")
- else (pillar or {}).get("_errors", __pillar__.get("_errors")) or None
- )
+ return None if kwargs.get("force") else (pillar or __pillar__).get("_errors")
def _wait(jid):
diff --git a/tests/pytests/integration/modules/state/test_state_pillar_errors.py b/tests/pytests/integration/modules/state/test_state_pillar_errors.py
new file mode 100644
index 0000000000..af65a05945
--- /dev/null
+++ b/tests/pytests/integration/modules/state/test_state_pillar_errors.py
@@ -0,0 +1,131 @@
+#!/usr/bin/python3
+
+import textwrap
+
+import pytest
+from saltfactories.utils.functional import StateResult
+
+pytestmark = [
+ pytest.mark.slow_test,
+]
+
+
+@pytest.fixture(scope="module")
+def reset_pillar(salt_call_cli):
+ try:
+ # Run tests
+ yield
+ finally:
+ # Refresh pillar once all tests are done.
+ ret = salt_call_cli.run("saltutil.refresh_pillar", wait=True)
+ assert ret.exitcode == 0
+ assert ret.json is True
+
+
+@pytest.fixture
+def testfile_path(tmp_path, base_env_state_tree_root_dir):
+ testfile = tmp_path / "testfile"
+ sls_contents = textwrap.dedent(
+ """
+ {}:
+ file:
+ - managed
+ - source: salt://testfile
+ - makedirs: true
+ """.format(testfile)
+ )
+ with pytest.helpers.temp_file(
+ "sls-id-test.sls", sls_contents, base_env_state_tree_root_dir
+ ):
+ yield testfile
+
+
+@pytest.mark.usefixtures("testfile_path", "reset_pillar")
+def test_state_apply_aborts_on_pillar_error(
+ salt_cli,
+ salt_minion,
+ base_env_pillar_tree_root_dir,
+):
+ """
+ Test state.apply with error in pillar.
+ """
+ pillar_top_file = textwrap.dedent(
+ """
+ base:
+ '{}':
+ - basic
+ """
+ ).format(salt_minion.id)
+ basic_pillar_file = textwrap.dedent(
+ """
+ syntax_error
+ """
+ )
+
+ with pytest.helpers.temp_file(
+ "top.sls", pillar_top_file, base_env_pillar_tree_root_dir
+ ), pytest.helpers.temp_file(
+ "basic.sls", basic_pillar_file, base_env_pillar_tree_root_dir
+ ):
+ expected_comment = [
+ "Pillar failed to render with the following messages:",
+ "SLS 'basic' does not render to a dictionary",
+ ]
+ shell_result = salt_cli.run(
+ "state.apply", "sls-id-test", minion_tgt=salt_minion.id
+ )
+ assert shell_result.exitcode == 1
+ assert shell_result.json == expected_comment
+
+
+@pytest.mark.usefixtures("testfile_path", "reset_pillar")
+def test_state_apply_continues_after_pillar_error_is_fixed(
+ salt_cli,
+ salt_minion,
+ base_env_pillar_tree_root_dir,
+):
+ """
+ Test state.apply with error in pillar.
+ """
+ pillar_top_file = textwrap.dedent(
+ """
+ base:
+ '{}':
+ - basic
+ """.format(salt_minion.id)
+ )
+ basic_pillar_file_error = textwrap.dedent(
+ """
+ syntax_error
+ """
+ )
+ basic_pillar_file = textwrap.dedent(
+ """
+ syntax_error: Fixed!
+ """
+ )
+
+ # save pillar render error in minion's in-memory pillar
+ with pytest.helpers.temp_file(
+ "top.sls", pillar_top_file, base_env_pillar_tree_root_dir
+ ), pytest.helpers.temp_file(
+ "basic.sls", basic_pillar_file_error, base_env_pillar_tree_root_dir
+ ):
+ shell_result = salt_cli.run(
+ "saltutil.refresh_pillar", minion_tgt=salt_minion.id
+ )
+ assert shell_result.exitcode == 0
+
+ # run state.apply with fixed pillar render error
+ with pytest.helpers.temp_file(
+ "top.sls", pillar_top_file, base_env_pillar_tree_root_dir
+ ), pytest.helpers.temp_file(
+ "basic.sls", basic_pillar_file, base_env_pillar_tree_root_dir
+ ):
+ shell_result = salt_cli.run(
+ "state.apply", "sls-id-test", minion_tgt=salt_minion.id
+ )
+ assert shell_result.exitcode == 0
+ state_result = StateResult(shell_result.json)
+ assert state_result.result is True
+ assert state_result.changes == {"diff": "New file", "mode": "0644"}
diff --git a/tests/pytests/unit/modules/state/test_state.py b/tests/pytests/unit/modules/state/test_state.py
index 02fd2dd307..30cda303cc 100644
--- a/tests/pytests/unit/modules/state/test_state.py
+++ b/tests/pytests/unit/modules/state/test_state.py
@@ -1,14 +1,16 @@
"""
:codeauthor: Rahul Handay <rahulha@saltstack.com>
"""
-
import datetime
import logging
import os
+from collections import namedtuple
import pytest
+
import salt.config
import salt.loader
+import salt.loader.context
import salt.modules.config as config
import salt.modules.state as state
import salt.state
@@ -1200,85 +1202,6 @@ def test_lock_saltenv():
)
-def test_get_pillar_errors_CC():
- """
- Test _get_pillar_errors function.
- CC: External clean, Internal clean
- :return:
- """
- for int_pillar, ext_pillar in [
- ({"foo": "bar"}, {"fred": "baz"}),
- ({"foo": "bar"}, None),
- ({}, {"fred": "baz"}),
- ]:
- with patch("salt.modules.state.__pillar__", int_pillar):
- for opts, res in [
- ({"force": True}, None),
- ({"force": False}, None),
- ({}, None),
- ]:
- assert res == state._get_pillar_errors(kwargs=opts, pillar=ext_pillar)
-
-
-def test_get_pillar_errors_EC():
- """
- Test _get_pillar_errors function.
- EC: External erroneous, Internal clean
- :return:
- """
- errors = ["failure", "everywhere"]
- for int_pillar, ext_pillar in [
- ({"foo": "bar"}, {"fred": "baz", "_errors": errors}),
- ({}, {"fred": "baz", "_errors": errors}),
- ]:
- with patch("salt.modules.state.__pillar__", int_pillar):
- for opts, res in [
- ({"force": True}, None),
- ({"force": False}, errors),
- ({}, errors),
- ]:
- assert res == state._get_pillar_errors(kwargs=opts, pillar=ext_pillar)
-
-
-def test_get_pillar_errors_EE():
- """
- Test _get_pillar_errors function.
- CC: External erroneous, Internal erroneous
- :return:
- """
- errors = ["failure", "everywhere"]
- for int_pillar, ext_pillar in [
- ({"foo": "bar", "_errors": errors}, {"fred": "baz", "_errors": errors})
- ]:
- with patch("salt.modules.state.__pillar__", int_pillar):
- for opts, res in [
- ({"force": True}, None),
- ({"force": False}, errors),
- ({}, errors),
- ]:
- assert res == state._get_pillar_errors(kwargs=opts, pillar=ext_pillar)
-
-
-def test_get_pillar_errors_CE():
- """
- Test _get_pillar_errors function.
- CC: External clean, Internal erroneous
- :return:
- """
- errors = ["failure", "everywhere"]
- for int_pillar, ext_pillar in [
- ({"foo": "bar", "_errors": errors}, {"fred": "baz"}),
- ({"foo": "bar", "_errors": errors}, None),
- ]:
- with patch("salt.modules.state.__pillar__", int_pillar):
- for opts, res in [
- ({"force": True}, None),
- ({"force": False}, errors),
- ({}, errors),
- ]:
- assert res == state._get_pillar_errors(kwargs=opts, pillar=ext_pillar)
-
-
def test_event():
"""
test state.event runner
@@ -1318,3 +1241,35 @@ def test_event():
if _expected in x.args[0]:
found = True
assert found is True
+
+
+PillarPair = namedtuple("PillarPair", ["in_memory", "fresh"])
+pillar_combinations = [
+ (PillarPair({"foo": "bar"}, {"fred": "baz"}), None),
+ (PillarPair({"foo": "bar"}, {"fred": "baz", "_errors": ["Failure"]}), ["Failure"]),
+ (PillarPair({"foo": "bar"}, None), None),
+ (PillarPair({"foo": "bar", "_errors": ["Failure"]}, None), ["Failure"]),
+ (PillarPair({"foo": "bar", "_errors": ["Failure"]}, {"fred": "baz"}), None),
+]
+
+
+@pytest.mark.parametrize("pillar,expected_errors", pillar_combinations)
+def test_get_pillar_errors(pillar: PillarPair, expected_errors):
+ """
+ test _get_pillar_errors function
+
+ There are three cases to consider:
+ 1. kwargs['force'] is True -> None, no matter what's in pillar/__pillar__
+ 2. pillar kwarg is available -> only check pillar, no matter what's in __pillar__
+ 3. pillar kwarg is not available -> check __pillar__
+ """
+ ctx = salt.loader.context.LoaderContext()
+ named_ctx = ctx.named_context("__pillar__", pillar.in_memory)
+ with patch("salt.modules.state.__pillar__", named_ctx, create=True):
+ assert (
+ state._get_pillar_errors(kwargs={"force": True}, pillar=pillar.fresh)
+ is None
+ )
+ assert (
+ state._get_pillar_errors(kwargs={}, pillar=pillar.fresh) == expected_errors
+ )
--
2.37.3

105
state.orchestrate_single-does-not-pass-pillar-none-4.patch
View File

@ -1,105 +0,0 @@
From 634e82874b17c38bd4d27c0c07a53c9e39e49968 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?=
<psuarezhernandez@suse.com>
Date: Wed, 9 Feb 2022 09:01:08 +0000
Subject: [PATCH] state.orchestrate_single does not pass pillar=None
(#488)
Passing a pillar to state.single can results in state functions not
working when they don't accept a pillar keyword argument. One example
where this is the case is salt.wait_for_event. When no pillar is
provided, it does not need to be passed.
Co-authored-by: Alexander Graul <agraul@suse.com>
---
changelog/61092.fixed | 3 ++
salt/runners/state.py | 10 ++++--
tests/pytests/unit/runners/test_state.py | 41 ++++++++++++++++++++++++
3 files changed, 51 insertions(+), 3 deletions(-)
create mode 100644 changelog/61092.fixed
create mode 100644 tests/pytests/unit/runners/test_state.py
diff --git a/changelog/61092.fixed b/changelog/61092.fixed
new file mode 100644
index 0000000000..6ca66839c9
--- /dev/null
+++ b/changelog/61092.fixed
@@ -0,0 +1,3 @@
+state.orchestrate_single only passes a pillar if it is set to the state
+function. This allows it to be used with state functions that don't accept a
+pillar keyword argument.
diff --git a/salt/runners/state.py b/salt/runners/state.py
index f8fc1b0944..5642204ce9 100644
--- a/salt/runners/state.py
+++ b/salt/runners/state.py
@@ -150,12 +150,16 @@ def orchestrate_single(fun, name, test=None, queue=False, pillar=None, **kwargs)
salt-run state.orchestrate_single fun=salt.wheel name=key.list_all
"""
- if pillar is not None and not isinstance(pillar, dict):
- raise SaltInvocationError("Pillar data must be formatted as a dictionary")
+ if pillar is not None:
+ if isinstance(pillar, dict):
+ kwargs["pillar"] = pillar
+ else:
+ raise SaltInvocationError("Pillar data must be formatted as a dictionary")
+
__opts__["file_client"] = "local"
minion = salt.minion.MasterMinion(__opts__)
running = minion.functions["state.single"](
- fun, name, test=None, queue=False, pillar=pillar, **kwargs
+ fun, name, test=None, queue=False, **kwargs
)
ret = {minion.opts["id"]: running}
__jid_event__.fire_event({"data": ret, "outputter": "highstate"}, "progress")
diff --git a/tests/pytests/unit/runners/test_state.py b/tests/pytests/unit/runners/test_state.py
new file mode 100644
index 0000000000..df0a718a41
--- /dev/null
+++ b/tests/pytests/unit/runners/test_state.py
@@ -0,0 +1,41 @@
+#!/usr/bin/python3
+
+import pytest
+from salt.runners import state as state_runner
+from tests.support.mock import Mock, patch
+
+
+@pytest.fixture
+def configure_loader_modules():
+ return {state_runner: {"__opts__": {}, "__jid_event__": Mock()}}
+
+
+def test_orchestrate_single_passes_pillar():
+ """
+ test state.orchestrate_single passes given pillar to state.single
+ """
+ mock_master_minion = Mock()
+ mock_state_single = Mock()
+ mock_master_minion.functions = {"state.single": mock_state_single}
+ mock_master_minion.opts = {"id": "dummy"}
+ test_pillar = {"test_entry": "exists"}
+ with patch("salt.minion.MasterMinion", Mock(return_value=mock_master_minion)):
+ state_runner.orchestrate_single(
+ fun="pillar.get", name="test_entry", pillar=test_pillar
+ )
+ assert mock_state_single.call_args.kwargs["pillar"] == test_pillar
+
+
+def test_orchestrate_single_does_not_pass_none_pillar():
+ """
+ test state.orchestrate_single does not pass pillar=None to state.single
+ """
+ mock_master_minion = Mock()
+ mock_state_single = Mock()
+ mock_master_minion.functions = {"state.single": mock_state_single}
+ mock_master_minion.opts = {"id": "dummy"}
+ with patch("salt.minion.MasterMinion", Mock(return_value=mock_master_minion)):
+ state_runner.orchestrate_single(
+ fun="pillar.get", name="test_entry", pillar=None
+ )
+ assert "pillar" not in mock_state_single.call_args.kwargs
--
2.37.3