From 323feae962c8bf2418ae7838db8472b8f85cc7cc8831d9d2ece77e3761d7085f Mon Sep 17 00:00:00 2001 From: Alexander Graul Date: Thu, 18 Jan 2024 13:41:50 +0000 Subject: [PATCH] Accepting request 1139717 from home:mczernek:branches:systemsmanagement:saltstack - Ensure that pillar refresh loads beacons from pillar without restart - Fix the aptpkg.py unit test failure - Prefer unittest.mock to python-mock in test suite - Added: * update-__pillar__-during-pillar_refresh.patch * fix-the-aptpkg.py-unit-test-failure.patch * prefer-unittest.mock-for-python-versions-that-are-su.patch OBS-URL: https://build.opensuse.org/request/show/1139717 OBS-URL: https://build.opensuse.org/package/show/systemsmanagement:saltstack/salt?expand=0&rev=225 --- _lastrevision | 2 +- fix-the-aptpkg.py-unit-test-failure.patch | 25 +++ ...mock-for-python-versions-that-are-su.patch | 135 ++++++++++++++ salt.changes | 12 ++ salt.spec | 7 + update-__pillar__-during-pillar_refresh.patch | 169 ++++++++++++++++++ 6 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 fix-the-aptpkg.py-unit-test-failure.patch create mode 100644 prefer-unittest.mock-for-python-versions-that-are-su.patch create mode 100644 update-__pillar__-during-pillar_refresh.patch diff --git a/_lastrevision b/_lastrevision index 69861cc..e901147 100644 --- a/_lastrevision +++ b/_lastrevision @@ -1 +1 @@ -ecaf86de2f64a2032b60880651dde534021c8954 \ No newline at end of file +ee71f172008f6c3aa9d80a26ee15f873ca17f471 \ No newline at end of file diff --git a/fix-the-aptpkg.py-unit-test-failure.patch b/fix-the-aptpkg.py-unit-test-failure.patch new file mode 100644 index 0000000..3d7c363 --- /dev/null +++ b/fix-the-aptpkg.py-unit-test-failure.patch @@ -0,0 +1,25 @@ +From 4bc3be7814daf5365d63b88f164f791ea53b418f Mon Sep 17 00:00:00 2001 +From: Marek Czernek +Date: Wed, 17 Jan 2024 15:04:53 +0100 +Subject: [PATCH] Fix the aptpkg.py unit test failure + +--- + salt/modules/aptpkg.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py +index 9885e9fb60..ad5450c415 100644 +--- a/salt/modules/aptpkg.py ++++ b/salt/modules/aptpkg.py +@@ -3128,7 +3128,7 @@ def expand_repo_def(**kwargs): + NOT USABLE IN THE CLI + """ + warn_until_date( +- "20240101", ++ "20250101", + "The pkg.expand_repo_def function is deprecated and set for removal " + "after {date}. This is only unsed internally by the apt pkg state " + "module. If that's not the case, please file an new issue requesting " +-- +2.43.0 + diff --git a/prefer-unittest.mock-for-python-versions-that-are-su.patch b/prefer-unittest.mock-for-python-versions-that-are-su.patch new file mode 100644 index 0000000..8b2bc39 --- /dev/null +++ b/prefer-unittest.mock-for-python-versions-that-are-su.patch @@ -0,0 +1,135 @@ +From 107de57586f0b0f784771543b942dfb6bb70453a Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Yeray=20Guti=C3=A9rrez=20Cedr=C3=A9s?= + +Date: Wed, 13 Dec 2023 11:03:45 +0000 +Subject: [PATCH] Prefer unittest.mock for Python versions that are + sufficient + +--- + requirements/pytest.txt | 2 +- + .../unit/cloud/clouds/test_dimensiondata.py | 4 +- + tests/pytests/unit/cloud/clouds/test_gce.py | 4 +- + tests/support/mock.py | 48 +++++++++---------- + 4 files changed, 25 insertions(+), 33 deletions(-) + +diff --git a/requirements/pytest.txt b/requirements/pytest.txt +index 5b67583a3d..0bead83f5b 100644 +--- a/requirements/pytest.txt ++++ b/requirements/pytest.txt +@@ -1,4 +1,4 @@ +-mock >= 3.0.0 ++mock >= 3.0.0; python_version < '3.8' + # PyTest + pytest >= 7.0.1; python_version <= "3.6" + pytest >= 7.2.0; python_version > "3.6" +diff --git a/tests/pytests/unit/cloud/clouds/test_dimensiondata.py b/tests/pytests/unit/cloud/clouds/test_dimensiondata.py +index e196805004..aab2e686f2 100644 +--- a/tests/pytests/unit/cloud/clouds/test_dimensiondata.py ++++ b/tests/pytests/unit/cloud/clouds/test_dimensiondata.py +@@ -11,7 +11,6 @@ from salt.cloud.clouds import dimensiondata + from salt.exceptions import SaltCloudSystemExit + from salt.utils.versions import Version + from tests.support.mock import MagicMock +-from tests.support.mock import __version__ as mock_version + from tests.support.mock import patch + + try: +@@ -144,8 +143,7 @@ def test_import(): + with patch("salt.config.check_driver_dependencies", return_value=True) as p: + get_deps = dimensiondata.get_dependencies() + assert get_deps is True +- if Version(mock_version) >= Version("2.0.0"): +- assert p.call_count >= 1 ++ assert p.call_count >= 1 + + + def test_provider_matches(): +diff --git a/tests/pytests/unit/cloud/clouds/test_gce.py b/tests/pytests/unit/cloud/clouds/test_gce.py +index 265818016e..ec1346a978 100644 +--- a/tests/pytests/unit/cloud/clouds/test_gce.py ++++ b/tests/pytests/unit/cloud/clouds/test_gce.py +@@ -13,7 +13,6 @@ from salt.cloud.clouds import gce + from salt.exceptions import SaltCloudSystemExit + from salt.utils.versions import Version + from tests.support.mock import MagicMock +-from tests.support.mock import __version__ as mock_version + from tests.support.mock import call, patch + + +@@ -281,8 +280,7 @@ def test_import(): + with patch("salt.config.check_driver_dependencies", return_value=True) as p: + get_deps = gce.get_dependencies() + assert get_deps is True +- if Version(mock_version) >= Version("2.0.0"): +- p.assert_called_once() ++ p.assert_called_once() + + + @pytest.mark.parametrize( +diff --git a/tests/support/mock.py b/tests/support/mock.py +index 2256ad8f5d..59e5fcbc8e 100644 +--- a/tests/support/mock.py ++++ b/tests/support/mock.py +@@ -18,37 +18,33 @@ import copy + import errno + import fnmatch + import sys +- +-# By these days, we should blowup if mock is not available +-import mock # pylint: disable=blacklisted-external-import +- +-# pylint: disable=no-name-in-module,no-member +-from mock import ( +- ANY, +- DEFAULT, +- FILTER_DIR, +- MagicMock, +- Mock, +- NonCallableMagicMock, +- NonCallableMock, +- PropertyMock, +- __version__, +- call, +- create_autospec, +- patch, +- sentinel, +-) ++import importlib ++ ++current_version = (sys.version_info.major, sys.version_info.minor) ++ ++# Prefer unittest.mock for Python versions that are sufficient ++if current_version >= (3,8): ++ mock = importlib.import_module('unittest.mock') ++else: ++ mock = importlib.import_module('mock') ++ ++ANY = mock.ANY ++DEFAULT = mock.DEFAULT ++FILTER_DIR = mock.FILTER_DIR ++MagicMock = mock.MagicMock ++Mock = mock.Mock ++NonCallableMagicMock = mock.NonCallableMagicMock ++NonCallableMock = mock.NonCallableMock ++PropertyMock = mock.PropertyMock ++call = mock.call ++create_autospec = mock.create_autospec ++patch = mock.patch ++sentinel = mock.sentinel + + import salt.utils.stringutils + + # pylint: disable=no-name-in-module,no-member + +- +-__mock_version = tuple( +- int(part) for part in mock.__version__.split(".") if part.isdigit() +-) # pylint: disable=no-member +- +- + class MockFH: + def __init__(self, filename, read_data, *args, **kwargs): + self.filename = filename +-- +2.41.0 + diff --git a/salt.changes b/salt.changes index f2c10a1..4c88728 100644 --- a/salt.changes +++ b/salt.changes @@ -1,3 +1,15 @@ +------------------------------------------------------------------- +Thu Jan 18 13:06:21 UTC 2024 - Marek Czernek + +- Ensure that pillar refresh loads beacons from pillar without restart +- Fix the aptpkg.py unit test failure +- Prefer unittest.mock to python-mock in test suite + +- Added: + * update-__pillar__-during-pillar_refresh.patch + * fix-the-aptpkg.py-unit-test-failure.patch + * prefer-unittest.mock-for-python-versions-that-are-su.patch + ------------------------------------------------------------------- Fri Dec 1 11:04:02 UTC 2023 - Pablo Suárez Hernández diff --git a/salt.spec b/salt.spec index c4dea34..7c0c2b4 100644 --- a/salt.spec +++ b/salt.spec @@ -329,6 +329,13 @@ Patch90: dereference-symlinks-to-set-proper-__cli-opt-bsc-121.patch Patch91: revert-make-sure-configured-user-is-properly-set-by-.patch # PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65488 Patch92: enable-keepalive-probes-for-salt-ssh-executions-bsc-.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65644 +Patch93: prefer-unittest.mock-for-python-versions-that-are-su.patch +# PATCH-FIX_OPENSUSE: https://github.com/openSUSE/salt/pull/620 +Patch94: fix-the-aptpkg.py-unit-test-failure.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65092 +Patch95: update-__pillar__-during-pillar_refresh.patch + ### IMPORTANT: The line below is used as a snippet marker. Do not touch it. ### SALT PATCHES LIST END diff --git a/update-__pillar__-during-pillar_refresh.patch b/update-__pillar__-during-pillar_refresh.patch new file mode 100644 index 0000000..88c09eb --- /dev/null +++ b/update-__pillar__-during-pillar_refresh.patch @@ -0,0 +1,169 @@ +From 3e7c5d95423491f83d0016eb7c02285cd0b1bcf4 Mon Sep 17 00:00:00 2001 +From: Marek Czernek +Date: Wed, 17 Jan 2024 15:39:41 +0100 +Subject: [PATCH] Update __pillar__ during pillar_refresh + +--- + changelog/63583.fixed.md | 1 + + salt/minion.py | 1 + + .../integration/modules/test_pillar.py | 110 +++++++++++++++++- + 3 files changed, 111 insertions(+), 1 deletion(-) + create mode 100644 changelog/63583.fixed.md + +diff --git a/changelog/63583.fixed.md b/changelog/63583.fixed.md +new file mode 100644 +index 0000000000..f1b6e32507 +--- /dev/null ++++ b/changelog/63583.fixed.md +@@ -0,0 +1 @@ ++Need to make sure we update __pillar__ during a pillar refresh to ensure that process_beacons has the updated beacons loaded from pillar. +diff --git a/salt/minion.py b/salt/minion.py +index 9597d6e63a..4db0d31bd4 100644 +--- a/salt/minion.py ++++ b/salt/minion.py +@@ -2498,6 +2498,7 @@ class Minion(MinionBase): + current_schedule, new_schedule + ) + self.opts["pillar"] = new_pillar ++ self.functions.pack["__pillar__"] = self.opts["pillar"] + finally: + async_pillar.destroy() + self.matchers_refresh() +diff --git a/tests/pytests/integration/modules/test_pillar.py b/tests/pytests/integration/modules/test_pillar.py +index 66f7b9e47b..5db9a1630a 100644 +--- a/tests/pytests/integration/modules/test_pillar.py ++++ b/tests/pytests/integration/modules/test_pillar.py +@@ -1,9 +1,14 @@ ++import logging + import pathlib + import time ++import types + + import attr + import pytest + ++log = logging.getLogger(__name__) ++ ++ + pytestmark = [ + pytest.mark.slow_test, + pytest.mark.windows_whitelisted, +@@ -210,7 +215,7 @@ class PillarRefresh: + "top.sls", top_file_contents + ) + self.minion_1_pillar = self.master.pillar_tree.base.temp_file( +- "minion-1-pillar.sls", "{}: true".format(self.pillar_key) ++ "minion-1-pillar.sls", f"{self.pillar_key}: true" + ) + self.top_file.__enter__() + self.minion_1_pillar.__enter__() +@@ -588,3 +593,106 @@ def test_pillar_ext_59975(salt_call_cli): + """ + ret = salt_call_cli.run("pillar.ext", '{"libvert": _}') + assert "ext_pillar_opts" in ret.data ++ ++ ++@pytest.fixture ++def event_listerner_timeout(grains): ++ if grains["os"] == "Windows": ++ if grains["osrelease"].startswith("2019"): ++ return types.SimpleNamespace(catch=120, miss=30) ++ return types.SimpleNamespace(catch=90, miss=10) ++ return types.SimpleNamespace(catch=60, miss=10) ++ ++ ++@pytest.mark.slow_test ++def test_pillar_refresh_pillar_beacons( ++ base_env_pillar_tree_root_dir, ++ salt_cli, ++ salt_minion, ++ salt_master, ++ event_listener, ++ event_listerner_timeout, ++): ++ """ ++ Ensure beacons jobs in pillar are started after ++ a pillar refresh and then not running when pillar ++ is cleared. ++ """ ++ ++ top_sls = """ ++ base: ++ '{}': ++ - test_beacons ++ """.format( ++ salt_minion.id ++ ) ++ ++ test_beacons_sls_empty = "" ++ ++ test_beacons_sls = """ ++ beacons: ++ status: ++ - loadavg: ++ - 1-min ++ """ ++ ++ assert salt_minion.is_running() ++ ++ top_tempfile = pytest.helpers.temp_file( ++ "top.sls", top_sls, base_env_pillar_tree_root_dir ++ ) ++ beacon_tempfile = pytest.helpers.temp_file( ++ "test_beacons.sls", test_beacons_sls_empty, base_env_pillar_tree_root_dir ++ ) ++ ++ with top_tempfile, beacon_tempfile: ++ # Calling refresh_pillar to update in-memory pillars ++ salt_cli.run("saltutil.refresh_pillar", wait=True, minion_tgt=salt_minion.id) ++ ++ # Ensure beacons start when pillar is refreshed ++ with salt_master.pillar_tree.base.temp_file( ++ "test_beacons.sls", test_beacons_sls ++ ): ++ # Calling refresh_pillar to update in-memory pillars ++ salt_cli.run( ++ "saltutil.refresh_pillar", wait=True, minion_tgt=salt_minion.id ++ ) ++ ++ # Give the beacons a chance to start ++ time.sleep(5) ++ ++ event_tag = f"salt/beacon/*/status/*" ++ start_time = time.time() ++ ++ event_pattern = (salt_master.id, event_tag) ++ matched_events = event_listener.wait_for_events( ++ [event_pattern], ++ after_time=start_time, ++ timeout=event_listerner_timeout.catch, ++ ) ++ ++ assert matched_events.found_all_events ++ ++ # Ensure beacons sttop when pillar is refreshed ++ with salt_master.pillar_tree.base.temp_file( ++ "test_beacons.sls", test_beacons_sls_empty ++ ): ++ # Calling refresh_pillar to update in-memory pillars ++ salt_cli.run( ++ "saltutil.refresh_pillar", wait=True, minion_tgt=salt_minion.id ++ ) ++ ++ # Give the beacons a chance to stop ++ time.sleep(5) ++ ++ event_tag = f"salt/beacon/*/status/*" ++ start_time = time.time() ++ ++ event_pattern = (salt_master.id, event_tag) ++ matched_events = event_listener.wait_for_events( ++ [event_pattern], ++ after_time=start_time, ++ timeout=event_listerner_timeout.miss, ++ ) ++ ++ assert not matched_events.found_all_events +-- +2.43.0 +