diff --git a/_lastrevision b/_lastrevision index e5f69c4..ad80ccc 100644 --- a/_lastrevision +++ b/_lastrevision @@ -1 +1 @@ -3becea2e5b00beff724c22a8ae320d4567031c7b \ No newline at end of file +d0c2f35ff4a0b21786b20c884cbb191ad2e63904 \ No newline at end of file diff --git a/allow-all-primitive-grain-types-for-autosign_grains-.patch b/allow-all-primitive-grain-types-for-autosign_grains-.patch new file mode 100644 index 0000000..f0af991 --- /dev/null +++ b/allow-all-primitive-grain-types-for-autosign_grains-.patch @@ -0,0 +1,97 @@ +From ae4e1d1cc15b3c510bdd774a1dfeff67c522324a Mon Sep 17 00:00:00 2001 +From: Marek Czernek +Date: Tue, 17 Oct 2023 13:05:00 +0200 +Subject: [PATCH] Allow all primitive grain types for autosign_grains + (#607) + +* Allow all primitive grain types for autosign_grains + +Signed-off-by: Marek Czernek + +* blacken daemons/masterapi.py and its test_auto_key + +Signed-off-by: Marek Czernek + +--------- + +Signed-off-by: Marek Czernek +Co-authored-by: Alexander Graul +--- + changelog/61416.fixed.md | 1 + + changelog/63708.fixed.md | 1 + + salt/daemons/masterapi.py | 2 +- + .../pytests/unit/daemons/masterapi/test_auto_key.py | 13 +++++++------ + 4 files changed, 10 insertions(+), 7 deletions(-) + create mode 100644 changelog/61416.fixed.md + create mode 100644 changelog/63708.fixed.md + +diff --git a/changelog/61416.fixed.md b/changelog/61416.fixed.md +new file mode 100644 +index 0000000000..3203a0a1c6 +--- /dev/null ++++ b/changelog/61416.fixed.md +@@ -0,0 +1 @@ ++Allow all primitive grain types for autosign_grains +diff --git a/changelog/63708.fixed.md b/changelog/63708.fixed.md +new file mode 100644 +index 0000000000..3203a0a1c6 +--- /dev/null ++++ b/changelog/63708.fixed.md +@@ -0,0 +1 @@ ++Allow all primitive grain types for autosign_grains +diff --git a/salt/daemons/masterapi.py b/salt/daemons/masterapi.py +index 3716c63d99..54aca64a76 100644 +--- a/salt/daemons/masterapi.py ++++ b/salt/daemons/masterapi.py +@@ -366,7 +366,7 @@ class AutoKey: + line = salt.utils.stringutils.to_unicode(line).strip() + if line.startswith("#"): + continue +- if autosign_grains[grain] == line: ++ if str(autosign_grains[grain]) == line: + return True + return False + +diff --git a/tests/pytests/unit/daemons/masterapi/test_auto_key.py b/tests/pytests/unit/daemons/masterapi/test_auto_key.py +index b3657b7f1b..54c3f22d2a 100644 +--- a/tests/pytests/unit/daemons/masterapi/test_auto_key.py ++++ b/tests/pytests/unit/daemons/masterapi/test_auto_key.py +@@ -17,11 +17,11 @@ def gen_permissions(owner="", group="", others=""): + """ + ret = 0 + for c in owner: +- ret |= getattr(stat, "S_I{}USR".format(c.upper()), 0) ++ ret |= getattr(stat, f"S_I{c.upper()}USR", 0) + for c in group: +- ret |= getattr(stat, "S_I{}GRP".format(c.upper()), 0) ++ ret |= getattr(stat, f"S_I{c.upper()}GRP", 0) + for c in others: +- ret |= getattr(stat, "S_I{}OTH".format(c.upper()), 0) ++ ret |= getattr(stat, f"S_I{c.upper()}OTH", 0) + return ret + + +@@ -256,16 +256,17 @@ def test_check_autosign_grains_no_autosign_grains_dir(auto_key): + _test_check_autosign_grains(test_func, auto_key, autosign_grains_dir=None) + + +-def test_check_autosign_grains_accept(auto_key): ++@pytest.mark.parametrize("grain_value", ["test_value", 123, True]) ++def test_check_autosign_grains_accept(grain_value, auto_key): + """ + Asserts that autosigning from grains passes when a matching grain value is in an + autosign_grain file. + """ + + def test_func(*args): +- assert auto_key.check_autosign_grains({"test_grain": "test_value"}) is True ++ assert auto_key.check_autosign_grains({"test_grain": grain_value}) is True + +- file_content = "#test_ignore\ntest_value" ++ file_content = f"#test_ignore\n{grain_value}" + _test_check_autosign_grains(test_func, auto_key, file_content=file_content) + + +-- +2.42.0 + diff --git a/allow-kwargs-for-fileserver-roots-update-bsc-1218482.patch b/allow-kwargs-for-fileserver-roots-update-bsc-1218482.patch new file mode 100644 index 0000000..f57584b --- /dev/null +++ b/allow-kwargs-for-fileserver-roots-update-bsc-1218482.patch @@ -0,0 +1,164 @@ +From 8ae54e8a0e12193507f1936f363c3438b4a006ee Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Yeray=20Guti=C3=A9rrez=20Cedr=C3=A9s?= + +Date: Tue, 23 Jan 2024 15:33:28 +0000 +Subject: [PATCH] Allow kwargs for fileserver roots update + (bsc#1218482) (#618) +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +* Allow kwargs for fileserver roots update (bsc#1218482) + +* Prevent exceptions with fileserver.update when called via state + +* Fix wrong logic and enhance tests around fileserver.update + +* Remove test which is not longer valid + +--------- + +Co-authored-by: Pablo Suárez Hernández +--- + changelog/65819.fixed.md | 1 + + salt/fileserver/roots.py | 8 ++-- + salt/runners/fileserver.py | 6 +++ + tests/integration/runners/test_fileserver.py | 40 ++++++++++++++++++-- + tests/pytests/unit/fileserver/test_roots.py | 2 +- + 5 files changed, 47 insertions(+), 10 deletions(-) + create mode 100644 changelog/65819.fixed.md + +diff --git a/changelog/65819.fixed.md b/changelog/65819.fixed.md +new file mode 100644 +index 0000000000..432f5c791c +--- /dev/null ++++ b/changelog/65819.fixed.md +@@ -0,0 +1 @@ ++Prevent exceptions with fileserver.update when called via state +diff --git a/salt/fileserver/roots.py b/salt/fileserver/roots.py +index 4880cbab9b..a02b597c6f 100644 +--- a/salt/fileserver/roots.py ++++ b/salt/fileserver/roots.py +@@ -193,9 +193,7 @@ def update(): + os.makedirs(mtime_map_path_dir) + with salt.utils.files.fopen(mtime_map_path, "wb") as fp_: + for file_path, mtime in new_mtime_map.items(): +- fp_.write( +- salt.utils.stringutils.to_bytes("{}:{}\n".format(file_path, mtime)) +- ) ++ fp_.write(salt.utils.stringutils.to_bytes(f"{file_path}:{mtime}\n")) + + if __opts__.get("fileserver_events", False): + # if there is a change, fire an event +@@ -326,11 +324,11 @@ def _file_lists(load, form): + return [] + list_cache = os.path.join( + list_cachedir, +- "{}.p".format(salt.utils.files.safe_filename_leaf(actual_saltenv)), ++ f"{salt.utils.files.safe_filename_leaf(actual_saltenv)}.p", + ) + w_lock = os.path.join( + list_cachedir, +- ".{}.w".format(salt.utils.files.safe_filename_leaf(actual_saltenv)), ++ f".{salt.utils.files.safe_filename_leaf(actual_saltenv)}.w", + ) + cache_match, refresh_cache, save_cache = salt.fileserver.check_file_list_cache( + __opts__, form, list_cache, w_lock +diff --git a/salt/runners/fileserver.py b/salt/runners/fileserver.py +index d75d7de0cf..1ed05b68ca 100644 +--- a/salt/runners/fileserver.py ++++ b/salt/runners/fileserver.py +@@ -350,6 +350,12 @@ def update(backend=None, **kwargs): + salt-run fileserver.update backend=git remotes=myrepo,yourrepo + """ + fileserver = salt.fileserver.Fileserver(__opts__) ++ ++ # Remove possible '__pub_user' in kwargs as it is not expected ++ # on "update" function for the different fileserver backends. ++ if "__pub_user" in kwargs: ++ del kwargs["__pub_user"] ++ + fileserver.update(back=backend, **kwargs) + return True + +diff --git a/tests/integration/runners/test_fileserver.py b/tests/integration/runners/test_fileserver.py +index ae8ab766aa..62f0da0c4a 100644 +--- a/tests/integration/runners/test_fileserver.py ++++ b/tests/integration/runners/test_fileserver.py +@@ -202,15 +202,31 @@ class FileserverTest(ShellCase): + fileserver.update + """ + ret = self.run_run_plus(fun="fileserver.update") +- self.assertTrue(ret["return"]) ++ self.assertTrue(ret["return"] is True) + + # Backend submitted as a string + ret = self.run_run_plus(fun="fileserver.update", backend="roots") +- self.assertTrue(ret["return"]) ++ self.assertTrue(ret["return"] is True) + + # Backend submitted as a list + ret = self.run_run_plus(fun="fileserver.update", backend=["roots"]) +- self.assertTrue(ret["return"]) ++ self.assertTrue(ret["return"] is True) ++ ++ # Possible '__pub_user' is removed from kwargs ++ ret = self.run_run_plus( ++ fun="fileserver.update", backend=["roots"], __pub_user="foo" ++ ) ++ self.assertTrue(ret["return"] is True) ++ ++ # Unknown arguments ++ ret = self.run_run_plus( ++ fun="fileserver.update", backend=["roots"], unknown_arg="foo" ++ ) ++ self.assertIn( ++ "Passed invalid arguments: update() got an unexpected keyword argument" ++ " 'unknown_arg'", ++ ret["return"], ++ ) + + # Other arguments are passed to backend + def mock_gitfs_update(remotes=None): +@@ -225,7 +241,23 @@ class FileserverTest(ShellCase): + ret = self.run_run_plus( + fun="fileserver.update", backend="gitfs", remotes="myrepo,yourrepo" + ) +- self.assertTrue(ret["return"]) ++ self.assertTrue(ret["return"] is True) ++ mock_backend_func.assert_called_once_with(remotes="myrepo,yourrepo") ++ ++ # Possible '__pub_user' arguments are removed from kwargs ++ mock_backend_func = create_autospec(mock_gitfs_update) ++ mock_return_value = { ++ "gitfs.envs": None, # This is needed to activate the backend ++ "gitfs.update": mock_backend_func, ++ } ++ with patch("salt.loader.fileserver", MagicMock(return_value=mock_return_value)): ++ ret = self.run_run_plus( ++ fun="fileserver.update", ++ backend="gitfs", ++ remotes="myrepo,yourrepo", ++ __pub_user="foo", ++ ) ++ self.assertTrue(ret["return"] is True) + mock_backend_func.assert_called_once_with(remotes="myrepo,yourrepo") + + # Unknown arguments are passed to backend +diff --git a/tests/pytests/unit/fileserver/test_roots.py b/tests/pytests/unit/fileserver/test_roots.py +index a8a80eea17..96bceb0fd3 100644 +--- a/tests/pytests/unit/fileserver/test_roots.py ++++ b/tests/pytests/unit/fileserver/test_roots.py +@@ -236,7 +236,7 @@ def test_update_mtime_map(): + # between Python releases. + lines_written = sorted(mtime_map_mock.write_calls()) + expected = sorted( +- salt.utils.stringutils.to_bytes("{key}:{val}\n".format(key=key, val=val)) ++ salt.utils.stringutils.to_bytes(f"{key}:{val}\n") + for key, val in new_mtime_map.items() + ) + assert lines_written == expected, lines_written +-- +2.43.0 + + diff --git a/dereference-symlinks-to-set-proper-__cli-opt-bsc-121.patch b/dereference-symlinks-to-set-proper-__cli-opt-bsc-121.patch new file mode 100644 index 0000000..aebc4ed --- /dev/null +++ b/dereference-symlinks-to-set-proper-__cli-opt-bsc-121.patch @@ -0,0 +1,101 @@ +From 9942c488b1e74f2c6f187fcef3556fe53382bb4c Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Mon, 13 Nov 2023 15:04:14 +0000 +Subject: [PATCH] Dereference symlinks to set proper __cli opt + (bsc#1215963) (#611) + +* Dereference symlinks to set proper __cli + +* Add changelog entry + +* Add unit tests to check path is expanded + +--------- + +Co-authored-by: vzhestkov +--- + changelog/65435.fixed.md | 1 + + salt/config/__init__.py | 8 ++++++-- + tests/pytests/unit/config/test_master_config.py | 13 +++++++++++++ + tests/pytests/unit/config/test_minion_config.py | 13 +++++++++++++ + 4 files changed, 33 insertions(+), 2 deletions(-) + create mode 100644 changelog/65435.fixed.md + create mode 100644 tests/pytests/unit/config/test_master_config.py + create mode 100644 tests/pytests/unit/config/test_minion_config.py + +diff --git a/changelog/65435.fixed.md b/changelog/65435.fixed.md +new file mode 100644 +index 0000000000..5fa532891d +--- /dev/null ++++ b/changelog/65435.fixed.md +@@ -0,0 +1 @@ ++Dereference symlinks to set proper __cli opt +diff --git a/salt/config/__init__.py b/salt/config/__init__.py +index 43182f3f92..d8258a4dbc 100644 +--- a/salt/config/__init__.py ++++ b/salt/config/__init__.py +@@ -3747,7 +3747,9 @@ def apply_minion_config( + ) + opts["fileserver_backend"][idx] = new_val + +- opts["__cli"] = salt.utils.stringutils.to_unicode(os.path.basename(sys.argv[0])) ++ opts["__cli"] = salt.utils.stringutils.to_unicode( ++ os.path.basename(salt.utils.path.expand(sys.argv[0])) ++ ) + + # No ID provided. Will getfqdn save us? + using_ip_for_id = False +@@ -3949,7 +3951,9 @@ def apply_master_config(overrides=None, defaults=None): + ) + opts["keep_acl_in_token"] = True + +- opts["__cli"] = salt.utils.stringutils.to_unicode(os.path.basename(sys.argv[0])) ++ opts["__cli"] = salt.utils.stringutils.to_unicode( ++ os.path.basename(salt.utils.path.expand(sys.argv[0])) ++ ) + + if "environment" in opts: + if opts["saltenv"] is not None: +diff --git a/tests/pytests/unit/config/test_master_config.py b/tests/pytests/unit/config/test_master_config.py +new file mode 100644 +index 0000000000..c9de8a7892 +--- /dev/null ++++ b/tests/pytests/unit/config/test_master_config.py +@@ -0,0 +1,13 @@ ++import salt.config ++from tests.support.mock import MagicMock, patch ++ ++ ++def test___cli_path_is_expanded(): ++ defaults = salt.config.DEFAULT_MASTER_OPTS.copy() ++ overrides = {} ++ with patch( ++ "salt.utils.path.expand", MagicMock(return_value="/path/to/testcli") ++ ) as expand_mock: ++ opts = salt.config.apply_master_config(overrides, defaults) ++ assert expand_mock.called ++ assert opts["__cli"] == "testcli" +diff --git a/tests/pytests/unit/config/test_minion_config.py b/tests/pytests/unit/config/test_minion_config.py +new file mode 100644 +index 0000000000..34aa84daa7 +--- /dev/null ++++ b/tests/pytests/unit/config/test_minion_config.py +@@ -0,0 +1,13 @@ ++import salt.config ++from tests.support.mock import MagicMock, patch ++ ++ ++def test___cli_path_is_expanded(): ++ defaults = salt.config.DEFAULT_MINION_OPTS.copy() ++ overrides = {} ++ with patch( ++ "salt.utils.path.expand", MagicMock(return_value="/path/to/testcli") ++ ) as expand_mock: ++ opts = salt.config.apply_minion_config(overrides, defaults) ++ assert expand_mock.called ++ assert opts["__cli"] == "testcli" +-- +2.42.0 + + diff --git a/enable-keepalive-probes-for-salt-ssh-executions-bsc-.patch b/enable-keepalive-probes-for-salt-ssh-executions-bsc-.patch new file mode 100644 index 0000000..9e41c16 --- /dev/null +++ b/enable-keepalive-probes-for-salt-ssh-executions-bsc-.patch @@ -0,0 +1,346 @@ +From 5303cc612bcbdb1ec45ede397ca1e2ca12ba3bd3 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Fri, 1 Dec 2023 10:59:30 +0000 +Subject: [PATCH] Enable "KeepAlive" probes for Salt SSH executions + (bsc#1211649) (#610) + +* Enable KeepAlive probes for Salt SSH connections (bsc#1211649) + +* Add tests for Salt SSH keepalive options + +* Add changelog file + +* Make changes suggested by pre-commit +--- + changelog/65488.added.md | 1 + + salt/client/ssh/__init__.py | 32 +++++++++--- + salt/client/ssh/client.py | 13 ++++- + salt/client/ssh/shell.py | 12 +++++ + salt/config/__init__.py | 6 +++ + salt/utils/parsers.py | 19 +++++++ + tests/pytests/unit/client/ssh/test_single.py | 55 ++++++++++++++++++++ + tests/pytests/unit/client/ssh/test_ssh.py | 3 ++ + 8 files changed, 133 insertions(+), 8 deletions(-) + create mode 100644 changelog/65488.added.md + +diff --git a/changelog/65488.added.md b/changelog/65488.added.md +new file mode 100644 +index 0000000000..78476cec11 +--- /dev/null ++++ b/changelog/65488.added.md +@@ -0,0 +1 @@ ++Enable "KeepAlive" probes for Salt SSH executions +diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py +index 1e143f9e30..1d8426b7c2 100644 +--- a/salt/client/ssh/__init__.py ++++ b/salt/client/ssh/__init__.py +@@ -50,8 +50,8 @@ import salt.utils.thin + import salt.utils.url + import salt.utils.verify + from salt._logging import LOG_LEVELS +-from salt._logging.mixins import MultiprocessingStateMixin + from salt._logging.impl import LOG_LOCK ++from salt._logging.mixins import MultiprocessingStateMixin + from salt.template import compile_template + from salt.utils.process import Process + from salt.utils.zeromq import zmq +@@ -307,6 +307,18 @@ class SSH(MultiprocessingStateMixin): + "ssh_timeout", salt.config.DEFAULT_MASTER_OPTS["ssh_timeout"] + ) + + self.opts.get("timeout", salt.config.DEFAULT_MASTER_OPTS["timeout"]), ++ "keepalive": self.opts.get( ++ "ssh_keepalive", ++ salt.config.DEFAULT_MASTER_OPTS["ssh_keepalive"], ++ ), ++ "keepalive_interval": self.opts.get( ++ "ssh_keepalive_interval", ++ salt.config.DEFAULT_MASTER_OPTS["ssh_keepalive_interval"], ++ ), ++ "keepalive_count_max": self.opts.get( ++ "ssh_keepalive_count_max", ++ salt.config.DEFAULT_MASTER_OPTS["ssh_keepalive_count_max"], ++ ), + "sudo": self.opts.get( + "ssh_sudo", salt.config.DEFAULT_MASTER_OPTS["ssh_sudo"] + ), +@@ -557,7 +569,7 @@ class SSH(MultiprocessingStateMixin): + mods=self.mods, + fsclient=self.fsclient, + thin=self.thin, +- **target ++ **target, + ) + if salt.utils.path.which("ssh-copy-id"): + # we have ssh-copy-id, use it! +@@ -573,7 +585,7 @@ class SSH(MultiprocessingStateMixin): + mods=self.mods, + fsclient=self.fsclient, + thin=self.thin, +- **target ++ **target, + ) + stdout, stderr, retcode = single.cmd_block() + try: +@@ -601,7 +613,7 @@ class SSH(MultiprocessingStateMixin): + fsclient=self.fsclient, + thin=self.thin, + mine=mine, +- **target ++ **target, + ) + ret = {"id": single.id} + stdout, stderr, retcode = single.run() +@@ -1022,7 +1034,10 @@ class Single: + remote_port_forwards=None, + winrm=False, + ssh_options=None, +- **kwargs ++ keepalive=True, ++ keepalive_interval=60, ++ keepalive_count_max=3, ++ **kwargs, + ): + # Get mine setting and mine_functions if defined in kwargs (from roster) + self.mine = mine +@@ -1081,6 +1096,9 @@ class Single: + "priv": priv, + "priv_passwd": priv_passwd, + "timeout": timeout, ++ "keepalive": keepalive, ++ "keepalive_interval": keepalive_interval, ++ "keepalive_count_max": keepalive_count_max, + "sudo": sudo, + "tty": tty, + "mods": self.mods, +@@ -1302,7 +1320,7 @@ class Single: + self.id, + fsclient=self.fsclient, + minion_opts=self.minion_opts, +- **self.target ++ **self.target, + ) + + opts_pkg = pre_wrapper["test.opts_pkg"]() # pylint: disable=E1102 +@@ -1388,7 +1406,7 @@ class Single: + self.id, + fsclient=self.fsclient, + minion_opts=self.minion_opts, +- **self.target ++ **self.target, + ) + wrapper.fsclient.opts["cachedir"] = opts["cachedir"] + self.wfuncs = salt.loader.ssh_wrapper(opts, wrapper, self.context) +diff --git a/salt/client/ssh/client.py b/salt/client/ssh/client.py +index 0b67598fc6..a00f5de423 100644 +--- a/salt/client/ssh/client.py ++++ b/salt/client/ssh/client.py +@@ -52,6 +52,9 @@ class SSHClient: + ("ssh_priv_passwd", str), + ("ssh_identities_only", bool), + ("ssh_remote_port_forwards", str), ++ ("ssh_keepalive", bool), ++ ("ssh_keepalive_interval", int), ++ ("ssh_keepalive_count_max", int), + ("ssh_options", list), + ("ssh_max_procs", int), + ("ssh_askpass", bool), +@@ -108,7 +111,15 @@ class SSHClient: + return sane_kwargs + + def _prep_ssh( +- self, tgt, fun, arg=(), timeout=None, tgt_type="glob", kwarg=None, context=None, **kwargs ++ self, ++ tgt, ++ fun, ++ arg=(), ++ timeout=None, ++ tgt_type="glob", ++ kwarg=None, ++ context=None, ++ **kwargs + ): + """ + Prepare the arguments +diff --git a/salt/client/ssh/shell.py b/salt/client/ssh/shell.py +index bc1ad034df..182e2c19e3 100644 +--- a/salt/client/ssh/shell.py ++++ b/salt/client/ssh/shell.py +@@ -85,6 +85,9 @@ class Shell: + remote_port_forwards=None, + winrm=False, + ssh_options=None, ++ keepalive=True, ++ keepalive_interval=None, ++ keepalive_count_max=None, + ): + self.opts = opts + # ssh , but scp [ (4, 9): + options.append("GSSAPIAuthentication=no") + options.append("ConnectTimeout={}".format(self.timeout)) ++ if self.keepalive: ++ options.append(f"ServerAliveInterval={self.keepalive_interval}") ++ options.append(f"ServerAliveCountMax={self.keepalive_count_max}") + if self.opts.get("ignore_host_keys"): + options.append("StrictHostKeyChecking=no") + if self.opts.get("no_host_keys"): +@@ -165,6 +174,9 @@ class Shell: + if self.opts["_ssh_version"] > (4, 9): + options.append("GSSAPIAuthentication=no") + options.append("ConnectTimeout={}".format(self.timeout)) ++ if self.keepalive: ++ options.append(f"ServerAliveInterval={self.keepalive_interval}") ++ options.append(f"ServerAliveCountMax={self.keepalive_count_max}") + if self.opts.get("ignore_host_keys"): + options.append("StrictHostKeyChecking=no") + if self.opts.get("no_host_keys"): +diff --git a/salt/config/__init__.py b/salt/config/__init__.py +index d8258a4dbc..68f2b0f674 100644 +--- a/salt/config/__init__.py ++++ b/salt/config/__init__.py +@@ -822,6 +822,9 @@ VALID_OPTS = immutabletypes.freeze( + "ssh_scan_ports": str, + "ssh_scan_timeout": float, + "ssh_identities_only": bool, ++ "ssh_keepalive": bool, ++ "ssh_keepalive_interval": int, ++ "ssh_keepalive_count_max": int, + "ssh_log_file": str, + "ssh_config_file": str, + "ssh_merge_pillar": bool, +@@ -1592,6 +1595,9 @@ DEFAULT_MASTER_OPTS = immutabletypes.freeze( + "ssh_scan_ports": "22", + "ssh_scan_timeout": 0.01, + "ssh_identities_only": False, ++ "ssh_keepalive": True, ++ "ssh_keepalive_interval": 60, ++ "ssh_keepalive_count_max": 3, + "ssh_log_file": os.path.join(salt.syspaths.LOGS_DIR, "ssh"), + "ssh_config_file": os.path.join(salt.syspaths.HOME_DIR, ".ssh", "config"), + "cluster_mode": False, +diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py +index dc125de7d7..6c7f9f2f66 100644 +--- a/salt/utils/parsers.py ++++ b/salt/utils/parsers.py +@@ -3383,6 +3383,25 @@ class SaltSSHOptionParser( + "-R parameters." + ), + ) ++ ssh_group.add_option( ++ "--disable-keepalive", ++ default=True, ++ action="store_false", ++ dest="ssh_keepalive", ++ help=( ++ "Disable KeepAlive probes (ServerAliveInterval) for the SSH connection." ++ ), ++ ) ++ ssh_group.add_option( ++ "--keepalive-interval", ++ dest="ssh_keepalive_interval", ++ help=("Define the value for ServerAliveInterval option."), ++ ) ++ ssh_group.add_option( ++ "--keepalive-count-max", ++ dest="ssh_keepalive_count_max", ++ help=("Define the value for ServerAliveCountMax option."), ++ ) + ssh_group.add_option( + "--ssh-option", + dest="ssh_options", +diff --git a/tests/pytests/unit/client/ssh/test_single.py b/tests/pytests/unit/client/ssh/test_single.py +index c88a1c2127..8d87da8700 100644 +--- a/tests/pytests/unit/client/ssh/test_single.py ++++ b/tests/pytests/unit/client/ssh/test_single.py +@@ -63,6 +63,61 @@ def test_single_opts(opts, target): + **target, + ) + ++ assert single.shell._ssh_opts() == "" ++ expected_cmd = ( ++ "ssh login1 " ++ "-o KbdInteractiveAuthentication=no -o " ++ "PasswordAuthentication=yes -o ConnectTimeout=65 -o ServerAliveInterval=60 " ++ "-o ServerAliveCountMax=3 -o Port=22 " ++ "-o IdentityFile=/etc/salt/pki/master/ssh/salt-ssh.rsa " ++ "-o User=root date +%s" ++ ) ++ assert single.shell._cmd_str("date +%s") == expected_cmd ++ ++ ++def test_single_opts_custom_keepalive_options(opts, target): ++ """Sanity check for ssh.Single options with custom keepalive""" ++ ++ single = ssh.Single( ++ opts, ++ opts["argv"], ++ "localhost", ++ mods={}, ++ fsclient=None, ++ thin=salt.utils.thin.thin_path(opts["cachedir"]), ++ mine=False, ++ keepalive_interval=15, ++ keepalive_count_max=5, ++ **target, ++ ) ++ ++ assert single.shell._ssh_opts() == "" ++ expected_cmd = ( ++ "ssh login1 " ++ "-o KbdInteractiveAuthentication=no -o " ++ "PasswordAuthentication=yes -o ConnectTimeout=65 -o ServerAliveInterval=15 " ++ "-o ServerAliveCountMax=5 -o Port=22 " ++ "-o IdentityFile=/etc/salt/pki/master/ssh/salt-ssh.rsa " ++ "-o User=root date +%s" ++ ) ++ assert single.shell._cmd_str("date +%s") == expected_cmd ++ ++ ++def test_single_opts_disable_keepalive(opts, target): ++ """Sanity check for ssh.Single options with custom keepalive""" ++ ++ single = ssh.Single( ++ opts, ++ opts["argv"], ++ "localhost", ++ mods={}, ++ fsclient=None, ++ thin=salt.utils.thin.thin_path(opts["cachedir"]), ++ mine=False, ++ keepalive=False, ++ **target, ++ ) ++ + assert single.shell._ssh_opts() == "" + expected_cmd = ( + "ssh login1 " +diff --git a/tests/pytests/unit/client/ssh/test_ssh.py b/tests/pytests/unit/client/ssh/test_ssh.py +index cece16026c..23223ba8ec 100644 +--- a/tests/pytests/unit/client/ssh/test_ssh.py ++++ b/tests/pytests/unit/client/ssh/test_ssh.py +@@ -78,6 +78,9 @@ def roster(): + ("ssh_scan_ports", "test", True), + ("ssh_scan_timeout", 1.0, True), + ("ssh_timeout", 1, False), ++ ("ssh_keepalive", True, True), ++ ("ssh_keepalive_interval", 30, True), ++ ("ssh_keepalive_count_max", 3, True), + ("ssh_log_file", "/tmp/test", True), + ("raw_shell", True, True), + ("refresh_cache", True, True), +-- +2.42.0 + + diff --git a/fix-calculation-of-sls-context-vars-when-trailing-do.patch b/fix-calculation-of-sls-context-vars-when-trailing-do.patch new file mode 100644 index 0000000..eb74a0e --- /dev/null +++ b/fix-calculation-of-sls-context-vars-when-trailing-do.patch @@ -0,0 +1,69 @@ +From 3403a7391df785be31b6fbe401a8229c2007ac19 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Mon, 2 Oct 2023 10:44:05 +0100 +Subject: [PATCH] Fix calculation of SLS context vars when trailing dots + on targetted sls/state (bsc#1213518) (#598) + +* Fix calculation of SLS context vars when trailing dots on targetted state + +* Add changelog file +--- + changelog/63411.fixed.md | 1 + + salt/utils/templates.py | 5 +++-- + tests/unit/utils/test_templates.py | 14 ++++++++++++++ + 3 files changed, 18 insertions(+), 2 deletions(-) + create mode 100644 changelog/63411.fixed.md + +diff --git a/changelog/63411.fixed.md b/changelog/63411.fixed.md +new file mode 100644 +index 0000000000..65340e3652 +--- /dev/null ++++ b/changelog/63411.fixed.md +@@ -0,0 +1 @@ ++Fix calculation of SLS context vars when trailing dots on targetted state +diff --git a/salt/utils/templates.py b/salt/utils/templates.py +index 4a8adf2a14..8639ea703e 100644 +--- a/salt/utils/templates.py ++++ b/salt/utils/templates.py +@@ -113,8 +113,9 @@ def generate_sls_context(tmplpath, sls): + + sls_context = {} + +- # Normalize SLS as path. +- slspath = sls.replace(".", "/") ++ # Normalize SLS as path and remove possible trailing slashes ++ # to prevent matching issues and wrong vars calculation ++ slspath = sls.replace(".", "/").rstrip("/") + + if tmplpath: + # Normalize template path +diff --git a/tests/unit/utils/test_templates.py b/tests/unit/utils/test_templates.py +index 4ba2f52d7b..264b4ae801 100644 +--- a/tests/unit/utils/test_templates.py ++++ b/tests/unit/utils/test_templates.py +@@ -320,6 +320,20 @@ class WrapRenderTestCase(TestCase): + slspath="foo", + ) + ++ def test_generate_sls_context__one_level_init_implicit_with_trailing_dot(self): ++ """generate_sls_context - Basic one level with implicit init.sls with trailing dot""" ++ self._test_generated_sls_context( ++ "/tmp/foo/init.sls", ++ "foo.", ++ tplfile="foo/init.sls", ++ tpldir="foo", ++ tpldot="foo", ++ slsdotpath="foo", ++ slscolonpath="foo", ++ sls_path="foo", ++ slspath="foo", ++ ) ++ + def test_generate_sls_context__one_level_init_explicit(self): + """generate_sls_context - Basic one level with explicit init.sls""" + self._test_generated_sls_context( +-- +2.42.0 + + diff --git a/fix-cve-2023-34049-bsc-1215157.patch b/fix-cve-2023-34049-bsc-1215157.patch new file mode 100644 index 0000000..03acd24 --- /dev/null +++ b/fix-cve-2023-34049-bsc-1215157.patch @@ -0,0 +1,1163 @@ +From b2baafcc96a2807cf7d34374904e1710a4f58b9f Mon Sep 17 00:00:00 2001 +From: Alexander Graul +Date: Tue, 31 Oct 2023 11:26:15 +0100 +Subject: [PATCH] Fix CVE-2023-34049 (bsc#1215157) + +Backport of https://github.com/saltstack/salt/pull/65482 +--- + salt/client/ssh/__init__.py | 56 +++- + tests/integration/modules/test_ssh.py | 3 +- + tests/integration/ssh/test_pre_flight.py | 132 -------- + .../integration/ssh/test_pre_flight.py | 315 ++++++++++++++++++ + tests/pytests/unit/client/ssh/test_single.py | 296 +++++++++++++--- + tests/pytests/unit/client/ssh/test_ssh.py | 110 ++++++ + 6 files changed, 727 insertions(+), 185 deletions(-) + delete mode 100644 tests/integration/ssh/test_pre_flight.py + create mode 100644 tests/pytests/integration/ssh/test_pre_flight.py + +diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py +index b120e0002e8..1e143f9e30c 100644 +--- a/salt/client/ssh/__init__.py ++++ b/salt/client/ssh/__init__.py +@@ -12,9 +12,11 @@ import hashlib + import logging + import multiprocessing + import os ++import pathlib + import queue + import re + import shlex ++import shutil + import subprocess + import sys + import tarfile +@@ -515,7 +517,14 @@ class SSH(MultiprocessingStateMixin): + if target.get("passwd", False) or self.opts["ssh_passwd"]: + self._key_deploy_run(host, target, False) + return ret +- if ret[host].get("stderr", "").count("Permission denied"): ++ stderr = ret[host].get("stderr", "") ++ # -failed to upload file- is detecting scp errors ++ # Errors to ignore when Permission denied is in the stderr. For example ++ # scp can get a permission denied on the target host, but they where ++ # able to accurate authenticate against the box ++ ignore_err = ["failed to upload file"] ++ check_err = [x for x in ignore_err if stderr.count(x)] ++ if "Permission denied" in stderr and not check_err: + target = self.targets[host] + # permission denied, attempt to auto deploy ssh key + print( +@@ -1137,11 +1146,30 @@ class Single: + """ + Run our pre_flight script before running any ssh commands + """ +- script = os.path.join(tempfile.gettempdir(), self.ssh_pre_file) +- +- self.shell.send(self.ssh_pre_flight, script) +- +- return self.execute_script(script, script_args=self.ssh_pre_flight_args) ++ with tempfile.NamedTemporaryFile() as temp: ++ # ensure we use copyfile to not copy the file attributes ++ # we want to ensure we use the perms set by the secure ++ # NamedTemporaryFile ++ try: ++ shutil.copyfile(self.ssh_pre_flight, temp.name) ++ except OSError as err: ++ return ( ++ "", ++ "Could not copy pre flight script to temporary path", ++ 1, ++ ) ++ target_script = f".{pathlib.Path(temp.name).name}" ++ log.trace("Copying the pre flight script to target") ++ stdout, stderr, retcode = self.shell.send(temp.name, target_script) ++ if retcode != 0: ++ # We could not copy the script to the target ++ log.error("Could not copy the pre flight script to target") ++ return stdout, stderr, retcode ++ ++ log.trace("Executing the pre flight script on target") ++ return self.execute_script( ++ target_script, script_args=self.ssh_pre_flight_args ++ ) + + def check_thin_dir(self): + """ +@@ -1531,18 +1559,20 @@ ARGS = {arguments}\n'''.format( + return self.shell.exec_cmd(cmd_str) + + # Write the shim to a temporary file in the default temp directory +- with tempfile.NamedTemporaryFile( +- mode="w+b", prefix="shim_", delete=False +- ) as shim_tmp_file: ++ with tempfile.NamedTemporaryFile(mode="w+b", delete=False) as shim_tmp_file: + shim_tmp_file.write(salt.utils.stringutils.to_bytes(cmd_str)) + + # Copy shim to target system, under $HOME/. +- target_shim_file = ".{}.{}".format( +- binascii.hexlify(os.urandom(6)).decode("ascii"), extension +- ) ++ target_shim_file = f".{pathlib.Path(shim_tmp_file.name).name}" ++ + if self.winrm: + target_shim_file = saltwinshell.get_target_shim_file(self, target_shim_file) +- self.shell.send(shim_tmp_file.name, target_shim_file, makedirs=True) ++ stdout, stderr, retcode = self.shell.send( ++ shim_tmp_file.name, target_shim_file, makedirs=True ++ ) ++ if retcode != 0: ++ log.error("Could not copy the shim script to target") ++ return stdout, stderr, retcode + + # Remove our shim file + try: +diff --git a/tests/integration/modules/test_ssh.py b/tests/integration/modules/test_ssh.py +index 0817877c86b..55586211622 100644 +--- a/tests/integration/modules/test_ssh.py ++++ b/tests/integration/modules/test_ssh.py +@@ -26,7 +26,8 @@ def check_status(): + return False + + +-@pytest.mark.windows_whitelisted ++# @pytest.mark.windows_whitelisted ++# De-whitelist windows since it's hanging on the newer windows golden images + @pytest.mark.skip_if_binaries_missing("ssh", "ssh-keygen", check_all=True) + class SSHModuleTest(ModuleCase): + """ +diff --git a/tests/integration/ssh/test_pre_flight.py b/tests/integration/ssh/test_pre_flight.py +deleted file mode 100644 +index 1598b3d51b5..00000000000 +--- a/tests/integration/ssh/test_pre_flight.py ++++ /dev/null +@@ -1,132 +0,0 @@ +-""" +-Test for ssh_pre_flight roster option +-""" +- +-import os +- +-import pytest +- +-import salt.utils.files +-from tests.support.case import SSHCase +-from tests.support.runtests import RUNTIME_VARS +- +- +-class SSHPreFlightTest(SSHCase): +- """ +- Test ssh_pre_flight roster option +- """ +- +- def setUp(self): +- super().setUp() +- self.roster = os.path.join(RUNTIME_VARS.TMP, "pre_flight_roster") +- self.data = { +- "ssh_pre_flight": os.path.join(RUNTIME_VARS.TMP, "ssh_pre_flight.sh") +- } +- self.test_script = os.path.join( +- RUNTIME_VARS.TMP, "test-pre-flight-script-worked.txt" +- ) +- +- def _create_roster(self, pre_flight_script_args=None): +- data = dict(self.data) +- if pre_flight_script_args: +- data["ssh_pre_flight_args"] = pre_flight_script_args +- +- self.custom_roster(self.roster, data) +- +- with salt.utils.files.fopen(data["ssh_pre_flight"], "w") as fp_: +- fp_.write("touch {}".format(self.test_script)) +- +- @pytest.mark.slow_test +- def test_ssh_pre_flight(self): +- """ +- test ssh when ssh_pre_flight is set +- ensure the script runs successfully +- """ +- self._create_roster() +- assert self.run_function("test.ping", roster_file=self.roster) +- +- assert os.path.exists(self.test_script) +- +- @pytest.mark.slow_test +- def test_ssh_run_pre_flight(self): +- """ +- test ssh when --pre-flight is passed to salt-ssh +- to ensure the script runs successfully +- """ +- self._create_roster() +- # make sure we previously ran a command so the thin dir exists +- self.run_function("test.ping", wipe=False) +- assert not os.path.exists(self.test_script) +- +- assert self.run_function( +- "test.ping", ssh_opts="--pre-flight", roster_file=self.roster, wipe=False +- ) +- assert os.path.exists(self.test_script) +- +- @pytest.mark.slow_test +- def test_ssh_run_pre_flight_args(self): +- """ +- test ssh when --pre-flight is passed to salt-ssh +- to ensure the script runs successfully passing some args +- """ +- self._create_roster(pre_flight_script_args="foobar test") +- # make sure we previously ran a command so the thin dir exists +- self.run_function("test.ping", wipe=False) +- assert not os.path.exists(self.test_script) +- +- assert self.run_function( +- "test.ping", ssh_opts="--pre-flight", roster_file=self.roster, wipe=False +- ) +- assert os.path.exists(self.test_script) +- +- @pytest.mark.slow_test +- def test_ssh_run_pre_flight_args_prevent_injection(self): +- """ +- test ssh when --pre-flight is passed to salt-ssh +- and evil arguments are used in order to produce shell injection +- """ +- injected_file = os.path.join(RUNTIME_VARS.TMP, "injection") +- self._create_roster( +- pre_flight_script_args="foobar; echo injected > {}".format(injected_file) +- ) +- # make sure we previously ran a command so the thin dir exists +- self.run_function("test.ping", wipe=False) +- assert not os.path.exists(self.test_script) +- assert not os.path.isfile(injected_file) +- +- assert self.run_function( +- "test.ping", ssh_opts="--pre-flight", roster_file=self.roster, wipe=False +- ) +- +- assert not os.path.isfile( +- injected_file +- ), "File injection suceeded. This shouldn't happend" +- +- @pytest.mark.slow_test +- def test_ssh_run_pre_flight_failure(self): +- """ +- test ssh_pre_flight when there is a failure +- in the script. +- """ +- self._create_roster() +- with salt.utils.files.fopen(self.data["ssh_pre_flight"], "w") as fp_: +- fp_.write("exit 2") +- +- ret = self.run_function( +- "test.ping", ssh_opts="--pre-flight", roster_file=self.roster, wipe=False +- ) +- assert ret["retcode"] == 2 +- +- def tearDown(self): +- """ +- make sure to clean up any old ssh directories +- """ +- files = [ +- self.roster, +- self.data["ssh_pre_flight"], +- self.test_script, +- os.path.join(RUNTIME_VARS.TMP, "injection"), +- ] +- for fp_ in files: +- if os.path.exists(fp_): +- os.remove(fp_) +diff --git a/tests/pytests/integration/ssh/test_pre_flight.py b/tests/pytests/integration/ssh/test_pre_flight.py +new file mode 100644 +index 00000000000..09c65d29430 +--- /dev/null ++++ b/tests/pytests/integration/ssh/test_pre_flight.py +@@ -0,0 +1,315 @@ ++""" ++Test for ssh_pre_flight roster option ++""" ++ ++try: ++ import grp ++ import pwd ++except ImportError: ++ # windows stacktraces on import of these modules ++ pass ++import os ++import pathlib ++import shutil ++import subprocess ++ ++import pytest ++import yaml ++from saltfactories.utils import random_string ++ ++import salt.utils.files ++ ++pytestmark = pytest.mark.skip_on_windows(reason="Salt-ssh not available on Windows") ++ ++ ++def _custom_roster(roster_file, roster_data): ++ with salt.utils.files.fopen(roster_file, "r") as fp: ++ data = salt.utils.yaml.safe_load(fp) ++ for key, item in roster_data.items(): ++ data["localhost"][key] = item ++ with salt.utils.files.fopen(roster_file, "w") as fp: ++ yaml.safe_dump(data, fp) ++ ++ ++@pytest.fixture ++def _create_roster(salt_ssh_roster_file, tmp_path): ++ ret = {} ++ ret["roster"] = salt_ssh_roster_file ++ ret["data"] = {"ssh_pre_flight": str(tmp_path / "ssh_pre_flight.sh")} ++ ret["test_script"] = str(tmp_path / "test-pre-flight-script-worked.txt") ++ ret["thin_dir"] = tmp_path / "thin_dir" ++ ++ with salt.utils.files.fopen(salt_ssh_roster_file, "r") as fp: ++ data = salt.utils.yaml.safe_load(fp) ++ pre_flight_script = ret["data"]["ssh_pre_flight"] ++ data["localhost"]["ssh_pre_flight"] = pre_flight_script ++ data["localhost"]["thin_dir"] = str(ret["thin_dir"]) ++ with salt.utils.files.fopen(salt_ssh_roster_file, "w") as fp: ++ yaml.safe_dump(data, fp) ++ ++ with salt.utils.files.fopen(pre_flight_script, "w") as fp: ++ fp.write("touch {}".format(ret["test_script"])) ++ ++ yield ret ++ if ret["thin_dir"].exists(): ++ shutil.rmtree(ret["thin_dir"]) ++ ++ ++@pytest.mark.slow_test ++def test_ssh_pre_flight(salt_ssh_cli, caplog, _create_roster): ++ """ ++ test ssh when ssh_pre_flight is set ++ ensure the script runs successfully ++ """ ++ ret = salt_ssh_cli.run("test.ping") ++ assert ret.returncode == 0 ++ ++ assert pathlib.Path(_create_roster["test_script"]).exists() ++ ++ ++@pytest.mark.slow_test ++def test_ssh_run_pre_flight(salt_ssh_cli, _create_roster): ++ """ ++ test ssh when --pre-flight is passed to salt-ssh ++ to ensure the script runs successfully ++ """ ++ # make sure we previously ran a command so the thin dir exists ++ ret = salt_ssh_cli.run("test.ping") ++ assert pathlib.Path(_create_roster["test_script"]).exists() ++ ++ # Now remeove the script to ensure pre_flight doesn't run ++ # without --pre-flight ++ pathlib.Path(_create_roster["test_script"]).unlink() ++ ++ assert salt_ssh_cli.run("test.ping").returncode == 0 ++ assert not pathlib.Path(_create_roster["test_script"]).exists() ++ ++ # Now ensure ++ ret = salt_ssh_cli.run( ++ "test.ping", ++ "--pre-flight", ++ ) ++ assert ret.returncode == 0 ++ assert pathlib.Path(_create_roster["test_script"]).exists() ++ ++ ++@pytest.mark.slow_test ++def test_ssh_run_pre_flight_args(salt_ssh_cli, _create_roster): ++ """ ++ test ssh when --pre-flight is passed to salt-ssh ++ to ensure the script runs successfully passing some args ++ """ ++ _custom_roster(salt_ssh_cli.roster_file, {"ssh_pre_flight_args": "foobar test"}) ++ # Create pre_flight script that accepts args ++ test_script = _create_roster["test_script"] ++ test_script_1 = pathlib.Path(test_script + "-foobar") ++ test_script_2 = pathlib.Path(test_script + "-test") ++ with salt.utils.files.fopen(_create_roster["data"]["ssh_pre_flight"], "w") as fp: ++ fp.write( ++ f""" ++ touch {str(test_script)}-$1 ++ touch {str(test_script)}-$2 ++ """ ++ ) ++ ret = salt_ssh_cli.run("test.ping") ++ assert ret.returncode == 0 ++ assert test_script_1.exists() ++ assert test_script_2.exists() ++ pathlib.Path(test_script_1).unlink() ++ pathlib.Path(test_script_2).unlink() ++ ++ ret = salt_ssh_cli.run("test.ping") ++ assert ret.returncode == 0 ++ assert not test_script_1.exists() ++ assert not test_script_2.exists() ++ ++ ret = salt_ssh_cli.run( ++ "test.ping", ++ "--pre-flight", ++ ) ++ assert ret.returncode == 0 ++ assert test_script_1.exists() ++ assert test_script_2.exists() ++ ++ ++@pytest.mark.slow_test ++def test_ssh_run_pre_flight_args_prevent_injection( ++ salt_ssh_cli, _create_roster, tmp_path ++): ++ """ ++ test ssh when --pre-flight is passed to salt-ssh ++ and evil arguments are used in order to produce shell injection ++ """ ++ injected_file = tmp_path / "injection" ++ _custom_roster( ++ salt_ssh_cli.roster_file, ++ {"ssh_pre_flight_args": f"foobar; echo injected > {str(injected_file)}"}, ++ ) ++ # Create pre_flight script that accepts args ++ test_script = _create_roster["test_script"] ++ test_script_1 = pathlib.Path(test_script + "-echo") ++ test_script_2 = pathlib.Path(test_script + "-foobar;") ++ with salt.utils.files.fopen(_create_roster["data"]["ssh_pre_flight"], "w") as fp: ++ fp.write( ++ f""" ++ touch {str(test_script)}-$1 ++ touch {str(test_script)}-$2 ++ """ ++ ) ++ ++ # make sure we previously ran a command so the thin dir exists ++ ret = salt_ssh_cli.run("test.ping") ++ assert ret.returncode == 0 ++ assert test_script_1.exists() ++ assert test_script_2.exists() ++ test_script_1.unlink() ++ test_script_2.unlink() ++ assert not injected_file.is_file() ++ ++ ret = salt_ssh_cli.run( ++ "test.ping", ++ "--pre-flight", ++ ) ++ assert ret.returncode == 0 ++ ++ assert test_script_1.exists() ++ assert test_script_2.exists() ++ assert not pathlib.Path( ++ injected_file ++ ).is_file(), "File injection suceeded. This shouldn't happend" ++ ++ ++@pytest.mark.flaky(max_runs=4) ++@pytest.mark.slow_test ++def test_ssh_run_pre_flight_failure(salt_ssh_cli, _create_roster): ++ """ ++ test ssh_pre_flight when there is a failure ++ in the script. ++ """ ++ with salt.utils.files.fopen(_create_roster["data"]["ssh_pre_flight"], "w") as fp_: ++ fp_.write("exit 2") ++ ++ ret = salt_ssh_cli.run( ++ "test.ping", ++ "--pre-flight", ++ ) ++ assert ret.data["retcode"] == 2 ++ ++ ++@pytest.fixture ++def account(): ++ username = random_string("test-account-", uppercase=False) ++ with pytest.helpers.create_account(username=username) as account: ++ yield account ++ ++ ++@pytest.mark.slow_test ++def test_ssh_pre_flight_script(salt_ssh_cli, caplog, _create_roster, tmp_path, account): ++ """ ++ Test to ensure user cannot create and run a script ++ with the expected pre_flight script path on target. ++ """ ++ try: ++ script = pathlib.Path.home() / "hacked" ++ tmp_preflight = pathlib.Path("/tmp", "ssh_pre_flight.sh") ++ tmp_preflight.write_text(f"touch {script}") ++ os.chown(tmp_preflight, account.info.uid, account.info.gid) ++ ret = salt_ssh_cli.run("test.ping") ++ assert not script.is_file() ++ assert ret.returncode == 0 ++ assert ret.stdout == '{\n"localhost": true\n}\n' ++ finally: ++ for _file in [script, tmp_preflight]: ++ if _file.is_file(): ++ _file.unlink() ++ ++ ++def demote(user_uid, user_gid): ++ def result(): ++ # os.setgid does not remove group membership, so we remove them here so they are REALLY non-root ++ os.setgroups([]) ++ os.setgid(user_gid) ++ os.setuid(user_uid) ++ ++ return result ++ ++ ++@pytest.mark.slow_test ++def test_ssh_pre_flight_perms(salt_ssh_cli, caplog, _create_roster, account): ++ """ ++ Test to ensure standard user cannot run pre flight script ++ on target when user sets wrong permissions (777) on ++ ssh_pre_flight script. ++ """ ++ try: ++ script = pathlib.Path("/tmp", "itworked") ++ preflight = pathlib.Path("/ssh_pre_flight.sh") ++ preflight.write_text(f"touch {str(script)}") ++ tmp_preflight = pathlib.Path("/tmp", preflight.name) ++ ++ _custom_roster(salt_ssh_cli.roster_file, {"ssh_pre_flight": str(preflight)}) ++ preflight.chmod(0o0777) ++ run_script = pathlib.Path("/run_script") ++ run_script.write_text( ++ f""" ++ x=1 ++ while [ $x -le 200000 ]; do ++ SCRIPT=`bash {str(tmp_preflight)} 2> /dev/null; echo $?` ++ if [ ${{SCRIPT}} == 0 ]; then ++ break ++ fi ++ x=$(( $x + 1 )) ++ done ++ """ ++ ) ++ run_script.chmod(0o0777) ++ # pylint: disable=W1509 ++ ret = subprocess.Popen( ++ ["sh", f"{run_script}"], ++ preexec_fn=demote(account.info.uid, account.info.gid), ++ stdout=None, ++ stderr=None, ++ stdin=None, ++ universal_newlines=True, ++ ) ++ # pylint: enable=W1509 ++ ret = salt_ssh_cli.run("test.ping") ++ assert ret.returncode == 0 ++ ++ # Lets make sure a different user other than root ++ # Didn't run the script ++ assert os.stat(script).st_uid != account.info.uid ++ assert script.is_file() ++ finally: ++ for _file in [script, preflight, tmp_preflight, run_script]: ++ if _file.is_file(): ++ _file.unlink() ++ ++ ++@pytest.mark.slow_test ++def test_ssh_run_pre_flight_target_file_perms(salt_ssh_cli, _create_roster, tmp_path): ++ """ ++ test ssh_pre_flight to ensure the target pre flight script ++ has the correct perms ++ """ ++ perms_file = tmp_path / "perms" ++ with salt.utils.files.fopen(_create_roster["data"]["ssh_pre_flight"], "w") as fp_: ++ fp_.write( ++ f""" ++ SCRIPT_NAME=$0 ++ stat -L -c "%a %G %U" $SCRIPT_NAME > {perms_file} ++ """ ++ ) ++ ++ ret = salt_ssh_cli.run( ++ "test.ping", ++ "--pre-flight", ++ ) ++ assert ret.returncode == 0 ++ with salt.utils.files.fopen(perms_file) as fp: ++ data = fp.read() ++ assert data.split()[0] == "600" ++ uid = os.getuid() ++ gid = os.getgid() ++ assert data.split()[1] == grp.getgrgid(gid).gr_name ++ assert data.split()[2] == pwd.getpwuid(uid).pw_name +diff --git a/tests/pytests/unit/client/ssh/test_single.py b/tests/pytests/unit/client/ssh/test_single.py +index f97519d5cc2..c88a1c2127f 100644 +--- a/tests/pytests/unit/client/ssh/test_single.py ++++ b/tests/pytests/unit/client/ssh/test_single.py +@@ -1,6 +1,5 @@ +-import os ++import logging + import re +-import tempfile + from textwrap import dedent + + import pytest +@@ -16,6 +15,8 @@ import salt.utils.yaml + from salt.client import ssh + from tests.support.mock import MagicMock, call, patch + ++log = logging.getLogger(__name__) ++ + + @pytest.fixture + def opts(tmp_path): +@@ -59,7 +60,7 @@ def test_single_opts(opts, target): + fsclient=None, + thin=salt.utils.thin.thin_path(opts["cachedir"]), + mine=False, +- **target ++ **target, + ) + + assert single.shell._ssh_opts() == "" +@@ -87,7 +88,7 @@ def test_run_with_pre_flight(opts, target, tmp_path): + fsclient=None, + thin=salt.utils.thin.thin_path(opts["cachedir"]), + mine=False, +- **target ++ **target, + ) + + cmd_ret = ("Success", "", 0) +@@ -122,7 +123,7 @@ def test_run_with_pre_flight_with_args(opts, target, tmp_path): + fsclient=None, + thin=salt.utils.thin.thin_path(opts["cachedir"]), + mine=False, +- **target ++ **target, + ) + + cmd_ret = ("Success", "foobar", 0) +@@ -156,7 +157,7 @@ def test_run_with_pre_flight_stderr(opts, target, tmp_path): + fsclient=None, + thin=salt.utils.thin.thin_path(opts["cachedir"]), + mine=False, +- **target ++ **target, + ) + + cmd_ret = ("", "Error running script", 1) +@@ -190,7 +191,7 @@ def test_run_with_pre_flight_script_doesnot_exist(opts, target, tmp_path): + fsclient=None, + thin=salt.utils.thin.thin_path(opts["cachedir"]), + mine=False, +- **target ++ **target, + ) + + cmd_ret = ("Success", "", 0) +@@ -224,7 +225,7 @@ def test_run_with_pre_flight_thin_dir_exists(opts, target, tmp_path): + fsclient=None, + thin=salt.utils.thin.thin_path(opts["cachedir"]), + mine=False, +- **target ++ **target, + ) + + cmd_ret = ("", "", 0) +@@ -242,6 +243,39 @@ def test_run_with_pre_flight_thin_dir_exists(opts, target, tmp_path): + assert ret == cmd_ret + + ++def test_run_ssh_pre_flight(opts, target, tmp_path): ++ """ ++ test Single.run_ssh_pre_flight function ++ """ ++ target["ssh_pre_flight"] = str(tmp_path / "script.sh") ++ single = ssh.Single( ++ opts, ++ opts["argv"], ++ "localhost", ++ mods={}, ++ fsclient=None, ++ thin=salt.utils.thin.thin_path(opts["cachedir"]), ++ mine=False, ++ **target, ++ ) ++ ++ cmd_ret = ("Success", "", 0) ++ mock_flight = MagicMock(return_value=cmd_ret) ++ mock_cmd = MagicMock(return_value=cmd_ret) ++ patch_flight = patch("salt.client.ssh.Single.run_ssh_pre_flight", mock_flight) ++ patch_cmd = patch("salt.client.ssh.Single.cmd_block", mock_cmd) ++ patch_exec_cmd = patch( ++ "salt.client.ssh.shell.Shell.exec_cmd", return_value=("", "", 1) ++ ) ++ patch_os = patch("os.path.exists", side_effect=[True]) ++ ++ with patch_os, patch_flight, patch_cmd, patch_exec_cmd: ++ ret = single.run() ++ mock_cmd.assert_called() ++ mock_flight.assert_called() ++ assert ret == cmd_ret ++ ++ + def test_execute_script(opts, target, tmp_path): + """ + test Single.execute_script() +@@ -255,7 +289,7 @@ def test_execute_script(opts, target, tmp_path): + thin=salt.utils.thin.thin_path(opts["cachedir"]), + mine=False, + winrm=False, +- **target ++ **target, + ) + + exp_ret = ("Success", "", 0) +@@ -273,7 +307,7 @@ def test_execute_script(opts, target, tmp_path): + ] == mock_cmd.call_args_list + + +-def test_shim_cmd(opts, target): ++def test_shim_cmd(opts, target, tmp_path): + """ + test Single.shim_cmd() + """ +@@ -287,7 +321,7 @@ def test_shim_cmd(opts, target): + mine=False, + winrm=False, + tty=True, +- **target ++ **target, + ) + + exp_ret = ("Success", "", 0) +@@ -295,21 +329,24 @@ def test_shim_cmd(opts, target): + patch_cmd = patch("salt.client.ssh.shell.Shell.exec_cmd", mock_cmd) + patch_send = patch("salt.client.ssh.shell.Shell.send", return_value=("", "", 0)) + patch_rand = patch("os.urandom", return_value=b"5\xd9l\xca\xc2\xff") ++ tmp_file = tmp_path / "tmp_file" ++ mock_tmp = MagicMock() ++ patch_tmp = patch("tempfile.NamedTemporaryFile", mock_tmp) ++ mock_tmp.return_value.__enter__.return_value.name = tmp_file + +- with patch_cmd, patch_rand, patch_send: ++ with patch_cmd, patch_tmp, patch_send: + ret = single.shim_cmd(cmd_str="echo test") + assert ret == exp_ret + assert [ +- call("/bin/sh '.35d96ccac2ff.py'"), +- call("rm '.35d96ccac2ff.py'"), ++ call(f"/bin/sh '.{tmp_file.name}'"), ++ call(f"rm '.{tmp_file.name}'"), + ] == mock_cmd.call_args_list + + +-def test_run_ssh_pre_flight(opts, target, tmp_path): ++def test_shim_cmd_copy_fails(opts, target, caplog): + """ +- test Single.run_ssh_pre_flight ++ test Single.shim_cmd() when copying the file fails + """ +- target["ssh_pre_flight"] = str(tmp_path / "script.sh") + single = ssh.Single( + opts, + opts["argv"], +@@ -320,24 +357,202 @@ def test_run_ssh_pre_flight(opts, target, tmp_path): + mine=False, + winrm=False, + tty=True, +- **target ++ **target, + ) + +- exp_ret = ("Success", "", 0) +- mock_cmd = MagicMock(return_value=exp_ret) ++ ret_cmd = ("Success", "", 0) ++ mock_cmd = MagicMock(return_value=ret_cmd) + patch_cmd = patch("salt.client.ssh.shell.Shell.exec_cmd", mock_cmd) +- patch_send = patch("salt.client.ssh.shell.Shell.send", return_value=exp_ret) +- exp_tmp = os.path.join( +- tempfile.gettempdir(), os.path.basename(target["ssh_pre_flight"]) ++ ret_send = ("", "General error in file copy", 1) ++ patch_send = patch("salt.client.ssh.shell.Shell.send", return_value=ret_send) ++ patch_rand = patch("os.urandom", return_value=b"5\xd9l\xca\xc2\xff") ++ ++ with patch_cmd, patch_rand, patch_send: ++ ret = single.shim_cmd(cmd_str="echo test") ++ assert ret == ret_send ++ assert "Could not copy the shim script to target" in caplog.text ++ mock_cmd.assert_not_called() ++ ++ ++def test_run_ssh_pre_flight_no_connect(opts, target, tmp_path, caplog): ++ """ ++ test Single.run_ssh_pre_flight when you ++ cannot connect to the target ++ """ ++ pre_flight = tmp_path / "script.sh" ++ pre_flight.write_text("") ++ target["ssh_pre_flight"] = str(pre_flight) ++ single = ssh.Single( ++ opts, ++ opts["argv"], ++ "localhost", ++ mods={}, ++ fsclient=None, ++ thin=salt.utils.thin.thin_path(opts["cachedir"]), ++ mine=False, ++ winrm=False, ++ tty=True, ++ **target, + ) ++ mock_exec_cmd = MagicMock(return_value=("", "", 1)) ++ patch_exec_cmd = patch("salt.client.ssh.shell.Shell.exec_cmd", mock_exec_cmd) ++ tmp_file = tmp_path / "tmp_file" ++ mock_tmp = MagicMock() ++ patch_tmp = patch("tempfile.NamedTemporaryFile", mock_tmp) ++ mock_tmp.return_value.__enter__.return_value.name = tmp_file ++ ret_send = ( ++ "", ++ "ssh: connect to host 192.168.1.186 port 22: No route to host\nscp: Connection closed\n", ++ 255, ++ ) ++ send_mock = MagicMock(return_value=ret_send) ++ patch_send = patch("salt.client.ssh.shell.Shell.send", send_mock) ++ ++ with caplog.at_level(logging.TRACE): ++ with patch_send, patch_exec_cmd, patch_tmp: ++ ret = single.run_ssh_pre_flight() ++ assert "Copying the pre flight script" in caplog.text ++ assert "Could not copy the pre flight script to target" in caplog.text ++ assert ret == ret_send ++ assert send_mock.call_args_list[0][0][0] == tmp_file ++ target_script = send_mock.call_args_list[0][0][1] ++ assert re.search(r".[a-z0-9]+", target_script) ++ mock_exec_cmd.assert_not_called() ++ ++ ++def test_run_ssh_pre_flight_permission_denied(opts, target, tmp_path): ++ """ ++ test Single.run_ssh_pre_flight when you ++ cannot copy script to the target due to ++ a permission denied error ++ """ ++ pre_flight = tmp_path / "script.sh" ++ pre_flight.write_text("") ++ target["ssh_pre_flight"] = str(pre_flight) ++ single = ssh.Single( ++ opts, ++ opts["argv"], ++ "localhost", ++ mods={}, ++ fsclient=None, ++ thin=salt.utils.thin.thin_path(opts["cachedir"]), ++ mine=False, ++ winrm=False, ++ tty=True, ++ **target, ++ ) ++ mock_exec_cmd = MagicMock(return_value=("", "", 1)) ++ patch_exec_cmd = patch("salt.client.ssh.shell.Shell.exec_cmd", mock_exec_cmd) ++ tmp_file = tmp_path / "tmp_file" ++ mock_tmp = MagicMock() ++ patch_tmp = patch("tempfile.NamedTemporaryFile", mock_tmp) ++ mock_tmp.return_value.__enter__.return_value.name = tmp_file ++ ret_send = ( ++ "", ++ 'scp: dest open "/tmp/preflight.sh": Permission denied\nscp: failed to upload file /etc/salt/preflight.sh to /tmp/preflight.sh\n', ++ 255, ++ ) ++ send_mock = MagicMock(return_value=ret_send) ++ patch_send = patch("salt.client.ssh.shell.Shell.send", send_mock) + +- with patch_cmd, patch_send: ++ with patch_send, patch_exec_cmd, patch_tmp: + ret = single.run_ssh_pre_flight() +- assert ret == exp_ret +- assert [ +- call("/bin/sh '{}'".format(exp_tmp)), +- call("rm '{}'".format(exp_tmp)), +- ] == mock_cmd.call_args_list ++ assert ret == ret_send ++ assert send_mock.call_args_list[0][0][0] == tmp_file ++ target_script = send_mock.call_args_list[0][0][1] ++ assert re.search(r".[a-z0-9]+", target_script) ++ mock_exec_cmd.assert_not_called() ++ ++ ++def test_run_ssh_pre_flight_connect(opts, target, tmp_path, caplog): ++ """ ++ test Single.run_ssh_pre_flight when you ++ can connect to the target ++ """ ++ pre_flight = tmp_path / "script.sh" ++ pre_flight.write_text("") ++ target["ssh_pre_flight"] = str(pre_flight) ++ single = ssh.Single( ++ opts, ++ opts["argv"], ++ "localhost", ++ mods={}, ++ fsclient=None, ++ thin=salt.utils.thin.thin_path(opts["cachedir"]), ++ mine=False, ++ winrm=False, ++ tty=True, ++ **target, ++ ) ++ ret_exec_cmd = ("", "", 1) ++ mock_exec_cmd = MagicMock(return_value=ret_exec_cmd) ++ patch_exec_cmd = patch("salt.client.ssh.shell.Shell.exec_cmd", mock_exec_cmd) ++ tmp_file = tmp_path / "tmp_file" ++ mock_tmp = MagicMock() ++ patch_tmp = patch("tempfile.NamedTemporaryFile", mock_tmp) ++ mock_tmp.return_value.__enter__.return_value.name = tmp_file ++ ret_send = ( ++ "", ++ "\rroot@192.168.1.187's password: \n\rpreflight.sh 0% 0 0.0KB/s --:-- ETA\rpreflight.sh 100% 20 2.7KB/s 00:00 \n", ++ 0, ++ ) ++ send_mock = MagicMock(return_value=ret_send) ++ patch_send = patch("salt.client.ssh.shell.Shell.send", send_mock) ++ ++ with caplog.at_level(logging.TRACE): ++ with patch_send, patch_exec_cmd, patch_tmp: ++ ret = single.run_ssh_pre_flight() ++ ++ assert "Executing the pre flight script on target" in caplog.text ++ assert ret == ret_exec_cmd ++ assert send_mock.call_args_list[0][0][0] == tmp_file ++ target_script = send_mock.call_args_list[0][0][1] ++ assert re.search(r".[a-z0-9]+", target_script) ++ mock_exec_cmd.assert_called() ++ ++ ++def test_run_ssh_pre_flight_shutil_fails(opts, target, tmp_path): ++ """ ++ test Single.run_ssh_pre_flight when cannot ++ copyfile with shutil ++ """ ++ pre_flight = tmp_path / "script.sh" ++ pre_flight.write_text("") ++ target["ssh_pre_flight"] = str(pre_flight) ++ single = ssh.Single( ++ opts, ++ opts["argv"], ++ "localhost", ++ mods={}, ++ fsclient=None, ++ thin=salt.utils.thin.thin_path(opts["cachedir"]), ++ mine=False, ++ winrm=False, ++ tty=True, ++ **target, ++ ) ++ ret_exec_cmd = ("", "", 1) ++ mock_exec_cmd = MagicMock(return_value=ret_exec_cmd) ++ patch_exec_cmd = patch("salt.client.ssh.shell.Shell.exec_cmd", mock_exec_cmd) ++ tmp_file = tmp_path / "tmp_file" ++ mock_tmp = MagicMock() ++ patch_tmp = patch("tempfile.NamedTemporaryFile", mock_tmp) ++ mock_tmp.return_value.__enter__.return_value.name = tmp_file ++ send_mock = MagicMock() ++ mock_shutil = MagicMock(side_effect=IOError("Permission Denied")) ++ patch_shutil = patch("shutil.copyfile", mock_shutil) ++ patch_send = patch("salt.client.ssh.shell.Shell.send", send_mock) ++ ++ with patch_send, patch_exec_cmd, patch_tmp, patch_shutil: ++ ret = single.run_ssh_pre_flight() ++ ++ assert ret == ( ++ "", ++ "Could not copy pre flight script to temporary path", ++ 1, ++ ) ++ mock_exec_cmd.assert_not_called() ++ send_mock.assert_not_called() + + + @pytest.mark.skip_on_windows(reason="SSH_PY_SHIM not set on windows") +@@ -355,7 +570,7 @@ def test_cmd_run_set_path(opts, target): + fsclient=None, + thin=salt.utils.thin.thin_path(opts["cachedir"]), + mine=False, +- **target ++ **target, + ) + + ret = single._cmd_str() +@@ -376,7 +591,7 @@ def test_cmd_run_not_set_path(opts, target): + fsclient=None, + thin=salt.utils.thin.thin_path(opts["cachedir"]), + mine=False, +- **target ++ **target, + ) + + ret = single._cmd_str() +@@ -395,7 +610,7 @@ def test_cmd_block_python_version_error(opts, target): + thin=salt.utils.thin.thin_path(opts["cachedir"]), + mine=False, + winrm=False, +- **target ++ **target, + ) + mock_shim = MagicMock( + return_value=(("", "ERROR: Unable to locate appropriate python command\n", 10)) +@@ -434,7 +649,9 @@ def test_run_with_pre_flight_args(opts, target, test_opts, tmp_path): + and script successfully runs + """ + opts["ssh_run_pre_flight"] = True +- target["ssh_pre_flight"] = str(tmp_path / "script.sh") ++ pre_flight_script = tmp_path / "script.sh" ++ pre_flight_script.write_text("") ++ target["ssh_pre_flight"] = str(pre_flight_script) + + if test_opts[0] is not None: + target["ssh_pre_flight_args"] = test_opts[0] +@@ -448,7 +665,7 @@ def test_run_with_pre_flight_args(opts, target, test_opts, tmp_path): + fsclient=None, + thin=salt.utils.thin.thin_path(opts["cachedir"]), + mine=False, +- **target ++ **target, + ) + + cmd_ret = ("Success", "", 0) +@@ -456,14 +673,15 @@ def test_run_with_pre_flight_args(opts, target, test_opts, tmp_path): + mock_exec_cmd = MagicMock(return_value=("", "", 0)) + patch_cmd = patch("salt.client.ssh.Single.cmd_block", mock_cmd) + patch_exec_cmd = patch("salt.client.ssh.shell.Shell.exec_cmd", mock_exec_cmd) +- patch_shell_send = patch("salt.client.ssh.shell.Shell.send", return_value=None) ++ patch_shell_send = patch( ++ "salt.client.ssh.shell.Shell.send", return_value=("", "", 0) ++ ) + patch_os = patch("os.path.exists", side_effect=[True]) + + with patch_os, patch_cmd, patch_exec_cmd, patch_shell_send: +- ret = single.run() +- assert mock_exec_cmd.mock_calls[0].args[ +- 0 +- ] == "/bin/sh '/tmp/script.sh'{}".format(expected_args) ++ single.run() ++ script_args = mock_exec_cmd.mock_calls[0].args[0] ++ assert re.search(r"\/bin\/sh '.[a-z0-9]+", script_args) + + + @pytest.mark.slow_test +diff --git a/tests/pytests/unit/client/ssh/test_ssh.py b/tests/pytests/unit/client/ssh/test_ssh.py +index 377aad9998c..cece16026cf 100644 +--- a/tests/pytests/unit/client/ssh/test_ssh.py ++++ b/tests/pytests/unit/client/ssh/test_ssh.py +@@ -339,3 +339,113 @@ def test_extra_filerefs(tmp_path, opts): + with patch("salt.roster.get_roster_file", MagicMock(return_value=roster)): + ssh_obj = client._prep_ssh(**ssh_opts) + assert ssh_obj.opts.get("extra_filerefs", None) == "salt://foobar" ++ ++ ++def test_key_deploy_permission_denied_scp(tmp_path, opts): ++ """ ++ test "key_deploy" function when ++ permission denied authentication error ++ when attempting to use scp to copy file ++ to target ++ """ ++ host = "localhost" ++ passwd = "password" ++ usr = "ssh-usr" ++ opts["ssh_user"] = usr ++ opts["tgt"] = host ++ ++ ssh_ret = { ++ host: { ++ "stdout": "\rroot@192.168.1.187's password: \n\rroot@192.168.1.187's password: \n\rroot@192.168.1.187's password: \n", ++ "stderr": "Permission denied, please try again.\nPermission denied, please try again.\nroot@192.168.1.187: Permission denied (publickey,gssapi-keyex,gssapi-with-micimport pudb; pu.dbassword).\nscp: Connection closed\n", ++ "retcode": 255, ++ } ++ } ++ key_run_ret = { ++ "localhost": { ++ "jid": "20230922155652279959", ++ "return": "test", ++ "retcode": 0, ++ "id": "test", ++ "fun": "cmd.run", ++ "fun_args": ["echo test"], ++ } ++ } ++ patch_roster_file = patch("salt.roster.get_roster_file", MagicMock(return_value="")) ++ with patch_roster_file: ++ client = ssh.SSH(opts) ++ patch_input = patch("builtins.input", side_effect=["y"]) ++ patch_getpass = patch("getpass.getpass", return_value=["password"]) ++ mock_key_run = MagicMock(return_value=key_run_ret) ++ patch_key_run = patch("salt.client.ssh.SSH._key_deploy_run", mock_key_run) ++ with patch_input, patch_getpass, patch_key_run: ++ ret = client.key_deploy(host, ssh_ret) ++ assert mock_key_run.call_args_list[0][0] == ( ++ host, ++ {"passwd": [passwd], "host": host, "user": usr}, ++ True, ++ ) ++ assert ret == key_run_ret ++ assert mock_key_run.call_count == 1 ++ ++ ++def test_key_deploy_permission_denied_file_scp(tmp_path, opts): ++ """ ++ test "key_deploy" function when permission denied ++ due to not having access to copy the file to the target ++ We do not want to deploy the key, because this is not ++ an authentication to the target error. ++ """ ++ host = "localhost" ++ passwd = "password" ++ usr = "ssh-usr" ++ opts["ssh_user"] = usr ++ opts["tgt"] = host ++ ++ mock_key_run = MagicMock(return_value=False) ++ patch_key_run = patch("salt.client.ssh.SSH._key_deploy_run", mock_key_run) ++ ++ ssh_ret = { ++ "localhost": { ++ "stdout": "", ++ "stderr": 'scp: dest open "/tmp/preflight.sh": Permission denied\nscp: failed to upload file /etc/salt/preflight.sh to /tmp/preflight.sh\n', ++ "retcode": 1, ++ } ++ } ++ patch_roster_file = patch("salt.roster.get_roster_file", MagicMock(return_value="")) ++ with patch_roster_file: ++ client = ssh.SSH(opts) ++ ret = client.key_deploy(host, ssh_ret) ++ assert ret == ssh_ret ++ assert mock_key_run.call_count == 0 ++ ++ ++def test_key_deploy_no_permission_denied(tmp_path, opts): ++ """ ++ test "key_deploy" function when no permission denied ++ is returned ++ """ ++ host = "localhost" ++ passwd = "password" ++ usr = "ssh-usr" ++ opts["ssh_user"] = usr ++ opts["tgt"] = host ++ ++ mock_key_run = MagicMock(return_value=False) ++ patch_key_run = patch("salt.client.ssh.SSH._key_deploy_run", mock_key_run) ++ ssh_ret = { ++ "localhost": { ++ "jid": "20230922161937998385", ++ "return": "test", ++ "retcode": 0, ++ "id": "test", ++ "fun": "cmd.run", ++ "fun_args": ["echo test"], ++ } ++ } ++ patch_roster_file = patch("salt.roster.get_roster_file", MagicMock(return_value="")) ++ with patch_roster_file: ++ client = ssh.SSH(opts) ++ ret = client.key_deploy(host, ssh_ret) ++ assert ret == ssh_ret ++ assert mock_key_run.call_count == 0 +-- +2.42.0 + diff --git a/fix-cve-2024-22231-and-cve-2024-22232-bsc-1219430-bs.patch b/fix-cve-2024-22231-and-cve-2024-22232-bsc-1219430-bs.patch new file mode 100644 index 0000000..ff8ee0a --- /dev/null +++ b/fix-cve-2024-22231-and-cve-2024-22232-bsc-1219430-bs.patch @@ -0,0 +1,544 @@ +From 5710bc3ff3887762182f8326bd74f40d3872a69f Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Thu, 1 Feb 2024 11:50:16 +0000 +Subject: [PATCH] Fix "CVE-2024-22231" and "CVE-2024-22232" + (bsc#1219430, bsc#1219431) (#621) + +* Fix CVE-2024-22231 and CVE-2024-22232 + +* Add changelogs for CVE-2024-22231 and CVE-2024-22232 + +* Fix linter issue + +* Add credit + +* Fix wart in patch + +* Clean up test fixtures + +* Fix test on windows + +* Update changelog file name + +* Fix fileroots tests + +--------- + +Co-authored-by: Daniel A. Wozniak +--- + changelog/565.security.md | 4 + + salt/fileserver/__init__.py | 9 +- + salt/fileserver/roots.py | 26 +++++ + salt/master.py | 15 ++- + tests/pytests/unit/fileserver/test_roots.py | 58 +++++++-- + tests/pytests/unit/test_fileserver.py | 123 ++++++++++++++++++++ + tests/pytests/unit/test_master.py | 33 ++++++ + tests/unit/test_fileserver.py | 79 ------------- + 8 files changed, 250 insertions(+), 97 deletions(-) + create mode 100644 changelog/565.security.md + create mode 100644 tests/pytests/unit/test_fileserver.py + delete mode 100644 tests/unit/test_fileserver.py + +diff --git a/changelog/565.security.md b/changelog/565.security.md +new file mode 100644 +index 00000000000..5d7ec8202ba +--- /dev/null ++++ b/changelog/565.security.md +@@ -0,0 +1,4 @@ ++CVE-2024-22231 Prevent directory traversal when creating syndic cache directory on the master ++CVE-2024-22232 Prevent directory traversal attacks in the master's serve_file method. ++These vulerablities were discovered and reported by: ++Yudi Zhao(Huawei Nebula Security Lab),Chenwei Jiang(Huawei Nebula Security Lab) +diff --git a/salt/fileserver/__init__.py b/salt/fileserver/__init__.py +index 99f12387f91..4eca98d14a4 100644 +--- a/salt/fileserver/__init__.py ++++ b/salt/fileserver/__init__.py +@@ -568,11 +568,6 @@ class Fileserver: + saltenv = salt.utils.stringutils.to_unicode(saltenv) + back = self.backends(back) + kwargs = {} +- fnd = {"path": "", "rel": ""} +- if os.path.isabs(path): +- return fnd +- if "../" in path: +- return fnd + if salt.utils.url.is_escaped(path): + # don't attempt to find URL query arguments in the path + path = salt.utils.url.unescape(path) +@@ -588,6 +583,10 @@ class Fileserver: + args = comp.split("=", 1) + kwargs[args[0]] = args[1] + ++ fnd = {"path": "", "rel": ""} ++ if os.path.isabs(path) or "../" in path: ++ return fnd ++ + if "env" in kwargs: + # "env" is not supported; Use "saltenv". + kwargs.pop("env") +diff --git a/salt/fileserver/roots.py b/salt/fileserver/roots.py +index a02b597c6f8..e2ea92029c3 100644 +--- a/salt/fileserver/roots.py ++++ b/salt/fileserver/roots.py +@@ -27,6 +27,7 @@ import salt.utils.hashutils + import salt.utils.path + import salt.utils.platform + import salt.utils.stringutils ++import salt.utils.verify + import salt.utils.versions + + log = logging.getLogger(__name__) +@@ -98,6 +99,11 @@ def find_file(path, saltenv="base", **kwargs): + if saltenv == "__env__": + root = root.replace("__env__", actual_saltenv) + full = os.path.join(root, path) ++ ++ # Refuse to serve file that is not under the root. ++ if not salt.utils.verify.clean_path(root, full, subdir=True): ++ continue ++ + if os.path.isfile(full) and not salt.fileserver.is_file_ignored(__opts__, full): + fnd["path"] = full + fnd["rel"] = path +@@ -128,6 +134,26 @@ def serve_file(load, fnd): + ret["dest"] = fnd["rel"] + gzip = load.get("gzip", None) + fpath = os.path.normpath(fnd["path"]) ++ ++ actual_saltenv = saltenv = load["saltenv"] ++ if saltenv not in __opts__["file_roots"]: ++ if "__env__" in __opts__["file_roots"]: ++ log.debug( ++ "salt environment '%s' maps to __env__ file_roots directory", saltenv ++ ) ++ saltenv = "__env__" ++ else: ++ return fnd ++ file_in_root = False ++ for root in __opts__["file_roots"][saltenv]: ++ if saltenv == "__env__": ++ root = root.replace("__env__", actual_saltenv) ++ # Refuse to serve file that is not under the root. ++ if salt.utils.verify.clean_path(root, fpath, subdir=True): ++ file_in_root = True ++ if not file_in_root: ++ return ret ++ + with salt.utils.files.fopen(fpath, "rb") as fp_: + fp_.seek(load["loc"]) + data = fp_.read(__opts__["file_buffer_size"]) +diff --git a/salt/master.py b/salt/master.py +index 3d2ba1e29de..425b4121481 100644 +--- a/salt/master.py ++++ b/salt/master.py +@@ -1038,7 +1038,10 @@ class MWorker(salt.utils.process.SignalHandlingProcess): + """ + key = payload["enc"] + load = payload["load"] +- ret = {"aes": self._handle_aes, "clear": self._handle_clear}[key](load) ++ if key == "aes": ++ ret = self._handle_aes(load) ++ else: ++ ret = self._handle_clear(load) + raise salt.ext.tornado.gen.Return(ret) + + def _post_stats(self, start, cmd): +@@ -1213,7 +1216,7 @@ class AESFuncs(TransportMethods): + "_dir_list", + "_symlink_list", + "_file_envs", +- "_ext_nodes", # To keep compatibility with old Salt minion versions ++ "_ext_nodes", # To keep compatibility with old Salt minion versions + ) + + def __init__(self, opts, context=None): +@@ -1746,10 +1749,16 @@ class AESFuncs(TransportMethods): + self.mminion.returners[fstr](load["jid"], load["load"]) + + # Register the syndic ++ ++ # We are creating a path using user suplied input. Use the ++ # clean_path to prevent a directory traversal. ++ root = os.path.join(self.opts["cachedir"], "syndics") + syndic_cache_path = os.path.join( + self.opts["cachedir"], "syndics", load["id"] + ) +- if not os.path.exists(syndic_cache_path): ++ if salt.utils.verify.clean_path( ++ root, syndic_cache_path ++ ) and not os.path.exists(syndic_cache_path): + path_name = os.path.split(syndic_cache_path)[0] + if not os.path.exists(path_name): + os.makedirs(path_name) +diff --git a/tests/pytests/unit/fileserver/test_roots.py b/tests/pytests/unit/fileserver/test_roots.py +index 96bceb0fd3d..c1660280bc5 100644 +--- a/tests/pytests/unit/fileserver/test_roots.py ++++ b/tests/pytests/unit/fileserver/test_roots.py +@@ -5,6 +5,7 @@ + import copy + import pathlib + import shutil ++import sys + import textwrap + + import pytest +@@ -28,14 +29,14 @@ def unicode_dirname(): + return "соль" + + +-@pytest.fixture(autouse=True) ++@pytest.fixture + def testfile(tmp_path): + fp = tmp_path / "testfile" + fp.write_text("This is a testfile") + return fp + + +-@pytest.fixture(autouse=True) ++@pytest.fixture + def tmp_state_tree(tmp_path, testfile, unicode_filename, unicode_dirname): + dirname = tmp_path / "roots_tmp_state_tree" + dirname.mkdir(parents=True, exist_ok=True) +@@ -54,11 +55,15 @@ def tmp_state_tree(tmp_path, testfile, unicode_filename, unicode_dirname): + + + @pytest.fixture +-def configure_loader_modules(tmp_state_tree, temp_salt_master): +- opts = temp_salt_master.config.copy() ++def testfilepath(tmp_state_tree, testfile): ++ return tmp_state_tree / testfile.name ++ ++ ++@pytest.fixture ++def configure_loader_modules(tmp_state_tree, master_opts): + overrides = {"file_roots": {"base": [str(tmp_state_tree)]}} +- opts.update(overrides) +- return {roots: {"__opts__": opts}} ++ master_opts.update(overrides) ++ return {roots: {"__opts__": master_opts}} + + + def test_file_list(unicode_filename): +@@ -75,17 +80,17 @@ def test_find_file(tmp_state_tree): + assert full_path_to_file == ret["path"] + + +-def test_serve_file(testfile): ++def test_serve_file(testfilepath): + with patch.dict(roots.__opts__, {"file_buffer_size": 262144}): + load = { + "saltenv": "base", +- "path": str(testfile), ++ "path": str(testfilepath), + "loc": 0, + } +- fnd = {"path": str(testfile), "rel": "testfile"} ++ fnd = {"path": str(testfilepath), "rel": "testfile"} + ret = roots.serve_file(load, fnd) + +- with salt.utils.files.fopen(str(testfile), "rb") as fp_: ++ with salt.utils.files.fopen(str(testfilepath), "rb") as fp_: + data = fp_.read() + + assert ret == {"data": data, "dest": "testfile"} +@@ -277,3 +282,36 @@ def test_update_mtime_map_unicode_error(tmp_path): + }, + "backend": "roots", + } ++ ++ ++def test_find_file_not_in_root(tmp_state_tree): ++ """ ++ Fileroots should never 'find' a file that is outside of it's root. ++ """ ++ badfile = pathlib.Path(tmp_state_tree).parent / "bar" ++ badfile.write_text("Bad file") ++ badpath = f"../bar" ++ ret = roots.find_file(badpath) ++ assert ret == {"path": "", "rel": ""} ++ badpath = f"{tmp_state_tree / '..' / 'bar'}" ++ ret = roots.find_file(badpath) ++ assert ret == {"path": "", "rel": ""} ++ ++ ++def test_serve_file_not_in_root(tmp_state_tree): ++ """ ++ Fileroots should never 'serve' a file that is outside of it's root. ++ """ ++ badfile = pathlib.Path(tmp_state_tree).parent / "bar" ++ badfile.write_text("Bad file") ++ badpath = f"../bar" ++ load = {"path": "salt://|..\\bar", "saltenv": "base", "loc": 0} ++ fnd = { ++ "path": f"{tmp_state_tree / '..' / 'bar'}", ++ "rel": f"{pathlib.Path('..') / 'bar'}", ++ } ++ ret = roots.serve_file(load, fnd) ++ if "win" in sys.platform: ++ assert ret == {"data": "", "dest": "..\\bar"} ++ else: ++ assert ret == {"data": "", "dest": "../bar"} +diff --git a/tests/pytests/unit/test_fileserver.py b/tests/pytests/unit/test_fileserver.py +new file mode 100644 +index 00000000000..8dd3ea0a27d +--- /dev/null ++++ b/tests/pytests/unit/test_fileserver.py +@@ -0,0 +1,123 @@ ++import datetime ++import os ++import time ++ ++import salt.fileserver ++import salt.utils.files ++ ++ ++def test_diff_with_diffent_keys(): ++ """ ++ Test that different maps are indeed reported different ++ """ ++ map1 = {"file1": 1234} ++ map2 = {"file2": 1234} ++ assert salt.fileserver.diff_mtime_map(map1, map2) is True ++ ++ ++def test_diff_with_diffent_values(): ++ """ ++ Test that different maps are indeed reported different ++ """ ++ map1 = {"file1": 12345} ++ map2 = {"file1": 1234} ++ assert salt.fileserver.diff_mtime_map(map1, map2) is True ++ ++ ++def test_whitelist(): ++ opts = { ++ "fileserver_backend": ["roots", "git", "s3fs", "hgfs", "svn"], ++ "extension_modules": "", ++ } ++ fs = salt.fileserver.Fileserver(opts) ++ assert sorted(fs.servers.whitelist) == sorted( ++ ["git", "gitfs", "hg", "hgfs", "svn", "svnfs", "roots", "s3fs"] ++ ), fs.servers.whitelist ++ ++ ++def test_future_file_list_cache_file_ignored(tmp_path): ++ opts = { ++ "fileserver_backend": ["roots"], ++ "cachedir": tmp_path, ++ "extension_modules": "", ++ } ++ ++ back_cachedir = os.path.join(tmp_path, "file_lists/roots") ++ os.makedirs(os.path.join(back_cachedir)) ++ ++ # Touch a couple files ++ for filename in ("base.p", "foo.txt"): ++ with salt.utils.files.fopen(os.path.join(back_cachedir, filename), "wb") as _f: ++ if filename == "base.p": ++ _f.write(b"\x80") ++ ++ # Set modification time to file list cache file to 1 year in the future ++ now = datetime.datetime.utcnow() ++ future = now + datetime.timedelta(days=365) ++ mod_time = time.mktime(future.timetuple()) ++ os.utime(os.path.join(back_cachedir, "base.p"), (mod_time, mod_time)) ++ ++ list_cache = os.path.join(back_cachedir, "base.p") ++ w_lock = os.path.join(back_cachedir, ".base.w") ++ ret = salt.fileserver.check_file_list_cache(opts, "files", list_cache, w_lock) ++ assert ( ++ ret[1] is True ++ ), "Cache file list cache file is not refreshed when future modification time" ++ ++ ++def test_file_server_url_escape(tmp_path): ++ (tmp_path / "srv").mkdir() ++ (tmp_path / "srv" / "salt").mkdir() ++ (tmp_path / "foo").mkdir() ++ (tmp_path / "foo" / "bar").write_text("Bad file") ++ fileroot = str(tmp_path / "srv" / "salt") ++ badfile = str(tmp_path / "foo" / "bar") ++ opts = { ++ "fileserver_backend": ["roots"], ++ "extension_modules": "", ++ "optimization_order": [ ++ 0, ++ ], ++ "file_roots": { ++ "base": [fileroot], ++ }, ++ "file_ignore_regex": "", ++ "file_ignore_glob": "", ++ } ++ fs = salt.fileserver.Fileserver(opts) ++ ret = fs.find_file( ++ "salt://|..\\..\\..\\foo/bar", ++ "base", ++ ) ++ assert ret == {"path": "", "rel": ""} ++ ++ ++def test_file_server_serve_url_escape(tmp_path): ++ (tmp_path / "srv").mkdir() ++ (tmp_path / "srv" / "salt").mkdir() ++ (tmp_path / "foo").mkdir() ++ (tmp_path / "foo" / "bar").write_text("Bad file") ++ fileroot = str(tmp_path / "srv" / "salt") ++ badfile = str(tmp_path / "foo" / "bar") ++ opts = { ++ "fileserver_backend": ["roots"], ++ "extension_modules": "", ++ "optimization_order": [ ++ 0, ++ ], ++ "file_roots": { ++ "base": [fileroot], ++ }, ++ "file_ignore_regex": "", ++ "file_ignore_glob": "", ++ "file_buffer_size": 2048, ++ } ++ fs = salt.fileserver.Fileserver(opts) ++ ret = fs.serve_file( ++ { ++ "path": "salt://|..\\..\\..\\foo/bar", ++ "saltenv": "base", ++ "loc": 0, ++ } ++ ) ++ assert ret == {"data": "", "dest": ""} +diff --git a/tests/pytests/unit/test_master.py b/tests/pytests/unit/test_master.py +index 98c796912aa..d338307d1f8 100644 +--- a/tests/pytests/unit/test_master.py ++++ b/tests/pytests/unit/test_master.py +@@ -1,3 +1,4 @@ ++import pathlib + import time + + import pytest +@@ -249,3 +250,35 @@ def test_mworker_pass_context(): + loadler_pillars_mock.call_args_list[0][1].get("pack").get("__context__") + == test_context + ) ++ ++ ++def test_syndic_return_cache_dir_creation(encrypted_requests): ++ """master's cachedir for a syndic will be created by AESFuncs._syndic_return method""" ++ cachedir = pathlib.Path(encrypted_requests.opts["cachedir"]) ++ assert not (cachedir / "syndics").exists() ++ encrypted_requests._syndic_return( ++ { ++ "id": "mamajama", ++ "jid": "", ++ "return": {}, ++ } ++ ) ++ assert (cachedir / "syndics").exists() ++ assert (cachedir / "syndics" / "mamajama").exists() ++ ++ ++def test_syndic_return_cache_dir_creation_traversal(encrypted_requests): ++ """ ++ master's AESFuncs._syndic_return method cachdir creation is not vulnerable to a directory traversal ++ """ ++ cachedir = pathlib.Path(encrypted_requests.opts["cachedir"]) ++ assert not (cachedir / "syndics").exists() ++ encrypted_requests._syndic_return( ++ { ++ "id": "../mamajama", ++ "jid": "", ++ "return": {}, ++ } ++ ) ++ assert not (cachedir / "syndics").exists() ++ assert not (cachedir / "mamajama").exists() +diff --git a/tests/unit/test_fileserver.py b/tests/unit/test_fileserver.py +deleted file mode 100644 +index c290b16b7e4..00000000000 +--- a/tests/unit/test_fileserver.py ++++ /dev/null +@@ -1,79 +0,0 @@ +-""" +- :codeauthor: Joao Mesquita +-""" +- +- +-import datetime +-import os +-import time +- +-import salt.utils.files +-from salt import fileserver +-from tests.support.helpers import with_tempdir +-from tests.support.mixins import LoaderModuleMockMixin +-from tests.support.unit import TestCase +- +- +-class MapDiffTestCase(TestCase): +- def test_diff_with_diffent_keys(self): +- """ +- Test that different maps are indeed reported different +- """ +- map1 = {"file1": 1234} +- map2 = {"file2": 1234} +- assert fileserver.diff_mtime_map(map1, map2) is True +- +- def test_diff_with_diffent_values(self): +- """ +- Test that different maps are indeed reported different +- """ +- map1 = {"file1": 12345} +- map2 = {"file1": 1234} +- assert fileserver.diff_mtime_map(map1, map2) is True +- +- +-class VCSBackendWhitelistCase(TestCase, LoaderModuleMockMixin): +- def setup_loader_modules(self): +- return {fileserver: {}} +- +- def test_whitelist(self): +- opts = { +- "fileserver_backend": ["roots", "git", "s3fs", "hgfs", "svn"], +- "extension_modules": "", +- } +- fs = fileserver.Fileserver(opts) +- assert sorted(fs.servers.whitelist) == sorted( +- ["git", "gitfs", "hg", "hgfs", "svn", "svnfs", "roots", "s3fs"] +- ), fs.servers.whitelist +- +- @with_tempdir() +- def test_future_file_list_cache_file_ignored(self, cachedir): +- opts = { +- "fileserver_backend": ["roots"], +- "cachedir": cachedir, +- "extension_modules": "", +- } +- +- back_cachedir = os.path.join(cachedir, "file_lists/roots") +- os.makedirs(os.path.join(back_cachedir)) +- +- # Touch a couple files +- for filename in ("base.p", "foo.txt"): +- with salt.utils.files.fopen( +- os.path.join(back_cachedir, filename), "wb" +- ) as _f: +- if filename == "base.p": +- _f.write(b"\x80") +- +- # Set modification time to file list cache file to 1 year in the future +- now = datetime.datetime.utcnow() +- future = now + datetime.timedelta(days=365) +- mod_time = time.mktime(future.timetuple()) +- os.utime(os.path.join(back_cachedir, "base.p"), (mod_time, mod_time)) +- +- list_cache = os.path.join(back_cachedir, "base.p") +- w_lock = os.path.join(back_cachedir, ".base.w") +- ret = fileserver.check_file_list_cache(opts, "files", list_cache, w_lock) +- assert ( +- ret[1] is True +- ), "Cache file list cache file is not refreshed when future modification time" +-- +2.43.0 + + diff --git a/fix-gitfs-__env__-and-improve-cache-cleaning-bsc-119.patch b/fix-gitfs-__env__-and-improve-cache-cleaning-bsc-119.patch new file mode 100644 index 0000000..4572426 --- /dev/null +++ b/fix-gitfs-__env__-and-improve-cache-cleaning-bsc-119.patch @@ -0,0 +1,2024 @@ +From a7c98ce490833ff232946b9715909161b6ba5a46 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Mon, 13 Nov 2023 11:24:09 +0000 +Subject: [PATCH] Fix gitfs "__env__" and improve cache cleaning + (bsc#1193948) (#608) + +* Fix __env__ and improve cache cleaning +* Move gitfs locks out of cachedir/.git + +--------- + +Co-authored-by: cmcmarrow +--- + changelog/65002.fixed.md | 1 + + changelog/65086.fixed.md | 1 + + salt/utils/cache.py | 32 ++ + salt/utils/gitfs.py | 307 +++++++++------ + .../functional/pillar/test_git_pillar.py | 262 +++++++++++++ + tests/pytests/functional/utils/test_cache.py | 83 ++++ + tests/pytests/functional/utils/test_gitfs.py | 275 +++++++++++++ + tests/pytests/functional/utils/test_pillar.py | 365 ++++++++++++++++++ + .../pytests/functional/utils/test_winrepo.py | 164 ++++++++ + tests/pytests/unit/test_minion.py | 31 +- + tests/pytests/unit/utils/test_gitfs.py | 18 +- + tests/unit/utils/test_gitfs.py | 21 +- + 12 files changed, 1393 insertions(+), 167 deletions(-) + create mode 100644 changelog/65002.fixed.md + create mode 100644 changelog/65086.fixed.md + create mode 100644 tests/pytests/functional/pillar/test_git_pillar.py + create mode 100644 tests/pytests/functional/utils/test_cache.py + create mode 100644 tests/pytests/functional/utils/test_gitfs.py + create mode 100644 tests/pytests/functional/utils/test_pillar.py + create mode 100644 tests/pytests/functional/utils/test_winrepo.py + +diff --git a/changelog/65002.fixed.md b/changelog/65002.fixed.md +new file mode 100644 +index 0000000000..86ed2d4bcc +--- /dev/null ++++ b/changelog/65002.fixed.md +@@ -0,0 +1 @@ ++Fix __env__ and improve cache cleaning see more info at pull #65017. +diff --git a/changelog/65086.fixed.md b/changelog/65086.fixed.md +new file mode 100644 +index 0000000000..292930f0fd +--- /dev/null ++++ b/changelog/65086.fixed.md +@@ -0,0 +1 @@ ++Moved gitfs locks to salt working dir to avoid lock wipes +diff --git a/salt/utils/cache.py b/salt/utils/cache.py +index a78a1f70fc..88e7fa2400 100644 +--- a/salt/utils/cache.py ++++ b/salt/utils/cache.py +@@ -6,6 +6,7 @@ import functools + import logging + import os + import re ++import shutil + import time + + import salt.config +@@ -15,6 +16,8 @@ import salt.utils.data + import salt.utils.dictupdate + import salt.utils.files + import salt.utils.msgpack ++import salt.utils.path ++import salt.version + from salt.utils.zeromq import zmq + + log = logging.getLogger(__name__) +@@ -345,3 +348,32 @@ def context_cache(func): + return func(*args, **kwargs) + + return context_cache_wrap ++ ++ ++def verify_cache_version(cache_path): ++ """ ++ Check that the cached version matches the Salt version. ++ If the cached version does not match the Salt version, wipe the cache. ++ ++ :return: ``True`` if cache version matches, otherwise ``False`` ++ """ ++ if not os.path.isdir(cache_path): ++ os.makedirs(cache_path) ++ with salt.utils.files.fopen( ++ salt.utils.path.join(cache_path, "cache_version"), "a+" ++ ) as file: ++ file.seek(0) ++ data = "\n".join(file.readlines()) ++ if data != salt.version.__version__: ++ log.warning(f"Cache version mismatch clearing: {repr(cache_path)}") ++ file.truncate(0) ++ file.write(salt.version.__version__) ++ for item in os.listdir(cache_path): ++ if item != "cache_version": ++ item_path = salt.utils.path.join(cache_path, item) ++ if os.path.isfile(item_path): ++ os.remove(item_path) ++ else: ++ shutil.rmtree(item_path) ++ return False ++ return True +diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py +index af61aa0dda..061647edac 100644 +--- a/salt/utils/gitfs.py ++++ b/salt/utils/gitfs.py +@@ -17,7 +17,6 @@ import os + import shlex + import shutil + import stat +-import string + import subprocess + import time + import weakref +@@ -26,6 +25,7 @@ from datetime import datetime + import salt.ext.tornado.ioloop + import salt.fileserver + import salt.syspaths ++import salt.utils.cache + import salt.utils.configparser + import salt.utils.data + import salt.utils.files +@@ -253,7 +253,6 @@ class GitProvider: + val_cb=lambda x, y: str(y), + ) + self.conf = copy.deepcopy(per_remote_defaults) +- + # Remove the 'salt://' from the beginning of any globally-defined + # per-saltenv mountpoints + for saltenv, saltenv_conf in self.global_saltenv.items(): +@@ -457,48 +456,38 @@ class GitProvider: + self.id, + ) + failhard(self.role) +- +- hash_type = getattr(hashlib, self.opts.get("hash_type", "md5")) +- # Generate full id. +- # Full id helps decrease the chances of collections in the gitfs cache. +- try: +- target = str(self.get_checkout_target()) +- except AttributeError: +- target = "" +- self._full_id = "-".join( +- [ +- getattr(self, "name", ""), +- self.id, +- getattr(self, "env", ""), +- getattr(self, "_root", ""), +- self.role, +- getattr(self, "base", ""), +- getattr(self, "branch", ""), +- target, +- ] ++ if hasattr(self, "name"): ++ self._cache_basehash = self.name ++ else: ++ hash_type = getattr(hashlib, self.opts.get("hash_type", "md5")) ++ # We loaded this data from yaml configuration files, so, its safe ++ # to use UTF-8 ++ self._cache_basehash = str( ++ base64.b64encode(hash_type(self.id.encode("utf-8")).digest()), ++ encoding="ascii", # base64 only outputs ascii ++ ).replace( ++ "/", "_" ++ ) # replace "/" with "_" to not cause trouble with file system ++ self._cache_hash = salt.utils.path.join(cache_root, self._cache_basehash) ++ self._cache_basename = "_" ++ if self.id.startswith("__env__"): ++ try: ++ self._cache_basename = self.get_checkout_target() ++ except AttributeError: ++ log.critical(f"__env__ cant generate basename: {self.role} {self.id}") ++ failhard(self.role) ++ self._cache_full_basename = salt.utils.path.join( ++ self._cache_basehash, self._cache_basename + ) +- # We loaded this data from yaml configuration files, so, its safe +- # to use UTF-8 +- base64_hash = str( +- base64.b64encode(hash_type(self._full_id.encode("utf-8")).digest()), +- encoding="ascii", # base64 only outputs ascii +- ).replace( +- "/", "_" +- ) # replace "/" with "_" to not cause trouble with file system +- +- # limit name length to 19, so we don't eat up all the path length for windows +- # this is due to pygit2 limitations +- # replace any unknown char with "_" to not cause trouble with file system +- name_chars = string.ascii_letters + string.digits + "-" +- cache_name = "".join( +- c if c in name_chars else "_" for c in getattr(self, "name", "")[:19] ++ self._cachedir = salt.utils.path.join(self._cache_hash, self._cache_basename) ++ self._salt_working_dir = salt.utils.path.join( ++ cache_root, "work", self._cache_full_basename + ) +- +- self.cachedir_basename = f"{cache_name}-{base64_hash}" +- self.cachedir = salt.utils.path.join(cache_root, self.cachedir_basename) +- self.linkdir = salt.utils.path.join(cache_root, "links", self.cachedir_basename) +- if not os.path.isdir(self.cachedir): +- os.makedirs(self.cachedir) ++ self._linkdir = salt.utils.path.join( ++ cache_root, "links", self._cache_full_basename ++ ) ++ if not os.path.isdir(self._cachedir): ++ os.makedirs(self._cachedir) + + try: + self.new = self.init_remote() +@@ -510,12 +499,32 @@ class GitProvider: + msg += " Perhaps git is not available." + log.critical(msg, exc_info=True) + failhard(self.role) ++ self.verify_auth() ++ self.setup_callbacks() ++ if not os.path.isdir(self._salt_working_dir): ++ os.makedirs(self._salt_working_dir) ++ self.fetch_request_check() ++ ++ def get_cache_basehash(self): ++ return self._cache_basehash ++ ++ def get_cache_hash(self): ++ return self._cache_hash + +- def full_id(self): +- return self._full_id ++ def get_cache_basename(self): ++ return self._cache_basename + +- def get_cachedir_basename(self): +- return self.cachedir_basename ++ def get_cache_full_basename(self): ++ return self._cache_full_basename ++ ++ def get_cachedir(self): ++ return self._cachedir ++ ++ def get_linkdir(self): ++ return self._linkdir ++ ++ def get_salt_working_dir(self): ++ return self._salt_working_dir + + def _get_envs_from_ref_paths(self, refs): + """ +@@ -557,7 +566,7 @@ class GitProvider: + return ret + + def _get_lock_file(self, lock_type="update"): +- return salt.utils.path.join(self.gitdir, lock_type + ".lk") ++ return salt.utils.path.join(self._salt_working_dir, lock_type + ".lk") + + @classmethod + def add_conf_overlay(cls, name): +@@ -644,7 +653,7 @@ class GitProvider: + # No need to pass an environment to self.root() here since per-saltenv + # configuration is a gitfs-only feature and check_root() is not used + # for gitfs. +- root_dir = salt.utils.path.join(self.cachedir, self.root()).rstrip(os.sep) ++ root_dir = salt.utils.path.join(self._cachedir, self.root()).rstrip(os.sep) + if os.path.isdir(root_dir): + return root_dir + log.error( +@@ -816,7 +825,7 @@ class GitProvider: + desired_refspecs, + ) + if refspecs != desired_refspecs: +- conf.set_multivar(remote_section, "fetch", self.refspecs) ++ conf.set_multivar(remote_section, "fetch", desired_refspecs) + log.debug( + "Refspecs for %s remote '%s' set to %s", + self.role, +@@ -1069,7 +1078,7 @@ class GitProvider: + """ + raise NotImplementedError() + +- def checkout(self): ++ def checkout(self, fetch_on_fail=True): + """ + This function must be overridden in a sub-class + """ +@@ -1192,6 +1201,21 @@ class GitProvider: + else: + self.url = self.id + ++ def fetch_request_check(self): ++ fetch_request = salt.utils.path.join(self._salt_working_dir, "fetch_request") ++ if os.path.isfile(fetch_request): ++ log.debug(f"Fetch request: {self._salt_working_dir}") ++ try: ++ os.remove(fetch_request) ++ except OSError as exc: ++ log.error( ++ f"Failed to remove Fetch request: {self._salt_working_dir} {exc}", ++ exc_info=True, ++ ) ++ self.fetch() ++ return True ++ return False ++ + @property + def linkdir_walk(self): + """ +@@ -1218,14 +1242,14 @@ class GitProvider: + dirs = [] + self._linkdir_walk.append( + ( +- salt.utils.path.join(self.linkdir, *parts[: idx + 1]), ++ salt.utils.path.join(self._linkdir, *parts[: idx + 1]), + dirs, + [], + ) + ) + try: + # The linkdir itself goes at the beginning +- self._linkdir_walk.insert(0, (self.linkdir, [parts[0]], [])) ++ self._linkdir_walk.insert(0, (self._linkdir, [parts[0]], [])) + except IndexError: + pass + return self._linkdir_walk +@@ -1275,13 +1299,17 @@ class GitPython(GitProvider): + role, + ) + +- def checkout(self): ++ def checkout(self, fetch_on_fail=True): + """ + Checkout the configured branch/tag. We catch an "Exception" class here + instead of a specific exception class because the exceptions raised by + GitPython when running these functions vary in different versions of + GitPython. ++ ++ fetch_on_fail ++ If checkout fails perform a fetch then try to checkout again. + """ ++ self.fetch_request_check() + tgt_ref = self.get_checkout_target() + try: + head_sha = self.repo.rev_parse("HEAD").hexsha +@@ -1345,6 +1373,15 @@ class GitPython(GitProvider): + except Exception: # pylint: disable=broad-except + continue + return self.check_root() ++ if fetch_on_fail: ++ log.debug( ++ "Failed to checkout %s from %s remote '%s': fetch and try again", ++ tgt_ref, ++ self.role, ++ self.id, ++ ) ++ self.fetch() ++ return self.checkout(fetch_on_fail=False) + log.error( + "Failed to checkout %s from %s remote '%s': remote ref does not exist", + tgt_ref, +@@ -1360,16 +1397,16 @@ class GitPython(GitProvider): + initialized by this function. + """ + new = False +- if not os.listdir(self.cachedir): ++ if not os.listdir(self._cachedir): + # Repo cachedir is empty, initialize a new repo there +- self.repo = git.Repo.init(self.cachedir) ++ self.repo = git.Repo.init(self._cachedir) + new = True + else: + # Repo cachedir exists, try to attach + try: +- self.repo = git.Repo(self.cachedir) ++ self.repo = git.Repo(self._cachedir) + except git.exc.InvalidGitRepositoryError: +- log.error(_INVALID_REPO, self.cachedir, self.url, self.role) ++ log.error(_INVALID_REPO, self._cachedir, self.url, self.role) + return new + + self.gitdir = salt.utils.path.join(self.repo.working_dir, ".git") +@@ -1603,10 +1640,14 @@ class Pygit2(GitProvider): + except AttributeError: + return obj.get_object() + +- def checkout(self): ++ def checkout(self, fetch_on_fail=True): + """ + Checkout the configured branch/tag ++ ++ fetch_on_fail ++ If checkout fails perform a fetch then try to checkout again. + """ ++ self.fetch_request_check() + tgt_ref = self.get_checkout_target() + local_ref = "refs/heads/" + tgt_ref + remote_ref = "refs/remotes/origin/" + tgt_ref +@@ -1796,6 +1837,15 @@ class Pygit2(GitProvider): + exc_info=True, + ) + return None ++ if fetch_on_fail: ++ log.debug( ++ "Failed to checkout %s from %s remote '%s': fetch and try again", ++ tgt_ref, ++ self.role, ++ self.id, ++ ) ++ self.fetch() ++ return self.checkout(fetch_on_fail=False) + log.error( + "Failed to checkout %s from %s remote '%s': remote ref does not exist", + tgt_ref, +@@ -1837,16 +1887,16 @@ class Pygit2(GitProvider): + home = os.path.expanduser("~") + pygit2.settings.search_path[pygit2.GIT_CONFIG_LEVEL_GLOBAL] = home + new = False +- if not os.listdir(self.cachedir): ++ if not os.listdir(self._cachedir): + # Repo cachedir is empty, initialize a new repo there +- self.repo = pygit2.init_repository(self.cachedir) ++ self.repo = pygit2.init_repository(self._cachedir) + new = True + else: + # Repo cachedir exists, try to attach + try: +- self.repo = pygit2.Repository(self.cachedir) ++ self.repo = pygit2.Repository(self._cachedir) + except KeyError: +- log.error(_INVALID_REPO, self.cachedir, self.url, self.role) ++ log.error(_INVALID_REPO, self._cachedir, self.url, self.role) + return new + + self.gitdir = salt.utils.path.join(self.repo.workdir, ".git") +@@ -2370,6 +2420,7 @@ class GitBase: + self.file_list_cachedir = salt.utils.path.join( + self.opts["cachedir"], "file_lists", self.role + ) ++ salt.utils.cache.verify_cache_version(self.cache_root) + if init_remotes: + self.init_remotes( + remotes if remotes is not None else [], +@@ -2442,8 +2493,6 @@ class GitBase: + ) + if hasattr(repo_obj, "repo"): + # Sanity check and assign the credential parameter +- repo_obj.verify_auth() +- repo_obj.setup_callbacks() + if self.opts["__role"] == "minion" and repo_obj.new: + # Perform initial fetch on masterless minion + repo_obj.fetch() +@@ -2492,7 +2541,7 @@ class GitBase: + # Don't allow collisions in cachedir naming + cachedir_map = {} + for repo in self.remotes: +- cachedir_map.setdefault(repo.cachedir, []).append(repo.id) ++ cachedir_map.setdefault(repo.get_cachedir(), []).append(repo.id) + + collisions = [x for x in cachedir_map if len(cachedir_map[x]) > 1] + if collisions: +@@ -2509,48 +2558,42 @@ class GitBase: + if any(x.new for x in self.remotes): + self.write_remote_map() + ++ def _remove_cache_dir(self, cache_dir): ++ try: ++ shutil.rmtree(cache_dir) ++ except OSError as exc: ++ log.error( ++ "Unable to remove old %s remote cachedir %s: %s", ++ self.role, ++ cache_dir, ++ exc, ++ ) ++ return False ++ log.debug("%s removed old cachedir %s", self.role, cache_dir) ++ return True ++ ++ def _iter_remote_hashes(self): ++ for item in os.listdir(self.cache_root): ++ if item in ("hash", "refs", "links", "work"): ++ continue ++ if os.path.isdir(salt.utils.path.join(self.cache_root, item)): ++ yield item ++ + def clear_old_remotes(self): + """ + Remove cache directories for remotes no longer configured + """ +- try: +- cachedir_ls = os.listdir(self.cache_root) +- except OSError: +- cachedir_ls = [] +- # Remove actively-used remotes from list +- for repo in self.remotes: +- try: +- cachedir_ls.remove(repo.cachedir_basename) +- except ValueError: +- pass +- to_remove = [] +- for item in cachedir_ls: +- if item in ("hash", "refs"): +- continue +- path = salt.utils.path.join(self.cache_root, item) +- if os.path.isdir(path): +- to_remove.append(path) +- failed = [] +- if to_remove: +- for rdir in to_remove: +- try: +- shutil.rmtree(rdir) +- except OSError as exc: +- log.error( +- "Unable to remove old %s remote cachedir %s: %s", +- self.role, +- rdir, +- exc, +- ) +- failed.append(rdir) +- else: +- log.debug("%s removed old cachedir %s", self.role, rdir) +- for fdir in failed: +- to_remove.remove(fdir) +- ret = bool(to_remove) +- if ret: ++ change = False ++ # Remove all hash dirs not part of this group ++ remote_set = {r.get_cache_basehash() for r in self.remotes} ++ for item in self._iter_remote_hashes(): ++ if item not in remote_set: ++ change = self._remove_cache_dir( ++ salt.utils.path.join(self.cache_root, item) or change ++ ) ++ if not change: + self.write_remote_map() +- return ret ++ return change + + def clear_cache(self): + """ +@@ -2609,6 +2652,27 @@ class GitBase: + name = getattr(repo, "name", None) + if not remotes or (repo.id, name) in remotes or name in remotes: + try: ++ # Find and place fetch_request file for all the other branches for this repo ++ repo_work_hash = os.path.split(repo.get_salt_working_dir())[0] ++ for branch in os.listdir(repo_work_hash): ++ # Don't place fetch request in current branch being updated ++ if branch == repo.get_cache_basename(): ++ continue ++ branch_salt_dir = salt.utils.path.join(repo_work_hash, branch) ++ fetch_path = salt.utils.path.join( ++ branch_salt_dir, "fetch_request" ++ ) ++ if os.path.isdir(branch_salt_dir): ++ try: ++ with salt.utils.files.fopen(fetch_path, "w"): ++ pass ++ except OSError as exc: # pylint: disable=broad-except ++ log.error( ++ f"Failed to make fetch request: {fetch_path} {exc}", ++ exc_info=True, ++ ) ++ else: ++ log.error(f"Failed to make fetch request: {fetch_path}") + if repo.fetch(): + # We can't just use the return value from repo.fetch() + # because the data could still have changed if old +@@ -2863,7 +2927,7 @@ class GitBase: + for repo in self.remotes: + fp_.write( + salt.utils.stringutils.to_str( +- "{} = {}\n".format(repo.cachedir_basename, repo.id) ++ "{} = {}\n".format(repo.get_cache_basehash(), repo.id) + ) + ) + except OSError: +@@ -2871,15 +2935,18 @@ class GitBase: + else: + log.info("Wrote new %s remote map to %s", self.role, remote_map) + +- def do_checkout(self, repo): ++ def do_checkout(self, repo, fetch_on_fail=True): + """ + Common code for git_pillar/winrepo to handle locking and checking out + of a repo. ++ ++ fetch_on_fail ++ If checkout fails perform a fetch then try to checkout again. + """ + time_start = time.time() + while time.time() - time_start <= 5: + try: +- return repo.checkout() ++ return repo.checkout(fetch_on_fail=fetch_on_fail) + except GitLockError as exc: + if exc.errno == errno.EEXIST: + time.sleep(0.1) +@@ -3274,14 +3341,17 @@ class GitPillar(GitBase): + + role = "git_pillar" + +- def checkout(self): ++ def checkout(self, fetch_on_fail=True): + """ + Checkout the targeted branches/tags from the git_pillar remotes ++ ++ fetch_on_fail ++ If checkout fails perform a fetch then try to checkout again. + """ + self.pillar_dirs = OrderedDict() + self.pillar_linked_dirs = [] + for repo in self.remotes: +- cachedir = self.do_checkout(repo) ++ cachedir = self.do_checkout(repo, fetch_on_fail=fetch_on_fail) + if cachedir is not None: + # Figure out which environment this remote should be assigned + if repo.branch == "__env__" and hasattr(repo, "all_saltenvs"): +@@ -3298,8 +3368,8 @@ class GitPillar(GitBase): + env = "base" if tgt == repo.base else tgt + if repo._mountpoint: + if self.link_mountpoint(repo): +- self.pillar_dirs[repo.linkdir] = env +- self.pillar_linked_dirs.append(repo.linkdir) ++ self.pillar_dirs[repo.get_linkdir()] = env ++ self.pillar_linked_dirs.append(repo.get_linkdir()) + else: + self.pillar_dirs[cachedir] = env + +@@ -3308,17 +3378,19 @@ class GitPillar(GitBase): + Ensure that the mountpoint is present in the correct location and + points at the correct path + """ +- lcachelink = salt.utils.path.join(repo.linkdir, repo._mountpoint) +- lcachedest = salt.utils.path.join(repo.cachedir, repo.root()).rstrip(os.sep) ++ lcachelink = salt.utils.path.join(repo.get_linkdir(), repo._mountpoint) ++ lcachedest = salt.utils.path.join(repo.get_cachedir(), repo.root()).rstrip( ++ os.sep ++ ) + wipe_linkdir = False + create_link = False + try: + with repo.gen_lock(lock_type="mountpoint", timeout=10): +- walk_results = list(os.walk(repo.linkdir, followlinks=False)) ++ walk_results = list(os.walk(repo.get_linkdir(), followlinks=False)) + if walk_results != repo.linkdir_walk: + log.debug( + "Results of walking %s differ from expected results", +- repo.linkdir, ++ repo.get_linkdir(), + ) + log.debug("Walk results: %s", walk_results) + log.debug("Expected results: %s", repo.linkdir_walk) +@@ -3379,7 +3451,7 @@ class GitPillar(GitBase): + # Wiping implies that we need to create the link + create_link = True + try: +- shutil.rmtree(repo.linkdir) ++ shutil.rmtree(repo.get_linkdir()) + except OSError: + pass + try: +@@ -3431,6 +3503,9 @@ class GitPillar(GitBase): + class WinRepo(GitBase): + """ + Functionality specific to the winrepo runner ++ ++ fetch_on_fail ++ If checkout fails perform a fetch then try to checkout again. + """ + + role = "winrepo" +@@ -3438,12 +3513,12 @@ class WinRepo(GitBase): + # out the repos. + winrepo_dirs = {} + +- def checkout(self): ++ def checkout(self, fetch_on_fail=True): + """ + Checkout the targeted branches/tags from the winrepo remotes + """ + self.winrepo_dirs = {} + for repo in self.remotes: +- cachedir = self.do_checkout(repo) ++ cachedir = self.do_checkout(repo, fetch_on_fail=fetch_on_fail) + if cachedir is not None: + self.winrepo_dirs[repo.id] = cachedir +diff --git a/tests/pytests/functional/pillar/test_git_pillar.py b/tests/pytests/functional/pillar/test_git_pillar.py +new file mode 100644 +index 0000000000..6fd3dee431 +--- /dev/null ++++ b/tests/pytests/functional/pillar/test_git_pillar.py +@@ -0,0 +1,262 @@ ++import pytest ++ ++from salt.pillar.git_pillar import ext_pillar ++from salt.utils.immutabletypes import ImmutableDict, ImmutableList ++from tests.support.mock import patch ++ ++pytestmark = [ ++ pytest.mark.slow_test, ++] ++ ++ ++try: ++ import git # pylint: disable=unused-import ++ ++ HAS_GITPYTHON = True ++except ImportError: ++ HAS_GITPYTHON = False ++ ++ ++try: ++ import pygit2 # pylint: disable=unused-import ++ ++ HAS_PYGIT2 = True ++except ImportError: ++ HAS_PYGIT2 = False ++ ++ ++skipif_no_gitpython = pytest.mark.skipif(not HAS_GITPYTHON, reason="Missing gitpython") ++skipif_no_pygit2 = pytest.mark.skipif(not HAS_PYGIT2, reason="Missing pygit2") ++ ++ ++@pytest.fixture ++def git_pillar_opts(salt_master, tmp_path): ++ opts = dict(salt_master.config) ++ opts["cachedir"] = str(tmp_path) ++ for key, item in opts.items(): ++ if isinstance(item, ImmutableDict): ++ opts[key] = dict(item) ++ elif isinstance(item, ImmutableList): ++ opts[key] = list(item) ++ return opts ++ ++ ++@pytest.fixture ++def gitpython_pillar_opts(git_pillar_opts): ++ git_pillar_opts["verified_git_pillar_provider"] = "gitpython" ++ return git_pillar_opts ++ ++ ++@pytest.fixture ++def pygit2_pillar_opts(git_pillar_opts): ++ git_pillar_opts["verified_git_pillar_provider"] = "pygit2" ++ return git_pillar_opts ++ ++ ++def _get_ext_pillar(minion, pillar_opts, grains, *repos): ++ with patch("salt.pillar.git_pillar.__opts__", pillar_opts, create=True): ++ with patch("salt.pillar.git_pillar.__grains__", grains, create=True): ++ return ext_pillar(minion, None, *repos) ++ ++ ++def _test_simple(pillar_opts, grains): ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ assert data == {"key": "value"} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_simple(gitpython_pillar_opts, grains): ++ _test_simple(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_simple(pygit2_pillar_opts, grains): ++ _test_simple(pygit2_pillar_opts, grains) ++ ++ ++def _test_missing_env(pillar_opts, grains): ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ { ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git": [ ++ {"env": "misssing"} ++ ] ++ }, ++ ) ++ assert data == {} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_missing_env(gitpython_pillar_opts, grains): ++ _test_missing_env(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_missing_env(pygit2_pillar_opts, grains): ++ _test_missing_env(pygit2_pillar_opts, grains) ++ ++ ++def _test_env(pillar_opts, grains): ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ { ++ "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git": [ ++ {"env": "other_env"} ++ ] ++ }, ++ ) ++ assert data == {"other": "env"} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_env(gitpython_pillar_opts, grains): ++ _test_env(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_env(pygit2_pillar_opts, grains): ++ _test_env(pygit2_pillar_opts, grains) ++ ++ ++def _test_branch(pillar_opts, grains): ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ "branch https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ assert data == {"key": "data"} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_branch(gitpython_pillar_opts, grains): ++ _test_branch(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_branch(pygit2_pillar_opts, grains): ++ _test_branch(pygit2_pillar_opts, grains) ++ ++ ++def _test_simple_dynamic(pillar_opts, grains): ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ assert data == {"key": "value"} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_simple_dynamic(gitpython_pillar_opts, grains): ++ _test_simple_dynamic(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_simple_dynamic(pygit2_pillar_opts, grains): ++ _test_simple_dynamic(pygit2_pillar_opts, grains) ++ ++ ++def _test_missing_env_dynamic(pillar_opts, grains): ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ { ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git": [ ++ {"env": "misssing"} ++ ] ++ }, ++ ) ++ assert data == {} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_missing_env_dynamic(gitpython_pillar_opts, grains): ++ _test_missing_env_dynamic(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_missing_env_dynamic(pygit2_pillar_opts, grains): ++ _test_missing_env_dynamic(pygit2_pillar_opts, grains) ++ ++ ++def _test_pillarenv_dynamic(pillar_opts, grains): ++ pillar_opts["pillarenv"] = "branch" ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ assert data == {"key": "data"} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_pillarenv_dynamic(gitpython_pillar_opts, grains): ++ _test_pillarenv_dynamic(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_pillarenv_dynamic(pygit2_pillar_opts, grains): ++ _test_pillarenv_dynamic(pygit2_pillar_opts, grains) ++ ++ ++def _test_multiple(pillar_opts, grains): ++ pillar_opts["pillarenv"] = "branch" ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ ) ++ assert data == {"key": "data"} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_multiple(gitpython_pillar_opts, grains): ++ _test_multiple(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_multiple(pygit2_pillar_opts, grains): ++ _test_multiple(pygit2_pillar_opts, grains) ++ ++ ++def _test_multiple_2(pillar_opts, grains): ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ ) ++ assert data == { ++ "key": "value", ++ "key1": "value1", ++ "key2": "value2", ++ "key4": "value4", ++ "data1": "d", ++ "data2": "d2", ++ } ++ ++ ++@skipif_no_gitpython ++def test_gitpython_multiple_2(gitpython_pillar_opts, grains): ++ _test_multiple_2(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_multiple_2(pygit2_pillar_opts, grains): ++ _test_multiple_2(pygit2_pillar_opts, grains) +diff --git a/tests/pytests/functional/utils/test_cache.py b/tests/pytests/functional/utils/test_cache.py +new file mode 100644 +index 0000000000..d405b8246f +--- /dev/null ++++ b/tests/pytests/functional/utils/test_cache.py +@@ -0,0 +1,83 @@ ++import os ++ ++import pytest ++ ++import salt.utils.cache ++import salt.utils.files ++import salt.utils.path ++import salt.version ++ ++_DUMMY_FILES = ( ++ "data.txt", ++ "foo.t2", ++ "bar.t3", ++ "nested/test", ++ "nested/cache.txt", ++ "n/n1/n2/n3/n4/n5", ++) ++ ++ ++def _make_dummy_files(tmp_path): ++ for full_path in _DUMMY_FILES: ++ full_path = salt.utils.path.join(tmp_path, full_path) ++ path, _ = os.path.split(full_path) ++ if not os.path.isdir(path): ++ os.makedirs(path) ++ with salt.utils.files.fopen(full_path, "w") as file: ++ file.write("data") ++ ++ ++def _dummy_files_exists(tmp_path): ++ """ ++ True if all files exists ++ False if all files are missing ++ None if some files exists and others are missing ++ """ ++ ret = None ++ for full_path in _DUMMY_FILES: ++ full_path = salt.utils.path.join(tmp_path, full_path) ++ is_file = os.path.isfile(full_path) ++ if ret is None: ++ ret = is_file ++ elif ret is not is_file: ++ return None # Some files are found and others are missing ++ return ret ++ ++ ++def test_verify_cache_version_bad_path(): ++ with pytest.raises(ValueError): ++ # cache version should fail if given bad file python ++ salt.utils.cache.verify_cache_version("\0/bad/path") ++ ++ ++def test_verify_cache_version(tmp_path): ++ # cache version should make dir if it does not exist ++ tmp_path = str(salt.utils.path.join(str(tmp_path), "work", "salt")) ++ cache_version = salt.utils.path.join(tmp_path, "cache_version") ++ ++ # check that cache clears when no cache_version is present ++ _make_dummy_files(tmp_path) ++ assert salt.utils.cache.verify_cache_version(tmp_path) is False ++ assert _dummy_files_exists(tmp_path) is False ++ ++ # check that cache_version has correct salt version ++ with salt.utils.files.fopen(cache_version, "r") as file: ++ assert "\n".join(file.readlines()) == salt.version.__version__ ++ ++ # check that cache does not get clear when check is called multiple times ++ _make_dummy_files(tmp_path) ++ for _ in range(3): ++ assert salt.utils.cache.verify_cache_version(tmp_path) is True ++ assert _dummy_files_exists(tmp_path) is True ++ ++ # check that cache clears when a different version is present ++ with salt.utils.files.fopen(cache_version, "w") as file: ++ file.write("-1") ++ assert salt.utils.cache.verify_cache_version(tmp_path) is False ++ assert _dummy_files_exists(tmp_path) is False ++ ++ # check that cache does not get clear when check is called multiple times ++ _make_dummy_files(tmp_path) ++ for _ in range(3): ++ assert salt.utils.cache.verify_cache_version(tmp_path) is True ++ assert _dummy_files_exists(tmp_path) is True +diff --git a/tests/pytests/functional/utils/test_gitfs.py b/tests/pytests/functional/utils/test_gitfs.py +new file mode 100644 +index 0000000000..30a5f147fa +--- /dev/null ++++ b/tests/pytests/functional/utils/test_gitfs.py +@@ -0,0 +1,275 @@ ++import os.path ++ ++import pytest ++ ++from salt.fileserver.gitfs import PER_REMOTE_ONLY, PER_REMOTE_OVERRIDES ++from salt.utils.gitfs import GitFS, GitPython, Pygit2 ++from salt.utils.immutabletypes import ImmutableDict, ImmutableList ++ ++pytestmark = [ ++ pytest.mark.slow_test, ++] ++ ++ ++try: ++ import git # pylint: disable=unused-import ++ ++ HAS_GITPYTHON = True ++except ImportError: ++ HAS_GITPYTHON = False ++ ++ ++try: ++ import pygit2 # pylint: disable=unused-import ++ ++ HAS_PYGIT2 = True ++except ImportError: ++ HAS_PYGIT2 = False ++ ++ ++skipif_no_gitpython = pytest.mark.skipif(not HAS_GITPYTHON, reason="Missing gitpython") ++skipif_no_pygit2 = pytest.mark.skipif(not HAS_PYGIT2, reason="Missing pygit2") ++ ++ ++@pytest.fixture ++def gitfs_opts(salt_factories, tmp_path): ++ config_defaults = {"cachedir": str(tmp_path)} ++ factory = salt_factories.salt_master_daemon( ++ "gitfs-functional-master", defaults=config_defaults ++ ) ++ config_defaults = dict(factory.config) ++ for key, item in config_defaults.items(): ++ if isinstance(item, ImmutableDict): ++ config_defaults[key] = dict(item) ++ elif isinstance(item, ImmutableList): ++ config_defaults[key] = list(item) ++ return config_defaults ++ ++ ++@pytest.fixture ++def gitpython_gitfs_opts(gitfs_opts): ++ gitfs_opts["verified_gitfs_provider"] = "gitpython" ++ GitFS.instance_map.clear() # wipe instance_map object map for clean run ++ return gitfs_opts ++ ++ ++@pytest.fixture ++def pygit2_gitfs_opts(gitfs_opts): ++ gitfs_opts["verified_gitfs_provider"] = "pygit2" ++ GitFS.instance_map.clear() # wipe instance_map object map for clean run ++ return gitfs_opts ++ ++ ++def _get_gitfs(opts, *remotes): ++ return GitFS( ++ opts, ++ remotes, ++ per_remote_overrides=PER_REMOTE_OVERRIDES, ++ per_remote_only=PER_REMOTE_ONLY, ++ ) ++ ++ ++def _test_gitfs_simple(gitfs_opts): ++ g = _get_gitfs( ++ gitfs_opts, ++ {"https://github.com/saltstack/salt-test-pillar-gitfs.git": [{"name": "bob"}]}, ++ ) ++ g.fetch_remotes() ++ assert len(g.remotes) == 1 ++ assert set(g.file_list({"saltenv": "main"})) == {".gitignore", "README.md"} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_gitfs_simple(gitpython_gitfs_opts): ++ _test_gitfs_simple(gitpython_gitfs_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_gitfs_simple(pygit2_gitfs_opts): ++ _test_gitfs_simple(pygit2_gitfs_opts) ++ ++ ++def _test_gitfs_simple_base(gitfs_opts): ++ g = _get_gitfs( ++ gitfs_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ g.fetch_remotes() ++ assert len(g.remotes) == 1 ++ assert set(g.file_list({"saltenv": "base"})) == { ++ ".gitignore", ++ "README.md", ++ "file.sls", ++ "top.sls", ++ } ++ ++ ++@skipif_no_gitpython ++def test_gitpython_gitfs_simple_base(gitpython_gitfs_opts): ++ _test_gitfs_simple_base(gitpython_gitfs_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_gitfs_simple_base(pygit2_gitfs_opts): ++ _test_gitfs_simple_base(pygit2_gitfs_opts) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_gitfs_provider(gitpython_gitfs_opts): ++ g = _get_gitfs( ++ gitpython_gitfs_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ assert len(g.remotes) == 1 ++ assert g.provider == "gitpython" ++ assert isinstance(g.remotes[0], GitPython) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_gitfs_provider(pygit2_gitfs_opts): ++ g = _get_gitfs( ++ pygit2_gitfs_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ assert len(g.remotes) == 1 ++ assert g.provider == "pygit2" ++ assert isinstance(g.remotes[0], Pygit2) ++ ++ ++def _test_gitfs_minion(gitfs_opts): ++ gitfs_opts["__role"] = "minion" ++ g = _get_gitfs( ++ gitfs_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ g.fetch_remotes() ++ assert len(g.remotes) == 1 ++ assert set(g.file_list({"saltenv": "base"})) == { ++ ".gitignore", ++ "README.md", ++ "file.sls", ++ "top.sls", ++ } ++ assert set(g.file_list({"saltenv": "main"})) == {".gitignore", "README.md"} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_gitfs_minion(gitpython_gitfs_opts): ++ _test_gitfs_minion(gitpython_gitfs_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_gitfs_minion(pygit2_gitfs_opts): ++ _test_gitfs_minion(pygit2_gitfs_opts) ++ ++ ++def _test_fetch_request_with_mountpoint(opts): ++ mpoint = [{"mountpoint": "salt/m"}] ++ p = _get_gitfs( ++ opts, ++ {"https://github.com/saltstack/salt-test-pillar-gitfs.git": mpoint}, ++ ) ++ p.fetch_remotes() ++ assert len(p.remotes) == 1 ++ repo = p.remotes[0] ++ assert repo.mountpoint("testmount") == "salt/m" ++ assert set(p.file_list({"saltenv": "testmount"})) == { ++ "salt/m/test_dir1/testfile3", ++ "salt/m/test_dir1/test_dir2/testfile2", ++ "salt/m/.gitignore", ++ "salt/m/README.md", ++ "salt/m/test_dir1/test_dir2/testfile1", ++ } ++ ++ ++@skipif_no_gitpython ++def test_gitpython_fetch_request_with_mountpoint(gitpython_gitfs_opts): ++ _test_fetch_request_with_mountpoint(gitpython_gitfs_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_fetch_request_with_mountpoint(pygit2_gitfs_opts): ++ _test_fetch_request_with_mountpoint(pygit2_gitfs_opts) ++ ++ ++def _test_name(opts): ++ p = _get_gitfs( ++ opts, ++ { ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git": [ ++ {"name": "name1"} ++ ] ++ }, ++ { ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git": [ ++ {"name": "name2"} ++ ] ++ }, ++ ) ++ p.fetch_remotes() ++ assert len(p.remotes) == 2 ++ repo = p.remotes[0] ++ repo2 = p.remotes[1] ++ assert repo.get_cache_basehash() == "name1" ++ assert repo2.get_cache_basehash() == "name2" ++ ++ ++@skipif_no_gitpython ++def test_gitpython_name(gitpython_gitfs_opts): ++ _test_name(gitpython_gitfs_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_name(pygit2_gitfs_opts): ++ _test_name(pygit2_gitfs_opts) ++ ++ ++def _test_remote_map(opts): ++ p = _get_gitfs( ++ opts, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ p.fetch_remotes() ++ assert len(p.remotes) == 1 ++ assert os.path.isfile(os.path.join(opts["cachedir"], "gitfs", "remote_map.txt")) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_remote_map(gitpython_gitfs_opts): ++ _test_remote_map(gitpython_gitfs_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_remote_map(pygit2_gitfs_opts): ++ _test_remote_map(pygit2_gitfs_opts) ++ ++ ++def _test_lock(opts): ++ g = _get_gitfs( ++ opts, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ g.fetch_remotes() ++ assert len(g.remotes) == 1 ++ repo = g.remotes[0] ++ assert repo.get_salt_working_dir() in repo._get_lock_file() ++ assert repo.lock() == ( ++ [ ++ "Set update lock for gitfs remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" ++ ], ++ [], ++ ) ++ assert os.path.isfile(repo._get_lock_file()) ++ assert repo.clear_lock() == ( ++ [ ++ "Removed update lock for gitfs remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" ++ ], ++ [], ++ ) ++ assert not os.path.isfile(repo._get_lock_file()) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_lock(gitpython_gitfs_opts): ++ _test_lock(gitpython_gitfs_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_lock(pygit2_gitfs_opts): ++ _test_lock(pygit2_gitfs_opts) +diff --git a/tests/pytests/functional/utils/test_pillar.py b/tests/pytests/functional/utils/test_pillar.py +new file mode 100644 +index 0000000000..143edbf6ff +--- /dev/null ++++ b/tests/pytests/functional/utils/test_pillar.py +@@ -0,0 +1,365 @@ ++import os ++ ++import pytest ++ ++from salt.pillar.git_pillar import GLOBAL_ONLY, PER_REMOTE_ONLY, PER_REMOTE_OVERRIDES ++from salt.utils.gitfs import GitPillar, GitPython, Pygit2 ++from salt.utils.immutabletypes import ImmutableDict, ImmutableList ++ ++pytestmark = [ ++ pytest.mark.slow_test, ++] ++ ++ ++try: ++ import git # pylint: disable=unused-import ++ ++ HAS_GITPYTHON = True ++except ImportError: ++ HAS_GITPYTHON = False ++ ++ ++try: ++ import pygit2 # pylint: disable=unused-import ++ ++ HAS_PYGIT2 = True ++except ImportError: ++ HAS_PYGIT2 = False ++ ++ ++skipif_no_gitpython = pytest.mark.skipif(not HAS_GITPYTHON, reason="Missing gitpython") ++skipif_no_pygit2 = pytest.mark.skipif(not HAS_PYGIT2, reason="Missing pygit2") ++ ++ ++@pytest.fixture ++def pillar_opts(salt_factories, tmp_path): ++ config_defaults = {"cachedir": str(tmp_path)} ++ factory = salt_factories.salt_master_daemon( ++ "pillar-functional-master", defaults=config_defaults ++ ) ++ config_defaults = dict(factory.config) ++ for key, item in config_defaults.items(): ++ if isinstance(item, ImmutableDict): ++ config_defaults[key] = dict(item) ++ elif isinstance(item, ImmutableList): ++ config_defaults[key] = list(item) ++ return config_defaults ++ ++ ++@pytest.fixture ++def gitpython_pillar_opts(pillar_opts): ++ pillar_opts["verified_git_pillar_provider"] = "gitpython" ++ return pillar_opts ++ ++ ++@pytest.fixture ++def pygit2_pillar_opts(pillar_opts): ++ pillar_opts["verified_git_pillar_provider"] = "pygit2" ++ return pillar_opts ++ ++ ++def _get_pillar(opts, *remotes): ++ return GitPillar( ++ opts, ++ remotes, ++ per_remote_overrides=PER_REMOTE_OVERRIDES, ++ per_remote_only=PER_REMOTE_ONLY, ++ global_only=GLOBAL_ONLY, ++ ) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_pillar_provider(gitpython_pillar_opts): ++ p = _get_pillar( ++ gitpython_pillar_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ assert len(p.remotes) == 1 ++ assert p.provider == "gitpython" ++ assert isinstance(p.remotes[0], GitPython) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_pillar_provider(pygit2_pillar_opts): ++ p = _get_pillar( ++ pygit2_pillar_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ assert len(p.remotes) == 1 ++ assert p.provider == "pygit2" ++ assert isinstance(p.remotes[0], Pygit2) ++ ++ ++def _test_env(opts): ++ p = _get_pillar( ++ opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ assert len(p.remotes) == 1 ++ p.checkout() ++ repo = p.remotes[0] ++ # test that two different pillarenvs can exist at the same time ++ files = set(os.listdir(repo.get_cachedir())) ++ for f in (".gitignore", "README.md", "file.sls", "top.sls"): ++ assert f in files ++ opts["pillarenv"] = "main" ++ p2 = _get_pillar( ++ opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ assert len(p.remotes) == 1 ++ p2.checkout() ++ repo2 = p2.remotes[0] ++ files = set(os.listdir(repo2.get_cachedir())) ++ for f in (".gitignore", "README.md"): ++ assert f in files ++ for f in ("file.sls", "top.sls", "back.sls", "rooms.sls"): ++ assert f not in files ++ assert repo.get_cachedir() != repo2.get_cachedir() ++ files = set(os.listdir(repo.get_cachedir())) ++ for f in (".gitignore", "README.md", "file.sls", "top.sls"): ++ assert f in files ++ ++ # double check cache paths ++ assert ( ++ repo.get_cache_hash() == repo2.get_cache_hash() ++ ) # __env__ repos share same hash ++ assert repo.get_cache_basename() != repo2.get_cache_basename() ++ assert repo.get_linkdir() != repo2.get_linkdir() ++ assert repo.get_salt_working_dir() != repo2.get_salt_working_dir() ++ assert repo.get_cache_basename() == "master" ++ assert repo2.get_cache_basename() == "main" ++ ++ assert repo.get_cache_basename() in repo.get_cachedir() ++ assert ( ++ os.path.join(repo.get_cache_basehash(), repo.get_cache_basename()) ++ == repo.get_cache_full_basename() ++ ) ++ assert repo.get_linkdir() not in repo.get_cachedir() ++ assert repo.get_salt_working_dir() not in repo.get_cachedir() ++ ++ ++@skipif_no_gitpython ++def test_gitpython_env(gitpython_pillar_opts): ++ _test_env(gitpython_pillar_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_env(pygit2_pillar_opts): ++ _test_env(pygit2_pillar_opts) ++ ++ ++def _test_checkout_fetch_on_fail(opts): ++ p = _get_pillar(opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git") ++ p.checkout(fetch_on_fail=False) # TODO write me ++ ++ ++@skipif_no_gitpython ++def test_gitpython_checkout_fetch_on_fail(gitpython_pillar_opts): ++ _test_checkout_fetch_on_fail(gitpython_pillar_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_checkout_fetch_on_fail(pygit2_pillar_opts): ++ _test_checkout_fetch_on_fail(pygit2_pillar_opts) ++ ++ ++def _test_multiple_repos(opts): ++ p = _get_pillar( ++ opts, ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "main https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "branch https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ ) ++ p.checkout() ++ assert len(p.remotes) == 5 ++ # make sure all repos dont share cache and working dir ++ assert len({r.get_cachedir() for r in p.remotes}) == 5 ++ assert len({r.get_salt_working_dir() for r in p.remotes}) == 5 ++ ++ p2 = _get_pillar( ++ opts, ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "main https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "branch https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ ) ++ p2.checkout() ++ assert len(p2.remotes) == 5 ++ # make sure that repos are given same cache dir ++ for repo, repo2 in zip(p.remotes, p2.remotes): ++ assert repo.get_cachedir() == repo2.get_cachedir() ++ assert repo.get_salt_working_dir() == repo2.get_salt_working_dir() ++ opts["pillarenv"] = "main" ++ p3 = _get_pillar( ++ opts, ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "main https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "branch https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ ) ++ p3.checkout() ++ # check that __env__ has different cache with different pillarenv ++ assert p.remotes[0].get_cachedir() != p3.remotes[0].get_cachedir() ++ assert p.remotes[1].get_cachedir() == p3.remotes[1].get_cachedir() ++ assert p.remotes[2].get_cachedir() == p3.remotes[2].get_cachedir() ++ assert p.remotes[3].get_cachedir() != p3.remotes[3].get_cachedir() ++ assert p.remotes[4].get_cachedir() == p3.remotes[4].get_cachedir() ++ ++ # check that other branch data is in cache ++ files = set(os.listdir(p.remotes[4].get_cachedir())) ++ for f in (".gitignore", "README.md", "file.sls", "top.sls", "other_env.sls"): ++ assert f in files ++ ++ ++@skipif_no_gitpython ++def test_gitpython_multiple_repos(gitpython_pillar_opts): ++ _test_multiple_repos(gitpython_pillar_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_multiple_repos(pygit2_pillar_opts): ++ _test_multiple_repos(pygit2_pillar_opts) ++ ++ ++def _test_fetch_request(opts): ++ p = _get_pillar( ++ opts, ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ ) ++ frequest = os.path.join(p.remotes[0].get_salt_working_dir(), "fetch_request") ++ frequest_other = os.path.join(p.remotes[1].get_salt_working_dir(), "fetch_request") ++ opts["pillarenv"] = "main" ++ p2 = _get_pillar( ++ opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ frequest2 = os.path.join(p2.remotes[0].get_salt_working_dir(), "fetch_request") ++ assert frequest != frequest2 ++ assert os.path.isfile(frequest) is False ++ assert os.path.isfile(frequest2) is False ++ assert os.path.isfile(frequest_other) is False ++ p.fetch_remotes() ++ assert os.path.isfile(frequest) is False ++ # fetch request was placed ++ assert os.path.isfile(frequest2) is True ++ p2.checkout() ++ # fetch request was found ++ assert os.path.isfile(frequest2) is False ++ p2.fetch_remotes() ++ assert os.path.isfile(frequest) is True ++ assert os.path.isfile(frequest2) is False ++ assert os.path.isfile(frequest_other) is False ++ for _ in range(3): ++ p2.fetch_remotes() ++ assert os.path.isfile(frequest) is True ++ assert os.path.isfile(frequest2) is False ++ assert os.path.isfile(frequest_other) is False ++ # fetch request should still be processed even on fetch_on_fail=False ++ p.checkout(fetch_on_fail=False) ++ assert os.path.isfile(frequest) is False ++ assert os.path.isfile(frequest2) is False ++ assert os.path.isfile(frequest_other) is False ++ ++ ++@skipif_no_gitpython ++def test_gitpython_fetch_request(gitpython_pillar_opts): ++ _test_fetch_request(gitpython_pillar_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_fetch_request(pygit2_pillar_opts): ++ _test_fetch_request(pygit2_pillar_opts) ++ ++ ++def _test_clear_old_remotes(opts): ++ p = _get_pillar( ++ opts, ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ ) ++ repo = p.remotes[0] ++ repo2 = p.remotes[1] ++ opts["pillarenv"] = "main" ++ p2 = _get_pillar( ++ opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ repo3 = p2.remotes[0] ++ assert os.path.isdir(repo.get_cachedir()) is True ++ assert os.path.isdir(repo2.get_cachedir()) is True ++ assert os.path.isdir(repo3.get_cachedir()) is True ++ p.clear_old_remotes() ++ assert os.path.isdir(repo.get_cachedir()) is True ++ assert os.path.isdir(repo2.get_cachedir()) is True ++ assert os.path.isdir(repo3.get_cachedir()) is True ++ p2.clear_old_remotes() ++ assert os.path.isdir(repo.get_cachedir()) is True ++ assert os.path.isdir(repo2.get_cachedir()) is False ++ assert os.path.isdir(repo3.get_cachedir()) is True ++ ++ ++@skipif_no_gitpython ++def test_gitpython_clear_old_remotes(gitpython_pillar_opts): ++ _test_clear_old_remotes(gitpython_pillar_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_clear_old_remotes(pygit2_pillar_opts): ++ _test_clear_old_remotes(pygit2_pillar_opts) ++ ++ ++def _test_remote_map(opts): ++ p = _get_pillar( ++ opts, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ p.fetch_remotes() ++ assert len(p.remotes) == 1 ++ assert os.path.isfile( ++ os.path.join(opts["cachedir"], "git_pillar", "remote_map.txt") ++ ) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_remote_map(gitpython_pillar_opts): ++ _test_remote_map(gitpython_pillar_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_remote_map(pygit2_pillar_opts): ++ _test_remote_map(pygit2_pillar_opts) ++ ++ ++def _test_lock(opts): ++ p = _get_pillar( ++ opts, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ p.fetch_remotes() ++ assert len(p.remotes) == 1 ++ repo = p.remotes[0] ++ assert repo.get_salt_working_dir() in repo._get_lock_file() ++ assert repo.lock() == ( ++ [ ++ "Set update lock for git_pillar remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" ++ ], ++ [], ++ ) ++ assert os.path.isfile(repo._get_lock_file()) ++ assert repo.clear_lock() == ( ++ [ ++ "Removed update lock for git_pillar remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" ++ ], ++ [], ++ ) ++ assert not os.path.isfile(repo._get_lock_file()) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_lock(gitpython_pillar_opts): ++ _test_lock(gitpython_pillar_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_lock(pygit2_pillar_opts): ++ _test_lock(pygit2_pillar_opts) +diff --git a/tests/pytests/functional/utils/test_winrepo.py b/tests/pytests/functional/utils/test_winrepo.py +new file mode 100644 +index 0000000000..117d995bba +--- /dev/null ++++ b/tests/pytests/functional/utils/test_winrepo.py +@@ -0,0 +1,164 @@ ++import os ++ ++import pytest ++ ++from salt.runners.winrepo import GLOBAL_ONLY, PER_REMOTE_ONLY, PER_REMOTE_OVERRIDES ++from salt.utils.gitfs import GitPython, Pygit2, WinRepo ++from salt.utils.immutabletypes import ImmutableDict, ImmutableList ++ ++pytestmark = [ ++ pytest.mark.slow_test, ++] ++ ++ ++try: ++ import git # pylint: disable=unused-import ++ ++ HAS_GITPYTHON = True ++except ImportError: ++ HAS_GITPYTHON = False ++ ++ ++try: ++ import pygit2 # pylint: disable=unused-import ++ ++ HAS_PYGIT2 = True ++except ImportError: ++ HAS_PYGIT2 = False ++ ++ ++skipif_no_gitpython = pytest.mark.skipif(not HAS_GITPYTHON, reason="Missing gitpython") ++skipif_no_pygit2 = pytest.mark.skipif(not HAS_PYGIT2, reason="Missing pygit2") ++ ++ ++@pytest.fixture ++def winrepo_opts(salt_factories, tmp_path): ++ config_defaults = {"cachedir": str(tmp_path)} ++ factory = salt_factories.salt_master_daemon( ++ "winrepo-functional-master", defaults=config_defaults ++ ) ++ config_defaults = dict(factory.config) ++ for key, item in config_defaults.items(): ++ if isinstance(item, ImmutableDict): ++ config_defaults[key] = dict(item) ++ elif isinstance(item, ImmutableList): ++ config_defaults[key] = list(item) ++ return config_defaults ++ ++ ++@pytest.fixture ++def gitpython_winrepo_opts(winrepo_opts): ++ winrepo_opts["verified_winrepo_provider"] = "gitpython" ++ return winrepo_opts ++ ++ ++@pytest.fixture ++def pygit2_winrepo_opts(winrepo_opts): ++ winrepo_opts["verified_winrepo_provider"] = "pygit2" ++ return winrepo_opts ++ ++ ++def _get_winrepo(opts, *remotes): ++ return WinRepo( ++ opts, ++ remotes, ++ per_remote_overrides=PER_REMOTE_OVERRIDES, ++ per_remote_only=PER_REMOTE_ONLY, ++ global_only=GLOBAL_ONLY, ++ ) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_winrepo_provider(gitpython_winrepo_opts): ++ w = _get_winrepo( ++ gitpython_winrepo_opts, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ assert len(w.remotes) == 1 ++ assert w.provider == "gitpython" ++ assert isinstance(w.remotes[0], GitPython) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_winrepo_provider(pygit2_winrepo_opts): ++ w = _get_winrepo( ++ pygit2_winrepo_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ assert len(w.remotes) == 1 ++ assert w.provider == "pygit2" ++ assert isinstance(w.remotes[0], Pygit2) ++ ++ ++def _test_winrepo_simple(opts): ++ w = _get_winrepo(opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git") ++ assert len(w.remotes) == 1 ++ w.checkout() ++ repo = w.remotes[0] ++ files = set(os.listdir(repo.get_cachedir())) ++ for f in (".gitignore", "README.md", "file.sls", "top.sls"): ++ assert f in files ++ ++ ++@skipif_no_gitpython ++def test_gitpython_winrepo_simple(gitpython_winrepo_opts): ++ _test_winrepo_simple(gitpython_winrepo_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_winrepo_simple(pygit2_winrepo_opts): ++ _test_winrepo_simple(pygit2_winrepo_opts) ++ ++ ++def _test_remote_map(opts): ++ p = _get_winrepo( ++ opts, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ p.fetch_remotes() ++ assert len(p.remotes) == 1 ++ assert os.path.isfile(os.path.join(opts["cachedir"], "winrepo", "remote_map.txt")) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_remote_map(gitpython_winrepo_opts): ++ _test_remote_map(gitpython_winrepo_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_remote_map(pygit2_winrepo_opts): ++ _test_remote_map(pygit2_winrepo_opts) ++ ++ ++def _test_lock(opts): ++ w = _get_winrepo( ++ opts, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ w.fetch_remotes() ++ assert len(w.remotes) == 1 ++ repo = w.remotes[0] ++ assert repo.get_salt_working_dir() in repo._get_lock_file() ++ assert repo.lock() == ( ++ [ ++ "Set update lock for winrepo remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" ++ ], ++ [], ++ ) ++ assert os.path.isfile(repo._get_lock_file()) ++ assert repo.clear_lock() == ( ++ [ ++ "Removed update lock for winrepo remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" ++ ], ++ [], ++ ) ++ assert not os.path.isfile(repo._get_lock_file()) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_lock(gitpython_winrepo_opts): ++ _test_lock(gitpython_winrepo_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_lock(pygit2_winrepo_opts): ++ _test_lock(pygit2_winrepo_opts) +diff --git a/tests/pytests/unit/test_minion.py b/tests/pytests/unit/test_minion.py +index 4508eaee95..740743194e 100644 +--- a/tests/pytests/unit/test_minion.py ++++ b/tests/pytests/unit/test_minion.py +@@ -21,35 +21,33 @@ from tests.support.mock import MagicMock, patch + log = logging.getLogger(__name__) + + +-def test_minion_load_grains_false(): ++def test_minion_load_grains_false(minion_opts): + """ + Minion does not generate grains when load_grains is False + """ +- opts = {"random_startup_delay": 0, "grains": {"foo": "bar"}} ++ minion_opts["grains"] = {"foo": "bar"} + with patch("salt.loader.grains") as grainsfunc: +- minion = salt.minion.Minion(opts, load_grains=False) +- assert minion.opts["grains"] == opts["grains"] ++ minion = salt.minion.Minion(minion_opts, load_grains=False) ++ assert minion.opts["grains"] == minion_opts["grains"] + grainsfunc.assert_not_called() + + +-def test_minion_load_grains_true(): ++def test_minion_load_grains_true(minion_opts): + """ + Minion generates grains when load_grains is True + """ +- opts = {"random_startup_delay": 0, "grains": {}} + with patch("salt.loader.grains") as grainsfunc: +- minion = salt.minion.Minion(opts, load_grains=True) ++ minion = salt.minion.Minion(minion_opts, load_grains=True) + assert minion.opts["grains"] != {} + grainsfunc.assert_called() + + +-def test_minion_load_grains_default(): ++def test_minion_load_grains_default(minion_opts): + """ + Minion load_grains defaults to True + """ +- opts = {"random_startup_delay": 0, "grains": {}} + with patch("salt.loader.grains") as grainsfunc: +- minion = salt.minion.Minion(opts) ++ minion = salt.minion.Minion(minion_opts) + assert minion.opts["grains"] != {} + grainsfunc.assert_called() + +@@ -91,24 +89,17 @@ def test_send_req_tries(req_channel, minion_opts): + + assert rtn == 30 + +- +-@patch("salt.channel.client.ReqChannel.factory") +-def test_mine_send_tries(req_channel_factory): ++def test_mine_send_tries(minion_opts): + channel_enter = MagicMock() + channel_enter.send.side_effect = lambda load, timeout, tries: tries + channel = MagicMock() + channel.__enter__.return_value = channel_enter + +- opts = { +- "random_startup_delay": 0, +- "grains": {}, +- "return_retry_tries": 20, +- "minion_sign_messages": False, +- } ++ minion_opts["return_retry_tries"] = 20 + with patch("salt.channel.client.ReqChannel.factory", return_value=channel), patch( + "salt.loader.grains" + ): +- minion = salt.minion.Minion(opts) ++ minion = salt.minion.Minion(minion_opts) + minion.tok = "token" + + data = {} +diff --git a/tests/pytests/unit/utils/test_gitfs.py b/tests/pytests/unit/utils/test_gitfs.py +index e9915de412..2bf627049f 100644 +--- a/tests/pytests/unit/utils/test_gitfs.py ++++ b/tests/pytests/unit/utils/test_gitfs.py +@@ -1,5 +1,4 @@ + import os +-import string + import time + + import pytest +@@ -214,11 +213,11 @@ def test_checkout_pygit2(_prepare_provider): + provider.init_remote() + provider.fetch() + provider.branch = "master" +- assert provider.cachedir in provider.checkout() ++ assert provider.get_cachedir() in provider.checkout() + provider.branch = "simple_tag" +- assert provider.cachedir in provider.checkout() ++ assert provider.get_cachedir() in provider.checkout() + provider.branch = "annotated_tag" +- assert provider.cachedir in provider.checkout() ++ assert provider.get_cachedir() in provider.checkout() + provider.branch = "does_not_exist" + assert provider.checkout() is None + +@@ -238,18 +237,9 @@ def test_checkout_pygit2_with_home_env_unset(_prepare_provider): + assert "HOME" in os.environ + + +-def test_full_id_pygit2(_prepare_provider): +- assert _prepare_provider.full_id().startswith("-") +- assert _prepare_provider.full_id().endswith("/pygit2-repo---gitfs-master--") +- +- + @pytest.mark.skipif(not HAS_PYGIT2, reason="This host lacks proper pygit2 support") + @pytest.mark.skip_on_windows( + reason="Skip Pygit2 on windows, due to pygit2 access error on windows" + ) + def test_get_cachedir_basename_pygit2(_prepare_provider): +- basename = _prepare_provider.get_cachedir_basename() +- assert len(basename) == 45 +- assert basename[0] == "-" +- # check that a valid base64 is given '/' -> '_' +- assert all(c in string.ascii_letters + string.digits + "+_=" for c in basename[1:]) ++ assert "_" == _prepare_provider.get_cache_basename() +diff --git a/tests/unit/utils/test_gitfs.py b/tests/unit/utils/test_gitfs.py +index 6d8e97a239..259ea056fc 100644 +--- a/tests/unit/utils/test_gitfs.py ++++ b/tests/unit/utils/test_gitfs.py +@@ -114,27 +114,14 @@ class TestGitBase(TestCase, AdaptedConfigurationTestCaseMixin): + self.assertTrue(self.main_class.remotes[0].fetched) + self.assertFalse(self.main_class.remotes[1].fetched) + +- def test_full_id(self): +- self.assertEqual( +- self.main_class.remotes[0].full_id(), "-file://repo1.git---gitfs-master--" +- ) +- +- def test_full_id_with_name(self): +- self.assertEqual( +- self.main_class.remotes[1].full_id(), +- "repo2-file://repo2.git---gitfs-master--", +- ) +- + def test_get_cachedir_basename(self): + self.assertEqual( +- self.main_class.remotes[0].get_cachedir_basename(), +- "-jXhnbGDemchtZwTwaD2s6VOaVvs98a7w+AtiYlmOVb0=", ++ self.main_class.remotes[0].get_cache_basename(), ++ "_", + ) +- +- def test_get_cachedir_base_with_name(self): + self.assertEqual( +- self.main_class.remotes[1].get_cachedir_basename(), +- "repo2-nuezpiDtjQRFC0ZJDByvi+F6Vb8ZhfoH41n_KFxTGsU=", ++ self.main_class.remotes[1].get_cache_basename(), ++ "_", + ) + + def test_git_provider_mp_lock(self): +-- +2.42.0 + + diff --git a/fix-optimization_order-opt-to-prevent-test-fails.patch b/fix-optimization_order-opt-to-prevent-test-fails.patch new file mode 100644 index 0000000..a3b99bc --- /dev/null +++ b/fix-optimization_order-opt-to-prevent-test-fails.patch @@ -0,0 +1,62 @@ +From aaf593d17f51a517e0adb6e9ec1c0d768ab5f855 Mon Sep 17 00:00:00 2001 +From: Victor Zhestkov +Date: Mon, 2 Oct 2023 14:24:27 +0200 +Subject: [PATCH] Fix optimization_order opt to prevent test fails + +--- + tests/pytests/unit/grains/test_core.py | 4 ++-- + tests/pytests/unit/loader/test_loader.py | 2 +- + tests/pytests/unit/test_config.py | 2 +- + 3 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/tests/pytests/unit/grains/test_core.py b/tests/pytests/unit/grains/test_core.py +index 993c723950..36545287b9 100644 +--- a/tests/pytests/unit/grains/test_core.py ++++ b/tests/pytests/unit/grains/test_core.py +@@ -156,7 +156,7 @@ def test_network_grains_secondary_ip(tmp_path): + opts = { + "cachedir": str(cache_dir), + "extension_modules": str(extmods), +- "optimization_order": [0], ++ "optimization_order": [0, 1, 2], + } + with patch("salt.utils.network.interfaces", side_effect=[data]): + grains = salt.loader.grain_funcs(opts) +@@ -243,7 +243,7 @@ def test_network_grains_cache(tmp_path): + opts = { + "cachedir": str(cache_dir), + "extension_modules": str(extmods), +- "optimization_order": [0], ++ "optimization_order": [0, 1, 2], + } + with patch( + "salt.utils.network.interfaces", side_effect=[call_1, call_2] +diff --git a/tests/pytests/unit/loader/test_loader.py b/tests/pytests/unit/loader/test_loader.py +index f4a4b51a58..86348749db 100644 +--- a/tests/pytests/unit/loader/test_loader.py ++++ b/tests/pytests/unit/loader/test_loader.py +@@ -57,7 +57,7 @@ def test_raw_mod_functions(): + "Ensure functions loaded by raw_mod are LoaderFunc instances" + opts = { + "extension_modules": "", +- "optimization_order": [0], ++ "optimization_order": [0, 1, 2], + } + ret = salt.loader.raw_mod(opts, "grains", "get") + for k, v in ret.items(): +diff --git a/tests/pytests/unit/test_config.py b/tests/pytests/unit/test_config.py +index cb343cb75e..76d5605360 100644 +--- a/tests/pytests/unit/test_config.py ++++ b/tests/pytests/unit/test_config.py +@@ -16,7 +16,7 @@ def test_call_id_function(tmp_path): + "cachedir": str(cache_dir), + "extension_modules": str(extmods), + "grains": {"osfinger": "meh"}, +- "optimization_order": [0], ++ "optimization_order": [0, 1, 2], + } + ret = salt.config.call_id_function(opts) + assert ret == "meh" +-- +2.42.0 + diff --git a/fix-the-aptpkg.py-unit-test-failure.patch b/fix-the-aptpkg.py-unit-test-failure.patch new file mode 100644 index 0000000..05b03db --- /dev/null +++ b/fix-the-aptpkg.py-unit-test-failure.patch @@ -0,0 +1,25 @@ +From 4bc3be7814daf5365d63b88f164f791ea53b418f Mon Sep 17 00:00:00 2001 +From: Marek Czernek +Date: Wed, 17 Jan 2024 15:04:53 +0100 +Subject: [PATCH] Fix the aptpkg.py unit test failure + +--- + salt/modules/aptpkg.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py +index 9885e9fb60..ad5450c415 100644 +--- a/salt/modules/aptpkg.py ++++ b/salt/modules/aptpkg.py +@@ -3128,7 +3128,7 @@ def expand_repo_def(**kwargs): + NOT USABLE IN THE CLI + """ + warn_until_date( +- "20240101", ++ "20250101", + "The pkg.expand_repo_def function is deprecated and set for removal " + "after {date}. This is only unsed internally by the apt pkg state " + "module. If that's not the case, please file an new issue requesting " +-- +2.43.0 + diff --git a/fixed-keyerror-in-logs-when-running-a-state-that-fai.patch b/fixed-keyerror-in-logs-when-running-a-state-that-fai.patch new file mode 100644 index 0000000..5343c0b --- /dev/null +++ b/fixed-keyerror-in-logs-when-running-a-state-that-fai.patch @@ -0,0 +1,121 @@ +From f41a8e2a142a8487e13af481990928e0afb5f15e Mon Sep 17 00:00:00 2001 +From: Victor Zhestkov +Date: Thu, 18 Jan 2024 17:02:03 +0100 +Subject: [PATCH] Fixed KeyError in logs when running a state that + fails. (#615) + +Co-authored-by: Megan Wilhite +--- + changelog/64231.fixed.md | 1 + + salt/master.py | 2 +- + salt/minion.py | 4 ++ + salt/utils/event.py | 3 +- + .../integration/states/test_state_test.py | 38 +++++++++++++++++++ + 5 files changed, 46 insertions(+), 2 deletions(-) + create mode 100644 changelog/64231.fixed.md + create mode 100644 tests/pytests/integration/states/test_state_test.py + +diff --git a/changelog/64231.fixed.md b/changelog/64231.fixed.md +new file mode 100644 +index 0000000000..0991c5a8b9 +--- /dev/null ++++ b/changelog/64231.fixed.md +@@ -0,0 +1 @@ ++Fixed KeyError in logs when running a state that fails. +diff --git a/salt/master.py b/salt/master.py +index fc243ef674..3d2ba1e29d 100644 +--- a/salt/master.py ++++ b/salt/master.py +@@ -1790,7 +1790,7 @@ class AESFuncs(TransportMethods): + def pub_ret(self, load): + """ + Request the return data from a specific jid, only allowed +- if the requesting minion also initialted the execution. ++ if the requesting minion also initiated the execution. + + :param dict load: The minion payload + +diff --git a/salt/minion.py b/salt/minion.py +index 4db0d31bd4..2ccd0cd5a9 100644 +--- a/salt/minion.py ++++ b/salt/minion.py +@@ -2022,6 +2022,8 @@ class Minion(MinionBase): + ret["jid"] = data["jid"] + ret["fun"] = data["fun"] + ret["fun_args"] = data["arg"] ++ if "user" in data: ++ ret["user"] = data["user"] + if "master_id" in data: + ret["master_id"] = data["master_id"] + if "metadata" in data: +@@ -2141,6 +2143,8 @@ class Minion(MinionBase): + ret["jid"] = data["jid"] + ret["fun"] = data["fun"] + ret["fun_args"] = data["arg"] ++ if "user" in data: ++ ret["user"] = data["user"] + if "metadata" in data: + ret["metadata"] = data["metadata"] + if minion_instance.connected: +diff --git a/salt/utils/event.py b/salt/utils/event.py +index 869e12a140..e6d7b00520 100644 +--- a/salt/utils/event.py ++++ b/salt/utils/event.py +@@ -902,7 +902,8 @@ class SaltEvent: + data["success"] = False + data["return"] = "Error: {}.{}".format(tags[0], tags[-1]) + data["fun"] = fun +- data["user"] = load["user"] ++ if "user" in load: ++ data["user"] = load["user"] + self.fire_event( + data, + tagify([load["jid"], "sub", load["id"], "error", fun], "job"), +diff --git a/tests/pytests/integration/states/test_state_test.py b/tests/pytests/integration/states/test_state_test.py +new file mode 100644 +index 0000000000..b2328a4c2b +--- /dev/null ++++ b/tests/pytests/integration/states/test_state_test.py +@@ -0,0 +1,38 @@ ++def test_failing_sls(salt_master, salt_minion, salt_cli, caplog): ++ """ ++ Test when running state.sls and the state fails. ++ When the master stores the job and attempts to send ++ an event a KeyError was previously being logged. ++ This test ensures we do not log an error when ++ attempting to send an event about a failing state. ++ """ ++ statesls = """ ++ test_state: ++ test.fail_without_changes: ++ - name: "bla" ++ """ ++ with salt_master.state_tree.base.temp_file("test_failure.sls", statesls): ++ ret = salt_cli.run("state.sls", "test_failure", minion_tgt=salt_minion.id) ++ for message in caplog.messages: ++ assert "Event iteration failed with" not in message ++ ++ ++def test_failing_sls_compound(salt_master, salt_minion, salt_cli, caplog): ++ """ ++ Test when running state.sls in a compound command and the state fails. ++ When the master stores the job and attempts to send ++ an event a KeyError was previously being logged. ++ This test ensures we do not log an error when ++ attempting to send an event about a failing state. ++ """ ++ statesls = """ ++ test_state: ++ test.fail_without_changes: ++ - name: "bla" ++ """ ++ with salt_master.state_tree.base.temp_file("test_failure.sls", statesls): ++ ret = salt_cli.run( ++ "state.sls,cmd.run", "test_failure,ls", minion_tgt=salt_minion.id ++ ) ++ for message in caplog.messages: ++ assert "Event iteration failed with" not in message +-- +2.43.0 + + diff --git a/implement-the-calling-for-batch-async-from-the-salt-.patch b/implement-the-calling-for-batch-async-from-the-salt-.patch new file mode 100644 index 0000000..d4e0af4 --- /dev/null +++ b/implement-the-calling-for-batch-async-from-the-salt-.patch @@ -0,0 +1,145 @@ +From 7ab208fd2d23eaa582cdbba912d4538d8c87e5f4 Mon Sep 17 00:00:00 2001 +From: Victor Zhestkov +Date: Mon, 2 Oct 2023 13:24:15 +0200 +Subject: [PATCH] Implement the calling for batch async from the salt + CLI + +* Implement calling batch async with salt CLI + +* Add the test for calling batch async with salt CLI +--- + salt/cli/salt.py | 53 ++++++++++++++++++++++++++++- + tests/pytests/unit/cli/test_salt.py | 50 +++++++++++++++++++++++++++ + 2 files changed, 102 insertions(+), 1 deletion(-) + create mode 100644 tests/pytests/unit/cli/test_salt.py + +diff --git a/salt/cli/salt.py b/salt/cli/salt.py +index f90057f668..e19cfa5ce6 100644 +--- a/salt/cli/salt.py ++++ b/salt/cli/salt.py +@@ -47,7 +47,12 @@ class SaltCMD(salt.utils.parsers.SaltCMDOptionParser): + self.exit(2, "{}\n".format(exc)) + return + +- if self.options.batch or self.options.static: ++ if self.options.batch and self.config["async"]: ++ # _run_batch_async() will just return the jid and exit ++ # Execution will not continue past this point ++ # in batch async mode. Batch async is handled by the master. ++ self._run_batch_async() ++ elif self.options.batch or self.options.static: + # _run_batch() will handle all output and + # exit with the appropriate error condition + # Execution will not continue past this point +@@ -296,6 +301,52 @@ class SaltCMD(salt.utils.parsers.SaltCMDOptionParser): + retcode = job_retcode + sys.exit(retcode) + ++ def _run_batch_async(self): ++ kwargs = { ++ "tgt": self.config["tgt"], ++ "fun": self.config["fun"], ++ "arg": self.config["arg"], ++ "timeout": self.options.timeout, ++ "show_timeout": self.options.show_timeout, ++ "show_jid": self.options.show_jid, ++ "batch": self.config["batch"], ++ } ++ tgt = kwargs.pop("tgt", "") ++ fun = kwargs.pop("fun", "") ++ ++ if self.config.get("eauth", ""): ++ kwargs.update( ++ { ++ "eauth": self.config["eauth"], ++ } ++ ) ++ for opt in ("username", "password"): ++ if opt in self.config: ++ kwargs[opt] = self.config[opt] ++ ++ try: ++ ret = self.local_client.run_job(tgt, fun, **kwargs) ++ except ( ++ AuthenticationError, ++ AuthorizationError, ++ SaltInvocationError, ++ EauthAuthenticationError, ++ SaltClientError, ++ ) as exc: ++ ret = str(exc) ++ self.exit(2, "ERROR: {}\n".format(exc)) ++ if "jid" in ret and "error" not in ret: ++ salt.utils.stringutils.print_cli( ++ "Executed command with job ID: {}".format(ret["jid"]) ++ ) ++ else: ++ self._output_ret(ret, self.config.get("output", "nested")) ++ ++ if "error" in ret: ++ sys.exit(1) ++ ++ sys.exit(0) ++ + def _print_errors_summary(self, errors): + if errors: + salt.utils.stringutils.print_cli("\n") +diff --git a/tests/pytests/unit/cli/test_salt.py b/tests/pytests/unit/cli/test_salt.py +new file mode 100644 +index 0000000000..d9f4b5b097 +--- /dev/null ++++ b/tests/pytests/unit/cli/test_salt.py +@@ -0,0 +1,50 @@ ++import pytest ++ ++from tests.support.mock import MagicMock, patch ++ ++ ++def test_saltcmd_batch_async_call(): ++ """ ++ Test calling batch async with salt CLI ++ """ ++ import salt.cli.salt ++ ++ local_client = MagicMock() ++ local_client.run_job = MagicMock(return_value={"jid": 123456}) ++ with pytest.raises(SystemExit) as exit_info, patch( ++ "sys.argv", ++ [ ++ "salt", ++ "--batch=10", ++ "--async", ++ "*", ++ "test.arg", ++ "arg1", ++ "arg2", ++ "kwarg1=val1", ++ ], ++ ), patch("salt.cli.salt.SaltCMD.process_config_dir", MagicMock), patch( ++ "salt.output.display_output", MagicMock() ++ ), patch( ++ "salt.client.get_local_client", return_value=local_client ++ ), patch( ++ "salt.utils.stringutils.print_cli", MagicMock() ++ ) as print_cli: ++ salt_cmd = salt.cli.salt.SaltCMD() ++ salt_cmd.config = { ++ "async": True, ++ "batch": 10, ++ "tgt": "*", ++ "fun": "test.arg", ++ "arg": ["arg1", "arg2", {"__kwarg__": True, "kwarg1": "val1"}], ++ } ++ salt_cmd._mixin_after_parsed_funcs = [] ++ salt_cmd.run() ++ ++ local_client.run_job.assert_called_once() ++ assert local_client.run_job.mock_calls[0].args[0] == "*" ++ assert local_client.run_job.mock_calls[0].args[1] == "test.arg" ++ assert local_client.run_job.mock_calls[0].kwargs["arg"] == ["arg1", "arg2", {"__kwarg__": True, "kwarg1": "val1"}] ++ assert local_client.run_job.mock_calls[0].kwargs["batch"] == 10 ++ print_cli.assert_called_once_with("Executed command with job ID: 123456") ++ assert exit_info.value.code == 0 +-- +2.42.0 + diff --git a/improve-pip-target-override-condition-with-venv_pip_.patch b/improve-pip-target-override-condition-with-venv_pip_.patch new file mode 100644 index 0000000..a3f1101 --- /dev/null +++ b/improve-pip-target-override-condition-with-venv_pip_.patch @@ -0,0 +1,113 @@ +From da938aa8a572138b5b9b1535c5c3d69326e5194e Mon Sep 17 00:00:00 2001 +From: Victor Zhestkov +Date: Thu, 18 Jan 2024 17:02:23 +0100 +Subject: [PATCH] Improve pip target override condition with + VENV_PIP_TARGET environment variable (bsc#1216850) (#613) + +* Improve pip target override condition + +* Improve pip test with different condition of overriding the target + +* Add changelog entry +--- + changelog/65562.fixed.md | 1 + + salt/modules/pip.py | 6 ++-- + tests/pytests/unit/modules/test_pip.py | 50 +++++++++++++++++--------- + 3 files changed, 38 insertions(+), 19 deletions(-) + create mode 100644 changelog/65562.fixed.md + +diff --git a/changelog/65562.fixed.md b/changelog/65562.fixed.md +new file mode 100644 +index 0000000000..ba483b4b77 +--- /dev/null ++++ b/changelog/65562.fixed.md +@@ -0,0 +1 @@ ++Improve the condition of overriding target for pip with VENV_PIP_TARGET environment variable. +diff --git a/salt/modules/pip.py b/salt/modules/pip.py +index a60bdca0bb..68a2a442a1 100644 +--- a/salt/modules/pip.py ++++ b/salt/modules/pip.py +@@ -857,9 +857,11 @@ def install( + cmd.extend(["--build", build]) + + # Use VENV_PIP_TARGET environment variable value as target +- # if set and no target specified on the function call ++ # if set and no target specified on the function call. ++ # Do not set target if bin_env specified, use default ++ # for specified binary environment or expect explicit target specification. + target_env = os.environ.get("VENV_PIP_TARGET", None) +- if target is None and target_env is not None: ++ if target is None and target_env is not None and bin_env is None: + target = target_env + + if target: +diff --git a/tests/pytests/unit/modules/test_pip.py b/tests/pytests/unit/modules/test_pip.py +index b7ad1ea3fd..c03e6ed292 100644 +--- a/tests/pytests/unit/modules/test_pip.py ++++ b/tests/pytests/unit/modules/test_pip.py +@@ -1738,28 +1738,44 @@ def test_when_version_is_called_with_a_user_it_should_be_passed_to_undelying_run + ) + + +-def test_install_target_from_VENV_PIP_TARGET_in_resulting_command(python_binary): ++@pytest.mark.parametrize( ++ "bin_env,target,target_env,expected_target", ++ [ ++ (None, None, None, None), ++ (None, "/tmp/foo", None, "/tmp/foo"), ++ (None, None, "/tmp/bar", "/tmp/bar"), ++ (None, "/tmp/foo", "/tmp/bar", "/tmp/foo"), ++ ("/tmp/venv", "/tmp/foo", None, "/tmp/foo"), ++ ("/tmp/venv", None, "/tmp/bar", None), ++ ("/tmp/venv", "/tmp/foo", "/tmp/bar", "/tmp/foo"), ++ ], ++) ++def test_install_target_from_VENV_PIP_TARGET_in_resulting_command( ++ python_binary, bin_env, target, target_env, expected_target ++): + pkg = "pep8" +- target = "/tmp/foo" +- target_env = "/tmp/bar" + mock = MagicMock(return_value={"retcode": 0, "stdout": ""}) + environment = os.environ.copy() +- environment["VENV_PIP_TARGET"] = target_env ++ real_get_pip_bin = pip._get_pip_bin ++ ++ def mock_get_pip_bin(bin_env): ++ if not bin_env: ++ return real_get_pip_bin(bin_env) ++ return [f"{bin_env}/bin/pip"] ++ ++ if target_env is not None: ++ environment["VENV_PIP_TARGET"] = target_env + with patch.dict(pip.__salt__, {"cmd.run_all": mock}), patch.object( + os, "environ", environment +- ): +- pip.install(pkg) +- expected = [*python_binary, "install", "--target", target_env, pkg] +- mock.assert_called_with( +- expected, +- saltenv="base", +- runas=None, +- use_vt=False, +- python_shell=False, +- ) +- mock.reset_mock() +- pip.install(pkg, target=target) +- expected = [*python_binary, "install", "--target", target, pkg] ++ ), patch.object(pip, "_get_pip_bin", mock_get_pip_bin): ++ pip.install(pkg, bin_env=bin_env, target=target) ++ expected_binary = python_binary ++ if bin_env is not None: ++ expected_binary = [f"{bin_env}/bin/pip"] ++ if expected_target is not None: ++ expected = [*expected_binary, "install", "--target", expected_target, pkg] ++ else: ++ expected = [*expected_binary, "install", pkg] + mock.assert_called_with( + expected, + saltenv="base", +-- +2.43.0 + + diff --git a/improve-salt.utils.json.find_json-bsc-1213293.patch b/improve-salt.utils.json.find_json-bsc-1213293.patch new file mode 100644 index 0000000..937a7fb --- /dev/null +++ b/improve-salt.utils.json.find_json-bsc-1213293.patch @@ -0,0 +1,204 @@ +From 4e6b445f2dbe8a79d220c697abff946e00b2e57b Mon Sep 17 00:00:00 2001 +From: Victor Zhestkov +Date: Mon, 2 Oct 2023 13:26:20 +0200 +Subject: [PATCH] Improve salt.utils.json.find_json (bsc#1213293) + +* Improve salt.utils.json.find_json + +* Move tests of find_json to pytest +--- + salt/utils/json.py | 39 +++++++- + tests/pytests/unit/utils/test_json.py | 122 ++++++++++++++++++++++++++ + 2 files changed, 158 insertions(+), 3 deletions(-) + create mode 100644 tests/pytests/unit/utils/test_json.py + +diff --git a/salt/utils/json.py b/salt/utils/json.py +index 33cdbf401d..0845b64694 100644 +--- a/salt/utils/json.py ++++ b/salt/utils/json.py +@@ -32,18 +32,51 @@ def find_json(raw): + """ + ret = {} + lines = __split(raw) ++ lengths = list(map(len, lines)) ++ starts = [] ++ ends = [] ++ ++ # Search for possible starts end ends of the json fragments + for ind, _ in enumerate(lines): ++ line = lines[ind].lstrip() ++ if line == "{" or line == "[": ++ starts.append((ind, line)) ++ if line == "}" or line == "]": ++ ends.append((ind, line)) ++ ++ # List all the possible pairs of starts and ends, ++ # and fill the length of each block to sort by size after ++ starts_ends = [] ++ for start, start_br in starts: ++ for end, end_br in reversed(ends): ++ if end > start and ( ++ (start_br == "{" and end_br == "}") ++ or (start_br == "[" and end_br == "]") ++ ): ++ starts_ends.append((start, end, sum(lengths[start : end + 1]))) ++ ++ # Iterate through all the possible pairs starting from the largest ++ starts_ends.sort(key=lambda x: (x[2], x[1] - x[0], x[0]), reverse=True) ++ for start, end, _ in starts_ends: ++ working = "\n".join(lines[start : end + 1]) + try: +- working = "\n".join(lines[ind:]) +- except UnicodeDecodeError: +- working = "\n".join(salt.utils.data.decode(lines[ind:])) ++ ret = json.loads(working) ++ except ValueError: ++ continue ++ if ret: ++ return ret + ++ # Fall back to old implementation for backward compatibility ++ # excpecting json after the text ++ for ind, _ in enumerate(lines): ++ working = "\n".join(lines[ind:]) + try: + ret = json.loads(working) + except ValueError: + continue + if ret: + return ret ++ + if not ret: + # Not json, raise an error + raise ValueError +diff --git a/tests/pytests/unit/utils/test_json.py b/tests/pytests/unit/utils/test_json.py +new file mode 100644 +index 0000000000..72b1023003 +--- /dev/null ++++ b/tests/pytests/unit/utils/test_json.py +@@ -0,0 +1,122 @@ ++""" ++Tests for salt.utils.json ++""" ++ ++import textwrap ++ ++import pytest ++ ++import salt.utils.json ++ ++ ++def test_find_json(): ++ some_junk_text = textwrap.dedent( ++ """ ++ Just some junk text ++ with multiline ++ """ ++ ) ++ some_warning_message = textwrap.dedent( ++ """ ++ [WARNING] Test warning message ++ """ ++ ) ++ test_small_json = textwrap.dedent( ++ """ ++ { ++ "local": true ++ } ++ """ ++ ) ++ test_sample_json = """ ++ { ++ "glossary": { ++ "title": "example glossary", ++ "GlossDiv": { ++ "title": "S", ++ "GlossList": { ++ "GlossEntry": { ++ "ID": "SGML", ++ "SortAs": "SGML", ++ "GlossTerm": "Standard Generalized Markup Language", ++ "Acronym": "SGML", ++ "Abbrev": "ISO 8879:1986", ++ "GlossDef": { ++ "para": "A meta-markup language, used to create markup languages such as DocBook.", ++ "GlossSeeAlso": ["GML", "XML"] ++ }, ++ "GlossSee": "markup" ++ } ++ } ++ } ++ } ++ } ++ """ ++ expected_ret = { ++ "glossary": { ++ "GlossDiv": { ++ "GlossList": { ++ "GlossEntry": { ++ "GlossDef": { ++ "GlossSeeAlso": ["GML", "XML"], ++ "para": ( ++ "A meta-markup language, used to create markup" ++ " languages such as DocBook." ++ ), ++ }, ++ "GlossSee": "markup", ++ "Acronym": "SGML", ++ "GlossTerm": "Standard Generalized Markup Language", ++ "SortAs": "SGML", ++ "Abbrev": "ISO 8879:1986", ++ "ID": "SGML", ++ } ++ }, ++ "title": "S", ++ }, ++ "title": "example glossary", ++ } ++ } ++ ++ # First test the valid JSON ++ ret = salt.utils.json.find_json(test_sample_json) ++ assert ret == expected_ret ++ ++ # Now pre-pend some garbage and re-test ++ garbage_prepend_json = f"{some_junk_text}{test_sample_json}" ++ ret = salt.utils.json.find_json(garbage_prepend_json) ++ assert ret == expected_ret ++ ++ # Now post-pend some garbage and re-test ++ garbage_postpend_json = f"{test_sample_json}{some_junk_text}" ++ ret = salt.utils.json.find_json(garbage_postpend_json) ++ assert ret == expected_ret ++ ++ # Now pre-pend some warning and re-test ++ warning_prepend_json = f"{some_warning_message}{test_sample_json}" ++ ret = salt.utils.json.find_json(warning_prepend_json) ++ assert ret == expected_ret ++ ++ # Now post-pend some warning and re-test ++ warning_postpend_json = f"{test_sample_json}{some_warning_message}" ++ ret = salt.utils.json.find_json(warning_postpend_json) ++ assert ret == expected_ret ++ ++ # Now put around some garbage and re-test ++ garbage_around_json = f"{some_junk_text}{test_sample_json}{some_junk_text}" ++ ret = salt.utils.json.find_json(garbage_around_json) ++ assert ret == expected_ret ++ ++ # Now pre-pend small json and re-test ++ small_json_pre_json = f"{test_small_json}{test_sample_json}" ++ ret = salt.utils.json.find_json(small_json_pre_json) ++ assert ret == expected_ret ++ ++ # Now post-pend small json and re-test ++ small_json_post_json = f"{test_sample_json}{test_small_json}" ++ ret = salt.utils.json.find_json(small_json_post_json) ++ assert ret == expected_ret ++ ++ # Test to see if a ValueError is raised if no JSON is passed in ++ with pytest.raises(ValueError): ++ ret = salt.utils.json.find_json(some_junk_text) +-- +2.42.0 + diff --git a/only-call-native_str-on-curl_debug-message-in-tornad.patch b/only-call-native_str-on-curl_debug-message-in-tornad.patch new file mode 100644 index 0000000..d720b33 --- /dev/null +++ b/only-call-native_str-on-curl_debug-message-in-tornad.patch @@ -0,0 +1,31 @@ +From b76b74bd9640adf3b6798e4de4b89aaa7af62c9f Mon Sep 17 00:00:00 2001 +From: Victor Zhestkov +Date: Mon, 2 Oct 2023 13:24:43 +0200 +Subject: [PATCH] Only call native_str on curl_debug message in tornado + when needed + +Co-authored-by: Ben Darnell +--- + salt/ext/tornado/curl_httpclient.py | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/salt/ext/tornado/curl_httpclient.py b/salt/ext/tornado/curl_httpclient.py +index 8652343cf7..9e4133fd13 100644 +--- a/salt/ext/tornado/curl_httpclient.py ++++ b/salt/ext/tornado/curl_httpclient.py +@@ -494,10 +494,11 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): + + def _curl_debug(self, debug_type, debug_msg): + debug_types = ('I', '<', '>', '<', '>') +- debug_msg = native_str(debug_msg) + if debug_type == 0: ++ debug_msg = native_str(debug_msg) + curl_log.debug('%s', debug_msg.strip()) + elif debug_type in (1, 2): ++ debug_msg = native_str(debug_msg) + for line in debug_msg.splitlines(): + curl_log.debug('%s %s', debug_types[debug_type], line) + elif debug_type == 4: +-- +2.42.0 + diff --git a/prefer-unittest.mock-for-python-versions-that-are-su.patch b/prefer-unittest.mock-for-python-versions-that-are-su.patch new file mode 100644 index 0000000..a419201 --- /dev/null +++ b/prefer-unittest.mock-for-python-versions-that-are-su.patch @@ -0,0 +1,135 @@ +From 107de57586f0b0f784771543b942dfb6bb70453a Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Yeray=20Guti=C3=A9rrez=20Cedr=C3=A9s?= + +Date: Wed, 13 Dec 2023 11:03:45 +0000 +Subject: [PATCH] Prefer unittest.mock for Python versions that are + sufficient + +--- + requirements/pytest.txt | 2 +- + .../unit/cloud/clouds/test_dimensiondata.py | 4 +- + tests/pytests/unit/cloud/clouds/test_gce.py | 4 +- + tests/support/mock.py | 48 +++++++++---------- + 4 files changed, 25 insertions(+), 33 deletions(-) + +diff --git a/requirements/pytest.txt b/requirements/pytest.txt +index 5b67583a3d..0bead83f5b 100644 +--- a/requirements/pytest.txt ++++ b/requirements/pytest.txt +@@ -1,4 +1,4 @@ +-mock >= 3.0.0 ++mock >= 3.0.0; python_version < '3.8' + # PyTest + pytest >= 7.0.1; python_version <= "3.6" + pytest >= 7.2.0; python_version > "3.6" +diff --git a/tests/pytests/unit/cloud/clouds/test_dimensiondata.py b/tests/pytests/unit/cloud/clouds/test_dimensiondata.py +index e196805004..aab2e686f2 100644 +--- a/tests/pytests/unit/cloud/clouds/test_dimensiondata.py ++++ b/tests/pytests/unit/cloud/clouds/test_dimensiondata.py +@@ -11,7 +11,6 @@ from salt.cloud.clouds import dimensiondata + from salt.exceptions import SaltCloudSystemExit + from salt.utils.versions import Version + from tests.support.mock import MagicMock +-from tests.support.mock import __version__ as mock_version + from tests.support.mock import patch + + try: +@@ -144,8 +143,7 @@ def test_import(): + with patch("salt.config.check_driver_dependencies", return_value=True) as p: + get_deps = dimensiondata.get_dependencies() + assert get_deps is True +- if Version(mock_version) >= Version("2.0.0"): +- assert p.call_count >= 1 ++ assert p.call_count >= 1 + + + def test_provider_matches(): +diff --git a/tests/pytests/unit/cloud/clouds/test_gce.py b/tests/pytests/unit/cloud/clouds/test_gce.py +index 265818016e..ec1346a978 100644 +--- a/tests/pytests/unit/cloud/clouds/test_gce.py ++++ b/tests/pytests/unit/cloud/clouds/test_gce.py +@@ -13,7 +13,6 @@ from salt.cloud.clouds import gce + from salt.exceptions import SaltCloudSystemExit + from salt.utils.versions import Version + from tests.support.mock import MagicMock +-from tests.support.mock import __version__ as mock_version + from tests.support.mock import call, patch + + +@@ -281,8 +280,7 @@ def test_import(): + with patch("salt.config.check_driver_dependencies", return_value=True) as p: + get_deps = gce.get_dependencies() + assert get_deps is True +- if Version(mock_version) >= Version("2.0.0"): +- p.assert_called_once() ++ p.assert_called_once() + + + @pytest.mark.parametrize( +diff --git a/tests/support/mock.py b/tests/support/mock.py +index 2256ad8f5d..59e5fcbc8e 100644 +--- a/tests/support/mock.py ++++ b/tests/support/mock.py +@@ -18,37 +18,33 @@ import copy + import errno + import fnmatch + import sys +- +-# By these days, we should blowup if mock is not available +-import mock # pylint: disable=blacklisted-external-import +- +-# pylint: disable=no-name-in-module,no-member +-from mock import ( +- ANY, +- DEFAULT, +- FILTER_DIR, +- MagicMock, +- Mock, +- NonCallableMagicMock, +- NonCallableMock, +- PropertyMock, +- __version__, +- call, +- create_autospec, +- patch, +- sentinel, +-) ++import importlib ++ ++current_version = (sys.version_info.major, sys.version_info.minor) ++ ++# Prefer unittest.mock for Python versions that are sufficient ++if current_version >= (3,8): ++ mock = importlib.import_module('unittest.mock') ++else: ++ mock = importlib.import_module('mock') ++ ++ANY = mock.ANY ++DEFAULT = mock.DEFAULT ++FILTER_DIR = mock.FILTER_DIR ++MagicMock = mock.MagicMock ++Mock = mock.Mock ++NonCallableMagicMock = mock.NonCallableMagicMock ++NonCallableMock = mock.NonCallableMock ++PropertyMock = mock.PropertyMock ++call = mock.call ++create_autospec = mock.create_autospec ++patch = mock.patch ++sentinel = mock.sentinel + + import salt.utils.stringutils + + # pylint: disable=no-name-in-module,no-member + +- +-__mock_version = tuple( +- int(part) for part in mock.__version__.split(".") if part.isdigit() +-) # pylint: disable=no-member +- +- + class MockFH: + def __init__(self, filename, read_data, *args, **kwargs): + self.filename = filename +-- +2.41.0 + diff --git a/revert-make-sure-configured-user-is-properly-set-by-.patch b/revert-make-sure-configured-user-is-properly-set-by-.patch new file mode 100644 index 0000000..f6bfcbc --- /dev/null +++ b/revert-make-sure-configured-user-is-properly-set-by-.patch @@ -0,0 +1,194 @@ +From d9980c8d2cfedfd6f08543face6ee7e34e9d1b54 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Thu, 16 Nov 2023 09:23:58 +0000 +Subject: [PATCH] Revert "Make sure configured user is properly set by + Salt (bsc#1210994) (#596)" (#614) + +This reverts commit 5ea4add5c8e2bed50b9825edfff7565e5f6124f3. +--- + pkg/common/salt-master.service | 1 - + pkg/old/deb/salt-master.service | 1 - + pkg/old/suse/salt-master.service | 1 - + salt/cli/daemons.py | 27 ------------------- + salt/cli/ssh.py | 8 ------ + salt/utils/verify.py | 4 +-- + .../integration/cli/test_salt_minion.py | 4 +-- + 7 files changed, 4 insertions(+), 42 deletions(-) + +diff --git a/pkg/common/salt-master.service b/pkg/common/salt-master.service +index 257ecc283f..377c87afeb 100644 +--- a/pkg/common/salt-master.service ++++ b/pkg/common/salt-master.service +@@ -8,7 +8,6 @@ LimitNOFILE=100000 + Type=notify + NotifyAccess=all + ExecStart=/usr/bin/salt-master +-User=salt + + [Install] + WantedBy=multi-user.target +diff --git a/pkg/old/deb/salt-master.service b/pkg/old/deb/salt-master.service +index f9dca296b4..b5d0cdd22c 100644 +--- a/pkg/old/deb/salt-master.service ++++ b/pkg/old/deb/salt-master.service +@@ -7,7 +7,6 @@ LimitNOFILE=16384 + Type=notify + NotifyAccess=all + ExecStart=/usr/bin/salt-master +-User=salt + + [Install] + WantedBy=multi-user.target +diff --git a/pkg/old/suse/salt-master.service b/pkg/old/suse/salt-master.service +index caabca511c..9e002d16ca 100644 +--- a/pkg/old/suse/salt-master.service ++++ b/pkg/old/suse/salt-master.service +@@ -8,7 +8,6 @@ LimitNOFILE=100000 + Type=simple + ExecStart=/usr/bin/salt-master + TasksMax=infinity +-User=salt + + [Install] + WantedBy=multi-user.target +diff --git a/salt/cli/daemons.py b/salt/cli/daemons.py +index c9ee9ced91..ecc05c919e 100644 +--- a/salt/cli/daemons.py ++++ b/salt/cli/daemons.py +@@ -7,7 +7,6 @@ import logging + import os + import warnings + +-import salt.defaults.exitcodes + import salt.utils.kinds as kinds + from salt.exceptions import SaltClientError, SaltSystemExit, get_error_message + from salt.utils import migrations +@@ -74,16 +73,6 @@ class DaemonsMixin: # pylint: disable=no-init + self.__class__.__name__, + ) + +- def verify_user(self): +- """ +- Verify Salt configured user for Salt and shutdown daemon if not valid. +- +- :return: +- """ +- if not check_user(self.config["user"]): +- self.action_log_info("Cannot switch to configured user for Salt. Exiting") +- self.shutdown(salt.defaults.exitcodes.EX_NOUSER) +- + def action_log_info(self, action): + """ + Say daemon starting. +@@ -189,10 +178,6 @@ class Master( + self.config["interface"] = ip_bracket(self.config["interface"]) + migrations.migrate_paths(self.config) + +- # Ensure configured user is valid and environment is properly set +- # before initializating rest of the stack. +- self.verify_user() +- + # Late import so logging works correctly + import salt.master + +@@ -305,10 +290,6 @@ class Minion( + + transport = self.config.get("transport").lower() + +- # Ensure configured user is valid and environment is properly set +- # before initializating rest of the stack. +- self.verify_user() +- + try: + # Late import so logging works correctly + import salt.minion +@@ -497,10 +478,6 @@ class ProxyMinion( + self.action_log_info("An instance is already running. Exiting") + self.shutdown(1) + +- # Ensure configured user is valid and environment is properly set +- # before initializating rest of the stack. +- self.verify_user() +- + # TODO: AIO core is separate from transport + # Late import so logging works correctly + import salt.minion +@@ -599,10 +576,6 @@ class Syndic( + + self.action_log_info('Setting up "{}"'.format(self.config["id"])) + +- # Ensure configured user is valid and environment is properly set +- # before initializating rest of the stack. +- self.verify_user() +- + # Late import so logging works correctly + import salt.minion + +diff --git a/salt/cli/ssh.py b/salt/cli/ssh.py +index 672f32b8c0..6048cb5f58 100644 +--- a/salt/cli/ssh.py ++++ b/salt/cli/ssh.py +@@ -1,9 +1,7 @@ + import sys + + import salt.client.ssh +-import salt.defaults.exitcodes + import salt.utils.parsers +-from salt.utils.verify import check_user + + + class SaltSSH(salt.utils.parsers.SaltSSHOptionParser): +@@ -17,11 +15,5 @@ class SaltSSH(salt.utils.parsers.SaltSSHOptionParser): + # that won't be used anyways with -H or --hosts + self.parse_args() + +- if not check_user(self.config["user"]): +- self.exit( +- salt.defaults.exitcodes.EX_NOUSER, +- "Cannot switch to configured user for Salt. Exiting", +- ) +- + ssh = salt.client.ssh.SSH(self.config) + ssh.run() +diff --git a/salt/utils/verify.py b/salt/utils/verify.py +index 7899fbe538..879128f231 100644 +--- a/salt/utils/verify.py ++++ b/salt/utils/verify.py +@@ -335,8 +335,8 @@ def check_user(user): + + # We could just reset the whole environment but let's just override + # the variables we can get from pwuser +- # We ensure HOME is always present and set according to pwuser +- os.environ["HOME"] = pwuser.pw_dir ++ if "HOME" in os.environ: ++ os.environ["HOME"] = pwuser.pw_dir + + if "SHELL" in os.environ: + os.environ["SHELL"] = pwuser.pw_shell +diff --git a/tests/pytests/integration/cli/test_salt_minion.py b/tests/pytests/integration/cli/test_salt_minion.py +index bde2dd51d7..c0d6013474 100644 +--- a/tests/pytests/integration/cli/test_salt_minion.py ++++ b/tests/pytests/integration/cli/test_salt_minion.py +@@ -41,7 +41,7 @@ def test_exit_status_unknown_user(salt_master, minion_id): + factory = salt_master.salt_minion_daemon( + minion_id, overrides={"user": "unknown-user"} + ) +- factory.start(start_timeout=30, max_start_attempts=1) ++ factory.start(start_timeout=10, max_start_attempts=1) + + assert exc.value.process_result.returncode == salt.defaults.exitcodes.EX_NOUSER + assert "The user is not available." in exc.value.process_result.stderr +@@ -53,7 +53,7 @@ def test_exit_status_unknown_argument(salt_master, minion_id): + """ + with pytest.raises(FactoryNotStarted) as exc: + factory = salt_master.salt_minion_daemon(minion_id) +- factory.start("--unknown-argument", start_timeout=30, max_start_attempts=1) ++ factory.start("--unknown-argument", start_timeout=10, max_start_attempts=1) + + assert exc.value.process_result.returncode == salt.defaults.exitcodes.EX_USAGE + assert "Usage" in exc.value.process_result.stderr +-- +2.42.0 + + diff --git a/salt.changes b/salt.changes index 3b34567..c2f997e 100644 --- a/salt.changes +++ b/salt.changes @@ -1,3 +1,54 @@ +------------------------------------------------------------------- +Thu Feb 1 14:48:40 UTC 2024 - Pablo Suárez Hernández + +- Prevent directory traversal when creating syndic cache directory + on the master (CVE-2024-22231, bsc#1219430) +- Prevent directory traversal attacks in the master's serve_file + method (CVE-2024-22232, bsc#1219431) +- Prevent exceptions with fileserver.update when called via state (bsc#1218482) +- Improve pip target override condition with VENV_PIP_TARGET + environment variable (bsc#1216850) +- Fixed KeyError in logs when running a state that fails +- Ensure that pillar refresh loads beacons from pillar without restart +- Fix the aptpkg.py unit test failure +- Prefer unittest.mock to python-mock in test suite +- Enable "KeepAlive" probes for Salt SSH executions (bsc#1211649) +- Revert changes to set Salt configured user early in the stack (bsc#1216284) +- Align behavior of some modules when using salt-call via symlink (bsc#1215963) +- Fix gitfs "__env__" and improve cache cleaning (bsc#1193948) +- Remove python-boto dependency for the python3-salt-testsuite package for Tumbleweed +- Randomize pre_flight_script path (CVE-2023-34049 bsc#1215157) +- Allow all primitive grain types for autosign_grains (bsc#1214477) +- Fix optimization_order opt to prevent testsuite fails +- Improve salt.utils.json.find_json to avoid fails (bsc#1213293) +- Use salt-call from salt bundle with transactional_update +- Only call native_str on curl_debug message in tornado when needed +- Implement the calling for batch async from the salt CLI +- Fix calculation of SLS context vars when trailing dots + on targetted sls/state (bsc#1213518) +- Rename salt-tests to python3-salt-testsuite + +- Added: + * enable-keepalive-probes-for-salt-ssh-executions-bsc-.patch + * allow-all-primitive-grain-types-for-autosign_grains-.patch + * fixed-keyerror-in-logs-when-running-a-state-that-fai.patch + * use-salt-call-from-salt-bundle-with-transactional_up.patch + * implement-the-calling-for-batch-async-from-the-salt-.patch + * fix-calculation-of-sls-context-vars-when-trailing-do.patch + * prefer-unittest.mock-for-python-versions-that-are-su.patch + * fix-cve-2023-34049-bsc-1215157.patch + * fix-gitfs-__env__-and-improve-cache-cleaning-bsc-119.patch + * allow-kwargs-for-fileserver-roots-update-bsc-1218482.patch + * dereference-symlinks-to-set-proper-__cli-opt-bsc-121.patch + * revert-make-sure-configured-user-is-properly-set-by-.patch + * fix-cve-2024-22231-and-cve-2024-22232-bsc-1219430-bs.patch + * improve-pip-target-override-condition-with-venv_pip_.patch + * only-call-native_str-on-curl_debug-message-in-tornad.patch + * update-__pillar__-during-pillar_refresh.patch + * improve-salt.utils.json.find_json-bsc-1213293.patch + * fix-the-aptpkg.py-unit-test-failure.patch + * fix-optimization_order-opt-to-prevent-test-fails.patch + ------------------------------------------------------------------- Wed Sep 20 15:04:34 UTC 2023 - Pablo Suárez Hernández diff --git a/salt.spec b/salt.spec index 808901d..2c9488d 100644 --- a/salt.spec +++ b/salt.spec @@ -296,7 +296,7 @@ Patch75: fix-tests-to-make-them-running-with-salt-testsuite.patch # PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/commit/f82860b8ad3ee786762fa02fa1a6eaf6e24dc8d4 # PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65020 Patch76: do-not-fail-on-bad-message-pack-message-bsc-1213441-.patch -# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/64510 +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/64510 (dropped at patch 91) Patch77: make-sure-configured-user-is-properly-set-by-salt-bs.patch # PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/64959 Patch78: fixed-gitfs-cachedir_basename-to-avoid-hash-collisio.patch @@ -304,6 +304,46 @@ Patch78: fixed-gitfs-cachedir_basename-to-avoid-hash-collisio.patch Patch79: revert-usage-of-long-running-req-channel-bsc-1213960.patch # PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65238 Patch80: write-salt-version-before-building-when-using-with-s.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65036 +Patch81: fix-calculation-of-sls-context-vars-when-trailing-do.patch +# PATCH-FIX_OPENSUSE: https://github.com/openSUSE/salt/pull/594 +Patch82: implement-the-calling-for-batch-async-from-the-salt-.patch +# PATCH-FIX_UPSTREAM: https://github.com/tornadoweb/tornado/pull/2277 +Patch83: only-call-native_str-on-curl_debug-message-in-tornad.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65204 +Patch84: use-salt-call-from-salt-bundle-with-transactional_up.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65181 +Patch85: improve-salt.utils.json.find_json-bsc-1213293.patch +# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/65266 +Patch86: fix-optimization_order-opt-to-prevent-test-fails.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65232 +Patch87: allow-all-primitive-grain-types-for-autosign_grains-.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65482 +Patch88: fix-cve-2023-34049-bsc-1215157.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65017 +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65136 +Patch89: fix-gitfs-__env__-and-improve-cache-cleaning-bsc-119.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65435 +Patch90: dereference-symlinks-to-set-proper-__cli-opt-bsc-121.patch +# PATCH-FIX_OPENSUSE: https://github.com/openSUSE/salt/pull/614 (revert patch 77) +Patch91: revert-make-sure-configured-user-is-properly-set-by-.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65488 +Patch92: enable-keepalive-probes-for-salt-ssh-executions-bsc-.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65644 +Patch93: prefer-unittest.mock-for-python-versions-that-are-su.patch +# PATCH-FIX_OPENSUSE: https://github.com/openSUSE/salt/pull/620 +Patch94: fix-the-aptpkg.py-unit-test-failure.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65092 +Patch95: update-__pillar__-during-pillar_refresh.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65969 +Patch96: fix-cve-2024-22231-and-cve-2024-22232-bsc-1219430-bs.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65009 +Patch97: fixed-keyerror-in-logs-when-running-a-state-that-fai.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65562 +Patch98: improve-pip-target-override-condition-with-venv_pip_.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65819 +Patch99: allow-kwargs-for-fileserver-roots-update-bsc-1218482.patch + ### IMPORTANT: The line below is used as a snippet marker. Do not touch it. ### SALT PATCHES LIST END @@ -659,12 +699,30 @@ Requires(pre): %fillup_prereq Salt ssh is a master running without zmq. it enables the management of minions over a ssh connection. -%package tests +%package -n python3-salt-testsuite Summary: Unit and integration tests for Salt Requires: %{name} = %{version}-%{release} +Requires: python3-CherryPy +Requires: python3-Genshi +Requires: python3-Mako +%if !0%{?suse_version} > 1600 || 0%{?centos} +Requires: python3-boto +%endif +Requires: python3-boto3 +Requires: python3-docker +Requires: python3-mock +Requires: python3-pygit2 +Requires: python3-pytest >= 7.0.1 +Requires: python3-pytest-httpserver +Requires: python3-pytest-salt-factories >= 1.0.0~rc21 +Requires: python3-pytest-subtests +Requires: python3-testinfra +Requires: python3-yamllint -%description tests -Collections of unit and integration tests for Salt +Obsoletes: %{name}-tests + +%description -n python3-salt-testsuite +Collection of unit, functional, and integration tests for %{name}. %if %{with bash_completion} %package bash-completion @@ -812,10 +870,12 @@ install -Dd -m 0755 %{buildroot}%{_sysconfdir}/logrotate.d/ install -Dpm 0644 salt/cli/support/profiles/* %{buildroot}%{python3_sitelib}/salt/cli/support/profiles # Install Salt tests -install -Dd -m 0750 %{buildroot}%{_datadir}/salt -install -Dd -m 0750 %{buildroot}%{_datadir}/salt/tests -cp -a tests/* %{buildroot}%{_datadir}/salt/tests/ -sed -i '1s=^#!/usr/bin/\(python\|env python\)[0-9.]*=#!/usr/bin/python3=' %{buildroot}%{_datadir}/salt/tests/runtests.py +install -Dd %{buildroot}%{python3_sitelib}/salt-testsuite +cp -a tests %{buildroot}%{python3_sitelib}/salt-testsuite/ +# Remove runtests.py which is not used as deprecated method of running the tests +rm %{buildroot}%{python3_sitelib}/salt-testsuite/tests/runtests.py +# Copy conf files to the testsuite as they are used by the tests +cp -a conf %{buildroot}%{python3_sitelib}/salt-testsuite/ ## Install Zypper plugins only on SUSE machines %if 0%{?suse_version} @@ -1392,7 +1452,10 @@ rm -f %{_localstatedir}/cache/salt/minion/thin/version %files -n python3-salt %defattr(-,root,root,-) -%{python3_sitelib}/* +%dir %{python3_sitelib}/salt +%dir %{python3_sitelib}/salt-*.egg-info +%{python3_sitelib}/salt/* +%{python3_sitelib}/salt-*.egg-info/* %exclude %{python3_sitelib}/salt/cloud/deploy/*.sh %if %{with docs} @@ -1401,10 +1464,8 @@ rm -f %{_localstatedir}/cache/salt/minion/thin/version %doc doc/_build/html %endif -%files tests -%dir %{_datadir}/salt/ -%dir %{_datadir}/salt/tests/ -%{_datadir}/salt/tests/* +%files -n python3-salt-testsuite +%{python3_sitelib}/salt-testsuite %if %{with bash_completion} %files bash-completion diff --git a/update-__pillar__-during-pillar_refresh.patch b/update-__pillar__-during-pillar_refresh.patch new file mode 100644 index 0000000..7476185 --- /dev/null +++ b/update-__pillar__-during-pillar_refresh.patch @@ -0,0 +1,169 @@ +From 3e7c5d95423491f83d0016eb7c02285cd0b1bcf4 Mon Sep 17 00:00:00 2001 +From: Marek Czernek +Date: Wed, 17 Jan 2024 15:39:41 +0100 +Subject: [PATCH] Update __pillar__ during pillar_refresh + +--- + changelog/63583.fixed.md | 1 + + salt/minion.py | 1 + + .../integration/modules/test_pillar.py | 110 +++++++++++++++++- + 3 files changed, 111 insertions(+), 1 deletion(-) + create mode 100644 changelog/63583.fixed.md + +diff --git a/changelog/63583.fixed.md b/changelog/63583.fixed.md +new file mode 100644 +index 0000000000..f1b6e32507 +--- /dev/null ++++ b/changelog/63583.fixed.md +@@ -0,0 +1 @@ ++Need to make sure we update __pillar__ during a pillar refresh to ensure that process_beacons has the updated beacons loaded from pillar. +diff --git a/salt/minion.py b/salt/minion.py +index 9597d6e63a..4db0d31bd4 100644 +--- a/salt/minion.py ++++ b/salt/minion.py +@@ -2498,6 +2498,7 @@ class Minion(MinionBase): + current_schedule, new_schedule + ) + self.opts["pillar"] = new_pillar ++ self.functions.pack["__pillar__"] = self.opts["pillar"] + finally: + async_pillar.destroy() + self.matchers_refresh() +diff --git a/tests/pytests/integration/modules/test_pillar.py b/tests/pytests/integration/modules/test_pillar.py +index 66f7b9e47b..5db9a1630a 100644 +--- a/tests/pytests/integration/modules/test_pillar.py ++++ b/tests/pytests/integration/modules/test_pillar.py +@@ -1,9 +1,14 @@ ++import logging + import pathlib + import time ++import types + + import attr + import pytest + ++log = logging.getLogger(__name__) ++ ++ + pytestmark = [ + pytest.mark.slow_test, + pytest.mark.windows_whitelisted, +@@ -210,7 +215,7 @@ class PillarRefresh: + "top.sls", top_file_contents + ) + self.minion_1_pillar = self.master.pillar_tree.base.temp_file( +- "minion-1-pillar.sls", "{}: true".format(self.pillar_key) ++ "minion-1-pillar.sls", f"{self.pillar_key}: true" + ) + self.top_file.__enter__() + self.minion_1_pillar.__enter__() +@@ -588,3 +593,106 @@ def test_pillar_ext_59975(salt_call_cli): + """ + ret = salt_call_cli.run("pillar.ext", '{"libvert": _}') + assert "ext_pillar_opts" in ret.data ++ ++ ++@pytest.fixture ++def event_listerner_timeout(grains): ++ if grains["os"] == "Windows": ++ if grains["osrelease"].startswith("2019"): ++ return types.SimpleNamespace(catch=120, miss=30) ++ return types.SimpleNamespace(catch=90, miss=10) ++ return types.SimpleNamespace(catch=60, miss=10) ++ ++ ++@pytest.mark.slow_test ++def test_pillar_refresh_pillar_beacons( ++ base_env_pillar_tree_root_dir, ++ salt_cli, ++ salt_minion, ++ salt_master, ++ event_listener, ++ event_listerner_timeout, ++): ++ """ ++ Ensure beacons jobs in pillar are started after ++ a pillar refresh and then not running when pillar ++ is cleared. ++ """ ++ ++ top_sls = """ ++ base: ++ '{}': ++ - test_beacons ++ """.format( ++ salt_minion.id ++ ) ++ ++ test_beacons_sls_empty = "" ++ ++ test_beacons_sls = """ ++ beacons: ++ status: ++ - loadavg: ++ - 1-min ++ """ ++ ++ assert salt_minion.is_running() ++ ++ top_tempfile = pytest.helpers.temp_file( ++ "top.sls", top_sls, base_env_pillar_tree_root_dir ++ ) ++ beacon_tempfile = pytest.helpers.temp_file( ++ "test_beacons.sls", test_beacons_sls_empty, base_env_pillar_tree_root_dir ++ ) ++ ++ with top_tempfile, beacon_tempfile: ++ # Calling refresh_pillar to update in-memory pillars ++ salt_cli.run("saltutil.refresh_pillar", wait=True, minion_tgt=salt_minion.id) ++ ++ # Ensure beacons start when pillar is refreshed ++ with salt_master.pillar_tree.base.temp_file( ++ "test_beacons.sls", test_beacons_sls ++ ): ++ # Calling refresh_pillar to update in-memory pillars ++ salt_cli.run( ++ "saltutil.refresh_pillar", wait=True, minion_tgt=salt_minion.id ++ ) ++ ++ # Give the beacons a chance to start ++ time.sleep(5) ++ ++ event_tag = f"salt/beacon/*/status/*" ++ start_time = time.time() ++ ++ event_pattern = (salt_master.id, event_tag) ++ matched_events = event_listener.wait_for_events( ++ [event_pattern], ++ after_time=start_time, ++ timeout=event_listerner_timeout.catch, ++ ) ++ ++ assert matched_events.found_all_events ++ ++ # Ensure beacons sttop when pillar is refreshed ++ with salt_master.pillar_tree.base.temp_file( ++ "test_beacons.sls", test_beacons_sls_empty ++ ): ++ # Calling refresh_pillar to update in-memory pillars ++ salt_cli.run( ++ "saltutil.refresh_pillar", wait=True, minion_tgt=salt_minion.id ++ ) ++ ++ # Give the beacons a chance to stop ++ time.sleep(5) ++ ++ event_tag = f"salt/beacon/*/status/*" ++ start_time = time.time() ++ ++ event_pattern = (salt_master.id, event_tag) ++ matched_events = event_listener.wait_for_events( ++ [event_pattern], ++ after_time=start_time, ++ timeout=event_listerner_timeout.miss, ++ ) ++ ++ assert not matched_events.found_all_events +-- +2.43.0 + diff --git a/use-salt-call-from-salt-bundle-with-transactional_up.patch b/use-salt-call-from-salt-bundle-with-transactional_up.patch new file mode 100644 index 0000000..3ee8d2f --- /dev/null +++ b/use-salt-call-from-salt-bundle-with-transactional_up.patch @@ -0,0 +1,103 @@ +From 0459d3f711eb9898f56a97d0bf0eb66fd1421a56 Mon Sep 17 00:00:00 2001 +From: Victor Zhestkov +Date: Mon, 2 Oct 2023 13:25:52 +0200 +Subject: [PATCH] Use salt-call from salt bundle with + transactional_update + +* Use salt-call from the bundle with transactional_update + +* Add test checking which salt-call is selected by executable +--- + salt/modules/transactional_update.py | 13 +++++- + .../unit/modules/test_transactional_update.py | 44 +++++++++++++++++++ + 2 files changed, 56 insertions(+), 1 deletion(-) + +diff --git a/salt/modules/transactional_update.py b/salt/modules/transactional_update.py +index 658ebccc6b..d6915475f5 100644 +--- a/salt/modules/transactional_update.py ++++ b/salt/modules/transactional_update.py +@@ -276,6 +276,9 @@ transaction. + """ + + import logging ++import os.path ++import pathlib ++import sys + + import salt.client.ssh.state + import salt.client.ssh.wrapper.state +@@ -941,10 +944,18 @@ def call(function, *args, **kwargs): + activate_transaction = kwargs.pop("activate_transaction", False) + + try: ++ # Set default salt-call command ++ salt_call_cmd = "salt-call" ++ python_exec_dir = os.path.dirname(sys.executable) ++ if "venv-salt-minion" in pathlib.Path(python_exec_dir).parts: ++ # If the module is executed with the Salt Bundle, ++ # use salt-call from the Salt Bundle ++ salt_call_cmd = os.path.join(python_exec_dir, "salt-call") ++ + safe_kwargs = salt.utils.args.clean_kwargs(**kwargs) + salt_argv = ( + [ +- "salt-call", ++ salt_call_cmd, + "--out", + "json", + "-l", +diff --git a/tests/pytests/unit/modules/test_transactional_update.py b/tests/pytests/unit/modules/test_transactional_update.py +index 5d9294c49b..dbd72fd74b 100644 +--- a/tests/pytests/unit/modules/test_transactional_update.py ++++ b/tests/pytests/unit/modules/test_transactional_update.py +@@ -670,3 +670,47 @@ def test_single_queue_true(): + "salt.modules.transactional_update.call", MagicMock(return_value="result") + ): + assert tu.single("pkg.installed", name="emacs", queue=True) == "result" ++ ++ ++@pytest.mark.parametrize( ++ "executable,salt_call_cmd", ++ [ ++ ("/usr/bin/python3", "salt-call"), ++ ( ++ "/usr/lib/venv-salt-minion/bin/python", ++ "/usr/lib/venv-salt-minion/bin/salt-call", ++ ), ++ ], ++) ++def test_call_which_salt_call_selected_with_executable(executable, salt_call_cmd): ++ """Test transactional_update.chroot which salt-call used""" ++ utils_mock = { ++ "json.find_json": MagicMock(return_value={"return": "result"}), ++ } ++ salt_mock = { ++ "cmd.run_all": MagicMock(return_value={"retcode": 0, "stdout": ""}), ++ } ++ with patch("sys.executable", executable), patch.dict( ++ tu.__utils__, utils_mock ++ ), patch.dict(tu.__salt__, salt_mock): ++ assert tu.call("test.ping") == "result" ++ ++ salt_mock["cmd.run_all"].assert_called_with( ++ [ ++ "transactional-update", ++ "--non-interactive", ++ "--drop-if-no-change", ++ "--no-selfupdate", ++ "--continue", ++ "--quiet", ++ "run", ++ salt_call_cmd, ++ "--out", ++ "json", ++ "-l", ++ "quiet", ++ "--no-return-event", ++ "--", ++ "test.ping", ++ ] ++ ) +-- +2.42.0 +