diff --git a/_lastrevision b/_lastrevision index 2786158..a693bb2 100644 --- a/_lastrevision +++ b/_lastrevision @@ -1 +1 @@ -d2d5f753be6e061cfb3d506641ceacd3b81b47f0 \ No newline at end of file +ca93a62c2cad9074f438fd562ea759079a0685c7 \ No newline at end of file diff --git a/dereference-symlinks-to-set-proper-__cli-opt-bsc-121.patch b/dereference-symlinks-to-set-proper-__cli-opt-bsc-121.patch new file mode 100644 index 0000000..55058a2 --- /dev/null +++ b/dereference-symlinks-to-set-proper-__cli-opt-bsc-121.patch @@ -0,0 +1,101 @@ +From 9942c488b1e74f2c6f187fcef3556fe53382bb4c Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Mon, 13 Nov 2023 15:04:14 +0000 +Subject: [PATCH] Dereference symlinks to set proper __cli opt + (bsc#1215963) (#611) + +* Dereference symlinks to set proper __cli + +* Add changelog entry + +* Add unit tests to check path is expanded + +--------- + +Co-authored-by: vzhestkov +--- + changelog/65435.fixed.md | 1 + + salt/config/__init__.py | 8 ++++++-- + tests/pytests/unit/config/test_master_config.py | 13 +++++++++++++ + tests/pytests/unit/config/test_minion_config.py | 13 +++++++++++++ + 4 files changed, 33 insertions(+), 2 deletions(-) + create mode 100644 changelog/65435.fixed.md + create mode 100644 tests/pytests/unit/config/test_master_config.py + create mode 100644 tests/pytests/unit/config/test_minion_config.py + +diff --git a/changelog/65435.fixed.md b/changelog/65435.fixed.md +new file mode 100644 +index 0000000000..5fa532891d +--- /dev/null ++++ b/changelog/65435.fixed.md +@@ -0,0 +1 @@ ++Dereference symlinks to set proper __cli opt +diff --git a/salt/config/__init__.py b/salt/config/__init__.py +index 43182f3f92..d8258a4dbc 100644 +--- a/salt/config/__init__.py ++++ b/salt/config/__init__.py +@@ -3747,7 +3747,9 @@ def apply_minion_config( + ) + opts["fileserver_backend"][idx] = new_val + +- opts["__cli"] = salt.utils.stringutils.to_unicode(os.path.basename(sys.argv[0])) ++ opts["__cli"] = salt.utils.stringutils.to_unicode( ++ os.path.basename(salt.utils.path.expand(sys.argv[0])) ++ ) + + # No ID provided. Will getfqdn save us? + using_ip_for_id = False +@@ -3949,7 +3951,9 @@ def apply_master_config(overrides=None, defaults=None): + ) + opts["keep_acl_in_token"] = True + +- opts["__cli"] = salt.utils.stringutils.to_unicode(os.path.basename(sys.argv[0])) ++ opts["__cli"] = salt.utils.stringutils.to_unicode( ++ os.path.basename(salt.utils.path.expand(sys.argv[0])) ++ ) + + if "environment" in opts: + if opts["saltenv"] is not None: +diff --git a/tests/pytests/unit/config/test_master_config.py b/tests/pytests/unit/config/test_master_config.py +new file mode 100644 +index 0000000000..c9de8a7892 +--- /dev/null ++++ b/tests/pytests/unit/config/test_master_config.py +@@ -0,0 +1,13 @@ ++import salt.config ++from tests.support.mock import MagicMock, patch ++ ++ ++def test___cli_path_is_expanded(): ++ defaults = salt.config.DEFAULT_MASTER_OPTS.copy() ++ overrides = {} ++ with patch( ++ "salt.utils.path.expand", MagicMock(return_value="/path/to/testcli") ++ ) as expand_mock: ++ opts = salt.config.apply_master_config(overrides, defaults) ++ assert expand_mock.called ++ assert opts["__cli"] == "testcli" +diff --git a/tests/pytests/unit/config/test_minion_config.py b/tests/pytests/unit/config/test_minion_config.py +new file mode 100644 +index 0000000000..34aa84daa7 +--- /dev/null ++++ b/tests/pytests/unit/config/test_minion_config.py +@@ -0,0 +1,13 @@ ++import salt.config ++from tests.support.mock import MagicMock, patch ++ ++ ++def test___cli_path_is_expanded(): ++ defaults = salt.config.DEFAULT_MINION_OPTS.copy() ++ overrides = {} ++ with patch( ++ "salt.utils.path.expand", MagicMock(return_value="/path/to/testcli") ++ ) as expand_mock: ++ opts = salt.config.apply_minion_config(overrides, defaults) ++ assert expand_mock.called ++ assert opts["__cli"] == "testcli" +-- +2.42.0 + + diff --git a/fix-gitfs-__env__-and-improve-cache-cleaning-bsc-119.patch b/fix-gitfs-__env__-and-improve-cache-cleaning-bsc-119.patch new file mode 100644 index 0000000..188743d --- /dev/null +++ b/fix-gitfs-__env__-and-improve-cache-cleaning-bsc-119.patch @@ -0,0 +1,2024 @@ +From a7c98ce490833ff232946b9715909161b6ba5a46 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Mon, 13 Nov 2023 11:24:09 +0000 +Subject: [PATCH] Fix gitfs "__env__" and improve cache cleaning + (bsc#1193948) (#608) + +* Fix __env__ and improve cache cleaning +* Move gitfs locks out of cachedir/.git + +--------- + +Co-authored-by: cmcmarrow +--- + changelog/65002.fixed.md | 1 + + changelog/65086.fixed.md | 1 + + salt/utils/cache.py | 32 ++ + salt/utils/gitfs.py | 307 +++++++++------ + .../functional/pillar/test_git_pillar.py | 262 +++++++++++++ + tests/pytests/functional/utils/test_cache.py | 83 ++++ + tests/pytests/functional/utils/test_gitfs.py | 275 +++++++++++++ + tests/pytests/functional/utils/test_pillar.py | 365 ++++++++++++++++++ + .../pytests/functional/utils/test_winrepo.py | 164 ++++++++ + tests/pytests/unit/test_minion.py | 31 +- + tests/pytests/unit/utils/test_gitfs.py | 18 +- + tests/unit/utils/test_gitfs.py | 21 +- + 12 files changed, 1393 insertions(+), 167 deletions(-) + create mode 100644 changelog/65002.fixed.md + create mode 100644 changelog/65086.fixed.md + create mode 100644 tests/pytests/functional/pillar/test_git_pillar.py + create mode 100644 tests/pytests/functional/utils/test_cache.py + create mode 100644 tests/pytests/functional/utils/test_gitfs.py + create mode 100644 tests/pytests/functional/utils/test_pillar.py + create mode 100644 tests/pytests/functional/utils/test_winrepo.py + +diff --git a/changelog/65002.fixed.md b/changelog/65002.fixed.md +new file mode 100644 +index 0000000000..86ed2d4bcc +--- /dev/null ++++ b/changelog/65002.fixed.md +@@ -0,0 +1 @@ ++Fix __env__ and improve cache cleaning see more info at pull #65017. +diff --git a/changelog/65086.fixed.md b/changelog/65086.fixed.md +new file mode 100644 +index 0000000000..292930f0fd +--- /dev/null ++++ b/changelog/65086.fixed.md +@@ -0,0 +1 @@ ++Moved gitfs locks to salt working dir to avoid lock wipes +diff --git a/salt/utils/cache.py b/salt/utils/cache.py +index a78a1f70fc..88e7fa2400 100644 +--- a/salt/utils/cache.py ++++ b/salt/utils/cache.py +@@ -6,6 +6,7 @@ import functools + import logging + import os + import re ++import shutil + import time + + import salt.config +@@ -15,6 +16,8 @@ import salt.utils.data + import salt.utils.dictupdate + import salt.utils.files + import salt.utils.msgpack ++import salt.utils.path ++import salt.version + from salt.utils.zeromq import zmq + + log = logging.getLogger(__name__) +@@ -345,3 +348,32 @@ def context_cache(func): + return func(*args, **kwargs) + + return context_cache_wrap ++ ++ ++def verify_cache_version(cache_path): ++ """ ++ Check that the cached version matches the Salt version. ++ If the cached version does not match the Salt version, wipe the cache. ++ ++ :return: ``True`` if cache version matches, otherwise ``False`` ++ """ ++ if not os.path.isdir(cache_path): ++ os.makedirs(cache_path) ++ with salt.utils.files.fopen( ++ salt.utils.path.join(cache_path, "cache_version"), "a+" ++ ) as file: ++ file.seek(0) ++ data = "\n".join(file.readlines()) ++ if data != salt.version.__version__: ++ log.warning(f"Cache version mismatch clearing: {repr(cache_path)}") ++ file.truncate(0) ++ file.write(salt.version.__version__) ++ for item in os.listdir(cache_path): ++ if item != "cache_version": ++ item_path = salt.utils.path.join(cache_path, item) ++ if os.path.isfile(item_path): ++ os.remove(item_path) ++ else: ++ shutil.rmtree(item_path) ++ return False ++ return True +diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py +index af61aa0dda..061647edac 100644 +--- a/salt/utils/gitfs.py ++++ b/salt/utils/gitfs.py +@@ -17,7 +17,6 @@ import os + import shlex + import shutil + import stat +-import string + import subprocess + import time + import weakref +@@ -26,6 +25,7 @@ from datetime import datetime + import salt.ext.tornado.ioloop + import salt.fileserver + import salt.syspaths ++import salt.utils.cache + import salt.utils.configparser + import salt.utils.data + import salt.utils.files +@@ -253,7 +253,6 @@ class GitProvider: + val_cb=lambda x, y: str(y), + ) + self.conf = copy.deepcopy(per_remote_defaults) +- + # Remove the 'salt://' from the beginning of any globally-defined + # per-saltenv mountpoints + for saltenv, saltenv_conf in self.global_saltenv.items(): +@@ -457,48 +456,38 @@ class GitProvider: + self.id, + ) + failhard(self.role) +- +- hash_type = getattr(hashlib, self.opts.get("hash_type", "md5")) +- # Generate full id. +- # Full id helps decrease the chances of collections in the gitfs cache. +- try: +- target = str(self.get_checkout_target()) +- except AttributeError: +- target = "" +- self._full_id = "-".join( +- [ +- getattr(self, "name", ""), +- self.id, +- getattr(self, "env", ""), +- getattr(self, "_root", ""), +- self.role, +- getattr(self, "base", ""), +- getattr(self, "branch", ""), +- target, +- ] ++ if hasattr(self, "name"): ++ self._cache_basehash = self.name ++ else: ++ hash_type = getattr(hashlib, self.opts.get("hash_type", "md5")) ++ # We loaded this data from yaml configuration files, so, its safe ++ # to use UTF-8 ++ self._cache_basehash = str( ++ base64.b64encode(hash_type(self.id.encode("utf-8")).digest()), ++ encoding="ascii", # base64 only outputs ascii ++ ).replace( ++ "/", "_" ++ ) # replace "/" with "_" to not cause trouble with file system ++ self._cache_hash = salt.utils.path.join(cache_root, self._cache_basehash) ++ self._cache_basename = "_" ++ if self.id.startswith("__env__"): ++ try: ++ self._cache_basename = self.get_checkout_target() ++ except AttributeError: ++ log.critical(f"__env__ cant generate basename: {self.role} {self.id}") ++ failhard(self.role) ++ self._cache_full_basename = salt.utils.path.join( ++ self._cache_basehash, self._cache_basename + ) +- # We loaded this data from yaml configuration files, so, its safe +- # to use UTF-8 +- base64_hash = str( +- base64.b64encode(hash_type(self._full_id.encode("utf-8")).digest()), +- encoding="ascii", # base64 only outputs ascii +- ).replace( +- "/", "_" +- ) # replace "/" with "_" to not cause trouble with file system +- +- # limit name length to 19, so we don't eat up all the path length for windows +- # this is due to pygit2 limitations +- # replace any unknown char with "_" to not cause trouble with file system +- name_chars = string.ascii_letters + string.digits + "-" +- cache_name = "".join( +- c if c in name_chars else "_" for c in getattr(self, "name", "")[:19] ++ self._cachedir = salt.utils.path.join(self._cache_hash, self._cache_basename) ++ self._salt_working_dir = salt.utils.path.join( ++ cache_root, "work", self._cache_full_basename + ) +- +- self.cachedir_basename = f"{cache_name}-{base64_hash}" +- self.cachedir = salt.utils.path.join(cache_root, self.cachedir_basename) +- self.linkdir = salt.utils.path.join(cache_root, "links", self.cachedir_basename) +- if not os.path.isdir(self.cachedir): +- os.makedirs(self.cachedir) ++ self._linkdir = salt.utils.path.join( ++ cache_root, "links", self._cache_full_basename ++ ) ++ if not os.path.isdir(self._cachedir): ++ os.makedirs(self._cachedir) + + try: + self.new = self.init_remote() +@@ -510,12 +499,32 @@ class GitProvider: + msg += " Perhaps git is not available." + log.critical(msg, exc_info=True) + failhard(self.role) ++ self.verify_auth() ++ self.setup_callbacks() ++ if not os.path.isdir(self._salt_working_dir): ++ os.makedirs(self._salt_working_dir) ++ self.fetch_request_check() ++ ++ def get_cache_basehash(self): ++ return self._cache_basehash ++ ++ def get_cache_hash(self): ++ return self._cache_hash + +- def full_id(self): +- return self._full_id ++ def get_cache_basename(self): ++ return self._cache_basename + +- def get_cachedir_basename(self): +- return self.cachedir_basename ++ def get_cache_full_basename(self): ++ return self._cache_full_basename ++ ++ def get_cachedir(self): ++ return self._cachedir ++ ++ def get_linkdir(self): ++ return self._linkdir ++ ++ def get_salt_working_dir(self): ++ return self._salt_working_dir + + def _get_envs_from_ref_paths(self, refs): + """ +@@ -557,7 +566,7 @@ class GitProvider: + return ret + + def _get_lock_file(self, lock_type="update"): +- return salt.utils.path.join(self.gitdir, lock_type + ".lk") ++ return salt.utils.path.join(self._salt_working_dir, lock_type + ".lk") + + @classmethod + def add_conf_overlay(cls, name): +@@ -644,7 +653,7 @@ class GitProvider: + # No need to pass an environment to self.root() here since per-saltenv + # configuration is a gitfs-only feature and check_root() is not used + # for gitfs. +- root_dir = salt.utils.path.join(self.cachedir, self.root()).rstrip(os.sep) ++ root_dir = salt.utils.path.join(self._cachedir, self.root()).rstrip(os.sep) + if os.path.isdir(root_dir): + return root_dir + log.error( +@@ -816,7 +825,7 @@ class GitProvider: + desired_refspecs, + ) + if refspecs != desired_refspecs: +- conf.set_multivar(remote_section, "fetch", self.refspecs) ++ conf.set_multivar(remote_section, "fetch", desired_refspecs) + log.debug( + "Refspecs for %s remote '%s' set to %s", + self.role, +@@ -1069,7 +1078,7 @@ class GitProvider: + """ + raise NotImplementedError() + +- def checkout(self): ++ def checkout(self, fetch_on_fail=True): + """ + This function must be overridden in a sub-class + """ +@@ -1192,6 +1201,21 @@ class GitProvider: + else: + self.url = self.id + ++ def fetch_request_check(self): ++ fetch_request = salt.utils.path.join(self._salt_working_dir, "fetch_request") ++ if os.path.isfile(fetch_request): ++ log.debug(f"Fetch request: {self._salt_working_dir}") ++ try: ++ os.remove(fetch_request) ++ except OSError as exc: ++ log.error( ++ f"Failed to remove Fetch request: {self._salt_working_dir} {exc}", ++ exc_info=True, ++ ) ++ self.fetch() ++ return True ++ return False ++ + @property + def linkdir_walk(self): + """ +@@ -1218,14 +1242,14 @@ class GitProvider: + dirs = [] + self._linkdir_walk.append( + ( +- salt.utils.path.join(self.linkdir, *parts[: idx + 1]), ++ salt.utils.path.join(self._linkdir, *parts[: idx + 1]), + dirs, + [], + ) + ) + try: + # The linkdir itself goes at the beginning +- self._linkdir_walk.insert(0, (self.linkdir, [parts[0]], [])) ++ self._linkdir_walk.insert(0, (self._linkdir, [parts[0]], [])) + except IndexError: + pass + return self._linkdir_walk +@@ -1275,13 +1299,17 @@ class GitPython(GitProvider): + role, + ) + +- def checkout(self): ++ def checkout(self, fetch_on_fail=True): + """ + Checkout the configured branch/tag. We catch an "Exception" class here + instead of a specific exception class because the exceptions raised by + GitPython when running these functions vary in different versions of + GitPython. ++ ++ fetch_on_fail ++ If checkout fails perform a fetch then try to checkout again. + """ ++ self.fetch_request_check() + tgt_ref = self.get_checkout_target() + try: + head_sha = self.repo.rev_parse("HEAD").hexsha +@@ -1345,6 +1373,15 @@ class GitPython(GitProvider): + except Exception: # pylint: disable=broad-except + continue + return self.check_root() ++ if fetch_on_fail: ++ log.debug( ++ "Failed to checkout %s from %s remote '%s': fetch and try again", ++ tgt_ref, ++ self.role, ++ self.id, ++ ) ++ self.fetch() ++ return self.checkout(fetch_on_fail=False) + log.error( + "Failed to checkout %s from %s remote '%s': remote ref does not exist", + tgt_ref, +@@ -1360,16 +1397,16 @@ class GitPython(GitProvider): + initialized by this function. + """ + new = False +- if not os.listdir(self.cachedir): ++ if not os.listdir(self._cachedir): + # Repo cachedir is empty, initialize a new repo there +- self.repo = git.Repo.init(self.cachedir) ++ self.repo = git.Repo.init(self._cachedir) + new = True + else: + # Repo cachedir exists, try to attach + try: +- self.repo = git.Repo(self.cachedir) ++ self.repo = git.Repo(self._cachedir) + except git.exc.InvalidGitRepositoryError: +- log.error(_INVALID_REPO, self.cachedir, self.url, self.role) ++ log.error(_INVALID_REPO, self._cachedir, self.url, self.role) + return new + + self.gitdir = salt.utils.path.join(self.repo.working_dir, ".git") +@@ -1603,10 +1640,14 @@ class Pygit2(GitProvider): + except AttributeError: + return obj.get_object() + +- def checkout(self): ++ def checkout(self, fetch_on_fail=True): + """ + Checkout the configured branch/tag ++ ++ fetch_on_fail ++ If checkout fails perform a fetch then try to checkout again. + """ ++ self.fetch_request_check() + tgt_ref = self.get_checkout_target() + local_ref = "refs/heads/" + tgt_ref + remote_ref = "refs/remotes/origin/" + tgt_ref +@@ -1796,6 +1837,15 @@ class Pygit2(GitProvider): + exc_info=True, + ) + return None ++ if fetch_on_fail: ++ log.debug( ++ "Failed to checkout %s from %s remote '%s': fetch and try again", ++ tgt_ref, ++ self.role, ++ self.id, ++ ) ++ self.fetch() ++ return self.checkout(fetch_on_fail=False) + log.error( + "Failed to checkout %s from %s remote '%s': remote ref does not exist", + tgt_ref, +@@ -1837,16 +1887,16 @@ class Pygit2(GitProvider): + home = os.path.expanduser("~") + pygit2.settings.search_path[pygit2.GIT_CONFIG_LEVEL_GLOBAL] = home + new = False +- if not os.listdir(self.cachedir): ++ if not os.listdir(self._cachedir): + # Repo cachedir is empty, initialize a new repo there +- self.repo = pygit2.init_repository(self.cachedir) ++ self.repo = pygit2.init_repository(self._cachedir) + new = True + else: + # Repo cachedir exists, try to attach + try: +- self.repo = pygit2.Repository(self.cachedir) ++ self.repo = pygit2.Repository(self._cachedir) + except KeyError: +- log.error(_INVALID_REPO, self.cachedir, self.url, self.role) ++ log.error(_INVALID_REPO, self._cachedir, self.url, self.role) + return new + + self.gitdir = salt.utils.path.join(self.repo.workdir, ".git") +@@ -2370,6 +2420,7 @@ class GitBase: + self.file_list_cachedir = salt.utils.path.join( + self.opts["cachedir"], "file_lists", self.role + ) ++ salt.utils.cache.verify_cache_version(self.cache_root) + if init_remotes: + self.init_remotes( + remotes if remotes is not None else [], +@@ -2442,8 +2493,6 @@ class GitBase: + ) + if hasattr(repo_obj, "repo"): + # Sanity check and assign the credential parameter +- repo_obj.verify_auth() +- repo_obj.setup_callbacks() + if self.opts["__role"] == "minion" and repo_obj.new: + # Perform initial fetch on masterless minion + repo_obj.fetch() +@@ -2492,7 +2541,7 @@ class GitBase: + # Don't allow collisions in cachedir naming + cachedir_map = {} + for repo in self.remotes: +- cachedir_map.setdefault(repo.cachedir, []).append(repo.id) ++ cachedir_map.setdefault(repo.get_cachedir(), []).append(repo.id) + + collisions = [x for x in cachedir_map if len(cachedir_map[x]) > 1] + if collisions: +@@ -2509,48 +2558,42 @@ class GitBase: + if any(x.new for x in self.remotes): + self.write_remote_map() + ++ def _remove_cache_dir(self, cache_dir): ++ try: ++ shutil.rmtree(cache_dir) ++ except OSError as exc: ++ log.error( ++ "Unable to remove old %s remote cachedir %s: %s", ++ self.role, ++ cache_dir, ++ exc, ++ ) ++ return False ++ log.debug("%s removed old cachedir %s", self.role, cache_dir) ++ return True ++ ++ def _iter_remote_hashes(self): ++ for item in os.listdir(self.cache_root): ++ if item in ("hash", "refs", "links", "work"): ++ continue ++ if os.path.isdir(salt.utils.path.join(self.cache_root, item)): ++ yield item ++ + def clear_old_remotes(self): + """ + Remove cache directories for remotes no longer configured + """ +- try: +- cachedir_ls = os.listdir(self.cache_root) +- except OSError: +- cachedir_ls = [] +- # Remove actively-used remotes from list +- for repo in self.remotes: +- try: +- cachedir_ls.remove(repo.cachedir_basename) +- except ValueError: +- pass +- to_remove = [] +- for item in cachedir_ls: +- if item in ("hash", "refs"): +- continue +- path = salt.utils.path.join(self.cache_root, item) +- if os.path.isdir(path): +- to_remove.append(path) +- failed = [] +- if to_remove: +- for rdir in to_remove: +- try: +- shutil.rmtree(rdir) +- except OSError as exc: +- log.error( +- "Unable to remove old %s remote cachedir %s: %s", +- self.role, +- rdir, +- exc, +- ) +- failed.append(rdir) +- else: +- log.debug("%s removed old cachedir %s", self.role, rdir) +- for fdir in failed: +- to_remove.remove(fdir) +- ret = bool(to_remove) +- if ret: ++ change = False ++ # Remove all hash dirs not part of this group ++ remote_set = {r.get_cache_basehash() for r in self.remotes} ++ for item in self._iter_remote_hashes(): ++ if item not in remote_set: ++ change = self._remove_cache_dir( ++ salt.utils.path.join(self.cache_root, item) or change ++ ) ++ if not change: + self.write_remote_map() +- return ret ++ return change + + def clear_cache(self): + """ +@@ -2609,6 +2652,27 @@ class GitBase: + name = getattr(repo, "name", None) + if not remotes or (repo.id, name) in remotes or name in remotes: + try: ++ # Find and place fetch_request file for all the other branches for this repo ++ repo_work_hash = os.path.split(repo.get_salt_working_dir())[0] ++ for branch in os.listdir(repo_work_hash): ++ # Don't place fetch request in current branch being updated ++ if branch == repo.get_cache_basename(): ++ continue ++ branch_salt_dir = salt.utils.path.join(repo_work_hash, branch) ++ fetch_path = salt.utils.path.join( ++ branch_salt_dir, "fetch_request" ++ ) ++ if os.path.isdir(branch_salt_dir): ++ try: ++ with salt.utils.files.fopen(fetch_path, "w"): ++ pass ++ except OSError as exc: # pylint: disable=broad-except ++ log.error( ++ f"Failed to make fetch request: {fetch_path} {exc}", ++ exc_info=True, ++ ) ++ else: ++ log.error(f"Failed to make fetch request: {fetch_path}") + if repo.fetch(): + # We can't just use the return value from repo.fetch() + # because the data could still have changed if old +@@ -2863,7 +2927,7 @@ class GitBase: + for repo in self.remotes: + fp_.write( + salt.utils.stringutils.to_str( +- "{} = {}\n".format(repo.cachedir_basename, repo.id) ++ "{} = {}\n".format(repo.get_cache_basehash(), repo.id) + ) + ) + except OSError: +@@ -2871,15 +2935,18 @@ class GitBase: + else: + log.info("Wrote new %s remote map to %s", self.role, remote_map) + +- def do_checkout(self, repo): ++ def do_checkout(self, repo, fetch_on_fail=True): + """ + Common code for git_pillar/winrepo to handle locking and checking out + of a repo. ++ ++ fetch_on_fail ++ If checkout fails perform a fetch then try to checkout again. + """ + time_start = time.time() + while time.time() - time_start <= 5: + try: +- return repo.checkout() ++ return repo.checkout(fetch_on_fail=fetch_on_fail) + except GitLockError as exc: + if exc.errno == errno.EEXIST: + time.sleep(0.1) +@@ -3274,14 +3341,17 @@ class GitPillar(GitBase): + + role = "git_pillar" + +- def checkout(self): ++ def checkout(self, fetch_on_fail=True): + """ + Checkout the targeted branches/tags from the git_pillar remotes ++ ++ fetch_on_fail ++ If checkout fails perform a fetch then try to checkout again. + """ + self.pillar_dirs = OrderedDict() + self.pillar_linked_dirs = [] + for repo in self.remotes: +- cachedir = self.do_checkout(repo) ++ cachedir = self.do_checkout(repo, fetch_on_fail=fetch_on_fail) + if cachedir is not None: + # Figure out which environment this remote should be assigned + if repo.branch == "__env__" and hasattr(repo, "all_saltenvs"): +@@ -3298,8 +3368,8 @@ class GitPillar(GitBase): + env = "base" if tgt == repo.base else tgt + if repo._mountpoint: + if self.link_mountpoint(repo): +- self.pillar_dirs[repo.linkdir] = env +- self.pillar_linked_dirs.append(repo.linkdir) ++ self.pillar_dirs[repo.get_linkdir()] = env ++ self.pillar_linked_dirs.append(repo.get_linkdir()) + else: + self.pillar_dirs[cachedir] = env + +@@ -3308,17 +3378,19 @@ class GitPillar(GitBase): + Ensure that the mountpoint is present in the correct location and + points at the correct path + """ +- lcachelink = salt.utils.path.join(repo.linkdir, repo._mountpoint) +- lcachedest = salt.utils.path.join(repo.cachedir, repo.root()).rstrip(os.sep) ++ lcachelink = salt.utils.path.join(repo.get_linkdir(), repo._mountpoint) ++ lcachedest = salt.utils.path.join(repo.get_cachedir(), repo.root()).rstrip( ++ os.sep ++ ) + wipe_linkdir = False + create_link = False + try: + with repo.gen_lock(lock_type="mountpoint", timeout=10): +- walk_results = list(os.walk(repo.linkdir, followlinks=False)) ++ walk_results = list(os.walk(repo.get_linkdir(), followlinks=False)) + if walk_results != repo.linkdir_walk: + log.debug( + "Results of walking %s differ from expected results", +- repo.linkdir, ++ repo.get_linkdir(), + ) + log.debug("Walk results: %s", walk_results) + log.debug("Expected results: %s", repo.linkdir_walk) +@@ -3379,7 +3451,7 @@ class GitPillar(GitBase): + # Wiping implies that we need to create the link + create_link = True + try: +- shutil.rmtree(repo.linkdir) ++ shutil.rmtree(repo.get_linkdir()) + except OSError: + pass + try: +@@ -3431,6 +3503,9 @@ class GitPillar(GitBase): + class WinRepo(GitBase): + """ + Functionality specific to the winrepo runner ++ ++ fetch_on_fail ++ If checkout fails perform a fetch then try to checkout again. + """ + + role = "winrepo" +@@ -3438,12 +3513,12 @@ class WinRepo(GitBase): + # out the repos. + winrepo_dirs = {} + +- def checkout(self): ++ def checkout(self, fetch_on_fail=True): + """ + Checkout the targeted branches/tags from the winrepo remotes + """ + self.winrepo_dirs = {} + for repo in self.remotes: +- cachedir = self.do_checkout(repo) ++ cachedir = self.do_checkout(repo, fetch_on_fail=fetch_on_fail) + if cachedir is not None: + self.winrepo_dirs[repo.id] = cachedir +diff --git a/tests/pytests/functional/pillar/test_git_pillar.py b/tests/pytests/functional/pillar/test_git_pillar.py +new file mode 100644 +index 0000000000..6fd3dee431 +--- /dev/null ++++ b/tests/pytests/functional/pillar/test_git_pillar.py +@@ -0,0 +1,262 @@ ++import pytest ++ ++from salt.pillar.git_pillar import ext_pillar ++from salt.utils.immutabletypes import ImmutableDict, ImmutableList ++from tests.support.mock import patch ++ ++pytestmark = [ ++ pytest.mark.slow_test, ++] ++ ++ ++try: ++ import git # pylint: disable=unused-import ++ ++ HAS_GITPYTHON = True ++except ImportError: ++ HAS_GITPYTHON = False ++ ++ ++try: ++ import pygit2 # pylint: disable=unused-import ++ ++ HAS_PYGIT2 = True ++except ImportError: ++ HAS_PYGIT2 = False ++ ++ ++skipif_no_gitpython = pytest.mark.skipif(not HAS_GITPYTHON, reason="Missing gitpython") ++skipif_no_pygit2 = pytest.mark.skipif(not HAS_PYGIT2, reason="Missing pygit2") ++ ++ ++@pytest.fixture ++def git_pillar_opts(salt_master, tmp_path): ++ opts = dict(salt_master.config) ++ opts["cachedir"] = str(tmp_path) ++ for key, item in opts.items(): ++ if isinstance(item, ImmutableDict): ++ opts[key] = dict(item) ++ elif isinstance(item, ImmutableList): ++ opts[key] = list(item) ++ return opts ++ ++ ++@pytest.fixture ++def gitpython_pillar_opts(git_pillar_opts): ++ git_pillar_opts["verified_git_pillar_provider"] = "gitpython" ++ return git_pillar_opts ++ ++ ++@pytest.fixture ++def pygit2_pillar_opts(git_pillar_opts): ++ git_pillar_opts["verified_git_pillar_provider"] = "pygit2" ++ return git_pillar_opts ++ ++ ++def _get_ext_pillar(minion, pillar_opts, grains, *repos): ++ with patch("salt.pillar.git_pillar.__opts__", pillar_opts, create=True): ++ with patch("salt.pillar.git_pillar.__grains__", grains, create=True): ++ return ext_pillar(minion, None, *repos) ++ ++ ++def _test_simple(pillar_opts, grains): ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ assert data == {"key": "value"} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_simple(gitpython_pillar_opts, grains): ++ _test_simple(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_simple(pygit2_pillar_opts, grains): ++ _test_simple(pygit2_pillar_opts, grains) ++ ++ ++def _test_missing_env(pillar_opts, grains): ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ { ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git": [ ++ {"env": "misssing"} ++ ] ++ }, ++ ) ++ assert data == {} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_missing_env(gitpython_pillar_opts, grains): ++ _test_missing_env(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_missing_env(pygit2_pillar_opts, grains): ++ _test_missing_env(pygit2_pillar_opts, grains) ++ ++ ++def _test_env(pillar_opts, grains): ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ { ++ "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git": [ ++ {"env": "other_env"} ++ ] ++ }, ++ ) ++ assert data == {"other": "env"} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_env(gitpython_pillar_opts, grains): ++ _test_env(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_env(pygit2_pillar_opts, grains): ++ _test_env(pygit2_pillar_opts, grains) ++ ++ ++def _test_branch(pillar_opts, grains): ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ "branch https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ assert data == {"key": "data"} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_branch(gitpython_pillar_opts, grains): ++ _test_branch(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_branch(pygit2_pillar_opts, grains): ++ _test_branch(pygit2_pillar_opts, grains) ++ ++ ++def _test_simple_dynamic(pillar_opts, grains): ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ assert data == {"key": "value"} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_simple_dynamic(gitpython_pillar_opts, grains): ++ _test_simple_dynamic(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_simple_dynamic(pygit2_pillar_opts, grains): ++ _test_simple_dynamic(pygit2_pillar_opts, grains) ++ ++ ++def _test_missing_env_dynamic(pillar_opts, grains): ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ { ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git": [ ++ {"env": "misssing"} ++ ] ++ }, ++ ) ++ assert data == {} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_missing_env_dynamic(gitpython_pillar_opts, grains): ++ _test_missing_env_dynamic(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_missing_env_dynamic(pygit2_pillar_opts, grains): ++ _test_missing_env_dynamic(pygit2_pillar_opts, grains) ++ ++ ++def _test_pillarenv_dynamic(pillar_opts, grains): ++ pillar_opts["pillarenv"] = "branch" ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ assert data == {"key": "data"} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_pillarenv_dynamic(gitpython_pillar_opts, grains): ++ _test_pillarenv_dynamic(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_pillarenv_dynamic(pygit2_pillar_opts, grains): ++ _test_pillarenv_dynamic(pygit2_pillar_opts, grains) ++ ++ ++def _test_multiple(pillar_opts, grains): ++ pillar_opts["pillarenv"] = "branch" ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ ) ++ assert data == {"key": "data"} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_multiple(gitpython_pillar_opts, grains): ++ _test_multiple(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_multiple(pygit2_pillar_opts, grains): ++ _test_multiple(pygit2_pillar_opts, grains) ++ ++ ++def _test_multiple_2(pillar_opts, grains): ++ data = _get_ext_pillar( ++ "minion", ++ pillar_opts, ++ grains, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ ) ++ assert data == { ++ "key": "value", ++ "key1": "value1", ++ "key2": "value2", ++ "key4": "value4", ++ "data1": "d", ++ "data2": "d2", ++ } ++ ++ ++@skipif_no_gitpython ++def test_gitpython_multiple_2(gitpython_pillar_opts, grains): ++ _test_multiple_2(gitpython_pillar_opts, grains) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_multiple_2(pygit2_pillar_opts, grains): ++ _test_multiple_2(pygit2_pillar_opts, grains) +diff --git a/tests/pytests/functional/utils/test_cache.py b/tests/pytests/functional/utils/test_cache.py +new file mode 100644 +index 0000000000..d405b8246f +--- /dev/null ++++ b/tests/pytests/functional/utils/test_cache.py +@@ -0,0 +1,83 @@ ++import os ++ ++import pytest ++ ++import salt.utils.cache ++import salt.utils.files ++import salt.utils.path ++import salt.version ++ ++_DUMMY_FILES = ( ++ "data.txt", ++ "foo.t2", ++ "bar.t3", ++ "nested/test", ++ "nested/cache.txt", ++ "n/n1/n2/n3/n4/n5", ++) ++ ++ ++def _make_dummy_files(tmp_path): ++ for full_path in _DUMMY_FILES: ++ full_path = salt.utils.path.join(tmp_path, full_path) ++ path, _ = os.path.split(full_path) ++ if not os.path.isdir(path): ++ os.makedirs(path) ++ with salt.utils.files.fopen(full_path, "w") as file: ++ file.write("data") ++ ++ ++def _dummy_files_exists(tmp_path): ++ """ ++ True if all files exists ++ False if all files are missing ++ None if some files exists and others are missing ++ """ ++ ret = None ++ for full_path in _DUMMY_FILES: ++ full_path = salt.utils.path.join(tmp_path, full_path) ++ is_file = os.path.isfile(full_path) ++ if ret is None: ++ ret = is_file ++ elif ret is not is_file: ++ return None # Some files are found and others are missing ++ return ret ++ ++ ++def test_verify_cache_version_bad_path(): ++ with pytest.raises(ValueError): ++ # cache version should fail if given bad file python ++ salt.utils.cache.verify_cache_version("\0/bad/path") ++ ++ ++def test_verify_cache_version(tmp_path): ++ # cache version should make dir if it does not exist ++ tmp_path = str(salt.utils.path.join(str(tmp_path), "work", "salt")) ++ cache_version = salt.utils.path.join(tmp_path, "cache_version") ++ ++ # check that cache clears when no cache_version is present ++ _make_dummy_files(tmp_path) ++ assert salt.utils.cache.verify_cache_version(tmp_path) is False ++ assert _dummy_files_exists(tmp_path) is False ++ ++ # check that cache_version has correct salt version ++ with salt.utils.files.fopen(cache_version, "r") as file: ++ assert "\n".join(file.readlines()) == salt.version.__version__ ++ ++ # check that cache does not get clear when check is called multiple times ++ _make_dummy_files(tmp_path) ++ for _ in range(3): ++ assert salt.utils.cache.verify_cache_version(tmp_path) is True ++ assert _dummy_files_exists(tmp_path) is True ++ ++ # check that cache clears when a different version is present ++ with salt.utils.files.fopen(cache_version, "w") as file: ++ file.write("-1") ++ assert salt.utils.cache.verify_cache_version(tmp_path) is False ++ assert _dummy_files_exists(tmp_path) is False ++ ++ # check that cache does not get clear when check is called multiple times ++ _make_dummy_files(tmp_path) ++ for _ in range(3): ++ assert salt.utils.cache.verify_cache_version(tmp_path) is True ++ assert _dummy_files_exists(tmp_path) is True +diff --git a/tests/pytests/functional/utils/test_gitfs.py b/tests/pytests/functional/utils/test_gitfs.py +new file mode 100644 +index 0000000000..30a5f147fa +--- /dev/null ++++ b/tests/pytests/functional/utils/test_gitfs.py +@@ -0,0 +1,275 @@ ++import os.path ++ ++import pytest ++ ++from salt.fileserver.gitfs import PER_REMOTE_ONLY, PER_REMOTE_OVERRIDES ++from salt.utils.gitfs import GitFS, GitPython, Pygit2 ++from salt.utils.immutabletypes import ImmutableDict, ImmutableList ++ ++pytestmark = [ ++ pytest.mark.slow_test, ++] ++ ++ ++try: ++ import git # pylint: disable=unused-import ++ ++ HAS_GITPYTHON = True ++except ImportError: ++ HAS_GITPYTHON = False ++ ++ ++try: ++ import pygit2 # pylint: disable=unused-import ++ ++ HAS_PYGIT2 = True ++except ImportError: ++ HAS_PYGIT2 = False ++ ++ ++skipif_no_gitpython = pytest.mark.skipif(not HAS_GITPYTHON, reason="Missing gitpython") ++skipif_no_pygit2 = pytest.mark.skipif(not HAS_PYGIT2, reason="Missing pygit2") ++ ++ ++@pytest.fixture ++def gitfs_opts(salt_factories, tmp_path): ++ config_defaults = {"cachedir": str(tmp_path)} ++ factory = salt_factories.salt_master_daemon( ++ "gitfs-functional-master", defaults=config_defaults ++ ) ++ config_defaults = dict(factory.config) ++ for key, item in config_defaults.items(): ++ if isinstance(item, ImmutableDict): ++ config_defaults[key] = dict(item) ++ elif isinstance(item, ImmutableList): ++ config_defaults[key] = list(item) ++ return config_defaults ++ ++ ++@pytest.fixture ++def gitpython_gitfs_opts(gitfs_opts): ++ gitfs_opts["verified_gitfs_provider"] = "gitpython" ++ GitFS.instance_map.clear() # wipe instance_map object map for clean run ++ return gitfs_opts ++ ++ ++@pytest.fixture ++def pygit2_gitfs_opts(gitfs_opts): ++ gitfs_opts["verified_gitfs_provider"] = "pygit2" ++ GitFS.instance_map.clear() # wipe instance_map object map for clean run ++ return gitfs_opts ++ ++ ++def _get_gitfs(opts, *remotes): ++ return GitFS( ++ opts, ++ remotes, ++ per_remote_overrides=PER_REMOTE_OVERRIDES, ++ per_remote_only=PER_REMOTE_ONLY, ++ ) ++ ++ ++def _test_gitfs_simple(gitfs_opts): ++ g = _get_gitfs( ++ gitfs_opts, ++ {"https://github.com/saltstack/salt-test-pillar-gitfs.git": [{"name": "bob"}]}, ++ ) ++ g.fetch_remotes() ++ assert len(g.remotes) == 1 ++ assert set(g.file_list({"saltenv": "main"})) == {".gitignore", "README.md"} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_gitfs_simple(gitpython_gitfs_opts): ++ _test_gitfs_simple(gitpython_gitfs_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_gitfs_simple(pygit2_gitfs_opts): ++ _test_gitfs_simple(pygit2_gitfs_opts) ++ ++ ++def _test_gitfs_simple_base(gitfs_opts): ++ g = _get_gitfs( ++ gitfs_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ g.fetch_remotes() ++ assert len(g.remotes) == 1 ++ assert set(g.file_list({"saltenv": "base"})) == { ++ ".gitignore", ++ "README.md", ++ "file.sls", ++ "top.sls", ++ } ++ ++ ++@skipif_no_gitpython ++def test_gitpython_gitfs_simple_base(gitpython_gitfs_opts): ++ _test_gitfs_simple_base(gitpython_gitfs_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_gitfs_simple_base(pygit2_gitfs_opts): ++ _test_gitfs_simple_base(pygit2_gitfs_opts) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_gitfs_provider(gitpython_gitfs_opts): ++ g = _get_gitfs( ++ gitpython_gitfs_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ assert len(g.remotes) == 1 ++ assert g.provider == "gitpython" ++ assert isinstance(g.remotes[0], GitPython) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_gitfs_provider(pygit2_gitfs_opts): ++ g = _get_gitfs( ++ pygit2_gitfs_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ assert len(g.remotes) == 1 ++ assert g.provider == "pygit2" ++ assert isinstance(g.remotes[0], Pygit2) ++ ++ ++def _test_gitfs_minion(gitfs_opts): ++ gitfs_opts["__role"] = "minion" ++ g = _get_gitfs( ++ gitfs_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ g.fetch_remotes() ++ assert len(g.remotes) == 1 ++ assert set(g.file_list({"saltenv": "base"})) == { ++ ".gitignore", ++ "README.md", ++ "file.sls", ++ "top.sls", ++ } ++ assert set(g.file_list({"saltenv": "main"})) == {".gitignore", "README.md"} ++ ++ ++@skipif_no_gitpython ++def test_gitpython_gitfs_minion(gitpython_gitfs_opts): ++ _test_gitfs_minion(gitpython_gitfs_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_gitfs_minion(pygit2_gitfs_opts): ++ _test_gitfs_minion(pygit2_gitfs_opts) ++ ++ ++def _test_fetch_request_with_mountpoint(opts): ++ mpoint = [{"mountpoint": "salt/m"}] ++ p = _get_gitfs( ++ opts, ++ {"https://github.com/saltstack/salt-test-pillar-gitfs.git": mpoint}, ++ ) ++ p.fetch_remotes() ++ assert len(p.remotes) == 1 ++ repo = p.remotes[0] ++ assert repo.mountpoint("testmount") == "salt/m" ++ assert set(p.file_list({"saltenv": "testmount"})) == { ++ "salt/m/test_dir1/testfile3", ++ "salt/m/test_dir1/test_dir2/testfile2", ++ "salt/m/.gitignore", ++ "salt/m/README.md", ++ "salt/m/test_dir1/test_dir2/testfile1", ++ } ++ ++ ++@skipif_no_gitpython ++def test_gitpython_fetch_request_with_mountpoint(gitpython_gitfs_opts): ++ _test_fetch_request_with_mountpoint(gitpython_gitfs_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_fetch_request_with_mountpoint(pygit2_gitfs_opts): ++ _test_fetch_request_with_mountpoint(pygit2_gitfs_opts) ++ ++ ++def _test_name(opts): ++ p = _get_gitfs( ++ opts, ++ { ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git": [ ++ {"name": "name1"} ++ ] ++ }, ++ { ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git": [ ++ {"name": "name2"} ++ ] ++ }, ++ ) ++ p.fetch_remotes() ++ assert len(p.remotes) == 2 ++ repo = p.remotes[0] ++ repo2 = p.remotes[1] ++ assert repo.get_cache_basehash() == "name1" ++ assert repo2.get_cache_basehash() == "name2" ++ ++ ++@skipif_no_gitpython ++def test_gitpython_name(gitpython_gitfs_opts): ++ _test_name(gitpython_gitfs_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_name(pygit2_gitfs_opts): ++ _test_name(pygit2_gitfs_opts) ++ ++ ++def _test_remote_map(opts): ++ p = _get_gitfs( ++ opts, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ p.fetch_remotes() ++ assert len(p.remotes) == 1 ++ assert os.path.isfile(os.path.join(opts["cachedir"], "gitfs", "remote_map.txt")) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_remote_map(gitpython_gitfs_opts): ++ _test_remote_map(gitpython_gitfs_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_remote_map(pygit2_gitfs_opts): ++ _test_remote_map(pygit2_gitfs_opts) ++ ++ ++def _test_lock(opts): ++ g = _get_gitfs( ++ opts, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ g.fetch_remotes() ++ assert len(g.remotes) == 1 ++ repo = g.remotes[0] ++ assert repo.get_salt_working_dir() in repo._get_lock_file() ++ assert repo.lock() == ( ++ [ ++ "Set update lock for gitfs remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" ++ ], ++ [], ++ ) ++ assert os.path.isfile(repo._get_lock_file()) ++ assert repo.clear_lock() == ( ++ [ ++ "Removed update lock for gitfs remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" ++ ], ++ [], ++ ) ++ assert not os.path.isfile(repo._get_lock_file()) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_lock(gitpython_gitfs_opts): ++ _test_lock(gitpython_gitfs_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_lock(pygit2_gitfs_opts): ++ _test_lock(pygit2_gitfs_opts) +diff --git a/tests/pytests/functional/utils/test_pillar.py b/tests/pytests/functional/utils/test_pillar.py +new file mode 100644 +index 0000000000..143edbf6ff +--- /dev/null ++++ b/tests/pytests/functional/utils/test_pillar.py +@@ -0,0 +1,365 @@ ++import os ++ ++import pytest ++ ++from salt.pillar.git_pillar import GLOBAL_ONLY, PER_REMOTE_ONLY, PER_REMOTE_OVERRIDES ++from salt.utils.gitfs import GitPillar, GitPython, Pygit2 ++from salt.utils.immutabletypes import ImmutableDict, ImmutableList ++ ++pytestmark = [ ++ pytest.mark.slow_test, ++] ++ ++ ++try: ++ import git # pylint: disable=unused-import ++ ++ HAS_GITPYTHON = True ++except ImportError: ++ HAS_GITPYTHON = False ++ ++ ++try: ++ import pygit2 # pylint: disable=unused-import ++ ++ HAS_PYGIT2 = True ++except ImportError: ++ HAS_PYGIT2 = False ++ ++ ++skipif_no_gitpython = pytest.mark.skipif(not HAS_GITPYTHON, reason="Missing gitpython") ++skipif_no_pygit2 = pytest.mark.skipif(not HAS_PYGIT2, reason="Missing pygit2") ++ ++ ++@pytest.fixture ++def pillar_opts(salt_factories, tmp_path): ++ config_defaults = {"cachedir": str(tmp_path)} ++ factory = salt_factories.salt_master_daemon( ++ "pillar-functional-master", defaults=config_defaults ++ ) ++ config_defaults = dict(factory.config) ++ for key, item in config_defaults.items(): ++ if isinstance(item, ImmutableDict): ++ config_defaults[key] = dict(item) ++ elif isinstance(item, ImmutableList): ++ config_defaults[key] = list(item) ++ return config_defaults ++ ++ ++@pytest.fixture ++def gitpython_pillar_opts(pillar_opts): ++ pillar_opts["verified_git_pillar_provider"] = "gitpython" ++ return pillar_opts ++ ++ ++@pytest.fixture ++def pygit2_pillar_opts(pillar_opts): ++ pillar_opts["verified_git_pillar_provider"] = "pygit2" ++ return pillar_opts ++ ++ ++def _get_pillar(opts, *remotes): ++ return GitPillar( ++ opts, ++ remotes, ++ per_remote_overrides=PER_REMOTE_OVERRIDES, ++ per_remote_only=PER_REMOTE_ONLY, ++ global_only=GLOBAL_ONLY, ++ ) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_pillar_provider(gitpython_pillar_opts): ++ p = _get_pillar( ++ gitpython_pillar_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ assert len(p.remotes) == 1 ++ assert p.provider == "gitpython" ++ assert isinstance(p.remotes[0], GitPython) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_pillar_provider(pygit2_pillar_opts): ++ p = _get_pillar( ++ pygit2_pillar_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ assert len(p.remotes) == 1 ++ assert p.provider == "pygit2" ++ assert isinstance(p.remotes[0], Pygit2) ++ ++ ++def _test_env(opts): ++ p = _get_pillar( ++ opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ assert len(p.remotes) == 1 ++ p.checkout() ++ repo = p.remotes[0] ++ # test that two different pillarenvs can exist at the same time ++ files = set(os.listdir(repo.get_cachedir())) ++ for f in (".gitignore", "README.md", "file.sls", "top.sls"): ++ assert f in files ++ opts["pillarenv"] = "main" ++ p2 = _get_pillar( ++ opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ assert len(p.remotes) == 1 ++ p2.checkout() ++ repo2 = p2.remotes[0] ++ files = set(os.listdir(repo2.get_cachedir())) ++ for f in (".gitignore", "README.md"): ++ assert f in files ++ for f in ("file.sls", "top.sls", "back.sls", "rooms.sls"): ++ assert f not in files ++ assert repo.get_cachedir() != repo2.get_cachedir() ++ files = set(os.listdir(repo.get_cachedir())) ++ for f in (".gitignore", "README.md", "file.sls", "top.sls"): ++ assert f in files ++ ++ # double check cache paths ++ assert ( ++ repo.get_cache_hash() == repo2.get_cache_hash() ++ ) # __env__ repos share same hash ++ assert repo.get_cache_basename() != repo2.get_cache_basename() ++ assert repo.get_linkdir() != repo2.get_linkdir() ++ assert repo.get_salt_working_dir() != repo2.get_salt_working_dir() ++ assert repo.get_cache_basename() == "master" ++ assert repo2.get_cache_basename() == "main" ++ ++ assert repo.get_cache_basename() in repo.get_cachedir() ++ assert ( ++ os.path.join(repo.get_cache_basehash(), repo.get_cache_basename()) ++ == repo.get_cache_full_basename() ++ ) ++ assert repo.get_linkdir() not in repo.get_cachedir() ++ assert repo.get_salt_working_dir() not in repo.get_cachedir() ++ ++ ++@skipif_no_gitpython ++def test_gitpython_env(gitpython_pillar_opts): ++ _test_env(gitpython_pillar_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_env(pygit2_pillar_opts): ++ _test_env(pygit2_pillar_opts) ++ ++ ++def _test_checkout_fetch_on_fail(opts): ++ p = _get_pillar(opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git") ++ p.checkout(fetch_on_fail=False) # TODO write me ++ ++ ++@skipif_no_gitpython ++def test_gitpython_checkout_fetch_on_fail(gitpython_pillar_opts): ++ _test_checkout_fetch_on_fail(gitpython_pillar_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_checkout_fetch_on_fail(pygit2_pillar_opts): ++ _test_checkout_fetch_on_fail(pygit2_pillar_opts) ++ ++ ++def _test_multiple_repos(opts): ++ p = _get_pillar( ++ opts, ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "main https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "branch https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ ) ++ p.checkout() ++ assert len(p.remotes) == 5 ++ # make sure all repos dont share cache and working dir ++ assert len({r.get_cachedir() for r in p.remotes}) == 5 ++ assert len({r.get_salt_working_dir() for r in p.remotes}) == 5 ++ ++ p2 = _get_pillar( ++ opts, ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "main https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "branch https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ ) ++ p2.checkout() ++ assert len(p2.remotes) == 5 ++ # make sure that repos are given same cache dir ++ for repo, repo2 in zip(p.remotes, p2.remotes): ++ assert repo.get_cachedir() == repo2.get_cachedir() ++ assert repo.get_salt_working_dir() == repo2.get_salt_working_dir() ++ opts["pillarenv"] = "main" ++ p3 = _get_pillar( ++ opts, ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "main https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "branch https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ ) ++ p3.checkout() ++ # check that __env__ has different cache with different pillarenv ++ assert p.remotes[0].get_cachedir() != p3.remotes[0].get_cachedir() ++ assert p.remotes[1].get_cachedir() == p3.remotes[1].get_cachedir() ++ assert p.remotes[2].get_cachedir() == p3.remotes[2].get_cachedir() ++ assert p.remotes[3].get_cachedir() != p3.remotes[3].get_cachedir() ++ assert p.remotes[4].get_cachedir() == p3.remotes[4].get_cachedir() ++ ++ # check that other branch data is in cache ++ files = set(os.listdir(p.remotes[4].get_cachedir())) ++ for f in (".gitignore", "README.md", "file.sls", "top.sls", "other_env.sls"): ++ assert f in files ++ ++ ++@skipif_no_gitpython ++def test_gitpython_multiple_repos(gitpython_pillar_opts): ++ _test_multiple_repos(gitpython_pillar_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_multiple_repos(pygit2_pillar_opts): ++ _test_multiple_repos(pygit2_pillar_opts) ++ ++ ++def _test_fetch_request(opts): ++ p = _get_pillar( ++ opts, ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ ) ++ frequest = os.path.join(p.remotes[0].get_salt_working_dir(), "fetch_request") ++ frequest_other = os.path.join(p.remotes[1].get_salt_working_dir(), "fetch_request") ++ opts["pillarenv"] = "main" ++ p2 = _get_pillar( ++ opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ frequest2 = os.path.join(p2.remotes[0].get_salt_working_dir(), "fetch_request") ++ assert frequest != frequest2 ++ assert os.path.isfile(frequest) is False ++ assert os.path.isfile(frequest2) is False ++ assert os.path.isfile(frequest_other) is False ++ p.fetch_remotes() ++ assert os.path.isfile(frequest) is False ++ # fetch request was placed ++ assert os.path.isfile(frequest2) is True ++ p2.checkout() ++ # fetch request was found ++ assert os.path.isfile(frequest2) is False ++ p2.fetch_remotes() ++ assert os.path.isfile(frequest) is True ++ assert os.path.isfile(frequest2) is False ++ assert os.path.isfile(frequest_other) is False ++ for _ in range(3): ++ p2.fetch_remotes() ++ assert os.path.isfile(frequest) is True ++ assert os.path.isfile(frequest2) is False ++ assert os.path.isfile(frequest_other) is False ++ # fetch request should still be processed even on fetch_on_fail=False ++ p.checkout(fetch_on_fail=False) ++ assert os.path.isfile(frequest) is False ++ assert os.path.isfile(frequest2) is False ++ assert os.path.isfile(frequest_other) is False ++ ++ ++@skipif_no_gitpython ++def test_gitpython_fetch_request(gitpython_pillar_opts): ++ _test_fetch_request(gitpython_pillar_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_fetch_request(pygit2_pillar_opts): ++ _test_fetch_request(pygit2_pillar_opts) ++ ++ ++def _test_clear_old_remotes(opts): ++ p = _get_pillar( ++ opts, ++ "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ "other https://github.com/saltstack/salt-test-pillar-gitfs-2.git", ++ ) ++ repo = p.remotes[0] ++ repo2 = p.remotes[1] ++ opts["pillarenv"] = "main" ++ p2 = _get_pillar( ++ opts, "__env__ https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ repo3 = p2.remotes[0] ++ assert os.path.isdir(repo.get_cachedir()) is True ++ assert os.path.isdir(repo2.get_cachedir()) is True ++ assert os.path.isdir(repo3.get_cachedir()) is True ++ p.clear_old_remotes() ++ assert os.path.isdir(repo.get_cachedir()) is True ++ assert os.path.isdir(repo2.get_cachedir()) is True ++ assert os.path.isdir(repo3.get_cachedir()) is True ++ p2.clear_old_remotes() ++ assert os.path.isdir(repo.get_cachedir()) is True ++ assert os.path.isdir(repo2.get_cachedir()) is False ++ assert os.path.isdir(repo3.get_cachedir()) is True ++ ++ ++@skipif_no_gitpython ++def test_gitpython_clear_old_remotes(gitpython_pillar_opts): ++ _test_clear_old_remotes(gitpython_pillar_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_clear_old_remotes(pygit2_pillar_opts): ++ _test_clear_old_remotes(pygit2_pillar_opts) ++ ++ ++def _test_remote_map(opts): ++ p = _get_pillar( ++ opts, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ p.fetch_remotes() ++ assert len(p.remotes) == 1 ++ assert os.path.isfile( ++ os.path.join(opts["cachedir"], "git_pillar", "remote_map.txt") ++ ) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_remote_map(gitpython_pillar_opts): ++ _test_remote_map(gitpython_pillar_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_remote_map(pygit2_pillar_opts): ++ _test_remote_map(pygit2_pillar_opts) ++ ++ ++def _test_lock(opts): ++ p = _get_pillar( ++ opts, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ p.fetch_remotes() ++ assert len(p.remotes) == 1 ++ repo = p.remotes[0] ++ assert repo.get_salt_working_dir() in repo._get_lock_file() ++ assert repo.lock() == ( ++ [ ++ "Set update lock for git_pillar remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" ++ ], ++ [], ++ ) ++ assert os.path.isfile(repo._get_lock_file()) ++ assert repo.clear_lock() == ( ++ [ ++ "Removed update lock for git_pillar remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" ++ ], ++ [], ++ ) ++ assert not os.path.isfile(repo._get_lock_file()) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_lock(gitpython_pillar_opts): ++ _test_lock(gitpython_pillar_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_lock(pygit2_pillar_opts): ++ _test_lock(pygit2_pillar_opts) +diff --git a/tests/pytests/functional/utils/test_winrepo.py b/tests/pytests/functional/utils/test_winrepo.py +new file mode 100644 +index 0000000000..117d995bba +--- /dev/null ++++ b/tests/pytests/functional/utils/test_winrepo.py +@@ -0,0 +1,164 @@ ++import os ++ ++import pytest ++ ++from salt.runners.winrepo import GLOBAL_ONLY, PER_REMOTE_ONLY, PER_REMOTE_OVERRIDES ++from salt.utils.gitfs import GitPython, Pygit2, WinRepo ++from salt.utils.immutabletypes import ImmutableDict, ImmutableList ++ ++pytestmark = [ ++ pytest.mark.slow_test, ++] ++ ++ ++try: ++ import git # pylint: disable=unused-import ++ ++ HAS_GITPYTHON = True ++except ImportError: ++ HAS_GITPYTHON = False ++ ++ ++try: ++ import pygit2 # pylint: disable=unused-import ++ ++ HAS_PYGIT2 = True ++except ImportError: ++ HAS_PYGIT2 = False ++ ++ ++skipif_no_gitpython = pytest.mark.skipif(not HAS_GITPYTHON, reason="Missing gitpython") ++skipif_no_pygit2 = pytest.mark.skipif(not HAS_PYGIT2, reason="Missing pygit2") ++ ++ ++@pytest.fixture ++def winrepo_opts(salt_factories, tmp_path): ++ config_defaults = {"cachedir": str(tmp_path)} ++ factory = salt_factories.salt_master_daemon( ++ "winrepo-functional-master", defaults=config_defaults ++ ) ++ config_defaults = dict(factory.config) ++ for key, item in config_defaults.items(): ++ if isinstance(item, ImmutableDict): ++ config_defaults[key] = dict(item) ++ elif isinstance(item, ImmutableList): ++ config_defaults[key] = list(item) ++ return config_defaults ++ ++ ++@pytest.fixture ++def gitpython_winrepo_opts(winrepo_opts): ++ winrepo_opts["verified_winrepo_provider"] = "gitpython" ++ return winrepo_opts ++ ++ ++@pytest.fixture ++def pygit2_winrepo_opts(winrepo_opts): ++ winrepo_opts["verified_winrepo_provider"] = "pygit2" ++ return winrepo_opts ++ ++ ++def _get_winrepo(opts, *remotes): ++ return WinRepo( ++ opts, ++ remotes, ++ per_remote_overrides=PER_REMOTE_OVERRIDES, ++ per_remote_only=PER_REMOTE_ONLY, ++ global_only=GLOBAL_ONLY, ++ ) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_winrepo_provider(gitpython_winrepo_opts): ++ w = _get_winrepo( ++ gitpython_winrepo_opts, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ assert len(w.remotes) == 1 ++ assert w.provider == "gitpython" ++ assert isinstance(w.remotes[0], GitPython) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_winrepo_provider(pygit2_winrepo_opts): ++ w = _get_winrepo( ++ pygit2_winrepo_opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git" ++ ) ++ assert len(w.remotes) == 1 ++ assert w.provider == "pygit2" ++ assert isinstance(w.remotes[0], Pygit2) ++ ++ ++def _test_winrepo_simple(opts): ++ w = _get_winrepo(opts, "https://github.com/saltstack/salt-test-pillar-gitfs.git") ++ assert len(w.remotes) == 1 ++ w.checkout() ++ repo = w.remotes[0] ++ files = set(os.listdir(repo.get_cachedir())) ++ for f in (".gitignore", "README.md", "file.sls", "top.sls"): ++ assert f in files ++ ++ ++@skipif_no_gitpython ++def test_gitpython_winrepo_simple(gitpython_winrepo_opts): ++ _test_winrepo_simple(gitpython_winrepo_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_winrepo_simple(pygit2_winrepo_opts): ++ _test_winrepo_simple(pygit2_winrepo_opts) ++ ++ ++def _test_remote_map(opts): ++ p = _get_winrepo( ++ opts, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ p.fetch_remotes() ++ assert len(p.remotes) == 1 ++ assert os.path.isfile(os.path.join(opts["cachedir"], "winrepo", "remote_map.txt")) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_remote_map(gitpython_winrepo_opts): ++ _test_remote_map(gitpython_winrepo_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_remote_map(pygit2_winrepo_opts): ++ _test_remote_map(pygit2_winrepo_opts) ++ ++ ++def _test_lock(opts): ++ w = _get_winrepo( ++ opts, ++ "https://github.com/saltstack/salt-test-pillar-gitfs.git", ++ ) ++ w.fetch_remotes() ++ assert len(w.remotes) == 1 ++ repo = w.remotes[0] ++ assert repo.get_salt_working_dir() in repo._get_lock_file() ++ assert repo.lock() == ( ++ [ ++ "Set update lock for winrepo remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" ++ ], ++ [], ++ ) ++ assert os.path.isfile(repo._get_lock_file()) ++ assert repo.clear_lock() == ( ++ [ ++ "Removed update lock for winrepo remote 'https://github.com/saltstack/salt-test-pillar-gitfs.git'" ++ ], ++ [], ++ ) ++ assert not os.path.isfile(repo._get_lock_file()) ++ ++ ++@skipif_no_gitpython ++def test_gitpython_lock(gitpython_winrepo_opts): ++ _test_lock(gitpython_winrepo_opts) ++ ++ ++@skipif_no_pygit2 ++def test_pygit2_lock(pygit2_winrepo_opts): ++ _test_lock(pygit2_winrepo_opts) +diff --git a/tests/pytests/unit/test_minion.py b/tests/pytests/unit/test_minion.py +index 4508eaee95..740743194e 100644 +--- a/tests/pytests/unit/test_minion.py ++++ b/tests/pytests/unit/test_minion.py +@@ -21,35 +21,33 @@ from tests.support.mock import MagicMock, patch + log = logging.getLogger(__name__) + + +-def test_minion_load_grains_false(): ++def test_minion_load_grains_false(minion_opts): + """ + Minion does not generate grains when load_grains is False + """ +- opts = {"random_startup_delay": 0, "grains": {"foo": "bar"}} ++ minion_opts["grains"] = {"foo": "bar"} + with patch("salt.loader.grains") as grainsfunc: +- minion = salt.minion.Minion(opts, load_grains=False) +- assert minion.opts["grains"] == opts["grains"] ++ minion = salt.minion.Minion(minion_opts, load_grains=False) ++ assert minion.opts["grains"] == minion_opts["grains"] + grainsfunc.assert_not_called() + + +-def test_minion_load_grains_true(): ++def test_minion_load_grains_true(minion_opts): + """ + Minion generates grains when load_grains is True + """ +- opts = {"random_startup_delay": 0, "grains": {}} + with patch("salt.loader.grains") as grainsfunc: +- minion = salt.minion.Minion(opts, load_grains=True) ++ minion = salt.minion.Minion(minion_opts, load_grains=True) + assert minion.opts["grains"] != {} + grainsfunc.assert_called() + + +-def test_minion_load_grains_default(): ++def test_minion_load_grains_default(minion_opts): + """ + Minion load_grains defaults to True + """ +- opts = {"random_startup_delay": 0, "grains": {}} + with patch("salt.loader.grains") as grainsfunc: +- minion = salt.minion.Minion(opts) ++ minion = salt.minion.Minion(minion_opts) + assert minion.opts["grains"] != {} + grainsfunc.assert_called() + +@@ -91,24 +89,17 @@ def test_send_req_tries(req_channel, minion_opts): + + assert rtn == 30 + +- +-@patch("salt.channel.client.ReqChannel.factory") +-def test_mine_send_tries(req_channel_factory): ++def test_mine_send_tries(minion_opts): + channel_enter = MagicMock() + channel_enter.send.side_effect = lambda load, timeout, tries: tries + channel = MagicMock() + channel.__enter__.return_value = channel_enter + +- opts = { +- "random_startup_delay": 0, +- "grains": {}, +- "return_retry_tries": 20, +- "minion_sign_messages": False, +- } ++ minion_opts["return_retry_tries"] = 20 + with patch("salt.channel.client.ReqChannel.factory", return_value=channel), patch( + "salt.loader.grains" + ): +- minion = salt.minion.Minion(opts) ++ minion = salt.minion.Minion(minion_opts) + minion.tok = "token" + + data = {} +diff --git a/tests/pytests/unit/utils/test_gitfs.py b/tests/pytests/unit/utils/test_gitfs.py +index e9915de412..2bf627049f 100644 +--- a/tests/pytests/unit/utils/test_gitfs.py ++++ b/tests/pytests/unit/utils/test_gitfs.py +@@ -1,5 +1,4 @@ + import os +-import string + import time + + import pytest +@@ -214,11 +213,11 @@ def test_checkout_pygit2(_prepare_provider): + provider.init_remote() + provider.fetch() + provider.branch = "master" +- assert provider.cachedir in provider.checkout() ++ assert provider.get_cachedir() in provider.checkout() + provider.branch = "simple_tag" +- assert provider.cachedir in provider.checkout() ++ assert provider.get_cachedir() in provider.checkout() + provider.branch = "annotated_tag" +- assert provider.cachedir in provider.checkout() ++ assert provider.get_cachedir() in provider.checkout() + provider.branch = "does_not_exist" + assert provider.checkout() is None + +@@ -238,18 +237,9 @@ def test_checkout_pygit2_with_home_env_unset(_prepare_provider): + assert "HOME" in os.environ + + +-def test_full_id_pygit2(_prepare_provider): +- assert _prepare_provider.full_id().startswith("-") +- assert _prepare_provider.full_id().endswith("/pygit2-repo---gitfs-master--") +- +- + @pytest.mark.skipif(not HAS_PYGIT2, reason="This host lacks proper pygit2 support") + @pytest.mark.skip_on_windows( + reason="Skip Pygit2 on windows, due to pygit2 access error on windows" + ) + def test_get_cachedir_basename_pygit2(_prepare_provider): +- basename = _prepare_provider.get_cachedir_basename() +- assert len(basename) == 45 +- assert basename[0] == "-" +- # check that a valid base64 is given '/' -> '_' +- assert all(c in string.ascii_letters + string.digits + "+_=" for c in basename[1:]) ++ assert "_" == _prepare_provider.get_cache_basename() +diff --git a/tests/unit/utils/test_gitfs.py b/tests/unit/utils/test_gitfs.py +index 6d8e97a239..259ea056fc 100644 +--- a/tests/unit/utils/test_gitfs.py ++++ b/tests/unit/utils/test_gitfs.py +@@ -114,27 +114,14 @@ class TestGitBase(TestCase, AdaptedConfigurationTestCaseMixin): + self.assertTrue(self.main_class.remotes[0].fetched) + self.assertFalse(self.main_class.remotes[1].fetched) + +- def test_full_id(self): +- self.assertEqual( +- self.main_class.remotes[0].full_id(), "-file://repo1.git---gitfs-master--" +- ) +- +- def test_full_id_with_name(self): +- self.assertEqual( +- self.main_class.remotes[1].full_id(), +- "repo2-file://repo2.git---gitfs-master--", +- ) +- + def test_get_cachedir_basename(self): + self.assertEqual( +- self.main_class.remotes[0].get_cachedir_basename(), +- "-jXhnbGDemchtZwTwaD2s6VOaVvs98a7w+AtiYlmOVb0=", ++ self.main_class.remotes[0].get_cache_basename(), ++ "_", + ) +- +- def test_get_cachedir_base_with_name(self): + self.assertEqual( +- self.main_class.remotes[1].get_cachedir_basename(), +- "repo2-nuezpiDtjQRFC0ZJDByvi+F6Vb8ZhfoH41n_KFxTGsU=", ++ self.main_class.remotes[1].get_cache_basename(), ++ "_", + ) + + def test_git_provider_mp_lock(self): +-- +2.42.0 + + diff --git a/salt.changes b/salt.changes index d1a1458..e86f805 100644 --- a/salt.changes +++ b/salt.changes @@ -1,3 +1,14 @@ +------------------------------------------------------------------- +Mon Nov 13 16:02:35 UTC 2023 - Pablo Suárez Hernández + +- Align behavior of some modules when using salt-call via symlink (bsc#1215963) +- Fix gitfs "__env__" and improve cache cleaning (bsc#1193948) +- Remove python-boto dependency for the python3-salt-testsuite package for Tumbleweed + +- Added: + * fix-gitfs-__env__-and-improve-cache-cleaning-bsc-119.patch + * dereference-symlinks-to-set-proper-__cli-opt-bsc-121.patch + ------------------------------------------------------------------- Tue Oct 31 11:51:17 UTC 2023 - Alexander Graul diff --git a/salt.spec b/salt.spec index 3770473..8d2dc76 100644 --- a/salt.spec +++ b/salt.spec @@ -320,6 +320,11 @@ Patch86: fix-optimization_order-opt-to-prevent-test-fails.patch Patch87: allow-all-primitive-grain-types-for-autosign_grains-.patch # PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65482 Patch88: fix-cve-2023-34049-bsc-1215157.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65017 +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65136 +Patch89: fix-gitfs-__env__-and-improve-cache-cleaning-bsc-119.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/65435 +Patch90: dereference-symlinks-to-set-proper-__cli-opt-bsc-121.patch ### IMPORTANT: The line below is used as a snippet marker. Do not touch it. ### SALT PATCHES LIST END @@ -681,7 +686,9 @@ Requires: %{name} = %{version}-%{release} Requires: python3-CherryPy Requires: python3-Genshi Requires: python3-Mako +%if !0%{?suse_version} > 1600 || 0%{?centos} Requires: python3-boto +%endif Requires: python3-boto3 Requires: python3-docker Requires: python3-mock