From a1fc5287d501a1ecdbd259e5bbdd4f7d5d06dd13 Mon Sep 17 00:00:00 2001 From: Alexander Graul Date: Fri, 28 Apr 2023 09:41:28 +0200 Subject: [PATCH] Make sure the file client is destroyed upon used Backport of https://github.com/saltstack/salt/pull/64113 --- salt/client/ssh/wrapper/saltcheck.py | 108 +++---- salt/fileclient.py | 11 - salt/modules/dockermod.py | 17 +- salt/pillar/__init__.py | 6 +- salt/states/ansiblegate.py | 11 +- salt/utils/asynchronous.py | 2 +- salt/utils/jinja.py | 53 ++- salt/utils/mako.py | 7 + salt/utils/templates.py | 303 +++++++++--------- .../integration/states/test_include.py | 40 +++ .../utils/jinja/test_salt_cache_loader.py | 47 ++- 11 files changed, 330 insertions(+), 275 deletions(-) create mode 100644 tests/pytests/integration/states/test_include.py diff --git a/salt/client/ssh/wrapper/saltcheck.py b/salt/client/ssh/wrapper/saltcheck.py index d47b5cf6883..b0b94593809 100644 --- a/salt/client/ssh/wrapper/saltcheck.py +++ b/salt/client/ssh/wrapper/saltcheck.py @@ -9,6 +9,7 @@ import tarfile import tempfile from contextlib import closing +import salt.fileclient import salt.utils.files import salt.utils.json import salt.utils.url @@ -28,65 +29,62 @@ def update_master_cache(states, saltenv="base"): # Setup for copying states to gendir gendir = tempfile.mkdtemp() trans_tar = salt.utils.files.mkstemp() - if "cp.fileclient_{}".format(id(__opts__)) not in __context__: - __context__[ - "cp.fileclient_{}".format(id(__opts__)) - ] = salt.fileclient.get_file_client(__opts__) - - # generate cp.list_states output and save to gendir - cp_output = salt.utils.json.dumps(__salt__["cp.list_states"]()) - cp_output_file = os.path.join(gendir, "cp_output.txt") - with salt.utils.files.fopen(cp_output_file, "w") as fp: - fp.write(cp_output) - - # cp state directories to gendir - already_processed = [] - sls_list = salt.utils.args.split_input(states) - for state_name in sls_list: - # generate low data for each state and save to gendir - state_low_file = os.path.join(gendir, state_name + ".low") - state_low_output = salt.utils.json.dumps( - __salt__["state.show_low_sls"](state_name) - ) - with salt.utils.files.fopen(state_low_file, "w") as fp: - fp.write(state_low_output) - - state_name = state_name.replace(".", os.sep) - if state_name in already_processed: - log.debug("Already cached state for %s", state_name) - else: - file_copy_file = os.path.join(gendir, state_name + ".copy") - log.debug("copying %s to %s", state_name, gendir) - qualified_name = salt.utils.url.create(state_name, saltenv) - # Duplicate cp.get_dir to gendir - copy_result = __context__["cp.fileclient_{}".format(id(__opts__))].get_dir( - qualified_name, gendir, saltenv + with salt.fileclient.get_file_client(__opts__) as cp_fileclient: + + # generate cp.list_states output and save to gendir + cp_output = salt.utils.json.dumps(__salt__["cp.list_states"]()) + cp_output_file = os.path.join(gendir, "cp_output.txt") + with salt.utils.files.fopen(cp_output_file, "w") as fp: + fp.write(cp_output) + + # cp state directories to gendir + already_processed = [] + sls_list = salt.utils.args.split_input(states) + for state_name in sls_list: + # generate low data for each state and save to gendir + state_low_file = os.path.join(gendir, state_name + ".low") + state_low_output = salt.utils.json.dumps( + __salt__["state.show_low_sls"](state_name) ) - if copy_result: - copy_result = [dir.replace(gendir, state_cache) for dir in copy_result] - copy_result_output = salt.utils.json.dumps(copy_result) - with salt.utils.files.fopen(file_copy_file, "w") as fp: - fp.write(copy_result_output) - already_processed.append(state_name) + with salt.utils.files.fopen(state_low_file, "w") as fp: + fp.write(state_low_output) + + state_name = state_name.replace(".", os.sep) + if state_name in already_processed: + log.debug("Already cached state for %s", state_name) else: - # If files were not copied, assume state.file.sls was given and just copy state - state_name = os.path.dirname(state_name) file_copy_file = os.path.join(gendir, state_name + ".copy") - if state_name in already_processed: - log.debug("Already cached state for %s", state_name) + log.debug("copying %s to %s", state_name, gendir) + qualified_name = salt.utils.url.create(state_name, saltenv) + # Duplicate cp.get_dir to gendir + copy_result = cp_fileclient.get_dir(qualified_name, gendir, saltenv) + if copy_result: + copy_result = [ + dir.replace(gendir, state_cache) for dir in copy_result + ] + copy_result_output = salt.utils.json.dumps(copy_result) + with salt.utils.files.fopen(file_copy_file, "w") as fp: + fp.write(copy_result_output) + already_processed.append(state_name) else: - qualified_name = salt.utils.url.create(state_name, saltenv) - copy_result = __context__[ - "cp.fileclient_{}".format(id(__opts__)) - ].get_dir(qualified_name, gendir, saltenv) - if copy_result: - copy_result = [ - dir.replace(gendir, state_cache) for dir in copy_result - ] - copy_result_output = salt.utils.json.dumps(copy_result) - with salt.utils.files.fopen(file_copy_file, "w") as fp: - fp.write(copy_result_output) - already_processed.append(state_name) + # If files were not copied, assume state.file.sls was given and just copy state + state_name = os.path.dirname(state_name) + file_copy_file = os.path.join(gendir, state_name + ".copy") + if state_name in already_processed: + log.debug("Already cached state for %s", state_name) + else: + qualified_name = salt.utils.url.create(state_name, saltenv) + copy_result = cp_fileclient.get_dir( + qualified_name, gendir, saltenv + ) + if copy_result: + copy_result = [ + dir.replace(gendir, state_cache) for dir in copy_result + ] + copy_result_output = salt.utils.json.dumps(copy_result) + with salt.utils.files.fopen(file_copy_file, "w") as fp: + fp.write(copy_result_output) + already_processed.append(state_name) # turn gendir into tarball and remove gendir try: diff --git a/salt/fileclient.py b/salt/fileclient.py index fef5154a0be..f01a86dd0d4 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -849,7 +849,6 @@ class Client: kwargs.pop("env") kwargs["saltenv"] = saltenv - url_data = urllib.parse.urlparse(url) sfn = self.cache_file(url, saltenv, cachedir=cachedir) if not sfn or not os.path.exists(sfn): return "" @@ -1165,13 +1164,8 @@ class RemoteClient(Client): if not salt.utils.platform.is_windows(): hash_server, stat_server = self.hash_and_stat_file(path, saltenv) - try: - mode_server = stat_server[0] - except (IndexError, TypeError): - mode_server = None else: hash_server = self.hash_file(path, saltenv) - mode_server = None # Check if file exists on server, before creating files and # directories @@ -1214,13 +1208,8 @@ class RemoteClient(Client): if dest2check and os.path.isfile(dest2check): if not salt.utils.platform.is_windows(): hash_local, stat_local = self.hash_and_stat_file(dest2check, saltenv) - try: - mode_local = stat_local[0] - except (IndexError, TypeError): - mode_local = None else: hash_local = self.hash_file(dest2check, saltenv) - mode_local = None if hash_local == hash_server: return dest2check diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index f7344b66ac6..69b722f0c95 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -6667,14 +6667,6 @@ def script_retcode( )["retcode"] -def _mk_fileclient(): - """ - Create a file client and add it to the context. - """ - if "cp.fileclient" not in __context__: - __context__["cp.fileclient"] = salt.fileclient.get_file_client(__opts__) - - def _generate_tmp_path(): return os.path.join("/tmp", "salt.docker.{}".format(uuid.uuid4().hex[:6])) @@ -6688,11 +6680,10 @@ def _prepare_trans_tar(name, sls_opts, mods=None, pillar=None, extra_filerefs="" # reuse it from salt.ssh, however this function should # be somewhere else refs = salt.client.ssh.state.lowstate_file_refs(chunks, extra_filerefs) - _mk_fileclient() - trans_tar = salt.client.ssh.state.prep_trans_tar( - __context__["cp.fileclient"], chunks, refs, pillar, name - ) - return trans_tar + with salt.fileclient.get_file_client(__opts__) as fileclient: + return salt.client.ssh.state.prep_trans_tar( + fileclient, chunks, refs, pillar, name + ) def _compile_state(sls_opts, mods=None): diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py index 0dfab4cc579..26312b3bd53 100644 --- a/salt/pillar/__init__.py +++ b/salt/pillar/__init__.py @@ -9,7 +9,6 @@ import logging import os import sys import traceback -import uuid import salt.channel.client import salt.ext.tornado.gen @@ -1351,6 +1350,11 @@ class Pillar: if hasattr(self, "_closing") and self._closing: return self._closing = True + if self.client: + try: + self.client.destroy() + except AttributeError: + pass # pylint: disable=W1701 def __del__(self): diff --git a/salt/states/ansiblegate.py b/salt/states/ansiblegate.py index 7fd4deb6c2a..9abd418c42c 100644 --- a/salt/states/ansiblegate.py +++ b/salt/states/ansiblegate.py @@ -32,12 +32,10 @@ state: - state: installed """ - import logging import os import sys -# Import salt modules import salt.fileclient import salt.utils.decorators.path from salt.utils.decorators import depends @@ -108,13 +106,6 @@ def __virtual__(): return __virtualname__ -def _client(): - """ - Get a fileclient - """ - return salt.fileclient.get_file_client(__opts__) - - def _changes(plays): """ Find changes in ansible return data @@ -171,7 +162,7 @@ def playbooks(name, rundir=None, git_repo=None, git_kwargs=None, ansible_kwargs= } if git_repo: if not isinstance(rundir, str) or not os.path.isdir(rundir): - with _client() as client: + with salt.fileclient.get_file_client(__opts__) as client: rundir = client._extrn_path(git_repo, "base") log.trace("rundir set to %s", rundir) if not isinstance(git_kwargs, dict): diff --git a/salt/utils/asynchronous.py b/salt/utils/asynchronous.py index 2a858feee98..0c645bbc3bb 100644 --- a/salt/utils/asynchronous.py +++ b/salt/utils/asynchronous.py @@ -131,7 +131,7 @@ class SyncWrapper: result = io_loop.run_sync(lambda: getattr(self.obj, key)(*args, **kwargs)) results.append(True) results.append(result) - except Exception as exc: # pylint: disable=broad-except + except Exception: # pylint: disable=broad-except results.append(False) results.append(sys.exc_info()) diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index fcc5aec497e..a6a8a279605 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -58,19 +58,6 @@ class SaltCacheLoader(BaseLoader): and only loaded once per loader instance. """ - _cached_pillar_client = None - _cached_client = None - - @classmethod - def shutdown(cls): - for attr in ("_cached_client", "_cached_pillar_client"): - client = getattr(cls, attr, None) - if client is not None: - # PillarClient and LocalClient objects do not have a destroy method - if hasattr(client, "destroy"): - client.destroy() - setattr(cls, attr, None) - def __init__( self, opts, @@ -93,8 +80,7 @@ class SaltCacheLoader(BaseLoader): log.debug("Jinja search path: %s", self.searchpath) self.cached = [] self._file_client = _file_client - # Instantiate the fileclient - self.file_client() + self._close_file_client = _file_client is None def file_client(self): """ @@ -108,18 +94,10 @@ class SaltCacheLoader(BaseLoader): or not hasattr(self._file_client, "opts") or self._file_client.opts["file_roots"] != self.opts["file_roots"] ): - attr = "_cached_pillar_client" if self.pillar_rend else "_cached_client" - cached_client = getattr(self, attr, None) - if ( - cached_client is None - or not hasattr(cached_client, "opts") - or cached_client.opts["file_roots"] != self.opts["file_roots"] - ): - cached_client = salt.fileclient.get_file_client( - self.opts, self.pillar_rend - ) - setattr(SaltCacheLoader, attr, cached_client) - self._file_client = cached_client + self._file_client = salt.fileclient.get_file_client( + self.opts, self.pillar_rend + ) + self._close_file_client = True return self._file_client def cache_file(self, template): @@ -221,6 +199,27 @@ class SaltCacheLoader(BaseLoader): # there is no template file within searchpaths raise TemplateNotFound(template) + def destroy(self): + if self._close_file_client is False: + return + if self._file_client is None: + return + file_client = self._file_client + self._file_client = None + + try: + file_client.destroy() + except AttributeError: + # PillarClient and LocalClient objects do not have a destroy method + pass + + def __enter__(self): + self.file_client() + return self + + def __exit__(self, *args): + self.destroy() + class PrintableDict(OrderedDict): """ diff --git a/salt/utils/mako.py b/salt/utils/mako.py index 69618de9837..037d5d86deb 100644 --- a/salt/utils/mako.py +++ b/salt/utils/mako.py @@ -97,3 +97,10 @@ if HAS_MAKO: self.cache[fpath] = self.file_client().get_file( fpath, "", True, self.saltenv ) + + def destroy(self): + if self.client: + try: + self.client.destroy() + except AttributeError: + pass diff --git a/salt/utils/templates.py b/salt/utils/templates.py index 4947b820a36..4a8adf2a14f 100644 --- a/salt/utils/templates.py +++ b/salt/utils/templates.py @@ -362,163 +362,169 @@ def render_jinja_tmpl(tmplstr, context, tmplpath=None): elif tmplstr.endswith("\n"): newline = "\n" - if not saltenv: - if tmplpath: - loader = jinja2.FileSystemLoader(os.path.dirname(tmplpath)) - else: - loader = salt.utils.jinja.SaltCacheLoader( - opts, - saltenv, - pillar_rend=context.get("_pillar_rend", False), - _file_client=file_client, - ) + try: + if not saltenv: + if tmplpath: + loader = jinja2.FileSystemLoader(os.path.dirname(tmplpath)) + else: + loader = salt.utils.jinja.SaltCacheLoader( + opts, + saltenv, + pillar_rend=context.get("_pillar_rend", False), + _file_client=file_client, + ) - env_args = {"extensions": [], "loader": loader} - - if hasattr(jinja2.ext, "with_"): - env_args["extensions"].append("jinja2.ext.with_") - if hasattr(jinja2.ext, "do"): - env_args["extensions"].append("jinja2.ext.do") - if hasattr(jinja2.ext, "loopcontrols"): - env_args["extensions"].append("jinja2.ext.loopcontrols") - env_args["extensions"].append(salt.utils.jinja.SerializerExtension) - - opt_jinja_env = opts.get("jinja_env", {}) - opt_jinja_sls_env = opts.get("jinja_sls_env", {}) - - opt_jinja_env = opt_jinja_env if isinstance(opt_jinja_env, dict) else {} - opt_jinja_sls_env = opt_jinja_sls_env if isinstance(opt_jinja_sls_env, dict) else {} - - # Pass through trim_blocks and lstrip_blocks Jinja parameters - # trim_blocks removes newlines around Jinja blocks - # lstrip_blocks strips tabs and spaces from the beginning of - # line to the start of a block. - if opts.get("jinja_trim_blocks", False): - log.debug("Jinja2 trim_blocks is enabled") - log.warning( - "jinja_trim_blocks is deprecated and will be removed in a future release," - " please use jinja_env and/or jinja_sls_env instead" - ) - opt_jinja_env["trim_blocks"] = True - opt_jinja_sls_env["trim_blocks"] = True - if opts.get("jinja_lstrip_blocks", False): - log.debug("Jinja2 lstrip_blocks is enabled") - log.warning( - "jinja_lstrip_blocks is deprecated and will be removed in a future release," - " please use jinja_env and/or jinja_sls_env instead" - ) - opt_jinja_env["lstrip_blocks"] = True - opt_jinja_sls_env["lstrip_blocks"] = True - - def opt_jinja_env_helper(opts, optname): - for k, v in opts.items(): - k = k.lower() - if hasattr(jinja2.defaults, k.upper()): - log.debug("Jinja2 environment %s was set to %s by %s", k, v, optname) - env_args[k] = v - else: - log.warning("Jinja2 environment %s is not recognized", k) + env_args = {"extensions": [], "loader": loader} - if "sls" in context and context["sls"] != "": - opt_jinja_env_helper(opt_jinja_sls_env, "jinja_sls_env") - else: - opt_jinja_env_helper(opt_jinja_env, "jinja_env") + if hasattr(jinja2.ext, "with_"): + env_args["extensions"].append("jinja2.ext.with_") + if hasattr(jinja2.ext, "do"): + env_args["extensions"].append("jinja2.ext.do") + if hasattr(jinja2.ext, "loopcontrols"): + env_args["extensions"].append("jinja2.ext.loopcontrols") + env_args["extensions"].append(salt.utils.jinja.SerializerExtension) - if opts.get("allow_undefined", False): - jinja_env = jinja2.sandbox.SandboxedEnvironment(**env_args) - else: - jinja_env = jinja2.sandbox.SandboxedEnvironment( - undefined=jinja2.StrictUndefined, **env_args - ) + opt_jinja_env = opts.get("jinja_env", {}) + opt_jinja_sls_env = opts.get("jinja_sls_env", {}) - indent_filter = jinja_env.filters.get("indent") - jinja_env.tests.update(JinjaTest.salt_jinja_tests) - jinja_env.filters.update(JinjaFilter.salt_jinja_filters) - if salt.utils.jinja.JINJA_VERSION >= Version("2.11"): - # Use the existing indent filter on Jinja versions where it's not broken - jinja_env.filters["indent"] = indent_filter - jinja_env.globals.update(JinjaGlobal.salt_jinja_globals) - - # globals - jinja_env.globals["odict"] = OrderedDict - jinja_env.globals["show_full_context"] = salt.utils.jinja.show_full_context - - jinja_env.tests["list"] = salt.utils.data.is_list - - decoded_context = {} - for key, value in context.items(): - if not isinstance(value, str): - if isinstance(value, NamedLoaderContext): - decoded_context[key] = value.value() - else: - decoded_context[key] = value - continue + opt_jinja_env = opt_jinja_env if isinstance(opt_jinja_env, dict) else {} + opt_jinja_sls_env = ( + opt_jinja_sls_env if isinstance(opt_jinja_sls_env, dict) else {} + ) - try: - decoded_context[key] = salt.utils.stringutils.to_unicode( - value, encoding=SLS_ENCODING + # Pass through trim_blocks and lstrip_blocks Jinja parameters + # trim_blocks removes newlines around Jinja blocks + # lstrip_blocks strips tabs and spaces from the beginning of + # line to the start of a block. + if opts.get("jinja_trim_blocks", False): + log.debug("Jinja2 trim_blocks is enabled") + log.warning( + "jinja_trim_blocks is deprecated and will be removed in a future release," + " please use jinja_env and/or jinja_sls_env instead" ) - except UnicodeDecodeError as ex: - log.debug( - "Failed to decode using default encoding (%s), trying system encoding", - SLS_ENCODING, + opt_jinja_env["trim_blocks"] = True + opt_jinja_sls_env["trim_blocks"] = True + if opts.get("jinja_lstrip_blocks", False): + log.debug("Jinja2 lstrip_blocks is enabled") + log.warning( + "jinja_lstrip_blocks is deprecated and will be removed in a future release," + " please use jinja_env and/or jinja_sls_env instead" ) - decoded_context[key] = salt.utils.data.decode(value) + opt_jinja_env["lstrip_blocks"] = True + opt_jinja_sls_env["lstrip_blocks"] = True + + def opt_jinja_env_helper(opts, optname): + for k, v in opts.items(): + k = k.lower() + if hasattr(jinja2.defaults, k.upper()): + log.debug( + "Jinja2 environment %s was set to %s by %s", k, v, optname + ) + env_args[k] = v + else: + log.warning("Jinja2 environment %s is not recognized", k) - jinja_env.globals.update(decoded_context) - try: - template = jinja_env.from_string(tmplstr) - output = template.render(**decoded_context) - except jinja2.exceptions.UndefinedError as exc: - trace = traceback.extract_tb(sys.exc_info()[2]) - line, out = _get_jinja_error(trace, context=decoded_context) - if not line: - tmplstr = "" - raise SaltRenderError("Jinja variable {}{}".format(exc, out), line, tmplstr) - except ( - jinja2.exceptions.TemplateRuntimeError, - jinja2.exceptions.TemplateSyntaxError, - jinja2.exceptions.SecurityError, - ) as exc: - trace = traceback.extract_tb(sys.exc_info()[2]) - line, out = _get_jinja_error(trace, context=decoded_context) - if not line: - tmplstr = "" - raise SaltRenderError( - "Jinja syntax error: {}{}".format(exc, out), line, tmplstr - ) - except (SaltInvocationError, CommandExecutionError) as exc: - trace = traceback.extract_tb(sys.exc_info()[2]) - line, out = _get_jinja_error(trace, context=decoded_context) - if not line: - tmplstr = "" - raise SaltRenderError( - "Problem running salt function in Jinja template: {}{}".format(exc, out), - line, - tmplstr, - ) - except Exception as exc: # pylint: disable=broad-except - tracestr = traceback.format_exc() - trace = traceback.extract_tb(sys.exc_info()[2]) - line, out = _get_jinja_error(trace, context=decoded_context) - if not line: - tmplstr = "" + if "sls" in context and context["sls"] != "": + opt_jinja_env_helper(opt_jinja_sls_env, "jinja_sls_env") else: - tmplstr += "\n{}".format(tracestr) - log.debug("Jinja Error") - log.debug("Exception:", exc_info=True) - log.debug("Out: %s", out) - log.debug("Line: %s", line) - log.debug("TmplStr: %s", tmplstr) - log.debug("TraceStr: %s", tracestr) + opt_jinja_env_helper(opt_jinja_env, "jinja_env") - raise SaltRenderError( - "Jinja error: {}{}".format(exc, out), line, tmplstr, trace=tracestr - ) + if opts.get("allow_undefined", False): + jinja_env = jinja2.sandbox.SandboxedEnvironment(**env_args) + else: + jinja_env = jinja2.sandbox.SandboxedEnvironment( + undefined=jinja2.StrictUndefined, **env_args + ) + + indent_filter = jinja_env.filters.get("indent") + jinja_env.tests.update(JinjaTest.salt_jinja_tests) + jinja_env.filters.update(JinjaFilter.salt_jinja_filters) + if salt.utils.jinja.JINJA_VERSION >= Version("2.11"): + # Use the existing indent filter on Jinja versions where it's not broken + jinja_env.filters["indent"] = indent_filter + jinja_env.globals.update(JinjaGlobal.salt_jinja_globals) + + # globals + jinja_env.globals["odict"] = OrderedDict + jinja_env.globals["show_full_context"] = salt.utils.jinja.show_full_context + + jinja_env.tests["list"] = salt.utils.data.is_list + + decoded_context = {} + for key, value in context.items(): + if not isinstance(value, str): + if isinstance(value, NamedLoaderContext): + decoded_context[key] = value.value() + else: + decoded_context[key] = value + continue + + try: + decoded_context[key] = salt.utils.stringutils.to_unicode( + value, encoding=SLS_ENCODING + ) + except UnicodeDecodeError: + log.debug( + "Failed to decode using default encoding (%s), trying system encoding", + SLS_ENCODING, + ) + decoded_context[key] = salt.utils.data.decode(value) + + jinja_env.globals.update(decoded_context) + try: + template = jinja_env.from_string(tmplstr) + output = template.render(**decoded_context) + except jinja2.exceptions.UndefinedError as exc: + trace = traceback.extract_tb(sys.exc_info()[2]) + line, out = _get_jinja_error(trace, context=decoded_context) + if not line: + tmplstr = "" + raise SaltRenderError("Jinja variable {}{}".format(exc, out), line, tmplstr) + except ( + jinja2.exceptions.TemplateRuntimeError, + jinja2.exceptions.TemplateSyntaxError, + jinja2.exceptions.SecurityError, + ) as exc: + trace = traceback.extract_tb(sys.exc_info()[2]) + line, out = _get_jinja_error(trace, context=decoded_context) + if not line: + tmplstr = "" + raise SaltRenderError( + "Jinja syntax error: {}{}".format(exc, out), line, tmplstr + ) + except (SaltInvocationError, CommandExecutionError) as exc: + trace = traceback.extract_tb(sys.exc_info()[2]) + line, out = _get_jinja_error(trace, context=decoded_context) + if not line: + tmplstr = "" + raise SaltRenderError( + "Problem running salt function in Jinja template: {}{}".format( + exc, out + ), + line, + tmplstr, + ) + except Exception as exc: # pylint: disable=broad-except + tracestr = traceback.format_exc() + trace = traceback.extract_tb(sys.exc_info()[2]) + line, out = _get_jinja_error(trace, context=decoded_context) + if not line: + tmplstr = "" + else: + tmplstr += "\n{}".format(tracestr) + log.debug("Jinja Error") + log.debug("Exception:", exc_info=True) + log.debug("Out: %s", out) + log.debug("Line: %s", line) + log.debug("TmplStr: %s", tmplstr) + log.debug("TraceStr: %s", tracestr) + + raise SaltRenderError( + "Jinja error: {}{}".format(exc, out), line, tmplstr, trace=tracestr + ) finally: - if loader and hasattr(loader, "_file_client"): - if hasattr(loader._file_client, "destroy"): - loader._file_client.destroy() + if loader and isinstance(loader, salt.utils.jinja.SaltCacheLoader): + loader.destroy() # Workaround a bug in Jinja that removes the final newline # (https://github.com/mitsuhiko/jinja2/issues/75) @@ -569,9 +575,8 @@ def render_mako_tmpl(tmplstr, context, tmplpath=None): except Exception: # pylint: disable=broad-except raise SaltRenderError(mako.exceptions.text_error_template().render()) finally: - if lookup and hasattr(lookup, "_file_client"): - if hasattr(lookup._file_client, "destroy"): - lookup._file_client.destroy() + if lookup and isinstance(lookup, SaltMakoTemplateLookup): + lookup.destroy() def render_wempy_tmpl(tmplstr, context, tmplpath=None): diff --git a/tests/pytests/integration/states/test_include.py b/tests/pytests/integration/states/test_include.py new file mode 100644 index 00000000000..f814328c5e4 --- /dev/null +++ b/tests/pytests/integration/states/test_include.py @@ -0,0 +1,40 @@ +""" +Integration tests for the jinja includes in states +""" +import logging + +import pytest + +log = logging.getLogger(__name__) + + +@pytest.mark.slow_test +def test_issue_64111(salt_master, salt_minion, salt_call_cli): + # This needs to be an integration test. A functional test does not trigger + # the issue fixed. + + macros_jinja = """ + {% macro a_jinja_macro(arg) -%} + {{ arg }} + {%- endmacro %} + """ + + init_sls = """ + include: + - common.file1 + """ + + file1_sls = """ + {% from 'common/macros.jinja' import a_jinja_macro with context %} + + a state id: + cmd.run: + - name: echo {{ a_jinja_macro("hello world") }} + """ + tf = salt_master.state_tree.base.temp_file + + with tf("common/macros.jinja", macros_jinja): + with tf("common/init.sls", init_sls): + with tf("common/file1.sls", file1_sls): + ret = salt_call_cli.run("state.apply", "common") + assert ret.returncode == 0 diff --git a/tests/pytests/unit/utils/jinja/test_salt_cache_loader.py b/tests/pytests/unit/utils/jinja/test_salt_cache_loader.py index 38c5ce5b724..e0f5fa158ff 100644 --- a/tests/pytests/unit/utils/jinja/test_salt_cache_loader.py +++ b/tests/pytests/unit/utils/jinja/test_salt_cache_loader.py @@ -15,7 +15,7 @@ import salt.utils.json # pylint: disable=unused-import import salt.utils.stringutils # pylint: disable=unused-import import salt.utils.yaml # pylint: disable=unused-import from salt.utils.jinja import SaltCacheLoader -from tests.support.mock import Mock, patch +from tests.support.mock import Mock, call, patch @pytest.fixture @@ -224,14 +224,45 @@ def test_file_client_kwarg(minion_opts, mock_file_client): assert loader._file_client is mock_file_client -def test_cache_loader_shutdown(minion_opts, mock_file_client): +def test_cache_loader_passed_file_client(minion_opts, mock_file_client): """ The shudown method can be called without raising an exception when the file_client does not have a destroy method """ - assert not hasattr(mock_file_client, "destroy") - mock_file_client.opts = minion_opts - loader = SaltCacheLoader(minion_opts, _file_client=mock_file_client) - assert loader._file_client is mock_file_client - # Shutdown method should not raise any exceptions - loader.shutdown() + # Test SaltCacheLoader creating and destroying the file client created + file_client = Mock() + with patch("salt.fileclient.get_file_client", return_value=file_client): + loader = SaltCacheLoader(minion_opts) + assert loader._file_client is None + with loader: + assert loader._file_client is file_client + assert loader._file_client is None + assert file_client.mock_calls == [call.destroy()] + + # Test SaltCacheLoader reusing the file client passed + file_client = Mock() + file_client.opts = {"file_roots": minion_opts["file_roots"]} + with patch("salt.fileclient.get_file_client", return_value=Mock()): + loader = SaltCacheLoader(minion_opts, _file_client=file_client) + assert loader._file_client is file_client + with loader: + assert loader._file_client is file_client + assert loader._file_client is file_client + assert file_client.mock_calls == [] + + # Test SaltCacheLoader creating a client even though a file client was + # passed because the "file_roots" option is different, and, as such, + # the destroy method on the new file client is called, but not on the + # file client passed in. + file_client = Mock() + file_client.opts = {"file_roots": ""} + new_file_client = Mock() + with patch("salt.fileclient.get_file_client", return_value=new_file_client): + loader = SaltCacheLoader(minion_opts, _file_client=file_client) + assert loader._file_client is file_client + with loader: + assert loader._file_client is not file_client + assert loader._file_client is new_file_client + assert loader._file_client is None + assert file_client.mock_calls == [] + assert new_file_client.mock_calls == [call.destroy()] -- 2.40.0