From aeed45986b7dbcdacaf33c541b6cae62bc76341f9908c6eba87c9dae01f5d722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klaus=20K=C3=A4mpf?= Date: Wed, 11 Apr 2018 16:47:08 +0000 Subject: [PATCH] Accepting request 594031 from systemsmanagement:saltstack:testing - Update to 2018.3.0 - Modified: * explore-module.run-response-to-catch-the-result-in-d.patch * add-saltssh-multi-version-support-across-python-inte.patch * run-salt-api-as-user-salt-bsc-1064520.patch * fix-openscap-push.patch * fix-decrease-loglevel-when-unable-to-resolve-addr.patch * fix-cp.push-empty-file.patch * make-it-possible-to-use-login-pull-and-push-from-mod.patch * avoid-excessive-syslogging-by-watchdog-cronjob-58.patch * feat-add-grain-for-all-fqdns.patch * fix-bsc-1065792.patch * run-salt-master-as-dedicated-salt-user.patch * move-log_file-option-to-changeable-defaults.patch * activate-all-beacons-sources-config-pillar-grains.patch * remove-obsolete-unicode-handling-in-pkg.info_install.patch - Add python-2.6 support to salt-ssh - Modified: * add-saltssh-multi-version-support-across-python-inte.patch - Update salt-ssh multiversion patch - Modified: * add-saltssh-multi-version-support-across-python-inte.patch - Removed: * require-same-major-version-while-minor-is-allowed-to.patch OBS-URL: https://build.opensuse.org/request/show/594031 OBS-URL: https://build.opensuse.org/package/show/systemsmanagement:saltstack/salt?expand=0&rev=117 --- 2018.3.0rc1.tar.gz | 3 - _lastrevision | 2 +- _service | 4 +- ...beacons-sources-config-pillar-grains.patch | 2 +- ...i-version-support-across-python-inte.patch | 1951 +++++++++++++++++ ...ve-syslogging-by-watchdog-cronjob-58.patch | 2 +- ...un-response-to-catch-the-result-in-d.patch | 131 ++ feat-add-grain-for-all-fqdns.patch | 8 +- fix-bsc-1065792.patch | 2 +- fix-cp.push-empty-file.patch | 2 +- ...loglevel-when-unable-to-resolve-addr.patch | 72 + fix-openscap-push.patch | 43 +- ...-to-use-login-pull-and-push-from-mod.patch | 119 + ...g_file-option-to-changeable-defaults.patch | 2 +- ...unicode-handling-in-pkg.info_install.patch | 18 +- run-salt-api-as-user-salt-bsc-1064520.patch | 2 +- run-salt-master-as-dedicated-salt-user.patch | 2 +- salt.changes | 129 ++ salt.spec | 31 +- v2018.3.0.tar.gz | 3 + 20 files changed, 2496 insertions(+), 32 deletions(-) delete mode 100644 2018.3.0rc1.tar.gz create mode 100644 add-saltssh-multi-version-support-across-python-inte.patch create mode 100644 explore-module.run-response-to-catch-the-result-in-d.patch create mode 100644 fix-decrease-loglevel-when-unable-to-resolve-addr.patch create mode 100644 make-it-possible-to-use-login-pull-and-push-from-mod.patch create mode 100644 v2018.3.0.tar.gz diff --git a/2018.3.0rc1.tar.gz b/2018.3.0rc1.tar.gz deleted file mode 100644 index 4f3f21f..0000000 --- a/2018.3.0rc1.tar.gz +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:90fb674a65567a3fd168d47eb95dcc5f9fee82ea201aff1d0fe91a2468d86900 -size 13577072 diff --git a/_lastrevision b/_lastrevision index a0e5a1a..98cd9fe 100644 --- a/_lastrevision +++ b/_lastrevision @@ -1 +1 @@ -HEAD +f43b8fb2425e3371decf3cde040c70ed15de375d \ No newline at end of file diff --git a/_service b/_service index beeadb9..c995eeb 100644 --- a/_service +++ b/_service @@ -12,8 +12,8 @@ codeload.github.com - saltstack/salt/tar.gz/2018.3.0rc1 - 2018.3.0rc1.tar.gz + saltstack/salt/tar.gz/v2018.3.0 + v2018.3.0.tar.gz diff --git a/activate-all-beacons-sources-config-pillar-grains.patch b/activate-all-beacons-sources-config-pillar-grains.patch index 968d357..39986ff 100644 --- a/activate-all-beacons-sources-config-pillar-grains.patch +++ b/activate-all-beacons-sources-config-pillar-grains.patch @@ -1,4 +1,4 @@ -From 8550c9cc51d227d4c32c844c7fe0526ae58f4874 Mon Sep 17 00:00:00 2001 +From 957ac8fe161db2c4b3b8fe8b84027bc15e144a49 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Tue, 17 Oct 2017 16:52:33 +0200 Subject: [PATCH] Activate all beacons sources: config/pillar/grains diff --git a/add-saltssh-multi-version-support-across-python-inte.patch b/add-saltssh-multi-version-support-across-python-inte.patch new file mode 100644 index 0000000..b656153 --- /dev/null +++ b/add-saltssh-multi-version-support-across-python-inte.patch @@ -0,0 +1,1951 @@ +From 7d3c1fee891a34f1e521228458ab113c3f6dabe1 Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Mon, 12 Mar 2018 12:01:39 +0100 +Subject: [PATCH] Add SaltSSH multi-version support across Python + interpeters. + +Bugfix: crashes when OPTIONS.saltdir is a file + +salt-ssh: allow server and client to run different python major version + +Handle non-directory on the /tmp + +Bugfix: prevent partial fileset removal in /tmp + +salt-ssh: compare checksums to detect newly generated thin on the server + +Reset time at thin unpack + +Bugfix: get a proper option for CLI and opts of wiping the tmp + +Add docstring to get_tops + +Remove unnecessary noise in imports + +Refactor get_tops collector + +Add logging to the get_tops + +Update call script + +Remove pre-caution + +Update log debug message for tops collector + +Reset default compression, if unknown is passed + +Refactor archive creation flow + +Add external shell-callable function to collect tops + +Simplify tops gathering, bugfix alternative to Py2 + +find working executable + +Add basic shareable module classifier + +Add proper error handler, unmuting exceptions during top collection + +Use common shared directory for compatible libraries + +fix searching for python versions + +Flatten error message string + +Bail-out immediately if <2.6 version detected + +Simplify shell cmd to get the version on Python 2.x + +Remove stub that was previously moved upfront + +Lintfix: PEP8 ident + +Add logging on the error, when Python-2 version cannot be detected properly + +Generate salt-call source, based on conditions + +Add logging on remove failure on thin.tgz archive + +Add config-based external tops gatherer + +Change signature to pass the extended configuration to the thin generator + +Update docstring to the salt-call generator + +Implement get namespaces inclusion to the salt-call script on the client machine + +Use new signature of the get call + +Implement namespace selector, based on the current Python interpreter version + +Add deps as a list, instead of a map + +Add debug logging + +Implement packaging an alternative version + +Update salt-call script so it swaps the namespace according to the configuration + +Compress thin.zip if zlib is available + +Fix a system exit error message + +Move compression fall-back operation + +Add debug logging prior to the thin archive removal + +Flatten the archive extension choice + +Lintfix: PEP8 an empty line required + +Bugfix: ZFS modules (zfs, zpool) crashes on non-ZFS systems + +Add unit test case for the Salt SSH parts + +Add unit test for missing dependencies on get_ext_tops + +Postpone inheritance implementation + +Refactor unit test for get_ext_tops + +Add unit test for get_ext_tops checks interpreter configuration + +Check python interpreter lock version + +Add unit test for get_ext_tops checks the python locked interepreter value + +Bugfix: report into warning log module name, not its config + +Add unit test for dependencies check python version lock (inherently) + +Mock os.path.isfile function + +Update warning logging information + +Add unit test for get_ext_tops module configuration validation + +Do not use list of dicts for namespaces, just dict for namespaces. + +Add unit test for get_ext_tops config verification + +Fix unit tests for the new config structure + +Add unit test for thin.gte call + +Add unit test for dependency path adding function + +Add unit test for thin_path function + +Add unit test for salt-call source generator + +Add unit test for get_ext_namespaces on empty configuration + +Add get_ext_namespaces for namespace extractions into a tuple for python version + +Remove unused variable + +Add unit test for getting namespace failure when python maj/min versions are not defined + +Add unit test to add tops based on the current interpreter + +Add unit test for get_tops with extra modules + +Add unit test for shared object modules top addition + +Add unit test for thin_sum hashing + +Add unit test for min_sum hashing + +Add unit test for gen_thin verify for 2.6 Python version is a minimum requirement + +Fix gen_thin exception on Python 3 + +Use object attribute instead of indeces. Remove an empty line. + +Add unit test for gen_thin compression type fallback + +Move helper functions up by the class code + +Update unit test doc + +Add check for correct archiving mode is opened + +Add unit test for gen_thin if control files are written correctly + +Update docstring for fake version info constructor method + +Add fake tarfile mock handler + +Mock-out missing methods inside gen_thin + +Move tarfile.open check to the end of the test + +Add unit test for tree addition to the archive + +Add shareable module to the gen_thin unit test + +Fix docstring + +Add unit test for an alternative version pack + +Lintfix + +Add documentation about updated Salt SSH features + +Fix typo + +Lintfix: PEP8 extra-line needed + +Make the command more readable + +Write all supported minimal python versions into a config file on the target machine + +Get supported Python executable based on the config py-map + +Add unit test for get_supported_py_config function typecheck + +Add unit test for get_supported_py_config function base tops + +Add unit test for get_supported_py_config function ext tops + +Fix unit test for catching "supported-versions" was written down + +Rephrase Salt SSH doc description + +Re-phrase docstring for alternative Salt installation + +require same major version while minor is allowed to be higher + +Bugfix: remove minor version from the namespaced, version-specific directory + +Fix unit tests for minor version removal of namespaced version-specific directory + +Initialise the options directly to be structure-ready object. + +Disable wiping if state is executed + +Properly mock a tempfile object + +Support Python 2.6 versions +--- + doc/topics/releases/fluorine.rst | 178 +++++++++++ + salt/client/ssh/__init__.py | 20 +- + salt/client/ssh/ssh_py_shim.py | 88 ++++-- + salt/client/ssh/wrapper/__init__.py | 2 +- + salt/modules/zfs.py | 4 +- + salt/modules/zpool.py | 4 +- + salt/utils/thin.py | 434 +++++++++++++++++++------- + tests/unit/utils/test_thin.py | 607 ++++++++++++++++++++++++++++++++++++ + 8 files changed, 1182 insertions(+), 155 deletions(-) + create mode 100644 doc/topics/releases/fluorine.rst + create mode 100644 tests/unit/utils/test_thin.py + +diff --git a/doc/topics/releases/fluorine.rst b/doc/topics/releases/fluorine.rst +new file mode 100644 +index 0000000000..40c69e25cc +--- /dev/null ++++ b/doc/topics/releases/fluorine.rst +@@ -0,0 +1,178 @@ ++:orphan: ++ ++====================================== ++Salt Release Notes - Codename Fluorine ++====================================== ++ ++ ++Minion Startup Events ++--------------------- ++ ++When a minion starts up it sends a notification on the event bus with a tag ++that looks like this: `salt/minion//start`. For historical reasons ++the minion also sends a similar event with an event tag like this: ++`minion_start`. This duplication can cause a lot of clutter on the event bus ++when there are many minions. Set `enable_legacy_startup_events: False` in the ++minion config to ensure only the `salt/minion//start` events are ++sent. ++ ++The new :conf_minion:`enable_legacy_startup_events` minion config option ++defaults to ``True``, but will be set to default to ``False`` beginning with ++the Neon release of Salt. ++ ++The Salt Syndic currently sends an old style `syndic_start` event as well. The ++syndic respects :conf_minion:`enable_legacy_startup_events` as well. ++ ++ ++Deprecations ++------------ ++ ++Module Deprecations ++=================== ++ ++The ``napalm_network`` module had the following changes: ++ ++- Support for the ``template_path`` has been removed in the ``load_template`` ++ function. This is because support for NAPALM native templates has been ++ dropped. ++ ++The ``trafficserver`` module had the following changes: ++ ++- Support for the ``match_var`` function was removed. Please use the ++ ``match_metric`` function instead. ++- Support for the ``read_var`` function was removed. Please use the ++ ``read_config`` function instead. ++- Support for the ``set_var`` function was removed. Please use the ++ ``set_config`` function instead. ++ ++The ``win_update`` module has been removed. It has been replaced by ``win_wua`` ++module. ++ ++The ``win_wua`` module had the following changes: ++ ++- Support for the ``download_update`` function has been removed. Please use the ++ ``download`` function instead. ++- Support for the ``download_updates`` function has been removed. Please use the ++ ``download`` function instead. ++- Support for the ``install_update`` function has been removed. Please use the ++ ``install`` function instead. ++- Support for the ``install_updates`` function has been removed. Please use the ++ ``install`` function instead. ++- Support for the ``list_update`` function has been removed. Please use the ++ ``get`` function instead. ++- Support for the ``list_updates`` function has been removed. Please use the ++ ``list`` function instead. ++ ++Pillar Deprecations ++=================== ++ ++The ``vault`` pillar had the following changes: ++ ++- Support for the ``profile`` argument was removed. Any options passed up until ++ and following the first ``path=`` are discarded. ++ ++Roster Deprecations ++=================== ++ ++The ``cache`` roster had the following changes: ++ ++- Support for ``roster_order`` as a list or tuple has been removed. As of the ++ ``Fluorine`` release, ``roster_order`` must be a dictionary. ++- The ``roster_order`` option now includes IPv6 in addition to IPv4 for the ++ ``private``, ``public``, ``global`` or ``local`` settings. The syntax for these ++ settings has changed to ``ipv4-*`` or ``ipv6-*``, respectively. ++ ++State Deprecations ++================== ++ ++The ``docker`` state has been removed. The following functions should be used ++instead. ++ ++- The ``docker.running`` function was removed. Please update applicable SLS files ++ to use the ``docker_container.running`` function instead. ++- The ``docker.stopped`` function was removed. Please update applicable SLS files ++ to use the ``docker_container.stopped`` function instead. ++- The ``docker.absent`` function was removed. Please update applicable SLS files ++ to use the ``docker_container.absent`` function instead. ++- The ``docker.absent`` function was removed. Please update applicable SLS files ++ to use the ``docker_container.absent`` function instead. ++- The ``docker.network_present`` function was removed. Please update applicable ++ SLS files to use the ``docker_network.present`` function instead. ++- The ``docker.network_absent`` function was removed. Please update applicable ++ SLS files to use the ``docker_network.absent`` function instead. ++- The ``docker.image_present`` function was removed. Please update applicable SLS ++ files to use the ``docker_image.present`` function instead. ++- The ``docker.image_absent`` function was removed. Please update applicable SLS ++ files to use the ``docker_image.absent`` function instead. ++- The ``docker.volume_present`` function was removed. Please update applicable SLS ++ files to use the ``docker_volume.present`` function instead. ++- The ``docker.volume_absent`` function was removed. Please update applicable SLS ++ files to use the ``docker_volume.absent`` function instead. ++ ++The ``docker_network`` state had the following changes: ++ ++- Support for the ``driver`` option has been removed from the ``absent`` function. ++ This option had no functionality in ``docker_network.absent``. ++ ++The ``git`` state had the following changes: ++ ++- Support for the ``ref`` option in the ``detached`` state has been removed. ++ Please use the ``rev`` option instead. ++ ++The ``k8s`` state has been removed. The following functions should be used ++instead: ++ ++- The ``k8s.label_absent`` function was removed. Please update applicable SLS ++ files to use the ``kubernetes.node_label_absent`` function instead. ++- The ``k8s.label_present`` function was removed. Please updated applicable SLS ++ files to use the ``kubernetes.node_label_present`` function instead. ++- The ``k8s.label_folder_absent`` function was removed. Please update applicable ++ SLS files to use the ``kubernetes.node_label_folder_absent`` function instead. ++ ++The ``netconfig`` state had the following changes: ++ ++- Support for the ``template_path`` option in the ``managed`` state has been ++ removed. This is because support for NAPALM native templates has been dropped. ++ ++The ``trafficserver`` state had the following changes: ++ ++- Support for the ``set_var`` function was removed. Please use the ``config`` ++ function instead. ++ ++The ``win_update`` state has been removed. Please use the ``win_wua`` state instead. ++ ++SaltSSH major updates ++===================== ++ ++SaltSSH now works across different major Python versions. Python 2.7 ~ Python 3.x ++are now supported transparently. Requirement is, however, that the SaltMaster should ++have installed Salt, including all related dependencies for Python 2 and Python 3. ++Everything needs to be importable from the respective Python environment. ++ ++SaltSSH can bundle up an arbitrary version of Salt. If there would be an old box for ++example, running an outdated and unsupported Python 2.6, it is still possible from ++a SaltMaster with Python 3.5 or newer to access it. This feature requires an additional ++configuration in /etc/salt/master as follows: ++ ++ ++.. code-block:: yaml ++ ++ ssh_ext_alternatives: ++ 2016.3: # Namespace, can be actually anything. ++ py-version: [2, 6] # Constraint to specific interpreter version ++ path: /opt/2016.3/salt # Main Salt installation ++ dependencies: # List of dependencies and their installation paths ++ jinja2: /opt/jinja2 ++ yaml: /opt/yaml ++ tornado: /opt/tornado ++ msgpack: /opt/msgpack ++ certifi: /opt/certifi ++ singledispatch: /opt/singledispatch.py ++ singledispatch_helpers: /opt/singledispatch_helpers.py ++ markupsafe: /opt/markupsafe ++ backports_abc: /opt/backports_abc.py ++ ++It is also possible to use several alternative versions of Salt. You can for instance generate ++a minimal tarball using runners and include that. But this is only possible, when such specific ++Salt version is also available on the Master machine, although does not need to be directly ++installed together with the older Python interpreter. +diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py +index f1c1ad9a22..ea5c700830 100644 +--- a/salt/client/ssh/__init__.py ++++ b/salt/client/ssh/__init__.py +@@ -150,14 +150,10 @@ EX_PYTHON_INVALID={EX_THIN_PYTHON_INVALID} + PYTHON_CMDS="python3 python27 python2.7 python26 python2.6 python2 python" + for py_cmd in $PYTHON_CMDS + do +- if command -v "$py_cmd" >/dev/null 2>&1 && "$py_cmd" -c \ +- "import sys; sys.exit(not (sys.version_info >= (2, 6) +- and sys.version_info[0] == {{HOST_PY_MAJOR}}));" ++ if command -v "$py_cmd" >/dev/null 2>&1 && "$py_cmd" -c "import sys; sys.exit(not (sys.version_info >= (2, 6)));" + then +- py_cmd_path=`"$py_cmd" -c \ +- 'from __future__ import print_function; +- import sys; print(sys.executable);'` +- cmdpath=$(command -v $py_cmd 2>/dev/null || which $py_cmd 2>/dev/null) ++ py_cmd_path=`"$py_cmd" -c 'from __future__ import print_function;import sys; print(sys.executable);'` ++ cmdpath=`command -v $py_cmd 2>/dev/null || which $py_cmd 2>/dev/null` + if file $cmdpath | grep "shell script" > /dev/null + then + ex_vars="'PATH', 'LD_LIBRARY_PATH', 'MANPATH', \ +@@ -323,7 +319,8 @@ class SSH(object): + extra_mods=self.opts.get('thin_extra_mods'), + overwrite=self.opts['regen_thin'], + python2_bin=self.opts['python2_bin'], +- python3_bin=self.opts['python3_bin']) ++ python3_bin=self.opts['python3_bin'], ++ extended_cfg=self.opts.get('ssh_ext_alternatives')) + self.mods = mod_data(self.fsclient) + + def _get_roster(self): +@@ -834,10 +831,10 @@ class Single(object): + + self.opts = opts + self.tty = tty +- if kwargs.get('wipe'): +- self.wipe = 'False' ++ if kwargs.get('disable_wipe'): ++ self.wipe = False + else: +- self.wipe = 'True' if self.opts.get('ssh_wipe') else 'False' ++ self.wipe = bool(self.opts.get('ssh_wipe')) + if kwargs.get('thin_dir'): + self.thin_dir = kwargs['thin_dir'] + elif self.winrm: +@@ -1168,7 +1165,6 @@ class Single(object): + if salt.log.LOG_LEVELS['debug'] >= salt.log.LOG_LEVELS[self.opts.get('log_level', 'info')]: + debug = '1' + arg_str = ''' +-OPTIONS = OBJ() + OPTIONS.config = \ + """ + {0} +diff --git a/salt/client/ssh/ssh_py_shim.py b/salt/client/ssh/ssh_py_shim.py +index e46220fc80..661a671b81 100644 +--- a/salt/client/ssh/ssh_py_shim.py ++++ b/salt/client/ssh/ssh_py_shim.py +@@ -16,11 +16,13 @@ import sys + import os + import stat + import subprocess ++import time + + THIN_ARCHIVE = 'salt-thin.tgz' + EXT_ARCHIVE = 'salt-ext_mods.tgz' + + # Keep these in sync with salt/defaults/exitcodes.py ++EX_THIN_PYTHON_INVALID = 10 + EX_THIN_DEPLOY = 11 + EX_THIN_CHECKSUM = 12 + EX_MOD_DEPLOY = 13 +@@ -28,14 +30,13 @@ EX_SCP_NOT_FOUND = 14 + EX_CANTCREAT = 73 + + +-class OBJ(object): ++class OptionsContainer(object): + ''' + An empty class for holding instance attribute values. + ''' +- pass + + +-OPTIONS = None ++OPTIONS = OptionsContainer() + ARGS = None + # The below line is where OPTIONS can be redefined with internal options + # (rather than cli arguments) when the shim is bundled by +@@ -128,7 +129,7 @@ def need_deployment(): + os.chmod(OPTIONS.saltdir, stt.st_mode | stat.S_IWGRP | stat.S_IRGRP | stat.S_IXGRP) + except OSError: + sys.stdout.write('\n\nUnable to set permissions on thin directory.\nIf sudo_user is set ' +- 'and is not root, be certain the user is in the same group\nas the login user') ++ 'and is not root, be certain the user is in the same group\nas the login user') + sys.exit(1) + + # Delimiter emitted on stdout *only* to indicate shim message to master. +@@ -161,11 +162,15 @@ def unpack_thin(thin_path): + old_umask = os.umask(0o077) + tfile.extractall(path=OPTIONS.saltdir) + tfile.close() +- os.umask(old_umask) ++ checksum_path = os.path.normpath(os.path.join(OPTIONS.saltdir, "thin_checksum")) ++ with open(checksum_path, 'w') as chk: ++ chk.write(OPTIONS.checksum + '\n') ++ os.umask(old_umask) # pylint: disable=blacklisted-function + try: + os.unlink(thin_path) + except OSError: + pass ++ reset_time(OPTIONS.saltdir) + + + def need_ext(): +@@ -199,6 +204,47 @@ def unpack_ext(ext_path): + shutil.move(ver_path, ver_dst) + + ++def reset_time(path='.', amt=None): ++ ''' ++ Reset atime/mtime on all files to prevent systemd swipes only part of the files in the /tmp. ++ ''' ++ if not amt: ++ amt = int(time.time()) ++ for fname in os.listdir(path): ++ fname = os.path.join(path, fname) ++ if os.path.isdir(fname): ++ reset_time(fname, amt=amt) ++ os.utime(fname, (amt, amt,)) ++ ++ ++def get_executable(): ++ ''' ++ Find executable which matches supported python version in the thin ++ ''' ++ pymap = {} ++ with open(os.path.join(OPTIONS.saltdir, 'supported-versions')) as _fp: ++ for line in _fp.readlines(): ++ ns, v_maj, v_min = line.strip().split(':') ++ pymap[ns] = (int(v_maj), int(v_min)) ++ ++ pycmds = (sys.executable, 'python3', 'python27', 'python2.7', 'python26', 'python2.6', 'python2', 'python') ++ for py_cmd in pycmds: ++ cmd = py_cmd + ' -c "import sys; sys.stdout.write(\'%s:%s\' % (sys.version_info[0], sys.version_info[1]))"' ++ stdout, _ = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True).communicate() ++ if sys.version_info[0] == 2 and sys.version_info[1] < 7: ++ stdout = stdout.decode(get_system_encoding(), "replace").strip() ++ else: ++ stdout = stdout.decode(encoding=get_system_encoding(), errors="replace").strip() ++ if not stdout: ++ continue ++ c_vn = tuple([int(x) for x in stdout.split(':')]) ++ for ns in pymap: ++ if c_vn[0] == pymap[ns][0] and c_vn >= pymap[ns] and os.path.exists(os.path.join(OPTIONS.saltdir, ns)): ++ return py_cmd ++ ++ sys.exit(EX_THIN_PYTHON_INVALID) ++ ++ + def main(argv): # pylint: disable=W0613 + ''' + Main program body +@@ -215,30 +261,30 @@ def main(argv): # pylint: disable=W0613 + if scpstat != 0: + sys.exit(EX_SCP_NOT_FOUND) + +- if not os.path.exists(OPTIONS.saltdir): +- need_deployment() +- +- if not os.path.isdir(OPTIONS.saltdir): ++ if os.path.exists(OPTIONS.saltdir) and not os.path.isdir(OPTIONS.saltdir): + sys.stderr.write( + 'ERROR: salt path "{0}" exists but is' + ' not a directory\n'.format(OPTIONS.saltdir) + ) + sys.exit(EX_CANTCREAT) + +- version_path = os.path.normpath(os.path.join(OPTIONS.saltdir, 'version')) +- if not os.path.exists(version_path) or not os.path.isfile(version_path): ++ if not os.path.exists(OPTIONS.saltdir): ++ need_deployment() ++ ++ checksum_path = os.path.normpath(os.path.join(OPTIONS.saltdir, 'thin_checksum')) ++ if not os.path.exists(checksum_path) or not os.path.isfile(checksum_path): + sys.stderr.write( + 'WARNING: Unable to locate current thin ' +- ' version: {0}.\n'.format(version_path) ++ ' checksum: {0}.\n'.format(checksum_path) + ) + need_deployment() +- with open(version_path, 'r') as vpo: +- cur_version = vpo.readline().strip() +- if cur_version != OPTIONS.version: ++ with open(checksum_path, 'r') as vpo: ++ cur_checksum = vpo.readline().strip() ++ if cur_checksum != OPTIONS.checksum: + sys.stderr.write( +- 'WARNING: current thin version {0}' ++ 'WARNING: current thin checksum {0}' + ' is not up-to-date with {1}.\n'.format( +- cur_version, OPTIONS.version ++ cur_checksum, OPTIONS.checksum + ) + ) + need_deployment() +@@ -270,7 +316,7 @@ def main(argv): # pylint: disable=W0613 + argv_prepared = ARGS + + salt_argv = [ +- sys.executable, ++ get_executable(), + salt_call_path, + '--retcode-passthrough', + '--local', +@@ -303,7 +349,10 @@ def main(argv): # pylint: disable=W0613 + if OPTIONS.tty: + # Returns bytes instead of string on python 3 + stdout, _ = subprocess.Popen(salt_argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() +- sys.stdout.write(stdout.decode(encoding=get_system_encoding(), errors="replace")) ++ if sys.version_info[0] == 2 and sys.version_info[1] < 7: ++ sys.stdout.write(stdout.decode(get_system_encoding(), "replace")) ++ else: ++ sys.stdout.write(stdout.decode(encoding=get_system_encoding(), errors="replace")) + sys.stdout.flush() + if OPTIONS.wipe: + shutil.rmtree(OPTIONS.saltdir) +@@ -315,5 +364,6 @@ def main(argv): # pylint: disable=W0613 + if OPTIONS.cmd_umask is not None: + os.umask(old_umask) + ++ + if __name__ == '__main__': + sys.exit(main(sys.argv)) +diff --git a/salt/client/ssh/wrapper/__init__.py b/salt/client/ssh/wrapper/__init__.py +index 04d751b51a..09f9344642 100644 +--- a/salt/client/ssh/wrapper/__init__.py ++++ b/salt/client/ssh/wrapper/__init__.py +@@ -113,7 +113,7 @@ class FunctionWrapper(object): + self.opts, + argv, + mods=self.mods, +- wipe=True, ++ disable_wipe=True, + fsclient=self.fsclient, + minion_opts=self.minion_opts, + **self.kwargs +diff --git a/salt/modules/zfs.py b/salt/modules/zfs.py +index bc54044b5c..d8fbfc76be 100644 +--- a/salt/modules/zfs.py ++++ b/salt/modules/zfs.py +@@ -37,10 +37,10 @@ def __virtual__(): + ''' + Only load when the platform has zfs support + ''' +- if __grains__['zfs_support']: ++ if __grains__.get('zfs_support'): + return __virtualname__ + else: +- return (False, "The zfs module cannot be loaded: zfs not supported") ++ return False, "The zfs module cannot be loaded: zfs not supported" + + + @decorators.memoize +diff --git a/salt/modules/zpool.py b/salt/modules/zpool.py +index f955175664..5e03418919 100644 +--- a/salt/modules/zpool.py ++++ b/salt/modules/zpool.py +@@ -31,10 +31,10 @@ def __virtual__(): + ''' + Only load when the platform has zfs support + ''' +- if __grains__['zfs_support']: ++ if __grains__.get('zfs_support'): + return __virtualname__ + else: +- return (False, "The zpool module cannot be loaded: zfs not supported") ++ return False, "The zpool module cannot be loaded: zfs not supported" + + + @salt.utils.decorators.memoize +diff --git a/salt/utils/thin.py b/salt/utils/thin.py +index 4c0969ea96..a6990d00b1 100644 +--- a/salt/utils/thin.py ++++ b/salt/utils/thin.py +@@ -8,11 +8,14 @@ from __future__ import absolute_import, print_function, unicode_literals + + import os + import sys ++import copy + import shutil + import tarfile + import zipfile + import tempfile + import subprocess ++import salt.utils.stringutils ++import logging + + # Import third party libs + import jinja2 +@@ -21,24 +24,26 @@ import msgpack + import salt.ext.six as _six + import tornado + ++try: ++ import zlib ++except ImportError: ++ zlib = None ++ + # pylint: disable=import-error,no-name-in-module + try: + import certifi +- HAS_CERTIFI = True + except ImportError: +- HAS_CERTIFI = False ++ certifi = None + + try: + import singledispatch +- HAS_SINGLEDISPATCH = True + except ImportError: +- HAS_SINGLEDISPATCH = False ++ singledispatch = None + + try: + import singledispatch_helpers +- HAS_SINGLEDISPATCH_HELPERS = True + except ImportError: +- HAS_SINGLEDISPATCH_HELPERS = False ++ singledispatch_helpers = None + + try: + import backports_abc +@@ -46,25 +51,22 @@ except ImportError: + import salt.ext.backports_abc as backports_abc + + try: ++ # New Jinja only + import markupsafe +- HAS_MARKUPSAFE = True + except ImportError: +- # Older jinja does not need markupsafe +- HAS_MARKUPSAFE = False ++ markupsafe = None + + # pylint: enable=import-error,no-name-in-module + + try: + # Older python where the backport from pypi is installed + from backports import ssl_match_hostname +- HAS_SSL_MATCH_HOSTNAME = True + except ImportError: + # Other older python we use our bundled copy + try: + from salt.ext import ssl_match_hostname +- HAS_SSL_MATCH_HOSTNAME = True + except ImportError: +- HAS_SSL_MATCH_HOSTNAME = False ++ ssl_match_hostname = None + + # Import salt libs + import salt +@@ -76,22 +78,52 @@ import salt.utils.stringutils + import salt.exceptions + import salt.version + +-SALTCALL = ''' ++log = logging.getLogger(__name__) ++ ++ ++def _get_salt_call(*dirs, **namespaces): ++ ''' ++ Return salt-call source, based on configuration. ++ This will include additional namespaces for another versions of Salt, ++ if needed (e.g. older interpreters etc). ++ ++ :dirs: List of directories to include in the system path ++ :namespaces: Dictionary of namespace ++ :return: ++ ''' ++ template = '''# -*- coding: utf-8 -*- + import os + import sys + +-sys.path.insert( +- 0, +- os.path.join( +- os.path.dirname(__file__), +- 'py{0[0]}'.format(sys.version_info) +- ) +-) ++# Namespaces is a map: {namespace: major/minor version}, like {'2016.11.4': [2, 6]} ++# Appears only when configured in Master configuration. ++namespaces = %namespaces% ++ ++# Default system paths alongside the namespaces ++syspaths = %dirs% ++syspaths.append('py{0}'.format(sys.version_info[0])) ++ ++curr_ver = (sys.version_info[0], sys.version_info[1],) ++ ++namespace = '' ++for ns in namespaces: ++ if curr_ver == tuple(namespaces[ns]): ++ namespace = ns ++ break ++ ++for base in syspaths: ++ sys.path.insert(0, os.path.join(os.path.dirname(__file__), ++ namespace and os.path.join(namespace, base) or base)) + +-from salt.scripts import salt_call + if __name__ == '__main__': ++ from salt.scripts import salt_call + salt_call() +-'''.encode('utf-8') ++''' ++ ++ for tgt, cnt in [('%dirs%', dirs), ('%namespaces%', namespaces)]: ++ template = template.replace(tgt, salt.utils.json.dumps(cnt)) ++ ++ return salt.utils.stringutils.to_bytes(template) + + + def thin_path(cachedir): +@@ -101,29 +133,136 @@ def thin_path(cachedir): + return os.path.join(cachedir, 'thin', 'thin.tgz') + + +-def get_tops(extra_mods='', so_mods=''): +- tops = [ +- os.path.dirname(salt.__file__), +- os.path.dirname(jinja2.__file__), +- os.path.dirname(yaml.__file__), +- os.path.dirname(tornado.__file__), +- os.path.dirname(msgpack.__file__), +- ] ++def _is_shareable(mod): ++ ''' ++ Return True if module is share-able between major Python versions. ++ ++ :param mod: ++ :return: ++ ''' ++ # This list is subject to change ++ shareable = ['salt', 'jinja2', ++ 'msgpack', 'certifi'] ++ ++ return os.path.basename(mod) in shareable ++ ++ ++def _add_dependency(container, obj): ++ ''' ++ Add a dependency to the top list. ++ ++ :param obj: ++ :param is_file: ++ :return: ++ ''' ++ if os.path.basename(obj.__file__).split('.')[0] == '__init__': ++ container.append(os.path.dirname(obj.__file__)) ++ else: ++ container.append(obj.__file__.replace('.pyc', '.py')) ++ ++ ++def gte(): ++ ''' ++ This function is called externally from the alternative ++ Python interpreter from within _get_tops function. ++ ++ :param extra_mods: ++ :param so_mods: ++ :return: ++ ''' ++ extra = salt.utils.json.loads(sys.argv[1]) ++ tops = get_tops(**extra) ++ ++ return salt.utils.json.dumps(tops, ensure_ascii=False) ++ ++ ++def get_ext_tops(config): ++ ''' ++ Get top directories for the dependencies, based on external configuration. ++ ++ :return: ++ ''' ++ alternatives = {} ++ required = ['jinja2', 'yaml', 'tornado', 'msgpack'] ++ tops = [] ++ for ns, cfg in salt.ext.six.iteritems(config or {}): ++ alternatives[ns] = cfg ++ locked_py_version = cfg.get('py-version') ++ err_msg = None ++ if not locked_py_version: ++ err_msg = 'Alternative Salt library: missing specific locked Python version' ++ elif not isinstance(locked_py_version, (tuple, list)): ++ err_msg = ('Alternative Salt library: specific locked Python version ' ++ 'should be a list of major/minor version') ++ if err_msg: ++ raise salt.exceptions.SaltSystemExit(err_msg) ++ ++ if cfg.get('dependencies') == 'inherit': ++ # TODO: implement inheritance of the modules from _here_ ++ raise NotImplementedError('This feature is not yet implemented') ++ else: ++ for dep in cfg.get('dependencies'): ++ mod = cfg['dependencies'][dep] or '' ++ if not mod: ++ log.warning('Module %s has missing configuration', dep) ++ continue ++ elif mod.endswith('.py') and not os.path.isfile(mod): ++ log.warning('Module %s configured with not a file or does not exist: %s', dep, mod) ++ continue ++ elif not mod.endswith('.py') and not os.path.isfile(os.path.join(mod, '__init__.py')): ++ log.warning('Module %s is not a Python importable module with %s', dep, mod) ++ continue ++ tops.append(mod) ++ ++ if dep in required: ++ required.pop(required.index(dep)) ++ ++ required = ', '.join(required) ++ if required: ++ msg = 'Missing dependencies for the alternative version' \ ++ ' in the external configuration: {}'.format(required) ++ log.error(msg) ++ raise salt.exceptions.SaltSystemExit(msg) ++ alternatives[ns]['dependencies'] = tops ++ return alternatives ++ ++ ++def _get_ext_namespaces(config): ++ ''' ++ Get namespaces from the existing configuration. + +- tops.append(_six.__file__.replace('.pyc', '.py')) +- tops.append(backports_abc.__file__.replace('.pyc', '.py')) ++ :param config: ++ :return: ++ ''' ++ namespaces = {} ++ if not config: ++ return namespaces ++ ++ for ns in config: ++ constraint_version = tuple(config[ns].get('py-version', [])) ++ if not constraint_version: ++ raise salt.exceptions.SaltSystemExit("An alternative version is configured, but not defined " ++ "to what Python's major/minor version it should be constrained.") ++ else: ++ namespaces[ns] = constraint_version + +- if HAS_CERTIFI: +- tops.append(os.path.dirname(certifi.__file__)) ++ return namespaces + +- if HAS_SINGLEDISPATCH: +- tops.append(singledispatch.__file__.replace('.pyc', '.py')) + +- if HAS_SINGLEDISPATCH_HELPERS: +- tops.append(singledispatch_helpers.__file__.replace('.pyc', '.py')) ++def get_tops(extra_mods='', so_mods=''): ++ ''' ++ Get top directories for the dependencies, based on Python interpreter. + +- if HAS_SSL_MATCH_HOSTNAME: +- tops.append(os.path.dirname(os.path.dirname(ssl_match_hostname.__file__))) ++ :param extra_mods: ++ :param so_mods: ++ :return: ++ ''' ++ tops = [] ++ for mod in [salt, jinja2, yaml, tornado, msgpack, certifi, singledispatch, ++ singledispatch_helpers, ssl_match_hostname, markupsafe, backports_abc]: ++ if mod: ++ log.debug('Adding module to the tops: "%s"', mod.__name__) ++ _add_dependency(tops, mod) + + for mod in [m for m in extra_mods.split(',') if m]: + if mod not in locals() and mod not in globals(): +@@ -135,28 +274,49 @@ def get_tops(extra_mods='', so_mods=''): + tops.append(moddir) + else: + tops.append(os.path.join(moddir, base + '.py')) +- except ImportError: +- # Not entirely sure this is the right thing, but the only +- # options seem to be 1) fail, 2) spew errors, or 3) pass. +- # Nothing else in here spits errors, and the markupsafe code +- # doesn't bail on import failure, so I followed that lead. +- # And of course, any other failure still S/T's. +- pass ++ except ImportError as err: ++ log.exception(err) ++ log.error('Unable to import extra-module "%s"', mod) ++ + for mod in [m for m in so_mods.split(',') if m]: + try: + locals()[mod] = __import__(mod) + tops.append(locals()[mod].__file__) +- except ImportError: +- pass # As per comment above +- if HAS_MARKUPSAFE: +- tops.append(os.path.dirname(markupsafe.__file__)) ++ except ImportError as err: ++ log.exception(err) ++ log.error('Unable to import so-module "%s"', mod) + + return tops + + ++def _get_supported_py_config(tops, extended_cfg): ++ ''' ++ Based on the Salt SSH configuration, create a YAML configuration ++ for the supported Python interpreter versions. This is then written into the thin.tgz ++ archive and then verified by salt.client.ssh.ssh_py_shim.get_executable() ++ ++ Note: Minimum default of 2.x versions is 2.7 and 3.x is 3.0, unless specified in namespaces. ++ ++ :return: ++ ''' ++ pymap = [] ++ for py_ver, tops in _six.iteritems(copy.deepcopy(tops)): ++ py_ver = int(py_ver) ++ if py_ver == 2: ++ pymap.append('py2:2:7') ++ elif py_ver == 3: ++ pymap.append('py3:3:0') ++ ++ for ns, cfg in _six.iteritems(copy.deepcopy(extended_cfg) or {}): ++ pymap.append('{}:{}:{}'.format(ns, *cfg.get('py-version'))) ++ pymap.append('') ++ ++ return salt.utils.stringutils.to_bytes(os.linesep.join(pymap)) ++ ++ + def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='', + python2_bin='python2', python3_bin='python3', absonly=True, +- compress='gzip'): ++ compress='gzip', extended_cfg=None): + ''' + Generate the salt-thin tarball and print the location of the tarball + Optional additional mods to include (e.g. mako) can be supplied as a comma +@@ -171,19 +331,24 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='', + salt-run thin.generate mako,wempy 1 + salt-run thin.generate overwrite=1 + ''' ++ if sys.version_info < (2, 6): ++ raise salt.exceptions.SaltSystemExit('The minimum required python version to run salt-ssh is "2.6".') ++ if compress not in ['gzip', 'zip']: ++ log.warning('Unknown compression type: "%s". Falling back to "gzip" compression.', compress) ++ compress = 'gzip' ++ + thindir = os.path.join(cachedir, 'thin') + if not os.path.isdir(thindir): + os.makedirs(thindir) +- if compress == 'gzip': +- thin_ext = 'tgz' +- elif compress == 'zip': +- thin_ext = 'zip' +- thintar = os.path.join(thindir, 'thin.' + thin_ext) ++ thintar = os.path.join(thindir, 'thin.' + (compress == 'gzip' and 'tgz' or 'zip')) + thinver = os.path.join(thindir, 'version') + pythinver = os.path.join(thindir, '.thin-gen-py-version') + salt_call = os.path.join(thindir, 'salt-call') ++ pymap_cfg = os.path.join(thindir, 'supported-versions') ++ + with salt.utils.files.fopen(salt_call, 'wb') as fp_: +- fp_.write(SALTCALL) ++ fp_.write(_get_salt_call('pyall', **_get_ext_namespaces(extended_cfg))) ++ + if os.path.isfile(thintar): + if not overwrite: + if os.path.isfile(thinver): +@@ -197,85 +362,88 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='', + + if overwrite: + try: ++ log.debug('Removing %s archive file', thintar) + os.remove(thintar) +- except OSError: +- pass ++ except OSError as exc: ++ log.error('Error while removing %s file: %s', thintar, exc) ++ if os.path.exists(thintar): ++ raise salt.exceptions.SaltSystemExit('Unable to remove %s file. See logs for details.', thintar) + else: + return thintar + if _six.PY3: + # Let's check for the minimum python 2 version requirement, 2.6 +- py_shell_cmd = ( +- python2_bin + ' -c \'from __future__ import print_function; import sys; ' +- 'print("{0}.{1}".format(*(sys.version_info[:2])));\'' +- ) ++ py_shell_cmd = "{} -c 'import sys;sys.stdout.write(\"%s.%s\\n\" % sys.version_info[:2]);'".format(python2_bin) + cmd = subprocess.Popen(py_shell_cmd, stdout=subprocess.PIPE, shell=True) + stdout, _ = cmd.communicate() + if cmd.returncode == 0: + py2_version = tuple(int(n) for n in stdout.decode('utf-8').strip().split('.')) + if py2_version < (2, 6): +- # Bail! + raise salt.exceptions.SaltSystemExit( + 'The minimum required python version to run salt-ssh is "2.6".' + 'The version reported by "{0}" is "{1}". Please try "salt-ssh ' +- '--python2-bin=".'.format(python2_bin, +- stdout.strip()) +- ) +- elif sys.version_info < (2, 6): +- # Bail! Though, how did we reached this far in the first place. +- raise salt.exceptions.SaltSystemExit( +- 'The minimum required python version to run salt-ssh is "2.6".' +- ) ++ '--python2-bin=".'.format(python2_bin, stdout.strip())) ++ else: ++ log.error('Unable to detect Python-2 version') ++ log.debug(stdout) + ++ tops_failure_msg = 'Failed %s tops for Python binary %s.' + tops_py_version_mapping = {} + tops = get_tops(extra_mods=extra_mods, so_mods=so_mods) +- if _six.PY2: +- tops_py_version_mapping['2'] = tops +- else: +- tops_py_version_mapping['3'] = tops ++ tops_py_version_mapping[sys.version_info.major] = tops + +- # TODO: Consider putting known py2 and py3 compatible libs in it's own sharable directory. +- # This would reduce the thin size. +- if _six.PY2 and sys.version_info[0] == 2: ++ # Collect tops, alternative to 2.x version ++ if _six.PY2 and sys.version_info.major == 2: + # Get python 3 tops +- py_shell_cmd = ( +- python3_bin + ' -c \'import sys; import json; import salt.utils.thin; ' +- 'print(json.dumps(salt.utils.thin.get_tops(**(json.loads(sys.argv[1]))), ensure_ascii=False)); exit(0);\' ' +- '\'{0}\''.format(salt.utils.json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods})) +- ) ++ py_shell_cmd = "{0} -c 'import salt.utils.thin as t;print(t.gte())' '{1}'".format( ++ python3_bin, salt.utils.json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods})) + cmd = subprocess.Popen(py_shell_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + stdout, stderr = cmd.communicate() + if cmd.returncode == 0: + try: + tops = salt.utils.json.loads(stdout) + tops_py_version_mapping['3'] = tops +- except ValueError: +- pass +- if _six.PY3 and sys.version_info[0] == 3: ++ except ValueError as err: ++ log.error(tops_failure_msg, 'parsing', python3_bin) ++ log.exception(err) ++ else: ++ log.error(tops_failure_msg, 'collecting', python3_bin) ++ log.debug(stderr) ++ ++ # Collect tops, alternative to 3.x version ++ if _six.PY3 and sys.version_info.major == 3: + # Get python 2 tops +- py_shell_cmd = ( +- python2_bin + ' -c \'from __future__ import print_function; ' +- 'import sys; import json; import salt.utils.thin; ' +- 'print(json.dumps(salt.utils.thin.get_tops(**(json.loads(sys.argv[1]))), ensure_ascii=False)); exit(0);\' ' +- '\'{0}\''.format(salt.utils.json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods})) +- ) ++ py_shell_cmd = "{0} -c 'import salt.utils.thin as t;print(t.gte())' '{1}'".format( ++ python2_bin, salt.utils.json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods})) + cmd = subprocess.Popen(py_shell_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + stdout, stderr = cmd.communicate() + if cmd.returncode == 0: + try: + tops = salt.utils.json.loads(stdout.decode('utf-8')) + tops_py_version_mapping['2'] = tops +- except ValueError: +- pass ++ except ValueError as err: ++ log.error(tops_failure_msg, 'parsing', python2_bin) ++ log.exception(err) ++ else: ++ log.error(tops_failure_msg, 'collecting', python2_bin) ++ log.debug(stderr) ++ ++ with salt.utils.files.fopen(pymap_cfg, 'wb') as fp_: ++ fp_.write(_get_supported_py_config(tops=tops_py_version_mapping, extended_cfg=extended_cfg)) + + if compress == 'gzip': + tfp = tarfile.open(thintar, 'w:gz', dereference=True) + elif compress == 'zip': +- tfp = zipfile.ZipFile(thintar, 'w') ++ tfp = zipfile.ZipFile(thintar, 'w', compression=zlib and zipfile.ZIP_DEFLATED or zipfile.ZIP_STORED) ++ tfp.add = tfp.write ++ + try: # cwd may not exist if it was removed but salt was run from it + start_dir = os.getcwd() + except OSError: + start_dir = None + tempdir = None ++ ++ # Pack default data ++ log.debug('Packing default libraries based on current Salt version') + for py_ver, tops in _six.iteritems(tops_py_version_mapping): + for top in tops: + if absonly and not os.path.isabs(top): +@@ -291,48 +459,76 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='', + egg.extractall(tempdir) + top = os.path.join(tempdir, base) + os.chdir(tempdir) ++ ++ site_pkg_dir = _is_shareable(base) and 'pyall' or 'py{}'.format(py_ver) ++ ++ log.debug('Packing "%s" to "%s" destination', base, site_pkg_dir) + if not os.path.isdir(top): + # top is a single file module + if os.path.exists(os.path.join(top_dirname, base)): +- if compress == 'gzip': +- tfp.add(base, arcname=os.path.join('py{0}'.format(py_ver), base)) +- elif compress == 'zip': +- tfp.write(base, arcname=os.path.join('py{0}'.format(py_ver), base)) ++ tfp.add(base, arcname=os.path.join(site_pkg_dir, base)) + continue + for root, dirs, files in salt.utils.path.os_walk(base, followlinks=True): + for name in files: + if not name.endswith(('.pyc', '.pyo')): +- if compress == 'gzip': +- tfp.add(os.path.join(root, name), +- arcname=os.path.join('py{0}'.format(py_ver), root, name)) +- elif compress == 'zip': ++ arcname = os.path.join(site_pkg_dir, root, name) ++ if hasattr(tfp, 'getinfo'): + try: + # This is a little slow but there's no clear way to detect duplicates +- tfp.getinfo(os.path.join('py{0}'.format(py_ver), root, name)) ++ tfp.getinfo(os.path.join(site_pkg_dir, root, name)) ++ arcname = None + except KeyError: +- tfp.write(os.path.join(root, name), arcname=os.path.join('py{0}'.format(py_ver), root, name)) ++ log.debug('ZIP: Unable to add "%s" with "getinfo"', arcname) ++ if arcname: ++ tfp.add(os.path.join(root, name), arcname=arcname) ++ + if tempdir is not None: + shutil.rmtree(tempdir) + tempdir = None ++ ++ # Pack alternative data ++ if extended_cfg: ++ log.debug('Packing libraries based on alternative Salt versions') ++ for ns, cfg in _six.iteritems(get_ext_tops(extended_cfg)): ++ tops = [cfg.get('path')] + cfg.get('dependencies') ++ py_ver_major, py_ver_minor = cfg.get('py-version') ++ for top in tops: ++ base, top_dirname = os.path.basename(top), os.path.dirname(top) ++ os.chdir(top_dirname) ++ site_pkg_dir = _is_shareable(base) and 'pyall' or 'py{0}'.format(py_ver_major) ++ log.debug('Packing alternative "%s" to "%s/%s" destination', base, ns, site_pkg_dir) ++ if not os.path.isdir(top): ++ # top is a single file module ++ if os.path.exists(os.path.join(top_dirname, base)): ++ tfp.add(base, arcname=os.path.join(ns, site_pkg_dir, base)) ++ continue ++ for root, dirs, files in salt.utils.path.os_walk(base, followlinks=True): ++ for name in files: ++ if not name.endswith(('.pyc', '.pyo')): ++ arcname = os.path.join(ns, site_pkg_dir, root, name) ++ if hasattr(tfp, 'getinfo'): ++ try: ++ tfp.getinfo(os.path.join(site_pkg_dir, root, name)) ++ arcname = None ++ except KeyError: ++ log.debug('ZIP: Unable to add "%s" with "getinfo"', arcname) ++ if arcname: ++ tfp.add(os.path.join(root, name), arcname=arcname) ++ + os.chdir(thindir) +- if compress == 'gzip': +- tfp.add('salt-call') +- elif compress == 'zip': +- tfp.write('salt-call') + with salt.utils.files.fopen(thinver, 'w+') as fp_: + fp_.write(salt.version.__version__) + with salt.utils.files.fopen(pythinver, 'w+') as fp_: +- fp_.write(str(sys.version_info[0])) # future lint: disable=blacklisted-function ++ fp_.write(str(sys.version_info.major)) # future lint: disable=blacklisted-function + os.chdir(os.path.dirname(thinver)) +- if compress == 'gzip': +- tfp.add('version') +- tfp.add('.thin-gen-py-version') +- elif compress == 'zip': +- tfp.write('version') +- tfp.write('.thin-gen-py-version') ++ ++ for fname in ['version', '.thin-gen-py-version', 'salt-call', 'supported-versions']: ++ tfp.add(fname) ++ + if start_dir: + os.chdir(start_dir) + tfp.close() ++ + return thintar + + +@@ -368,7 +564,7 @@ def gen_min(cachedir, extra_mods='', overwrite=False, so_mods='', + pyminver = os.path.join(mindir, '.min-gen-py-version') + salt_call = os.path.join(mindir, 'salt-call') + with salt.utils.files.fopen(salt_call, 'wb') as fp_: +- fp_.write(SALTCALL) ++ fp_.write(_get_salt_call()) + if os.path.isfile(mintar): + if not overwrite: + if os.path.isfile(minver): +diff --git a/tests/unit/utils/test_thin.py b/tests/unit/utils/test_thin.py +new file mode 100644 +index 0000000000..8157eefed8 +--- /dev/null ++++ b/tests/unit/utils/test_thin.py +@@ -0,0 +1,607 @@ ++# -*- coding: utf-8 -*- ++''' ++ :codeauthor: :email:`Bo Maryniuk ` ++''' ++from __future__ import absolute_import, print_function, unicode_literals ++ ++import os ++import sys ++from tests.support.unit import TestCase, skipIf ++from tests.support.mock import ( ++ NO_MOCK, ++ NO_MOCK_REASON, ++ MagicMock, ++ patch) ++ ++import salt.exceptions ++from salt.utils import thin ++from salt.utils import json ++import salt.utils.stringutils ++from salt.utils.stringutils import to_bytes as bts ++ ++try: ++ import pytest ++except ImportError: ++ pytest = None ++ ++ ++@skipIf(NO_MOCK, NO_MOCK_REASON) ++@skipIf(pytest is None, 'PyTest is missing') ++class SSHThinTestCase(TestCase): ++ ''' ++ TestCase for SaltSSH-related parts. ++ ''' ++ def _popen(self, return_value=None, side_effect=None, returncode=0): ++ ''' ++ Fake subprocess.Popen ++ ++ :return: ++ ''' ++ ++ proc = MagicMock() ++ proc.communicate = MagicMock(return_value=return_value, side_effect=side_effect) ++ proc.returncode = returncode ++ popen = MagicMock(return_value=proc) ++ ++ return popen ++ ++ def _version_info(self, major=None, minor=None): ++ ''' ++ Fake version info. ++ ++ :param major: ++ :param minor: ++ :return: ++ ''' ++ class VersionInfo(tuple): ++ pass ++ ++ vi = VersionInfo([major, minor]) ++ vi.major = major or sys.version_info.major ++ vi.minor = minor or sys.version_info.minor ++ ++ return vi ++ ++ def _tarfile(self, getinfo=False): ++ ''' ++ Fake tarfile handler. ++ ++ :return: ++ ''' ++ spec = ['add', 'close'] ++ if getinfo: ++ spec.append('getinfo') ++ ++ tf = MagicMock() ++ tf.open = MagicMock(return_value=MagicMock(spec=spec)) ++ ++ return tf ++ ++ ++ @patch('salt.exceptions.SaltSystemExit', Exception) ++ @patch('salt.utils.thin.log', MagicMock()) ++ @patch('salt.utils.thin.os.path.isfile', MagicMock(return_value=False)) ++ def test_get_ext_tops_cfg_missing_dependencies(self): ++ ''' ++ Test thin.get_ext_tops contains all required dependencies. ++ ++ :return: ++ ''' ++ cfg = {'namespace': {'py-version': [0, 0], 'path': '/foo', 'dependencies': []}} ++ ++ with pytest.raises(Exception) as err: ++ thin.get_ext_tops(cfg) ++ assert 'Missing dependencies' in str(err) ++ assert thin.log.error.called ++ assert 'Missing dependencies' in thin.log.error.call_args[0][0] ++ assert 'jinja2, yaml, tornado, msgpack' in thin.log.error.call_args[0][0] ++ ++ @patch('salt.exceptions.SaltSystemExit', Exception) ++ @patch('salt.utils.thin.log', MagicMock()) ++ @patch('salt.utils.thin.os.path.isfile', MagicMock(return_value=False)) ++ def test_get_ext_tops_cfg_missing_interpreter(self): ++ ''' ++ Test thin.get_ext_tops contains interpreter configuration. ++ ++ :return: ++ ''' ++ cfg = {'namespace': {'path': '/foo', ++ 'dependencies': []}} ++ with pytest.raises(salt.exceptions.SaltSystemExit) as err: ++ thin.get_ext_tops(cfg) ++ assert 'missing specific locked Python version' in str(err) ++ ++ @patch('salt.exceptions.SaltSystemExit', Exception) ++ @patch('salt.utils.thin.log', MagicMock()) ++ @patch('salt.utils.thin.os.path.isfile', MagicMock(return_value=False)) ++ def test_get_ext_tops_cfg_wrong_interpreter(self): ++ ''' ++ Test thin.get_ext_tops contains correct interpreter configuration. ++ ++ :return: ++ ''' ++ cfg = {'namespace': {'path': '/foo', ++ 'py-version': 2, ++ 'dependencies': []}} ++ ++ with pytest.raises(salt.exceptions.SaltSystemExit) as err: ++ thin.get_ext_tops(cfg) ++ assert 'specific locked Python version should be a list of major/minor version' in str(err) ++ ++ @patch('salt.exceptions.SaltSystemExit', Exception) ++ @patch('salt.utils.thin.log', MagicMock()) ++ @patch('salt.utils.thin.os.path.isfile', MagicMock(return_value=False)) ++ def test_get_ext_tops_cfg_interpreter(self): ++ ''' ++ Test thin.get_ext_tops interpreter configuration. ++ ++ :return: ++ ''' ++ cfg = {'namespace': {'path': '/foo', ++ 'py-version': [2, 6], ++ 'dependencies': {'jinja2': '', ++ 'yaml': '', ++ 'tornado': '', ++ 'msgpack': ''}}} ++ ++ with pytest.raises(salt.exceptions.SaltSystemExit): ++ thin.get_ext_tops(cfg) ++ assert len(thin.log.warning.mock_calls) == 4 ++ assert sorted([x[1][1] for x in thin.log.warning.mock_calls]) == ['jinja2', 'msgpack', 'tornado', 'yaml'] ++ assert 'Module test has missing configuration' == thin.log.warning.mock_calls[0][1][0] % 'test' ++ ++ @patch('salt.exceptions.SaltSystemExit', Exception) ++ @patch('salt.utils.thin.log', MagicMock()) ++ @patch('salt.utils.thin.os.path.isfile', MagicMock(return_value=False)) ++ def test_get_ext_tops_dependency_config_check(self): ++ ''' ++ Test thin.get_ext_tops dependencies are importable ++ ++ :return: ++ ''' ++ cfg = {'namespace': {'path': '/foo', ++ 'py-version': [2, 6], ++ 'dependencies': {'jinja2': '/jinja/foo.py', ++ 'yaml': '/yaml/', ++ 'tornado': '/tornado/wrong.rb', ++ 'msgpack': 'msgpack.sh'}}} ++ ++ with pytest.raises(salt.exceptions.SaltSystemExit) as err: ++ thin.get_ext_tops(cfg) ++ assert 'Missing dependencies for the alternative version in the external configuration' in str(err) ++ ++ messages = {} ++ for cl in thin.log.warning.mock_calls: ++ messages[cl[1][1]] = cl[1][0] % (cl[1][1], cl[1][2]) ++ for mod in ['tornado', 'yaml', 'msgpack']: ++ assert 'not a Python importable module' in messages[mod] ++ assert 'configured with not a file or does not exist' in messages['jinja2'] ++ ++ @patch('salt.exceptions.SaltSystemExit', Exception) ++ @patch('salt.utils.thin.log', MagicMock()) ++ @patch('salt.utils.thin.os.path.isfile', MagicMock(return_value=True)) ++ def test_get_ext_tops_config_pass(self): ++ ''' ++ Test thin.get_ext_tops configuration ++ ++ :return: ++ ''' ++ cfg = {'namespace': {'path': '/foo', ++ 'py-version': [2, 6], ++ 'dependencies': {'jinja2': '/jinja/foo.py', ++ 'yaml': '/yaml/', ++ 'tornado': '/tornado/tornado.py', ++ 'msgpack': 'msgpack.py'}}} ++ assert cfg == thin.get_ext_tops(cfg) ++ ++ @patch('salt.utils.thin.sys.argv', [None, '{"foo": "bar"}']) ++ @patch('salt.utils.thin.get_tops', lambda **kw: kw) ++ def test_gte(self): ++ ''' ++ Test thin.gte external call for processing the info about tops per interpreter. ++ ++ :return: ++ ''' ++ assert json.loads(thin.gte()).get('foo') == 'bar' ++ ++ def test_add_dep_path(self): ++ ''' ++ Test thin._add_dependency function to setup dependency paths ++ :return: ++ ''' ++ container = [] ++ for pth in ['/foo/bar.py', '/something/else/__init__.py']: ++ thin._add_dependency(container, type(str('obj'), (), {'__file__': pth})()) ++ assert '__init__' not in container[1] ++ assert container == ['/foo/bar.py', '/something/else'] ++ ++ def test_thin_path(self): ++ ''' ++ Test thin.thin_path returns the expected path. ++ ++ :return: ++ ''' ++ assert thin.thin_path('/path/to') == '/path/to/thin/thin.tgz' ++ ++ def test_get_salt_call_script(self): ++ ''' ++ Test get salt-call script rendered. ++ ++ :return: ++ ''' ++ out = thin._get_salt_call('foo', 'bar', py26=[2, 6], py27=[2, 7], py34=[3, 4]) ++ for line in salt.utils.stringutils.to_str(out).split(os.linesep): ++ if line.startswith('namespaces = {'): ++ data = json.loads(line.replace('namespaces = ', '').strip()) ++ assert data.get('py26') == [2, 6] ++ assert data.get('py27') == [2, 7] ++ assert data.get('py34') == [3, 4] ++ if line.startswith('syspaths = '): ++ data = json.loads(line.replace('syspaths = ', '')) ++ assert data == ['foo', 'bar'] ++ ++ def test_get_ext_namespaces_empty(self): ++ ''' ++ Test thin._get_ext_namespaces function returns an empty dictionary on nothing ++ :return: ++ ''' ++ for obj in [None, {}, []]: ++ assert thin._get_ext_namespaces(obj) == {} ++ ++ def test_get_ext_namespaces(self): ++ ''' ++ Test thin._get_ext_namespaces function returns namespaces properly out of the config. ++ :return: ++ ''' ++ cfg = {'ns': {'py-version': [2, 7]}} ++ assert thin._get_ext_namespaces(cfg).get('ns') == (2, 7,) ++ assert isinstance(thin._get_ext_namespaces(cfg).get('ns'), tuple) ++ ++ def test_get_ext_namespaces_failure(self): ++ ''' ++ Test thin._get_ext_namespaces function raises an exception ++ if python major/minor version is not configured. ++ :return: ++ ''' ++ with pytest.raises(salt.exceptions.SaltSystemExit): ++ thin._get_ext_namespaces({'ns': {}}) ++ ++ @patch('salt.utils.thin.salt', type(str('salt'), (), {'__file__': '/site-packages/salt'})) ++ @patch('salt.utils.thin.jinja2', type(str('jinja2'), (), {'__file__': '/site-packages/jinja2'})) ++ @patch('salt.utils.thin.yaml', type(str('yaml'), (), {'__file__': '/site-packages/yaml'})) ++ @patch('salt.utils.thin.tornado', type(str('tornado'), (), {'__file__': '/site-packages/tornado'})) ++ @patch('salt.utils.thin.msgpack', type(str('msgpack'), (), {'__file__': '/site-packages/msgpack'})) ++ @patch('salt.utils.thin.certifi', type(str('certifi'), (), {'__file__': '/site-packages/certifi'})) ++ @patch('salt.utils.thin.singledispatch', type(str('singledispatch'), (), {'__file__': '/site-packages/sdp'})) ++ @patch('salt.utils.thin.singledispatch_helpers', type(str('singledispatch_helpers'), (), {'__file__': '/site-packages/sdp_hlp'})) ++ @patch('salt.utils.thin.ssl_match_hostname', type(str('ssl_match_hostname'), (), {'__file__': '/site-packages/ssl_mh'})) ++ @patch('salt.utils.thin.markupsafe', type(str('markupsafe'), (), {'__file__': '/site-packages/markupsafe'})) ++ @patch('salt.utils.thin.backports_abc', type(str('backports_abc'), (), {'__file__': '/site-packages/backports_abc'})) ++ @patch('salt.utils.thin.log', MagicMock()) ++ def test_get_tops(self): ++ ''' ++ Test thin.get_tops to get top directories, based on the interpreter. ++ :return: ++ ''' ++ base_tops = ['/site-packages/salt', '/site-packages/jinja2', '/site-packages/yaml', ++ '/site-packages/tornado', '/site-packages/msgpack', '/site-packages/certifi', ++ '/site-packages/sdp', '/site-packages/sdp_hlp', '/site-packages/ssl_mh', ++ '/site-packages/markupsafe', '/site-packages/backports_abc'] ++ ++ tops = thin.get_tops() ++ assert len(tops) == len(base_tops) ++ assert sorted(tops) == sorted(base_tops) ++ ++ @patch('salt.utils.thin.salt', type(str('salt'), (), {'__file__': '/site-packages/salt'})) ++ @patch('salt.utils.thin.jinja2', type(str('jinja2'), (), {'__file__': '/site-packages/jinja2'})) ++ @patch('salt.utils.thin.yaml', type(str('yaml'), (), {'__file__': '/site-packages/yaml'})) ++ @patch('salt.utils.thin.tornado', type(str('tornado'), (), {'__file__': '/site-packages/tornado'})) ++ @patch('salt.utils.thin.msgpack', type(str('msgpack'), (), {'__file__': '/site-packages/msgpack'})) ++ @patch('salt.utils.thin.certifi', type(str('certifi'), (), {'__file__': '/site-packages/certifi'})) ++ @patch('salt.utils.thin.singledispatch', type(str('singledispatch'), (), {'__file__': '/site-packages/sdp'})) ++ @patch('salt.utils.thin.singledispatch_helpers', type(str('singledispatch_helpers'), (), {'__file__': '/site-packages/sdp_hlp'})) ++ @patch('salt.utils.thin.ssl_match_hostname', type(str('ssl_match_hostname'), (), {'__file__': '/site-packages/ssl_mh'})) ++ @patch('salt.utils.thin.markupsafe', type(str('markupsafe'), (), {'__file__': '/site-packages/markupsafe'})) ++ @patch('salt.utils.thin.backports_abc', type(str('backports_abc'), (), {'__file__': '/site-packages/backports_abc'})) ++ @patch('salt.utils.thin.log', MagicMock()) ++ def test_get_tops_extra_mods(self): ++ ''' ++ Test thin.get_tops to get extra-modules alongside the top directories, based on the interpreter. ++ :return: ++ ''' ++ base_tops = ['/site-packages/salt', '/site-packages/jinja2', '/site-packages/yaml', ++ '/site-packages/tornado', '/site-packages/msgpack', '/site-packages/certifi', ++ '/site-packages/sdp', '/site-packages/sdp_hlp', '/site-packages/ssl_mh', ++ '/site-packages/markupsafe', '/site-packages/backports_abc', '/custom/foo', '/custom/bar.py'] ++ builtins = sys.version_info.major == 3 and 'builtins' or '__builtin__' ++ with patch('{}.__import__'.format(builtins), ++ MagicMock(side_effect=[type(str('foo'), (), {'__file__': '/custom/foo/__init__.py'}), ++ type(str('bar'), (), {'__file__': '/custom/bar'})])): ++ tops = thin.get_tops(extra_mods='foo,bar') ++ assert len(tops) == len(base_tops) ++ assert sorted(tops) == sorted(base_tops) ++ ++ @patch('salt.utils.thin.salt', type(str('salt'), (), {'__file__': '/site-packages/salt'})) ++ @patch('salt.utils.thin.jinja2', type(str('jinja2'), (), {'__file__': '/site-packages/jinja2'})) ++ @patch('salt.utils.thin.yaml', type(str('yaml'), (), {'__file__': '/site-packages/yaml'})) ++ @patch('salt.utils.thin.tornado', type(str('tornado'), (), {'__file__': '/site-packages/tornado'})) ++ @patch('salt.utils.thin.msgpack', type(str('msgpack'), (), {'__file__': '/site-packages/msgpack'})) ++ @patch('salt.utils.thin.certifi', type(str('certifi'), (), {'__file__': '/site-packages/certifi'})) ++ @patch('salt.utils.thin.singledispatch', type(str('singledispatch'), (), {'__file__': '/site-packages/sdp'})) ++ @patch('salt.utils.thin.singledispatch_helpers', type(str('singledispatch_helpers'), (), {'__file__': '/site-packages/sdp_hlp'})) ++ @patch('salt.utils.thin.ssl_match_hostname', type(str('ssl_match_hostname'), (), {'__file__': '/site-packages/ssl_mh'})) ++ @patch('salt.utils.thin.markupsafe', type(str('markupsafe'), (), {'__file__': '/site-packages/markupsafe'})) ++ @patch('salt.utils.thin.backports_abc', type(str('backports_abc'), (), {'__file__': '/site-packages/backports_abc'})) ++ @patch('salt.utils.thin.log', MagicMock()) ++ def test_get_tops_so_mods(self): ++ ''' ++ Test thin.get_tops to get extra-modules alongside the top directories, based on the interpreter. ++ :return: ++ ''' ++ base_tops = ['/site-packages/salt', '/site-packages/jinja2', '/site-packages/yaml', ++ '/site-packages/tornado', '/site-packages/msgpack', '/site-packages/certifi', ++ '/site-packages/sdp', '/site-packages/sdp_hlp', '/site-packages/ssl_mh', ++ '/site-packages/markupsafe', '/site-packages/backports_abc', '/custom/foo.so', '/custom/bar.so'] ++ builtins = sys.version_info.major == 3 and 'builtins' or '__builtin__' ++ with patch('{}.__import__'.format(builtins), ++ MagicMock(side_effect=[type(str('salt'), (), {'__file__': '/custom/foo.so'}), ++ type(str('salt'), (), {'__file__': '/custom/bar.so'})])): ++ tops = thin.get_tops(so_mods='foo,bar') ++ assert len(tops) == len(base_tops) ++ assert sorted(tops) == sorted(base_tops) ++ ++ @patch('salt.utils.thin.gen_thin', MagicMock(return_value='/path/to/thin/thin.tgz')) ++ @patch('salt.utils.hashutils.get_hash', MagicMock(return_value=12345)) ++ def test_thin_sum(self): ++ ''' ++ Test thin.thin_sum function. ++ ++ :return: ++ ''' ++ assert thin.thin_sum('/cachedir', form='sha256') == 12345 ++ thin.salt.utils.hashutils.get_hash.assert_called() ++ assert thin.salt.utils.hashutils.get_hash.call_count == 1 ++ ++ path, form = thin.salt.utils.hashutils.get_hash.call_args[0] ++ assert path == '/path/to/thin/thin.tgz' ++ assert form == 'sha256' ++ ++ @patch('salt.utils.thin.gen_min', MagicMock(return_value='/path/to/thin/min.tgz')) ++ @patch('salt.utils.hashutils.get_hash', MagicMock(return_value=12345)) ++ def test_min_sum(self): ++ ''' ++ Test thin.thin_sum function. ++ ++ :return: ++ ''' ++ assert thin.min_sum('/cachedir', form='sha256') == 12345 ++ thin.salt.utils.hashutils.get_hash.assert_called() ++ assert thin.salt.utils.hashutils.get_hash.call_count == 1 ++ ++ path, form = thin.salt.utils.hashutils.get_hash.call_args[0] ++ assert path == '/path/to/thin/min.tgz' ++ assert form == 'sha256' ++ ++ @patch('salt.utils.thin.sys.version_info', (2, 5)) ++ @patch('salt.exceptions.SaltSystemExit', Exception) ++ def test_gen_thin_fails_ancient_python_version(self): ++ ''' ++ Test thin.gen_thin function raises an exception ++ if Python major/minor version is lower than 2.6 ++ ++ :return: ++ ''' ++ with pytest.raises(salt.exceptions.SaltSystemExit) as err: ++ thin.sys.exc_clear = lambda: None ++ thin.gen_thin('') ++ assert 'The minimum required python version to run salt-ssh is "2.6"' in str(err) ++ ++ @patch('salt.exceptions.SaltSystemExit', Exception) ++ @patch('salt.utils.thin.log', MagicMock()) ++ @patch('salt.utils.thin.os.makedirs', MagicMock()) ++ @patch('salt.utils.files.fopen', MagicMock()) ++ @patch('salt.utils.thin._get_salt_call', MagicMock()) ++ @patch('salt.utils.thin._get_ext_namespaces', MagicMock()) ++ @patch('salt.utils.thin.get_tops', MagicMock(return_value=['/foo3', '/bar3'])) ++ @patch('salt.utils.thin.get_ext_tops', MagicMock(return_value={})) ++ @patch('salt.utils.thin.os.path.isfile', MagicMock()) ++ @patch('salt.utils.thin.os.path.isdir', MagicMock(return_value=True)) ++ @patch('salt.utils.thin.log', MagicMock()) ++ @patch('salt.utils.thin.os.remove', MagicMock()) ++ @patch('salt.utils.thin.os.path.exists', MagicMock()) ++ @patch('salt.utils.path.os_walk', MagicMock(return_value=[])) ++ @patch('salt.utils.thin.subprocess.Popen', ++ _popen(None, side_effect=[(bts('2.7'), bts('')), (bts('["/foo27", "/bar27"]'), bts(''))])) ++ @patch('salt.utils.thin.tarfile', MagicMock()) ++ @patch('salt.utils.thin.zipfile', MagicMock()) ++ @patch('salt.utils.thin.os.getcwd', MagicMock()) ++ @patch('salt.utils.thin.os.chdir', MagicMock()) ++ @patch('salt.utils.thin.tempfile', MagicMock(mkdtemp=MagicMock(return_value=''))) ++ @patch('salt.utils.thin.shutil', MagicMock()) ++ @patch('salt.utils.thin._six.PY3', True) ++ @patch('salt.utils.thin._six.PY2', False) ++ @patch('salt.utils.thin.sys.version_info', _version_info(None, 3, 6)) ++ def test_gen_thin_compression_fallback_py3(self): ++ ''' ++ Test thin.gen_thin function if fallbacks to the gzip compression, once setup wrong. ++ NOTE: Py2 version of this test is not required, as code shares the same spot across the versions. ++ ++ :return: ++ ''' ++ thin.gen_thin('', compress='arj') ++ thin.log.warning.assert_called() ++ pt, msg = thin.log.warning.mock_calls[0][1] ++ assert pt % msg == 'Unknown compression type: "arj". Falling back to "gzip" compression.' ++ thin.zipfile.ZipFile.assert_not_called() ++ thin.tarfile.open.assert_called() ++ ++ @patch('salt.exceptions.SaltSystemExit', Exception) ++ @patch('salt.utils.thin.log', MagicMock()) ++ @patch('salt.utils.thin.os.makedirs', MagicMock()) ++ @patch('salt.utils.files.fopen', MagicMock()) ++ @patch('salt.utils.thin._get_salt_call', MagicMock()) ++ @patch('salt.utils.thin._get_ext_namespaces', MagicMock()) ++ @patch('salt.utils.thin.get_tops', MagicMock(return_value=['/foo3', '/bar3'])) ++ @patch('salt.utils.thin.get_ext_tops', MagicMock(return_value={})) ++ @patch('salt.utils.thin.os.path.isfile', MagicMock()) ++ @patch('salt.utils.thin.os.path.isdir', MagicMock(return_value=False)) ++ @patch('salt.utils.thin.log', MagicMock()) ++ @patch('salt.utils.thin.os.remove', MagicMock()) ++ @patch('salt.utils.thin.os.path.exists', MagicMock()) ++ @patch('salt.utils.path.os_walk', MagicMock(return_value=[])) ++ @patch('salt.utils.thin.subprocess.Popen', ++ _popen(None, side_effect=[(bts('2.7'), bts('')), (bts('["/foo27", "/bar27"]'), bts(''))])) ++ @patch('salt.utils.thin.tarfile', MagicMock()) ++ @patch('salt.utils.thin.zipfile', MagicMock()) ++ @patch('salt.utils.thin.os.getcwd', MagicMock()) ++ @patch('salt.utils.thin.os.chdir', MagicMock()) ++ @patch('salt.utils.thin.tempfile', MagicMock(mkdtemp=MagicMock(return_value=''))) ++ @patch('salt.utils.thin.shutil', MagicMock()) ++ @patch('salt.utils.thin._six.PY3', True) ++ @patch('salt.utils.thin._six.PY2', False) ++ @patch('salt.utils.thin.sys.version_info', _version_info(None, 3, 6)) ++ def test_gen_thin_control_files_written_py3(self): ++ ''' ++ Test thin.gen_thin function if control files are written (version, salt-call etc). ++ NOTE: Py2 version of this test is not required, as code shares the same spot across the versions. ++ ++ :return: ++ ''' ++ thin.gen_thin('') ++ arc_name, arc_mode = thin.tarfile.method_calls[0][1] ++ assert arc_name == 'thin/thin.tgz' ++ assert arc_mode == 'w:gz' ++ for idx, fname in enumerate(['version', '.thin-gen-py-version', 'salt-call', 'supported-versions']): ++ assert thin.tarfile.open().method_calls[idx + 4][1][0] == fname ++ thin.tarfile.open().close.assert_called() ++ ++ @patch('salt.exceptions.SaltSystemExit', Exception) ++ @patch('salt.utils.thin.log', MagicMock()) ++ @patch('salt.utils.thin.os.makedirs', MagicMock()) ++ @patch('salt.utils.files.fopen', MagicMock()) ++ @patch('salt.utils.thin._get_salt_call', MagicMock()) ++ @patch('salt.utils.thin._get_ext_namespaces', MagicMock()) ++ @patch('salt.utils.thin.get_tops', MagicMock(return_value=['/salt', '/bar3'])) ++ @patch('salt.utils.thin.get_ext_tops', MagicMock(return_value={})) ++ @patch('salt.utils.thin.os.path.isfile', MagicMock()) ++ @patch('salt.utils.thin.os.path.isdir', MagicMock(return_value=True)) ++ @patch('salt.utils.thin.log', MagicMock()) ++ @patch('salt.utils.thin.os.remove', MagicMock()) ++ @patch('salt.utils.thin.os.path.exists', MagicMock()) ++ @patch('salt.utils.path.os_walk', ++ MagicMock(return_value=(('root', [], ['r1', 'r2', 'r3']), ('root2', [], ['r4', 'r5', 'r6'])))) ++ @patch('salt.utils.thin.subprocess.Popen', ++ _popen(None, side_effect=[(bts('2.7'), bts('')), (bts('["/foo27", "/bar27"]'), bts(''))])) ++ @patch('salt.utils.thin.tarfile', _tarfile(None)) ++ @patch('salt.utils.thin.zipfile', MagicMock()) ++ @patch('salt.utils.thin.os.getcwd', MagicMock()) ++ @patch('salt.utils.thin.os.chdir', MagicMock()) ++ @patch('salt.utils.thin.tempfile', MagicMock()) ++ @patch('salt.utils.thin.shutil', MagicMock()) ++ @patch('salt.utils.thin._six.PY3', True) ++ @patch('salt.utils.thin._six.PY2', False) ++ @patch('salt.utils.thin.sys.version_info', _version_info(None, 3, 6)) ++ def test_gen_thin_main_content_files_written_py3(self): ++ ''' ++ Test thin.gen_thin function if main content files are written. ++ NOTE: Py2 version of this test is not required, as code shares the same spot across the versions. ++ ++ :return: ++ ''' ++ thin.gen_thin('') ++ files = [ ++ 'py2/root/r1', 'py2/root/r2', 'py2/root/r3', 'py2/root2/r4', 'py2/root2/r5', 'py2/root2/r6', ++ 'py2/root/r1', 'py2/root/r2', 'py2/root/r3', 'py2/root2/r4', 'py2/root2/r5', 'py2/root2/r6', ++ 'py3/root/r1', 'py3/root/r2', 'py3/root/r3', 'py3/root2/r4', 'py3/root2/r5', 'py3/root2/r6', ++ 'pyall/root/r1', 'pyall/root/r2', 'pyall/root/r3', 'pyall/root2/r4', 'pyall/root2/r5', 'pyall/root2/r6' ++ ] ++ for cl in thin.tarfile.open().method_calls[:-5]: ++ arcname = cl[2].get('arcname') ++ assert arcname in files ++ files.pop(files.index(arcname)) ++ assert not bool(files) ++ ++ @patch('salt.exceptions.SaltSystemExit', Exception) ++ @patch('salt.utils.thin.log', MagicMock()) ++ @patch('salt.utils.thin.os.makedirs', MagicMock()) ++ @patch('salt.utils.files.fopen', MagicMock()) ++ @patch('salt.utils.thin._get_salt_call', MagicMock()) ++ @patch('salt.utils.thin._get_ext_namespaces', MagicMock()) ++ @patch('salt.utils.thin.get_tops', MagicMock(return_value=[])) ++ @patch('salt.utils.thin.get_ext_tops', ++ MagicMock(return_value={'namespace': {'py-version': [2, 7], ++ 'path': '/opt/2015.8/salt', ++ 'dependencies': ['/opt/certifi', '/opt/whatever']}})) ++ @patch('salt.utils.thin.os.path.isfile', MagicMock()) ++ @patch('salt.utils.thin.os.path.isdir', MagicMock(return_value=True)) ++ @patch('salt.utils.thin.log', MagicMock()) ++ @patch('salt.utils.thin.os.remove', MagicMock()) ++ @patch('salt.utils.thin.os.path.exists', MagicMock()) ++ @patch('salt.utils.path.os_walk', ++ MagicMock(return_value=(('root', [], ['r1', 'r2', 'r3']), ('root2', [], ['r4', 'r5', 'r6'])))) ++ @patch('salt.utils.thin.subprocess.Popen', ++ _popen(None, side_effect=[(bts('2.7'), bts('')), (bts('["/foo27", "/bar27"]'), bts(''))])) ++ @patch('salt.utils.thin.tarfile', _tarfile(None)) ++ @patch('salt.utils.thin.zipfile', MagicMock()) ++ @patch('salt.utils.thin.os.getcwd', MagicMock()) ++ @patch('salt.utils.thin.os.chdir', MagicMock()) ++ @patch('salt.utils.thin.tempfile', MagicMock(mkdtemp=MagicMock(return_value=''))) ++ @patch('salt.utils.thin.shutil', MagicMock()) ++ @patch('salt.utils.thin._six.PY3', True) ++ @patch('salt.utils.thin._six.PY2', False) ++ @patch('salt.utils.thin.sys.version_info', _version_info(None, 3, 6)) ++ def test_gen_thin_ext_alternative_content_files_written_py3(self): ++ ''' ++ Test thin.gen_thin function if external alternative content files are written. ++ NOTE: Py2 version of this test is not required, as code shares the same spot across the versions. ++ ++ :return: ++ ''' ++ thin.gen_thin('') ++ files = ['namespace/pyall/root/r1', 'namespace/pyall/root/r2', 'namespace/pyall/root/r3', ++ 'namespace/pyall/root2/r4', 'namespace/pyall/root2/r5', 'namespace/pyall/root2/r6', ++ 'namespace/pyall/root/r1', 'namespace/pyall/root/r2', 'namespace/pyall/root/r3', ++ 'namespace/pyall/root2/r4', 'namespace/pyall/root2/r5', 'namespace/pyall/root2/r6', ++ 'namespace/py2/root/r1', 'namespace/py2/root/r2', 'namespace/py2/root/r3', ++ 'namespace/py2/root2/r4', 'namespace/py2/root2/r5', 'namespace/py2/root2/r6' ++ ] ++ for idx, cl in enumerate(thin.tarfile.open().method_calls[12:-5]): ++ arcname = cl[2].get('arcname') ++ assert arcname in files ++ files.pop(files.index(arcname)) ++ assert not bool(files) ++ ++ def test_get_supported_py_config_typecheck(self): ++ ''' ++ Test collecting proper py-versions. Should return bytes type. ++ :return: ++ ''' ++ tops = {} ++ ext_cfg = {} ++ out = thin._get_supported_py_config(tops=tops, extended_cfg=ext_cfg) ++ assert type(salt.utils.stringutils.to_bytes('')) == type(out) ++ ++ def test_get_supported_py_config_base_tops(self): ++ ''' ++ Test collecting proper py-versions. Should return proper base tops. ++ :return: ++ ''' ++ tops = {'3': ['/groundkeepers', '/stole'], '2': ['/the-root', '/password']} ++ ext_cfg = {} ++ out = salt.utils.stringutils.to_str(thin._get_supported_py_config( ++ tops=tops, extended_cfg=ext_cfg)).strip().split('\n') ++ assert len(out) == 2 ++ for t_line in ['py3:3:0', 'py2:2:7']: ++ assert t_line in out ++ ++ def test_get_supported_py_config_ext_tops(self): ++ ''' ++ Test collecting proper py-versions. Should return proper ext conf tops. ++ :return: ++ ''' ++ tops = {} ++ ext_cfg = {'solar-interference': {'py-version': [2, 6]}, 'second-system-effect': {'py-version': [2, 7]}} ++ out = salt.utils.stringutils.to_str(thin._get_supported_py_config( ++ tops=tops, extended_cfg=ext_cfg)).strip().split('\n') ++ for t_line in ['second-system-effect:2:7', 'solar-interference:2:6']: ++ assert t_line in out +-- +2.16.2 + + diff --git a/avoid-excessive-syslogging-by-watchdog-cronjob-58.patch b/avoid-excessive-syslogging-by-watchdog-cronjob-58.patch index 78c4917..8f33ff3 100644 --- a/avoid-excessive-syslogging-by-watchdog-cronjob-58.patch +++ b/avoid-excessive-syslogging-by-watchdog-cronjob-58.patch @@ -1,4 +1,4 @@ -From 5dfc5ebd799d5c9a4588d9e1a27b813b8c35ce25 Mon Sep 17 00:00:00 2001 +From edb1c95fa06b8bb1d7e6d91beaaddec6d22c966b Mon Sep 17 00:00:00 2001 From: Hubert Mantel Date: Mon, 27 Nov 2017 13:55:13 +0100 Subject: [PATCH] avoid excessive syslogging by watchdog cronjob (#58) diff --git a/explore-module.run-response-to-catch-the-result-in-d.patch b/explore-module.run-response-to-catch-the-result-in-d.patch new file mode 100644 index 0000000..0b23ccd --- /dev/null +++ b/explore-module.run-response-to-catch-the-result-in-d.patch @@ -0,0 +1,131 @@ +From 8c6b77bfd913b3b47d3d4206ec0a9e08754b6f93 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Wed, 7 Mar 2018 09:42:46 +0000 +Subject: [PATCH] Explore 'module.run' response to catch the 'result' in + depth + +Fix Python3 and pylint issue + +Rename and fix recursive method + +Add new unit test to check state.apply within module.run +--- + salt/states/module.py | 18 ++++++++++++ + tests/unit/states/test_module.py | 62 ++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 80 insertions(+) + +diff --git a/salt/states/module.py b/salt/states/module.py +index fda8bdf17a..2190ffa3d2 100644 +--- a/salt/states/module.py ++++ b/salt/states/module.py +@@ -531,7 +531,25 @@ def _get_result(func_ret, changes): + res = changes_ret.get('result', {}) + elif changes_ret.get('retcode', 0) != 0: + res = False ++ # Explore dict in depth to determine if there is a ++ # 'result' key set to False which sets the global ++ # state result. ++ else: ++ res = _get_dict_result(changes_ret) + + return res + ++ ++def _get_dict_result(node): ++ ret = True ++ for key, val in six.iteritems(node): ++ if key == 'result' and val is False: ++ ret = False ++ break ++ elif isinstance(val, dict): ++ ret = _get_dict_result(val) ++ if ret is False: ++ break ++ return ret ++ + mod_watch = salt.utils.functools.alias_function(run, 'mod_watch') +diff --git a/tests/unit/states/test_module.py b/tests/unit/states/test_module.py +index 12ad54f979..bf4ddcc5b4 100644 +--- a/tests/unit/states/test_module.py ++++ b/tests/unit/states/test_module.py +@@ -25,6 +25,57 @@ log = logging.getLogger(__name__) + + CMD = 'foo.bar' + ++STATE_APPLY_RET = { ++ 'module_|-test2_|-state.apply_|-run': { ++ 'comment': 'Module function state.apply executed', ++ 'name': 'state.apply', ++ 'start_time': '16:11:48.818932', ++ 'result': False, ++ 'duration': 179.439, ++ '__run_num__': 0, ++ 'changes': { ++ 'ret': { ++ 'module_|-test3_|-state.apply_|-run': { ++ 'comment': 'Module function state.apply executed', ++ 'name': 'state.apply', ++ 'start_time': '16:11:48.904796', ++ 'result': True, ++ 'duration': 89.522, ++ '__run_num__': 0, ++ 'changes': { ++ 'ret': { ++ 'module_|-test4_|-cmd.run_|-run': { ++ 'comment': 'Module function cmd.run executed', ++ 'name': 'cmd.run', ++ 'start_time': '16:11:48.988574', ++ 'result': True, ++ 'duration': 4.543, ++ '__run_num__': 0, ++ 'changes': { ++ 'ret': 'Wed Mar 7 16:11:48 CET 2018' ++ }, ++ '__id__': 'test4' ++ } ++ } ++ }, ++ '__id__': 'test3' ++ }, ++ 'module_|-test3_fail_|-test3_fail_|-run': { ++ 'comment': 'Module function test3_fail is not available', ++ 'name': 'test3_fail', ++ 'start_time': '16:11:48.994607', ++ 'result': False, ++ 'duration': 0.466, ++ '__run_num__': 1, ++ 'changes': {}, ++ '__id__': 'test3_fail' ++ } ++ } ++ }, ++ '__id__': 'test2' ++ } ++} ++ + + def _mocked_func_named(name, names=('Fred', 'Swen',)): + ''' +@@ -140,6 +191,17 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): + if ret['comment'] != '{0}: Success'.format(CMD) or not ret['result']: + self.fail('module.run failed: {0}'.format(ret)) + ++ def test_run_state_apply_result_false(self): ++ ''' ++ Tests the 'result' of module.run that calls state.apply execution module ++ :return: ++ ''' ++ with patch.dict(module.__salt__, {"state.apply": MagicMock(return_value=STATE_APPLY_RET)}): ++ with patch.dict(module.__opts__, {'use_deprecated': ['module.run']}): ++ ret = module.run(**{"name": "state.apply", 'mods': 'test2'}) ++ if ret['result']: ++ self.fail('module.run did not report false result: {0}'.format(ret)) ++ + def test_run_unexpected_keywords(self): + with patch.dict(module.__salt__, {CMD: _mocked_func_args}): + with patch.dict(module.__opts__, {'use_superseded': ['module.run']}): +-- +2.16.2 + + diff --git a/feat-add-grain-for-all-fqdns.patch b/feat-add-grain-for-all-fqdns.patch index 7bdf9da..b9315b0 100644 --- a/feat-add-grain-for-all-fqdns.patch +++ b/feat-add-grain-for-all-fqdns.patch @@ -1,4 +1,4 @@ -From 067b63ccbc6942a0399eb43d7fdf314b5c9b417f Mon Sep 17 00:00:00 2001 +From 0449bead92ff763d186f5e524556f82c618d652c Mon Sep 17 00:00:00 2001 From: Michele Bologna Date: Thu, 14 Dec 2017 18:20:02 +0100 Subject: [PATCH] Feat: add grain for all FQDNs @@ -21,10 +21,10 @@ https://github.com/saltstack/salt/pull/45060 3 files changed, 29 insertions(+) diff --git a/salt/grains/core.py b/salt/grains/core.py -index 9352987abd..f8e36a895e 100644 +index b7d446676e..96b7ce2cf2 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py -@@ -1890,6 +1890,33 @@ def append_domain(): +@@ -1888,6 +1888,33 @@ def append_domain(): return grain @@ -71,7 +71,7 @@ index 709f882b45..aa7bd44202 100644 'groupname', 'host', diff --git a/tests/unit/grains/test_core.py b/tests/unit/grains/test_core.py -index e781fadefe..dba8d082c5 100644 +index 50babe3ed3..47c9cdd35b 100644 --- a/tests/unit/grains/test_core.py +++ b/tests/unit/grains/test_core.py @@ -7,6 +7,7 @@ diff --git a/fix-bsc-1065792.patch b/fix-bsc-1065792.patch index a064415..b4184ee 100644 --- a/fix-bsc-1065792.patch +++ b/fix-bsc-1065792.patch @@ -1,4 +1,4 @@ -From 1d2030c6d002ef293c7a71c77db1617fdb5a8cb0 Mon Sep 17 00:00:00 2001 +From 27d0e8b7e7c1eae68ef6dc972ea0f091d18cd92e Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Thu, 14 Dec 2017 16:21:40 +0100 Subject: [PATCH] Fix bsc#1065792 diff --git a/fix-cp.push-empty-file.patch b/fix-cp.push-empty-file.patch index d9aeeca..ea3a799 100644 --- a/fix-cp.push-empty-file.patch +++ b/fix-cp.push-empty-file.patch @@ -1,4 +1,4 @@ -From e91e2db15b99ef677cb112bb817275ed60a1f0e1 Mon Sep 17 00:00:00 2001 +From 74ca7c3fd6a42f95f9d702ef2847a1f76399db5f Mon Sep 17 00:00:00 2001 From: Mihai Dinca Date: Wed, 7 Mar 2018 13:11:16 +0100 Subject: [PATCH] Fix cp.push empty file diff --git a/fix-decrease-loglevel-when-unable-to-resolve-addr.patch b/fix-decrease-loglevel-when-unable-to-resolve-addr.patch new file mode 100644 index 0000000..717b109 --- /dev/null +++ b/fix-decrease-loglevel-when-unable-to-resolve-addr.patch @@ -0,0 +1,72 @@ +From 14128fc65bf007bbb5b27b3eedec30b7f729bfbd Mon Sep 17 00:00:00 2001 +From: Michele Bologna +Date: Tue, 20 Mar 2018 19:27:36 +0100 +Subject: [PATCH] Fix: decrease loglevel when unable to resolve addr + +Upstream PR: https://github.com/saltstack/salt/pull/46575 + +It occurs that when the machine has multiple interfaces without an associated FQDN, Salt logs are polluted by this error. +Some examples: + +``` +caasp-admin:~ # uptime + 09:08am up 0:13, 2 users, load average: 1.30, 1.37, 0.98 +caasp-admin:~ # docker logs $(docker ps | grep salt-master | awk '{print $1}') 2>&1 | grep "Exception during resolving address" | wc -l +528 +``` + +``` +caasp-admin:~ # docker exec -it $(docker ps | grep salt-master | awk '{print $1}') salt '*' cmd.run uptime +b24f41eb4cc94624862ca0c9e8afcd15: + 09:08am up 0:11, 0 users, load average: 1.26, 0.83, 0.40 +admin: + 09:08am up 0:13, 2 users, load average: 1.33, 1.37, 0.99 +ba8c76af029043a39ba917f7ab2af796: + 09:08am up 0:12, 0 users, load average: 0.84, 0.63, 0.32 +7b7aa52158524556a0c46ae57569ce93: + 09:08am up 0:11, 1 user, load average: 1.05, 0.77, 0.38 +5ab0e18cbd084e9088a928a17edb86cb: + 09:08am up 0:10, 0 users, load average: 0.12, 0.25, 0.20 +1756c9cd9a9a402b91d8636400d1e512: + 09:08am up 0:09, 0 users, load average: 0.12, 0.23, 0.14 +ca: + 09:08am up 0:13, 0 users, load average: 1.33, 1.37, 0.99 +caasp-admin:~ # docker exec -it $(docker ps | grep salt-master | awk '{print $1}') salt '*' cmd.run "bash -c 'cat /var/log/salt/minion | grep \"Exception during resolving address\" | wc -l'" +admin: + 63 +ba8c76af029043a39ba917f7ab2af796: + 47 +5ab0e18cbd084e9088a928a17edb86cb: + 55 +7b7aa52158524556a0c46ae57569ce93: + 59 +b24f41eb4cc94624862ca0c9e8afcd15: + 47 +1756c9cd9a9a402b91d8636400d1e512: + 59 +ca: + 25 +``` + +This patch changes the log level of the exception to INFO, since the resolve-unable problem is not blocking. +--- + salt/grains/core.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/salt/grains/core.py b/salt/grains/core.py +index 96b7ce2cf2..17a7d9819a 100644 +--- a/salt/grains/core.py ++++ b/salt/grains/core.py +@@ -1909,7 +1909,7 @@ def fqdns(): + fqdns.add(socket.gethostbyaddr(ip)[0]) + except (socket.error, socket.herror, + socket.gaierror, socket.timeout) as e: +- log.error("Exception during resolving address: " + str(e)) ++ log.info("Exception during resolving address: " + str(e)) + + grains['fqdns'] = list(fqdns) + return grains +-- +2.16.2 + + diff --git a/fix-openscap-push.patch b/fix-openscap-push.patch index 45c1330..69b69cb 100644 --- a/fix-openscap-push.patch +++ b/fix-openscap-push.patch @@ -1,11 +1,12 @@ -From 7259333c3652d5208258632532a9151648c9cb4d Mon Sep 17 00:00:00 2001 +From 589d90117783a126dce695cf76a3b8fc2953f8b6 Mon Sep 17 00:00:00 2001 From: Mihai Dinca Date: Fri, 2 Mar 2018 17:17:58 +0100 Subject: [PATCH] Fix openscap push --- - salt/modules/openscap.py | 4 +--- - 1 file changed, 1 insertion(+), 3 deletions(-) + salt/modules/openscap.py | 4 +--- + tests/unit/modules/test_openscap.py | 10 +++++----- + 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/salt/modules/openscap.py b/salt/modules/openscap.py index c5b51a1846..e3190e1e11 100644 @@ -29,6 +30,42 @@ index c5b51a1846..e3190e1e11 100644 shutil.rmtree(tempdir, ignore_errors=True) upload_dir = tempdir +diff --git a/tests/unit/modules/test_openscap.py b/tests/unit/modules/test_openscap.py +index eb8ad1225b..6e17148de1 100644 +--- a/tests/unit/modules/test_openscap.py ++++ b/tests/unit/modules/test_openscap.py +@@ -28,8 +28,10 @@ class OpenscapTestCase(TestCase): + policy_file = '/usr/share/openscap/policy-file-xccdf.xml' + + def setUp(self): ++ import salt.modules.openscap ++ salt.modules.openscap.__salt__ = MagicMock() + patchers = [ +- patch('salt.modules.openscap.Caller', MagicMock()), ++ patch('salt.modules.openscap.__salt__', MagicMock()), + patch('salt.modules.openscap.shutil.rmtree', Mock()), + patch( + 'salt.modules.openscap.tempfile.mkdtemp', +@@ -68,8 +70,7 @@ class OpenscapTestCase(TestCase): + cwd=openscap.tempfile.mkdtemp.return_value, + stderr=PIPE, + stdout=PIPE) +- openscap.Caller().cmd.assert_called_once_with( +- 'cp.push_dir', self.random_temp_dir) ++ openscap.__salt__['cp.push_dir'].assert_called_once_with(self.random_temp_dir) + self.assertEqual(openscap.shutil.rmtree.call_count, 1) + self.assertEqual( + response, +@@ -106,8 +107,7 @@ class OpenscapTestCase(TestCase): + cwd=openscap.tempfile.mkdtemp.return_value, + stderr=PIPE, + stdout=PIPE) +- openscap.Caller().cmd.assert_called_once_with( +- 'cp.push_dir', self.random_temp_dir) ++ openscap.__salt__['cp.push_dir'].assert_called_once_with(self.random_temp_dir) + self.assertEqual(openscap.shutil.rmtree.call_count, 1) + self.assertEqual( + response, -- 2.16.2 diff --git a/make-it-possible-to-use-login-pull-and-push-from-mod.patch b/make-it-possible-to-use-login-pull-and-push-from-mod.patch new file mode 100644 index 0000000..8d5d401 --- /dev/null +++ b/make-it-possible-to-use-login-pull-and-push-from-mod.patch @@ -0,0 +1,119 @@ +From d0b7808f63a32c15249a8adbed048859dfac21a8 Mon Sep 17 00:00:00 2001 +From: Michael Calmer +Date: Thu, 22 Mar 2018 08:56:58 +0100 +Subject: [PATCH] make it possible to use login, pull and push from + module.run and detect errors + +when using state.apply module.run doing docker operations retcode +is tracked to find out if the call was successful or not. + +add unit test for failed login +--- + salt/modules/dockermod.py | 14 ++++++++++---- + tests/unit/modules/test_dockermod.py | 20 ++++++++++++++++++++ + 2 files changed, 30 insertions(+), 4 deletions(-) + +diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py +index 23cff8806b..c20b73452b 100644 +--- a/salt/modules/dockermod.py ++++ b/salt/modules/dockermod.py +@@ -1354,7 +1354,7 @@ def login(*registries): + # information is added to the config.json, since docker-py isn't designed + # to do so. + registry_auth = __pillar__.get('docker-registries', {}) +- ret = {} ++ ret = {'retcode': 0} + errors = ret.setdefault('Errors', []) + if not isinstance(registry_auth, dict): + errors.append('\'docker-registries\' Pillar value must be a dictionary') +@@ -1412,6 +1412,8 @@ def login(*registries): + errors.append(login_cmd['stderr']) + elif login_cmd['stdout']: + errors.append(login_cmd['stdout']) ++ if errors: ++ ret['retcode'] = 1 + return ret + + +@@ -4490,7 +4492,7 @@ def pull(image, + + time_started = time.time() + response = _client_wrapper('pull', image, **kwargs) +- ret = {'Time_Elapsed': time.time() - time_started} ++ ret = {'Time_Elapsed': time.time() - time_started, 'retcode': 0} + _clear_context() + + if not response: +@@ -4523,6 +4525,7 @@ def pull(image, + + if errors: + ret['Errors'] = errors ++ ret['retcode'] = 1 + return ret + + +@@ -4585,7 +4588,7 @@ def push(image, + + time_started = time.time() + response = _client_wrapper('push', image, **kwargs) +- ret = {'Time_Elapsed': time.time() - time_started} ++ ret = {'Time_Elapsed': time.time() - time_started, 'retcode': 0} + _clear_context() + + if not response: +@@ -4617,6 +4620,7 @@ def push(image, + + if errors: + ret['Errors'] = errors ++ ret['retcode'] = 1 + return ret + + +@@ -4688,9 +4692,11 @@ def rmi(*names, **kwargs): + + _clear_context() + ret = {'Layers': [x for x in pre_images if x not in images(all=True)], +- 'Tags': [x for x in pre_tags if x not in list_tags()]} ++ 'Tags': [x for x in pre_tags if x not in list_tags()], ++ 'retcode': 0} + if errors: + ret['Errors'] = errors ++ ret['retcode'] = 1 + return ret + + +diff --git a/tests/unit/modules/test_dockermod.py b/tests/unit/modules/test_dockermod.py +index 4e061ce369..77c4bcfb85 100644 +--- a/tests/unit/modules/test_dockermod.py ++++ b/tests/unit/modules/test_dockermod.py +@@ -64,6 +64,26 @@ class DockerTestCase(TestCase, LoaderModuleMockMixin): + ''' + docker_mod.__context__.pop('docker.client', None) + ++ def test_failed_login(self): ++ ''' ++ Check that when docker.login failed a retcode other then 0 ++ is part of the return. ++ ''' ++ client = Mock() ++ get_client_mock = MagicMock(return_value=client) ++ ref_out = { ++ 'stdout': '', ++ 'stderr': 'login failed', ++ 'retcode': 1 ++ } ++ with patch.dict(docker_mod.__pillar__, {'docker-registries': {'portus.example.com:5000': ++ {'username': 'admin', 'password': 'linux12345', 'email': 'tux@example.com'}}}): ++ with patch.object(docker_mod, '_get_client', get_client_mock): ++ with patch.dict(docker_mod.__salt__, {'cmd.run_all': MagicMock(return_value=ref_out)}): ++ ret = docker_mod.login('portus.example.com:5000') ++ self.assertTrue('retcode' in ret) ++ self.assertTrue(ret['retcode'] > 0) ++ + def test_ps_with_host_true(self): + ''' + Check that docker.ps called with host is ``True``, +-- +2.16.2 + + diff --git a/move-log_file-option-to-changeable-defaults.patch b/move-log_file-option-to-changeable-defaults.patch index 2c2d252..849089c 100644 --- a/move-log_file-option-to-changeable-defaults.patch +++ b/move-log_file-option-to-changeable-defaults.patch @@ -1,4 +1,4 @@ -From 76706e2b241c932e4067487139325dc16ca08477 Mon Sep 17 00:00:00 2001 +From f77ae8d0426e551d6249b097850da0ed4ff7276d Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Sun, 11 Feb 2018 19:15:27 +0100 Subject: [PATCH] move log_file option to changeable defaults diff --git a/remove-obsolete-unicode-handling-in-pkg.info_install.patch b/remove-obsolete-unicode-handling-in-pkg.info_install.patch index f369137..47e4bee 100644 --- a/remove-obsolete-unicode-handling-in-pkg.info_install.patch +++ b/remove-obsolete-unicode-handling-in-pkg.info_install.patch @@ -1,26 +1,30 @@ -From a29071d597553184ea39f7c5783b0bd4f29fab2b Mon Sep 17 00:00:00 2001 +From dc262b912c63ed0d3152a01c9eaaa3ec3f8e0f7e Mon Sep 17 00:00:00 2001 From: Mihai Dinca Date: Tue, 13 Feb 2018 16:11:20 +0100 Subject: [PATCH] Remove obsolete unicode handling in pkg.info_installed --- - salt/modules/zypper.py | 11 +---------- - 1 file changed, 1 insertion(+), 10 deletions(-) + salt/modules/zypper.py | 15 +++++---------- + 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py -index 51d01c3fc9..659d8858f0 100644 +index 51d01c3fc9..16fc877684 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py -@@ -309,7 +309,7 @@ class _Zypper(object): +@@ -309,7 +309,11 @@ class _Zypper(object): if self.error_msg and not self.__no_raise and not self.__ignore_repo_failure: raise CommandExecutionError('Zypper command failure: {0}'.format(self.error_msg)) - return self._is_xml_mode() and dom.parseString(self.__call_result['stdout']) or self.__call_result['stdout'] -+ return self._is_xml_mode() and dom.parseString(self.__call_result['stdout'].encode('utf-8')) or self.__call_result['stdout'] ++ return ( ++ self._is_xml_mode() and ++ dom.parseString(salt.utils.stringutils.to_str(self.__call_result['stdout'])) or ++ self.__call_result['stdout'] ++ ) __zypper__ = _Zypper() -@@ -482,15 +482,6 @@ def info_installed(*names, **kwargs): +@@ -482,15 +486,6 @@ def info_installed(*names, **kwargs): t_nfo = dict() # Translate dpkg-specific keys to a common structure for key, value in six.iteritems(pkg_nfo): diff --git a/run-salt-api-as-user-salt-bsc-1064520.patch b/run-salt-api-as-user-salt-bsc-1064520.patch index e4c2364..9e68400 100644 --- a/run-salt-api-as-user-salt-bsc-1064520.patch +++ b/run-salt-api-as-user-salt-bsc-1064520.patch @@ -1,4 +1,4 @@ -From ff376d2811248384e3a41e69404d2c66affd5279 Mon Sep 17 00:00:00 2001 +From 92f41027bc08be3e14a47bbf7f43205a60606643 Mon Sep 17 00:00:00 2001 From: Christian Lanig Date: Mon, 27 Nov 2017 13:10:26 +0100 Subject: [PATCH] Run salt-api as user salt (bsc#1064520) diff --git a/run-salt-master-as-dedicated-salt-user.patch b/run-salt-master-as-dedicated-salt-user.patch index 7f3f583..8319161 100644 --- a/run-salt-master-as-dedicated-salt-user.patch +++ b/run-salt-master-as-dedicated-salt-user.patch @@ -1,4 +1,4 @@ -From 1df8d3011ebe645c25be8541cbc40ac2b700dfcf Mon Sep 17 00:00:00 2001 +From 04906c9a9c1b9fdbc6854a017e92525acd167bc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klaus=20K=C3=A4mpf?= Date: Wed, 20 Jan 2016 11:01:06 +0100 Subject: [PATCH] Run salt master as dedicated salt user diff --git a/salt.changes b/salt.changes index b0aea59..8928018 100644 --- a/salt.changes +++ b/salt.changes @@ -1,3 +1,132 @@ +------------------------------------------------------------------- +Fri Apr 6 16:58:59 UTC 2018 - Mihai Dinca + +- Update to 2018.3.0 + + +- Modified: + * explore-module.run-response-to-catch-the-result-in-d.patch + * add-saltssh-multi-version-support-across-python-inte.patch + * run-salt-api-as-user-salt-bsc-1064520.patch + * fix-openscap-push.patch + * fix-decrease-loglevel-when-unable-to-resolve-addr.patch + * fix-cp.push-empty-file.patch + * make-it-possible-to-use-login-pull-and-push-from-mod.patch + * avoid-excessive-syslogging-by-watchdog-cronjob-58.patch + * feat-add-grain-for-all-fqdns.patch + * fix-bsc-1065792.patch + * run-salt-master-as-dedicated-salt-user.patch + * move-log_file-option-to-changeable-defaults.patch + * activate-all-beacons-sources-config-pillar-grains.patch + * remove-obsolete-unicode-handling-in-pkg.info_install.patch + +------------------------------------------------------------------- +Thu Apr 5 15:58:22 UTC 2018 - Mihai Dinca + +- Add python-2.6 support to salt-ssh + +- Modified: + * add-saltssh-multi-version-support-across-python-inte.patch + +------------------------------------------------------------------- +Wed Apr 4 16:32:10 UTC 2018 - Mihai Dinca + +- Update salt-ssh multiversion patch + +- Modified: + * add-saltssh-multi-version-support-across-python-inte.patch + +- Removed: + * require-same-major-version-while-minor-is-allowed-to.patch + +------------------------------------------------------------------- +Wed Mar 28 12:18:08 UTC 2018 - Mihai Dinca + +- Add iprout/net-tools dependency + + +------------------------------------------------------------------- +Wed Mar 28 11:57:30 UTC 2018 - Michael Calmer + +- salt-ssh: require same major version while minor is allowed to be + +- Added: + * require-same-major-version-while-minor-is-allowed-to.patch + +- Modified: + * explore-module.run-response-to-catch-the-result-in-d.patch + * add-saltssh-multi-version-support-across-python-inte.patch + * run-salt-api-as-user-salt-bsc-1064520.patch + * fix-openscap-push.patch + * fix-decrease-loglevel-when-unable-to-resolve-addr.patch + * fix-cp.push-empty-file.patch + * make-it-possible-to-use-login-pull-and-push-from-mod.patch + * avoid-excessive-syslogging-by-watchdog-cronjob-58.patch + * feat-add-grain-for-all-fqdns.patch + * fix-bsc-1065792.patch + * run-salt-master-as-dedicated-salt-user.patch + * move-log_file-option-to-changeable-defaults.patch + * activate-all-beacons-sources-config-pillar-grains.patch + * remove-obsolete-unicode-handling-in-pkg.info_install.patch + +------------------------------------------------------------------- +Tue Mar 27 16:29:08 UTC 2018 - Mihai Dinca + +- Add SaltSSH multi-version support across Python interpeters. + +- Added: + * add-saltssh-multi-version-support-across-python-inte.patch + +------------------------------------------------------------------- +Fri Mar 23 18:12:09 UTC 2018 - Mihai Dinca + +- Fix zypper.info_installed 'ascii' issue + +- Modified: + * explore-module.run-response-to-catch-the-result-in-d.patch + * fix-openscap-push.patch + * fix-decrease-loglevel-when-unable-to-resolve-addr.patch + * fix-cp.push-empty-file.patch + * make-it-possible-to-use-login-pull-and-push-from-mod.patch + * move-log_file-option-to-changeable-defaults.patch + * remove-obsolete-unicode-handling-in-pkg.info_install.patch + +------------------------------------------------------------------- +Fri Mar 23 16:19:42 UTC 2018 - Mihai Dinca + +- Update openscap push patch to include the test fixes + +- Modified: + * explore-module.run-response-to-catch-the-result-in-d.patch + * fix-openscap-push.patch + * fix-decrease-loglevel-when-unable-to-resolve-addr.patch + * fix-cp.push-empty-file.patch + * make-it-possible-to-use-login-pull-and-push-from-mod.patch + * move-log_file-option-to-changeable-defaults.patch + +------------------------------------------------------------------- +Thu Mar 22 14:40:50 UTC 2018 - Pablo Suárez Hernández + +- Explore 'module.run' state module output in depth to catch "result" properly + +- Added: + * explore-module.run-response-to-catch-the-result-in-d.patch + +------------------------------------------------------------------- +Thu Mar 22 09:10:33 UTC 2018 - Michael Calmer + +- make it possible to use docker login, pull and push from module.run and detect errors +- Added: + * make-it-possible-to-use-login-pull-and-push-from-mod.patch + +------------------------------------------------------------------- +Tue Mar 20 19:15:38 UTC 2018 - michele.bologna@suse.com + +- Fix logging with FQDNs + +- Added: + * fix-decrease-loglevel-when-unable-to-resolve-addr.patch + ------------------------------------------------------------------- Wed Mar 14 09:37:07 UTC 2018 - Mihai Dinca diff --git a/salt.spec b/salt.spec index c58fb6e..2d67b34 100644 --- a/salt.spec +++ b/salt.spec @@ -52,14 +52,13 @@ %bcond_with builddocs Name: salt -Version: 2018.1.99 +Version: 2018.3.0 Release: 0 Summary: A parallel remote execution system License: Apache-2.0 Group: System/Management Url: http://saltstack.org/ -# Source: https://github.com/saltstack/salt/archive/v%{version}.tar.gz -Source: https://github.com/saltstack/salt/archive/2018.3.0rc1.tar.gz +Source: https://github.com/saltstack/salt/archive/v%{version}.tar.gz Source1: README.SUSE Source2: salt-tmpfiles.d Source3: html.tar.bz2 @@ -79,9 +78,17 @@ Patch8: fix-openscap-push.patch Patch9: move-log_file-option-to-changeable-defaults.patch # PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/46416 Patch10: fix-cp.push-empty-file.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/46575 +Patch11: fix-decrease-loglevel-when-unable-to-resolve-addr.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/46643 +Patch12: make-it-possible-to-use-login-pull-and-push-from-mod.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/46413 +Patch13: explore-module.run-response-to-catch-the-result-in-d.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/46684 +Patch14: add-saltssh-multi-version-support-across-python-inte.patch # BuildRoot: %{_tmppath}/%{name}-%{version}-build -BuildRoot: %{_tmppath}/%{name}-2018.3.0rc1-build +BuildRoot: %{_tmppath}/%{name}-%{version}-build BuildRequires: logrotate %if 0%{?suse_version} > 1020 BuildRequires: fdupes @@ -106,6 +113,16 @@ Requires(pre): dbus Requires: logrotate Requires: procps +%if 0%{?suse_version} >= 1500 +Requires: iproute2 +%else +Requires: net-tools +%endif + +%if 0%{?rhel} +Requires: iproute +%endif + %if %{with systemd} BuildRequires: systemd %{?systemd_requires} @@ -524,7 +541,7 @@ Zsh command line completion support for %{name}. %prep # %setup -q -n salt-%{version} -%setup -q -n salt-2018.3.0rc1 +%setup -q -n salt-%{version} cp %{S:1} . cp %{S:5} ./.travis.yml %patch1 -p1 @@ -537,6 +554,10 @@ cp %{S:5} ./.travis.yml %patch8 -p1 %patch9 -p1 %patch10 -p1 +%patch11 -p1 +%patch12 -p1 +%patch13 -p1 +%patch14 -p1 %build %if 0%{?build_py2} diff --git a/v2018.3.0.tar.gz b/v2018.3.0.tar.gz new file mode 100644 index 0000000..df8e8b6 --- /dev/null +++ b/v2018.3.0.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4310936a99a330fb67d86d430189831b8b7e064357a8faabebd5e0115a7e0dfc +size 13469511