2018-07-30 13:52:13 +02:00
|
|
|
From 23aba97ccbdf9952f6a8107a8d90b40d0d2c41ee Mon Sep 17 00:00:00 2001
|
2018-04-11 18:47:08 +02:00
|
|
|
From: Bo Maryniuk <bo@suse.de>
|
|
|
|
Date: Mon, 12 Mar 2018 12:01:39 +0100
|
|
|
|
Subject: [PATCH] Add SaltSSH multi-version support across Python
|
|
|
|
interpeters.
|
|
|
|
|
|
|
|
Bugfix: crashes when OPTIONS.saltdir is a file
|
|
|
|
|
|
|
|
salt-ssh: allow server and client to run different python major version
|
|
|
|
|
|
|
|
Handle non-directory on the /tmp
|
|
|
|
|
|
|
|
Bugfix: prevent partial fileset removal in /tmp
|
|
|
|
|
|
|
|
salt-ssh: compare checksums to detect newly generated thin on the server
|
|
|
|
|
|
|
|
Reset time at thin unpack
|
|
|
|
|
|
|
|
Bugfix: get a proper option for CLI and opts of wiping the tmp
|
|
|
|
|
|
|
|
Add docstring to get_tops
|
|
|
|
|
|
|
|
Remove unnecessary noise in imports
|
|
|
|
|
|
|
|
Refactor get_tops collector
|
|
|
|
|
|
|
|
Add logging to the get_tops
|
|
|
|
|
|
|
|
Update call script
|
|
|
|
|
|
|
|
Remove pre-caution
|
|
|
|
|
|
|
|
Update log debug message for tops collector
|
|
|
|
|
|
|
|
Reset default compression, if unknown is passed
|
|
|
|
|
|
|
|
Refactor archive creation flow
|
|
|
|
|
|
|
|
Add external shell-callable function to collect tops
|
|
|
|
|
|
|
|
Simplify tops gathering, bugfix alternative to Py2
|
|
|
|
|
|
|
|
find working executable
|
|
|
|
|
|
|
|
Add basic shareable module classifier
|
|
|
|
|
|
|
|
Add proper error handler, unmuting exceptions during top collection
|
|
|
|
|
|
|
|
Use common shared directory for compatible libraries
|
|
|
|
|
|
|
|
fix searching for python versions
|
|
|
|
|
|
|
|
Flatten error message string
|
|
|
|
|
|
|
|
Bail-out immediately if <2.6 version detected
|
|
|
|
|
|
|
|
Simplify shell cmd to get the version on Python 2.x
|
|
|
|
|
|
|
|
Remove stub that was previously moved upfront
|
|
|
|
|
|
|
|
Lintfix: PEP8 ident
|
|
|
|
|
|
|
|
Add logging on the error, when Python-2 version cannot be detected properly
|
|
|
|
|
|
|
|
Generate salt-call source, based on conditions
|
|
|
|
|
|
|
|
Add logging on remove failure on thin.tgz archive
|
|
|
|
|
|
|
|
Add config-based external tops gatherer
|
|
|
|
|
|
|
|
Change signature to pass the extended configuration to the thin generator
|
|
|
|
|
|
|
|
Update docstring to the salt-call generator
|
|
|
|
|
|
|
|
Implement get namespaces inclusion to the salt-call script on the client machine
|
|
|
|
|
|
|
|
Use new signature of the get call
|
|
|
|
|
|
|
|
Implement namespace selector, based on the current Python interpreter version
|
|
|
|
|
|
|
|
Add deps as a list, instead of a map
|
|
|
|
|
|
|
|
Add debug logging
|
|
|
|
|
|
|
|
Implement packaging an alternative version
|
|
|
|
|
|
|
|
Update salt-call script so it swaps the namespace according to the configuration
|
|
|
|
|
|
|
|
Compress thin.zip if zlib is available
|
|
|
|
|
|
|
|
Fix a system exit error message
|
|
|
|
|
|
|
|
Move compression fall-back operation
|
|
|
|
|
|
|
|
Add debug logging prior to the thin archive removal
|
|
|
|
|
|
|
|
Flatten the archive extension choice
|
|
|
|
|
|
|
|
Lintfix: PEP8 an empty line required
|
|
|
|
|
|
|
|
Bugfix: ZFS modules (zfs, zpool) crashes on non-ZFS systems
|
|
|
|
|
|
|
|
Add unit test case for the Salt SSH parts
|
|
|
|
|
|
|
|
Add unit test for missing dependencies on get_ext_tops
|
|
|
|
|
|
|
|
Postpone inheritance implementation
|
|
|
|
|
|
|
|
Refactor unit test for get_ext_tops
|
|
|
|
|
|
|
|
Add unit test for get_ext_tops checks interpreter configuration
|
|
|
|
|
|
|
|
Check python interpreter lock version
|
|
|
|
|
|
|
|
Add unit test for get_ext_tops checks the python locked interepreter value
|
|
|
|
|
|
|
|
Bugfix: report into warning log module name, not its config
|
|
|
|
|
|
|
|
Add unit test for dependencies check python version lock (inherently)
|
|
|
|
|
|
|
|
Mock os.path.isfile function
|
|
|
|
|
|
|
|
Update warning logging information
|
|
|
|
|
|
|
|
Add unit test for get_ext_tops module configuration validation
|
|
|
|
|
|
|
|
Do not use list of dicts for namespaces, just dict for namespaces.
|
|
|
|
|
|
|
|
Add unit test for get_ext_tops config verification
|
|
|
|
|
|
|
|
Fix unit tests for the new config structure
|
|
|
|
|
|
|
|
Add unit test for thin.gte call
|
|
|
|
|
|
|
|
Add unit test for dependency path adding function
|
|
|
|
|
|
|
|
Add unit test for thin_path function
|
|
|
|
|
|
|
|
Add unit test for salt-call source generator
|
|
|
|
|
|
|
|
Add unit test for get_ext_namespaces on empty configuration
|
|
|
|
|
|
|
|
Add get_ext_namespaces for namespace extractions into a tuple for python version
|
|
|
|
|
|
|
|
Remove unused variable
|
|
|
|
|
|
|
|
Add unit test for getting namespace failure when python maj/min versions are not defined
|
|
|
|
|
|
|
|
Add unit test to add tops based on the current interpreter
|
|
|
|
|
|
|
|
Add unit test for get_tops with extra modules
|
|
|
|
|
|
|
|
Add unit test for shared object modules top addition
|
|
|
|
|
|
|
|
Add unit test for thin_sum hashing
|
|
|
|
|
|
|
|
Add unit test for min_sum hashing
|
|
|
|
|
|
|
|
Add unit test for gen_thin verify for 2.6 Python version is a minimum requirement
|
|
|
|
|
|
|
|
Fix gen_thin exception on Python 3
|
|
|
|
|
|
|
|
Use object attribute instead of indeces. Remove an empty line.
|
|
|
|
|
|
|
|
Add unit test for gen_thin compression type fallback
|
|
|
|
|
|
|
|
Move helper functions up by the class code
|
|
|
|
|
|
|
|
Update unit test doc
|
|
|
|
|
|
|
|
Add check for correct archiving mode is opened
|
|
|
|
|
|
|
|
Add unit test for gen_thin if control files are written correctly
|
|
|
|
|
|
|
|
Update docstring for fake version info constructor method
|
|
|
|
|
|
|
|
Add fake tarfile mock handler
|
|
|
|
|
|
|
|
Mock-out missing methods inside gen_thin
|
|
|
|
|
|
|
|
Move tarfile.open check to the end of the test
|
|
|
|
|
|
|
|
Add unit test for tree addition to the archive
|
|
|
|
|
|
|
|
Add shareable module to the gen_thin unit test
|
|
|
|
|
|
|
|
Fix docstring
|
|
|
|
|
|
|
|
Add unit test for an alternative version pack
|
|
|
|
|
|
|
|
Lintfix
|
|
|
|
|
|
|
|
Add documentation about updated Salt SSH features
|
|
|
|
|
|
|
|
Fix typo
|
|
|
|
|
|
|
|
Lintfix: PEP8 extra-line needed
|
|
|
|
|
|
|
|
Make the command more readable
|
|
|
|
|
|
|
|
Write all supported minimal python versions into a config file on the target machine
|
|
|
|
|
|
|
|
Get supported Python executable based on the config py-map
|
|
|
|
|
|
|
|
Add unit test for get_supported_py_config function typecheck
|
|
|
|
|
|
|
|
Add unit test for get_supported_py_config function base tops
|
|
|
|
|
|
|
|
Add unit test for get_supported_py_config function ext tops
|
|
|
|
|
|
|
|
Fix unit test for catching "supported-versions" was written down
|
|
|
|
|
|
|
|
Rephrase Salt SSH doc description
|
|
|
|
|
|
|
|
Re-phrase docstring for alternative Salt installation
|
|
|
|
|
|
|
|
require same major version while minor is allowed to be higher
|
|
|
|
|
|
|
|
Bugfix: remove minor version from the namespaced, version-specific directory
|
|
|
|
|
|
|
|
Fix unit tests for minor version removal of namespaced version-specific directory
|
|
|
|
|
|
|
|
Initialise the options directly to be structure-ready object.
|
|
|
|
|
|
|
|
Disable wiping if state is executed
|
|
|
|
|
|
|
|
Properly mock a tempfile object
|
|
|
|
|
|
|
|
Support Python 2.6 versions
|
2018-04-30 14:06:44 +02:00
|
|
|
|
|
|
|
Add digest collector for file trees etc
|
|
|
|
|
|
|
|
Bufix: recurse calls damages the configuration (reference problem)
|
|
|
|
|
|
|
|
Collect digest of the code
|
|
|
|
|
|
|
|
Get code checksum into the shim options
|
|
|
|
|
|
|
|
Get all the code content, not just Python sources
|
|
|
|
|
|
|
|
Bugfix: Python3 compat - string required instead of bytes
|
|
|
|
|
|
|
|
Lintfix: too many empty lines
|
|
|
|
|
|
|
|
Lintfix: blocked function used
|
|
|
|
|
|
|
|
Bugfix: key error master_tops_first
|
|
|
|
|
|
|
|
Fix unit tests for the checksum generator
|
|
|
|
|
|
|
|
Use code checksum to update thin archive on client's cache
|
|
|
|
|
|
|
|
Lintfix
|
|
|
|
|
|
|
|
Set master_top_first to False by default
|
2018-04-11 18:47:08 +02:00
|
|
|
---
|
|
|
|
doc/topics/releases/fluorine.rst | 178 +++++++++++
|
2018-07-30 13:52:13 +02:00
|
|
|
salt/client/ssh/__init__.py | 60 ++--
|
|
|
|
salt/client/ssh/ssh_py_shim.py | 93 ++++--
|
2018-04-11 18:47:08 +02:00
|
|
|
salt/client/ssh/wrapper/__init__.py | 2 +-
|
2018-04-30 14:06:44 +02:00
|
|
|
salt/config/__init__.py | 1 +
|
|
|
|
salt/state.py | 2 +-
|
|
|
|
salt/utils/hashutils.py | 37 +++
|
|
|
|
salt/utils/thin.py | 450 +++++++++++++++++++-------
|
|
|
|
tests/unit/utils/test_thin.py | 612 ++++++++++++++++++++++++++++++++++++
|
2018-07-30 13:52:13 +02:00
|
|
|
9 files changed, 1258 insertions(+), 177 deletions(-)
|
2018-04-11 18:47:08 +02:00
|
|
|
create mode 100644 doc/topics/releases/fluorine.rst
|
|
|
|
create mode 100644 tests/unit/utils/test_thin.py
|
|
|
|
|
|
|
|
diff --git a/doc/topics/releases/fluorine.rst b/doc/topics/releases/fluorine.rst
|
|
|
|
new file mode 100644
|
|
|
|
index 0000000000..40c69e25cc
|
|
|
|
--- /dev/null
|
|
|
|
+++ b/doc/topics/releases/fluorine.rst
|
|
|
|
@@ -0,0 +1,178 @@
|
|
|
|
+:orphan:
|
|
|
|
+
|
|
|
|
+======================================
|
|
|
|
+Salt Release Notes - Codename Fluorine
|
|
|
|
+======================================
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+Minion Startup Events
|
|
|
|
+---------------------
|
|
|
|
+
|
|
|
|
+When a minion starts up it sends a notification on the event bus with a tag
|
|
|
|
+that looks like this: `salt/minion/<minion_id>/start`. For historical reasons
|
|
|
|
+the minion also sends a similar event with an event tag like this:
|
|
|
|
+`minion_start`. This duplication can cause a lot of clutter on the event bus
|
|
|
|
+when there are many minions. Set `enable_legacy_startup_events: False` in the
|
|
|
|
+minion config to ensure only the `salt/minion/<minion_id>/start` events are
|
|
|
|
+sent.
|
|
|
|
+
|
|
|
|
+The new :conf_minion:`enable_legacy_startup_events` minion config option
|
|
|
|
+defaults to ``True``, but will be set to default to ``False`` beginning with
|
|
|
|
+the Neon release of Salt.
|
|
|
|
+
|
|
|
|
+The Salt Syndic currently sends an old style `syndic_start` event as well. The
|
|
|
|
+syndic respects :conf_minion:`enable_legacy_startup_events` as well.
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+Deprecations
|
|
|
|
+------------
|
|
|
|
+
|
|
|
|
+Module Deprecations
|
|
|
|
+===================
|
|
|
|
+
|
|
|
|
+The ``napalm_network`` module had the following changes:
|
|
|
|
+
|
|
|
|
+- Support for the ``template_path`` has been removed in the ``load_template``
|
|
|
|
+ function. This is because support for NAPALM native templates has been
|
|
|
|
+ dropped.
|
|
|
|
+
|
|
|
|
+The ``trafficserver`` module had the following changes:
|
|
|
|
+
|
|
|
|
+- Support for the ``match_var`` function was removed. Please use the
|
|
|
|
+ ``match_metric`` function instead.
|
|
|
|
+- Support for the ``read_var`` function was removed. Please use the
|
|
|
|
+ ``read_config`` function instead.
|
|
|
|
+- Support for the ``set_var`` function was removed. Please use the
|
|
|
|
+ ``set_config`` function instead.
|
|
|
|
+
|
|
|
|
+The ``win_update`` module has been removed. It has been replaced by ``win_wua``
|
|
|
|
+module.
|
|
|
|
+
|
|
|
|
+The ``win_wua`` module had the following changes:
|
|
|
|
+
|
|
|
|
+- Support for the ``download_update`` function has been removed. Please use the
|
|
|
|
+ ``download`` function instead.
|
|
|
|
+- Support for the ``download_updates`` function has been removed. Please use the
|
|
|
|
+ ``download`` function instead.
|
|
|
|
+- Support for the ``install_update`` function has been removed. Please use the
|
|
|
|
+ ``install`` function instead.
|
|
|
|
+- Support for the ``install_updates`` function has been removed. Please use the
|
|
|
|
+ ``install`` function instead.
|
|
|
|
+- Support for the ``list_update`` function has been removed. Please use the
|
|
|
|
+ ``get`` function instead.
|
|
|
|
+- Support for the ``list_updates`` function has been removed. Please use the
|
|
|
|
+ ``list`` function instead.
|
|
|
|
+
|
|
|
|
+Pillar Deprecations
|
|
|
|
+===================
|
|
|
|
+
|
|
|
|
+The ``vault`` pillar had the following changes:
|
|
|
|
+
|
|
|
|
+- Support for the ``profile`` argument was removed. Any options passed up until
|
|
|
|
+ and following the first ``path=`` are discarded.
|
|
|
|
+
|
|
|
|
+Roster Deprecations
|
|
|
|
+===================
|
|
|
|
+
|
|
|
|
+The ``cache`` roster had the following changes:
|
|
|
|
+
|
|
|
|
+- Support for ``roster_order`` as a list or tuple has been removed. As of the
|
|
|
|
+ ``Fluorine`` release, ``roster_order`` must be a dictionary.
|
|
|
|
+- The ``roster_order`` option now includes IPv6 in addition to IPv4 for the
|
|
|
|
+ ``private``, ``public``, ``global`` or ``local`` settings. The syntax for these
|
|
|
|
+ settings has changed to ``ipv4-*`` or ``ipv6-*``, respectively.
|
|
|
|
+
|
|
|
|
+State Deprecations
|
|
|
|
+==================
|
|
|
|
+
|
|
|
|
+The ``docker`` state has been removed. The following functions should be used
|
|
|
|
+instead.
|
|
|
|
+
|
|
|
|
+- The ``docker.running`` function was removed. Please update applicable SLS files
|
|
|
|
+ to use the ``docker_container.running`` function instead.
|
|
|
|
+- The ``docker.stopped`` function was removed. Please update applicable SLS files
|
|
|
|
+ to use the ``docker_container.stopped`` function instead.
|
|
|
|
+- The ``docker.absent`` function was removed. Please update applicable SLS files
|
|
|
|
+ to use the ``docker_container.absent`` function instead.
|
|
|
|
+- The ``docker.absent`` function was removed. Please update applicable SLS files
|
|
|
|
+ to use the ``docker_container.absent`` function instead.
|
|
|
|
+- The ``docker.network_present`` function was removed. Please update applicable
|
|
|
|
+ SLS files to use the ``docker_network.present`` function instead.
|
|
|
|
+- The ``docker.network_absent`` function was removed. Please update applicable
|
|
|
|
+ SLS files to use the ``docker_network.absent`` function instead.
|
|
|
|
+- The ``docker.image_present`` function was removed. Please update applicable SLS
|
|
|
|
+ files to use the ``docker_image.present`` function instead.
|
|
|
|
+- The ``docker.image_absent`` function was removed. Please update applicable SLS
|
|
|
|
+ files to use the ``docker_image.absent`` function instead.
|
|
|
|
+- The ``docker.volume_present`` function was removed. Please update applicable SLS
|
|
|
|
+ files to use the ``docker_volume.present`` function instead.
|
|
|
|
+- The ``docker.volume_absent`` function was removed. Please update applicable SLS
|
|
|
|
+ files to use the ``docker_volume.absent`` function instead.
|
|
|
|
+
|
|
|
|
+The ``docker_network`` state had the following changes:
|
|
|
|
+
|
|
|
|
+- Support for the ``driver`` option has been removed from the ``absent`` function.
|
|
|
|
+ This option had no functionality in ``docker_network.absent``.
|
|
|
|
+
|
|
|
|
+The ``git`` state had the following changes:
|
|
|
|
+
|
|
|
|
+- Support for the ``ref`` option in the ``detached`` state has been removed.
|
|
|
|
+ Please use the ``rev`` option instead.
|
|
|
|
+
|
|
|
|
+The ``k8s`` state has been removed. The following functions should be used
|
|
|
|
+instead:
|
|
|
|
+
|
|
|
|
+- The ``k8s.label_absent`` function was removed. Please update applicable SLS
|
|
|
|
+ files to use the ``kubernetes.node_label_absent`` function instead.
|
|
|
|
+- The ``k8s.label_present`` function was removed. Please updated applicable SLS
|
|
|
|
+ files to use the ``kubernetes.node_label_present`` function instead.
|
|
|
|
+- The ``k8s.label_folder_absent`` function was removed. Please update applicable
|
|
|
|
+ SLS files to use the ``kubernetes.node_label_folder_absent`` function instead.
|
|
|
|
+
|
|
|
|
+The ``netconfig`` state had the following changes:
|
|
|
|
+
|
|
|
|
+- Support for the ``template_path`` option in the ``managed`` state has been
|
|
|
|
+ removed. This is because support for NAPALM native templates has been dropped.
|
|
|
|
+
|
|
|
|
+The ``trafficserver`` state had the following changes:
|
|
|
|
+
|
|
|
|
+- Support for the ``set_var`` function was removed. Please use the ``config``
|
|
|
|
+ function instead.
|
|
|
|
+
|
|
|
|
+The ``win_update`` state has been removed. Please use the ``win_wua`` state instead.
|
|
|
|
+
|
|
|
|
+SaltSSH major updates
|
|
|
|
+=====================
|
|
|
|
+
|
|
|
|
+SaltSSH now works across different major Python versions. Python 2.7 ~ Python 3.x
|
|
|
|
+are now supported transparently. Requirement is, however, that the SaltMaster should
|
|
|
|
+have installed Salt, including all related dependencies for Python 2 and Python 3.
|
|
|
|
+Everything needs to be importable from the respective Python environment.
|
|
|
|
+
|
|
|
|
+SaltSSH can bundle up an arbitrary version of Salt. If there would be an old box for
|
|
|
|
+example, running an outdated and unsupported Python 2.6, it is still possible from
|
|
|
|
+a SaltMaster with Python 3.5 or newer to access it. This feature requires an additional
|
|
|
|
+configuration in /etc/salt/master as follows:
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+.. code-block:: yaml
|
|
|
|
+
|
|
|
|
+ ssh_ext_alternatives:
|
|
|
|
+ 2016.3: # Namespace, can be actually anything.
|
|
|
|
+ py-version: [2, 6] # Constraint to specific interpreter version
|
|
|
|
+ path: /opt/2016.3/salt # Main Salt installation
|
|
|
|
+ dependencies: # List of dependencies and their installation paths
|
|
|
|
+ jinja2: /opt/jinja2
|
|
|
|
+ yaml: /opt/yaml
|
|
|
|
+ tornado: /opt/tornado
|
|
|
|
+ msgpack: /opt/msgpack
|
|
|
|
+ certifi: /opt/certifi
|
|
|
|
+ singledispatch: /opt/singledispatch.py
|
|
|
|
+ singledispatch_helpers: /opt/singledispatch_helpers.py
|
|
|
|
+ markupsafe: /opt/markupsafe
|
|
|
|
+ backports_abc: /opt/backports_abc.py
|
|
|
|
+
|
|
|
|
+It is also possible to use several alternative versions of Salt. You can for instance generate
|
|
|
|
+a minimal tarball using runners and include that. But this is only possible, when such specific
|
|
|
|
+Salt version is also available on the Master machine, although does not need to be directly
|
|
|
|
+installed together with the older Python interpreter.
|
|
|
|
diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py
|
2018-07-30 13:52:13 +02:00
|
|
|
index 141e1c6850..f1300b5698 100644
|
2018-04-11 18:47:08 +02:00
|
|
|
--- a/salt/client/ssh/__init__.py
|
|
|
|
+++ b/salt/client/ssh/__init__.py
|
2018-07-30 13:52:13 +02:00
|
|
|
@@ -150,9 +150,7 @@ EX_PYTHON_INVALID={EX_THIN_PYTHON_INVALID}
|
2018-04-11 18:47:08 +02:00
|
|
|
PYTHON_CMDS="python3 python27 python2.7 python26 python2.6 python2 python"
|
|
|
|
for py_cmd in $PYTHON_CMDS
|
|
|
|
do
|
|
|
|
- if command -v "$py_cmd" >/dev/null 2>&1 && "$py_cmd" -c \
|
|
|
|
- "import sys; sys.exit(not (sys.version_info >= (2, 6)
|
|
|
|
- and sys.version_info[0] == {{HOST_PY_MAJOR}}));"
|
|
|
|
+ if command -v "$py_cmd" >/dev/null 2>&1 && "$py_cmd" -c "import sys; sys.exit(not (sys.version_info >= (2, 6)));"
|
|
|
|
then
|
2018-07-30 13:52:13 +02:00
|
|
|
py_cmd_path=`"$py_cmd" -c \
|
|
|
|
'from __future__ import print_function;
|
|
|
|
@@ -323,7 +321,8 @@ class SSH(object):
|
2018-04-11 18:47:08 +02:00
|
|
|
extra_mods=self.opts.get('thin_extra_mods'),
|
|
|
|
overwrite=self.opts['regen_thin'],
|
|
|
|
python2_bin=self.opts['python2_bin'],
|
|
|
|
- python3_bin=self.opts['python3_bin'])
|
|
|
|
+ python3_bin=self.opts['python3_bin'],
|
|
|
|
+ extended_cfg=self.opts.get('ssh_ext_alternatives'))
|
|
|
|
self.mods = mod_data(self.fsclient)
|
|
|
|
|
|
|
|
def _get_roster(self):
|
2018-07-30 13:52:13 +02:00
|
|
|
@@ -850,10 +849,10 @@ class Single(object):
|
2018-04-11 18:47:08 +02:00
|
|
|
|
|
|
|
self.opts = opts
|
|
|
|
self.tty = tty
|
|
|
|
- if kwargs.get('wipe'):
|
|
|
|
- self.wipe = 'False'
|
|
|
|
+ if kwargs.get('disable_wipe'):
|
|
|
|
+ self.wipe = False
|
|
|
|
else:
|
|
|
|
- self.wipe = 'True' if self.opts.get('ssh_wipe') else 'False'
|
|
|
|
+ self.wipe = bool(self.opts.get('ssh_wipe'))
|
|
|
|
if kwargs.get('thin_dir'):
|
|
|
|
self.thin_dir = kwargs['thin_dir']
|
|
|
|
elif self.winrm:
|
2018-07-30 13:52:13 +02:00
|
|
|
@@ -1178,38 +1177,39 @@ class Single(object):
|
2018-04-30 14:06:44 +02:00
|
|
|
cachedir = self.opts['_caller_cachedir']
|
|
|
|
else:
|
|
|
|
cachedir = self.opts['cachedir']
|
|
|
|
- thin_sum = salt.utils.thin.thin_sum(cachedir, 'sha1')
|
|
|
|
+ thin_code_digest, thin_sum = salt.utils.thin.thin_sum(cachedir, 'sha1')
|
|
|
|
debug = ''
|
|
|
|
if not self.opts.get('log_level'):
|
|
|
|
self.opts['log_level'] = 'info'
|
2018-04-11 18:47:08 +02:00
|
|
|
if salt.log.LOG_LEVELS['debug'] >= salt.log.LOG_LEVELS[self.opts.get('log_level', 'info')]:
|
|
|
|
debug = '1'
|
|
|
|
arg_str = '''
|
|
|
|
-OPTIONS = OBJ()
|
|
|
|
OPTIONS.config = \
|
|
|
|
"""
|
2018-04-30 14:06:44 +02:00
|
|
|
-{0}
|
|
|
|
+{config}
|
|
|
|
"""
|
|
|
|
-OPTIONS.delimiter = '{1}'
|
|
|
|
-OPTIONS.saltdir = '{2}'
|
|
|
|
-OPTIONS.checksum = '{3}'
|
|
|
|
-OPTIONS.hashfunc = '{4}'
|
|
|
|
-OPTIONS.version = '{5}'
|
|
|
|
-OPTIONS.ext_mods = '{6}'
|
|
|
|
-OPTIONS.wipe = {7}
|
|
|
|
-OPTIONS.tty = {8}
|
|
|
|
-OPTIONS.cmd_umask = {9}
|
|
|
|
-ARGS = {10}\n'''.format(self.minion_config,
|
|
|
|
- RSTR,
|
|
|
|
- self.thin_dir,
|
|
|
|
- thin_sum,
|
|
|
|
- 'sha1',
|
|
|
|
- salt.version.__version__,
|
|
|
|
- self.mods.get('version', ''),
|
|
|
|
- self.wipe,
|
|
|
|
- self.tty,
|
|
|
|
- self.cmd_umask,
|
|
|
|
- self.argv)
|
|
|
|
+OPTIONS.delimiter = '{delimeter}'
|
|
|
|
+OPTIONS.saltdir = '{saltdir}'
|
|
|
|
+OPTIONS.checksum = '{checksum}'
|
|
|
|
+OPTIONS.hashfunc = '{hashfunc}'
|
|
|
|
+OPTIONS.version = '{version}'
|
|
|
|
+OPTIONS.ext_mods = '{ext_mods}'
|
|
|
|
+OPTIONS.wipe = {wipe}
|
|
|
|
+OPTIONS.tty = {tty}
|
|
|
|
+OPTIONS.cmd_umask = {cmd_umask}
|
|
|
|
+OPTIONS.code_checksum = {code_checksum}
|
|
|
|
+ARGS = {arguments}\n'''.format(config=self.minion_config,
|
|
|
|
+ delimeter=RSTR,
|
|
|
|
+ saltdir=self.thin_dir,
|
|
|
|
+ checksum=thin_sum,
|
|
|
|
+ hashfunc='sha1',
|
|
|
|
+ version=salt.version.__version__,
|
|
|
|
+ ext_mods=self.mods.get('version', ''),
|
|
|
|
+ wipe=self.wipe,
|
|
|
|
+ tty=self.tty,
|
|
|
|
+ cmd_umask=self.cmd_umask,
|
|
|
|
+ code_checksum=thin_code_digest,
|
|
|
|
+ arguments=self.argv)
|
|
|
|
py_code = SSH_PY_SHIM.replace('#%%OPTS', arg_str)
|
|
|
|
if six.PY2:
|
|
|
|
py_code_enc = py_code.encode('base64')
|
2018-04-11 18:47:08 +02:00
|
|
|
diff --git a/salt/client/ssh/ssh_py_shim.py b/salt/client/ssh/ssh_py_shim.py
|
2018-07-30 13:52:13 +02:00
|
|
|
index 5e5dbdc55e..92ede14930 100644
|
2018-04-11 18:47:08 +02:00
|
|
|
--- a/salt/client/ssh/ssh_py_shim.py
|
|
|
|
+++ b/salt/client/ssh/ssh_py_shim.py
|
|
|
|
@@ -16,11 +16,13 @@ import sys
|
|
|
|
import os
|
|
|
|
import stat
|
|
|
|
import subprocess
|
|
|
|
+import time
|
|
|
|
|
|
|
|
THIN_ARCHIVE = 'salt-thin.tgz'
|
|
|
|
EXT_ARCHIVE = 'salt-ext_mods.tgz'
|
|
|
|
|
|
|
|
# Keep these in sync with salt/defaults/exitcodes.py
|
|
|
|
+EX_THIN_PYTHON_INVALID = 10
|
|
|
|
EX_THIN_DEPLOY = 11
|
|
|
|
EX_THIN_CHECKSUM = 12
|
|
|
|
EX_MOD_DEPLOY = 13
|
|
|
|
@@ -28,14 +30,13 @@ EX_SCP_NOT_FOUND = 14
|
|
|
|
EX_CANTCREAT = 73
|
|
|
|
|
|
|
|
|
|
|
|
-class OBJ(object):
|
|
|
|
+class OptionsContainer(object):
|
|
|
|
'''
|
|
|
|
An empty class for holding instance attribute values.
|
|
|
|
'''
|
|
|
|
- pass
|
|
|
|
|
|
|
|
|
|
|
|
-OPTIONS = None
|
|
|
|
+OPTIONS = OptionsContainer()
|
|
|
|
ARGS = None
|
|
|
|
# The below line is where OPTIONS can be redefined with internal options
|
|
|
|
# (rather than cli arguments) when the shim is bundled by
|
2018-07-30 13:52:13 +02:00
|
|
|
@@ -130,7 +131,7 @@ def need_deployment():
|
2018-04-11 18:47:08 +02:00
|
|
|
os.chmod(OPTIONS.saltdir, stt.st_mode | stat.S_IWGRP | stat.S_IRGRP | stat.S_IXGRP)
|
|
|
|
except OSError:
|
|
|
|
sys.stdout.write('\n\nUnable to set permissions on thin directory.\nIf sudo_user is set '
|
|
|
|
- 'and is not root, be certain the user is in the same group\nas the login user')
|
|
|
|
+ 'and is not root, be certain the user is in the same group\nas the login user')
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
# Delimiter emitted on stdout *only* to indicate shim message to master.
|
2018-07-30 13:52:13 +02:00
|
|
|
@@ -163,11 +164,15 @@ def unpack_thin(thin_path):
|
|
|
|
old_umask = os.umask(0o077) # pylint: disable=blacklisted-function
|
2018-04-11 18:47:08 +02:00
|
|
|
tfile.extractall(path=OPTIONS.saltdir)
|
|
|
|
tfile.close()
|
|
|
|
+ checksum_path = os.path.normpath(os.path.join(OPTIONS.saltdir, "thin_checksum"))
|
|
|
|
+ with open(checksum_path, 'w') as chk:
|
|
|
|
+ chk.write(OPTIONS.checksum + '\n')
|
2018-07-30 13:52:13 +02:00
|
|
|
os.umask(old_umask) # pylint: disable=blacklisted-function
|
2018-04-11 18:47:08 +02:00
|
|
|
try:
|
|
|
|
os.unlink(thin_path)
|
|
|
|
except OSError:
|
|
|
|
pass
|
|
|
|
+ reset_time(OPTIONS.saltdir)
|
|
|
|
|
|
|
|
|
|
|
|
def need_ext():
|
2018-07-30 13:52:13 +02:00
|
|
|
@@ -201,6 +206,47 @@ def unpack_ext(ext_path):
|
2018-04-11 18:47:08 +02:00
|
|
|
shutil.move(ver_path, ver_dst)
|
|
|
|
|
|
|
|
|
|
|
|
+def reset_time(path='.', amt=None):
|
|
|
|
+ '''
|
|
|
|
+ Reset atime/mtime on all files to prevent systemd swipes only part of the files in the /tmp.
|
|
|
|
+ '''
|
|
|
|
+ if not amt:
|
|
|
|
+ amt = int(time.time())
|
|
|
|
+ for fname in os.listdir(path):
|
|
|
|
+ fname = os.path.join(path, fname)
|
|
|
|
+ if os.path.isdir(fname):
|
|
|
|
+ reset_time(fname, amt=amt)
|
|
|
|
+ os.utime(fname, (amt, amt,))
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def get_executable():
|
|
|
|
+ '''
|
|
|
|
+ Find executable which matches supported python version in the thin
|
|
|
|
+ '''
|
|
|
|
+ pymap = {}
|
|
|
|
+ with open(os.path.join(OPTIONS.saltdir, 'supported-versions')) as _fp:
|
|
|
|
+ for line in _fp.readlines():
|
|
|
|
+ ns, v_maj, v_min = line.strip().split(':')
|
|
|
|
+ pymap[ns] = (int(v_maj), int(v_min))
|
|
|
|
+
|
|
|
|
+ pycmds = (sys.executable, 'python3', 'python27', 'python2.7', 'python26', 'python2.6', 'python2', 'python')
|
|
|
|
+ for py_cmd in pycmds:
|
|
|
|
+ cmd = py_cmd + ' -c "import sys; sys.stdout.write(\'%s:%s\' % (sys.version_info[0], sys.version_info[1]))"'
|
|
|
|
+ stdout, _ = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True).communicate()
|
|
|
|
+ if sys.version_info[0] == 2 and sys.version_info[1] < 7:
|
|
|
|
+ stdout = stdout.decode(get_system_encoding(), "replace").strip()
|
|
|
|
+ else:
|
|
|
|
+ stdout = stdout.decode(encoding=get_system_encoding(), errors="replace").strip()
|
|
|
|
+ if not stdout:
|
|
|
|
+ continue
|
|
|
|
+ c_vn = tuple([int(x) for x in stdout.split(':')])
|
|
|
|
+ for ns in pymap:
|
|
|
|
+ if c_vn[0] == pymap[ns][0] and c_vn >= pymap[ns] and os.path.exists(os.path.join(OPTIONS.saltdir, ns)):
|
|
|
|
+ return py_cmd
|
|
|
|
+
|
|
|
|
+ sys.exit(EX_THIN_PYTHON_INVALID)
|
|
|
|
+
|
|
|
|
+
|
|
|
|
def main(argv): # pylint: disable=W0613
|
|
|
|
'''
|
|
|
|
Main program body
|
2018-07-30 13:52:13 +02:00
|
|
|
@@ -217,32 +263,25 @@ def main(argv): # pylint: disable=W0613
|
2018-04-11 18:47:08 +02:00
|
|
|
if scpstat != 0:
|
|
|
|
sys.exit(EX_SCP_NOT_FOUND)
|
|
|
|
|
|
|
|
- if not os.path.exists(OPTIONS.saltdir):
|
|
|
|
- need_deployment()
|
|
|
|
-
|
|
|
|
- if not os.path.isdir(OPTIONS.saltdir):
|
|
|
|
+ if os.path.exists(OPTIONS.saltdir) and not os.path.isdir(OPTIONS.saltdir):
|
|
|
|
sys.stderr.write(
|
|
|
|
'ERROR: salt path "{0}" exists but is'
|
|
|
|
' not a directory\n'.format(OPTIONS.saltdir)
|
|
|
|
)
|
|
|
|
sys.exit(EX_CANTCREAT)
|
|
|
|
|
|
|
|
- version_path = os.path.normpath(os.path.join(OPTIONS.saltdir, 'version'))
|
|
|
|
- if not os.path.exists(version_path) or not os.path.isfile(version_path):
|
2018-04-30 14:06:44 +02:00
|
|
|
- sys.stderr.write(
|
|
|
|
- 'WARNING: Unable to locate current thin '
|
2018-04-11 18:47:08 +02:00
|
|
|
- ' version: {0}.\n'.format(version_path)
|
2018-04-30 14:06:44 +02:00
|
|
|
- )
|
|
|
|
+ if not os.path.exists(OPTIONS.saltdir):
|
2018-04-11 18:47:08 +02:00
|
|
|
need_deployment()
|
|
|
|
- with open(version_path, 'r') as vpo:
|
|
|
|
- cur_version = vpo.readline().strip()
|
|
|
|
- if cur_version != OPTIONS.version:
|
2018-04-30 14:06:44 +02:00
|
|
|
- sys.stderr.write(
|
2018-04-11 18:47:08 +02:00
|
|
|
- 'WARNING: current thin version {0}'
|
2018-04-30 14:06:44 +02:00
|
|
|
- ' is not up-to-date with {1}.\n'.format(
|
2018-04-11 18:47:08 +02:00
|
|
|
- cur_version, OPTIONS.version
|
2018-04-30 14:06:44 +02:00
|
|
|
- )
|
|
|
|
- )
|
|
|
|
+
|
|
|
|
+ code_checksum_path = os.path.normpath(os.path.join(OPTIONS.saltdir, 'code-checksum'))
|
|
|
|
+ if not os.path.exists(code_checksum_path) or not os.path.isfile(code_checksum_path):
|
|
|
|
+ sys.stderr.write('WARNING: Unable to locate current code checksum: {0}.\n'.format(code_checksum_path))
|
|
|
|
+ need_deployment()
|
|
|
|
+ with open(code_checksum_path, 'r') as vpo:
|
|
|
|
+ cur_code_cs = vpo.readline().strip()
|
|
|
|
+ if cur_code_cs != OPTIONS.code_checksum:
|
|
|
|
+ sys.stderr.write('WARNING: current code checksum {0} is different to {1}.\n'.format(cur_code_cs,
|
|
|
|
+ OPTIONS.code_checksum))
|
2018-04-11 18:47:08 +02:00
|
|
|
need_deployment()
|
2018-04-30 14:06:44 +02:00
|
|
|
# Salt thin exists and is up-to-date - fall through and use it
|
|
|
|
|
2018-07-30 13:52:13 +02:00
|
|
|
@@ -272,7 +311,7 @@ def main(argv): # pylint: disable=W0613
|
2018-04-11 18:47:08 +02:00
|
|
|
argv_prepared = ARGS
|
|
|
|
|
|
|
|
salt_argv = [
|
|
|
|
- sys.executable,
|
|
|
|
+ get_executable(),
|
|
|
|
salt_call_path,
|
|
|
|
'--retcode-passthrough',
|
|
|
|
'--local',
|
2018-07-30 13:52:13 +02:00
|
|
|
@@ -305,7 +344,10 @@ def main(argv): # pylint: disable=W0613
|
2018-04-11 18:47:08 +02:00
|
|
|
if OPTIONS.tty:
|
|
|
|
# Returns bytes instead of string on python 3
|
|
|
|
stdout, _ = subprocess.Popen(salt_argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
|
|
|
|
- sys.stdout.write(stdout.decode(encoding=get_system_encoding(), errors="replace"))
|
|
|
|
+ if sys.version_info[0] == 2 and sys.version_info[1] < 7:
|
|
|
|
+ sys.stdout.write(stdout.decode(get_system_encoding(), "replace"))
|
|
|
|
+ else:
|
|
|
|
+ sys.stdout.write(stdout.decode(encoding=get_system_encoding(), errors="replace"))
|
|
|
|
sys.stdout.flush()
|
|
|
|
if OPTIONS.wipe:
|
|
|
|
shutil.rmtree(OPTIONS.saltdir)
|
2018-07-30 13:52:13 +02:00
|
|
|
@@ -317,5 +359,6 @@ def main(argv): # pylint: disable=W0613
|
2018-04-11 18:47:08 +02:00
|
|
|
if OPTIONS.cmd_umask is not None:
|
2018-07-30 13:52:13 +02:00
|
|
|
os.umask(old_umask) # pylint: disable=blacklisted-function
|
2018-04-11 18:47:08 +02:00
|
|
|
|
|
|
|
+
|
|
|
|
if __name__ == '__main__':
|
|
|
|
sys.exit(main(sys.argv))
|
|
|
|
diff --git a/salt/client/ssh/wrapper/__init__.py b/salt/client/ssh/wrapper/__init__.py
|
|
|
|
index 04d751b51a..09f9344642 100644
|
|
|
|
--- a/salt/client/ssh/wrapper/__init__.py
|
|
|
|
+++ b/salt/client/ssh/wrapper/__init__.py
|
|
|
|
@@ -113,7 +113,7 @@ class FunctionWrapper(object):
|
|
|
|
self.opts,
|
|
|
|
argv,
|
|
|
|
mods=self.mods,
|
|
|
|
- wipe=True,
|
|
|
|
+ disable_wipe=True,
|
|
|
|
fsclient=self.fsclient,
|
|
|
|
minion_opts=self.minion_opts,
|
|
|
|
**self.kwargs
|
2018-04-30 14:06:44 +02:00
|
|
|
diff --git a/salt/config/__init__.py b/salt/config/__init__.py
|
2018-07-30 13:52:13 +02:00
|
|
|
index 289991771d..432364b201 100644
|
2018-04-30 14:06:44 +02:00
|
|
|
--- a/salt/config/__init__.py
|
|
|
|
+++ b/salt/config/__init__.py
|
2018-07-30 13:52:13 +02:00
|
|
|
@@ -1663,6 +1663,7 @@ DEFAULT_MASTER_OPTS = {
|
2018-04-30 14:06:44 +02:00
|
|
|
'state_top': 'top.sls',
|
|
|
|
'state_top_saltenv': None,
|
|
|
|
'master_tops': {},
|
|
|
|
+ 'master_tops_first': False,
|
|
|
|
'order_masters': False,
|
|
|
|
'job_cache': True,
|
|
|
|
'ext_job_cache': '',
|
|
|
|
diff --git a/salt/state.py b/salt/state.py
|
2018-07-30 13:52:13 +02:00
|
|
|
index 09709347b1..e7288bce2e 100644
|
2018-04-30 14:06:44 +02:00
|
|
|
--- a/salt/state.py
|
|
|
|
+++ b/salt/state.py
|
2018-07-30 13:52:13 +02:00
|
|
|
@@ -3383,7 +3383,7 @@ class BaseHighState(object):
|
2018-04-30 14:06:44 +02:00
|
|
|
ext_matches = self._master_tops()
|
|
|
|
for saltenv in ext_matches:
|
|
|
|
top_file_matches = matches.get(saltenv, [])
|
|
|
|
- if self.opts['master_tops_first']:
|
|
|
|
+ if self.opts.get('master_tops_first'):
|
|
|
|
first = ext_matches[saltenv]
|
|
|
|
second = top_file_matches
|
|
|
|
else:
|
|
|
|
diff --git a/salt/utils/hashutils.py b/salt/utils/hashutils.py
|
2018-07-30 13:52:13 +02:00
|
|
|
index b42a60d222..ee01be7377 100644
|
2018-04-30 14:06:44 +02:00
|
|
|
--- a/salt/utils/hashutils.py
|
|
|
|
+++ b/salt/utils/hashutils.py
|
|
|
|
@@ -9,6 +9,7 @@ import base64
|
|
|
|
import hashlib
|
|
|
|
import hmac
|
|
|
|
import random
|
|
|
|
+import os
|
|
|
|
|
|
|
|
# Import Salt libs
|
|
|
|
from salt.ext import six
|
2018-07-30 13:52:13 +02:00
|
|
|
@@ -150,3 +151,39 @@ def get_hash(path, form='sha256', chunk_size=65536):
|
2018-04-30 14:06:44 +02:00
|
|
|
for chunk in iter(lambda: ifile.read(chunk_size), b''):
|
|
|
|
hash_obj.update(chunk)
|
|
|
|
return hash_obj.hexdigest()
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+class DigestCollector(object):
|
|
|
|
+ '''
|
|
|
|
+ Class to collect digest of the file tree.
|
|
|
|
+ '''
|
|
|
|
+
|
|
|
|
+ def __init__(self, form='sha256', buff=0x10000):
|
|
|
|
+ '''
|
|
|
|
+ Constructor of the class.
|
|
|
|
+ :param form:
|
|
|
|
+ '''
|
|
|
|
+ self.__digest = hasattr(hashlib, form) and getattr(hashlib, form)() or None
|
|
|
|
+ if self.__digest is None:
|
|
|
|
+ raise ValueError('Invalid hash type: {0}'.format(form))
|
|
|
|
+ self.__buff = buff
|
|
|
|
+
|
|
|
|
+ def add(self, path):
|
|
|
|
+ '''
|
|
|
|
+ Update digest with the file content by path.
|
|
|
|
+
|
|
|
|
+ :param path:
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ with salt.utils.files.fopen(path, 'rb') as ifile:
|
|
|
|
+ for chunk in iter(lambda: ifile.read(self.__buff), b''):
|
|
|
|
+ self.__digest.update(chunk)
|
|
|
|
+
|
|
|
|
+ def digest(self):
|
|
|
|
+ '''
|
|
|
|
+ Get digest.
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+
|
|
|
|
+ return salt.utils.stringutils.to_str(self.__digest.hexdigest() + os.linesep)
|
2018-04-11 18:47:08 +02:00
|
|
|
diff --git a/salt/utils/thin.py b/salt/utils/thin.py
|
2018-04-30 14:06:44 +02:00
|
|
|
index 4c0969ea96..e4b878eb19 100644
|
2018-04-11 18:47:08 +02:00
|
|
|
--- a/salt/utils/thin.py
|
|
|
|
+++ b/salt/utils/thin.py
|
|
|
|
@@ -8,11 +8,14 @@ from __future__ import absolute_import, print_function, unicode_literals
|
|
|
|
|
|
|
|
import os
|
|
|
|
import sys
|
|
|
|
+import copy
|
|
|
|
import shutil
|
|
|
|
import tarfile
|
|
|
|
import zipfile
|
|
|
|
import tempfile
|
|
|
|
import subprocess
|
|
|
|
+import salt.utils.stringutils
|
|
|
|
+import logging
|
|
|
|
|
|
|
|
# Import third party libs
|
|
|
|
import jinja2
|
|
|
|
@@ -21,24 +24,26 @@ import msgpack
|
|
|
|
import salt.ext.six as _six
|
|
|
|
import tornado
|
|
|
|
|
|
|
|
+try:
|
|
|
|
+ import zlib
|
|
|
|
+except ImportError:
|
|
|
|
+ zlib = None
|
|
|
|
+
|
|
|
|
# pylint: disable=import-error,no-name-in-module
|
|
|
|
try:
|
|
|
|
import certifi
|
|
|
|
- HAS_CERTIFI = True
|
|
|
|
except ImportError:
|
|
|
|
- HAS_CERTIFI = False
|
|
|
|
+ certifi = None
|
|
|
|
|
|
|
|
try:
|
|
|
|
import singledispatch
|
|
|
|
- HAS_SINGLEDISPATCH = True
|
|
|
|
except ImportError:
|
|
|
|
- HAS_SINGLEDISPATCH = False
|
|
|
|
+ singledispatch = None
|
|
|
|
|
|
|
|
try:
|
|
|
|
import singledispatch_helpers
|
|
|
|
- HAS_SINGLEDISPATCH_HELPERS = True
|
|
|
|
except ImportError:
|
|
|
|
- HAS_SINGLEDISPATCH_HELPERS = False
|
|
|
|
+ singledispatch_helpers = None
|
|
|
|
|
|
|
|
try:
|
|
|
|
import backports_abc
|
|
|
|
@@ -46,25 +51,22 @@ except ImportError:
|
|
|
|
import salt.ext.backports_abc as backports_abc
|
|
|
|
|
|
|
|
try:
|
|
|
|
+ # New Jinja only
|
|
|
|
import markupsafe
|
|
|
|
- HAS_MARKUPSAFE = True
|
|
|
|
except ImportError:
|
|
|
|
- # Older jinja does not need markupsafe
|
|
|
|
- HAS_MARKUPSAFE = False
|
|
|
|
+ markupsafe = None
|
|
|
|
|
|
|
|
# pylint: enable=import-error,no-name-in-module
|
|
|
|
|
|
|
|
try:
|
|
|
|
# Older python where the backport from pypi is installed
|
|
|
|
from backports import ssl_match_hostname
|
|
|
|
- HAS_SSL_MATCH_HOSTNAME = True
|
|
|
|
except ImportError:
|
|
|
|
# Other older python we use our bundled copy
|
|
|
|
try:
|
|
|
|
from salt.ext import ssl_match_hostname
|
|
|
|
- HAS_SSL_MATCH_HOSTNAME = True
|
|
|
|
except ImportError:
|
|
|
|
- HAS_SSL_MATCH_HOSTNAME = False
|
|
|
|
+ ssl_match_hostname = None
|
|
|
|
|
|
|
|
# Import salt libs
|
|
|
|
import salt
|
|
|
|
@@ -76,22 +78,52 @@ import salt.utils.stringutils
|
|
|
|
import salt.exceptions
|
|
|
|
import salt.version
|
|
|
|
|
|
|
|
-SALTCALL = '''
|
|
|
|
+log = logging.getLogger(__name__)
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def _get_salt_call(*dirs, **namespaces):
|
|
|
|
+ '''
|
|
|
|
+ Return salt-call source, based on configuration.
|
|
|
|
+ This will include additional namespaces for another versions of Salt,
|
|
|
|
+ if needed (e.g. older interpreters etc).
|
|
|
|
+
|
|
|
|
+ :dirs: List of directories to include in the system path
|
|
|
|
+ :namespaces: Dictionary of namespace
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ template = '''# -*- coding: utf-8 -*-
|
|
|
|
import os
|
|
|
|
import sys
|
|
|
|
|
|
|
|
-sys.path.insert(
|
|
|
|
- 0,
|
|
|
|
- os.path.join(
|
|
|
|
- os.path.dirname(__file__),
|
|
|
|
- 'py{0[0]}'.format(sys.version_info)
|
|
|
|
- )
|
|
|
|
-)
|
|
|
|
+# Namespaces is a map: {namespace: major/minor version}, like {'2016.11.4': [2, 6]}
|
|
|
|
+# Appears only when configured in Master configuration.
|
|
|
|
+namespaces = %namespaces%
|
|
|
|
+
|
|
|
|
+# Default system paths alongside the namespaces
|
|
|
|
+syspaths = %dirs%
|
|
|
|
+syspaths.append('py{0}'.format(sys.version_info[0]))
|
|
|
|
+
|
|
|
|
+curr_ver = (sys.version_info[0], sys.version_info[1],)
|
|
|
|
+
|
|
|
|
+namespace = ''
|
|
|
|
+for ns in namespaces:
|
|
|
|
+ if curr_ver == tuple(namespaces[ns]):
|
|
|
|
+ namespace = ns
|
|
|
|
+ break
|
|
|
|
+
|
|
|
|
+for base in syspaths:
|
|
|
|
+ sys.path.insert(0, os.path.join(os.path.dirname(__file__),
|
|
|
|
+ namespace and os.path.join(namespace, base) or base))
|
|
|
|
|
|
|
|
-from salt.scripts import salt_call
|
|
|
|
if __name__ == '__main__':
|
|
|
|
+ from salt.scripts import salt_call
|
|
|
|
salt_call()
|
|
|
|
-'''.encode('utf-8')
|
|
|
|
+'''
|
|
|
|
+
|
|
|
|
+ for tgt, cnt in [('%dirs%', dirs), ('%namespaces%', namespaces)]:
|
|
|
|
+ template = template.replace(tgt, salt.utils.json.dumps(cnt))
|
|
|
|
+
|
|
|
|
+ return salt.utils.stringutils.to_bytes(template)
|
|
|
|
|
|
|
|
|
|
|
|
def thin_path(cachedir):
|
2018-04-30 14:06:44 +02:00
|
|
|
@@ -101,29 +133,137 @@ def thin_path(cachedir):
|
2018-04-11 18:47:08 +02:00
|
|
|
return os.path.join(cachedir, 'thin', 'thin.tgz')
|
|
|
|
|
|
|
|
|
|
|
|
-def get_tops(extra_mods='', so_mods=''):
|
|
|
|
- tops = [
|
|
|
|
- os.path.dirname(salt.__file__),
|
|
|
|
- os.path.dirname(jinja2.__file__),
|
|
|
|
- os.path.dirname(yaml.__file__),
|
|
|
|
- os.path.dirname(tornado.__file__),
|
|
|
|
- os.path.dirname(msgpack.__file__),
|
|
|
|
- ]
|
|
|
|
+def _is_shareable(mod):
|
|
|
|
+ '''
|
|
|
|
+ Return True if module is share-able between major Python versions.
|
|
|
|
+
|
|
|
|
+ :param mod:
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ # This list is subject to change
|
|
|
|
+ shareable = ['salt', 'jinja2',
|
|
|
|
+ 'msgpack', 'certifi']
|
|
|
|
+
|
|
|
|
+ return os.path.basename(mod) in shareable
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def _add_dependency(container, obj):
|
|
|
|
+ '''
|
|
|
|
+ Add a dependency to the top list.
|
2018-04-30 14:06:44 +02:00
|
|
|
|
|
|
|
- tops.append(_six.__file__.replace('.pyc', '.py'))
|
|
|
|
- tops.append(backports_abc.__file__.replace('.pyc', '.py'))
|
2018-04-11 18:47:08 +02:00
|
|
|
+ :param obj:
|
|
|
|
+ :param is_file:
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ if os.path.basename(obj.__file__).split('.')[0] == '__init__':
|
|
|
|
+ container.append(os.path.dirname(obj.__file__))
|
|
|
|
+ else:
|
|
|
|
+ container.append(obj.__file__.replace('.pyc', '.py'))
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def gte():
|
|
|
|
+ '''
|
|
|
|
+ This function is called externally from the alternative
|
|
|
|
+ Python interpreter from within _get_tops function.
|
2018-04-30 14:06:44 +02:00
|
|
|
|
|
|
|
- if HAS_CERTIFI:
|
|
|
|
- tops.append(os.path.dirname(certifi.__file__))
|
2018-04-11 18:47:08 +02:00
|
|
|
+ :param extra_mods:
|
|
|
|
+ :param so_mods:
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ extra = salt.utils.json.loads(sys.argv[1])
|
|
|
|
+ tops = get_tops(**extra)
|
2018-04-30 14:06:44 +02:00
|
|
|
|
|
|
|
- if HAS_SINGLEDISPATCH:
|
|
|
|
- tops.append(singledispatch.__file__.replace('.pyc', '.py'))
|
2018-04-11 18:47:08 +02:00
|
|
|
+ return salt.utils.json.dumps(tops, ensure_ascii=False)
|
2018-04-30 14:06:44 +02:00
|
|
|
|
|
|
|
- if HAS_SINGLEDISPATCH_HELPERS:
|
|
|
|
- tops.append(singledispatch_helpers.__file__.replace('.pyc', '.py'))
|
|
|
|
|
|
|
|
- if HAS_SSL_MATCH_HOSTNAME:
|
|
|
|
- tops.append(os.path.dirname(os.path.dirname(ssl_match_hostname.__file__)))
|
2018-04-11 18:47:08 +02:00
|
|
|
+def get_ext_tops(config):
|
|
|
|
+ '''
|
|
|
|
+ Get top directories for the dependencies, based on external configuration.
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
2018-04-30 14:06:44 +02:00
|
|
|
+ config = copy.deepcopy(config)
|
2018-04-11 18:47:08 +02:00
|
|
|
+ alternatives = {}
|
|
|
|
+ required = ['jinja2', 'yaml', 'tornado', 'msgpack']
|
|
|
|
+ tops = []
|
|
|
|
+ for ns, cfg in salt.ext.six.iteritems(config or {}):
|
|
|
|
+ alternatives[ns] = cfg
|
|
|
|
+ locked_py_version = cfg.get('py-version')
|
|
|
|
+ err_msg = None
|
|
|
|
+ if not locked_py_version:
|
|
|
|
+ err_msg = 'Alternative Salt library: missing specific locked Python version'
|
|
|
|
+ elif not isinstance(locked_py_version, (tuple, list)):
|
|
|
|
+ err_msg = ('Alternative Salt library: specific locked Python version '
|
|
|
|
+ 'should be a list of major/minor version')
|
|
|
|
+ if err_msg:
|
|
|
|
+ raise salt.exceptions.SaltSystemExit(err_msg)
|
|
|
|
+
|
|
|
|
+ if cfg.get('dependencies') == 'inherit':
|
|
|
|
+ # TODO: implement inheritance of the modules from _here_
|
|
|
|
+ raise NotImplementedError('This feature is not yet implemented')
|
|
|
|
+ else:
|
|
|
|
+ for dep in cfg.get('dependencies'):
|
|
|
|
+ mod = cfg['dependencies'][dep] or ''
|
|
|
|
+ if not mod:
|
|
|
|
+ log.warning('Module %s has missing configuration', dep)
|
|
|
|
+ continue
|
|
|
|
+ elif mod.endswith('.py') and not os.path.isfile(mod):
|
|
|
|
+ log.warning('Module %s configured with not a file or does not exist: %s', dep, mod)
|
|
|
|
+ continue
|
|
|
|
+ elif not mod.endswith('.py') and not os.path.isfile(os.path.join(mod, '__init__.py')):
|
|
|
|
+ log.warning('Module %s is not a Python importable module with %s', dep, mod)
|
|
|
|
+ continue
|
|
|
|
+ tops.append(mod)
|
|
|
|
+
|
|
|
|
+ if dep in required:
|
|
|
|
+ required.pop(required.index(dep))
|
|
|
|
+
|
|
|
|
+ required = ', '.join(required)
|
|
|
|
+ if required:
|
|
|
|
+ msg = 'Missing dependencies for the alternative version' \
|
|
|
|
+ ' in the external configuration: {}'.format(required)
|
|
|
|
+ log.error(msg)
|
|
|
|
+ raise salt.exceptions.SaltSystemExit(msg)
|
|
|
|
+ alternatives[ns]['dependencies'] = tops
|
|
|
|
+ return alternatives
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def _get_ext_namespaces(config):
|
|
|
|
+ '''
|
|
|
|
+ Get namespaces from the existing configuration.
|
2018-04-30 14:06:44 +02:00
|
|
|
+
|
2018-04-11 18:47:08 +02:00
|
|
|
+ :param config:
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ namespaces = {}
|
|
|
|
+ if not config:
|
|
|
|
+ return namespaces
|
|
|
|
+
|
|
|
|
+ for ns in config:
|
|
|
|
+ constraint_version = tuple(config[ns].get('py-version', []))
|
|
|
|
+ if not constraint_version:
|
|
|
|
+ raise salt.exceptions.SaltSystemExit("An alternative version is configured, but not defined "
|
|
|
|
+ "to what Python's major/minor version it should be constrained.")
|
|
|
|
+ else:
|
|
|
|
+ namespaces[ns] = constraint_version
|
2018-04-30 14:06:44 +02:00
|
|
|
+
|
2018-04-11 18:47:08 +02:00
|
|
|
+ return namespaces
|
2018-04-30 14:06:44 +02:00
|
|
|
+
|
|
|
|
+
|
2018-04-11 18:47:08 +02:00
|
|
|
+def get_tops(extra_mods='', so_mods=''):
|
|
|
|
+ '''
|
|
|
|
+ Get top directories for the dependencies, based on Python interpreter.
|
2018-04-30 14:06:44 +02:00
|
|
|
+
|
2018-04-11 18:47:08 +02:00
|
|
|
+ :param extra_mods:
|
|
|
|
+ :param so_mods:
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ tops = []
|
|
|
|
+ for mod in [salt, jinja2, yaml, tornado, msgpack, certifi, singledispatch,
|
|
|
|
+ singledispatch_helpers, ssl_match_hostname, markupsafe, backports_abc]:
|
|
|
|
+ if mod:
|
|
|
|
+ log.debug('Adding module to the tops: "%s"', mod.__name__)
|
|
|
|
+ _add_dependency(tops, mod)
|
|
|
|
|
|
|
|
for mod in [m for m in extra_mods.split(',') if m]:
|
|
|
|
if mod not in locals() and mod not in globals():
|
2018-04-30 14:06:44 +02:00
|
|
|
@@ -135,28 +275,49 @@ def get_tops(extra_mods='', so_mods=''):
|
2018-04-11 18:47:08 +02:00
|
|
|
tops.append(moddir)
|
|
|
|
else:
|
|
|
|
tops.append(os.path.join(moddir, base + '.py'))
|
|
|
|
- except ImportError:
|
|
|
|
- # Not entirely sure this is the right thing, but the only
|
|
|
|
- # options seem to be 1) fail, 2) spew errors, or 3) pass.
|
|
|
|
- # Nothing else in here spits errors, and the markupsafe code
|
|
|
|
- # doesn't bail on import failure, so I followed that lead.
|
|
|
|
- # And of course, any other failure still S/T's.
|
|
|
|
- pass
|
|
|
|
+ except ImportError as err:
|
|
|
|
+ log.exception(err)
|
|
|
|
+ log.error('Unable to import extra-module "%s"', mod)
|
|
|
|
+
|
|
|
|
for mod in [m for m in so_mods.split(',') if m]:
|
|
|
|
try:
|
|
|
|
locals()[mod] = __import__(mod)
|
|
|
|
tops.append(locals()[mod].__file__)
|
|
|
|
- except ImportError:
|
|
|
|
- pass # As per comment above
|
|
|
|
- if HAS_MARKUPSAFE:
|
|
|
|
- tops.append(os.path.dirname(markupsafe.__file__))
|
|
|
|
+ except ImportError as err:
|
|
|
|
+ log.exception(err)
|
|
|
|
+ log.error('Unable to import so-module "%s"', mod)
|
|
|
|
|
|
|
|
return tops
|
|
|
|
|
|
|
|
|
|
|
|
+def _get_supported_py_config(tops, extended_cfg):
|
|
|
|
+ '''
|
|
|
|
+ Based on the Salt SSH configuration, create a YAML configuration
|
|
|
|
+ for the supported Python interpreter versions. This is then written into the thin.tgz
|
|
|
|
+ archive and then verified by salt.client.ssh.ssh_py_shim.get_executable()
|
|
|
|
+
|
|
|
|
+ Note: Minimum default of 2.x versions is 2.7 and 3.x is 3.0, unless specified in namespaces.
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ pymap = []
|
|
|
|
+ for py_ver, tops in _six.iteritems(copy.deepcopy(tops)):
|
|
|
|
+ py_ver = int(py_ver)
|
|
|
|
+ if py_ver == 2:
|
|
|
|
+ pymap.append('py2:2:7')
|
|
|
|
+ elif py_ver == 3:
|
|
|
|
+ pymap.append('py3:3:0')
|
|
|
|
+
|
|
|
|
+ for ns, cfg in _six.iteritems(copy.deepcopy(extended_cfg) or {}):
|
|
|
|
+ pymap.append('{}:{}:{}'.format(ns, *cfg.get('py-version')))
|
|
|
|
+ pymap.append('')
|
|
|
|
+
|
|
|
|
+ return salt.utils.stringutils.to_bytes(os.linesep.join(pymap))
|
|
|
|
+
|
|
|
|
+
|
|
|
|
def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='',
|
|
|
|
python2_bin='python2', python3_bin='python3', absonly=True,
|
|
|
|
- compress='gzip'):
|
|
|
|
+ compress='gzip', extended_cfg=None):
|
|
|
|
'''
|
|
|
|
Generate the salt-thin tarball and print the location of the tarball
|
|
|
|
Optional additional mods to include (e.g. mako) can be supplied as a comma
|
2018-04-30 14:06:44 +02:00
|
|
|
@@ -171,19 +332,26 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='',
|
2018-04-11 18:47:08 +02:00
|
|
|
salt-run thin.generate mako,wempy 1
|
|
|
|
salt-run thin.generate overwrite=1
|
|
|
|
'''
|
|
|
|
+ if sys.version_info < (2, 6):
|
|
|
|
+ raise salt.exceptions.SaltSystemExit('The minimum required python version to run salt-ssh is "2.6".')
|
|
|
|
+ if compress not in ['gzip', 'zip']:
|
|
|
|
+ log.warning('Unknown compression type: "%s". Falling back to "gzip" compression.', compress)
|
|
|
|
+ compress = 'gzip'
|
|
|
|
+
|
|
|
|
thindir = os.path.join(cachedir, 'thin')
|
|
|
|
if not os.path.isdir(thindir):
|
|
|
|
os.makedirs(thindir)
|
|
|
|
- if compress == 'gzip':
|
|
|
|
- thin_ext = 'tgz'
|
|
|
|
- elif compress == 'zip':
|
|
|
|
- thin_ext = 'zip'
|
|
|
|
- thintar = os.path.join(thindir, 'thin.' + thin_ext)
|
|
|
|
+ thintar = os.path.join(thindir, 'thin.' + (compress == 'gzip' and 'tgz' or 'zip'))
|
|
|
|
thinver = os.path.join(thindir, 'version')
|
|
|
|
pythinver = os.path.join(thindir, '.thin-gen-py-version')
|
|
|
|
salt_call = os.path.join(thindir, 'salt-call')
|
|
|
|
+ pymap_cfg = os.path.join(thindir, 'supported-versions')
|
2018-04-30 14:06:44 +02:00
|
|
|
+ code_checksum = os.path.join(thindir, 'code-checksum')
|
|
|
|
+ digest_collector = salt.utils.hashutils.DigestCollector()
|
2018-04-11 18:47:08 +02:00
|
|
|
+
|
|
|
|
with salt.utils.files.fopen(salt_call, 'wb') as fp_:
|
|
|
|
- fp_.write(SALTCALL)
|
|
|
|
+ fp_.write(_get_salt_call('pyall', **_get_ext_namespaces(extended_cfg)))
|
|
|
|
+
|
|
|
|
if os.path.isfile(thintar):
|
|
|
|
if not overwrite:
|
|
|
|
if os.path.isfile(thinver):
|
2018-04-30 14:06:44 +02:00
|
|
|
@@ -197,85 +365,88 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='',
|
2018-04-11 18:47:08 +02:00
|
|
|
|
|
|
|
if overwrite:
|
|
|
|
try:
|
|
|
|
+ log.debug('Removing %s archive file', thintar)
|
|
|
|
os.remove(thintar)
|
|
|
|
- except OSError:
|
|
|
|
- pass
|
|
|
|
+ except OSError as exc:
|
|
|
|
+ log.error('Error while removing %s file: %s', thintar, exc)
|
|
|
|
+ if os.path.exists(thintar):
|
|
|
|
+ raise salt.exceptions.SaltSystemExit('Unable to remove %s file. See logs for details.', thintar)
|
|
|
|
else:
|
|
|
|
return thintar
|
|
|
|
if _six.PY3:
|
|
|
|
# Let's check for the minimum python 2 version requirement, 2.6
|
|
|
|
- py_shell_cmd = (
|
|
|
|
- python2_bin + ' -c \'from __future__ import print_function; import sys; '
|
|
|
|
- 'print("{0}.{1}".format(*(sys.version_info[:2])));\''
|
|
|
|
- )
|
|
|
|
+ py_shell_cmd = "{} -c 'import sys;sys.stdout.write(\"%s.%s\\n\" % sys.version_info[:2]);'".format(python2_bin)
|
|
|
|
cmd = subprocess.Popen(py_shell_cmd, stdout=subprocess.PIPE, shell=True)
|
|
|
|
stdout, _ = cmd.communicate()
|
|
|
|
if cmd.returncode == 0:
|
|
|
|
py2_version = tuple(int(n) for n in stdout.decode('utf-8').strip().split('.'))
|
|
|
|
if py2_version < (2, 6):
|
|
|
|
- # Bail!
|
|
|
|
raise salt.exceptions.SaltSystemExit(
|
|
|
|
'The minimum required python version to run salt-ssh is "2.6".'
|
|
|
|
'The version reported by "{0}" is "{1}". Please try "salt-ssh '
|
|
|
|
- '--python2-bin=<path-to-python-2.6-binary-or-higher>".'.format(python2_bin,
|
|
|
|
- stdout.strip())
|
|
|
|
- )
|
|
|
|
- elif sys.version_info < (2, 6):
|
|
|
|
- # Bail! Though, how did we reached this far in the first place.
|
|
|
|
- raise salt.exceptions.SaltSystemExit(
|
|
|
|
- 'The minimum required python version to run salt-ssh is "2.6".'
|
|
|
|
- )
|
|
|
|
+ '--python2-bin=<path-to-python-2.6-binary-or-higher>".'.format(python2_bin, stdout.strip()))
|
|
|
|
+ else:
|
|
|
|
+ log.error('Unable to detect Python-2 version')
|
|
|
|
+ log.debug(stdout)
|
|
|
|
|
|
|
|
+ tops_failure_msg = 'Failed %s tops for Python binary %s.'
|
|
|
|
tops_py_version_mapping = {}
|
|
|
|
tops = get_tops(extra_mods=extra_mods, so_mods=so_mods)
|
|
|
|
- if _six.PY2:
|
|
|
|
- tops_py_version_mapping['2'] = tops
|
|
|
|
- else:
|
|
|
|
- tops_py_version_mapping['3'] = tops
|
|
|
|
+ tops_py_version_mapping[sys.version_info.major] = tops
|
|
|
|
|
|
|
|
- # TODO: Consider putting known py2 and py3 compatible libs in it's own sharable directory.
|
|
|
|
- # This would reduce the thin size.
|
|
|
|
- if _six.PY2 and sys.version_info[0] == 2:
|
|
|
|
+ # Collect tops, alternative to 2.x version
|
|
|
|
+ if _six.PY2 and sys.version_info.major == 2:
|
|
|
|
# Get python 3 tops
|
|
|
|
- py_shell_cmd = (
|
|
|
|
- python3_bin + ' -c \'import sys; import json; import salt.utils.thin; '
|
|
|
|
- 'print(json.dumps(salt.utils.thin.get_tops(**(json.loads(sys.argv[1]))), ensure_ascii=False)); exit(0);\' '
|
|
|
|
- '\'{0}\''.format(salt.utils.json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods}))
|
|
|
|
- )
|
|
|
|
+ py_shell_cmd = "{0} -c 'import salt.utils.thin as t;print(t.gte())' '{1}'".format(
|
|
|
|
+ python3_bin, salt.utils.json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods}))
|
|
|
|
cmd = subprocess.Popen(py_shell_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
|
|
|
|
stdout, stderr = cmd.communicate()
|
|
|
|
if cmd.returncode == 0:
|
|
|
|
try:
|
|
|
|
tops = salt.utils.json.loads(stdout)
|
|
|
|
tops_py_version_mapping['3'] = tops
|
|
|
|
- except ValueError:
|
|
|
|
- pass
|
|
|
|
- if _six.PY3 and sys.version_info[0] == 3:
|
|
|
|
+ except ValueError as err:
|
|
|
|
+ log.error(tops_failure_msg, 'parsing', python3_bin)
|
|
|
|
+ log.exception(err)
|
|
|
|
+ else:
|
|
|
|
+ log.error(tops_failure_msg, 'collecting', python3_bin)
|
|
|
|
+ log.debug(stderr)
|
|
|
|
+
|
|
|
|
+ # Collect tops, alternative to 3.x version
|
|
|
|
+ if _six.PY3 and sys.version_info.major == 3:
|
|
|
|
# Get python 2 tops
|
|
|
|
- py_shell_cmd = (
|
|
|
|
- python2_bin + ' -c \'from __future__ import print_function; '
|
|
|
|
- 'import sys; import json; import salt.utils.thin; '
|
|
|
|
- 'print(json.dumps(salt.utils.thin.get_tops(**(json.loads(sys.argv[1]))), ensure_ascii=False)); exit(0);\' '
|
|
|
|
- '\'{0}\''.format(salt.utils.json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods}))
|
|
|
|
- )
|
|
|
|
+ py_shell_cmd = "{0} -c 'import salt.utils.thin as t;print(t.gte())' '{1}'".format(
|
|
|
|
+ python2_bin, salt.utils.json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods}))
|
|
|
|
cmd = subprocess.Popen(py_shell_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
|
|
|
|
stdout, stderr = cmd.communicate()
|
|
|
|
if cmd.returncode == 0:
|
|
|
|
try:
|
|
|
|
tops = salt.utils.json.loads(stdout.decode('utf-8'))
|
|
|
|
tops_py_version_mapping['2'] = tops
|
|
|
|
- except ValueError:
|
|
|
|
- pass
|
|
|
|
+ except ValueError as err:
|
|
|
|
+ log.error(tops_failure_msg, 'parsing', python2_bin)
|
|
|
|
+ log.exception(err)
|
|
|
|
+ else:
|
|
|
|
+ log.error(tops_failure_msg, 'collecting', python2_bin)
|
|
|
|
+ log.debug(stderr)
|
|
|
|
+
|
|
|
|
+ with salt.utils.files.fopen(pymap_cfg, 'wb') as fp_:
|
|
|
|
+ fp_.write(_get_supported_py_config(tops=tops_py_version_mapping, extended_cfg=extended_cfg))
|
|
|
|
|
|
|
|
if compress == 'gzip':
|
|
|
|
tfp = tarfile.open(thintar, 'w:gz', dereference=True)
|
|
|
|
elif compress == 'zip':
|
|
|
|
- tfp = zipfile.ZipFile(thintar, 'w')
|
|
|
|
+ tfp = zipfile.ZipFile(thintar, 'w', compression=zlib and zipfile.ZIP_DEFLATED or zipfile.ZIP_STORED)
|
|
|
|
+ tfp.add = tfp.write
|
|
|
|
+
|
|
|
|
try: # cwd may not exist if it was removed but salt was run from it
|
|
|
|
start_dir = os.getcwd()
|
|
|
|
except OSError:
|
|
|
|
start_dir = None
|
|
|
|
tempdir = None
|
|
|
|
+
|
|
|
|
+ # Pack default data
|
|
|
|
+ log.debug('Packing default libraries based on current Salt version')
|
|
|
|
for py_ver, tops in _six.iteritems(tops_py_version_mapping):
|
|
|
|
for top in tops:
|
|
|
|
if absonly and not os.path.isabs(top):
|
2018-04-30 14:06:44 +02:00
|
|
|
@@ -291,48 +462,80 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='',
|
2018-04-11 18:47:08 +02:00
|
|
|
egg.extractall(tempdir)
|
|
|
|
top = os.path.join(tempdir, base)
|
|
|
|
os.chdir(tempdir)
|
|
|
|
+
|
|
|
|
+ site_pkg_dir = _is_shareable(base) and 'pyall' or 'py{}'.format(py_ver)
|
|
|
|
+
|
|
|
|
+ log.debug('Packing "%s" to "%s" destination', base, site_pkg_dir)
|
|
|
|
if not os.path.isdir(top):
|
|
|
|
# top is a single file module
|
|
|
|
if os.path.exists(os.path.join(top_dirname, base)):
|
|
|
|
- if compress == 'gzip':
|
|
|
|
- tfp.add(base, arcname=os.path.join('py{0}'.format(py_ver), base))
|
|
|
|
- elif compress == 'zip':
|
|
|
|
- tfp.write(base, arcname=os.path.join('py{0}'.format(py_ver), base))
|
|
|
|
+ tfp.add(base, arcname=os.path.join(site_pkg_dir, base))
|
|
|
|
continue
|
|
|
|
for root, dirs, files in salt.utils.path.os_walk(base, followlinks=True):
|
|
|
|
for name in files:
|
|
|
|
if not name.endswith(('.pyc', '.pyo')):
|
|
|
|
- if compress == 'gzip':
|
|
|
|
- tfp.add(os.path.join(root, name),
|
|
|
|
- arcname=os.path.join('py{0}'.format(py_ver), root, name))
|
|
|
|
- elif compress == 'zip':
|
2018-04-30 14:06:44 +02:00
|
|
|
+ digest_collector.add(os.path.join(root, name))
|
2018-04-11 18:47:08 +02:00
|
|
|
+ arcname = os.path.join(site_pkg_dir, root, name)
|
|
|
|
+ if hasattr(tfp, 'getinfo'):
|
|
|
|
try:
|
|
|
|
# This is a little slow but there's no clear way to detect duplicates
|
|
|
|
- tfp.getinfo(os.path.join('py{0}'.format(py_ver), root, name))
|
|
|
|
+ tfp.getinfo(os.path.join(site_pkg_dir, root, name))
|
|
|
|
+ arcname = None
|
|
|
|
except KeyError:
|
|
|
|
- tfp.write(os.path.join(root, name), arcname=os.path.join('py{0}'.format(py_ver), root, name))
|
|
|
|
+ log.debug('ZIP: Unable to add "%s" with "getinfo"', arcname)
|
|
|
|
+ if arcname:
|
|
|
|
+ tfp.add(os.path.join(root, name), arcname=arcname)
|
|
|
|
+
|
|
|
|
if tempdir is not None:
|
|
|
|
shutil.rmtree(tempdir)
|
|
|
|
tempdir = None
|
|
|
|
+
|
|
|
|
+ # Pack alternative data
|
|
|
|
+ if extended_cfg:
|
|
|
|
+ log.debug('Packing libraries based on alternative Salt versions')
|
|
|
|
+ for ns, cfg in _six.iteritems(get_ext_tops(extended_cfg)):
|
|
|
|
+ tops = [cfg.get('path')] + cfg.get('dependencies')
|
|
|
|
+ py_ver_major, py_ver_minor = cfg.get('py-version')
|
|
|
|
+ for top in tops:
|
|
|
|
+ base, top_dirname = os.path.basename(top), os.path.dirname(top)
|
|
|
|
+ os.chdir(top_dirname)
|
|
|
|
+ site_pkg_dir = _is_shareable(base) and 'pyall' or 'py{0}'.format(py_ver_major)
|
|
|
|
+ log.debug('Packing alternative "%s" to "%s/%s" destination', base, ns, site_pkg_dir)
|
|
|
|
+ if not os.path.isdir(top):
|
|
|
|
+ # top is a single file module
|
|
|
|
+ if os.path.exists(os.path.join(top_dirname, base)):
|
|
|
|
+ tfp.add(base, arcname=os.path.join(ns, site_pkg_dir, base))
|
|
|
|
+ continue
|
|
|
|
+ for root, dirs, files in salt.utils.path.os_walk(base, followlinks=True):
|
|
|
|
+ for name in files:
|
|
|
|
+ if not name.endswith(('.pyc', '.pyo')):
|
2018-04-30 14:06:44 +02:00
|
|
|
+ digest_collector.add(os.path.join(root, name))
|
2018-04-11 18:47:08 +02:00
|
|
|
+ arcname = os.path.join(ns, site_pkg_dir, root, name)
|
|
|
|
+ if hasattr(tfp, 'getinfo'):
|
|
|
|
+ try:
|
|
|
|
+ tfp.getinfo(os.path.join(site_pkg_dir, root, name))
|
|
|
|
+ arcname = None
|
|
|
|
+ except KeyError:
|
|
|
|
+ log.debug('ZIP: Unable to add "%s" with "getinfo"', arcname)
|
|
|
|
+ if arcname:
|
|
|
|
+ tfp.add(os.path.join(root, name), arcname=arcname)
|
|
|
|
+
|
|
|
|
os.chdir(thindir)
|
|
|
|
- if compress == 'gzip':
|
|
|
|
- tfp.add('salt-call')
|
|
|
|
- elif compress == 'zip':
|
|
|
|
- tfp.write('salt-call')
|
|
|
|
with salt.utils.files.fopen(thinver, 'w+') as fp_:
|
|
|
|
fp_.write(salt.version.__version__)
|
|
|
|
with salt.utils.files.fopen(pythinver, 'w+') as fp_:
|
|
|
|
- fp_.write(str(sys.version_info[0])) # future lint: disable=blacklisted-function
|
|
|
|
+ fp_.write(str(sys.version_info.major)) # future lint: disable=blacklisted-function
|
2018-04-30 14:06:44 +02:00
|
|
|
+ with salt.utils.files.fopen(code_checksum, 'w+') as fp_:
|
|
|
|
+ fp_.write(digest_collector.digest())
|
2018-04-11 18:47:08 +02:00
|
|
|
os.chdir(os.path.dirname(thinver))
|
|
|
|
- if compress == 'gzip':
|
|
|
|
- tfp.add('version')
|
|
|
|
- tfp.add('.thin-gen-py-version')
|
|
|
|
- elif compress == 'zip':
|
|
|
|
- tfp.write('version')
|
|
|
|
- tfp.write('.thin-gen-py-version')
|
|
|
|
+
|
2018-04-30 14:06:44 +02:00
|
|
|
+ for fname in ['version', '.thin-gen-py-version', 'salt-call', 'supported-versions', 'code-checksum']:
|
2018-04-11 18:47:08 +02:00
|
|
|
+ tfp.add(fname)
|
|
|
|
+
|
|
|
|
if start_dir:
|
|
|
|
os.chdir(start_dir)
|
|
|
|
tfp.close()
|
|
|
|
+
|
|
|
|
return thintar
|
|
|
|
|
|
|
|
|
2018-04-30 14:06:44 +02:00
|
|
|
@@ -341,7 +544,14 @@ def thin_sum(cachedir, form='sha1'):
|
|
|
|
Return the checksum of the current thin tarball
|
|
|
|
'''
|
|
|
|
thintar = gen_thin(cachedir)
|
|
|
|
- return salt.utils.hashutils.get_hash(thintar, form)
|
|
|
|
+ code_checksum_path = os.path.join(cachedir, 'thin', 'code-checksum')
|
|
|
|
+ if os.path.isfile(code_checksum_path):
|
|
|
|
+ with salt.utils.fopen(code_checksum_path, 'r') as fh:
|
|
|
|
+ code_checksum = "'{0}'".format(fh.read().strip())
|
|
|
|
+ else:
|
|
|
|
+ code_checksum = "'0'"
|
|
|
|
+
|
|
|
|
+ return code_checksum, salt.utils.hashutils.get_hash(thintar, form)
|
|
|
|
|
|
|
|
|
|
|
|
def gen_min(cachedir, extra_mods='', overwrite=False, so_mods='',
|
|
|
|
@@ -368,7 +578,7 @@ def gen_min(cachedir, extra_mods='', overwrite=False, so_mods='',
|
2018-04-11 18:47:08 +02:00
|
|
|
pyminver = os.path.join(mindir, '.min-gen-py-version')
|
|
|
|
salt_call = os.path.join(mindir, 'salt-call')
|
|
|
|
with salt.utils.files.fopen(salt_call, 'wb') as fp_:
|
|
|
|
- fp_.write(SALTCALL)
|
|
|
|
+ fp_.write(_get_salt_call())
|
|
|
|
if os.path.isfile(mintar):
|
|
|
|
if not overwrite:
|
|
|
|
if os.path.isfile(minver):
|
|
|
|
diff --git a/tests/unit/utils/test_thin.py b/tests/unit/utils/test_thin.py
|
|
|
|
new file mode 100644
|
2018-04-30 14:06:44 +02:00
|
|
|
index 0000000000..549d48a703
|
2018-04-11 18:47:08 +02:00
|
|
|
--- /dev/null
|
|
|
|
+++ b/tests/unit/utils/test_thin.py
|
2018-04-30 14:06:44 +02:00
|
|
|
@@ -0,0 +1,612 @@
|
2018-04-11 18:47:08 +02:00
|
|
|
+# -*- coding: utf-8 -*-
|
|
|
|
+'''
|
|
|
|
+ :codeauthor: :email:`Bo Maryniuk <bo@suse.de>`
|
|
|
|
+'''
|
|
|
|
+from __future__ import absolute_import, print_function, unicode_literals
|
|
|
|
+
|
|
|
|
+import os
|
|
|
|
+import sys
|
|
|
|
+from tests.support.unit import TestCase, skipIf
|
|
|
|
+from tests.support.mock import (
|
|
|
|
+ NO_MOCK,
|
|
|
|
+ NO_MOCK_REASON,
|
|
|
|
+ MagicMock,
|
|
|
|
+ patch)
|
|
|
|
+
|
|
|
|
+import salt.exceptions
|
|
|
|
+from salt.utils import thin
|
|
|
|
+from salt.utils import json
|
|
|
|
+import salt.utils.stringutils
|
|
|
|
+from salt.utils.stringutils import to_bytes as bts
|
|
|
|
+
|
|
|
|
+try:
|
|
|
|
+ import pytest
|
|
|
|
+except ImportError:
|
|
|
|
+ pytest = None
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+@skipIf(NO_MOCK, NO_MOCK_REASON)
|
|
|
|
+@skipIf(pytest is None, 'PyTest is missing')
|
|
|
|
+class SSHThinTestCase(TestCase):
|
|
|
|
+ '''
|
|
|
|
+ TestCase for SaltSSH-related parts.
|
|
|
|
+ '''
|
|
|
|
+ def _popen(self, return_value=None, side_effect=None, returncode=0):
|
|
|
|
+ '''
|
|
|
|
+ Fake subprocess.Popen
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+
|
|
|
|
+ proc = MagicMock()
|
|
|
|
+ proc.communicate = MagicMock(return_value=return_value, side_effect=side_effect)
|
|
|
|
+ proc.returncode = returncode
|
|
|
|
+ popen = MagicMock(return_value=proc)
|
|
|
|
+
|
|
|
|
+ return popen
|
|
|
|
+
|
|
|
|
+ def _version_info(self, major=None, minor=None):
|
|
|
|
+ '''
|
|
|
|
+ Fake version info.
|
|
|
|
+
|
|
|
|
+ :param major:
|
|
|
|
+ :param minor:
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ class VersionInfo(tuple):
|
|
|
|
+ pass
|
|
|
|
+
|
|
|
|
+ vi = VersionInfo([major, minor])
|
|
|
|
+ vi.major = major or sys.version_info.major
|
|
|
|
+ vi.minor = minor or sys.version_info.minor
|
|
|
|
+
|
|
|
|
+ return vi
|
|
|
|
+
|
|
|
|
+ def _tarfile(self, getinfo=False):
|
|
|
|
+ '''
|
|
|
|
+ Fake tarfile handler.
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ spec = ['add', 'close']
|
|
|
|
+ if getinfo:
|
|
|
|
+ spec.append('getinfo')
|
|
|
|
+
|
|
|
|
+ tf = MagicMock()
|
|
|
|
+ tf.open = MagicMock(return_value=MagicMock(spec=spec))
|
|
|
|
+
|
|
|
|
+ return tf
|
|
|
|
+
|
|
|
|
+ @patch('salt.exceptions.SaltSystemExit', Exception)
|
|
|
|
+ @patch('salt.utils.thin.log', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.path.isfile', MagicMock(return_value=False))
|
|
|
|
+ def test_get_ext_tops_cfg_missing_dependencies(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.get_ext_tops contains all required dependencies.
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ cfg = {'namespace': {'py-version': [0, 0], 'path': '/foo', 'dependencies': []}}
|
|
|
|
+
|
|
|
|
+ with pytest.raises(Exception) as err:
|
|
|
|
+ thin.get_ext_tops(cfg)
|
|
|
|
+ assert 'Missing dependencies' in str(err)
|
|
|
|
+ assert thin.log.error.called
|
|
|
|
+ assert 'Missing dependencies' in thin.log.error.call_args[0][0]
|
|
|
|
+ assert 'jinja2, yaml, tornado, msgpack' in thin.log.error.call_args[0][0]
|
|
|
|
+
|
|
|
|
+ @patch('salt.exceptions.SaltSystemExit', Exception)
|
|
|
|
+ @patch('salt.utils.thin.log', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.path.isfile', MagicMock(return_value=False))
|
|
|
|
+ def test_get_ext_tops_cfg_missing_interpreter(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.get_ext_tops contains interpreter configuration.
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ cfg = {'namespace': {'path': '/foo',
|
|
|
|
+ 'dependencies': []}}
|
|
|
|
+ with pytest.raises(salt.exceptions.SaltSystemExit) as err:
|
|
|
|
+ thin.get_ext_tops(cfg)
|
|
|
|
+ assert 'missing specific locked Python version' in str(err)
|
|
|
|
+
|
|
|
|
+ @patch('salt.exceptions.SaltSystemExit', Exception)
|
|
|
|
+ @patch('salt.utils.thin.log', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.path.isfile', MagicMock(return_value=False))
|
|
|
|
+ def test_get_ext_tops_cfg_wrong_interpreter(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.get_ext_tops contains correct interpreter configuration.
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ cfg = {'namespace': {'path': '/foo',
|
|
|
|
+ 'py-version': 2,
|
|
|
|
+ 'dependencies': []}}
|
|
|
|
+
|
|
|
|
+ with pytest.raises(salt.exceptions.SaltSystemExit) as err:
|
|
|
|
+ thin.get_ext_tops(cfg)
|
|
|
|
+ assert 'specific locked Python version should be a list of major/minor version' in str(err)
|
|
|
|
+
|
|
|
|
+ @patch('salt.exceptions.SaltSystemExit', Exception)
|
|
|
|
+ @patch('salt.utils.thin.log', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.path.isfile', MagicMock(return_value=False))
|
|
|
|
+ def test_get_ext_tops_cfg_interpreter(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.get_ext_tops interpreter configuration.
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ cfg = {'namespace': {'path': '/foo',
|
|
|
|
+ 'py-version': [2, 6],
|
|
|
|
+ 'dependencies': {'jinja2': '',
|
|
|
|
+ 'yaml': '',
|
|
|
|
+ 'tornado': '',
|
|
|
|
+ 'msgpack': ''}}}
|
|
|
|
+
|
|
|
|
+ with pytest.raises(salt.exceptions.SaltSystemExit):
|
|
|
|
+ thin.get_ext_tops(cfg)
|
|
|
|
+ assert len(thin.log.warning.mock_calls) == 4
|
|
|
|
+ assert sorted([x[1][1] for x in thin.log.warning.mock_calls]) == ['jinja2', 'msgpack', 'tornado', 'yaml']
|
|
|
|
+ assert 'Module test has missing configuration' == thin.log.warning.mock_calls[0][1][0] % 'test'
|
|
|
|
+
|
|
|
|
+ @patch('salt.exceptions.SaltSystemExit', Exception)
|
|
|
|
+ @patch('salt.utils.thin.log', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.path.isfile', MagicMock(return_value=False))
|
|
|
|
+ def test_get_ext_tops_dependency_config_check(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.get_ext_tops dependencies are importable
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ cfg = {'namespace': {'path': '/foo',
|
|
|
|
+ 'py-version': [2, 6],
|
|
|
|
+ 'dependencies': {'jinja2': '/jinja/foo.py',
|
|
|
|
+ 'yaml': '/yaml/',
|
|
|
|
+ 'tornado': '/tornado/wrong.rb',
|
|
|
|
+ 'msgpack': 'msgpack.sh'}}}
|
|
|
|
+
|
|
|
|
+ with pytest.raises(salt.exceptions.SaltSystemExit) as err:
|
|
|
|
+ thin.get_ext_tops(cfg)
|
|
|
|
+ assert 'Missing dependencies for the alternative version in the external configuration' in str(err)
|
|
|
|
+
|
|
|
|
+ messages = {}
|
|
|
|
+ for cl in thin.log.warning.mock_calls:
|
|
|
|
+ messages[cl[1][1]] = cl[1][0] % (cl[1][1], cl[1][2])
|
|
|
|
+ for mod in ['tornado', 'yaml', 'msgpack']:
|
|
|
|
+ assert 'not a Python importable module' in messages[mod]
|
|
|
|
+ assert 'configured with not a file or does not exist' in messages['jinja2']
|
|
|
|
+
|
|
|
|
+ @patch('salt.exceptions.SaltSystemExit', Exception)
|
|
|
|
+ @patch('salt.utils.thin.log', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.path.isfile', MagicMock(return_value=True))
|
|
|
|
+ def test_get_ext_tops_config_pass(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.get_ext_tops configuration
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ cfg = {'namespace': {'path': '/foo',
|
|
|
|
+ 'py-version': [2, 6],
|
|
|
|
+ 'dependencies': {'jinja2': '/jinja/foo.py',
|
|
|
|
+ 'yaml': '/yaml/',
|
|
|
|
+ 'tornado': '/tornado/tornado.py',
|
|
|
|
+ 'msgpack': 'msgpack.py'}}}
|
2018-04-30 14:06:44 +02:00
|
|
|
+ out = thin.get_ext_tops(cfg)
|
|
|
|
+ assert out['namespace']['py-version'] == cfg['namespace']['py-version']
|
|
|
|
+ assert out['namespace']['path'] == cfg['namespace']['path']
|
|
|
|
+ assert sorted(out['namespace']['dependencies']) == sorted(['/tornado/tornado.py',
|
|
|
|
+ '/jinja/foo.py', '/yaml/', 'msgpack.py'])
|
2018-04-11 18:47:08 +02:00
|
|
|
+
|
|
|
|
+ @patch('salt.utils.thin.sys.argv', [None, '{"foo": "bar"}'])
|
|
|
|
+ @patch('salt.utils.thin.get_tops', lambda **kw: kw)
|
|
|
|
+ def test_gte(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.gte external call for processing the info about tops per interpreter.
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ assert json.loads(thin.gte()).get('foo') == 'bar'
|
|
|
|
+
|
|
|
|
+ def test_add_dep_path(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin._add_dependency function to setup dependency paths
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ container = []
|
|
|
|
+ for pth in ['/foo/bar.py', '/something/else/__init__.py']:
|
|
|
|
+ thin._add_dependency(container, type(str('obj'), (), {'__file__': pth})())
|
|
|
|
+ assert '__init__' not in container[1]
|
|
|
|
+ assert container == ['/foo/bar.py', '/something/else']
|
|
|
|
+
|
|
|
|
+ def test_thin_path(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.thin_path returns the expected path.
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ assert thin.thin_path('/path/to') == '/path/to/thin/thin.tgz'
|
|
|
|
+
|
|
|
|
+ def test_get_salt_call_script(self):
|
|
|
|
+ '''
|
|
|
|
+ Test get salt-call script rendered.
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ out = thin._get_salt_call('foo', 'bar', py26=[2, 6], py27=[2, 7], py34=[3, 4])
|
|
|
|
+ for line in salt.utils.stringutils.to_str(out).split(os.linesep):
|
|
|
|
+ if line.startswith('namespaces = {'):
|
|
|
|
+ data = json.loads(line.replace('namespaces = ', '').strip())
|
|
|
|
+ assert data.get('py26') == [2, 6]
|
|
|
|
+ assert data.get('py27') == [2, 7]
|
|
|
|
+ assert data.get('py34') == [3, 4]
|
|
|
|
+ if line.startswith('syspaths = '):
|
|
|
|
+ data = json.loads(line.replace('syspaths = ', ''))
|
|
|
|
+ assert data == ['foo', 'bar']
|
|
|
|
+
|
|
|
|
+ def test_get_ext_namespaces_empty(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin._get_ext_namespaces function returns an empty dictionary on nothing
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ for obj in [None, {}, []]:
|
|
|
|
+ assert thin._get_ext_namespaces(obj) == {}
|
|
|
|
+
|
|
|
|
+ def test_get_ext_namespaces(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin._get_ext_namespaces function returns namespaces properly out of the config.
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ cfg = {'ns': {'py-version': [2, 7]}}
|
|
|
|
+ assert thin._get_ext_namespaces(cfg).get('ns') == (2, 7,)
|
|
|
|
+ assert isinstance(thin._get_ext_namespaces(cfg).get('ns'), tuple)
|
|
|
|
+
|
|
|
|
+ def test_get_ext_namespaces_failure(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin._get_ext_namespaces function raises an exception
|
|
|
|
+ if python major/minor version is not configured.
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ with pytest.raises(salt.exceptions.SaltSystemExit):
|
|
|
|
+ thin._get_ext_namespaces({'ns': {}})
|
|
|
|
+
|
|
|
|
+ @patch('salt.utils.thin.salt', type(str('salt'), (), {'__file__': '/site-packages/salt'}))
|
|
|
|
+ @patch('salt.utils.thin.jinja2', type(str('jinja2'), (), {'__file__': '/site-packages/jinja2'}))
|
|
|
|
+ @patch('salt.utils.thin.yaml', type(str('yaml'), (), {'__file__': '/site-packages/yaml'}))
|
|
|
|
+ @patch('salt.utils.thin.tornado', type(str('tornado'), (), {'__file__': '/site-packages/tornado'}))
|
|
|
|
+ @patch('salt.utils.thin.msgpack', type(str('msgpack'), (), {'__file__': '/site-packages/msgpack'}))
|
|
|
|
+ @patch('salt.utils.thin.certifi', type(str('certifi'), (), {'__file__': '/site-packages/certifi'}))
|
|
|
|
+ @patch('salt.utils.thin.singledispatch', type(str('singledispatch'), (), {'__file__': '/site-packages/sdp'}))
|
|
|
|
+ @patch('salt.utils.thin.singledispatch_helpers', type(str('singledispatch_helpers'), (), {'__file__': '/site-packages/sdp_hlp'}))
|
|
|
|
+ @patch('salt.utils.thin.ssl_match_hostname', type(str('ssl_match_hostname'), (), {'__file__': '/site-packages/ssl_mh'}))
|
|
|
|
+ @patch('salt.utils.thin.markupsafe', type(str('markupsafe'), (), {'__file__': '/site-packages/markupsafe'}))
|
|
|
|
+ @patch('salt.utils.thin.backports_abc', type(str('backports_abc'), (), {'__file__': '/site-packages/backports_abc'}))
|
|
|
|
+ @patch('salt.utils.thin.log', MagicMock())
|
|
|
|
+ def test_get_tops(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.get_tops to get top directories, based on the interpreter.
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ base_tops = ['/site-packages/salt', '/site-packages/jinja2', '/site-packages/yaml',
|
|
|
|
+ '/site-packages/tornado', '/site-packages/msgpack', '/site-packages/certifi',
|
|
|
|
+ '/site-packages/sdp', '/site-packages/sdp_hlp', '/site-packages/ssl_mh',
|
|
|
|
+ '/site-packages/markupsafe', '/site-packages/backports_abc']
|
|
|
|
+
|
|
|
|
+ tops = thin.get_tops()
|
|
|
|
+ assert len(tops) == len(base_tops)
|
|
|
|
+ assert sorted(tops) == sorted(base_tops)
|
|
|
|
+
|
|
|
|
+ @patch('salt.utils.thin.salt', type(str('salt'), (), {'__file__': '/site-packages/salt'}))
|
|
|
|
+ @patch('salt.utils.thin.jinja2', type(str('jinja2'), (), {'__file__': '/site-packages/jinja2'}))
|
|
|
|
+ @patch('salt.utils.thin.yaml', type(str('yaml'), (), {'__file__': '/site-packages/yaml'}))
|
|
|
|
+ @patch('salt.utils.thin.tornado', type(str('tornado'), (), {'__file__': '/site-packages/tornado'}))
|
|
|
|
+ @patch('salt.utils.thin.msgpack', type(str('msgpack'), (), {'__file__': '/site-packages/msgpack'}))
|
|
|
|
+ @patch('salt.utils.thin.certifi', type(str('certifi'), (), {'__file__': '/site-packages/certifi'}))
|
|
|
|
+ @patch('salt.utils.thin.singledispatch', type(str('singledispatch'), (), {'__file__': '/site-packages/sdp'}))
|
|
|
|
+ @patch('salt.utils.thin.singledispatch_helpers', type(str('singledispatch_helpers'), (), {'__file__': '/site-packages/sdp_hlp'}))
|
|
|
|
+ @patch('salt.utils.thin.ssl_match_hostname', type(str('ssl_match_hostname'), (), {'__file__': '/site-packages/ssl_mh'}))
|
|
|
|
+ @patch('salt.utils.thin.markupsafe', type(str('markupsafe'), (), {'__file__': '/site-packages/markupsafe'}))
|
|
|
|
+ @patch('salt.utils.thin.backports_abc', type(str('backports_abc'), (), {'__file__': '/site-packages/backports_abc'}))
|
|
|
|
+ @patch('salt.utils.thin.log', MagicMock())
|
|
|
|
+ def test_get_tops_extra_mods(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.get_tops to get extra-modules alongside the top directories, based on the interpreter.
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ base_tops = ['/site-packages/salt', '/site-packages/jinja2', '/site-packages/yaml',
|
|
|
|
+ '/site-packages/tornado', '/site-packages/msgpack', '/site-packages/certifi',
|
|
|
|
+ '/site-packages/sdp', '/site-packages/sdp_hlp', '/site-packages/ssl_mh',
|
|
|
|
+ '/site-packages/markupsafe', '/site-packages/backports_abc', '/custom/foo', '/custom/bar.py']
|
|
|
|
+ builtins = sys.version_info.major == 3 and 'builtins' or '__builtin__'
|
|
|
|
+ with patch('{}.__import__'.format(builtins),
|
|
|
|
+ MagicMock(side_effect=[type(str('foo'), (), {'__file__': '/custom/foo/__init__.py'}),
|
|
|
|
+ type(str('bar'), (), {'__file__': '/custom/bar'})])):
|
|
|
|
+ tops = thin.get_tops(extra_mods='foo,bar')
|
|
|
|
+ assert len(tops) == len(base_tops)
|
|
|
|
+ assert sorted(tops) == sorted(base_tops)
|
|
|
|
+
|
|
|
|
+ @patch('salt.utils.thin.salt', type(str('salt'), (), {'__file__': '/site-packages/salt'}))
|
|
|
|
+ @patch('salt.utils.thin.jinja2', type(str('jinja2'), (), {'__file__': '/site-packages/jinja2'}))
|
|
|
|
+ @patch('salt.utils.thin.yaml', type(str('yaml'), (), {'__file__': '/site-packages/yaml'}))
|
|
|
|
+ @patch('salt.utils.thin.tornado', type(str('tornado'), (), {'__file__': '/site-packages/tornado'}))
|
|
|
|
+ @patch('salt.utils.thin.msgpack', type(str('msgpack'), (), {'__file__': '/site-packages/msgpack'}))
|
|
|
|
+ @patch('salt.utils.thin.certifi', type(str('certifi'), (), {'__file__': '/site-packages/certifi'}))
|
|
|
|
+ @patch('salt.utils.thin.singledispatch', type(str('singledispatch'), (), {'__file__': '/site-packages/sdp'}))
|
|
|
|
+ @patch('salt.utils.thin.singledispatch_helpers', type(str('singledispatch_helpers'), (), {'__file__': '/site-packages/sdp_hlp'}))
|
|
|
|
+ @patch('salt.utils.thin.ssl_match_hostname', type(str('ssl_match_hostname'), (), {'__file__': '/site-packages/ssl_mh'}))
|
|
|
|
+ @patch('salt.utils.thin.markupsafe', type(str('markupsafe'), (), {'__file__': '/site-packages/markupsafe'}))
|
|
|
|
+ @patch('salt.utils.thin.backports_abc', type(str('backports_abc'), (), {'__file__': '/site-packages/backports_abc'}))
|
|
|
|
+ @patch('salt.utils.thin.log', MagicMock())
|
|
|
|
+ def test_get_tops_so_mods(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.get_tops to get extra-modules alongside the top directories, based on the interpreter.
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ base_tops = ['/site-packages/salt', '/site-packages/jinja2', '/site-packages/yaml',
|
|
|
|
+ '/site-packages/tornado', '/site-packages/msgpack', '/site-packages/certifi',
|
|
|
|
+ '/site-packages/sdp', '/site-packages/sdp_hlp', '/site-packages/ssl_mh',
|
|
|
|
+ '/site-packages/markupsafe', '/site-packages/backports_abc', '/custom/foo.so', '/custom/bar.so']
|
|
|
|
+ builtins = sys.version_info.major == 3 and 'builtins' or '__builtin__'
|
|
|
|
+ with patch('{}.__import__'.format(builtins),
|
|
|
|
+ MagicMock(side_effect=[type(str('salt'), (), {'__file__': '/custom/foo.so'}),
|
|
|
|
+ type(str('salt'), (), {'__file__': '/custom/bar.so'})])):
|
|
|
|
+ tops = thin.get_tops(so_mods='foo,bar')
|
|
|
|
+ assert len(tops) == len(base_tops)
|
|
|
|
+ assert sorted(tops) == sorted(base_tops)
|
|
|
|
+
|
|
|
|
+ @patch('salt.utils.thin.gen_thin', MagicMock(return_value='/path/to/thin/thin.tgz'))
|
|
|
|
+ @patch('salt.utils.hashutils.get_hash', MagicMock(return_value=12345))
|
|
|
|
+ def test_thin_sum(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.thin_sum function.
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
2018-04-30 14:06:44 +02:00
|
|
|
+ assert thin.thin_sum('/cachedir', form='sha256')[1] == 12345
|
2018-04-11 18:47:08 +02:00
|
|
|
+ thin.salt.utils.hashutils.get_hash.assert_called()
|
|
|
|
+ assert thin.salt.utils.hashutils.get_hash.call_count == 1
|
|
|
|
+
|
|
|
|
+ path, form = thin.salt.utils.hashutils.get_hash.call_args[0]
|
|
|
|
+ assert path == '/path/to/thin/thin.tgz'
|
|
|
|
+ assert form == 'sha256'
|
|
|
|
+
|
|
|
|
+ @patch('salt.utils.thin.gen_min', MagicMock(return_value='/path/to/thin/min.tgz'))
|
|
|
|
+ @patch('salt.utils.hashutils.get_hash', MagicMock(return_value=12345))
|
|
|
|
+ def test_min_sum(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.thin_sum function.
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ assert thin.min_sum('/cachedir', form='sha256') == 12345
|
|
|
|
+ thin.salt.utils.hashutils.get_hash.assert_called()
|
|
|
|
+ assert thin.salt.utils.hashutils.get_hash.call_count == 1
|
|
|
|
+
|
|
|
|
+ path, form = thin.salt.utils.hashutils.get_hash.call_args[0]
|
|
|
|
+ assert path == '/path/to/thin/min.tgz'
|
|
|
|
+ assert form == 'sha256'
|
|
|
|
+
|
|
|
|
+ @patch('salt.utils.thin.sys.version_info', (2, 5))
|
|
|
|
+ @patch('salt.exceptions.SaltSystemExit', Exception)
|
|
|
|
+ def test_gen_thin_fails_ancient_python_version(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.gen_thin function raises an exception
|
|
|
|
+ if Python major/minor version is lower than 2.6
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ with pytest.raises(salt.exceptions.SaltSystemExit) as err:
|
|
|
|
+ thin.sys.exc_clear = lambda: None
|
|
|
|
+ thin.gen_thin('')
|
|
|
|
+ assert 'The minimum required python version to run salt-ssh is "2.6"' in str(err)
|
|
|
|
+
|
|
|
|
+ @patch('salt.exceptions.SaltSystemExit', Exception)
|
|
|
|
+ @patch('salt.utils.thin.log', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.makedirs', MagicMock())
|
|
|
|
+ @patch('salt.utils.files.fopen', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin._get_salt_call', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin._get_ext_namespaces', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.get_tops', MagicMock(return_value=['/foo3', '/bar3']))
|
|
|
|
+ @patch('salt.utils.thin.get_ext_tops', MagicMock(return_value={}))
|
|
|
|
+ @patch('salt.utils.thin.os.path.isfile', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.path.isdir', MagicMock(return_value=True))
|
|
|
|
+ @patch('salt.utils.thin.log', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.remove', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.path.exists', MagicMock())
|
|
|
|
+ @patch('salt.utils.path.os_walk', MagicMock(return_value=[]))
|
|
|
|
+ @patch('salt.utils.thin.subprocess.Popen',
|
|
|
|
+ _popen(None, side_effect=[(bts('2.7'), bts('')), (bts('["/foo27", "/bar27"]'), bts(''))]))
|
|
|
|
+ @patch('salt.utils.thin.tarfile', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.zipfile', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.getcwd', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.chdir', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.tempfile', MagicMock(mkdtemp=MagicMock(return_value='')))
|
|
|
|
+ @patch('salt.utils.thin.shutil', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin._six.PY3', True)
|
|
|
|
+ @patch('salt.utils.thin._six.PY2', False)
|
|
|
|
+ @patch('salt.utils.thin.sys.version_info', _version_info(None, 3, 6))
|
|
|
|
+ def test_gen_thin_compression_fallback_py3(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.gen_thin function if fallbacks to the gzip compression, once setup wrong.
|
|
|
|
+ NOTE: Py2 version of this test is not required, as code shares the same spot across the versions.
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ thin.gen_thin('', compress='arj')
|
|
|
|
+ thin.log.warning.assert_called()
|
|
|
|
+ pt, msg = thin.log.warning.mock_calls[0][1]
|
|
|
|
+ assert pt % msg == 'Unknown compression type: "arj". Falling back to "gzip" compression.'
|
|
|
|
+ thin.zipfile.ZipFile.assert_not_called()
|
|
|
|
+ thin.tarfile.open.assert_called()
|
|
|
|
+
|
|
|
|
+ @patch('salt.exceptions.SaltSystemExit', Exception)
|
|
|
|
+ @patch('salt.utils.thin.log', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.makedirs', MagicMock())
|
|
|
|
+ @patch('salt.utils.files.fopen', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin._get_salt_call', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin._get_ext_namespaces', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.get_tops', MagicMock(return_value=['/foo3', '/bar3']))
|
|
|
|
+ @patch('salt.utils.thin.get_ext_tops', MagicMock(return_value={}))
|
|
|
|
+ @patch('salt.utils.thin.os.path.isfile', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.path.isdir', MagicMock(return_value=False))
|
|
|
|
+ @patch('salt.utils.thin.log', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.remove', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.path.exists', MagicMock())
|
|
|
|
+ @patch('salt.utils.path.os_walk', MagicMock(return_value=[]))
|
|
|
|
+ @patch('salt.utils.thin.subprocess.Popen',
|
|
|
|
+ _popen(None, side_effect=[(bts('2.7'), bts('')), (bts('["/foo27", "/bar27"]'), bts(''))]))
|
|
|
|
+ @patch('salt.utils.thin.tarfile', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.zipfile', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.getcwd', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.chdir', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.tempfile', MagicMock(mkdtemp=MagicMock(return_value='')))
|
|
|
|
+ @patch('salt.utils.thin.shutil', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin._six.PY3', True)
|
|
|
|
+ @patch('salt.utils.thin._six.PY2', False)
|
|
|
|
+ @patch('salt.utils.thin.sys.version_info', _version_info(None, 3, 6))
|
|
|
|
+ def test_gen_thin_control_files_written_py3(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.gen_thin function if control files are written (version, salt-call etc).
|
|
|
|
+ NOTE: Py2 version of this test is not required, as code shares the same spot across the versions.
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ thin.gen_thin('')
|
|
|
|
+ arc_name, arc_mode = thin.tarfile.method_calls[0][1]
|
|
|
|
+ assert arc_name == 'thin/thin.tgz'
|
|
|
|
+ assert arc_mode == 'w:gz'
|
|
|
|
+ for idx, fname in enumerate(['version', '.thin-gen-py-version', 'salt-call', 'supported-versions']):
|
|
|
|
+ assert thin.tarfile.open().method_calls[idx + 4][1][0] == fname
|
|
|
|
+ thin.tarfile.open().close.assert_called()
|
|
|
|
+
|
|
|
|
+ @patch('salt.exceptions.SaltSystemExit', Exception)
|
|
|
|
+ @patch('salt.utils.thin.log', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.makedirs', MagicMock())
|
|
|
|
+ @patch('salt.utils.files.fopen', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin._get_salt_call', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin._get_ext_namespaces', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.get_tops', MagicMock(return_value=['/salt', '/bar3']))
|
|
|
|
+ @patch('salt.utils.thin.get_ext_tops', MagicMock(return_value={}))
|
|
|
|
+ @patch('salt.utils.thin.os.path.isfile', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.path.isdir', MagicMock(return_value=True))
|
|
|
|
+ @patch('salt.utils.thin.log', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.remove', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.path.exists', MagicMock())
|
|
|
|
+ @patch('salt.utils.path.os_walk',
|
|
|
|
+ MagicMock(return_value=(('root', [], ['r1', 'r2', 'r3']), ('root2', [], ['r4', 'r5', 'r6']))))
|
|
|
|
+ @patch('salt.utils.thin.subprocess.Popen',
|
|
|
|
+ _popen(None, side_effect=[(bts('2.7'), bts('')), (bts('["/foo27", "/bar27"]'), bts(''))]))
|
|
|
|
+ @patch('salt.utils.thin.tarfile', _tarfile(None))
|
|
|
|
+ @patch('salt.utils.thin.zipfile', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.getcwd', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.chdir', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.tempfile', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.shutil', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin._six.PY3', True)
|
|
|
|
+ @patch('salt.utils.thin._six.PY2', False)
|
|
|
|
+ @patch('salt.utils.thin.sys.version_info', _version_info(None, 3, 6))
|
2018-04-30 14:06:44 +02:00
|
|
|
+ @patch('salt.utils.hashutils.DigestCollector', MagicMock())
|
2018-04-11 18:47:08 +02:00
|
|
|
+ def test_gen_thin_main_content_files_written_py3(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.gen_thin function if main content files are written.
|
|
|
|
+ NOTE: Py2 version of this test is not required, as code shares the same spot across the versions.
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ thin.gen_thin('')
|
|
|
|
+ files = [
|
|
|
|
+ 'py2/root/r1', 'py2/root/r2', 'py2/root/r3', 'py2/root2/r4', 'py2/root2/r5', 'py2/root2/r6',
|
|
|
|
+ 'py2/root/r1', 'py2/root/r2', 'py2/root/r3', 'py2/root2/r4', 'py2/root2/r5', 'py2/root2/r6',
|
|
|
|
+ 'py3/root/r1', 'py3/root/r2', 'py3/root/r3', 'py3/root2/r4', 'py3/root2/r5', 'py3/root2/r6',
|
|
|
|
+ 'pyall/root/r1', 'pyall/root/r2', 'pyall/root/r3', 'pyall/root2/r4', 'pyall/root2/r5', 'pyall/root2/r6'
|
|
|
|
+ ]
|
2018-04-30 14:06:44 +02:00
|
|
|
+ for cl in thin.tarfile.open().method_calls[:-6]:
|
2018-04-11 18:47:08 +02:00
|
|
|
+ arcname = cl[2].get('arcname')
|
|
|
|
+ assert arcname in files
|
|
|
|
+ files.pop(files.index(arcname))
|
|
|
|
+ assert not bool(files)
|
|
|
|
+
|
|
|
|
+ @patch('salt.exceptions.SaltSystemExit', Exception)
|
|
|
|
+ @patch('salt.utils.thin.log', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.makedirs', MagicMock())
|
|
|
|
+ @patch('salt.utils.files.fopen', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin._get_salt_call', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin._get_ext_namespaces', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.get_tops', MagicMock(return_value=[]))
|
|
|
|
+ @patch('salt.utils.thin.get_ext_tops',
|
|
|
|
+ MagicMock(return_value={'namespace': {'py-version': [2, 7],
|
|
|
|
+ 'path': '/opt/2015.8/salt',
|
|
|
|
+ 'dependencies': ['/opt/certifi', '/opt/whatever']}}))
|
|
|
|
+ @patch('salt.utils.thin.os.path.isfile', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.path.isdir', MagicMock(return_value=True))
|
|
|
|
+ @patch('salt.utils.thin.log', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.remove', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.path.exists', MagicMock())
|
|
|
|
+ @patch('salt.utils.path.os_walk',
|
|
|
|
+ MagicMock(return_value=(('root', [], ['r1', 'r2', 'r3']), ('root2', [], ['r4', 'r5', 'r6']))))
|
|
|
|
+ @patch('salt.utils.thin.subprocess.Popen',
|
|
|
|
+ _popen(None, side_effect=[(bts('2.7'), bts('')), (bts('["/foo27", "/bar27"]'), bts(''))]))
|
|
|
|
+ @patch('salt.utils.thin.tarfile', _tarfile(None))
|
|
|
|
+ @patch('salt.utils.thin.zipfile', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.getcwd', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.os.chdir', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin.tempfile', MagicMock(mkdtemp=MagicMock(return_value='')))
|
|
|
|
+ @patch('salt.utils.thin.shutil', MagicMock())
|
|
|
|
+ @patch('salt.utils.thin._six.PY3', True)
|
|
|
|
+ @patch('salt.utils.thin._six.PY2', False)
|
|
|
|
+ @patch('salt.utils.thin.sys.version_info', _version_info(None, 3, 6))
|
2018-04-30 14:06:44 +02:00
|
|
|
+ @patch('salt.utils.hashutils.DigestCollector', MagicMock())
|
2018-04-11 18:47:08 +02:00
|
|
|
+ def test_gen_thin_ext_alternative_content_files_written_py3(self):
|
|
|
|
+ '''
|
|
|
|
+ Test thin.gen_thin function if external alternative content files are written.
|
|
|
|
+ NOTE: Py2 version of this test is not required, as code shares the same spot across the versions.
|
|
|
|
+
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ thin.gen_thin('')
|
|
|
|
+ files = ['namespace/pyall/root/r1', 'namespace/pyall/root/r2', 'namespace/pyall/root/r3',
|
|
|
|
+ 'namespace/pyall/root2/r4', 'namespace/pyall/root2/r5', 'namespace/pyall/root2/r6',
|
|
|
|
+ 'namespace/pyall/root/r1', 'namespace/pyall/root/r2', 'namespace/pyall/root/r3',
|
|
|
|
+ 'namespace/pyall/root2/r4', 'namespace/pyall/root2/r5', 'namespace/pyall/root2/r6',
|
|
|
|
+ 'namespace/py2/root/r1', 'namespace/py2/root/r2', 'namespace/py2/root/r3',
|
|
|
|
+ 'namespace/py2/root2/r4', 'namespace/py2/root2/r5', 'namespace/py2/root2/r6'
|
|
|
|
+ ]
|
2018-04-30 14:06:44 +02:00
|
|
|
+ for idx, cl in enumerate(thin.tarfile.open().method_calls[12:-6]):
|
2018-04-11 18:47:08 +02:00
|
|
|
+ arcname = cl[2].get('arcname')
|
|
|
|
+ assert arcname in files
|
|
|
|
+ files.pop(files.index(arcname))
|
|
|
|
+ assert not bool(files)
|
|
|
|
+
|
|
|
|
+ def test_get_supported_py_config_typecheck(self):
|
|
|
|
+ '''
|
|
|
|
+ Test collecting proper py-versions. Should return bytes type.
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ tops = {}
|
|
|
|
+ ext_cfg = {}
|
|
|
|
+ out = thin._get_supported_py_config(tops=tops, extended_cfg=ext_cfg)
|
|
|
|
+ assert type(salt.utils.stringutils.to_bytes('')) == type(out)
|
|
|
|
+
|
|
|
|
+ def test_get_supported_py_config_base_tops(self):
|
|
|
|
+ '''
|
|
|
|
+ Test collecting proper py-versions. Should return proper base tops.
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ tops = {'3': ['/groundkeepers', '/stole'], '2': ['/the-root', '/password']}
|
|
|
|
+ ext_cfg = {}
|
|
|
|
+ out = salt.utils.stringutils.to_str(thin._get_supported_py_config(
|
|
|
|
+ tops=tops, extended_cfg=ext_cfg)).strip().split('\n')
|
|
|
|
+ assert len(out) == 2
|
|
|
|
+ for t_line in ['py3:3:0', 'py2:2:7']:
|
|
|
|
+ assert t_line in out
|
|
|
|
+
|
|
|
|
+ def test_get_supported_py_config_ext_tops(self):
|
|
|
|
+ '''
|
|
|
|
+ Test collecting proper py-versions. Should return proper ext conf tops.
|
|
|
|
+ :return:
|
|
|
|
+ '''
|
|
|
|
+ tops = {}
|
|
|
|
+ ext_cfg = {'solar-interference': {'py-version': [2, 6]}, 'second-system-effect': {'py-version': [2, 7]}}
|
|
|
|
+ out = salt.utils.stringutils.to_str(thin._get_supported_py_config(
|
|
|
|
+ tops=tops, extended_cfg=ext_cfg)).strip().split('\n')
|
|
|
|
+ for t_line in ['second-system-effect:2:7', 'solar-interference:2:6']:
|
|
|
|
+ assert t_line in out
|
|
|
|
--
|
2018-07-30 13:52:13 +02:00
|
|
|
2.13.7
|
2018-04-11 18:47:08 +02:00
|
|
|
|
|
|
|
|