diff --git a/_lastrevision b/_lastrevision index e229c4b..047b3de 100644 --- a/_lastrevision +++ b/_lastrevision @@ -1 +1 @@ -ebc77d067d9fa300bdc5bb5dcccaa09e1787f688 \ No newline at end of file +2460cb78e6bda580f2567781e060a3e6c6ba25de \ No newline at end of file diff --git a/add-cpe_name-for-osversion-grain-parsing-u-49946.patch b/add-cpe_name-for-osversion-grain-parsing-u-49946.patch new file mode 100644 index 0000000..8b8826a --- /dev/null +++ b/add-cpe_name-for-osversion-grain-parsing-u-49946.patch @@ -0,0 +1,194 @@ +From 3bad9e211c2e76ddac48f7c8ff1632e32e0a256e Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Tue, 9 Oct 2018 14:08:50 +0200 +Subject: [PATCH] Add CPE_NAME for osversion* grain parsing (U#49946) + +Remove unnecessary linebreak + +Override VERSION_ID from os-release, if CPE_NAME is given + +Add unit test for WFN format of CPE_NAME + +Add unit test for v2.3 of CPE format + +Add unit test for broken CPE_NAME + +Prevent possible crash if CPE_NAME is wrongly written in the distro + +Add part parsing + +Keep CPE_NAME only for opensuse series + +Remove linebreak + +Expand unit test to verify part name + +Fix proper part name in the string-bound CPE +--- + salt/grains/core.py | 43 +++++++++++++++++++++--- + tests/unit/grains/test_core.py | 60 +++++++++++++++++++++++++++++----- + 2 files changed, 90 insertions(+), 13 deletions(-) + +diff --git a/salt/grains/core.py b/salt/grains/core.py +index 80eebd1c05..e41ab4e0ae 100644 +--- a/salt/grains/core.py ++++ b/salt/grains/core.py +@@ -1355,6 +1355,34 @@ def _parse_os_release(os_release_files): + return data + + ++def _parse_cpe_name(cpe): ++ ''' ++ Parse CPE_NAME data from the os-release ++ ++ Info: https://csrc.nist.gov/projects/security-content-automation-protocol/scap-specifications/cpe ++ ++ :param cpe: ++ :return: ++ ''' ++ part = { ++ 'o': 'operating system', ++ 'h': 'hardware', ++ 'a': 'application', ++ } ++ ret = {} ++ cpe = (cpe or '').split(':') ++ if len(cpe) > 4 and cpe[0] == 'cpe': ++ if cpe[1].startswith('/'): # WFN to URI ++ ret['vendor'], ret['product'], ret['version'] = cpe[2:5] ++ ret['phase'] = cpe[5] if len(cpe) > 5 else None ++ ret['part'] = part.get(cpe[1][1:]) ++ elif len(cpe) == 13 and cpe[1] == '2.3': # WFN to a string ++ ret['vendor'], ret['product'], ret['version'], ret['phase'] = [x if x != '*' else None for x in cpe[3:7]] ++ ret['part'] = part.get(cpe[2]) ++ ++ return ret ++ ++ + def os_data(): + ''' + Return grains pertaining to the operating system +@@ -1554,13 +1582,20 @@ def os_data(): + codename = codename_match.group(1) + grains['lsb_distrib_codename'] = codename + if 'CPE_NAME' in os_release: +- if ":suse:" in os_release['CPE_NAME'] or ":opensuse:" in os_release['CPE_NAME']: ++ cpe = _parse_cpe_name(os_release['CPE_NAME']) ++ if not cpe: ++ log.error('Broken CPE_NAME format in /etc/os-release!') ++ elif cpe.get('vendor', '').lower() in ['suse', 'opensuse']: + grains['os'] = "SUSE" + # openSUSE `osfullname` grain normalization + if os_release.get("NAME") == "openSUSE Leap": + grains['osfullname'] = "Leap" + elif os_release.get("VERSION") == "Tumbleweed": + grains['osfullname'] = os_release["VERSION"] ++ # Override VERSION_ID, if CPE_NAME around ++ if cpe.get('version') and cpe.get('vendor') == 'opensuse': # Keep VERSION_ID for SLES ++ grains['lsb_distrib_release'] = cpe['version'] ++ + elif os.path.isfile('/etc/SuSE-release'): + grains['lsb_distrib_id'] = 'SUSE' + version = '' +@@ -1666,8 +1701,7 @@ def os_data(): + # Commit introducing this comment should be reverted after the upstream bug is released. + if 'CentOS Linux 7' in grains.get('lsb_distrib_codename', ''): + grains.pop('lsb_distrib_release', None) +- grains['osrelease'] = \ +- grains.get('lsb_distrib_release', osrelease).strip() ++ grains['osrelease'] = grains.get('lsb_distrib_release', osrelease).strip() + grains['oscodename'] = grains.get('lsb_distrib_codename', '').strip() or oscodename + if 'Red Hat' in grains['oscodename']: + grains['oscodename'] = oscodename +@@ -1702,8 +1736,7 @@ def os_data(): + r'((?:Open|Oracle )?Solaris|OpenIndiana|OmniOS) (Development)?' + r'\s*(\d+\.?\d*|v\d+)\s?[A-Z]*\s?(r\d+|\d+\/\d+|oi_\S+|snv_\S+)?' + ) +- osname, development, osmajorrelease, osminorrelease = \ +- release_re.search(rel_data).groups() ++ osname, development, osmajorrelease, osminorrelease = release_re.search(rel_data).groups() + except AttributeError: + # Set a blank osrelease grain and fallback to 'Solaris' + # as the 'os' grain. +diff --git a/tests/unit/grains/test_core.py b/tests/unit/grains/test_core.py +index e973428add..2ab32ef41b 100644 +--- a/tests/unit/grains/test_core.py ++++ b/tests/unit/grains/test_core.py +@@ -62,10 +62,11 @@ class CoreGrainsTestCase(TestCase, LoaderModuleMockMixin): + def test_parse_etc_os_release(self, path_isfile_mock): + path_isfile_mock.side_effect = lambda x: x == "/usr/lib/os-release" + with salt.utils.files.fopen(os.path.join(OS_RELEASE_DIR, "ubuntu-17.10")) as os_release_file: +- os_release_content = os_release_file.readlines() +- with patch("salt.utils.files.fopen", mock_open()) as os_release_file: +- os_release_file.return_value.__iter__.return_value = os_release_content +- os_release = core._parse_os_release(["/etc/os-release", "/usr/lib/os-release"]) ++ os_release_content = os_release_file.read() ++ with patch("salt.utils.files.fopen", mock_open(read_data=os_release_content)): ++ os_release = core._parse_os_release( ++ '/etc/os-release', ++ '/usr/lib/os-release') + self.assertEqual(os_release, { + "NAME": "Ubuntu", + "VERSION": "17.10 (Artful Aardvark)", +@@ -81,10 +82,53 @@ class CoreGrainsTestCase(TestCase, LoaderModuleMockMixin): + "UBUNTU_CODENAME": "artful", + }) + +- @patch("os.path.isfile") +- def test_missing_os_release(self, path_isfile_mock): +- path_isfile_mock.return_value = False +- os_release = core._parse_os_release(["/etc/os-release", "/usr/lib/os-release"]) ++ def test_parse_cpe_name_wfn(self): ++ ''' ++ Parse correct CPE_NAME data WFN formatted ++ :return: ++ ''' ++ for cpe, cpe_ret in [('cpe:/o:opensuse:leap:15.0', ++ {'phase': None, 'version': '15.0', 'product': 'leap', ++ 'vendor': 'opensuse', 'part': 'operating system'}), ++ ('cpe:/o:vendor:product:42:beta', ++ {'phase': 'beta', 'version': '42', 'product': 'product', ++ 'vendor': 'vendor', 'part': 'operating system'})]: ++ ret = core._parse_cpe_name(cpe) ++ for key in cpe_ret: ++ assert key in ret ++ assert cpe_ret[key] == ret[key] ++ ++ def test_parse_cpe_name_v23(self): ++ ''' ++ Parse correct CPE_NAME data v2.3 formatted ++ :return: ++ ''' ++ for cpe, cpe_ret in [('cpe:2.3:o:microsoft:windows_xp:5.1.601:beta:*:*:*:*:*:*', ++ {'phase': 'beta', 'version': '5.1.601', 'product': 'windows_xp', ++ 'vendor': 'microsoft', 'part': 'operating system'}), ++ ('cpe:2.3:h:corellian:millenium_falcon:1.0:*:*:*:*:*:*:*', ++ {'phase': None, 'version': '1.0', 'product': 'millenium_falcon', ++ 'vendor': 'corellian', 'part': 'hardware'}), ++ ('cpe:2.3:*:dark_empire:light_saber:3.0:beta:*:*:*:*:*:*', ++ {'phase': 'beta', 'version': '3.0', 'product': 'light_saber', ++ 'vendor': 'dark_empire', 'part': None})]: ++ ret = core._parse_cpe_name(cpe) ++ for key in cpe_ret: ++ assert key in ret ++ assert cpe_ret[key] == ret[key] ++ ++ def test_parse_cpe_name_broken(self): ++ ''' ++ Parse broken CPE_NAME data ++ :return: ++ ''' ++ for cpe in ['cpe:broken', 'cpe:broken:in:all:ways:*:*:*:*', ++ 'cpe:x:still:broken:123', 'who:/knows:what:is:here']: ++ assert core._parse_cpe_name(cpe) == {} ++ ++ def test_missing_os_release(self): ++ with patch('salt.utils.files.fopen', mock_open(read_data={})): ++ os_release = core._parse_os_release('/etc/os-release', '/usr/lib/os-release') + self.assertEqual(os_release, {}) + + @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') +-- +2.19.0 + + diff --git a/add-hold-unhold-functions.patch b/add-hold-unhold-functions.patch new file mode 100644 index 0000000..b1def6b --- /dev/null +++ b/add-hold-unhold-functions.patch @@ -0,0 +1,144 @@ +From ba5171ce35b733a1f7997b4ea038998802b67298 Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Thu, 6 Dec 2018 16:26:23 +0100 +Subject: [PATCH] Add hold/unhold functions + +Add unhold function + +Add warnings +--- + salt/modules/zypper.py | 88 +++++++++++++++++++++++++++++++++++++++++- + 1 file changed, 87 insertions(+), 1 deletion(-) + +diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py +index 6845e44ab6..773354b2f3 100644 +--- a/salt/modules/zypper.py ++++ b/salt/modules/zypper.py +@@ -41,6 +41,7 @@ import salt.utils.pkg + import salt.utils.pkg.rpm + import salt.utils.stringutils + import salt.utils.systemd ++import salt.utils.versions + from salt.utils.versions import LooseVersion + from salt.exceptions import CommandExecutionError, MinionError, SaltInvocationError + +@@ -1738,7 +1739,7 @@ def clean_locks(): + return out + + +-def remove_lock(packages, **kwargs): # pylint: disable=unused-argument ++def unhold(name=None, pkgs=None, **kwargs): + ''' + Remove specified package lock. + +@@ -1750,7 +1751,47 @@ def remove_lock(packages, **kwargs): # pylint: disable=unused-argument + salt '*' pkg.remove_lock ,, + salt '*' pkg.remove_lock pkgs='["foo", "bar"]' + ''' ++ ret = {} ++ if (not name and not pkgs) or (name and pkgs): ++ raise CommandExecutionError('Name or packages must be specified.') ++ elif name: ++ pkgs = [name] ++ ++ locks = list_locks() ++ try: ++ pkgs = list(__salt__['pkg_resource.parse_targets'](pkgs)[0].keys()) ++ except MinionError as exc: ++ raise CommandExecutionError(exc) ++ ++ removed = [] ++ missing = [] ++ for pkg in pkgs: ++ if locks.get(pkg): ++ removed.append(pkg) ++ ret[pkg]['comment'] = 'Package {0} is no longer held.'.format(pkg) ++ else: ++ missing.append(pkg) ++ ret[pkg]['comment'] = 'Package {0} unable to be unheld.'.format(pkg) ++ ++ if removed: ++ __zypper__.call('rl', *removed) ++ ++ return ret ++ ++ ++def remove_lock(packages, **kwargs): # pylint: disable=unused-argument ++ ''' ++ Remove specified package lock. ++ ++ CLI Example: ++ ++ .. code-block:: bash + ++ salt '*' pkg.remove_lock ++ salt '*' pkg.remove_lock ,, ++ salt '*' pkg.remove_lock pkgs='["foo", "bar"]' ++ ''' ++ salt.utils.versions.warn_until('Sodium', 'This function is deprecated. Please use unhold() instead.') + locks = list_locks() + try: + packages = list(__salt__['pkg_resource.parse_targets'](packages)[0].keys()) +@@ -1771,6 +1812,50 @@ def remove_lock(packages, **kwargs): # pylint: disable=unused-argument + return {'removed': len(removed), 'not_found': missing} + + ++def hold(name=None, pkgs=None, **kwargs): ++ ''' ++ Add a package lock. Specify packages to lock by exact name. ++ ++ CLI Example: ++ ++ .. code-block:: bash ++ ++ salt '*' pkg.add_lock ++ salt '*' pkg.add_lock ,, ++ salt '*' pkg.add_lock pkgs='["foo", "bar"]' ++ ++ :param name: ++ :param pkgs: ++ :param kwargs: ++ :return: ++ ''' ++ ret = {} ++ if (not name and not pkgs) or (name and pkgs): ++ raise CommandExecutionError('Name or packages must be specified.') ++ elif name: ++ pkgs = [name] ++ ++ locks = list_locks() ++ added = [] ++ try: ++ pkgs = list(__salt__['pkg_resource.parse_targets'](pkgs)[0].keys()) ++ except MinionError as exc: ++ raise CommandExecutionError(exc) ++ ++ for pkg in pkgs: ++ ret[pkg] = {'name': pkg, 'changes': {}, 'result': False, 'comment': ''} ++ if not locks.get(pkg): ++ added.append(pkg) ++ ret[pkg]['comment'] = 'Package {0} is now being held.'.format(pkg) ++ else: ++ ret[pkg]['comment'] = 'Package {0} is already set to be held.'.format(pkg) ++ ++ if added: ++ __zypper__.call('al', *added) ++ ++ return ret ++ ++ + def add_lock(packages, **kwargs): # pylint: disable=unused-argument + ''' + Add a package lock. Specify packages to lock by exact name. +@@ -1783,6 +1868,7 @@ def add_lock(packages, **kwargs): # pylint: disable=unused-argument + salt '*' pkg.add_lock ,, + salt '*' pkg.add_lock pkgs='["foo", "bar"]' + ''' ++ salt.utils.versions.warn_until('Sodium', 'This function is deprecated. Please use hold() instead.') + locks = list_locks() + added = [] + try: +-- +2.20.1 + + diff --git a/add-multi-file-support-and-globbing-to-the-filetree-.patch b/add-multi-file-support-and-globbing-to-the-filetree-.patch new file mode 100644 index 0000000..b8a3dda --- /dev/null +++ b/add-multi-file-support-and-globbing-to-the-filetree-.patch @@ -0,0 +1,116 @@ +From 671bb9d48e120c806ca1f6f176b0ada43b1e7594 Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Fri, 12 Oct 2018 16:20:40 +0200 +Subject: [PATCH] Add multi-file support and globbing to the filetree + (U#50018) + +Add more possible logs + +Support multiple files grabbing + +Collect system logs and boot logs + +Support globbing in filetree +--- + salt/cli/support/intfunc.py | 49 ++++++++++++++++----------- + salt/cli/support/profiles/default.yml | 7 ++++ + 2 files changed, 37 insertions(+), 19 deletions(-) + +diff --git a/salt/cli/support/intfunc.py b/salt/cli/support/intfunc.py +index 2727cd6394..f15f4d4097 100644 +--- a/salt/cli/support/intfunc.py ++++ b/salt/cli/support/intfunc.py +@@ -6,6 +6,7 @@ Internal functions. + + from __future__ import absolute_import, print_function, unicode_literals + import os ++import glob + from salt.cli.support.console import MessagesOutput + import salt.utils.files + +@@ -13,7 +14,7 @@ import salt.utils.files + out = MessagesOutput() + + +-def filetree(collector, path): ++def filetree(collector, *paths): + ''' + Add all files in the tree. If the "path" is a file, + only that file will be added. +@@ -21,22 +22,32 @@ def filetree(collector, path): + :param path: File or directory + :return: + ''' +- if not path: +- out.error('Path not defined', ident=2) +- else: +- # The filehandler needs to be explicitly passed here, so PyLint needs to accept that. +- # pylint: disable=W8470 +- if os.path.isfile(path): +- filename = os.path.basename(path) +- try: +- file_ref = salt.utils.files.fopen(path) # pylint: disable=W +- out.put('Add {}'.format(filename), indent=2) +- collector.add(filename) +- collector.link(title=path, path=file_ref) +- except Exception as err: +- out.error(err, ident=4) +- # pylint: enable=W8470 ++ _paths = [] ++ # Unglob ++ for path in paths: ++ _paths += glob.glob(path) ++ for path in set(_paths): ++ if not path: ++ out.error('Path not defined', ident=2) ++ elif not os.path.exists(path): ++ out.warning('Path {} does not exists'.format(path)) + else: +- for fname in os.listdir(path): +- fname = os.path.join(path, fname) +- filetree(collector, fname) ++ # The filehandler needs to be explicitly passed here, so PyLint needs to accept that. ++ # pylint: disable=W8470 ++ if os.path.isfile(path): ++ filename = os.path.basename(path) ++ try: ++ file_ref = salt.utils.files.fopen(path) # pylint: disable=W ++ out.put('Add {}'.format(filename), indent=2) ++ collector.add(filename) ++ collector.link(title=path, path=file_ref) ++ except Exception as err: ++ out.error(err, ident=4) ++ # pylint: enable=W8470 ++ else: ++ try: ++ for fname in os.listdir(path): ++ fname = os.path.join(path, fname) ++ filetree(collector, [fname]) ++ except Exception as err: ++ out.error(err, ident=4) +diff --git a/salt/cli/support/profiles/default.yml b/salt/cli/support/profiles/default.yml +index 01d9a26193..3defb5eef3 100644 +--- a/salt/cli/support/profiles/default.yml ++++ b/salt/cli/support/profiles/default.yml +@@ -62,10 +62,17 @@ general-health: + - ps.top: + info: Top CPU consuming processes + ++boot_log: ++ - filetree: ++ info: Collect boot logs ++ args: ++ - /var/log/boot.* ++ + system.log: + # This works on any file system object. + - filetree: + info: Add system log + args: + - /var/log/syslog ++ - /var/log/messages + +-- +2.19.0 + + diff --git a/add-supportconfig-module-for-remote-calls-and-saltss.patch b/add-supportconfig-module-for-remote-calls-and-saltss.patch new file mode 100644 index 0000000..38c1d15 --- /dev/null +++ b/add-supportconfig-module-for-remote-calls-and-saltss.patch @@ -0,0 +1,1405 @@ +From 2bb024d871acaf5726eeb6e89fb83785605b4c83 Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Fri, 19 Oct 2018 15:44:47 +0200 +Subject: [PATCH] Add supportconfig module for remote calls and SaltSSH + +Add log collector for remote purposes + +Implement default archive name + +Fix imports + +Implement runner function + +Remove targets data collector function as it is now called by a module instead + +Add external method decorator marker + +Add utility class for detecting exportable methods + +Mark run method as an external function + +Implement function setter + +Fix imports + +Setup config from __opts__ + +Use utility class + +Remove utils class + +Allow specify profile from the API parameter directly + +Rename module by virtual name + +Bypass parent subclass + +Implement profiles listing (local only for now) + +Specify profile from the state/call + +Set default or personalised archive name + +Add archives lister + +Add personalised name element to the archive name + +Use proper args/kwargs to the exported function + +Add archives deletion function + +Change log level when debugging rendered profiles + +Add ability to directly pass profile source when taking local data + +Add pillar profile support + +Remove extra-line + +Fix header + +Change output format for deleting archives + +Refactor logger output format + +Add time/milliseconds to each log notification + +Fix imports + +Switch output destination by context + +Add last archive function + +Lintfix + +Return consistent type + +Change output format for deleted archives report + +Implement report archive syncing to the reporting node + +Send multiple files at once via rsync, instead of send one after another + +Add sync stats formatter + +Change signature: cleanup -> move. Update docstring. + +Flush empty data from the output format + +Report archfiles activity + +Refactor imports + +Do not remove retcode if it is EX_OK + +Do not raise rsync error for undefined archives. + +Update header + +Add salt-support state module + +Move all functions into a callable class object + +Support __call__ function in state and command modules as default entrance that does not need to be specified in SLS state syntax + +Access from the outside only allowed class methods + +Pre-create destination of the archive, preventing single archive copied as a group name + +Handle functions exceptions + +Add unit test scaffold + +Add LogCollector UT for testing regular message + +Add LogCollector UT for testing INFO message + +Add LogCollector UT for testing WARNING message + +Replace hardcoded variables with defined constants + +Add LogCollector UT for testing ERROR message + +Test title attribute in msg method of LogCollector + +Add UT for LogCollector on highlighter method + +Add UT for LogCollector on put method + +Fix docstrings + +Add UT for archive name generator + +Add UT for custom archive name + +Fix docstring for the UT + +Add UT for checking profiles list format + +Add Unit Test for existing archives listing + +Add UT for the last archive function + +Create instance of the support class + +Add UT for successfully deleting all archives + +Add UT for deleting archives with failures + +Add UI for formatting sync stats and order preservation + +Add UT for testing sync failure when no archives has been specified + +Add UT for last picked archive has not found + +Add UT for last specified archive was not found + +Bugfix: do not create an array with None element in it + +Fix UT for found bugfix + +Add UT for syncing no archives failure + +Add UT for sync function + +Add UT for run support function + +Fix docstring for function "run" + +lintfix: use 'salt.support.mock' and 'patch()' + +Rewrite subdirectory creation and do not rely on Python3-only code + +Lintfix: remove unused imports + +Lintfix: regexp strings + +Break-down oneliner if/else clause + +Use ordered dictionary to preserve order of the state. + +This has transparent effect to the current process: OrderedDict is the +same as just Python dict, except it is preserving order of the state +chunks. + +Refactor state processing class. + +Add __call__ function to process single-id syntax + +Add backward-compatibility with default SLS syntax (id-per-call) + +Lintfix: E1120 no value in argument 'name' for class constructor + +Remove unused import + +Check last function by full name +--- + salt/cli/support/__init__.py | 2 +- + salt/cli/support/collector.py | 12 +- + salt/loader.py | 6 +- + salt/modules/saltsupport.py | 381 ++++++++++++++++++++++++ + salt/state.py | 34 ++- + salt/states/saltsupport.py | 206 +++++++++++++ + salt/utils/args.py | 4 +- + salt/utils/decorators/__init__.py | 24 ++ + tests/unit/modules/test_saltsupport.py | 394 +++++++++++++++++++++++++ + 9 files changed, 1044 insertions(+), 19 deletions(-) + create mode 100644 salt/modules/saltsupport.py + create mode 100644 salt/states/saltsupport.py + create mode 100644 tests/unit/modules/test_saltsupport.py + +diff --git a/salt/cli/support/__init__.py b/salt/cli/support/__init__.py +index 6a98a2d656..0a48b0a081 100644 +--- a/salt/cli/support/__init__.py ++++ b/salt/cli/support/__init__.py +@@ -40,7 +40,7 @@ def get_profile(profile, caller, runner): + if os.path.exists(profile_path): + try: + rendered_template = _render_profile(profile_path, caller, runner) +- log.trace('\n{d}\n{t}\n{d}\n'.format(d='-' * 80, t=rendered_template)) ++ log.debug('\n{d}\n{t}\n{d}\n'.format(d='-' * 80, t=rendered_template)) + data.update(yaml.load(rendered_template)) + except Exception as ex: + log.debug(ex, exc_info=True) +diff --git a/salt/cli/support/collector.py b/salt/cli/support/collector.py +index a4343297b6..cbae189aea 100644 +--- a/salt/cli/support/collector.py ++++ b/salt/cli/support/collector.py +@@ -354,7 +354,7 @@ class SaltSupport(salt.utils.parsers.SaltSupportOptionParser): + + return data + +- def collect_local_data(self): ++ def collect_local_data(self, profile=None, profile_source=None): + ''' + Collects master system data. + :return: +@@ -375,7 +375,7 @@ class SaltSupport(salt.utils.parsers.SaltSupportOptionParser): + ''' + return self._extract_return(self._local_run({'fun': func, 'arg': args, 'kwarg': kwargs})) + +- scenario = salt.cli.support.get_profile(self.config['support_profile'], call, run) ++ scenario = profile_source or salt.cli.support.get_profile(profile or self.config['support_profile'], call, run) + for category_name in scenario: + self.out.put(category_name) + self.collector.add(category_name) +@@ -415,13 +415,6 @@ class SaltSupport(salt.utils.parsers.SaltSupportOptionParser): + + return action_name.split(':')[0] or None + +- def collect_targets_data(self): +- ''' +- Collects minion targets data +- :return: +- ''' +- # TODO: remote collector? +- + def _cleanup(self): + ''' + Cleanup if crash/exception +@@ -511,7 +504,6 @@ class SaltSupport(salt.utils.parsers.SaltSupportOptionParser): + self.collector.open() + self.collect_local_data() + self.collect_internal_data() +- self.collect_targets_data() + self.collector.close() + + archive_path = self.collector.archive_path +diff --git a/salt/loader.py b/salt/loader.py +index ae024ccac9..094a816d11 100644 +--- a/salt/loader.py ++++ b/salt/loader.py +@@ -1570,8 +1570,10 @@ class LazyLoader(salt.utils.lazy.LazyDict): + )) + + for attr in getattr(mod, '__load__', dir(mod)): +- if attr.startswith('_'): +- # private functions are skipped ++ if attr.startswith('_') and attr != '__call__': ++ # private functions are skipped, ++ # except __call__ which is default entrance ++ # for multi-function batch-like state syntax + continue + func = getattr(mod, attr) + if not inspect.isfunction(func) and not isinstance(func, functools.partial): +diff --git a/salt/modules/saltsupport.py b/salt/modules/saltsupport.py +new file mode 100644 +index 0000000000..750b2655d6 +--- /dev/null ++++ b/salt/modules/saltsupport.py +@@ -0,0 +1,381 @@ ++# -*- coding: utf-8 -*- ++# ++# Author: Bo Maryniuk ++# ++# Copyright 2018 SUSE LLC ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++''' ++:codeauthor: :email:`Bo Maryniuk ` ++ ++Module to run salt-support within Salt. ++''' ++# pylint: disable=W0231,W0221 ++ ++from __future__ import unicode_literals, print_function, absolute_import ++ ++import tempfile ++import re ++import os ++import sys ++import time ++import datetime ++import logging ++ ++import salt.cli.support.intfunc ++import salt.utils.decorators ++import salt.utils.path ++import salt.cli.support ++import salt.exceptions ++import salt.utils.stringutils ++import salt.defaults.exitcodes ++import salt.utils.odict ++import salt.utils.dictupdate ++ ++from salt.cli.support.collector import SaltSupport, SupportDataCollector ++ ++__virtualname__ = 'support' ++log = logging.getLogger(__name__) ++ ++ ++class LogCollector(object): ++ ''' ++ Output collector. ++ ''' ++ INFO = 'info' ++ WARNING = 'warning' ++ ERROR = 'error' ++ ++ class MessagesList(list): ++ def append(self, obj): ++ list.append(self, '{} - {}'.format(datetime.datetime.utcnow().strftime('%T.%f')[:-3], obj)) ++ __call__ = append ++ ++ def __init__(self): ++ self.messages = { ++ self.INFO: self.MessagesList(), ++ self.WARNING: self.MessagesList(), ++ self.ERROR: self.MessagesList(), ++ } ++ ++ def msg(self, message, *args, **kwargs): ++ title = kwargs.get('title') ++ if title: ++ message = '{}: {}'.format(title, message) ++ self.messages[self.INFO](message) ++ ++ def info(self, message, *args, **kwargs): ++ self.msg(message) ++ ++ def warning(self, message, *args, **kwargs): ++ self.messages[self.WARNING](message) ++ ++ def error(self, message, *args, **kwargs): ++ self.messages[self.ERROR](message) ++ ++ def put(self, message, *args, **kwargs): ++ self.messages[self.INFO](message) ++ ++ def highlight(self, message, *values, **kwargs): ++ self.msg(message.format(*values)) ++ ++ ++class SaltSupportModule(SaltSupport): ++ ''' ++ Salt Support module class. ++ ''' ++ def __init__(self): ++ ''' ++ Constructor ++ ''' ++ self.config = self.setup_config() ++ ++ def setup_config(self): ++ ''' ++ Return current configuration ++ :return: ++ ''' ++ return __opts__ ++ ++ def _get_archive_name(self, archname=None): ++ ''' ++ Create default archive name. ++ ++ :return: ++ ''' ++ archname = re.sub('[^a-z0-9]', '', (archname or '').lower()) or 'support' ++ for grain in ['fqdn', 'host', 'localhost', 'nodename']: ++ host = __grains__.get(grain) ++ if host: ++ break ++ if not host: ++ host = 'localhost' ++ ++ return os.path.join(tempfile.gettempdir(), ++ '{hostname}-{archname}-{date}-{time}.bz2'.format(archname=archname, ++ hostname=host, ++ date=time.strftime('%Y%m%d'), ++ time=time.strftime('%H%M%S'))) ++ ++ @salt.utils.decorators.external ++ def profiles(self): ++ ''' ++ Get list of profiles. ++ ++ :return: ++ ''' ++ return { ++ 'standard': salt.cli.support.get_profiles(self.config), ++ 'custom': [], ++ } ++ ++ @salt.utils.decorators.external ++ def archives(self): ++ ''' ++ Get list of existing archives. ++ :return: ++ ''' ++ arc_files = [] ++ tmpdir = tempfile.gettempdir() ++ for filename in os.listdir(tmpdir): ++ mtc = re.match(r'\w+-\w+-\d+-\d+\.bz2', filename) ++ if mtc and len(filename) == mtc.span()[-1]: ++ arc_files.append(os.path.join(tmpdir, filename)) ++ ++ return arc_files ++ ++ @salt.utils.decorators.external ++ def last_archive(self): ++ ''' ++ Get the last available archive ++ :return: ++ ''' ++ archives = {} ++ for archive in self.archives(): ++ archives[int(archive.split('.')[0].split('-')[-1])] = archive ++ ++ return archives and archives[max(archives)] or None ++ ++ @salt.utils.decorators.external ++ def delete_archives(self, *archives): ++ ''' ++ Delete archives ++ :return: ++ ''' ++ # Remove paths ++ _archives = [] ++ for archive in archives: ++ _archives.append(os.path.basename(archive)) ++ archives = _archives[:] ++ ++ ret = {'files': {}, 'errors': {}} ++ for archive in self.archives(): ++ arc_dir = os.path.dirname(archive) ++ archive = os.path.basename(archive) ++ if archives and archive in archives or not archives: ++ archive = os.path.join(arc_dir, archive) ++ try: ++ os.unlink(archive) ++ ret['files'][archive] = 'removed' ++ except Exception as err: ++ ret['errors'][archive] = str(err) ++ ret['files'][archive] = 'left' ++ ++ return ret ++ ++ def format_sync_stats(self, cnt): ++ ''' ++ Format stats of the sync output. ++ ++ :param cnt: ++ :return: ++ ''' ++ stats = salt.utils.odict.OrderedDict() ++ if cnt.get('retcode') == salt.defaults.exitcodes.EX_OK: ++ for line in cnt.get('stdout', '').split(os.linesep): ++ line = line.split(': ') ++ if len(line) == 2: ++ stats[line[0].lower().replace(' ', '_')] = line[1] ++ cnt['transfer'] = stats ++ del cnt['stdout'] ++ ++ # Remove empty ++ empty_sections = [] ++ for section in cnt: ++ if not cnt[section] and section != 'retcode': ++ empty_sections.append(section) ++ for section in empty_sections: ++ del cnt[section] ++ ++ return cnt ++ ++ @salt.utils.decorators.depends('rsync') ++ @salt.utils.decorators.external ++ def sync(self, group, name=None, host=None, location=None, move=False, all=False): ++ ''' ++ Sync the latest archive to the host on given location. ++ ++ CLI Example: ++ ++ .. code-block:: bash ++ ++ salt '*' support.sync group=test ++ salt '*' support.sync group=test name=/tmp/myspecial-12345-67890.bz2 ++ salt '*' support.sync group=test name=/tmp/myspecial-12345-67890.bz2 host=allmystuff.lan ++ salt '*' support.sync group=test name=/tmp/myspecial-12345-67890.bz2 host=allmystuff.lan location=/opt/ ++ ++ :param group: name of the local directory to which sync is going to put the result files ++ :param name: name of the archive. Latest, if not specified. ++ :param host: name of the destination host for rsync. Default is master, if not specified. ++ :param location: local destination directory, default temporary if not specified ++ :param move: move archive file[s]. Default is False. ++ :param all: work with all available archives. Default is False (i.e. latest available) ++ ++ :return: ++ ''' ++ tfh, tfn = tempfile.mkstemp() ++ processed_archives = [] ++ src_uri = uri = None ++ ++ last_arc = self.last_archive() ++ if name: ++ archives = [name] ++ elif all: ++ archives = self.archives() ++ elif last_arc: ++ archives = [last_arc] ++ else: ++ archives = [] ++ ++ for name in archives: ++ err = None ++ if not name: ++ err = 'No support archive has been defined.' ++ elif not os.path.exists(name): ++ err = 'Support archive "{}" was not found'.format(name) ++ if err is not None: ++ log.error(err) ++ raise salt.exceptions.SaltInvocationError(err) ++ ++ if not uri: ++ src_uri = os.path.dirname(name) ++ uri = '{host}:{loc}'.format(host=host or __opts__['master'], ++ loc=os.path.join(location or tempfile.gettempdir(), group)) ++ ++ os.write(tfh, salt.utils.stringutils.to_bytes(os.path.basename(name))) ++ os.write(tfh, salt.utils.stringutils.to_bytes(os.linesep)) ++ processed_archives.append(name) ++ log.debug('Syncing {filename} to {uri}'.format(filename=name, uri=uri)) ++ os.close(tfh) ++ ++ if not processed_archives: ++ raise salt.exceptions.SaltInvocationError('No archives found to transfer.') ++ ++ ret = __salt__['rsync.rsync'](src=src_uri, dst=uri, additional_opts=['--stats', '--files-from={}'.format(tfn)]) ++ ret['files'] = {} ++ for name in processed_archives: ++ if move: ++ salt.utils.dictupdate.update(ret, self.delete_archives(name)) ++ log.debug('Deleting {filename}'.format(filename=name)) ++ ret['files'][name] = 'moved' ++ else: ++ ret['files'][name] = 'copied' ++ ++ try: ++ os.unlink(tfn) ++ except (OSError, IOError) as err: ++ log.error('Cannot remove temporary rsync file {fn}: {err}'.format(fn=tfn, err=err)) ++ ++ return self.format_sync_stats(ret) ++ ++ @salt.utils.decorators.external ++ def run(self, profile='default', pillar=None, archive=None, output='nested'): ++ ''' ++ Run Salt Support on the minion. ++ ++ profile ++ Set available profile name. Default is "default". ++ ++ pillar ++ Set available profile from the pillars. ++ ++ archive ++ Override archive name. Default is "support". This results to "hostname-support-YYYYMMDD-hhmmss.bz2". ++ ++ output ++ Change the default outputter. Default is "nested". ++ ++ CLI Example: ++ ++ .. code-block:: bash ++ ++ salt '*' support.run ++ salt '*' support.run profile=network ++ salt '*' support.run pillar=something_special ++ ''' ++ class outputswitch(object): ++ ''' ++ Output switcher on context ++ ''' ++ def __init__(self, output_device): ++ self._tmp_out = output_device ++ self._orig_out = None ++ ++ def __enter__(self): ++ self._orig_out = salt.cli.support.intfunc.out ++ salt.cli.support.intfunc.out = self._tmp_out ++ ++ def __exit__(self, *args): ++ salt.cli.support.intfunc.out = self._orig_out ++ ++ self.out = LogCollector() ++ with outputswitch(self.out): ++ self.collector = SupportDataCollector(archive or self._get_archive_name(archname=archive), output) ++ self.collector.out = self.out ++ self.collector.open() ++ self.collect_local_data(profile=profile, profile_source=__pillar__.get(pillar)) ++ self.collect_internal_data() ++ self.collector.close() ++ ++ return {'archive': self.collector.archive_path, ++ 'messages': self.out.messages} ++ ++ ++def __virtual__(): ++ ''' ++ Set method references as module functions aliases ++ :return: ++ ''' ++ support = SaltSupportModule() ++ ++ def _set_function(obj): ++ ''' ++ Create a Salt function for the SaltSupport class. ++ ''' ++ def _cmd(*args, **kwargs): ++ ''' ++ Call support method as a function from the Salt. ++ ''' ++ _kwargs = {} ++ for kw in kwargs: ++ if not kw.startswith('__'): ++ _kwargs[kw] = kwargs[kw] ++ return obj(*args, **_kwargs) ++ _cmd.__doc__ = obj.__doc__ ++ return _cmd ++ ++ for m_name in dir(support): ++ obj = getattr(support, m_name) ++ if getattr(obj, 'external', False): ++ setattr(sys.modules[__name__], m_name, _set_function(obj)) ++ ++ return __virtualname__ +diff --git a/salt/state.py b/salt/state.py +index e7288bce2e..b4b2a00601 100644 +--- a/salt/state.py ++++ b/salt/state.py +@@ -1315,8 +1315,9 @@ class State(object): + names = [] + if state.startswith('__'): + continue +- chunk = {'state': state, +- 'name': name} ++ chunk = OrderedDict() ++ chunk['state'] = state ++ chunk['name'] = name + if orchestration_jid is not None: + chunk['__orchestration_jid__'] = orchestration_jid + if '__sls__' in body: +@@ -1901,8 +1902,12 @@ class State(object): + ret = self.call_parallel(cdata, low) + else: + self.format_slots(cdata) +- ret = self.states[cdata['full']](*cdata['args'], +- **cdata['kwargs']) ++ if cdata['full'].split('.')[-1] == '__call__': ++ # __call__ requires OrderedDict to preserve state order ++ # kwargs are also invalid overall ++ ret = self.states[cdata['full']](cdata['args'], module=None, state=cdata['kwargs']) ++ else: ++ ret = self.states[cdata['full']](*cdata['args'], **cdata['kwargs']) + self.states.inject_globals = {} + if 'check_cmd' in low and '{0[state]}.mod_run_check_cmd'.format(low) not in self.states: + ret.update(self._run_check_cmd(low)) +@@ -2729,10 +2734,31 @@ class State(object): + running.update(errors) + return running + ++ def inject_default_call(self, high): ++ ''' ++ Sets .call function to a state, if not there. ++ ++ :param high: ++ :return: ++ ''' ++ for chunk in high: ++ state = high[chunk] ++ for state_ref in state: ++ needs_default = True ++ for argset in state[state_ref]: ++ if isinstance(argset, six.string_types): ++ needs_default = False ++ break ++ if needs_default: ++ order = state[state_ref].pop(-1) ++ state[state_ref].append('__call__') ++ state[state_ref].append(order) ++ + def call_high(self, high, orchestration_jid=None): + ''' + Process a high data call and ensure the defined states. + ''' ++ self.inject_default_call(high) + errors = [] + # If there is extension data reconcile it + high, ext_errors = self.reconcile_extend(high) +diff --git a/salt/states/saltsupport.py b/salt/states/saltsupport.py +new file mode 100644 +index 0000000000..f245f7f137 +--- /dev/null ++++ b/salt/states/saltsupport.py +@@ -0,0 +1,206 @@ ++# -*- coding: utf-8 -*- ++# ++# Author: Bo Maryniuk ++# ++# Copyright 2018 SUSE LLC ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++ ++r''' ++:codeauthor: :email:`Bo Maryniuk ` ++ ++Execution of Salt Support from within states ++============================================ ++ ++State to collect support data from the systems: ++ ++.. code-block:: yaml ++ ++ examine_my_systems: ++ support.taken: ++ - profile: default ++ ++ support.collected: ++ - group: somewhere ++ - move: true ++ ++''' ++from __future__ import absolute_import, print_function, unicode_literals ++import logging ++import os ++import tempfile ++ ++# Import salt modules ++import salt.fileclient ++import salt.utils.decorators.path ++import salt.exceptions ++import salt.utils.odict ++ ++log = logging.getLogger(__name__) ++__virtualname__ = 'support' ++ ++ ++class SaltSupportState(object): ++ ''' ++ Salt-support. ++ ''' ++ EXPORTED = ['collected', 'taken'] ++ ++ def get_kwargs(self, data): ++ kwargs = {} ++ for keyset in data: ++ kwargs.update(keyset) ++ ++ return kwargs ++ ++ def __call__(self, state): ++ ''' ++ Call support. ++ ++ :param args: ++ :param kwargs: ++ :return: ++ ''' ++ ret = { ++ 'name': state.pop('name'), ++ 'changes': {}, ++ 'result': True, ++ 'comment': '', ++ } ++ ++ out = {} ++ functions = ['Functions:'] ++ try: ++ for ref_func, ref_kwargs in state.items(): ++ if ref_func not in self.EXPORTED: ++ raise salt.exceptions.SaltInvocationError('Function {} is not found'.format(ref_func)) ++ out[ref_func] = getattr(self, ref_func)(**self.get_kwargs(ref_kwargs)) ++ functions.append(' - {}'.format(ref_func)) ++ ret['comment'] = '\n'.join(functions) ++ except Exception as ex: ++ ret['comment'] = str(ex) ++ ret['result'] = False ++ ret['changes'] = out ++ ++ return ret ++ ++ def check_destination(self, location, group): ++ ''' ++ Check destination for the archives. ++ :return: ++ ''' ++ # Pre-create destination, since rsync will ++ # put one file named as group ++ try: ++ destination = os.path.join(location, group) ++ if os.path.exists(destination) and not os.path.isdir(destination): ++ raise salt.exceptions.SaltException('Destination "{}" should be directory!'.format(destination)) ++ if not os.path.exists(destination): ++ os.makedirs(destination) ++ log.debug('Created destination directory for archives: %s', destination) ++ else: ++ log.debug('Archives destination directory %s already exists', destination) ++ except OSError as err: ++ log.error(err) ++ ++ def collected(self, group, filename=None, host=None, location=None, move=True, all=True): ++ ''' ++ Sync archives to a central place. ++ ++ :param name: ++ :param group: ++ :param filename: ++ :param host: ++ :param location: ++ :param move: ++ :param all: ++ :return: ++ ''' ++ ret = { ++ 'name': 'support.collected', ++ 'changes': {}, ++ 'result': True, ++ 'comment': '', ++ } ++ location = location or tempfile.gettempdir() ++ self.check_destination(location, group) ++ ret['changes'] = __salt__['support.sync'](group, name=filename, host=host, ++ location=location, move=move, all=all) ++ ++ return ret ++ ++ def taken(self, profile='default', pillar=None, archive=None, output='nested'): ++ ''' ++ Takes minion support config data. ++ ++ :param profile: ++ :param pillar: ++ :param archive: ++ :param output: ++ :return: ++ ''' ++ ret = { ++ 'name': 'support.taken', ++ 'changes': {}, ++ 'result': True, ++ } ++ ++ result = __salt__['support.run'](profile=profile, pillar=pillar, archive=archive, output=output) ++ if result.get('archive'): ++ ret['comment'] = 'Information about this system has been saved to {} file.'.format(result['archive']) ++ ret['changes']['archive'] = result['archive'] ++ ret['changes']['messages'] = {} ++ for key in ['info', 'error', 'warning']: ++ if result.get('messages', {}).get(key): ++ ret['changes']['messages'][key] = result['messages'][key] ++ else: ++ ret['comment'] = '' ++ ++ return ret ++ ++ ++_support_state = SaltSupportState() ++ ++ ++def __call__(*args, **kwargs): ++ ''' ++ SLS single-ID syntax processing. ++ ++ module: ++ This module reference, equals to sys.modules[__name__] ++ ++ state: ++ Compiled state in preserved order. The function supposed to look ++ at first level array of functions. ++ ++ :param cdata: ++ :param kwargs: ++ :return: ++ ''' ++ return _support_state(kwargs.get('state', {})) ++ ++ ++def taken(name, profile='default', pillar=None, archive=None, output='nested'): ++ return _support_state.taken(profile=profile, pillar=pillar, ++ archive=archive, output=output) ++ ++ ++def collected(name, group, filename=None, host=None, location=None, move=True, all=True): ++ return _support_state.collected(group=group, filename=filename, ++ host=host, location=location, move=move, all=all) ++ ++ ++def __virtual__(): ++ ''' ++ Salt Support state ++ ''' ++ return __virtualname__ +diff --git a/salt/utils/args.py b/salt/utils/args.py +index a3d8099c7f..19de7d5d39 100644 +--- a/salt/utils/args.py ++++ b/salt/utils/args.py +@@ -19,7 +19,7 @@ import salt.utils.data + import salt.utils.jid + import salt.utils.versions + import salt.utils.yaml +- ++from salt.utils.odict import OrderedDict + + if six.PY3: + KWARG_REGEX = re.compile(r'^([^\d\W][\w.-]*)=(?!=)(.*)$', re.UNICODE) +@@ -409,7 +409,7 @@ def format_call(fun, + ret = initial_ret is not None and initial_ret or {} + + ret['args'] = [] +- ret['kwargs'] = {} ++ ret['kwargs'] = OrderedDict() + + aspec = get_function_argspec(fun, is_class_method=is_class_method) + +diff --git a/salt/utils/decorators/__init__.py b/salt/utils/decorators/__init__.py +index 81d1812833..c5da5b6d4b 100644 +--- a/salt/utils/decorators/__init__.py ++++ b/salt/utils/decorators/__init__.py +@@ -596,3 +596,27 @@ def ensure_unicode_args(function): + else: + return function(*args, **kwargs) + return wrapped ++ ++ ++def external(func): ++ ''' ++ Mark function as external. ++ ++ :param func: ++ :return: ++ ''' ++ ++ def f(*args, **kwargs): ++ ''' ++ Stub. ++ ++ :param args: ++ :param kwargs: ++ :return: ++ ''' ++ return func(*args, **kwargs) ++ ++ f.external = True ++ f.__doc__ = func.__doc__ ++ ++ return f +diff --git a/tests/unit/modules/test_saltsupport.py b/tests/unit/modules/test_saltsupport.py +new file mode 100644 +index 0000000000..7bd652a90e +--- /dev/null ++++ b/tests/unit/modules/test_saltsupport.py +@@ -0,0 +1,394 @@ ++# -*- coding: utf-8 -*- ++''' ++ :codeauthor: Bo Maryniuk ++''' ++ ++# Import Python libs ++from __future__ import absolute_import, print_function, unicode_literals ++ ++# Import Salt Testing Libs ++from tests.support.mixins import LoaderModuleMockMixin ++from tests.support.unit import TestCase, skipIf ++from tests.support.mock import patch, MagicMock, NO_MOCK, NO_MOCK_REASON ++from salt.modules import saltsupport ++import salt.exceptions ++import datetime ++ ++try: ++ import pytest ++except ImportError: ++ pytest = None ++ ++ ++@skipIf(not bool(pytest), 'Pytest required') ++@skipIf(NO_MOCK, NO_MOCK_REASON) ++class SaltSupportModuleTestCase(TestCase, LoaderModuleMockMixin): ++ ''' ++ Test cases for salt.modules.support::SaltSupportModule ++ ''' ++ def setup_loader_modules(self): ++ return {saltsupport: {}} ++ ++ @patch('tempfile.gettempdir', MagicMock(return_value='/mnt/storage')) ++ @patch('salt.modules.saltsupport.__grains__', {'fqdn': 'c-3po'}) ++ @patch('time.strftime', MagicMock(return_value='000')) ++ def test_get_archive_name(self): ++ ''' ++ Test archive name construction. ++ ++ :return: ++ ''' ++ support = saltsupport.SaltSupportModule() ++ assert support._get_archive_name() == '/mnt/storage/c-3po-support-000-000.bz2' ++ ++ @patch('tempfile.gettempdir', MagicMock(return_value='/mnt/storage')) ++ @patch('salt.modules.saltsupport.__grains__', {'fqdn': 'c-3po'}) ++ @patch('time.strftime', MagicMock(return_value='000')) ++ def test_get_custom_archive_name(self): ++ ''' ++ Test get custom archive name. ++ ++ :return: ++ ''' ++ support = saltsupport.SaltSupportModule() ++ temp_name = support._get_archive_name(archname='Darth Wader') ++ assert temp_name == '/mnt/storage/c-3po-darthwader-000-000.bz2' ++ temp_name = support._get_archive_name(archname='Яйця з сіллю') ++ assert temp_name == '/mnt/storage/c-3po-support-000-000.bz2' ++ temp_name = support._get_archive_name(archname='!@#$%^&*()Fillip J. Fry') ++ assert temp_name == '/mnt/storage/c-3po-fillipjfry-000-000.bz2' ++ ++ @patch('salt.cli.support.get_profiles', MagicMock(return_value={'message': 'Feature was not beta tested'})) ++ def test_profiles_format(self): ++ ''' ++ Test profiles format. ++ ++ :return: ++ ''' ++ support = saltsupport.SaltSupportModule() ++ profiles = support.profiles() ++ assert 'custom' in profiles ++ assert 'standard' in profiles ++ assert 'message' in profiles['standard'] ++ assert profiles['custom'] == [] ++ assert profiles['standard']['message'] == 'Feature was not beta tested' ++ ++ @patch('tempfile.gettempdir', MagicMock(return_value='/mnt/storage')) ++ @patch('os.listdir', MagicMock(return_value=['one-support-000-000.bz2', 'two-support-111-111.bz2', 'trash.bz2', ++ 'hostname-000-000.bz2', 'three-support-wrong222-222.bz2', ++ '000-support-000-000.bz2'])) ++ def test_get_existing_archives(self): ++ ''' ++ Get list of existing archives. ++ ++ :return: ++ ''' ++ support = saltsupport.SaltSupportModule() ++ out = support.archives() ++ assert len(out) == 3 ++ for name in ['/mnt/storage/one-support-000-000.bz2', '/mnt/storage/two-support-111-111.bz2', ++ '/mnt/storage/000-support-000-000.bz2']: ++ assert name in out ++ ++ def test_last_archive(self): ++ ''' ++ Get last archive name ++ :return: ++ ''' ++ support = saltsupport.SaltSupportModule() ++ support.archives = MagicMock(return_value=['/mnt/storage/one-support-000-000.bz2', ++ '/mnt/storage/two-support-111-111.bz2', ++ '/mnt/storage/three-support-222-222.bz2']) ++ assert support.last_archive() == '/mnt/storage/three-support-222-222.bz2' ++ ++ @patch('os.unlink', MagicMock(return_value=True)) ++ def test_delete_all_archives_success(self): ++ ''' ++ Test delete archives ++ :return: ++ ''' ++ support = saltsupport.SaltSupportModule() ++ support.archives = MagicMock(return_value=['/mnt/storage/one-support-000-000.bz2', ++ '/mnt/storage/two-support-111-111.bz2', ++ '/mnt/storage/three-support-222-222.bz2']) ++ ret = support.delete_archives() ++ assert 'files' in ret ++ assert 'errors' in ret ++ assert not bool(ret['errors']) ++ assert bool(ret['files']) ++ assert isinstance(ret['errors'], dict) ++ assert isinstance(ret['files'], dict) ++ ++ for arc in support.archives(): ++ assert ret['files'][arc] == 'removed' ++ ++ @patch('os.unlink', MagicMock(return_value=False, side_effect=[OSError('Decreasing electron flux'), ++ OSError('Solar flares interference'), ++ None])) ++ def test_delete_all_archives_failure(self): ++ ''' ++ Test delete archives failure ++ :return: ++ ''' ++ support = saltsupport.SaltSupportModule() ++ support.archives = MagicMock(return_value=['/mnt/storage/one-support-000-000.bz2', ++ '/mnt/storage/two-support-111-111.bz2', ++ '/mnt/storage/three-support-222-222.bz2']) ++ ret = support.delete_archives() ++ assert 'files' in ret ++ assert 'errors' in ret ++ assert bool(ret['errors']) ++ assert bool(ret['files']) ++ assert isinstance(ret['errors'], dict) ++ assert isinstance(ret['files'], dict) ++ ++ assert ret['files']['/mnt/storage/three-support-222-222.bz2'] == 'removed' ++ assert ret['files']['/mnt/storage/one-support-000-000.bz2'] == 'left' ++ assert ret['files']['/mnt/storage/two-support-111-111.bz2'] == 'left' ++ ++ assert len(ret['errors']) == 2 ++ assert ret['errors']['/mnt/storage/one-support-000-000.bz2'] == 'Decreasing electron flux' ++ assert ret['errors']['/mnt/storage/two-support-111-111.bz2'] == 'Solar flares interference' ++ ++ def test_format_sync_stats(self): ++ ''' ++ Test format rsync stats for preserving ordering of the keys ++ ++ :return: ++ ''' ++ support = saltsupport.SaltSupportModule() ++ stats = ''' ++robot: Bender ++cute: Leela ++weird: Zoidberg ++professor: Farnsworth ++ ''' ++ f_stats = support.format_sync_stats({'retcode': 0, 'stdout': stats}) ++ assert list(f_stats['transfer'].keys()) == ['robot', 'cute', 'weird', 'professor'] ++ assert list(f_stats['transfer'].values()) == ['Bender', 'Leela', 'Zoidberg', 'Farnsworth'] ++ ++ @patch('tempfile.mkstemp', MagicMock(return_value=(0, 'dummy'))) ++ @patch('os.close', MagicMock()) ++ def test_sync_no_archives_failure(self): ++ ''' ++ Test sync failed when no archives specified. ++ ++ :return: ++ ''' ++ support = saltsupport.SaltSupportModule() ++ support.archives = MagicMock(return_value=[]) ++ ++ with pytest.raises(salt.exceptions.SaltInvocationError) as err: ++ support.sync('group-name') ++ assert 'No archives found to transfer' in str(err) ++ ++ @patch('tempfile.mkstemp', MagicMock(return_value=(0, 'dummy'))) ++ @patch('os.path.exists', MagicMock(return_value=False)) ++ def test_sync_last_picked_archive_not_found_failure(self): ++ ''' ++ Test sync failed when archive was not found (last picked) ++ ++ :return: ++ ''' ++ support = saltsupport.SaltSupportModule() ++ support.archives = MagicMock(return_value=['/mnt/storage/one-support-000-000.bz2', ++ '/mnt/storage/two-support-111-111.bz2', ++ '/mnt/storage/three-support-222-222.bz2']) ++ ++ with pytest.raises(salt.exceptions.SaltInvocationError) as err: ++ support.sync('group-name') ++ assert ' Support archive "/mnt/storage/three-support-222-222.bz2" was not found' in str(err) ++ ++ @patch('tempfile.mkstemp', MagicMock(return_value=(0, 'dummy'))) ++ @patch('os.path.exists', MagicMock(return_value=False)) ++ def test_sync_specified_archive_not_found_failure(self): ++ ''' ++ Test sync failed when archive was not found (last picked) ++ ++ :return: ++ ''' ++ support = saltsupport.SaltSupportModule() ++ support.archives = MagicMock(return_value=['/mnt/storage/one-support-000-000.bz2', ++ '/mnt/storage/two-support-111-111.bz2', ++ '/mnt/storage/three-support-222-222.bz2']) ++ ++ with pytest.raises(salt.exceptions.SaltInvocationError) as err: ++ support.sync('group-name', name='lost.bz2') ++ assert ' Support archive "lost.bz2" was not found' in str(err) ++ ++ @patch('tempfile.mkstemp', MagicMock(return_value=(0, 'dummy'))) ++ @patch('os.path.exists', MagicMock(return_value=False)) ++ @patch('os.close', MagicMock()) ++ def test_sync_no_archive_to_transfer_failure(self): ++ ''' ++ Test sync failed when no archive was found to transfer ++ ++ :return: ++ ''' ++ support = saltsupport.SaltSupportModule() ++ support.archives = MagicMock(return_value=[]) ++ with pytest.raises(salt.exceptions.SaltInvocationError) as err: ++ support.sync('group-name', all=True) ++ assert 'No archives found to transfer' in str(err) ++ ++ @patch('tempfile.mkstemp', MagicMock(return_value=(0, 'dummy'))) ++ @patch('os.path.exists', MagicMock(return_value=True)) ++ @patch('os.close', MagicMock()) ++ @patch('os.write', MagicMock()) ++ @patch('os.unlink', MagicMock()) ++ @patch('salt.modules.saltsupport.__salt__', {'rsync.rsync': MagicMock(return_value={})}) ++ def test_sync_archives(self): ++ ''' ++ Test sync archives ++ :return: ++ ''' ++ support = saltsupport.SaltSupportModule() ++ support.archives = MagicMock(return_value=['/mnt/storage/one-support-000-000.bz2', ++ '/mnt/storage/two-support-111-111.bz2', ++ '/mnt/storage/three-support-222-222.bz2']) ++ out = support.sync('group-name', host='buzz', all=True, move=False) ++ assert 'files' in out ++ for arc_name in out['files']: ++ assert out['files'][arc_name] == 'copied' ++ assert saltsupport.os.unlink.call_count == 1 ++ assert saltsupport.os.unlink.call_args_list[0][0][0] == 'dummy' ++ calls = [] ++ for call in saltsupport.os.write.call_args_list: ++ assert len(call) == 2 ++ calls.append(call[0]) ++ assert calls == [(0, b'one-support-000-000.bz2'), ++ (0, b'\n'), (0, b'two-support-111-111.bz2'), (0, b'\n'), ++ (0, b'three-support-222-222.bz2'), (0, b'\n')] ++ ++ @patch('salt.modules.saltsupport.__pillar__', {}) ++ @patch('salt.modules.saltsupport.SupportDataCollector', MagicMock()) ++ def test_run_support(self): ++ ''' ++ Test run support ++ :return: ++ ''' ++ saltsupport.SupportDataCollector(None, None).archive_path = 'dummy' ++ support = saltsupport.SaltSupportModule() ++ support.collect_internal_data = MagicMock() ++ support.collect_local_data = MagicMock() ++ out = support.run() ++ ++ for section in ['messages', 'archive']: ++ assert section in out ++ assert out['archive'] == 'dummy' ++ for section in ['warning', 'error', 'info']: ++ assert section in out['messages'] ++ ld_call = support.collect_local_data.call_args_list[0][1] ++ assert 'profile' in ld_call ++ assert ld_call['profile'] == 'default' ++ assert 'profile_source' in ld_call ++ assert ld_call['profile_source'] is None ++ assert support.collector.open.call_count == 1 ++ assert support.collector.close.call_count == 1 ++ assert support.collect_internal_data.call_count == 1 ++ ++ ++@skipIf(not bool(pytest), 'Pytest required') ++@skipIf(NO_MOCK, NO_MOCK_REASON) ++class LogCollectorTestCase(TestCase, LoaderModuleMockMixin): ++ ''' ++ Test cases for salt.modules.support::LogCollector ++ ''' ++ def setup_loader_modules(self): ++ return {saltsupport: {}} ++ ++ def test_msg(self): ++ ''' ++ Test message to the log collector. ++ ++ :return: ++ ''' ++ utcmock = MagicMock() ++ utcmock.utcnow = MagicMock(return_value=datetime.datetime.utcfromtimestamp(0)) ++ with patch('datetime.datetime', utcmock): ++ msg = 'Upgrading /dev/null device' ++ out = saltsupport.LogCollector() ++ out.msg(msg, title='Here') ++ assert saltsupport.LogCollector.INFO in out.messages ++ assert type(out.messages[saltsupport.LogCollector.INFO]) == saltsupport.LogCollector.MessagesList ++ assert out.messages[saltsupport.LogCollector.INFO] == ['00:00:00.000 - {0}: {1}'.format('Here', msg)] ++ ++ def test_info_message(self): ++ ''' ++ Test info message to the log collector. ++ ++ :return: ++ ''' ++ utcmock = MagicMock() ++ utcmock.utcnow = MagicMock(return_value=datetime.datetime.utcfromtimestamp(0)) ++ with patch('datetime.datetime', utcmock): ++ msg = 'SIMM crosstalk during tectonic stress' ++ out = saltsupport.LogCollector() ++ out.info(msg) ++ assert saltsupport.LogCollector.INFO in out.messages ++ assert type(out.messages[saltsupport.LogCollector.INFO]) == saltsupport.LogCollector.MessagesList ++ assert out.messages[saltsupport.LogCollector.INFO] == ['00:00:00.000 - {}'.format(msg)] ++ ++ def test_put_message(self): ++ ''' ++ Test put message to the log collector. ++ ++ :return: ++ ''' ++ utcmock = MagicMock() ++ utcmock.utcnow = MagicMock(return_value=datetime.datetime.utcfromtimestamp(0)) ++ with patch('datetime.datetime', utcmock): ++ msg = 'Webmaster kidnapped by evil cult' ++ out = saltsupport.LogCollector() ++ out.put(msg) ++ assert saltsupport.LogCollector.INFO in out.messages ++ assert type(out.messages[saltsupport.LogCollector.INFO]) == saltsupport.LogCollector.MessagesList ++ assert out.messages[saltsupport.LogCollector.INFO] == ['00:00:00.000 - {}'.format(msg)] ++ ++ def test_warning_message(self): ++ ''' ++ Test warning message to the log collector. ++ ++ :return: ++ ''' ++ utcmock = MagicMock() ++ utcmock.utcnow = MagicMock(return_value=datetime.datetime.utcfromtimestamp(0)) ++ with patch('datetime.datetime', utcmock): ++ msg = 'Your e-mail is now being delivered by USPS' ++ out = saltsupport.LogCollector() ++ out.warning(msg) ++ assert saltsupport.LogCollector.WARNING in out.messages ++ assert type(out.messages[saltsupport.LogCollector.WARNING]) == saltsupport.LogCollector.MessagesList ++ assert out.messages[saltsupport.LogCollector.WARNING] == ['00:00:00.000 - {}'.format(msg)] ++ ++ def test_error_message(self): ++ ''' ++ Test error message to the log collector. ++ ++ :return: ++ ''' ++ utcmock = MagicMock() ++ utcmock.utcnow = MagicMock(return_value=datetime.datetime.utcfromtimestamp(0)) ++ with patch('datetime.datetime', utcmock): ++ msg = 'Learning curve appears to be fractal' ++ out = saltsupport.LogCollector() ++ out.error(msg) ++ assert saltsupport.LogCollector.ERROR in out.messages ++ assert type(out.messages[saltsupport.LogCollector.ERROR]) == saltsupport.LogCollector.MessagesList ++ assert out.messages[saltsupport.LogCollector.ERROR] == ['00:00:00.000 - {}'.format(msg)] ++ ++ def test_hl_message(self): ++ ''' ++ Test highlighter message to the log collector. ++ ++ :return: ++ ''' ++ utcmock = MagicMock() ++ utcmock.utcnow = MagicMock(return_value=datetime.datetime.utcfromtimestamp(0)) ++ with patch('datetime.datetime', utcmock): ++ out = saltsupport.LogCollector() ++ out.highlight('The {} TTYs became {} TTYs and vice versa', 'real', 'pseudo') ++ assert saltsupport.LogCollector.INFO in out.messages ++ assert type(out.messages[saltsupport.LogCollector.INFO]) == saltsupport.LogCollector.MessagesList ++ assert out.messages[saltsupport.LogCollector.INFO] == ['00:00:00.000 - The real TTYs became ' ++ 'pseudo TTYs and vice versa'] +-- +2.20.1 + + diff --git a/azurefs-gracefully-handle-attributeerror.patch b/azurefs-gracefully-handle-attributeerror.patch new file mode 100644 index 0000000..fa03205 --- /dev/null +++ b/azurefs-gracefully-handle-attributeerror.patch @@ -0,0 +1,31 @@ +From 326e649ef1f14b609916f0e9ce75e29a5e7f4d05 Mon Sep 17 00:00:00 2001 +From: Robert Munteanu +Date: Mon, 19 Nov 2018 17:52:34 +0100 +Subject: [PATCH] azurefs: gracefully handle AttributeError + +It is possible that the azure.storage object has no __version__ defined. +In that case, prevent console spam with unhandled AttributeError +messages and instead consider that Azure support is not present. + +Problem was encountered on openSUSE Tumbleweed. +--- + salt/fileserver/azurefs.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/salt/fileserver/azurefs.py b/salt/fileserver/azurefs.py +index c266323fbe..a42c10c594 100644 +--- a/salt/fileserver/azurefs.py ++++ b/salt/fileserver/azurefs.py +@@ -68,7 +68,7 @@ try: + if LooseVersion(azure.storage.__version__) < LooseVersion('0.20.0'): + raise ImportError('azure.storage.__version__ must be >= 0.20.0') + HAS_AZURE = True +-except ImportError: ++except (ImportError, AttributeError): + HAS_AZURE = False + + # Import third party libs +-- +2.20.1 + + diff --git a/bugfix-any-unicode-string-of-length-16-will-raise-ty.patch b/bugfix-any-unicode-string-of-length-16-will-raise-ty.patch new file mode 100644 index 0000000..aa453d5 --- /dev/null +++ b/bugfix-any-unicode-string-of-length-16-will-raise-ty.patch @@ -0,0 +1,27 @@ +From e82dc4c556497b612d31b65e60b34c979c957424 Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Fri, 5 Oct 2018 12:02:08 +0200 +Subject: [PATCH] Bugfix: any unicode string of length 16 will raise + TypeError instead of ValueError + +--- + salt/_compat.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/salt/_compat.py b/salt/_compat.py +index 0576210afc..71963a4ead 100644 +--- a/salt/_compat.py ++++ b/salt/_compat.py +@@ -192,7 +192,7 @@ class IPv6AddressScoped(ipaddress.IPv6Address): + if len(data) == 16 and ':' not in data: + try: + packed = bool(int(str(bytearray(data)).encode('hex'), 16)) +- except ValueError: ++ except (ValueError, TypeError): + pass + + return packed +-- +2.17.1 + + diff --git a/debian-info_installed-compatibility-50453.patch b/debian-info_installed-compatibility-50453.patch new file mode 100644 index 0000000..566f529 --- /dev/null +++ b/debian-info_installed-compatibility-50453.patch @@ -0,0 +1,642 @@ +From 9b2473001dcf25c53dff469d3ffb38113e0402eb Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Tue, 20 Nov 2018 16:06:31 +0100 +Subject: [PATCH] Debian info_installed compatibility (#50453) + +Remove unused variable + +Get unit ticks installation time + +Pass on unix ticks installation date time + +Implement function to figure out package build time + +Unify arch attribute + +Add 'attr' support. + +Use attr parameter in aptpkg + +Add 'all_versions' output structure backward compatibility + +Fix docstring + +Add UT for generic test of function 'info' + +Add UT for 'info' function with the parameter 'attr' + +Add UT for info_installed's 'attr' param + +Fix docstring + +Add returned type check + +Add UT for info_installed with 'all_versions=True' output structure + +Refactor UT for 'owner' function + +Refactor UT: move to decorators, add more checks + +Schedule TODO for next refactoring of UT 'show' function + +Refactor UT: get rid of old assertion way, flatten tests + +Refactor UT: move to native assertions, cleanup noise, flatten complexity for better visibility what is tested + +Lintfix: too many empty lines + +Adjust architecture getter according to the lowpkg info + +Fix wrong Git merge: missing function signature +--- + salt/modules/aptpkg.py | 20 +++- + salt/modules/dpkg.py | 93 +++++++++++++-- + tests/unit/modules/test_aptpkg.py | 189 +++++++++++++++++------------- + tests/unit/modules/test_dpkg.py | 69 +++++++++++ + 4 files changed, 274 insertions(+), 97 deletions(-) + +diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py +index 90b99c44b9..dc27903230 100644 +--- a/salt/modules/aptpkg.py ++++ b/salt/modules/aptpkg.py +@@ -2800,6 +2800,15 @@ def info_installed(*names, **kwargs): + + .. versionadded:: 2016.11.3 + ++ attr ++ Comma-separated package attributes. If no 'attr' is specified, all available attributes returned. ++ ++ Valid attributes are: ++ version, vendor, release, build_date, build_date_time_t, install_date, install_date_time_t, ++ build_host, group, source_rpm, arch, epoch, size, license, signature, packager, url, summary, description. ++ ++ .. versionadded:: Neon ++ + CLI example: + + .. code-block:: bash +@@ -2810,11 +2819,15 @@ def info_installed(*names, **kwargs): + ''' + kwargs = salt.utils.args.clean_kwargs(**kwargs) + failhard = kwargs.pop('failhard', True) ++ kwargs.pop('errors', None) # Only for compatibility with RPM ++ attr = kwargs.pop('attr', None) # Package attributes to return ++ all_versions = kwargs.pop('all_versions', False) # This is for backward compatible structure only ++ + if kwargs: + salt.utils.args.invalid_kwargs(kwargs) + + ret = dict() +- for pkg_name, pkg_nfo in __salt__['lowpkg.info'](*names, failhard=failhard).items(): ++ for pkg_name, pkg_nfo in __salt__['lowpkg.info'](*names, failhard=failhard, attr=attr).items(): + t_nfo = dict() + # Translate dpkg-specific keys to a common structure + for key, value in pkg_nfo.items(): +@@ -2831,7 +2844,10 @@ def info_installed(*names, **kwargs): + else: + t_nfo[key] = value + +- ret[pkg_name] = t_nfo ++ if all_versions: ++ ret.setdefault(pkg_name, []).append(t_nfo) ++ else: ++ ret[pkg_name] = t_nfo + + return ret + +diff --git a/salt/modules/dpkg.py b/salt/modules/dpkg.py +index 03be5f821a..26ca5dcf5a 100644 +--- a/salt/modules/dpkg.py ++++ b/salt/modules/dpkg.py +@@ -252,6 +252,38 @@ def file_dict(*packages): + return {'errors': errors, 'packages': ret} + + ++def _get_pkg_build_time(name): ++ ''' ++ Get package build time, if possible. ++ ++ :param name: ++ :return: ++ ''' ++ iso_time = iso_time_t = None ++ changelog_dir = os.path.join('/usr/share/doc', name) ++ if os.path.exists(changelog_dir): ++ for fname in os.listdir(changelog_dir): ++ try: ++ iso_time_t = int(os.path.getmtime(os.path.join(changelog_dir, fname))) ++ iso_time = datetime.datetime.utcfromtimestamp(iso_time_t).isoformat() + 'Z' ++ break ++ except OSError: ++ pass ++ ++ # Packager doesn't care about Debian standards, therefore Plan B: brute-force it. ++ if not iso_time: ++ for pkg_f_path in __salt__['cmd.run']('dpkg-query -L {}'.format(name)).splitlines(): ++ if 'changelog' in pkg_f_path.lower() and os.path.exists(pkg_f_path): ++ try: ++ iso_time_t = int(os.path.getmtime(pkg_f_path)) ++ iso_time = datetime.datetime.utcfromtimestamp(iso_time_t).isoformat() + 'Z' ++ break ++ except OSError: ++ pass ++ ++ return iso_time, iso_time_t ++ ++ + def _get_pkg_info(*packages, **kwargs): + ''' + Return list of package information. If 'packages' parameter is empty, +@@ -274,7 +306,7 @@ def _get_pkg_info(*packages, **kwargs): + ret = [] + cmd = "dpkg-query -W -f='package:" + bin_var + "\\n" \ + "revision:${binary:Revision}\\n" \ +- "architecture:${Architecture}\\n" \ ++ "arch:${Architecture}\\n" \ + "maintainer:${Maintainer}\\n" \ + "summary:${Summary}\\n" \ + "source:${source:Package}\\n" \ +@@ -307,9 +339,14 @@ def _get_pkg_info(*packages, **kwargs): + key, value = pkg_info_line.split(":", 1) + if value: + pkg_data[key] = value +- install_date = _get_pkg_install_time(pkg_data.get('package')) +- if install_date: +- pkg_data['install_date'] = install_date ++ install_date, install_date_t = _get_pkg_install_time(pkg_data.get('package'), pkg_data.get('arch')) ++ if install_date: ++ pkg_data['install_date'] = install_date ++ pkg_data['install_date_time_t'] = install_date_t # Unix ticks ++ build_date, build_date_t = _get_pkg_build_time(pkg_data.get('package')) ++ if build_date: ++ pkg_data['build_date'] = build_date ++ pkg_data['build_date_time_t'] = build_date_t + pkg_data['description'] = pkg_descr.split(":", 1)[-1] + ret.append(pkg_data) + +@@ -335,19 +372,32 @@ def _get_pkg_license(pkg): + return ", ".join(sorted(licenses)) + + +-def _get_pkg_install_time(pkg): ++def _get_pkg_install_time(pkg, arch): + ''' + Return package install time, based on the /var/lib/dpkg/info/.list + + :return: + ''' +- iso_time = None ++ iso_time = iso_time_t = None ++ loc_root = '/var/lib/dpkg/info' + if pkg is not None: +- location = "/var/lib/dpkg/info/{0}.list".format(pkg) +- if os.path.exists(location): +- iso_time = datetime.datetime.utcfromtimestamp(int(os.path.getmtime(location))).isoformat() + "Z" ++ locations = [] ++ if arch is not None and arch != 'all': ++ locations.append(os.path.join(loc_root, '{0}:{1}.list'.format(pkg, arch))) ++ ++ locations.append(os.path.join(loc_root, '{0}.list'.format(pkg))) ++ for location in locations: ++ try: ++ iso_time_t = int(os.path.getmtime(location)) ++ iso_time = datetime.datetime.utcfromtimestamp(iso_time_t).isoformat() + 'Z' ++ break ++ except OSError: ++ pass + +- return iso_time ++ if iso_time is None: ++ log.debug('Unable to get package installation time for package "%s".', pkg) ++ ++ return iso_time, iso_time_t + + + def _get_pkg_ds_avail(): +@@ -397,6 +447,15 @@ def info(*packages, **kwargs): + + .. versionadded:: 2016.11.3 + ++ attr ++ Comma-separated package attributes. If no 'attr' is specified, all available attributes returned. ++ ++ Valid attributes are: ++ version, vendor, release, build_date, build_date_time_t, install_date, install_date_time_t, ++ build_host, group, source_rpm, arch, epoch, size, license, signature, packager, url, summary, description. ++ ++ .. versionadded:: Neon ++ + CLI example: + + .. code-block:: bash +@@ -411,6 +470,10 @@ def info(*packages, **kwargs): + + kwargs = salt.utils.args.clean_kwargs(**kwargs) + failhard = kwargs.pop('failhard', True) ++ attr = kwargs.pop('attr', None) or None ++ if attr: ++ attr = attr.split(',') ++ + if kwargs: + salt.utils.args.invalid_kwargs(kwargs) + +@@ -430,6 +493,14 @@ def info(*packages, **kwargs): + lic = _get_pkg_license(pkg['package']) + if lic: + pkg['license'] = lic +- ret[pkg['package']] = pkg ++ ++ # Remove keys that aren't in attrs ++ pkg_name = pkg['package'] ++ if attr: ++ for k in list(pkg.keys())[:]: ++ if k not in attr: ++ del pkg[k] ++ ++ ret[pkg_name] = pkg + + return ret +diff --git a/tests/unit/modules/test_aptpkg.py b/tests/unit/modules/test_aptpkg.py +index c0e26cfcd4..5352e39982 100644 +--- a/tests/unit/modules/test_aptpkg.py ++++ b/tests/unit/modules/test_aptpkg.py +@@ -13,12 +13,14 @@ import copy + # Import Salt Testing Libs + from tests.support.mixins import LoaderModuleMockMixin + from tests.support.unit import TestCase, skipIf +-from tests.support.mock import MagicMock, patch, NO_MOCK, NO_MOCK_REASON ++from tests.support.mock import Mock, MagicMock, patch, NO_MOCK, NO_MOCK_REASON + + # Import Salt Libs + from salt.ext import six + from salt.exceptions import CommandExecutionError, SaltInvocationError + import salt.modules.aptpkg as aptpkg ++import pytest ++import textwrap + + + APT_KEY_LIST = r''' +@@ -142,51 +144,39 @@ class AptPkgTestCase(TestCase, LoaderModuleMockMixin): + def setup_loader_modules(self): + return {aptpkg: {}} + ++ @patch('salt.modules.aptpkg.__salt__', ++ {'pkg_resource.version': MagicMock(return_value=LOWPKG_INFO['wget']['version'])}) + def test_version(self): + ''' + Test - Returns a string representing the package version or an empty string if + not installed. + ''' +- version = LOWPKG_INFO['wget']['version'] +- mock = MagicMock(return_value=version) +- with patch.dict(aptpkg.__salt__, {'pkg_resource.version': mock}): +- self.assertEqual(aptpkg.version(*['wget']), version) ++ assert aptpkg.version(*['wget']) == aptpkg.__salt__['pkg_resource.version']() + ++ @patch('salt.modules.aptpkg.latest_version', MagicMock(return_value='')) + def test_upgrade_available(self): + ''' + Test - Check whether or not an upgrade is available for a given package. + ''' +- with patch('salt.modules.aptpkg.latest_version', +- MagicMock(return_value='')): +- self.assertFalse(aptpkg.upgrade_available('wget')) ++ assert not aptpkg.upgrade_available('wget') + ++ @patch('salt.modules.aptpkg.get_repo_keys', MagicMock(return_value=REPO_KEYS)) ++ @patch('salt.modules.aptpkg.__salt__', {'cmd.run_all': MagicMock(return_value={'retcode': 0, 'stdout': 'OK'})}) + def test_add_repo_key(self): + ''' + Test - Add a repo key. + ''' +- with patch('salt.modules.aptpkg.get_repo_keys', +- MagicMock(return_value=REPO_KEYS)): +- mock = MagicMock(return_value={ +- 'retcode': 0, +- 'stdout': 'OK' +- }) +- with patch.dict(aptpkg.__salt__, {'cmd.run_all': mock}): +- self.assertTrue(aptpkg.add_repo_key(keyserver='keyserver.ubuntu.com', +- keyid='FBB75451')) ++ assert aptpkg.add_repo_key(keyserver='keyserver.ubuntu.com', keyid='FBB75451') + ++ @patch('salt.modules.aptpkg.get_repo_keys', MagicMock(return_value=REPO_KEYS)) ++ @patch('salt.modules.aptpkg.__salt__', {'cmd.run_all': MagicMock(return_value={'retcode': 0, 'stdout': 'OK'})}) + def test_add_repo_key_failed(self): + ''' + Test - Add a repo key using incomplete input data. + ''' +- with patch('salt.modules.aptpkg.get_repo_keys', +- MagicMock(return_value=REPO_KEYS)): +- kwargs = {'keyserver': 'keyserver.ubuntu.com'} +- mock = MagicMock(return_value={ +- 'retcode': 0, +- 'stdout': 'OK' +- }) +- with patch.dict(aptpkg.__salt__, {'cmd.run_all': mock}): +- self.assertRaises(SaltInvocationError, aptpkg.add_repo_key, **kwargs) ++ with pytest.raises(SaltInvocationError) as ex: ++ aptpkg.add_repo_key(keyserver='keyserver.ubuntu.com') ++ assert ' No keyid or keyid too short for keyserver: keyserver.ubuntu.com' in str(ex) + + def test_get_repo_keys(self): + ''' +@@ -199,35 +189,31 @@ class AptPkgTestCase(TestCase, LoaderModuleMockMixin): + with patch.dict(aptpkg.__salt__, {'cmd.run_all': mock}): + self.assertEqual(aptpkg.get_repo_keys(), REPO_KEYS) + ++ @patch('salt.modules.aptpkg.__salt__', {'lowpkg.file_dict': MagicMock(return_value=LOWPKG_FILES)}) + def test_file_dict(self): + ''' + Test - List the files that belong to a package, grouped by package. + ''' +- mock = MagicMock(return_value=LOWPKG_FILES) +- with patch.dict(aptpkg.__salt__, {'lowpkg.file_dict': mock}): +- self.assertEqual(aptpkg.file_dict('wget'), LOWPKG_FILES) ++ assert aptpkg.file_dict('wget') == LOWPKG_FILES + ++ @patch('salt.modules.aptpkg.__salt__', { ++ 'lowpkg.file_list': MagicMock(return_value={'errors': LOWPKG_FILES['errors'], ++ 'files': LOWPKG_FILES['packages']['wget']})}) + def test_file_list(self): + ''' +- Test - List the files that belong to a package. ++ Test 'file_list' function, which is just an alias to the lowpkg 'file_list' ++ + ''' +- files = { +- 'errors': LOWPKG_FILES['errors'], +- 'files': LOWPKG_FILES['packages']['wget'], +- } +- mock = MagicMock(return_value=files) +- with patch.dict(aptpkg.__salt__, {'lowpkg.file_list': mock}): +- self.assertEqual(aptpkg.file_list('wget'), files) ++ assert aptpkg.file_list('wget') == aptpkg.__salt__['lowpkg.file_list']() + ++ @patch('salt.modules.aptpkg.__salt__', {'cmd.run_stdout': MagicMock(return_value='wget\t\t\t\t\t\tinstall')}) + def test_get_selections(self): + ''' + Test - View package state from the dpkg database. + ''' +- selections = {'install': ['wget']} +- mock = MagicMock(return_value='wget\t\t\t\t\t\tinstall') +- with patch.dict(aptpkg.__salt__, {'cmd.run_stdout': mock}): +- self.assertEqual(aptpkg.get_selections('wget'), selections) ++ assert aptpkg.get_selections('wget') == {'install': ['wget']} + ++ @patch('salt.modules.aptpkg.__salt__', {'lowpkg.info': MagicMock(return_value=LOWPKG_INFO)}) + def test_info_installed(self): + ''' + Test - Return the information of the named package(s) installed on the system. +@@ -243,19 +229,72 @@ class AptPkgTestCase(TestCase, LoaderModuleMockMixin): + if installed['wget'].get(names[name], False): + installed['wget'][name] = installed['wget'].pop(names[name]) + +- mock = MagicMock(return_value=LOWPKG_INFO) +- with patch.dict(aptpkg.__salt__, {'lowpkg.info': mock}): +- self.assertEqual(aptpkg.info_installed('wget'), installed) ++ assert aptpkg.info_installed('wget') == installed ++ ++ @patch('salt.modules.aptpkg.__salt__', {'lowpkg.info': MagicMock(return_value=LOWPKG_INFO)}) ++ def test_info_installed_attr(self): ++ ''' ++ Test info_installed 'attr'. ++ This doesn't test 'attr' behaviour per se, since the underlying function is in dpkg. ++ The test should simply not raise exceptions for invalid parameter. ++ ++ :return: ++ ''' ++ ret = aptpkg.info_installed('emacs', attr='foo,bar') ++ assert isinstance(ret, dict) ++ assert 'wget' in ret ++ assert isinstance(ret['wget'], dict) ++ ++ wget_pkg = ret['wget'] ++ expected_pkg = {'url': 'http://www.gnu.org/software/wget/', ++ 'packager': 'Ubuntu Developers ', 'name': 'wget', ++ 'install_date': '2016-08-30T22:20:15Z', 'description': 'retrieves files from the web', ++ 'version': '1.15-1ubuntu1.14.04.2', 'architecture': 'amd64', 'group': 'web', 'source': 'wget'} ++ for k in wget_pkg: ++ assert k in expected_pkg ++ assert wget_pkg[k] == expected_pkg[k] ++ ++ @patch('salt.modules.aptpkg.__salt__', {'lowpkg.info': MagicMock(return_value=LOWPKG_INFO)}) ++ def test_info_installed_all_versions(self): ++ ''' ++ Test info_installed 'all_versions'. ++ Since Debian won't return same name packages with the different names, ++ this should just return different structure, backward compatible with ++ the RPM equivalents. ++ ++ :return: ++ ''' ++ print() ++ ret = aptpkg.info_installed('emacs', all_versions=True) ++ assert isinstance(ret, dict) ++ assert 'wget' in ret ++ assert isinstance(ret['wget'], list) + ++ pkgs = ret['wget'] ++ ++ assert len(pkgs) == 1 ++ assert isinstance(pkgs[0], dict) ++ ++ wget_pkg = pkgs[0] ++ expected_pkg = {'url': 'http://www.gnu.org/software/wget/', ++ 'packager': 'Ubuntu Developers ', 'name': 'wget', ++ 'install_date': '2016-08-30T22:20:15Z', 'description': 'retrieves files from the web', ++ 'version': '1.15-1ubuntu1.14.04.2', 'architecture': 'amd64', 'group': 'web', 'source': 'wget'} ++ for k in wget_pkg: ++ assert k in expected_pkg ++ assert wget_pkg[k] == expected_pkg[k] ++ ++ @patch('salt.modules.aptpkg.__salt__', {'cmd.run_stdout': MagicMock(return_value='wget: /usr/bin/wget')}) + def test_owner(self): + ''' + Test - Return the name of the package that owns the file. + ''' +- paths = ['/usr/bin/wget'] +- mock = MagicMock(return_value='wget: /usr/bin/wget') +- with patch.dict(aptpkg.__salt__, {'cmd.run_stdout': mock}): +- self.assertEqual(aptpkg.owner(*paths), 'wget') ++ assert aptpkg.owner('/usr/bin/wget') == 'wget' + ++ @patch('salt.utils.pkg.clear_rtag', MagicMock()) ++ @patch('salt.modules.aptpkg.__salt__', {'cmd.run_all': MagicMock(return_value={'retcode': 0, ++ 'stdout': APT_Q_UPDATE}), ++ 'config.get': MagicMock(return_value=False)}) + def test_refresh_db(self): + ''' + Test - Updates the APT database to latest packages based upon repositories. +@@ -267,26 +306,20 @@ class AptPkgTestCase(TestCase, LoaderModuleMockMixin): + 'http://security.ubuntu.com trusty-security/main amd64 Packages': True, + 'http://security.ubuntu.com trusty-security/main i386 Packages': True + } +- mock = MagicMock(return_value={ +- 'retcode': 0, +- 'stdout': APT_Q_UPDATE +- }) +- with patch('salt.utils.pkg.clear_rtag', MagicMock()): +- with patch.dict(aptpkg.__salt__, {'cmd.run_all': mock}): +- self.assertEqual(aptpkg.refresh_db(), refresh_db) ++ assert aptpkg.refresh_db() == refresh_db + ++ @patch('salt.utils.pkg.clear_rtag', MagicMock()) ++ @patch('salt.modules.aptpkg.__salt__', {'cmd.run_all': MagicMock(return_value={'retcode': 0, ++ 'stdout': APT_Q_UPDATE_ERROR}), ++ 'config.get': MagicMock(return_value=False)}) + def test_refresh_db_failed(self): + ''' + Test - Update the APT database using unreachable repositories. + ''' +- kwargs = {'failhard': True} +- mock = MagicMock(return_value={ +- 'retcode': 0, +- 'stdout': APT_Q_UPDATE_ERROR +- }) +- with patch('salt.utils.pkg.clear_rtag', MagicMock()): +- with patch.dict(aptpkg.__salt__, {'cmd.run_all': mock}): +- self.assertRaises(CommandExecutionError, aptpkg.refresh_db, **kwargs) ++ with pytest.raises(CommandExecutionError) as err: ++ aptpkg.refresh_db(failhard=True) ++ assert 'Error getting repos' in str(err) ++ assert 'http://security.ubuntu.com trusty InRelease, http://security.ubuntu.com trusty Release.gpg' in str(err) + + def test_autoremove(self): + ''' +@@ -306,38 +339,26 @@ class AptPkgTestCase(TestCase, LoaderModuleMockMixin): + self.assertEqual(aptpkg.autoremove(list_only=True), list()) + self.assertEqual(aptpkg.autoremove(list_only=True, purge=True), list()) + ++ @patch('salt.modules.aptpkg._uninstall', MagicMock(return_value=UNINSTALL)) + def test_remove(self): + ''' + Test - Remove packages. + ''' +- with patch('salt.modules.aptpkg._uninstall', +- MagicMock(return_value=UNINSTALL)): +- self.assertEqual(aptpkg.remove(name='tmux'), UNINSTALL) ++ assert aptpkg.remove(name='tmux') == UNINSTALL + ++ @patch('salt.modules.aptpkg._uninstall', MagicMock(return_value=UNINSTALL)) + def test_purge(self): + ''' + Test - Remove packages along with all configuration files. + ''' +- with patch('salt.modules.aptpkg._uninstall', +- MagicMock(return_value=UNINSTALL)): +- self.assertEqual(aptpkg.purge(name='tmux'), UNINSTALL) ++ assert aptpkg.purge(name='tmux') == UNINSTALL + ++ @patch('salt.utils.pkg.clear_rtag', MagicMock()) ++ @patch('salt.modules.aptpkg.list_pkgs', MagicMock(return_value=UNINSTALL)) ++ @patch.multiple(aptpkg, **{'__salt__': {'config.get': MagicMock(return_value=True), ++ 'cmd.run_all': MagicMock(return_value={'retcode': 0, 'stdout': UPGRADE})}}) + def test_upgrade(self): + ''' + Test - Upgrades all packages. + ''' +- with patch('salt.utils.pkg.clear_rtag', MagicMock()): +- with patch('salt.modules.aptpkg.list_pkgs', +- MagicMock(return_value=UNINSTALL)): +- mock_cmd = MagicMock(return_value={ +- 'retcode': 0, +- 'stdout': UPGRADE +- }) +- patch_kwargs = { +- '__salt__': { +- 'config.get': MagicMock(return_value=True), +- 'cmd.run_all': mock_cmd +- } +- } +- with patch.multiple(aptpkg, **patch_kwargs): +- self.assertEqual(aptpkg.upgrade(), dict()) ++ assert aptpkg.upgrade() == {} +diff --git a/tests/unit/modules/test_dpkg.py b/tests/unit/modules/test_dpkg.py +index fcfa7caf77..1acfd89ccf 100644 +--- a/tests/unit/modules/test_dpkg.py ++++ b/tests/unit/modules/test_dpkg.py +@@ -25,6 +25,30 @@ class DpkgTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test cases for salt.modules.dpkg + ''' ++ dselect_pkg = { ++ 'emacs': {'priority': 'optional', 'filename': 'pool/main/e/emacs-defaults/emacs_46.1_all.deb', ++ 'description': 'GNU Emacs editor (metapackage)', 'md5sum': '766eb2cee55ba0122dac64c4cea04445', ++ 'sha256': 'd172289b9a1608820eddad85c7ffc15f346a6e755c3120de0f64739c4bbc44ce', ++ 'description-md5': '21fb7da111336097a2378959f6d6e6a8', ++ 'bugs': 'https://bugs.launchpad.net/springfield/+filebug', ++ 'depends': 'emacs24 | emacs24-lucid | emacs24-nox', 'origin': 'Simpsons', 'version': '46.1', ++ 'task': 'ubuntu-usb, edubuntu-usb', 'original-maintainer': 'Homer Simpson ', ++ 'package': 'emacs', 'architecture': 'all', 'size': '1692', ++ 'sha1': '9271bcec53c1f7373902b1e594d9fc0359616407', 'source': 'emacs-defaults', ++ 'maintainer': 'Simpsons Developers ', 'supported': '9m', ++ 'section': 'editors', 'installed-size': '25'} ++ } ++ ++ pkgs_info = [ ++ {'version': '46.1', 'arch': 'all', 'build_date': '2014-08-07T16:51:48Z', 'install_date_time_t': 1481745778, ++ 'section': 'editors', 'description': 'GNU Emacs editor (metapackage)\n GNU Emacs is the extensible ' ++ 'self-documenting text editor.\n This is a metapackage that will always ' ++ 'depend on the latest\n recommended Emacs release.\n', ++ 'package': 'emacs', 'source': 'emacs-defaults', ++ 'maintainer': 'Simpsons Developers ', ++ 'build_date_time_t': 1407430308, 'installed_size': '25', 'install_date': '2016-12-14T20:02:58Z'} ++ ] ++ + def setup_loader_modules(self): + return {dpkg: {}} + +@@ -102,3 +126,48 @@ class DpkgTestCase(TestCase, LoaderModuleMockMixin): + 'stdout': 'Salt'}) + with patch.dict(dpkg.__salt__, {'cmd.run_all': mock}): + self.assertEqual(dpkg.file_dict('httpd'), 'Error: error') ++ ++ @patch('salt.modules.dpkg._get_pkg_ds_avail', MagicMock(return_value=dselect_pkg)) ++ @patch('salt.modules.dpkg._get_pkg_info', MagicMock(return_value=pkgs_info)) ++ @patch('salt.modules.dpkg._get_pkg_license', MagicMock(return_value='BSD v3')) ++ def test_info(self): ++ ''' ++ Test info ++ :return: ++ ''' ++ ret = dpkg.info('emacs') ++ ++ assert isinstance(ret, dict) ++ assert len(ret.keys()) == 1 ++ assert 'emacs' in ret ++ ++ pkg_data = ret['emacs'] ++ ++ assert isinstance(pkg_data, dict) ++ for pkg_section in ['section', 'architecture', 'original-maintainer', 'maintainer', 'package', 'installed-size', ++ 'build_date_time_t', 'sha256', 'origin', 'build_date', 'size', 'source', 'version', ++ 'install_date_time_t', 'license', 'priority', 'description', 'md5sum', 'supported', ++ 'filename', 'sha1', 'install_date', 'arch']: ++ assert pkg_section in pkg_data ++ ++ assert pkg_data['section'] == 'editors' ++ assert pkg_data['maintainer'] == 'Simpsons Developers ' ++ assert pkg_data['license'] == 'BSD v3' ++ ++ @patch('salt.modules.dpkg._get_pkg_ds_avail', MagicMock(return_value=dselect_pkg)) ++ @patch('salt.modules.dpkg._get_pkg_info', MagicMock(return_value=pkgs_info)) ++ @patch('salt.modules.dpkg._get_pkg_license', MagicMock(return_value='BSD v3')) ++ def test_info_attr(self): ++ ''' ++ Test info with 'attr' parameter ++ :return: ++ ''' ++ ret = dpkg.info('emacs', attr='arch,license,version') ++ assert isinstance(ret, dict) ++ assert 'emacs' in ret ++ for attr in ['arch', 'license', 'version']: ++ assert attr in ret['emacs'] ++ ++ assert ret['emacs']['arch'] == 'all' ++ assert ret['emacs']['license'] == 'BSD v3' ++ assert ret['emacs']['version'] == '46.1' +-- +2.19.1 + + diff --git a/decide-if-the-source-should-be-actually-skipped.patch b/decide-if-the-source-should-be-actually-skipped.patch new file mode 100644 index 0000000..815788a --- /dev/null +++ b/decide-if-the-source-should-be-actually-skipped.patch @@ -0,0 +1,54 @@ +From 5eacdf8fef35cdd05cae1b65485b3f820c86bc68 Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Tue, 4 Dec 2018 16:39:08 +0100 +Subject: [PATCH] Decide if the source should be actually skipped + +--- + salt/modules/aptpkg.py | 23 ++++++++++++++++++++++- + 1 file changed, 22 insertions(+), 1 deletion(-) + +diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py +index dc27903230..42d606926f 100644 +--- a/salt/modules/aptpkg.py ++++ b/salt/modules/aptpkg.py +@@ -1698,6 +1698,27 @@ def list_repo_pkgs(*args, **kwargs): # pylint: disable=unused-import + return ret + + ++def _skip_source(source): ++ ''' ++ Decide to skip source or not. ++ ++ :param source: ++ :return: ++ ''' ++ if source.invalid: ++ if source.uri and source.type and source.type in ("deb", "deb-src", "rpm", "rpm-src"): ++ pieces = source.mysplit(source.line) ++ if pieces[1].strip()[0] == "[": ++ options = pieces.pop(1).strip("[]").split() ++ if len(options) > 0: ++ log.debug("Source %s will be included although is marked invalid", source.uri) ++ return False ++ return True ++ else: ++ return True ++ return False ++ ++ + def list_repos(): + ''' + Lists all repos in the sources.list (and sources.lists.d) files +@@ -1713,7 +1734,7 @@ def list_repos(): + repos = {} + sources = sourceslist.SourcesList() + for source in sources.list: +- if source.invalid: ++ if _skip_source(source): + continue + repo = {} + repo['file'] = source.file +-- +2.20.1 + + diff --git a/do-not-load-pip-state-if-there-is-no-3rd-party-depen.patch b/do-not-load-pip-state-if-there-is-no-3rd-party-depen.patch new file mode 100644 index 0000000..0149d8e --- /dev/null +++ b/do-not-load-pip-state-if-there-is-no-3rd-party-depen.patch @@ -0,0 +1,73 @@ +From 7727ab13e3492b722b316469cc912d9dd64f063e Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Fri, 21 Sep 2018 17:31:39 +0200 +Subject: [PATCH] Do not load pip state if there is no 3rd party + dependencies + +Safe import 3rd party dependency +--- + salt/modules/pip.py | 12 ++++++++++-- + salt/states/pip_state.py | 9 +++++---- + 2 files changed, 15 insertions(+), 6 deletions(-) + +diff --git a/salt/modules/pip.py b/salt/modules/pip.py +index f1a2e42433..85844f098b 100644 +--- a/salt/modules/pip.py ++++ b/salt/modules/pip.py +@@ -79,7 +79,10 @@ from __future__ import absolute_import, print_function, unicode_literals + # Import python libs + import logging + import os +-import pkg_resources ++try: ++ import pkg_resources ++except ImportError: ++ pkg_resources = None + import re + import shutil + import sys +@@ -116,7 +119,12 @@ def __virtual__(): + entire filesystem. If it's not installed in a conventional location, the + user is required to provide the location of pip each time it is used. + ''' +- return 'pip' ++ if pkg_resources is None: ++ ret = False, 'Package dependency "pkg_resource" is missing' ++ else: ++ ret = 'pip' ++ ++ return ret + + + def _clear_context(bin_env=None): +diff --git a/salt/states/pip_state.py b/salt/states/pip_state.py +index ab58fbd5fc..afe41d7fc8 100644 +--- a/salt/states/pip_state.py ++++ b/salt/states/pip_state.py +@@ -23,7 +23,10 @@ requisite to a pkg.installed state for the package which provides pip + from __future__ import absolute_import, print_function, unicode_literals + import re + import logging +-import pkg_resources ++try: ++ import pkg_resources ++except ImportError: ++ pkg_resources = None + + # Import salt libs + import salt.utils.versions +@@ -71,9 +74,7 @@ def __virtual__(): + ''' + Only load if the pip module is available in __salt__ + ''' +- if 'pip.list' in __salt__: +- return __virtualname__ +- return False ++ return 'pip.list' in __salt__ and __virtualname__ or False + + + def _find_key(prefix, pip_list): +-- +2.19.0 + + diff --git a/don-t-error-on-retcode-0-in-libcrypto.openssl_init_c.patch b/don-t-error-on-retcode-0-in-libcrypto.openssl_init_c.patch new file mode 100644 index 0000000..0a754c4 --- /dev/null +++ b/don-t-error-on-retcode-0-in-libcrypto.openssl_init_c.patch @@ -0,0 +1,33 @@ +From 34089db15e7d3a1e361789f04613d0a13138dea0 Mon Sep 17 00:00:00 2001 +From: rallytime +Date: Fri, 13 Jul 2018 12:42:46 -0400 +Subject: [PATCH] Don't error on retcode 0 in + libcrypto.OPENSSL_init_crypto call + +Fixes #46884 +--- + salt/utils/rsax931.py | 7 +++---- + 1 file changed, 3 insertions(+), 4 deletions(-) + +diff --git a/salt/utils/rsax931.py b/salt/utils/rsax931.py +index 168c02734b..6bfef41bd3 100644 +--- a/salt/utils/rsax931.py ++++ b/salt/utils/rsax931.py +@@ -71,10 +71,9 @@ def _init_libcrypto(): + libcrypto.RSA_public_decrypt.argtypes = (c_int, c_char_p, c_char_p, c_void_p, c_int) + + try: +- if libcrypto.OPENSSL_init_crypto(OPENSSL_INIT_NO_LOAD_CONFIG | +- OPENSSL_INIT_ADD_ALL_CIPHERS | +- OPENSSL_INIT_ADD_ALL_DIGESTS, None) != 1: +- raise OSError("Failed to initialize OpenSSL library (OPENSSL_init_crypto failed)") ++ libcrypto.OPENSSL_init_crypto(OPENSSL_INIT_NO_LOAD_CONFIG | ++ OPENSSL_INIT_ADD_ALL_CIPHERS | ++ OPENSSL_INIT_ADD_ALL_DIGESTS, None) + except AttributeError: + # Support for OpenSSL < 1.1 (OPENSSL_API_COMPAT < 0x10100000L) + libcrypto.OPENSSL_no_config() +-- +2.19.2 + + diff --git a/early-feature-support-config.patch b/early-feature-support-config.patch new file mode 100644 index 0000000..39bf87f --- /dev/null +++ b/early-feature-support-config.patch @@ -0,0 +1,1996 @@ +From 7c72c080c6b47cccecdf058cfe1be8f465d568a6 Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Tue, 10 Jul 2018 12:06:33 +0200 +Subject: [PATCH] early feature: support-config + +Add support script function + +Add salt-support starter + +Initial support wrapper + +Add data collector skeleton + +Add default scenario of the support configuration + +Add main flow for the collector. + +Move support library to its own package + +Add default support collection scenario + +Add logging + +Handle CLI error. + +Update format of the default support scenario + +Default archive name + +Finalise local data collection + +Write archive from memory objects. + +Add colored console outputter for salt-support. + +Use colored outputter + +Add message output class + +Remove try/except capture from the scripts and move to the runner directly + +Implement output highlighter methods for CLI output + +Move scenarios to profiles + +Get return section from the output. Tolerate raw data. + +Implement internal data collector + +Add network stack examination to the default profile + +Add an internal filetree function + +Add a method to discard current session + +Add a method to link a static file to the resulting archive + +Implement internal function caller + +Add internal functions + +Add default root for the one-file support data + +Set output device + +Separate dynamic data and static files on the fs + +Update color theme + +Add ident to the error message + +Report rejected files with the ident + +Reuse system error exceptions and reduce stat on the file check + +Use socket name of the host machine + +Add options for profile and archive settings + +Use archive name from options. + +Get profile by config/options + +Cleanup broken archive on crash/exception + +Use profile from the options/configuration + +Add more colored messages :-) + +Initial implementation of get static profiles + +Update docstring + +Move PostgreSQL profile to its own + +Handle profile listing, do not yield sys.exit on specific module + +Add network profile + +Add Salt's profile + +Uncomment package profile + +Allow several profiles to be specified + +Remove comments, add parameter to get more profiles + +Implement existing configuration finder + +Add options to handle unit configurations + +Pre-parse options prior run() to choose proper configuration target + +Handle arg parse generic errors, unit mis-choose + +Let cleanup be aware of pre-config state + +Fix imports + +Handle exit codes properly + +Allow to overwrite existing archive + +Use py2/3 exceptions equally + +Include exit exception on debugging + +Render profiles as Jinja2, add basic recursive caller to the template of the profile + +Add "users" profile + +Implement basic caller for the profile template + +Add table output renderer + +Fix typo + +Remove table outputter + +Allow default outputters and specify outputters inside the profile + +Remove group.getent from the loop per each user + +Add table outputter to network profile + +Add text outputter to hostname/fqdn data + +Remove network part from the default profile. Add text/table outputters. + +Fix Py3 compat + +Collect status (initial) + +Avoid irrelevant to profile files + +Add job profiles + +Add profile template trace + +Add inspection through the runners + +Allow parameters in callers and runners + +Handle non-dict iterables + +Highlight template content in the trace log + +Add return extractor from the local call returns + +Move local runner to its own namespace + +Lintfix: PEP8 + +Remove duplicate code + +Fix caller return + +Add description tag to the scenario + +Add generic colored message + +Add wrapping function. NOTE: it should be refactored with the other similar functions + +Print description while processing the scenario + +Turn off default profile and print help instead + +Move command-line check before collector + +Do not verify archive if help needs to be printed + +Add console output unit test for indent output + +Fix docstring + +Rename test class + +Refactor test to add setup/teardown + +Add unit test to verify indent + +Use direct constants instead of encoded strings + +Add unit test for color indent rotation check + +Add a test case for Collector class + +Add unit test for closing the archive + +Add unit test for add/write sections on the collector object + +Add test for linking an external file + +Cleanup tests on tear-down method + +Add call count check + +Add unit test for support collection section discard + +Add unittest for SaltSupport's function config preparation + +Fix docstring + +Add unit test for local caller + +Add unit test for local runner + +Add unit test for internal function call + +Add unit test for getting an action description from the action meta + +Add unit test for internal function call + +Add unit test for return extration + +Add unit test for determine action type from the action meta + +Add unit test for cleanup routine + +Fix typo of method name + +Add unit test for check existing archive + +Add test suite for profile testing + +Add unit test for default profile is YAML-parseable + +Add unit test for user template profile rendering + +Update unit test for all non-template profiles parse check + +Add function to render a Jinja2 template by name + +Use template rendering function + +Add unit test on jobs-trace template for runner + +Move function above the tests + +Add current logfile, if defined in configuration + +Bugfix: ignore logfile, if path was not found or not defined or is None + +Lintfix: iteration over .keys() + +Remove template "salt" from non-template checks + +Lintfix: use salt.utils.files.fopen for resource leak prevention + +Lintfix: PEP8 E302: expected 2 blank lines, found 0 + +Lintfix: use salt.utils.files.fopen instead of open + +Lintfix: PEP8 E303: too many blank lines (3) + +Lintfix: Uses of an external blacklisted import 'six': Please use 'import salt.ext.six as six' + +Lintfix: use salt.utils.files.fopen instead of open + +Fix unit tests + +Fix six import + +Mute pylint: file handler explicitly needed + +Lintfix: explicitly close filehandle + +Lintfix: mute fopen warning + +Remove development stub. Ughh... + +Removed blacklist of pkg_resources +--- + salt/cli/support/__init__.py | 65 +++ + salt/cli/support/collector.py | 495 ++++++++++++++++++++++ + salt/cli/support/console.py | 165 ++++++++ + salt/cli/support/intfunc.py | 42 ++ + salt/cli/support/localrunner.py | 34 ++ + salt/cli/support/profiles/default.yml | 71 ++++ + salt/cli/support/profiles/jobs-active.yml | 3 + + salt/cli/support/profiles/jobs-last.yml | 3 + + salt/cli/support/profiles/jobs-trace.yml | 7 + + salt/cli/support/profiles/network.yml | 27 ++ + salt/cli/support/profiles/postgres.yml | 11 + + salt/cli/support/profiles/salt.yml | 9 + + salt/cli/support/profiles/users.yml | 22 + + salt/scripts.py | 14 + + salt/utils/parsers.py | 65 +++ + scripts/salt-support | 11 + + setup.py | 2 + + tests/unit/cli/test_support.py | 477 +++++++++++++++++++++ + 18 files changed, 1523 insertions(+) + create mode 100644 salt/cli/support/__init__.py + create mode 100644 salt/cli/support/collector.py + create mode 100644 salt/cli/support/console.py + create mode 100644 salt/cli/support/intfunc.py + create mode 100644 salt/cli/support/localrunner.py + create mode 100644 salt/cli/support/profiles/default.yml + create mode 100644 salt/cli/support/profiles/jobs-active.yml + create mode 100644 salt/cli/support/profiles/jobs-last.yml + create mode 100644 salt/cli/support/profiles/jobs-trace.yml + create mode 100644 salt/cli/support/profiles/network.yml + create mode 100644 salt/cli/support/profiles/postgres.yml + create mode 100644 salt/cli/support/profiles/salt.yml + create mode 100644 salt/cli/support/profiles/users.yml + create mode 100755 scripts/salt-support + create mode 100644 tests/unit/cli/test_support.py + +diff --git a/salt/cli/support/__init__.py b/salt/cli/support/__init__.py +new file mode 100644 +index 0000000000..6a98a2d656 +--- /dev/null ++++ b/salt/cli/support/__init__.py +@@ -0,0 +1,65 @@ ++# coding=utf-8 ++''' ++Get default scenario of the support. ++''' ++from __future__ import print_function, unicode_literals, absolute_import ++import yaml ++import os ++import salt.exceptions ++import jinja2 ++import logging ++ ++log = logging.getLogger(__name__) ++ ++ ++def _render_profile(path, caller, runner): ++ ''' ++ Render profile as Jinja2. ++ :param path: ++ :return: ++ ''' ++ env = jinja2.Environment(loader=jinja2.FileSystemLoader(os.path.dirname(path)), trim_blocks=False) ++ return env.get_template(os.path.basename(path)).render(salt=caller, runners=runner).strip() ++ ++ ++def get_profile(profile, caller, runner): ++ ''' ++ Get profile. ++ ++ :param profile: ++ :return: ++ ''' ++ profiles = profile.split(',') ++ data = {} ++ for profile in profiles: ++ if os.path.basename(profile) == profile: ++ profile = profile.split('.')[0] # Trim extension if someone added it ++ profile_path = os.path.join(os.path.dirname(__file__), 'profiles', profile + '.yml') ++ else: ++ profile_path = profile ++ if os.path.exists(profile_path): ++ try: ++ rendered_template = _render_profile(profile_path, caller, runner) ++ log.trace('\n{d}\n{t}\n{d}\n'.format(d='-' * 80, t=rendered_template)) ++ data.update(yaml.load(rendered_template)) ++ except Exception as ex: ++ log.debug(ex, exc_info=True) ++ raise salt.exceptions.SaltException('Rendering profile failed: {}'.format(ex)) ++ else: ++ raise salt.exceptions.SaltException('Profile "{}" is not found.'.format(profile)) ++ ++ return data ++ ++ ++def get_profiles(config): ++ ''' ++ Get available profiles. ++ ++ :return: ++ ''' ++ profiles = [] ++ for profile_name in os.listdir(os.path.join(os.path.dirname(__file__), 'profiles')): ++ if profile_name.endswith('.yml'): ++ profiles.append(profile_name.split('.')[0]) ++ ++ return sorted(profiles) +diff --git a/salt/cli/support/collector.py b/salt/cli/support/collector.py +new file mode 100644 +index 0000000000..478d07e13b +--- /dev/null ++++ b/salt/cli/support/collector.py +@@ -0,0 +1,495 @@ ++# coding=utf-8 ++from __future__ import absolute_import, print_function, unicode_literals ++import os ++import sys ++import copy ++import yaml ++import json ++import logging ++import tarfile ++import time ++import salt.ext.six as six ++ ++if six.PY2: ++ import exceptions ++else: ++ import builtins as exceptions ++ from io import IOBase as file ++ ++from io import BytesIO ++ ++import salt.utils.stringutils ++import salt.utils.parsers ++import salt.utils.verify ++import salt.utils.platform ++import salt.utils.process ++import salt.exceptions ++import salt.defaults.exitcodes ++import salt.cli.caller ++import salt.cli.support ++import salt.cli.support.console ++import salt.cli.support.intfunc ++import salt.cli.support.localrunner ++import salt.output.table_out ++import salt.runner ++import salt.utils.files ++ ++ ++salt.output.table_out.__opts__ = {} ++log = logging.getLogger(__name__) ++ ++ ++class SupportDataCollector(object): ++ ''' ++ Data collector. It behaves just like another outputter, ++ except it grabs the data to the archive files. ++ ''' ++ def __init__(self, name, output): ++ ''' ++ constructor of the data collector ++ :param name: ++ :param path: ++ :param format: ++ ''' ++ self.archive_path = name ++ self.__default_outputter = output ++ self.__format = format ++ self.__arch = None ++ self.__current_section = None ++ self.__current_section_name = None ++ self.__default_root = time.strftime('%Y.%m.%d-%H.%M.%S-snapshot') ++ self.out = salt.cli.support.console.MessagesOutput() ++ ++ def open(self): ++ ''' ++ Opens archive. ++ :return: ++ ''' ++ if self.__arch is not None: ++ raise salt.exceptions.SaltException('Archive already opened.') ++ self.__arch = tarfile.TarFile.bz2open(self.archive_path, 'w') ++ ++ def close(self): ++ ''' ++ Closes the archive. ++ :return: ++ ''' ++ if self.__arch is None: ++ raise salt.exceptions.SaltException('Archive already closed') ++ self._flush_content() ++ self.__arch.close() ++ self.__arch = None ++ ++ def _flush_content(self): ++ ''' ++ Flush content to the archive ++ :return: ++ ''' ++ if self.__current_section is not None: ++ buff = BytesIO() ++ buff._dirty = False ++ for action_return in self.__current_section: ++ for title, ret_data in action_return.items(): ++ if isinstance(ret_data, file): ++ self.out.put(ret_data.name, indent=4) ++ self.__arch.add(ret_data.name, arcname=ret_data.name) ++ else: ++ buff.write(salt.utils.stringutils.to_bytes(title + '\n')) ++ buff.write(salt.utils.stringutils.to_bytes(('-' * len(title)) + '\n\n')) ++ buff.write(salt.utils.stringutils.to_bytes(ret_data)) ++ buff.write(salt.utils.stringutils.to_bytes('\n\n\n')) ++ buff._dirty = True ++ if buff._dirty: ++ buff.seek(0) ++ tar_info = tarfile.TarInfo(name="{}/{}".format(self.__default_root, self.__current_section_name)) ++ if not hasattr(buff, 'getbuffer'): # Py2's BytesIO is older ++ buff.getbuffer = buff.getvalue ++ tar_info.size = len(buff.getbuffer()) ++ self.__arch.addfile(tarinfo=tar_info, fileobj=buff) ++ ++ def add(self, name): ++ ''' ++ Start a new section. ++ :param name: ++ :return: ++ ''' ++ if self.__current_section: ++ self._flush_content() ++ self.discard_current(name) ++ ++ def discard_current(self, name=None): ++ ''' ++ Discard current section ++ :return: ++ ''' ++ self.__current_section = [] ++ self.__current_section_name = name ++ ++ def write(self, title, data, output=None): ++ ''' ++ Add a data to the current opened section. ++ :return: ++ ''' ++ if not isinstance(data, (dict, list, tuple)): ++ data = {'raw-content': str(data)} ++ output = output or self.__default_outputter ++ ++ if output != 'null': ++ try: ++ if isinstance(data, dict) and 'return' in data: ++ data = data['return'] ++ content = salt.output.try_printout(data, output, {'extension_modules': '', 'color': False}) ++ except Exception: # Fall-back to just raw YAML ++ content = None ++ else: ++ content = None ++ ++ if content is None: ++ data = json.loads(json.dumps(data)) ++ if isinstance(data, dict) and data.get('return'): ++ data = data.get('return') ++ content = yaml.safe_dump(data, default_flow_style=False, indent=4) ++ ++ self.__current_section.append({title: content}) ++ ++ def link(self, title, path): ++ ''' ++ Add a static file on the file system. ++ ++ :param title: ++ :param path: ++ :return: ++ ''' ++ # The filehandler needs to be explicitly passed here, so PyLint needs to accept that. ++ # pylint: disable=W8470 ++ if not isinstance(path, file): ++ path = salt.utils.files.fopen(path) ++ self.__current_section.append({title: path}) ++ # pylint: enable=W8470 ++ ++ ++class SaltSupport(salt.utils.parsers.SaltSupportOptionParser): ++ ''' ++ Class to run Salt Support subsystem. ++ ''' ++ RUNNER_TYPE = 'run' ++ CALL_TYPE = 'call' ++ ++ def _setup_fun_config(self, fun_conf): ++ ''' ++ Setup function configuration. ++ ++ :param conf: ++ :return: ++ ''' ++ conf = copy.deepcopy(self.config) ++ conf['file_client'] = 'local' ++ conf['fun'] = '' ++ conf['arg'] = [] ++ conf['kwarg'] = {} ++ conf['cache_jobs'] = False ++ conf['print_metadata'] = False ++ conf.update(fun_conf) ++ conf['fun'] = conf['fun'].split(':')[-1] # Discard typing prefix ++ ++ return conf ++ ++ def _get_runner(self, conf): ++ ''' ++ Get & setup runner. ++ ++ :param conf: ++ :return: ++ ''' ++ conf = self._setup_fun_config(copy.deepcopy(conf)) ++ if not getattr(self, '_runner', None): ++ self._runner = salt.cli.support.localrunner.LocalRunner(conf) ++ else: ++ self._runner.opts = conf ++ return self._runner ++ ++ def _get_caller(self, conf): ++ ''' ++ Get & setup caller from the factory. ++ ++ :param conf: ++ :return: ++ ''' ++ conf = self._setup_fun_config(copy.deepcopy(conf)) ++ if not getattr(self, '_caller', None): ++ self._caller = salt.cli.caller.Caller.factory(conf) ++ else: ++ self._caller.opts = conf ++ return self._caller ++ ++ def _local_call(self, call_conf): ++ ''' ++ Execute local call ++ ''' ++ try: ++ ret = self._get_caller(call_conf).call() ++ except SystemExit: ++ ret = 'Data is not available at this moment' ++ self.out.error(ret) ++ except Exception as ex: ++ ret = 'Unhandled exception occurred: {}'.format(ex) ++ log.debug(ex, exc_info=True) ++ self.out.error(ret) ++ ++ return ret ++ ++ def _local_run(self, run_conf): ++ ''' ++ Execute local runner ++ ++ :param run_conf: ++ :return: ++ ''' ++ try: ++ ret = self._get_runner(run_conf).run() ++ except SystemExit: ++ ret = 'Runner is not available at this moment' ++ self.out.error(ret) ++ except Exception as ex: ++ ret = 'Unhandled exception occurred: {}'.format(ex) ++ log.debug(ex, exc_info=True) ++ ++ return ret ++ ++ def _internal_function_call(self, call_conf): ++ ''' ++ Call internal function. ++ ++ :param call_conf: ++ :return: ++ ''' ++ def stub(*args, **kwargs): ++ message = 'Function {} is not available'.format(call_conf['fun']) ++ self.out.error(message) ++ log.debug('Attempt to run "{fun}" with {arg} arguments and {kwargs} parameters.'.format(**call_conf)) ++ return message ++ ++ return getattr(salt.cli.support.intfunc, ++ call_conf['fun'], stub)(self.collector, ++ *call_conf['arg'], ++ **call_conf['kwargs']) ++ ++ def _get_action(self, action_meta): ++ ''' ++ Parse action and turn into a calling point. ++ :param action_meta: ++ :return: ++ ''' ++ conf = { ++ 'fun': list(action_meta.keys())[0], ++ 'arg': [], ++ 'kwargs': {}, ++ } ++ if not len(conf['fun'].split('.')) - 1: ++ conf['salt.int.intfunc'] = True ++ ++ action_meta = action_meta[conf['fun']] ++ info = action_meta.get('info', 'Action for {}'.format(conf['fun'])) ++ for arg in action_meta.get('args') or []: ++ if not isinstance(arg, dict): ++ conf['arg'].append(arg) ++ else: ++ conf['kwargs'].update(arg) ++ ++ return info, action_meta.get('output'), conf ++ ++ def collect_internal_data(self): ++ ''' ++ Dumps current running pillars, configuration etc. ++ :return: ++ ''' ++ section = 'configuration' ++ self.out.put(section) ++ self.collector.add(section) ++ self.out.put('Saving config', indent=2) ++ self.collector.write('General Configuration', self.config) ++ self.out.put('Saving pillars', indent=2) ++ self.collector.write('Active Pillars', self._local_call({'fun': 'pillar.items'})) ++ ++ section = 'highstate' ++ self.out.put(section) ++ self.collector.add(section) ++ self.out.put('Saving highstate', indent=2) ++ self.collector.write('Rendered highstate', self._local_call({'fun': 'state.show_highstate'})) ++ ++ def _extract_return(self, data): ++ ''' ++ Extracts return data from the results. ++ ++ :param data: ++ :return: ++ ''' ++ if isinstance(data, dict): ++ data = data.get('return', data) ++ ++ return data ++ ++ def collect_local_data(self): ++ ''' ++ Collects master system data. ++ :return: ++ ''' ++ def call(func, *args, **kwargs): ++ ''' ++ Call wrapper for templates ++ :param func: ++ :return: ++ ''' ++ return self._extract_return(self._local_call({'fun': func, 'arg': args, 'kwarg': kwargs})) ++ ++ def run(func, *args, **kwargs): ++ ''' ++ Runner wrapper for templates ++ :param func: ++ :return: ++ ''' ++ return self._extract_return(self._local_run({'fun': func, 'arg': args, 'kwarg': kwargs})) ++ ++ scenario = salt.cli.support.get_profile(self.config['support_profile'], call, run) ++ for category_name in scenario: ++ self.out.put(category_name) ++ self.collector.add(category_name) ++ for action in scenario[category_name]: ++ if not action: ++ continue ++ action_name = next(iter(action)) ++ if not isinstance(action[action_name], six.string_types): ++ info, output, conf = self._get_action(action) ++ action_type = self._get_action_type(action) # run: for runners ++ if action_type == self.RUNNER_TYPE: ++ self.out.put('Running {}'.format(info.lower()), indent=2) ++ self.collector.write(info, self._local_run(conf), output=output) ++ elif action_type == self.CALL_TYPE: ++ if not conf.get('salt.int.intfunc'): ++ self.out.put('Collecting {}'.format(info.lower()), indent=2) ++ self.collector.write(info, self._local_call(conf), output=output) ++ else: ++ self.collector.discard_current() ++ self._internal_function_call(conf) ++ else: ++ self.out.error('Unknown action type "{}" for action: {}'.format(action_type, action)) ++ else: ++ # TODO: This needs to be moved then to the utils. ++ # But the code is not yet there (other PRs) ++ self.out.msg('\n'.join(salt.cli.support.console.wrap(action[action_name])), ident=2) ++ ++ def _get_action_type(self, action): ++ ''' ++ Get action type. ++ :param action: ++ :return: ++ ''' ++ action_name = next(iter(action or {'': None})) ++ if ':' not in action_name: ++ action_name = '{}:{}'.format(self.CALL_TYPE, action_name) ++ ++ return action_name.split(':')[0] or None ++ ++ def collect_targets_data(self): ++ ''' ++ Collects minion targets data ++ :return: ++ ''' ++ # TODO: remote collector? ++ ++ def _cleanup(self): ++ ''' ++ Cleanup if crash/exception ++ :return: ++ ''' ++ if (hasattr(self, 'config') ++ and self.config.get('support_archive') ++ and os.path.exists(self.config['support_archive'])): ++ self.out.warning('Terminated earlier, cleaning up') ++ os.unlink(self.config['support_archive']) ++ ++ def _check_existing_archive(self): ++ ''' ++ Check if archive exists or not. If exists and --force was not specified, ++ bail out. Otherwise remove it and move on. ++ ++ :return: ++ ''' ++ if os.path.exists(self.config['support_archive']): ++ if self.config['support_archive_force_overwrite']: ++ self.out.warning('Overwriting existing archive: {}'.format(self.config['support_archive'])) ++ os.unlink(self.config['support_archive']) ++ ret = True ++ else: ++ self.out.warning('File {} already exists.'.format(self.config['support_archive'])) ++ ret = False ++ else: ++ ret = True ++ ++ return ret ++ ++ def run(self): ++ exit_code = salt.defaults.exitcodes.EX_OK ++ self.out = salt.cli.support.console.MessagesOutput() ++ try: ++ self.parse_args() ++ except (Exception, SystemExit) as ex: ++ if not isinstance(ex, exceptions.SystemExit): ++ exit_code = salt.defaults.exitcodes.EX_GENERIC ++ self.out.error(ex) ++ elif isinstance(ex, exceptions.SystemExit): ++ exit_code = ex.code ++ else: ++ exit_code = salt.defaults.exitcodes.EX_GENERIC ++ self.out.error(ex) ++ else: ++ if self.config['log_level'] not in ('quiet', ): ++ self.setup_logfile_logger() ++ salt.utils.verify.verify_log(self.config) ++ salt.cli.support.log = log # Pass update logger so trace is available ++ ++ if self.config['support_profile_list']: ++ self.out.put('List of available profiles:') ++ for idx, profile in enumerate(salt.cli.support.get_profiles(self.config)): ++ msg_template = ' {}. '.format(idx + 1) + '{}' ++ self.out.highlight(msg_template, profile) ++ exit_code = salt.defaults.exitcodes.EX_OK ++ elif self.config['support_show_units']: ++ self.out.put('List of available units:') ++ for idx, unit in enumerate(self.find_existing_configs(None)): ++ msg_template = ' {}. '.format(idx + 1) + '{}' ++ self.out.highlight(msg_template, unit) ++ exit_code = salt.defaults.exitcodes.EX_OK ++ else: ++ if not self.config['support_profile']: ++ self.print_help() ++ raise SystemExit() ++ ++ if self._check_existing_archive(): ++ try: ++ self.collector = SupportDataCollector(self.config['support_archive'], ++ output=self.config['support_output_format']) ++ except Exception as ex: ++ self.out.error(ex) ++ exit_code = salt.defaults.exitcodes.EX_GENERIC ++ log.debug(ex, exc_info=True) ++ else: ++ try: ++ self.collector.open() ++ self.collect_local_data() ++ self.collect_internal_data() ++ self.collect_targets_data() ++ self.collector.close() ++ ++ archive_path = self.collector.archive_path ++ self.out.highlight('\nSupport data has been written to "{}" file.\n', ++ archive_path, _main='YELLOW') ++ except Exception as ex: ++ self.out.error(ex) ++ log.debug(ex, exc_info=True) ++ exit_code = salt.defaults.exitcodes.EX_SOFTWARE ++ ++ if exit_code: ++ self._cleanup() ++ ++ sys.exit(exit_code) +diff --git a/salt/cli/support/console.py b/salt/cli/support/console.py +new file mode 100644 +index 0000000000..fb6992d657 +--- /dev/null ++++ b/salt/cli/support/console.py +@@ -0,0 +1,165 @@ ++# coding=utf-8 ++''' ++Collection of tools to report messages to console. ++ ++NOTE: This is subject to incorporate other formatting bits ++ from all around everywhere and then to be moved to utils. ++''' ++ ++from __future__ import absolute_import, print_function, unicode_literals ++ ++import sys ++import os ++import salt.utils.color ++import textwrap ++ ++ ++class IndentOutput(object): ++ ''' ++ Paint different indends in different output. ++ ''' ++ def __init__(self, conf=None, device=sys.stdout): ++ if conf is None: ++ conf = {0: 'CYAN', 2: 'GREEN', 4: 'LIGHT_BLUE', 6: 'BLUE'} ++ self._colors_conf = conf ++ self._device = device ++ self._colors = salt.utils.color.get_colors() ++ self._default_color = 'GREEN' ++ self._default_hl_color = 'LIGHT_GREEN' ++ ++ def put(self, message, indent=0): ++ ''' ++ Print message with an indent. ++ ++ :param message: ++ :param indent: ++ :return: ++ ''' ++ color = self._colors_conf.get(indent + indent % 2, self._colors_conf.get(0, self._default_color)) ++ ++ for chunk in [' ' * indent, self._colors[color], message, self._colors['ENDC']]: ++ self._device.write(str(chunk)) ++ self._device.write(os.linesep) ++ self._device.flush() ++ ++ ++class MessagesOutput(IndentOutput): ++ ''' ++ Messages output to the CLI. ++ ''' ++ def msg(self, message, title=None, title_color=None, color='BLUE', ident=0): ++ ''' ++ Hint message. ++ ++ :param message: ++ :param title: ++ :param title_color: ++ :param color: ++ :param ident: ++ :return: ++ ''' ++ if title and not title_color: ++ title_color = color ++ if title_color and not title: ++ title_color = None ++ ++ self.__colored_output(title, message, title_color, color, ident=ident) ++ ++ def info(self, message, ident=0): ++ ''' ++ Write an info message to the CLI. ++ ++ :param message: ++ :param ident: ++ :return: ++ ''' ++ self.__colored_output('Info', message, 'GREEN', 'LIGHT_GREEN', ident=ident) ++ ++ def warning(self, message, ident=0): ++ ''' ++ Write a warning message to the CLI. ++ ++ :param message: ++ :param ident: ++ :return: ++ ''' ++ self.__colored_output('Warning', message, 'YELLOW', 'LIGHT_YELLOW', ident=ident) ++ ++ def error(self, message, ident=0): ++ ''' ++ Write an error message to the CLI. ++ ++ :param message: ++ :param ident ++ :return: ++ ''' ++ self.__colored_output('Error', message, 'RED', 'LIGHT_RED', ident=ident) ++ ++ def __colored_output(self, title, message, title_color, message_color, ident=0): ++ if title and not title.endswith(':'): ++ _linesep = title.endswith(os.linesep) ++ title = '{}:{}'.format(title.strip(), _linesep and os.linesep or ' ') ++ ++ for chunk in [title_color and self._colors[title_color] or None, ' ' * ident, ++ title, self._colors[message_color], message, self._colors['ENDC']]: ++ if chunk: ++ self._device.write(str(chunk)) ++ self._device.write(os.linesep) ++ self._device.flush() ++ ++ def highlight(self, message, *values, **colors): ++ ''' ++ Highlighter works the way that message parameter is a template, ++ the "values" is a list of arguments going one after another as values there. ++ And so the "colors" should designate either highlight color or alternate for each. ++ ++ Example: ++ ++ highlight('Hello {}, there! It is {}.', 'user', 'daytime', _main='GREEN', _highlight='RED') ++ highlight('Hello {}, there! It is {}.', 'user', 'daytime', _main='GREEN', _highlight='RED', 'daytime'='YELLOW') ++ ++ First example will highlight all the values in the template with the red color. ++ Second example will highlight the second value with the yellow color. ++ ++ Usage: ++ ++ colors: ++ _main: Sets the main color (or default is used) ++ _highlight: Sets the alternative color for everything ++ 'any phrase' that is the same in the "values" can override color. ++ ++ :param message: ++ :param formatted: ++ :param colors: ++ :return: ++ ''' ++ ++ m_color = colors.get('_main', self._default_color) ++ h_color = colors.get('_highlight', self._default_hl_color) ++ ++ _values = [] ++ for value in values: ++ _values.append('{p}{c}{r}'.format(p=self._colors[colors.get(value, h_color)], ++ c=value, r=self._colors[m_color])) ++ self._device.write('{s}{m}{e}'.format(s=self._colors[m_color], ++ m=message.format(*_values), e=self._colors['ENDC'])) ++ self._device.write(os.linesep) ++ self._device.flush() ++ ++ ++def wrap(txt, width=80, ident=0): ++ ''' ++ Wrap text to the required dimensions and clean it up, prepare for display. ++ ++ :param txt: ++ :param width: ++ :return: ++ ''' ++ ident = ' ' * ident ++ txt = (txt or '').replace(os.linesep, ' ').strip() ++ ++ wrapper = textwrap.TextWrapper() ++ wrapper.fix_sentence_endings = False ++ wrapper.initial_indent = wrapper.subsequent_indent = ident ++ ++ return wrapper.wrap(txt) +diff --git a/salt/cli/support/intfunc.py b/salt/cli/support/intfunc.py +new file mode 100644 +index 0000000000..2727cd6394 +--- /dev/null ++++ b/salt/cli/support/intfunc.py +@@ -0,0 +1,42 @@ ++# coding=utf-8 ++''' ++Internal functions. ++''' ++# Maybe this needs to be a modules in a future? ++ ++from __future__ import absolute_import, print_function, unicode_literals ++import os ++from salt.cli.support.console import MessagesOutput ++import salt.utils.files ++ ++ ++out = MessagesOutput() ++ ++ ++def filetree(collector, path): ++ ''' ++ Add all files in the tree. If the "path" is a file, ++ only that file will be added. ++ ++ :param path: File or directory ++ :return: ++ ''' ++ if not path: ++ out.error('Path not defined', ident=2) ++ else: ++ # The filehandler needs to be explicitly passed here, so PyLint needs to accept that. ++ # pylint: disable=W8470 ++ if os.path.isfile(path): ++ filename = os.path.basename(path) ++ try: ++ file_ref = salt.utils.files.fopen(path) # pylint: disable=W ++ out.put('Add {}'.format(filename), indent=2) ++ collector.add(filename) ++ collector.link(title=path, path=file_ref) ++ except Exception as err: ++ out.error(err, ident=4) ++ # pylint: enable=W8470 ++ else: ++ for fname in os.listdir(path): ++ fname = os.path.join(path, fname) ++ filetree(collector, fname) +diff --git a/salt/cli/support/localrunner.py b/salt/cli/support/localrunner.py +new file mode 100644 +index 0000000000..26deb883bc +--- /dev/null ++++ b/salt/cli/support/localrunner.py +@@ -0,0 +1,34 @@ ++# coding=utf-8 ++''' ++Local Runner ++''' ++ ++from __future__ import print_function, absolute_import, unicode_literals ++import salt.runner ++import salt.utils.platform ++import salt.utils.process ++import logging ++ ++log = logging.getLogger(__name__) ++ ++ ++class LocalRunner(salt.runner.Runner): ++ ''' ++ Runner class that changes its default behaviour. ++ ''' ++ ++ def _proc_function(self, fun, low, user, tag, jid, daemonize=True): ++ ''' ++ Same as original _proc_function in AsyncClientMixin, ++ except it calls "low" without firing a print event. ++ ''' ++ if daemonize and not salt.utils.platform.is_windows(): ++ salt.log.setup.shutdown_multiprocessing_logging() ++ salt.utils.process.daemonize() ++ salt.log.setup.setup_multiprocessing_logging() ++ ++ low['__jid__'] = jid ++ low['__user__'] = user ++ low['__tag__'] = tag ++ ++ return self.low(fun, low, print_event=False, full_return=False) +diff --git a/salt/cli/support/profiles/default.yml b/salt/cli/support/profiles/default.yml +new file mode 100644 +index 0000000000..01d9a26193 +--- /dev/null ++++ b/salt/cli/support/profiles/default.yml +@@ -0,0 +1,71 @@ ++sysinfo: ++ - description: | ++ Get the Salt grains of the current system. ++ - grains.items: ++ info: System grains ++ ++packages: ++ - description: | ++ Fetch list of all the installed packages. ++ - pkg.list_pkgs: ++ info: Installed packages ++ ++repositories: ++ - pkg.list_repos: ++ info: Available repositories ++ ++upgrades: ++ - pkg.list_upgrades: ++ info: Possible upgrades ++ ++## TODO: Some data here belongs elsewhere and also is duplicated ++status: ++ - status.version: ++ info: Status version ++ - status.cpuinfo: ++ info: CPU information ++ - status.cpustats: ++ info: CPU stats ++ - status.diskstats: ++ info: Disk stats ++ - status.loadavg: ++ info: Average load of the current system ++ - status.uptime: ++ info: Uptime of the machine ++ - status.meminfo: ++ info: Information about memory ++ - status.vmstats: ++ info: Virtual memory stats ++ - status.netdev: ++ info: Network device stats ++ - status.nproc: ++ info: Number of processing units available on this system ++ - status.procs: ++ info: Process data ++ ++general-health: ++ - ps.boot_time: ++ info: System Boot Time ++ - ps.swap_memory: ++ info: Swap Memory ++ output: txt ++ - ps.cpu_times: ++ info: CPU times ++ - ps.disk_io_counters: ++ info: Disk IO counters ++ - ps.disk_partition_usage: ++ info: Disk partition usage ++ output: table ++ - ps.disk_partitions: ++ info: Disk partitions ++ output: table ++ - ps.top: ++ info: Top CPU consuming processes ++ ++system.log: ++ # This works on any file system object. ++ - filetree: ++ info: Add system log ++ args: ++ - /var/log/syslog ++ +diff --git a/salt/cli/support/profiles/jobs-active.yml b/salt/cli/support/profiles/jobs-active.yml +new file mode 100644 +index 0000000000..508c54ece7 +--- /dev/null ++++ b/salt/cli/support/profiles/jobs-active.yml +@@ -0,0 +1,3 @@ ++jobs-active: ++ - run:jobs.active: ++ info: List of all actively running jobs +diff --git a/salt/cli/support/profiles/jobs-last.yml b/salt/cli/support/profiles/jobs-last.yml +new file mode 100644 +index 0000000000..e3b719f552 +--- /dev/null ++++ b/salt/cli/support/profiles/jobs-last.yml +@@ -0,0 +1,3 @@ ++jobs-last: ++ - run:jobs.last_run: ++ info: List all detectable jobs and associated functions +diff --git a/salt/cli/support/profiles/jobs-trace.yml b/salt/cli/support/profiles/jobs-trace.yml +new file mode 100644 +index 0000000000..00b28e0502 +--- /dev/null ++++ b/salt/cli/support/profiles/jobs-trace.yml +@@ -0,0 +1,7 @@ ++jobs-details: ++ {% for job in runners('jobs.list_jobs') %} ++ - run:jobs.list_job: ++ info: Details on JID {{job}} ++ args: ++ - {{job}} ++ {% endfor %} +diff --git a/salt/cli/support/profiles/network.yml b/salt/cli/support/profiles/network.yml +new file mode 100644 +index 0000000000..268f02e61f +--- /dev/null ++++ b/salt/cli/support/profiles/network.yml +@@ -0,0 +1,27 @@ ++network: ++ - network.get_hostname: ++ info: Hostname ++ output: txt ++ - network.get_fqdn: ++ info: FQDN ++ output: txt ++ - network.default_route: ++ info: Default route ++ output: table ++ - network.interfaces: ++ info: All the available interfaces ++ output: table ++ - network.subnets: ++ info: List of IPv4 subnets ++ - network.subnets6: ++ info: List of IPv6 subnets ++ - network.routes: ++ info: Network configured routes from routing tables ++ output: table ++ - network.netstat: ++ info: Information on open ports and states ++ output: table ++ - network.active_tcp: ++ info: All running TCP connections ++ - network.arp: ++ info: ARP table +diff --git a/salt/cli/support/profiles/postgres.yml b/salt/cli/support/profiles/postgres.yml +new file mode 100644 +index 0000000000..2238752c7a +--- /dev/null ++++ b/salt/cli/support/profiles/postgres.yml +@@ -0,0 +1,11 @@ ++system.log: ++ - filetree: ++ info: Add system log ++ args: ++ - /var/log/syslog ++ ++etc/postgres: ++ - filetree: ++ info: Pick entire /etc/postgresql ++ args: ++ - /etc/postgresql +diff --git a/salt/cli/support/profiles/salt.yml b/salt/cli/support/profiles/salt.yml +new file mode 100644 +index 0000000000..4b18d98870 +--- /dev/null ++++ b/salt/cli/support/profiles/salt.yml +@@ -0,0 +1,9 @@ ++sysinfo: ++ - grains.items: ++ info: System grains ++ ++logfile: ++ - filetree: ++ info: Add current logfile ++ args: ++ - {{salt('config.get', 'log_file')}} +diff --git a/salt/cli/support/profiles/users.yml b/salt/cli/support/profiles/users.yml +new file mode 100644 +index 0000000000..391acdb606 +--- /dev/null ++++ b/salt/cli/support/profiles/users.yml +@@ -0,0 +1,22 @@ ++all-users: ++ {%for uname in salt('user.list_users') %} ++ - user.info: ++ info: Information about "{{uname}}" ++ args: ++ - {{uname}} ++ - user.list_groups: ++ info: List groups for user "{{uname}}" ++ args: ++ - {{uname}} ++ - shadow.info: ++ info: Shadow information about user "{{uname}}" ++ args: ++ - {{uname}} ++ - cron.raw_cron: ++ info: Cron for user "{{uname}}" ++ args: ++ - {{uname}} ++ {%endfor%} ++ - group.getent: ++ info: List of all available groups ++ output: table +diff --git a/salt/scripts.py b/salt/scripts.py +index e677368a0f..7a1b4bb133 100644 +--- a/salt/scripts.py ++++ b/salt/scripts.py +@@ -508,3 +508,17 @@ def salt_extend(extension, name, description, salt_dir, merge): + description=description, + salt_dir=salt_dir, + merge=merge) ++ ++ ++def salt_support(): ++ ''' ++ Run Salt Support that collects system data, logs etc for debug and support purposes. ++ :return: ++ ''' ++ ++ import salt.cli.support.collector ++ if '' in sys.path: ++ sys.path.remove('') ++ client = salt.cli.support.collector.SaltSupport() ++ _install_signal_handlers(client) ++ client.run() +diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py +index 9a7f27ac11..56a8961c3a 100644 +--- a/salt/utils/parsers.py ++++ b/salt/utils/parsers.py +@@ -20,6 +20,7 @@ import getpass + import logging + import optparse + import traceback ++import tempfile + from functools import partial + + +@@ -34,6 +35,7 @@ import salt.utils.data + import salt.utils.files + import salt.utils.jid + import salt.utils.kinds as kinds ++import salt.utils.network + import salt.utils.platform + import salt.utils.process + import salt.utils.stringutils +@@ -1863,6 +1865,69 @@ class SyndicOptionParser(six.with_metaclass(OptionParserMeta, + self.get_config_file_path('minion')) + + ++class SaltSupportOptionParser(six.with_metaclass(OptionParserMeta, OptionParser, ConfigDirMixIn, ++ MergeConfigMixIn, LogLevelMixIn, TimeoutMixIn)): ++ default_timeout = 5 ++ description = 'Salt Support is a program to collect all support data: logs, system configuration etc.' ++ usage = '%prog [options] \'\' [arguments]' ++ # ConfigDirMixIn config filename attribute ++ _config_filename_ = 'master' ++ ++ # LogLevelMixIn attributes ++ _default_logging_level_ = config.DEFAULT_MASTER_OPTS['log_level'] ++ _default_logging_logfile_ = config.DEFAULT_MASTER_OPTS['log_file'] ++ ++ def _mixin_setup(self): ++ self.add_option('-P', '--show-profiles', default=False, action='store_true', ++ dest='support_profile_list', help='Show available profiles') ++ self.add_option('-p', '--profile', default='', dest='support_profile', ++ help='Specify support profile or comma-separated profiles, e.g.: "salt,network"') ++ support_archive = '{t}/{h}-support.tar.bz2'.format(t=tempfile.gettempdir(), ++ h=salt.utils.network.get_fqhostname()) ++ self.add_option('-a', '--archive', default=support_archive, dest='support_archive', ++ help=('Specify name of the resulting support archive. ' ++ 'Default is "{f}".'.format(f=support_archive))) ++ self.add_option('-u', '--unit', default='', dest='support_unit', ++ help='Specify examined unit (default "master").') ++ self.add_option('-U', '--show-units', default=False, action='store_true', dest='support_show_units', ++ help='Show available units') ++ self.add_option('-f', '--force', default=False, action='store_true', dest='support_archive_force_overwrite', ++ help='Force overwrite existing archive, if exists') ++ self.add_option('-o', '--out', default='null', dest='support_output_format', ++ help=('Set the default output using the specified outputter, ' ++ 'unless profile does not overrides this. Default: "yaml".')) ++ ++ def find_existing_configs(self, default): ++ ''' ++ Find configuration files on the system. ++ :return: ++ ''' ++ configs = [] ++ for cfg in [default, self._config_filename_, 'minion', 'proxy', 'cloud', 'spm']: ++ if not cfg: ++ continue ++ config_path = self.get_config_file_path(cfg) ++ if os.path.exists(config_path): ++ configs.append(cfg) ++ ++ if default and default not in configs: ++ raise SystemExit('Unknown configuration unit: {}'.format(default)) ++ ++ return configs ++ ++ def setup_config(self, cfg=None): ++ ''' ++ Open suitable config file. ++ :return: ++ ''' ++ _opts, _args = optparse.OptionParser.parse_args(self) ++ configs = self.find_existing_configs(_opts.support_unit) ++ if cfg not in configs: ++ cfg = configs[0] ++ ++ return config.master_config(self.get_config_file_path(cfg)) ++ ++ + class SaltCMDOptionParser(six.with_metaclass(OptionParserMeta, + OptionParser, + ConfigDirMixIn, +diff --git a/scripts/salt-support b/scripts/salt-support +new file mode 100755 +index 0000000000..48ce141c67 +--- /dev/null ++++ b/scripts/salt-support +@@ -0,0 +1,11 @@ ++#!/usr/bin/env python ++''' ++Salt support is to collect logs, ++debug data and system information ++for support purposes. ++''' ++ ++from salt.scripts import salt_support ++ ++if __name__ == '__main__': ++ salt_support() +diff --git a/setup.py b/setup.py +index 0841c93553..8d0dcaa4b8 100755 +--- a/setup.py ++++ b/setup.py +@@ -1015,6 +1015,7 @@ class SaltDistribution(distutils.dist.Distribution): + 'scripts/salt-key', + 'scripts/salt-master', + 'scripts/salt-minion', ++ 'scripts/salt-support', + 'scripts/salt-ssh', + 'scripts/salt-syndic', + 'scripts/salt-unity', +@@ -1052,6 +1053,7 @@ class SaltDistribution(distutils.dist.Distribution): + 'salt-key = salt.scripts:salt_key', + 'salt-master = salt.scripts:salt_master', + 'salt-minion = salt.scripts:salt_minion', ++ 'salt-support = salt.scripts:salt_support', + 'salt-ssh = salt.scripts:salt_ssh', + 'salt-syndic = salt.scripts:salt_syndic', + 'salt-unity = salt.scripts:salt_unity', +diff --git a/tests/unit/cli/test_support.py b/tests/unit/cli/test_support.py +new file mode 100644 +index 0000000000..85ea957d79 +--- /dev/null ++++ b/tests/unit/cli/test_support.py +@@ -0,0 +1,477 @@ ++# -*- coding: utf-8 -*- ++''' ++ :codeauthor: Bo Maryniuk ++''' ++ ++from __future__ import absolute_import, print_function, unicode_literals ++ ++from tests.support.unit import skipIf, TestCase ++from tests.support.mock import MagicMock, patch, NO_MOCK, NO_MOCK_REASON ++ ++from salt.cli.support.console import IndentOutput ++from salt.cli.support.collector import SupportDataCollector, SaltSupport ++from salt.utils.color import get_colors ++from salt.utils.stringutils import to_bytes ++import salt.exceptions ++import salt.cli.support.collector ++import salt.utils.files ++import os ++import yaml ++import jinja2 ++ ++try: ++ import pytest ++except ImportError: ++ pytest = None ++ ++ ++@skipIf(not bool(pytest), 'Pytest needs to be installed') ++@skipIf(NO_MOCK, NO_MOCK_REASON) ++class SaltSupportIndentOutputTestCase(TestCase): ++ ''' ++ Unit Tests for the salt-support indent output. ++ ''' ++ ++ def setUp(self): ++ ''' ++ Setup test ++ :return: ++ ''' ++ ++ self.message = 'Stubborn processes on dumb terminal' ++ self.device = MagicMock() ++ self.iout = IndentOutput(device=self.device) ++ self.colors = get_colors() ++ ++ def tearDown(self): ++ ''' ++ Remove instances after test run ++ :return: ++ ''' ++ del self.message ++ del self.device ++ del self.iout ++ del self.colors ++ ++ def test_standard_output(self): ++ ''' ++ Test console standard output. ++ ''' ++ self.iout.put(self.message) ++ assert self.device.write.called ++ assert self.device.write.call_count == 5 ++ for idx, data in enumerate(['', str(self.colors['CYAN']), self.message, str(self.colors['ENDC']), '\n']): ++ assert self.device.write.call_args_list[idx][0][0] == data ++ ++ def test_indent_output(self): ++ ''' ++ Test indent distance. ++ :return: ++ ''' ++ self.iout.put(self.message, indent=10) ++ for idx, data in enumerate([' ' * 10, str(self.colors['CYAN']), self.message, str(self.colors['ENDC']), '\n']): ++ assert self.device.write.call_args_list[idx][0][0] == data ++ ++ def test_color_config(self): ++ ''' ++ Test color config changes on each ident. ++ :return: ++ ''' ++ ++ conf = {0: 'MAGENTA', 2: 'RED', 4: 'WHITE', 6: 'YELLOW'} ++ self.iout = IndentOutput(conf=conf, device=self.device) ++ for indent in sorted(list(conf)): ++ self.iout.put(self.message, indent=indent) ++ ++ step = 1 ++ for ident_key in sorted(list(conf)): ++ assert str(self.device.write.call_args_list[step][0][0]) == str(self.colors[conf[ident_key]]) ++ step += 5 ++ ++ ++@skipIf(not bool(pytest), 'Pytest needs to be installed') ++@skipIf(NO_MOCK, NO_MOCK_REASON) ++class SaltSupportCollectorTestCase(TestCase): ++ ''' ++ Collector tests. ++ ''' ++ def setUp(self): ++ ''' ++ Setup the test case ++ :return: ++ ''' ++ self.archive_path = '/highway/to/hell' ++ self.output_device = MagicMock() ++ self.collector = SupportDataCollector(self.archive_path, self.output_device) ++ ++ def tearDown(self): ++ ''' ++ Tear down the test case elements ++ :return: ++ ''' ++ del self.collector ++ del self.archive_path ++ del self.output_device ++ ++ @patch('salt.cli.support.collector.tarfile.TarFile', MagicMock()) ++ def test_archive_open(self): ++ ''' ++ Test archive is opened. ++ ++ :return: ++ ''' ++ self.collector.open() ++ assert self.collector.archive_path == self.archive_path ++ with pytest.raises(salt.exceptions.SaltException) as err: ++ self.collector.open() ++ assert 'Archive already opened' in str(err) ++ ++ @patch('salt.cli.support.collector.tarfile.TarFile', MagicMock()) ++ def test_archive_close(self): ++ ''' ++ Test archive is opened. ++ ++ :return: ++ ''' ++ self.collector.open() ++ self.collector._flush_content = lambda: None ++ self.collector.close() ++ assert self.collector.archive_path == self.archive_path ++ with pytest.raises(salt.exceptions.SaltException) as err: ++ self.collector.close() ++ assert 'Archive already closed' in str(err) ++ ++ def test_archive_addwrite(self): ++ ''' ++ Test add to the archive a section and write to it. ++ ++ :return: ++ ''' ++ archive = MagicMock() ++ with patch('salt.cli.support.collector.tarfile.TarFile', archive): ++ self.collector.open() ++ self.collector.add('foo') ++ self.collector.write(title='title', data='data', output='null') ++ self.collector._flush_content() ++ ++ assert (archive.bz2open().addfile.call_args[1]['fileobj'].read() ++ == to_bytes('title\n-----\n\nraw-content: data\n\n\n\n')) ++ ++ @patch('salt.utils.files.fopen', MagicMock(return_value='path=/dev/null')) ++ def test_archive_addlink(self): ++ ''' ++ Test add to the archive a section and link an external file or directory to it. ++ ++ :return: ++ ''' ++ archive = MagicMock() ++ with patch('salt.cli.support.collector.tarfile.TarFile', archive): ++ self.collector.open() ++ self.collector.add('foo') ++ self.collector.link(title='Backup Path', path='/path/to/backup.config') ++ self.collector._flush_content() ++ ++ assert archive.bz2open().addfile.call_count == 1 ++ assert (archive.bz2open().addfile.call_args[1]['fileobj'].read() ++ == to_bytes('Backup Path\n-----------\n\npath=/dev/null\n\n\n')) ++ ++ @patch('salt.utils.files.fopen', MagicMock(return_value='path=/dev/null')) ++ def test_archive_discard_section(self): ++ ''' ++ Test discard a section from the archive. ++ ++ :return: ++ ''' ++ archive = MagicMock() ++ with patch('salt.cli.support.collector.tarfile.TarFile', archive): ++ self.collector.open() ++ self.collector.add('solar-interference') ++ self.collector.link(title='Thermal anomaly', path='/path/to/another/great.config') ++ self.collector.add('foo') ++ self.collector.link(title='Backup Path', path='/path/to/backup.config') ++ self.collector._flush_content() ++ assert archive.bz2open().addfile.call_count == 2 ++ assert (archive.bz2open().addfile.mock_calls[0][2]['fileobj'].read() ++ == to_bytes('Thermal anomaly\n---------------\n\npath=/dev/null\n\n\n')) ++ self.collector.close() ++ ++ archive = MagicMock() ++ with patch('salt.cli.support.collector.tarfile.TarFile', archive): ++ self.collector.open() ++ self.collector.add('solar-interference') ++ self.collector.link(title='Thermal anomaly', path='/path/to/another/great.config') ++ self.collector.discard_current() ++ self.collector.add('foo') ++ self.collector.link(title='Backup Path', path='/path/to/backup.config') ++ self.collector._flush_content() ++ assert archive.bz2open().addfile.call_count == 2 ++ assert (archive.bz2open().addfile.mock_calls[0][2]['fileobj'].read() ++ == to_bytes('Backup Path\n-----------\n\npath=/dev/null\n\n\n')) ++ self.collector.close() ++ ++ ++@skipIf(not bool(pytest), 'Pytest needs to be installed') ++@skipIf(NO_MOCK, NO_MOCK_REASON) ++class SaltSupportRunnerTestCase(TestCase): ++ ''' ++ Test runner class. ++ ''' ++ ++ def setUp(self): ++ ''' ++ Set up test suite. ++ :return: ++ ''' ++ self.archive_path = '/dev/null' ++ self.output_device = MagicMock() ++ self.runner = SaltSupport() ++ self.runner.collector = SupportDataCollector(self.archive_path, self.output_device) ++ ++ def tearDown(self): ++ ''' ++ Tear down. ++ ++ :return: ++ ''' ++ del self.archive_path ++ del self.output_device ++ del self.runner ++ ++ def test_function_config(self): ++ ''' ++ Test function config formation. ++ ++ :return: ++ ''' ++ self.runner.config = {} ++ msg = 'Electromagnetic energy loss' ++ assert self.runner._setup_fun_config({'description': msg}) == {'print_metadata': False, ++ 'file_client': 'local', ++ 'fun': '', 'kwarg': {}, ++ 'description': msg, ++ 'cache_jobs': False, 'arg': []} ++ ++ def test_local_caller(self): ++ ''' ++ Test local caller. ++ ++ :return: ++ ''' ++ msg = 'Because of network lag due to too many people playing deathmatch' ++ caller = MagicMock() ++ caller().call = MagicMock(return_value=msg) ++ ++ self.runner._get_caller = caller ++ self.runner.out = MagicMock() ++ assert self.runner._local_call({}) == msg ++ ++ caller().call = MagicMock(side_effect=SystemExit) ++ assert self.runner._local_call({}) == 'Data is not available at this moment' ++ ++ err_msg = "The UPS doesn't have a battery backup." ++ caller().call = MagicMock(side_effect=Exception(err_msg)) ++ assert self.runner._local_call({}) == "Unhandled exception occurred: The UPS doesn't have a battery backup." ++ ++ def test_local_runner(self): ++ ''' ++ Test local runner. ++ ++ :return: ++ ''' ++ msg = 'Big to little endian conversion error' ++ runner = MagicMock() ++ runner().run = MagicMock(return_value=msg) ++ ++ self.runner._get_runner = runner ++ self.runner.out = MagicMock() ++ assert self.runner._local_run({}) == msg ++ ++ runner().run = MagicMock(side_effect=SystemExit) ++ assert self.runner._local_run({}) == 'Runner is not available at this moment' ++ ++ err_msg = 'Trojan horse ran out of hay' ++ runner().run = MagicMock(side_effect=Exception(err_msg)) ++ assert self.runner._local_run({}) == 'Unhandled exception occurred: Trojan horse ran out of hay' ++ ++ @patch('salt.cli.support.intfunc', MagicMock(spec=[])) ++ def test_internal_function_call_stub(self): ++ ''' ++ Test missing internal function call is handled accordingly. ++ ++ :return: ++ ''' ++ self.runner.out = MagicMock() ++ out = self.runner._internal_function_call({'fun': 'everythingisawesome', ++ 'arg': [], 'kwargs': {}}) ++ assert out == 'Function everythingisawesome is not available' ++ ++ def test_internal_function_call(self): ++ ''' ++ Test missing internal function call is handled accordingly. ++ ++ :return: ++ ''' ++ msg = 'Internet outage' ++ intfunc = MagicMock() ++ intfunc.everythingisawesome = MagicMock(return_value=msg) ++ self.runner.out = MagicMock() ++ with patch('salt.cli.support.intfunc', intfunc): ++ out = self.runner._internal_function_call({'fun': 'everythingisawesome', ++ 'arg': [], 'kwargs': {}}) ++ assert out == msg ++ ++ def test_get_action(self): ++ ''' ++ Test action meta gets parsed. ++ ++ :return: ++ ''' ++ action_meta = {'run:jobs.list_jobs_filter': {'info': 'List jobs filter', 'args': [1]}} ++ assert self.runner._get_action(action_meta) == ('List jobs filter', None, ++ {'fun': 'run:jobs.list_jobs_filter', 'kwargs': {}, 'arg': [1]}) ++ action_meta = {'user.info': {'info': 'Information about "usbmux"', 'args': ['usbmux']}} ++ assert self.runner._get_action(action_meta) == ('Information about "usbmux"', None, ++ {'fun': 'user.info', 'kwargs': {}, 'arg': ['usbmux']}) ++ ++ def test_extract_return(self): ++ ''' ++ Test extract return from the output. ++ ++ :return: ++ ''' ++ out = {'key': 'value'} ++ assert self.runner._extract_return(out) == out ++ assert self.runner._extract_return({'return': out}) == out ++ ++ def test_get_action_type(self): ++ ''' ++ Test action meta determines action type. ++ ++ :return: ++ ''' ++ action_meta = {'run:jobs.list_jobs_filter': {'info': 'List jobs filter', 'args': [1]}} ++ assert self.runner._get_action_type(action_meta) == 'run' ++ ++ action_meta = {'user.info': {'info': 'Information about "usbmux"', 'args': ['usbmux']}} ++ assert self.runner._get_action_type(action_meta) == 'call' ++ ++ @patch('os.path.exists', MagicMock(return_value=True)) ++ def test_cleanup(self): ++ ''' ++ Test cleanup routine. ++ ++ :return: ++ ''' ++ arch = '/tmp/killme.zip' ++ unlink = MagicMock() ++ with patch('os.unlink', unlink): ++ self.runner.config = {'support_archive': arch} ++ self.runner.out = MagicMock() ++ self.runner._cleanup() ++ ++ assert self.runner.out.warning.call_args[0][0] == 'Terminated earlier, cleaning up' ++ unlink.assert_called_once_with(arch) ++ ++ @patch('os.path.exists', MagicMock(return_value=True)) ++ def test_check_existing_archive(self): ++ ''' ++ Test check existing archive. ++ ++ :return: ++ ''' ++ arch = '/tmp/endothermal-recalibration.zip' ++ unlink = MagicMock() ++ with patch('os.unlink', unlink), patch('os.path.exists', MagicMock(return_value=False)): ++ self.runner.config = {'support_archive': '', ++ 'support_archive_force_overwrite': True} ++ self.runner.out = MagicMock() ++ assert self.runner._check_existing_archive() ++ assert self.runner.out.warning.call_count == 0 ++ ++ with patch('os.unlink', unlink): ++ self.runner.config = {'support_archive': arch, ++ 'support_archive_force_overwrite': False} ++ self.runner.out = MagicMock() ++ assert not self.runner._check_existing_archive() ++ assert self.runner.out.warning.call_args[0][0] == 'File {} already exists.'.format(arch) ++ ++ with patch('os.unlink', unlink): ++ self.runner.config = {'support_archive': arch, ++ 'support_archive_force_overwrite': True} ++ self.runner.out = MagicMock() ++ assert self.runner._check_existing_archive() ++ assert self.runner.out.warning.call_args[0][0] == 'Overwriting existing archive: {}'.format(arch) ++ ++ ++@skipIf(not bool(pytest), 'Pytest needs to be installed') ++@skipIf(NO_MOCK, NO_MOCK_REASON) ++class ProfileIntegrityTestCase(TestCase): ++ ''' ++ Default profile integrity ++ ''' ++ def setUp(self): ++ ''' ++ Set up test suite. ++ ++ :return: ++ ''' ++ self.profiles = {} ++ profiles = os.path.join(os.path.dirname(salt.cli.support.collector.__file__), 'profiles') ++ for profile in os.listdir(profiles): ++ self.profiles[profile.split('.')[0]] = os.path.join(profiles, profile) ++ ++ def tearDown(self): ++ ''' ++ Tear down test suite. ++ ++ :return: ++ ''' ++ del self.profiles ++ ++ def _render_template_to_yaml(self, name, *args, **kwargs): ++ ''' ++ Get template referene for rendering. ++ :return: ++ ''' ++ with salt.utils.files.fopen(self.profiles[name]) as t_fh: ++ template = t_fh.read() ++ return yaml.load(jinja2.Environment().from_string(template).render(*args, **kwargs)) ++ ++ def test_non_template_profiles_parseable(self): ++ ''' ++ Test shipped default profile is YAML parse-able. ++ ++ :return: ++ ''' ++ for t_name in ['default', 'jobs-active', 'jobs-last', 'network', 'postgres']: ++ with salt.utils.files.fopen(self.profiles[t_name]) as ref: ++ try: ++ yaml.load(ref) ++ parsed = True ++ except Exception: ++ parsed = False ++ assert parsed ++ ++ def test_users_template_profile(self): ++ ''' ++ Test users template profile. ++ ++ :return: ++ ''' ++ users_data = self._render_template_to_yaml('users', salt=MagicMock(return_value=['pokemon'])) ++ assert len(users_data['all-users']) == 5 ++ for user_data in users_data['all-users']: ++ for tgt in ['user.list_groups', 'shadow.info', 'cron.raw_cron']: ++ if tgt in user_data: ++ assert user_data[tgt]['args'] == ['pokemon'] ++ ++ def test_jobs_trace_template_profile(self): ++ ''' ++ Test jobs-trace template profile. ++ ++ :return: ++ ''' ++ jobs_trace = self._render_template_to_yaml('jobs-trace', runners=MagicMock(return_value=['0000'])) ++ assert len(jobs_trace['jobs-details']) == 1 ++ assert jobs_trace['jobs-details'][0]['run:jobs.list_job']['info'] == 'Details on JID 0000' ++ assert jobs_trace['jobs-details'][0]['run:jobs.list_job']['args'] == [0] +-- +2.17.1 + + diff --git a/fix-async-call-to-process-manager.patch b/fix-async-call-to-process-manager.patch new file mode 100644 index 0000000..ed37284 --- /dev/null +++ b/fix-async-call-to-process-manager.patch @@ -0,0 +1,35 @@ +From b276ee7373e88d05c01912a9d9d3a44a5d17bab6 Mon Sep 17 00:00:00 2001 +From: Daniel Wallace +Date: Mon, 13 Aug 2018 13:55:37 -0500 +Subject: [PATCH] fix async call to process manager + +--- + salt/minion.py | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/salt/minion.py b/salt/minion.py +index 9c05a646ea..8b8fd797d1 100644 +--- a/salt/minion.py ++++ b/salt/minion.py +@@ -923,7 +923,7 @@ class MinionManager(MinionBase): + install_zmq() + self.io_loop = ZMQDefaultLoop.current() + self.process_manager = ProcessManager(name='MultiMinionProcessManager') +- self.io_loop.spawn_callback(self.process_manager.run, **{'async': True}) # Tornado backward compat ++ self.io_loop.spawn_callback(self.process_manager.run, **{'asynchronous': True}) # Tornado backward compat + + def __del__(self): + self.destroy() +@@ -1120,7 +1120,7 @@ class Minion(MinionBase): + time.sleep(sleep_time) + + self.process_manager = ProcessManager(name='MinionProcessManager') +- self.io_loop.spawn_callback(self.process_manager.run, **{'async': True}) ++ self.io_loop.spawn_callback(self.process_manager.run, **{'asynchronous': True}) + # We don't have the proxy setup yet, so we can't start engines + # Engines need to be able to access __proxy__ + if not salt.utils.platform.is_proxy(): +-- +2.17.1 + + diff --git a/fix-git_pillar-merging-across-multiple-__env__-repos.patch b/fix-git_pillar-merging-across-multiple-__env__-repos.patch new file mode 100644 index 0000000..91b790c --- /dev/null +++ b/fix-git_pillar-merging-across-multiple-__env__-repos.patch @@ -0,0 +1,328 @@ +From 49f8f296edf4655e2be7e564745931692ae939b7 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Tue, 6 Nov 2018 16:38:54 +0000 +Subject: [PATCH] Fix git_pillar merging across multiple __env__ + repositories (bsc#1112874) + +Resolve target branch when using __env__ + +Test git ext_pillar across multiple repos using __env__ + +Remove unicode references +--- + salt/utils/gitfs.py | 2 +- + tests/integration/pillar/test_git_pillar.py | 144 ++++++++++++++++++++ + tests/support/gitfs.py | 66 ++++++++- + 3 files changed, 209 insertions(+), 3 deletions(-) + +diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py +index 6963f40226..11af741e35 100644 +--- a/salt/utils/gitfs.py ++++ b/salt/utils/gitfs.py +@@ -2975,7 +2975,7 @@ class GitPillar(GitBase): + if repo.env: + env = repo.env + else: +- env = 'base' if repo.branch == repo.base else repo.branch ++ env = 'base' if repo.branch == repo.base else repo.get_checkout_target() + if repo._mountpoint: + if self.link_mountpoint(repo): + self.pillar_dirs[repo.linkdir] = env +diff --git a/tests/integration/pillar/test_git_pillar.py b/tests/integration/pillar/test_git_pillar.py +index e97e720bab..e052782311 100644 +--- a/tests/integration/pillar/test_git_pillar.py ++++ b/tests/integration/pillar/test_git_pillar.py +@@ -358,6 +358,38 @@ class GitPythonMixin(object): + "available on the salt master"]} + ) + ++ def test_includes_enabled_solves___env___with_mountpoint(self): ++ ''' ++ Test with git_pillar_includes enabled and using "__env__" as the branch ++ name for the configured repositories. ++ The "gitinfo" repository contains top.sls file with a local reference ++ and also referencing external "nowhere.foo" which is provided by "webinfo" ++ repository mounted as "nowhere". ++ ''' ++ ret = self.get_pillar('''\ ++ file_ignore_regex: [] ++ file_ignore_glob: [] ++ git_pillar_provider: gitpython ++ cachedir: {cachedir} ++ extension_modules: {extmods} ++ ext_pillar: ++ - git: ++ - __env__ {url_extra_repo}: ++ - name: gitinfo ++ - __env__ {url}: ++ - name: webinfo ++ - mountpoint: nowhere ++ ''') ++ self.assertEqual( ++ ret, ++ {'branch': 'master', ++ 'motd': 'The force will be with you. Always.', ++ 'mylist': ['master'], ++ 'mydict': {'master': True, ++ 'nested_list': ['master'], ++ 'nested_dict': {'master': True}}} ++ ) ++ + + @destructiveTest + @skipIf(NO_MOCK, NO_MOCK_REASON) +@@ -413,7 +445,12 @@ class TestGitPythonAuthenticatedHTTP(TestGitPythonHTTP, GitPythonMixin): + username=cls.username, + password=cls.password, + port=cls.nginx_port) ++ cls.url_extra_repo = 'http://{username}:{password}@127.0.0.1:{port}/extra_repo.git'.format( ++ username=cls.username, ++ password=cls.password, ++ port=cls.nginx_port) + cls.ext_opts['url'] = cls.url ++ cls.ext_opts['url_extra_repo'] = cls.url_extra_repo + cls.ext_opts['username'] = cls.username + cls.ext_opts['password'] = cls.password + +@@ -1192,6 +1229,40 @@ class TestPygit2SSH(GitPillarSSHTestBase): + ''') + self.assertEqual(ret, expected) + ++ def test_includes_enabled_solves___env___with_mountpoint(self): ++ ''' ++ Test with git_pillar_includes enabled and using "__env__" as the branch ++ name for the configured repositories. ++ The "gitinfo" repository contains top.sls file with a local reference ++ and also referencing external "nowhere.foo" which is provided by "webinfo" ++ repository mounted as "nowhere". ++ ''' ++ ret = self.get_pillar('''\ ++ file_ignore_regex: [] ++ file_ignore_glob: [] ++ git_pillar_provider: pygit2 ++ git_pillar_pubkey: {pubkey_nopass} ++ git_pillar_privkey: {privkey_nopass} ++ cachedir: {cachedir} ++ extension_modules: {extmods} ++ ext_pillar: ++ - git: ++ - __env__ {url_extra_repo}: ++ - name: gitinfo ++ - __env__ {url}: ++ - name: webinfo ++ - mountpoint: nowhere ++ ''') ++ self.assertEqual( ++ ret, ++ {'branch': 'master', ++ 'motd': 'The force will be with you. Always.', ++ 'mylist': ['master'], ++ 'mydict': {'master': True, ++ 'nested_list': ['master'], ++ 'nested_dict': {'master': True}}} ++ ) ++ + + @skipIf(NO_MOCK, NO_MOCK_REASON) + @skipIf(_windows_or_mac(), 'minion is windows or mac') +@@ -1439,6 +1510,38 @@ class TestPygit2HTTP(GitPillarHTTPTestBase): + ''') + self.assertEqual(ret, expected) + ++ def test_includes_enabled_solves___env___with_mountpoint(self): ++ ''' ++ Test with git_pillar_includes enabled and using "__env__" as the branch ++ name for the configured repositories. ++ The "gitinfo" repository contains top.sls file with a local reference ++ and also referencing external "nowhere.foo" which is provided by "webinfo" ++ repository mounted as "nowhere". ++ ''' ++ ret = self.get_pillar('''\ ++ file_ignore_regex: [] ++ file_ignore_glob: [] ++ git_pillar_provider: pygit2 ++ cachedir: {cachedir} ++ extension_modules: {extmods} ++ ext_pillar: ++ - git: ++ - __env__ {url_extra_repo}: ++ - name: gitinfo ++ - __env__ {url}: ++ - name: webinfo ++ - mountpoint: nowhere ++ ''') ++ self.assertEqual( ++ ret, ++ {'branch': 'master', ++ 'motd': 'The force will be with you. Always.', ++ 'mylist': ['master'], ++ 'mydict': {'master': True, ++ 'nested_list': ['master'], ++ 'nested_dict': {'master': True}}} ++ ) ++ + + @skipIf(NO_MOCK, NO_MOCK_REASON) + @skipIf(_windows_or_mac(), 'minion is windows or mac') +@@ -1887,3 +1990,44 @@ class TestPygit2AuthenticatedHTTP(GitPillarHTTPTestBase): + - env: base + ''') + self.assertEqual(ret, expected) ++ ++ def test_includes_enabled_solves___env___with_mountpoint(self): ++ ''' ++ Test with git_pillar_includes enabled and using "__env__" as the branch ++ name for the configured repositories. ++ The "gitinfo" repository contains top.sls file with a local reference ++ and also referencing external "nowhere.foo" which is provided by "webinfo" ++ repository mounted as "nowhere". ++ ''' ++ ret = self.get_pillar('''\ ++ file_ignore_regex: [] ++ file_ignore_glob: [] ++ git_pillar_provider: pygit2 ++ git_pillar_user: {user} ++ git_pillar_password: {password} ++ git_pillar_insecure_auth: True ++ cachedir: {cachedir} ++ extension_modules: {extmods} ++ ext_pillar: ++ - git: ++ - __env__ {url_extra_repo}: ++ - name: gitinfo ++ - user: {user} ++ - password: {password} ++ - insecure_auth: True ++ - __env__ {url}: ++ - name: webinfo ++ - mountpoint: nowhere ++ - user: {user} ++ - password: {password} ++ - insecure_auth: True ++ ''') ++ self.assertEqual( ++ ret, ++ {'branch': 'master', ++ 'motd': 'The force will be with you. Always.', ++ 'mylist': ['master'], ++ 'mydict': {'master': True, ++ 'nested_list': ['master'], ++ 'nested_dict': {'master': True}}} ++ ) +diff --git a/tests/support/gitfs.py b/tests/support/gitfs.py +index 2afd31539d..e645c50a86 100644 +--- a/tests/support/gitfs.py ++++ b/tests/support/gitfs.py +@@ -133,9 +133,13 @@ class SSHDMixin(ModuleCase, ProcessManager, SaltReturnAssertsMixin): + cls.url = 'ssh://{username}@127.0.0.1:{port}/~/repo.git'.format( + username=cls.username, + port=cls.sshd_port) ++ cls.url_extra_repo = 'ssh://{username}@127.0.0.1:{port}/~/extra_repo.git'.format( ++ username=cls.username, ++ port=cls.sshd_port) + home = '/root/.ssh' + cls.ext_opts = { + 'url': cls.url, ++ 'url_extra_repo': cls.url_extra_repo, + 'privkey_nopass': os.path.join(home, cls.id_rsa_nopass), + 'pubkey_nopass': os.path.join(home, cls.id_rsa_nopass + '.pub'), + 'privkey_withpass': os.path.join(home, cls.id_rsa_withpass), +@@ -193,7 +197,8 @@ class WebserverMixin(ModuleCase, ProcessManager, SaltReturnAssertsMixin): + # get_unused_localhost_port() return identical port numbers. + cls.uwsgi_port = get_unused_localhost_port() + cls.url = 'http://127.0.0.1:{port}/repo.git'.format(port=cls.nginx_port) +- cls.ext_opts = {'url': cls.url} ++ cls.url_extra_repo = 'http://127.0.0.1:{port}/extra_repo.git'.format(port=cls.nginx_port) ++ cls.ext_opts = {'url': cls.url, 'url_extra_repo': cls.url_extra_repo} + # Add auth params if present (if so this will trigger the spawned + # server to turn on HTTP basic auth). + for credential_param in ('user', 'password'): +@@ -250,7 +255,7 @@ class GitTestBase(ModuleCase): + Base class for all gitfs/git_pillar tests. Must be subclassed and paired + with either SSHDMixin or WebserverMixin to provide the server. + ''' +- case = port = bare_repo = admin_repo = None ++ case = port = bare_repo = base_extra_repo = admin_repo = admin_extra_repo = None + maxDiff = None + git_opts = '-c user.name="Foo Bar" -c user.email=foo@bar.com' + ext_opts = {} +@@ -465,6 +470,61 @@ class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin): + ''')) + _push('top_only', 'add top_only branch') + ++ def make_extra_repo(self, root_dir, user='root'): ++ self.bare_extra_repo = os.path.join(root_dir, 'extra_repo.git') ++ self.admin_extra_repo = os.path.join(root_dir, 'admin_extra') ++ ++ for dirname in (self.bare_extra_repo, self.admin_extra_repo): ++ shutil.rmtree(dirname, ignore_errors=True) ++ ++ # Create bare extra repo ++ self.run_function( ++ 'git.init', ++ [self.bare_extra_repo], ++ user=user, ++ bare=True) ++ ++ # Clone bare repo ++ self.run_function( ++ 'git.clone', ++ [self.admin_extra_repo], ++ url=self.bare_extra_repo, ++ user=user) ++ ++ def _push(branch, message): ++ self.run_function( ++ 'git.add', ++ [self.admin_extra_repo, '.'], ++ user=user) ++ self.run_function( ++ 'git.commit', ++ [self.admin_extra_repo, message], ++ user=user, ++ git_opts=self.git_opts, ++ ) ++ self.run_function( ++ 'git.push', ++ [self.admin_extra_repo], ++ remote='origin', ++ ref=branch, ++ user=user, ++ ) ++ ++ with salt.utils.files.fopen( ++ os.path.join(self.admin_extra_repo, 'top.sls'), 'w') as fp_: ++ fp_.write(textwrap.dedent('''\ ++ "{{saltenv}}": ++ '*': ++ - motd ++ - nowhere.foo ++ ''')) ++ with salt.utils.files.fopen( ++ os.path.join(self.admin_extra_repo, 'motd.sls'), 'w') as fp_: ++ fp_.write(textwrap.dedent('''\ ++ motd: The force will be with you. Always. ++ ''')) ++ _push('master', 'initial commit') ++ + + class GitPillarSSHTestBase(GitPillarTestBase, SSHDMixin): + ''' +@@ -533,6 +593,7 @@ class GitPillarSSHTestBase(GitPillarTestBase, SSHDMixin): + ) + ) + self.make_repo(root_dir, user=self.username) ++ self.make_extra_repo(root_dir, user=self.username) + + def get_pillar(self, ext_pillar_conf): + ''' +@@ -579,3 +640,4 @@ class GitPillarHTTPTestBase(GitPillarTestBase, WebserverMixin): + self.spawn_server() # pylint: disable=E1120 + + self.make_repo(self.repo_dir) ++ self.make_extra_repo(self.repo_dir) +-- +2.17.1 + + diff --git a/fix-index-error-when-running-on-python-3.patch b/fix-index-error-when-running-on-python-3.patch new file mode 100644 index 0000000..0d15705 --- /dev/null +++ b/fix-index-error-when-running-on-python-3.patch @@ -0,0 +1,37 @@ +From 5502f05fac89330ab26d04e29d3aa6d36ab928c5 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Thu, 20 Sep 2018 11:51:58 +0100 +Subject: [PATCH] Fix index error when running on Python 3 + +Fix wrong queryformat for zypper list_provides +--- + salt/modules/zypper.py | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py +index 695bce4f4e..e4423cf1fc 100644 +--- a/salt/modules/zypper.py ++++ b/salt/modules/zypper.py +@@ -2314,7 +2314,7 @@ def list_provides(**kwargs): + ''' + ret = __context__.get('pkg.list_provides') + if not ret: +- cmd = ['rpm', '-qa', '--queryformat', '[%{PROVIDES}_|-%{NAME}\n]'] ++ cmd = ['rpm', '-qa', '--queryformat', '%{PROVIDES}_|-%{NAME}\n'] + ret = dict() + for line in __salt__['cmd.run'](cmd, output_loglevel='trace', python_shell=False).splitlines(): + provide, realname = line.split('_|-') +@@ -2379,7 +2379,7 @@ def resolve_capabilities(pkgs, refresh, **kwargs): + try: + result = search(name, provides=True, match='exact') + if len(result) == 1: +- name = result.keys()[0] ++ name = next(iter(result.keys())) + elif len(result) > 1: + log.warn("Found ambiguous match for capability '%s'.", pkg) + except CommandExecutionError as exc: +-- +2.17.1 + + diff --git a/fix-ipv6-scope-bsc-1108557.patch b/fix-ipv6-scope-bsc-1108557.patch new file mode 100644 index 0000000..1381712 --- /dev/null +++ b/fix-ipv6-scope-bsc-1108557.patch @@ -0,0 +1,659 @@ +From 0509f0b0f1e880e7651e2a33cf5b70ef1930a3ff Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Fri, 28 Sep 2018 15:22:33 +0200 +Subject: [PATCH] Fix IPv6 scope (bsc#1108557) + +Fix ipaddress imports + +Remove unused import + +Fix ipaddress import + +Fix unicode imports in compat + +Override standard IPv6Address class + +Check version via object + +Isolate Py2 and Py3 mode + +Add logging + +Add debugging to the ip_address method (py2 and py3) + +Remove multiple returns and add check for address syntax + +Remove unnecessary variable for import detection + +Remove duplicated code + +Remove unnecessary operator + +Remove multiple returns + +Use ternary operator instead + +Remove duplicated code + +Move docstrings to their native places + +Add real exception message + +Add logging to the ip_interface + +Add scope on str + +Lintfix: mute not called constructors + +Add extra detection for hexadecimal packed bytes on Python2. This cannot be detected with type comparison, because bytes == str and at the same time bytes != str if compatibility is not around + +Fix py2 case where the same class cannot initialise itself on Python2 via super. + +Simplify checking clause + +Do not use introspection for method swap + +Fix wrong type swap + +Add Py3.4 old implementation's fix + +Lintfix + +Lintfix refactor: remove duplicate returns as not needed + +Revert method remapping with pylint updates + +Remove unnecessary manipulation with IPv6 scope outside of the IPv6Address object instance + +Lintfix: W0611 + +Reverse skipping tests: if no ipaddress +--- + salt/_compat.py | 287 +++++++++++++++++++++++------ + salt/cloud/clouds/saltify.py | 5 +- + salt/cloud/clouds/vagrant.py | 9 +- + salt/ext/win_inet_pton.py | 2 +- + salt/minion.py | 5 +- + salt/modules/ipset.py | 5 +- + salt/modules/network.py | 5 +- + salt/modules/vagrant.py | 6 +- + salt/utils/dns.py | 11 +- + salt/utils/minions.py | 5 +- + tests/unit/grains/test_core.py | 5 +- + tests/unit/modules/test_network.py | 15 +- + 12 files changed, 245 insertions(+), 115 deletions(-) + +diff --git a/salt/_compat.py b/salt/_compat.py +index 9b10646ace..0576210afc 100644 +--- a/salt/_compat.py ++++ b/salt/_compat.py +@@ -2,18 +2,21 @@ + ''' + Salt compatibility code + ''' +-# pylint: disable=import-error,unused-import,invalid-name ++# pylint: disable=import-error,unused-import,invalid-name,W0231,W0233 + + # Import python libs +-from __future__ import absolute_import ++from __future__ import absolute_import, unicode_literals, print_function + import sys + import types ++import logging + + # Import 3rd-party libs +-from salt.ext.six import binary_type, string_types, text_type ++from salt.exceptions import SaltException ++from salt.ext.six import binary_type, string_types, text_type, integer_types + from salt.ext.six.moves import cStringIO, StringIO + +-HAS_XML = True ++log = logging.getLogger(__name__) ++ + try: + # Python >2.5 + import xml.etree.cElementTree as ElementTree +@@ -31,11 +34,10 @@ except Exception: + import elementtree.ElementTree as ElementTree + except Exception: + ElementTree = None +- HAS_XML = False + + + # True if we are running on Python 3. +-PY3 = sys.version_info[0] == 3 ++PY3 = sys.version_info.major == 3 + + + if PY3: +@@ -45,13 +47,12 @@ else: + import exceptions + + +-if HAS_XML: ++if ElementTree is not None: + if not hasattr(ElementTree, 'ParseError'): + class ParseError(Exception): + ''' + older versions of ElementTree do not have ParseError + ''' +- pass + + ElementTree.ParseError = ParseError + +@@ -61,9 +62,7 @@ def text_(s, encoding='latin-1', errors='strict'): + If ``s`` is an instance of ``binary_type``, return + ``s.decode(encoding, errors)``, otherwise return ``s`` + ''' +- if isinstance(s, binary_type): +- return s.decode(encoding, errors) +- return s ++ return s.decode(encoding, errors) if isinstance(s, binary_type) else s + + + def bytes_(s, encoding='latin-1', errors='strict'): +@@ -71,57 +70,37 @@ def bytes_(s, encoding='latin-1', errors='strict'): + If ``s`` is an instance of ``text_type``, return + ``s.encode(encoding, errors)``, otherwise return ``s`` + ''' +- if isinstance(s, text_type): +- return s.encode(encoding, errors) +- return s ++ return s.encode(encoding, errors) if isinstance(s, text_type) else s + + +-if PY3: +- def ascii_native_(s): +- if isinstance(s, text_type): +- s = s.encode('ascii') +- return str(s, 'ascii', 'strict') +-else: +- def ascii_native_(s): +- if isinstance(s, text_type): +- s = s.encode('ascii') +- return str(s) ++def ascii_native_(s): ++ ''' ++ Python 3: If ``s`` is an instance of ``text_type``, return ++ ``s.encode('ascii')``, otherwise return ``str(s, 'ascii', 'strict')`` + +-ascii_native_.__doc__ = ''' +-Python 3: If ``s`` is an instance of ``text_type``, return +-``s.encode('ascii')``, otherwise return ``str(s, 'ascii', 'strict')`` ++ Python 2: If ``s`` is an instance of ``text_type``, return ++ ``s.encode('ascii')``, otherwise return ``str(s)`` ++ ''' ++ if isinstance(s, text_type): ++ s = s.encode('ascii') + +-Python 2: If ``s`` is an instance of ``text_type``, return +-``s.encode('ascii')``, otherwise return ``str(s)`` +-''' ++ return str(s, 'ascii', 'strict') if PY3 else s + + +-if PY3: +- def native_(s, encoding='latin-1', errors='strict'): +- ''' +- If ``s`` is an instance of ``text_type``, return +- ``s``, otherwise return ``str(s, encoding, errors)`` +- ''' +- if isinstance(s, text_type): +- return s +- return str(s, encoding, errors) +-else: +- def native_(s, encoding='latin-1', errors='strict'): +- ''' +- If ``s`` is an instance of ``text_type``, return +- ``s.encode(encoding, errors)``, otherwise return ``str(s)`` +- ''' +- if isinstance(s, text_type): +- return s.encode(encoding, errors) +- return str(s) ++def native_(s, encoding='latin-1', errors='strict'): ++ ''' ++ Python 3: If ``s`` is an instance of ``text_type``, return ``s``, otherwise ++ return ``str(s, encoding, errors)`` + +-native_.__doc__ = ''' +-Python 3: If ``s`` is an instance of ``text_type``, return ``s``, otherwise +-return ``str(s, encoding, errors)`` ++ Python 2: If ``s`` is an instance of ``text_type``, return ++ ``s.encode(encoding, errors)``, otherwise return ``str(s)`` ++ ''' ++ if PY3: ++ out = s if isinstance(s, text_type) else str(s, encoding, errors) ++ else: ++ out = s.encode(encoding, errors) if isinstance(s, text_type) else str(s) + +-Python 2: If ``s`` is an instance of ``text_type``, return +-``s.encode(encoding, errors)``, otherwise return ``str(s)`` +-''' ++ return out + + + def string_io(data=None): # cStringIO can't handle unicode +@@ -133,7 +112,199 @@ def string_io(data=None): # cStringIO can't handle unicode + except (UnicodeEncodeError, TypeError): + return StringIO(data) + +-if PY3: +- import ipaddress +-else: +- import salt.ext.ipaddress as ipaddress ++ ++try: ++ if PY3: ++ import ipaddress ++ else: ++ import salt.ext.ipaddress as ipaddress ++except ImportError: ++ ipaddress = None ++ ++ ++class IPv6AddressScoped(ipaddress.IPv6Address): ++ ''' ++ Represent and manipulate single IPv6 Addresses. ++ Scope-aware version ++ ''' ++ def __init__(self, address): ++ ''' ++ Instantiate a new IPv6 address object. Scope is moved to an attribute 'scope'. ++ ++ Args: ++ address: A string or integer representing the IP ++ ++ Additionally, an integer can be passed, so ++ IPv6Address('2001:db8::') == IPv6Address(42540766411282592856903984951653826560) ++ or, more generally ++ IPv6Address(int(IPv6Address('2001:db8::'))) == IPv6Address('2001:db8::') ++ ++ Raises: ++ AddressValueError: If address isn't a valid IPv6 address. ++ ++ :param address: ++ ''' ++ # pylint: disable-all ++ if not hasattr(self, '_is_packed_binary'): ++ # This method (below) won't be around for some Python 3 versions ++ # and we need check this differently anyway ++ self._is_packed_binary = lambda p: isinstance(p, bytes) ++ # pylint: enable-all ++ ++ if isinstance(address, string_types) and '%' in address: ++ buff = address.split('%') ++ if len(buff) != 2: ++ raise SaltException('Invalid IPv6 address: "{}"'.format(address)) ++ address, self.__scope = buff ++ else: ++ self.__scope = None ++ ++ if sys.version_info.major == 2: ++ ipaddress._BaseAddress.__init__(self, address) ++ ipaddress._BaseV6.__init__(self, address) ++ else: ++ # Python 3.4 fix. Versions higher are simply not affected ++ # https://github.com/python/cpython/blob/3.4/Lib/ipaddress.py#L543-L544 ++ self._version = 6 ++ self._max_prefixlen = ipaddress.IPV6LENGTH ++ ++ # Efficient constructor from integer. ++ if isinstance(address, integer_types): ++ self._check_int_address(address) ++ self._ip = address ++ elif self._is_packed_binary(address): ++ self._check_packed_address(address, 16) ++ self._ip = ipaddress._int_from_bytes(address, 'big') ++ else: ++ address = str(address) ++ if '/' in address: ++ raise ipaddress.AddressValueError("Unexpected '/' in {}".format(address)) ++ self._ip = self._ip_int_from_string(address) ++ ++ def _is_packed_binary(self, data): ++ ''' ++ Check if data is hexadecimal packed ++ ++ :param data: ++ :return: ++ ''' ++ packed = False ++ if len(data) == 16 and ':' not in data: ++ try: ++ packed = bool(int(str(bytearray(data)).encode('hex'), 16)) ++ except ValueError: ++ pass ++ ++ return packed ++ ++ @property ++ def scope(self): ++ ''' ++ Return scope of IPv6 address. ++ ++ :return: ++ ''' ++ return self.__scope ++ ++ def __str__(self): ++ return text_type(self._string_from_ip_int(self._ip) + ++ ('%' + self.scope if self.scope is not None else '')) ++ ++ ++class IPv6InterfaceScoped(ipaddress.IPv6Interface, IPv6AddressScoped): ++ ''' ++ Update ++ ''' ++ def __init__(self, address): ++ if isinstance(address, (bytes, int)): ++ IPv6AddressScoped.__init__(self, address) ++ self.network = ipaddress.IPv6Network(self._ip) ++ self._prefixlen = self._max_prefixlen ++ return ++ ++ addr = ipaddress._split_optional_netmask(address) ++ IPv6AddressScoped.__init__(self, addr[0]) ++ self.network = ipaddress.IPv6Network(address, strict=False) ++ self.netmask = self.network.netmask ++ self._prefixlen = self.network._prefixlen ++ self.hostmask = self.network.hostmask ++ ++ ++def ip_address(address): ++ """Take an IP string/int and return an object of the correct type. ++ ++ Args: ++ address: A string or integer, the IP address. Either IPv4 or ++ IPv6 addresses may be supplied; integers less than 2**32 will ++ be considered to be IPv4 by default. ++ ++ Returns: ++ An IPv4Address or IPv6Address object. ++ ++ Raises: ++ ValueError: if the *address* passed isn't either a v4 or a v6 ++ address ++ ++ """ ++ try: ++ return ipaddress.IPv4Address(address) ++ except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err: ++ log.debug('Error while parsing IPv4 address: %s', address) ++ log.debug(err) ++ ++ try: ++ return IPv6AddressScoped(address) ++ except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err: ++ log.debug('Error while parsing IPv6 address: %s', address) ++ log.debug(err) ++ ++ if isinstance(address, bytes): ++ raise ipaddress.AddressValueError('{} does not appear to be an IPv4 or IPv6 address. ' ++ 'Did you pass in a bytes (str in Python 2) instead ' ++ 'of a unicode object?'.format(repr(address))) ++ ++ raise ValueError('{} does not appear to be an IPv4 or IPv6 address'.format(repr(address))) ++ ++ ++def ip_interface(address): ++ """Take an IP string/int and return an object of the correct type. ++ ++ Args: ++ address: A string or integer, the IP address. Either IPv4 or ++ IPv6 addresses may be supplied; integers less than 2**32 will ++ be considered to be IPv4 by default. ++ ++ Returns: ++ An IPv4Interface or IPv6Interface object. ++ ++ Raises: ++ ValueError: if the string passed isn't either a v4 or a v6 ++ address. ++ ++ Notes: ++ The IPv?Interface classes describe an Address on a particular ++ Network, so they're basically a combination of both the Address ++ and Network classes. ++ ++ """ ++ try: ++ return ipaddress.IPv4Interface(address) ++ except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err: ++ log.debug('Error while getting IPv4 interface for address %s', address) ++ log.debug(err) ++ ++ try: ++ return ipaddress.IPv6Interface(address) ++ except (ipaddress.AddressValueError, ipaddress.NetmaskValueError) as err: ++ log.debug('Error while getting IPv6 interface for address %s', address) ++ log.debug(err) ++ ++ raise ValueError('{} does not appear to be an IPv4 or IPv6 interface'.format(address)) ++ ++ ++if ipaddress: ++ ipaddress.IPv6Address = IPv6AddressScoped ++ if sys.version_info.major == 2: ++ ipaddress.IPv6Interface = IPv6InterfaceScoped ++ ipaddress.ip_address = ip_address ++ ipaddress.ip_interface = ip_interface +diff --git a/salt/cloud/clouds/saltify.py b/salt/cloud/clouds/saltify.py +index c9cc281b42..e0e56349a0 100644 +--- a/salt/cloud/clouds/saltify.py ++++ b/salt/cloud/clouds/saltify.py +@@ -27,10 +27,7 @@ import salt.utils.cloud + import salt.config as config + import salt.client + import salt.ext.six as six +-if six.PY3: +- import ipaddress +-else: +- import salt.ext.ipaddress as ipaddress ++from salt._compat import ipaddress + + from salt.exceptions import SaltCloudException, SaltCloudSystemExit + +diff --git a/salt/cloud/clouds/vagrant.py b/salt/cloud/clouds/vagrant.py +index a24170c78a..0fe410eb91 100644 +--- a/salt/cloud/clouds/vagrant.py ++++ b/salt/cloud/clouds/vagrant.py +@@ -25,13 +25,8 @@ import tempfile + import salt.utils + import salt.config as config + import salt.client +-import salt.ext.six as six +-if six.PY3: +- import ipaddress +-else: +- import salt.ext.ipaddress as ipaddress +-from salt.exceptions import SaltCloudException, SaltCloudSystemExit, \ +- SaltInvocationError ++from salt._compat import ipaddress ++from salt.exceptions import SaltCloudException, SaltCloudSystemExit, SaltInvocationError + + # Get logging started + log = logging.getLogger(__name__) +diff --git a/salt/ext/win_inet_pton.py b/salt/ext/win_inet_pton.py +index 1204bede10..89aba14ce9 100644 +--- a/salt/ext/win_inet_pton.py ++++ b/salt/ext/win_inet_pton.py +@@ -9,7 +9,7 @@ from __future__ import absolute_import + import socket + import ctypes + import os +-import ipaddress ++from salt._compat import ipaddress + import salt.ext.six as six + + +diff --git a/salt/minion.py b/salt/minion.py +index 17e11c0ebe..9c05a646ea 100644 +--- a/salt/minion.py ++++ b/salt/minion.py +@@ -26,10 +26,7 @@ from binascii import crc32 + # Import Salt Libs + # pylint: disable=import-error,no-name-in-module,redefined-builtin + from salt.ext import six +-if six.PY3: +- import ipaddress +-else: +- import salt.ext.ipaddress as ipaddress ++from salt._compat import ipaddress + from salt.ext.six.moves import range + from salt.utils.zeromq import zmq, ZMQDefaultLoop, install_zmq, ZMQ_VERSION_INFO + +diff --git a/salt/modules/ipset.py b/salt/modules/ipset.py +index 7047e84c29..1a0fa0044d 100644 +--- a/salt/modules/ipset.py ++++ b/salt/modules/ipset.py +@@ -13,10 +13,7 @@ from salt.ext.six.moves import map, range + import salt.utils.path + + # Import third-party libs +-if six.PY3: +- import ipaddress +-else: +- import salt.ext.ipaddress as ipaddress ++from salt._compat import ipaddress + + # Set up logging + log = logging.getLogger(__name__) +diff --git a/salt/modules/network.py b/salt/modules/network.py +index 92893572a6..60f586f6bc 100644 +--- a/salt/modules/network.py ++++ b/salt/modules/network.py +@@ -26,10 +26,7 @@ from salt.exceptions import CommandExecutionError + # Import 3rd-party libs + from salt.ext import six + from salt.ext.six.moves import range # pylint: disable=import-error,no-name-in-module,redefined-builtin +-if six.PY3: +- import ipaddress +-else: +- import salt.ext.ipaddress as ipaddress ++from salt._compat import ipaddress + + + log = logging.getLogger(__name__) +diff --git a/salt/modules/vagrant.py b/salt/modules/vagrant.py +index 0592dede55..0f518c2602 100644 +--- a/salt/modules/vagrant.py ++++ b/salt/modules/vagrant.py +@@ -39,11 +39,7 @@ import salt.utils.path + import salt.utils.stringutils + from salt.exceptions import CommandExecutionError, SaltInvocationError + import salt.ext.six as six +- +-if six.PY3: +- import ipaddress +-else: +- import salt.ext.ipaddress as ipaddress ++from salt._compat import ipaddress + + log = logging.getLogger(__name__) + +diff --git a/salt/utils/dns.py b/salt/utils/dns.py +index db08bcb7ac..40011016fd 100644 +--- a/salt/utils/dns.py ++++ b/salt/utils/dns.py +@@ -1029,18 +1029,13 @@ def parse_resolv(src='/etc/resolv.conf'): + try: + (directive, arg) = (line[0].lower(), line[1:]) + # Drop everything after # or ; (comments) +- arg = list(itertools.takewhile( +- lambda x: x[0] not in ('#', ';'), arg)) +- ++ arg = list(itertools.takewhile(lambda x: x[0] not in ('#', ';'), arg)) + if directive == 'nameserver': +- # Split the scope (interface) if it is present +- addr, scope = arg[0].split('%', 1) if '%' in arg[0] else (arg[0], '') ++ addr = arg[0] + try: + ip_addr = ipaddress.ip_address(addr) + version = ip_addr.version +- # Rejoin scope after address validation +- if scope: +- ip_addr = '%'.join((str(ip_addr), scope)) ++ ip_addr = str(ip_addr) + if ip_addr not in nameservers: + nameservers.append(ip_addr) + if version == 4 and ip_addr not in ip4_nameservers: +diff --git a/salt/utils/minions.py b/salt/utils/minions.py +index bb0cbaa589..f282464eee 100644 +--- a/salt/utils/minions.py ++++ b/salt/utils/minions.py +@@ -26,10 +26,7 @@ import salt.cache + from salt.ext import six + + # Import 3rd-party libs +-if six.PY3: +- import ipaddress +-else: +- import salt.ext.ipaddress as ipaddress ++from salt._compat import ipaddress + HAS_RANGE = False + try: + import seco.range # pylint: disable=import-error +diff --git a/tests/unit/grains/test_core.py b/tests/unit/grains/test_core.py +index dd7d5b06f8..e973428add 100644 +--- a/tests/unit/grains/test_core.py ++++ b/tests/unit/grains/test_core.py +@@ -32,10 +32,7 @@ import salt.grains.core as core + + # Import 3rd-party libs + from salt.ext import six +-if six.PY3: +- import ipaddress +-else: +- import salt.ext.ipaddress as ipaddress ++from salt._compat import ipaddress + + log = logging.getLogger(__name__) + +diff --git a/tests/unit/modules/test_network.py b/tests/unit/modules/test_network.py +index 865f15f3e3..50fa629276 100644 +--- a/tests/unit/modules/test_network.py ++++ b/tests/unit/modules/test_network.py +@@ -20,20 +20,11 @@ from tests.support.mock import ( + ) + + # Import Salt Libs +-from salt.ext import six + import salt.utils.network + import salt.utils.path + import salt.modules.network as network + from salt.exceptions import CommandExecutionError +-if six.PY2: +- import salt.ext.ipaddress as ipaddress +- HAS_IPADDRESS = True +-else: +- try: +- import ipaddress +- HAS_IPADDRESS = True +- except ImportError: +- HAS_IPADDRESS = False ++from salt._compat import ipaddress + + + @skipIf(NO_MOCK, NO_MOCK_REASON) +@@ -278,7 +269,7 @@ class NetworkTestCase(TestCase, LoaderModuleMockMixin): + self.assertDictEqual(network.connect('host', 'port'), + {'comment': ret, 'result': True}) + +- @skipIf(HAS_IPADDRESS is False, 'unable to import \'ipaddress\'') ++ @skipIf(not bool(ipaddress), 'unable to import \'ipaddress\'') + def test_is_private(self): + ''' + Test for Check if the given IP address is a private address +@@ -290,7 +281,7 @@ class NetworkTestCase(TestCase, LoaderModuleMockMixin): + return_value=True): + self.assertTrue(network.is_private('::1')) + +- @skipIf(HAS_IPADDRESS is False, 'unable to import \'ipaddress\'') ++ @skipIf(not bool(ipaddress), 'unable to import \'ipaddress\'') + def test_is_loopback(self): + ''' + Test for Check if the given IP address is a loopback address +-- +2.19.0 + + diff --git a/fix-issue-2068-test.patch b/fix-issue-2068-test.patch new file mode 100644 index 0000000..c70d265 --- /dev/null +++ b/fix-issue-2068-test.patch @@ -0,0 +1,52 @@ +From 2916f2f3e7c6af07148863281ffaf07df21f21da Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Wed, 9 Jan 2019 16:08:19 +0100 +Subject: [PATCH] Fix issue #2068 test + +Skip injecting `__call__` if chunk is not dict. + +This also fixes `integration/modules/test_state.py:StateModuleTest.test_exclude` that tests `include` and `exclude` state directives containing the only list of strings. + +Minor update: more correct is-dict check. +--- + salt/state.py | 9 ++++++--- + 1 file changed, 6 insertions(+), 3 deletions(-) + +diff --git a/salt/state.py b/salt/state.py +index b4b2a00601..815ebaec24 100644 +--- a/salt/state.py ++++ b/salt/state.py +@@ -25,6 +25,7 @@ import traceback + import re + import time + import random ++import collections + + # Import salt libs + import salt.loader +@@ -2743,16 +2744,18 @@ class State(object): + ''' + for chunk in high: + state = high[chunk] ++ if not isinstance(state, collections.Mapping): ++ continue + for state_ref in state: + needs_default = True ++ if not isinstance(state[state_ref], list): ++ continue + for argset in state[state_ref]: + if isinstance(argset, six.string_types): + needs_default = False + break + if needs_default: +- order = state[state_ref].pop(-1) +- state[state_ref].append('__call__') +- state[state_ref].append(order) ++ state[state_ref].insert(-1, '__call__') + + def call_high(self, high, orchestration_jid=None): + ''' +-- +2.20.1 + + diff --git a/fix-latin1-encoding-problems-on-file-module-bsc-1116.patch b/fix-latin1-encoding-problems-on-file-module-bsc-1116.patch new file mode 100644 index 0000000..5f9c966 --- /dev/null +++ b/fix-latin1-encoding-problems-on-file-module-bsc-1116.patch @@ -0,0 +1,941 @@ +From 140388e51e5b5b7ee33b776269bce67046cce32f Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Tue, 4 Dec 2018 16:16:18 +0000 +Subject: [PATCH] Fix latin1 encoding problems on file module + (bsc#1116837) + +_get_line_indent renamed to _set_line_indent + +_regex_to_static refactored to work on lists + +line function refactored to work on list + +Added _set_line_eol and _get_eol functions + +Setting end of line + +Make tests green + +test_line_insert_end fixed + +/sr.* pattern should raise exception + +file.line function refactored + +Make integration green. Added test for mode ensure insert before first line + +Fixed file permissions + +Removed regex compilation + +Comprehensions converting to unicode replaced by salt.utils.data.decode_list + +Empty match on delete or replace not causing IndexError exception + +List comprehension replaced + +Added comments + +Add get_diff to salt.utils.stringutils + +Make to_unicode/to_str/to_bytes helpers attempt latin-1 + +Also allow for multiple encodings to be passed + +Use new get_diff helper in file module + +Use BASE_FILES instead of redundant STATE_DIR + +Add integration test for latin-1 file diffs + +PY3 scoping fix + +In PY3 the caught exceptions now drop out of scope when leaving the for +loop. + +Add unit test for latin-1 fallback, multi-encoding + +Skip pylint false-positives + +Fix incorrect use of __salt__ when __utils__ is needed + +Add stringutils.get_diff to mocks + +Only try latin-1 from get_diff instead of by default + +Fix to_unicode test + +Since latin-1 is not being automatically decoded, we need to explicitly +pass it on the test. + +Revert "Use BASE_FILES instead of redundant STATE_DIR" + +This reverts commit ba524c81b6ae6091259157cec1259f5a7fb776c0. +--- + salt/modules/file.py | 224 +++++++++--------- + salt/modules/win_file.py | 14 +- + salt/utils/stringutils.py | 118 ++++++--- + .../files/file/base/issue-48777/new.html | 5 + + .../files/file/base/issue-48777/old.html | 4 + + tests/integration/states/test_file.py | 23 ++ + tests/unit/modules/test_file.py | 102 +++++++- + tests/unit/utils/test_stringutils.py | 14 ++ + 8 files changed, 348 insertions(+), 156 deletions(-) + create mode 100644 tests/integration/files/file/base/issue-48777/new.html + create mode 100644 tests/integration/files/file/base/issue-48777/old.html + +diff --git a/salt/modules/file.py b/salt/modules/file.py +index 1b4b7e0e46..1ad0fef1ea 100644 +--- a/salt/modules/file.py ++++ b/salt/modules/file.py +@@ -12,7 +12,6 @@ from __future__ import absolute_import, print_function, unicode_literals + + # Import python libs + import datetime +-import difflib + import errno + import fileinput + import fnmatch +@@ -61,6 +60,7 @@ import salt.utils.stringutils + import salt.utils.templates + import salt.utils.url + import salt.utils.user ++import salt.utils.data + from salt.exceptions import CommandExecutionError, MinionError, SaltInvocationError, get_error_message as _get_error_message + from salt.utils.files import HASHES, HASHES_REVMAP + +@@ -1570,7 +1570,7 @@ def comment_line(path, + check_perms(path, None, pre_user, pre_group, pre_mode) + + # Return a diff using the two dictionaries +- return ''.join(difflib.unified_diff(orig_file, new_file)) ++ return __utils__['stringutils.get_diff'](orig_file, new_file) + + + def _get_flags(flags): +@@ -1722,18 +1722,19 @@ def _regex_to_static(src, regex): + return None + + try: +- src = re.search(regex, src, re.M) ++ compiled = re.compile(regex, re.DOTALL) ++ src = [line for line in src if compiled.search(line) or line.count(regex)] + except Exception as ex: + raise CommandExecutionError("{0}: '{1}'".format(_get_error_message(ex), regex)) + +- return src and src.group().rstrip('\r') or regex ++ return src and src or [] + + +-def _assert_occurrence(src, probe, target, amount=1): ++def _assert_occurrence(probe, target, amount=1): + ''' + Raise an exception, if there are different amount of specified occurrences in src. + ''' +- occ = src.count(probe) ++ occ = len(probe) + if occ > amount: + msg = 'more than' + elif occ < amount: +@@ -1749,7 +1750,7 @@ def _assert_occurrence(src, probe, target, amount=1): + return occ + + +-def _get_line_indent(src, line, indent): ++def _set_line_indent(src, line, indent): + ''' + Indent the line with the source line. + ''' +@@ -1762,7 +1763,36 @@ def _get_line_indent(src, line, indent): + break + idt.append(c) + +- return ''.join(idt) + line.strip() ++ return ''.join(idt) + line.lstrip() ++ ++ ++def _get_eol(line): ++ match = re.search('((? -1 and not file_line == content) else file_line) +- for file_line in body.split(os.linesep)]) ++ body = [] ++ elif mode == 'delete' and match: ++ body = [line for line in body if line != match[0]] ++ elif mode == 'replace' and match: ++ idx = body.index(match[0]) ++ file_line = body.pop(idx) ++ body.insert(idx, _set_line_indent(file_line, content, indent)) + elif mode == 'insert': + if not location and not before and not after: + raise CommandExecutionError('On insert must be defined either "location" or "before/after" conditions.') + + if not location: + if before and after: +- _assert_occurrence(body, before, 'before') +- _assert_occurrence(body, after, 'after') ++ _assert_occurrence(before, 'before') ++ _assert_occurrence(after, 'after') ++ + out = [] +- lines = body.split(os.linesep) + in_range = False +- for line in lines: +- if line.find(after) > -1: ++ for line in body: ++ if line == after[0]: + in_range = True +- elif line.find(before) > -1 and in_range: +- out.append(_get_line_indent(line, content, indent)) ++ elif line == before[0] and in_range: ++ cnd = _set_line_indent(line, content, indent) ++ out.append(cnd) + out.append(line) +- body = os.linesep.join(out) ++ body = out + + if before and not after: +- _assert_occurrence(body, before, 'before') +- out = [] +- lines = body.split(os.linesep) +- for idx in range(len(lines)): +- _line = lines[idx] +- if _line.find(before) > -1: +- cnd = _get_line_indent(_line, content, indent) +- if not idx or (idx and _starts_till(lines[idx - 1], cnd) < 0): # Job for replace instead +- out.append(cnd) +- out.append(_line) +- body = os.linesep.join(out) ++ _assert_occurrence(before, 'before') ++ ++ idx = body.index(before[0]) ++ body = _insert_line_before(idx, body, content, indent) + + elif after and not before: +- _assert_occurrence(body, after, 'after') +- out = [] +- lines = body.split(os.linesep) +- for idx, _line in enumerate(lines): +- out.append(_line) +- cnd = _get_line_indent(_line, content, indent) +- # No duplicates or append, if "after" is the last line +- if (_line.find(after) > -1 and +- (lines[((idx + 1) < len(lines)) and idx + 1 or idx].strip() != cnd or +- idx + 1 == len(lines))): +- out.append(cnd) +- body = os.linesep.join(out) ++ _assert_occurrence(after, 'after') ++ ++ idx = body.index(after[0]) ++ body = _insert_line_after(idx, body, content, indent) + + else: + if location == 'start': +- body = os.linesep.join((content, body)) ++ if body: ++ body.insert(0, _set_line_eol(body[0], content)) ++ else: ++ body.append(content + os.linesep) + elif location == 'end': +- body = os.linesep.join((body, _get_line_indent(body[-1], content, indent) if body else content)) ++ body.append(_set_line_indent(body[-1], content, indent) if body else content) + + elif mode == 'ensure': +- after = after and after.strip() +- before = before and before.strip() + + if before and after: +- _assert_occurrence(body, before, 'before') +- _assert_occurrence(body, after, 'after') ++ _assert_occurrence(before, 'before') ++ _assert_occurrence(after, 'after') + +- is_there = bool(body.count(content)) ++ is_there = bool([l for l in body if l.count(content)]) + if not is_there: +- out = [] +- body = body.split(os.linesep) +- for idx, line in enumerate(body): +- out.append(line) +- if line.find(content) > -1: +- is_there = True +- if not is_there: +- if idx < (len(body) - 1) and line.find(after) > -1 and body[idx + 1].find(before) > -1: +- out.append(content) +- elif line.find(after) > -1: +- raise CommandExecutionError('Found more than one line between ' +- 'boundaries "before" and "after".') +- body = os.linesep.join(out) ++ idx = body.index(after[0]) ++ if idx < (len(body) - 1) and body[idx + 1] == before[0]: ++ cnd = _set_line_indent(body[idx], content, indent) ++ body.insert(idx + 1, cnd) ++ else: ++ raise CommandExecutionError('Found more than one line between ' ++ 'boundaries "before" and "after".') + + elif before and not after: +- _assert_occurrence(body, before, 'before') +- body = body.split(os.linesep) +- out = [] +- for idx in range(len(body)): +- if body[idx].find(before) > -1: +- prev = (idx > 0 and idx or 1) - 1 +- out.append(_get_line_indent(body[idx], content, indent)) +- if _starts_till(out[prev], content) > -1: +- del out[prev] +- out.append(body[idx]) +- body = os.linesep.join(out) ++ _assert_occurrence(before, 'before') ++ ++ idx = body.index(before[0]) ++ body = _insert_line_before(idx, body, content, indent) + + elif not before and after: +- _assert_occurrence(body, after, 'after') +- body = body.split(os.linesep) +- skip = None +- out = [] +- for idx in range(len(body)): +- if skip != body[idx]: +- out.append(body[idx]) +- +- if body[idx].find(after) > -1: +- next_line = idx + 1 < len(body) and body[idx + 1] or None +- if next_line is not None and _starts_till(next_line, content) > -1: +- skip = next_line +- out.append(_get_line_indent(body[idx], content, indent)) +- body = os.linesep.join(out) ++ _assert_occurrence(after, 'after') ++ ++ idx = body.index(after[0]) ++ body = _insert_line_after(idx, body, content, indent) + + else: + raise CommandExecutionError("Wrong conditions? " + "Unable to ensure line without knowing " + "where to put it before and/or after.") + +- changed = body_before != hashlib.sha256(salt.utils.stringutils.to_bytes(body)).hexdigest() ++ if body: ++ for idx, line in enumerate(body): ++ if not _get_eol(line) and idx+1 < len(body): ++ prev = idx and idx-1 or 1 ++ body[idx] = _set_line_eol(body[prev], line) ++ # We do not need empty line at the end anymore ++ if '' == body[-1]: ++ body.pop() ++ ++ changed = body_before != hashlib.sha256(salt.utils.stringutils.to_bytes(''.join(body))).hexdigest() + + if backup and changed and __opts__['test'] is False: + try: +@@ -2032,20 +2040,15 @@ def line(path, content=None, match=None, mode=None, location=None, + if changed: + if show_changes: + with salt.utils.files.fopen(path, 'r') as fp_: +- path_content = [salt.utils.stringutils.to_unicode(x) +- for x in fp_.read().splitlines(True)] +- changes_diff = ''.join(difflib.unified_diff( +- path_content, +- [salt.utils.stringutils.to_unicode(x) +- for x in body.splitlines(True)] +- )) ++ path_content = salt.utils.data.decode_list(fp_.read().splitlines(True)) ++ changes_diff = __utils__['stringutils.get_diff'](path_content, body) + if __opts__['test'] is False: + fh_ = None + try: + # Make sure we match the file mode from salt.utils.files.fopen + mode = 'wb' if six.PY2 and salt.utils.platform.is_windows() else 'w' + fh_ = salt.utils.atomicfile.atomic_open(path, mode) +- fh_.write(body) ++ fh_.write(''.join(body)) + finally: + if fh_: + fh_.close() +@@ -2419,18 +2422,15 @@ def replace(path, + if not dry_run and not salt.utils.platform.is_windows(): + check_perms(path, None, pre_user, pre_group, pre_mode) + +- def get_changes(): +- orig_file_as_str = [salt.utils.stringutils.to_unicode(x) for x in orig_file] +- new_file_as_str = [salt.utils.stringutils.to_unicode(x) for x in new_file] +- return ''.join(difflib.unified_diff(orig_file_as_str, new_file_as_str)) ++ differences = __utils__['stringutils.get_diff'](orig_file, new_file) + + if show_changes: +- return get_changes() ++ return differences + + # We may have found a regex line match but don't need to change the line + # (for situations where the pattern also matches the repl). Revert the + # has_changes flag to False if the final result is unchanged. +- if not get_changes(): ++ if not differences: + has_changes = False + + return has_changes +@@ -2684,7 +2684,7 @@ def blockreplace(path, + ) + + if block_found: +- diff = ''.join(difflib.unified_diff(orig_file, new_file)) ++ diff = __utils__['stringutils.get_diff'](orig_file, new_file) + has_changes = diff is not '' + if has_changes and not dry_run: + # changes detected +@@ -5003,11 +5003,7 @@ def get_diff(file1, + else: + if show_filenames: + args.extend(files) +- ret = ''.join( +- difflib.unified_diff( +- *salt.utils.data.decode(args) +- ) +- ) ++ ret = __utils__['stringutils.get_diff'](*args) + return ret + return '' + +diff --git a/salt/modules/win_file.py b/salt/modules/win_file.py +index d321bd538e..0f5c908c8f 100644 +--- a/salt/modules/win_file.py ++++ b/salt/modules/win_file.py +@@ -58,8 +58,9 @@ from salt.modules.file import (check_hash, # pylint: disable=W0611 + RE_FLAG_TABLE, blockreplace, prepend, seek_read, seek_write, rename, + lstat, path_exists_glob, write, pardir, join, HASHES, HASHES_REVMAP, + comment, uncomment, _add_flags, comment_line, _regex_to_static, +- _get_line_indent, apply_template_on_contents, dirname, basename, +- list_backups_dir, _assert_occurrence, _starts_till) ++ _set_line_indent, apply_template_on_contents, dirname, basename, ++ list_backups_dir, _assert_occurrence, _starts_till, _set_line_eol, _get_eol, ++ _insert_line_after, _insert_line_before) + from salt.modules.file import normpath as normpath_ + + from salt.utils.functools import namespaced_function as _namespaced_function +@@ -116,8 +117,9 @@ def __virtual__(): + global blockreplace, prepend, seek_read, seek_write, rename, lstat + global write, pardir, join, _add_flags, apply_template_on_contents + global path_exists_glob, comment, uncomment, _mkstemp_copy +- global _regex_to_static, _get_line_indent, dirname, basename ++ global _regex_to_static, _set_line_indent, dirname, basename + global list_backups_dir, normpath_, _assert_occurrence, _starts_till ++ global _insert_line_before, _insert_line_after, _set_line_eol, _get_eol + + replace = _namespaced_function(replace, globals()) + search = _namespaced_function(search, globals()) +@@ -172,7 +174,11 @@ def __virtual__(): + uncomment = _namespaced_function(uncomment, globals()) + comment_line = _namespaced_function(comment_line, globals()) + _regex_to_static = _namespaced_function(_regex_to_static, globals()) +- _get_line_indent = _namespaced_function(_get_line_indent, globals()) ++ _set_line_indent = _namespaced_function(_set_line_indent, globals()) ++ _set_line_eol = _namespaced_function(_set_line_eol, globals()) ++ _get_eol = _namespaced_function(_get_eol, globals()) ++ _insert_line_after = _namespaced_function(_insert_line_after, globals()) ++ _insert_line_before = _namespaced_function(_insert_line_before, globals()) + _mkstemp_copy = _namespaced_function(_mkstemp_copy, globals()) + _add_flags = _namespaced_function(_add_flags, globals()) + apply_template_on_contents = _namespaced_function(apply_template_on_contents, globals()) +diff --git a/salt/utils/stringutils.py b/salt/utils/stringutils.py +index 2909d4aebe..f84fda70a5 100644 +--- a/salt/utils/stringutils.py ++++ b/salt/utils/stringutils.py +@@ -6,6 +6,7 @@ Functions for manipulating or otherwise processing strings + # Import Python libs + from __future__ import absolute_import, print_function, unicode_literals + import base64 ++import difflib + import errno + import fnmatch + import logging +@@ -31,21 +32,32 @@ def to_bytes(s, encoding=None, errors='strict'): + Given bytes, bytearray, str, or unicode (python 2), return bytes (str for + python 2) + ''' ++ if encoding is None: ++ # Try utf-8 first, and fall back to detected encoding ++ encoding = ('utf-8', __salt_system_encoding__) ++ if not isinstance(encoding, (tuple, list)): ++ encoding = (encoding,) ++ ++ if not encoding: ++ raise ValueError('encoding cannot be empty') ++ ++ exc = None + if six.PY3: + if isinstance(s, bytes): + return s + if isinstance(s, bytearray): + return bytes(s) + if isinstance(s, six.string_types): +- if encoding: +- return s.encode(encoding, errors) +- else: ++ for enc in encoding: + try: +- # Try UTF-8 first +- return s.encode('utf-8', errors) +- except UnicodeEncodeError: +- # Fall back to detected encoding +- return s.encode(__salt_system_encoding__, errors) ++ return s.encode(enc, errors) ++ except UnicodeEncodeError as err: ++ exc = err ++ continue ++ # The only way we get this far is if a UnicodeEncodeError was ++ # raised, otherwise we would have already returned (or raised some ++ # other exception). ++ raise exc # pylint: disable=raising-bad-type + raise TypeError('expected bytes, bytearray, or str') + else: + return to_str(s, encoding, errors) +@@ -61,35 +73,48 @@ def to_str(s, encoding=None, errors='strict', normalize=False): + except TypeError: + return s + ++ if encoding is None: ++ # Try utf-8 first, and fall back to detected encoding ++ encoding = ('utf-8', __salt_system_encoding__) ++ if not isinstance(encoding, (tuple, list)): ++ encoding = (encoding,) ++ ++ if not encoding: ++ raise ValueError('encoding cannot be empty') ++ + # This shouldn't be six.string_types because if we're on PY2 and we already + # have a string, we should just return it. + if isinstance(s, str): + return _normalize(s) ++ ++ exc = None + if six.PY3: + if isinstance(s, (bytes, bytearray)): +- if encoding: +- return _normalize(s.decode(encoding, errors)) +- else: ++ for enc in encoding: + try: +- # Try UTF-8 first +- return _normalize(s.decode('utf-8', errors)) +- except UnicodeDecodeError: +- # Fall back to detected encoding +- return _normalize(s.decode(__salt_system_encoding__, errors)) ++ return _normalize(s.decode(enc, errors)) ++ except UnicodeDecodeError as err: ++ exc = err ++ continue ++ # The only way we get this far is if a UnicodeDecodeError was ++ # raised, otherwise we would have already returned (or raised some ++ # other exception). ++ raise exc # pylint: disable=raising-bad-type + raise TypeError('expected str, bytes, or bytearray not {}'.format(type(s))) + else: + if isinstance(s, bytearray): + return str(s) # future lint: disable=blacklisted-function + if isinstance(s, unicode): # pylint: disable=incompatible-py3-code,undefined-variable +- if encoding: +- return _normalize(s).encode(encoding, errors) +- else: ++ for enc in encoding: + try: +- # Try UTF-8 first +- return _normalize(s).encode('utf-8', errors) +- except UnicodeEncodeError: +- # Fall back to detected encoding +- return _normalize(s).encode(__salt_system_encoding__, errors) ++ return _normalize(s).encode(enc, errors) ++ except UnicodeEncodeError as err: ++ exc = err ++ continue ++ # The only way we get this far is if a UnicodeDecodeError was ++ # raised, otherwise we would have already returned (or raised some ++ # other exception). ++ raise exc # pylint: disable=raising-bad-type + raise TypeError('expected str, bytearray, or unicode') + + +@@ -100,6 +125,16 @@ def to_unicode(s, encoding=None, errors='strict', normalize=False): + def _normalize(s): + return unicodedata.normalize('NFC', s) if normalize else s + ++ if encoding is None: ++ # Try utf-8 first, and fall back to detected encoding ++ encoding = ('utf-8', __salt_system_encoding__) ++ if not isinstance(encoding, (tuple, list)): ++ encoding = (encoding,) ++ ++ if not encoding: ++ raise ValueError('encoding cannot be empty') ++ ++ exc = None + if six.PY3: + if isinstance(s, str): + return _normalize(s) +@@ -113,15 +148,16 @@ def to_unicode(s, encoding=None, errors='strict', normalize=False): + if isinstance(s, unicode): # pylint: disable=incompatible-py3-code + return _normalize(s) + elif isinstance(s, (str, bytearray)): +- if encoding: +- return _normalize(s.decode(encoding, errors)) +- else: ++ for enc in encoding: + try: +- # Try UTF-8 first +- return _normalize(s.decode('utf-8', errors)) +- except UnicodeDecodeError: +- # Fall back to detected encoding +- return _normalize(s.decode(__salt_system_encoding__, errors)) ++ return _normalize(s.decode(enc, errors)) ++ except UnicodeDecodeError as err: ++ exc = err ++ continue ++ # The only way we get this far is if a UnicodeDecodeError was ++ # raised, otherwise we would have already returned (or raised some ++ # other exception). ++ raise exc # pylint: disable=raising-bad-type + raise TypeError('expected str or bytearray') + + +@@ -513,3 +549,21 @@ def get_context(template, line, num_lines=5, marker=None): + buf[error_line_in_context] += marker + + return '---\n{0}\n---'.format('\n'.join(buf)) ++ ++ ++def get_diff(a, b, *args, **kwargs): ++ ''' ++ Perform diff on two iterables containing lines from two files, and return ++ the diff as as string. Lines are normalized to str types to avoid issues ++ with unicode on PY2. ++ ''' ++ encoding = ('utf-8', 'latin-1', __salt_system_encoding__) ++ # Late import to avoid circular import ++ import salt.utils.data ++ return ''.join( ++ difflib.unified_diff( ++ salt.utils.data.decode_list(a, encoding=encoding), ++ salt.utils.data.decode_list(b, encoding=encoding), ++ *args, **kwargs ++ ) ++ ) +diff --git a/tests/integration/files/file/base/issue-48777/new.html b/tests/integration/files/file/base/issue-48777/new.html +new file mode 100644 +index 0000000000..2d5c1ae744 +--- /dev/null ++++ b/tests/integration/files/file/base/issue-48777/new.html +@@ -0,0 +1,5 @@ ++ ++ ++rksmrgs ++ ++ +diff --git a/tests/integration/files/file/base/issue-48777/old.html b/tests/integration/files/file/base/issue-48777/old.html +new file mode 100644 +index 0000000000..7879e1ce9f +--- /dev/null ++++ b/tests/integration/files/file/base/issue-48777/old.html +@@ -0,0 +1,4 @@ ++ ++ ++ ++ +diff --git a/tests/integration/states/test_file.py b/tests/integration/states/test_file.py +index 9064ba7cc1..30ad39de6b 100644 +--- a/tests/integration/states/test_file.py ++++ b/tests/integration/states/test_file.py +@@ -656,6 +656,29 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): + self.assertIn( + 'does not exist', ret['comment']) + ++ def test_managed_latin1_diff(self): ++ ''' ++ Tests that latin-1 file contents are represented properly in the diff ++ ''' ++ name = os.path.join(TMP, 'local_latin1_diff') ++ # Lay down the initial file ++ ret = self.run_state( ++ 'file.managed', ++ name=name, ++ source='salt://issue-48777/old.html') ++ ret = ret[next(iter(ret))] ++ assert ret['result'] is True, ret ++ ++ # Replace it with the new file and check the diff ++ ret = self.run_state( ++ 'file.managed', ++ name=name, ++ source='salt://issue-48777/new.html') ++ ret = ret[next(iter(ret))] ++ assert ret['result'] is True, ret ++ diff_lines = ret['changes']['diff'].split('\n') ++ assert '+räksmörgås' in diff_lines, diff_lines ++ + def test_directory(self): + ''' + file.directory +diff --git a/tests/unit/modules/test_file.py b/tests/unit/modules/test_file.py +index b157a577e5..66acaf9cb6 100644 +--- a/tests/unit/modules/test_file.py ++++ b/tests/unit/modules/test_file.py +@@ -57,7 +57,10 @@ class FileReplaceTestCase(TestCase, LoaderModuleMockMixin): + 'grains': {}, + }, + '__grains__': {'kernel': 'Linux'}, +- '__utils__': {'files.is_text': MagicMock(return_value=True)}, ++ '__utils__': { ++ 'files.is_text': MagicMock(return_value=True), ++ 'stringutils.get_diff': salt.utils.stringutils.get_diff, ++ }, + } + } + +@@ -235,7 +238,12 @@ class FileBlockReplaceTestCase(TestCase, LoaderModuleMockMixin): + 'grains': {}, + }, + '__grains__': {'kernel': 'Linux'}, +- '__utils__': {'files.is_text': MagicMock(return_value=True)}, ++ '__utils__': { ++ 'files.is_binary': MagicMock(return_value=False), ++ 'files.is_text': MagicMock(return_value=True), ++ 'files.get_encoding': MagicMock(return_value='utf-8'), ++ 'stringutils.get_diff': salt.utils.stringutils.get_diff, ++ }, + } + } + +@@ -528,7 +536,10 @@ class FileModuleTestCase(TestCase, LoaderModuleMockMixin): + 'cachedir': 'tmp', + 'grains': {}, + }, +- '__grains__': {'kernel': 'Linux'} ++ '__grains__': {'kernel': 'Linux'}, ++ '__utils__': { ++ 'stringutils.get_diff': salt.utils.stringutils.get_diff, ++ }, + } + } + +@@ -907,7 +918,10 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): + 'cachedir': 'tmp', + 'grains': {}, + }, +- '__grains__': {'kernel': 'Linux'} ++ '__grains__': {'kernel': 'Linux'}, ++ '__utils__': { ++ 'stringutils.get_diff': salt.utils.stringutils.get_diff, ++ }, + } + } + +@@ -930,6 +944,29 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): + self.assertIn('Cannot find text to {0}'.format(mode), + _log.warning.call_args_list[0][0][0]) + ++ @patch('os.path.realpath', MagicMock()) ++ @patch('os.path.isfile', MagicMock(return_value=True)) ++ @patch('os.stat', MagicMock()) ++ def test_line_delete_no_match(self): ++ ''' ++ Tests that when calling file.line with ``mode=delete``, ++ with not matching pattern to delete returns False ++ :return: ++ ''' ++ file_content = os.linesep.join([ ++ 'file_roots:', ++ ' base:', ++ ' - /srv/salt', ++ ' - /srv/custom' ++ ]) ++ match = 'not matching' ++ for mode in ['delete', 'replace']: ++ files_fopen = mock_open(read_data=file_content) ++ with patch('salt.utils.files.fopen', files_fopen): ++ atomic_opener = mock_open() ++ with patch('salt.utils.atomicfile.atomic_open', atomic_opener): ++ self.assertFalse(filemod.line('foo', content='foo', match=match, mode=mode)) ++ + @patch('os.path.realpath', MagicMock()) + @patch('os.path.isfile', MagicMock(return_value=True)) + def test_line_modecheck_failure(self): +@@ -1082,7 +1119,7 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): + ' - /srv/sugar' + ]) + cfg_content = '- /srv/custom' +- for before_line in ['/srv/salt', '/srv/sa.*t', '/sr.*']: ++ for before_line in ['/srv/salt', '/srv/sa.*t']: + files_fopen = mock_open(read_data=file_content) + with patch('salt.utils.files.fopen', files_fopen): + atomic_opener = mock_open() +@@ -1092,6 +1129,32 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): + self.assertEqual(atomic_opener().write.call_args_list[0][0][0], + file_modified) + ++ @patch('os.path.realpath', MagicMock()) ++ @patch('os.path.isfile', MagicMock(return_value=True)) ++ @patch('os.stat', MagicMock()) ++ def test_line_assert_exception_pattern(self): ++ ''' ++ Test for file.line for exception on insert with too general pattern. ++ ++ :return: ++ ''' ++ file_content = os.linesep.join([ ++ 'file_roots:', ++ ' base:', ++ ' - /srv/salt', ++ ' - /srv/sugar' ++ ]) ++ cfg_content = '- /srv/custom' ++ for before_line in ['/sr.*']: ++ files_fopen = mock_open(read_data=file_content) ++ with patch('salt.utils.files.fopen', files_fopen): ++ atomic_opener = mock_open() ++ with patch('salt.utils.atomicfile.atomic_open', atomic_opener): ++ with self.assertRaises(CommandExecutionError) as cm: ++ filemod.line('foo', content=cfg_content, before=before_line, mode='insert') ++ self.assertEqual(cm.exception.strerror, ++ 'Found more than expected occurrences in "before" expression') ++ + @patch('os.path.realpath', MagicMock()) + @patch('os.path.isfile', MagicMock(return_value=True)) + @patch('os.stat', MagicMock()) +@@ -1179,7 +1242,7 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): + ' base:', + ' - /srv/salt', + ' - /srv/sugar', +- cfg_content ++ ' ' + cfg_content + ]) + files_fopen = mock_open(read_data=file_content) + with patch('salt.utils.files.fopen', files_fopen): +@@ -1273,6 +1336,33 @@ class FilemodLineTests(TestCase, LoaderModuleMockMixin): + self.assertEqual(atomic_opener().write.call_args_list[0][0][0], + file_modified) + ++ @patch('os.path.realpath', MagicMock()) ++ @patch('os.path.isfile', MagicMock(return_value=True)) ++ @patch('os.stat', MagicMock()) ++ def test_line_insert_ensure_before_first_line(self): ++ ''' ++ Test for file.line for insertion ensuring the line is before first line ++ :return: ++ ''' ++ cfg_content = '#!/bin/bash' ++ file_content = os.linesep.join([ ++ '/etc/init.d/someservice restart', ++ 'exit 0' ++ ]) ++ file_modified = os.linesep.join([ ++ cfg_content, ++ '/etc/init.d/someservice restart', ++ 'exit 0' ++ ]) ++ files_fopen = mock_open(read_data=file_content) ++ with patch('salt.utils.files.fopen', files_fopen): ++ atomic_opener = mock_open() ++ with patch('salt.utils.atomicfile.atomic_open', atomic_opener): ++ filemod.line('foo', content=cfg_content, before='/etc/init.d/someservice restart', mode='ensure') ++ self.assertEqual(len(atomic_opener().write.call_args_list), 1) ++ self.assertEqual(atomic_opener().write.call_args_list[0][0][0], ++ file_modified) ++ + @patch('os.path.realpath', MagicMock()) + @patch('os.path.isfile', MagicMock(return_value=True)) + @patch('os.stat', MagicMock()) +diff --git a/tests/unit/utils/test_stringutils.py b/tests/unit/utils/test_stringutils.py +index 9c8fd4f7c3..852f558793 100644 +--- a/tests/unit/utils/test_stringutils.py ++++ b/tests/unit/utils/test_stringutils.py +@@ -18,6 +18,9 @@ STR = BYTES = UNICODE.encode('utf-8') + # code points. Do not modify it. + EGGS = '\u044f\u0438\u0306\u0446\u0430' + ++LATIN1_UNICODE = 'räksmörgås' ++LATIN1_BYTES = LATIN1_UNICODE.encode('latin-1') ++ + + class StringutilsTestCase(TestCase): + def test_contains_whitespace(self): +@@ -134,6 +137,13 @@ class StringutilsTestCase(TestCase): + 'яйца' + ) + ++ self.assertEqual( ++ salt.utils.stringutils.to_unicode( ++ LATIN1_BYTES, encoding='latin-1' ++ ), ++ LATIN1_UNICODE ++ ) ++ + if six.PY3: + self.assertEqual(salt.utils.stringutils.to_unicode('plugh'), 'plugh') + self.assertEqual(salt.utils.stringutils.to_unicode('áéíóúý'), 'áéíóúý') +@@ -150,6 +160,10 @@ class StringutilsTestCase(TestCase): + with patch.object(builtins, '__salt_system_encoding__', 'CP1252'): + self.assertEqual(salt.utils.stringutils.to_unicode('Ψ'.encode('utf-8')), 'Ψ') + ++ def test_to_unicode_multi_encoding(self): ++ result = salt.utils.stringutils.to_unicode(LATIN1_BYTES, encoding=('utf-8', 'latin1')) ++ assert result == LATIN1_UNICODE ++ + def test_build_whitespace_split_regex(self): + expected_regex = '(?m)^(?:[\\s]+)?Lorem(?:[\\s]+)?ipsum(?:[\\s]+)?dolor(?:[\\s]+)?sit(?:[\\s]+)?amet\\,' \ + '(?:[\\s]+)?$' +-- +2.17.1 + + diff --git a/fix-unit-test-for-grains-core.patch b/fix-unit-test-for-grains-core.patch new file mode 100644 index 0000000..3d3004a --- /dev/null +++ b/fix-unit-test-for-grains-core.patch @@ -0,0 +1,42 @@ +From 7ffa39cd80393f2a3ed5cd75793b134b9d939cf9 Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Thu, 11 Oct 2018 16:20:40 +0200 +Subject: [PATCH] Fix unit test for grains core + +--- + tests/unit/grains/test_core.py | 11 +++++------ + 1 file changed, 5 insertions(+), 6 deletions(-) + +diff --git a/tests/unit/grains/test_core.py b/tests/unit/grains/test_core.py +index 2ab32ef41b..4923ee00b0 100644 +--- a/tests/unit/grains/test_core.py ++++ b/tests/unit/grains/test_core.py +@@ -62,11 +62,10 @@ class CoreGrainsTestCase(TestCase, LoaderModuleMockMixin): + def test_parse_etc_os_release(self, path_isfile_mock): + path_isfile_mock.side_effect = lambda x: x == "/usr/lib/os-release" + with salt.utils.files.fopen(os.path.join(OS_RELEASE_DIR, "ubuntu-17.10")) as os_release_file: +- os_release_content = os_release_file.read() +- with patch("salt.utils.files.fopen", mock_open(read_data=os_release_content)): +- os_release = core._parse_os_release( +- '/etc/os-release', +- '/usr/lib/os-release') ++ os_release_content = os_release_file.readlines() ++ with patch("salt.utils.files.fopen", mock_open()) as os_release_file: ++ os_release_file.return_value.__iter__.return_value = os_release_content ++ os_release = core._parse_os_release(["/etc/os-release", "/usr/lib/os-release"]) + self.assertEqual(os_release, { + "NAME": "Ubuntu", + "VERSION": "17.10 (Artful Aardvark)", +@@ -128,7 +127,7 @@ class CoreGrainsTestCase(TestCase, LoaderModuleMockMixin): + + def test_missing_os_release(self): + with patch('salt.utils.files.fopen', mock_open(read_data={})): +- os_release = core._parse_os_release('/etc/os-release', '/usr/lib/os-release') ++ os_release = core._parse_os_release(['/etc/os-release', '/usr/lib/os-release']) + self.assertEqual(os_release, {}) + + @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') +-- +2.19.0 + + diff --git a/fixes-cve-2018-15750-cve-2018-15751.patch b/fixes-cve-2018-15750-cve-2018-15751.patch new file mode 100644 index 0000000..f9ef657 --- /dev/null +++ b/fixes-cve-2018-15750-cve-2018-15751.patch @@ -0,0 +1,143 @@ +From 43b1f8fb6608c944812bc5bcd9da407624409ac7 Mon Sep 17 00:00:00 2001 +From: Erik Johnson +Date: Fri, 24 Aug 2018 10:35:55 -0500 +Subject: [PATCH] Fixes: CVE-2018-15750, CVE-2018-15751 + +Ensure that tokens are hex to avoid hanging/errors in cherrypy + +Add empty token salt-api integration tests + +Handle Auth exceptions in run_job + +Update tornado test to correct authentication message +--- + salt/client/__init__.py | 8 ++++ + salt/netapi/rest_cherrypy/app.py | 13 ++++++- + .../netapi/rest_cherrypy/test_app.py | 39 +++++++++++++++++++ + .../netapi/rest_tornado/test_app.py | 2 +- + 4 files changed, 60 insertions(+), 2 deletions(-) + +diff --git a/salt/client/__init__.py b/salt/client/__init__.py +index dcbc1473e1..77f2a963f7 100644 +--- a/salt/client/__init__.py ++++ b/salt/client/__init__.py +@@ -349,6 +349,10 @@ class LocalClient(object): + raise SaltClientError( + 'The salt master could not be contacted. Is master running?' + ) ++ except AuthenticationError as err: ++ raise AuthenticationError(err) ++ except AuthorizationError as err: ++ raise AuthorizationError(err) + except Exception as general_exception: + # Convert to generic client error and pass along message + raise SaltClientError(general_exception) +@@ -415,6 +419,10 @@ class LocalClient(object): + raise SaltClientError( + 'The salt master could not be contacted. Is master running?' + ) ++ except AuthenticationError as err: ++ raise AuthenticationError(err) ++ except AuthorizationError as err: ++ raise AuthorizationError(err) + except Exception as general_exception: + # Convert to generic client error and pass along message + raise SaltClientError(general_exception) +diff --git a/salt/netapi/rest_cherrypy/app.py b/salt/netapi/rest_cherrypy/app.py +index 78ea3c3fef..c272674146 100644 +--- a/salt/netapi/rest_cherrypy/app.py ++++ b/salt/netapi/rest_cherrypy/app.py +@@ -1167,6 +1167,13 @@ class LowDataAdapter(object): + if token: + chunk['token'] = token + ++ if 'token' in chunk: ++ # Make sure that auth token is hex ++ try: ++ int(chunk['token'], 16) ++ except (TypeError, ValueError): ++ raise cherrypy.HTTPError(401, 'Invalid token') ++ + if client: + chunk['client'] = client + +@@ -2167,7 +2174,11 @@ class Events(object): + + :return bool: True if valid, False if not valid. + ''' +- if auth_token is None: ++ # Make sure that auth token is hex. If it's None, or something other ++ # than hex, this will raise a ValueError. ++ try: ++ int(auth_token, 16) ++ except ValueError: + return False + + # First check if the given token is in our session table; if so it's a +diff --git a/tests/integration/netapi/rest_cherrypy/test_app.py b/tests/integration/netapi/rest_cherrypy/test_app.py +index 000b7418bf..5865510fd7 100644 +--- a/tests/integration/netapi/rest_cherrypy/test_app.py ++++ b/tests/integration/netapi/rest_cherrypy/test_app.py +@@ -124,6 +124,45 @@ class TestRun(cptc.BaseRestCherryPyTest): + }) + self.assertEqual(response.status, '401 Unauthorized') + ++ def test_run_empty_token(self): ++ ''' ++ Test the run URL with empty token ++ ''' ++ cmd = dict(self.low, **{'token': ''}) ++ body = urlencode(cmd) ++ ++ request, response = self.request('/run', method='POST', body=body, ++ headers={ ++ 'content-type': 'application/x-www-form-urlencoded' ++ }) ++ assert response.status == '401 Unauthorized' ++ ++ def test_run_empty_token_upercase(self): ++ ''' ++ Test the run URL with empty token with upercase characters ++ ''' ++ cmd = dict(self.low, **{'ToKen': ''}) ++ body = urlencode(cmd) ++ ++ request, response = self.request('/run', method='POST', body=body, ++ headers={ ++ 'content-type': 'application/x-www-form-urlencoded' ++ }) ++ assert response.status == '401 Unauthorized' ++ ++ def test_run_wrong_token(self): ++ ''' ++ Test the run URL with incorrect token ++ ''' ++ cmd = dict(self.low, **{'token': 'bad'}) ++ body = urlencode(cmd) ++ ++ request, response = self.request('/run', method='POST', body=body, ++ headers={ ++ 'content-type': 'application/x-www-form-urlencoded' ++ }) ++ assert response.status == '401 Unauthorized' ++ + + class TestWebhookDisableAuth(cptc.BaseRestCherryPyTest): + +diff --git a/tests/integration/netapi/rest_tornado/test_app.py b/tests/integration/netapi/rest_tornado/test_app.py +index beb085db1e..01abd354a7 100644 +--- a/tests/integration/netapi/rest_tornado/test_app.py ++++ b/tests/integration/netapi/rest_tornado/test_app.py +@@ -237,7 +237,7 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase): + self.assertEqual(len(ret), 3) # make sure we got 3 responses + self.assertIn('jid', ret[0]) # the first 2 are regular returns + self.assertIn('jid', ret[1]) +- self.assertIn('Authentication error occurred.', ret[2]) # bad auth ++ self.assertIn('Failed to authenticate', ret[2]) # bad auth + self.assertEqual(ret[0]['minions'], sorted(['minion', 'sub_minion', 'localhost'])) + self.assertEqual(ret[1]['minions'], sorted(['minion', 'sub_minion', 'localhost'])) + +-- +2.17.1 + + diff --git a/fixing-issue-when-a-valid-token-is-generated-even-wh.patch b/fixing-issue-when-a-valid-token-is-generated-even-wh.patch new file mode 100644 index 0000000..7f2d8b4 --- /dev/null +++ b/fixing-issue-when-a-valid-token-is-generated-even-wh.patch @@ -0,0 +1,37 @@ +From 6c85da9a53e9dd022c96a199be4e3bdd280543d6 Mon Sep 17 00:00:00 2001 +From: "Gareth J. Greenaway" +Date: Thu, 2 Aug 2018 15:35:24 -0700 +Subject: [PATCH] Fixing issue when a valid token is generated even when + invalid user credentials are passed. This change verifies that the binddn + credentials are valid, then verifies that the username & password (if not + None) are also valid. + +--- + salt/auth/ldap.py | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +diff --git a/salt/auth/ldap.py b/salt/auth/ldap.py +index cbfb03a2f2..0b9aa69fe4 100644 +--- a/salt/auth/ldap.py ++++ b/salt/auth/ldap.py +@@ -283,9 +283,15 @@ def auth(username, password): + log.error('LDAP authentication requires python-ldap module') + return False + +- # If bind credentials are configured, use them instead of user's ++ # If bind credentials are configured, verify that we can a valid bind + if _config('binddn', mandatory=False) and _config('bindpw', mandatory=False): + bind = _bind_for_search(anonymous=_config('anonymous', mandatory=False)) ++ ++ # If username & password are not None, attempt to verify they are valid ++ if bind and username and password: ++ bind = _bind(username, password, ++ anonymous=_config('auth_by_group_membership_only', mandatory=False) ++ and _config('anonymous', mandatory=False)) + else: + bind = _bind(username, password, + anonymous=_config('auth_by_group_membership_only', mandatory=False) +-- +2.19.0 + + diff --git a/get-os_arch-also-without-rpm-package-installed.patch b/get-os_arch-also-without-rpm-package-installed.patch new file mode 100644 index 0000000..22b95b5 --- /dev/null +++ b/get-os_arch-also-without-rpm-package-installed.patch @@ -0,0 +1,143 @@ +From 2e0abe6d12aa2657a4febed3a80b8c4cf104487a Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Wed, 14 Nov 2018 17:36:23 +0100 +Subject: [PATCH] Get os_arch also without RPM package installed + +backport pkg.rpm test + +Add pkg.rpm unit test case + +Fix docstring + +Add UT for getting OS architecture fallback, when no RPM found (initrd, e.g.) + +Add UT for OS architecture detection on fallback, when no CPU arch can be determined + +Add UT for OS arch detection when no CPU arch or machine can be determined + +Remove unsupported testcase +--- + salt/utils/pkg/rpm.py | 18 ++++++--- + tests/unit/utils/test_pkg.py | 72 ++++++++++++++++++++++++++++++++++++ + 2 files changed, 84 insertions(+), 6 deletions(-) + create mode 100644 tests/unit/utils/test_pkg.py + +diff --git a/salt/utils/pkg/rpm.py b/salt/utils/pkg/rpm.py +index 94e231da4b..bb8c3fb589 100644 +--- a/salt/utils/pkg/rpm.py ++++ b/salt/utils/pkg/rpm.py +@@ -9,7 +9,9 @@ import collections + import datetime + import logging + import subprocess ++import platform + import salt.utils.stringutils ++import salt.utils.path + + # Import 3rd-party libs + from salt.ext import six +@@ -42,12 +44,16 @@ def get_osarch(): + ''' + Get the os architecture using rpm --eval + ''' +- ret = subprocess.Popen( +- 'rpm --eval "%{_host_cpu}"', +- shell=True, +- close_fds=True, +- stdout=subprocess.PIPE, +- stderr=subprocess.PIPE).communicate()[0] ++ if salt.utils.path.which('rpm'): ++ ret = subprocess.Popen( ++ 'rpm --eval "%{_host_cpu}"', ++ shell=True, ++ close_fds=True, ++ stdout=subprocess.PIPE, ++ stderr=subprocess.PIPE).communicate()[0] ++ else: ++ ret = ''.join(list(filter(None, platform.uname()[-2:]))[-1:]) ++ + return salt.utils.stringutils.to_str(ret).strip() or 'unknown' + + +diff --git a/tests/unit/utils/test_pkg.py b/tests/unit/utils/test_pkg.py +new file mode 100644 +index 0000000000..361e0bf92f +--- /dev/null ++++ b/tests/unit/utils/test_pkg.py +@@ -0,0 +1,72 @@ ++# -*- coding: utf-8 -*- ++ ++from __future__ import absolute_import, unicode_literals, print_function ++ ++from tests.support.unit import TestCase, skipIf ++from tests.support.mock import Mock, MagicMock, patch, NO_MOCK, NO_MOCK_REASON ++import salt.utils.pkg ++from salt.utils.pkg import rpm ++ ++try: ++ import pytest ++except ImportError: ++ pytest = None ++ ++ ++@skipIf(NO_MOCK, NO_MOCK_REASON) ++@skipIf(pytest is None, 'PyTest is missing') ++class PkgRPMTestCase(TestCase): ++ ''' ++ Test case for pkg.rpm utils ++ ''' ++ ++ @patch('salt.utils.path.which', MagicMock(return_value=True)) ++ def test_get_osarch_by_rpm(self): ++ ''' ++ Get os_arch if RPM package is installed. ++ :return: ++ ''' ++ subprocess_mock = MagicMock() ++ subprocess_mock.Popen = MagicMock() ++ subprocess_mock.Popen().communicate = MagicMock(return_value=['Z80']) ++ with patch('salt.utils.pkg.rpm.subprocess', subprocess_mock): ++ assert rpm.get_osarch() == 'Z80' ++ assert subprocess_mock.Popen.call_count == 2 # One within the mock ++ assert subprocess_mock.Popen.call_args[1]['close_fds'] ++ assert subprocess_mock.Popen.call_args[1]['shell'] ++ assert len(subprocess_mock.Popen.call_args_list) == 2 ++ assert subprocess_mock.Popen.call_args[0][0] == 'rpm --eval "%{_host_cpu}"' ++ ++ @patch('salt.utils.path.which', MagicMock(return_value=False)) ++ @patch('salt.utils.pkg.rpm.subprocess', MagicMock(return_value=False)) ++ @patch('salt.utils.pkg.rpm.platform.uname', MagicMock( ++ return_value=('Sinclair BASIC', 'motophone', '1982 Sinclair Research Ltd', '1.0', 'ZX81', 'Z80'))) ++ def test_get_osarch_by_platform(self): ++ ''' ++ Get os_arch if RPM package is not installed (inird image, for example). ++ :return: ++ ''' ++ assert rpm.get_osarch() == 'Z80' ++ ++ @patch('salt.utils.path.which', MagicMock(return_value=False)) ++ @patch('salt.utils.pkg.rpm.subprocess', MagicMock(return_value=False)) ++ @patch('salt.utils.pkg.rpm.platform.uname', MagicMock( ++ return_value=('Sinclair BASIC', 'motophone', '1982 Sinclair Research Ltd', '1.0', 'ZX81', ''))) ++ def test_get_osarch_by_platform_no_cpu_arch(self): ++ ''' ++ Get os_arch if RPM package is not installed (inird image, for example) but cpu arch cannot be determined. ++ :return: ++ ''' ++ assert rpm.get_osarch() == 'ZX81' ++ ++ @patch('salt.utils.path.which', MagicMock(return_value=False)) ++ @patch('salt.utils.pkg.rpm.subprocess', MagicMock(return_value=False)) ++ @patch('salt.utils.pkg.rpm.platform.uname', MagicMock( ++ return_value=('Sinclair BASIC', 'motophone', '1982 Sinclair Research Ltd', '1.0', '', ''))) ++ def test_get_osarch_by_platform_no_cpu_arch_no_machine(self): ++ ''' ++ Get os_arch if RPM package is not installed (inird image, for example) ++ where both cpu arch and machine cannot be determined. ++ :return: ++ ''' ++ assert rpm.get_osarch() == 'unknown' +-- +2.19.1 + + diff --git a/get-os_family-for-rpm-distros-from-the-rpm-macros.-u.patch b/get-os_family-for-rpm-distros-from-the-rpm-macros.-u.patch new file mode 100644 index 0000000..5a3d552 --- /dev/null +++ b/get-os_family-for-rpm-distros-from-the-rpm-macros.-u.patch @@ -0,0 +1,65 @@ +From 943a258da3ed460f173968b0a92b95f2e63ab669 Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Mon, 8 Oct 2018 12:48:24 +0200 +Subject: [PATCH] Get os_family for RPM distros from the RPM macros. + (U#49930) + +Strip and stringify the return for the osarch + +Fix imports +--- + salt/grains/core.py | 8 +++++--- + salt/utils/pkg/rpm.py | 3 ++- + 2 files changed, 7 insertions(+), 4 deletions(-) + +diff --git a/salt/grains/core.py b/salt/grains/core.py +index 6aaf38096d..80eebd1c05 100644 +--- a/salt/grains/core.py ++++ b/salt/grains/core.py +@@ -49,6 +49,8 @@ import salt.utils.path + import salt.utils.platform + import salt.utils.stringutils + import salt.utils.versions ++import salt.utils.pkg.rpm ++ + from salt.ext import six + from salt.ext.six.moves import range + +@@ -1776,9 +1778,9 @@ def os_data(): + # architecture. + if grains.get('os_family') == 'Debian': + osarch = __salt__['cmd.run']('dpkg --print-architecture').strip() +- elif grains.get('os_family') == 'RedHat': +- osarch = __salt__['cmd.run']('rpm --eval %{_host_cpu}').strip() +- elif grains.get('os_family') == 'NILinuxRT': ++ elif grains.get('os_family') in ['RedHat', 'Suse']: ++ osarch = salt.utils.pkg.rpm.get_osarch() ++ elif grains.get('os_family') in ('NILinuxRT', 'Poky'): + archinfo = {} + for line in __salt__['cmd.run']('opkg print-architecture').splitlines(): + if line.startswith('arch'): +diff --git a/salt/utils/pkg/rpm.py b/salt/utils/pkg/rpm.py +index 987edab894..94e231da4b 100644 +--- a/salt/utils/pkg/rpm.py ++++ b/salt/utils/pkg/rpm.py +@@ -9,6 +9,7 @@ import collections + import datetime + import logging + import subprocess ++import salt.utils.stringutils + + # Import 3rd-party libs + from salt.ext import six +@@ -47,7 +48,7 @@ def get_osarch(): + close_fds=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate()[0] +- return ret or 'unknown' ++ return salt.utils.stringutils.to_str(ret).strip() or 'unknown' + + + def check_32(arch, osarch=None): +-- +2.19.0 + + diff --git a/improved-handling-of-ldap-group-id.patch b/improved-handling-of-ldap-group-id.patch new file mode 100644 index 0000000..4b8b948 --- /dev/null +++ b/improved-handling-of-ldap-group-id.patch @@ -0,0 +1,38 @@ +From d0234ed977ca860b3a3a6a587a6972bbaf5ae345 Mon Sep 17 00:00:00 2001 +From: Raine Curtis +Date: Mon, 9 Jul 2018 09:55:30 -0600 +Subject: [PATCH] Improved handling of LDAP group id + +gid is casted to int, which should be the case. Otherwise an error +is returned. +--- + salt/states/group.py | 11 +++++++++-- + 1 file changed, 9 insertions(+), 2 deletions(-) + +diff --git a/salt/states/group.py b/salt/states/group.py +index 6a720757e8..acf775134c 100644 +--- a/salt/states/group.py ++++ b/salt/states/group.py +@@ -72,9 +72,16 @@ def _changes(name, + delusers = [salt.utils.win_functions.get_sam_name(user).lower() for user in delusers] + + change = {} ++ ret = {} + if gid: +- if lgrp['gid'] != gid: +- change['gid'] = gid ++ try: ++ gid = int(gid) ++ if lgrp['gid'] != gid: ++ change['gid'] = gid ++ except (TypeError, ValueError): ++ ret['result'] = False ++ ret['comment'] = 'Invalid gid' ++ return ret + + if members: + # -- if new member list if different than the current +-- +2.19.1 + + diff --git a/loosen-azure-sdk-dependencies-in-azurearm-cloud-driv.patch b/loosen-azure-sdk-dependencies-in-azurearm-cloud-driv.patch new file mode 100644 index 0000000..339cf3f --- /dev/null +++ b/loosen-azure-sdk-dependencies-in-azurearm-cloud-driv.patch @@ -0,0 +1,63 @@ +From 9d9fb3fd787b40d9d27ad7c5eb69fa0cd4f5a304 Mon Sep 17 00:00:00 2001 +From: Joachim Gleissner +Date: Tue, 18 Sep 2018 15:07:13 +0200 +Subject: [PATCH] loosen azure sdk dependencies in azurearm cloud driver + +Remove dependency to azure-cli, which is not used at all. +Use azure-storage-sdk as fallback if multiapi version is not available. + +remove unused import from azurearm driver +--- + salt/cloud/clouds/azurearm.py | 14 ++++++++------ + 1 file changed, 8 insertions(+), 6 deletions(-) + +diff --git a/salt/cloud/clouds/azurearm.py b/salt/cloud/clouds/azurearm.py +index 8b9a9e8903..50e5ce1f62 100644 +--- a/salt/cloud/clouds/azurearm.py ++++ b/salt/cloud/clouds/azurearm.py +@@ -67,6 +67,7 @@ import logging + import pprint + import base64 + import collections ++import pkgutil + import salt.cache + import salt.config as config + import salt.utils.cloud +@@ -74,7 +75,6 @@ import salt.utils.data + import salt.utils.files + import salt.utils.stringutils + import salt.utils.yaml +-from salt.utils.versions import LooseVersion + from salt.ext import six + import salt.version + from salt.exceptions import ( +@@ -125,9 +125,12 @@ try: + from azure.mgmt.storage import StorageManagementClient + from azure.mgmt.web import WebSiteManagementClient + from msrestazure.azure_exceptions import CloudError +- from azure.multiapi.storage.v2016_05_31 import CloudStorageAccount +- from azure.cli import core +- HAS_LIBS = LooseVersion(core.__version__) >= LooseVersion("2.0.12") ++ if pkgutil.find_loader('azure.multiapi'): ++ # use multiapi version if available ++ from azure.multiapi.storage.v2016_05_31 import CloudStorageAccount ++ else: ++ from azure.storage import CloudStorageAccount ++ HAS_LIBS = True + except ImportError: + pass + # pylint: enable=wrong-import-position,wrong-import-order +@@ -160,8 +163,7 @@ def __virtual__(): + False, + 'The following dependencies are required to use the AzureARM driver: ' + 'Microsoft Azure SDK for Python >= 2.0rc5, ' +- 'Microsoft Azure Storage SDK for Python >= 0.32, ' +- 'Microsoft Azure CLI >= 2.0.12' ++ 'Microsoft Azure Storage SDK for Python >= 0.32' + ) + + global cache # pylint: disable=global-statement,invalid-name +-- +2.17.1 + + diff --git a/make-aptpkg.list_repos-compatible-on-enabled-disable.patch b/make-aptpkg.list_repos-compatible-on-enabled-disable.patch new file mode 100644 index 0000000..a899087 --- /dev/null +++ b/make-aptpkg.list_repos-compatible-on-enabled-disable.patch @@ -0,0 +1,26 @@ +From 350b0aa4ead80ac50047c08121bc09bddc05341d Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Fri, 16 Nov 2018 10:54:12 +0100 +Subject: [PATCH] Make aptpkg.list_repos compatible on enabled/disabled + output + +--- + salt/modules/aptpkg.py | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py +index 175ef2ed06..90b99c44b9 100644 +--- a/salt/modules/aptpkg.py ++++ b/salt/modules/aptpkg.py +@@ -1719,6 +1719,7 @@ def list_repos(): + repo['file'] = source.file + repo['comps'] = getattr(source, 'comps', []) + repo['disabled'] = source.disabled ++ repo['enabled'] = not repo['disabled'] # This is for compatibility with the other modules + repo['dist'] = source.dist + repo['type'] = source.type + repo['uri'] = source.uri.rstrip('/') +-- +2.19.1 + + diff --git a/make-profiles-a-package.patch b/make-profiles-a-package.patch new file mode 100644 index 0000000..421ccc1 --- /dev/null +++ b/make-profiles-a-package.patch @@ -0,0 +1,27 @@ +From 155aa52dca9272db492990ad737256dada1c4364 Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Mon, 8 Oct 2018 17:52:07 +0200 +Subject: [PATCH] Make profiles a package. + +Add UTF-8 encoding + +Add a docstring +--- + salt/cli/support/profiles/__init__.py | 4 ++++ + 1 file changed, 4 insertions(+) + create mode 100644 salt/cli/support/profiles/__init__.py + +diff --git a/salt/cli/support/profiles/__init__.py b/salt/cli/support/profiles/__init__.py +new file mode 100644 +index 0000000000..b86aef30b8 +--- /dev/null ++++ b/salt/cli/support/profiles/__init__.py +@@ -0,0 +1,4 @@ ++# coding=utf-8 ++''' ++Profiles for salt-support. ++''' +-- +2.19.0 + + diff --git a/preserving-signature-in-module.run-state-u-50049.patch b/preserving-signature-in-module.run-state-u-50049.patch new file mode 100644 index 0000000..3d00d00 --- /dev/null +++ b/preserving-signature-in-module.run-state-u-50049.patch @@ -0,0 +1,89 @@ +From 318b4e0cd2efb02f26392bfe2d354a3ff5d21cbc Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Mon, 15 Oct 2018 17:26:16 +0200 +Subject: [PATCH] Preserving signature in "module.run" state (U#50049) + +Add unit test for _call_function on signature aligning named arguments + +Add unit test for _call_function routine for unnamed positional arguments + +Remove redundant docstrings + +Add different test function signature with the same outcome + +Replace standalone function with lambda-proxy for signatures only +--- + salt/states/module.py | 7 +++++-- + tests/unit/states/test_module.py | 27 +++++++++++++++++++++++++++ + 2 files changed, 32 insertions(+), 2 deletions(-) + +diff --git a/salt/states/module.py b/salt/states/module.py +index 2190ffa3d2..90b1d0a5f5 100644 +--- a/salt/states/module.py ++++ b/salt/states/module.py +@@ -323,7 +323,7 @@ def _call_function(name, returner=None, **kwargs): + + # func_args is initialized to a list of positional arguments that the function to be run accepts + func_args = argspec.args[:len(argspec.args or []) - len(argspec.defaults or [])] +- arg_type, na_type, kw_type = [], {}, False ++ arg_type, kw_to_arg_type, na_type, kw_type = [], {}, {}, False + for funcset in reversed(kwargs.get('func_args') or []): + if not isinstance(funcset, dict): + # We are just receiving a list of args to the function to be run, so just append +@@ -334,13 +334,16 @@ def _call_function(name, returner=None, **kwargs): + # We are going to pass in a keyword argument. The trick here is to make certain + # that if we find that in the *args* list that we pass it there and not as a kwarg + if kwarg_key in func_args: +- arg_type.append(funcset[kwarg_key]) ++ kw_to_arg_type[kwarg_key] = funcset[kwarg_key] + continue + else: + # Otherwise, we're good and just go ahead and pass the keyword/value pair into + # the kwargs list to be run. + func_kw.update(funcset) + arg_type.reverse() ++ for arg in func_args: ++ if arg in kw_to_arg_type: ++ arg_type.append(kw_to_arg_type[arg]) + _exp_prm = len(argspec.args or []) - len(argspec.defaults or []) + _passed_prm = len(arg_type) + missing = [] +diff --git a/tests/unit/states/test_module.py b/tests/unit/states/test_module.py +index bf4ddcc5b4..25082d4bb4 100644 +--- a/tests/unit/states/test_module.py ++++ b/tests/unit/states/test_module.py +@@ -324,3 +324,30 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): + self.assertIn(comment, ret['comment']) + self.assertIn('world', ret['comment']) + self.assertIn('hello', ret['comment']) ++ ++ def test_call_function_named_args(self): ++ ''' ++ Test _call_function routine when params are named. Their position ordering should not matter. ++ ++ :return: ++ ''' ++ with patch.dict(module.__salt__, ++ {'testfunc': lambda a, b, c, *args, **kwargs: (a, b, c, args, kwargs)}, clear=True): ++ assert module._call_function('testfunc', func_args=[{'a': 1}, {'b': 2}, {'c': 3}]) == (1, 2, 3, (), {}) ++ assert module._call_function('testfunc', func_args=[{'c': 3}, {'a': 1}, {'b': 2}]) == (1, 2, 3, (), {}) ++ ++ with patch.dict(module.__salt__, ++ {'testfunc': lambda c, a, b, *args, **kwargs: (a, b, c, args, kwargs)}, clear=True): ++ assert module._call_function('testfunc', func_args=[{'a': 1}, {'b': 2}, {'c': 3}]) == (1, 2, 3, (), {}) ++ assert module._call_function('testfunc', func_args=[{'c': 3}, {'a': 1}, {'b': 2}]) == (1, 2, 3, (), {}) ++ ++ def test_call_function_ordered_args(self): ++ ''' ++ Test _call_function routine when params are not named. Their position should matter. ++ ++ :return: ++ ''' ++ with patch.dict(module.__salt__, ++ {'testfunc': lambda a, b, c, *args, **kwargs: (a, b, c, args, kwargs)}, clear=True): ++ assert module._call_function('testfunc', func_args=[1, 2, 3]) == (1, 2, 3, (), {}) ++ assert module._call_function('testfunc', func_args=[3, 1, 2]) == (3, 1, 2, (), {}) +-- +2.19.0 + + diff --git a/remove-arch-from-name-when-pkg.list_pkgs-is-called-w.patch b/remove-arch-from-name-when-pkg.list_pkgs-is-called-w.patch new file mode 100644 index 0000000..83a65f6 --- /dev/null +++ b/remove-arch-from-name-when-pkg.list_pkgs-is-called-w.patch @@ -0,0 +1,744 @@ +From 6488d91acb6f470bfa2b66ac8100cb67d6367612 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Mon, 19 Nov 2018 11:46:26 +0000 +Subject: [PATCH] Remove arch from name when pkg.list_pkgs is called with + 'attr' (bsc#1114029) + +Add unit tests for pkg_resource.format_pkg_list + +Fix pylint issues + +Refactor: Return requested attr even if empty + +Add corner cases on package names to unit tests + +Fix Zypper/Yum unit test after returning empty requested attrs + +Add Yum/Zypper list_pkgs unit tests for multiple versions reported + +Compare testing items properly to avoid unwanted failures + +Use assertCountEqual when running on Python3 + +Add missing import for the six module + +Strip architecture from package name in aptpkg module + +Use parse_arch_from_name if available on the virtual pkg module + +Adapt unit tests after introducing parse_arch_from_name + +Use PKG_ARCH_SEPARATOR in pkg.normalize_name method + +Add pkg_resource to setup loader modules. Fix pylint + +Remove unnecessary lambda + +Return None instead empty string for arch and release in pkg.list_pkgs +--- + salt/modules/aptpkg.py | 38 ++++++++ + salt/modules/pkg_resource.py | 17 +++- + salt/modules/yumpkg.py | 32 ++++++- + salt/modules/zypper.py | 29 +++++- + tests/unit/modules/test_pkg_resource.py | 116 ++++++++++++++++++++++++ + tests/unit/modules/test_yumpkg.py | 85 ++++++++++++++++- + tests/unit/modules/test_zypper.py | 81 ++++++++++++++++- + 7 files changed, 382 insertions(+), 16 deletions(-) + +diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py +index 42d606926f..1fd4883f2c 100644 +--- a/salt/modules/aptpkg.py ++++ b/salt/modules/aptpkg.py +@@ -77,6 +77,7 @@ except ImportError: + # pylint: enable=import-error + + APT_LISTS_PATH = "/var/lib/apt/lists" ++PKG_ARCH_SEPARATOR = ':' + + # Source format for urllib fallback on PPA handling + LP_SRC_FORMAT = 'deb http://ppa.launchpad.net/{0}/{1}/ubuntu {2} main' +@@ -218,6 +219,43 @@ def _warn_software_properties(repo): + log.warning('Best guess at ppa format: %s', repo) + + ++def normalize_name(name): ++ ''' ++ Strips the architecture from the specified package name, if necessary. ++ ++ CLI Example: ++ ++ .. code-block:: bash ++ ++ salt '*' pkg.normalize_name zsh:amd64 ++ ''' ++ try: ++ name, arch = name.rsplit(PKG_ARCH_SEPARATOR, 1) ++ except ValueError: ++ return name ++ return name ++ ++ ++def parse_arch_from_name(name): ++ ''' ++ Parse name and architecture from the specified package name. ++ ++ CLI Example: ++ ++ .. code-block:: bash ++ ++ salt '*' pkg.parse_arch_from_name zsh:amd64 ++ ''' ++ try: ++ _name, _arch = name.rsplit(PKG_ARCH_SEPARATOR, 1) ++ except ValueError: ++ _name, _arch = name, None ++ return { ++ 'name': _name, ++ 'arch': _arch ++ } ++ ++ + def latest_version(*names, **kwargs): + ''' + Return the latest version of the named package available for upgrade or +diff --git a/salt/modules/pkg_resource.py b/salt/modules/pkg_resource.py +index 9b0a8287f5..0c872f1805 100644 +--- a/salt/modules/pkg_resource.py ++++ b/salt/modules/pkg_resource.py +@@ -311,22 +311,31 @@ def format_pkg_list(packages, versions_as_list, attr): + ''' + ret = copy.deepcopy(packages) + if attr: ++ ret_attr = {} + requested_attr = set(['epoch', 'version', 'release', 'arch', + 'install_date', 'install_date_time_t']) + + if attr != 'all': +- requested_attr &= set(attr + ['version']) ++ requested_attr &= set(attr + ['version'] + ['arch']) + + for name in ret: ++ _parse_arch_from_name = __salt__.get('pkg.parse_arch_from_name', lambda pkgname: {'name': pkgname, 'arch': None}) ++ name_arch_d = _parse_arch_from_name(name) ++ _name = name_arch_d['name'] ++ _arch = name_arch_d['arch'] ++ + versions = [] ++ pkgname = None + for all_attr in ret[name]: + filtered_attr = {} + for key in requested_attr: +- if all_attr[key]: ++ if key in all_attr: + filtered_attr[key] = all_attr[key] + versions.append(filtered_attr) +- ret[name] = versions +- return ret ++ if _name and filtered_attr.get('arch', None) == _arch: ++ pkgname = _name ++ ret_attr.setdefault(pkgname or name, []).extend(versions) ++ return ret_attr + + for name in ret: + ret[name] = [format_version(d['epoch'], d['version'], d['release']) +diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py +index 51832bf883..cf50d1a4c4 100644 +--- a/salt/modules/yumpkg.py ++++ b/salt/modules/yumpkg.py +@@ -65,6 +65,8 @@ log = logging.getLogger(__name__) + + __HOLD_PATTERN = r'[\w+]+(?:[.-][^-]+)*' + ++PKG_ARCH_SEPARATOR = '.' ++ + # Define the module's virtual name + __virtualname__ = 'pkg' + +@@ -397,7 +399,7 @@ def normalize_name(name): + salt '*' pkg.normalize_name zsh.x86_64 + ''' + try: +- arch = name.rsplit('.', 1)[-1] ++ arch = name.rsplit(PKG_ARCH_SEPARATOR, 1)[-1] + if arch not in salt.utils.pkg.rpm.ARCHES + ('noarch',): + return name + except ValueError: +@@ -408,6 +410,30 @@ def normalize_name(name): + return name + + ++def parse_arch_from_name(name): ++ ''' ++ Parse name and architecture from the specified package name. ++ ++ CLI Example: ++ ++ .. code-block:: bash ++ ++ salt '*' pkg.parse_arch_from_name zsh.x86_64 ++ ''' ++ _name, _arch = None, None ++ try: ++ _name, _arch = name.rsplit(PKG_ARCH_SEPARATOR, 1) ++ except ValueError: ++ pass ++ if _arch not in salt.utils.pkg.rpm.ARCHES + ('noarch',): ++ _name = name ++ _arch = None ++ return { ++ 'name': _name, ++ 'arch': _arch ++ } ++ ++ + def latest_version(*names, **kwargs): + ''' + Return the latest version of the named package available for upgrade or +@@ -647,8 +673,8 @@ def list_pkgs(versions_as_list=False, **kwargs): + if pkginfo is not None: + # see rpm version string rules available at https://goo.gl/UGKPNd + pkgver = pkginfo.version +- epoch = '' +- release = '' ++ epoch = None ++ release = None + if ':' in pkgver: + epoch, pkgver = pkgver.split(":", 1) + if '-' in pkgver: +diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py +index 773354b2f3..ae66e4709d 100644 +--- a/salt/modules/zypper.py ++++ b/salt/modules/zypper.py +@@ -52,6 +52,7 @@ ZYPP_HOME = '/etc/zypp' + LOCKS = '{0}/locks'.format(ZYPP_HOME) + REPOS = '{0}/repos.d'.format(ZYPP_HOME) + DEFAULT_PRIORITY = 99 ++PKG_ARCH_SEPARATOR = '.' + + # Define the module's virtual name + __virtualname__ = 'pkg' +@@ -588,6 +589,30 @@ def info_available(*names, **kwargs): + return ret + + ++def parse_arch_from_name(name): ++ ''' ++ Parse name and architecture from the specified package name. ++ ++ CLI Example: ++ ++ .. code-block:: bash ++ ++ salt '*' pkg.parse_arch_from_name zsh.x86_64 ++ ''' ++ _name, _arch = None, None ++ try: ++ _name, _arch = name.rsplit(PKG_ARCH_SEPARATOR, 1) ++ except ValueError: ++ pass ++ if _arch not in salt.utils.pkg.rpm.ARCHES + ('noarch',): ++ _name = name ++ _arch = None ++ return { ++ 'name': _name, ++ 'arch': _arch ++ } ++ ++ + def latest_version(*names, **kwargs): + ''' + Return the latest version of the named package available for upgrade or +@@ -756,8 +781,8 @@ def list_pkgs(versions_as_list=False, **kwargs): + if pkginfo is not None: + # see rpm version string rules available at https://goo.gl/UGKPNd + pkgver = pkginfo.version +- epoch = '' +- release = '' ++ epoch = None ++ release = None + if ':' in pkgver: + epoch, pkgver = pkgver.split(":", 1) + if '-' in pkgver: +diff --git a/tests/unit/modules/test_pkg_resource.py b/tests/unit/modules/test_pkg_resource.py +index dd3ae9a1ac..2cfd6bb16a 100644 +--- a/tests/unit/modules/test_pkg_resource.py ++++ b/tests/unit/modules/test_pkg_resource.py +@@ -129,6 +129,122 @@ class PkgresTestCase(TestCase, LoaderModuleMockMixin): + ''' + self.assertIsNone(pkg_resource.sort_pkglist({})) + ++ def test_format_pkg_list_no_attr(self): ++ ''' ++ Test to output format of the package list with no attr parameter. ++ ''' ++ packages = { ++ 'glibc': [{'version': '2.12', 'epoch': None, 'release': '1.212.el6', 'arch': 'x86_64'}], ++ 'glibc.i686': [{'version': '2.12', 'epoch': None, 'release': '1.212.el6', 'arch': 'i686'}], ++ 'foobar': [ ++ {'version': '1.2.0', 'epoch': '2', 'release': '7', 'arch': 'x86_64'}, ++ {'version': '1.2.3', 'epoch': '2', 'release': '27', 'arch': 'x86_64'}, ++ ], ++ 'foobar.something': [{'version': '1.1', 'epoch': '3', 'release': '23.1', 'arch': 'i686'}], ++ 'foobar.': [{'version': '1.1', 'epoch': '3', 'release': '23.1', 'arch': 'i686'}] ++ } ++ expected_pkg_list = { ++ 'glibc': '2.12-1.212.el6', ++ 'glibc.i686': '2.12-1.212.el6', ++ 'foobar': '2:1.2.0-7,2:1.2.3-27', ++ 'foobar.something': '3:1.1-23.1', ++ 'foobar.': '3:1.1-23.1', ++ } ++ if six.PY3: ++ self.assertCountEqual(pkg_resource.format_pkg_list(packages, False, None), expected_pkg_list) ++ else: ++ self.assertItemsEqual(pkg_resource.format_pkg_list(packages, False, None), expected_pkg_list) ++ ++ def test_format_pkg_list_with_attr(self): ++ ''' ++ Test to output format of the package list with attr parameter. ++ In this case, any redundant "arch" reference will be removed from the package name since it's ++ include as part of the requested attr. ++ ''' ++ NAME_ARCH_MAPPING = { ++ 'glibc': { ++ 'name': 'glibc', ++ 'arch': None ++ }, ++ 'glibc.i686': { ++ 'name': 'glibc', ++ 'arch': 'i686' ++ }, ++ 'foobar': { ++ 'name': 'foobar', ++ 'arch': None ++ }, ++ 'foobar.something': { ++ 'name': 'foobar.something', ++ 'arch': None ++ }, ++ 'foobar.': { ++ 'name': 'foobar.', ++ 'arch': None ++ } ++ } ++ packages = { ++ 'glibc': [{'version': '2.12', 'epoch': None, 'release': '1.212.el6', 'arch': 'x86_64'}], ++ 'glibc.i686': [{'version': '2.12', 'epoch': None, 'release': '1.212.el6', 'arch': 'i686'}], ++ 'foobar': [ ++ {'version': '1.2.0', 'epoch': '2', 'release': '7', 'arch': 'x86_64'}, ++ {'version': '1.2.3', 'epoch': '2', 'release': '27', 'arch': 'x86_64'}, ++ ], ++ 'foobar.something': [{'version': '1.1', 'epoch': '3', 'release': '23.1', 'arch': 'i686'}], ++ 'foobar.': [{'version': '1.1', 'epoch': '3', 'release': '23.1', 'arch': 'i686'}] ++ } ++ expected_pkg_list = { ++ 'glibc': [ ++ { ++ 'arch': 'x86_64', ++ 'release': '1.212.el6', ++ 'epoch': None, ++ 'version': '2.12' ++ }, ++ { ++ 'arch': 'i686', ++ 'release': '1.212.el6', ++ 'epoch': None, ++ 'version': '2.12' ++ } ++ ], ++ 'foobar': [ ++ { ++ 'arch': 'x86_64', ++ 'release': '7', ++ 'epoch': '2', ++ 'version': '1.2.0' ++ }, ++ { ++ 'arch': 'x86_64', ++ 'release': '27', ++ 'epoch': '2', ++ 'version': '1.2.3' ++ } ++ ], ++ 'foobar.': [ ++ { ++ 'arch': 'i686', ++ 'release': '23.1', ++ 'epoch': '3', ++ 'version': '1.1' ++ } ++ ], ++ 'foobar.something': [ ++ { ++ 'arch': 'i686', ++ 'release': '23.1', ++ 'epoch': '3', ++ 'version': '1.1' ++ } ++ ] ++ } ++ with patch.dict(pkg_resource.__salt__, {'pkg.parse_arch_from_name': NAME_ARCH_MAPPING.get}): ++ if six.PY3: ++ self.assertCountEqual(pkg_resource.format_pkg_list(packages, False, attr=['epoch', 'release']), expected_pkg_list) ++ else: ++ self.assertItemsEqual(pkg_resource.format_pkg_list(packages, False, attr=['epoch', 'release']), expected_pkg_list) ++ + def test_stringify(self): + ''' + Test to takes a dict of package name/version information +diff --git a/tests/unit/modules/test_yumpkg.py b/tests/unit/modules/test_yumpkg.py +index c73f2582b9..324c2c8b66 100644 +--- a/tests/unit/modules/test_yumpkg.py ++++ b/tests/unit/modules/test_yumpkg.py +@@ -16,6 +16,7 @@ from tests.support.mock import ( + ) + + # Import Salt libs ++from salt.ext import six + import salt.modules.yumpkg as yumpkg + import salt.modules.pkg_resource as pkg_resource + +@@ -69,7 +70,8 @@ class YumTestCase(TestCase, LoaderModuleMockMixin): + 'os_family': 'RedHat', + 'osmajorrelease': 7, + }, +- } ++ }, ++ pkg_resource: {} + } + + def test_list_pkgs(self): +@@ -100,7 +102,8 @@ class YumTestCase(TestCase, LoaderModuleMockMixin): + patch.dict(yumpkg.__salt__, {'cmd.run': MagicMock(return_value=os.linesep.join(rpm_out))}), \ + patch.dict(yumpkg.__salt__, {'pkg_resource.add_pkg': _add_data}), \ + patch.dict(yumpkg.__salt__, {'pkg_resource.format_pkg_list': pkg_resource.format_pkg_list}), \ +- patch.dict(yumpkg.__salt__, {'pkg_resource.stringify': MagicMock()}): ++ patch.dict(yumpkg.__salt__, {'pkg_resource.stringify': MagicMock()}), \ ++ patch.dict(pkg_resource.__salt__, {'pkg.parse_arch_from_name': yumpkg.parse_arch_from_name}): + pkgs = yumpkg.list_pkgs(versions_as_list=True) + for pkg_name, pkg_version in { + 'python-urlgrabber': '3.10-8.el7', +@@ -147,7 +150,8 @@ class YumTestCase(TestCase, LoaderModuleMockMixin): + patch.dict(yumpkg.__salt__, {'cmd.run': MagicMock(return_value=os.linesep.join(rpm_out))}), \ + patch.dict(yumpkg.__salt__, {'pkg_resource.add_pkg': _add_data}), \ + patch.dict(yumpkg.__salt__, {'pkg_resource.format_pkg_list': pkg_resource.format_pkg_list}), \ +- patch.dict(yumpkg.__salt__, {'pkg_resource.stringify': MagicMock()}): ++ patch.dict(yumpkg.__salt__, {'pkg_resource.stringify': MagicMock()}), \ ++ patch.dict(pkg_resource.__salt__, {'pkg.parse_arch_from_name': yumpkg.parse_arch_from_name}): + pkgs = yumpkg.list_pkgs(attr=['epoch', 'release', 'arch', 'install_date_time_t']) + for pkg_name, pkg_attr in { + 'python-urlgrabber': { +@@ -155,54 +159,63 @@ class YumTestCase(TestCase, LoaderModuleMockMixin): + 'release': '8.el7', + 'arch': 'noarch', + 'install_date_time_t': 1487838471, ++ 'epoch': None + }, + 'alsa-lib': { + 'version': '1.1.1', + 'release': '1.el7', + 'arch': 'x86_64', + 'install_date_time_t': 1487838475, ++ 'epoch': None + }, + 'gnupg2': { + 'version': '2.0.22', + 'release': '4.el7', + 'arch': 'x86_64', + 'install_date_time_t': 1487838477, ++ 'epoch': None + }, + 'rpm-python': { + 'version': '4.11.3', + 'release': '21.el7', + 'arch': 'x86_64', + 'install_date_time_t': 1487838477, ++ 'epoch': None + }, + 'pygpgme': { + 'version': '0.3', + 'release': '9.el7', + 'arch': 'x86_64', + 'install_date_time_t': 1487838478, ++ 'epoch': None + }, + 'yum': { + 'version': '3.4.3', + 'release': '150.el7.centos', + 'arch': 'noarch', + 'install_date_time_t': 1487838479, ++ 'epoch': None + }, + 'lzo': { + 'version': '2.06', + 'release': '8.el7', + 'arch': 'x86_64', + 'install_date_time_t': 1487838479, ++ 'epoch': None + }, + 'qrencode-libs': { + 'version': '3.4.1', + 'release': '3.el7', + 'arch': 'x86_64', + 'install_date_time_t': 1487838480, ++ 'epoch': None + }, + 'ustr': { + 'version': '1.0.4', + 'release': '16.el7', + 'arch': 'x86_64', + 'install_date_time_t': 1487838480, ++ 'epoch': None + }, + 'shadow-utils': { + 'epoch': '2', +@@ -216,22 +229,88 @@ class YumTestCase(TestCase, LoaderModuleMockMixin): + 'release': '33.el7', + 'arch': 'x86_64', + 'install_date_time_t': 1487838484, ++ 'epoch': None + }, + 'openssh': { + 'version': '6.6.1p1', + 'release': '33.el7_3', + 'arch': 'x86_64', + 'install_date_time_t': 1487838485, ++ 'epoch': None + }, + 'virt-what': { + 'version': '1.13', + 'release': '8.el7', + 'install_date_time_t': 1487838486, + 'arch': 'x86_64', ++ 'epoch': None + }}.items(): ++ + self.assertTrue(pkgs.get(pkg_name)) + self.assertEqual(pkgs[pkg_name], [pkg_attr]) + ++ def test_list_pkgs_with_attr_multiple_versions(self): ++ ''' ++ Test packages listing with the attr parameter reporting multiple version installed ++ ++ :return: ++ ''' ++ def _add_data(data, key, value): ++ data.setdefault(key, []).append(value) ++ ++ rpm_out = [ ++ 'glibc_|-(none)_|-2.12_|-1.212.el6_|-i686_|-(none)_|-1542394210' ++ 'glibc_|-(none)_|-2.12_|-1.212.el6_|-x86_64_|-(none)_|-1542394204', ++ 'virt-what_|-(none)_|-1.13_|-8.el7_|-x86_64_|-(none)_|-1487838486', ++ 'virt-what_|-(none)_|-1.10_|-2.el7_|-x86_64_|-(none)_|-1387838486', ++ ] ++ with patch.dict(yumpkg.__grains__, {'osarch': 'x86_64'}), \ ++ patch.dict(yumpkg.__salt__, {'cmd.run': MagicMock(return_value=os.linesep.join(rpm_out))}), \ ++ patch.dict(yumpkg.__salt__, {'pkg_resource.add_pkg': _add_data}), \ ++ patch.dict(yumpkg.__salt__, {'pkg_resource.format_pkg_list': pkg_resource.format_pkg_list}), \ ++ patch.dict(yumpkg.__salt__, {'pkg_resource.stringify': MagicMock()}), \ ++ patch.dict(pkg_resource.__salt__, {'pkg.parse_arch_from_name': yumpkg.parse_arch_from_name}): ++ pkgs = yumpkg.list_pkgs(attr=['epoch', 'release', 'arch', 'install_date_time_t']) ++ expected_pkg_list = { ++ 'glibc': [ ++ { ++ 'version': '2.12', ++ 'release': '1.212.el6', ++ 'install_date_time_t': 1542394210, ++ 'arch': 'i686', ++ 'epoch': None ++ }, ++ { ++ 'version': '2.12', ++ 'release': '1.212.el6', ++ 'install_date_time_t': 1542394204, ++ 'arch': 'x86_64', ++ 'epoch': None ++ } ++ ], ++ 'virt-what': [ ++ { ++ 'version': '1.10', ++ 'release': '2.el7', ++ 'install_date_time_t': 1387838486, ++ 'arch': 'x86_64', ++ 'epoch': None ++ }, ++ { ++ 'version': '1.13', ++ 'release': '8.el7', ++ 'install_date_time_t': 1487838486, ++ 'arch': 'x86_64', ++ 'epoch': None ++ } ++ ] ++ } ++ for pkgname, pkginfo in pkgs.items(): ++ if six.PY3: ++ self.assertCountEqual(pkginfo, expected_pkg_list[pkgname]) ++ else: ++ self.assertItemsEqual(pkginfo, expected_pkg_list[pkgname]) ++ + def test_latest_version_with_options(self): + with patch.object(yumpkg, 'list_pkgs', MagicMock(return_value={})): + +diff --git a/tests/unit/modules/test_zypper.py b/tests/unit/modules/test_zypper.py +index 424438c8bf..a60e209b2c 100644 +--- a/tests/unit/modules/test_zypper.py ++++ b/tests/unit/modules/test_zypper.py +@@ -61,7 +61,7 @@ class ZypperTestCase(TestCase, LoaderModuleMockMixin): + ''' + + def setup_loader_modules(self): +- return {zypper: {'rpm': None}} ++ return {zypper: {'rpm': None}, pkg_resource: {}} + + def setUp(self): + self.new_repo_config = dict( +@@ -603,7 +603,8 @@ Repository 'DUMMY' not found by its alias, number, or URI. + patch.dict(zypper.__grains__, {'osarch': 'x86_64'}), \ + patch.dict(zypper.__salt__, {'pkg_resource.add_pkg': _add_data}), \ + patch.dict(zypper.__salt__, {'pkg_resource.format_pkg_list': pkg_resource.format_pkg_list}), \ +- patch.dict(zypper.__salt__, {'pkg_resource.stringify': MagicMock()}): ++ patch.dict(zypper.__salt__, {'pkg_resource.stringify': MagicMock()}), \ ++ patch.dict(pkg_resource.__salt__, {'pkg.parse_arch_from_name': zypper.parse_arch_from_name}): + pkgs = zypper.list_pkgs(attr=['epoch', 'release', 'arch', 'install_date_time_t']) + self.assertFalse(pkgs.get('gpg-pubkey', False)) + for pkg_name, pkg_attr in { +@@ -612,58 +613,130 @@ Repository 'DUMMY' not found by its alias, number, or URI. + 'release': '129.686', + 'arch': 'noarch', + 'install_date_time_t': 1498636511, ++ 'epoch': None, + }], + 'yast2-ftp-server': [{ + 'version': '3.1.8', + 'release': '8.1', + 'arch': 'x86_64', + 'install_date_time_t': 1499257798, ++ 'epoch': None, + }], + 'protobuf-java': [{ + 'version': '2.6.1', + 'release': '3.1.develHead', + 'install_date_time_t': 1499257756, + 'arch': 'noarch', ++ 'epoch': None, + }], + 'susemanager-build-keys-web': [{ + 'version': '12.0', + 'release': '5.1.develHead', + 'arch': 'noarch', + 'install_date_time_t': 1498636510, ++ 'epoch': None, + }], + 'apache-commons-cli': [{ + 'version': '1.2', + 'release': '1.233', + 'arch': 'noarch', + 'install_date_time_t': 1498636510, ++ 'epoch': None, + }], + 'kernel-default': [{ + 'version': '4.4.138', + 'release': '94.39.1', + 'arch': 'x86_64', +- 'install_date_time_t': 1529936067 ++ 'install_date_time_t': 1529936067, ++ 'epoch': None, + }, + { + 'version': '4.4.73', + 'release': '5.1', + 'arch': 'x86_64', + 'install_date_time_t': 1503572639, ++ 'epoch': None, + }], +- 'perseus-dummy.i586': [{ ++ 'perseus-dummy': [{ + 'version': '1.1', + 'release': '1.1', + 'arch': 'i586', + 'install_date_time_t': 1529936062, ++ 'epoch': None, + }], + 'jose4j': [{ + 'arch': 'noarch', + 'version': '0.4.4', + 'release': '2.1.develHead', + 'install_date_time_t': 1499257756, ++ 'epoch': None, + }]}.items(): + self.assertTrue(pkgs.get(pkg_name)) + self.assertEqual(pkgs[pkg_name], pkg_attr) + ++ def test_list_pkgs_with_attr_multiple_versions(self): ++ ''' ++ Test packages listing with the attr parameter reporting multiple version installed ++ ++ :return: ++ ''' ++ def _add_data(data, key, value): ++ data.setdefault(key, []).append(value) ++ ++ rpm_out = [ ++ 'glibc_|-2.12_|-1.212.el6_|-i686_|-_|-1542394210', ++ 'glibc_|-2.12_|-1.212.el6_|-x86_64_|-_|-1542394204', ++ 'virt-what_|-1.13_|-8.el7_|-x86_64_|-_|-1487838486', ++ 'virt-what_|-1.10_|-2.el7_|-x86_64_|-_|-1387838486', ++ ] ++ ++ with patch.dict(zypper.__salt__, {'cmd.run': MagicMock(return_value=os.linesep.join(rpm_out))}), \ ++ patch.dict(zypper.__grains__, {'osarch': 'x86_64'}), \ ++ patch.dict(zypper.__salt__, {'pkg_resource.add_pkg': _add_data}), \ ++ patch.dict(zypper.__salt__, {'pkg_resource.format_pkg_list': pkg_resource.format_pkg_list}), \ ++ patch.dict(zypper.__salt__, {'pkg_resource.stringify': MagicMock()}), \ ++ patch.dict(pkg_resource.__salt__, {'pkg.parse_arch_from_name': zypper.parse_arch_from_name}): ++ pkgs = zypper.list_pkgs(attr=['epoch', 'release', 'arch', 'install_date_time_t']) ++ expected_pkg_list = { ++ 'glibc': [ ++ { ++ 'version': '2.12', ++ 'release': '1.212.el6', ++ 'install_date_time_t': 1542394210, ++ 'arch': 'i686', ++ 'epoch': None ++ }, ++ { ++ 'version': '2.12', ++ 'release': '1.212.el6', ++ 'install_date_time_t': 1542394204, ++ 'arch': 'x86_64', ++ 'epoch': None ++ } ++ ], ++ 'virt-what': [ ++ { ++ 'version': '1.10', ++ 'release': '2.el7', ++ 'install_date_time_t': 1387838486, ++ 'arch': 'x86_64', ++ 'epoch': None ++ }, ++ { ++ 'version': '1.13', ++ 'release': '8.el7', ++ 'install_date_time_t': 1487838486, ++ 'arch': 'x86_64', ++ 'epoch': None ++ } ++ ] ++ } ++ for pkgname, pkginfo in pkgs.items(): ++ if six.PY3: ++ self.assertCountEqual(pkginfo, expected_pkg_list[pkgname]) ++ else: ++ self.assertItemsEqual(pkginfo, expected_pkg_list[pkgname]) ++ + def test_list_patches(self): + ''' + Test advisory patches listing. +-- +2.17.1 + + diff --git a/retire-md5-checksum-for-pkg-mgmt-plugins.patch b/retire-md5-checksum-for-pkg-mgmt-plugins.patch new file mode 100644 index 0000000..33f009e --- /dev/null +++ b/retire-md5-checksum-for-pkg-mgmt-plugins.patch @@ -0,0 +1,43 @@ +From 0908344fae3edda3372ee03820ea30ebcfe8980e Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Thu, 13 Sep 2018 12:00:55 +0200 +Subject: [PATCH] Retire MD5 checksum for pkg mgmt plugins + +Use SHA256 algorithm for zyppnotify plugin + +Remove an empty line +--- + scripts/suse/yum/plugins/yumnotify.py | 2 +- + scripts/suse/zypper/plugins/commit/zyppnotify | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/scripts/suse/yum/plugins/yumnotify.py b/scripts/suse/yum/plugins/yumnotify.py +index 268e1e9531..dd2485c886 100644 +--- a/scripts/suse/yum/plugins/yumnotify.py ++++ b/scripts/suse/yum/plugins/yumnotify.py +@@ -32,7 +32,7 @@ def _get_checksum(): + Returns: + hexdigest + """ +- digest = hashlib.md5() ++ digest = hashlib.sha256() + with open(RPM_PATH, "rb") as rpm_db_fh: + while True: + buff = rpm_db_fh.read(0x1000) +diff --git a/scripts/suse/zypper/plugins/commit/zyppnotify b/scripts/suse/zypper/plugins/commit/zyppnotify +index 268298b108..b64badb119 100755 +--- a/scripts/suse/zypper/plugins/commit/zyppnotify ++++ b/scripts/suse/zypper/plugins/commit/zyppnotify +@@ -35,7 +35,7 @@ class DriftDetector(Plugin): + Returns: + hexdigest + ''' +- digest = hashlib.md5() ++ digest = hashlib.sha256() + with open(self.rpm_path, "rb") as rpm_db_fh: + while True: + buff = rpm_db_fh.read(0x1000) +-- +2.20.1 + + diff --git a/return-the-expected-powerpc-os-arch-bsc-1117995.patch b/return-the-expected-powerpc-os-arch-bsc-1117995.patch new file mode 100644 index 0000000..53a57c2 --- /dev/null +++ b/return-the-expected-powerpc-os-arch-bsc-1117995.patch @@ -0,0 +1,31 @@ +From 2cbc403b422a699cd948ed6218fce28fa901f5fa Mon Sep 17 00:00:00 2001 +From: Mihai Dinca +Date: Thu, 13 Dec 2018 12:17:35 +0100 +Subject: [PATCH] Return the expected powerpc os arch (bsc#1117995) + +--- + salt/utils/pkg/rpm.py | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/salt/utils/pkg/rpm.py b/salt/utils/pkg/rpm.py +index bb8c3fb589..828b0cecda 100644 +--- a/salt/utils/pkg/rpm.py ++++ b/salt/utils/pkg/rpm.py +@@ -53,8 +53,11 @@ def get_osarch(): + stderr=subprocess.PIPE).communicate()[0] + else: + ret = ''.join(list(filter(None, platform.uname()[-2:]))[-1:]) +- +- return salt.utils.stringutils.to_str(ret).strip() or 'unknown' ++ ret = salt.utils.stringutils.to_str(ret).strip() or 'unknown' ++ ARCH_FIXES_MAPPING = { ++ "powerpc64le": "ppc64le" ++ } ++ return ARCH_FIXES_MAPPING.get(ret, ret) + + + def check_32(arch, osarch=None): +-- +2.20.1 + + diff --git a/salt.changes b/salt.changes index d612c8a..d61a0cf 100644 --- a/salt.changes +++ b/salt.changes @@ -1,3 +1,253 @@ +------------------------------------------------------------------- +Wed Jan 16 16:28:09 UTC 2019 - psuarezhernandez@suse.com + +- Do not restrict the Python version to < 3.7 + +------------------------------------------------------------------- +Tue Jan 15 09:47:12 UTC 2019 - bo@suse.de + +- Fix integration tests in state compiler (U#2068) + +- Added: + * fix-issue-2068-test.patch + +------------------------------------------------------------------- +Fri Jan 11 13:23:13 UTC 2019 - psuarezhernandez@suse.com + +- Fix "pkg.list_pkgs" output when using "attr" to take the arch into account (bsc#1114029) + +- Added: + * remove-arch-from-name-when-pkg.list_pkgs-is-called-w.patch + +------------------------------------------------------------------- +Thu Jan 10 12:52:09 UTC 2019 - mdinca + +- Fix powerpc null server_id_arch (bsc#1117995) + +- Added: + * return-the-expected-powerpc-os-arch-bsc-1117995.patch + +------------------------------------------------------------------- +Thu Jan 10 09:53:33 UTC 2019 - bo@suse.de + +- Fix module 'azure.storage' has no attribute '__version__' + (bsc#1121091) + +- Added: + * azurefs-gracefully-handle-attributeerror.patch + +------------------------------------------------------------------- +Fri Jan 4 13:29:50 UTC 2019 - bo@suse.de + +- Add supportconfig module and states for minions and SaltSSH + +- Added: + * add-supportconfig-module-for-remote-calls-and-saltss.patch + +------------------------------------------------------------------- +Thu Jan 3 16:35:30 UTC 2019 - bo@suse.de + +- Fix FIPS enabled RES clients (bsc#1099887) + +- Added: + * retire-md5-checksum-for-pkg-mgmt-plugins.patch + +------------------------------------------------------------------- +Thu Jan 3 15:48:20 UTC 2019 - bo@suse.de + +- Add hold/unhold functions. Fix Debian repo "signed-by". + +- Added: + * decide-if-the-source-should-be-actually-skipped.patch + * add-hold-unhold-functions.patch + +------------------------------------------------------------------- +Tue Dec 4 16:28:21 UTC 2018 - psuarezhernandez@suse.com + +- Fix latin1 encoding problems on file module (bsc#1116837) + +- Added: + * fix-latin1-encoding-problems-on-file-module-bsc-1116.patch + +------------------------------------------------------------------- +Fri Nov 30 13:14:19 UTC 2018 - bo@suse.de + +- Don't error on retcode 0 in libcrypto.OPENSSL_init_crypto + +- Added: + * don-t-error-on-retcode-0-in-libcrypto.openssl_init_c.patch + +------------------------------------------------------------------- +Tue Nov 20 15:33:39 UTC 2018 - bo@suse.de + +- Debian info_installed compatibility (U#50453) + +- Added: + * debian-info_installed-compatibility-50453.patch + +------------------------------------------------------------------- +Fri Nov 16 14:17:45 UTC 2018 - bo@suse.de + +- Add compatibility with other package modules for "list_repos" function +- Bugfix: unable to detect os arch when RPM is not installed (bsc#1114197) + +- Added: + * make-aptpkg.list_repos-compatible-on-enabled-disable.patch + * get-os_arch-also-without-rpm-package-installed.patch + +------------------------------------------------------------------- +Thu Nov 8 09:32:49 UTC 2018 - psuarezhernandez@suse.com + +- Fix git_pillar merging across multiple __env__ repositories (bsc#1112874) + +- Added: + * fix-git_pillar-merging-across-multiple-__env__-repos.patch + +------------------------------------------------------------------- +Wed Oct 31 14:52:31 UTC 2018 - bo@suse.de + +- Fix LDAP authentication issue when a valid token is generated + by the salt-api even when invalid user credentials are passed. + (U#48901) + +- Added: + * fixing-issue-when-a-valid-token-is-generated-even-wh.patch + +------------------------------------------------------------------- +Tue Oct 30 10:48:23 UTC 2018 - Jochen Breuer + +- Improved handling of LDAP group id. gid is no longer treated as a + string, which could have lead to faulty group creations. (bsc#1113784) + +- Added: + * improved-handling-of-ldap-group-id.patch + +------------------------------------------------------------------- +Thu Oct 25 13:04:42 UTC 2018 - psuarezhernandez@suse.com + +- Fix remote command execution and incorrect access control + when using salt-api. (bsc#1113699) (CVE-2018-15751) +- Fix Directory traversal vulnerability when using salt-api. + Allows an attacker to determine what files exist on + a server when querying /run or /events. (bsc#1113698) (CVE-2018-15750) + +- Added: + * fixes-cve-2018-15750-cve-2018-15751.patch + +------------------------------------------------------------------- +Thu Oct 18 13:17:33 UTC 2018 - bo@suse.de + +- Add multi-file support and globbing to the filetree (U#50018) + +- Added: + * add-multi-file-support-and-globbing-to-the-filetree-.patch + +------------------------------------------------------------------- +Wed Oct 17 15:21:17 UTC 2018 - bo@suse.de + +- Bugfix: supportconfig non-root permission issues (U#50095) + +- Added: + * support-config-non-root-permission-issues-fixes-u-50.patch + +------------------------------------------------------------------- +Wed Oct 17 14:18:09 UTC 2018 - bo@suse.de + +- Open profiles permissions to everyone for read-only + +------------------------------------------------------------------- +Tue Oct 16 15:26:16 UTC 2018 - bo@suse.de + +- Preserving signature in "module.run" state (U#50049) + +- Added: + * preserving-signature-in-module.run-state-u-50049.patch + +------------------------------------------------------------------- +Fri Oct 12 11:48:40 UTC 2018 - bo@suse.de + +- Install default salt-support profiles + +------------------------------------------------------------------- +Thu Oct 11 15:04:30 UTC 2018 - bo@suse.de + +- Fix unit tests due to merger failure +- Add CPE_NAME for osversion* grain parsing +- Get os_family for RPM distros from the RPM macros +- Install support profiles + +- Added: + * get-os_family-for-rpm-distros-from-the-rpm-macros.-u.patch + * add-cpe_name-for-osversion-grain-parsing-u-49946.patch + * make-profiles-a-package.patch + * fix-unit-test-for-grains-core.patch + +------------------------------------------------------------------- +Tue Oct 9 14:50:25 UTC 2018 - psuarezhernandez@suse.com + +- Bugfix: any unicode string of length 16 will raise TypeError + +- Added: + * bugfix-any-unicode-string-of-length-16-will-raise-ty.patch + +------------------------------------------------------------------- +Mon Oct 8 08:52:23 UTC 2018 - psuarezhernandez@suse.com + +- Fix async call to process manager (bsc#1110938) +- Early feature: Salt support-config (salt-support) + +- Added: + * fix-async-call-to-process-manager.patch + * early-feature-support-config.patch + +------------------------------------------------------------------- +Mon Oct 1 16:03:27 UTC 2018 - bo@suse.de + +- Fix IPv6 scope (bsc#1108557) + +- Added: + * fix-ipv6-scope-bsc-1108557.patch + +------------------------------------------------------------------- +Fri Sep 28 12:37:02 UTC 2018 - bo@suse.de + +- Handle zypper ZYPPER_EXIT_NO_REPOS exit code (bsc#1108834, bsc#1109893) + +- Added: + * update-error-list-for-zypper.patch + +------------------------------------------------------------------- +Mon Sep 24 15:49:47 UTC 2018 - bo@suse.de + +- Bugfix for pkg_resources crash (bsc#1104491) + +- Added: + * do-not-load-pip-state-if-there-is-no-3rd-party-depen.patch + +------------------------------------------------------------------- +Fri Sep 21 15:39:49 UTC 2018 - psuarezhernandez@suse.com + +- Fix loosen azure sdk dependencies in azurearm cloud driver (bsc#1107333) + +- Added: + * loosen-azure-sdk-dependencies-in-azurearm-cloud-driv.patch + +------------------------------------------------------------------- +Thu Sep 20 11:25:57 UTC 2018 - psuarezhernandez@suse.com + +- Fix broken "resolve_capabilities" on Python 3 (bsc#1108995) + +- Added: + * fix-index-error-when-running-on-python-3.patch + +------------------------------------------------------------------- +Wed Sep 19 13:06:21 UTC 2018 - psuarezhernandez@suse.com + +- Allow empty service_account_private_key in GCE driver (bsc#1108969) + +- Added: + * support-use-of-gce-instance-credentials-109.patch + ------------------------------------------------------------------- Tue Sep 18 14:28:13 UTC 2018 - mihai.dinca@suse.com diff --git a/salt.spec b/salt.spec index a9e53bb..a937c00 100644 --- a/salt.spec +++ b/salt.spec @@ -15,7 +15,6 @@ # Please submit bugfixes or comments via http://bugs.opensuse.org/ # - %if 0%{?suse_version} >= 1320 # SLE15 %global build_py3 1 @@ -150,6 +149,76 @@ Patch41: change-stringio-import-in-python2-to-import-the-clas.patch Patch42: use-adler32-algorithm-to-compute-string-checksums.patch # PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/49497 Patch43: x509-fixes-111.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/49605 +Patch44: support-use-of-gce-instance-credentials-109.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/49722 +Patch45: fix-index-error-when-running-on-python-3.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/49696 +Patch46: loosen-azure-sdk-dependencies-in-azurearm-cloud-driv.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/49737 +Patch47: do-not-load-pip-state-if-there-is-no-3rd-party-depen.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/49806 +Patch48: update-error-list-for-zypper.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/49815 +Patch49: fix-ipv6-scope-bsc-1108557.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/49480 +Patch50: early-feature-support-config.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/49085 +Patch51: fix-async-call-to-process-manager.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/49908 +Patch52: bugfix-any-unicode-string-of-length-16-will-raise-ty.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/49936 +Patch53: make-profiles-a-package.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/49930 +Patch54: get-os_family-for-rpm-distros-from-the-rpm-macros.-u.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/49946 +Patch55: add-cpe_name-for-osversion-grain-parsing-u-49946.patch +# PATCH-FIX_OPENSUSE: Fix unit test for grains core +Patch56: fix-unit-test-for-grains-core.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/50049 +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/50072 +Patch57: preserving-signature-in-module.run-state-u-50049.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/50095 +Patch58: support-config-non-root-permission-issues-fixes-u-50.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/50018 +Patch59: add-multi-file-support-and-globbing-to-the-filetree-.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/49761 +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/50201 +Patch60: fixes-cve-2018-15750-cve-2018-15751.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/48491 +Patch61: improved-handling-of-ldap-group-id.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/48901 +Patch62: fixing-issue-when-a-valid-token-is-generated-even-wh.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/50417 +Patch63: fix-git_pillar-merging-across-multiple-__env__-repos.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/50523 +Patch64: get-os_arch-also-without-rpm-package-installed.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/50392 +Patch65: make-aptpkg.list_repos-compatible-on-enabled-disable.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/50453 +Patch66: debian-info_installed-compatibility-50453.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/48580 +Patch67: don-t-error-on-retcode-0-in-libcrypto.openssl_init_c.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/48503 +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/48934 +Patch68: fix-latin1-encoding-problems-on-file-module-bsc-1116.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/50742 +Patch69: decide-if-the-source-should-be-actually-skipped.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/50773 +Patch70: add-hold-unhold-functions.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/49639 +Patch71: retire-md5-checksum-for-pkg-mgmt-plugins.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/50401 +# NOTE: This is a techpreview as well as in Fluorine! Release only in Neon. +Patch72: add-supportconfig-module-for-remote-calls-and-saltss.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/50567 +Patch73: azurefs-gracefully-handle-attributeerror.patch +# PATCH_FIX_OPENSUSE: https://github.com/openSUSE/salt/pull/116 +Patch74: return-the-expected-powerpc-os-arch-bsc-1117995.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/51108 +Patch75: remove-arch-from-name-when-pkg.list_pkgs-is-called-w.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/51119 +Patch76: fix-issue-2068-test.patch # BuildRoot: %{_tmppath}/%{name}-%{version}-build BuildRoot: %{_tmppath}/%{name}-%{version}-build @@ -235,12 +304,12 @@ BuildRequires: python-devel >= 2.7 # requirements/base.txt %if 0%{?rhel} BuildRequires: python-jinja2 -BuildRequires: python-markupsafe BuildRequires: python-yaml +BuildRequires: python-markupsafe %else BuildRequires: python-Jinja2 -BuildRequires: python-MarkupSafe BuildRequires: python-PyYAML +BuildRequires: python-MarkupSafe %endif BuildRequires: python-futures >= 2.0 @@ -273,16 +342,16 @@ Requires: python-certifi # requirements/base.txt %if 0%{?rhel} Requires: python-jinja2 -Requires: python-markupsafe Requires: python-yaml +Requires: python-markupsafe Requires: yum %if 0%{?rhel} == 6 Requires: yum-plugin-security %endif %else Requires: python-Jinja2 -Requires: python-MarkupSafe Requires: python-PyYAML +Requires: python-MarkupSafe %endif Requires: python-futures >= 2.0 @@ -322,7 +391,7 @@ Summary: python3 library for salt Group: System/Management Requires: %{name} = %{version}-%{release} BuildRequires: python-rpm-macros -BuildRequires: python3 < 3.7 +BuildRequires: python3 BuildRequires: python3-devel # requirements/base.txt %if 0%{?rhel} @@ -353,7 +422,7 @@ BuildRequires: python3-xml %if %{with builddocs} BuildRequires: python3-sphinx %endif -Requires: python3 < 3.7 +Requires: python3 # %if ! 0%{?suse_version} > 1110 Requires: python3-certifi @@ -651,6 +720,39 @@ cp %{S:5} ./.travis.yml %patch41 -p1 %patch42 -p1 %patch43 -p1 +%patch44 -p1 +%patch45 -p1 +%patch46 -p1 +%patch47 -p1 +%patch48 -p1 +%patch49 -p1 +%patch50 -p1 +%patch51 -p1 +%patch52 -p1 +%patch53 -p1 +%patch54 -p1 +%patch55 -p1 +%patch56 -p1 +%patch57 -p1 +%patch58 -p1 +%patch59 -p1 +%patch60 -p1 +%patch61 -p1 +%patch62 -p1 +%patch63 -p1 +%patch64 -p1 +%patch65 -p1 +%patch66 -p1 +%patch67 -p1 +%patch68 -p1 +%patch69 -p1 +%patch70 -p1 +%patch71 -p1 +%patch72 -p1 +%patch73 -p1 +%patch74 -p1 +%patch75 -p1 +%patch76 -p1 %build %if 0%{?build_py2} @@ -739,6 +841,15 @@ install -Dd -m 0750 %{buildroot}%{_sysconfdir}/salt/pki/master/minions_pre install -Dd -m 0750 %{buildroot}%{_sysconfdir}/salt/pki/master/minions_rejected install -Dd -m 0750 %{buildroot}%{_sysconfdir}/salt/pki/minion +# Install salt-support profiles +%if 0%{?build_py2} +install -Dpm 0644 salt/cli/support/profiles/* %{buildroot}%{python_sitelib}/salt/cli/support/profiles +%endif +%if 0%{?build_py3} +install -Dpm 0644 salt/cli/support/profiles/* %{buildroot}%{python3_sitelib}/salt/cli/support/profiles +%endif + + ## Install Zypper plugins only on SUSE machines %if 0%{?suse_version} install -Dd -m 0750 %{buildroot}%{_prefix}/lib/zypp/plugins/commit @@ -1287,6 +1398,7 @@ rm -f %{_localstatedir}/cache/salt/minion/thin/version %defattr(-,root,root,-) %{_bindir}/spm %{_bindir}/salt-call +%{_bindir}/salt-support %{_bindir}/salt-unity %{_mandir}/man1/salt-unity.1.gz %{_mandir}/man1/salt-call.1.gz @@ -1353,3 +1465,5 @@ rm -f %{_localstatedir}/cache/salt/minion/thin/version %endif %changelog + + diff --git a/support-config-non-root-permission-issues-fixes-u-50.patch b/support-config-non-root-permission-issues-fixes-u-50.patch new file mode 100644 index 0000000..a5aa8bf --- /dev/null +++ b/support-config-non-root-permission-issues-fixes-u-50.patch @@ -0,0 +1,106 @@ +From 1113909fe9ab0509ebe439051238d6a4f95d3c54 Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Wed, 17 Oct 2018 14:10:47 +0200 +Subject: [PATCH] Support-config non-root permission issues fixes + (U#50095) + +Do not crash if there is no configuration available at all + +Handle CLI and log errors + +Catch overwriting exiting archive error by other users + +Suppress excessive tracebacks on error log level +--- + salt/cli/support/collector.py | 39 ++++++++++++++++++++++++++++++++--- + salt/utils/parsers.py | 2 +- + 2 files changed, 37 insertions(+), 4 deletions(-) + +diff --git a/salt/cli/support/collector.py b/salt/cli/support/collector.py +index 478d07e13b..a4343297b6 100644 +--- a/salt/cli/support/collector.py ++++ b/salt/cli/support/collector.py +@@ -125,6 +125,31 @@ class SupportDataCollector(object): + self.__current_section = [] + self.__current_section_name = name + ++ def _printout(self, data, output): ++ ''' ++ Use salt outputter to printout content. ++ ++ :return: ++ ''' ++ opts = {'extension_modules': '', 'color': False} ++ try: ++ printout = salt.output.get_printout(output, opts)(data) ++ if printout is not None: ++ return printout.rstrip() ++ except (KeyError, AttributeError, TypeError) as err: ++ log.debug(err, exc_info=True) ++ try: ++ printout = salt.output.get_printout('nested', opts)(data) ++ if printout is not None: ++ return printout.rstrip() ++ except (KeyError, AttributeError, TypeError) as err: ++ log.debug(err, exc_info=True) ++ printout = salt.output.get_printout('raw', opts)(data) ++ if printout is not None: ++ return printout.rstrip() ++ ++ return salt.output.try_printout(data, output, opts) ++ + def write(self, title, data, output=None): + ''' + Add a data to the current opened section. +@@ -138,7 +163,7 @@ class SupportDataCollector(object): + try: + if isinstance(data, dict) and 'return' in data: + data = data['return'] +- content = salt.output.try_printout(data, output, {'extension_modules': '', 'color': False}) ++ content = self._printout(data, output) + except Exception: # Fall-back to just raw YAML + content = None + else: +@@ -406,7 +431,11 @@ class SaltSupport(salt.utils.parsers.SaltSupportOptionParser): + and self.config.get('support_archive') + and os.path.exists(self.config['support_archive'])): + self.out.warning('Terminated earlier, cleaning up') +- os.unlink(self.config['support_archive']) ++ try: ++ os.unlink(self.config['support_archive']) ++ except Exception as err: ++ log.debug(err) ++ self.out.error('{} while cleaning up.'.format(err)) + + def _check_existing_archive(self): + ''' +@@ -418,7 +447,11 @@ class SaltSupport(salt.utils.parsers.SaltSupportOptionParser): + if os.path.exists(self.config['support_archive']): + if self.config['support_archive_force_overwrite']: + self.out.warning('Overwriting existing archive: {}'.format(self.config['support_archive'])) +- os.unlink(self.config['support_archive']) ++ try: ++ os.unlink(self.config['support_archive']) ++ except Exception as err: ++ log.debug(err) ++ self.out.error('{} while trying to overwrite existing archive.'.format(err)) + ret = True + else: + self.out.warning('File {} already exists.'.format(self.config['support_archive'])) +diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py +index 56a8961c3a..058346a9f4 100644 +--- a/salt/utils/parsers.py ++++ b/salt/utils/parsers.py +@@ -1922,7 +1922,7 @@ class SaltSupportOptionParser(six.with_metaclass(OptionParserMeta, OptionParser, + ''' + _opts, _args = optparse.OptionParser.parse_args(self) + configs = self.find_existing_configs(_opts.support_unit) +- if cfg not in configs: ++ if configs and cfg not in configs: + cfg = configs[0] + + return config.master_config(self.get_config_file_path(cfg)) +-- +2.19.0 + + diff --git a/support-use-of-gce-instance-credentials-109.patch b/support-use-of-gce-instance-credentials-109.patch new file mode 100644 index 0000000..04e80eb --- /dev/null +++ b/support-use-of-gce-instance-credentials-109.patch @@ -0,0 +1,33 @@ +From 4571116a54ff51683cb695ce795f04f8b318b440 Mon Sep 17 00:00:00 2001 +From: jgleissner +Date: Wed, 19 Sep 2018 14:37:12 +0200 +Subject: [PATCH] Support use of GCE instance credentials (#109) + +* Integration of MSI authentication with azurearm cloud driver (#105) + +* allow empty service_account_private_key in GCE driver + +Passing an emoty service_account_private_key to libcloud will enable +authentication using instance credentials, which is used by CaaSP in GCE. +--- + salt/cloud/clouds/gce.py | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/salt/cloud/clouds/gce.py b/salt/cloud/clouds/gce.py +index 75109491be..1018e36ed5 100644 +--- a/salt/cloud/clouds/gce.py ++++ b/salt/cloud/clouds/gce.py +@@ -134,7 +134,8 @@ def __virtual__(): + + parameters = details['gce'] + pathname = os.path.expanduser(parameters['service_account_private_key']) +- if salt.utils.cloud.check_key_path_and_mode( ++ # empty pathname will tell libcloud to use instance credentials ++ if pathname and salt.utils.cloud.check_key_path_and_mode( + provider, pathname + ) is False: + return False +-- +2.17.1 + + diff --git a/update-error-list-for-zypper.patch b/update-error-list-for-zypper.patch new file mode 100644 index 0000000..3987c08 --- /dev/null +++ b/update-error-list-for-zypper.patch @@ -0,0 +1,62 @@ +From 71e7ecfbb07cf14680a2a39de48a6e60cd20cb07 Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Wed, 26 Sep 2018 17:54:53 +0200 +Subject: [PATCH] Update error list for zypper + +Add error logging +--- + salt/modules/zypper.py | 30 ++++++++++++++++++++++++++++-- + 1 file changed, 28 insertions(+), 2 deletions(-) + +diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py +index e4423cf1fc..6845e44ab6 100644 +--- a/salt/modules/zypper.py ++++ b/salt/modules/zypper.py +@@ -75,7 +75,25 @@ class _Zypper(object): + Allows serial zypper calls (first came, first won). + ''' + +- SUCCESS_EXIT_CODES = [0, 100, 101, 102, 103] ++ SUCCESS_EXIT_CODES = { ++ 0: 'Successful run of zypper with no special info.', ++ 100: 'Patches are available for installation.', ++ 101: 'Security patches are available for installation.', ++ 102: 'Installation successful, reboot required.', ++ 103: 'Installation succesful, restart of the package manager itself required.', ++ } ++ ++ WARNING_EXIT_CODES = { ++ 6: 'No repositories are defined.', ++ 7: 'The ZYPP library is locked.', ++ 106: 'Some repository had to be disabled temporarily because it failed to refresh. ' ++ 'You should check your repository configuration (e.g. zypper ref -f).', ++ 107: 'Installation basically succeeded, but some of the packages %post install scripts returned an error. ' ++ 'These packages were successfully unpacked to disk and are registered in the rpm database, ' ++ 'but due to the failed install script they may not work as expected. The failed scripts output might ' ++ 'reveal what actually went wrong. Any scripts output is also logged to /var/log/zypp/history.' ++ } ++ + LOCK_EXIT_CODE = 7 + XML_DIRECTIVES = ['-x', '--xmlout'] + ZYPPER_LOCK = '/var/run/zypp.pid' +@@ -188,7 +206,15 @@ class _Zypper(object): + + :return: + ''' +- return self.exit_code not in self.SUCCESS_EXIT_CODES ++ if self.exit_code: ++ msg = self.SUCCESS_EXIT_CODES.get(self.exit_code) ++ if msg: ++ log.info(msg) ++ msg = self.WARNING_EXIT_CODES.get(self.exit_code) ++ if msg: ++ log.warning(msg) ++ ++ return self.exit_code not in self.SUCCESS_EXIT_CODES and self.exit_code not in self.WARNING_EXIT_CODES + + def _is_lock(self): + ''' +-- +2.19.0 + +