From 08133114a128752878edf92ff72edb218ea612a8823fd3b3810aee2e9763b7fd Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Fri, 13 Oct 2017 13:57:09 +0000 Subject: [PATCH] Accepting request 533951 from systemsmanagement:saltstack:testing - Add possibility to generate _version.py at the build time for raw builds: https://github.com/saltstack/salt/pull/43955 - Added: * enable-with-salt-version-parameter-for-setup.py-scri.patch - Update to 2017.7.2 See https://docs.saltstack.com/en/develop/topics/releases/2017.7.2.html for full changelog - Fix for CVE-2017-14695 (bsc#1062462) - Fix for CVE-2017-14696 (bsc#1062464) - Fix salt target-type field returns "String" for existing jids but an empty "Array" for non existing jids. (issue #1711) - Added: * bugfix-always-return-a-string-list-on-unknown-job-ta.patch - Fixed minion resource exhaustion when many functions are being executed in parallel (bsc#1059758) - Added: * introduce-process_count_max-minion-configuration-par.patch * multiprocessing-minion-option-documentation-fixes.patch - Remove 'TasksTask' attribute from salt-master.service in older versions of systemd (bsc#985112) - Provide custom SUSE salt-master.service file. - Fix wrong version reported by Salt (bsc#1061407) - list_pkgs: add parameter for returned attribute selection (bsc#1052264) - Adding the leftover for zypper and yum list_pkgs functionality. - Use $HOME to get the user home directory instead using '~' char (bsc#1042749) OBS-URL: https://build.opensuse.org/request/show/533951 OBS-URL: https://build.opensuse.org/package/show/systemsmanagement:saltstack/salt?expand=0&rev=96 --- ...turn-a-string-list-on-unknown-job-ta.patch | 69 ++ ...-version-parameter-for-setup.py-scri.patch | 76 ++ ...s_count_max-minion-configuration-par.patch | 211 +++++ ...rameter-for-returned-attribute-selec.patch | 792 ++++++++++++++++++ ...ng-minion-option-documentation-fixes.patch | 47 ++ salt.changes | 57 ++ salt.spec | 38 +- ...the-user-home-directory-instead-usin.patch | 29 + v2017.7.1.tar.gz | 3 - v2017.7.2.tar.gz | 3 + 10 files changed, 1318 insertions(+), 7 deletions(-) create mode 100644 bugfix-always-return-a-string-list-on-unknown-job-ta.patch create mode 100644 enable-with-salt-version-parameter-for-setup.py-scri.patch create mode 100644 introduce-process_count_max-minion-configuration-par.patch create mode 100644 list_pkgs-add-parameter-for-returned-attribute-selec.patch create mode 100644 multiprocessing-minion-option-documentation-fixes.patch create mode 100644 use-home-to-get-the-user-home-directory-instead-usin.patch delete mode 100644 v2017.7.1.tar.gz create mode 100644 v2017.7.2.tar.gz diff --git a/bugfix-always-return-a-string-list-on-unknown-job-ta.patch b/bugfix-always-return-a-string-list-on-unknown-job-ta.patch new file mode 100644 index 0000000..7a54502 --- /dev/null +++ b/bugfix-always-return-a-string-list-on-unknown-job-ta.patch @@ -0,0 +1,69 @@ +From 49a4e807fb1cb844cec7b7c11b37f6c276f899e4 Mon Sep 17 00:00:00 2001 +From: Bo Maryniuk +Date: Mon, 9 Oct 2017 17:57:48 +0200 +Subject: [PATCH] Bugfix: always return a string "list" on unknown job + target type. + +--- + salt/returners/couchbase_return.py | 2 +- + salt/returners/postgres_local_cache.py | 2 +- + salt/runners/jobs.py | 2 +- + salt/utils/jid.py | 2 +- + 4 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/salt/returners/couchbase_return.py b/salt/returners/couchbase_return.py +index 24c3a9105a..f5adecc2e7 100644 +--- a/salt/returners/couchbase_return.py ++++ b/salt/returners/couchbase_return.py +@@ -309,7 +309,7 @@ def _format_job_instance(job): + 'Arguments': list(job.get('arg', [])), + # unlikely but safeguard from invalid returns + 'Target': job.get('tgt', 'unknown-target'), +- 'Target-type': job.get('tgt_type', []), ++ 'Target-type': job.get('tgt_type', 'list'), + 'User': job.get('user', 'root')} + + if 'metadata' in job: +diff --git a/salt/returners/postgres_local_cache.py b/salt/returners/postgres_local_cache.py +index 422f8c77c7..28dc2f565c 100644 +--- a/salt/returners/postgres_local_cache.py ++++ b/salt/returners/postgres_local_cache.py +@@ -180,7 +180,7 @@ def _format_job_instance(job): + 'Arguments': json.loads(job.get('arg', '[]')), + # unlikely but safeguard from invalid returns + 'Target': job.get('tgt', 'unknown-target'), +- 'Target-type': job.get('tgt_type', []), ++ 'Target-type': job.get('tgt_type', 'list'), + 'User': job.get('user', 'root')} + # TODO: Add Metadata support when it is merged from develop + return ret +diff --git a/salt/runners/jobs.py b/salt/runners/jobs.py +index 82abd56eae..fae7942e38 100644 +--- a/salt/runners/jobs.py ++++ b/salt/runners/jobs.py +@@ -542,7 +542,7 @@ def _format_job_instance(job): + 'Arguments': list(job.get('arg', [])), + # unlikely but safeguard from invalid returns + 'Target': job.get('tgt', 'unknown-target'), +- 'Target-type': job.get('tgt_type', []), ++ 'Target-type': job.get('tgt_type', 'list'), + 'User': job.get('user', 'root')} + + if 'metadata' in job: +diff --git a/salt/utils/jid.py b/salt/utils/jid.py +index 3f4ef296a2..4dbf0d2c6f 100644 +--- a/salt/utils/jid.py ++++ b/salt/utils/jid.py +@@ -65,7 +65,7 @@ def format_job_instance(job): + 'Arguments': list(job.get('arg', [])), + # unlikely but safeguard from invalid returns + 'Target': job.get('tgt', 'unknown-target'), +- 'Target-type': job.get('tgt_type', []), ++ 'Target-type': job.get('tgt_type', 'list'), + 'User': job.get('user', 'root')} + + if 'metadata' in job: +-- +2.14.2 + + diff --git a/enable-with-salt-version-parameter-for-setup.py-scri.patch b/enable-with-salt-version-parameter-for-setup.py-scri.patch new file mode 100644 index 0000000..f4275cb --- /dev/null +++ b/enable-with-salt-version-parameter-for-setup.py-scri.patch @@ -0,0 +1,76 @@ +From 1949261a504fd01e057b41126d78f142f4977204 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Fri, 6 Oct 2017 17:12:15 +0100 +Subject: [PATCH] Enable '--with-salt-version' parameter for setup.py + script + +--- + setup.py | 20 ++++++++++++++++++-- + 1 file changed, 18 insertions(+), 2 deletions(-) + +diff --git a/setup.py b/setup.py +index effdc2f230..519f753401 100755 +--- a/setup.py ++++ b/setup.py +@@ -183,17 +183,22 @@ class WriteSaltVersion(Command): + ''' + + def run(self): +- if not os.path.exists(SALT_VERSION_HARDCODED): ++ if not os.path.exists(SALT_VERSION_HARDCODED) or self.distribution.with_salt_version: + # Write the version file + if getattr(self.distribution, 'salt_version_hardcoded_path', None) is None: + print('This command is not meant to be called on it\'s own') + exit(1) + ++ if not self.distribution.with_salt_version: ++ salt_version = __saltstack_version__ ++ else: ++ salt_version = SaltStackVersion.parse(self.distribution.with_salt_version) ++ + # pylint: disable=E0602 + open(self.distribution.salt_version_hardcoded_path, 'w').write( + INSTALL_VERSION_TEMPLATE.format( + date=DATE, +- full_version_info=__saltstack_version__.full_info ++ full_version_info=salt_version.full_info + ) + ) + # pylint: enable=E0602 +@@ -731,6 +736,13 @@ class Build(build): + def run(self): + # Run build.run function + build.run(self) ++ if getattr(self.distribution, 'with_salt_version', False): ++ # Write the hardcoded salt version module salt/_version.py ++ self.distribution.salt_version_hardcoded_path = os.path.join( ++ self.build_lib, 'salt', '_version.py' ++ ) ++ self.run_command('write_salt_version') ++ + if getattr(self.distribution, 'running_salt_install', False): + # If our install attribute is present and set to True, we'll go + # ahead and write our install time python modules. +@@ -839,6 +851,7 @@ class SaltDistribution(distutils.dist.Distribution): + ('ssh-packaging', None, 'Run in SSH packaging mode'), + ('salt-transport=', None, 'The transport to prepare salt for. Choices are \'zeromq\' ' + '\'raet\' or \'both\'. Defaults to \'zeromq\'', 'zeromq')] + [ ++ ('with-salt-version=', None, 'Set a fixed version for Salt instead calculating it'), + # Salt's Paths Configuration Settings + ('salt-root-dir=', None, + 'Salt\'s pre-configured root directory'), +@@ -893,6 +906,9 @@ class SaltDistribution(distutils.dist.Distribution): + self.salt_spm_pillar_dir = None + self.salt_spm_reactor_dir = None + ++ # Salt version ++ self.with_salt_version = None ++ + self.name = 'salt-ssh' if PACKAGED_FOR_SALT_SSH else 'salt' + self.salt_version = __version__ # pylint: disable=undefined-variable + self.description = 'Portable, distributed, remote execution and configuration management system' +-- +2.14.2 + + diff --git a/introduce-process_count_max-minion-configuration-par.patch b/introduce-process_count_max-minion-configuration-par.patch new file mode 100644 index 0000000..5cf33c5 --- /dev/null +++ b/introduce-process_count_max-minion-configuration-par.patch @@ -0,0 +1,211 @@ +From dd3490c7e56a0aca17f73dde1684d469fe5582df Mon Sep 17 00:00:00 2001 +From: Silvio Moioli +Date: Wed, 20 Sep 2017 14:33:33 +0200 +Subject: [PATCH] Introduce process_count_max minion configuration + parameter + +This allows users to limit the number of processes or threads a minion +will start in response to published messages, prevents resource +exhaustion in case a high number of concurrent jobs is scheduled in a +short time. + +process_count_max: add defaults and documentation + +process_count_max: adapt existing unit tests + +process_count_max: add unit test + +process_count_max: disable by default +--- + conf/minion | 6 +++++ + doc/ref/configuration/minion.rst | 17 +++++++++++++ + salt/config/__init__.py | 4 +++ + salt/minion.py | 10 ++++++++ + tests/unit/test_minion.py | 53 +++++++++++++++++++++++++++++++++++++--- + 5 files changed, 87 insertions(+), 3 deletions(-) + +diff --git a/conf/minion b/conf/minion +index b1122c9e52..6d2b43da1b 100644 +--- a/conf/minion ++++ b/conf/minion +@@ -671,6 +671,12 @@ + # for a full explanation. + #multiprocessing: True + ++# Limit the maximum amount of processes or threads created by salt-minion. ++# This is useful to avoid resource exhaustion in case the minion receives more ++# publications than it is able to handle, as it limits the number of spawned ++# processes or threads. -1 is the default and disables the limit. ++#process_count_max: -1 ++ + + ##### Logging settings ##### + ########################################## +diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst +index 5cc72f1daf..b935f86656 100644 +--- a/doc/ref/configuration/minion.rst ++++ b/doc/ref/configuration/minion.rst +@@ -2179,6 +2179,23 @@ executed in a thread. + + multiprocessing: True + ++.. conf_minion:: process_count_max ++ ++``process_count_max`` ++------- ++ ++.. versionadded:: Oxygen ++ ++Default: ``-1`` ++ ++Limit the maximum amount of processes or threads created by ``salt-minion``. ++This is useful to avoid resource exhaustion in case the minion receives more ++publications than it is able to handle, as it limits the number of spawned ++processes or threads. ``-1`` is the default and disables the limit. ++ ++.. code-block:: yaml ++ ++ process_count_max: -1 + + .. _minion-logging-settings: + +diff --git a/salt/config/__init__.py b/salt/config/__init__.py +index b5b7f2a1f7..e843d8c79d 100644 +--- a/salt/config/__init__.py ++++ b/salt/config/__init__.py +@@ -328,6 +328,9 @@ VALID_OPTS = { + # Whether or not processes should be forked when needed. The alternative is to use threading. + 'multiprocessing': bool, + ++ # Maximum number of concurrently active processes at any given point in time ++ 'process_count_max': int, ++ + # Whether or not the salt minion should run scheduled mine updates + 'mine_enabled': bool, + +@@ -1193,6 +1196,7 @@ DEFAULT_MINION_OPTS = { + 'auto_accept': True, + 'autosign_timeout': 120, + 'multiprocessing': True, ++ 'process_count_max': -1, + 'mine_enabled': True, + 'mine_return_job': False, + 'mine_interval': 60, +diff --git a/salt/minion.py b/salt/minion.py +index 3c5046ee93..9e7301aafa 100644 +--- a/salt/minion.py ++++ b/salt/minion.py +@@ -1288,6 +1288,7 @@ class Minion(MinionBase): + self._send_req_async(load, timeout, callback=lambda f: None) # pylint: disable=unexpected-keyword-arg + return True + ++ @tornado.gen.coroutine + def _handle_decoded_payload(self, data): + ''' + Override this method if you wish to handle the decoded data +@@ -1319,6 +1320,15 @@ class Minion(MinionBase): + self.functions, self.returners, self.function_errors, self.executors = self._load_modules() + self.schedule.functions = self.functions + self.schedule.returners = self.returners ++ ++ process_count_max = self.opts.get('process_count_max') ++ if process_count_max > 0: ++ process_count = len(salt.utils.minion.running(self.opts)) ++ while process_count >= process_count_max: ++ log.warn("Maximum number of processes reached while executing jid {0}, waiting...".format(data['jid'])) ++ yield tornado.gen.sleep(10) ++ process_count = len(salt.utils.minion.running(self.opts)) ++ + # We stash an instance references to allow for the socket + # communication in Windows. You can't pickle functions, and thus + # python needs to be able to reconstruct the reference on the other +diff --git a/tests/unit/test_minion.py b/tests/unit/test_minion.py +index 535dfeedfc..6c9dca13cd 100644 +--- a/tests/unit/test_minion.py ++++ b/tests/unit/test_minion.py +@@ -18,6 +18,7 @@ from salt.utils import event + from salt.exceptions import SaltSystemExit + import salt.syspaths + import tornado ++from salt.ext.six.moves import range + + __opts__ = {} + +@@ -69,7 +70,7 @@ class MinionTestCase(TestCase): + mock_jid_queue = [123] + try: + minion = salt.minion.Minion(mock_opts, jid_queue=copy.copy(mock_jid_queue), io_loop=tornado.ioloop.IOLoop()) +- ret = minion._handle_decoded_payload(mock_data) ++ ret = minion._handle_decoded_payload(mock_data).result() + self.assertEqual(minion.jid_queue, mock_jid_queue) + self.assertIsNone(ret) + finally: +@@ -98,7 +99,7 @@ class MinionTestCase(TestCase): + # Call the _handle_decoded_payload function and update the mock_jid_queue to include the new + # mock_jid. The mock_jid should have been added to the jid_queue since the mock_jid wasn't + # previously included. The minion's jid_queue attribute and the mock_jid_queue should be equal. +- minion._handle_decoded_payload(mock_data) ++ minion._handle_decoded_payload(mock_data).result() + mock_jid_queue.append(mock_jid) + self.assertEqual(minion.jid_queue, mock_jid_queue) + finally: +@@ -126,8 +127,54 @@ class MinionTestCase(TestCase): + + # Call the _handle_decoded_payload function and check that the queue is smaller by one item + # and contains the new jid +- minion._handle_decoded_payload(mock_data) ++ minion._handle_decoded_payload(mock_data).result() + self.assertEqual(len(minion.jid_queue), 2) + self.assertEqual(minion.jid_queue, [456, 789]) + finally: + minion.destroy() ++ ++ def test_process_count_max(self): ++ ''' ++ Tests that the _handle_decoded_payload function does not spawn more than the configured amount of processes, ++ as per process_count_max. ++ ''' ++ with patch('salt.minion.Minion.ctx', MagicMock(return_value={})), \ ++ patch('salt.utils.process.SignalHandlingMultiprocessingProcess.start', MagicMock(return_value=True)), \ ++ patch('salt.utils.process.SignalHandlingMultiprocessingProcess.join', MagicMock(return_value=True)), \ ++ patch('salt.utils.minion.running', MagicMock(return_value=[])), \ ++ patch('tornado.gen.sleep', MagicMock(return_value=tornado.concurrent.Future())): ++ process_count_max = 10 ++ mock_opts = salt.config.DEFAULT_MINION_OPTS ++ mock_opts['minion_jid_queue_hwm'] = 100 ++ mock_opts["process_count_max"] = process_count_max ++ ++ try: ++ io_loop = tornado.ioloop.IOLoop() ++ minion = salt.minion.Minion(mock_opts, jid_queue=[], io_loop=io_loop) ++ ++ # mock gen.sleep to throw a special Exception when called, so that we detect it ++ class SleepCalledEception(Exception): ++ """Thrown when sleep is called""" ++ pass ++ tornado.gen.sleep.return_value.set_exception(SleepCalledEception()) ++ ++ # up until process_count_max: gen.sleep does not get called, processes are started normally ++ for i in range(process_count_max): ++ mock_data = {'fun': 'foo.bar', ++ 'jid': i} ++ io_loop.run_sync(lambda data=mock_data: minion._handle_decoded_payload(data)) ++ self.assertEqual(salt.utils.process.SignalHandlingMultiprocessingProcess.start.call_count, i + 1) ++ self.assertEqual(len(minion.jid_queue), i + 1) ++ salt.utils.minion.running.return_value += [i] ++ ++ # above process_count_max: gen.sleep does get called, JIDs are created but no new processes are started ++ mock_data = {'fun': 'foo.bar', ++ 'jid': process_count_max + 1} ++ ++ self.assertRaises(SleepCalledEception, ++ lambda: io_loop.run_sync(lambda: minion._handle_decoded_payload(mock_data))) ++ self.assertEqual(salt.utils.process.SignalHandlingMultiprocessingProcess.start.call_count, ++ process_count_max) ++ self.assertEqual(len(minion.jid_queue), process_count_max + 1) ++ finally: ++ minion.destroy() +-- +2.14.2 + + diff --git a/list_pkgs-add-parameter-for-returned-attribute-selec.patch b/list_pkgs-add-parameter-for-returned-attribute-selec.patch new file mode 100644 index 0000000..ac0307a --- /dev/null +++ b/list_pkgs-add-parameter-for-returned-attribute-selec.patch @@ -0,0 +1,792 @@ +From 2996d5ab680b2b3c243a2216e13326750bc37ac8 Mon Sep 17 00:00:00 2001 +From: Silvio Moioli +Date: Thu, 13 Jul 2017 15:59:01 +0200 +Subject: [PATCH] list_pkgs: add parameter for returned attribute selection + (bsc#1052264) + +zypper.list_pkgs: + * It adds a new optional parameter to list_pkg in the zypper module to return more data than the version (original reason is that for SUSE Manager integration we also need arch and install_date). Format is the same of existing method info_installed. + +yumpkg.list_pkgs: + * It adds a new optional parameter to list_pkg, originally added to the the zypper module via PR #42310, to yumpkg providing the same functionality and interface to the yum package manager. +--- + salt/modules/pkg_resource.py | 42 +++++++++ + salt/modules/yumpkg.py | 102 +++++++++++++++++----- + salt/modules/zypper.py | 94 ++++++++++++++------ + salt/utils/pkg/rpm.py | 22 +++-- + tests/unit/modules/test_yumpkg.py | 174 ++++++++++++++++++++++++++++++++++++++ + tests/unit/modules/test_zypper.py | 105 ++++++++++++++++++----- + 6 files changed, 466 insertions(+), 73 deletions(-) + create mode 100644 tests/unit/modules/test_yumpkg.py + +diff --git a/salt/modules/pkg_resource.py b/salt/modules/pkg_resource.py +index 928dccae7d..a9f396b212 100644 +--- a/salt/modules/pkg_resource.py ++++ b/salt/modules/pkg_resource.py +@@ -5,6 +5,7 @@ Resources needed by pkg providers + + # Import python libs + from __future__ import absolute_import ++import copy + import fnmatch + import logging + import os +@@ -306,3 +307,44 @@ def check_extra_requirements(pkgname, pkgver): + return __salt__['pkg.check_extra_requirements'](pkgname, pkgver) + + return True ++ ++ ++def format_pkg_list(packages, versions_as_list, attr): ++ ''' ++ Formats packages according to parameters for list_pkgs. ++ ''' ++ ret = copy.deepcopy(packages) ++ if attr: ++ requested_attr = set(['epoch', 'version', 'release', 'arch', ++ 'install_date', 'install_date_time_t']) ++ ++ if attr != 'all': ++ requested_attr &= set(attr + ['version']) ++ ++ for name in ret: ++ versions = [] ++ for all_attr in ret[name]: ++ filtered_attr = {} ++ for key in requested_attr: ++ if all_attr[key]: ++ filtered_attr[key] = all_attr[key] ++ versions.append(filtered_attr) ++ ret[name] = versions ++ return ret ++ ++ for name in ret: ++ ret[name] = [format_version(d['epoch'], d['version'], d['release']) ++ for d in ret[name]] ++ if not versions_as_list: ++ stringify(ret) ++ return ret ++ ++ ++def format_version(epoch, version, release): ++ ''' ++ Formats a version string for list_pkgs. ++ ''' ++ full_version = '{0}:{1}'.format(epoch, version) if epoch else version ++ if release: ++ full_version += '-{0}'.format(release) ++ return full_version +diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py +index e659d1b79c..8ffc63a74e 100644 +--- a/salt/modules/yumpkg.py ++++ b/salt/modules/yumpkg.py +@@ -17,7 +17,6 @@ Support for YUM/DNF + # Import python libs + from __future__ import absolute_import + import contextlib +-import copy + import datetime + import fnmatch + import itertools +@@ -584,15 +583,35 @@ def version_cmp(pkg1, pkg2, ignore_epoch=False): + + def list_pkgs(versions_as_list=False, **kwargs): + ''' +- List the packages currently installed in a dict:: ++ List the packages currently installed as a dict. By default, the dict ++ contains versions as a comma separated string:: + +- {'': ''} ++ {'': '[,...]'} ++ ++ versions_as_list: ++ If set to true, the versions are provided as a list ++ ++ {'': ['', '']} ++ ++ attr: ++ If a list of package attributes is specified, returned value will ++ contain them in addition to version, eg.:: ++ ++ {'': [{'version' : 'version', 'arch' : 'arch'}]} ++ ++ Valid attributes are: ``epoch``, ``version``, ``release``, ``arch``, ++ ``install_date``, ``install_date_time_t``. ++ ++ If ``all`` is specified, all valid attributes will be returned. ++ ++ .. versionadded:: Oxygen + + CLI Example: + + .. code-block:: bash + + salt '*' pkg.list_pkgs ++ salt '*' pkg.list_pkgs attr='["version", "arch"]' + ''' + versions_as_list = salt.utils.is_true(versions_as_list) + # not yet implemented or not applicable +@@ -600,17 +619,14 @@ def list_pkgs(versions_as_list=False, **kwargs): + for x in ('removed', 'purge_desired')]): + return {} + ++ attr = kwargs.get("attr") + if 'pkg.list_pkgs' in __context__: +- if versions_as_list: +- return __context__['pkg.list_pkgs'] +- else: +- ret = copy.deepcopy(__context__['pkg.list_pkgs']) +- __salt__['pkg_resource.stringify'](ret) +- return ret ++ cached = __context__['pkg.list_pkgs'] ++ return __salt__['pkg_resource.format_pkg_list'](cached, versions_as_list, attr) + + ret = {} + cmd = ['rpm', '-qa', '--queryformat', +- salt.utils.pkg.rpm.QUERYFORMAT.replace('%{REPOID}', '(none)\n')] ++ salt.utils.pkg.rpm.QUERYFORMAT.replace('%{REPOID}', '(none)') + '\n'] + output = __salt__['cmd.run'](cmd, + python_shell=False, + output_loglevel='trace') +@@ -620,15 +636,25 @@ def list_pkgs(versions_as_list=False, **kwargs): + osarch=__grains__['osarch'] + ) + if pkginfo is not None: +- __salt__['pkg_resource.add_pkg'](ret, +- pkginfo.name, +- pkginfo.version) +- +- __salt__['pkg_resource.sort_pkglist'](ret) +- __context__['pkg.list_pkgs'] = copy.deepcopy(ret) +- if not versions_as_list: +- __salt__['pkg_resource.stringify'](ret) +- return ret ++ # see rpm version string rules available at https://goo.gl/UGKPNd ++ pkgver = pkginfo.version ++ epoch = '' ++ release = '' ++ if ':' in pkgver: ++ epoch, pkgver = pkgver.split(":", 1) ++ if '-' in pkgver: ++ pkgver, release = pkgver.split("-", 1) ++ all_attr = {'epoch': epoch, 'version': pkgver, 'release': release, ++ 'arch': pkginfo.arch, 'install_date': pkginfo.install_date, ++ 'install_date_time_t': pkginfo.install_date_time_t} ++ __salt__['pkg_resource.add_pkg'](ret, pkginfo.name, all_attr) ++ ++ for pkgname in ret: ++ ret[pkgname] = sorted(ret[pkgname], key=lambda d: d['version']) ++ ++ __context__['pkg.list_pkgs'] = ret ++ ++ return __salt__['pkg_resource.format_pkg_list'](ret, versions_as_list, attr) + + + def list_repo_pkgs(*args, **kwargs): +@@ -1205,11 +1231,42 @@ def install(name=None, + + .. versionadded:: 2014.7.0 + ++ diff_attr: ++ If a list of package attributes is specified, returned value will ++ contain them, eg.:: ++ ++ {'': { ++ 'old': { ++ 'version': '', ++ 'arch': ''}, ++ ++ 'new': { ++ 'version': '', ++ 'arch': ''}}} ++ ++ Valid attributes are: ``epoch``, ``version``, ``release``, ``arch``, ++ ``install_date``, ``install_date_time_t``. ++ ++ If ``all`` is specified, all valid attributes will be returned. ++ ++ .. versionadded:: Oxygen + + Returns a dict containing the new package names and versions:: + + {'': {'old': '', + 'new': ''}} ++ ++ If an attribute list in diff_attr is specified, the dict will also contain ++ any specified attribute, eg.:: ++ ++ {'': { ++ 'old': { ++ 'version': '', ++ 'arch': ''}, ++ ++ 'new': { ++ 'version': '', ++ 'arch': ''}}} + ''' + repo_arg = _get_repo_options(**kwargs) + exclude_arg = _get_excludes_option(**kwargs) +@@ -1238,10 +1295,11 @@ def install(name=None, + log.warning('"version" parameter will be ignored for multiple ' + 'package targets') + +- old = list_pkgs(versions_as_list=False) if not downloadonly else list_downloaded() ++ diff_attr = kwargs.get("diff_attr") ++ old = list_pkgs(versions_as_list=False, attr=diff_attr) if not downloadonly else list_downloaded() + # Use of __context__ means no duplicate work here, just accessing + # information already in __context__ from the previous call to list_pkgs() +- old_as_list = list_pkgs(versions_as_list=True) if not downloadonly else list_downloaded() ++ old_as_list = list_pkgs(versions_as_list=True, attr=diff_attr) if not downloadonly else list_downloaded() + + to_install = [] + to_downgrade = [] +@@ -1544,7 +1602,7 @@ def install(name=None, + errors.append(out['stdout']) + + __context__.pop('pkg.list_pkgs', None) +- new = list_pkgs(versions_as_list=False) if not downloadonly else list_downloaded() ++ new = list_pkgs(versions_as_list=False, attr=diff_attr) if not downloadonly else list_downloaded() + + ret = salt.utils.compare_dicts(old, new) + +diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py +index 4ede437c30..b440af08a4 100644 +--- a/salt/modules/zypper.py ++++ b/salt/modules/zypper.py +@@ -14,7 +14,6 @@ Package support for openSUSE via the zypper package manager + + # Import python libs + from __future__ import absolute_import +-import copy + import fnmatch + import logging + import re +@@ -652,8 +651,8 @@ def version_cmp(ver1, ver2, ignore_epoch=False): + + def list_pkgs(versions_as_list=False, **kwargs): + ''' +- List the packages currently installed as a dict with versions +- as a comma separated string:: ++ List the packages currently installed as a dict. By default, the dict ++ contains versions as a comma separated string:: + + {'': '[,...]'} + +@@ -662,6 +661,19 @@ def list_pkgs(versions_as_list=False, **kwargs): + + {'': ['', '']} + ++ attr: ++ If a list of package attributes is specified, returned value will ++ contain them in addition to version, eg.:: ++ ++ {'': [{'version' : 'version', 'arch' : 'arch'}]} ++ ++ Valid attributes are: ``epoch``, ``version``, ``release``, ``arch``, ++ ``install_date``, ``install_date_time_t``. ++ ++ If ``all`` is specified, all valid attributes will be returned. ++ ++ .. versionadded:: Oxygen ++ + removed: + not supported + +@@ -673,6 +685,7 @@ def list_pkgs(versions_as_list=False, **kwargs): + .. code-block:: bash + + salt '*' pkg.list_pkgs ++ salt '*' pkg.list_pkgs attr='["version", "arch"]' + ''' + versions_as_list = salt.utils.is_true(versions_as_list) + # not yet implemented or not applicable +@@ -680,30 +693,30 @@ def list_pkgs(versions_as_list=False, **kwargs): + for x in ('removed', 'purge_desired')]): + return {} + ++ attr = kwargs.get("attr") + if 'pkg.list_pkgs' in __context__: +- if versions_as_list: +- return __context__['pkg.list_pkgs'] +- else: +- ret = copy.deepcopy(__context__['pkg.list_pkgs']) +- __salt__['pkg_resource.stringify'](ret) +- return ret ++ cached = __context__['pkg.list_pkgs'] ++ return __salt__['pkg_resource.format_pkg_list'](cached, versions_as_list, attr) + +- cmd = ['rpm', '-qa', '--queryformat', '%{NAME}_|-%{VERSION}_|-%{RELEASE}_|-%|EPOCH?{%{EPOCH}}:{}|\\n'] ++ cmd = ['rpm', '-qa', '--queryformat', ( ++ "%{NAME}_|-%{VERSION}_|-%{RELEASE}_|-%{ARCH}_|-" ++ "%|EPOCH?{%{EPOCH}}:{}|_|-%{INSTALLTIME}\\n")] + ret = {} + for line in __salt__['cmd.run'](cmd, output_loglevel='trace', python_shell=False).splitlines(): +- name, pkgver, rel, epoch = line.split('_|-') +- if epoch: +- pkgver = '{0}:{1}'.format(epoch, pkgver) +- if rel: +- pkgver += '-{0}'.format(rel) +- __salt__['pkg_resource.add_pkg'](ret, name, pkgver) +- +- __salt__['pkg_resource.sort_pkglist'](ret) +- __context__['pkg.list_pkgs'] = copy.deepcopy(ret) +- if not versions_as_list: +- __salt__['pkg_resource.stringify'](ret) ++ name, pkgver, rel, arch, epoch, install_time = line.split('_|-') ++ install_date = datetime.datetime.utcfromtimestamp(int(install_time)).isoformat() + "Z" ++ install_date_time_t = int(install_time) + +- return ret ++ all_attr = {'epoch': epoch, 'version': pkgver, 'release': rel, 'arch': arch, ++ 'install_date': install_date, 'install_date_time_t': install_date_time_t} ++ __salt__['pkg_resource.add_pkg'](ret, name, all_attr) ++ ++ for pkgname in ret: ++ ret[pkgname] = sorted(ret[pkgname], key=lambda d: d['version']) ++ ++ __context__['pkg.list_pkgs'] = ret ++ ++ return __salt__['pkg_resource.format_pkg_list'](ret, versions_as_list, attr) + + + def _get_configured_repos(): +@@ -1069,11 +1082,43 @@ def install(name=None, + Zypper returns error code 106 if one of the repositories are not available for various reasons. + In case to set strict check, this parameter needs to be set to True. Default: False. + ++ diff_attr: ++ If a list of package attributes is specified, returned value will ++ contain them, eg.:: ++ ++ {'': { ++ 'old': { ++ 'version': '', ++ 'arch': ''}, ++ ++ 'new': { ++ 'version': '', ++ 'arch': ''}}} ++ ++ Valid attributes are: ``epoch``, ``version``, ``release``, ``arch``, ++ ``install_date``, ``install_date_time_t``. ++ ++ If ``all`` is specified, all valid attributes will be returned. ++ ++ .. versionadded:: Oxygen ++ + + Returns a dict containing the new package names and versions:: + + {'': {'old': '', + 'new': ''}} ++ ++ If an attribute list is specified in ``diff_attr``, the dict will also contain ++ any specified attribute, eg.:: ++ ++ {'': { ++ 'old': { ++ 'version': '', ++ 'arch': ''}, ++ ++ 'new': { ++ 'version': '', ++ 'arch': ''}}} + ''' + if refresh: + refresh_db() +@@ -1117,7 +1162,8 @@ def install(name=None, + else: + targets = pkg_params + +- old = list_pkgs() if not downloadonly else list_downloaded() ++ diff_attr = kwargs.get("diff_attr") ++ old = list_pkgs(attr=diff_attr) if not downloadonly else list_downloaded() + downgrades = [] + if fromrepo: + fromrepoopt = ['--force', '--force-resolution', '--from', fromrepo] +@@ -1155,7 +1201,7 @@ def install(name=None, + __zypper__(no_repo_failure=ignore_repo_failure).call(*cmd) + + __context__.pop('pkg.list_pkgs', None) +- new = list_pkgs() if not downloadonly else list_downloaded() ++ new = list_pkgs(attr=diff_attr) if not downloadonly else list_downloaded() + + # Handle packages which report multiple new versions + # (affects only kernel packages at this point) +diff --git a/salt/utils/pkg/rpm.py b/salt/utils/pkg/rpm.py +index 0d5c21a82f..7ac7db6316 100644 +--- a/salt/utils/pkg/rpm.py ++++ b/salt/utils/pkg/rpm.py +@@ -6,6 +6,7 @@ Common functions for working with RPM packages + # Import python libs + from __future__ import absolute_import + import collections ++import datetime + import logging + import subprocess + +@@ -34,7 +35,7 @@ ARCHES = ARCHES_64 + ARCHES_32 + ARCHES_PPC + ARCHES_S390 + \ + ARCHES_ALPHA + ARCHES_ARM + ARCHES_SH + + # EPOCHNUM can't be used until RHEL5 is EOL as it is not present +-QUERYFORMAT = '%{NAME}_|-%{EPOCH}_|-%{VERSION}_|-%{RELEASE}_|-%{ARCH}_|-%{REPOID}' ++QUERYFORMAT = '%{NAME}_|-%{EPOCH}_|-%{VERSION}_|-%{RELEASE}_|-%{ARCH}_|-%{REPOID}_|-%{INSTALLTIME}' + + + def get_osarch(): +@@ -59,15 +60,17 @@ def check_32(arch, osarch=None): + return all(x in ARCHES_32 for x in (osarch, arch)) + + +-def pkginfo(name, version, arch, repoid): ++def pkginfo(name, version, arch, repoid, install_date=None, install_date_time_t=None): + ''' + Build and return a pkginfo namedtuple + ''' + pkginfo_tuple = collections.namedtuple( + 'PkgInfo', +- ('name', 'version', 'arch', 'repoid') ++ ('name', 'version', 'arch', 'repoid', 'install_date', ++ 'install_date_time_t') + ) +- return pkginfo_tuple(name, version, arch, repoid) ++ return pkginfo_tuple(name, version, arch, repoid, install_date, ++ install_date_time_t) + + + def resolve_name(name, arch, osarch=None): +@@ -89,7 +92,7 @@ def parse_pkginfo(line, osarch=None): + pkginfo namedtuple. + ''' + try: +- name, epoch, version, release, arch, repoid = line.split('_|-') ++ name, epoch, version, release, arch, repoid, install_time = line.split('_|-') + # Handle unpack errors (should never happen with the queryformat we are + # using, but can't hurt to be careful). + except ValueError: +@@ -101,7 +104,14 @@ def parse_pkginfo(line, osarch=None): + if epoch not in ('(none)', '0'): + version = ':'.join((epoch, version)) + +- return pkginfo(name, version, arch, repoid) ++ if install_time not in ('(none)', '0'): ++ install_date = datetime.datetime.utcfromtimestamp(int(install_time)).isoformat() + "Z" ++ install_date_time_t = int(install_time) ++ else: ++ install_date = None ++ install_date_time_t = None ++ ++ return pkginfo(name, version, arch, repoid, install_date, install_date_time_t) + + + def combine_comments(comments): +diff --git a/tests/unit/modules/test_yumpkg.py b/tests/unit/modules/test_yumpkg.py +new file mode 100644 +index 0000000000..cf754d6289 +--- /dev/null ++++ b/tests/unit/modules/test_yumpkg.py +@@ -0,0 +1,174 @@ ++# -*- coding: utf-8 -*- ++ ++# Import Python Libs ++from __future__ import absolute_import ++import os ++ ++# Import Salt Testing Libs ++from tests.support.mixins import LoaderModuleMockMixin ++from tests.support.unit import TestCase, skipIf ++from tests.support.mock import ( ++ MagicMock, ++ patch, ++ NO_MOCK, ++ NO_MOCK_REASON ++) ++ ++# Import Salt libs ++import salt.modules.yumpkg as yumpkg ++import salt.modules.pkg_resource as pkg_resource ++ ++ ++@skipIf(NO_MOCK, NO_MOCK_REASON) ++class YumTestCase(TestCase, LoaderModuleMockMixin): ++ ''' ++ Test cases for salt.modules.yumpkg ++ ''' ++ def setup_loader_modules(self): ++ return {yumpkg: {'rpm': None}} ++ ++ def test_list_pkgs(self): ++ ''' ++ Test packages listing. ++ ++ :return: ++ ''' ++ def _add_data(data, key, value): ++ data.setdefault(key, []).append(value) ++ ++ rpm_out = [ ++ 'python-urlgrabber_|-(none)_|-3.10_|-8.el7_|-noarch_|-(none)_|-1487838471', ++ 'alsa-lib_|-(none)_|-1.1.1_|-1.el7_|-x86_64_|-(none)_|-1487838475', ++ 'gnupg2_|-(none)_|-2.0.22_|-4.el7_|-x86_64_|-(none)_|-1487838477', ++ 'rpm-python_|-(none)_|-4.11.3_|-21.el7_|-x86_64_|-(none)_|-1487838477', ++ 'pygpgme_|-(none)_|-0.3_|-9.el7_|-x86_64_|-(none)_|-1487838478', ++ 'yum_|-(none)_|-3.4.3_|-150.el7.centos_|-noarch_|-(none)_|-1487838479', ++ 'lzo_|-(none)_|-2.06_|-8.el7_|-x86_64_|-(none)_|-1487838479', ++ 'qrencode-libs_|-(none)_|-3.4.1_|-3.el7_|-x86_64_|-(none)_|-1487838480', ++ 'ustr_|-(none)_|-1.0.4_|-16.el7_|-x86_64_|-(none)_|-1487838480', ++ 'shadow-utils_|-2_|-4.1.5.1_|-24.el7_|-x86_64_|-(none)_|-1487838481', ++ 'util-linux_|-(none)_|-2.23.2_|-33.el7_|-x86_64_|-(none)_|-1487838484', ++ 'openssh_|-(none)_|-6.6.1p1_|-33.el7_3_|-x86_64_|-(none)_|-1487838485', ++ 'virt-what_|-(none)_|-1.13_|-8.el7_|-x86_64_|-(none)_|-1487838486', ++ ] ++ with patch.dict(yumpkg.__grains__, {'osarch': 'x86_64'}), \ ++ patch.dict(yumpkg.__salt__, {'cmd.run': MagicMock(return_value=os.linesep.join(rpm_out))}), \ ++ patch.dict(yumpkg.__salt__, {'pkg_resource.add_pkg': _add_data}), \ ++ patch.dict(yumpkg.__salt__, {'pkg_resource.format_pkg_list': pkg_resource.format_pkg_list}), \ ++ patch.dict(yumpkg.__salt__, {'pkg_resource.stringify': MagicMock()}): ++ pkgs = yumpkg.list_pkgs(versions_as_list=True) ++ for pkg_name, pkg_version in { ++ 'python-urlgrabber': '3.10-8.el7', ++ 'alsa-lib': '1.1.1-1.el7', ++ 'gnupg2': '2.0.22-4.el7', ++ 'rpm-python': '4.11.3-21.el7', ++ 'pygpgme': '0.3-9.el7', ++ 'yum': '3.4.3-150.el7.centos', ++ 'lzo': '2.06-8.el7', ++ 'qrencode-libs': '3.4.1-3.el7', ++ 'ustr': '1.0.4-16.el7', ++ 'shadow-utils': '2:4.1.5.1-24.el7', ++ 'util-linux': '2.23.2-33.el7', ++ 'openssh': '6.6.1p1-33.el7_3', ++ 'virt-what': '1.13-8.el7'}.items(): ++ self.assertTrue(pkgs.get(pkg_name)) ++ self.assertEqual(pkgs[pkg_name], [pkg_version]) ++ ++ def test_list_pkgs_with_attr(self): ++ ''' ++ Test packages listing with the attr parameter ++ ++ :return: ++ ''' ++ def _add_data(data, key, value): ++ data.setdefault(key, []).append(value) ++ ++ rpm_out = [ ++ 'python-urlgrabber_|-(none)_|-3.10_|-8.el7_|-noarch_|-(none)_|-1487838471', ++ 'alsa-lib_|-(none)_|-1.1.1_|-1.el7_|-x86_64_|-(none)_|-1487838475', ++ 'gnupg2_|-(none)_|-2.0.22_|-4.el7_|-x86_64_|-(none)_|-1487838477', ++ 'rpm-python_|-(none)_|-4.11.3_|-21.el7_|-x86_64_|-(none)_|-1487838477', ++ 'pygpgme_|-(none)_|-0.3_|-9.el7_|-x86_64_|-(none)_|-1487838478', ++ 'yum_|-(none)_|-3.4.3_|-150.el7.centos_|-noarch_|-(none)_|-1487838479', ++ 'lzo_|-(none)_|-2.06_|-8.el7_|-x86_64_|-(none)_|-1487838479', ++ 'qrencode-libs_|-(none)_|-3.4.1_|-3.el7_|-x86_64_|-(none)_|-1487838480', ++ 'ustr_|-(none)_|-1.0.4_|-16.el7_|-x86_64_|-(none)_|-1487838480', ++ 'shadow-utils_|-2_|-4.1.5.1_|-24.el7_|-x86_64_|-(none)_|-1487838481', ++ 'util-linux_|-(none)_|-2.23.2_|-33.el7_|-x86_64_|-(none)_|-1487838484', ++ 'openssh_|-(none)_|-6.6.1p1_|-33.el7_3_|-x86_64_|-(none)_|-1487838485', ++ 'virt-what_|-(none)_|-1.13_|-8.el7_|-x86_64_|-(none)_|-1487838486', ++ ] ++ with patch.dict(yumpkg.__grains__, {'osarch': 'x86_64'}), \ ++ patch.dict(yumpkg.__salt__, {'cmd.run': MagicMock(return_value=os.linesep.join(rpm_out))}), \ ++ patch.dict(yumpkg.__salt__, {'pkg_resource.add_pkg': _add_data}), \ ++ patch.dict(yumpkg.__salt__, {'pkg_resource.format_pkg_list': pkg_resource.format_pkg_list}), \ ++ patch.dict(yumpkg.__salt__, {'pkg_resource.stringify': MagicMock()}): ++ pkgs = yumpkg.list_pkgs(attr=['arch', 'install_date_time_t']) ++ for pkg_name, pkg_attr in { ++ 'python-urlgrabber': { ++ 'version': '3.10-8.el7', ++ 'arch': 'noarch', ++ 'install_date_time_t': 1487838471, ++ }, ++ 'alsa-lib': { ++ 'version': '1.1.1-1.el7', ++ 'arch': 'x86_64', ++ 'install_date_time_t': 1487838475, ++ }, ++ 'gnupg2': { ++ 'version': '2.0.22-4.el7', ++ 'arch': 'x86_64', ++ 'install_date_time_t': 1487838477, ++ }, ++ 'rpm-python': { ++ 'version': '4.11.3-21.el7', ++ 'arch': 'x86_64', ++ 'install_date_time_t': 1487838477, ++ }, ++ 'pygpgme': { ++ 'version': '0.3-9.el7', ++ 'arch': 'x86_64', ++ 'install_date_time_t': 1487838478, ++ }, ++ 'yum': { ++ 'version': '3.4.3-150.el7.centos', ++ 'arch': 'noarch', ++ 'install_date_time_t': 1487838479, ++ }, ++ 'lzo': { ++ 'version': '2.06-8.el7', ++ 'arch': 'x86_64', ++ 'install_date_time_t': 1487838479, ++ }, ++ 'qrencode-libs': { ++ 'version': '3.4.1-3.el7', ++ 'arch': 'x86_64', ++ 'install_date_time_t': 1487838480, ++ }, ++ 'ustr': { ++ 'version': '1.0.4-16.el7', ++ 'arch': 'x86_64', ++ 'install_date_time_t': 1487838480, ++ }, ++ 'shadow-utils': { ++ 'version': '2:4.1.5.1-24.el7', ++ 'arch': 'x86_64', ++ 'install_date_time_t': 1487838481, ++ }, ++ 'util-linux': { ++ 'version': '2.23.2-33.el7', ++ 'arch': 'x86_64', ++ 'install_date_time_t': 1487838484, ++ }, ++ 'openssh': { ++ 'version': '6.6.1p1-33.el7_3', ++ 'arch': 'x86_64', ++ 'install_date_time_t': 1487838485, ++ }, ++ 'virt-what': { ++ 'version': '1.13-8.el7', ++ 'arch': 'x86_64', ++ 'install_date_time_t': 1487838486, ++ }}.items(): ++ self.assertTrue(pkgs.get(pkg_name)) ++ self.assertEqual(pkgs[pkg_name], [pkg_attr]) +diff --git a/tests/unit/modules/test_zypper.py b/tests/unit/modules/test_zypper.py +index f3403e6e1c..41f3845646 100644 +--- a/tests/unit/modules/test_zypper.py ++++ b/tests/unit/modules/test_zypper.py +@@ -23,6 +23,7 @@ from tests.support.mock import ( + # Import Salt libs + import salt.utils + import salt.modules.zypper as zypper ++import salt.modules.pkg_resource as pkg_resource + from salt.exceptions import CommandExecutionError + + # Import 3rd-party libs +@@ -486,30 +487,92 @@ Repository 'DUMMY' not found by its alias, number, or URI. + :return: + ''' + def _add_data(data, key, value): +- data[key] = value ++ data.setdefault(key, []).append(value) + + rpm_out = [ +- 'protobuf-java_|-2.6.1_|-3.1.develHead_|-', +- 'yast2-ftp-server_|-3.1.8_|-8.1_|-', +- 'jose4j_|-0.4.4_|-2.1.develHead_|-', +- 'apache-commons-cli_|-1.2_|-1.233_|-', +- 'jakarta-commons-discovery_|-0.4_|-129.686_|-', +- 'susemanager-build-keys-web_|-12.0_|-5.1.develHead_|-', ++ 'protobuf-java_|-2.6.1_|-3.1.develHead_|-noarch_|-_|-1499257756', ++ 'yast2-ftp-server_|-3.1.8_|-8.1_|-x86_64_|-_|-1499257798', ++ 'jose4j_|-0.4.4_|-2.1.develHead_|-noarch_|-_|-1499257756', ++ 'apache-commons-cli_|-1.2_|-1.233_|-noarch_|-_|-1498636510', ++ 'jakarta-commons-discovery_|-0.4_|-129.686_|-noarch_|-_|-1498636511', ++ 'susemanager-build-keys-web_|-12.0_|-5.1.develHead_|-noarch_|-_|-1498636510', + ] +- with patch.dict(zypper.__salt__, {'cmd.run': MagicMock(return_value=os.linesep.join(rpm_out))}): +- with patch.dict(zypper.__salt__, {'pkg_resource.add_pkg': _add_data}): +- with patch.dict(zypper.__salt__, {'pkg_resource.sort_pkglist': MagicMock()}): +- with patch.dict(zypper.__salt__, {'pkg_resource.stringify': MagicMock()}): +- pkgs = zypper.list_pkgs() +- for pkg_name, pkg_version in { +- 'jakarta-commons-discovery': '0.4-129.686', +- 'yast2-ftp-server': '3.1.8-8.1', +- 'protobuf-java': '2.6.1-3.1.develHead', +- 'susemanager-build-keys-web': '12.0-5.1.develHead', +- 'apache-commons-cli': '1.2-1.233', +- 'jose4j': '0.4.4-2.1.develHead'}.items(): +- self.assertTrue(pkgs.get(pkg_name)) +- self.assertEqual(pkgs[pkg_name], pkg_version) ++ with patch.dict(zypper.__salt__, {'cmd.run': MagicMock(return_value=os.linesep.join(rpm_out))}), \ ++ patch.dict(zypper.__salt__, {'pkg_resource.add_pkg': _add_data}), \ ++ patch.dict(zypper.__salt__, {'pkg_resource.format_pkg_list': pkg_resource.format_pkg_list}), \ ++ patch.dict(zypper.__salt__, {'pkg_resource.stringify': MagicMock()}): ++ pkgs = zypper.list_pkgs(versions_as_list=True) ++ for pkg_name, pkg_version in { ++ 'jakarta-commons-discovery': '0.4-129.686', ++ 'yast2-ftp-server': '3.1.8-8.1', ++ 'protobuf-java': '2.6.1-3.1.develHead', ++ 'susemanager-build-keys-web': '12.0-5.1.develHead', ++ 'apache-commons-cli': '1.2-1.233', ++ 'jose4j': '0.4.4-2.1.develHead'}.items(): ++ self.assertTrue(pkgs.get(pkg_name)) ++ self.assertEqual(pkgs[pkg_name], [pkg_version]) ++ ++ def test_list_pkgs_with_attr(self): ++ ''' ++ Test packages listing with the attr parameter ++ ++ :return: ++ ''' ++ def _add_data(data, key, value): ++ data.setdefault(key, []).append(value) ++ ++ rpm_out = [ ++ 'protobuf-java_|-2.6.1_|-3.1.develHead_|-noarch_|-_|-1499257756', ++ 'yast2-ftp-server_|-3.1.8_|-8.1_|-x86_64_|-_|-1499257798', ++ 'jose4j_|-0.4.4_|-2.1.develHead_|-noarch_|-_|-1499257756', ++ 'apache-commons-cli_|-1.2_|-1.233_|-noarch_|-_|-1498636510', ++ 'jakarta-commons-discovery_|-0.4_|-129.686_|-noarch_|-_|-1498636511', ++ 'susemanager-build-keys-web_|-12.0_|-5.1.develHead_|-noarch_|-_|-1498636510', ++ ] ++ with patch.dict(zypper.__salt__, {'cmd.run': MagicMock(return_value=os.linesep.join(rpm_out))}), \ ++ patch.dict(zypper.__salt__, {'pkg_resource.add_pkg': _add_data}), \ ++ patch.dict(zypper.__salt__, {'pkg_resource.format_pkg_list': pkg_resource.format_pkg_list}), \ ++ patch.dict(zypper.__salt__, {'pkg_resource.stringify': MagicMock()}): ++ pkgs = zypper.list_pkgs(attr=['epoch', 'release', 'arch', 'install_date_time_t']) ++ for pkg_name, pkg_attr in { ++ 'jakarta-commons-discovery': { ++ 'version': '0.4', ++ 'release': '129.686', ++ 'arch': 'noarch', ++ 'install_date_time_t': 1498636511, ++ }, ++ 'yast2-ftp-server': { ++ 'version': '3.1.8', ++ 'release': '8.1', ++ 'arch': 'x86_64', ++ 'install_date_time_t': 1499257798, ++ }, ++ 'protobuf-java': { ++ 'version': '2.6.1', ++ 'release': '3.1.develHead', ++ 'install_date_time_t': 1499257756, ++ 'arch': 'noarch', ++ }, ++ 'susemanager-build-keys-web': { ++ 'version': '12.0', ++ 'release': '5.1.develHead', ++ 'arch': 'noarch', ++ 'install_date_time_t': 1498636510, ++ }, ++ 'apache-commons-cli': { ++ 'version': '1.2', ++ 'release': '1.233', ++ 'arch': 'noarch', ++ 'install_date_time_t': 1498636510, ++ }, ++ 'jose4j': { ++ 'arch': 'noarch', ++ 'version': '0.4.4', ++ 'release': '2.1.develHead', ++ 'install_date_time_t': 1499257756, ++ }}.items(): ++ self.assertTrue(pkgs.get(pkg_name)) ++ self.assertEqual(pkgs[pkg_name], [pkg_attr]) + + def test_list_patches(self): + ''' +-- +2.12.2 + + diff --git a/multiprocessing-minion-option-documentation-fixes.patch b/multiprocessing-minion-option-documentation-fixes.patch new file mode 100644 index 0000000..19105d1 --- /dev/null +++ b/multiprocessing-minion-option-documentation-fixes.patch @@ -0,0 +1,47 @@ +From 6c5b8be3d14814903abc70b5605c87277dad39db Mon Sep 17 00:00:00 2001 +From: Silvio Moioli +Date: Wed, 20 Sep 2017 14:32:47 +0200 +Subject: [PATCH] multiprocessing minion option: documentation fixes + +--- + doc/man/salt.7 | 1 + + doc/ref/configuration/minion.rst | 7 +++++-- + 2 files changed, 6 insertions(+), 2 deletions(-) + +diff --git a/doc/man/salt.7 b/doc/man/salt.7 +index d6cfe937a1..86c463b771 100644 +--- a/doc/man/salt.7 ++++ b/doc/man/salt.7 +@@ -10795,6 +10795,7 @@ cmd_whitelist_glob: + .UNINDENT + .UNINDENT + .SS Thread Settings ++.SS \fBmultiprocessing\fP + .sp + Default: \fBTrue\fP + .sp +diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst +index cd554268c1..5cc72f1daf 100644 +--- a/doc/ref/configuration/minion.rst ++++ b/doc/ref/configuration/minion.rst +@@ -2164,11 +2164,14 @@ Thread Settings + + .. conf_minion:: multiprocessing + ++``multiprocessing`` ++------- ++ + Default: ``True`` + +-If `multiprocessing` is enabled when a minion receives a ++If ``multiprocessing`` is enabled when a minion receives a + publication a new process is spawned and the command is executed therein. +-Conversely, if `multiprocessing` is disabled the new publication will be run ++Conversely, if ``multiprocessing`` is disabled the new publication will be run + executed in a thread. + + +-- +2.14.2 + + diff --git a/salt.changes b/salt.changes index f412371..1d26cd7 100644 --- a/salt.changes +++ b/salt.changes @@ -1,3 +1,60 @@ +------------------------------------------------------------------- +Wed Oct 11 14:15:17 UTC 2017 - bmaryniuk@suse.com + +- Add possibility to generate _version.py at the build time for + raw builds: https://github.com/saltstack/salt/pull/43955 + +- Added: + * enable-with-salt-version-parameter-for-setup.py-scri.patch + +------------------------------------------------------------------- +Tue Oct 10 15:31:36 UTC 2017 - bmaryniuk@suse.com + +- Update to 2017.7.2 + See https://docs.saltstack.com/en/develop/topics/releases/2017.7.2.html + for full changelog +- Fix for CVE-2017-14695 (bsc#1062462) +- Fix for CVE-2017-14696 (bsc#1062464) +- Fix salt target-type field returns "String" for existing + jids but an empty "Array" for non existing jids. (issue #1711) + +- Added: + * bugfix-always-return-a-string-list-on-unknown-job-ta.patch + + +------------------------------------------------------------------- +Fri Oct 6 13:53:59 UTC 2017 - bmaryniuk@suse.com + +- Fixed minion resource exhaustion when many functions are being + executed in parallel (bsc#1059758) + +- Added: + * introduce-process_count_max-minion-configuration-par.patch + * multiprocessing-minion-option-documentation-fixes.patch + +------------------------------------------------------------------- +Thu Oct 5 15:37:06 UTC 2017 - pablo.suarezhernandez@suse.com + +- Remove 'TasksTask' attribute from salt-master.service in older + versions of systemd (bsc#985112) +- Provide custom SUSE salt-master.service file. + +------------------------------------------------------------------- +Tue Oct 3 11:54:02 UTC 2017 - pablo.suarezhernandez@suse.com + +- Fix wrong version reported by Salt (bsc#1061407) + +------------------------------------------------------------------- +Tue Sep 12 07:26:03 UTC 2017 - pablo.suarezhernandez@suse.com + +- list_pkgs: add parameter for returned attribute selection (bsc#1052264) +- Adding the leftover for zypper and yum list_pkgs functionality. +- Use $HOME to get the user home directory instead using '~' char (bsc#1042749) + +- Added: + * list_pkgs-add-parameter-for-returned-attribute-selec.patch + * use-home-to-get-the-user-home-directory-instead-usin.patch + ------------------------------------------------------------------- Wed Aug 16 09:26:01 UTC 2017 - bmaryniuk@suse.com diff --git a/salt.spec b/salt.spec index 9340c91..ca45e89 100644 --- a/salt.spec +++ b/salt.spec @@ -36,19 +36,36 @@ %bcond_with builddocs Name: salt -Version: 2017.7.1 +Version: 2017.7.2 Release: 0 Summary: A parallel remote execution system License: Apache-2.0 Group: System/Management Url: http://saltstack.org/ -Source: https://github.com/saltstack/salt/archive/v2017.7.1.tar.gz +Source: https://github.com/saltstack/salt/archive/v%{version}.tar.gz Source1: README.SUSE Source2: salt-tmpfiles.d Source3: html.tar.bz2 Source4: update-documentation.sh Source5: travis.yml +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/42310 +# https://github.com/saltstack/salt/pull/42718 +# https://github.com/saltstack/salt/pull/43195 +# https://github.com/saltstack/salt/pull/43214 +# https://github.com/saltstack/salt/pull/43281 +Patch1: list_pkgs-add-parameter-for-returned-attribute-selec.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/43441 +Patch2: use-home-to-get-the-user-home-directory-instead-usin.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/43663 +Patch3: multiprocessing-minion-option-documentation-fixes.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/43669 +Patch4: introduce-process_count_max-minion-configuration-par.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/ +Patch5: bugfix-always-return-a-string-list-on-unknown-job-ta.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/43955 +Patch6: enable-with-salt-version-parameter-for-setup.py-scri.patch + BuildRoot: %{_tmppath}/%{name}-%{version}-build BuildRequires: fdupes BuildRequires: logrotate @@ -386,8 +403,16 @@ Zsh command line completion support for %{name}. cp %{S:1} . cp %{S:5} ./.travis.yml +%patch1 -p1 +%patch2 -p1 +%patch3 -p1 +%patch4 -p1 +%patch5 -p1 +%patch6 -p1 + %build -%{__python} setup.py --salt-transport=both build +%{__python} setup.py --with-salt-version=%{version} --salt-transport=both build +cp ./build/lib/salt/_version.py ./salt %if %{with docs} && %{without builddocs} # extract docs from the tarball @@ -459,7 +484,7 @@ install scripts/suse/yum/plugins/yumnotify.conf %{buildroot}/etc/yum/pluginconf. ## install init and systemd scripts %if %{with systemd} -install -Dpm 0644 pkg/salt-master.service %{buildroot}%{_unitdir}/salt-master.service +install -Dpm 0644 pkg/suse/salt-master.service %{buildroot}%{_unitdir}/salt-master.service %if 0%{?suse_version} install -Dpm 0644 pkg/suse/salt-minion.service %{buildroot}%{_unitdir}/salt-minion.service %else @@ -692,6 +717,11 @@ if [ $1 -eq 2 ] ; then true fi %if %{with systemd} +if [ `rpm -q systemd --queryformat="%{VERSION}"` -lt 228 ]; then + # On systemd < 228 the 'TasksTask' attribute is not available. + # Removing TasksMax from salt-master.service on SLE12SP1 LTSS (bsc#985112) + sed -i '/TasksMax=infinity/d' %{_unitdir}/salt-master.service +fi %if 0%{?suse_version} %service_add_post salt-master.service %fillup_only diff --git a/use-home-to-get-the-user-home-directory-instead-usin.patch b/use-home-to-get-the-user-home-directory-instead-usin.patch new file mode 100644 index 0000000..b7df7a5 --- /dev/null +++ b/use-home-to-get-the-user-home-directory-instead-usin.patch @@ -0,0 +1,29 @@ +From 2dfd28560f3825dc512822ba9d01d67070d5175b Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Mon, 11 Sep 2017 19:57:28 +0200 +Subject: [PATCH] Use $HOME to get the user home directory instead using + '~' char + +--- + pkg/salt.bash | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/pkg/salt.bash b/pkg/salt.bash +index 480361fe23..00174c072f 100644 +--- a/pkg/salt.bash ++++ b/pkg/salt.bash +@@ -35,7 +35,8 @@ _salt_get_keys(){ + } + + _salt(){ +- local _salt_cache_functions=${SALT_COMP_CACHE_FUNCTIONS:='~/.cache/salt-comp-cache_functions'} ++ CACHE_DIR="$HOME/.cache/salt-comp-cache_functions" ++ local _salt_cache_functions=${SALT_COMP_CACHE_FUNCTIONS:=$CACHE_DIR} + local _salt_cache_timeout=${SALT_COMP_CACHE_TIMEOUT:='last hour'} + + if [ ! -d "$(dirname ${_salt_cache_functions})" ]; then +-- +2.12.2 + + diff --git a/v2017.7.1.tar.gz b/v2017.7.1.tar.gz deleted file mode 100644 index 11a67e2..0000000 --- a/v2017.7.1.tar.gz +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:232b7fbe27f07670e8dc9f869bffc7681d780f3b1711926ac812391b05f272b8 -size 11420619 diff --git a/v2017.7.2.tar.gz b/v2017.7.2.tar.gz new file mode 100644 index 0000000..b757ea3 --- /dev/null +++ b/v2017.7.2.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9f9dd9ddd129ddadadf963178383b50c32283aeb1c338d9c23cc01b11722db2 +size 11483585