diff --git a/_lastrevision b/_lastrevision index 2e10b0a..c639742 100644 --- a/_lastrevision +++ b/_lastrevision @@ -1 +1 @@ -191f12d1f56ce5b5645ae420da0392941bcb2de4 \ No newline at end of file +09133c4494ccc374b1ec76de50772e1efee3f00e \ No newline at end of file diff --git a/do-not-override-jid-on-returners-only-sending-back-t.patch b/do-not-override-jid-on-returners-only-sending-back-t.patch new file mode 100644 index 0000000..0566ba6 --- /dev/null +++ b/do-not-override-jid-on-returners-only-sending-back-t.patch @@ -0,0 +1,38 @@ +From c49a9aca3519d1baef2f9d82963a6e80eabb26d4 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= + +Date: Fri, 4 May 2018 09:34:13 +0100 +Subject: [PATCH] Do not override jid on returners, only sending back to + master (bsc#1092373) + +--- + salt/utils/schedule.py | 12 +++++++----- + 1 file changed, 7 insertions(+), 5 deletions(-) + +diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py +index 6cb3ce0ef8..e81ae143bd 100644 +--- a/salt/utils/schedule.py ++++ b/salt/utils/schedule.py +@@ -753,11 +753,13 @@ class Schedule(object): + else: + # Send back to master so the job is included in the job list + mret = ret.copy() +- mret['jid'] = 'req' +- if data.get('return_job') == 'nocache': +- # overwrite 'req' to signal to master that +- # this job shouldn't be stored +- mret['jid'] = 'nocache' ++ # No returners defined, so we're only sending back to the master ++ if not data_returner and not self.schedule_returner: ++ mret['jid'] = 'req' ++ if data.get('return_job') == 'nocache': ++ # overwrite 'req' to signal to master that ++ # this job shouldn't be stored ++ mret['jid'] = 'nocache' + load = {'cmd': '_return', 'id': self.opts['id']} + for key, value in six.iteritems(mret): + load[key] = value +-- +2.15.1 + + diff --git a/enable-passing-a-unix_socket-for-mysql-returners-bsc.patch b/enable-passing-a-unix_socket-for-mysql-returners-bsc.patch new file mode 100644 index 0000000..e1fef6a --- /dev/null +++ b/enable-passing-a-unix_socket-for-mysql-returners-bsc.patch @@ -0,0 +1,74 @@ +From cdfb19b6c2801a7d5a883492a0ccc57c803dcd7f Mon Sep 17 00:00:00 2001 +From: Maximilian Meister +Date: Thu, 3 May 2018 15:52:23 +0200 +Subject: [PATCH] enable passing a unix_socket for mysql returners + (bsc#1091371) + +quick fix for: + https://bugzilla.suse.com/show_bug.cgi?id=1091371 + +the upstream patch will go through some bigger refactoring of +the mysql drivers to be cleaner + +this patch should only be temporary and can be dropped again once +the refactor is done upstream + +Signed-off-by: Maximilian Meister +--- + salt/returners/mysql.py | 11 ++++++++--- + 1 file changed, 8 insertions(+), 3 deletions(-) + +diff --git a/salt/returners/mysql.py b/salt/returners/mysql.py +index 85892cb06c..a286731d5c 100644 +--- a/salt/returners/mysql.py ++++ b/salt/returners/mysql.py +@@ -18,6 +18,7 @@ config. These are the defaults: + mysql.pass: 'salt' + mysql.db: 'salt' + mysql.port: 3306 ++ mysql.unix_socket: '/tmp/mysql.sock' + + SSL is optional. The defaults are set to None. If you do not want to use SSL, + either exclude these options or set them to None. +@@ -43,6 +44,7 @@ optional. The following ssl options are simply for illustration purposes: + alternative.mysql.ssl_ca: '/etc/pki/mysql/certs/localhost.pem' + alternative.mysql.ssl_cert: '/etc/pki/mysql/certs/localhost.crt' + alternative.mysql.ssl_key: '/etc/pki/mysql/certs/localhost.key' ++ alternative.mysql.unix_socket: '/tmp/mysql.sock' + + Should you wish the returner data to be cleaned out every so often, set + `keep_jobs` to the number of hours for the jobs to live in the tables. +@@ -198,7 +200,8 @@ def _get_options(ret=None): + 'port': 3306, + 'ssl_ca': None, + 'ssl_cert': None, +- 'ssl_key': None} ++ 'ssl_key': None, ++ 'unix_socket': '/tmp/mysql.sock'} + + attrs = {'host': 'host', + 'user': 'user', +@@ -207,7 +210,8 @@ def _get_options(ret=None): + 'port': 'port', + 'ssl_ca': 'ssl_ca', + 'ssl_cert': 'ssl_cert', +- 'ssl_key': 'ssl_key'} ++ 'ssl_key': 'ssl_key', ++ 'unix_socket': 'unix_socket'} + + _options = salt.returners.get_returner_options(__virtualname__, + ret, +@@ -261,7 +265,8 @@ def _get_serv(ret=None, commit=False): + passwd=_options.get('pass'), + db=_options.get('db'), + port=_options.get('port'), +- ssl=ssl_options) ++ ssl=ssl_options, ++ unix_socket=_options.get('unix_socket')) + + try: + __context__['mysql_returner_conn'] = conn +-- +2.13.6 + + diff --git a/extra-filerefs-include-files-even-if-no-refs-in-stat.patch b/extra-filerefs-include-files-even-if-no-refs-in-stat.patch new file mode 100644 index 0000000..38c772f --- /dev/null +++ b/extra-filerefs-include-files-even-if-no-refs-in-stat.patch @@ -0,0 +1,32 @@ +From 544dfd7dbaa1c837b75976f15ad67159b1bdedbb Mon Sep 17 00:00:00 2001 +From: Matei Albu +Date: Sun, 6 May 2018 11:56:18 +0200 +Subject: [PATCH] --extra-filerefs include files even if no refs in + states to apply + +Fixes #47496 +(cherry picked from commit d67239a) +--- + salt/client/ssh/state.py | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/salt/client/ssh/state.py b/salt/client/ssh/state.py +index 8fa11d031e..08d4846bb6 100644 +--- a/salt/client/ssh/state.py ++++ b/salt/client/ssh/state.py +@@ -135,9 +135,9 @@ def lowstate_file_refs(chunks, extras=''): + elif state.startswith('__'): + continue + crefs.extend(salt_refs(chunk[state])) ++ if saltenv not in refs: ++ refs[saltenv] = [] + if crefs: +- if saltenv not in refs: +- refs[saltenv] = [] + refs[saltenv].append(crefs) + if extras: + extra_refs = extras.split(',') +-- +2.13.6 + + diff --git a/fix-for-ec2-rate-limit-failures.patch b/fix-for-ec2-rate-limit-failures.patch new file mode 100644 index 0000000..c8fd3db --- /dev/null +++ b/fix-for-ec2-rate-limit-failures.patch @@ -0,0 +1,66 @@ +From b0273e150b29fb44d72246918792117b485441de Mon Sep 17 00:00:00 2001 +From: Daniel Wallace +Date: Wed, 25 Apr 2018 11:13:15 -0500 +Subject: [PATCH] Fix for EC2 Rate Limit Failures + +Fix for ec2 rate limit failures described here: https://bugzilla.suse.com/show_bug.cgi?id=1088888 +--- + salt/utils/aws.py | 22 ++++++++++++++++------ + 1 file changed, 16 insertions(+), 6 deletions(-) + +diff --git a/salt/utils/aws.py b/salt/utils/aws.py +index 059450e7ca..912f1466ba 100644 +--- a/salt/utils/aws.py ++++ b/salt/utils/aws.py +@@ -20,6 +20,7 @@ import hmac + import logging + import salt.config + import re ++import random + from salt.ext import six + + # Import Salt libs +@@ -442,8 +443,9 @@ def query(params=None, setname=None, requesturl=None, location=None, + ) + headers = {} + +- attempts = 5 +- while attempts > 0: ++ MAX_RETRIES = 6 ++ attempts = 0 ++ while attempts < MAX_RETRIES: + log.debug('AWS Request: %s', requesturl) + log.trace('AWS Request Parameters: %s', params_with_headers) + try: +@@ -461,15 +463,23 @@ def query(params=None, setname=None, requesturl=None, location=None, + + # check to see if we should retry the query + err_code = data.get('Errors', {}).get('Error', {}).get('Code', '') +- if attempts > 0 and err_code and err_code in AWS_RETRY_CODES: +- attempts -= 1 ++ if attempts < MAX_RETRIES and err_code and err_code in AWS_RETRY_CODES: ++ attempts += 1 + log.error( + 'AWS Response Status Code and Error: [%s %s] %s; ' + 'Attempts remaining: %s', + exc.response.status_code, exc, data, attempts + ) +- # Wait a bit before continuing to prevent throttling +- time.sleep(2) ++ # backoff an exponential amount of time to throttle requests ++ # during "API Rate Exceeded" failures as suggested by the AWS documentation here: ++ # https://docs.aws.amazon.com/AWSEC2/latest/APIReference/query-api-troubleshooting.html ++ # and also here: ++ # https://docs.aws.amazon.com/general/latest/gr/api-retries.html ++ # Failure to implement this approach results in a failure rate of >30% when using salt-cloud with ++ # "--parallel" when creating 50 or more instances with a fixed delay of 2 seconds. ++ # A failure rate of >10% is observed when using the salt-api with an asyncronous client ++ # specified (runner_async). ++ time.sleep(random.uniform(1, 2**attempts)) + continue + + log.error( +-- +2.13.6 + + diff --git a/option-to-merge-current-pillar-with-opts-pillar-duri.patch b/option-to-merge-current-pillar-with-opts-pillar-duri.patch new file mode 100644 index 0000000..49f64d7 --- /dev/null +++ b/option-to-merge-current-pillar-with-opts-pillar-duri.patch @@ -0,0 +1,101 @@ +From 0cfa4f2a1cf559f87286069691a2766cb24f6076 Mon Sep 17 00:00:00 2001 +From: Matei Albu +Date: Sun, 6 May 2018 21:15:58 +0200 +Subject: [PATCH] Option to merge current pillar with opts['pillar'] + during pillar compile + +Fixes #47501 +(cherry picked from commit 2f1485e) +--- + doc/ref/configuration/minion.rst | 28 ++++++++++++++++++++++++++++ + salt/config/__init__.py | 4 +++- + salt/pillar/__init__.py | 7 +++++++ + 3 files changed, 38 insertions(+), 1 deletion(-) + +diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst +index 9683a0a20a..75ad26c723 100644 +--- a/doc/ref/configuration/minion.rst ++++ b/doc/ref/configuration/minion.rst +@@ -3219,3 +3219,31 @@ URL of the repository: + Replace ```` with the SHA1 hash of a commit ID. Specifying a commit + ID is useful in that it allows one to revert back to a previous version in the + event that an error is introduced in the latest revision of the repo. ++ ++``ssh_merge_pillar`` ++-------------------- ++ ++.. versionadded:: 2018.3.2 ++ ++Default: ``True`` ++ ++Merges the compiled pillar data with the pillar data already available globally. ++This is useful when using ``salt-ssh`` or ``salt-call --local`` and overriding the pillar ++data in a state file: ++ ++.. code-block:: yaml ++ ++ apply_showpillar: ++ module.run: ++ - name: state.apply ++ - mods: ++ - showpillar ++ - kwargs: ++ pillar: ++ test: "foo bar" ++ ++If set to ``True`` the ``showpillar`` state will have access to the ++global pillar data. ++ ++If set to ``False`` only the overriding pillar data will be available ++to the ``showpillar`` state. +diff --git a/salt/config/__init__.py b/salt/config/__init__.py +index b3de3820b0..82d3dfa07f 100644 +--- a/salt/config/__init__.py ++++ b/salt/config/__init__.py +@@ -983,6 +983,7 @@ VALID_OPTS = { + 'ssh_identities_only': bool, + 'ssh_log_file': six.string_types, + 'ssh_config_file': six.string_types, ++ 'ssh_merge_pillar': bool, + + # Enable ioflo verbose logging. Warning! Very verbose! + 'ioflo_verbose': int, +@@ -1476,6 +1477,7 @@ DEFAULT_MINION_OPTS = { + }, + 'discovery': False, + 'schedule': {}, ++ 'ssh_merge_pillar': True + } + + DEFAULT_MASTER_OPTS = { +@@ -2078,7 +2080,7 @@ def _validate_ssh_minion_opts(opts): + + for opt_name in list(ssh_minion_opts): + if re.match('^[a-z0-9]+fs_', opt_name, flags=re.IGNORECASE) \ +- or 'pillar' in opt_name \ ++ or ('pillar' in opt_name and not 'ssh_merge_pillar' == opt_name) \ + or opt_name in ('fileserver_backend',): + log.warning( + '\'%s\' is not a valid ssh_minion_opts parameter, ignoring', +diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py +index 388b774434..5940b7c105 100644 +--- a/salt/pillar/__init__.py ++++ b/salt/pillar/__init__.py +@@ -993,6 +993,13 @@ class Pillar(object): + mopts['file_roots'] = self.actual_file_roots + mopts['saltversion'] = __version__ + pillar['master'] = mopts ++ if 'pillar' in self.opts and self.opts.get('ssh_merge_pillar', False): ++ pillar = merge( ++ self.opts['pillar'], ++ pillar, ++ self.merge_strategy, ++ self.opts.get('renderer', 'yaml'), ++ self.opts.get('pillar_merge_lists', False)) + if errors: + for error in errors: + log.critical('Pillar render error: %s', error) +-- +2.13.6 + + diff --git a/salt.changes b/salt.changes index 2534e90..be46054 100644 --- a/salt.changes +++ b/salt.changes @@ -1,3 +1,29 @@ +------------------------------------------------------------------- +Wed May 16 10:57:17 UTC 2018 - Jochen Breuer + +- No more AWS EC2 rate limitations in salt-cloud (bsc#1088888) +- MySQL returner now also allows to use Unix sockets (bsc#1091371) + +- Added: + * fix-for-ec2-rate-limit-failures.patch + * enable-passing-a-unix_socket-for-mysql-returners-bsc.patch + +------------------------------------------------------------------- +Thu May 10 09:12:24 UTC 2018 - Pablo Suárez Hernández + +- Do not override jid on returners, only sending back to master (bsc#1092373) +- Fixes for salt-ssh: + - Option --extra-filerefs doesn't add all files to the state + archive + - Pillar completely overwritten (not merged) when doing + module.run + state.apply with pillar in kwargs +- Remove minion/thin/version if exists to force thin regeneration (bsc#1092161) + +- Added: + * extra-filerefs-include-files-even-if-no-refs-in-stat.patch + * do-not-override-jid-on-returners-only-sending-back-t.patch + * option-to-merge-current-pillar-with-opts-pillar-duri.patch + ------------------------------------------------------------------- Fri May 4 12:17:07 UTC 2018 - Jochen Breuer diff --git a/salt.spec b/salt.spec index ab1cdde..f11d52a 100644 --- a/salt.spec +++ b/salt.spec @@ -98,7 +98,16 @@ Patch18: provide-kwargs-to-pkg_resource.parse_targets-require.patch Patch19: initialize-__context__-retcode-for-functions-handled.patch # PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/47232 Patch20: fixed-usage-of-ipaddress.patch - +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/47497 +Patch21: extra-filerefs-include-files-even-if-no-refs-in-stat.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/47504 +Patch22: option-to-merge-current-pillar-with-opts-pillar-duri.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/47471 +Patch23: do-not-override-jid-on-returners-only-sending-back-t.patch +# PATCH-FIX_OPENSUSE bsc#1091371 +Patch24: enable-passing-a-unix_socket-for-mysql-returners-bsc.patch +# PATCH-FIX_UPSTREAM https://github.com/saltstack/salt/pull/47211 +Patch25: fix-for-ec2-rate-limit-failures.patch # BuildRoot: %{_tmppath}/%{name}-%{version}-build BuildRoot: %{_tmppath}/%{name}-%{version}-build @@ -577,6 +586,11 @@ cp %{S:5} ./.travis.yml %patch18 -p1 %patch19 -p1 %patch20 -p1 +%patch21 -p1 +%patch22 -p1 +%patch23 -p1 +%patch24 -p1 +%patch25 -p1 %build %if 0%{?build_py2} @@ -1071,11 +1085,13 @@ fi %posttrans -n python2-salt # force re-generate a new thin.tgz rm -f %{_localstatedir}/cache/salt/master/thin/version +rm -f %{_localstatedir}/cache/salt/minion/thin/version %if 0%{?build_py3} %posttrans -n python3-salt # force re-generate a new thin.tgz rm -f %{_localstatedir}/cache/salt/master/thin/version +rm -f %{_localstatedir}/cache/salt/minion/thin/version %endif %files api