Accepting request 801830 from systemsmanagement:saltstack

- Python 3.8 compatibility changes
- msgpack support for version >= 1.0.0 (bsc#1171257)
- Added:
  * python3.8-compatibility-pr-s-235.patch
  * msgpack-support-versions-1.0.0.patch

- Prevent sporious "salt-api" stuck processes when managing SSH minions
  because of logging deadlock (bsc#1159284)
- Avoid segfault from "salt-api" under certain conditions of heavy load
  managing SSH minions (bsc#1169604)
- Added:
  * prevent-logging-deadlock-on-salt-api-subprocesses-bs.patch
  * make-lazyloader.__init__-call-to-_refresh_file_mappi.patch

OBS-URL: https://build.opensuse.org/request/show/801830
OBS-URL: https://build.opensuse.org/package/show/openSUSE:Factory/salt?expand=0&rev=104

_lastrevision

@@ -1 +1 @@
-52475d78d58db61df54486af13c5eb4055c2d12e
+4b7e8cbf000c33224e9cd7a7eb581954ca3dbbec

make-lazyloader.__init__-call-to-_refresh_file_mappi.patch

@@ -0,0 +1,29 @@
+From 6af6a52165c70c3be7c8d339a3dd5e539f3c1772 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?=
+ <psuarezhernandez@suse.com>
+Date: Thu, 23 Apr 2020 09:54:53 +0100
+Subject: [PATCH] Make LazyLoader.__init__ call to
+ _refresh_file_mapping thread-safe (bsc#1169604)
+
+---
+ salt/loader.py | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/salt/loader.py b/salt/loader.py
+index 5bd4773645c77a133701982e19d19739be00a38f..54dadb0b513dbaa4914b0d4b1d343dde709699ad 100644
+--- a/salt/loader.py
++++ b/salt/loader.py
+@@ -1251,7 +1251,8 @@ class LazyLoader(salt.utils.lazy.LazyDict):
+             self.suffix_order.append(suffix)
+ 
+         self._lock = threading.RLock()
+-        self._refresh_file_mapping()
++        with self._lock:
++            self._refresh_file_mapping()
+ 
+         super(LazyLoader, self).__init__()  # late init the lazy loader
+         # create all of the import namespaces
+-- 
+2.23.0
+
+

msgpack-support-versions-1.0.0.patch

@@ -0,0 +1,72 @@
+From ef23c1d53e99e19e5b03658aa62b67cfef9adce5 Mon Sep 17 00:00:00 2001
+From: Alberto Planas <aplanas@suse.com>
+Date: Thu, 7 May 2020 12:40:55 +0200
+Subject: [PATCH] msgpack: support versions >= 1.0.0
+
+A recent change in msgpack >= 1.0.0, update the default value for the
+parameter `raw` to False.  This change breaks Salt for those versions.
+
+This patch add the parameter `raw=True` to all the unpack operations,
+restoring the old default.
+
+Fix bsc#1171257
+
+(cherry picked from commit 1b3939fb01fc3405d8d222f118617220aecee092)
+---
+ salt/utils/msgpack.py | 20 +++++++++++++++++---
+ 1 file changed, 17 insertions(+), 3 deletions(-)
+
+diff --git a/salt/utils/msgpack.py b/salt/utils/msgpack.py
+index 4b5a256513..027fe81a18 100644
+--- a/salt/utils/msgpack.py
++++ b/salt/utils/msgpack.py
+@@ -69,12 +69,26 @@ def _sanitize_msgpack_kwargs(kwargs):
+     return kwargs
+ 
+ 
++def _sanitize_msgpack_unpack_kwargs(kwargs):
++    """
++    Clean up msgpack keyword arguments for unpack operations, based on
++    the version
++    https://github.com/msgpack/msgpack-python/blob/master/ChangeLog.rst
++    """
++    assert isinstance(kwargs, dict)
++    if version >= (1, 0, 0) and kwargs.get("raw", None) is None:
++        log.info("adding `raw=True` argument to msgpack call")
++        kwargs["raw"] = True
++
++    return _sanitize_msgpack_kwargs(kwargs)
++
++
+ class Unpacker(msgpack.Unpacker):
+     '''
+     Wraps the msgpack.Unpacker and removes non-relevant arguments
+     '''
+     def __init__(self, *args, **kwargs):
+-        msgpack.Unpacker.__init__(self, *args, **_sanitize_msgpack_kwargs(kwargs))
++        msgpack.Unpacker.__init__(self, *args, **_sanitize_msgpack_unpack_kwargs(kwargs))
+ 
+ 
+ def pack(o, stream, **kwargs):
+@@ -113,7 +127,7 @@ def unpack(stream, **kwargs):
+     By default, this function uses the msgpack module and falls back to
+     msgpack_pure, if the msgpack is not available.
+     '''
+-    return msgpack.unpack(stream, **_sanitize_msgpack_kwargs(kwargs))
++    return msgpack.unpack(stream, **_sanitize_msgpack_unpack_kwargs(kwargs))
+ 
+ 
+ def unpackb(packed, **kwargs):
+@@ -125,7 +139,7 @@ def unpackb(packed, **kwargs):
+     By default, this function uses the msgpack module and falls back to
+     msgpack_pure.
+     '''
+-    return msgpack.unpackb(packed, **_sanitize_msgpack_kwargs(kwargs))
++    return msgpack.unpackb(packed, **_sanitize_msgpack_unpack_kwargs(kwargs))
+ 
+ 
+ # alias for compatibility to simplejson/marshal/pickle.
+-- 
+2.26.1
+
+

prevent-logging-deadlock-on-salt-api-subprocesses-bs.patch

@@ -0,0 +1,256 @@
+From 217c5ba75b5de813ddf769e7eeebe4027c1c9a70 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?=
+ <psuarezhernandez@suse.com>
+Date: Wed, 22 Jan 2020 08:19:55 +0000
+Subject: [PATCH] Prevent logging deadlock on salt-api subprocesses
+ (bsc#1159284)
+
+---
+ salt/_logging/impl.py         | 60 ++++++++++++++++++++---------------
+ salt/client/ssh/__init__.py   | 16 +++++++---
+ salt/client/ssh/client.py     |  9 +++++-
+ salt/client/ssh/wrapper/cp.py |  2 +-
+ salt/loader.py                |  2 +-
+ salt/utils/lazy.py            |  5 ++-
+ 6 files changed, 61 insertions(+), 33 deletions(-)
+
+diff --git a/salt/_logging/impl.py b/salt/_logging/impl.py
+index 347259bcf506705e2ea1a24da030a7132eb8a527..fdfabf6d3b16619350107bd01f2c3a606fb93262 100644
+--- a/salt/_logging/impl.py
++++ b/salt/_logging/impl.py
+@@ -19,6 +19,7 @@ PROFILE = logging.PROFILE = 15
+ TRACE = logging.TRACE = 5
+ GARBAGE = logging.GARBAGE = 1
+ QUIET = logging.QUIET = 1000
++DEBUG = logging.DEBUG = 10
+ 
+ # Import Salt libs
+ from salt._logging.handlers import StreamHandler
+@@ -187,11 +188,11 @@ class SaltLoggingClass(six.with_metaclass(LoggingMixinMeta, LOGGING_LOGGER_CLASS
+         '''
+         instance = super(SaltLoggingClass, cls).__new__(cls)
+ 
+-        try:
+-            max_logger_length = len(max(
+-                list(logging.Logger.manager.loggerDict), key=len
+-            ))
+-            for handler in logging.root.handlers:
++        max_logger_length = len(max(
++            list(logging.Logger.manager.loggerDict), key=len
++        ))
++        for handler in logging.root.handlers:
++            try:
+                 if handler in (LOGGING_NULL_HANDLER,
+                                LOGGING_STORE_HANDLER,
+                                LOGGING_TEMP_HANDLER):
+@@ -210,18 +211,15 @@ class SaltLoggingClass(six.with_metaclass(LoggingMixinMeta, LOGGING_LOGGER_CLASS
+                 match = MODNAME_PATTERN.search(fmt)
+                 if not match:
+                     # Not matched. Release handler and return.
+-                    handler.release()
+                     return instance
+ 
+                 if 'digits' not in match.groupdict():
+                     # No digits group. Release handler and return.
+-                    handler.release()
+                     return instance
+ 
+                 digits = match.group('digits')
+                 if not digits or not (digits and digits.isdigit()):
+                     # No valid digits. Release handler and return.
+-                    handler.release()
+                     return instance
+ 
+                 if int(digits) < max_logger_length:
+@@ -233,9 +231,14 @@ class SaltLoggingClass(six.with_metaclass(LoggingMixinMeta, LOGGING_LOGGER_CLASS
+                     )
+                     handler.setFormatter(formatter)
+                 handler.release()
+-        except ValueError:
+-            # There are no registered loggers yet
+-            pass
++            except ValueError:
++                # There are no registered loggers yet
++                pass
++            finally:
++                try:
++                    handler.release()
++                except:
++                    pass
+         return instance
+ 
+     def _log(self, level, msg, args, exc_info=None,
+@@ -278,20 +281,26 @@ class SaltLoggingClass(six.with_metaclass(LoggingMixinMeta, LOGGING_LOGGER_CLASS
+         else:
+             extra['exc_info_on_loglevel'] = exc_info_on_loglevel
+ 
+-        if sys.version_info < (3,):
+-            LOGGING_LOGGER_CLASS._log(
+-                self, level, msg, args, exc_info=exc_info, extra=extra
+-            )
+-        elif sys.version_info < (3, 8):
+-            LOGGING_LOGGER_CLASS._log(
+-                self, level, msg, args, exc_info=exc_info, extra=extra,
+-                stack_info=stack_info
+-            )
+-        else:
+-            LOGGING_LOGGER_CLASS._log(
+-                self, level, msg, args, exc_info=exc_info, extra=extra,
+-                stack_info=stack_info, stacklevel=stacklevel
+-            )
++        try:
++            logging._acquireLock()
++            if sys.version_info < (3,):
++                LOGGING_LOGGER_CLASS._log(
++                    self, level, msg, args, exc_info=exc_info, extra=extra
++                )
++            elif sys.version_info < (3, 8):
++                LOGGING_LOGGER_CLASS._log(
++                    self, level, msg, args, exc_info=exc_info, extra=extra,
++                    stack_info=stack_info
++                )
++            else:
++                LOGGING_LOGGER_CLASS._log(
++                    self, level, msg, args, exc_info=exc_info, extra=extra,
++                    stack_info=stack_info, stacklevel=stacklevel
++                )
++        except:
++            pass
++        finally:
++            logging._releaseLock()
+ 
+     def makeRecord(self, name, level, fn, lno, msg, args, exc_info,
+                    func=None, extra=None, sinfo=None):
+@@ -393,6 +402,7 @@ if logging.getLoggerClass() is not SaltLoggingClass:
+     logging.addLevelName(PROFILE, 'PROFILE')
+     logging.addLevelName(TRACE, 'TRACE')
+     logging.addLevelName(GARBAGE, 'GARBAGE')
++    logging.addLevelName(DEBUG, 'DEBUG')
+ 
+     # ----- REMOVE ON REFACTORING COMPLETE -------------------------------------------------------------------------->
+     if not logging.root.handlers:
+diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py
+index d9e91b0f50bfaa76d519fcaa4bdc868bce80f554..e8aad093e0f6df32faa16a838f1db2c6746e1b8e 100644
+--- a/salt/client/ssh/__init__.py
++++ b/salt/client/ssh/__init__.py
+@@ -520,7 +520,9 @@ class SSH(object):
+                 mine=mine,
+                 **target)
+         ret = {'id': single.id}
++        logging._acquireLock()
+         stdout, stderr, retcode = single.run()
++        logging._releaseLock()
+         # This job is done, yield
+         try:
+             data = salt.utils.json.find_json(stdout)
+@@ -586,10 +588,16 @@ class SSH(object):
+                         self.targets[host],
+                         mine,
+                         )
+-                routine = Process(
+-                                target=self.handle_routine,
+-                                args=args)
+-                routine.start()
++                try:
++                    logging._acquireLock()
++                    routine = Process(
++                                    target=self.handle_routine,
++                                    args=args)
++                    routine.start()
++                except:
++                    pass
++                finally:
++                    logging._releaseLock()
+                 running[host] = {'thread': routine}
+                 continue
+             ret = {}
+diff --git a/salt/client/ssh/client.py b/salt/client/ssh/client.py
+index e8e634ca12d85f1e1a9e047f43eac8c041cc5666..d4a89cf4fbbde5282597dc6b82c66dde4288edf1 100644
+--- a/salt/client/ssh/client.py
++++ b/salt/client/ssh/client.py
+@@ -6,6 +6,8 @@ import os
+ import copy
+ import logging
+ import random
++import time
++import multiprocessing
+ 
+ # Import Salt libs
+ import salt.config
+@@ -15,6 +17,7 @@ from salt.exceptions import SaltClientError  # Temporary
+ 
+ log = logging.getLogger(__name__)
+ 
++_LOCK = multiprocessing.Lock()
+ 
+ class SSHClient(object):
+     '''
+@@ -61,7 +64,11 @@ class SSHClient(object):
+         opts['selected_target_option'] = tgt_type
+         opts['tgt'] = tgt
+         opts['arg'] = arg
+-        return salt.client.ssh.SSH(opts)
++        _LOCK.acquire()
++        ret = salt.client.ssh.SSH(opts)
++        time.sleep(0.01)
++        _LOCK.release()
++        return ret
+ 
+     def cmd_iter(
+             self,
+diff --git a/salt/client/ssh/wrapper/cp.py b/salt/client/ssh/wrapper/cp.py
+index 894e62f94c87ae5b68c1f82fc3e80ec8f25ac118..9bf0c150a071b4bfa780fe0293e3d8e93ab8e6ef 100644
+--- a/salt/client/ssh/wrapper/cp.py
++++ b/salt/client/ssh/wrapper/cp.py
+@@ -4,7 +4,7 @@ Wrap the cp module allowing for managed ssh file transfers
+ '''
+ # Import Python libs
+ from __future__ import absolute_import, print_function
+-import logging
++import salt.log.setup as logging
+ import os
+ 
+ # Import salt libs
+diff --git a/salt/loader.py b/salt/loader.py
+index 54dadb0b513dbaa4914b0d4b1d343dde709699ad..b824a70a0cc40128f3271f70f676f1551194236c 100644
+--- a/salt/loader.py
++++ b/salt/loader.py
+@@ -11,7 +11,7 @@ import os
+ import re
+ import sys
+ import time
+-import logging
++import salt.log.setup as logging
+ import inspect
+ import tempfile
+ import functools
+diff --git a/salt/utils/lazy.py b/salt/utils/lazy.py
+index 3cd6489d2d8c50ec4e6eb70c50407f1084db377b..bb4b38e1a3cfa05945cd438fc9d30e7c47c3391b 100644
+--- a/salt/utils/lazy.py
++++ b/salt/utils/lazy.py
+@@ -5,7 +5,8 @@ Lazily-evaluated data structures, primarily used by Salt's loader
+ 
+ # Import Python Libs
+ from __future__ import absolute_import, unicode_literals
+-import logging
++import salt.log.setup as logging
++import time
+ import salt.exceptions
+ 
+ try:
+@@ -102,9 +103,11 @@ class LazyDict(MutableMapping):
+             # load the item
+             if self._load(key):
+                 log.debug('LazyLoaded %s', key)
++                time.sleep(0.0001)
+                 return self._dict[key]
+             else:
+                 log.debug('Could not LazyLoad %s: %s', key, self.missing_fun_string(key))
++                time.sleep(0.0001)
+                 raise KeyError(key)
+         else:
+             return self._dict[key]
+-- 
+2.23.0
+
+

salt.changes

@@ -1,3 +1,25 @@
+-------------------------------------------------------------------
+Fri May  8 14:24:19 UTC 2020 - Jochen Breuer <jbreuer@suse.de>
+
+- Python 3.8 compatibility changes
+- msgpack support for version >= 1.0.0 (bsc#1171257)
+
+- Added:
+  * python3.8-compatibility-pr-s-235.patch
+  * msgpack-support-versions-1.0.0.patch
+
+-------------------------------------------------------------------
+Thu May  7 15:36:38 UTC 2020 - Pablo Suárez Hernández <pablo.suarezhernandez@suse.com>
+
+- Prevent sporious "salt-api" stuck processes when managing SSH minions
+  because of logging deadlock (bsc#1159284)
+- Avoid segfault from "salt-api" under certain conditions of heavy load
+  managing SSH minions (bsc#1169604)
+
+- Added:
+  * prevent-logging-deadlock-on-salt-api-subprocesses-bs.patch
+  * make-lazyloader.__init__-call-to-_refresh_file_mappi.patch
+
 -------------------------------------------------------------------
 Thu Apr 30 13:24:35 UTC 2020 - Pablo Suárez Hernández <pablo.suarezhernandez@suse.com>
 

salt.spec

@@ -300,6 +300,14 @@ Patch107:     fixed-bug-lvm-has-no-parttion-type.-the-scipt-later-.patch
 Patch108:     remove-vendored-backports-abc-from-requirements.patch
 # PATCH-FIX_OPENSUSE: https://github.com/openSUSE/salt/commit/a5ef829408685d9e65eaa24bba40d221adffaa95
 Patch109:     fix-typo-in-minion_runner-for-aesfuncs-exposed-metho.patch
+# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/57119
+Patch110:     make-lazyloader.__init__-call-to-_refresh_file_mappi.patch
+# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/57123
+Patch111:     prevent-logging-deadlock-on-salt-api-subprocesses-bs.patch
+# PATCH-FIX_UPSTREAM: https://github.com/saltstack/salt/pull/57122
+Patch112:     msgpack-support-versions-1.0.0.patch
+# PATCH-FIX_OPENSUSE: https://github.com/openSUSE/salt/pull/235
+Patch113:     python3.8-compatibility-pr-s-235.patch
 
 BuildRoot:      %{_tmppath}/%{name}-%{version}-build
 BuildRequires:  logrotate
@@ -916,6 +924,10 @@ cp %{S:5} ./.travis.yml
 %patch107 -p1
 %patch108 -p1
 %patch109 -p1
+%patch110 -p1
+%patch111 -p1
+%patch112 -p1
+%patch113 -p1
 
 %build
 %if 0%{?build_py2}