saltbundle/venv-salt-minion
SHA256
7
0
Fork 2
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
bundle_testing
venv-salt-minion/fix-salt-for-python-3.11.patch
Pablo Suárez Hernández f673ca273e Fix Salt for Python > 3.11 (bsc#1252285) (bsc#1252244) 
* Use external tornado on Python > 3.11
* Make tls and x509 to use python-cryptography
* Remove usage of spwd
2025-11-10 09:23:21 +00:00

6675 lines
253 KiB
Diff
Raw Permalink Blame History

From e784a3d4d35e3e54804e212752ae8eed789b2142 Mon Sep 17 00:00:00 2001
From: Marek Czernek <marek.czernek@suse.com>
Date: Fri, 7 Nov 2025 17:12:31 +0100
Subject: [PATCH] Fix Salt for Python > 3.11
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* Do not use ConfigParser.readfp in nilrt_ip module
* Use external tornado with newest Python
* Migrate batch_async to external tornado
* Mutating locals is unsupported in Py >=3.13
* modules/tls.py: Use cryptography for create_pkcs12
* modules/x509: Use cryptography for create_crl
* Remove use of spwd, removed module in py3.13
* Fix issues around closing IOLoop
* Fix error if future is already done
* Fix io_loop closing in tpc.py and http.py
Partial cherry-pick of opensuse/salt@c4677c6eb4532212406cdffd935e66a97c68490f
* Remove unexpected callback keyword
* Fix fileclient inability to download a file
* Remove mentions of StackContext
* Ensure current_ioloop works for vendored and ext tornado
* Fix test_linux_shadow tests
Cherry-pick of https://github.com/saltstack/salt/pull/68402
* Test saltnado fail fix
* Fix test_base_api_handler test
* Fix tests_webhook_handler test
* Fix salt fileclient for old and new tornado
* Raise Python version for vendored tornado
* Add compatibility with StackContext
* Implement fixme in tls
* Apply cosmetic suggestion
Co-authored-by: Victor Zhestkov <vzhestkov@suse.com>
---------
Co-authored-by: Alexander Graul <agraul@suse.com>
Co-authored-by: Toyam Cox <Vaelatern@voidlinux.org>
Co-authored-by: Pablo Suárez Hernández <psuarezhernandez@suse.com>
Co-authored-by: Victor Zhestkov <vzhestkov@suse.com>
---
salt/__init__.py | 2 +-
salt/channel/client.py | 53 +--
salt/channel/server.py | 43 +--
salt/cli/batch_async.py | 38 +--
salt/client/__init__.py | 15 +-
salt/client/mixins.py | 14 +-
salt/crypt.py | 23 +-
salt/engines/ircbot.py | 15 +-
salt/engines/webhook.py | 15 +-
salt/fileclient.py | 9 +-
salt/master.py | 11 +-
salt/metaproxy/deltaproxy.py | 15 +-
salt/metaproxy/proxy.py | 14 +-
salt/minion.py | 97 +++---
salt/modules/linux_shadow.py | 29 +-
salt/modules/nilrt_ip.py | 6 +-
salt/modules/tls.py | 78 +++--
salt/modules/x509.py | 82 +----
salt/netapi/rest_tornado/__init__.py | 12 +-
salt/netapi/rest_tornado/saltnado.py | 94 +++---
.../rest_tornado/saltnado_websockets.py | 10 +-
salt/pillar/__init__.py | 10 +-
salt/transport/base.py | 6 +-
salt/transport/ipc.py | 96 +++---
salt/transport/tcp.py | 113 ++++---
salt/transport/zeromq.py | 40 +--
salt/utils/asynchronous.py | 106 +++++-
salt/utils/event.py | 26 +-
salt/utils/gitfs.py | 4 +-
salt/utils/http.py | 27 +-
salt/utils/process.py | 2 +-
salt/utils/thin.py | 8 +-
tests/integration/minion/test_minion_cache.py | 2 +-
tests/integration/modules/test_gem.py | 2 +-
.../netapi/rest_tornado/test_app.py | 10 +-
tests/pytests/conftest.py | 6 +-
.../functional/channel/test_req_channel.py | 10 +-
.../pytests/functional/channel/test_server.py | 8 +-
.../netapi/rest_cherrypy/conftest.py | 4 +-
.../netapi/rest_cherrypy/test_auth.py | 2 +-
.../netapi/rest_cherrypy/test_auth_pam.py | 2 +-
.../netapi/rest_cherrypy/test_out_formats.py | 2 +-
.../netapi/rest_tornado/test_auth_handler.py | 2 +-
.../rest_tornado/test_auth_handler_pam.py | 2 +-
.../rest_tornado/test_base_api_handler.py | 11 +-
.../netapi/rest_tornado/test_utils.py | 4 +-
.../rest_tornado/test_websockets_handler.py | 4 +-
.../functional/transport/ipc/test_client.py | 2 +-
.../transport/ipc/test_subscriber.py | 8 +-
.../transport/server/test_req_channel.py | 10 +-
.../transport/tcp/test_message_client.py | 18 +-
.../utils/test_async_event_publisher.py | 2 +-
.../netapi/rest_cherrypy/conftest.py | 4 +-
.../netapi/rest_cherrypy/test_auth.py | 2 +-
.../netapi/rest_cherrypy/test_run.py | 2 +-
.../rest_tornado/test_events_api_handler.py | 4 +-
.../rest_tornado/test_minions_api_handler.py | 2 +-
.../netapi/rest_tornado/test_root_handler.py | 2 +-
tests/pytests/unit/channel/test_server.py | 4 +-
tests/pytests/unit/cli/test_batch_async.py | 40 +--
.../unit/fileserver/gitfs/test_gitfs.py | 4 +-
.../fileserver/gitfs/test_gitfs_config.py | 4 +-
.../pytests/unit/modules/test_linux_shadow.py | 24 +-
tests/pytests/unit/test_ext_importers.py | 51 ---
tests/pytests/unit/test_minion.py | 50 +--
tests/pytests/unit/transport/test_ipc.py | 4 +-
tests/pytests/unit/transport/test_tcp.py | 28 +-
tests/pytests/unit/transport/test_zeromq.py | 44 +--
tests/pytests/unit/utils/event/test_event.py | 16 +-
.../unit/utils/event/test_event_return.py | 2 +-
tests/pytests/unit/utils/test_asynchronous.py | 12 +-
tests/support/helpers.py | 20 +-
tests/support/netapi.py | 16 +-
tests/support/pytest/transport.py | 20 +-
tests/unit/modules/test_random_org.py | 2 +-
.../unit/netapi/rest_tornado/test_saltnado.py | 314 +++++++++---------
tests/unit/test_proxy_minion.py | 10 +-
tests/unit/transport/mixins.py | 6 +-
tests/unit/transport/test_ipc.py | 12 +-
tests/unit/transport/test_tcp.py | 10 +-
tests/unit/utils/test_context.py | 165 +--------
tests/unit/utils/test_gitfs.py | 4 +-
tests/unit/utils/test_http.py | 2 +-
83 files changed, 1000 insertions(+), 1104 deletions(-)
delete mode 100644 tests/pytests/unit/test_ext_importers.py
diff --git a/salt/__init__.py b/salt/__init__.py
index b5fe3677c22..16d5063fa87 100644
--- a/salt/__init__.py
+++ b/salt/__init__.py
@@ -12,7 +12,7 @@ if sys.version_info < (3,):
)
sys.stderr.flush()
-USE_VENDORED_TORNADO = True
+USE_VENDORED_TORNADO = sys.version_info < (3,12)
class TornadoImporter:
diff --git a/salt/channel/client.py b/salt/channel/client.py
index 25f4af7689d..56878333ed8 100644
--- a/salt/channel/client.py
+++ b/salt/channel/client.py
@@ -10,10 +10,11 @@ import os
import time
import uuid
+import tornado.gen
+import tornado.ioloop
+
import salt.crypt
import salt.exceptions
-import salt.ext.tornado.gen
-import salt.ext.tornado.ioloop
import salt.payload
import salt.transport.frame
import salt.utils.event
@@ -121,7 +122,7 @@ class AsyncReqChannel:
opts["master_uri"] = kwargs["master_uri"]
io_loop = kwargs.get("io_loop")
if io_loop is None:
- io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+ io_loop = tornado.ioloop.IOLoop.current()
timeout = opts.get("request_channel_timeout", REQUEST_CHANNEL_TIMEOUT)
tries = opts.get("request_channel_tries", REQUEST_CHANNEL_TRIES)
@@ -201,7 +202,7 @@ class AsyncReqChannel:
ret["id"] = self.opts["id"]
return ret
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _send_with_retry(self, load, tries, timeout):
_try = 1
while True:
@@ -218,9 +219,9 @@ class AsyncReqChannel:
else:
_try += 1
continue
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def crypted_transfer_decode_dictentry(
self,
load,
@@ -274,12 +275,12 @@ class AsyncReqChannel:
# Validate the nonce.
if data["nonce"] != nonce:
raise salt.crypt.AuthenticationError("Pillar nonce verification failed.")
- raise salt.ext.tornado.gen.Return(data["pillar"])
+ raise tornado.gen.Return(data["pillar"])
def verify_signature(self, data, sig):
return salt.crypt.verify_signature(self.master_pubkey_path, data, sig)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _crypted_transfer(self, load, timeout, raw=False):
"""
Send a load across the wire, with encryption
@@ -294,7 +295,7 @@ class AsyncReqChannel:
:param int timeout: The number of seconds on a response before failing
"""
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _do_transfer():
# Yield control to the caller. When send() completes, resume by populating data with the Future.result
nonce = uuid.uuid4().hex
@@ -311,7 +312,7 @@ class AsyncReqChannel:
if not raw or self.ttype == "tcp": # XXX Why is this needed for tcp
data = salt.transport.frame.decode_embedded_strs(data)
- raise salt.ext.tornado.gen.Return(data)
+ raise tornado.gen.Return(data)
if not self.auth.authenticated:
# Return control back to the caller, resume when authentication succeeds
@@ -323,9 +324,9 @@ class AsyncReqChannel:
# If auth error, return control back to the caller, continue when authentication succeeds
yield self.auth.authenticate()
ret = yield _do_transfer()
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _uncrypted_transfer(self, load, timeout):
"""
Send a load across the wire in cleartext
@@ -338,9 +339,9 @@ class AsyncReqChannel:
timeout=timeout,
)
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def send(self, load, tries=None, timeout=None, raw=False):
"""
Send a request, return a future which will complete when we send the message
@@ -370,7 +371,7 @@ class AsyncReqChannel:
else:
_try += 1
continue
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
def close(self):
"""
@@ -424,7 +425,7 @@ class AsyncPubChannel:
io_loop = kwargs.get("io_loop")
if io_loop is None:
- io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+ io_loop = tornado.ioloop.IOLoop.current()
auth = salt.crypt.AsyncAuth(opts, io_loop=io_loop)
transport = salt.transport.publish_client(opts, io_loop)
@@ -445,7 +446,7 @@ class AsyncPubChannel:
def crypt(self):
return "aes" if self.auth else "clear"
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def connect(self):
"""
Return a future which completes when connected to the remote publisher
@@ -487,7 +488,7 @@ class AsyncPubChannel:
if callback is None:
return self.transport.on_recv(None)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def wrap_callback(messages):
payload = yield self.transport._decode_messages(messages)
decoded = yield self._decode_payload(payload)
@@ -504,7 +505,7 @@ class AsyncPubChannel:
"version": 3,
}
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def send_id(self, tok, force_auth):
"""
Send the minion id to the master so that the master may better
@@ -514,13 +515,13 @@ class AsyncPubChannel:
"""
load = {"id": self.opts["id"], "tok": tok}
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _do_transfer():
msg = self._package_load(self.auth.crypticle.dumps(load))
package = salt.transport.frame.frame_msg(msg, header=None)
yield self.transport.send(package)
- raise salt.ext.tornado.gen.Return(True)
+ raise tornado.gen.Return(True)
if force_auth or not self.auth.authenticated:
count = 0
@@ -536,13 +537,13 @@ class AsyncPubChannel:
count += 1
try:
ret = yield _do_transfer()
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
except salt.crypt.AuthenticationError:
yield self.auth.authenticate()
ret = yield _do_transfer()
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def connect_callback(self, result):
if self._closing:
return
@@ -614,7 +615,7 @@ class AsyncPubChannel:
"Message signature failed to validate."
)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _decode_payload(self, payload):
# we need to decrypt it
log.trace("Decoding payload: %s", payload)
@@ -626,7 +627,7 @@ class AsyncPubChannel:
yield self.auth.authenticate()
payload["load"] = self.auth.crypticle.loads(payload["load"])
- raise salt.ext.tornado.gen.Return(payload)
+ raise tornado.gen.Return(payload)
def __enter__(self):
return self
diff --git a/salt/channel/server.py b/salt/channel/server.py
index 1d7cc9c577f..a9151440c2b 100644
--- a/salt/channel/server.py
+++ b/salt/channel/server.py
@@ -12,8 +12,9 @@ import pathlib
import shutil
import time
+import tornado.gen
+
import salt.crypt
-import salt.ext.tornado.gen
import salt.master
import salt.payload
import salt.transport.frame
@@ -128,7 +129,7 @@ class ReqServerChannel:
if hasattr(self.transport, "post_fork"):
self.transport.post_fork(self.handle_message, io_loop)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def handle_message(self, payload):
if (
not isinstance(payload, dict)
@@ -136,7 +137,7 @@ class ReqServerChannel:
or "load" not in payload
):
log.warn("bad load received on socket")
- raise salt.ext.tornado.gen.Return("bad load")
+ raise tornado.gen.Return("bad load")
version = payload.get("version", 0)
try:
payload = self._decode_payload(payload, version)
@@ -151,7 +152,7 @@ class ReqServerChannel:
)
else:
log.error("Bad load from minion: %s: %s", exc_type, exc)
- raise salt.ext.tornado.gen.Return("bad load")
+ raise tornado.gen.Return("bad load")
# TODO helper functions to normalize payload?
if not isinstance(payload, dict) or not isinstance(payload.get("load"), dict):
@@ -160,16 +161,16 @@ class ReqServerChannel:
payload,
payload.get("load"),
)
- raise salt.ext.tornado.gen.Return("payload and load must be a dict")
+ raise tornado.gen.Return("payload and load must be a dict")
try:
id_ = payload["load"].get("id", "")
if "\0" in id_:
log.error("Payload contains an id with a null byte: %s", payload)
- raise salt.ext.tornado.gen.Return("bad load: id contains a null byte")
+ raise tornado.gen.Return("bad load: id contains a null byte")
except TypeError:
log.error("Payload contains non-string id: %s", payload)
- raise salt.ext.tornado.gen.Return(
+ raise tornado.gen.Return(
"bad load: id {} is not a string".format(id_)
)
@@ -184,7 +185,7 @@ class ReqServerChannel:
ret = self._auth(payload["load"], sign_messages, version)
if self.opts.get("master_stats", False):
yield self.payload_handler({"cmd": "_auth", "_start": start})
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
if payload["enc"] == "aes":
nonce = None
@@ -202,7 +203,7 @@ class ReqServerChannel:
ttl,
self.opts["request_server_ttl"],
)
- raise salt.ext.tornado.gen.Return("bad load")
+ raise tornado.gen.Return("bad load")
if payload["id"] != payload["load"]["id"]:
log.warning(
@@ -210,18 +211,18 @@ class ReqServerChannel:
payload["load"]["id"],
payload["id"],
)
- raise salt.ext.tornado.gen.Return("bad load")
+ raise tornado.gen.Return("bad load")
if not salt.utils.verify.valid_id(self.opts, payload["load"]["id"]):
log.warning(
"Request contains invalid minion id '%s'", payload["load"]["id"]
)
- raise salt.ext.tornado.gen.Return("bad load")
+ raise tornado.gen.Return("bad load")
if not self.validate_token(payload, required=True):
- raise salt.ext.tornado.gen.Return("bad load")
+ raise tornado.gen.Return("bad load")
# The token won't always be present in the payload for v2 and
# below, but if it is we always wanto validate it.
elif not self.validate_token(payload, required=False):
- raise salt.ext.tornado.gen.Return("bad load")
+ raise tornado.gen.Return("bad load")
# TODO: test
try:
@@ -231,22 +232,22 @@ class ReqServerChannel:
except Exception as e: # pylint: disable=broad-except
# always attempt to return an error to the minion
log.error("Some exception handling a payload from minion", exc_info=True)
- raise salt.ext.tornado.gen.Return("Some exception handling minion payload")
+ raise tornado.gen.Return("Some exception handling minion payload")
req_fun = req_opts.get("fun", "send")
if req_fun == "send_clear":
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
elif req_fun == "send":
if version > 2:
- raise salt.ext.tornado.gen.Return(
+ raise tornado.gen.Return(
salt.crypt.Crypticle(self.opts, self.session_key(id_)).dumps(
ret, nonce
)
)
else:
- raise salt.ext.tornado.gen.Return(self.crypticle.dumps(ret, nonce))
+ raise tornado.gen.Return(self.crypticle.dumps(ret, nonce))
elif req_fun == "send_private":
- raise salt.ext.tornado.gen.Return(
+ raise tornado.gen.Return(
self._encrypt_private(
ret,
req_opts["key"],
@@ -257,7 +258,7 @@ class ReqServerChannel:
)
log.error("Unknown req_fun %s", req_fun)
# always attempt to return an error to the minion
- raise salt.ext.tornado.gen.Return("Server-side exception handling payload")
+ raise tornado.gen.Return("Server-side exception handling payload")
def _encrypt_private(
self,
@@ -985,7 +986,7 @@ class PubServerChannel:
data, salt.utils.event.tagify("present", "presence")
)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def publish_payload(self, load, *args):
unpacked_package = self.wrap_payload(load)
try:
@@ -998,7 +999,7 @@ class PubServerChannel:
ret = yield self.transport.publish_payload(payload, topic_list)
else:
ret = yield self.transport.publish_payload(payload)
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
def wrap_payload(self, load):
payload = {"enc": "aes"}
diff --git a/salt/cli/batch_async.py b/salt/cli/batch_async.py
index 92215d0e04a..9e0ecddda54 100644
--- a/salt/cli/batch_async.py
+++ b/salt/cli/batch_async.py
@@ -6,10 +6,10 @@ import logging
import re
import salt.client
-import salt.ext.tornado
+import tornado
import salt.utils.event
from salt.cli.batch import batch_get_eauth, batch_get_opts, get_bnum
-from salt.ext.tornado.iostream import StreamClosedError
+from tornado.iostream import StreamClosedError
log = logging.getLogger(__name__)
@@ -111,14 +111,14 @@ class SharedEventsChannel:
if not self._subscribers[subscriber_id]:
del self._subscribers[subscriber_id]
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def __handle_close(self):
if not self._subscriptions:
return
log.warning("Master Event Subscriber was closed. Trying to reconnect...")
yield self.__reconnect_subscriber()
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def __handle_event(self, raw):
if self.master_event is None:
return
@@ -138,7 +138,7 @@ class SharedEventsChannel:
exc_info=True,
)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def __reconnect_subscriber(self):
if self.master_event.subscriber.connected() or self._reconnecting_subscriber:
return
@@ -167,7 +167,7 @@ class SharedEventsChannel:
self._reconnecting_subscriber = False
return
if _try < max_tries:
- yield salt.ext.tornado.gen.sleep(self._subscriber_reconnect_interval)
+ yield tornado.gen.sleep(self._subscriber_reconnect_interval)
_try += 1
self._reconnecting_subscriber = False
@@ -229,7 +229,7 @@ class BatchAsync:
self.extra_job_kwargs[kwarg] = kwargs[kwarg]
elif kwarg in opts:
self.extra_job_kwargs[kwarg] = opts[kwarg]
- self.io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+ self.io_loop = tornado.ioloop.IOLoop.current()
self.events_channel = _get_shared_events_channel(opts, self.io_loop).use(
id(self)
)
@@ -278,7 +278,7 @@ class BatchAsync:
self.batch_jid, "batch_run", id(self), self.__event_handler
)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def __event_handler(self, tag, data, op):
# IMPORTANT: This function must run fast and not wait for any other task,
# otherwise it would cause events to be stuck.
@@ -321,7 +321,7 @@ class BatchAsync:
)
return set(list(to_run)[:next_batch_size])
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def check_find_job(self, batch_minions, jid):
"""
Check if the job with specified ``jid`` was finished on the minions
@@ -346,7 +346,7 @@ class BatchAsync:
self.find_job_returned = self.find_job_returned.difference(running)
yield self.find_job(running)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def find_job(self, minions):
"""
Find if the job was finished on the minions
@@ -377,7 +377,7 @@ class BatchAsync:
listen=False,
**self.eauth,
)
- yield salt.ext.tornado.gen.sleep(self.opts["gather_job_timeout"])
+ yield tornado.gen.sleep(self.opts["gather_job_timeout"])
if self.event:
yield self.check_find_job(not_done, jid)
except Exception as ex: # pylint: disable=W0703
@@ -388,7 +388,7 @@ class BatchAsync:
)
self.close_safe()
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def start(self):
"""
Start the batch execution
@@ -419,7 +419,7 @@ class BatchAsync:
self.start_batch,
)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def start_batch(self):
"""
Fire `salt/batch/*/start` and continue batch with `run_next`
@@ -442,7 +442,7 @@ class BatchAsync:
if self.event:
yield self.run_next()
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def end_batch(self):
"""
End the batch and call safe closing
@@ -469,7 +469,7 @@ class BatchAsync:
# release to the IOLoop to allow the event to be published
# before closing batch async execution
- yield salt.ext.tornado.gen.sleep(0.03)
+ yield tornado.gen.sleep(0.03)
self.close_safe()
def close_safe(self):
@@ -481,7 +481,7 @@ class BatchAsync:
_destroy_unused_shared_events_channel()
self.event = None
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def schedule_next(self):
log.trace("[%s] BatchAsync.schedule_next called", self.batch_jid)
if self.scheduled:
@@ -498,11 +498,11 @@ class BatchAsync:
self.batch_jid,
self.batch_delay,
)
- yield salt.ext.tornado.gen.sleep(self.batch_delay)
+ yield tornado.gen.sleep(self.batch_delay)
if self.event:
yield self.run_next()
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def run_next(self):
"""
Continue batch execution with the next targets
@@ -535,7 +535,7 @@ class BatchAsync:
**self.extra_job_kwargs,
)
- yield salt.ext.tornado.gen.sleep(self.opts["timeout"])
+ yield tornado.gen.sleep(self.opts["timeout"])
# The batch can be done already at this point, which means no self.event
if self.event and self.active.intersection(next_batch):
diff --git a/salt/client/__init__.py b/salt/client/__init__.py
index b2617e4554d..be5939e58f6 100644
--- a/salt/client/__init__.py
+++ b/salt/client/__init__.py
@@ -25,11 +25,12 @@ import sys
import time
from datetime import datetime
+import tornado.gen
+
import salt.cache
import salt.channel.client
import salt.config
import salt.defaults.exitcodes
-import salt.ext.tornado.gen
import salt.loader
import salt.payload
import salt.syspaths as syspaths
@@ -416,7 +417,7 @@ class LocalClient:
)
return _res["minions"]
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def run_job_async(
self,
tgt,
@@ -473,7 +474,7 @@ class LocalClient:
# Convert to generic client error and pass along message
raise SaltClientError(general_exception)
- raise salt.ext.tornado.gen.Return(self._check_pub_data(pub_data, listen=listen))
+ raise tornado.gen.Return(self._check_pub_data(pub_data, listen=listen))
def cmd_async(
self, tgt, fun, arg=(), tgt_type="glob", ret="", jid="", kwarg=None, **kwargs
@@ -1924,7 +1925,7 @@ class LocalClient:
return {"jid": payload["load"]["jid"], "minions": payload["load"]["minions"]}
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def pub_async(
self,
tgt,
@@ -2005,7 +2006,7 @@ class LocalClient:
# and try again if the key has changed
key = self.__read_master_key()
if key == self.key:
- raise salt.ext.tornado.gen.Return(payload)
+ raise tornado.gen.Return(payload)
self.key = key
payload_kwargs["key"] = self.key
payload = yield channel.send(payload_kwargs)
@@ -2023,9 +2024,9 @@ class LocalClient:
raise PublishError(error)
if not payload:
- raise salt.ext.tornado.gen.Return(payload)
+ raise tornado.gen.Return(payload)
- raise salt.ext.tornado.gen.Return(
+ raise tornado.gen.Return(
{"jid": payload["load"]["jid"], "minions": payload["load"]["minions"]}
)
diff --git a/salt/client/mixins.py b/salt/client/mixins.py
index 7cdae88ae8a..431d67d1b9b 100644
--- a/salt/client/mixins.py
+++ b/salt/client/mixins.py
@@ -14,7 +14,6 @@ from collections.abc import Mapping, MutableMapping
import salt._logging
import salt.channel.client
import salt.exceptions
-import salt.ext.tornado.stack_context
import salt.minion
import salt.output
import salt.utils.args
@@ -30,6 +29,12 @@ import salt.utils.state
import salt.utils.user
import salt.utils.versions
+from salt import USE_VENDORED_TORNADO
+if USE_VENDORED_TORNADO:
+ from salt.ext.tornado.stack_context import StackContext
+else:
+ from contextlib import nullcontext as StackContext
+
log = logging.getLogger(__name__)
CLIENT_INTERNAL_KEYWORDS = frozenset(
@@ -379,10 +384,7 @@ class SyncClientMixin(ClientStateMixin):
data["fun_args"] = list(args) + ([kwargs] if kwargs else [])
func_globals["__jid_event__"].fire_event(data, "new")
- # Initialize a context for executing the method.
- with salt.ext.tornado.stack_context.StackContext(
- self.functions.context_dict.clone
- ):
+ with StackContext(self.functions.context_dict.clone):
func = self.functions[fun]
try:
data["return"] = func(*args, **kwargs)
@@ -394,6 +396,7 @@ class SyncClientMixin(ClientStateMixin):
)
try:
data["success"] = self.context.get("retcode", 0) == 0
+
except AttributeError:
# Assume a True result if no context attribute
data["success"] = True
@@ -402,6 +405,7 @@ class SyncClientMixin(ClientStateMixin):
data["success"] = salt.utils.state.check_result(
data["return"]["data"]
)
+
except (Exception, SystemExit) as ex: # pylint: disable=broad-except
if isinstance(ex, salt.exceptions.NotImplemented):
data["return"] = str(ex)
diff --git a/salt/crypt.py b/salt/crypt.py
index 29fd159b48c..f1214caebba 100644
--- a/salt/crypt.py
+++ b/salt/crypt.py
@@ -22,9 +22,10 @@ import traceback
import uuid
import weakref
+import tornado.gen
+
import salt.channel.client
import salt.defaults.exitcodes
-import salt.ext.tornado.gen
import salt.payload
import salt.utils.crypt
import salt.utils.decorators
@@ -497,7 +498,7 @@ class AsyncAuth:
Only create one instance of AsyncAuth per __key()
"""
# do we have any mapping for this io_loop
- io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current()
+ io_loop = io_loop or tornado.ioloop.IOLoop.current()
if io_loop not in AsyncAuth.instance_map:
AsyncAuth.instance_map[io_loop] = weakref.WeakValueDictionary()
loop_instance_map = AsyncAuth.instance_map[io_loop]
@@ -548,7 +549,7 @@ class AsyncAuth:
if not os.path.isfile(self.pub_path):
self.get_keys()
- self.io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current()
+ self.io_loop = io_loop or tornado.ioloop.IOLoop.current()
salt.utils.crypt.reinit_crypto()
key = self.__key(self.opts)
@@ -558,7 +559,7 @@ class AsyncAuth:
self._creds = creds
self._crypticle = Crypticle(self.opts, creds["aes"])
self._session_crypticle = Crypticle(self.opts, creds["session"])
- self._authenticate_future = salt.ext.tornado.concurrent.Future()
+ self._authenticate_future = tornado.concurrent.Future()
self._authenticate_future.set_result(True)
else:
self.authenticate()
@@ -617,7 +618,7 @@ class AsyncAuth:
):
future = self._authenticate_future
else:
- future = salt.ext.tornado.concurrent.Future()
+ future = tornado.concurrent.Future()
self._authenticate_future = future
self.io_loop.add_callback(self._authenticate)
@@ -631,7 +632,7 @@ class AsyncAuth:
return future
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _authenticate(self):
"""
Authenticate with the master, this method breaks the functional
@@ -682,7 +683,7 @@ class AsyncAuth:
log.info(
"Waiting %s seconds before retry.", acceptance_wait_time
)
- yield salt.ext.tornado.gen.sleep(acceptance_wait_time)
+ yield tornado.gen.sleep(acceptance_wait_time)
if acceptance_wait_time < acceptance_wait_time_max:
acceptance_wait_time += acceptance_wait_time
log.debug(
@@ -733,7 +734,7 @@ class AsyncAuth:
salt.utils.event.tagify(prefix="auth", suffix="creds"),
)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def sign_in(self, timeout=60, safe=True, tries=1, channel=None):
"""
Send a sign in request to the master, sets the key information and
@@ -774,9 +775,9 @@ class AsyncAuth:
except SaltReqTimeoutError as e:
if safe:
log.warning("SaltReqTimeoutError: %s", e)
- raise salt.ext.tornado.gen.Return("retry")
+ raise tornado.gen.Return("retry")
if self.opts.get("detect_mode") is True:
- raise salt.ext.tornado.gen.Return("retry")
+ raise tornado.gen.Return("retry")
else:
raise SaltClientError(
"Attempt to authenticate with the salt master failed with timeout"
@@ -786,7 +787,7 @@ class AsyncAuth:
if close_channel:
channel.close()
ret = self.handle_signin_response(sign_in_payload, payload)
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
def handle_signin_response(self, sign_in_payload, payload):
auth = {}
diff --git a/salt/engines/ircbot.py b/salt/engines/ircbot.py
index 1dab78dbbc5..982c860c262 100644
--- a/salt/engines/ircbot.py
+++ b/salt/engines/ircbot.py
@@ -61,8 +61,9 @@ import socket
import ssl
from collections import namedtuple
-import salt.ext.tornado.ioloop
-import salt.ext.tornado.iostream
+import tornado.ioloop
+import tornado.iostream
+
import salt.utils.event
log = logging.getLogger(__name__)
@@ -101,18 +102,18 @@ class IRCClient:
self.allow_hosts = allow_hosts
self.allow_nicks = allow_nicks
self.disable_query = disable_query
- self.io_loop = salt.ext.tornado.ioloop.IOLoop(make_current=False)
+ self.io_loop = tornado.ioloop.IOLoop(make_current=False)
self.io_loop.make_current()
self._connect()
def _connect(self):
_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
if self.ssl is True:
- self._stream = salt.ext.tornado.iostream.SSLIOStream(
+ self._stream = tornado.iostream.SSLIOStream(
_sock, ssl_options={"cert_reqs": ssl.CERT_NONE}
)
else:
- self._stream = salt.ext.tornado.iostream.IOStream(_sock)
+ self._stream = tornado.iostream.IOStream(_sock)
self._stream.set_close_callback(self.on_closed)
self._stream.connect((self.host, self.port), self.on_connect)
@@ -218,11 +219,11 @@ class IRCClient:
event = self._event(raw)
if event.code == "PING":
- salt.ext.tornado.ioloop.IOLoop.current().spawn_callback(
+ tornado.ioloop.IOLoop.current().spawn_callback(
self.send_message, "PONG {}".format(event.line)
)
elif event.code == "PRIVMSG":
- salt.ext.tornado.ioloop.IOLoop.current().spawn_callback(
+ tornado.ioloop.IOLoop.current().spawn_callback(
self._privmsg, event
)
self.read_messages()
diff --git a/salt/engines/webhook.py b/salt/engines/webhook.py
index 1424d4a27dc..fdd2ca2dfbb 100644
--- a/salt/engines/webhook.py
+++ b/salt/engines/webhook.py
@@ -2,9 +2,10 @@
Send events from webhook api
"""
-import salt.ext.tornado.httpserver
-import salt.ext.tornado.ioloop
-import salt.ext.tornado.web
+import tornado.httpserver
+import tornado.ioloop
+import tornado.web
+
import salt.utils.event
@@ -64,7 +65,7 @@ def start(address=None, port=5000, ssl_crt=None, ssl_key=None):
__salt__["event.send"](tag, msg)
class WebHook(
- salt.ext.tornado.web.RequestHandler
+ tornado.web.RequestHandler
): # pylint: disable=abstract-method
def post(self, tag): # pylint: disable=arguments-differ
body = self.request.body
@@ -75,13 +76,13 @@ def start(address=None, port=5000, ssl_crt=None, ssl_key=None):
}
fire("salt/engines/hook/" + tag, payload)
- application = salt.ext.tornado.web.Application([(r"/(.*)", WebHook)])
+ application = tornado.web.Application([(r"/(.*)", WebHook)])
ssl_options = None
if all([ssl_crt, ssl_key]):
ssl_options = {"certfile": ssl_crt, "keyfile": ssl_key}
- io_loop = salt.ext.tornado.ioloop.IOLoop(make_current=False)
+ io_loop = tornado.ioloop.IOLoop(make_current=False)
io_loop.make_current()
- http_server = salt.ext.tornado.httpserver.HTTPServer(
+ http_server = tornado.httpserver.HTTPServer(
application, ssl_options=ssl_options
)
http_server.listen(port, address=address)
diff --git a/salt/fileclient.py b/salt/fileclient.py
index 4ed341221cd..82e0fc2a824 100644
--- a/salt/fileclient.py
+++ b/salt/fileclient.py
@@ -12,6 +12,9 @@ import string
import urllib.error
import urllib.parse
+from salt import USE_VENDORED_TORNADO
+from tornado.httputil import HTTPHeaders, HTTPInputError, parse_response_start_line
+
import salt.channel.client
import salt.client
import salt.crypt
@@ -32,11 +35,6 @@ import salt.utils.url
import salt.utils.verify
import salt.utils.versions
from salt.exceptions import CommandExecutionError, MinionError
-from salt.ext.tornado.httputil import (
- HTTPHeaders,
- HTTPInputError,
- parse_response_start_line,
-)
from salt.utils.openstack.swift import SaltSwift
log = logging.getLogger(__name__)
@@ -747,6 +745,7 @@ class Client:
# Check the status line of the HTTP request
if write_body[0] is None:
try:
+ hdr = hdr if USE_VENDORED_TORNADO else hdr.strip()
hdr = parse_response_start_line(hdr)
except HTTPInputError:
# Not the first line, do nothing
diff --git a/salt/master.py b/salt/master.py
index ba7c751d4b4..09ce7d36a7e 100644
--- a/salt/master.py
+++ b/salt/master.py
@@ -17,6 +17,8 @@ import sys
import threading
import time
+import tornado.gen
+
import salt.acl
import salt.auth
import salt.channel.server
@@ -27,7 +29,6 @@ import salt.daemons.masterapi
import salt.defaults.exitcodes
import salt.engines
import salt.exceptions
-import salt.ext.tornado.gen
import salt.key
import salt.minion
import salt.payload
@@ -1002,7 +1003,7 @@ class MWorker(salt.utils.process.SignalHandlingProcess):
"""
Bind to the local port
"""
- self.io_loop = salt.ext.tornado.ioloop.IOLoop()
+ self.io_loop = tornado.ioloop.IOLoop()
self.io_loop.make_current()
for req_channel in self.req_channels:
req_channel.post_fork(
@@ -1014,7 +1015,7 @@ class MWorker(salt.utils.process.SignalHandlingProcess):
# Tornado knows what to do
pass
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _handle_payload(self, payload):
"""
The _handle_payload method is the key method used to figure out what
@@ -1047,7 +1048,7 @@ class MWorker(salt.utils.process.SignalHandlingProcess):
ret = self._handle_aes(load)
else:
ret = self._handle_clear(load)
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
def _post_stats(self, start, cmd):
"""
@@ -2137,7 +2138,7 @@ class ClearFuncs(TransportMethods):
lambda: self._prep_jid(clear_load, {}),
batch_load,
)
- ioloop = salt.ext.tornado.ioloop.IOLoop.current()
+ ioloop = tornado.ioloop.IOLoop.current()
ioloop.add_callback(batch.start)
return {
diff --git a/salt/metaproxy/deltaproxy.py b/salt/metaproxy/deltaproxy.py
index c3003b368f7..fe71c321ea2 100644
--- a/salt/metaproxy/deltaproxy.py
+++ b/salt/metaproxy/deltaproxy.py
@@ -10,6 +10,9 @@ import threading
import traceback
import types
+import tornado.gen # pylint: disable=F0401
+import tornado.ioloop # pylint: disable=F0401
+
import salt
import salt._logging
import salt.beacons
@@ -19,8 +22,6 @@ import salt.config
import salt.crypt
import salt.defaults.exitcodes
import salt.engines
-import salt.ext.tornado.gen # pylint: disable=F0401
-import salt.ext.tornado.ioloop # pylint: disable=F0401
import salt.loader
import salt.minion
import salt.payload
@@ -55,6 +56,12 @@ from salt.minion import ProxyMinion
from salt.utils.event import tagify
from salt.utils.process import SignalHandlingProcess, default_signals
+from salt import USE_VENDORED_TORNADO
+if USE_VENDORED_TORNADO:
+ from salt.ext.tornado.stack_context import StackContext
+else:
+ from contextlib import nullcontext as StackContext
+
log = logging.getLogger(__name__)
@@ -561,7 +568,7 @@ def target(cls, minion_instance, opts, data, connected):
uid = salt.utils.user.get_uid(user=opts.get("user", None))
minion_instance.proc_dir = salt.minion.get_proc_dir(opts["cachedir"], uid=uid)
- with salt.ext.tornado.stack_context.StackContext(minion_instance.ctx):
+ with StackContext(minion_instance.ctx):
if isinstance(data["fun"], tuple) or isinstance(data["fun"], list):
ProxyMinion._thread_multi_return(minion_instance, opts, data)
else:
@@ -1014,7 +1021,7 @@ def handle_decoded_payload(self, data):
data["jid"],
)
once_logged = True
- yield salt.ext.tornado.gen.sleep(0.5)
+ yield tornado.gen.sleep(0.5)
process_count = self.subprocess_list.count
# We stash an instance references to allow for the socket
diff --git a/salt/metaproxy/proxy.py b/salt/metaproxy/proxy.py
index a399c15ef16..4699e4b3de3 100644
--- a/salt/metaproxy/proxy.py
+++ b/salt/metaproxy/proxy.py
@@ -16,8 +16,8 @@ import salt.client
import salt.crypt
import salt.defaults.exitcodes
import salt.engines
-import salt.ext.tornado.gen # pylint: disable=F0401
-import salt.ext.tornado.ioloop # pylint: disable=F0401
+import tornado.gen # pylint: disable=F0401
+import tornado.ioloop # pylint: disable=F0401
import salt.loader
import salt.minion
import salt.payload
@@ -52,6 +52,12 @@ from salt.minion import ProxyMinion
from salt.utils.event import tagify
from salt.utils.process import SignalHandlingProcess, default_signals
+from salt import USE_VENDORED_TORNADO
+if USE_VENDORED_TORNADO:
+ from salt.ext.tornado.stack_context import StackContext
+else:
+ from contextlib import nullcontext as StackContext
+
log = logging.getLogger(__name__)
@@ -380,7 +386,7 @@ def target(cls, minion_instance, opts, data, connected):
opts["cachedir"], uid=uid
)
- with salt.ext.tornado.stack_context.StackContext(minion_instance.ctx):
+ with StackContext(minion_instance.ctx):
if isinstance(data["fun"], tuple) or isinstance(data["fun"], list):
ProxyMinion._thread_multi_return(minion_instance, opts, data)
else:
@@ -804,7 +810,7 @@ def handle_decoded_payload(self, data):
"Maximum number of processes reached while executing jid %s, waiting...",
data["jid"],
)
- yield salt.ext.tornado.gen.sleep(10)
+ yield tornado.gen.sleep(10)
process_count = len(salt.utils.minion.running(self.opts))
# We stash an instance references to allow for the socket
diff --git a/salt/minion.py b/salt/minion.py
index d9201e20109..805e893341c 100644
--- a/salt/minion.py
+++ b/salt/minion.py
@@ -26,9 +26,9 @@ import salt.crypt
import salt.defaults.events
import salt.defaults.exitcodes
import salt.engines
-import salt.ext.tornado
-import salt.ext.tornado.gen
-import salt.ext.tornado.ioloop
+import tornado
+import tornado.gen
+import tornado.ioloop
import salt.loader
import salt.loader.lazy
import salt.payload
@@ -101,6 +101,11 @@ try:
except ImportError:
HAS_WIN_FUNCTIONS = False
+from salt import USE_VENDORED_TORNADO
+if USE_VENDORED_TORNADO:
+ from salt.ext.tornado.stack_context import ExceptionStackContext
+else:
+ from contextlib import nullcontext as ExceptionStackContext
log = logging.getLogger(__name__)
@@ -510,7 +515,7 @@ class MinionBase:
) # pylint: disable=no-member
return []
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def eval_master(self, opts, timeout=60, safe=True, failed=False, failback=False):
"""
Evaluates and returns a tuple of the current master address and the pub_channel.
@@ -531,7 +536,7 @@ class MinionBase:
if opts["master_type"] == "disable":
log.warning("Master is set to disable, skipping connection")
self.connected = False
- raise salt.ext.tornado.gen.Return((None, None))
+ raise tornado.gen.Return((None, None))
# Run masters discovery over SSDP. This may modify the whole configuration,
# depending of the networking and sets of masters.
@@ -701,7 +706,7 @@ class MinionBase:
if attempts != 0:
# Give up a little time between connection attempts
# to allow the IOLoop to run any other scheduled tasks.
- yield salt.ext.tornado.gen.sleep(opts["acceptance_wait_time"])
+ yield tornado.gen.sleep(opts["acceptance_wait_time"])
attempts += 1
if tries > 0:
log.debug("Connecting to master. Attempt %s of %s", attempts, tries)
@@ -770,7 +775,7 @@ class MinionBase:
else:
self.tok = pub_channel.auth.gen_token(b"salt")
self.connected = True
- raise salt.ext.tornado.gen.Return((opts["master"], pub_channel))
+ raise tornado.gen.Return((opts["master"], pub_channel))
# single master sign in
else:
@@ -784,7 +789,7 @@ class MinionBase:
if attempts != 0:
# Give up a little time between connection attempts
# to allow the IOLoop to run any other scheduled tasks.
- yield salt.ext.tornado.gen.sleep(opts["acceptance_wait_time"])
+ yield tornado.gen.sleep(opts["acceptance_wait_time"])
attempts += 1
if tries > 0:
log.debug("Connecting to master. Attempt %s of %s", attempts, tries)
@@ -816,7 +821,7 @@ class MinionBase:
yield pub_channel.connect()
self.tok = pub_channel.auth.gen_token(b"salt")
self.connected = True
- raise salt.ext.tornado.gen.Return((opts["master"], pub_channel))
+ raise tornado.gen.Return((opts["master"], pub_channel))
except SaltClientError:
if pub_channel:
pub_channel.close()
@@ -924,7 +929,7 @@ class SMinion(MinionBase):
if self.opts.get("file_client", "remote") == "remote" or self.opts.get(
"use_master_when_local", False
):
- io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+ io_loop = tornado.ioloop.IOLoop.current()
io_loop.run_sync(lambda: self.eval_master(self.opts, failed=True))
self.gen_modules(initial_load=True, context=context)
@@ -1027,7 +1032,7 @@ class MinionManager(MinionBase):
self.minions = []
self.jid_queue = []
- self.io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+ self.io_loop = tornado.ioloop.IOLoop.current()
self.process_manager = ProcessManager(name="MultiMinionProcessManager")
self.io_loop.spawn_callback(
self.process_manager.run, **{"asynchronous": True}
@@ -1053,7 +1058,7 @@ class MinionManager(MinionBase):
self.event.subscribe("")
self.event.set_event_handler(self.handle_event)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def handle_event(self, package):
for minion in self.minions:
minion.handle_event(package)
@@ -1117,7 +1122,7 @@ class MinionManager(MinionBase):
self.io_loop.spawn_callback(self._connect_minion, minion)
self.io_loop.call_later(timeout, self._check_minions)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _connect_minion(self, minion):
"""
Create a minion, and asynchronously connect it to a master
@@ -1149,7 +1154,7 @@ class MinionManager(MinionBase):
last = time.time()
if auth_wait < self.max_auth_wait:
auth_wait += self.auth_wait
- yield salt.ext.tornado.gen.sleep(auth_wait) # TODO: log?
+ yield tornado.gen.sleep(auth_wait) # TODO: log?
except SaltMasterUnresolvableError:
err = (
"Master address: '{}' could not be resolved. Invalid or"
@@ -1166,7 +1171,7 @@ class MinionManager(MinionBase):
minion.opts["master"],
exc_info=True,
)
- yield salt.ext.tornado.gen.sleep(retry_wait)
+ yield tornado.gen.sleep(retry_wait)
if retry_wait < max_retry_wait:
retry_wait += retry_wait_inc
@@ -1255,7 +1260,7 @@ class Minion(MinionBase):
self.periodic_callbacks = {}
if io_loop is None:
- self.io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+ self.io_loop = tornado.ioloop.IOLoop.current()
else:
self.io_loop = io_loop
@@ -1358,7 +1363,7 @@ class Minion(MinionBase):
if timeout and self._sync_connect_master_success is False:
raise SaltDaemonNotRunning("Failed to connect to the salt-master")
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def connect_master(self, failed=False):
"""
Return a future which will complete when you are connected to a master
@@ -1369,7 +1374,7 @@ class Minion(MinionBase):
yield self._post_master_init(master)
# TODO: better name...
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _post_master_init(self, master):
"""
Function to finish init after connecting to a master
@@ -1596,7 +1601,7 @@ class Minion(MinionBase):
load, timeout=timeout, tries=self.opts["return_retry_tries"]
)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _send_req_async(self, load, timeout):
if self.opts["minion_sign_messages"]:
log.trace("Signing event to be published onto the bus.")
@@ -1610,7 +1615,7 @@ class Minion(MinionBase):
ret = yield channel.send(
load, timeout=timeout, tries=self.opts["return_retry_tries"]
)
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
def _fire_master(
self,
@@ -1675,13 +1680,13 @@ class Minion(MinionBase):
timeout_handler = handle_timeout
- with salt.ext.tornado.stack_context.ExceptionStackContext(timeout_handler):
+ with ExceptionStackContext(timeout_handler):
# pylint: disable=unexpected-keyword-arg
- self._send_req_async(load, timeout, callback=lambda f: None)
+ self._send_req_async(load, timeout)
# pylint: enable=unexpected-keyword-arg
return True
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _handle_decoded_payload(self, data):
"""
Override this method if you wish to handle the decoded data
@@ -1739,7 +1744,7 @@ class Minion(MinionBase):
" waiting...",
data["jid"],
)
- yield salt.ext.tornado.gen.sleep(10)
+ yield tornado.gen.sleep(10)
process_count = len(salt.utils.minion.running(self.opts))
# We stash an instance references to allow for the socket
@@ -2252,10 +2257,10 @@ class Minion(MinionBase):
timeout_handler()
return ""
else:
- with salt.ext.tornado.stack_context.ExceptionStackContext(timeout_handler):
+ with ExceptionStackContext(timeout_handler):
# pylint: disable=unexpected-keyword-arg
ret_val = self._send_req_async(
- load, timeout=timeout, callback=lambda f: None
+ load, timeout=timeout
)
# pylint: enable=unexpected-keyword-arg
@@ -2343,10 +2348,10 @@ class Minion(MinionBase):
timeout_handler()
return ""
else:
- with salt.ext.tornado.stack_context.ExceptionStackContext(timeout_handler):
+ with ExceptionStackContext(timeout_handler):
# pylint: disable=unexpected-keyword-arg
ret_val = self._send_req_async(
- load, timeout=timeout, callback=lambda f: None
+ load, timeout=timeout
)
# pylint: enable=unexpected-keyword-arg
@@ -2483,7 +2488,7 @@ class Minion(MinionBase):
return pillar_schedule
# TODO: only allow one future in flight at a time?
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def pillar_refresh(self, force_refresh=False, clean_cache=False):
"""
Refresh the pillar
@@ -2671,19 +2676,19 @@ class Minion(MinionBase):
log.warning("Unable to send mine data to master.")
return None
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def handle_event(self, package):
"""
Handle an event from the epull_sock (all local minion events)
"""
if not self.ready:
- raise salt.ext.tornado.gen.Return()
+ raise tornado.gen.Return()
tag, data = salt.utils.event.SaltEvent.unpack(package)
if "proxy_target" in data and self.opts.get("metaproxy") == "deltaproxy":
proxy_target = data["proxy_target"]
if proxy_target not in self.deltaproxy_objs:
- raise salt.ext.tornado.gen.Return()
+ raise tornado.gen.Return()
_minion = self.deltaproxy_objs[proxy_target]
else:
_minion = self
@@ -2746,7 +2751,7 @@ class Minion(MinionBase):
and data["master"] != self.opts["master"]
):
# not mine master, ignore
- raise salt.ext.tornado.gen.Return()
+ raise tornado.gen.Return()
if tag.startswith(master_event(type="failback")):
# if the master failback event is not for the top master, raise an exception
if data["master"] != self.opts["master_list"][0]:
@@ -3035,7 +3040,7 @@ class Minion(MinionBase):
"""
if name in self.periodic_callbacks:
return False
- self.periodic_callbacks[name] = salt.ext.tornado.ioloop.PeriodicCallback(
+ self.periodic_callbacks[name] = tornado.ioloop.PeriodicCallback(
method,
interval * 1000,
)
@@ -3273,7 +3278,7 @@ class Syndic(Minion):
log.warning("Unable to forward pub data: %s", args[1])
return True
- with salt.ext.tornado.stack_context.ExceptionStackContext(timeout_handler):
+ with ExceptionStackContext(timeout_handler):
self.local.pub_async(
data["tgt"],
data["fun"],
@@ -3299,7 +3304,7 @@ class Syndic(Minion):
load, timeout=timeout, tries=self.opts["return_retry_tries"]
)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _send_req_async(self, load, timeout):
if self.opts["minion_sign_messages"]:
log.trace("Signing event to be published onto the bus.")
@@ -3353,7 +3358,7 @@ class Syndic(Minion):
# In the future, we could add support for some clearfuncs, but
# the syndic currently has no need.
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def reconnect(self):
if hasattr(self, "pub_channel"):
self.pub_channel.on_recv(None)
@@ -3370,7 +3375,7 @@ class Syndic(Minion):
self.pub_channel.on_recv(self._process_cmd_socket)
log.info("Minion is ready to receive requests!")
- raise salt.ext.tornado.gen.Return(self)
+ raise tornado.gen.Return(self)
def destroy(self):
"""
@@ -3429,7 +3434,7 @@ class SyndicManager(MinionBase):
self.jid_forward_cache = set()
if io_loop is None:
- self.io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+ self.io_loop = tornado.ioloop.IOLoop.current()
else:
self.io_loop = io_loop
@@ -3456,7 +3461,7 @@ class SyndicManager(MinionBase):
s_opts["master"] = master
self._syndics[master] = self._connect_syndic(s_opts)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _connect_syndic(self, opts):
"""
Create a syndic, and asynchronously connect it to a master
@@ -3492,7 +3497,7 @@ class SyndicManager(MinionBase):
last = time.time()
if auth_wait < self.max_auth_wait:
auth_wait += self.auth_wait
- yield salt.ext.tornado.gen.sleep(auth_wait) # TODO: log?
+ yield tornado.gen.sleep(auth_wait) # TODO: log?
except (KeyboardInterrupt, SystemExit): # pylint: disable=try-except-raise
raise
except Exception: # pylint: disable=broad-except
@@ -3503,7 +3508,7 @@ class SyndicManager(MinionBase):
exc_info=True,
)
- raise salt.ext.tornado.gen.Return(syndic)
+ raise tornado.gen.Return(syndic)
def _mark_master_dead(self, master):
"""
@@ -3635,7 +3640,7 @@ class SyndicManager(MinionBase):
self.io_loop.add_future(future, self.reconnect_event_bus)
# forward events every syndic_event_forward_timeout
- self.forward_events = salt.ext.tornado.ioloop.PeriodicCallback(
+ self.forward_events = tornado.ioloop.PeriodicCallback(
self._forward_events,
self.opts["syndic_event_forward_timeout"] * 1000,
)
@@ -3790,7 +3795,7 @@ class ProxyMinion(Minion):
"""
# TODO: better name...
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _post_master_init(self, master):
"""
Function to finish init after connecting to a master
@@ -3808,7 +3813,7 @@ class ProxyMinion(Minion):
mp_call = _metaproxy_call(self.opts, "post_master_init")
return mp_call(self, master)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def subproxy_post_master_init(self, minion_id, uid):
"""
Function to finish init for the sub proxies
@@ -3837,7 +3842,7 @@ class ProxyMinion(Minion):
mp_call = _metaproxy_call(self.opts, "handle_payload")
return mp_call(self, payload)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _handle_decoded_payload(self, data):
mp_call = _metaproxy_call(self.opts, "handle_decoded_payload")
return mp_call(self, data)
diff --git a/salt/modules/linux_shadow.py b/salt/modules/linux_shadow.py
index aa149ac4c8e..0104b6299b1 100644
--- a/salt/modules/linux_shadow.py
+++ b/salt/modules/linux_shadow.py
@@ -8,6 +8,7 @@ Manage the shadow file on Linux systems
<module-provider-override>`.
"""
+import collections
import datetime
import functools
import logging
@@ -17,11 +18,6 @@ import salt.utils.data
import salt.utils.files
from salt.exceptions import CommandExecutionError
-try:
- import spwd
-except ImportError:
- pass
-
try:
import salt.utils.pycrypto
@@ -34,6 +30,21 @@ __virtualname__ = "shadow"
log = logging.getLogger(__name__)
+struct_spwd = collections.namedtuple(
+ "struct_spwd",
+ [
+ "sp_namp",
+ "sp_pwdp",
+ "sp_lstchg",
+ "sp_min",
+ "sp_max",
+ "sp_warn",
+ "sp_inact",
+ "sp_expire",
+ "sp_flag",
+ ]
+)
+
def __virtual__():
return __virtualname__ if __grains__.get("kernel", "") == "Linux" else False
@@ -71,7 +82,7 @@ def info(name, root=None):
if root is not None:
getspnam = functools.partial(_getspnam, root=root)
else:
- getspnam = functools.partial(spwd.getspnam)
+ getspnam = functools.partial(_getspnam, root="/")
try:
data = getspnam(name)
@@ -509,7 +520,7 @@ def list_users(root=None):
if root is not None:
getspall = functools.partial(_getspall, root=root)
else:
- getspall = functools.partial(spwd.getspall)
+ getspall = functools.partial(_getspall, root="/")
return sorted(
user.sp_namp if hasattr(user, "sp_namp") else user.sp_nam for user in getspall()
@@ -529,7 +540,7 @@ def _getspnam(name, root=None):
# Generate a getspnam compatible output
for i in range(2, 9):
comps[i] = int(comps[i]) if comps[i] else -1
- return spwd.struct_spwd(comps)
+ return struct_spwd(*comps)
raise KeyError
@@ -545,4 +556,4 @@ def _getspall(root=None):
# Generate a getspall compatible output
for i in range(2, 9):
comps[i] = int(comps[i]) if comps[i] else -1
- yield spwd.struct_spwd(comps)
+ yield struct_spwd(*comps)
diff --git a/salt/modules/nilrt_ip.py b/salt/modules/nilrt_ip.py
index 84612d7a17d..d29e3f118c2 100644
--- a/salt/modules/nilrt_ip.py
+++ b/salt/modules/nilrt_ip.py
@@ -271,7 +271,7 @@ def _load_config(section, options, default_value="", filename=INI_FILE):
config_parser = configparser.RawConfigParser(
dict_type=CaseInsensitiveDict, converters={"unquoted": _remove_quotes}
)
- config_parser.readfp(config_file)
+ config_parser.read_file(config_file)
for option in options:
results[option] = config_parser.getunquoted(
section, option, fallback=default_value
@@ -576,7 +576,7 @@ def _change_dhcp_config(interface, enable_dhcp=True, filename=INTERFACES_CONFIG)
if os.path.exists(filename):
try:
with salt.utils.files.fopen(filename, "r") as config_file:
- parser.readfp(config_file)
+ parser.read_file(config_file)
except configparser.MissingSectionHeaderError:
pass
interface = pyiface.Interface(name=interface)
@@ -858,7 +858,7 @@ def _configure_static_interface(interface, **settings):
if os.path.exists(INTERFACES_CONFIG):
try:
with salt.utils.files.fopen(INTERFACES_CONFIG, "r") as config_file:
- parser.readfp(config_file)
+ parser.read_file(config_file)
except configparser.MissingSectionHeaderError:
pass
hwaddr = interface.hwaddr[:-1]
diff --git a/salt/modules/tls.py b/salt/modules/tls.py
index b74b765cfbb..9d29bd1e9bd 100644
--- a/salt/modules/tls.py
+++ b/salt/modules/tls.py
@@ -120,6 +120,11 @@ HAS_SSL = False
X509_EXT_ENABLED = True
try:
import OpenSSL
+ import cryptography.x509
+ import cryptography.exceptions
+
+ import cryptography.hazmat.primitives.serialization as cryptography_serialization
+ import cryptography.hazmat.primitives.serialization.pkcs12 as cryptography_pkcs12
HAS_SSL = True
OpenSSL_version = Version(OpenSSL.__dict__.get("__version__", "0.0"))
@@ -1590,49 +1595,52 @@ def create_pkcs12(ca_name, CN, passphrase="", cacert_path=None, replace=False):
salt '*' tls.create_pkcs12 test localhost
"""
set_ca_path(cacert_path)
- if not replace and os.path.exists(
- "{}/{}/certs/{}.p12".format(cert_base_path(), ca_name, CN)
- ):
+ p12_path = f"{cert_base_path()}/{ca_name}/certs/{CN}.p12"
+ ca_cert_path = f"{cert_base_path()}/{ca_name}/{ca_name}_ca_cert.crt"
+ cert_path = f"{cert_base_path()}/{ca_name}/certs/{CN}.crt"
+ priv_key_path = f"{cert_base_path()}/{ca_name}/certs/{CN}.key"
+
+ if not replace and os.path.exists(p12_path):
return 'Certificate "{}" already exists'.format(CN)
try:
- with salt.utils.files.fopen(
- "{0}/{1}/{1}_ca_cert.crt".format(cert_base_path(), ca_name)
- ) as fhr:
- ca_cert = OpenSSL.crypto.load_certificate(
- OpenSSL.crypto.FILETYPE_PEM, fhr.read()
- )
+ with salt.utils.files.fopen(ca_cert_path, "rb") as fhr:
+ ca_cert = cryptography.x509.load_pem_x509_certificate(fhr.read())
except OSError:
return 'There is no CA named "{}"'.format(ca_name)
+ except ValueError as e:
+ return f'Could not load CA {ca_cert_path}: {e}'
try:
- with salt.utils.files.fopen(
- "{}/{}/certs/{}.crt".format(cert_base_path(), ca_name, CN)
- ) as fhr:
- cert = OpenSSL.crypto.load_certificate(
- OpenSSL.crypto.FILETYPE_PEM, fhr.read()
- )
- with salt.utils.files.fopen(
- "{}/{}/certs/{}.key".format(cert_base_path(), ca_name, CN)
- ) as fhr:
- key = OpenSSL.crypto.load_privatekey(
- OpenSSL.crypto.FILETYPE_PEM, fhr.read()
+ with salt.utils.files.fopen(cert_path, "rb") as fhr:
+ cert = cryptography.x509.load_pem_x509_certificate(fhr.read())
+ with salt.utils.files.fopen(priv_key_path, "rb") as fhr:
+ key = cryptography_serialization.load_pem_private_key(
+ fhr.read(),
+ password=None,
)
except OSError:
return 'There is no certificate that matches the CN "{}"'.format(CN)
+ except ValueError as e:
+ return f'Could not load certificate {cert_path}: {e}'
- pkcs12 = OpenSSL.crypto.PKCS12()
-
- pkcs12.set_certificate(cert)
- pkcs12.set_ca_certificates([ca_cert])
- pkcs12.set_privatekey(key)
-
- with salt.utils.files.fopen(
- "{}/{}/certs/{}.p12".format(cert_base_path(), ca_name, CN), "wb"
- ) as ofile:
- ofile.write(
- pkcs12.export(passphrase=salt.utils.stringutils.to_bytes(passphrase))
+ if passphrase:
+ encryption_algorithm = cryptography_serialization.BestAvailableEncryption(
+ salt.utils.stringutils.to_bytes(passphrase)
)
+ else:
+ encryption_algorithm = cryptography_serialization.NoEncryption()
+
+ pkcs12 = cryptography_pkcs12.serialize_key_and_certificates(
+ name=salt.utils.stringutils.to_bytes(CN),
+ key=key,
+ cert=cert,
+ cas=[ca_cert],
+ encryption_algorithm=encryption_algorithm,
+ )
+
+ with salt.utils.files.fopen(p12_path, "wb") as ofile:
+ ofile.write(pkcs12)
return 'Created PKCS#12 Certificate for "{0}": "{1}/{2}/certs/{0}.p12"'.format(
CN,
@@ -1929,10 +1937,10 @@ def revoke_cert(
)
except ValueError:
ret["retcode"] = 1
- ret[
- "comment"
- ] = "Revocation date '{}' does not matchformat '{}'".format(
- revoke_date, two_digit_year_fmt
+ ret["comment"] = (
+ "Revocation date '{}' does not matchformat '{}'".format(
+ revoke_date, two_digit_year_fmt
+ )
)
return ret
elif index_serial_subject in line:
diff --git a/salt/modules/x509.py b/salt/modules/x509.py
index 9687f63d1bb..164541fc761 100644
--- a/salt/modules/x509.py
+++ b/salt/modules/x509.py
@@ -32,11 +32,13 @@ import tempfile
import salt.exceptions
import salt.utils.data
+import salt.utils.dictupdate
import salt.utils.files
import salt.utils.path
import salt.utils.platform
import salt.utils.stringutils
import salt.utils.versions
+import salt.utils.x509 as x509util
from salt.state import STATE_INTERNAL_KEYWORDS as _STATE_INTERNAL_KEYWORDS
from salt.utils.odict import OrderedDict
@@ -983,86 +985,38 @@ def create_crl(
raise salt.exceptions.SaltInvocationError(
"Could not load OpenSSL module, OpenSSL unavailable"
)
- crl = OpenSSL.crypto.CRL()
if revoked is None:
revoked = []
for rev_item in revoked:
- if "certificate" in rev_item:
- rev_cert = read_certificate(rev_item["certificate"])
- rev_item["serial_number"] = rev_cert["Serial Number"]
- rev_item["not_after"] = rev_cert["Not After"]
-
- serial_number = rev_item["serial_number"].replace(":", "")
- # OpenSSL bindings requires this to be a non-unicode string
- serial_number = salt.utils.stringutils.to_bytes(serial_number)
-
- if "not_after" in rev_item and not include_expired:
- not_after = datetime.datetime.strptime(
- rev_item["not_after"], "%Y-%m-%d %H:%M:%S"
- )
- if datetime.datetime.now() > not_after:
- continue
-
- if "revocation_date" not in rev_item:
- rev_item["revocation_date"] = datetime.datetime.now().strftime(
- "%Y-%m-%d %H:%M:%S"
- )
-
- rev_date = datetime.datetime.strptime(
- rev_item["revocation_date"], "%Y-%m-%d %H:%M:%S"
- )
- rev_date = rev_date.strftime("%Y%m%d%H%M%SZ")
- rev_date = salt.utils.stringutils.to_bytes(rev_date)
-
- rev = OpenSSL.crypto.Revoked()
- rev.set_serial(salt.utils.stringutils.to_bytes(serial_number))
- rev.set_rev_date(salt.utils.stringutils.to_bytes(rev_date))
-
if "reason" in rev_item:
- # Same here for OpenSSL bindings and non-unicode strings
- reason = salt.utils.stringutils.to_bytes(rev_item["reason"])
- rev.set_reason(reason)
-
- crl.add_revoked(rev)
+ salt.utils.dictupdate.set_dict_key_value(
+ rev_item, "extensions:CRLReason", rev_item["reason"]
+ )
- signing_cert = _text_or_file(signing_cert)
- cert = OpenSSL.crypto.load_certificate(
- OpenSSL.crypto.FILETYPE_PEM, get_pem_entry(signing_cert, pem_type="CERTIFICATE")
- )
- signing_private_key = _get_private_key_obj(
- signing_private_key, passphrase=signing_private_key_passphrase
- ).as_pem(cipher=None)
- key = OpenSSL.crypto.load_privatekey(
- OpenSSL.crypto.FILETYPE_PEM, get_pem_entry(signing_private_key)
+ builder, private_key_obj = x509util.build_crl(
+ signing_private_key=signing_private_key,
+ signing_private_key_passphrase=signing_private_key_passphrase,
+ include_expired=include_expired,
+ revoked=revoked,
+ signing_cert=signing_cert,
+ days_valid=days_valid,
)
- export_kwargs = {
- "cert": cert,
- "key": key,
- "type": OpenSSL.crypto.FILETYPE_PEM,
- "days": days_valid,
- }
if digest:
- export_kwargs["digest"] = salt.utils.stringutils.to_bytes(digest)
+ hashing_algorithm = x509util.get_hashing_algorithm(digest)
else:
log.warning("No digest specified. The default md5 digest will be used.")
+ hashing_algorithm = x509util.get_hashing_algorithm("MD5")
- try:
- crltext = crl.export(**export_kwargs)
- except (TypeError, ValueError):
- log.warning(
- "Error signing crl with specified digest. Are you using "
- "pyopenssl 0.15 or newer? The default md5 digest will be used."
- )
- export_kwargs.pop("digest", None)
- crltext = crl.export(**export_kwargs)
+ crl = builder.sign(private_key_obj, algorithm=hashing_algorithm)
+ crl_bytes = crl.public_bytes(x509util.serialization.Encoding.PEM)
if text:
- return crltext
+ return crl_bytes.decode()
- return write_pem(text=crltext, path=path, pem_type="X509 CRL")
+ return write_pem(text=crl_bytes, path=path, pem_type="X509 CRL")
def sign_remote_certificate(argdic, **kwargs):
diff --git a/salt/netapi/rest_tornado/__init__.py b/salt/netapi/rest_tornado/__init__.py
index 67336d0adaa..b4f5fb4422b 100644
--- a/salt/netapi/rest_tornado/__init__.py
+++ b/salt/netapi/rest_tornado/__init__.py
@@ -13,9 +13,9 @@ log = logging.getLogger(__virtualname__)
min_tornado_version = "4.0"
has_tornado = False
try:
- import salt.ext.tornado
+ import tornado
- if Version(salt.ext.tornado.version) >= Version(min_tornado_version):
+ if Version(tornado.version) >= Version(min_tornado_version):
has_tornado = True
else:
log.error("rest_tornado requires at least tornado %s", min_tornado_version)
@@ -74,7 +74,7 @@ def get_application(opts):
(formatted_events_pattern, saltnado_websockets.FormattedEventsHandler),
]
- application = salt.ext.tornado.web.Application(
+ application = tornado.web.Application(
paths, debug=mod_opts.get("debug", False)
)
@@ -117,9 +117,9 @@ def start():
ssl_opts.update({"keyfile": mod_opts["ssl_key"]})
kwargs["ssl_options"] = ssl_opts
- import salt.ext.tornado.httpserver
+ import tornado.httpserver
- http_server = salt.ext.tornado.httpserver.HTTPServer(
+ http_server = tornado.httpserver.HTTPServer(
get_application(__opts__), **kwargs
)
try:
@@ -136,6 +136,6 @@ def start():
raise SystemExit(1)
try:
- salt.ext.tornado.ioloop.IOLoop.current().start()
+ tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt:
raise SystemExit(0)
diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py
index 44c4089ccbb..ec7e16a36ee 100644
--- a/salt/netapi/rest_tornado/saltnado.py
+++ b/salt/netapi/rest_tornado/saltnado.py
@@ -194,11 +194,11 @@ from copy import copy
import salt.auth
import salt.client
-import salt.ext.tornado.escape
-import salt.ext.tornado.gen
-import salt.ext.tornado.httpserver
-import salt.ext.tornado.ioloop
-import salt.ext.tornado.web
+import tornado.escape
+import tornado.gen
+import tornado.httpserver
+import tornado.ioloop
+import tornado.web
import salt.netapi
import salt.runner
import salt.utils.args
@@ -211,7 +211,7 @@ from salt.exceptions import (
AuthorizationError,
EauthAuthenticationError,
)
-from salt.ext.tornado.concurrent import Future
+from tornado.concurrent import Future
from salt.utils.event import tagify
_json = salt.utils.json.import_json()
@@ -277,7 +277,7 @@ class EventListener:
opts["sock_dir"],
opts=opts,
listen=True,
- io_loop=salt.ext.tornado.ioloop.IOLoop.current(),
+ io_loop=tornado.ioloop.IOLoop.current(),
)
# tag -> list of futures
@@ -302,7 +302,7 @@ class EventListener:
self._timeout_future(tag, matcher, future)
# remove the timeout
if future in self.timeout_map:
- salt.ext.tornado.ioloop.IOLoop.current().remove_timeout(
+ tornado.ioloop.IOLoop.current().remove_timeout(
self.timeout_map[future]
)
del self.timeout_map[future]
@@ -336,7 +336,7 @@ class EventListener:
if callback is not None:
def handle_future(future):
- salt.ext.tornado.ioloop.IOLoop.current().add_callback(
+ tornado.ioloop.IOLoop.current().add_callback(
callback, future
) # pylint: disable=E1102
@@ -346,7 +346,7 @@ class EventListener:
self.request_map[request].append((tag, matcher, future))
if timeout:
- timeout_future = salt.ext.tornado.ioloop.IOLoop.current().call_later(
+ timeout_future = tornado.ioloop.IOLoop.current().call_later(
timeout, self._timeout_future, tag, matcher, future
)
self.timeout_map[future] = timeout_future
@@ -391,18 +391,22 @@ class EventListener:
future.set_result({"data": data, "tag": mtag})
self.tag_map[(tag, matcher)].remove(future)
if future in self.timeout_map:
- salt.ext.tornado.ioloop.IOLoop.current().remove_timeout(
+ tornado.ioloop.IOLoop.current().remove_timeout(
self.timeout_map[future]
)
del self.timeout_map[future]
-class BaseSaltAPIHandler(salt.ext.tornado.web.RequestHandler): # pylint: disable=W0223
+class BaseSaltAPIHandler(tornado.web.RequestHandler): # pylint: disable=W0223
ct_out_map = (
("application/json", _json_dumps),
("application/x-yaml", salt.utils.yaml.safe_dump),
)
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._auto_finish = False
+
def _verify_client(self, low):
"""
Verify that the client is in fact one we have
@@ -555,7 +559,7 @@ class BaseSaltAPIHandler(salt.ext.tornado.web.RequestHandler): # pylint: disabl
try:
# Use cgi.parse_header to correctly separate parameters from value
value, parameters = cgi.parse_header(self.request.headers["Content-Type"])
- return ct_in_map[value](salt.ext.tornado.escape.native_str(data))
+ return ct_in_map[value](tornado.escape.native_str(data))
except KeyError:
self.send_error(406)
except ValueError:
@@ -659,6 +663,7 @@ class SaltAuthHandler(BaseSaltAPIHandler): # pylint: disable=W0223
ret = {"status": "401 Unauthorized", "return": "Please log in"}
self.write(self.serialize(ret))
+ self.finish()
# TODO: make asynchronous? Underlying library isn't... and we ARE making disk calls :(
def post(self): # pylint: disable=arguments-differ
@@ -785,6 +790,7 @@ class SaltAuthHandler(BaseSaltAPIHandler): # pylint: disable=W0223
}
self.write(self.serialize(ret))
+ self.finish()
class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
@@ -828,8 +834,9 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
"""
ret = {"clients": list(self.saltclients.keys()), "return": "Welcome"}
self.write(self.serialize(ret))
+ self.finish()
- @salt.ext.tornado.web.asynchronous
+ @tornado.gen.coroutine
def post(self): # pylint: disable=arguments-differ
"""
Send one or more Salt commands (lowstates) in the request body
@@ -907,7 +914,7 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
self.disbatch()
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def disbatch(self):
"""
Disbatch all lowstates to the appropriate clients
@@ -950,7 +957,7 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
except RuntimeError:
pass # Do we need any logging here?
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def get_minion_returns(
self, events, is_finished, is_timed_out, min_wait_time, minions
):
@@ -980,11 +987,11 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
# When finished entire routine, cleanup other futures and return result
if f is is_finished or f is is_timed_out:
cancel_inflight_futures()
- raise salt.ext.tornado.gen.Return(chunk_ret)
+ raise tornado.gen.Return(chunk_ret)
elif f is min_wait_time:
if not more_todo():
cancel_inflight_futures()
- raise salt.ext.tornado.gen.Return(chunk_ret)
+ raise tornado.gen.Return(chunk_ret)
continue
f_result = f.result()
@@ -1000,7 +1007,7 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
# if there are no more minions to wait for, then we are done
if not more_todo() and min_wait_time.done():
cancel_inflight_futures()
- raise salt.ext.tornado.gen.Return(chunk_ret)
+ raise tornado.gen.Return(chunk_ret)
except TimeoutException:
pass
@@ -1008,7 +1015,7 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
if f in events:
events.remove(f)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _disbatch_local(self, chunk):
"""
Dispatch local client commands
@@ -1057,7 +1064,7 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
future.set_result(None)
except Exception: # pylint: disable=broad-except
pass
- raise salt.ext.tornado.gen.Return(
+ raise tornado.gen.Return(
"No minions matched the target. No command was sent, no jid was"
" assigned."
)
@@ -1078,19 +1085,19 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
# wait syndic a while to avoid missing published events
if self.application.opts["order_masters"]:
- min_wait_time = salt.ext.tornado.gen.sleep(
+ min_wait_time = tornado.gen.sleep(
self.application.opts["syndic_wait"]
)
# To ensure job_not_running and all_return are terminated by each other, communicate using a future
- is_finished = salt.ext.tornado.gen.Future()
- is_timed_out = salt.ext.tornado.gen.sleep(
+ is_finished = tornado.gen.Future()
+ is_timed_out = tornado.gen.sleep(
self.application.opts["gather_job_timeout"]
)
# ping until the job is not running, while doing so, if we see new minions returning
# that they are running the job, add them to the list
- salt.ext.tornado.ioloop.IOLoop.current().spawn_callback(
+ tornado.ioloop.IOLoop.current().spawn_callback(
self.job_not_running,
pub_data["jid"],
chunk["tgt"],
@@ -1106,9 +1113,9 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
min_wait_time=min_wait_time,
minions=minions,
)
- raise salt.ext.tornado.gen.Return(result)
+ raise tornado.gen.Return(result)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def job_not_running(self, jid, tgt, tgt_type, minions, is_finished):
"""
Return a future which will complete once jid (passed in) is no longer
@@ -1133,11 +1140,11 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
if f is is_finished:
if not event.done():
event.set_result(None)
- raise salt.ext.tornado.gen.Return(True)
+ raise tornado.gen.Return(True)
event = f.result()
except TimeoutException:
if not minion_running or is_finished.done():
- raise salt.ext.tornado.gen.Return(True)
+ raise tornado.gen.Return(True)
else:
ping_pub_data = yield self.saltclients["local"](
tgt, "saltutil.find_job", [jid], tgt_type=tgt_type
@@ -1153,7 +1160,7 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
minions[event["data"]["id"]] = False
minion_running = True
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _disbatch_local_async(self, chunk):
"""
Disbatch local client_async commands
@@ -1164,9 +1171,9 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
*f_call.get("args", ()), **f_call.get("kwargs", {})
)
- raise salt.ext.tornado.gen.Return(pub_data)
+ raise tornado.gen.Return(pub_data)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _disbatch_runner(self, chunk):
"""
Disbatch runner client commands
@@ -1179,25 +1186,25 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223
# only return the return data
ret = event if full_return else event["data"]["return"]
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
except TimeoutException:
- raise salt.ext.tornado.gen.Return("Timeout waiting for runner to execute")
+ raise tornado.gen.Return("Timeout waiting for runner to execute")
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _disbatch_runner_async(self, chunk):
"""
Disbatch runner client_async commands
"""
pub_data = self.saltclients["runner"](chunk)
- raise salt.ext.tornado.gen.Return(pub_data)
+ raise tornado.gen.Return(pub_data)
# salt.utils.args.format_call doesn't work for functions having the
- # annotation salt.ext.tornado.gen.coroutine
+ # annotation tornado.gen.coroutine
def _format_call_run_job_async(self, chunk):
f_call = salt.utils.args.format_call(
salt.client.LocalClient.run_job, chunk, is_class_method=True
)
- f_call.get("kwargs", {})["io_loop"] = salt.ext.tornado.ioloop.IOLoop.current()
+ f_call.get("kwargs", {})["io_loop"] = tornado.ioloop.IOLoop.current()
return f_call
@@ -1206,7 +1213,7 @@ class MinionSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223
A convenience endpoint for minion related functions
"""
- @salt.ext.tornado.web.asynchronous
+ @tornado.gen.coroutine
def get(self, mid=None): # pylint: disable=W0221
"""
A convenience URL for getting lists of minions or getting minion
@@ -1254,7 +1261,7 @@ class MinionSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223
self.lowstate = [{"client": "local", "tgt": mid or "*", "fun": "grains.items"}]
self.disbatch()
- @salt.ext.tornado.web.asynchronous
+ @tornado.gen.coroutine
def post(self):
"""
Start an execution command and immediately return the job id
@@ -1332,7 +1339,7 @@ class JobsSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223
A convenience endpoint for job cache data
"""
- @salt.ext.tornado.web.asynchronous
+ @tornado.gen.coroutine
def get(self, jid=None): # pylint: disable=W0221
"""
A convenience URL for getting lists of previously run jobs or getting
@@ -1432,7 +1439,7 @@ class RunSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223
Endpoint to run commands without normal session handling
"""
- @salt.ext.tornado.web.asynchronous
+ @tornado.gen.coroutine
def post(self):
"""
Run commands bypassing the :ref:`normal session handling
@@ -1504,7 +1511,7 @@ class EventsSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223
.. seealso:: :ref:`events`
"""
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def get(self):
r"""
An HTTP stream of the Salt master event bus
@@ -1790,6 +1797,7 @@ class WebhookSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223
)
self.write(self.serialize({"success": ret}))
+ self.finish()
def _check_cors_origin(origin, allowed_origins):
diff --git a/salt/netapi/rest_tornado/saltnado_websockets.py b/salt/netapi/rest_tornado/saltnado_websockets.py
index 08f4b7ad157..98d64000909 100644
--- a/salt/netapi/rest_tornado/saltnado_websockets.py
+++ b/salt/netapi/rest_tornado/saltnado_websockets.py
@@ -291,8 +291,8 @@ Setup
import logging
-import salt.ext.tornado.gen
-import salt.ext.tornado.websocket
+import tornado.gen
+import tornado.websocket
import salt.netapi
import salt.utils.json
@@ -306,7 +306,7 @@ log = logging.getLogger(__name__)
class AllEventsHandler(
- salt.ext.tornado.websocket.WebSocketHandler
+ tornado.websocket.WebSocketHandler
): # pylint: disable=W0223,W0232
"""
Server side websocket handler.
@@ -335,7 +335,7 @@ class AllEventsHandler(
"""
self.connected = False
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def on_message(self, message):
"""Listens for a "websocket client ready" message.
Once that message is received an asynchronous job
@@ -387,7 +387,7 @@ class AllEventsHandler(
class FormattedEventsHandler(AllEventsHandler): # pylint: disable=W0223,W0232
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def on_message(self, message):
"""Listens for a "websocket client ready" message.
Once that message is received an asynchronous job
diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py
index 8f6bd90d0a4..0b532aca5ed 100644
--- a/salt/pillar/__init__.py
+++ b/salt/pillar/__init__.py
@@ -12,7 +12,7 @@ import time
import traceback
import salt.channel.client
-import salt.ext.tornado.gen
+import tornado.gen
import salt.fileclient
import salt.loader
import salt.minion
@@ -243,7 +243,7 @@ class AsyncRemotePillar(RemotePillarMixin):
self._closing = False
self.clean_cache = clean_cache
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def compile_pillar(self):
"""
Return a future which will contain the pillar data from the master
@@ -286,7 +286,7 @@ class AsyncRemotePillar(RemotePillarMixin):
log.error(msg)
# raise an exception! Pillar isn't empty, we can't sync it!
raise SaltClientError(msg)
- raise salt.ext.tornado.gen.Return(ret_pillar)
+ raise tornado.gen.Return(ret_pillar)
def destroy(self):
if hasattr(self, "_closing") and self._closing:
@@ -1392,7 +1392,7 @@ class Pillar:
# TODO: actually migrate from Pillar to AsyncPillar to allow for futures in
# ext_pillar etc.
class AsyncPillar(Pillar):
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def compile_pillar(self, ext=True):
ret = super().compile_pillar(ext=ext)
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
diff --git a/salt/transport/base.py b/salt/transport/base.py
index 014a9731d59..86a2c882deb 100644
--- a/salt/transport/base.py
+++ b/salt/transport/base.py
@@ -1,4 +1,4 @@
-import salt.ext.tornado.gen
+import tornado.gen
TRANSPORTS = (
"zeromq",
@@ -103,7 +103,7 @@ class RequestClient:
def __init__(self, opts, io_loop, **kwargs):
pass
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def send(self, load, timeout=60):
"""
Send a request message and return the reply from the server.
@@ -211,7 +211,7 @@ class PublishClient:
"""
raise NotImplementedError
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def connect(self, publish_port, connect_callback=None, disconnect_callback=None):
"""
Create a network connection to the the PublishServer or broker.
diff --git a/salt/transport/ipc.py b/salt/transport/ipc.py
index 6631781c5cf..d016c8c8b19 100644
--- a/salt/transport/ipc.py
+++ b/salt/transport/ipc.py
@@ -7,18 +7,18 @@ import logging
import socket
import time
-import salt.ext.tornado
-import salt.ext.tornado.concurrent
-import salt.ext.tornado.gen
-import salt.ext.tornado.ioloop
-import salt.ext.tornado.netutil
+import tornado
+import tornado.concurrent
+import tornado.gen
+import tornado.ioloop
+import tornado.netutil
import salt.transport.client
import salt.transport.frame
import salt.utils.msgpack
-from salt.ext.tornado.ioloop import IOLoop
-from salt.ext.tornado.ioloop import TimeoutError as TornadoTimeoutError
-from salt.ext.tornado.iostream import IOStream, StreamClosedError
-from salt.ext.tornado.locks import Lock
+from tornado.ioloop import IOLoop
+from tornado.ioloop import TimeoutError as TornadoTimeoutError
+from tornado.iostream import IOStream, StreamClosedError
+from tornado.locks import Lock
log = logging.getLogger(__name__)
@@ -32,7 +32,7 @@ def future_with_timeout_callback(future):
future._future_with_timeout._done_callback(future)
-class FutureWithTimeout(salt.ext.tornado.concurrent.Future):
+class FutureWithTimeout(tornado.concurrent.Future):
def __init__(self, io_loop, future, timeout):
super().__init__()
self.io_loop = io_loop
@@ -112,7 +112,7 @@ class IPCServer:
# Placeholders for attributes to be populated by method calls
self.sock = None
- self.io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current()
+ self.io_loop = io_loop or tornado.ioloop.IOLoop.current()
self._closing = False
def start(self):
@@ -131,16 +131,16 @@ class IPCServer:
# Based on default used in tornado.netutil.bind_sockets()
self.sock.listen(128)
else:
- self.sock = salt.ext.tornado.netutil.bind_unix_socket(self.socket_path)
+ self.sock = tornado.netutil.bind_unix_socket(self.socket_path)
with salt.utils.asynchronous.current_ioloop(self.io_loop):
- salt.ext.tornado.netutil.add_accept_handler(
+ tornado.netutil.add_accept_handler(
self.sock,
self.handle_connection,
)
self._started = True
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def handle_stream(self, stream):
"""
Override this to handle the streams as they arrive
@@ -151,14 +151,14 @@ class IPCServer:
for additional details.
"""
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _null(msg):
- raise salt.ext.tornado.gen.Return(None)
+ raise tornado.gen.Return(None)
def write_callback(stream, header):
if header.get("mid"):
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def return_message(msg):
pack = salt.transport.frame.frame_msg_ipc(
msg,
@@ -273,7 +273,7 @@ class IPCClient:
to the server.
"""
- self.io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current()
+ self.io_loop = io_loop or tornado.ioloop.IOLoop.current()
self.socket_path = socket_path
self._closing = False
self.stream = None
@@ -299,7 +299,7 @@ class IPCClient:
if self._connecting_future is not None:
# read previous future result to prevent the "unhandled future exception" error
self._connecting_future.exception() # pylint: disable=E0203
- future = salt.ext.tornado.concurrent.Future()
+ future = tornado.concurrent.Future()
self._connecting_future = future
self._connect(timeout)
@@ -313,7 +313,7 @@ class IPCClient:
return future
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _connect(self, timeout=None):
"""
Connect to a running IPCServer
@@ -339,7 +339,7 @@ class IPCClient:
try:
log.trace("IPCClient: Connecting to socket: %s", self.socket_path)
yield self.stream.connect(sock_addr)
- if self._connecting_future is not None:
+ if self._connecting_future is not None and not self._connecting_future.done():
self._connecting_future.set_result(True)
break
except Exception as e: # pylint: disable=broad-except
@@ -350,11 +350,11 @@ class IPCClient:
if self.stream is not None:
self.stream.close()
self.stream = None
- if self._connecting_future is not None:
+ if self._connecting_future is not None and not self._connecting_future.done():
self._connecting_future.set_exception(e)
break
- yield salt.ext.tornado.gen.sleep(1)
+ yield tornado.gen.sleep(1)
def close(self):
"""
@@ -366,7 +366,7 @@ class IPCClient:
return
self._closing = True
- if self._connecting_future is not None:
+ if self._connecting_future is not None and not self._connecting_future.done():
try:
self._connecting_future.set_result(True)
self._connecting_future.exception() # pylint: disable=E0203
@@ -414,13 +414,13 @@ class IPCMessageClient(IPCClient):
IMPORTANT: The below example also assumes a running IOLoop process.
# Import Tornado libs
- import salt.ext.tornado.ioloop
+ import tornado.ioloop
# Import Salt libs
import salt.config
import salt.transport.ipc
- io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+ io_loop = tornado.ioloop.IOLoop.current()
ipc_server_socket_path = '/var/run/ipc_server.ipc'
@@ -442,7 +442,7 @@ class IPCMessageClient(IPCClient):
"close",
]
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def send(self, msg, timeout=None, tries=None):
"""
Send a message to an IPC socket
@@ -499,7 +499,7 @@ class IPCMessageClient(IPCClient):
_try += 1
if _try > tries or (due_time is not None and cur_time > due_time):
return
- yield salt.ext.tornado.gen.sleep(
+ yield tornado.gen.sleep(
1
if due_time is None
else (due_time - cur_time) / max(tries - _try + 1, 1)
@@ -517,12 +517,12 @@ class IPCMessageServer(IPCServer):
a console:
# Import Tornado libs
- import salt.ext.tornado.ioloop
+ import tornado.ioloop
# Import Salt libs
import salt.transport.ipc
- io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+ io_loop = tornado.ioloop.IOLoop.current()
ipc_server_socket_path = '/var/run/ipc_server.ipc'
ipc_server = salt.transport.ipc.IPCMessageServer(ipc_server_socket_path, io_loop=io_loop,
payload_handler=print_to_console)
@@ -583,19 +583,19 @@ class IPCMessagePublisher:
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.setblocking(0)
self.sock.bind(("127.0.0.1", self.socket_path))
- # Based on default used in salt.ext.tornado.netutil.bind_sockets()
+ # Based on default used in tornado.netutil.bind_sockets()
self.sock.listen(128)
else:
- self.sock = salt.ext.tornado.netutil.bind_unix_socket(self.socket_path)
+ self.sock = tornado.netutil.bind_unix_socket(self.socket_path)
with salt.utils.asynchronous.current_ioloop(self.io_loop):
- salt.ext.tornado.netutil.add_accept_handler(
+ tornado.netutil.add_accept_handler(
self.sock,
self.handle_connection,
)
self._started = True
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _write(self, stream, pack):
try:
yield stream.write(pack)
@@ -674,7 +674,7 @@ class IPCMessageSubscriber(IPCClient):
IMPORTANT: The below example also assumes the IOLoop is NOT running.
# Import Tornado libs
- import salt.ext.tornado.ioloop
+ import tornado.ioloop
# Import Salt libs
import salt.config
@@ -682,7 +682,7 @@ class IPCMessageSubscriber(IPCClient):
# Create a new IO Loop.
# We know that this new IO Loop is not currently running.
- io_loop = salt.ext.tornado.ioloop.IOLoop()
+ io_loop = tornado.ioloop.IOLoop()
ipc_publisher_socket_path = '/var/run/ipc_publisher.ipc'
@@ -711,13 +711,13 @@ class IPCMessageSubscriber(IPCClient):
self._read_in_progress = Lock()
self.callbacks = set()
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _read(self, timeout, callback=None):
try:
try:
yield self._read_in_progress.acquire(timeout=0.00000001)
- except salt.ext.tornado.gen.TimeoutError:
- raise salt.ext.tornado.gen.Return(None)
+ except tornado.gen.TimeoutError:
+ raise tornado.gen.Return(None)
exc_to_raise = None
ret = None
@@ -772,12 +772,12 @@ class IPCMessageSubscriber(IPCClient):
if exc_to_raise is not None:
raise exc_to_raise # pylint: disable=E0702
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
# Handle ctrl+c gracefully
except TypeError:
pass
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def read(self, timeout):
"""
Asynchronously read messages and invoke a callback when they are ready.
@@ -785,7 +785,7 @@ class IPCMessageSubscriber(IPCClient):
"""
if self._saved_data:
res = self._saved_data.pop(0)
- raise salt.ext.tornado.gen.Return(res)
+ raise tornado.gen.Return(res)
while not self.connected():
try:
yield self.connect(timeout=5)
@@ -794,12 +794,12 @@ class IPCMessageSubscriber(IPCClient):
"Subscriber closed stream on IPC %s before connect",
self.socket_path,
)
- yield salt.ext.tornado.gen.sleep(1)
+ yield tornado.gen.sleep(1)
except Exception as exc: # pylint: disable=broad-except
log.error("Exception occurred while Subscriber connecting: %s", exc)
- yield salt.ext.tornado.gen.sleep(1)
+ yield tornado.gen.sleep(1)
res = yield self._read(timeout)
- raise salt.ext.tornado.gen.Return(res)
+ raise tornado.gen.Return(res)
def read_sync(self, timeout=None):
"""
@@ -819,7 +819,7 @@ class IPCMessageSubscriber(IPCClient):
for callback in self.callbacks:
self.io_loop.spawn_callback(callback, raw)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def read_async(self):
"""
Asynchronously read messages and invoke a callback when they are ready.
@@ -834,10 +834,10 @@ class IPCMessageSubscriber(IPCClient):
"Subscriber closed stream on IPC %s before connect",
self.socket_path,
)
- yield salt.ext.tornado.gen.sleep(1)
+ yield tornado.gen.sleep(1)
except Exception as exc: # pylint: disable=broad-except
log.error("Exception occurred while Subscriber connecting: %s", exc)
- yield salt.ext.tornado.gen.sleep(1)
+ yield tornado.gen.sleep(1)
yield self._read(None, self.__run_callbacks)
def close(self):
diff --git a/salt/transport/tcp.py b/salt/transport/tcp.py
index 6a9e1138940..488178e90f5 100644
--- a/salt/transport/tcp.py
+++ b/salt/transport/tcp.py
@@ -16,13 +16,13 @@ import socket
import threading
import urllib
-import salt.ext.tornado
-import salt.ext.tornado.concurrent
-import salt.ext.tornado.gen
-import salt.ext.tornado.iostream
-import salt.ext.tornado.netutil
-import salt.ext.tornado.tcpclient
-import salt.ext.tornado.tcpserver
+import tornado
+import tornado.concurrent
+import tornado.gen
+import tornado.iostream
+import tornado.netutil
+import tornado.tcpclient
+import tornado.tcpserver
import salt.master
import salt.payload
import salt.transport.client
@@ -43,7 +43,7 @@ else:
USE_LOAD_BALANCER = False
if USE_LOAD_BALANCER:
- import salt.ext.tornado.util
+ import tornado.util
from salt.utils.process import SignalHandlingProcess
log = logging.getLogger(__name__)
@@ -141,7 +141,7 @@ if USE_LOAD_BALANCER:
"""
# TODO: opts!
- # Based on default used in salt.ext.tornado.netutil.bind_sockets()
+ # Based on default used in tornado.netutil.bind_sockets()
backlog = 128
def __init__(self, opts, socket_queue, **kwargs):
@@ -187,7 +187,7 @@ if USE_LOAD_BALANCER:
# but it was closed while still in the accept queue.
# (observed on FreeBSD).
if (
- salt.ext.tornado.util.errno_from_exception(e)
+ tornado.util.errno_from_exception(e)
== errno.ECONNABORTED
):
continue
@@ -200,8 +200,8 @@ class Resolver:
@classmethod
def _config_resolver(cls, num_threads=10):
- salt.ext.tornado.netutil.Resolver.configure(
- "salt.ext.tornado.netutil.ThreadedResolver", num_threads=num_threads
+ tornado.netutil.Resolver.configure(
+ "tornado.netutil.ThreadedResolver", num_threads=num_threads
)
cls._resolver_configured = True
@@ -240,7 +240,7 @@ class TCPPubClient(salt.transport.base.PublishClient):
# pylint: enable=W1701
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def connect(self, publish_port, connect_callback=None, disconnect_callback=None):
self.publish_port = publish_port
self.message_client = MessageClient(
@@ -256,7 +256,7 @@ class TCPPubClient(salt.transport.base.PublishClient):
yield self.message_client.connect() # wait for the client to be connected
self.connected = True
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _decode_messages(self, messages):
if not isinstance(messages, dict):
# TODO: For some reason we need to decode here for things
@@ -265,9 +265,9 @@ class TCPPubClient(salt.transport.base.PublishClient):
body = salt.transport.frame.decode_embedded_strs(body)
else:
body = messages
- raise salt.ext.tornado.gen.Return(body)
+ raise tornado.gen.Return(body)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def send(self, msg):
yield self.message_client._stream.write(msg)
@@ -365,6 +365,7 @@ class TCPReqServer(salt.transport.base.DaemonizedRequestServer):
message_handler: function to call with your payloads
"""
self.message_handler = message_handler
+ log.info("ReqServer workers %s", socket)
with salt.utils.asynchronous.current_ioloop(io_loop):
if USE_LOAD_BALANCER:
@@ -388,7 +389,7 @@ class TCPReqServer(salt.transport.base.DaemonizedRequestServer):
self.req_server.add_socket(self._socket)
self._socket.listen(self.backlog)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def handle_message(self, stream, payload, header=None):
payload = self.decode_payload(payload)
reply = yield self.message_handler(payload)
@@ -398,7 +399,7 @@ class TCPReqServer(salt.transport.base.DaemonizedRequestServer):
return payload
-class SaltMessageServer(salt.ext.tornado.tcpserver.TCPServer):
+class SaltMessageServer(tornado.tcpserver.TCPServer):
"""
Raw TCP server which will receive all of the TCP streams and re-assemble
messages that are sent through to us
@@ -406,7 +407,7 @@ class SaltMessageServer(salt.ext.tornado.tcpserver.TCPServer):
def __init__(self, message_handler, *args, **kwargs):
io_loop = (
- kwargs.pop("io_loop", None) or salt.ext.tornado.ioloop.IOLoop.current()
+ kwargs.pop("io_loop", None) or tornado.ioloop.IOLoop.current()
)
self._closing = False
super().__init__(*args, **kwargs)
@@ -414,12 +415,12 @@ class SaltMessageServer(salt.ext.tornado.tcpserver.TCPServer):
self.clients = []
self.message_handler = message_handler
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def handle_stream( # pylint: disable=arguments-differ
self,
stream,
address,
- _StreamClosedError=salt.ext.tornado.iostream.StreamClosedError,
+ _StreamClosedError=tornado.iostream.StreamClosedError,
):
"""
Handle incoming streams and add messages to the incoming queue
@@ -501,7 +502,7 @@ if USE_LOAD_BALANCER:
break
continue
# 'self.io_loop' initialized in super class
- # 'salt.ext.tornado.tcpserver.TCPServer'.
+ # 'tornado.tcpserver.TCPServer'.
# 'self._handle_connection' defined in same super class.
self.io_loop.spawn_callback(
self._handle_connection, client_socket, address
@@ -510,7 +511,7 @@ if USE_LOAD_BALANCER:
pass
-class TCPClientKeepAlive(salt.ext.tornado.tcpclient.TCPClient):
+class TCPClientKeepAlive(tornado.tcpclient.TCPClient):
"""
Override _create_stream() in TCPClient to enable keep alive support.
"""
@@ -533,10 +534,10 @@ class TCPClientKeepAlive(salt.ext.tornado.tcpclient.TCPClient):
# after one connection has completed.
sock = _get_socket(self.opts)
_set_tcp_keepalive(sock, self.opts)
- stream = salt.ext.tornado.iostream.IOStream(
+ stream = tornado.iostream.IOStream(
sock, max_buffer_size=max_buffer_size
)
- if salt.ext.tornado.version_info < (5,):
+ if tornado.version_info < (5,):
return stream.connect(addr)
return stream, stream.connect(addr)
@@ -568,7 +569,7 @@ class MessageClient:
self.source_port = source_port
self.connect_callback = connect_callback
self.disconnect_callback = disconnect_callback
- self.io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current()
+ self.io_loop = io_loop or tornado.ioloop.IOLoop.current()
with salt.utils.asynchronous.current_ioloop(self.io_loop):
self._tcp_client = TCPClientKeepAlive(opts, resolver=resolver)
self._mid = 1
@@ -581,7 +582,7 @@ class MessageClient:
self._on_recv = None
self._closing = False
self._closed = False
- self._connecting_future = salt.ext.tornado.concurrent.Future()
+ self._connecting_future = tornado.concurrent.Future()
self._stream_return_running = False
self._stream = None
@@ -596,9 +597,12 @@ class MessageClient:
if self._closing:
return
self._closing = True
- self.io_loop.add_timeout(1, self.check_close)
+ try:
+ self.io_loop.add_timeout(1, self.check_close)
+ except RuntimeError:
+ pass
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def check_close(self):
if not self.send_future_map:
self._tcp_client.close()
@@ -606,7 +610,10 @@ class MessageClient:
self._closing = False
self._closed = True
else:
- self.io_loop.add_timeout(1, self.check_close)
+ try:
+ self.io_loop.add_timeout(1, self.check_close)
+ except RuntimeError:
+ pass
# pylint: disable=W1701
def __del__(self):
@@ -614,7 +621,7 @@ class MessageClient:
# pylint: enable=W1701
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def getstream(self, **kwargs):
if self.source_ip or self.source_port:
kwargs = {
@@ -639,10 +646,10 @@ class MessageClient:
exc,
self.backoff,
)
- yield salt.ext.tornado.gen.sleep(self.backoff)
- raise salt.ext.tornado.gen.Return(stream)
+ yield tornado.gen.sleep(self.backoff)
+ raise tornado.gen.Return(stream)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def connect(self):
if self._stream is None:
self._stream = yield self.getstream()
@@ -652,7 +659,7 @@ class MessageClient:
if self.connect_callback:
self.connect_callback(True)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _stream_return(self):
self._stream_return_running = True
unpacker = salt.utils.msgpack.Unpacker()
@@ -678,7 +685,7 @@ class MessageClient:
" tracking",
message_id,
)
- except salt.ext.tornado.iostream.StreamClosedError as e:
+ except tornado.iostream.StreamClosedError as e:
log.debug(
"tcp stream to %s:%s closed, unable to recv",
self.host,
@@ -764,14 +771,14 @@ class MessageClient:
if future is not None:
future.set_exception(SaltReqTimeoutError("Message timed out"))
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def send(self, msg, timeout=None, callback=None, raw=False):
if self._closing:
raise ClosingError()
message_id = self._message_id()
header = {"mid": message_id}
- future = salt.ext.tornado.concurrent.Future()
+ future = tornado.concurrent.Future()
if callback is not None:
@@ -791,7 +798,7 @@ class MessageClient:
item = salt.transport.frame.frame_msg(msg, header=header)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _do_send():
yield self.connect()
# If the _stream is None, we failed to connect.
@@ -802,7 +809,7 @@ class MessageClient:
# out before we are able to connect.
self.io_loop.add_callback(_do_send)
recv = yield future
- raise salt.ext.tornado.gen.Return(recv)
+ raise tornado.gen.Return(recv)
class Subscriber:
@@ -839,7 +846,7 @@ class Subscriber:
# pylint: enable=W1701
-class PubServer(salt.ext.tornado.tcpserver.TCPServer):
+class PubServer(tornado.tcpserver.TCPServer):
"""
TCP publisher
"""
@@ -875,7 +882,7 @@ class PubServer(salt.ext.tornado.tcpserver.TCPServer):
# pylint: enable=W1701
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _stream_read(self, client):
unpacker = salt.utils.msgpack.Unpacker()
while not self._closing:
@@ -888,7 +895,7 @@ class PubServer(salt.ext.tornado.tcpserver.TCPServer):
body = framed_msg["body"]
if self.presence_callback:
self.presence_callback(client, body)
- except salt.ext.tornado.iostream.StreamClosedError as e:
+ except tornado.iostream.StreamClosedError as e:
log.debug("tcp stream to %s closed, unable to recv", client.address)
client.close()
self.remove_presence_callback(client)
@@ -907,7 +914,7 @@ class PubServer(salt.ext.tornado.tcpserver.TCPServer):
self.io_loop.spawn_callback(self._stream_read, client)
# TODO: ACK the publish through IPC
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def publish_payload(self, package, topic_list=None):
log.trace("TCP PubServer sending payload: %s \n\n %r", package, topic_list)
payload = salt.transport.frame.frame_msg(package)
@@ -922,7 +929,7 @@ class PubServer(salt.ext.tornado.tcpserver.TCPServer):
yield client.stream.write(payload)
sent = True
# self.io_loop.add_future(f, lambda f: True)
- except salt.ext.tornado.iostream.StreamClosedError:
+ except tornado.iostream.StreamClosedError:
to_remove.append(client)
if not sent:
log.debug("Publish target %s not connected %r", topic, self.clients)
@@ -931,7 +938,7 @@ class PubServer(salt.ext.tornado.tcpserver.TCPServer):
try:
# Write the packed str
yield client.stream.write(payload)
- except salt.ext.tornado.iostream.StreamClosedError:
+ except tornado.iostream.StreamClosedError:
to_remove.append(client)
for client in to_remove:
log.debug(
@@ -949,7 +956,7 @@ class TCPPublishServer(salt.transport.base.DaemonizedPublishServer):
"""
# TODO: opts!
- # Based on default used in salt.ext.tornado.netutil.bind_sockets()
+ # Based on default used in tornado.netutil.bind_sockets()
backlog = 128
def __init__(self, opts):
@@ -975,7 +982,7 @@ class TCPPublishServer(salt.transport.base.DaemonizedPublishServer):
"""
Bind to the interface specified in the configuration file
"""
- io_loop = salt.ext.tornado.ioloop.IOLoop()
+ io_loop = tornado.ioloop.IOLoop()
io_loop.make_current()
# Spin up the publisher
@@ -1027,10 +1034,10 @@ class TCPPublishServer(salt.transport.base.DaemonizedPublishServer):
"""
process_manager.add_process(self.publish_daemon, name=self.__class__.__name__)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def publish_payload(self, payload, *args):
ret = yield self.pub_server.publish_payload(payload, *args)
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
def publish(self, payload, **kwargs):
"""
@@ -1080,14 +1087,14 @@ class TCPReqClient(salt.transport.base.RequestClient):
source_port=opts.get("source_ret_port"),
)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def connect(self):
yield self.message_client.connect()
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def send(self, load, timeout=60):
ret = yield self.message_client.send(load, timeout=timeout)
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
def close(self):
self.message_client.close()
diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py
index 89f705190e0..730af297790 100644
--- a/salt/transport/zeromq.py
+++ b/salt/transport/zeromq.py
@@ -14,10 +14,10 @@ from random import randint
import zmq.error
import zmq.eventloop.zmqstream
-import salt.ext.tornado
-import salt.ext.tornado.concurrent
-import salt.ext.tornado.gen
-import salt.ext.tornado.ioloop
+import tornado
+import tornado.concurrent
+import tornado.gen
+import tornado.ioloop
import salt.payload
import salt.transport.base
import salt.utils.files
@@ -204,7 +204,7 @@ class PublishClient(salt.transport.base.PublishClient):
self.close()
# TODO: this is the time to see if we are connected, maybe use the req channel to guess?
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def connect(self, publish_port, connect_callback=None, disconnect_callback=None):
self.publish_port = publish_port
log.debug(
@@ -233,7 +233,7 @@ class PublishClient(salt.transport.base.PublishClient):
source_port=self.opts.get("source_publish_port"),
)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _decode_messages(self, messages):
"""
Take the zmq messages, decrypt/decode them into a payload
@@ -255,7 +255,7 @@ class PublishClient(salt.transport.base.PublishClient):
and message_target not in ("broadcast", "syndic")
):
log.debug("Publish received for not this minion: %s", message_target)
- raise salt.ext.tornado.gen.Return(None)
+ raise tornado.gen.Return(None)
payload = salt.payload.loads(messages[1])
else:
raise Exception(
@@ -265,7 +265,7 @@ class PublishClient(salt.transport.base.PublishClient):
)
# Yield control back to the caller. When the payload has been decoded, assign
# the decoded payload to 'ret' and resume operation
- raise salt.ext.tornado.gen.Return(payload)
+ raise tornado.gen.Return(payload)
@property
def stream(self):
@@ -286,7 +286,7 @@ class PublishClient(salt.transport.base.PublishClient):
"""
return self.stream.on_recv(callback)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def send(self, msg):
self.stream.send(msg, noblock=True)
@@ -433,7 +433,7 @@ class RequestServer(salt.transport.base.DaemonizedRequestServer):
self.message_handler = message_handler
self.stream.on_recv_stream(self.handle_message)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def handle_message(self, stream, payload):
try:
payload = self.decode_payload(payload)
@@ -515,7 +515,7 @@ class AsyncReqMessageClient:
self.addr = addr
self.linger = linger
if io_loop is None:
- self.io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+ self.io_loop = tornado.ioloop.IOLoop.current()
else:
self.io_loop = io_loop
@@ -598,12 +598,12 @@ class AsyncReqMessageClient:
if future is not None:
future.set_exception(SaltReqTimeoutError("Message timed out"))
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def send(self, message, timeout=None, callback=None):
"""
Return a future which will be completed when the message has a response
"""
- future = salt.ext.tornado.concurrent.Future()
+ future = tornado.concurrent.Future()
message = salt.payload.dumps(message)
@@ -635,7 +635,7 @@ class AsyncReqMessageClient:
self.stream.on_recv(mark_future)
yield self.stream.send(message)
recv = yield future
- raise salt.ext.tornado.gen.Return(recv)
+ raise tornado.gen.Return(recv)
class ZeroMQSocketMonitor:
@@ -719,7 +719,7 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer):
self.opts = opts
def connect(self):
- return salt.ext.tornado.gen.sleep(5)
+ return tornado.gen.sleep(5)
def publish_daemon(
self,
@@ -731,7 +731,7 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer):
This method represents the Publish Daemon process. It is intended to be
run in a thread or process as it creates and runs an it's own ioloop.
"""
- ioloop = salt.ext.tornado.ioloop.IOLoop()
+ ioloop = tornado.ioloop.IOLoop()
ioloop.make_current()
self.io_loop = ioloop
context = zmq.Context(1)
@@ -767,7 +767,7 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer):
with salt.utils.files.set_umask(0o177):
pull_sock.bind(self.pull_uri)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def on_recv(packages):
for package in packages:
payload = salt.payload.loads(package)
@@ -796,7 +796,7 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer):
def pub_uri(self):
return "tcp://{interface}:{publish_port}".format(**self.opts)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def publish_payload(self, payload, topic_list=None):
payload = salt.payload.dumps(payload)
if self.opts["zmq_filtering"]:
@@ -929,11 +929,11 @@ class RequestClient(salt.transport.base.RequestClient):
def connect(self):
self.message_client.connect()
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def send(self, load, timeout=60):
self.connect()
ret = yield self.message_client.send(load, timeout=timeout)
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
def close(self):
self.message_client.close()
diff --git a/salt/utils/asynchronous.py b/salt/utils/asynchronous.py
index 55a50cbcbff..0429bf9fcc2 100644
--- a/salt/utils/asynchronous.py
+++ b/salt/utils/asynchronous.py
@@ -2,11 +2,16 @@
Helpers/utils for working with tornado asynchronous stuff
"""
+import asyncio
import contextlib
import logging
+import sys
+import threading
-import salt.ext.tornado.concurrent
-import salt.ext.tornado.ioloop
+import tornado.concurrent
+import tornado.ioloop
+
+from salt import USE_VENDORED_TORNADO
log = logging.getLogger(__name__)
@@ -16,12 +21,24 @@ def current_ioloop(io_loop):
"""
A context manager that will set the current ioloop to io_loop for the context
"""
- orig_loop = salt.ext.tornado.ioloop.IOLoop.current()
- io_loop.make_current()
+ try:
+ orig_loop = tornado.ioloop.IOLoop.current()
+ except RuntimeError:
+ orig_loop = None
+ if USE_VENDORED_TORNADO:
+ io_loop.make_current()
+ else:
+ asyncio.set_event_loop(io_loop.asyncio_loop)
try:
yield
finally:
- orig_loop.make_current()
+ if orig_loop:
+ if USE_VENDORED_TORNADO:
+ orig_loop.make_current()
+ else:
+ asyncio.set_event_loop(orig_loop.asyncio_loop)
+ else:
+ asyncio.set_event_loop(None)
class SyncWrapper:
@@ -31,7 +48,7 @@ class SyncWrapper:
This is uses as a simple wrapper, for example:
asynchronous = AsyncClass()
- # this method would reguarly return a future
+ # this method would regularly return a future
future = asynchronous.async_method()
sync = SyncWrapper(async_factory_method, (arg1, arg2), {'kwarg1': 'val'})
@@ -48,7 +65,13 @@ class SyncWrapper:
close_methods=None,
loop_kwarg=None,
):
- self.io_loop = salt.ext.tornado.ioloop.IOLoop()
+ if USE_VENDORED_TORNADO:
+ self.io_loop = tornado.ioloop.IOLoop()
+ else:
+ self.asyncio_loop = asyncio.new_event_loop()
+ self.io_loop = tornado.ioloop.IOLoop(
+ asyncio_loop=self.asyncio_loop, make_current=False
+ )
if args is None:
args = []
if kwargs is None:
@@ -61,7 +84,8 @@ class SyncWrapper:
self.cls = cls
if loop_kwarg:
kwargs[self.loop_kwarg] = self.io_loop
- self.obj = cls(*args, **kwargs)
+ with current_ioloop(self.io_loop):
+ self.obj = cls(*args, **kwargs)
self._async_methods = list(
set(async_methods + getattr(self.obj, "async_methods", []))
)
@@ -79,7 +103,7 @@ class SyncWrapper:
self._async_methods += self.obj._coroutines
def __repr__(self):
- return "<SyncWrapper(cls={})".format(self.cls)
+ return f"<SyncWrapper(cls={self.cls})"
def close(self):
for method in self._close_methods:
@@ -99,7 +123,16 @@ class SyncWrapper:
log.exception("Exception encountered while running stop method")
io_loop = self.io_loop
io_loop.stop()
- io_loop.close(all_fds=True)
+ try:
+ io_loop.close(all_fds=True)
+ except (KeyError, RuntimeError):
+ pass
+ try:
+ if hasattr(self, "asyncio_loop"):
+ self.asyncio_loop.close()
+ except (KeyError, RuntimeError):
+ pass
+
def __getattr__(self, key):
if key in self._async_methods:
@@ -108,14 +141,63 @@ class SyncWrapper:
def _wrap(self, key):
def wrap(*args, **kwargs):
- return self.io_loop.run_sync(
- lambda: getattr(self.obj, key)(*args, **kwargs)
+ if USE_VENDORED_TORNADO:
+ return self.io_loop.run_sync(
+ lambda: getattr(self.obj, key)(*args, **kwargs)
+ )
+ try:
+ asyncio.get_running_loop()
+ except RuntimeError:
+ # asyncio.get_running_loop() raises RuntimeError
+ # if there is no running loop, so we can run the method
+ # directly with no detaching it to the distinct thread.
+ # It will make SyncWrapper way faster for the cases
+ # when there are no nested SyncWrapper objects used.
+ return self.io_loop.run_sync(
+ lambda: getattr(self.obj, key)(*args, **kwargs)
+ )
+ results = []
+ thread = threading.Thread(
+ target=self._target,
+ args=(key, args, kwargs, results, self.asyncio_loop),
)
+ thread.start()
+ thread.join()
+ if results[0]:
+ return results[1]
+ else:
+ exc_info = results[1]
+ raise exc_info[1].with_traceback(exc_info[2])
return wrap
+ def _target(self, key, args, kwargs, results, asyncio_loop):
+ asyncio.set_event_loop(asyncio_loop)
+ io_loop = tornado.ioloop.IOLoop.current()
+ try:
+ result = io_loop.run_sync(lambda: getattr(self.obj, key)(*args, **kwargs))
+ results.append(True)
+ results.append(result)
+ except Exception: # pylint: disable=broad-except
+ results.append(False)
+ results.append(sys.exc_info())
+
def __enter__(self):
+ if hasattr(self.obj, "__aenter__"):
+ ret = self._wrap("__aenter__")()
+ if ret == self.obj:
+ return self
+ else:
+ return ret
+ elif hasattr(self.obj, "__enter__"):
+ ret = self.obj.__enter__()
+ if ret == self.obj:
+ return self
+ else:
+ return ret
return self
def __exit__(self, exc_type, exc_val, tb):
+ if hasattr(self.obj, "__aexit__"):
+ self._wrap("__aexit__")(exc_type, exc_val, tb)
self.close()
diff --git a/salt/utils/event.py b/salt/utils/event.py
index c973926db93..964dc5caa85 100644
--- a/salt/utils/event.py
+++ b/salt/utils/event.py
@@ -63,8 +63,8 @@ from collections.abc import MutableMapping
import salt.channel.client
import salt.config
import salt.defaults.exitcodes
-import salt.ext.tornado.ioloop
-import salt.ext.tornado.iostream
+import tornado.ioloop
+import tornado.iostream
import salt.payload
import salt.transport.ipc
import salt.utils.asynchronous
@@ -237,7 +237,7 @@ class SaltEvent:
self.io_loop = io_loop
self._run_io_loop_sync = False
else:
- self.io_loop = salt.ext.tornado.ioloop.IOLoop()
+ self.io_loop = tornado.ioloop.IOLoop()
self._run_io_loop_sync = True
self.cpub = False
self.cpush = False
@@ -376,7 +376,7 @@ class SaltEvent:
try:
self.subscriber.connect(timeout=timeout)
self.cpub = True
- except salt.ext.tornado.iostream.StreamClosedError:
+ except tornado.iostream.StreamClosedError:
log.error("Encountered StreamClosedException")
except OSError as exc:
if exc.errno != errno.ENOENT:
@@ -431,7 +431,7 @@ class SaltEvent:
try:
self.pusher.connect(timeout=timeout)
self.cpush = True
- except salt.ext.tornado.iostream.StreamClosedError as exc:
+ except tornado.iostream.StreamClosedError as exc:
log.debug("Unable to connect pusher: %s", exc)
except Exception as exc: # pylint: disable=broad-except
log.error(
@@ -589,7 +589,7 @@ class SaltEvent:
ret = {"data": data, "tag": mtag}
except KeyboardInterrupt:
return {"tag": "salt/event/exit", "data": {}}
- except salt.ext.tornado.iostream.StreamClosedError:
+ except tornado.iostream.StreamClosedError:
if self.raise_errors:
raise
else:
@@ -686,7 +686,7 @@ class SaltEvent:
try:
ret = self._get_event(wait, tag, match_func, no_block)
break
- except salt.ext.tornado.iostream.StreamClosedError:
+ except tornado.iostream.StreamClosedError:
self.close_pub()
self.connect_pub(timeout=wait)
continue
@@ -742,7 +742,7 @@ class SaltEvent:
continue
yield data
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def fire_event_async(self, data, tag, cb=None, timeout=1000):
"""
Send a single event into the publisher with payload dict "data" and
@@ -1109,7 +1109,7 @@ class AsyncEventPublisher:
default_minion_sock_dir = self.opts["sock_dir"]
self.opts.update(opts)
- self.io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current()
+ self.io_loop = io_loop or tornado.ioloop.IOLoop.current()
self._closing = False
self.publisher = None
self.puller = None
@@ -1227,7 +1227,7 @@ class EventPublisher(salt.utils.process.SignalHandlingProcess):
)
os.nice(self.opts["event_publisher_niceness"])
- self.io_loop = salt.ext.tornado.ioloop.IOLoop()
+ self.io_loop = tornado.ioloop.IOLoop()
with salt.utils.asynchronous.current_ioloop(self.io_loop):
if self.opts["ipc_mode"] == "tcp":
epub_uri = int(self.opts["tcp_master_pub_port"])
@@ -1290,7 +1290,11 @@ class EventPublisher(salt.utils.process.SignalHandlingProcess):
self.puller.close()
self.puller = None
if self.io_loop is not None:
- self.io_loop.close()
+ try:
+ self.io_loop.stop()
+ self.io_loop.close()
+ except RuntimeError:
+ pass
self.io_loop = None
def _handle_signals(self, signum, sigframe):
diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py
index d597c17b870..f0a3e6ea5cf 100644
--- a/salt/utils/gitfs.py
+++ b/salt/utils/gitfs.py
@@ -23,7 +23,7 @@ import time
import weakref
from datetime import datetime
-import salt.ext.tornado.ioloop
+import tornado.ioloop
import salt.fileserver
import salt.syspaths
import salt.utils.cache
@@ -3081,7 +3081,7 @@ class GitFS(GitBase):
exited.
"""
# No need to get the ioloop reference if we're not initializing remotes
- io_loop = salt.ext.tornado.ioloop.IOLoop.current() if init_remotes else None
+ io_loop = tornado.ioloop.IOLoop.current() if init_remotes else None
if not init_remotes or io_loop not in cls.instance_map:
# We only evaluate the second condition in this if statement if
# we're initializing remotes, so we won't get here unless io_loop
diff --git a/salt/utils/http.py b/salt/utils/http.py
index 26f2e85c2ee..9d0669a85bd 100644
--- a/salt/utils/http.py
+++ b/salt/utils/http.py
@@ -23,8 +23,8 @@ import xml.etree.ElementTree as ET
import zlib
import salt.config
-import salt.ext.tornado.httputil
-import salt.ext.tornado.simple_httpclient
+import tornado.httputil
+import tornado.simple_httpclient
import salt.loader
import salt.syspaths
import salt.utils.args
@@ -38,7 +38,7 @@ import salt.utils.stringutils
import salt.utils.xmlutil as xml
import salt.utils.yaml
import salt.version
-from salt.ext.tornado.httpclient import HTTPClient
+from tornado.httpclient import HTTPClient
from salt.template import compile_template
from salt.utils.decorators.jinja import jinja_filter
@@ -63,7 +63,7 @@ except ImportError:
try:
- import salt.ext.tornado.curl_httpclient
+ import tornado.curl_httpclient
HAS_CURL_HTTPCLIENT = True
except ImportError:
@@ -213,7 +213,7 @@ def query(
# Some libraries don't support separation of url and GET parameters
# Don't need a try/except block, since Salt depends on tornado
- url_full = salt.ext.tornado.httputil.url_concat(url, params) if params else url
+ url_full = tornado.httputil.url_concat(url, params) if params else url
if ca_bundle is None:
ca_bundle = get_ca_bundle(opts)
@@ -564,16 +564,16 @@ def query(
log.error(ret["error"])
return ret
- salt.ext.tornado.httpclient.AsyncHTTPClient.configure(
+ tornado.httpclient.AsyncHTTPClient.configure(
"tornado.curl_httpclient.CurlAsyncHTTPClient"
)
client_argspec = salt.utils.args.get_function_argspec(
- salt.ext.tornado.curl_httpclient.CurlAsyncHTTPClient.initialize
+ tornado.curl_httpclient.CurlAsyncHTTPClient.initialize
)
else:
- salt.ext.tornado.httpclient.AsyncHTTPClient.configure(None)
+ tornado.httpclient.AsyncHTTPClient.configure(None)
client_argspec = salt.utils.args.get_function_argspec(
- salt.ext.tornado.simple_httpclient.SimpleAsyncHTTPClient.initialize
+ tornado.simple_httpclient.SimpleAsyncHTTPClient.initialize
)
supports_max_body_size = "max_body_size" in client_argspec.args
@@ -612,15 +612,18 @@ def query(
else HTTPClient()
)
result = download_client.fetch(url_full, **req_kwargs)
- except salt.ext.tornado.httpclient.HTTPError as exc:
+ except tornado.httpclient.HTTPError as exc:
ret["status"] = exc.code
ret["error"] = str(exc)
return ret
except (socket.herror, OSError, socket.timeout, socket.gaierror) as exc:
if status is True:
ret["status"] = 0
- ret["error"] = str(exc)
- log.debug("Cannot perform 'http.query': %s - %s", url_full, ret["error"])
+ if not raise_error:
+ ret["body"] = None
+ else:
+ ret["error"] = str(exc)
+ log.debug("Cannot perform 'http.query': %s - %s", url_full, ret["error"])
return ret
if stream is True or handle is True:
diff --git a/salt/utils/process.py b/salt/utils/process.py
index 6c393706acd..9d7b53aa112 100644
--- a/salt/utils/process.py
+++ b/salt/utils/process.py
@@ -26,7 +26,7 @@ import salt.utils.files
import salt.utils.path
import salt.utils.platform
import salt.utils.versions
-from salt.ext.tornado import gen
+from tornado import gen
log = logging.getLogger(__name__)
diff --git a/salt/utils/thin.py b/salt/utils/thin.py
index 80766a286a9..eb7363984d9 100644
--- a/salt/utils/thin.py
+++ b/salt/utils/thin.py
@@ -24,7 +24,7 @@ import yaml
import salt
import salt.exceptions
-import salt.ext.tornado as tornado
+import tornado
import salt.utils.files
import salt.utils.hashutils
import salt.utils.json
@@ -450,8 +450,7 @@ def get_tops(extra_mods="", so_mods=""):
for mod in [m for m in extra_mods.split(",") if m]:
if mod not in locals() and mod not in globals():
try:
- locals()[mod] = __import__(mod)
- moddir, modname = os.path.split(locals()[mod].__file__)
+ moddir, modname = os.path.split(__import__(mod).__file__)
base, _ = os.path.splitext(modname)
if base == "__init__":
tops.append(moddir)
@@ -464,8 +463,7 @@ def get_tops(extra_mods="", so_mods=""):
for mod in [m for m in so_mods.split(",") if m]:
try:
- locals()[mod] = __import__(mod)
- tops.append(locals()[mod].__file__)
+ tops.append(__import__(mod).__file__)
except ImportError as err:
log.error('Unable to import so-module "%s"', mod, exc_info=True)
diff --git a/tests/integration/minion/test_minion_cache.py b/tests/integration/minion/test_minion_cache.py
index 039c1e51c94..33125f8bccb 100644
--- a/tests/integration/minion/test_minion_cache.py
+++ b/tests/integration/minion/test_minion_cache.py
@@ -35,7 +35,7 @@ class BasePillarTest(ModuleCase):
}
with patch("salt.loader.grains", return_value={}), patch(
"salt.minion.SMinion.gen_modules"
- ), patch("salt.ext.tornado.ioloop.IOLoop.current"):
+ ), patch("tornado.ioloop.IOLoop.current"):
minion = salt.minion.SMinion(opts)
self.assertTrue("pillar" in os.listdir(tempdir))
pillar_cache = os.path.join(tempdir, "pillar")
diff --git a/tests/integration/modules/test_gem.py b/tests/integration/modules/test_gem.py
index f484e8c3c1d..7c988800018 100644
--- a/tests/integration/modules/test_gem.py
+++ b/tests/integration/modules/test_gem.py
@@ -4,7 +4,7 @@ Integration tests for Ruby Gem module
import pytest
-from salt.ext.tornado.httpclient import HTTPClient
+from tornado.httpclient import HTTPClient
from tests.support.case import ModuleCase
diff --git a/tests/integration/netapi/rest_tornado/test_app.py b/tests/integration/netapi/rest_tornado/test_app.py
index 65bb4ad6463..be5cfb3e547 100644
--- a/tests/integration/netapi/rest_tornado/test_app.py
+++ b/tests/integration/netapi/rest_tornado/test_app.py
@@ -5,11 +5,11 @@ import time
import pytest
import salt.auth
-import salt.ext.tornado.escape
-import salt.ext.tornado.web
+import tornado.escape
+import tornado.web
import salt.utils.json
import salt.utils.stringutils
-from salt.ext.tornado.testing import AsyncHTTPTestCase
+from tornado.testing import AsyncHTTPTestCase
from salt.netapi.rest_tornado import saltnado
from tests.support.helpers import TstSuiteLoggingHandler, patched_environ
from tests.support.mixins import AdaptedConfigurationTestCaseMixin
@@ -96,7 +96,7 @@ class SaltnadoIntegrationTestsBase(
del self.patched_environ
def build_tornado_app(self, urls):
- application = salt.ext.tornado.web.Application(urls, debug=True)
+ application = tornado.web.Application(urls, debug=True)
application.auth = self.auth
application.opts = self.opts
@@ -112,7 +112,7 @@ class SaltnadoIntegrationTestsBase(
if response.headers.get("Content-Type") == "application/json":
response._body = response.body.decode("utf-8")
else:
- response._body = salt.ext.tornado.escape.native_str(response.body)
+ response._body = tornado.escape.native_str(response.body)
return response
def fetch(self, path, **kwargs):
diff --git a/tests/pytests/conftest.py b/tests/pytests/conftest.py
index 324bd116303..dc7c4aff81b 100644
--- a/tests/pytests/conftest.py
+++ b/tests/pytests/conftest.py
@@ -19,7 +19,7 @@ import pytest
from pytestshellutils.utils import ports
from saltfactories.utils import random_string
-import salt.ext.tornado.ioloop
+import tornado.ioloop
import salt.utils.files
import salt.utils.platform
from salt.serializers import yaml
@@ -622,7 +622,7 @@ def pytest_pyfunc_call(pyfuncitem):
try:
loop = funcargs["io_loop"]
except KeyError:
- loop = salt.ext.tornado.ioloop.IOLoop.current()
+ loop = tornado.ioloop.IOLoop.current()
__tracebackhide__ = True
@@ -637,7 +637,7 @@ def io_loop():
"""
Create new io loop for each test, and tear it down after.
"""
- loop = salt.ext.tornado.ioloop.IOLoop()
+ loop = tornado.ioloop.IOLoop()
loop.make_current()
try:
yield loop
diff --git a/tests/pytests/functional/channel/test_req_channel.py b/tests/pytests/functional/channel/test_req_channel.py
index 4f5f52d66f4..5f46c4d3971 100644
--- a/tests/pytests/functional/channel/test_req_channel.py
+++ b/tests/pytests/functional/channel/test_req_channel.py
@@ -13,7 +13,7 @@ import salt.channel.server
import salt.config
import salt.crypt
import salt.exceptions
-import salt.ext.tornado.gen
+import tornado.gen
import salt.master
import salt.utils.platform
import salt.utils.process
@@ -64,7 +64,7 @@ class ReqServerChannelProcess(salt.utils.process.SignalHandlingProcess):
),
}
- self.io_loop = salt.ext.tornado.ioloop.IOLoop()
+ self.io_loop = tornado.ioloop.IOLoop()
self.io_loop.make_current()
self.req_server_channel.post_fork(self._handle_payload, io_loop=self.io_loop)
self.io_loop.add_callback(self.running.set)
@@ -100,11 +100,11 @@ class ReqServerChannelProcess(salt.utils.process.SignalHandlingProcess):
terminate_process(pid=pid, kill_children=True, slow_stop=False)
self.process_manager = None
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _handle_payload(self, payload):
if self.req_channel_crypt == "clear":
- raise salt.ext.tornado.gen.Return((payload, {"fun": "send_clear"}))
- raise salt.ext.tornado.gen.Return((payload, {"fun": "send"}))
+ raise tornado.gen.Return((payload, {"fun": "send_clear"}))
+ raise tornado.gen.Return((payload, {"fun": "send"}))
@pytest.fixture
diff --git a/tests/pytests/functional/channel/test_server.py b/tests/pytests/functional/channel/test_server.py
index 30bf2d35cba..37a75c16dd0 100644
--- a/tests/pytests/functional/channel/test_server.py
+++ b/tests/pytests/functional/channel/test_server.py
@@ -14,8 +14,8 @@ from saltfactories.utils import random_string
import salt.channel.client
import salt.channel.server
import salt.config
-import salt.ext.tornado.gen
-import salt.ext.tornado.ioloop
+import tornado.gen
+import tornado.ioloop
import salt.master
import salt.utils.platform
import salt.utils.process
@@ -117,7 +117,7 @@ def master_secrets():
salt.master.SMaster.secrets.pop("aes")
-@salt.ext.tornado.gen.coroutine
+@tornado.gen.coroutine
def _connect_and_publish(
io_loop, channel_minion_id, channel, server, received, timeout=60
):
@@ -134,7 +134,7 @@ def _connect_and_publish(
server.publish({"tgt_type": "glob", "tgt": [channel_minion_id], "WTF": "SON"})
start = time.time()
while time.time() - start < timeout:
- yield salt.ext.tornado.gen.sleep(1)
+ yield tornado.gen.sleep(1)
io_loop.stop()
diff --git a/tests/pytests/functional/netapi/rest_cherrypy/conftest.py b/tests/pytests/functional/netapi/rest_cherrypy/conftest.py
index e25ce0cf13f..90b43f938ed 100644
--- a/tests/pytests/functional/netapi/rest_cherrypy/conftest.py
+++ b/tests/pytests/functional/netapi/rest_cherrypy/conftest.py
@@ -1,6 +1,6 @@
import pytest
-import salt.ext.tornado.wsgi
+import tornado.wsgi
import salt.netapi.rest_cherrypy.app
import tests.support.netapi as netapi
from tests.support.mock import patch
@@ -23,7 +23,7 @@ def app(client_config, load_auth):
with patch("salt.netapi.NetapiClient._is_master_running", return_value=True), patch(
"salt.auth.Resolver.mk_token", load_auth.mk_token
):
- yield salt.ext.tornado.wsgi.WSGIContainer(
+ yield tornado.wsgi.WSGIContainer(
cherrypy.Application(app, "/", config=cherry_opts)
)
diff --git a/tests/pytests/functional/netapi/rest_cherrypy/test_auth.py b/tests/pytests/functional/netapi/rest_cherrypy/test_auth.py
index eb6e155487b..08bff51dc68 100644
--- a/tests/pytests/functional/netapi/rest_cherrypy/test_auth.py
+++ b/tests/pytests/functional/netapi/rest_cherrypy/test_auth.py
@@ -3,7 +3,7 @@ import urllib.parse
import pytest
import salt.utils.json
-from salt.ext.tornado.httpclient import HTTPError
+from tornado.httpclient import HTTPError
async def test_get_root_noauth(http_client):
diff --git a/tests/pytests/functional/netapi/rest_cherrypy/test_auth_pam.py b/tests/pytests/functional/netapi/rest_cherrypy/test_auth_pam.py
index f275b1510e3..d292424a61c 100644
--- a/tests/pytests/functional/netapi/rest_cherrypy/test_auth_pam.py
+++ b/tests/pytests/functional/netapi/rest_cherrypy/test_auth_pam.py
@@ -2,7 +2,7 @@ import urllib.parse
import pytest
-from salt.ext.tornado.httpclient import HTTPError
+from tornado.httpclient import HTTPError
pytestmark = [
pytest.mark.destructive_test,
diff --git a/tests/pytests/functional/netapi/rest_cherrypy/test_out_formats.py b/tests/pytests/functional/netapi/rest_cherrypy/test_out_formats.py
index 444bced9be1..2579f339276 100644
--- a/tests/pytests/functional/netapi/rest_cherrypy/test_out_formats.py
+++ b/tests/pytests/functional/netapi/rest_cherrypy/test_out_formats.py
@@ -1,6 +1,6 @@
import pytest
-from salt.ext.tornado.httpclient import HTTPError
+from tornado.httpclient import HTTPError
@pytest.fixture
diff --git a/tests/pytests/functional/netapi/rest_tornado/test_auth_handler.py b/tests/pytests/functional/netapi/rest_tornado/test_auth_handler.py
index 91e04ee0578..3ca335d4630 100644
--- a/tests/pytests/functional/netapi/rest_tornado/test_auth_handler.py
+++ b/tests/pytests/functional/netapi/rest_tornado/test_auth_handler.py
@@ -4,7 +4,7 @@ import pytest
import salt.utils.json
import salt.utils.yaml
-from salt.ext.tornado.httpclient import HTTPError
+from tornado.httpclient import HTTPError
from salt.netapi.rest_tornado import saltnado
pytestmark = [
diff --git a/tests/pytests/functional/netapi/rest_tornado/test_auth_handler_pam.py b/tests/pytests/functional/netapi/rest_tornado/test_auth_handler_pam.py
index ffd9871339b..006e4fb0ab4 100644
--- a/tests/pytests/functional/netapi/rest_tornado/test_auth_handler_pam.py
+++ b/tests/pytests/functional/netapi/rest_tornado/test_auth_handler_pam.py
@@ -4,7 +4,7 @@ import pytest
import salt.utils.json
import salt.utils.yaml
-from salt.ext.tornado.httpclient import HTTPError
+from tornado.httpclient import HTTPError
from salt.netapi.rest_tornado import saltnado
pytestmark = [
diff --git a/tests/pytests/functional/netapi/rest_tornado/test_base_api_handler.py b/tests/pytests/functional/netapi/rest_tornado/test_base_api_handler.py
index 78ece2b3d59..5d549fd6ff0 100644
--- a/tests/pytests/functional/netapi/rest_tornado/test_base_api_handler.py
+++ b/tests/pytests/functional/netapi/rest_tornado/test_base_api_handler.py
@@ -4,7 +4,7 @@ import pytest
import salt.utils.json
import salt.utils.yaml
-from salt.ext.tornado.httpclient import HTTPError
+from tornado.httpclient import HTTPError
from salt.netapi.rest_tornado import saltnado
@@ -27,6 +27,7 @@ class StubHandler(saltnado.BaseSaltAPIHandler): # pylint: disable=abstract-meth
ret_dict[attr] = getattr(self, attr)
self.write(self.serialize(ret_dict))
+ self.finish()
@pytest.fixture
@@ -37,7 +38,7 @@ def app_urls():
]
-async def test_accept_content_type(http_client, content_type_map, subtests):
+async def test_accept_content_type(http_client, content_type_map, subtests, io_loop):
"""
Test the base handler's accept picking
"""
@@ -94,7 +95,7 @@ async def test_token(http_client):
# send a token as a cookie
response = await http_client.fetch(
- "/", headers={"Cookie": "{}=foo".format(saltnado.AUTH_COOKIE_NAME)}
+ "/", headers={"Cookie": f"{saltnado.AUTH_COOKIE_NAME}=foo"}
)
token = salt.utils.json.loads(response.body)["token"]
assert token == "foo"
@@ -104,7 +105,7 @@ async def test_token(http_client):
"/",
headers={
saltnado.AUTH_TOKEN_HEADER: "foo",
- "Cookie": "{}=bar".format(saltnado.AUTH_COOKIE_NAME),
+ "Cookie": f"{saltnado.AUTH_COOKIE_NAME}=bar",
},
)
token = salt.utils.json.loads(response.body)["token"]
@@ -367,7 +368,7 @@ async def test_cors_preflight_request(http_client, app):
assert response.code == 204
-async def test_cors_origin_url_with_arguments(app, http_client):
+async def test_cors_origin_url_with_arguments(io_loop, app, http_client):
"""
Check that preflight requests works with url with components
like jobs or minions endpoints.
diff --git a/tests/pytests/functional/netapi/rest_tornado/test_utils.py b/tests/pytests/functional/netapi/rest_tornado/test_utils.py
index 69edce7da50..a32de85de1d 100644
--- a/tests/pytests/functional/netapi/rest_tornado/test_utils.py
+++ b/tests/pytests/functional/netapi/rest_tornado/test_utils.py
@@ -1,4 +1,4 @@
-import salt.ext.tornado.concurrent
+import tornado.concurrent
from salt.netapi.rest_tornado import saltnado
@@ -9,7 +9,7 @@ async def test_any_future():
# create a few futures
futures = []
for _ in range(3):
- future = salt.ext.tornado.concurrent.Future()
+ future = tornado.concurrent.Future()
futures.append(future)
# create an any future, make sure it isn't immediately done
diff --git a/tests/pytests/functional/netapi/rest_tornado/test_websockets_handler.py b/tests/pytests/functional/netapi/rest_tornado/test_websockets_handler.py
index d039e75d29b..c6673ccd310 100644
--- a/tests/pytests/functional/netapi/rest_tornado/test_websockets_handler.py
+++ b/tests/pytests/functional/netapi/rest_tornado/test_websockets_handler.py
@@ -6,8 +6,8 @@ import pytest
import salt.netapi.rest_tornado as rest_tornado
import salt.utils.json
import salt.utils.yaml
-from salt.ext.tornado.httpclient import HTTPError, HTTPRequest
-from salt.ext.tornado.websocket import websocket_connect
+from tornado.httpclient import HTTPError, HTTPRequest
+from tornado.websocket import websocket_connect
pytestmark = [
pytest.mark.destructive_test,
diff --git a/tests/pytests/functional/transport/ipc/test_client.py b/tests/pytests/functional/transport/ipc/test_client.py
index df6f2372011..fb064843e41 100644
--- a/tests/pytests/functional/transport/ipc/test_client.py
+++ b/tests/pytests/functional/transport/ipc/test_client.py
@@ -5,7 +5,7 @@ import pytest
import salt.transport.ipc
import salt.utils.platform
-from salt.ext.tornado import locks
+from tornado import locks
pytestmark = [
# Windows does not support POSIX IPC
diff --git a/tests/pytests/functional/transport/ipc/test_subscriber.py b/tests/pytests/functional/transport/ipc/test_subscriber.py
index 07676305cb4..5bb9ea486ab 100644
--- a/tests/pytests/functional/transport/ipc/test_subscriber.py
+++ b/tests/pytests/functional/transport/ipc/test_subscriber.py
@@ -4,10 +4,10 @@ import attr
import pytest
import salt.channel.server
-import salt.ext.tornado.gen
+import tornado.gen
import salt.transport.ipc
import salt.utils.platform
-from salt.ext.tornado import locks
+from tornado import locks
pytestmark = [
# Windows does not support POSIX IPC
@@ -110,9 +110,9 @@ async def test_basic_send(channel):
# XXX: IPCClient connect and connected methods need to be cleaned up as
# this should not be needed.
while not channel.subscriber._connecting_future.done():
- await salt.ext.tornado.gen.sleep(0.01)
+ await tornado.gen.sleep(0.01)
while not channel.subscriber.connected():
- await salt.ext.tornado.gen.sleep(0.01)
+ await tornado.gen.sleep(0.01)
assert channel.subscriber.connected()
await channel.publish(msg)
ret = await channel.read()
diff --git a/tests/pytests/functional/transport/server/test_req_channel.py b/tests/pytests/functional/transport/server/test_req_channel.py
index 0ea71318f59..6bdab71ee03 100644
--- a/tests/pytests/functional/transport/server/test_req_channel.py
+++ b/tests/pytests/functional/transport/server/test_req_channel.py
@@ -9,7 +9,7 @@ import salt.channel.client
import salt.channel.server
import salt.config
import salt.exceptions
-import salt.ext.tornado.gen
+import tornado.gen
import salt.master
import salt.transport.client
import salt.transport.server
@@ -57,7 +57,7 @@ class ReqServerChannelProcess(salt.utils.process.SignalHandlingProcess):
),
}
- self.io_loop = salt.ext.tornado.ioloop.IOLoop()
+ self.io_loop = tornado.ioloop.IOLoop()
self.io_loop.make_current()
self.req_server_channel.post_fork(self._handle_payload, io_loop=self.io_loop)
self.io_loop.add_callback(self.running.set)
@@ -93,17 +93,17 @@ class ReqServerChannelProcess(salt.utils.process.SignalHandlingProcess):
terminate_process(pid=pid, kill_children=True, slow_stop=False)
self.process_manager = None
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _handle_payload(self, payload):
if self.req_channel_crypt == "clear":
- raise salt.ext.tornado.gen.Return((payload, {"fun": "send_clear"}))
+ raise tornado.gen.Return((payload, {"fun": "send_clear"}))
for key in (
"id",
"ts",
"tok",
):
payload["load"].pop(key, None)
- raise salt.ext.tornado.gen.Return((payload, {"fun": "send"}))
+ raise tornado.gen.Return((payload, {"fun": "send"}))
@pytest.fixture
diff --git a/tests/pytests/functional/transport/tcp/test_message_client.py b/tests/pytests/functional/transport/tcp/test_message_client.py
index 292f389000d..b989f951488 100644
--- a/tests/pytests/functional/transport/tcp/test_message_client.py
+++ b/tests/pytests/functional/transport/tcp/test_message_client.py
@@ -2,9 +2,9 @@ import logging
import pytest
-import salt.ext.tornado.gen
-import salt.ext.tornado.iostream
-import salt.ext.tornado.tcpserver
+import tornado.gen
+import tornado.iostream
+import tornado.tcpserver
import salt.transport.tcp
import salt.utils.msgpack
@@ -21,7 +21,7 @@ def config():
@pytest.fixture
def server(config):
- class TestServer(salt.ext.tornado.tcpserver.TCPServer):
+ class TestServer(tornado.tcpserver.TCPServer):
send = []
disconnect = False
@@ -31,10 +31,10 @@ def server(config):
msg = self.send.pop(0)
try:
await stream.write(msg)
- except salt.ext.tornado.iostream.StreamClosedError:
+ except tornado.iostream.StreamClosedError:
break
else:
- await salt.ext.tornado.gen.sleep(1)
+ await tornado.gen.sleep(1)
stream.close()
server = TestServer()
@@ -81,14 +81,14 @@ async def test_message_client_reconnect(io_loop, config, client, server):
server.send.append(partial)
while not received:
- await salt.ext.tornado.gen.sleep(1)
+ await tornado.gen.sleep(1)
assert received == [msg]
# The message client has unpacked one msg and there is a partial msg left in
# the unpacker. Closing the stream now leaves the unpacker in a bad state
# since the rest of the partil message will never be received.
server.disconnect = True
- await salt.ext.tornado.gen.sleep(1)
+ await tornado.gen.sleep(1)
server.disconnect = False
received = []
@@ -97,5 +97,5 @@ async def test_message_client_reconnect(io_loop, config, client, server):
# rest of this test would fail.
server.send.append(pmsg)
while not received:
- await salt.ext.tornado.gen.sleep(1)
+ await tornado.gen.sleep(1)
assert received == [msg, msg]
diff --git a/tests/pytests/functional/utils/test_async_event_publisher.py b/tests/pytests/functional/utils/test_async_event_publisher.py
index aa773b452ab..12f4e68889a 100644
--- a/tests/pytests/functional/utils/test_async_event_publisher.py
+++ b/tests/pytests/functional/utils/test_async_event_publisher.py
@@ -2,7 +2,7 @@ import pytest
import zmq
import salt.config
-import salt.ext.tornado.ioloop
+import tornado.ioloop
import salt.utils.event
import salt.utils.stringutils
diff --git a/tests/pytests/integration/netapi/rest_cherrypy/conftest.py b/tests/pytests/integration/netapi/rest_cherrypy/conftest.py
index bc0eb017bce..213b0390ae2 100644
--- a/tests/pytests/integration/netapi/rest_cherrypy/conftest.py
+++ b/tests/pytests/integration/netapi/rest_cherrypy/conftest.py
@@ -1,6 +1,6 @@
import pytest
-import salt.ext.tornado.wsgi
+import tornado.wsgi
import salt.netapi.rest_cherrypy.app
import tests.support.netapi as netapi
@@ -22,7 +22,7 @@ def client_config(client_config, netapi_port, request):
def app(client_config, load_auth, salt_minion):
app, _, cherry_opts = salt.netapi.rest_cherrypy.app.get_app(client_config)
- return salt.ext.tornado.wsgi.WSGIContainer(
+ return tornado.wsgi.WSGIContainer(
cherrypy.Application(app, "/", config=cherry_opts)
)
diff --git a/tests/pytests/integration/netapi/rest_cherrypy/test_auth.py b/tests/pytests/integration/netapi/rest_cherrypy/test_auth.py
index ab58d70b5ed..068e7075ede 100644
--- a/tests/pytests/integration/netapi/rest_cherrypy/test_auth.py
+++ b/tests/pytests/integration/netapi/rest_cherrypy/test_auth.py
@@ -3,7 +3,7 @@ import urllib.parse
import pytest
import salt.utils.json
-from salt.ext.tornado.httpclient import HTTPError
+from tornado.httpclient import HTTPError
async def test_get_root_noauth(http_client):
diff --git a/tests/pytests/integration/netapi/rest_cherrypy/test_run.py b/tests/pytests/integration/netapi/rest_cherrypy/test_run.py
index e012d13c593..f9a3f7aa61d 100644
--- a/tests/pytests/integration/netapi/rest_cherrypy/test_run.py
+++ b/tests/pytests/integration/netapi/rest_cherrypy/test_run.py
@@ -2,7 +2,7 @@ import urllib.parse
import pytest
-from salt.ext.tornado.httpclient import HTTPError
+from tornado.httpclient import HTTPError
@pytest.mark.netapi_client_data(["local"])
diff --git a/tests/pytests/integration/netapi/rest_tornado/test_events_api_handler.py b/tests/pytests/integration/netapi/rest_tornado/test_events_api_handler.py
index 188e4603492..a307767be94 100644
--- a/tests/pytests/integration/netapi/rest_tornado/test_events_api_handler.py
+++ b/tests/pytests/integration/netapi/rest_tornado/test_events_api_handler.py
@@ -2,7 +2,7 @@ from functools import partial
import pytest
-import salt.ext.tornado.gen
+import tornado.gen
from salt.netapi.rest_tornado import saltnado
# TODO: run all the same tests from the root handler, but for now since they are
@@ -45,6 +45,6 @@ async def test_get(http_client, io_loop, app):
)
while len(events_fired) < 5:
- await salt.ext.tornado.gen.sleep(1)
+ await tornado.gen.sleep(1)
assert len(events_fired) >= 5
diff --git a/tests/pytests/integration/netapi/rest_tornado/test_minions_api_handler.py b/tests/pytests/integration/netapi/rest_tornado/test_minions_api_handler.py
index 05146b45a45..fffee6cd8b4 100644
--- a/tests/pytests/integration/netapi/rest_tornado/test_minions_api_handler.py
+++ b/tests/pytests/integration/netapi/rest_tornado/test_minions_api_handler.py
@@ -1,7 +1,7 @@
import pytest
import salt.utils.json
-from salt.ext.tornado.httpclient import HTTPError
+from tornado.httpclient import HTTPError
from salt.netapi.rest_tornado import saltnado
diff --git a/tests/pytests/integration/netapi/rest_tornado/test_root_handler.py b/tests/pytests/integration/netapi/rest_tornado/test_root_handler.py
index 9dd4cbd6aab..7d93c1d6a17 100644
--- a/tests/pytests/integration/netapi/rest_tornado/test_root_handler.py
+++ b/tests/pytests/integration/netapi/rest_tornado/test_root_handler.py
@@ -1,7 +1,7 @@
import pytest
import salt.utils.json
-from salt.ext.tornado.httpclient import HTTPError
+from tornado.httpclient import HTTPError
from salt.netapi.rest_tornado import saltnado
diff --git a/tests/pytests/unit/channel/test_server.py b/tests/pytests/unit/channel/test_server.py
index 3f6262f89b1..806b82bbaaf 100644
--- a/tests/pytests/unit/channel/test_server.py
+++ b/tests/pytests/unit/channel/test_server.py
@@ -3,7 +3,7 @@ import time
import pytest
import salt.channel.server as server
-import salt.ext.tornado.gen
+import tornado.gen
from tests.support.mock import MagicMock, patch
@@ -19,7 +19,7 @@ def test__auth_cmd_stats_passing(master_opts):
time.sleep(0.03)
return fake_ret
- future = salt.ext.tornado.gen.Future()
+ future = tornado.gen.Future()
future.set_result({})
with patch.object(req_server_channel, "_auth", _auth_mock):
diff --git a/tests/pytests/unit/cli/test_batch_async.py b/tests/pytests/unit/cli/test_batch_async.py
index be8de692e64..cd73b8d883b 100644
--- a/tests/pytests/unit/cli/test_batch_async.py
+++ b/tests/pytests/unit/cli/test_batch_async.py
@@ -1,6 +1,6 @@
import pytest
-import salt.ext.tornado
+import tornado
from salt.cli.batch_async import BatchAsync, batch_async_required
from tests.support.mock import MagicMock, patch
@@ -79,12 +79,12 @@ def test_batch_size(batch):
def test_batch_start_on_batch_presence_ping_timeout(batch):
- future_ret = salt.ext.tornado.gen.Future()
+ future_ret = tornado.gen.Future()
future_ret.set_result({"minions": ["foo", "bar"]})
- future = salt.ext.tornado.gen.Future()
+ future = tornado.gen.Future()
future.set_result({})
with patch.object(batch, "events_channel", MagicMock()), patch(
- "salt.ext.tornado.gen.sleep", return_value=future
+ "tornado.gen.sleep", return_value=future
), patch.object(batch, "io_loop", MagicMock()), patch.object(
batch, "start_batch", return_value=future
) as start_batch_mock:
@@ -108,13 +108,13 @@ def test_batch_start_on_batch_presence_ping_timeout(batch):
def test_batch_start_on_gather_job_timeout(batch):
- future = salt.ext.tornado.gen.Future()
+ future = tornado.gen.Future()
future.set_result({})
- future_ret = salt.ext.tornado.gen.Future()
+ future_ret = tornado.gen.Future()
future_ret.set_result({"minions": ["foo", "bar"]})
batch.batch_presence_ping_timeout = None
with patch.object(batch, "events_channel", MagicMock()), patch(
- "salt.ext.tornado.gen.sleep", return_value=future
+ "tornado.gen.sleep", return_value=future
), patch.object(batch, "io_loop", MagicMock()), patch.object(
batch, "start_batch", return_value=future
), patch.object(
@@ -149,7 +149,7 @@ def test_batch_fire_start_event(batch):
def test_start_batch_calls_next(batch):
batch.initialized = False
- future = salt.ext.tornado.gen.Future()
+ future = tornado.gen.Future()
future.set_result({})
with patch.object(batch, "event", MagicMock()), patch.object(
batch, "events_channel", MagicMock()
@@ -195,9 +195,9 @@ def test_batch_next(batch):
batch.opts["fun"] = "my.fun"
batch.opts["arg"] = []
batch.batch_size = 2
- future = salt.ext.tornado.gen.Future()
+ future = tornado.gen.Future()
future.set_result({})
- with patch("salt.ext.tornado.gen.sleep", return_value=future), patch.object(
+ with patch("tornado.gen.sleep", return_value=future), patch.object(
batch, "events_channel", MagicMock()
), patch.object(batch, "_get_next", return_value={"foo", "bar"}), patch.object(
batch, "find_job", return_value=future
@@ -283,7 +283,7 @@ def test_batch__event_handler_ping_return(batch):
def test_batch__event_handler_batch_run_return(batch):
- future = salt.ext.tornado.gen.Future()
+ future = tornado.gen.Future()
future.set_result({})
with patch.object(batch, "io_loop", MagicMock()), patch.object(
batch, "schedule_next", return_value=future
@@ -307,7 +307,7 @@ def test_batch__event_handler_find_job_return(batch):
def test_batch_run_next_end_batch_when_no_next(batch):
- future = salt.ext.tornado.gen.Future()
+ future = tornado.gen.Future()
future.set_result({})
with patch.object(batch, "_get_next", return_value={}), patch.object(
batch, "end_batch", return_value=future
@@ -317,10 +317,10 @@ def test_batch_run_next_end_batch_when_no_next(batch):
def test_batch_find_job(batch):
- future = salt.ext.tornado.gen.Future()
+ future = tornado.gen.Future()
future.set_result({})
batch.minions = {"foo", "bar"}
- with patch("salt.ext.tornado.gen.sleep", return_value=future), patch.object(
+ with patch("tornado.gen.sleep", return_value=future), patch.object(
batch, "check_find_job", return_value=future
) as check_find_job_mock, patch.object(batch, "jid_gen", return_value="1236"):
batch.events_channel.local_client.run_job_async.return_value = future
@@ -333,10 +333,10 @@ def test_batch_find_job(batch):
def test_batch_find_job_with_done_minions(batch):
batch.done_minions = {"bar"}
- future = salt.ext.tornado.gen.Future()
+ future = tornado.gen.Future()
future.set_result({})
batch.minions = {"foo", "bar"}
- with patch("salt.ext.tornado.gen.sleep", return_value=future), patch.object(
+ with patch("tornado.gen.sleep", return_value=future), patch.object(
batch, "check_find_job", return_value=future
) as check_find_job_mock, patch.object(batch, "jid_gen", return_value="1236"):
batch.events_channel.local_client.run_job_async.return_value = future
@@ -350,7 +350,7 @@ def test_batch_find_job_with_done_minions(batch):
def test_batch_check_find_job_did_not_return(batch):
batch.active = {"foo"}
batch.find_job_returned = set()
- future = salt.ext.tornado.gen.Future()
+ future = tornado.gen.Future()
future.set_result({})
with patch.object(batch, "find_job", return_value=future) as find_job_mock:
batch.check_find_job({"foo"}, jid="1234")
@@ -361,7 +361,7 @@ def test_batch_check_find_job_did_not_return(batch):
def test_batch_check_find_job_did_return(batch):
batch.find_job_returned = {"foo"}
- future = salt.ext.tornado.gen.Future()
+ future = tornado.gen.Future()
future.set_result({})
with patch.object(batch, "find_job", return_value=future) as find_job_mock:
batch.check_find_job({"foo"}, jid="1234")
@@ -384,7 +384,7 @@ def test_batch_check_find_job_multiple_states(batch):
# both not yet done but only 'foo' responded to find_job
not_done = {"foo", "bar"}
- future = salt.ext.tornado.gen.Future()
+ future = tornado.gen.Future()
future.set_result({})
with patch.object(batch, "schedule_next", return_value=future), patch.object(
@@ -403,7 +403,7 @@ def test_batch_check_find_job_multiple_states(batch):
def test_only_on_run_next_is_scheduled(batch):
- future = salt.ext.tornado.gen.Future()
+ future = tornado.gen.Future()
future.set_result({})
batch.scheduled = True
with patch.object(batch, "run_next", return_value=future) as run_next_mock:
diff --git a/tests/pytests/unit/fileserver/gitfs/test_gitfs.py b/tests/pytests/unit/fileserver/gitfs/test_gitfs.py
index 7896fd93bb7..18d3893a5d0 100644
--- a/tests/pytests/unit/fileserver/gitfs/test_gitfs.py
+++ b/tests/pytests/unit/fileserver/gitfs/test_gitfs.py
@@ -25,7 +25,7 @@ import pathlib
import pytest
-import salt.ext.tornado.ioloop
+import tornado.ioloop
import salt.fileserver.gitfs as gitfs
import salt.utils.files
import salt.utils.gitfs
@@ -127,7 +127,7 @@ def testfile(tmp_path):
def repo_dir(tmp_path, unicode_dirname, tag_name, unicode_filename):
try:
del salt.utils.gitfs.GitFS.instance_map[
- salt.ext.tornado.ioloop.IOLoop.current()
+ tornado.ioloop.IOLoop.current()
]
except KeyError:
pass
diff --git a/tests/pytests/unit/fileserver/gitfs/test_gitfs_config.py b/tests/pytests/unit/fileserver/gitfs/test_gitfs_config.py
index afd0cfdc9e2..ca502f716f3 100644
--- a/tests/pytests/unit/fileserver/gitfs/test_gitfs_config.py
+++ b/tests/pytests/unit/fileserver/gitfs/test_gitfs_config.py
@@ -2,7 +2,7 @@ import textwrap
import pytest
-import salt.ext.tornado.ioloop
+import tornado.ioloop
import salt.fileserver.gitfs as gitfs
import salt.utils.files
import salt.utils.gitfs
@@ -74,7 +74,7 @@ def configure_loader_modules(tmp_path):
def clear_instance_map():
try:
del salt.utils.gitfs.GitFS.instance_map[
- salt.ext.tornado.ioloop.IOLoop.current()
+ tornado.ioloop.IOLoop.current()
]
except KeyError:
pass
diff --git a/tests/pytests/unit/modules/test_linux_shadow.py b/tests/pytests/unit/modules/test_linux_shadow.py
index 5d977fc5212..723e3a8ad26 100644
--- a/tests/pytests/unit/modules/test_linux_shadow.py
+++ b/tests/pytests/unit/modules/test_linux_shadow.py
@@ -1,6 +1,7 @@
"""
- :codeauthor: Erik Johnson <erik@saltstack.com>
+:codeauthor: Erik Johnson <erik@saltstack.com>
"""
+
import types
import pytest
@@ -174,6 +175,11 @@ def test_info(password):
Test if info shows the correct user information
"""
+ data = {
+ "/etc/shadow": f"foo:{password.pw_hash}:31337:0:99999:7:::",
+ "*": Exception("Attempted to open something other than /etc/shadow"),
+ }
+
# First test is with a succesful call
expected_result = [
("expire", -1),
@@ -185,10 +191,7 @@ def test_info(password):
("passwd", password.pw_hash),
("warn", 7),
]
- getspnam_return = spwd.struct_spwd(
- ["foo", password.pw_hash, 31337, 0, 99999, 7, -1, -1, -1]
- )
- with patch("spwd.getspnam", return_value=getspnam_return):
+ with patch("salt.utils.files.fopen", mock_open(read_data=data)):
result = shadow.info("foo")
assert expected_result == sorted(result.items(), key=lambda x: x[0])
@@ -203,15 +206,8 @@ def test_info(password):
("passwd", ""),
("warn", ""),
]
- # We get KeyError exception for non-existent users in glibc based systems
- getspnam_return = KeyError
- with patch("spwd.getspnam", side_effect=getspnam_return):
- result = shadow.info("foo")
- assert expected_result == sorted(result.items(), key=lambda x: x[0])
- # And FileNotFoundError in musl based systems
- getspnam_return = FileNotFoundError
- with patch("spwd.getspnam", side_effect=getspnam_return):
- result = shadow.info("foo")
+ with patch("salt.utils.files.fopen", mock_open(read_data=data)):
+ result = shadow.info("bar")
assert expected_result == sorted(result.items(), key=lambda x: x[0])
diff --git a/tests/pytests/unit/test_ext_importers.py b/tests/pytests/unit/test_ext_importers.py
deleted file mode 100644
index e81ef234a92..00000000000
--- a/tests/pytests/unit/test_ext_importers.py
+++ /dev/null
@@ -1,51 +0,0 @@
-import logging
-import os
-import subprocess
-import sys
-
-import pytest
-
-import salt
-
-log = logging.getLogger(__name__)
-
-
-def test_tornado_import_override(tmp_path):
- """
- Ensure we are not using any non vendor'ed tornado
- """
- test_source = """
- from __future__ import absolute_import, print_function
- import salt
- import tornado
- print(tornado.__name__)
- """
- tornado_source = """
- foo = 'bar'
- """
- with pytest.helpers.temp_file(
- "test.py", directory=tmp_path, contents=test_source
- ) as test_source_path, pytest.helpers.temp_file(
- "tornado.py", directory=tmp_path, contents=tornado_source
- ):
- env = os.environ.copy()
- env["PYTHONPATH"] = os.pathsep.join(sys.path)
- ret = subprocess.run(
- [sys.executable, str(test_source_path)],
- stderr=subprocess.PIPE,
- stdout=subprocess.PIPE,
- env=env,
- shell=False,
- check=False,
- universal_newlines=True,
- )
- assert ret.returncode == 0
- assert ret.stdout.strip() == "salt.ext.tornado"
-
-
-def test_regression_56063():
- importer = salt.TornadoImporter()
- try:
- importer.find_module("tornado")
- except TypeError:
- assert False, "TornadoImporter raised type error when one argument passed"
diff --git a/tests/pytests/unit/test_minion.py b/tests/pytests/unit/test_minion.py
index 017c28d1634..7c5c14c0182 100644
--- a/tests/pytests/unit/test_minion.py
+++ b/tests/pytests/unit/test_minion.py
@@ -4,9 +4,9 @@ import os
import pytest
-import salt.ext.tornado
-import salt.ext.tornado.gen
-import salt.ext.tornado.testing
+import tornado
+import tornado.gen
+import tornado.testing
import salt.minion
import salt.syspaths
import salt.utils.crypt
@@ -57,7 +57,7 @@ def test_minion_load_grains_default(minion_opts):
[
(
"salt.channel.client.AsyncReqChannel.factory",
- lambda load, timeout, tries: salt.ext.tornado.gen.maybe_future(tries),
+ lambda load, timeout, tries: tornado.gen.maybe_future(tries),
),
(
"salt.channel.client.ReqChannel.factory",
@@ -261,7 +261,7 @@ def test_handle_decoded_payload_jid_match_in_jid_queue(minion_opts):
minion = salt.minion.Minion(
minion_opts,
jid_queue=copy.copy(mock_jid_queue),
- io_loop=salt.ext.tornado.ioloop.IOLoop(),
+ io_loop=tornado.ioloop.IOLoop(),
)
try:
ret = minion._handle_decoded_payload(mock_data).result()
@@ -290,7 +290,7 @@ def test_handle_decoded_payload_jid_queue_addition(minion_opts):
minion = salt.minion.Minion(
minion_opts,
jid_queue=copy.copy(mock_jid_queue),
- io_loop=salt.ext.tornado.ioloop.IOLoop(),
+ io_loop=tornado.ioloop.IOLoop(),
)
try:
@@ -327,7 +327,7 @@ def test_handle_decoded_payload_jid_queue_reduced_minion_jid_queue_hwm(minion_op
minion = salt.minion.Minion(
minion_opts,
jid_queue=copy.copy(mock_jid_queue),
- io_loop=salt.ext.tornado.ioloop.IOLoop(),
+ io_loop=tornado.ioloop.IOLoop(),
)
try:
@@ -359,15 +359,15 @@ def test_process_count_max(minion_opts):
), patch(
"salt.utils.minion.running", MagicMock(return_value=[])
), patch(
- "salt.ext.tornado.gen.sleep",
- MagicMock(return_value=salt.ext.tornado.concurrent.Future()),
+ "tornado.gen.sleep",
+ MagicMock(return_value=tornado.concurrent.Future()),
):
process_count_max = 10
minion_opts["__role"] = "minion"
minion_opts["minion_jid_queue_hwm"] = 100
minion_opts["process_count_max"] = process_count_max
- io_loop = salt.ext.tornado.ioloop.IOLoop()
+ io_loop = tornado.ioloop.IOLoop()
minion = salt.minion.Minion(minion_opts, jid_queue=[], io_loop=io_loop)
try:
@@ -375,7 +375,7 @@ def test_process_count_max(minion_opts):
class SleepCalledException(Exception):
"""Thrown when sleep is called"""
- salt.ext.tornado.gen.sleep.return_value.set_exception(
+ tornado.gen.sleep.return_value.set_exception(
SleepCalledException()
)
@@ -425,7 +425,7 @@ def test_beacons_before_connect(minion_opts):
MagicMock(return_value=True),
):
minion_opts["beacons_before_connect"] = True
- io_loop = salt.ext.tornado.ioloop.IOLoop()
+ io_loop = tornado.ioloop.IOLoop()
io_loop.make_current()
minion = salt.minion.Minion(minion_opts, io_loop=io_loop)
try:
@@ -458,7 +458,7 @@ def test_scheduler_before_connect(minion_opts):
MagicMock(return_value=True),
):
minion_opts["scheduler_before_connect"] = True
- io_loop = salt.ext.tornado.ioloop.IOLoop()
+ io_loop = tornado.ioloop.IOLoop()
io_loop.make_current()
minion = salt.minion.Minion(minion_opts, io_loop=io_loop)
try:
@@ -489,7 +489,7 @@ def test_minion_module_refresh(minion_opts):
try:
minion = salt.minion.Minion(
minion_opts,
- io_loop=salt.ext.tornado.ioloop.IOLoop(),
+ io_loop=tornado.ioloop.IOLoop(),
)
minion.schedule = salt.utils.schedule.Schedule(
minion_opts, {}, returners={}
@@ -517,7 +517,7 @@ def test_minion_module_refresh_beacons_refresh(minion_opts):
try:
minion = salt.minion.Minion(
minion_opts,
- io_loop=salt.ext.tornado.ioloop.IOLoop(),
+ io_loop=tornado.ioloop.IOLoop(),
)
minion.schedule = salt.utils.schedule.Schedule(
minion_opts, {}, returners={}
@@ -547,7 +547,7 @@ def test_when_ping_interval_is_set_the_callback_should_be_added_to_periodic_call
MagicMock(return_value=True),
):
minion_opts["ping_interval"] = 10
- io_loop = salt.ext.tornado.ioloop.IOLoop()
+ io_loop = tornado.ioloop.IOLoop()
io_loop.make_current()
minion = salt.minion.Minion(minion_opts, io_loop=io_loop)
try:
@@ -569,7 +569,7 @@ def test_when_passed_start_event_grains(minion_opts):
# provide mock opts an os grain since we'll look for it later.
minion_opts["grains"]["os"] = "linux"
minion_opts["start_event_grains"] = ["os"]
- io_loop = salt.ext.tornado.ioloop.IOLoop()
+ io_loop = tornado.ioloop.IOLoop()
io_loop.make_current()
minion = salt.minion.Minion(minion_opts, io_loop=io_loop)
try:
@@ -588,7 +588,7 @@ def test_when_passed_start_event_grains(minion_opts):
@pytest.mark.slow_test
def test_when_not_passed_start_event_grains(minion_opts):
- io_loop = salt.ext.tornado.ioloop.IOLoop()
+ io_loop = tornado.ioloop.IOLoop()
io_loop.make_current()
minion = salt.minion.Minion(minion_opts, io_loop=io_loop)
try:
@@ -605,7 +605,7 @@ def test_when_not_passed_start_event_grains(minion_opts):
@pytest.mark.slow_test
def test_when_other_events_fired_and_start_event_grains_are_set(minion_opts):
minion_opts["start_event_grains"] = ["os"]
- io_loop = salt.ext.tornado.ioloop.IOLoop()
+ io_loop = tornado.ioloop.IOLoop()
io_loop.make_current()
minion = salt.minion.Minion(minion_opts, io_loop=io_loop)
try:
@@ -643,7 +643,7 @@ def test_gen_modules_executors(minion_opts):
"""
Ensure gen_modules is called with the correct arguments #54429
"""
- io_loop = salt.ext.tornado.ioloop.IOLoop()
+ io_loop = tornado.ioloop.IOLoop()
io_loop.make_current()
minion = salt.minion.Minion(minion_opts, io_loop=io_loop)
@@ -669,7 +669,7 @@ def test_reinit_crypto_on_fork(minion_opts):
minion_opts["multiprocessing"] = True
with patch("salt.utils.process.default_signals"):
- io_loop = salt.ext.tornado.ioloop.IOLoop()
+ io_loop = tornado.ioloop.IOLoop()
io_loop.make_current()
minion = salt.minion.Minion(minion_opts, io_loop=io_loop)
@@ -710,7 +710,7 @@ def test_minion_manage_schedule(minion_opts):
"salt.utils.process.SignalHandlingProcess.join",
MagicMock(return_value=True),
):
- io_loop = salt.ext.tornado.ioloop.IOLoop()
+ io_loop = tornado.ioloop.IOLoop()
io_loop.make_current()
with patch("salt.utils.schedule.clean_proc_dir", MagicMock(return_value=None)):
@@ -770,7 +770,7 @@ def test_minion_manage_beacons(minion_opts):
try:
minion_opts["beacons"] = {}
- io_loop = salt.ext.tornado.ioloop.IOLoop()
+ io_loop = tornado.ioloop.IOLoop()
io_loop.make_current()
mock_functions = {"test.ping": None}
@@ -940,7 +940,7 @@ def test_minion_grains_refresh_pre_exec_false(minion_opts):
minion = salt.minion.Minion(
minion_opts,
jid_queue=None,
- io_loop=salt.ext.tornado.ioloop.IOLoop(),
+ io_loop=tornado.ioloop.IOLoop(),
load_grains=False,
)
try:
@@ -963,7 +963,7 @@ def test_minion_grains_refresh_pre_exec_true(minion_opts):
minion = salt.minion.Minion(
minion_opts,
jid_queue=None,
- io_loop=salt.ext.tornado.ioloop.IOLoop(),
+ io_loop=tornado.ioloop.IOLoop(),
load_grains=False,
)
try:
diff --git a/tests/pytests/unit/transport/test_ipc.py b/tests/pytests/unit/transport/test_ipc.py
index 5a687836161..36a6e8f70b4 100644
--- a/tests/pytests/unit/transport/test_ipc.py
+++ b/tests/pytests/unit/transport/test_ipc.py
@@ -1,7 +1,7 @@
import pytest
from pytestshellutils.utils import ports
-import salt.ext.tornado.iostream
+import tornado.iostream
import salt.transport.ipc
import salt.utils.asynchronous
import salt.utils.platform
@@ -31,6 +31,6 @@ async def test_ipc_connect_sync_wrapped(io_loop, tmp_path):
kwargs={"io_loop": io_loop},
loop_kwarg="io_loop",
)
- with pytest.raises(salt.ext.tornado.iostream.StreamClosedError):
+ with pytest.raises(tornado.iostream.StreamClosedError):
# Don't `await subscriber.connect()`, that's the purpose of the SyncWrapper
subscriber.connect()
diff --git a/tests/pytests/unit/transport/test_tcp.py b/tests/pytests/unit/transport/test_tcp.py
index bcfb71f5590..537395d04c5 100644
--- a/tests/pytests/unit/transport/test_tcp.py
+++ b/tests/pytests/unit/transport/test_tcp.py
@@ -8,7 +8,7 @@ from pytestshellutils.utils import ports
import salt.channel.server
import salt.exceptions
-import salt.ext.tornado
+import tornado
import salt.transport.tcp
from tests.support.mock import MagicMock, PropertyMock, patch
@@ -31,9 +31,9 @@ def fake_crypto():
@pytest.fixture
def fake_authd():
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def return_nothing():
- raise salt.ext.tornado.gen.Return()
+ raise tornado.gen.Return()
with patch(
"salt.crypt.AsyncAuth.authenticated", new_callable=PropertyMock
@@ -85,7 +85,7 @@ def test_message_client_cleanup_on_close(client_socket, temp_salt_master):
"""
test message client cleanup on close
"""
- orig_loop = salt.ext.tornado.ioloop.IOLoop()
+ orig_loop = tornado.ioloop.IOLoop()
orig_loop.make_current()
opts = dict(temp_salt_master.config.copy(), transport="tcp")
@@ -241,7 +241,7 @@ def test_tcp_pub_server_channel_publish_filtering_str_list(temp_salt_master):
@pytest.fixture(scope="function")
def salt_message_client():
- io_loop_mock = MagicMock(spec=salt.ext.tornado.ioloop.IOLoop)
+ io_loop_mock = MagicMock(spec=tornado.ioloop.IOLoop)
io_loop_mock.call_later.side_effect = lambda *args, **kwargs: (args, kwargs)
client = salt.transport.tcp.MessageClient(
@@ -346,7 +346,7 @@ def test_timeout_message_unknown_future(salt_message_client):
# if we do have the actual future stored under the id, but it's none
# we shouldn't fail as well
message_id = 1
- future = salt.ext.tornado.concurrent.Future()
+ future = tornado.concurrent.Future()
future.attempts = 1
future.tries = 1
salt_message_client.send_future_map[message_id] = future
@@ -367,16 +367,16 @@ def xtest_client_reconnect_backoff(client_socket):
client.close()
assert t == 5
return
- # return salt.ext.tornado.gen.sleep()
+ # return tornado.gen.sleep()
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def connect(*args, **kwargs):
raise Exception("err")
client._tcp_client.connect = connect
try:
- with patch("salt.ext.tornado.gen.sleep", side_effect=_sleep):
+ with patch("tornado.gen.sleep", side_effect=_sleep):
client.io_loop.run_sync(client.connect)
finally:
client.close()
@@ -456,7 +456,7 @@ def test_presence_events_callback_passed(temp_salt_master, salt_message_client):
def test_presence_removed_on_stream_closed():
opts = {"presence_events": True}
- io_loop_mock = MagicMock(spec=salt.ext.tornado.ioloop.IOLoop)
+ io_loop_mock = MagicMock(spec=tornado.ioloop.IOLoop)
with patch("salt.master.AESFuncs.__init__", return_value=None):
server = salt.transport.tcp.PubServer(opts, io_loop=io_loop_mock)
@@ -464,12 +464,12 @@ def test_presence_removed_on_stream_closed():
server.remove_presence_callback = MagicMock()
client = salt.transport.tcp.Subscriber(
- salt.ext.tornado.iostream.IOStream, "1.2.3.4"
+ tornado.iostream.IOStream, "1.2.3.4"
)
client._closing = True
server.clients = {client}
- io_loop = salt.ext.tornado.ioloop.IOLoop.current()
+ io_loop = tornado.ioloop.IOLoop.current()
package = {
"topic_lst": [],
"payload": "test-payload",
@@ -477,8 +477,8 @@ def test_presence_removed_on_stream_closed():
with patch("salt.transport.frame.frame_msg", return_value="framed-payload"):
with patch(
- "salt.ext.tornado.iostream.BaseIOStream.write",
- side_effect=salt.ext.tornado.iostream.StreamClosedError(),
+ "tornado.iostream.BaseIOStream.write",
+ side_effect=tornado.iostream.StreamClosedError(),
):
io_loop.run_sync(functools.partial(server.publish_payload, package, None))
diff --git a/tests/pytests/unit/transport/test_zeromq.py b/tests/pytests/unit/transport/test_zeromq.py
index fd0e4b19772..95216cc9c48 100644
--- a/tests/pytests/unit/transport/test_zeromq.py
+++ b/tests/pytests/unit/transport/test_zeromq.py
@@ -19,8 +19,8 @@ import salt.channel.server
import salt.config
import salt.crypt
import salt.exceptions
-import salt.ext.tornado.gen
-import salt.ext.tornado.ioloop
+import tornado.gen
+import tornado.ioloop
import salt.transport.zeromq
import salt.utils.platform
import salt.utils.process
@@ -344,14 +344,14 @@ def run_loop_in_thread(loop, evt):
"""
loop.make_current()
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def stopper():
- yield salt.ext.tornado.gen.sleep(0.1)
+ yield tornado.gen.sleep(0.1)
while True:
if not evt.is_set():
loop.stop()
break
- yield salt.ext.tornado.gen.sleep(0.3)
+ yield tornado.gen.sleep(0.3)
loop.add_callback(evt.set)
loop.add_callback(stopper)
@@ -383,7 +383,7 @@ class MockSaltMinionMaster:
self.server_channel = salt.channel.server.ReqServerChannel.factory(master_opts)
self.server_channel.pre_fork(self.process_manager)
- self.io_loop = salt.ext.tornado.ioloop.IOLoop()
+ self.io_loop = tornado.ioloop.IOLoop()
self.evt = threading.Event()
self.server_channel.post_fork(self._handle_payload, io_loop=self.io_loop)
self.server_thread = threading.Thread(
@@ -426,13 +426,13 @@ class MockSaltMinionMaster:
# pylint: enable=W1701
@classmethod
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _handle_payload(cls, payload):
"""
TODO: something besides echo
"""
cls.mock._handle_payload_hook()
- raise salt.ext.tornado.gen.Return((payload, {"fun": "send_clear"}))
+ raise tornado.gen.Return((payload, {"fun": "send_clear"}))
@pytest.mark.parametrize("message", ["", [], ()])
@@ -462,7 +462,7 @@ def test_serverside_exception(temp_salt_minion, temp_salt_master):
"""
with MockSaltMinionMaster(temp_salt_minion, temp_salt_master) as minion_master:
with patch.object(minion_master.mock, "_handle_payload_hook") as _mock:
- _mock.side_effect = salt.ext.tornado.gen.Return(({}, {"fun": "madeup-fun"}))
+ _mock.side_effect = tornado.gen.Return(({}, {"fun": "madeup-fun"}))
ret = minion_master.channel.send({}, timeout=5, tries=1)
assert ret == "Server-side exception handling payload"
@@ -485,7 +485,7 @@ def test_zeromq_async_pub_channel_publish_port(temp_salt_master):
sign_pub_messages=False,
)
opts["master_uri"] = "tcp://{interface}:{publish_port}".format(**opts)
- ioloop = salt.ext.tornado.ioloop.IOLoop()
+ ioloop = tornado.ioloop.IOLoop()
transport = salt.transport.zeromq.PublishClient(opts, ioloop)
with transport:
patch_socket = MagicMock(return_value=True)
@@ -527,7 +527,7 @@ def test_zeromq_async_pub_channel_filtering_decode_message_no_match(
)
opts["master_uri"] = "tcp://{interface}:{publish_port}".format(**opts)
- ioloop = salt.ext.tornado.ioloop.IOLoop()
+ ioloop = tornado.ioloop.IOLoop()
channel = salt.transport.zeromq.PublishClient(opts, ioloop)
with channel:
with patch(
@@ -574,7 +574,7 @@ def test_zeromq_async_pub_channel_filtering_decode_message(
)
opts["master_uri"] = "tcp://{interface}:{publish_port}".format(**opts)
- ioloop = salt.ext.tornado.ioloop.IOLoop()
+ ioloop = tornado.ioloop.IOLoop()
channel = salt.transport.zeromq.PublishClient(opts, ioloop)
with channel:
with patch(
@@ -587,7 +587,7 @@ def test_zeromq_async_pub_channel_filtering_decode_message(
def test_req_server_chan_encrypt_v2(pki_dir, master_opts):
- loop = salt.ext.tornado.ioloop.IOLoop.current()
+ loop = tornado.ioloop.IOLoop.current()
master_opts.update(
{
"worker_threads": 1,
@@ -633,7 +633,7 @@ def test_req_server_chan_encrypt_v2(pki_dir, master_opts):
def test_req_server_chan_encrypt_v1(pki_dir, master_opts):
- loop = salt.ext.tornado.ioloop.IOLoop.current()
+ loop = tornado.ioloop.IOLoop.current()
master_opts.update(
{
"worker_threads": 1,
@@ -747,14 +747,14 @@ async def test_req_chan_decode_data_dict_entry_v2(pki_dir, master_opts, minion_o
client.auth.session_crypticle.loads = auth.session_crypticle.loads
client.transport = MagicMock()
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def mocksend(msg, timeout=60, tries=3):
client.transport.msg = msg
load = client.auth.session_crypticle.loads(msg["load"])
ret = server._encrypt_private(
pillar_data, dictkey, target, nonce=load["nonce"], sign_messages=True
)
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
client.transport.send = mocksend
@@ -819,10 +819,10 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_nonce(pki_dir, master_opts
pillar_data, dictkey, target, nonce=badnonce, sign_messages=True
)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def mocksend(msg, timeout=60, tries=3):
client.transport.msg = msg
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
client.transport.send = mocksend
@@ -890,7 +890,7 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_signature(pki_dir, master_
client.auth.session_crypticle.loads = auth.session_crypticle.loads
client.transport = MagicMock()
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def mocksend(msg, timeout=60, tries=3):
client.transport.msg = msg
load = client.auth.session_crypticle.loads(msg["load"])
@@ -912,7 +912,7 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_signature(pki_dir, master_
data["pillar"] = {"pillar1": "bar"}
signed_msg["data"] = salt.payload.dumps(data)
ret[dictkey] = pcrypt.dumps(signed_msg)
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
client.transport.send = mocksend
@@ -980,7 +980,7 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_key(pki_dir, master_opts,
client.auth.session_crypticle.loads = auth.session_crypticle.loads
client.transport = MagicMock()
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def mocksend(msg, timeout=60, tries=3):
client.transport.msg = msg
load = client.auth.session_crypticle.loads(msg["load"])
@@ -1009,7 +1009,7 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_key(pki_dir, master_opts,
else:
cipher = PKCS1_OAEP.new(pub)
ret["key"] = cipher.encrypt(key)
- raise salt.ext.tornado.gen.Return(ret)
+ raise tornado.gen.Return(ret)
client.transport.send = mocksend
diff --git a/tests/pytests/unit/utils/event/test_event.py b/tests/pytests/unit/utils/event/test_event.py
index fa9e420a933..3b261f95c8d 100644
--- a/tests/pytests/unit/utils/event/test_event.py
+++ b/tests/pytests/unit/utils/event/test_event.py
@@ -8,8 +8,8 @@ import pytest
import zmq.eventloop.ioloop
import salt.config
-import salt.ext.tornado.ioloop
-import salt.ext.tornado.iostream
+import tornado.ioloop
+import tornado.iostream
import salt.utils.event
import salt.utils.stringutils
from salt.exceptions import SaltDeserializationError
@@ -304,12 +304,12 @@ def test_connect_pull_should_debug_log_on_StreamClosedError():
salt.utils.event.log, "debug", autospec=True
) as mock_log_debug:
mock_pusher.connect.side_effect = (
- salt.ext.tornado.iostream.StreamClosedError
+ tornado.iostream.StreamClosedError
)
event.connect_pull()
call = mock_log_debug.mock_calls[0]
assert call.args[0] == "Unable to connect pusher: %s"
- assert isinstance(call.args[1], salt.ext.tornado.iostream.StreamClosedError)
+ assert isinstance(call.args[1], tornado.iostream.StreamClosedError)
assert call.args[1].args[0] == "Stream is closed"
@@ -329,7 +329,7 @@ def test_connect_pull_should_error_log_on_other_errors(error):
call = mock_log_error.mock_calls[0]
assert call.args[0] == "Unable to connect pusher: %s"
assert not isinstance(
- call.args[1], salt.ext.tornado.iostream.StreamClosedError
+ call.args[1], tornado.iostream.StreamClosedError
)
@@ -456,7 +456,7 @@ def test_event_single_timeout_tries(sock_dir):
write_calls_count = 0
real_stream_write = None
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def write_mock(pack):
nonlocal write_calls_count
nonlocal real_stream_write
@@ -464,7 +464,7 @@ def test_event_single_timeout_tries(sock_dir):
if write_calls_count > 3:
yield real_stream_write(pack)
else:
- raise salt.ext.tornado.iostream.StreamClosedError()
+ raise tornado.iostream.StreamClosedError()
with eventpublisher_process(str(sock_dir)), salt.utils.event.MasterEvent(
str(sock_dir), listen=True
@@ -480,7 +480,7 @@ def test_event_single_timeout_tries(sock_dir):
), patch.object(
me.pusher,
"connect",
- side_effect=salt.ext.tornado.iostream.StreamClosedError,
+ side_effect=tornado.iostream.StreamClosedError,
), patch.object(
me.pusher.stream,
"write",
diff --git a/tests/pytests/unit/utils/event/test_event_return.py b/tests/pytests/unit/utils/event/test_event_return.py
index e9548c701f4..67cee0d1350 100644
--- a/tests/pytests/unit/utils/event/test_event_return.py
+++ b/tests/pytests/unit/utils/event/test_event_return.py
@@ -1,7 +1,7 @@
import pytest
from pytestshellutils.utils.processes import terminate_process
-import salt.ext.tornado.ioloop
+import tornado.ioloop
import salt.utils.event
import salt.utils.stringutils
diff --git a/tests/pytests/unit/utils/test_asynchronous.py b/tests/pytests/unit/utils/test_asynchronous.py
index 2b5613e2bfe..b8b86d2ce03 100644
--- a/tests/pytests/unit/utils/test_asynchronous.py
+++ b/tests/pytests/unit/utils/test_asynchronous.py
@@ -1,8 +1,12 @@
+import asyncio
+
import tornado.gen
import tornado.ioloop
import salt.utils.asynchronous as asynchronous
+from salt import USE_VENDORED_TORNADO
+
class HelperA:
@@ -41,7 +45,11 @@ def test_helpers():
"""
Test that the helper classes do what we expect within a regular asynchronous env
"""
- io_loop = tornado.ioloop.IOLoop(make_current=False)
+ if USE_VENDORED_TORNADO:
+ io_loop = tornado.ioloop.IOLoop(make_current=False)
+ else:
+ asyncio_loop = asyncio.new_event_loop()
+ io_loop = tornado.ioloop.IOLoop(asyncio_loop=asyncio_loop, make_current=False)
ret = io_loop.run_sync(lambda: HelperA().sleep())
assert ret is True
@@ -89,4 +97,4 @@ def test_double_sameloop():
a = asynchronous.SyncWrapper(HelperA)
sync = asynchronous.SyncWrapper(HelperB, (a,))
ret = sync.sleep()
- assert ret is False
+ assert ret is False
\ No newline at end of file
diff --git a/tests/support/helpers.py b/tests/support/helpers.py
index 3556e08853b..5dc4041ff4d 100644
--- a/tests/support/helpers.py
+++ b/tests/support/helpers.py
@@ -37,8 +37,8 @@ from pytestshellutils.exceptions import ProcessFailed
from pytestshellutils.utils import ports
from pytestshellutils.utils.processes import ProcessResult
-import salt.ext.tornado.ioloop
-import salt.ext.tornado.web
+import tornado.ioloop
+import tornado.web
import salt.utils.files
import salt.utils.platform
import salt.utils.pycrypto
@@ -1277,7 +1277,7 @@ def http_basic_auth(login_cb=lambda username, password: False):
.. code-block:: python
@http_basic_auth(lambda u, p: u == 'foo' and p == 'bar')
- class AuthenticatedHandler(salt.ext.tornado.web.RequestHandler):
+ class AuthenticatedHandler(tornado.web.RequestHandler):
pass
"""
@@ -1422,7 +1422,7 @@ class Webserver:
self.port = port
self.wait = wait
self.handler = (
- handler if handler is not None else salt.ext.tornado.web.StaticFileHandler
+ handler if handler is not None else tornado.web.StaticFileHandler
)
self.web_root = None
self.ssl_opts = ssl_opts
@@ -1431,14 +1431,14 @@ class Webserver:
"""
Threading target which stands up the tornado application
"""
- self.ioloop = salt.ext.tornado.ioloop.IOLoop()
+ self.ioloop = tornado.ioloop.IOLoop()
self.ioloop.make_current()
- if self.handler == salt.ext.tornado.web.StaticFileHandler:
- self.application = salt.ext.tornado.web.Application(
+ if self.handler == tornado.web.StaticFileHandler:
+ self.application = tornado.web.Application(
[(r"/(.*)", self.handler, {"path": self.root})]
)
else:
- self.application = salt.ext.tornado.web.Application(
+ self.application = tornado.web.Application(
[(r"/(.*)", self.handler)]
)
self.application.listen(self.port, ssl_options=self.ssl_opts)
@@ -1515,7 +1515,7 @@ class Webserver:
self.stop()
-class SaveRequestsPostHandler(salt.ext.tornado.web.RequestHandler):
+class SaveRequestsPostHandler(tornado.web.RequestHandler):
"""
Save all requests sent to the server.
"""
@@ -1535,7 +1535,7 @@ class SaveRequestsPostHandler(salt.ext.tornado.web.RequestHandler):
raise NotImplementedError()
-class MirrorPostHandler(salt.ext.tornado.web.RequestHandler):
+class MirrorPostHandler(tornado.web.RequestHandler):
"""
Mirror a POST body back to the client
"""
diff --git a/tests/support/netapi.py b/tests/support/netapi.py
index 91fbad5f1a4..5095a5998d4 100644
--- a/tests/support/netapi.py
+++ b/tests/support/netapi.py
@@ -4,12 +4,12 @@ import socket
import attr
import salt.auth
-import salt.ext.tornado.escape
-import salt.ext.tornado.web
-from salt.ext.tornado import netutil
-from salt.ext.tornado.httpclient import AsyncHTTPClient, HTTPError
-from salt.ext.tornado.httpserver import HTTPServer
-from salt.ext.tornado.ioloop import TimeoutError as IOLoopTimeoutError
+import tornado.escape
+import tornado.web
+from tornado import netutil
+from tornado.httpclient import AsyncHTTPClient, HTTPError
+from tornado.httpserver import HTTPServer
+from tornado.ioloop import TimeoutError as IOLoopTimeoutError
from salt.netapi.rest_tornado import saltnado
log = logging.getLogger(__name__)
@@ -46,7 +46,7 @@ class TestsHttpClient:
if response.headers.get("Content-Type") == "application/json":
response._body = response.body.decode("utf-8")
else:
- response._body = salt.ext.tornado.escape.native_str(response.body)
+ response._body = tornado.escape.native_str(response.body)
return response
@@ -115,7 +115,7 @@ def auth_token(load_auth, auth_creds):
def build_tornado_app(
urls, load_auth, client_config, minion_config, setup_event_listener=False
):
- application = salt.ext.tornado.web.Application(urls, debug=True)
+ application = tornado.web.Application(urls, debug=True)
application.auth = load_auth
application.opts = client_config
diff --git a/tests/support/pytest/transport.py b/tests/support/pytest/transport.py
index eaa8adc8bd4..592cbd05d46 100644
--- a/tests/support/pytest/transport.py
+++ b/tests/support/pytest/transport.py
@@ -10,9 +10,9 @@ from pytestshellutils.utils.processes import terminate_process
import salt.channel.server
import salt.exceptions
-import salt.ext.tornado.gen
-import salt.ext.tornado.ioloop
-import salt.ext.tornado.iostream
+import tornado.gen
+import tornado.ioloop
+import tornado.iostream
import salt.master
import salt.utils.msgpack
import salt.utils.process
@@ -93,9 +93,9 @@ class Collector(salt.utils.process.SignalHandlingProcess):
time.sleep(1)
else:
break
- self.sock = salt.ext.tornado.iostream.IOStream(sock)
+ self.sock = tornado.iostream.IOStream(sock)
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _recv(self):
if self.transport == "zeromq":
# test_zeromq_filtering requires catching the
@@ -103,19 +103,19 @@ class Collector(salt.utils.process.SignalHandlingProcess):
try:
payload = self.sock.recv(zmq.NOBLOCK)
serial_payload = salt.payload.loads(payload)
- raise salt.ext.tornado.gen.Return(serial_payload)
+ raise tornado.gen.Return(serial_payload)
except (zmq.ZMQError, salt.exceptions.SaltDeserializationError):
raise RecvError("ZMQ Error")
else:
for msg in self.unpacker:
- raise salt.ext.tornado.gen.Return(msg["body"])
+ raise tornado.gen.Return(msg["body"])
byts = yield self.sock.read_bytes(8096, partial=True)
self.unpacker.feed(byts)
for msg in self.unpacker:
- raise salt.ext.tornado.gen.Return(msg["body"])
+ raise tornado.gen.Return(msg["body"])
raise RecvError("TCP Error")
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def _run(self, loop):
try:
self._setup_listener()
@@ -165,7 +165,7 @@ class Collector(salt.utils.process.SignalHandlingProcess):
Gather results until then number of seconds specified by timeout passes
without receiving a message
"""
- loop = salt.ext.tornado.ioloop.IOLoop()
+ loop = tornado.ioloop.IOLoop()
loop.add_callback(self._run, loop)
loop.start()
diff --git a/tests/unit/modules/test_random_org.py b/tests/unit/modules/test_random_org.py
index 30e98ca0802..2c3340c30e8 100644
--- a/tests/unit/modules/test_random_org.py
+++ b/tests/unit/modules/test_random_org.py
@@ -5,7 +5,7 @@
import pytest
import salt.modules.random_org as random_org
-from salt.ext.tornado.httpclient import HTTPClient
+from tornado.httpclient import HTTPClient
from tests.support.mixins import LoaderModuleMockMixin
from tests.support.unit import TestCase
diff --git a/tests/unit/netapi/rest_tornado/test_saltnado.py b/tests/unit/netapi/rest_tornado/test_saltnado.py
index c4758e700ab..43367df13f9 100644
--- a/tests/unit/netapi/rest_tornado/test_saltnado.py
+++ b/tests/unit/netapi/rest_tornado/test_saltnado.py
@@ -1,10 +1,10 @@
-import salt.ext.tornado
-import salt.ext.tornado.testing
+import tornado
+import tornado.testing
import salt.netapi.rest_tornado.saltnado as saltnado
from tests.support.mock import MagicMock, patch
-class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
+class TestJobNotRunning(tornado.testing.AsyncTestCase):
def setUp(self):
super().setUp()
self.mock = MagicMock()
@@ -23,11 +23,11 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
self.handler.lowstate = []
self.handler.content_type = "text/plain"
self.handler.dumper = lambda x: x
- f = salt.ext.tornado.gen.Future()
+ f = tornado.gen.Future()
f.set_result({"jid": f, "minions": []})
self.handler.saltclients.update({"local": lambda *args, **kwargs: f})
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_disbatch_has_already_finished_then_writing_return_should_not_fail(
self,
):
@@ -37,7 +37,7 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
# Asserting that it doesn't raise anything is... the default behavior
# for a test.
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_disbatch_has_already_finished_then_finishing_should_not_fail(self):
self.handler.finish()
result = yield self.handler.disbatch()
@@ -45,12 +45,12 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
# Asserting that it doesn't raise anything is... the default behavior
# for a test.
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_event_times_out_and_minion_is_not_running_result_should_be_True(self):
- fut = salt.ext.tornado.gen.Future()
+ fut = tornado.gen.Future()
fut.set_exception(saltnado.TimeoutException())
self.mock.event_listener.get_event.return_value = fut
- wrong_future = salt.ext.tornado.gen.Future()
+ wrong_future = tornado.gen.Future()
result = yield self.handler.job_not_running(
jid=42, tgt="*", tgt_type="glob", minions=[], is_finished=wrong_future
@@ -58,14 +58,14 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
self.assertTrue(result)
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_event_times_out_and_minion_is_not_running_minion_data_should_not_be_set(
self,
):
- fut = salt.ext.tornado.gen.Future()
+ fut = tornado.gen.Future()
fut.set_exception(saltnado.TimeoutException())
self.mock.event_listener.get_event.return_value = fut
- wrong_future = salt.ext.tornado.gen.Future()
+ wrong_future = tornado.gen.Future()
minions = {}
result = yield self.handler.job_not_running(
@@ -74,20 +74,20 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
assert not minions
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_event_finally_finishes_and_returned_minion_not_in_minions_it_should_be_set_to_False(
self,
):
expected_id = 42
- no_data_event = salt.ext.tornado.gen.Future()
+ no_data_event = tornado.gen.Future()
no_data_event.set_result({"data": {}})
- empty_return_event = salt.ext.tornado.gen.Future()
+ empty_return_event = tornado.gen.Future()
empty_return_event.set_result({"data": {"return": {}}})
- actual_return_event = salt.ext.tornado.gen.Future()
+ actual_return_event = tornado.gen.Future()
actual_return_event.set_result(
{"data": {"return": {"something happened here": "OK?"}, "id": expected_id}}
)
- timed_out_event = salt.ext.tornado.gen.Future()
+ timed_out_event = tornado.gen.Future()
timed_out_event.set_exception(saltnado.TimeoutException())
self.mock.event_listener.get_event.side_effect = [
no_data_event,
@@ -103,27 +103,27 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
tgt="*",
tgt_type="fnord",
minions=minions,
- is_finished=salt.ext.tornado.gen.Future(),
+ is_finished=tornado.gen.Future(),
)
self.assertFalse(minions[expected_id])
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_event_finally_finishes_and_returned_minion_already_in_minions_it_should_not_be_changed(
self,
):
expected_id = 42
expected_value = object()
minions = {expected_id: expected_value}
- no_data_event = salt.ext.tornado.gen.Future()
+ no_data_event = tornado.gen.Future()
no_data_event.set_result({"data": {}})
- empty_return_event = salt.ext.tornado.gen.Future()
+ empty_return_event = tornado.gen.Future()
empty_return_event.set_result({"data": {"return": {}}})
- actual_return_event = salt.ext.tornado.gen.Future()
+ actual_return_event = tornado.gen.Future()
actual_return_event.set_result(
{"data": {"return": {"something happened here": "OK?"}, "id": expected_id}}
)
- timed_out_event = salt.ext.tornado.gen.Future()
+ timed_out_event = tornado.gen.Future()
timed_out_event.set_exception(saltnado.TimeoutException())
self.mock.event_listener.get_event.side_effect = [
no_data_event,
@@ -138,22 +138,22 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
tgt="*",
tgt_type="fnord",
minions=minions,
- is_finished=salt.ext.tornado.gen.Future(),
+ is_finished=tornado.gen.Future(),
)
self.assertIs(minions[expected_id], expected_value)
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_event_returns_early_and_finally_times_out_result_should_be_True(self):
- no_data_event = salt.ext.tornado.gen.Future()
+ no_data_event = tornado.gen.Future()
no_data_event.set_result({"data": {}})
- empty_return_event = salt.ext.tornado.gen.Future()
+ empty_return_event = tornado.gen.Future()
empty_return_event.set_result({"data": {"return": {}}})
- actual_return_event = salt.ext.tornado.gen.Future()
+ actual_return_event = tornado.gen.Future()
actual_return_event.set_result(
{"data": {"return": {"something happened here": "OK?"}, "id": "fnord"}}
)
- timed_out_event = salt.ext.tornado.gen.Future()
+ timed_out_event = tornado.gen.Future()
timed_out_event.set_exception(saltnado.TimeoutException())
self.mock.event_listener.get_event.side_effect = [
no_data_event,
@@ -168,21 +168,21 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
tgt="*",
tgt_type="fnord",
minions={},
- is_finished=salt.ext.tornado.gen.Future(),
+ is_finished=tornado.gen.Future(),
)
self.assertTrue(result)
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_event_finishes_but_is_finished_is_done_then_result_should_be_True(
self,
):
expected_minion_id = "fnord"
expected_minion_value = object()
- no_data_event = salt.ext.tornado.gen.Future()
+ no_data_event = tornado.gen.Future()
no_data_event.set_result({"data": {}})
- empty_return_event = salt.ext.tornado.gen.Future()
+ empty_return_event = tornado.gen.Future()
empty_return_event.set_result({"data": {"return": {}}})
- actual_return_event = salt.ext.tornado.gen.Future()
+ actual_return_event = tornado.gen.Future()
actual_return_event.set_result(
{
"data": {
@@ -191,11 +191,11 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
}
}
)
- is_finished = salt.ext.tornado.gen.Future()
+ is_finished = tornado.gen.Future()
def abort(*args, **kwargs):
yield actual_return_event
- f = salt.ext.tornado.gen.Future()
+ f = tornado.gen.Future()
f.set_exception(saltnado.TimeoutException())
is_finished.set_result("This is done")
yield f
@@ -218,14 +218,14 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
self.assertTrue(len(minions) == 1, str(minions))
self.assertIs(minions[expected_minion_id], expected_minion_value)
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_is_finished_times_out_before_event_finishes_result_should_be_True(
self,
):
# Other test times out with event - this one should time out for is_finished
- finished = salt.ext.tornado.gen.Future()
+ finished = tornado.gen.Future()
finished.set_exception(saltnado.TimeoutException())
- wrong_future = salt.ext.tornado.gen.Future()
+ wrong_future = tornado.gen.Future()
self.mock.event_listener.get_event.return_value = wrong_future
result = yield self.handler.job_not_running(
@@ -234,13 +234,13 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
self.assertTrue(result)
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_is_finished_times_out_before_event_finishes_event_should_have_result_set_to_None(
self,
):
- finished = salt.ext.tornado.gen.Future()
+ finished = tornado.gen.Future()
finished.set_exception(saltnado.TimeoutException())
- wrong_future = salt.ext.tornado.gen.Future()
+ wrong_future = tornado.gen.Future()
self.mock.event_listener.get_event.return_value = wrong_future
result = yield self.handler.job_not_running(
@@ -251,7 +251,7 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase):
# TODO: I think we can extract seUp into a superclass -W. Werner, 2020-11-03
-class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
+class TestGetMinionReturns(tornado.testing.AsyncTestCase):
def setUp(self):
super().setUp()
self.mock = MagicMock()
@@ -265,22 +265,22 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
"gather_job_timeout": 10.001,
}
self.handler = saltnado.SaltAPIHandler(self.mock, self.mock)
- f = salt.ext.tornado.gen.Future()
+ f = tornado.gen.Future()
f.set_result({"jid": f, "minions": []})
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_if_finished_before_any_events_return_then_result_should_be_empty_dictionary(
self,
):
expected_result = {}
- xxx = salt.ext.tornado.gen.Future()
+ xxx = tornado.gen.Future()
xxx.set_result(None)
- is_finished = salt.ext.tornado.gen.Future()
+ is_finished = tornado.gen.Future()
is_finished.set_result(None)
actual_result = yield self.handler.get_minion_returns(
events=[],
is_finished=is_finished,
- is_timed_out=salt.ext.tornado.gen.Future(),
+ is_timed_out=tornado.gen.Future(),
min_wait_time=xxx,
minions={},
)
@@ -288,7 +288,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
# TODO: Copy above - test with timed out -W. Werner, 2020-11-05
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_if_is_finished_after_events_return_then_result_should_contain_event_result_data(
self,
):
@@ -296,15 +296,15 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
"minion1": {"fnord": "this is some fnordish data"},
"minion2": {"fnord": "this is some other fnordish data"},
}
- xxx = salt.ext.tornado.gen.Future()
+ xxx = tornado.gen.Future()
xxx.set_result(None)
- is_finished = salt.ext.tornado.gen.Future()
+ is_finished = tornado.gen.Future()
# XXX what do I do here?
events = [
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
]
events[0].set_result(
{
@@ -323,7 +323,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
actual_result = yield self.handler.get_minion_returns(
events=events,
is_finished=is_finished,
- is_timed_out=salt.ext.tornado.gen.Future(),
+ is_timed_out=tornado.gen.Future(),
min_wait_time=xxx,
minions={
"minion1": False,
@@ -334,7 +334,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
assert actual_result == expected_result
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_if_timed_out_after_events_return_then_result_should_contain_event_result_data(
self,
):
@@ -342,15 +342,15 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
"minion1": {"fnord": "this is some fnordish data"},
"minion2": {"fnord": "this is some other fnordish data"},
}
- xxx = salt.ext.tornado.gen.Future()
+ xxx = tornado.gen.Future()
xxx.set_result(None)
- is_timed_out = salt.ext.tornado.gen.Future()
+ is_timed_out = tornado.gen.Future()
# XXX what do I do here?
events = [
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
]
events[0].set_result(
{
@@ -368,7 +368,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
actual_result = yield self.handler.get_minion_returns(
events=events,
- is_finished=salt.ext.tornado.gen.Future(),
+ is_finished=tornado.gen.Future(),
is_timed_out=is_timed_out,
min_wait_time=xxx,
minions={
@@ -380,7 +380,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
assert actual_result == expected_result
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_if_wait_timer_is_not_done_even_though_results_are_then_data_should_not_yet_be_returned(
self,
):
@@ -388,18 +388,18 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
"one": {"fnordy one": "one has some data"},
"two": {"fnordy two": "two has some data"},
}
- events = [salt.ext.tornado.gen.Future(), salt.ext.tornado.gen.Future()]
+ events = [tornado.gen.Future(), tornado.gen.Future()]
events[0].set_result(
{"tag": "fnord", "data": {"id": "one", "return": expected_result["one"]}}
)
events[1].set_result(
{"tag": "fnord", "data": {"id": "two", "return": expected_result["two"]}}
)
- wait_timer = salt.ext.tornado.gen.Future()
+ wait_timer = tornado.gen.Future()
fut = self.handler.get_minion_returns(
events=events,
- is_finished=salt.ext.tornado.gen.Future(),
- is_timed_out=salt.ext.tornado.gen.Future(),
+ is_finished=tornado.gen.Future(),
+ is_timed_out=tornado.gen.Future(),
min_wait_time=wait_timer,
minions={"one": False, "two": False},
)
@@ -408,7 +408,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
yield fut
self.io_loop.spawn_callback(boop)
- yield salt.ext.tornado.gen.sleep(0.1)
+ yield tornado.gen.sleep(0.1)
assert not fut.done()
@@ -417,30 +417,30 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
assert actual_result == expected_result
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_is_finished_any_other_futures_should_be_canceled(self):
events = [
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
]
- is_finished = salt.ext.tornado.gen.Future()
+ is_finished = tornado.gen.Future()
is_finished.set_result(None)
yield self.handler.get_minion_returns(
events=events,
is_finished=is_finished,
- is_timed_out=salt.ext.tornado.gen.Future(),
- min_wait_time=salt.ext.tornado.gen.Future(),
+ is_timed_out=tornado.gen.Future(),
+ min_wait_time=tornado.gen.Future(),
minions={"one": False, "two": False},
)
are_done = [event.done() for event in events]
assert all(are_done)
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_an_event_times_out_then_we_should_not_enter_an_infinite_loop(self):
# NOTE: this test will enter an infinite loop if the code is broken. I
# was not able to figure out a way to ensure that the test exits with
@@ -451,27 +451,27 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
# TimeoutException.
events = [
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
]
# Arguably any event would work, but 3 isn't the first, so it
# gives us a little more confidence that this test is testing
# correctly
events[3].set_exception(saltnado.TimeoutException())
- times_out_later = salt.ext.tornado.gen.Future()
+ times_out_later = tornado.gen.Future()
# 0.5s should be long enough that the test gets through doing other
# things before hitting this timeout, which will cancel all the
# in-flight futures.
self.io_loop.call_later(0.5, lambda: times_out_later.set_result(None))
yield self.handler.get_minion_returns(
events=events,
- is_finished=salt.ext.tornado.gen.Future(),
+ is_finished=tornado.gen.Future(),
is_timed_out=times_out_later,
- min_wait_time=salt.ext.tornado.gen.Future(),
+ min_wait_time=tornado.gen.Future(),
minions={"one": False, "two": False},
)
@@ -482,7 +482,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
assert all(are_done)
assert times_out_later.done()
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_is_timed_out_any_other_futures_should_be_canceled(self):
# There is some question about whether this test is or should be
# necessary. Or if it's meaningful. The code that this is testing
@@ -491,46 +491,46 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
# That being said, the worst case is that this is just a duplicate
# or irrelevant test, and can be removed.
events = [
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
]
- is_timed_out = salt.ext.tornado.gen.Future()
+ is_timed_out = tornado.gen.Future()
is_timed_out.set_result(None)
yield self.handler.get_minion_returns(
events=events,
- is_finished=salt.ext.tornado.gen.Future(),
+ is_finished=tornado.gen.Future(),
is_timed_out=is_timed_out,
- min_wait_time=salt.ext.tornado.gen.Future(),
+ min_wait_time=tornado.gen.Future(),
minions={"one": False, "two": False},
)
are_done = [event.done() for event in events]
assert all(are_done)
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_min_wait_time_and_nothing_todo_any_other_futures_should_be_canceled(
self,
):
events = [
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
- salt.ext.tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
+ tornado.gen.Future(),
]
- is_finished = salt.ext.tornado.gen.Future()
- min_wait_time = salt.ext.tornado.gen.Future()
+ is_finished = tornado.gen.Future()
+ min_wait_time = tornado.gen.Future()
self.io_loop.call_later(0.2, lambda: min_wait_time.set_result(None))
yield self.handler.get_minion_returns(
events=events,
is_finished=is_finished,
- is_timed_out=salt.ext.tornado.gen.Future(),
+ is_timed_out=tornado.gen.Future(),
min_wait_time=min_wait_time,
minions={"one": True, "two": True},
)
@@ -538,37 +538,37 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
are_done = [event.done() for event in events] + [is_finished.done()]
assert all(are_done)
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_is_finished_but_not_is_timed_out_then_timed_out_should_not_be_set_to_done(
self,
):
- events = [salt.ext.tornado.gen.Future()]
- is_timed_out = salt.ext.tornado.gen.Future()
- is_finished = salt.ext.tornado.gen.Future()
+ events = [tornado.gen.Future()]
+ is_timed_out = tornado.gen.Future()
+ is_finished = tornado.gen.Future()
is_finished.set_result(None)
yield self.handler.get_minion_returns(
events=events,
is_finished=is_finished,
is_timed_out=is_timed_out,
- min_wait_time=salt.ext.tornado.gen.Future(),
+ min_wait_time=tornado.gen.Future(),
minions={"one": False, "two": False},
)
assert not is_timed_out.done()
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_min_wait_time_and_all_completed_but_not_is_timed_out_then_timed_out_should_not_be_set_to_done(
self,
):
- events = [salt.ext.tornado.gen.Future()]
- is_timed_out = salt.ext.tornado.gen.Future()
- min_wait_time = salt.ext.tornado.gen.Future()
+ events = [tornado.gen.Future()]
+ is_timed_out = tornado.gen.Future()
+ min_wait_time = tornado.gen.Future()
self.io_loop.call_later(0.2, lambda: min_wait_time.set_result(None))
yield self.handler.get_minion_returns(
events=events,
- is_finished=salt.ext.tornado.gen.Future(),
+ is_finished=tornado.gen.Future(),
is_timed_out=is_timed_out,
min_wait_time=min_wait_time,
minions={"one": True},
@@ -576,21 +576,21 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
assert not is_timed_out.done()
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_things_are_completed_but_not_timed_out_then_timed_out_event_should_not_be_done(
self,
):
events = [
- salt.ext.tornado.gen.Future(),
+ tornado.gen.Future(),
]
events[0].set_result({"tag": "fnord", "data": {"id": "one", "return": {}}})
- min_wait_time = salt.ext.tornado.gen.Future()
+ min_wait_time = tornado.gen.Future()
min_wait_time.set_result(None)
- is_timed_out = salt.ext.tornado.gen.Future()
+ is_timed_out = tornado.gen.Future()
yield self.handler.get_minion_returns(
events=events,
- is_finished=salt.ext.tornado.gen.Future(),
+ is_finished=tornado.gen.Future(),
is_timed_out=is_timed_out,
min_wait_time=min_wait_time,
minions={"one": True},
@@ -599,7 +599,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase):
assert not is_timed_out.done()
-class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
+class TestDisbatchLocal(tornado.testing.AsyncTestCase):
def setUp(self):
super().setUp()
self.mock = MagicMock()
@@ -614,12 +614,12 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
}
self.handler = saltnado.SaltAPIHandler(self.mock, self.mock)
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_is_timed_out_is_set_before_other_events_are_completed_then_result_should_be_empty_dictionary(
self,
):
- completed_event = salt.ext.tornado.gen.Future()
- never_completed = salt.ext.tornado.gen.Future()
+ completed_event = tornado.gen.Future()
+ never_completed = tornado.gen.Future()
# TODO: We may need to tweak these values to get them close enough but not so far away -W. Werner, 2020-11-17
gather_timeout = 0.1
event_timeout = gather_timeout + 0.05
@@ -642,7 +642,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
self.io_loop.call_later(event_timeout, completer)
- f = salt.ext.tornado.gen.Future()
+ f = tornado.gen.Future()
f.set_result({"jid": "42", "minions": []})
with patch.object(
self.handler.application.event_listener,
@@ -660,12 +660,12 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
assert result == {}
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_is_finished_is_set_before_events_return_then_no_data_should_be_returned(
self,
):
- completed_event = salt.ext.tornado.gen.Future()
- never_completed = salt.ext.tornado.gen.Future()
+ completed_event = tornado.gen.Future()
+ never_completed = tornado.gen.Future()
gather_timeout = 2
event_timeout = gather_timeout - 1
@@ -692,7 +692,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
assert finished is not None
finished.set_result(42)
- f = salt.ext.tornado.gen.Future()
+ f = tornado.gen.Future()
f.set_result({"jid": "42", "minions": []})
with patch.object(
self.handler.application.event_listener,
@@ -715,13 +715,13 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
assert result == {}
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_is_finished_then_all_collected_data_should_be_returned(self):
- completed_event = salt.ext.tornado.gen.Future()
- never_completed = salt.ext.tornado.gen.Future()
+ completed_event = tornado.gen.Future()
+ never_completed = tornado.gen.Future()
# This timeout should never be reached
gather_timeout = 42
- completed_events = [salt.ext.tornado.gen.Future() for _ in range(5)]
+ completed_events = [tornado.gen.Future() for _ in range(5)]
for i, event in enumerate(completed_events):
event.set_result(
{
@@ -732,7 +732,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
},
}
)
- uncompleted_events = [salt.ext.tornado.gen.Future() for _ in range(5)]
+ uncompleted_events = [tornado.gen.Future() for _ in range(5)]
events = iter(completed_events + uncompleted_events)
expected_result = {
"fnord 0": "return from fnord 0",
@@ -753,7 +753,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
assert finished is not None
finished.set_result(42)
- f = salt.ext.tornado.gen.Future()
+ f = tornado.gen.Future()
f.set_result({"jid": "42", "minions": ["non-existent minion"]})
with patch.object(
self.handler.application.event_listener,
@@ -776,16 +776,16 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
assert result == expected_result
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_is_timed_out_then_all_collected_data_should_be_returned(self):
- completed_event = salt.ext.tornado.gen.Future()
- never_completed = salt.ext.tornado.gen.Future()
+ completed_event = tornado.gen.Future()
+ never_completed = tornado.gen.Future()
# 2s is probably enough for any kind of computer to manage to
# do all the other processing. We could maybe reduce this - just
# depends on how slow of a system we're running on.
# TODO: Maybe we should have a test helper/fixture that benchmarks the system and gets a reasonable timeout? -W. Werner, 2020-11-19
gather_timeout = 2
- completed_events = [salt.ext.tornado.gen.Future() for _ in range(5)]
+ completed_events = [tornado.gen.Future() for _ in range(5)]
for i, event in enumerate(completed_events):
event.set_result(
{
@@ -796,7 +796,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
},
}
)
- uncompleted_events = [salt.ext.tornado.gen.Future() for _ in range(5)]
+ uncompleted_events = [tornado.gen.Future() for _ in range(5)]
events = iter(completed_events + uncompleted_events)
expected_result = {
"fnord 0": "return from fnord 0",
@@ -812,7 +812,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
else:
return next(events)
- f = salt.ext.tornado.gen.Future()
+ f = tornado.gen.Future()
f.set_result({"jid": "42", "minions": ["non-existent minion"]})
with patch.object(
self.handler.application.event_listener,
@@ -830,13 +830,13 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
assert result == expected_result
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_minions_all_return_then_all_collected_data_should_be_returned(self):
- completed_event = salt.ext.tornado.gen.Future()
- never_completed = salt.ext.tornado.gen.Future()
+ completed_event = tornado.gen.Future()
+ never_completed = tornado.gen.Future()
# Timeout is something ridiculously high - it should never be reached
gather_timeout = 20
- completed_events = [salt.ext.tornado.gen.Future() for _ in range(10)]
+ completed_events = [tornado.gen.Future() for _ in range(10)]
events_by_id = {}
for i, event in enumerate(completed_events):
id_ = f"fnord {i}"
@@ -864,7 +864,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
tag = kwargs.get("tag", "").rpartition("/")[-1]
return events_by_id.get(tag, never_completed)
- f = salt.ext.tornado.gen.Future()
+ f = tornado.gen.Future()
f.set_result(
{
"jid": "42",
@@ -887,15 +887,15 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
assert result == expected_result
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_when_min_wait_time_has_not_passed_then_disbatch_should_not_return_expected_data_until_time_has_passed(
self,
):
- completed_event = salt.ext.tornado.gen.Future()
- never_completed = salt.ext.tornado.gen.Future()
- wait_timer = salt.ext.tornado.gen.Future()
+ completed_event = tornado.gen.Future()
+ never_completed = tornado.gen.Future()
+ wait_timer = tornado.gen.Future()
gather_timeout = 20
- completed_events = [salt.ext.tornado.gen.Future() for _ in range(10)]
+ completed_events = [tornado.gen.Future() for _ in range(10)]
events_by_id = {}
# Setup some real-enough looking return data
for i, event in enumerate(completed_events):
@@ -943,11 +943,11 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
# The fake sleep is necessary so that we can return our own
# min_wait_time future. The fakeo_timer object is how we signal
# which one we need to be returning.
- orig_sleep = salt.ext.tornado.gen.sleep
+ orig_sleep = tornado.gen.sleep
fakeo_timer = object()
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def fake_sleep(timer):
# only return our fake min_wait_time future when the sentinel
# value is provided. Otherwise it's just a number.
@@ -956,7 +956,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
else:
yield orig_sleep(timer)
- f = salt.ext.tornado.gen.Future()
+ f = tornado.gen.Future()
f.set_result(
{
"jid": "42",
@@ -981,7 +981,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
"order_masters": True,
},
), patch(
- "salt.ext.tornado.gen.sleep",
+ "tornado.gen.sleep",
autospec=True,
side_effect=fake_sleep,
), patch.dict(
@@ -1009,7 +1009,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase):
yield fut
self.io_loop.spawn_callback(boop)
- yield salt.ext.tornado.gen.sleep(0.1)
+ yield tornado.gen.sleep(0.1)
# here, all the minions should be complete (i.e. "True")
assert all(minions[m_id] for m_id in minions)
# But _disbatch_local is not returned yet because min_wait_time has not passed
diff --git a/tests/unit/test_proxy_minion.py b/tests/unit/test_proxy_minion.py
index bc3e867619f..92fc68ad4b1 100644
--- a/tests/unit/test_proxy_minion.py
+++ b/tests/unit/test_proxy_minion.py
@@ -13,8 +13,8 @@ import pytest
from saltfactories.utils import random_string
import salt.config
-import salt.ext.tornado
-import salt.ext.tornado.testing
+import tornado
+import tornado.testing
import salt.metaproxy.proxy
import salt.minion
import salt.syspaths
@@ -38,7 +38,7 @@ class ProxyMinionTestCase(TestCase):
proxy_minion = salt.minion.ProxyMinion(
mock_opts,
jid_queue=copy.copy(mock_jid_queue),
- io_loop=salt.ext.tornado.ioloop.IOLoop(),
+ io_loop=tornado.ioloop.IOLoop(),
)
mock_metaproxy_call = MagicMock()
with patch(
@@ -65,7 +65,7 @@ class ProxyMinionTestCase(TestCase):
proxy_minion = salt.minion.ProxyMinion(
mock_opts,
jid_queue=copy.copy(mock_jid_queue),
- io_loop=salt.ext.tornado.ioloop.IOLoop(),
+ io_loop=tornado.ioloop.IOLoop(),
)
mock_metaproxy_call = MagicMock()
with patch(
@@ -93,7 +93,7 @@ class ProxyMinionTestCase(TestCase):
proxy_minion = salt.minion.ProxyMinion(
mock_opts,
jid_queue=copy.copy(mock_jid_queue),
- io_loop=salt.ext.tornado.ioloop.IOLoop(),
+ io_loop=tornado.ioloop.IOLoop(),
)
mock_metaproxy_call = MagicMock()
with patch(
diff --git a/tests/unit/transport/mixins.py b/tests/unit/transport/mixins.py
index 65b1c7a9498..c757f37861f 100644
--- a/tests/unit/transport/mixins.py
+++ b/tests/unit/transport/mixins.py
@@ -1,4 +1,4 @@
-import salt.ext.tornado.gen
+import tornado.gen
def run_loop_in_thread(loop, evt):
@@ -7,13 +7,13 @@ def run_loop_in_thread(loop, evt):
"""
loop.make_current()
- @salt.ext.tornado.gen.coroutine
+ @tornado.gen.coroutine
def stopper():
while True:
if evt.is_set():
loop.stop()
break
- yield salt.ext.tornado.gen.sleep(0.3)
+ yield tornado.gen.sleep(0.3)
loop.add_callback(stopper)
try:
diff --git a/tests/unit/transport/test_ipc.py b/tests/unit/transport/test_ipc.py
index 72c32a2382c..dff6f77f2d9 100644
--- a/tests/unit/transport/test_ipc.py
+++ b/tests/unit/transport/test_ipc.py
@@ -10,12 +10,12 @@ import pytest
import salt.config
import salt.exceptions
-import salt.ext.tornado.gen
-import salt.ext.tornado.ioloop
-import salt.ext.tornado.testing
+import tornado.gen
+import tornado.ioloop
+import tornado.testing
import salt.transport.ipc
import salt.utils.platform
-from salt.ext.tornado.iostream import StreamClosedError
+from tornado.iostream import StreamClosedError
from tests.support.runtests import RUNTIME_VARS
pytestmark = [
@@ -28,7 +28,7 @@ log = logging.getLogger(__name__)
@pytest.mark.skip_on_windows(reason="Windows does not support Posix IPC")
-class IPCMessagePubSubCase(salt.ext.tornado.testing.AsyncTestCase):
+class IPCMessagePubSubCase(tornado.testing.AsyncTestCase):
"""
Test all of the clear msg stuff
"""
@@ -124,7 +124,7 @@ class IPCMessagePubSubCase(salt.ext.tornado.testing.AsyncTestCase):
self.assertEqual(ret1, "TEST")
self.assertEqual(ret2, "TEST")
- @salt.ext.tornado.testing.gen_test
+ @tornado.testing.gen_test
def test_async_reading_streamclosederror(self):
client1 = self.sub_channel
call_cnt = []
diff --git a/tests/unit/transport/test_tcp.py b/tests/unit/transport/test_tcp.py
index dbe88081859..934a0400933 100644
--- a/tests/unit/transport/test_tcp.py
+++ b/tests/unit/transport/test_tcp.py
@@ -12,12 +12,12 @@ import salt.channel.client
import salt.channel.server
import salt.config
import salt.exceptions
-import salt.ext.tornado.concurrent
-import salt.ext.tornado.gen
-import salt.ext.tornado.ioloop
+import tornado.concurrent
+import tornado.gen
+import tornado.ioloop
import salt.utils.platform
import salt.utils.process
-from salt.ext.tornado.testing import AsyncTestCase
+from tornado.testing import AsyncTestCase
from tests.support.mixins import AdaptedConfigurationTestCaseMixin
from tests.unit.transport.mixins import run_loop_in_thread
@@ -82,7 +82,7 @@ class AsyncPubServerTest(AsyncTestCase, AdaptedConfigurationTestCaseMixin):
cls.master_config
)
cls.req_server_channel.pre_fork(cls.process_manager)
- cls.io_loop = salt.ext.tornado.ioloop.IOLoop()
+ cls.io_loop = tornado.ioloop.IOLoop()
cls.stop = threading.Event()
cls.req_server_channel.post_fork(cls._handle_payload, io_loop=cls.io_loop)
cls.server_thread = threading.Thread(
diff --git a/tests/unit/utils/test_context.py b/tests/unit/utils/test_context.py
index abea69fbf62..79029b1e4d7 100644
--- a/tests/unit/utils/test_context.py
+++ b/tests/unit/utils/test_context.py
@@ -3,172 +3,11 @@ tests.unit.context_test
~~~~~~~~~~~~~~~~~~~~~~~
"""
-import threading
-import time
-
-import pytest
-
-import salt.ext.tornado.gen
-import salt.ext.tornado.stack_context
import salt.utils.json
-from salt.ext.tornado.testing import AsyncTestCase, gen_test
-from salt.utils.context import ContextDict, NamespacedDictWrapper
+from salt.utils.context import NamespacedDictWrapper
from tests.support.unit import TestCase
-class ContextDictTests(AsyncTestCase):
- # how many threads/coroutines to run at a time
- num_concurrent_tasks = 5
-
- def setUp(self):
- super().setUp()
- self.cd = ContextDict()
- # set a global value
- self.cd["foo"] = "global"
-
- @pytest.mark.slow_test
- def test_threads(self):
- """Verify that ContextDict overrides properly within threads"""
- rets = []
-
- def tgt(x, s):
- inner_ret = []
- over = self.cd.clone()
-
- inner_ret.append(self.cd.get("foo"))
- with over:
- inner_ret.append(over.get("foo"))
- over["foo"] = x
- inner_ret.append(over.get("foo"))
- time.sleep(s)
- inner_ret.append(over.get("foo"))
- rets.append(inner_ret)
-
- threads = []
- for x in range(0, self.num_concurrent_tasks):
- s = self.num_concurrent_tasks - x
- t = threading.Thread(target=tgt, args=(x, s))
- t.start()
- threads.append(t)
-
- for t in threads:
- t.join()
-
- for r in rets:
- self.assertEqual(r[0], r[1])
- self.assertEqual(r[2], r[3])
-
- @gen_test
- @pytest.mark.slow_test
- def test_coroutines(self):
- """Verify that ContextDict overrides properly within coroutines"""
-
- @salt.ext.tornado.gen.coroutine
- def secondary_coroutine(over):
- raise salt.ext.tornado.gen.Return(over.get("foo"))
-
- @salt.ext.tornado.gen.coroutine
- def tgt(x, s, over):
- inner_ret = []
- # first grab the global
- inner_ret.append(self.cd.get("foo"))
- # grab the child's global (should match)
- inner_ret.append(over.get("foo"))
- # override the global
- over["foo"] = x
- inner_ret.append(over.get("foo"))
- # sleep for some time to let other coroutines do this section of code
- yield salt.ext.tornado.gen.sleep(s)
- # get the value of the global again.
- inner_ret.append(over.get("foo"))
- # Call another coroutine to verify that we keep our context
- r = yield secondary_coroutine(over)
- inner_ret.append(r)
- raise salt.ext.tornado.gen.Return(inner_ret)
-
- futures = []
-
- for x in range(0, self.num_concurrent_tasks):
- s = self.num_concurrent_tasks - x
- over = self.cd.clone()
-
- # pylint: disable=cell-var-from-loop
- f = salt.ext.tornado.stack_context.run_with_stack_context(
- salt.ext.tornado.stack_context.StackContext(lambda: over),
- lambda: tgt(x, s / 5.0, over),
- )
- # pylint: enable=cell-var-from-loop
- futures.append(f)
-
- wait_iterator = salt.ext.tornado.gen.WaitIterator(*futures)
- while not wait_iterator.done():
- r = yield wait_iterator.next() # pylint: disable=incompatible-py3-code
- self.assertEqual(r[0], r[1]) # verify that the global value remails
- self.assertEqual(r[2], r[3]) # verify that the override sticks locally
- self.assertEqual(
- r[3], r[4]
- ) # verify that the override sticks across coroutines
-
- def test_basic(self):
- """Test that the contextDict is a dict"""
- # ensure we get the global value
- self.assertEqual(
- dict(self.cd),
- {"foo": "global"},
- )
-
- def test_override(self):
- over = self.cd.clone()
- over["bar"] = "global"
- self.assertEqual(
- dict(over),
- {"foo": "global", "bar": "global"},
- )
- self.assertEqual(
- dict(self.cd),
- {"foo": "global"},
- )
- with over:
- self.assertEqual(
- dict(over),
- {"foo": "global", "bar": "global"},
- )
- self.assertEqual(
- dict(self.cd),
- {"foo": "global", "bar": "global"},
- )
- over["bar"] = "baz"
- self.assertEqual(
- dict(over),
- {"foo": "global", "bar": "baz"},
- )
- self.assertEqual(
- dict(self.cd),
- {"foo": "global", "bar": "baz"},
- )
- self.assertEqual(
- dict(over),
- {"foo": "global", "bar": "baz"},
- )
- self.assertEqual(
- dict(self.cd),
- {"foo": "global"},
- )
-
- def test_multiple_contexts(self):
- cds = []
- for x in range(0, 10):
- cds.append(self.cd.clone(bar=x))
- for x, cd in enumerate(cds):
- self.assertNotIn("bar", self.cd)
- with cd:
- self.assertEqual(
- dict(self.cd),
- {"bar": x, "foo": "global"},
- )
- self.assertNotIn("bar", self.cd)
-
-
class NamespacedDictWrapperTests(TestCase):
PREFIX = "prefix"
@@ -193,4 +32,4 @@ class NamespacedDictWrapperTests(TestCase):
def test_json_dumps_multiple_key(self):
self._dict["prefix"] = {"foo": {"bar": "baz"}}
w = NamespacedDictWrapper(self._dict, ("prefix", "foo"))
- self.assertEqual(salt.utils.json.dumps(w), '{"bar": "baz"}')
+ self.assertEqual(salt.utils.json.dumps(w), '{"bar": "baz"}')
\ No newline at end of file
diff --git a/tests/unit/utils/test_gitfs.py b/tests/unit/utils/test_gitfs.py
index 6e99a219ca3..48c0d0472d4 100644
--- a/tests/unit/utils/test_gitfs.py
+++ b/tests/unit/utils/test_gitfs.py
@@ -7,7 +7,7 @@ import tempfile
import pytest
-import salt.ext.tornado.ioloop
+import tornado.ioloop
import salt.fileserver.gitfs
import salt.utils.files
import salt.utils.gitfs
@@ -20,7 +20,7 @@ from tests.support.unit import TestCase
def _clear_instance_map():
try:
del salt.utils.gitfs.GitFS.instance_map[
- salt.ext.tornado.ioloop.IOLoop.current()
+ tornado.ioloop.IOLoop.current()
]
except KeyError:
pass
diff --git a/tests/unit/utils/test_http.py b/tests/unit/utils/test_http.py
index d9a84f9582a..c1bd5c16589 100644
--- a/tests/unit/utils/test_http.py
+++ b/tests/unit/utils/test_http.py
@@ -15,7 +15,7 @@ from tests.support.runtests import RUNTIME_VARS
from tests.support.unit import TestCase
try:
- import salt.ext.tornado.curl_httpclient # pylint: disable=unused-import
+ import tornado.curl_httpclient # pylint: disable=unused-import
HAS_CURL = True
except ImportError:
--
2.51.1
Reference in New Issue View Git Blame Copy Permalink