From e784a3d4d35e3e54804e212752ae8eed789b2142 Mon Sep 17 00:00:00 2001 From: Marek Czernek Date: Fri, 7 Nov 2025 17:12:31 +0100 Subject: [PATCH] Fix Salt for Python > 3.11 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Do not use ConfigParser.readfp in nilrt_ip module * Use external tornado with newest Python * Migrate batch_async to external tornado * Mutating locals is unsupported in Py >=3.13 * modules/tls.py: Use cryptography for create_pkcs12 * modules/x509: Use cryptography for create_crl * Remove use of spwd, removed module in py3.13 * Fix issues around closing IOLoop * Fix error if future is already done * Fix io_loop closing in tpc.py and http.py Partial cherry-pick of opensuse/salt@c4677c6eb4532212406cdffd935e66a97c68490f * Remove unexpected callback keyword * Fix fileclient inability to download a file * Remove mentions of StackContext * Ensure current_ioloop works for vendored and ext tornado * Fix test_linux_shadow tests Cherry-pick of https://github.com/saltstack/salt/pull/68402 * Test saltnado fail fix * Fix test_base_api_handler test * Fix tests_webhook_handler test * Fix salt fileclient for old and new tornado * Raise Python version for vendored tornado * Add compatibility with StackContext * Implement fixme in tls * Apply cosmetic suggestion Co-authored-by: Victor Zhestkov --------- Co-authored-by: Alexander Graul Co-authored-by: Toyam Cox Co-authored-by: Pablo Suárez Hernández Co-authored-by: Victor Zhestkov --- salt/__init__.py | 2 +- salt/channel/client.py | 53 +-- salt/channel/server.py | 43 +-- salt/cli/batch_async.py | 38 +-- salt/client/__init__.py | 15 +- salt/client/mixins.py | 14 +- salt/crypt.py | 23 +- salt/engines/ircbot.py | 15 +- salt/engines/webhook.py | 15 +- salt/fileclient.py | 9 +- salt/master.py | 11 +- salt/metaproxy/deltaproxy.py | 15 +- salt/metaproxy/proxy.py | 14 +- salt/minion.py | 97 +++--- salt/modules/linux_shadow.py | 29 +- salt/modules/nilrt_ip.py | 6 +- salt/modules/tls.py | 78 +++-- salt/modules/x509.py | 82 +---- salt/netapi/rest_tornado/__init__.py | 12 +- salt/netapi/rest_tornado/saltnado.py | 94 +++--- .../rest_tornado/saltnado_websockets.py | 10 +- salt/pillar/__init__.py | 10 +- salt/transport/base.py | 6 +- salt/transport/ipc.py | 96 +++--- salt/transport/tcp.py | 113 ++++--- salt/transport/zeromq.py | 40 +-- salt/utils/asynchronous.py | 106 +++++- salt/utils/event.py | 26 +- salt/utils/gitfs.py | 4 +- salt/utils/http.py | 27 +- salt/utils/process.py | 2 +- salt/utils/thin.py | 8 +- tests/integration/minion/test_minion_cache.py | 2 +- tests/integration/modules/test_gem.py | 2 +- .../netapi/rest_tornado/test_app.py | 10 +- tests/pytests/conftest.py | 6 +- .../functional/channel/test_req_channel.py | 10 +- .../pytests/functional/channel/test_server.py | 8 +- .../netapi/rest_cherrypy/conftest.py | 4 +- .../netapi/rest_cherrypy/test_auth.py | 2 +- .../netapi/rest_cherrypy/test_auth_pam.py | 2 +- .../netapi/rest_cherrypy/test_out_formats.py | 2 +- .../netapi/rest_tornado/test_auth_handler.py | 2 +- .../rest_tornado/test_auth_handler_pam.py | 2 +- .../rest_tornado/test_base_api_handler.py | 11 +- .../netapi/rest_tornado/test_utils.py | 4 +- .../rest_tornado/test_websockets_handler.py | 4 +- .../functional/transport/ipc/test_client.py | 2 +- .../transport/ipc/test_subscriber.py | 8 +- .../transport/server/test_req_channel.py | 10 +- .../transport/tcp/test_message_client.py | 18 +- .../utils/test_async_event_publisher.py | 2 +- .../netapi/rest_cherrypy/conftest.py | 4 +- .../netapi/rest_cherrypy/test_auth.py | 2 +- .../netapi/rest_cherrypy/test_run.py | 2 +- .../rest_tornado/test_events_api_handler.py | 4 +- .../rest_tornado/test_minions_api_handler.py | 2 +- .../netapi/rest_tornado/test_root_handler.py | 2 +- tests/pytests/unit/channel/test_server.py | 4 +- tests/pytests/unit/cli/test_batch_async.py | 40 +-- .../unit/fileserver/gitfs/test_gitfs.py | 4 +- .../fileserver/gitfs/test_gitfs_config.py | 4 +- .../pytests/unit/modules/test_linux_shadow.py | 24 +- tests/pytests/unit/test_ext_importers.py | 51 --- tests/pytests/unit/test_minion.py | 50 +-- tests/pytests/unit/transport/test_ipc.py | 4 +- tests/pytests/unit/transport/test_tcp.py | 28 +- tests/pytests/unit/transport/test_zeromq.py | 44 +-- tests/pytests/unit/utils/event/test_event.py | 16 +- .../unit/utils/event/test_event_return.py | 2 +- tests/pytests/unit/utils/test_asynchronous.py | 12 +- tests/support/helpers.py | 20 +- tests/support/netapi.py | 16 +- tests/support/pytest/transport.py | 20 +- tests/unit/modules/test_random_org.py | 2 +- .../unit/netapi/rest_tornado/test_saltnado.py | 314 +++++++++--------- tests/unit/test_proxy_minion.py | 10 +- tests/unit/transport/mixins.py | 6 +- tests/unit/transport/test_ipc.py | 12 +- tests/unit/transport/test_tcp.py | 10 +- tests/unit/utils/test_context.py | 165 +-------- tests/unit/utils/test_gitfs.py | 4 +- tests/unit/utils/test_http.py | 2 +- 83 files changed, 1000 insertions(+), 1104 deletions(-) delete mode 100644 tests/pytests/unit/test_ext_importers.py diff --git a/salt/__init__.py b/salt/__init__.py index b5fe3677c22..16d5063fa87 100644 --- a/salt/__init__.py +++ b/salt/__init__.py @@ -12,7 +12,7 @@ if sys.version_info < (3,): ) sys.stderr.flush() -USE_VENDORED_TORNADO = True +USE_VENDORED_TORNADO = sys.version_info < (3,12) class TornadoImporter: diff --git a/salt/channel/client.py b/salt/channel/client.py index 25f4af7689d..56878333ed8 100644 --- a/salt/channel/client.py +++ b/salt/channel/client.py @@ -10,10 +10,11 @@ import os import time import uuid +import tornado.gen +import tornado.ioloop + import salt.crypt import salt.exceptions -import salt.ext.tornado.gen -import salt.ext.tornado.ioloop import salt.payload import salt.transport.frame import salt.utils.event @@ -121,7 +122,7 @@ class AsyncReqChannel: opts["master_uri"] = kwargs["master_uri"] io_loop = kwargs.get("io_loop") if io_loop is None: - io_loop = salt.ext.tornado.ioloop.IOLoop.current() + io_loop = tornado.ioloop.IOLoop.current() timeout = opts.get("request_channel_timeout", REQUEST_CHANNEL_TIMEOUT) tries = opts.get("request_channel_tries", REQUEST_CHANNEL_TRIES) @@ -201,7 +202,7 @@ class AsyncReqChannel: ret["id"] = self.opts["id"] return ret - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _send_with_retry(self, load, tries, timeout): _try = 1 while True: @@ -218,9 +219,9 @@ class AsyncReqChannel: else: _try += 1 continue - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def crypted_transfer_decode_dictentry( self, load, @@ -274,12 +275,12 @@ class AsyncReqChannel: # Validate the nonce. if data["nonce"] != nonce: raise salt.crypt.AuthenticationError("Pillar nonce verification failed.") - raise salt.ext.tornado.gen.Return(data["pillar"]) + raise tornado.gen.Return(data["pillar"]) def verify_signature(self, data, sig): return salt.crypt.verify_signature(self.master_pubkey_path, data, sig) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _crypted_transfer(self, load, timeout, raw=False): """ Send a load across the wire, with encryption @@ -294,7 +295,7 @@ class AsyncReqChannel: :param int timeout: The number of seconds on a response before failing """ - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _do_transfer(): # Yield control to the caller. When send() completes, resume by populating data with the Future.result nonce = uuid.uuid4().hex @@ -311,7 +312,7 @@ class AsyncReqChannel: if not raw or self.ttype == "tcp": # XXX Why is this needed for tcp data = salt.transport.frame.decode_embedded_strs(data) - raise salt.ext.tornado.gen.Return(data) + raise tornado.gen.Return(data) if not self.auth.authenticated: # Return control back to the caller, resume when authentication succeeds @@ -323,9 +324,9 @@ class AsyncReqChannel: # If auth error, return control back to the caller, continue when authentication succeeds yield self.auth.authenticate() ret = yield _do_transfer() - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _uncrypted_transfer(self, load, timeout): """ Send a load across the wire in cleartext @@ -338,9 +339,9 @@ class AsyncReqChannel: timeout=timeout, ) - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def send(self, load, tries=None, timeout=None, raw=False): """ Send a request, return a future which will complete when we send the message @@ -370,7 +371,7 @@ class AsyncReqChannel: else: _try += 1 continue - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) def close(self): """ @@ -424,7 +425,7 @@ class AsyncPubChannel: io_loop = kwargs.get("io_loop") if io_loop is None: - io_loop = salt.ext.tornado.ioloop.IOLoop.current() + io_loop = tornado.ioloop.IOLoop.current() auth = salt.crypt.AsyncAuth(opts, io_loop=io_loop) transport = salt.transport.publish_client(opts, io_loop) @@ -445,7 +446,7 @@ class AsyncPubChannel: def crypt(self): return "aes" if self.auth else "clear" - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def connect(self): """ Return a future which completes when connected to the remote publisher @@ -487,7 +488,7 @@ class AsyncPubChannel: if callback is None: return self.transport.on_recv(None) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def wrap_callback(messages): payload = yield self.transport._decode_messages(messages) decoded = yield self._decode_payload(payload) @@ -504,7 +505,7 @@ class AsyncPubChannel: "version": 3, } - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def send_id(self, tok, force_auth): """ Send the minion id to the master so that the master may better @@ -514,13 +515,13 @@ class AsyncPubChannel: """ load = {"id": self.opts["id"], "tok": tok} - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _do_transfer(): msg = self._package_load(self.auth.crypticle.dumps(load)) package = salt.transport.frame.frame_msg(msg, header=None) yield self.transport.send(package) - raise salt.ext.tornado.gen.Return(True) + raise tornado.gen.Return(True) if force_auth or not self.auth.authenticated: count = 0 @@ -536,13 +537,13 @@ class AsyncPubChannel: count += 1 try: ret = yield _do_transfer() - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) except salt.crypt.AuthenticationError: yield self.auth.authenticate() ret = yield _do_transfer() - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def connect_callback(self, result): if self._closing: return @@ -614,7 +615,7 @@ class AsyncPubChannel: "Message signature failed to validate." ) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _decode_payload(self, payload): # we need to decrypt it log.trace("Decoding payload: %s", payload) @@ -626,7 +627,7 @@ class AsyncPubChannel: yield self.auth.authenticate() payload["load"] = self.auth.crypticle.loads(payload["load"]) - raise salt.ext.tornado.gen.Return(payload) + raise tornado.gen.Return(payload) def __enter__(self): return self diff --git a/salt/channel/server.py b/salt/channel/server.py index 1d7cc9c577f..a9151440c2b 100644 --- a/salt/channel/server.py +++ b/salt/channel/server.py @@ -12,8 +12,9 @@ import pathlib import shutil import time +import tornado.gen + import salt.crypt -import salt.ext.tornado.gen import salt.master import salt.payload import salt.transport.frame @@ -128,7 +129,7 @@ class ReqServerChannel: if hasattr(self.transport, "post_fork"): self.transport.post_fork(self.handle_message, io_loop) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def handle_message(self, payload): if ( not isinstance(payload, dict) @@ -136,7 +137,7 @@ class ReqServerChannel: or "load" not in payload ): log.warn("bad load received on socket") - raise salt.ext.tornado.gen.Return("bad load") + raise tornado.gen.Return("bad load") version = payload.get("version", 0) try: payload = self._decode_payload(payload, version) @@ -151,7 +152,7 @@ class ReqServerChannel: ) else: log.error("Bad load from minion: %s: %s", exc_type, exc) - raise salt.ext.tornado.gen.Return("bad load") + raise tornado.gen.Return("bad load") # TODO helper functions to normalize payload? if not isinstance(payload, dict) or not isinstance(payload.get("load"), dict): @@ -160,16 +161,16 @@ class ReqServerChannel: payload, payload.get("load"), ) - raise salt.ext.tornado.gen.Return("payload and load must be a dict") + raise tornado.gen.Return("payload and load must be a dict") try: id_ = payload["load"].get("id", "") if "\0" in id_: log.error("Payload contains an id with a null byte: %s", payload) - raise salt.ext.tornado.gen.Return("bad load: id contains a null byte") + raise tornado.gen.Return("bad load: id contains a null byte") except TypeError: log.error("Payload contains non-string id: %s", payload) - raise salt.ext.tornado.gen.Return( + raise tornado.gen.Return( "bad load: id {} is not a string".format(id_) ) @@ -184,7 +185,7 @@ class ReqServerChannel: ret = self._auth(payload["load"], sign_messages, version) if self.opts.get("master_stats", False): yield self.payload_handler({"cmd": "_auth", "_start": start}) - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) if payload["enc"] == "aes": nonce = None @@ -202,7 +203,7 @@ class ReqServerChannel: ttl, self.opts["request_server_ttl"], ) - raise salt.ext.tornado.gen.Return("bad load") + raise tornado.gen.Return("bad load") if payload["id"] != payload["load"]["id"]: log.warning( @@ -210,18 +211,18 @@ class ReqServerChannel: payload["load"]["id"], payload["id"], ) - raise salt.ext.tornado.gen.Return("bad load") + raise tornado.gen.Return("bad load") if not salt.utils.verify.valid_id(self.opts, payload["load"]["id"]): log.warning( "Request contains invalid minion id '%s'", payload["load"]["id"] ) - raise salt.ext.tornado.gen.Return("bad load") + raise tornado.gen.Return("bad load") if not self.validate_token(payload, required=True): - raise salt.ext.tornado.gen.Return("bad load") + raise tornado.gen.Return("bad load") # The token won't always be present in the payload for v2 and # below, but if it is we always wanto validate it. elif not self.validate_token(payload, required=False): - raise salt.ext.tornado.gen.Return("bad load") + raise tornado.gen.Return("bad load") # TODO: test try: @@ -231,22 +232,22 @@ class ReqServerChannel: except Exception as e: # pylint: disable=broad-except # always attempt to return an error to the minion log.error("Some exception handling a payload from minion", exc_info=True) - raise salt.ext.tornado.gen.Return("Some exception handling minion payload") + raise tornado.gen.Return("Some exception handling minion payload") req_fun = req_opts.get("fun", "send") if req_fun == "send_clear": - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) elif req_fun == "send": if version > 2: - raise salt.ext.tornado.gen.Return( + raise tornado.gen.Return( salt.crypt.Crypticle(self.opts, self.session_key(id_)).dumps( ret, nonce ) ) else: - raise salt.ext.tornado.gen.Return(self.crypticle.dumps(ret, nonce)) + raise tornado.gen.Return(self.crypticle.dumps(ret, nonce)) elif req_fun == "send_private": - raise salt.ext.tornado.gen.Return( + raise tornado.gen.Return( self._encrypt_private( ret, req_opts["key"], @@ -257,7 +258,7 @@ class ReqServerChannel: ) log.error("Unknown req_fun %s", req_fun) # always attempt to return an error to the minion - raise salt.ext.tornado.gen.Return("Server-side exception handling payload") + raise tornado.gen.Return("Server-side exception handling payload") def _encrypt_private( self, @@ -985,7 +986,7 @@ class PubServerChannel: data, salt.utils.event.tagify("present", "presence") ) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def publish_payload(self, load, *args): unpacked_package = self.wrap_payload(load) try: @@ -998,7 +999,7 @@ class PubServerChannel: ret = yield self.transport.publish_payload(payload, topic_list) else: ret = yield self.transport.publish_payload(payload) - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) def wrap_payload(self, load): payload = {"enc": "aes"} diff --git a/salt/cli/batch_async.py b/salt/cli/batch_async.py index 92215d0e04a..9e0ecddda54 100644 --- a/salt/cli/batch_async.py +++ b/salt/cli/batch_async.py @@ -6,10 +6,10 @@ import logging import re import salt.client -import salt.ext.tornado +import tornado import salt.utils.event from salt.cli.batch import batch_get_eauth, batch_get_opts, get_bnum -from salt.ext.tornado.iostream import StreamClosedError +from tornado.iostream import StreamClosedError log = logging.getLogger(__name__) @@ -111,14 +111,14 @@ class SharedEventsChannel: if not self._subscribers[subscriber_id]: del self._subscribers[subscriber_id] - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def __handle_close(self): if not self._subscriptions: return log.warning("Master Event Subscriber was closed. Trying to reconnect...") yield self.__reconnect_subscriber() - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def __handle_event(self, raw): if self.master_event is None: return @@ -138,7 +138,7 @@ class SharedEventsChannel: exc_info=True, ) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def __reconnect_subscriber(self): if self.master_event.subscriber.connected() or self._reconnecting_subscriber: return @@ -167,7 +167,7 @@ class SharedEventsChannel: self._reconnecting_subscriber = False return if _try < max_tries: - yield salt.ext.tornado.gen.sleep(self._subscriber_reconnect_interval) + yield tornado.gen.sleep(self._subscriber_reconnect_interval) _try += 1 self._reconnecting_subscriber = False @@ -229,7 +229,7 @@ class BatchAsync: self.extra_job_kwargs[kwarg] = kwargs[kwarg] elif kwarg in opts: self.extra_job_kwargs[kwarg] = opts[kwarg] - self.io_loop = salt.ext.tornado.ioloop.IOLoop.current() + self.io_loop = tornado.ioloop.IOLoop.current() self.events_channel = _get_shared_events_channel(opts, self.io_loop).use( id(self) ) @@ -278,7 +278,7 @@ class BatchAsync: self.batch_jid, "batch_run", id(self), self.__event_handler ) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def __event_handler(self, tag, data, op): # IMPORTANT: This function must run fast and not wait for any other task, # otherwise it would cause events to be stuck. @@ -321,7 +321,7 @@ class BatchAsync: ) return set(list(to_run)[:next_batch_size]) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def check_find_job(self, batch_minions, jid): """ Check if the job with specified ``jid`` was finished on the minions @@ -346,7 +346,7 @@ class BatchAsync: self.find_job_returned = self.find_job_returned.difference(running) yield self.find_job(running) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def find_job(self, minions): """ Find if the job was finished on the minions @@ -377,7 +377,7 @@ class BatchAsync: listen=False, **self.eauth, ) - yield salt.ext.tornado.gen.sleep(self.opts["gather_job_timeout"]) + yield tornado.gen.sleep(self.opts["gather_job_timeout"]) if self.event: yield self.check_find_job(not_done, jid) except Exception as ex: # pylint: disable=W0703 @@ -388,7 +388,7 @@ class BatchAsync: ) self.close_safe() - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def start(self): """ Start the batch execution @@ -419,7 +419,7 @@ class BatchAsync: self.start_batch, ) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def start_batch(self): """ Fire `salt/batch/*/start` and continue batch with `run_next` @@ -442,7 +442,7 @@ class BatchAsync: if self.event: yield self.run_next() - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def end_batch(self): """ End the batch and call safe closing @@ -469,7 +469,7 @@ class BatchAsync: # release to the IOLoop to allow the event to be published # before closing batch async execution - yield salt.ext.tornado.gen.sleep(0.03) + yield tornado.gen.sleep(0.03) self.close_safe() def close_safe(self): @@ -481,7 +481,7 @@ class BatchAsync: _destroy_unused_shared_events_channel() self.event = None - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def schedule_next(self): log.trace("[%s] BatchAsync.schedule_next called", self.batch_jid) if self.scheduled: @@ -498,11 +498,11 @@ class BatchAsync: self.batch_jid, self.batch_delay, ) - yield salt.ext.tornado.gen.sleep(self.batch_delay) + yield tornado.gen.sleep(self.batch_delay) if self.event: yield self.run_next() - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def run_next(self): """ Continue batch execution with the next targets @@ -535,7 +535,7 @@ class BatchAsync: **self.extra_job_kwargs, ) - yield salt.ext.tornado.gen.sleep(self.opts["timeout"]) + yield tornado.gen.sleep(self.opts["timeout"]) # The batch can be done already at this point, which means no self.event if self.event and self.active.intersection(next_batch): diff --git a/salt/client/__init__.py b/salt/client/__init__.py index b2617e4554d..be5939e58f6 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -25,11 +25,12 @@ import sys import time from datetime import datetime +import tornado.gen + import salt.cache import salt.channel.client import salt.config import salt.defaults.exitcodes -import salt.ext.tornado.gen import salt.loader import salt.payload import salt.syspaths as syspaths @@ -416,7 +417,7 @@ class LocalClient: ) return _res["minions"] - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def run_job_async( self, tgt, @@ -473,7 +474,7 @@ class LocalClient: # Convert to generic client error and pass along message raise SaltClientError(general_exception) - raise salt.ext.tornado.gen.Return(self._check_pub_data(pub_data, listen=listen)) + raise tornado.gen.Return(self._check_pub_data(pub_data, listen=listen)) def cmd_async( self, tgt, fun, arg=(), tgt_type="glob", ret="", jid="", kwarg=None, **kwargs @@ -1924,7 +1925,7 @@ class LocalClient: return {"jid": payload["load"]["jid"], "minions": payload["load"]["minions"]} - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def pub_async( self, tgt, @@ -2005,7 +2006,7 @@ class LocalClient: # and try again if the key has changed key = self.__read_master_key() if key == self.key: - raise salt.ext.tornado.gen.Return(payload) + raise tornado.gen.Return(payload) self.key = key payload_kwargs["key"] = self.key payload = yield channel.send(payload_kwargs) @@ -2023,9 +2024,9 @@ class LocalClient: raise PublishError(error) if not payload: - raise salt.ext.tornado.gen.Return(payload) + raise tornado.gen.Return(payload) - raise salt.ext.tornado.gen.Return( + raise tornado.gen.Return( {"jid": payload["load"]["jid"], "minions": payload["load"]["minions"]} ) diff --git a/salt/client/mixins.py b/salt/client/mixins.py index 7cdae88ae8a..431d67d1b9b 100644 --- a/salt/client/mixins.py +++ b/salt/client/mixins.py @@ -14,7 +14,6 @@ from collections.abc import Mapping, MutableMapping import salt._logging import salt.channel.client import salt.exceptions -import salt.ext.tornado.stack_context import salt.minion import salt.output import salt.utils.args @@ -30,6 +29,12 @@ import salt.utils.state import salt.utils.user import salt.utils.versions +from salt import USE_VENDORED_TORNADO +if USE_VENDORED_TORNADO: + from salt.ext.tornado.stack_context import StackContext +else: + from contextlib import nullcontext as StackContext + log = logging.getLogger(__name__) CLIENT_INTERNAL_KEYWORDS = frozenset( @@ -379,10 +384,7 @@ class SyncClientMixin(ClientStateMixin): data["fun_args"] = list(args) + ([kwargs] if kwargs else []) func_globals["__jid_event__"].fire_event(data, "new") - # Initialize a context for executing the method. - with salt.ext.tornado.stack_context.StackContext( - self.functions.context_dict.clone - ): + with StackContext(self.functions.context_dict.clone): func = self.functions[fun] try: data["return"] = func(*args, **kwargs) @@ -394,6 +396,7 @@ class SyncClientMixin(ClientStateMixin): ) try: data["success"] = self.context.get("retcode", 0) == 0 + except AttributeError: # Assume a True result if no context attribute data["success"] = True @@ -402,6 +405,7 @@ class SyncClientMixin(ClientStateMixin): data["success"] = salt.utils.state.check_result( data["return"]["data"] ) + except (Exception, SystemExit) as ex: # pylint: disable=broad-except if isinstance(ex, salt.exceptions.NotImplemented): data["return"] = str(ex) diff --git a/salt/crypt.py b/salt/crypt.py index 29fd159b48c..f1214caebba 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -22,9 +22,10 @@ import traceback import uuid import weakref +import tornado.gen + import salt.channel.client import salt.defaults.exitcodes -import salt.ext.tornado.gen import salt.payload import salt.utils.crypt import salt.utils.decorators @@ -497,7 +498,7 @@ class AsyncAuth: Only create one instance of AsyncAuth per __key() """ # do we have any mapping for this io_loop - io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current() + io_loop = io_loop or tornado.ioloop.IOLoop.current() if io_loop not in AsyncAuth.instance_map: AsyncAuth.instance_map[io_loop] = weakref.WeakValueDictionary() loop_instance_map = AsyncAuth.instance_map[io_loop] @@ -548,7 +549,7 @@ class AsyncAuth: if not os.path.isfile(self.pub_path): self.get_keys() - self.io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current() + self.io_loop = io_loop or tornado.ioloop.IOLoop.current() salt.utils.crypt.reinit_crypto() key = self.__key(self.opts) @@ -558,7 +559,7 @@ class AsyncAuth: self._creds = creds self._crypticle = Crypticle(self.opts, creds["aes"]) self._session_crypticle = Crypticle(self.opts, creds["session"]) - self._authenticate_future = salt.ext.tornado.concurrent.Future() + self._authenticate_future = tornado.concurrent.Future() self._authenticate_future.set_result(True) else: self.authenticate() @@ -617,7 +618,7 @@ class AsyncAuth: ): future = self._authenticate_future else: - future = salt.ext.tornado.concurrent.Future() + future = tornado.concurrent.Future() self._authenticate_future = future self.io_loop.add_callback(self._authenticate) @@ -631,7 +632,7 @@ class AsyncAuth: return future - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _authenticate(self): """ Authenticate with the master, this method breaks the functional @@ -682,7 +683,7 @@ class AsyncAuth: log.info( "Waiting %s seconds before retry.", acceptance_wait_time ) - yield salt.ext.tornado.gen.sleep(acceptance_wait_time) + yield tornado.gen.sleep(acceptance_wait_time) if acceptance_wait_time < acceptance_wait_time_max: acceptance_wait_time += acceptance_wait_time log.debug( @@ -733,7 +734,7 @@ class AsyncAuth: salt.utils.event.tagify(prefix="auth", suffix="creds"), ) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def sign_in(self, timeout=60, safe=True, tries=1, channel=None): """ Send a sign in request to the master, sets the key information and @@ -774,9 +775,9 @@ class AsyncAuth: except SaltReqTimeoutError as e: if safe: log.warning("SaltReqTimeoutError: %s", e) - raise salt.ext.tornado.gen.Return("retry") + raise tornado.gen.Return("retry") if self.opts.get("detect_mode") is True: - raise salt.ext.tornado.gen.Return("retry") + raise tornado.gen.Return("retry") else: raise SaltClientError( "Attempt to authenticate with the salt master failed with timeout" @@ -786,7 +787,7 @@ class AsyncAuth: if close_channel: channel.close() ret = self.handle_signin_response(sign_in_payload, payload) - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) def handle_signin_response(self, sign_in_payload, payload): auth = {} diff --git a/salt/engines/ircbot.py b/salt/engines/ircbot.py index 1dab78dbbc5..982c860c262 100644 --- a/salt/engines/ircbot.py +++ b/salt/engines/ircbot.py @@ -61,8 +61,9 @@ import socket import ssl from collections import namedtuple -import salt.ext.tornado.ioloop -import salt.ext.tornado.iostream +import tornado.ioloop +import tornado.iostream + import salt.utils.event log = logging.getLogger(__name__) @@ -101,18 +102,18 @@ class IRCClient: self.allow_hosts = allow_hosts self.allow_nicks = allow_nicks self.disable_query = disable_query - self.io_loop = salt.ext.tornado.ioloop.IOLoop(make_current=False) + self.io_loop = tornado.ioloop.IOLoop(make_current=False) self.io_loop.make_current() self._connect() def _connect(self): _sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) if self.ssl is True: - self._stream = salt.ext.tornado.iostream.SSLIOStream( + self._stream = tornado.iostream.SSLIOStream( _sock, ssl_options={"cert_reqs": ssl.CERT_NONE} ) else: - self._stream = salt.ext.tornado.iostream.IOStream(_sock) + self._stream = tornado.iostream.IOStream(_sock) self._stream.set_close_callback(self.on_closed) self._stream.connect((self.host, self.port), self.on_connect) @@ -218,11 +219,11 @@ class IRCClient: event = self._event(raw) if event.code == "PING": - salt.ext.tornado.ioloop.IOLoop.current().spawn_callback( + tornado.ioloop.IOLoop.current().spawn_callback( self.send_message, "PONG {}".format(event.line) ) elif event.code == "PRIVMSG": - salt.ext.tornado.ioloop.IOLoop.current().spawn_callback( + tornado.ioloop.IOLoop.current().spawn_callback( self._privmsg, event ) self.read_messages() diff --git a/salt/engines/webhook.py b/salt/engines/webhook.py index 1424d4a27dc..fdd2ca2dfbb 100644 --- a/salt/engines/webhook.py +++ b/salt/engines/webhook.py @@ -2,9 +2,10 @@ Send events from webhook api """ -import salt.ext.tornado.httpserver -import salt.ext.tornado.ioloop -import salt.ext.tornado.web +import tornado.httpserver +import tornado.ioloop +import tornado.web + import salt.utils.event @@ -64,7 +65,7 @@ def start(address=None, port=5000, ssl_crt=None, ssl_key=None): __salt__["event.send"](tag, msg) class WebHook( - salt.ext.tornado.web.RequestHandler + tornado.web.RequestHandler ): # pylint: disable=abstract-method def post(self, tag): # pylint: disable=arguments-differ body = self.request.body @@ -75,13 +76,13 @@ def start(address=None, port=5000, ssl_crt=None, ssl_key=None): } fire("salt/engines/hook/" + tag, payload) - application = salt.ext.tornado.web.Application([(r"/(.*)", WebHook)]) + application = tornado.web.Application([(r"/(.*)", WebHook)]) ssl_options = None if all([ssl_crt, ssl_key]): ssl_options = {"certfile": ssl_crt, "keyfile": ssl_key} - io_loop = salt.ext.tornado.ioloop.IOLoop(make_current=False) + io_loop = tornado.ioloop.IOLoop(make_current=False) io_loop.make_current() - http_server = salt.ext.tornado.httpserver.HTTPServer( + http_server = tornado.httpserver.HTTPServer( application, ssl_options=ssl_options ) http_server.listen(port, address=address) diff --git a/salt/fileclient.py b/salt/fileclient.py index 4ed341221cd..82e0fc2a824 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -12,6 +12,9 @@ import string import urllib.error import urllib.parse +from salt import USE_VENDORED_TORNADO +from tornado.httputil import HTTPHeaders, HTTPInputError, parse_response_start_line + import salt.channel.client import salt.client import salt.crypt @@ -32,11 +35,6 @@ import salt.utils.url import salt.utils.verify import salt.utils.versions from salt.exceptions import CommandExecutionError, MinionError -from salt.ext.tornado.httputil import ( - HTTPHeaders, - HTTPInputError, - parse_response_start_line, -) from salt.utils.openstack.swift import SaltSwift log = logging.getLogger(__name__) @@ -747,6 +745,7 @@ class Client: # Check the status line of the HTTP request if write_body[0] is None: try: + hdr = hdr if USE_VENDORED_TORNADO else hdr.strip() hdr = parse_response_start_line(hdr) except HTTPInputError: # Not the first line, do nothing diff --git a/salt/master.py b/salt/master.py index ba7c751d4b4..09ce7d36a7e 100644 --- a/salt/master.py +++ b/salt/master.py @@ -17,6 +17,8 @@ import sys import threading import time +import tornado.gen + import salt.acl import salt.auth import salt.channel.server @@ -27,7 +29,6 @@ import salt.daemons.masterapi import salt.defaults.exitcodes import salt.engines import salt.exceptions -import salt.ext.tornado.gen import salt.key import salt.minion import salt.payload @@ -1002,7 +1003,7 @@ class MWorker(salt.utils.process.SignalHandlingProcess): """ Bind to the local port """ - self.io_loop = salt.ext.tornado.ioloop.IOLoop() + self.io_loop = tornado.ioloop.IOLoop() self.io_loop.make_current() for req_channel in self.req_channels: req_channel.post_fork( @@ -1014,7 +1015,7 @@ class MWorker(salt.utils.process.SignalHandlingProcess): # Tornado knows what to do pass - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _handle_payload(self, payload): """ The _handle_payload method is the key method used to figure out what @@ -1047,7 +1048,7 @@ class MWorker(salt.utils.process.SignalHandlingProcess): ret = self._handle_aes(load) else: ret = self._handle_clear(load) - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) def _post_stats(self, start, cmd): """ @@ -2137,7 +2138,7 @@ class ClearFuncs(TransportMethods): lambda: self._prep_jid(clear_load, {}), batch_load, ) - ioloop = salt.ext.tornado.ioloop.IOLoop.current() + ioloop = tornado.ioloop.IOLoop.current() ioloop.add_callback(batch.start) return { diff --git a/salt/metaproxy/deltaproxy.py b/salt/metaproxy/deltaproxy.py index c3003b368f7..fe71c321ea2 100644 --- a/salt/metaproxy/deltaproxy.py +++ b/salt/metaproxy/deltaproxy.py @@ -10,6 +10,9 @@ import threading import traceback import types +import tornado.gen # pylint: disable=F0401 +import tornado.ioloop # pylint: disable=F0401 + import salt import salt._logging import salt.beacons @@ -19,8 +22,6 @@ import salt.config import salt.crypt import salt.defaults.exitcodes import salt.engines -import salt.ext.tornado.gen # pylint: disable=F0401 -import salt.ext.tornado.ioloop # pylint: disable=F0401 import salt.loader import salt.minion import salt.payload @@ -55,6 +56,12 @@ from salt.minion import ProxyMinion from salt.utils.event import tagify from salt.utils.process import SignalHandlingProcess, default_signals +from salt import USE_VENDORED_TORNADO +if USE_VENDORED_TORNADO: + from salt.ext.tornado.stack_context import StackContext +else: + from contextlib import nullcontext as StackContext + log = logging.getLogger(__name__) @@ -561,7 +568,7 @@ def target(cls, minion_instance, opts, data, connected): uid = salt.utils.user.get_uid(user=opts.get("user", None)) minion_instance.proc_dir = salt.minion.get_proc_dir(opts["cachedir"], uid=uid) - with salt.ext.tornado.stack_context.StackContext(minion_instance.ctx): + with StackContext(minion_instance.ctx): if isinstance(data["fun"], tuple) or isinstance(data["fun"], list): ProxyMinion._thread_multi_return(minion_instance, opts, data) else: @@ -1014,7 +1021,7 @@ def handle_decoded_payload(self, data): data["jid"], ) once_logged = True - yield salt.ext.tornado.gen.sleep(0.5) + yield tornado.gen.sleep(0.5) process_count = self.subprocess_list.count # We stash an instance references to allow for the socket diff --git a/salt/metaproxy/proxy.py b/salt/metaproxy/proxy.py index a399c15ef16..4699e4b3de3 100644 --- a/salt/metaproxy/proxy.py +++ b/salt/metaproxy/proxy.py @@ -16,8 +16,8 @@ import salt.client import salt.crypt import salt.defaults.exitcodes import salt.engines -import salt.ext.tornado.gen # pylint: disable=F0401 -import salt.ext.tornado.ioloop # pylint: disable=F0401 +import tornado.gen # pylint: disable=F0401 +import tornado.ioloop # pylint: disable=F0401 import salt.loader import salt.minion import salt.payload @@ -52,6 +52,12 @@ from salt.minion import ProxyMinion from salt.utils.event import tagify from salt.utils.process import SignalHandlingProcess, default_signals +from salt import USE_VENDORED_TORNADO +if USE_VENDORED_TORNADO: + from salt.ext.tornado.stack_context import StackContext +else: + from contextlib import nullcontext as StackContext + log = logging.getLogger(__name__) @@ -380,7 +386,7 @@ def target(cls, minion_instance, opts, data, connected): opts["cachedir"], uid=uid ) - with salt.ext.tornado.stack_context.StackContext(minion_instance.ctx): + with StackContext(minion_instance.ctx): if isinstance(data["fun"], tuple) or isinstance(data["fun"], list): ProxyMinion._thread_multi_return(minion_instance, opts, data) else: @@ -804,7 +810,7 @@ def handle_decoded_payload(self, data): "Maximum number of processes reached while executing jid %s, waiting...", data["jid"], ) - yield salt.ext.tornado.gen.sleep(10) + yield tornado.gen.sleep(10) process_count = len(salt.utils.minion.running(self.opts)) # We stash an instance references to allow for the socket diff --git a/salt/minion.py b/salt/minion.py index d9201e20109..805e893341c 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -26,9 +26,9 @@ import salt.crypt import salt.defaults.events import salt.defaults.exitcodes import salt.engines -import salt.ext.tornado -import salt.ext.tornado.gen -import salt.ext.tornado.ioloop +import tornado +import tornado.gen +import tornado.ioloop import salt.loader import salt.loader.lazy import salt.payload @@ -101,6 +101,11 @@ try: except ImportError: HAS_WIN_FUNCTIONS = False +from salt import USE_VENDORED_TORNADO +if USE_VENDORED_TORNADO: + from salt.ext.tornado.stack_context import ExceptionStackContext +else: + from contextlib import nullcontext as ExceptionStackContext log = logging.getLogger(__name__) @@ -510,7 +515,7 @@ class MinionBase: ) # pylint: disable=no-member return [] - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def eval_master(self, opts, timeout=60, safe=True, failed=False, failback=False): """ Evaluates and returns a tuple of the current master address and the pub_channel. @@ -531,7 +536,7 @@ class MinionBase: if opts["master_type"] == "disable": log.warning("Master is set to disable, skipping connection") self.connected = False - raise salt.ext.tornado.gen.Return((None, None)) + raise tornado.gen.Return((None, None)) # Run masters discovery over SSDP. This may modify the whole configuration, # depending of the networking and sets of masters. @@ -701,7 +706,7 @@ class MinionBase: if attempts != 0: # Give up a little time between connection attempts # to allow the IOLoop to run any other scheduled tasks. - yield salt.ext.tornado.gen.sleep(opts["acceptance_wait_time"]) + yield tornado.gen.sleep(opts["acceptance_wait_time"]) attempts += 1 if tries > 0: log.debug("Connecting to master. Attempt %s of %s", attempts, tries) @@ -770,7 +775,7 @@ class MinionBase: else: self.tok = pub_channel.auth.gen_token(b"salt") self.connected = True - raise salt.ext.tornado.gen.Return((opts["master"], pub_channel)) + raise tornado.gen.Return((opts["master"], pub_channel)) # single master sign in else: @@ -784,7 +789,7 @@ class MinionBase: if attempts != 0: # Give up a little time between connection attempts # to allow the IOLoop to run any other scheduled tasks. - yield salt.ext.tornado.gen.sleep(opts["acceptance_wait_time"]) + yield tornado.gen.sleep(opts["acceptance_wait_time"]) attempts += 1 if tries > 0: log.debug("Connecting to master. Attempt %s of %s", attempts, tries) @@ -816,7 +821,7 @@ class MinionBase: yield pub_channel.connect() self.tok = pub_channel.auth.gen_token(b"salt") self.connected = True - raise salt.ext.tornado.gen.Return((opts["master"], pub_channel)) + raise tornado.gen.Return((opts["master"], pub_channel)) except SaltClientError: if pub_channel: pub_channel.close() @@ -924,7 +929,7 @@ class SMinion(MinionBase): if self.opts.get("file_client", "remote") == "remote" or self.opts.get( "use_master_when_local", False ): - io_loop = salt.ext.tornado.ioloop.IOLoop.current() + io_loop = tornado.ioloop.IOLoop.current() io_loop.run_sync(lambda: self.eval_master(self.opts, failed=True)) self.gen_modules(initial_load=True, context=context) @@ -1027,7 +1032,7 @@ class MinionManager(MinionBase): self.minions = [] self.jid_queue = [] - self.io_loop = salt.ext.tornado.ioloop.IOLoop.current() + self.io_loop = tornado.ioloop.IOLoop.current() self.process_manager = ProcessManager(name="MultiMinionProcessManager") self.io_loop.spawn_callback( self.process_manager.run, **{"asynchronous": True} @@ -1053,7 +1058,7 @@ class MinionManager(MinionBase): self.event.subscribe("") self.event.set_event_handler(self.handle_event) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def handle_event(self, package): for minion in self.minions: minion.handle_event(package) @@ -1117,7 +1122,7 @@ class MinionManager(MinionBase): self.io_loop.spawn_callback(self._connect_minion, minion) self.io_loop.call_later(timeout, self._check_minions) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _connect_minion(self, minion): """ Create a minion, and asynchronously connect it to a master @@ -1149,7 +1154,7 @@ class MinionManager(MinionBase): last = time.time() if auth_wait < self.max_auth_wait: auth_wait += self.auth_wait - yield salt.ext.tornado.gen.sleep(auth_wait) # TODO: log? + yield tornado.gen.sleep(auth_wait) # TODO: log? except SaltMasterUnresolvableError: err = ( "Master address: '{}' could not be resolved. Invalid or" @@ -1166,7 +1171,7 @@ class MinionManager(MinionBase): minion.opts["master"], exc_info=True, ) - yield salt.ext.tornado.gen.sleep(retry_wait) + yield tornado.gen.sleep(retry_wait) if retry_wait < max_retry_wait: retry_wait += retry_wait_inc @@ -1255,7 +1260,7 @@ class Minion(MinionBase): self.periodic_callbacks = {} if io_loop is None: - self.io_loop = salt.ext.tornado.ioloop.IOLoop.current() + self.io_loop = tornado.ioloop.IOLoop.current() else: self.io_loop = io_loop @@ -1358,7 +1363,7 @@ class Minion(MinionBase): if timeout and self._sync_connect_master_success is False: raise SaltDaemonNotRunning("Failed to connect to the salt-master") - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def connect_master(self, failed=False): """ Return a future which will complete when you are connected to a master @@ -1369,7 +1374,7 @@ class Minion(MinionBase): yield self._post_master_init(master) # TODO: better name... - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _post_master_init(self, master): """ Function to finish init after connecting to a master @@ -1596,7 +1601,7 @@ class Minion(MinionBase): load, timeout=timeout, tries=self.opts["return_retry_tries"] ) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _send_req_async(self, load, timeout): if self.opts["minion_sign_messages"]: log.trace("Signing event to be published onto the bus.") @@ -1610,7 +1615,7 @@ class Minion(MinionBase): ret = yield channel.send( load, timeout=timeout, tries=self.opts["return_retry_tries"] ) - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) def _fire_master( self, @@ -1675,13 +1680,13 @@ class Minion(MinionBase): timeout_handler = handle_timeout - with salt.ext.tornado.stack_context.ExceptionStackContext(timeout_handler): + with ExceptionStackContext(timeout_handler): # pylint: disable=unexpected-keyword-arg - self._send_req_async(load, timeout, callback=lambda f: None) + self._send_req_async(load, timeout) # pylint: enable=unexpected-keyword-arg return True - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _handle_decoded_payload(self, data): """ Override this method if you wish to handle the decoded data @@ -1739,7 +1744,7 @@ class Minion(MinionBase): " waiting...", data["jid"], ) - yield salt.ext.tornado.gen.sleep(10) + yield tornado.gen.sleep(10) process_count = len(salt.utils.minion.running(self.opts)) # We stash an instance references to allow for the socket @@ -2252,10 +2257,10 @@ class Minion(MinionBase): timeout_handler() return "" else: - with salt.ext.tornado.stack_context.ExceptionStackContext(timeout_handler): + with ExceptionStackContext(timeout_handler): # pylint: disable=unexpected-keyword-arg ret_val = self._send_req_async( - load, timeout=timeout, callback=lambda f: None + load, timeout=timeout ) # pylint: enable=unexpected-keyword-arg @@ -2343,10 +2348,10 @@ class Minion(MinionBase): timeout_handler() return "" else: - with salt.ext.tornado.stack_context.ExceptionStackContext(timeout_handler): + with ExceptionStackContext(timeout_handler): # pylint: disable=unexpected-keyword-arg ret_val = self._send_req_async( - load, timeout=timeout, callback=lambda f: None + load, timeout=timeout ) # pylint: enable=unexpected-keyword-arg @@ -2483,7 +2488,7 @@ class Minion(MinionBase): return pillar_schedule # TODO: only allow one future in flight at a time? - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def pillar_refresh(self, force_refresh=False, clean_cache=False): """ Refresh the pillar @@ -2671,19 +2676,19 @@ class Minion(MinionBase): log.warning("Unable to send mine data to master.") return None - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def handle_event(self, package): """ Handle an event from the epull_sock (all local minion events) """ if not self.ready: - raise salt.ext.tornado.gen.Return() + raise tornado.gen.Return() tag, data = salt.utils.event.SaltEvent.unpack(package) if "proxy_target" in data and self.opts.get("metaproxy") == "deltaproxy": proxy_target = data["proxy_target"] if proxy_target not in self.deltaproxy_objs: - raise salt.ext.tornado.gen.Return() + raise tornado.gen.Return() _minion = self.deltaproxy_objs[proxy_target] else: _minion = self @@ -2746,7 +2751,7 @@ class Minion(MinionBase): and data["master"] != self.opts["master"] ): # not mine master, ignore - raise salt.ext.tornado.gen.Return() + raise tornado.gen.Return() if tag.startswith(master_event(type="failback")): # if the master failback event is not for the top master, raise an exception if data["master"] != self.opts["master_list"][0]: @@ -3035,7 +3040,7 @@ class Minion(MinionBase): """ if name in self.periodic_callbacks: return False - self.periodic_callbacks[name] = salt.ext.tornado.ioloop.PeriodicCallback( + self.periodic_callbacks[name] = tornado.ioloop.PeriodicCallback( method, interval * 1000, ) @@ -3273,7 +3278,7 @@ class Syndic(Minion): log.warning("Unable to forward pub data: %s", args[1]) return True - with salt.ext.tornado.stack_context.ExceptionStackContext(timeout_handler): + with ExceptionStackContext(timeout_handler): self.local.pub_async( data["tgt"], data["fun"], @@ -3299,7 +3304,7 @@ class Syndic(Minion): load, timeout=timeout, tries=self.opts["return_retry_tries"] ) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _send_req_async(self, load, timeout): if self.opts["minion_sign_messages"]: log.trace("Signing event to be published onto the bus.") @@ -3353,7 +3358,7 @@ class Syndic(Minion): # In the future, we could add support for some clearfuncs, but # the syndic currently has no need. - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def reconnect(self): if hasattr(self, "pub_channel"): self.pub_channel.on_recv(None) @@ -3370,7 +3375,7 @@ class Syndic(Minion): self.pub_channel.on_recv(self._process_cmd_socket) log.info("Minion is ready to receive requests!") - raise salt.ext.tornado.gen.Return(self) + raise tornado.gen.Return(self) def destroy(self): """ @@ -3429,7 +3434,7 @@ class SyndicManager(MinionBase): self.jid_forward_cache = set() if io_loop is None: - self.io_loop = salt.ext.tornado.ioloop.IOLoop.current() + self.io_loop = tornado.ioloop.IOLoop.current() else: self.io_loop = io_loop @@ -3456,7 +3461,7 @@ class SyndicManager(MinionBase): s_opts["master"] = master self._syndics[master] = self._connect_syndic(s_opts) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _connect_syndic(self, opts): """ Create a syndic, and asynchronously connect it to a master @@ -3492,7 +3497,7 @@ class SyndicManager(MinionBase): last = time.time() if auth_wait < self.max_auth_wait: auth_wait += self.auth_wait - yield salt.ext.tornado.gen.sleep(auth_wait) # TODO: log? + yield tornado.gen.sleep(auth_wait) # TODO: log? except (KeyboardInterrupt, SystemExit): # pylint: disable=try-except-raise raise except Exception: # pylint: disable=broad-except @@ -3503,7 +3508,7 @@ class SyndicManager(MinionBase): exc_info=True, ) - raise salt.ext.tornado.gen.Return(syndic) + raise tornado.gen.Return(syndic) def _mark_master_dead(self, master): """ @@ -3635,7 +3640,7 @@ class SyndicManager(MinionBase): self.io_loop.add_future(future, self.reconnect_event_bus) # forward events every syndic_event_forward_timeout - self.forward_events = salt.ext.tornado.ioloop.PeriodicCallback( + self.forward_events = tornado.ioloop.PeriodicCallback( self._forward_events, self.opts["syndic_event_forward_timeout"] * 1000, ) @@ -3790,7 +3795,7 @@ class ProxyMinion(Minion): """ # TODO: better name... - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _post_master_init(self, master): """ Function to finish init after connecting to a master @@ -3808,7 +3813,7 @@ class ProxyMinion(Minion): mp_call = _metaproxy_call(self.opts, "post_master_init") return mp_call(self, master) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def subproxy_post_master_init(self, minion_id, uid): """ Function to finish init for the sub proxies @@ -3837,7 +3842,7 @@ class ProxyMinion(Minion): mp_call = _metaproxy_call(self.opts, "handle_payload") return mp_call(self, payload) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _handle_decoded_payload(self, data): mp_call = _metaproxy_call(self.opts, "handle_decoded_payload") return mp_call(self, data) diff --git a/salt/modules/linux_shadow.py b/salt/modules/linux_shadow.py index aa149ac4c8e..0104b6299b1 100644 --- a/salt/modules/linux_shadow.py +++ b/salt/modules/linux_shadow.py @@ -8,6 +8,7 @@ Manage the shadow file on Linux systems `. """ +import collections import datetime import functools import logging @@ -17,11 +18,6 @@ import salt.utils.data import salt.utils.files from salt.exceptions import CommandExecutionError -try: - import spwd -except ImportError: - pass - try: import salt.utils.pycrypto @@ -34,6 +30,21 @@ __virtualname__ = "shadow" log = logging.getLogger(__name__) +struct_spwd = collections.namedtuple( + "struct_spwd", + [ + "sp_namp", + "sp_pwdp", + "sp_lstchg", + "sp_min", + "sp_max", + "sp_warn", + "sp_inact", + "sp_expire", + "sp_flag", + ] +) + def __virtual__(): return __virtualname__ if __grains__.get("kernel", "") == "Linux" else False @@ -71,7 +82,7 @@ def info(name, root=None): if root is not None: getspnam = functools.partial(_getspnam, root=root) else: - getspnam = functools.partial(spwd.getspnam) + getspnam = functools.partial(_getspnam, root="/") try: data = getspnam(name) @@ -509,7 +520,7 @@ def list_users(root=None): if root is not None: getspall = functools.partial(_getspall, root=root) else: - getspall = functools.partial(spwd.getspall) + getspall = functools.partial(_getspall, root="/") return sorted( user.sp_namp if hasattr(user, "sp_namp") else user.sp_nam for user in getspall() @@ -529,7 +540,7 @@ def _getspnam(name, root=None): # Generate a getspnam compatible output for i in range(2, 9): comps[i] = int(comps[i]) if comps[i] else -1 - return spwd.struct_spwd(comps) + return struct_spwd(*comps) raise KeyError @@ -545,4 +556,4 @@ def _getspall(root=None): # Generate a getspall compatible output for i in range(2, 9): comps[i] = int(comps[i]) if comps[i] else -1 - yield spwd.struct_spwd(comps) + yield struct_spwd(*comps) diff --git a/salt/modules/nilrt_ip.py b/salt/modules/nilrt_ip.py index 84612d7a17d..d29e3f118c2 100644 --- a/salt/modules/nilrt_ip.py +++ b/salt/modules/nilrt_ip.py @@ -271,7 +271,7 @@ def _load_config(section, options, default_value="", filename=INI_FILE): config_parser = configparser.RawConfigParser( dict_type=CaseInsensitiveDict, converters={"unquoted": _remove_quotes} ) - config_parser.readfp(config_file) + config_parser.read_file(config_file) for option in options: results[option] = config_parser.getunquoted( section, option, fallback=default_value @@ -576,7 +576,7 @@ def _change_dhcp_config(interface, enable_dhcp=True, filename=INTERFACES_CONFIG) if os.path.exists(filename): try: with salt.utils.files.fopen(filename, "r") as config_file: - parser.readfp(config_file) + parser.read_file(config_file) except configparser.MissingSectionHeaderError: pass interface = pyiface.Interface(name=interface) @@ -858,7 +858,7 @@ def _configure_static_interface(interface, **settings): if os.path.exists(INTERFACES_CONFIG): try: with salt.utils.files.fopen(INTERFACES_CONFIG, "r") as config_file: - parser.readfp(config_file) + parser.read_file(config_file) except configparser.MissingSectionHeaderError: pass hwaddr = interface.hwaddr[:-1] diff --git a/salt/modules/tls.py b/salt/modules/tls.py index b74b765cfbb..9d29bd1e9bd 100644 --- a/salt/modules/tls.py +++ b/salt/modules/tls.py @@ -120,6 +120,11 @@ HAS_SSL = False X509_EXT_ENABLED = True try: import OpenSSL + import cryptography.x509 + import cryptography.exceptions + + import cryptography.hazmat.primitives.serialization as cryptography_serialization + import cryptography.hazmat.primitives.serialization.pkcs12 as cryptography_pkcs12 HAS_SSL = True OpenSSL_version = Version(OpenSSL.__dict__.get("__version__", "0.0")) @@ -1590,49 +1595,52 @@ def create_pkcs12(ca_name, CN, passphrase="", cacert_path=None, replace=False): salt '*' tls.create_pkcs12 test localhost """ set_ca_path(cacert_path) - if not replace and os.path.exists( - "{}/{}/certs/{}.p12".format(cert_base_path(), ca_name, CN) - ): + p12_path = f"{cert_base_path()}/{ca_name}/certs/{CN}.p12" + ca_cert_path = f"{cert_base_path()}/{ca_name}/{ca_name}_ca_cert.crt" + cert_path = f"{cert_base_path()}/{ca_name}/certs/{CN}.crt" + priv_key_path = f"{cert_base_path()}/{ca_name}/certs/{CN}.key" + + if not replace and os.path.exists(p12_path): return 'Certificate "{}" already exists'.format(CN) try: - with salt.utils.files.fopen( - "{0}/{1}/{1}_ca_cert.crt".format(cert_base_path(), ca_name) - ) as fhr: - ca_cert = OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_PEM, fhr.read() - ) + with salt.utils.files.fopen(ca_cert_path, "rb") as fhr: + ca_cert = cryptography.x509.load_pem_x509_certificate(fhr.read()) except OSError: return 'There is no CA named "{}"'.format(ca_name) + except ValueError as e: + return f'Could not load CA {ca_cert_path}: {e}' try: - with salt.utils.files.fopen( - "{}/{}/certs/{}.crt".format(cert_base_path(), ca_name, CN) - ) as fhr: - cert = OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_PEM, fhr.read() - ) - with salt.utils.files.fopen( - "{}/{}/certs/{}.key".format(cert_base_path(), ca_name, CN) - ) as fhr: - key = OpenSSL.crypto.load_privatekey( - OpenSSL.crypto.FILETYPE_PEM, fhr.read() + with salt.utils.files.fopen(cert_path, "rb") as fhr: + cert = cryptography.x509.load_pem_x509_certificate(fhr.read()) + with salt.utils.files.fopen(priv_key_path, "rb") as fhr: + key = cryptography_serialization.load_pem_private_key( + fhr.read(), + password=None, ) except OSError: return 'There is no certificate that matches the CN "{}"'.format(CN) + except ValueError as e: + return f'Could not load certificate {cert_path}: {e}' - pkcs12 = OpenSSL.crypto.PKCS12() - - pkcs12.set_certificate(cert) - pkcs12.set_ca_certificates([ca_cert]) - pkcs12.set_privatekey(key) - - with salt.utils.files.fopen( - "{}/{}/certs/{}.p12".format(cert_base_path(), ca_name, CN), "wb" - ) as ofile: - ofile.write( - pkcs12.export(passphrase=salt.utils.stringutils.to_bytes(passphrase)) + if passphrase: + encryption_algorithm = cryptography_serialization.BestAvailableEncryption( + salt.utils.stringutils.to_bytes(passphrase) ) + else: + encryption_algorithm = cryptography_serialization.NoEncryption() + + pkcs12 = cryptography_pkcs12.serialize_key_and_certificates( + name=salt.utils.stringutils.to_bytes(CN), + key=key, + cert=cert, + cas=[ca_cert], + encryption_algorithm=encryption_algorithm, + ) + + with salt.utils.files.fopen(p12_path, "wb") as ofile: + ofile.write(pkcs12) return 'Created PKCS#12 Certificate for "{0}": "{1}/{2}/certs/{0}.p12"'.format( CN, @@ -1929,10 +1937,10 @@ def revoke_cert( ) except ValueError: ret["retcode"] = 1 - ret[ - "comment" - ] = "Revocation date '{}' does not matchformat '{}'".format( - revoke_date, two_digit_year_fmt + ret["comment"] = ( + "Revocation date '{}' does not matchformat '{}'".format( + revoke_date, two_digit_year_fmt + ) ) return ret elif index_serial_subject in line: diff --git a/salt/modules/x509.py b/salt/modules/x509.py index 9687f63d1bb..164541fc761 100644 --- a/salt/modules/x509.py +++ b/salt/modules/x509.py @@ -32,11 +32,13 @@ import tempfile import salt.exceptions import salt.utils.data +import salt.utils.dictupdate import salt.utils.files import salt.utils.path import salt.utils.platform import salt.utils.stringutils import salt.utils.versions +import salt.utils.x509 as x509util from salt.state import STATE_INTERNAL_KEYWORDS as _STATE_INTERNAL_KEYWORDS from salt.utils.odict import OrderedDict @@ -983,86 +985,38 @@ def create_crl( raise salt.exceptions.SaltInvocationError( "Could not load OpenSSL module, OpenSSL unavailable" ) - crl = OpenSSL.crypto.CRL() if revoked is None: revoked = [] for rev_item in revoked: - if "certificate" in rev_item: - rev_cert = read_certificate(rev_item["certificate"]) - rev_item["serial_number"] = rev_cert["Serial Number"] - rev_item["not_after"] = rev_cert["Not After"] - - serial_number = rev_item["serial_number"].replace(":", "") - # OpenSSL bindings requires this to be a non-unicode string - serial_number = salt.utils.stringutils.to_bytes(serial_number) - - if "not_after" in rev_item and not include_expired: - not_after = datetime.datetime.strptime( - rev_item["not_after"], "%Y-%m-%d %H:%M:%S" - ) - if datetime.datetime.now() > not_after: - continue - - if "revocation_date" not in rev_item: - rev_item["revocation_date"] = datetime.datetime.now().strftime( - "%Y-%m-%d %H:%M:%S" - ) - - rev_date = datetime.datetime.strptime( - rev_item["revocation_date"], "%Y-%m-%d %H:%M:%S" - ) - rev_date = rev_date.strftime("%Y%m%d%H%M%SZ") - rev_date = salt.utils.stringutils.to_bytes(rev_date) - - rev = OpenSSL.crypto.Revoked() - rev.set_serial(salt.utils.stringutils.to_bytes(serial_number)) - rev.set_rev_date(salt.utils.stringutils.to_bytes(rev_date)) - if "reason" in rev_item: - # Same here for OpenSSL bindings and non-unicode strings - reason = salt.utils.stringutils.to_bytes(rev_item["reason"]) - rev.set_reason(reason) - - crl.add_revoked(rev) + salt.utils.dictupdate.set_dict_key_value( + rev_item, "extensions:CRLReason", rev_item["reason"] + ) - signing_cert = _text_or_file(signing_cert) - cert = OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_PEM, get_pem_entry(signing_cert, pem_type="CERTIFICATE") - ) - signing_private_key = _get_private_key_obj( - signing_private_key, passphrase=signing_private_key_passphrase - ).as_pem(cipher=None) - key = OpenSSL.crypto.load_privatekey( - OpenSSL.crypto.FILETYPE_PEM, get_pem_entry(signing_private_key) + builder, private_key_obj = x509util.build_crl( + signing_private_key=signing_private_key, + signing_private_key_passphrase=signing_private_key_passphrase, + include_expired=include_expired, + revoked=revoked, + signing_cert=signing_cert, + days_valid=days_valid, ) - export_kwargs = { - "cert": cert, - "key": key, - "type": OpenSSL.crypto.FILETYPE_PEM, - "days": days_valid, - } if digest: - export_kwargs["digest"] = salt.utils.stringutils.to_bytes(digest) + hashing_algorithm = x509util.get_hashing_algorithm(digest) else: log.warning("No digest specified. The default md5 digest will be used.") + hashing_algorithm = x509util.get_hashing_algorithm("MD5") - try: - crltext = crl.export(**export_kwargs) - except (TypeError, ValueError): - log.warning( - "Error signing crl with specified digest. Are you using " - "pyopenssl 0.15 or newer? The default md5 digest will be used." - ) - export_kwargs.pop("digest", None) - crltext = crl.export(**export_kwargs) + crl = builder.sign(private_key_obj, algorithm=hashing_algorithm) + crl_bytes = crl.public_bytes(x509util.serialization.Encoding.PEM) if text: - return crltext + return crl_bytes.decode() - return write_pem(text=crltext, path=path, pem_type="X509 CRL") + return write_pem(text=crl_bytes, path=path, pem_type="X509 CRL") def sign_remote_certificate(argdic, **kwargs): diff --git a/salt/netapi/rest_tornado/__init__.py b/salt/netapi/rest_tornado/__init__.py index 67336d0adaa..b4f5fb4422b 100644 --- a/salt/netapi/rest_tornado/__init__.py +++ b/salt/netapi/rest_tornado/__init__.py @@ -13,9 +13,9 @@ log = logging.getLogger(__virtualname__) min_tornado_version = "4.0" has_tornado = False try: - import salt.ext.tornado + import tornado - if Version(salt.ext.tornado.version) >= Version(min_tornado_version): + if Version(tornado.version) >= Version(min_tornado_version): has_tornado = True else: log.error("rest_tornado requires at least tornado %s", min_tornado_version) @@ -74,7 +74,7 @@ def get_application(opts): (formatted_events_pattern, saltnado_websockets.FormattedEventsHandler), ] - application = salt.ext.tornado.web.Application( + application = tornado.web.Application( paths, debug=mod_opts.get("debug", False) ) @@ -117,9 +117,9 @@ def start(): ssl_opts.update({"keyfile": mod_opts["ssl_key"]}) kwargs["ssl_options"] = ssl_opts - import salt.ext.tornado.httpserver + import tornado.httpserver - http_server = salt.ext.tornado.httpserver.HTTPServer( + http_server = tornado.httpserver.HTTPServer( get_application(__opts__), **kwargs ) try: @@ -136,6 +136,6 @@ def start(): raise SystemExit(1) try: - salt.ext.tornado.ioloop.IOLoop.current().start() + tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt: raise SystemExit(0) diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py index 44c4089ccbb..ec7e16a36ee 100644 --- a/salt/netapi/rest_tornado/saltnado.py +++ b/salt/netapi/rest_tornado/saltnado.py @@ -194,11 +194,11 @@ from copy import copy import salt.auth import salt.client -import salt.ext.tornado.escape -import salt.ext.tornado.gen -import salt.ext.tornado.httpserver -import salt.ext.tornado.ioloop -import salt.ext.tornado.web +import tornado.escape +import tornado.gen +import tornado.httpserver +import tornado.ioloop +import tornado.web import salt.netapi import salt.runner import salt.utils.args @@ -211,7 +211,7 @@ from salt.exceptions import ( AuthorizationError, EauthAuthenticationError, ) -from salt.ext.tornado.concurrent import Future +from tornado.concurrent import Future from salt.utils.event import tagify _json = salt.utils.json.import_json() @@ -277,7 +277,7 @@ class EventListener: opts["sock_dir"], opts=opts, listen=True, - io_loop=salt.ext.tornado.ioloop.IOLoop.current(), + io_loop=tornado.ioloop.IOLoop.current(), ) # tag -> list of futures @@ -302,7 +302,7 @@ class EventListener: self._timeout_future(tag, matcher, future) # remove the timeout if future in self.timeout_map: - salt.ext.tornado.ioloop.IOLoop.current().remove_timeout( + tornado.ioloop.IOLoop.current().remove_timeout( self.timeout_map[future] ) del self.timeout_map[future] @@ -336,7 +336,7 @@ class EventListener: if callback is not None: def handle_future(future): - salt.ext.tornado.ioloop.IOLoop.current().add_callback( + tornado.ioloop.IOLoop.current().add_callback( callback, future ) # pylint: disable=E1102 @@ -346,7 +346,7 @@ class EventListener: self.request_map[request].append((tag, matcher, future)) if timeout: - timeout_future = salt.ext.tornado.ioloop.IOLoop.current().call_later( + timeout_future = tornado.ioloop.IOLoop.current().call_later( timeout, self._timeout_future, tag, matcher, future ) self.timeout_map[future] = timeout_future @@ -391,18 +391,22 @@ class EventListener: future.set_result({"data": data, "tag": mtag}) self.tag_map[(tag, matcher)].remove(future) if future in self.timeout_map: - salt.ext.tornado.ioloop.IOLoop.current().remove_timeout( + tornado.ioloop.IOLoop.current().remove_timeout( self.timeout_map[future] ) del self.timeout_map[future] -class BaseSaltAPIHandler(salt.ext.tornado.web.RequestHandler): # pylint: disable=W0223 +class BaseSaltAPIHandler(tornado.web.RequestHandler): # pylint: disable=W0223 ct_out_map = ( ("application/json", _json_dumps), ("application/x-yaml", salt.utils.yaml.safe_dump), ) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._auto_finish = False + def _verify_client(self, low): """ Verify that the client is in fact one we have @@ -555,7 +559,7 @@ class BaseSaltAPIHandler(salt.ext.tornado.web.RequestHandler): # pylint: disabl try: # Use cgi.parse_header to correctly separate parameters from value value, parameters = cgi.parse_header(self.request.headers["Content-Type"]) - return ct_in_map[value](salt.ext.tornado.escape.native_str(data)) + return ct_in_map[value](tornado.escape.native_str(data)) except KeyError: self.send_error(406) except ValueError: @@ -659,6 +663,7 @@ class SaltAuthHandler(BaseSaltAPIHandler): # pylint: disable=W0223 ret = {"status": "401 Unauthorized", "return": "Please log in"} self.write(self.serialize(ret)) + self.finish() # TODO: make asynchronous? Underlying library isn't... and we ARE making disk calls :( def post(self): # pylint: disable=arguments-differ @@ -785,6 +790,7 @@ class SaltAuthHandler(BaseSaltAPIHandler): # pylint: disable=W0223 } self.write(self.serialize(ret)) + self.finish() class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 @@ -828,8 +834,9 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 """ ret = {"clients": list(self.saltclients.keys()), "return": "Welcome"} self.write(self.serialize(ret)) + self.finish() - @salt.ext.tornado.web.asynchronous + @tornado.gen.coroutine def post(self): # pylint: disable=arguments-differ """ Send one or more Salt commands (lowstates) in the request body @@ -907,7 +914,7 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 self.disbatch() - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def disbatch(self): """ Disbatch all lowstates to the appropriate clients @@ -950,7 +957,7 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 except RuntimeError: pass # Do we need any logging here? - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def get_minion_returns( self, events, is_finished, is_timed_out, min_wait_time, minions ): @@ -980,11 +987,11 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 # When finished entire routine, cleanup other futures and return result if f is is_finished or f is is_timed_out: cancel_inflight_futures() - raise salt.ext.tornado.gen.Return(chunk_ret) + raise tornado.gen.Return(chunk_ret) elif f is min_wait_time: if not more_todo(): cancel_inflight_futures() - raise salt.ext.tornado.gen.Return(chunk_ret) + raise tornado.gen.Return(chunk_ret) continue f_result = f.result() @@ -1000,7 +1007,7 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 # if there are no more minions to wait for, then we are done if not more_todo() and min_wait_time.done(): cancel_inflight_futures() - raise salt.ext.tornado.gen.Return(chunk_ret) + raise tornado.gen.Return(chunk_ret) except TimeoutException: pass @@ -1008,7 +1015,7 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 if f in events: events.remove(f) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _disbatch_local(self, chunk): """ Dispatch local client commands @@ -1057,7 +1064,7 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 future.set_result(None) except Exception: # pylint: disable=broad-except pass - raise salt.ext.tornado.gen.Return( + raise tornado.gen.Return( "No minions matched the target. No command was sent, no jid was" " assigned." ) @@ -1078,19 +1085,19 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 # wait syndic a while to avoid missing published events if self.application.opts["order_masters"]: - min_wait_time = salt.ext.tornado.gen.sleep( + min_wait_time = tornado.gen.sleep( self.application.opts["syndic_wait"] ) # To ensure job_not_running and all_return are terminated by each other, communicate using a future - is_finished = salt.ext.tornado.gen.Future() - is_timed_out = salt.ext.tornado.gen.sleep( + is_finished = tornado.gen.Future() + is_timed_out = tornado.gen.sleep( self.application.opts["gather_job_timeout"] ) # ping until the job is not running, while doing so, if we see new minions returning # that they are running the job, add them to the list - salt.ext.tornado.ioloop.IOLoop.current().spawn_callback( + tornado.ioloop.IOLoop.current().spawn_callback( self.job_not_running, pub_data["jid"], chunk["tgt"], @@ -1106,9 +1113,9 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 min_wait_time=min_wait_time, minions=minions, ) - raise salt.ext.tornado.gen.Return(result) + raise tornado.gen.Return(result) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def job_not_running(self, jid, tgt, tgt_type, minions, is_finished): """ Return a future which will complete once jid (passed in) is no longer @@ -1133,11 +1140,11 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 if f is is_finished: if not event.done(): event.set_result(None) - raise salt.ext.tornado.gen.Return(True) + raise tornado.gen.Return(True) event = f.result() except TimeoutException: if not minion_running or is_finished.done(): - raise salt.ext.tornado.gen.Return(True) + raise tornado.gen.Return(True) else: ping_pub_data = yield self.saltclients["local"]( tgt, "saltutil.find_job", [jid], tgt_type=tgt_type @@ -1153,7 +1160,7 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 minions[event["data"]["id"]] = False minion_running = True - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _disbatch_local_async(self, chunk): """ Disbatch local client_async commands @@ -1164,9 +1171,9 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 *f_call.get("args", ()), **f_call.get("kwargs", {}) ) - raise salt.ext.tornado.gen.Return(pub_data) + raise tornado.gen.Return(pub_data) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _disbatch_runner(self, chunk): """ Disbatch runner client commands @@ -1179,25 +1186,25 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 # only return the return data ret = event if full_return else event["data"]["return"] - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) except TimeoutException: - raise salt.ext.tornado.gen.Return("Timeout waiting for runner to execute") + raise tornado.gen.Return("Timeout waiting for runner to execute") - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _disbatch_runner_async(self, chunk): """ Disbatch runner client_async commands """ pub_data = self.saltclients["runner"](chunk) - raise salt.ext.tornado.gen.Return(pub_data) + raise tornado.gen.Return(pub_data) # salt.utils.args.format_call doesn't work for functions having the - # annotation salt.ext.tornado.gen.coroutine + # annotation tornado.gen.coroutine def _format_call_run_job_async(self, chunk): f_call = salt.utils.args.format_call( salt.client.LocalClient.run_job, chunk, is_class_method=True ) - f_call.get("kwargs", {})["io_loop"] = salt.ext.tornado.ioloop.IOLoop.current() + f_call.get("kwargs", {})["io_loop"] = tornado.ioloop.IOLoop.current() return f_call @@ -1206,7 +1213,7 @@ class MinionSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223 A convenience endpoint for minion related functions """ - @salt.ext.tornado.web.asynchronous + @tornado.gen.coroutine def get(self, mid=None): # pylint: disable=W0221 """ A convenience URL for getting lists of minions or getting minion @@ -1254,7 +1261,7 @@ class MinionSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223 self.lowstate = [{"client": "local", "tgt": mid or "*", "fun": "grains.items"}] self.disbatch() - @salt.ext.tornado.web.asynchronous + @tornado.gen.coroutine def post(self): """ Start an execution command and immediately return the job id @@ -1332,7 +1339,7 @@ class JobsSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223 A convenience endpoint for job cache data """ - @salt.ext.tornado.web.asynchronous + @tornado.gen.coroutine def get(self, jid=None): # pylint: disable=W0221 """ A convenience URL for getting lists of previously run jobs or getting @@ -1432,7 +1439,7 @@ class RunSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223 Endpoint to run commands without normal session handling """ - @salt.ext.tornado.web.asynchronous + @tornado.gen.coroutine def post(self): """ Run commands bypassing the :ref:`normal session handling @@ -1504,7 +1511,7 @@ class EventsSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223 .. seealso:: :ref:`events` """ - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def get(self): r""" An HTTP stream of the Salt master event bus @@ -1790,6 +1797,7 @@ class WebhookSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223 ) self.write(self.serialize({"success": ret})) + self.finish() def _check_cors_origin(origin, allowed_origins): diff --git a/salt/netapi/rest_tornado/saltnado_websockets.py b/salt/netapi/rest_tornado/saltnado_websockets.py index 08f4b7ad157..98d64000909 100644 --- a/salt/netapi/rest_tornado/saltnado_websockets.py +++ b/salt/netapi/rest_tornado/saltnado_websockets.py @@ -291,8 +291,8 @@ Setup import logging -import salt.ext.tornado.gen -import salt.ext.tornado.websocket +import tornado.gen +import tornado.websocket import salt.netapi import salt.utils.json @@ -306,7 +306,7 @@ log = logging.getLogger(__name__) class AllEventsHandler( - salt.ext.tornado.websocket.WebSocketHandler + tornado.websocket.WebSocketHandler ): # pylint: disable=W0223,W0232 """ Server side websocket handler. @@ -335,7 +335,7 @@ class AllEventsHandler( """ self.connected = False - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def on_message(self, message): """Listens for a "websocket client ready" message. Once that message is received an asynchronous job @@ -387,7 +387,7 @@ class AllEventsHandler( class FormattedEventsHandler(AllEventsHandler): # pylint: disable=W0223,W0232 - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def on_message(self, message): """Listens for a "websocket client ready" message. Once that message is received an asynchronous job diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py index 8f6bd90d0a4..0b532aca5ed 100644 --- a/salt/pillar/__init__.py +++ b/salt/pillar/__init__.py @@ -12,7 +12,7 @@ import time import traceback import salt.channel.client -import salt.ext.tornado.gen +import tornado.gen import salt.fileclient import salt.loader import salt.minion @@ -243,7 +243,7 @@ class AsyncRemotePillar(RemotePillarMixin): self._closing = False self.clean_cache = clean_cache - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def compile_pillar(self): """ Return a future which will contain the pillar data from the master @@ -286,7 +286,7 @@ class AsyncRemotePillar(RemotePillarMixin): log.error(msg) # raise an exception! Pillar isn't empty, we can't sync it! raise SaltClientError(msg) - raise salt.ext.tornado.gen.Return(ret_pillar) + raise tornado.gen.Return(ret_pillar) def destroy(self): if hasattr(self, "_closing") and self._closing: @@ -1392,7 +1392,7 @@ class Pillar: # TODO: actually migrate from Pillar to AsyncPillar to allow for futures in # ext_pillar etc. class AsyncPillar(Pillar): - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def compile_pillar(self, ext=True): ret = super().compile_pillar(ext=ext) - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) diff --git a/salt/transport/base.py b/salt/transport/base.py index 014a9731d59..86a2c882deb 100644 --- a/salt/transport/base.py +++ b/salt/transport/base.py @@ -1,4 +1,4 @@ -import salt.ext.tornado.gen +import tornado.gen TRANSPORTS = ( "zeromq", @@ -103,7 +103,7 @@ class RequestClient: def __init__(self, opts, io_loop, **kwargs): pass - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def send(self, load, timeout=60): """ Send a request message and return the reply from the server. @@ -211,7 +211,7 @@ class PublishClient: """ raise NotImplementedError - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def connect(self, publish_port, connect_callback=None, disconnect_callback=None): """ Create a network connection to the the PublishServer or broker. diff --git a/salt/transport/ipc.py b/salt/transport/ipc.py index 6631781c5cf..d016c8c8b19 100644 --- a/salt/transport/ipc.py +++ b/salt/transport/ipc.py @@ -7,18 +7,18 @@ import logging import socket import time -import salt.ext.tornado -import salt.ext.tornado.concurrent -import salt.ext.tornado.gen -import salt.ext.tornado.ioloop -import salt.ext.tornado.netutil +import tornado +import tornado.concurrent +import tornado.gen +import tornado.ioloop +import tornado.netutil import salt.transport.client import salt.transport.frame import salt.utils.msgpack -from salt.ext.tornado.ioloop import IOLoop -from salt.ext.tornado.ioloop import TimeoutError as TornadoTimeoutError -from salt.ext.tornado.iostream import IOStream, StreamClosedError -from salt.ext.tornado.locks import Lock +from tornado.ioloop import IOLoop +from tornado.ioloop import TimeoutError as TornadoTimeoutError +from tornado.iostream import IOStream, StreamClosedError +from tornado.locks import Lock log = logging.getLogger(__name__) @@ -32,7 +32,7 @@ def future_with_timeout_callback(future): future._future_with_timeout._done_callback(future) -class FutureWithTimeout(salt.ext.tornado.concurrent.Future): +class FutureWithTimeout(tornado.concurrent.Future): def __init__(self, io_loop, future, timeout): super().__init__() self.io_loop = io_loop @@ -112,7 +112,7 @@ class IPCServer: # Placeholders for attributes to be populated by method calls self.sock = None - self.io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current() + self.io_loop = io_loop or tornado.ioloop.IOLoop.current() self._closing = False def start(self): @@ -131,16 +131,16 @@ class IPCServer: # Based on default used in tornado.netutil.bind_sockets() self.sock.listen(128) else: - self.sock = salt.ext.tornado.netutil.bind_unix_socket(self.socket_path) + self.sock = tornado.netutil.bind_unix_socket(self.socket_path) with salt.utils.asynchronous.current_ioloop(self.io_loop): - salt.ext.tornado.netutil.add_accept_handler( + tornado.netutil.add_accept_handler( self.sock, self.handle_connection, ) self._started = True - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def handle_stream(self, stream): """ Override this to handle the streams as they arrive @@ -151,14 +151,14 @@ class IPCServer: for additional details. """ - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _null(msg): - raise salt.ext.tornado.gen.Return(None) + raise tornado.gen.Return(None) def write_callback(stream, header): if header.get("mid"): - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def return_message(msg): pack = salt.transport.frame.frame_msg_ipc( msg, @@ -273,7 +273,7 @@ class IPCClient: to the server. """ - self.io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current() + self.io_loop = io_loop or tornado.ioloop.IOLoop.current() self.socket_path = socket_path self._closing = False self.stream = None @@ -299,7 +299,7 @@ class IPCClient: if self._connecting_future is not None: # read previous future result to prevent the "unhandled future exception" error self._connecting_future.exception() # pylint: disable=E0203 - future = salt.ext.tornado.concurrent.Future() + future = tornado.concurrent.Future() self._connecting_future = future self._connect(timeout) @@ -313,7 +313,7 @@ class IPCClient: return future - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _connect(self, timeout=None): """ Connect to a running IPCServer @@ -339,7 +339,7 @@ class IPCClient: try: log.trace("IPCClient: Connecting to socket: %s", self.socket_path) yield self.stream.connect(sock_addr) - if self._connecting_future is not None: + if self._connecting_future is not None and not self._connecting_future.done(): self._connecting_future.set_result(True) break except Exception as e: # pylint: disable=broad-except @@ -350,11 +350,11 @@ class IPCClient: if self.stream is not None: self.stream.close() self.stream = None - if self._connecting_future is not None: + if self._connecting_future is not None and not self._connecting_future.done(): self._connecting_future.set_exception(e) break - yield salt.ext.tornado.gen.sleep(1) + yield tornado.gen.sleep(1) def close(self): """ @@ -366,7 +366,7 @@ class IPCClient: return self._closing = True - if self._connecting_future is not None: + if self._connecting_future is not None and not self._connecting_future.done(): try: self._connecting_future.set_result(True) self._connecting_future.exception() # pylint: disable=E0203 @@ -414,13 +414,13 @@ class IPCMessageClient(IPCClient): IMPORTANT: The below example also assumes a running IOLoop process. # Import Tornado libs - import salt.ext.tornado.ioloop + import tornado.ioloop # Import Salt libs import salt.config import salt.transport.ipc - io_loop = salt.ext.tornado.ioloop.IOLoop.current() + io_loop = tornado.ioloop.IOLoop.current() ipc_server_socket_path = '/var/run/ipc_server.ipc' @@ -442,7 +442,7 @@ class IPCMessageClient(IPCClient): "close", ] - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def send(self, msg, timeout=None, tries=None): """ Send a message to an IPC socket @@ -499,7 +499,7 @@ class IPCMessageClient(IPCClient): _try += 1 if _try > tries or (due_time is not None and cur_time > due_time): return - yield salt.ext.tornado.gen.sleep( + yield tornado.gen.sleep( 1 if due_time is None else (due_time - cur_time) / max(tries - _try + 1, 1) @@ -517,12 +517,12 @@ class IPCMessageServer(IPCServer): a console: # Import Tornado libs - import salt.ext.tornado.ioloop + import tornado.ioloop # Import Salt libs import salt.transport.ipc - io_loop = salt.ext.tornado.ioloop.IOLoop.current() + io_loop = tornado.ioloop.IOLoop.current() ipc_server_socket_path = '/var/run/ipc_server.ipc' ipc_server = salt.transport.ipc.IPCMessageServer(ipc_server_socket_path, io_loop=io_loop, payload_handler=print_to_console) @@ -583,19 +583,19 @@ class IPCMessagePublisher: self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.sock.setblocking(0) self.sock.bind(("127.0.0.1", self.socket_path)) - # Based on default used in salt.ext.tornado.netutil.bind_sockets() + # Based on default used in tornado.netutil.bind_sockets() self.sock.listen(128) else: - self.sock = salt.ext.tornado.netutil.bind_unix_socket(self.socket_path) + self.sock = tornado.netutil.bind_unix_socket(self.socket_path) with salt.utils.asynchronous.current_ioloop(self.io_loop): - salt.ext.tornado.netutil.add_accept_handler( + tornado.netutil.add_accept_handler( self.sock, self.handle_connection, ) self._started = True - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _write(self, stream, pack): try: yield stream.write(pack) @@ -674,7 +674,7 @@ class IPCMessageSubscriber(IPCClient): IMPORTANT: The below example also assumes the IOLoop is NOT running. # Import Tornado libs - import salt.ext.tornado.ioloop + import tornado.ioloop # Import Salt libs import salt.config @@ -682,7 +682,7 @@ class IPCMessageSubscriber(IPCClient): # Create a new IO Loop. # We know that this new IO Loop is not currently running. - io_loop = salt.ext.tornado.ioloop.IOLoop() + io_loop = tornado.ioloop.IOLoop() ipc_publisher_socket_path = '/var/run/ipc_publisher.ipc' @@ -711,13 +711,13 @@ class IPCMessageSubscriber(IPCClient): self._read_in_progress = Lock() self.callbacks = set() - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _read(self, timeout, callback=None): try: try: yield self._read_in_progress.acquire(timeout=0.00000001) - except salt.ext.tornado.gen.TimeoutError: - raise salt.ext.tornado.gen.Return(None) + except tornado.gen.TimeoutError: + raise tornado.gen.Return(None) exc_to_raise = None ret = None @@ -772,12 +772,12 @@ class IPCMessageSubscriber(IPCClient): if exc_to_raise is not None: raise exc_to_raise # pylint: disable=E0702 - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) # Handle ctrl+c gracefully except TypeError: pass - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def read(self, timeout): """ Asynchronously read messages and invoke a callback when they are ready. @@ -785,7 +785,7 @@ class IPCMessageSubscriber(IPCClient): """ if self._saved_data: res = self._saved_data.pop(0) - raise salt.ext.tornado.gen.Return(res) + raise tornado.gen.Return(res) while not self.connected(): try: yield self.connect(timeout=5) @@ -794,12 +794,12 @@ class IPCMessageSubscriber(IPCClient): "Subscriber closed stream on IPC %s before connect", self.socket_path, ) - yield salt.ext.tornado.gen.sleep(1) + yield tornado.gen.sleep(1) except Exception as exc: # pylint: disable=broad-except log.error("Exception occurred while Subscriber connecting: %s", exc) - yield salt.ext.tornado.gen.sleep(1) + yield tornado.gen.sleep(1) res = yield self._read(timeout) - raise salt.ext.tornado.gen.Return(res) + raise tornado.gen.Return(res) def read_sync(self, timeout=None): """ @@ -819,7 +819,7 @@ class IPCMessageSubscriber(IPCClient): for callback in self.callbacks: self.io_loop.spawn_callback(callback, raw) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def read_async(self): """ Asynchronously read messages and invoke a callback when they are ready. @@ -834,10 +834,10 @@ class IPCMessageSubscriber(IPCClient): "Subscriber closed stream on IPC %s before connect", self.socket_path, ) - yield salt.ext.tornado.gen.sleep(1) + yield tornado.gen.sleep(1) except Exception as exc: # pylint: disable=broad-except log.error("Exception occurred while Subscriber connecting: %s", exc) - yield salt.ext.tornado.gen.sleep(1) + yield tornado.gen.sleep(1) yield self._read(None, self.__run_callbacks) def close(self): diff --git a/salt/transport/tcp.py b/salt/transport/tcp.py index 6a9e1138940..488178e90f5 100644 --- a/salt/transport/tcp.py +++ b/salt/transport/tcp.py @@ -16,13 +16,13 @@ import socket import threading import urllib -import salt.ext.tornado -import salt.ext.tornado.concurrent -import salt.ext.tornado.gen -import salt.ext.tornado.iostream -import salt.ext.tornado.netutil -import salt.ext.tornado.tcpclient -import salt.ext.tornado.tcpserver +import tornado +import tornado.concurrent +import tornado.gen +import tornado.iostream +import tornado.netutil +import tornado.tcpclient +import tornado.tcpserver import salt.master import salt.payload import salt.transport.client @@ -43,7 +43,7 @@ else: USE_LOAD_BALANCER = False if USE_LOAD_BALANCER: - import salt.ext.tornado.util + import tornado.util from salt.utils.process import SignalHandlingProcess log = logging.getLogger(__name__) @@ -141,7 +141,7 @@ if USE_LOAD_BALANCER: """ # TODO: opts! - # Based on default used in salt.ext.tornado.netutil.bind_sockets() + # Based on default used in tornado.netutil.bind_sockets() backlog = 128 def __init__(self, opts, socket_queue, **kwargs): @@ -187,7 +187,7 @@ if USE_LOAD_BALANCER: # but it was closed while still in the accept queue. # (observed on FreeBSD). if ( - salt.ext.tornado.util.errno_from_exception(e) + tornado.util.errno_from_exception(e) == errno.ECONNABORTED ): continue @@ -200,8 +200,8 @@ class Resolver: @classmethod def _config_resolver(cls, num_threads=10): - salt.ext.tornado.netutil.Resolver.configure( - "salt.ext.tornado.netutil.ThreadedResolver", num_threads=num_threads + tornado.netutil.Resolver.configure( + "tornado.netutil.ThreadedResolver", num_threads=num_threads ) cls._resolver_configured = True @@ -240,7 +240,7 @@ class TCPPubClient(salt.transport.base.PublishClient): # pylint: enable=W1701 - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def connect(self, publish_port, connect_callback=None, disconnect_callback=None): self.publish_port = publish_port self.message_client = MessageClient( @@ -256,7 +256,7 @@ class TCPPubClient(salt.transport.base.PublishClient): yield self.message_client.connect() # wait for the client to be connected self.connected = True - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _decode_messages(self, messages): if not isinstance(messages, dict): # TODO: For some reason we need to decode here for things @@ -265,9 +265,9 @@ class TCPPubClient(salt.transport.base.PublishClient): body = salt.transport.frame.decode_embedded_strs(body) else: body = messages - raise salt.ext.tornado.gen.Return(body) + raise tornado.gen.Return(body) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def send(self, msg): yield self.message_client._stream.write(msg) @@ -365,6 +365,7 @@ class TCPReqServer(salt.transport.base.DaemonizedRequestServer): message_handler: function to call with your payloads """ self.message_handler = message_handler + log.info("ReqServer workers %s", socket) with salt.utils.asynchronous.current_ioloop(io_loop): if USE_LOAD_BALANCER: @@ -388,7 +389,7 @@ class TCPReqServer(salt.transport.base.DaemonizedRequestServer): self.req_server.add_socket(self._socket) self._socket.listen(self.backlog) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def handle_message(self, stream, payload, header=None): payload = self.decode_payload(payload) reply = yield self.message_handler(payload) @@ -398,7 +399,7 @@ class TCPReqServer(salt.transport.base.DaemonizedRequestServer): return payload -class SaltMessageServer(salt.ext.tornado.tcpserver.TCPServer): +class SaltMessageServer(tornado.tcpserver.TCPServer): """ Raw TCP server which will receive all of the TCP streams and re-assemble messages that are sent through to us @@ -406,7 +407,7 @@ class SaltMessageServer(salt.ext.tornado.tcpserver.TCPServer): def __init__(self, message_handler, *args, **kwargs): io_loop = ( - kwargs.pop("io_loop", None) or salt.ext.tornado.ioloop.IOLoop.current() + kwargs.pop("io_loop", None) or tornado.ioloop.IOLoop.current() ) self._closing = False super().__init__(*args, **kwargs) @@ -414,12 +415,12 @@ class SaltMessageServer(salt.ext.tornado.tcpserver.TCPServer): self.clients = [] self.message_handler = message_handler - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def handle_stream( # pylint: disable=arguments-differ self, stream, address, - _StreamClosedError=salt.ext.tornado.iostream.StreamClosedError, + _StreamClosedError=tornado.iostream.StreamClosedError, ): """ Handle incoming streams and add messages to the incoming queue @@ -501,7 +502,7 @@ if USE_LOAD_BALANCER: break continue # 'self.io_loop' initialized in super class - # 'salt.ext.tornado.tcpserver.TCPServer'. + # 'tornado.tcpserver.TCPServer'. # 'self._handle_connection' defined in same super class. self.io_loop.spawn_callback( self._handle_connection, client_socket, address @@ -510,7 +511,7 @@ if USE_LOAD_BALANCER: pass -class TCPClientKeepAlive(salt.ext.tornado.tcpclient.TCPClient): +class TCPClientKeepAlive(tornado.tcpclient.TCPClient): """ Override _create_stream() in TCPClient to enable keep alive support. """ @@ -533,10 +534,10 @@ class TCPClientKeepAlive(salt.ext.tornado.tcpclient.TCPClient): # after one connection has completed. sock = _get_socket(self.opts) _set_tcp_keepalive(sock, self.opts) - stream = salt.ext.tornado.iostream.IOStream( + stream = tornado.iostream.IOStream( sock, max_buffer_size=max_buffer_size ) - if salt.ext.tornado.version_info < (5,): + if tornado.version_info < (5,): return stream.connect(addr) return stream, stream.connect(addr) @@ -568,7 +569,7 @@ class MessageClient: self.source_port = source_port self.connect_callback = connect_callback self.disconnect_callback = disconnect_callback - self.io_loop = io_loop or salt.ext.tornado.ioloop.IOLoop.current() + self.io_loop = io_loop or tornado.ioloop.IOLoop.current() with salt.utils.asynchronous.current_ioloop(self.io_loop): self._tcp_client = TCPClientKeepAlive(opts, resolver=resolver) self._mid = 1 @@ -581,7 +582,7 @@ class MessageClient: self._on_recv = None self._closing = False self._closed = False - self._connecting_future = salt.ext.tornado.concurrent.Future() + self._connecting_future = tornado.concurrent.Future() self._stream_return_running = False self._stream = None @@ -596,9 +597,12 @@ class MessageClient: if self._closing: return self._closing = True - self.io_loop.add_timeout(1, self.check_close) + try: + self.io_loop.add_timeout(1, self.check_close) + except RuntimeError: + pass - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def check_close(self): if not self.send_future_map: self._tcp_client.close() @@ -606,7 +610,10 @@ class MessageClient: self._closing = False self._closed = True else: - self.io_loop.add_timeout(1, self.check_close) + try: + self.io_loop.add_timeout(1, self.check_close) + except RuntimeError: + pass # pylint: disable=W1701 def __del__(self): @@ -614,7 +621,7 @@ class MessageClient: # pylint: enable=W1701 - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def getstream(self, **kwargs): if self.source_ip or self.source_port: kwargs = { @@ -639,10 +646,10 @@ class MessageClient: exc, self.backoff, ) - yield salt.ext.tornado.gen.sleep(self.backoff) - raise salt.ext.tornado.gen.Return(stream) + yield tornado.gen.sleep(self.backoff) + raise tornado.gen.Return(stream) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def connect(self): if self._stream is None: self._stream = yield self.getstream() @@ -652,7 +659,7 @@ class MessageClient: if self.connect_callback: self.connect_callback(True) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _stream_return(self): self._stream_return_running = True unpacker = salt.utils.msgpack.Unpacker() @@ -678,7 +685,7 @@ class MessageClient: " tracking", message_id, ) - except salt.ext.tornado.iostream.StreamClosedError as e: + except tornado.iostream.StreamClosedError as e: log.debug( "tcp stream to %s:%s closed, unable to recv", self.host, @@ -764,14 +771,14 @@ class MessageClient: if future is not None: future.set_exception(SaltReqTimeoutError("Message timed out")) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def send(self, msg, timeout=None, callback=None, raw=False): if self._closing: raise ClosingError() message_id = self._message_id() header = {"mid": message_id} - future = salt.ext.tornado.concurrent.Future() + future = tornado.concurrent.Future() if callback is not None: @@ -791,7 +798,7 @@ class MessageClient: item = salt.transport.frame.frame_msg(msg, header=header) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _do_send(): yield self.connect() # If the _stream is None, we failed to connect. @@ -802,7 +809,7 @@ class MessageClient: # out before we are able to connect. self.io_loop.add_callback(_do_send) recv = yield future - raise salt.ext.tornado.gen.Return(recv) + raise tornado.gen.Return(recv) class Subscriber: @@ -839,7 +846,7 @@ class Subscriber: # pylint: enable=W1701 -class PubServer(salt.ext.tornado.tcpserver.TCPServer): +class PubServer(tornado.tcpserver.TCPServer): """ TCP publisher """ @@ -875,7 +882,7 @@ class PubServer(salt.ext.tornado.tcpserver.TCPServer): # pylint: enable=W1701 - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _stream_read(self, client): unpacker = salt.utils.msgpack.Unpacker() while not self._closing: @@ -888,7 +895,7 @@ class PubServer(salt.ext.tornado.tcpserver.TCPServer): body = framed_msg["body"] if self.presence_callback: self.presence_callback(client, body) - except salt.ext.tornado.iostream.StreamClosedError as e: + except tornado.iostream.StreamClosedError as e: log.debug("tcp stream to %s closed, unable to recv", client.address) client.close() self.remove_presence_callback(client) @@ -907,7 +914,7 @@ class PubServer(salt.ext.tornado.tcpserver.TCPServer): self.io_loop.spawn_callback(self._stream_read, client) # TODO: ACK the publish through IPC - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def publish_payload(self, package, topic_list=None): log.trace("TCP PubServer sending payload: %s \n\n %r", package, topic_list) payload = salt.transport.frame.frame_msg(package) @@ -922,7 +929,7 @@ class PubServer(salt.ext.tornado.tcpserver.TCPServer): yield client.stream.write(payload) sent = True # self.io_loop.add_future(f, lambda f: True) - except salt.ext.tornado.iostream.StreamClosedError: + except tornado.iostream.StreamClosedError: to_remove.append(client) if not sent: log.debug("Publish target %s not connected %r", topic, self.clients) @@ -931,7 +938,7 @@ class PubServer(salt.ext.tornado.tcpserver.TCPServer): try: # Write the packed str yield client.stream.write(payload) - except salt.ext.tornado.iostream.StreamClosedError: + except tornado.iostream.StreamClosedError: to_remove.append(client) for client in to_remove: log.debug( @@ -949,7 +956,7 @@ class TCPPublishServer(salt.transport.base.DaemonizedPublishServer): """ # TODO: opts! - # Based on default used in salt.ext.tornado.netutil.bind_sockets() + # Based on default used in tornado.netutil.bind_sockets() backlog = 128 def __init__(self, opts): @@ -975,7 +982,7 @@ class TCPPublishServer(salt.transport.base.DaemonizedPublishServer): """ Bind to the interface specified in the configuration file """ - io_loop = salt.ext.tornado.ioloop.IOLoop() + io_loop = tornado.ioloop.IOLoop() io_loop.make_current() # Spin up the publisher @@ -1027,10 +1034,10 @@ class TCPPublishServer(salt.transport.base.DaemonizedPublishServer): """ process_manager.add_process(self.publish_daemon, name=self.__class__.__name__) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def publish_payload(self, payload, *args): ret = yield self.pub_server.publish_payload(payload, *args) - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) def publish(self, payload, **kwargs): """ @@ -1080,14 +1087,14 @@ class TCPReqClient(salt.transport.base.RequestClient): source_port=opts.get("source_ret_port"), ) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def connect(self): yield self.message_client.connect() - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def send(self, load, timeout=60): ret = yield self.message_client.send(load, timeout=timeout) - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) def close(self): self.message_client.close() diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index 89f705190e0..730af297790 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -14,10 +14,10 @@ from random import randint import zmq.error import zmq.eventloop.zmqstream -import salt.ext.tornado -import salt.ext.tornado.concurrent -import salt.ext.tornado.gen -import salt.ext.tornado.ioloop +import tornado +import tornado.concurrent +import tornado.gen +import tornado.ioloop import salt.payload import salt.transport.base import salt.utils.files @@ -204,7 +204,7 @@ class PublishClient(salt.transport.base.PublishClient): self.close() # TODO: this is the time to see if we are connected, maybe use the req channel to guess? - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def connect(self, publish_port, connect_callback=None, disconnect_callback=None): self.publish_port = publish_port log.debug( @@ -233,7 +233,7 @@ class PublishClient(salt.transport.base.PublishClient): source_port=self.opts.get("source_publish_port"), ) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _decode_messages(self, messages): """ Take the zmq messages, decrypt/decode them into a payload @@ -255,7 +255,7 @@ class PublishClient(salt.transport.base.PublishClient): and message_target not in ("broadcast", "syndic") ): log.debug("Publish received for not this minion: %s", message_target) - raise salt.ext.tornado.gen.Return(None) + raise tornado.gen.Return(None) payload = salt.payload.loads(messages[1]) else: raise Exception( @@ -265,7 +265,7 @@ class PublishClient(salt.transport.base.PublishClient): ) # Yield control back to the caller. When the payload has been decoded, assign # the decoded payload to 'ret' and resume operation - raise salt.ext.tornado.gen.Return(payload) + raise tornado.gen.Return(payload) @property def stream(self): @@ -286,7 +286,7 @@ class PublishClient(salt.transport.base.PublishClient): """ return self.stream.on_recv(callback) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def send(self, msg): self.stream.send(msg, noblock=True) @@ -433,7 +433,7 @@ class RequestServer(salt.transport.base.DaemonizedRequestServer): self.message_handler = message_handler self.stream.on_recv_stream(self.handle_message) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def handle_message(self, stream, payload): try: payload = self.decode_payload(payload) @@ -515,7 +515,7 @@ class AsyncReqMessageClient: self.addr = addr self.linger = linger if io_loop is None: - self.io_loop = salt.ext.tornado.ioloop.IOLoop.current() + self.io_loop = tornado.ioloop.IOLoop.current() else: self.io_loop = io_loop @@ -598,12 +598,12 @@ class AsyncReqMessageClient: if future is not None: future.set_exception(SaltReqTimeoutError("Message timed out")) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def send(self, message, timeout=None, callback=None): """ Return a future which will be completed when the message has a response """ - future = salt.ext.tornado.concurrent.Future() + future = tornado.concurrent.Future() message = salt.payload.dumps(message) @@ -635,7 +635,7 @@ class AsyncReqMessageClient: self.stream.on_recv(mark_future) yield self.stream.send(message) recv = yield future - raise salt.ext.tornado.gen.Return(recv) + raise tornado.gen.Return(recv) class ZeroMQSocketMonitor: @@ -719,7 +719,7 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer): self.opts = opts def connect(self): - return salt.ext.tornado.gen.sleep(5) + return tornado.gen.sleep(5) def publish_daemon( self, @@ -731,7 +731,7 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer): This method represents the Publish Daemon process. It is intended to be run in a thread or process as it creates and runs an it's own ioloop. """ - ioloop = salt.ext.tornado.ioloop.IOLoop() + ioloop = tornado.ioloop.IOLoop() ioloop.make_current() self.io_loop = ioloop context = zmq.Context(1) @@ -767,7 +767,7 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer): with salt.utils.files.set_umask(0o177): pull_sock.bind(self.pull_uri) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def on_recv(packages): for package in packages: payload = salt.payload.loads(package) @@ -796,7 +796,7 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer): def pub_uri(self): return "tcp://{interface}:{publish_port}".format(**self.opts) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def publish_payload(self, payload, topic_list=None): payload = salt.payload.dumps(payload) if self.opts["zmq_filtering"]: @@ -929,11 +929,11 @@ class RequestClient(salt.transport.base.RequestClient): def connect(self): self.message_client.connect() - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def send(self, load, timeout=60): self.connect() ret = yield self.message_client.send(load, timeout=timeout) - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) def close(self): self.message_client.close() diff --git a/salt/utils/asynchronous.py b/salt/utils/asynchronous.py index 55a50cbcbff..0429bf9fcc2 100644 --- a/salt/utils/asynchronous.py +++ b/salt/utils/asynchronous.py @@ -2,11 +2,16 @@ Helpers/utils for working with tornado asynchronous stuff """ +import asyncio import contextlib import logging +import sys +import threading -import salt.ext.tornado.concurrent -import salt.ext.tornado.ioloop +import tornado.concurrent +import tornado.ioloop + +from salt import USE_VENDORED_TORNADO log = logging.getLogger(__name__) @@ -16,12 +21,24 @@ def current_ioloop(io_loop): """ A context manager that will set the current ioloop to io_loop for the context """ - orig_loop = salt.ext.tornado.ioloop.IOLoop.current() - io_loop.make_current() + try: + orig_loop = tornado.ioloop.IOLoop.current() + except RuntimeError: + orig_loop = None + if USE_VENDORED_TORNADO: + io_loop.make_current() + else: + asyncio.set_event_loop(io_loop.asyncio_loop) try: yield finally: - orig_loop.make_current() + if orig_loop: + if USE_VENDORED_TORNADO: + orig_loop.make_current() + else: + asyncio.set_event_loop(orig_loop.asyncio_loop) + else: + asyncio.set_event_loop(None) class SyncWrapper: @@ -31,7 +48,7 @@ class SyncWrapper: This is uses as a simple wrapper, for example: asynchronous = AsyncClass() - # this method would reguarly return a future + # this method would regularly return a future future = asynchronous.async_method() sync = SyncWrapper(async_factory_method, (arg1, arg2), {'kwarg1': 'val'}) @@ -48,7 +65,13 @@ class SyncWrapper: close_methods=None, loop_kwarg=None, ): - self.io_loop = salt.ext.tornado.ioloop.IOLoop() + if USE_VENDORED_TORNADO: + self.io_loop = tornado.ioloop.IOLoop() + else: + self.asyncio_loop = asyncio.new_event_loop() + self.io_loop = tornado.ioloop.IOLoop( + asyncio_loop=self.asyncio_loop, make_current=False + ) if args is None: args = [] if kwargs is None: @@ -61,7 +84,8 @@ class SyncWrapper: self.cls = cls if loop_kwarg: kwargs[self.loop_kwarg] = self.io_loop - self.obj = cls(*args, **kwargs) + with current_ioloop(self.io_loop): + self.obj = cls(*args, **kwargs) self._async_methods = list( set(async_methods + getattr(self.obj, "async_methods", [])) ) @@ -79,7 +103,7 @@ class SyncWrapper: self._async_methods += self.obj._coroutines def __repr__(self): - return "= 5 diff --git a/tests/pytests/integration/netapi/rest_tornado/test_minions_api_handler.py b/tests/pytests/integration/netapi/rest_tornado/test_minions_api_handler.py index 05146b45a45..fffee6cd8b4 100644 --- a/tests/pytests/integration/netapi/rest_tornado/test_minions_api_handler.py +++ b/tests/pytests/integration/netapi/rest_tornado/test_minions_api_handler.py @@ -1,7 +1,7 @@ import pytest import salt.utils.json -from salt.ext.tornado.httpclient import HTTPError +from tornado.httpclient import HTTPError from salt.netapi.rest_tornado import saltnado diff --git a/tests/pytests/integration/netapi/rest_tornado/test_root_handler.py b/tests/pytests/integration/netapi/rest_tornado/test_root_handler.py index 9dd4cbd6aab..7d93c1d6a17 100644 --- a/tests/pytests/integration/netapi/rest_tornado/test_root_handler.py +++ b/tests/pytests/integration/netapi/rest_tornado/test_root_handler.py @@ -1,7 +1,7 @@ import pytest import salt.utils.json -from salt.ext.tornado.httpclient import HTTPError +from tornado.httpclient import HTTPError from salt.netapi.rest_tornado import saltnado diff --git a/tests/pytests/unit/channel/test_server.py b/tests/pytests/unit/channel/test_server.py index 3f6262f89b1..806b82bbaaf 100644 --- a/tests/pytests/unit/channel/test_server.py +++ b/tests/pytests/unit/channel/test_server.py @@ -3,7 +3,7 @@ import time import pytest import salt.channel.server as server -import salt.ext.tornado.gen +import tornado.gen from tests.support.mock import MagicMock, patch @@ -19,7 +19,7 @@ def test__auth_cmd_stats_passing(master_opts): time.sleep(0.03) return fake_ret - future = salt.ext.tornado.gen.Future() + future = tornado.gen.Future() future.set_result({}) with patch.object(req_server_channel, "_auth", _auth_mock): diff --git a/tests/pytests/unit/cli/test_batch_async.py b/tests/pytests/unit/cli/test_batch_async.py index be8de692e64..cd73b8d883b 100644 --- a/tests/pytests/unit/cli/test_batch_async.py +++ b/tests/pytests/unit/cli/test_batch_async.py @@ -1,6 +1,6 @@ import pytest -import salt.ext.tornado +import tornado from salt.cli.batch_async import BatchAsync, batch_async_required from tests.support.mock import MagicMock, patch @@ -79,12 +79,12 @@ def test_batch_size(batch): def test_batch_start_on_batch_presence_ping_timeout(batch): - future_ret = salt.ext.tornado.gen.Future() + future_ret = tornado.gen.Future() future_ret.set_result({"minions": ["foo", "bar"]}) - future = salt.ext.tornado.gen.Future() + future = tornado.gen.Future() future.set_result({}) with patch.object(batch, "events_channel", MagicMock()), patch( - "salt.ext.tornado.gen.sleep", return_value=future + "tornado.gen.sleep", return_value=future ), patch.object(batch, "io_loop", MagicMock()), patch.object( batch, "start_batch", return_value=future ) as start_batch_mock: @@ -108,13 +108,13 @@ def test_batch_start_on_batch_presence_ping_timeout(batch): def test_batch_start_on_gather_job_timeout(batch): - future = salt.ext.tornado.gen.Future() + future = tornado.gen.Future() future.set_result({}) - future_ret = salt.ext.tornado.gen.Future() + future_ret = tornado.gen.Future() future_ret.set_result({"minions": ["foo", "bar"]}) batch.batch_presence_ping_timeout = None with patch.object(batch, "events_channel", MagicMock()), patch( - "salt.ext.tornado.gen.sleep", return_value=future + "tornado.gen.sleep", return_value=future ), patch.object(batch, "io_loop", MagicMock()), patch.object( batch, "start_batch", return_value=future ), patch.object( @@ -149,7 +149,7 @@ def test_batch_fire_start_event(batch): def test_start_batch_calls_next(batch): batch.initialized = False - future = salt.ext.tornado.gen.Future() + future = tornado.gen.Future() future.set_result({}) with patch.object(batch, "event", MagicMock()), patch.object( batch, "events_channel", MagicMock() @@ -195,9 +195,9 @@ def test_batch_next(batch): batch.opts["fun"] = "my.fun" batch.opts["arg"] = [] batch.batch_size = 2 - future = salt.ext.tornado.gen.Future() + future = tornado.gen.Future() future.set_result({}) - with patch("salt.ext.tornado.gen.sleep", return_value=future), patch.object( + with patch("tornado.gen.sleep", return_value=future), patch.object( batch, "events_channel", MagicMock() ), patch.object(batch, "_get_next", return_value={"foo", "bar"}), patch.object( batch, "find_job", return_value=future @@ -283,7 +283,7 @@ def test_batch__event_handler_ping_return(batch): def test_batch__event_handler_batch_run_return(batch): - future = salt.ext.tornado.gen.Future() + future = tornado.gen.Future() future.set_result({}) with patch.object(batch, "io_loop", MagicMock()), patch.object( batch, "schedule_next", return_value=future @@ -307,7 +307,7 @@ def test_batch__event_handler_find_job_return(batch): def test_batch_run_next_end_batch_when_no_next(batch): - future = salt.ext.tornado.gen.Future() + future = tornado.gen.Future() future.set_result({}) with patch.object(batch, "_get_next", return_value={}), patch.object( batch, "end_batch", return_value=future @@ -317,10 +317,10 @@ def test_batch_run_next_end_batch_when_no_next(batch): def test_batch_find_job(batch): - future = salt.ext.tornado.gen.Future() + future = tornado.gen.Future() future.set_result({}) batch.minions = {"foo", "bar"} - with patch("salt.ext.tornado.gen.sleep", return_value=future), patch.object( + with patch("tornado.gen.sleep", return_value=future), patch.object( batch, "check_find_job", return_value=future ) as check_find_job_mock, patch.object(batch, "jid_gen", return_value="1236"): batch.events_channel.local_client.run_job_async.return_value = future @@ -333,10 +333,10 @@ def test_batch_find_job(batch): def test_batch_find_job_with_done_minions(batch): batch.done_minions = {"bar"} - future = salt.ext.tornado.gen.Future() + future = tornado.gen.Future() future.set_result({}) batch.minions = {"foo", "bar"} - with patch("salt.ext.tornado.gen.sleep", return_value=future), patch.object( + with patch("tornado.gen.sleep", return_value=future), patch.object( batch, "check_find_job", return_value=future ) as check_find_job_mock, patch.object(batch, "jid_gen", return_value="1236"): batch.events_channel.local_client.run_job_async.return_value = future @@ -350,7 +350,7 @@ def test_batch_find_job_with_done_minions(batch): def test_batch_check_find_job_did_not_return(batch): batch.active = {"foo"} batch.find_job_returned = set() - future = salt.ext.tornado.gen.Future() + future = tornado.gen.Future() future.set_result({}) with patch.object(batch, "find_job", return_value=future) as find_job_mock: batch.check_find_job({"foo"}, jid="1234") @@ -361,7 +361,7 @@ def test_batch_check_find_job_did_not_return(batch): def test_batch_check_find_job_did_return(batch): batch.find_job_returned = {"foo"} - future = salt.ext.tornado.gen.Future() + future = tornado.gen.Future() future.set_result({}) with patch.object(batch, "find_job", return_value=future) as find_job_mock: batch.check_find_job({"foo"}, jid="1234") @@ -384,7 +384,7 @@ def test_batch_check_find_job_multiple_states(batch): # both not yet done but only 'foo' responded to find_job not_done = {"foo", "bar"} - future = salt.ext.tornado.gen.Future() + future = tornado.gen.Future() future.set_result({}) with patch.object(batch, "schedule_next", return_value=future), patch.object( @@ -403,7 +403,7 @@ def test_batch_check_find_job_multiple_states(batch): def test_only_on_run_next_is_scheduled(batch): - future = salt.ext.tornado.gen.Future() + future = tornado.gen.Future() future.set_result({}) batch.scheduled = True with patch.object(batch, "run_next", return_value=future) as run_next_mock: diff --git a/tests/pytests/unit/fileserver/gitfs/test_gitfs.py b/tests/pytests/unit/fileserver/gitfs/test_gitfs.py index 7896fd93bb7..18d3893a5d0 100644 --- a/tests/pytests/unit/fileserver/gitfs/test_gitfs.py +++ b/tests/pytests/unit/fileserver/gitfs/test_gitfs.py @@ -25,7 +25,7 @@ import pathlib import pytest -import salt.ext.tornado.ioloop +import tornado.ioloop import salt.fileserver.gitfs as gitfs import salt.utils.files import salt.utils.gitfs @@ -127,7 +127,7 @@ def testfile(tmp_path): def repo_dir(tmp_path, unicode_dirname, tag_name, unicode_filename): try: del salt.utils.gitfs.GitFS.instance_map[ - salt.ext.tornado.ioloop.IOLoop.current() + tornado.ioloop.IOLoop.current() ] except KeyError: pass diff --git a/tests/pytests/unit/fileserver/gitfs/test_gitfs_config.py b/tests/pytests/unit/fileserver/gitfs/test_gitfs_config.py index afd0cfdc9e2..ca502f716f3 100644 --- a/tests/pytests/unit/fileserver/gitfs/test_gitfs_config.py +++ b/tests/pytests/unit/fileserver/gitfs/test_gitfs_config.py @@ -2,7 +2,7 @@ import textwrap import pytest -import salt.ext.tornado.ioloop +import tornado.ioloop import salt.fileserver.gitfs as gitfs import salt.utils.files import salt.utils.gitfs @@ -74,7 +74,7 @@ def configure_loader_modules(tmp_path): def clear_instance_map(): try: del salt.utils.gitfs.GitFS.instance_map[ - salt.ext.tornado.ioloop.IOLoop.current() + tornado.ioloop.IOLoop.current() ] except KeyError: pass diff --git a/tests/pytests/unit/modules/test_linux_shadow.py b/tests/pytests/unit/modules/test_linux_shadow.py index 5d977fc5212..723e3a8ad26 100644 --- a/tests/pytests/unit/modules/test_linux_shadow.py +++ b/tests/pytests/unit/modules/test_linux_shadow.py @@ -1,6 +1,7 @@ """ - :codeauthor: Erik Johnson +:codeauthor: Erik Johnson """ + import types import pytest @@ -174,6 +175,11 @@ def test_info(password): Test if info shows the correct user information """ + data = { + "/etc/shadow": f"foo:{password.pw_hash}:31337:0:99999:7:::", + "*": Exception("Attempted to open something other than /etc/shadow"), + } + # First test is with a succesful call expected_result = [ ("expire", -1), @@ -185,10 +191,7 @@ def test_info(password): ("passwd", password.pw_hash), ("warn", 7), ] - getspnam_return = spwd.struct_spwd( - ["foo", password.pw_hash, 31337, 0, 99999, 7, -1, -1, -1] - ) - with patch("spwd.getspnam", return_value=getspnam_return): + with patch("salt.utils.files.fopen", mock_open(read_data=data)): result = shadow.info("foo") assert expected_result == sorted(result.items(), key=lambda x: x[0]) @@ -203,15 +206,8 @@ def test_info(password): ("passwd", ""), ("warn", ""), ] - # We get KeyError exception for non-existent users in glibc based systems - getspnam_return = KeyError - with patch("spwd.getspnam", side_effect=getspnam_return): - result = shadow.info("foo") - assert expected_result == sorted(result.items(), key=lambda x: x[0]) - # And FileNotFoundError in musl based systems - getspnam_return = FileNotFoundError - with patch("spwd.getspnam", side_effect=getspnam_return): - result = shadow.info("foo") + with patch("salt.utils.files.fopen", mock_open(read_data=data)): + result = shadow.info("bar") assert expected_result == sorted(result.items(), key=lambda x: x[0]) diff --git a/tests/pytests/unit/test_ext_importers.py b/tests/pytests/unit/test_ext_importers.py deleted file mode 100644 index e81ef234a92..00000000000 --- a/tests/pytests/unit/test_ext_importers.py +++ /dev/null @@ -1,51 +0,0 @@ -import logging -import os -import subprocess -import sys - -import pytest - -import salt - -log = logging.getLogger(__name__) - - -def test_tornado_import_override(tmp_path): - """ - Ensure we are not using any non vendor'ed tornado - """ - test_source = """ - from __future__ import absolute_import, print_function - import salt - import tornado - print(tornado.__name__) - """ - tornado_source = """ - foo = 'bar' - """ - with pytest.helpers.temp_file( - "test.py", directory=tmp_path, contents=test_source - ) as test_source_path, pytest.helpers.temp_file( - "tornado.py", directory=tmp_path, contents=tornado_source - ): - env = os.environ.copy() - env["PYTHONPATH"] = os.pathsep.join(sys.path) - ret = subprocess.run( - [sys.executable, str(test_source_path)], - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - env=env, - shell=False, - check=False, - universal_newlines=True, - ) - assert ret.returncode == 0 - assert ret.stdout.strip() == "salt.ext.tornado" - - -def test_regression_56063(): - importer = salt.TornadoImporter() - try: - importer.find_module("tornado") - except TypeError: - assert False, "TornadoImporter raised type error when one argument passed" diff --git a/tests/pytests/unit/test_minion.py b/tests/pytests/unit/test_minion.py index 017c28d1634..7c5c14c0182 100644 --- a/tests/pytests/unit/test_minion.py +++ b/tests/pytests/unit/test_minion.py @@ -4,9 +4,9 @@ import os import pytest -import salt.ext.tornado -import salt.ext.tornado.gen -import salt.ext.tornado.testing +import tornado +import tornado.gen +import tornado.testing import salt.minion import salt.syspaths import salt.utils.crypt @@ -57,7 +57,7 @@ def test_minion_load_grains_default(minion_opts): [ ( "salt.channel.client.AsyncReqChannel.factory", - lambda load, timeout, tries: salt.ext.tornado.gen.maybe_future(tries), + lambda load, timeout, tries: tornado.gen.maybe_future(tries), ), ( "salt.channel.client.ReqChannel.factory", @@ -261,7 +261,7 @@ def test_handle_decoded_payload_jid_match_in_jid_queue(minion_opts): minion = salt.minion.Minion( minion_opts, jid_queue=copy.copy(mock_jid_queue), - io_loop=salt.ext.tornado.ioloop.IOLoop(), + io_loop=tornado.ioloop.IOLoop(), ) try: ret = minion._handle_decoded_payload(mock_data).result() @@ -290,7 +290,7 @@ def test_handle_decoded_payload_jid_queue_addition(minion_opts): minion = salt.minion.Minion( minion_opts, jid_queue=copy.copy(mock_jid_queue), - io_loop=salt.ext.tornado.ioloop.IOLoop(), + io_loop=tornado.ioloop.IOLoop(), ) try: @@ -327,7 +327,7 @@ def test_handle_decoded_payload_jid_queue_reduced_minion_jid_queue_hwm(minion_op minion = salt.minion.Minion( minion_opts, jid_queue=copy.copy(mock_jid_queue), - io_loop=salt.ext.tornado.ioloop.IOLoop(), + io_loop=tornado.ioloop.IOLoop(), ) try: @@ -359,15 +359,15 @@ def test_process_count_max(minion_opts): ), patch( "salt.utils.minion.running", MagicMock(return_value=[]) ), patch( - "salt.ext.tornado.gen.sleep", - MagicMock(return_value=salt.ext.tornado.concurrent.Future()), + "tornado.gen.sleep", + MagicMock(return_value=tornado.concurrent.Future()), ): process_count_max = 10 minion_opts["__role"] = "minion" minion_opts["minion_jid_queue_hwm"] = 100 minion_opts["process_count_max"] = process_count_max - io_loop = salt.ext.tornado.ioloop.IOLoop() + io_loop = tornado.ioloop.IOLoop() minion = salt.minion.Minion(minion_opts, jid_queue=[], io_loop=io_loop) try: @@ -375,7 +375,7 @@ def test_process_count_max(minion_opts): class SleepCalledException(Exception): """Thrown when sleep is called""" - salt.ext.tornado.gen.sleep.return_value.set_exception( + tornado.gen.sleep.return_value.set_exception( SleepCalledException() ) @@ -425,7 +425,7 @@ def test_beacons_before_connect(minion_opts): MagicMock(return_value=True), ): minion_opts["beacons_before_connect"] = True - io_loop = salt.ext.tornado.ioloop.IOLoop() + io_loop = tornado.ioloop.IOLoop() io_loop.make_current() minion = salt.minion.Minion(minion_opts, io_loop=io_loop) try: @@ -458,7 +458,7 @@ def test_scheduler_before_connect(minion_opts): MagicMock(return_value=True), ): minion_opts["scheduler_before_connect"] = True - io_loop = salt.ext.tornado.ioloop.IOLoop() + io_loop = tornado.ioloop.IOLoop() io_loop.make_current() minion = salt.minion.Minion(minion_opts, io_loop=io_loop) try: @@ -489,7 +489,7 @@ def test_minion_module_refresh(minion_opts): try: minion = salt.minion.Minion( minion_opts, - io_loop=salt.ext.tornado.ioloop.IOLoop(), + io_loop=tornado.ioloop.IOLoop(), ) minion.schedule = salt.utils.schedule.Schedule( minion_opts, {}, returners={} @@ -517,7 +517,7 @@ def test_minion_module_refresh_beacons_refresh(minion_opts): try: minion = salt.minion.Minion( minion_opts, - io_loop=salt.ext.tornado.ioloop.IOLoop(), + io_loop=tornado.ioloop.IOLoop(), ) minion.schedule = salt.utils.schedule.Schedule( minion_opts, {}, returners={} @@ -547,7 +547,7 @@ def test_when_ping_interval_is_set_the_callback_should_be_added_to_periodic_call MagicMock(return_value=True), ): minion_opts["ping_interval"] = 10 - io_loop = salt.ext.tornado.ioloop.IOLoop() + io_loop = tornado.ioloop.IOLoop() io_loop.make_current() minion = salt.minion.Minion(minion_opts, io_loop=io_loop) try: @@ -569,7 +569,7 @@ def test_when_passed_start_event_grains(minion_opts): # provide mock opts an os grain since we'll look for it later. minion_opts["grains"]["os"] = "linux" minion_opts["start_event_grains"] = ["os"] - io_loop = salt.ext.tornado.ioloop.IOLoop() + io_loop = tornado.ioloop.IOLoop() io_loop.make_current() minion = salt.minion.Minion(minion_opts, io_loop=io_loop) try: @@ -588,7 +588,7 @@ def test_when_passed_start_event_grains(minion_opts): @pytest.mark.slow_test def test_when_not_passed_start_event_grains(minion_opts): - io_loop = salt.ext.tornado.ioloop.IOLoop() + io_loop = tornado.ioloop.IOLoop() io_loop.make_current() minion = salt.minion.Minion(minion_opts, io_loop=io_loop) try: @@ -605,7 +605,7 @@ def test_when_not_passed_start_event_grains(minion_opts): @pytest.mark.slow_test def test_when_other_events_fired_and_start_event_grains_are_set(minion_opts): minion_opts["start_event_grains"] = ["os"] - io_loop = salt.ext.tornado.ioloop.IOLoop() + io_loop = tornado.ioloop.IOLoop() io_loop.make_current() minion = salt.minion.Minion(minion_opts, io_loop=io_loop) try: @@ -643,7 +643,7 @@ def test_gen_modules_executors(minion_opts): """ Ensure gen_modules is called with the correct arguments #54429 """ - io_loop = salt.ext.tornado.ioloop.IOLoop() + io_loop = tornado.ioloop.IOLoop() io_loop.make_current() minion = salt.minion.Minion(minion_opts, io_loop=io_loop) @@ -669,7 +669,7 @@ def test_reinit_crypto_on_fork(minion_opts): minion_opts["multiprocessing"] = True with patch("salt.utils.process.default_signals"): - io_loop = salt.ext.tornado.ioloop.IOLoop() + io_loop = tornado.ioloop.IOLoop() io_loop.make_current() minion = salt.minion.Minion(minion_opts, io_loop=io_loop) @@ -710,7 +710,7 @@ def test_minion_manage_schedule(minion_opts): "salt.utils.process.SignalHandlingProcess.join", MagicMock(return_value=True), ): - io_loop = salt.ext.tornado.ioloop.IOLoop() + io_loop = tornado.ioloop.IOLoop() io_loop.make_current() with patch("salt.utils.schedule.clean_proc_dir", MagicMock(return_value=None)): @@ -770,7 +770,7 @@ def test_minion_manage_beacons(minion_opts): try: minion_opts["beacons"] = {} - io_loop = salt.ext.tornado.ioloop.IOLoop() + io_loop = tornado.ioloop.IOLoop() io_loop.make_current() mock_functions = {"test.ping": None} @@ -940,7 +940,7 @@ def test_minion_grains_refresh_pre_exec_false(minion_opts): minion = salt.minion.Minion( minion_opts, jid_queue=None, - io_loop=salt.ext.tornado.ioloop.IOLoop(), + io_loop=tornado.ioloop.IOLoop(), load_grains=False, ) try: @@ -963,7 +963,7 @@ def test_minion_grains_refresh_pre_exec_true(minion_opts): minion = salt.minion.Minion( minion_opts, jid_queue=None, - io_loop=salt.ext.tornado.ioloop.IOLoop(), + io_loop=tornado.ioloop.IOLoop(), load_grains=False, ) try: diff --git a/tests/pytests/unit/transport/test_ipc.py b/tests/pytests/unit/transport/test_ipc.py index 5a687836161..36a6e8f70b4 100644 --- a/tests/pytests/unit/transport/test_ipc.py +++ b/tests/pytests/unit/transport/test_ipc.py @@ -1,7 +1,7 @@ import pytest from pytestshellutils.utils import ports -import salt.ext.tornado.iostream +import tornado.iostream import salt.transport.ipc import salt.utils.asynchronous import salt.utils.platform @@ -31,6 +31,6 @@ async def test_ipc_connect_sync_wrapped(io_loop, tmp_path): kwargs={"io_loop": io_loop}, loop_kwarg="io_loop", ) - with pytest.raises(salt.ext.tornado.iostream.StreamClosedError): + with pytest.raises(tornado.iostream.StreamClosedError): # Don't `await subscriber.connect()`, that's the purpose of the SyncWrapper subscriber.connect() diff --git a/tests/pytests/unit/transport/test_tcp.py b/tests/pytests/unit/transport/test_tcp.py index bcfb71f5590..537395d04c5 100644 --- a/tests/pytests/unit/transport/test_tcp.py +++ b/tests/pytests/unit/transport/test_tcp.py @@ -8,7 +8,7 @@ from pytestshellutils.utils import ports import salt.channel.server import salt.exceptions -import salt.ext.tornado +import tornado import salt.transport.tcp from tests.support.mock import MagicMock, PropertyMock, patch @@ -31,9 +31,9 @@ def fake_crypto(): @pytest.fixture def fake_authd(): - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def return_nothing(): - raise salt.ext.tornado.gen.Return() + raise tornado.gen.Return() with patch( "salt.crypt.AsyncAuth.authenticated", new_callable=PropertyMock @@ -85,7 +85,7 @@ def test_message_client_cleanup_on_close(client_socket, temp_salt_master): """ test message client cleanup on close """ - orig_loop = salt.ext.tornado.ioloop.IOLoop() + orig_loop = tornado.ioloop.IOLoop() orig_loop.make_current() opts = dict(temp_salt_master.config.copy(), transport="tcp") @@ -241,7 +241,7 @@ def test_tcp_pub_server_channel_publish_filtering_str_list(temp_salt_master): @pytest.fixture(scope="function") def salt_message_client(): - io_loop_mock = MagicMock(spec=salt.ext.tornado.ioloop.IOLoop) + io_loop_mock = MagicMock(spec=tornado.ioloop.IOLoop) io_loop_mock.call_later.side_effect = lambda *args, **kwargs: (args, kwargs) client = salt.transport.tcp.MessageClient( @@ -346,7 +346,7 @@ def test_timeout_message_unknown_future(salt_message_client): # if we do have the actual future stored under the id, but it's none # we shouldn't fail as well message_id = 1 - future = salt.ext.tornado.concurrent.Future() + future = tornado.concurrent.Future() future.attempts = 1 future.tries = 1 salt_message_client.send_future_map[message_id] = future @@ -367,16 +367,16 @@ def xtest_client_reconnect_backoff(client_socket): client.close() assert t == 5 return - # return salt.ext.tornado.gen.sleep() + # return tornado.gen.sleep() - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def connect(*args, **kwargs): raise Exception("err") client._tcp_client.connect = connect try: - with patch("salt.ext.tornado.gen.sleep", side_effect=_sleep): + with patch("tornado.gen.sleep", side_effect=_sleep): client.io_loop.run_sync(client.connect) finally: client.close() @@ -456,7 +456,7 @@ def test_presence_events_callback_passed(temp_salt_master, salt_message_client): def test_presence_removed_on_stream_closed(): opts = {"presence_events": True} - io_loop_mock = MagicMock(spec=salt.ext.tornado.ioloop.IOLoop) + io_loop_mock = MagicMock(spec=tornado.ioloop.IOLoop) with patch("salt.master.AESFuncs.__init__", return_value=None): server = salt.transport.tcp.PubServer(opts, io_loop=io_loop_mock) @@ -464,12 +464,12 @@ def test_presence_removed_on_stream_closed(): server.remove_presence_callback = MagicMock() client = salt.transport.tcp.Subscriber( - salt.ext.tornado.iostream.IOStream, "1.2.3.4" + tornado.iostream.IOStream, "1.2.3.4" ) client._closing = True server.clients = {client} - io_loop = salt.ext.tornado.ioloop.IOLoop.current() + io_loop = tornado.ioloop.IOLoop.current() package = { "topic_lst": [], "payload": "test-payload", @@ -477,8 +477,8 @@ def test_presence_removed_on_stream_closed(): with patch("salt.transport.frame.frame_msg", return_value="framed-payload"): with patch( - "salt.ext.tornado.iostream.BaseIOStream.write", - side_effect=salt.ext.tornado.iostream.StreamClosedError(), + "tornado.iostream.BaseIOStream.write", + side_effect=tornado.iostream.StreamClosedError(), ): io_loop.run_sync(functools.partial(server.publish_payload, package, None)) diff --git a/tests/pytests/unit/transport/test_zeromq.py b/tests/pytests/unit/transport/test_zeromq.py index fd0e4b19772..95216cc9c48 100644 --- a/tests/pytests/unit/transport/test_zeromq.py +++ b/tests/pytests/unit/transport/test_zeromq.py @@ -19,8 +19,8 @@ import salt.channel.server import salt.config import salt.crypt import salt.exceptions -import salt.ext.tornado.gen -import salt.ext.tornado.ioloop +import tornado.gen +import tornado.ioloop import salt.transport.zeromq import salt.utils.platform import salt.utils.process @@ -344,14 +344,14 @@ def run_loop_in_thread(loop, evt): """ loop.make_current() - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def stopper(): - yield salt.ext.tornado.gen.sleep(0.1) + yield tornado.gen.sleep(0.1) while True: if not evt.is_set(): loop.stop() break - yield salt.ext.tornado.gen.sleep(0.3) + yield tornado.gen.sleep(0.3) loop.add_callback(evt.set) loop.add_callback(stopper) @@ -383,7 +383,7 @@ class MockSaltMinionMaster: self.server_channel = salt.channel.server.ReqServerChannel.factory(master_opts) self.server_channel.pre_fork(self.process_manager) - self.io_loop = salt.ext.tornado.ioloop.IOLoop() + self.io_loop = tornado.ioloop.IOLoop() self.evt = threading.Event() self.server_channel.post_fork(self._handle_payload, io_loop=self.io_loop) self.server_thread = threading.Thread( @@ -426,13 +426,13 @@ class MockSaltMinionMaster: # pylint: enable=W1701 @classmethod - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _handle_payload(cls, payload): """ TODO: something besides echo """ cls.mock._handle_payload_hook() - raise salt.ext.tornado.gen.Return((payload, {"fun": "send_clear"})) + raise tornado.gen.Return((payload, {"fun": "send_clear"})) @pytest.mark.parametrize("message", ["", [], ()]) @@ -462,7 +462,7 @@ def test_serverside_exception(temp_salt_minion, temp_salt_master): """ with MockSaltMinionMaster(temp_salt_minion, temp_salt_master) as minion_master: with patch.object(minion_master.mock, "_handle_payload_hook") as _mock: - _mock.side_effect = salt.ext.tornado.gen.Return(({}, {"fun": "madeup-fun"})) + _mock.side_effect = tornado.gen.Return(({}, {"fun": "madeup-fun"})) ret = minion_master.channel.send({}, timeout=5, tries=1) assert ret == "Server-side exception handling payload" @@ -485,7 +485,7 @@ def test_zeromq_async_pub_channel_publish_port(temp_salt_master): sign_pub_messages=False, ) opts["master_uri"] = "tcp://{interface}:{publish_port}".format(**opts) - ioloop = salt.ext.tornado.ioloop.IOLoop() + ioloop = tornado.ioloop.IOLoop() transport = salt.transport.zeromq.PublishClient(opts, ioloop) with transport: patch_socket = MagicMock(return_value=True) @@ -527,7 +527,7 @@ def test_zeromq_async_pub_channel_filtering_decode_message_no_match( ) opts["master_uri"] = "tcp://{interface}:{publish_port}".format(**opts) - ioloop = salt.ext.tornado.ioloop.IOLoop() + ioloop = tornado.ioloop.IOLoop() channel = salt.transport.zeromq.PublishClient(opts, ioloop) with channel: with patch( @@ -574,7 +574,7 @@ def test_zeromq_async_pub_channel_filtering_decode_message( ) opts["master_uri"] = "tcp://{interface}:{publish_port}".format(**opts) - ioloop = salt.ext.tornado.ioloop.IOLoop() + ioloop = tornado.ioloop.IOLoop() channel = salt.transport.zeromq.PublishClient(opts, ioloop) with channel: with patch( @@ -587,7 +587,7 @@ def test_zeromq_async_pub_channel_filtering_decode_message( def test_req_server_chan_encrypt_v2(pki_dir, master_opts): - loop = salt.ext.tornado.ioloop.IOLoop.current() + loop = tornado.ioloop.IOLoop.current() master_opts.update( { "worker_threads": 1, @@ -633,7 +633,7 @@ def test_req_server_chan_encrypt_v2(pki_dir, master_opts): def test_req_server_chan_encrypt_v1(pki_dir, master_opts): - loop = salt.ext.tornado.ioloop.IOLoop.current() + loop = tornado.ioloop.IOLoop.current() master_opts.update( { "worker_threads": 1, @@ -747,14 +747,14 @@ async def test_req_chan_decode_data_dict_entry_v2(pki_dir, master_opts, minion_o client.auth.session_crypticle.loads = auth.session_crypticle.loads client.transport = MagicMock() - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def mocksend(msg, timeout=60, tries=3): client.transport.msg = msg load = client.auth.session_crypticle.loads(msg["load"]) ret = server._encrypt_private( pillar_data, dictkey, target, nonce=load["nonce"], sign_messages=True ) - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) client.transport.send = mocksend @@ -819,10 +819,10 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_nonce(pki_dir, master_opts pillar_data, dictkey, target, nonce=badnonce, sign_messages=True ) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def mocksend(msg, timeout=60, tries=3): client.transport.msg = msg - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) client.transport.send = mocksend @@ -890,7 +890,7 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_signature(pki_dir, master_ client.auth.session_crypticle.loads = auth.session_crypticle.loads client.transport = MagicMock() - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def mocksend(msg, timeout=60, tries=3): client.transport.msg = msg load = client.auth.session_crypticle.loads(msg["load"]) @@ -912,7 +912,7 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_signature(pki_dir, master_ data["pillar"] = {"pillar1": "bar"} signed_msg["data"] = salt.payload.dumps(data) ret[dictkey] = pcrypt.dumps(signed_msg) - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) client.transport.send = mocksend @@ -980,7 +980,7 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_key(pki_dir, master_opts, client.auth.session_crypticle.loads = auth.session_crypticle.loads client.transport = MagicMock() - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def mocksend(msg, timeout=60, tries=3): client.transport.msg = msg load = client.auth.session_crypticle.loads(msg["load"]) @@ -1009,7 +1009,7 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_key(pki_dir, master_opts, else: cipher = PKCS1_OAEP.new(pub) ret["key"] = cipher.encrypt(key) - raise salt.ext.tornado.gen.Return(ret) + raise tornado.gen.Return(ret) client.transport.send = mocksend diff --git a/tests/pytests/unit/utils/event/test_event.py b/tests/pytests/unit/utils/event/test_event.py index fa9e420a933..3b261f95c8d 100644 --- a/tests/pytests/unit/utils/event/test_event.py +++ b/tests/pytests/unit/utils/event/test_event.py @@ -8,8 +8,8 @@ import pytest import zmq.eventloop.ioloop import salt.config -import salt.ext.tornado.ioloop -import salt.ext.tornado.iostream +import tornado.ioloop +import tornado.iostream import salt.utils.event import salt.utils.stringutils from salt.exceptions import SaltDeserializationError @@ -304,12 +304,12 @@ def test_connect_pull_should_debug_log_on_StreamClosedError(): salt.utils.event.log, "debug", autospec=True ) as mock_log_debug: mock_pusher.connect.side_effect = ( - salt.ext.tornado.iostream.StreamClosedError + tornado.iostream.StreamClosedError ) event.connect_pull() call = mock_log_debug.mock_calls[0] assert call.args[0] == "Unable to connect pusher: %s" - assert isinstance(call.args[1], salt.ext.tornado.iostream.StreamClosedError) + assert isinstance(call.args[1], tornado.iostream.StreamClosedError) assert call.args[1].args[0] == "Stream is closed" @@ -329,7 +329,7 @@ def test_connect_pull_should_error_log_on_other_errors(error): call = mock_log_error.mock_calls[0] assert call.args[0] == "Unable to connect pusher: %s" assert not isinstance( - call.args[1], salt.ext.tornado.iostream.StreamClosedError + call.args[1], tornado.iostream.StreamClosedError ) @@ -456,7 +456,7 @@ def test_event_single_timeout_tries(sock_dir): write_calls_count = 0 real_stream_write = None - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def write_mock(pack): nonlocal write_calls_count nonlocal real_stream_write @@ -464,7 +464,7 @@ def test_event_single_timeout_tries(sock_dir): if write_calls_count > 3: yield real_stream_write(pack) else: - raise salt.ext.tornado.iostream.StreamClosedError() + raise tornado.iostream.StreamClosedError() with eventpublisher_process(str(sock_dir)), salt.utils.event.MasterEvent( str(sock_dir), listen=True @@ -480,7 +480,7 @@ def test_event_single_timeout_tries(sock_dir): ), patch.object( me.pusher, "connect", - side_effect=salt.ext.tornado.iostream.StreamClosedError, + side_effect=tornado.iostream.StreamClosedError, ), patch.object( me.pusher.stream, "write", diff --git a/tests/pytests/unit/utils/event/test_event_return.py b/tests/pytests/unit/utils/event/test_event_return.py index e9548c701f4..67cee0d1350 100644 --- a/tests/pytests/unit/utils/event/test_event_return.py +++ b/tests/pytests/unit/utils/event/test_event_return.py @@ -1,7 +1,7 @@ import pytest from pytestshellutils.utils.processes import terminate_process -import salt.ext.tornado.ioloop +import tornado.ioloop import salt.utils.event import salt.utils.stringutils diff --git a/tests/pytests/unit/utils/test_asynchronous.py b/tests/pytests/unit/utils/test_asynchronous.py index 2b5613e2bfe..b8b86d2ce03 100644 --- a/tests/pytests/unit/utils/test_asynchronous.py +++ b/tests/pytests/unit/utils/test_asynchronous.py @@ -1,8 +1,12 @@ +import asyncio + import tornado.gen import tornado.ioloop import salt.utils.asynchronous as asynchronous +from salt import USE_VENDORED_TORNADO + class HelperA: @@ -41,7 +45,11 @@ def test_helpers(): """ Test that the helper classes do what we expect within a regular asynchronous env """ - io_loop = tornado.ioloop.IOLoop(make_current=False) + if USE_VENDORED_TORNADO: + io_loop = tornado.ioloop.IOLoop(make_current=False) + else: + asyncio_loop = asyncio.new_event_loop() + io_loop = tornado.ioloop.IOLoop(asyncio_loop=asyncio_loop, make_current=False) ret = io_loop.run_sync(lambda: HelperA().sleep()) assert ret is True @@ -89,4 +97,4 @@ def test_double_sameloop(): a = asynchronous.SyncWrapper(HelperA) sync = asynchronous.SyncWrapper(HelperB, (a,)) ret = sync.sleep() - assert ret is False + assert ret is False \ No newline at end of file diff --git a/tests/support/helpers.py b/tests/support/helpers.py index 3556e08853b..5dc4041ff4d 100644 --- a/tests/support/helpers.py +++ b/tests/support/helpers.py @@ -37,8 +37,8 @@ from pytestshellutils.exceptions import ProcessFailed from pytestshellutils.utils import ports from pytestshellutils.utils.processes import ProcessResult -import salt.ext.tornado.ioloop -import salt.ext.tornado.web +import tornado.ioloop +import tornado.web import salt.utils.files import salt.utils.platform import salt.utils.pycrypto @@ -1277,7 +1277,7 @@ def http_basic_auth(login_cb=lambda username, password: False): .. code-block:: python @http_basic_auth(lambda u, p: u == 'foo' and p == 'bar') - class AuthenticatedHandler(salt.ext.tornado.web.RequestHandler): + class AuthenticatedHandler(tornado.web.RequestHandler): pass """ @@ -1422,7 +1422,7 @@ class Webserver: self.port = port self.wait = wait self.handler = ( - handler if handler is not None else salt.ext.tornado.web.StaticFileHandler + handler if handler is not None else tornado.web.StaticFileHandler ) self.web_root = None self.ssl_opts = ssl_opts @@ -1431,14 +1431,14 @@ class Webserver: """ Threading target which stands up the tornado application """ - self.ioloop = salt.ext.tornado.ioloop.IOLoop() + self.ioloop = tornado.ioloop.IOLoop() self.ioloop.make_current() - if self.handler == salt.ext.tornado.web.StaticFileHandler: - self.application = salt.ext.tornado.web.Application( + if self.handler == tornado.web.StaticFileHandler: + self.application = tornado.web.Application( [(r"/(.*)", self.handler, {"path": self.root})] ) else: - self.application = salt.ext.tornado.web.Application( + self.application = tornado.web.Application( [(r"/(.*)", self.handler)] ) self.application.listen(self.port, ssl_options=self.ssl_opts) @@ -1515,7 +1515,7 @@ class Webserver: self.stop() -class SaveRequestsPostHandler(salt.ext.tornado.web.RequestHandler): +class SaveRequestsPostHandler(tornado.web.RequestHandler): """ Save all requests sent to the server. """ @@ -1535,7 +1535,7 @@ class SaveRequestsPostHandler(salt.ext.tornado.web.RequestHandler): raise NotImplementedError() -class MirrorPostHandler(salt.ext.tornado.web.RequestHandler): +class MirrorPostHandler(tornado.web.RequestHandler): """ Mirror a POST body back to the client """ diff --git a/tests/support/netapi.py b/tests/support/netapi.py index 91fbad5f1a4..5095a5998d4 100644 --- a/tests/support/netapi.py +++ b/tests/support/netapi.py @@ -4,12 +4,12 @@ import socket import attr import salt.auth -import salt.ext.tornado.escape -import salt.ext.tornado.web -from salt.ext.tornado import netutil -from salt.ext.tornado.httpclient import AsyncHTTPClient, HTTPError -from salt.ext.tornado.httpserver import HTTPServer -from salt.ext.tornado.ioloop import TimeoutError as IOLoopTimeoutError +import tornado.escape +import tornado.web +from tornado import netutil +from tornado.httpclient import AsyncHTTPClient, HTTPError +from tornado.httpserver import HTTPServer +from tornado.ioloop import TimeoutError as IOLoopTimeoutError from salt.netapi.rest_tornado import saltnado log = logging.getLogger(__name__) @@ -46,7 +46,7 @@ class TestsHttpClient: if response.headers.get("Content-Type") == "application/json": response._body = response.body.decode("utf-8") else: - response._body = salt.ext.tornado.escape.native_str(response.body) + response._body = tornado.escape.native_str(response.body) return response @@ -115,7 +115,7 @@ def auth_token(load_auth, auth_creds): def build_tornado_app( urls, load_auth, client_config, minion_config, setup_event_listener=False ): - application = salt.ext.tornado.web.Application(urls, debug=True) + application = tornado.web.Application(urls, debug=True) application.auth = load_auth application.opts = client_config diff --git a/tests/support/pytest/transport.py b/tests/support/pytest/transport.py index eaa8adc8bd4..592cbd05d46 100644 --- a/tests/support/pytest/transport.py +++ b/tests/support/pytest/transport.py @@ -10,9 +10,9 @@ from pytestshellutils.utils.processes import terminate_process import salt.channel.server import salt.exceptions -import salt.ext.tornado.gen -import salt.ext.tornado.ioloop -import salt.ext.tornado.iostream +import tornado.gen +import tornado.ioloop +import tornado.iostream import salt.master import salt.utils.msgpack import salt.utils.process @@ -93,9 +93,9 @@ class Collector(salt.utils.process.SignalHandlingProcess): time.sleep(1) else: break - self.sock = salt.ext.tornado.iostream.IOStream(sock) + self.sock = tornado.iostream.IOStream(sock) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _recv(self): if self.transport == "zeromq": # test_zeromq_filtering requires catching the @@ -103,19 +103,19 @@ class Collector(salt.utils.process.SignalHandlingProcess): try: payload = self.sock.recv(zmq.NOBLOCK) serial_payload = salt.payload.loads(payload) - raise salt.ext.tornado.gen.Return(serial_payload) + raise tornado.gen.Return(serial_payload) except (zmq.ZMQError, salt.exceptions.SaltDeserializationError): raise RecvError("ZMQ Error") else: for msg in self.unpacker: - raise salt.ext.tornado.gen.Return(msg["body"]) + raise tornado.gen.Return(msg["body"]) byts = yield self.sock.read_bytes(8096, partial=True) self.unpacker.feed(byts) for msg in self.unpacker: - raise salt.ext.tornado.gen.Return(msg["body"]) + raise tornado.gen.Return(msg["body"]) raise RecvError("TCP Error") - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def _run(self, loop): try: self._setup_listener() @@ -165,7 +165,7 @@ class Collector(salt.utils.process.SignalHandlingProcess): Gather results until then number of seconds specified by timeout passes without receiving a message """ - loop = salt.ext.tornado.ioloop.IOLoop() + loop = tornado.ioloop.IOLoop() loop.add_callback(self._run, loop) loop.start() diff --git a/tests/unit/modules/test_random_org.py b/tests/unit/modules/test_random_org.py index 30e98ca0802..2c3340c30e8 100644 --- a/tests/unit/modules/test_random_org.py +++ b/tests/unit/modules/test_random_org.py @@ -5,7 +5,7 @@ import pytest import salt.modules.random_org as random_org -from salt.ext.tornado.httpclient import HTTPClient +from tornado.httpclient import HTTPClient from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import TestCase diff --git a/tests/unit/netapi/rest_tornado/test_saltnado.py b/tests/unit/netapi/rest_tornado/test_saltnado.py index c4758e700ab..43367df13f9 100644 --- a/tests/unit/netapi/rest_tornado/test_saltnado.py +++ b/tests/unit/netapi/rest_tornado/test_saltnado.py @@ -1,10 +1,10 @@ -import salt.ext.tornado -import salt.ext.tornado.testing +import tornado +import tornado.testing import salt.netapi.rest_tornado.saltnado as saltnado from tests.support.mock import MagicMock, patch -class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase): +class TestJobNotRunning(tornado.testing.AsyncTestCase): def setUp(self): super().setUp() self.mock = MagicMock() @@ -23,11 +23,11 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase): self.handler.lowstate = [] self.handler.content_type = "text/plain" self.handler.dumper = lambda x: x - f = salt.ext.tornado.gen.Future() + f = tornado.gen.Future() f.set_result({"jid": f, "minions": []}) self.handler.saltclients.update({"local": lambda *args, **kwargs: f}) - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_disbatch_has_already_finished_then_writing_return_should_not_fail( self, ): @@ -37,7 +37,7 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase): # Asserting that it doesn't raise anything is... the default behavior # for a test. - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_disbatch_has_already_finished_then_finishing_should_not_fail(self): self.handler.finish() result = yield self.handler.disbatch() @@ -45,12 +45,12 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase): # Asserting that it doesn't raise anything is... the default behavior # for a test. - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_event_times_out_and_minion_is_not_running_result_should_be_True(self): - fut = salt.ext.tornado.gen.Future() + fut = tornado.gen.Future() fut.set_exception(saltnado.TimeoutException()) self.mock.event_listener.get_event.return_value = fut - wrong_future = salt.ext.tornado.gen.Future() + wrong_future = tornado.gen.Future() result = yield self.handler.job_not_running( jid=42, tgt="*", tgt_type="glob", minions=[], is_finished=wrong_future @@ -58,14 +58,14 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase): self.assertTrue(result) - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_event_times_out_and_minion_is_not_running_minion_data_should_not_be_set( self, ): - fut = salt.ext.tornado.gen.Future() + fut = tornado.gen.Future() fut.set_exception(saltnado.TimeoutException()) self.mock.event_listener.get_event.return_value = fut - wrong_future = salt.ext.tornado.gen.Future() + wrong_future = tornado.gen.Future() minions = {} result = yield self.handler.job_not_running( @@ -74,20 +74,20 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase): assert not minions - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_event_finally_finishes_and_returned_minion_not_in_minions_it_should_be_set_to_False( self, ): expected_id = 42 - no_data_event = salt.ext.tornado.gen.Future() + no_data_event = tornado.gen.Future() no_data_event.set_result({"data": {}}) - empty_return_event = salt.ext.tornado.gen.Future() + empty_return_event = tornado.gen.Future() empty_return_event.set_result({"data": {"return": {}}}) - actual_return_event = salt.ext.tornado.gen.Future() + actual_return_event = tornado.gen.Future() actual_return_event.set_result( {"data": {"return": {"something happened here": "OK?"}, "id": expected_id}} ) - timed_out_event = salt.ext.tornado.gen.Future() + timed_out_event = tornado.gen.Future() timed_out_event.set_exception(saltnado.TimeoutException()) self.mock.event_listener.get_event.side_effect = [ no_data_event, @@ -103,27 +103,27 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase): tgt="*", tgt_type="fnord", minions=minions, - is_finished=salt.ext.tornado.gen.Future(), + is_finished=tornado.gen.Future(), ) self.assertFalse(minions[expected_id]) - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_event_finally_finishes_and_returned_minion_already_in_minions_it_should_not_be_changed( self, ): expected_id = 42 expected_value = object() minions = {expected_id: expected_value} - no_data_event = salt.ext.tornado.gen.Future() + no_data_event = tornado.gen.Future() no_data_event.set_result({"data": {}}) - empty_return_event = salt.ext.tornado.gen.Future() + empty_return_event = tornado.gen.Future() empty_return_event.set_result({"data": {"return": {}}}) - actual_return_event = salt.ext.tornado.gen.Future() + actual_return_event = tornado.gen.Future() actual_return_event.set_result( {"data": {"return": {"something happened here": "OK?"}, "id": expected_id}} ) - timed_out_event = salt.ext.tornado.gen.Future() + timed_out_event = tornado.gen.Future() timed_out_event.set_exception(saltnado.TimeoutException()) self.mock.event_listener.get_event.side_effect = [ no_data_event, @@ -138,22 +138,22 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase): tgt="*", tgt_type="fnord", minions=minions, - is_finished=salt.ext.tornado.gen.Future(), + is_finished=tornado.gen.Future(), ) self.assertIs(minions[expected_id], expected_value) - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_event_returns_early_and_finally_times_out_result_should_be_True(self): - no_data_event = salt.ext.tornado.gen.Future() + no_data_event = tornado.gen.Future() no_data_event.set_result({"data": {}}) - empty_return_event = salt.ext.tornado.gen.Future() + empty_return_event = tornado.gen.Future() empty_return_event.set_result({"data": {"return": {}}}) - actual_return_event = salt.ext.tornado.gen.Future() + actual_return_event = tornado.gen.Future() actual_return_event.set_result( {"data": {"return": {"something happened here": "OK?"}, "id": "fnord"}} ) - timed_out_event = salt.ext.tornado.gen.Future() + timed_out_event = tornado.gen.Future() timed_out_event.set_exception(saltnado.TimeoutException()) self.mock.event_listener.get_event.side_effect = [ no_data_event, @@ -168,21 +168,21 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase): tgt="*", tgt_type="fnord", minions={}, - is_finished=salt.ext.tornado.gen.Future(), + is_finished=tornado.gen.Future(), ) self.assertTrue(result) - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_event_finishes_but_is_finished_is_done_then_result_should_be_True( self, ): expected_minion_id = "fnord" expected_minion_value = object() - no_data_event = salt.ext.tornado.gen.Future() + no_data_event = tornado.gen.Future() no_data_event.set_result({"data": {}}) - empty_return_event = salt.ext.tornado.gen.Future() + empty_return_event = tornado.gen.Future() empty_return_event.set_result({"data": {"return": {}}}) - actual_return_event = salt.ext.tornado.gen.Future() + actual_return_event = tornado.gen.Future() actual_return_event.set_result( { "data": { @@ -191,11 +191,11 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase): } } ) - is_finished = salt.ext.tornado.gen.Future() + is_finished = tornado.gen.Future() def abort(*args, **kwargs): yield actual_return_event - f = salt.ext.tornado.gen.Future() + f = tornado.gen.Future() f.set_exception(saltnado.TimeoutException()) is_finished.set_result("This is done") yield f @@ -218,14 +218,14 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase): self.assertTrue(len(minions) == 1, str(minions)) self.assertIs(minions[expected_minion_id], expected_minion_value) - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_is_finished_times_out_before_event_finishes_result_should_be_True( self, ): # Other test times out with event - this one should time out for is_finished - finished = salt.ext.tornado.gen.Future() + finished = tornado.gen.Future() finished.set_exception(saltnado.TimeoutException()) - wrong_future = salt.ext.tornado.gen.Future() + wrong_future = tornado.gen.Future() self.mock.event_listener.get_event.return_value = wrong_future result = yield self.handler.job_not_running( @@ -234,13 +234,13 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase): self.assertTrue(result) - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_is_finished_times_out_before_event_finishes_event_should_have_result_set_to_None( self, ): - finished = salt.ext.tornado.gen.Future() + finished = tornado.gen.Future() finished.set_exception(saltnado.TimeoutException()) - wrong_future = salt.ext.tornado.gen.Future() + wrong_future = tornado.gen.Future() self.mock.event_listener.get_event.return_value = wrong_future result = yield self.handler.job_not_running( @@ -251,7 +251,7 @@ class TestJobNotRunning(salt.ext.tornado.testing.AsyncTestCase): # TODO: I think we can extract seUp into a superclass -W. Werner, 2020-11-03 -class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): +class TestGetMinionReturns(tornado.testing.AsyncTestCase): def setUp(self): super().setUp() self.mock = MagicMock() @@ -265,22 +265,22 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): "gather_job_timeout": 10.001, } self.handler = saltnado.SaltAPIHandler(self.mock, self.mock) - f = salt.ext.tornado.gen.Future() + f = tornado.gen.Future() f.set_result({"jid": f, "minions": []}) - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_if_finished_before_any_events_return_then_result_should_be_empty_dictionary( self, ): expected_result = {} - xxx = salt.ext.tornado.gen.Future() + xxx = tornado.gen.Future() xxx.set_result(None) - is_finished = salt.ext.tornado.gen.Future() + is_finished = tornado.gen.Future() is_finished.set_result(None) actual_result = yield self.handler.get_minion_returns( events=[], is_finished=is_finished, - is_timed_out=salt.ext.tornado.gen.Future(), + is_timed_out=tornado.gen.Future(), min_wait_time=xxx, minions={}, ) @@ -288,7 +288,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): # TODO: Copy above - test with timed out -W. Werner, 2020-11-05 - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_if_is_finished_after_events_return_then_result_should_contain_event_result_data( self, ): @@ -296,15 +296,15 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): "minion1": {"fnord": "this is some fnordish data"}, "minion2": {"fnord": "this is some other fnordish data"}, } - xxx = salt.ext.tornado.gen.Future() + xxx = tornado.gen.Future() xxx.set_result(None) - is_finished = salt.ext.tornado.gen.Future() + is_finished = tornado.gen.Future() # XXX what do I do here? events = [ - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), ] events[0].set_result( { @@ -323,7 +323,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): actual_result = yield self.handler.get_minion_returns( events=events, is_finished=is_finished, - is_timed_out=salt.ext.tornado.gen.Future(), + is_timed_out=tornado.gen.Future(), min_wait_time=xxx, minions={ "minion1": False, @@ -334,7 +334,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): assert actual_result == expected_result - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_if_timed_out_after_events_return_then_result_should_contain_event_result_data( self, ): @@ -342,15 +342,15 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): "minion1": {"fnord": "this is some fnordish data"}, "minion2": {"fnord": "this is some other fnordish data"}, } - xxx = salt.ext.tornado.gen.Future() + xxx = tornado.gen.Future() xxx.set_result(None) - is_timed_out = salt.ext.tornado.gen.Future() + is_timed_out = tornado.gen.Future() # XXX what do I do here? events = [ - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), ] events[0].set_result( { @@ -368,7 +368,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): actual_result = yield self.handler.get_minion_returns( events=events, - is_finished=salt.ext.tornado.gen.Future(), + is_finished=tornado.gen.Future(), is_timed_out=is_timed_out, min_wait_time=xxx, minions={ @@ -380,7 +380,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): assert actual_result == expected_result - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_if_wait_timer_is_not_done_even_though_results_are_then_data_should_not_yet_be_returned( self, ): @@ -388,18 +388,18 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): "one": {"fnordy one": "one has some data"}, "two": {"fnordy two": "two has some data"}, } - events = [salt.ext.tornado.gen.Future(), salt.ext.tornado.gen.Future()] + events = [tornado.gen.Future(), tornado.gen.Future()] events[0].set_result( {"tag": "fnord", "data": {"id": "one", "return": expected_result["one"]}} ) events[1].set_result( {"tag": "fnord", "data": {"id": "two", "return": expected_result["two"]}} ) - wait_timer = salt.ext.tornado.gen.Future() + wait_timer = tornado.gen.Future() fut = self.handler.get_minion_returns( events=events, - is_finished=salt.ext.tornado.gen.Future(), - is_timed_out=salt.ext.tornado.gen.Future(), + is_finished=tornado.gen.Future(), + is_timed_out=tornado.gen.Future(), min_wait_time=wait_timer, minions={"one": False, "two": False}, ) @@ -408,7 +408,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): yield fut self.io_loop.spawn_callback(boop) - yield salt.ext.tornado.gen.sleep(0.1) + yield tornado.gen.sleep(0.1) assert not fut.done() @@ -417,30 +417,30 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): assert actual_result == expected_result - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_is_finished_any_other_futures_should_be_canceled(self): events = [ - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), ] - is_finished = salt.ext.tornado.gen.Future() + is_finished = tornado.gen.Future() is_finished.set_result(None) yield self.handler.get_minion_returns( events=events, is_finished=is_finished, - is_timed_out=salt.ext.tornado.gen.Future(), - min_wait_time=salt.ext.tornado.gen.Future(), + is_timed_out=tornado.gen.Future(), + min_wait_time=tornado.gen.Future(), minions={"one": False, "two": False}, ) are_done = [event.done() for event in events] assert all(are_done) - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_an_event_times_out_then_we_should_not_enter_an_infinite_loop(self): # NOTE: this test will enter an infinite loop if the code is broken. I # was not able to figure out a way to ensure that the test exits with @@ -451,27 +451,27 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): # TimeoutException. events = [ - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), ] # Arguably any event would work, but 3 isn't the first, so it # gives us a little more confidence that this test is testing # correctly events[3].set_exception(saltnado.TimeoutException()) - times_out_later = salt.ext.tornado.gen.Future() + times_out_later = tornado.gen.Future() # 0.5s should be long enough that the test gets through doing other # things before hitting this timeout, which will cancel all the # in-flight futures. self.io_loop.call_later(0.5, lambda: times_out_later.set_result(None)) yield self.handler.get_minion_returns( events=events, - is_finished=salt.ext.tornado.gen.Future(), + is_finished=tornado.gen.Future(), is_timed_out=times_out_later, - min_wait_time=salt.ext.tornado.gen.Future(), + min_wait_time=tornado.gen.Future(), minions={"one": False, "two": False}, ) @@ -482,7 +482,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): assert all(are_done) assert times_out_later.done() - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_is_timed_out_any_other_futures_should_be_canceled(self): # There is some question about whether this test is or should be # necessary. Or if it's meaningful. The code that this is testing @@ -491,46 +491,46 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): # That being said, the worst case is that this is just a duplicate # or irrelevant test, and can be removed. events = [ - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), ] - is_timed_out = salt.ext.tornado.gen.Future() + is_timed_out = tornado.gen.Future() is_timed_out.set_result(None) yield self.handler.get_minion_returns( events=events, - is_finished=salt.ext.tornado.gen.Future(), + is_finished=tornado.gen.Future(), is_timed_out=is_timed_out, - min_wait_time=salt.ext.tornado.gen.Future(), + min_wait_time=tornado.gen.Future(), minions={"one": False, "two": False}, ) are_done = [event.done() for event in events] assert all(are_done) - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_min_wait_time_and_nothing_todo_any_other_futures_should_be_canceled( self, ): events = [ - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), - salt.ext.tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), + tornado.gen.Future(), ] - is_finished = salt.ext.tornado.gen.Future() - min_wait_time = salt.ext.tornado.gen.Future() + is_finished = tornado.gen.Future() + min_wait_time = tornado.gen.Future() self.io_loop.call_later(0.2, lambda: min_wait_time.set_result(None)) yield self.handler.get_minion_returns( events=events, is_finished=is_finished, - is_timed_out=salt.ext.tornado.gen.Future(), + is_timed_out=tornado.gen.Future(), min_wait_time=min_wait_time, minions={"one": True, "two": True}, ) @@ -538,37 +538,37 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): are_done = [event.done() for event in events] + [is_finished.done()] assert all(are_done) - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_is_finished_but_not_is_timed_out_then_timed_out_should_not_be_set_to_done( self, ): - events = [salt.ext.tornado.gen.Future()] - is_timed_out = salt.ext.tornado.gen.Future() - is_finished = salt.ext.tornado.gen.Future() + events = [tornado.gen.Future()] + is_timed_out = tornado.gen.Future() + is_finished = tornado.gen.Future() is_finished.set_result(None) yield self.handler.get_minion_returns( events=events, is_finished=is_finished, is_timed_out=is_timed_out, - min_wait_time=salt.ext.tornado.gen.Future(), + min_wait_time=tornado.gen.Future(), minions={"one": False, "two": False}, ) assert not is_timed_out.done() - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_min_wait_time_and_all_completed_but_not_is_timed_out_then_timed_out_should_not_be_set_to_done( self, ): - events = [salt.ext.tornado.gen.Future()] - is_timed_out = salt.ext.tornado.gen.Future() - min_wait_time = salt.ext.tornado.gen.Future() + events = [tornado.gen.Future()] + is_timed_out = tornado.gen.Future() + min_wait_time = tornado.gen.Future() self.io_loop.call_later(0.2, lambda: min_wait_time.set_result(None)) yield self.handler.get_minion_returns( events=events, - is_finished=salt.ext.tornado.gen.Future(), + is_finished=tornado.gen.Future(), is_timed_out=is_timed_out, min_wait_time=min_wait_time, minions={"one": True}, @@ -576,21 +576,21 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): assert not is_timed_out.done() - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_things_are_completed_but_not_timed_out_then_timed_out_event_should_not_be_done( self, ): events = [ - salt.ext.tornado.gen.Future(), + tornado.gen.Future(), ] events[0].set_result({"tag": "fnord", "data": {"id": "one", "return": {}}}) - min_wait_time = salt.ext.tornado.gen.Future() + min_wait_time = tornado.gen.Future() min_wait_time.set_result(None) - is_timed_out = salt.ext.tornado.gen.Future() + is_timed_out = tornado.gen.Future() yield self.handler.get_minion_returns( events=events, - is_finished=salt.ext.tornado.gen.Future(), + is_finished=tornado.gen.Future(), is_timed_out=is_timed_out, min_wait_time=min_wait_time, minions={"one": True}, @@ -599,7 +599,7 @@ class TestGetMinionReturns(salt.ext.tornado.testing.AsyncTestCase): assert not is_timed_out.done() -class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): +class TestDisbatchLocal(tornado.testing.AsyncTestCase): def setUp(self): super().setUp() self.mock = MagicMock() @@ -614,12 +614,12 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): } self.handler = saltnado.SaltAPIHandler(self.mock, self.mock) - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_is_timed_out_is_set_before_other_events_are_completed_then_result_should_be_empty_dictionary( self, ): - completed_event = salt.ext.tornado.gen.Future() - never_completed = salt.ext.tornado.gen.Future() + completed_event = tornado.gen.Future() + never_completed = tornado.gen.Future() # TODO: We may need to tweak these values to get them close enough but not so far away -W. Werner, 2020-11-17 gather_timeout = 0.1 event_timeout = gather_timeout + 0.05 @@ -642,7 +642,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): self.io_loop.call_later(event_timeout, completer) - f = salt.ext.tornado.gen.Future() + f = tornado.gen.Future() f.set_result({"jid": "42", "minions": []}) with patch.object( self.handler.application.event_listener, @@ -660,12 +660,12 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): assert result == {} - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_is_finished_is_set_before_events_return_then_no_data_should_be_returned( self, ): - completed_event = salt.ext.tornado.gen.Future() - never_completed = salt.ext.tornado.gen.Future() + completed_event = tornado.gen.Future() + never_completed = tornado.gen.Future() gather_timeout = 2 event_timeout = gather_timeout - 1 @@ -692,7 +692,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): assert finished is not None finished.set_result(42) - f = salt.ext.tornado.gen.Future() + f = tornado.gen.Future() f.set_result({"jid": "42", "minions": []}) with patch.object( self.handler.application.event_listener, @@ -715,13 +715,13 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): assert result == {} - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_is_finished_then_all_collected_data_should_be_returned(self): - completed_event = salt.ext.tornado.gen.Future() - never_completed = salt.ext.tornado.gen.Future() + completed_event = tornado.gen.Future() + never_completed = tornado.gen.Future() # This timeout should never be reached gather_timeout = 42 - completed_events = [salt.ext.tornado.gen.Future() for _ in range(5)] + completed_events = [tornado.gen.Future() for _ in range(5)] for i, event in enumerate(completed_events): event.set_result( { @@ -732,7 +732,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): }, } ) - uncompleted_events = [salt.ext.tornado.gen.Future() for _ in range(5)] + uncompleted_events = [tornado.gen.Future() for _ in range(5)] events = iter(completed_events + uncompleted_events) expected_result = { "fnord 0": "return from fnord 0", @@ -753,7 +753,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): assert finished is not None finished.set_result(42) - f = salt.ext.tornado.gen.Future() + f = tornado.gen.Future() f.set_result({"jid": "42", "minions": ["non-existent minion"]}) with patch.object( self.handler.application.event_listener, @@ -776,16 +776,16 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): assert result == expected_result - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_is_timed_out_then_all_collected_data_should_be_returned(self): - completed_event = salt.ext.tornado.gen.Future() - never_completed = salt.ext.tornado.gen.Future() + completed_event = tornado.gen.Future() + never_completed = tornado.gen.Future() # 2s is probably enough for any kind of computer to manage to # do all the other processing. We could maybe reduce this - just # depends on how slow of a system we're running on. # TODO: Maybe we should have a test helper/fixture that benchmarks the system and gets a reasonable timeout? -W. Werner, 2020-11-19 gather_timeout = 2 - completed_events = [salt.ext.tornado.gen.Future() for _ in range(5)] + completed_events = [tornado.gen.Future() for _ in range(5)] for i, event in enumerate(completed_events): event.set_result( { @@ -796,7 +796,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): }, } ) - uncompleted_events = [salt.ext.tornado.gen.Future() for _ in range(5)] + uncompleted_events = [tornado.gen.Future() for _ in range(5)] events = iter(completed_events + uncompleted_events) expected_result = { "fnord 0": "return from fnord 0", @@ -812,7 +812,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): else: return next(events) - f = salt.ext.tornado.gen.Future() + f = tornado.gen.Future() f.set_result({"jid": "42", "minions": ["non-existent minion"]}) with patch.object( self.handler.application.event_listener, @@ -830,13 +830,13 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): assert result == expected_result - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_minions_all_return_then_all_collected_data_should_be_returned(self): - completed_event = salt.ext.tornado.gen.Future() - never_completed = salt.ext.tornado.gen.Future() + completed_event = tornado.gen.Future() + never_completed = tornado.gen.Future() # Timeout is something ridiculously high - it should never be reached gather_timeout = 20 - completed_events = [salt.ext.tornado.gen.Future() for _ in range(10)] + completed_events = [tornado.gen.Future() for _ in range(10)] events_by_id = {} for i, event in enumerate(completed_events): id_ = f"fnord {i}" @@ -864,7 +864,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): tag = kwargs.get("tag", "").rpartition("/")[-1] return events_by_id.get(tag, never_completed) - f = salt.ext.tornado.gen.Future() + f = tornado.gen.Future() f.set_result( { "jid": "42", @@ -887,15 +887,15 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): assert result == expected_result - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_when_min_wait_time_has_not_passed_then_disbatch_should_not_return_expected_data_until_time_has_passed( self, ): - completed_event = salt.ext.tornado.gen.Future() - never_completed = salt.ext.tornado.gen.Future() - wait_timer = salt.ext.tornado.gen.Future() + completed_event = tornado.gen.Future() + never_completed = tornado.gen.Future() + wait_timer = tornado.gen.Future() gather_timeout = 20 - completed_events = [salt.ext.tornado.gen.Future() for _ in range(10)] + completed_events = [tornado.gen.Future() for _ in range(10)] events_by_id = {} # Setup some real-enough looking return data for i, event in enumerate(completed_events): @@ -943,11 +943,11 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): # The fake sleep is necessary so that we can return our own # min_wait_time future. The fakeo_timer object is how we signal # which one we need to be returning. - orig_sleep = salt.ext.tornado.gen.sleep + orig_sleep = tornado.gen.sleep fakeo_timer = object() - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def fake_sleep(timer): # only return our fake min_wait_time future when the sentinel # value is provided. Otherwise it's just a number. @@ -956,7 +956,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): else: yield orig_sleep(timer) - f = salt.ext.tornado.gen.Future() + f = tornado.gen.Future() f.set_result( { "jid": "42", @@ -981,7 +981,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): "order_masters": True, }, ), patch( - "salt.ext.tornado.gen.sleep", + "tornado.gen.sleep", autospec=True, side_effect=fake_sleep, ), patch.dict( @@ -1009,7 +1009,7 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): yield fut self.io_loop.spawn_callback(boop) - yield salt.ext.tornado.gen.sleep(0.1) + yield tornado.gen.sleep(0.1) # here, all the minions should be complete (i.e. "True") assert all(minions[m_id] for m_id in minions) # But _disbatch_local is not returned yet because min_wait_time has not passed diff --git a/tests/unit/test_proxy_minion.py b/tests/unit/test_proxy_minion.py index bc3e867619f..92fc68ad4b1 100644 --- a/tests/unit/test_proxy_minion.py +++ b/tests/unit/test_proxy_minion.py @@ -13,8 +13,8 @@ import pytest from saltfactories.utils import random_string import salt.config -import salt.ext.tornado -import salt.ext.tornado.testing +import tornado +import tornado.testing import salt.metaproxy.proxy import salt.minion import salt.syspaths @@ -38,7 +38,7 @@ class ProxyMinionTestCase(TestCase): proxy_minion = salt.minion.ProxyMinion( mock_opts, jid_queue=copy.copy(mock_jid_queue), - io_loop=salt.ext.tornado.ioloop.IOLoop(), + io_loop=tornado.ioloop.IOLoop(), ) mock_metaproxy_call = MagicMock() with patch( @@ -65,7 +65,7 @@ class ProxyMinionTestCase(TestCase): proxy_minion = salt.minion.ProxyMinion( mock_opts, jid_queue=copy.copy(mock_jid_queue), - io_loop=salt.ext.tornado.ioloop.IOLoop(), + io_loop=tornado.ioloop.IOLoop(), ) mock_metaproxy_call = MagicMock() with patch( @@ -93,7 +93,7 @@ class ProxyMinionTestCase(TestCase): proxy_minion = salt.minion.ProxyMinion( mock_opts, jid_queue=copy.copy(mock_jid_queue), - io_loop=salt.ext.tornado.ioloop.IOLoop(), + io_loop=tornado.ioloop.IOLoop(), ) mock_metaproxy_call = MagicMock() with patch( diff --git a/tests/unit/transport/mixins.py b/tests/unit/transport/mixins.py index 65b1c7a9498..c757f37861f 100644 --- a/tests/unit/transport/mixins.py +++ b/tests/unit/transport/mixins.py @@ -1,4 +1,4 @@ -import salt.ext.tornado.gen +import tornado.gen def run_loop_in_thread(loop, evt): @@ -7,13 +7,13 @@ def run_loop_in_thread(loop, evt): """ loop.make_current() - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def stopper(): while True: if evt.is_set(): loop.stop() break - yield salt.ext.tornado.gen.sleep(0.3) + yield tornado.gen.sleep(0.3) loop.add_callback(stopper) try: diff --git a/tests/unit/transport/test_ipc.py b/tests/unit/transport/test_ipc.py index 72c32a2382c..dff6f77f2d9 100644 --- a/tests/unit/transport/test_ipc.py +++ b/tests/unit/transport/test_ipc.py @@ -10,12 +10,12 @@ import pytest import salt.config import salt.exceptions -import salt.ext.tornado.gen -import salt.ext.tornado.ioloop -import salt.ext.tornado.testing +import tornado.gen +import tornado.ioloop +import tornado.testing import salt.transport.ipc import salt.utils.platform -from salt.ext.tornado.iostream import StreamClosedError +from tornado.iostream import StreamClosedError from tests.support.runtests import RUNTIME_VARS pytestmark = [ @@ -28,7 +28,7 @@ log = logging.getLogger(__name__) @pytest.mark.skip_on_windows(reason="Windows does not support Posix IPC") -class IPCMessagePubSubCase(salt.ext.tornado.testing.AsyncTestCase): +class IPCMessagePubSubCase(tornado.testing.AsyncTestCase): """ Test all of the clear msg stuff """ @@ -124,7 +124,7 @@ class IPCMessagePubSubCase(salt.ext.tornado.testing.AsyncTestCase): self.assertEqual(ret1, "TEST") self.assertEqual(ret2, "TEST") - @salt.ext.tornado.testing.gen_test + @tornado.testing.gen_test def test_async_reading_streamclosederror(self): client1 = self.sub_channel call_cnt = [] diff --git a/tests/unit/transport/test_tcp.py b/tests/unit/transport/test_tcp.py index dbe88081859..934a0400933 100644 --- a/tests/unit/transport/test_tcp.py +++ b/tests/unit/transport/test_tcp.py @@ -12,12 +12,12 @@ import salt.channel.client import salt.channel.server import salt.config import salt.exceptions -import salt.ext.tornado.concurrent -import salt.ext.tornado.gen -import salt.ext.tornado.ioloop +import tornado.concurrent +import tornado.gen +import tornado.ioloop import salt.utils.platform import salt.utils.process -from salt.ext.tornado.testing import AsyncTestCase +from tornado.testing import AsyncTestCase from tests.support.mixins import AdaptedConfigurationTestCaseMixin from tests.unit.transport.mixins import run_loop_in_thread @@ -82,7 +82,7 @@ class AsyncPubServerTest(AsyncTestCase, AdaptedConfigurationTestCaseMixin): cls.master_config ) cls.req_server_channel.pre_fork(cls.process_manager) - cls.io_loop = salt.ext.tornado.ioloop.IOLoop() + cls.io_loop = tornado.ioloop.IOLoop() cls.stop = threading.Event() cls.req_server_channel.post_fork(cls._handle_payload, io_loop=cls.io_loop) cls.server_thread = threading.Thread( diff --git a/tests/unit/utils/test_context.py b/tests/unit/utils/test_context.py index abea69fbf62..79029b1e4d7 100644 --- a/tests/unit/utils/test_context.py +++ b/tests/unit/utils/test_context.py @@ -3,172 +3,11 @@ tests.unit.context_test ~~~~~~~~~~~~~~~~~~~~~~~ """ -import threading -import time - -import pytest - -import salt.ext.tornado.gen -import salt.ext.tornado.stack_context import salt.utils.json -from salt.ext.tornado.testing import AsyncTestCase, gen_test -from salt.utils.context import ContextDict, NamespacedDictWrapper +from salt.utils.context import NamespacedDictWrapper from tests.support.unit import TestCase -class ContextDictTests(AsyncTestCase): - # how many threads/coroutines to run at a time - num_concurrent_tasks = 5 - - def setUp(self): - super().setUp() - self.cd = ContextDict() - # set a global value - self.cd["foo"] = "global" - - @pytest.mark.slow_test - def test_threads(self): - """Verify that ContextDict overrides properly within threads""" - rets = [] - - def tgt(x, s): - inner_ret = [] - over = self.cd.clone() - - inner_ret.append(self.cd.get("foo")) - with over: - inner_ret.append(over.get("foo")) - over["foo"] = x - inner_ret.append(over.get("foo")) - time.sleep(s) - inner_ret.append(over.get("foo")) - rets.append(inner_ret) - - threads = [] - for x in range(0, self.num_concurrent_tasks): - s = self.num_concurrent_tasks - x - t = threading.Thread(target=tgt, args=(x, s)) - t.start() - threads.append(t) - - for t in threads: - t.join() - - for r in rets: - self.assertEqual(r[0], r[1]) - self.assertEqual(r[2], r[3]) - - @gen_test - @pytest.mark.slow_test - def test_coroutines(self): - """Verify that ContextDict overrides properly within coroutines""" - - @salt.ext.tornado.gen.coroutine - def secondary_coroutine(over): - raise salt.ext.tornado.gen.Return(over.get("foo")) - - @salt.ext.tornado.gen.coroutine - def tgt(x, s, over): - inner_ret = [] - # first grab the global - inner_ret.append(self.cd.get("foo")) - # grab the child's global (should match) - inner_ret.append(over.get("foo")) - # override the global - over["foo"] = x - inner_ret.append(over.get("foo")) - # sleep for some time to let other coroutines do this section of code - yield salt.ext.tornado.gen.sleep(s) - # get the value of the global again. - inner_ret.append(over.get("foo")) - # Call another coroutine to verify that we keep our context - r = yield secondary_coroutine(over) - inner_ret.append(r) - raise salt.ext.tornado.gen.Return(inner_ret) - - futures = [] - - for x in range(0, self.num_concurrent_tasks): - s = self.num_concurrent_tasks - x - over = self.cd.clone() - - # pylint: disable=cell-var-from-loop - f = salt.ext.tornado.stack_context.run_with_stack_context( - salt.ext.tornado.stack_context.StackContext(lambda: over), - lambda: tgt(x, s / 5.0, over), - ) - # pylint: enable=cell-var-from-loop - futures.append(f) - - wait_iterator = salt.ext.tornado.gen.WaitIterator(*futures) - while not wait_iterator.done(): - r = yield wait_iterator.next() # pylint: disable=incompatible-py3-code - self.assertEqual(r[0], r[1]) # verify that the global value remails - self.assertEqual(r[2], r[3]) # verify that the override sticks locally - self.assertEqual( - r[3], r[4] - ) # verify that the override sticks across coroutines - - def test_basic(self): - """Test that the contextDict is a dict""" - # ensure we get the global value - self.assertEqual( - dict(self.cd), - {"foo": "global"}, - ) - - def test_override(self): - over = self.cd.clone() - over["bar"] = "global" - self.assertEqual( - dict(over), - {"foo": "global", "bar": "global"}, - ) - self.assertEqual( - dict(self.cd), - {"foo": "global"}, - ) - with over: - self.assertEqual( - dict(over), - {"foo": "global", "bar": "global"}, - ) - self.assertEqual( - dict(self.cd), - {"foo": "global", "bar": "global"}, - ) - over["bar"] = "baz" - self.assertEqual( - dict(over), - {"foo": "global", "bar": "baz"}, - ) - self.assertEqual( - dict(self.cd), - {"foo": "global", "bar": "baz"}, - ) - self.assertEqual( - dict(over), - {"foo": "global", "bar": "baz"}, - ) - self.assertEqual( - dict(self.cd), - {"foo": "global"}, - ) - - def test_multiple_contexts(self): - cds = [] - for x in range(0, 10): - cds.append(self.cd.clone(bar=x)) - for x, cd in enumerate(cds): - self.assertNotIn("bar", self.cd) - with cd: - self.assertEqual( - dict(self.cd), - {"bar": x, "foo": "global"}, - ) - self.assertNotIn("bar", self.cd) - - class NamespacedDictWrapperTests(TestCase): PREFIX = "prefix" @@ -193,4 +32,4 @@ class NamespacedDictWrapperTests(TestCase): def test_json_dumps_multiple_key(self): self._dict["prefix"] = {"foo": {"bar": "baz"}} w = NamespacedDictWrapper(self._dict, ("prefix", "foo")) - self.assertEqual(salt.utils.json.dumps(w), '{"bar": "baz"}') + self.assertEqual(salt.utils.json.dumps(w), '{"bar": "baz"}') \ No newline at end of file diff --git a/tests/unit/utils/test_gitfs.py b/tests/unit/utils/test_gitfs.py index 6e99a219ca3..48c0d0472d4 100644 --- a/tests/unit/utils/test_gitfs.py +++ b/tests/unit/utils/test_gitfs.py @@ -7,7 +7,7 @@ import tempfile import pytest -import salt.ext.tornado.ioloop +import tornado.ioloop import salt.fileserver.gitfs import salt.utils.files import salt.utils.gitfs @@ -20,7 +20,7 @@ from tests.support.unit import TestCase def _clear_instance_map(): try: del salt.utils.gitfs.GitFS.instance_map[ - salt.ext.tornado.ioloop.IOLoop.current() + tornado.ioloop.IOLoop.current() ] except KeyError: pass diff --git a/tests/unit/utils/test_http.py b/tests/unit/utils/test_http.py index d9a84f9582a..c1bd5c16589 100644 --- a/tests/unit/utils/test_http.py +++ b/tests/unit/utils/test_http.py @@ -15,7 +15,7 @@ from tests.support.runtests import RUNTIME_VARS from tests.support.unit import TestCase try: - import salt.ext.tornado.curl_httpclient # pylint: disable=unused-import + import tornado.curl_httpclient # pylint: disable=unused-import HAS_CURL = True except ImportError: -- 2.51.1