salt/skip-certain-tests-if-necessary-and-mark-some-flaky-.patch

From c02adf9e721dbebf0b50f202d2e86723e5642514 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?=
 <psuarezhernandez@suse.com>
Date: Wed, 12 Jun 2024 09:38:43 +0100
Subject: [PATCH] Skip certain tests if necessary and mark some flaky
 ones (#657)

* Small alignments with upstream

* Skip tests if necessary and mark some flaky ones

* Add extra flaky test
---
 tests/integration/modules/test_status.py              |  1 +
 tests/pytests/functional/cache/test_consul.py         |  7 +------
 tests/pytests/functional/cache/test_mysql.py          |  6 +++---
 tests/pytests/functional/modules/state/test_state.py  |  3 +++
 tests/pytests/functional/modules/test_pkg.py          |  8 ++++++++
 .../pytests/functional/modules/test_virtualenv_mod.py |  5 +++++
 tests/pytests/functional/states/file/test_managed.py  |  2 +-
 tests/pytests/functional/states/pkgrepo/test_suse.py  | 10 ++++++++++
 .../functional/states/test_docker_container.py        |  7 +++++++
 tests/pytests/functional/states/test_npm.py           |  5 +++++
 tests/pytests/functional/states/test_pip_state.py     |  4 ++++
 tests/pytests/functional/states/test_user.py          | 11 ++++++++++-
 tests/pytests/integration/cli/test_syndic_eauth.py    |  2 --
 tests/pytests/integration/daemons/test_memory_leak.py |  2 +-
 tests/support/pytest/mysql.py                         |  8 ++++++++
 15 files changed, 67 insertions(+), 14 deletions(-)

diff --git a/tests/integration/modules/test_status.py b/tests/integration/modules/test_status.py
index 73ce4817e82..a7265672ebe 100644
--- a/tests/integration/modules/test_status.py
+++ b/tests/integration/modules/test_status.py
@@ -44,6 +44,7 @@ class StatusModuleTest(ModuleCase):
         self.assertTrue(isinstance(ret, int))
 
     @pytest.mark.slow_test
+    @pytest.mark.flaky(max_runs=4)
     def test_status_diskusage(self):
         """
         status.diskusage
diff --git a/tests/pytests/functional/cache/test_consul.py b/tests/pytests/functional/cache/test_consul.py
index 996a1e932b6..19ad6f4aae0 100644
--- a/tests/pytests/functional/cache/test_consul.py
+++ b/tests/pytests/functional/cache/test_consul.py
@@ -8,14 +8,13 @@ from saltfactories.utils import random_string
 
 import salt.cache
 import salt.loader
-from salt.utils.versions import Version
 from tests.pytests.functional.cache.helpers import run_common_cache_tests
 
 pytest.importorskip(
     "consul",
     reason="Please install python-consul package to use consul data cache driver",
 )
-docker = pytest.importorskip("docker")
+docker = pytest.importorskip("docker", minversion="4.0.0")
 
 log = logging.getLogger(__name__)
 
@@ -25,10 +24,6 @@ pytestmark = [
     pytest.mark.slow_test,
     pytest.mark.skip_if_binaries_missing("dockerd"),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
-    pytest.mark.skipif(
-        Version(docker.__version__) < Version("4.0.0"),
-        reason="Test does not work in this version of docker-py",
-    ),
 ]
 
 
diff --git a/tests/pytests/functional/cache/test_mysql.py b/tests/pytests/functional/cache/test_mysql.py
index 93c6c7c6f6f..6cf3cc49534 100644
--- a/tests/pytests/functional/cache/test_mysql.py
+++ b/tests/pytests/functional/cache/test_mysql.py
@@ -5,11 +5,12 @@ import pytest
 
 import salt.cache
 import salt.loader
+import salt.modules.mysql
 from salt.utils.versions import Version
 from tests.pytests.functional.cache.helpers import run_common_cache_tests
 from tests.support.pytest.mysql import *  # pylint: disable=wildcard-import,unused-wildcard-import
 
-docker = pytest.importorskip("docker")
+pytest.importorskip("docker", minversion="4.0.0")
 
 log = logging.getLogger(__name__)
 
@@ -20,8 +21,7 @@ pytestmark = [
     pytest.mark.skip_if_binaries_missing("dockerd"),
     pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
     pytest.mark.skipif(
-        Version(docker.__version__) < Version("4.0.0"),
-        reason="Test does not work in this version of docker-py",
+        not salt.modules.mysql.MySQLdb, reason="Missing python MySQLdb library"
     ),
 ]
 
diff --git a/tests/pytests/functional/modules/state/test_state.py b/tests/pytests/functional/modules/state/test_state.py
index 7640afaa882..57aa53001ab 100644
--- a/tests/pytests/functional/modules/state/test_state.py
+++ b/tests/pytests/functional/modules/state/test_state.py
@@ -714,6 +714,7 @@ def test_retry_option_success(state, state_tree, tmp_path):
 @pytest.mark.skip_on_windows(
     reason="Skipped until parallel states can be fixed on Windows"
 )
+@pytest.mark.xfail(reason="This test is flaky")
 def test_retry_option_success_parallel(state, state_tree, tmp_path):
     """
     test a state with the retry option that should return True immediately (i.e. no retries)
@@ -753,6 +754,7 @@ def test_retry_option_success_parallel(state, state_tree, tmp_path):
 
 
 @pytest.mark.slow_test
+@pytest.mark.xfail(reason="This test is flaky")
 def test_retry_option_eventual_success(state, state_tree, tmp_path):
     """
     test a state with the retry option that should return True, eventually
@@ -801,6 +803,7 @@ def test_retry_option_eventual_success(state, state_tree, tmp_path):
     reason="Skipped until parallel states can be fixed on Windows"
 )
 @pytest.mark.slow_test
+@pytest.mark.xfail(reason="This test is flaky")
 def test_retry_option_eventual_success_parallel(state, state_tree, tmp_path):
     """
     test a state with the retry option that should return True, eventually
diff --git a/tests/pytests/functional/modules/test_pkg.py b/tests/pytests/functional/modules/test_pkg.py
index 7cedd32bf6c..82d0801965d 100644
--- a/tests/pytests/functional/modules/test_pkg.py
+++ b/tests/pytests/functional/modules/test_pkg.py
@@ -232,6 +232,10 @@ def test_which(modules):
 @pytest.mark.requires_salt_modules("pkg.version", "pkg.install", "pkg.remove")
 @pytest.mark.slow_test
 @pytest.mark.requires_network
+@pytest.mark.skipif(
+    bool(salt.utils.path.which("transactional-update")),
+    reason="Skipping on transactional systems",
+)
 def test_install_remove(modules, test_pkg, refresh_db):
     """
     successfully install and uninstall a package
@@ -272,6 +276,10 @@ def test_install_remove(modules, test_pkg, refresh_db):
 @pytest.mark.slow_test
 @pytest.mark.requires_network
 @pytest.mark.requires_salt_states("pkg.installed")
+@pytest.mark.skipif(
+    bool(salt.utils.path.which("transactional-update")),
+    reason="Skipping on transactional systems",
+)
 def test_hold_unhold(grains, modules, states, test_pkg, refresh_db):
     """
     test holding and unholding a package
diff --git a/tests/pytests/functional/modules/test_virtualenv_mod.py b/tests/pytests/functional/modules/test_virtualenv_mod.py
index 7d8398e149b..2b6abf91e23 100644
--- a/tests/pytests/functional/modules/test_virtualenv_mod.py
+++ b/tests/pytests/functional/modules/test_virtualenv_mod.py
@@ -2,6 +2,7 @@ import shutil
 
 import pytest
 
+import salt.utils.path
 from salt.modules.virtualenv_mod import KNOWN_BINARY_NAMES
 
 pytestmark = [
@@ -63,6 +64,10 @@ def test_clear(virtualenv, venv_dir, modules):
     assert "pep8" not in packages
 
 
+@pytest.mark.skipif(
+    bool(salt.utils.path.which("transactional-update")),
+    reason="Skipping on transactional systems",
+)
 def test_virtualenv_ver(virtualenv, venv_dir):
     ret = virtualenv.create(str(venv_dir))
     assert ret
diff --git a/tests/pytests/functional/states/file/test_managed.py b/tests/pytests/functional/states/file/test_managed.py
index 9678fb63432..1b904c33543 100644
--- a/tests/pytests/functional/states/file/test_managed.py
+++ b/tests/pytests/functional/states/file/test_managed.py
@@ -658,7 +658,6 @@ def test_issue_8947_utf8_sls(modules, tmp_path, state_tree, subtests):
 @pytest.mark.skip_if_not_root
 @pytest.mark.skip_on_windows(reason="Windows does not support setuid. Skipping.")
 def test_owner_after_setuid(file, modules, tmp_path, state_file_account):
-
     """
     Test to check file user/group after setting setuid or setgid.
     Because Python os.chown() does reset the setuid/setgid to 0.
@@ -767,6 +766,7 @@ def test_file_managed_keep_source_false_http(
 
 
 @pytest.mark.parametrize("verify_ssl", [True, False])
+@pytest.mark.flaky(max_runs=4)
 def test_verify_ssl_https_source(file, tmp_path, ssl_webserver, verify_ssl):
     """
     test verify_ssl when its False and True when managing
diff --git a/tests/pytests/functional/states/pkgrepo/test_suse.py b/tests/pytests/functional/states/pkgrepo/test_suse.py
index 19ba928ce6e..3bafeedc941 100644
--- a/tests/pytests/functional/states/pkgrepo/test_suse.py
+++ b/tests/pytests/functional/states/pkgrepo/test_suse.py
@@ -1,5 +1,7 @@
 import pytest
 
+import salt.utils.path
+
 pytestmark = [
     pytest.mark.destructive_test,
     pytest.mark.skip_if_not_root,
@@ -80,6 +82,10 @@ def suse_state_tree(grains, pkgrepo, state_tree):
 
 
 @pytest.mark.requires_salt_states("pkgrepo.managed", "pkgrepo.absent")
+@pytest.mark.skipif(
+    bool(salt.utils.path.which("transactional-update")),
+    reason="Skipping on transactional systems",
+)
 def test_pkgrepo_managed_absent(grains, modules, subtests, suse_state_tree):
     """
     Test adding and removing a repository
@@ -134,6 +140,10 @@ def test_pkgrepo_managed_absent(grains, modules, subtests, suse_state_tree):
 
 
 @pytest.mark.requires_salt_states("pkgrepo.managed")
+@pytest.mark.skipif(
+    bool(salt.utils.path.which("transactional-update")),
+    reason="Skipping on transactional systems",
+)
 def test_pkgrepo_managed_modify(grains, modules, subtests, suse_state_tree):
     """
     Test adding and modifying a repository
diff --git a/tests/pytests/functional/states/test_docker_container.py b/tests/pytests/functional/states/test_docker_container.py
index 2267399891e..539a2acbf1a 100644
--- a/tests/pytests/functional/states/test_docker_container.py
+++ b/tests/pytests/functional/states/test_docker_container.py
@@ -26,6 +26,7 @@ pytestmark = [
     pytest.mark.slow_test,
     pytest.mark.skip_on_freebsd(reason="No Docker on FreeBSD available"),
     pytest.mark.skip_if_binaries_missing("busybox", reason="Busybox not installed"),
+    pytest.mark.skip_if_binaries_missing("ldd", reason="ldd is missing"),
     pytest.mark.skip_if_binaries_missing(
         "docker", "dockerd", reason="Docker not installed"
     ),
@@ -172,6 +173,12 @@ def image(tmp_path_factory):
         # Somehow the above skip_if_binaries_missing marker for docker
         # only get's evaluated after this fixture?!?
         pytest.skip("The `docker` binary is not available")
+    if not salt.modules.cmdmod.retcode(
+        "ldd {}".format(salt.utils.path.which("busybox"))
+    ):
+        pytest.skip(
+            "`busybox` appears to be a dynamic executable, please use busybox-static"
+        )
     container_build_dir = tmp_path_factory.mktemp("busybox")
     image_name = random_string("salt-busybox-", uppercase=False)
 
diff --git a/tests/pytests/functional/states/test_npm.py b/tests/pytests/functional/states/test_npm.py
index 2899b7985a1..54db6042716 100644
--- a/tests/pytests/functional/states/test_npm.py
+++ b/tests/pytests/functional/states/test_npm.py
@@ -3,6 +3,7 @@ import shutil
 
 import pytest
 
+import salt.utils.path
 from salt.exceptions import CommandExecutionError
 from salt.utils.versions import Version
 
@@ -10,6 +11,10 @@ pytestmark = [
     pytest.mark.slow_test,
     pytest.mark.destructive_test,
     pytest.mark.requires_network,
+    pytest.mark.skipif(
+        bool(salt.utils.path.which("transactional-update")),
+        reason="Skipping on transactional systems",
+    ),
 ]
 
 MAX_NPM_VERSION = "5.0.0"
diff --git a/tests/pytests/functional/states/test_pip_state.py b/tests/pytests/functional/states/test_pip_state.py
index 1921751b5dc..1f2080f1f86 100644
--- a/tests/pytests/functional/states/test_pip_state.py
+++ b/tests/pytests/functional/states/test_pip_state.py
@@ -80,6 +80,10 @@ def create_virtualenv(modules):
 
 
 @pytest.mark.slow_test
+@pytest.mark.skipif(
+    bool(salt.utils.path.which("transactional-update")),
+    reason="Skipping on transactional systems",
+)
 def test_pip_installed_removed(modules, states):
     """
     Tests installed and removed states
diff --git a/tests/pytests/functional/states/test_user.py b/tests/pytests/functional/states/test_user.py
index 96b1ec55c88..5eac093ef44 100644
--- a/tests/pytests/functional/states/test_user.py
+++ b/tests/pytests/functional/states/test_user.py
@@ -13,6 +13,7 @@ import pytest
 from saltfactories.utils import random_string
 
 import salt.utils.files
+import salt.utils.path
 import salt.utils.platform
 
 try:
@@ -137,7 +138,7 @@ def test_user_present_nondefault(grains, modules, states, username, user_home):
     if not salt.utils.platform.is_darwin() and not salt.utils.platform.is_windows():
         assert user_home.is_dir()
 
-    if grains["os_family"] in ("Suse",):
+    if grains["os_family"] in ("Suse",) and not grains.get("transactional", False):
         expected_group_name = "users"
     elif grains["os_family"] == "MacOS":
         expected_group_name = "staff"
@@ -380,6 +381,10 @@ def test_user_present_existing(states, username):
 
 
 @pytest.mark.skip_unless_on_linux(reason="underlying functionality only runs on Linux")
+@pytest.mark.skipif(
+    bool(salt.utils.path.which("transactional-update")),
+    reason="Skipping on transactional systems",
+)
 def test_user_present_change_groups(modules, states, username, group_1, group_2):
     ret = states.user.present(
         name=username,
@@ -404,6 +409,10 @@ def test_user_present_change_groups(modules, states, username, group_1, group_2)
 
 
 @pytest.mark.skip_unless_on_linux(reason="underlying functionality only runs on Linux")
+@pytest.mark.skipif(
+    bool(salt.utils.path.which("transactional-update")),
+    reason="Skipping on transactional systems",
+)
 def test_user_present_change_optional_groups(
     modules, states, username, group_1, group_2
 ):
diff --git a/tests/pytests/integration/cli/test_syndic_eauth.py b/tests/pytests/integration/cli/test_syndic_eauth.py
index 8dcdd3fbd28..dde4c25bc91 100644
--- a/tests/pytests/integration/cli/test_syndic_eauth.py
+++ b/tests/pytests/integration/cli/test_syndic_eauth.py
@@ -6,8 +6,6 @@ import time
 
 import pytest
 
-from tests.conftest import CODE_DIR
-
 docker = pytest.importorskip("docker", minversion="4.0.0")
 
 INSIDE_CONTAINER = os.getenv("HOSTNAME", "") == "salt-test-container"
diff --git a/tests/pytests/integration/daemons/test_memory_leak.py b/tests/pytests/integration/daemons/test_memory_leak.py
index f2c5307f1a5..869ce72f588 100644
--- a/tests/pytests/integration/daemons/test_memory_leak.py
+++ b/tests/pytests/integration/daemons/test_memory_leak.py
@@ -49,7 +49,7 @@ def file_add_delete_sls(testfile_path, base_env_state_tree_root_dir):
 
 @pytest.mark.skip_on_darwin(reason="MacOS is a spawning platform, won't work")
 @pytest.mark.skipif(GITHUB_ACTIONS, reason="Test is failing in GitHub Actions")
-@pytest.mark.flaky(max_runs=10)
+@pytest.mark.xfail(reason="This test is flaky")
 def test_memory_leak(salt_cli, salt_minion, file_add_delete_sls):
     max_usg = None
 
diff --git a/tests/support/pytest/mysql.py b/tests/support/pytest/mysql.py
index 218c38686e7..6195d53d212 100644
--- a/tests/support/pytest/mysql.py
+++ b/tests/support/pytest/mysql.py
@@ -5,11 +5,19 @@ import attr
 import pytest
 from saltfactories.utils import random_string
 
+import salt.modules.mysql
+
 # This `pytest.importorskip` here actually works because this module
 # is imported into test modules, otherwise, the skipping would just fail
 pytest.importorskip("docker")
 import docker.errors  # isort:skip  pylint: disable=3rd-party-module-not-gated
 
+pytestmark = [
+    pytest.mark.skipif(
+        not salt.modules.mysql.MySQLdb, reason="Missing python MySQLdb library"
+    )
+]
+
 log = logging.getLogger(__name__)
 
 
-- 
2.44.0