diff --git a/python-pylint.changes b/python-pylint.changes
index 3cbb0c0..b4da957 100644
--- a/python-pylint.changes
+++ b/python-pylint.changes
@@ -1,3 +1,8 @@
+-------------------------------------------------------------------
+Thu Jun  1 07:23:25 UTC 2023 - Ben Greiner <code@bnavigator.de>
+
+- Update test setup: rerun flaky pyreverse tests
+
 -------------------------------------------------------------------
 Mon May  8 15:31:49 UTC 2023 - Dirk Müller <dmueller@suse.com>
 
diff --git a/python-pylint.spec b/python-pylint.spec
index 9681502..8c35aa6 100644
--- a/python-pylint.spec
+++ b/python-pylint.spec
@@ -59,7 +59,7 @@ BuildRequires:  %{python_module typing-extensions >= 3.10}
 # /SECTION
 # SECTION test deps
 BuildRequires:  %{python_module GitPython > 3}
-BuildRequires:  %{python_module pytest-benchmark}
+BuildRequires:  %{python_module pytest-rerunfailures}
 BuildRequires:  %{python_module pytest-timeout}
 BuildRequires:  %{python_module pytest-xdist}
 BuildRequires:  %{python_module pytest}
@@ -106,7 +106,8 @@ done
 %if %{with tests}
 %check
 export LC_ALL="en_US.UTF-8"
-%pytest %{?jobs:-n %jobs} --benchmark-disable -k "not test_linter_with_unpickleable_plugins_is_pickleable"
+# reruns: tests/pyreverse is incredibly non-deterministic in failures
+%pytest -n auto --ignore tests/benchmark --reruns 5 -rsfER -k "not test_linter_with_unpickleable_plugins_is_pickleable"
 %endif
 
 %post