diff --git a/allow-bokeh3.patch b/allow-bokeh3.patch new file mode 100644 index 0000000..9795657 --- /dev/null +++ b/allow-bokeh3.patch @@ -0,0 +1,13 @@ +Index: distributed-2022.11.1/distributed/versions.py +=================================================================== +--- distributed-2022.11.1.orig/distributed/versions.py ++++ distributed-2022.11.1/distributed/versions.py +@@ -13,7 +13,7 @@ from types import ModuleType + from typing import Any + + MIN_BOKEH_VERSION = "2.4.2" +-MAX_BOKEH_VERSION = "2.4.3" ++MAX_BOKEH_VERSION = "4" + + required_packages = [ + ("dask", lambda p: p.__version__), diff --git a/distributed-2022.10.0-gh.tar.gz b/distributed-2022.10.0-gh.tar.gz deleted file mode 100644 index 25b3848..0000000 --- a/distributed-2022.10.0-gh.tar.gz +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7ae192269e40d76d7af24efa9ce2161048db2be1f66d8d7319e07107bfceba4f -size 1754746 diff --git a/distributed-2022.11.1-gh.tar.gz b/distributed-2022.11.1-gh.tar.gz new file mode 100644 index 0000000..9c6e7a4 --- /dev/null +++ b/distributed-2022.11.1-gh.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2e0e17d5f7addc5049829b3f84802bde9a89a8bed196b8a689d7c4f980e3a22 +size 1785156 diff --git a/distributed-pr7286-tornado-6-2.patch b/distributed-pr7286-tornado-6-2.patch new file mode 100644 index 0000000..d2ad862 --- /dev/null +++ b/distributed-pr7286-tornado-6-2.patch @@ -0,0 +1,204 @@ +From 76f2c6d58187ee263b7df5db8eb8dc6829458c3e Mon Sep 17 00:00:00 2001 +From: Thomas Grainger +Date: Thu, 10 Nov 2022 12:09:38 +0000 +Subject: [PATCH 1/3] upgrade to tornado 6.2 + +Co-Authored-By: Lawrence Mitchell +--- + continuous_integration/environment-3.10.yaml | 2 +- + continuous_integration/environment-3.8.yaml | 2 +- + continuous_integration/environment-3.9.yaml | 2 +- + requirements.txt | 2 +- + 4 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/continuous_integration/environment-3.10.yaml b/continuous_integration/environment-3.10.yaml +index e5549dd19d..8cc9615cda 100644 +--- a/continuous_integration/environment-3.10.yaml ++++ b/continuous_integration/environment-3.10.yaml +@@ -38,7 +38,7 @@ dependencies: + - sortedcollections + - tblib + - toolz +- - tornado<6.2 ++ - tornado >=6.2 + - zict # overridden by git tip below + - zstandard >=0.9.0 + - pip: +diff --git a/continuous_integration/environment-3.8.yaml b/continuous_integration/environment-3.8.yaml +index 353be57689..fa5cbefe96 100644 +--- a/continuous_integration/environment-3.8.yaml ++++ b/continuous_integration/environment-3.8.yaml +@@ -41,7 +41,7 @@ dependencies: + - sortedcollections + - tblib + - toolz +- - tornado<6.2 ++ - tornado >=6.2 + - zict + - zstandard >=0.9.0 + - pip: +diff --git a/continuous_integration/environment-3.9.yaml b/continuous_integration/environment-3.9.yaml +index 46362af94b..dfd82c00f0 100644 +--- a/continuous_integration/environment-3.9.yaml ++++ b/continuous_integration/environment-3.9.yaml +@@ -44,7 +44,7 @@ dependencies: + - tblib + - toolz + - torchvision # Only tested here +- - tornado<6.2 ++ - tornado >=6.2 + - zict + - zstandard >=0.9.0 + - pip: +diff --git a/requirements.txt b/requirements.txt +index 6d0f0cdbca..8e7f7180b0 100644 +--- a/requirements.txt ++++ b/requirements.txt +@@ -9,7 +9,7 @@ psutil >= 5.0 + sortedcontainers !=2.0.0, !=2.0.1 + tblib >= 1.6.0 + toolz >= 0.10.0 +-tornado >= 6.0.3, <6.2 ++tornado >= 6.0.3 + urllib3 + zict >= 0.1.3 + pyyaml + +From 9f89068f1dde4afa0bfd2677f7e377cf4878edde Mon Sep 17 00:00:00 2001 +From: Lawrence Mitchell +Date: Thu, 13 Oct 2022 10:49:18 +0100 +Subject: [PATCH 2/3] Filter implicit deprecation warnings from tornado as well + +tornado 6.2 deprecates functionality that relies on deprecated asyncio +calls (get_event_loop and friends); a few tests still use a pattern of +implicitly creating a new loop (and test that appropriate deprecation +warnings are uttered), so just catch these new tornado warnings for +now, until the functionality is removed in distributed. +--- + distributed/deploy/tests/test_adaptive.py | 1 + + distributed/deploy/tests/test_spec_cluster.py | 1 + + distributed/tests/test_client.py | 4 ++++ + distributed/tests/test_client_loop.py | 2 ++ + distributed/tests/test_utils.py | 2 ++ + 5 files changed, 10 insertions(+) + +diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py +index d88615a756..59ec50eaab 100644 +--- a/distributed/deploy/tests/test_adaptive.py ++++ b/distributed/deploy/tests/test_adaptive.py +@@ -281,6 +281,7 @@ async def test_no_more_workers_than_tasks(): + + + @pytest.mark.filterwarnings("ignore:There is no current event loop:DeprecationWarning") ++@pytest.mark.filterwarnings("ignore:make_current is deprecated:DeprecationWarning") + def test_basic_no_loop(cleanup): + loop = None + try: +diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py +index 31b95e2802..e0e7301fcf 100644 +--- a/distributed/deploy/tests/test_spec_cluster.py ++++ b/distributed/deploy/tests/test_spec_cluster.py +@@ -83,6 +83,7 @@ def test_spec_sync(loop): + + + @pytest.mark.filterwarnings("ignore:There is no current event loop:DeprecationWarning") ++@pytest.mark.filterwarnings("ignore:make_current is deprecated:DeprecationWarning") + def test_loop_started_in_constructor(cleanup): + # test that SpecCluster.__init__ starts a loop in another thread + cluster = SpecCluster(worker_spec, scheduler=scheduler, loop=None) +diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py +index 590fc13fd7..85e28262e7 100644 +--- a/distributed/tests/test_client.py ++++ b/distributed/tests/test_client.py +@@ -2889,6 +2889,7 @@ async def test_startup_close_startup(s, a, b): + + + @pytest.mark.filterwarnings("ignore:There is no current event loop:DeprecationWarning") ++@pytest.mark.filterwarnings("ignore:make_current is deprecated:DeprecationWarning") + def test_startup_close_startup_sync(loop): + with cluster() as (s, [a, b]): + with Client(s["address"], loop=loop) as c: +@@ -5613,6 +5614,8 @@ async def test_future_auto_inform(c, s, a, b): + + + @pytest.mark.filterwarnings("ignore:There is no current event loop:DeprecationWarning") ++@pytest.mark.filterwarnings("ignore:make_current is deprecated:DeprecationWarning") ++@pytest.mark.filterwarnings("ignore:clear_current is deprecated:DeprecationWarning") + def test_client_async_before_loop_starts(cleanup): + async def close(): + async with client: +@@ -6924,6 +6927,7 @@ async def test_workers_collection_restriction(c, s, a, b): + + + @pytest.mark.filterwarnings("ignore:There is no current event loop:DeprecationWarning") ++@pytest.mark.filterwarnings("ignore:make_current is deprecated:DeprecationWarning") + @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)]) + async def test_get_client_functions_spawn_clusters(c, s, a): + # see gh4565 +diff --git a/distributed/tests/test_client_loop.py b/distributed/tests/test_client_loop.py +index 35bb3a7e86..9d8316cdd0 100644 +--- a/distributed/tests/test_client_loop.py ++++ b/distributed/tests/test_client_loop.py +@@ -30,6 +30,7 @@ def _check_cluster_and_client_loop(loop): + + # Test if Client stops LoopRunner on close. + @pytest.mark.filterwarnings("ignore:There is no current event loop:DeprecationWarning") ++@pytest.mark.filterwarnings("ignore:make_current is deprecated:DeprecationWarning") + def test_close_loop_sync_start_new_loop(cleanup): + with _check_loop_runner(): + _check_cluster_and_client_loop(loop=None) +@@ -37,6 +38,7 @@ def test_close_loop_sync_start_new_loop(cleanup): + + # Test if Client stops LoopRunner on close. + @pytest.mark.filterwarnings("ignore:There is no current event loop:DeprecationWarning") ++@pytest.mark.filterwarnings("ignore:make_current is deprecated:DeprecationWarning") + def test_close_loop_sync_use_running_loop(cleanup): + with _check_loop_runner(): + # Start own loop or use current thread's one. +diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py +index 853ba578f1..b2dae76fdb 100644 +--- a/distributed/tests/test_utils.py ++++ b/distributed/tests/test_utils.py +@@ -405,6 +405,7 @@ def assert_not_running(loop): + + + @pytest.mark.filterwarnings("ignore:There is no current event loop:DeprecationWarning") ++@pytest.mark.filterwarnings("ignore:make_current is deprecated:DeprecationWarning") + def test_loop_runner(loop_in_thread): + # Implicit loop + loop = IOLoop() +@@ -488,6 +489,7 @@ def test_loop_runner(loop_in_thread): + + + @pytest.mark.filterwarnings("ignore:There is no current event loop:DeprecationWarning") ++@pytest.mark.filterwarnings("ignore:make_current is deprecated:DeprecationWarning") + def test_two_loop_runners(loop_in_thread): + # Loop runners tied to the same loop should cooperate + + +From fd74414f63d95f54bfcc25112b9d5b7d7f3168a8 Mon Sep 17 00:00:00 2001 +From: Thomas Grainger +Date: Thu, 10 Nov 2022 15:01:59 +0000 +Subject: [PATCH 3/3] get loop from fixture + +--- + distributed/tests/test_client.py | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py +index 85e28262e7..a063dd4eea 100644 +--- a/distributed/tests/test_client.py ++++ b/distributed/tests/test_client.py +@@ -6593,9 +6593,9 @@ async def test_as_completed_condition_loop(c, s, a, b): + sys.version_info >= (3, 10), + reason="On Py3.10+ semaphore._loop is not bound until .acquire() blocks", + ) +-def test_client_connectionpool_semaphore_loop(s, a, b): +- with Client(s["address"]) as c: +- assert c.rpc.semaphore._loop is c.loop.asyncio_loop ++def test_client_connectionpool_semaphore_loop(s, a, b, loop): ++ with Client(s["address"], loop=loop) as c: ++ assert c.rpc.semaphore._loop is loop.asyncio_loop + + + @pytest.mark.slow diff --git a/python-distributed.changes b/python-distributed.changes index dc3f84b..5131070 100644 --- a/python-distributed.changes +++ b/python-distributed.changes @@ -1,3 +1,150 @@ +------------------------------------------------------------------- +Sun Nov 20 10:38:22 UTC 2022 - Ben Greiner + +- Update to version 2022.11.1 + ## Enhancements + * Restrict bokeh=3 support (GH#7329) Gabe Joseph (ignored in rpm + fixed by bokek 3.0.2, see gh#dask/dask#9659) + * Respect death timeout when waiting for scheduler file (GH#7296) + Florian Jetter + * Always raise exception if P2PShuffle s send fails (GH#7317) + Hendrik Makait + ## Maintenance + * Remove exception handling from transitions (GH#7316) crusaderky + * Turn private functions into private SchedulerState methods + (GH#7260) Hendrik Makait + * Bump toolz minimum version to 0.10.0 (GH#7309) Sam Grayson +- Release 2022.11.0 + ## Note + * This release changes the default scheduling mode to use + queuing. This will significantly reduce cluster memory use in + most cases, and generally improve stability and performance. + Learn more here and please provide any feedback on this + discussion. + * In rare cases, this could make some workloads slower. See the + documentation for more information, and how to switch back to + the old mode. + ## New Features + * Add ForwardOutput worker plugin to forward stdout and stderr to + client. (GH#7297) Hendrik Makait + * Duration counters on prefix level (GH#7288) Florian Jetter + * Include button for launching JupyterLab layout in repr + (GH#7218) Ian Rose + ## Enhancements + * Support MIG parsing during CUDA context creation in UCX + initialization (GH#6720) Peter Andreas Entschev + * Handle /metrics endpoint without prometheus-client installed + (GH#7234) Hendrik Makait + * Add support for unpacking namedtuples in remote data (GH#7282) + Andrew + * Enable queuing by default (GH#7279) Florian Jetter + * Fix exists->``exist`` typo in scheduler error messages + (GH#7281) Matthew Plough + * If there’s an exception in the Client async context manager + body then close fast (GH#6920) Thomas Grainger + ## Bug Fixes + * Ensure category is optional when logging "warn" events + (GH#7169) James Bourbeau + * Edge and impossible transitions to memory (GH#7205) crusaderky + ## Maintenance + * Allow bokeh=3 (GH#5648) James Bourbeau + * Fix typos in P2P shuffle code (GH#7304) Hendrik Makait + * Reenable test_bad_disk (GH#7300) Florian Jetter + * Reduce max-runs in test reports (GH#7299) Florian Jetter + * Revert idle classification when worker-saturation is set + (GH#7278) Florian Jetter + * Fix flaky deadline_expiration (GH#7287) Florian Jetter + * Rewrite of P2P control flow (GH#7268) Florian Jetter + * Bump minimum bokeh version to 2.4.2 (GH#7271) James Bourbeau + * Remove deprecated code calls to IOLoop.make_current() (GH#7240) + Thomas Grainger + * Improved test for balancing expensive tasks (GH#7272) Hendrik + Makait + * Refactor semaphore._Watch into general-purpose Deadline utility + (GH#7238) Hendrik Makait + * Require Click 7.0+ (GH#7226) jakirkham + * Drop tests (GH#7269) Hendrik Makait + * Replace test_(do_not_)steal_communication_heavy_tasks tests + with more robust versions (GH#7243) Hendrik Makait + * xfail test_bad_disk (GH#7265) crusaderky + * Move transition_log from Scheduler to SchedulerState (GH#7254) + crusaderky + * Remove Scheduler.log (GH#7258) crusaderky + * Use latest pickle (GH#5826) jakirkham + * Polish parsing of worker-saturation from config (GH#7255) + crusaderky + * Avoid expensive occupancy calculation when unused (GH#7257) + Gabe Joseph + * Un-skip test_nested_compute (GH#7247) Gabe Joseph + * Review test_do_not_steal_communication_heavy_tasks (GH#7250) + crusaderky + * Fix test_stress_creation_and_deletion (GH#7215) crusaderky + * Raise exceptions in Server.handle_stream instead of + swallowing/logging (GH#7162) Hendrik Makait + * Fix _update_scheduler_info hanging failed tests (GH#7225) Gabe + Joseph + * Bump xarray-contrib/ci-trigger from 1.1 to 1.2 (GH#7232) +- Release 2022.10.2 + * Reverted a bug where Bokeh was accidentally made non-optional + (GH#7230) Oliver Holworthy + * Schedule a queued task when a task secedes (GH#7224) Gabe + Joseph + * This was a hotfix release +- Release 2022.10.1 + ## New Features + * Add Client.restart_workers method (GH#7154) James Bourbeau + * Implement PackageInstall plugin for pip and conda (GH#7126) + Hendrik Makait + ## Enhancements + * Add prometheus collector for work-stealing (GH#7206) Hendrik + Makait + * Track reason of workers closing and restarting (GH#7166) + Hendrik Makait + * Show no-worker on task progress bar (GH#7171) Florian Jetter + * Set OPENBLAS_NUM_THREADS by default (GH#7177) James Bourbeau + * Optionally provide local directory to data constructor + (GH#7153) Lawrence Mitchell + * Introduce distributed.comm.ucx.environment config slot + (GH#7164) Lawrence Mitchell + * Log information about memory limit (GH#7160) Florian Jetter + * Improve log messages on scheduler for restart (GH#7150) Florian + Jetter + * More comprehensive WorkerState task counters (GH#7167) + crusaderky + ## Bug Fixes + * Update minimum bokeh version message (GH#7172) James Bourbeau + * Revamped implementations of remote print() and warn(), fixing + #7095 (GH#7129) Max Bane + ## Maintenance + * Temporarily restrict bokeh<3 (GH#7219) James Bourbeau + * Make Scheduler.reschedule private (GH#7216) crusaderky + * Fix decide_worker_rootish_queuing_disabled assert (GH#7065) + Gabe Joseph + * Fix flaky test_include_communication_in_occupancy (GH#7212) + Gabe Joseph + * Do not raise on leaked websockets (GH#7199) Florian Jetter + * Update nightly recipes with CLI tests, dependency changes + (GH#7201) Charles Blackmon-Luca + * Make p2p shuffle submodules private (GH#7186) Florian Jetter + * Backport tornado PeriodicCallback (GH#7165) Florian Jetter + * Fix mypy failure on CI (GH#7198) Florian Jetter + * User a layer for p2p shuffle (GH#7180) Florian Jetter + * Type annotations for shuffle (GH#7185) Florian Jetter + * Do not close worker on comm error in heartbeat (GH#7163) + Hendrik Makait + * Errors when setting TCP timeouts log as error (GH#7161) Florian + Jetter + * Remove incorrect advice from pre-commit config (GH#7159) + crusaderky + * Use functionalities network for codecov uploader (GH#7148) + Florian Jetter + * Use counter metric type where appropriate, incoming_count was + reporting bytes (GH#7125) Nat Tabris +- Add allow-bokeh3.patch -- raise upper bokeh version, reverts part + of gh#dask/distributed#7329, see also gh#dask/dask#9659 +- Replace support-tornado-6-2.patch with + distributed-pr7286-tornado-6-2.patch -- gh#dask/distributed#7286 + ------------------------------------------------------------------- Fri Oct 21 13:22:30 UTC 2022 - Ben Greiner diff --git a/python-distributed.spec b/python-distributed.spec index 8ddb37e..c863512 100644 --- a/python-distributed.spec +++ b/python-distributed.spec @@ -39,32 +39,33 @@ %if "%{flavor}" == "" %bcond_with test %endif -%{?!python_module:%define python_module() python3-%{**}} -%define skip_python2 1 # use this to run tests with xdist in parallel, unfortunately fails server side %bcond_with paralleltests + Name: python-distributed%{psuffix} # ===> Note: python-dask MUST be updated in sync with python-distributed! <=== -Version: 2022.10.0 +Version: 2022.11.1 Release: 0 Summary: Library for distributed computing with Python License: BSD-3-Clause URL: https://distributed.dask.org Source: https://github.com/dask/distributed/archive/refs/tags/%{version}.tar.gz#/distributed-%{version}-gh.tar.gz Source99: python-distributed-rpmlintrc +# PATCH-FIX-UPSTREAM allow-bokeh3.patch -- raise upper bokeh version, reverts part of gh#dask/distributed#7329, see also gh#dask/dask#9659 +Patch1: allow-bokeh3.patch +# PATCH-FIX-UPSTREAM distributed-pr7286-tornado-6-2.patch gh#dask/distributed#7286 +Patch2: distributed-pr7286-tornado-6-2.patch # PATCH-FIX-OPENSUSE distributed-ignore-off.patch -- ignore that we can't probe addresses on obs, code@bnavigator.de -Patch1: distributed-ignore-offline.patch +Patch3: distributed-ignore-offline.patch # PATCH-FIX-OPENSUSE distributed-ignore-thread-leaks.patch -- ignore leaking threads on obs, code@bnavigator.de -Patch2: distributed-ignore-thread-leaks.patch -# PATCH-FIX-OPENSUSE Ignore two deprecations introduced by Tornado 6.2 -Patch3: support-tornado-6-2.patch +Patch4: distributed-ignore-thread-leaks.patch BuildRequires: %{python_module base >= 3.8} BuildRequires: %{python_module setuptools} BuildRequires: fdupes BuildRequires: python-rpm-macros Requires: python-PyYAML Requires: python-certifi -Requires: python-click >= 6.6 +Requires: python-click >= 7.0 Requires: python-cloudpickle >= 1.5.0 Requires: python-dask = %{version} Requires: python-locket >= 1.0.0 @@ -73,7 +74,7 @@ Requires: python-packaging >= 20.0 Requires: python-psutil >= 5.0 Requires: python-sortedcontainers Requires: python-tblib -Requires: python-toolz >= 0.8.2 +Requires: python-toolz >= 0.10.0 Requires: python-tornado >= 6.2 Requires: python-urllib3 Requires: python-zict >= 0.1.3 @@ -81,7 +82,7 @@ Requires(post): update-alternatives Requires(postun):update-alternatives BuildArch: noarch %if %{with test} -BuildRequires: %{python_module bokeh} +BuildRequires: %{python_module bokeh >= 2.4.2} BuildRequires: %{python_module dask-complete = %{version}} BuildRequires: %{python_module distributed = %{version}} BuildRequires: %{python_module ipykernel} @@ -154,16 +155,11 @@ donttest+=" or (test_variable and test_variable_in_task)" donttest+=" or (test_worker and test_worker_reconnects_mid_compute)" # server-side fail due to the non-network warning in a subprocess where the patched filter does not apply donttest+=" or (test_client and test_quiet_close_process)" -# creates OOM aborts on some obs workers -donttest+=" or (test_steal and steal_communication_heavy_tasks)" - if [[ $(getconf LONG_BIT) -eq 32 ]]; then # OverflowError -- https://github.com/dask/distributed/issues/5252 donttest+=" or test_ensure_spilled_immediately" donttest+=" or test_value_raises_during_spilling" donttest+=" or test_fail_to_pickle_execute_1" - # https://github.com/dask/distributed/issues/7174 - donttest+=" or (test_steal and steal_communication_heavy_tasks)" # https://github.com/dask/distributed/issues/7175 donttest+=" or (test_sizeof_error and larger)" fi diff --git a/support-tornado-6-2.patch b/support-tornado-6-2.patch deleted file mode 100644 index a584f0a..0000000 --- a/support-tornado-6-2.patch +++ /dev/null @@ -1,13 +0,0 @@ -Index: distributed-2022.8.1/setup.cfg -=================================================================== ---- distributed-2022.8.1.orig/setup.cfg -+++ distributed-2022.8.1/setup.cfg -@@ -58,6 +58,8 @@ filterwarnings = - error - ignore:Please use `dok_matrix` from the `scipy\.sparse` namespace, the `scipy\.sparse\.dok` namespace is deprecated.:DeprecationWarning - ignore:elementwise comparison failed. this will raise an error in the future:DeprecationWarning -+ ignore:clear_current is deprecated:DeprecationWarning -+ ignore:make_current is deprecated; start the event loop first:DeprecationWarning - ignore:unclosed