diff --git a/distributed-2021.2.0.tar.gz b/distributed-2021.2.0.tar.gz deleted file mode 100644 index d39c9cb..0000000 --- a/distributed-2021.2.0.tar.gz +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9150b99b2c28e7c238c486b117154abd545c6990080d084aaff02f803f86f75e -size 693844 diff --git a/distributed-2021.3.0.tar.gz b/distributed-2021.3.0.tar.gz new file mode 100644 index 0000000..0e4210d --- /dev/null +++ b/distributed-2021.3.0.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:427fe7e047bbad7413c65d57e56fa4d6b1cf921fd206a3d2caab94bf9153c3db +size 694209 diff --git a/importorskip-numpy-tests.patch b/importorskip-numpy-tests.patch index fa6f22d..686790c 100644 --- a/importorskip-numpy-tests.patch +++ b/importorskip-numpy-tests.patch @@ -1,76 +1,148 @@ -Index: distributed-2021.2.0/distributed/protocol/tests/test_collection.py -=================================================================== ---- distributed-2021.2.0.orig/distributed/protocol/tests/test_collection.py -+++ distributed-2021.2.0/distributed/protocol/tests/test_collection.py +diff --git a/distributed/protocol/tests/test_arrow.py b/distributed/protocol/tests/test_arrow.py +index e86bfa6f..3d2c66af 100644 +--- a/distributed/protocol/tests/test_arrow.py ++++ b/distributed/protocol/tests/test_arrow.py @@ -1,7 +1,7 @@ +-import pandas as pd + import pytest + + pa = pytest.importorskip("pyarrow") ++pd = pytest.importorskip("pandas") + + import distributed + from distributed.utils_test import gen_cluster +diff --git a/distributed/protocol/tests/test_collection.py b/distributed/protocol/tests/test_collection.py +index ddb8a44b..f9f5bf79 100644 +--- a/distributed/protocol/tests/test_collection.py ++++ b/distributed/protocol/tests/test_collection.py +@@ -1,8 +1,8 @@ import pytest from distributed.protocol import serialize, deserialize - import pandas as pd +-import pandas as pd -import numpy as np -+np = pytest.importorskip("numpy") ++np = pytest.importorskip("numpy") ++pd = pytest.importorskip("pandas") @pytest.mark.parametrize("collection", [tuple, dict, list]) -Index: distributed-2021.2.0/distributed/protocol/tests/test_keras.py -=================================================================== ---- distributed-2021.2.0.orig/distributed/protocol/tests/test_keras.py -+++ distributed-2021.2.0/distributed/protocol/tests/test_keras.py -@@ -1,4 +1,4 @@ + @pytest.mark.parametrize( +diff --git a/distributed/protocol/tests/test_collection_cuda.py b/distributed/protocol/tests/test_collection_cuda.py +index e2602795..19484a87 100644 +--- a/distributed/protocol/tests/test_collection_cuda.py ++++ b/distributed/protocol/tests/test_collection_cuda.py +@@ -2,7 +2,6 @@ import pytest + + from distributed.protocol import serialize, deserialize + from dask.dataframe.utils import assert_eq +-import pandas as pd + + + @pytest.mark.parametrize("collection", [tuple, dict]) +@@ -37,14 +36,14 @@ def test_serialize_cupy(collection, y, y_serializer): + @pytest.mark.parametrize("collection", [tuple, dict]) + @pytest.mark.parametrize( + "df2,df2_serializer", +- [(pd.DataFrame({"C": [3, 4, 5], "D": [2.5, 3.5, 4.5]}), "cuda"), (None, "pickle")], ++ [({"C": [3, 4, 5], "D": [2.5, 3.5, 4.5]}, "cuda"), (None, "pickle")], + ) + def test_serialize_pandas_pandas(collection, df2, df2_serializer): + cudf = pytest.importorskip("cudf") +- ++ pd = pytest.importorskip("pandas") + df1 = cudf.DataFrame({"A": [1, 2, None], "B": [1.0, 2.0, None]}) +- if df2 is not None: +- df2 = cudf.from_pandas(df2) ++ if df2 is not None: ++ df2 = cudf.from_pandas(pd.DataFrame(df2)) + if issubclass(collection, dict): + header, frames = serialize( + {"df1": df1, "df2": df2}, serializers=("cuda", "dask", "pickle") +diff --git a/distributed/protocol/tests/test_keras.py b/distributed/protocol/tests/test_keras.py +index da8cdf63..05f3c7ca 100644 +--- a/distributed/protocol/tests/test_keras.py ++++ b/distributed/protocol/tests/test_keras.py +@@ -1,13 +1,13 @@ -import numpy as np -+np = pytest.importorskip("numpy") - from numpy.testing import assert_allclose +-from numpy.testing import assert_allclose import pytest -Index: distributed-2021.2.0/distributed/protocol/tests/test_numpy.py -=================================================================== ---- distributed-2021.2.0.orig/distributed/protocol/tests/test_numpy.py -+++ distributed-2021.2.0/distributed/protocol/tests/test_numpy.py -@@ -1,6 +1,6 @@ + keras = pytest.importorskip("keras") ++np = pytest.importorskip("numpy") + + from distributed.protocol import serialize, deserialize, dumps, loads, to_serialize + + + def test_serialize_deserialize_model(): ++ from numpy.testing import assert_allclose + model = keras.models.Sequential() + model.add(keras.layers.Dense(5, input_dim=3)) + model.add(keras.layers.Dense(2)) +diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py +index c52e4f5b..ebb709dd 100644 +--- a/distributed/protocol/tests/test_numpy.py ++++ b/distributed/protocol/tests/test_numpy.py +@@ -1,8 +1,7 @@ from zlib import crc32 -import numpy as np -+np = pytest.importorskip("numpy") import pytest - +- ++np = pytest.importorskip("numpy") from distributed.protocol import ( -Index: distributed-2021.2.0/distributed/protocol/tests/test_pandas.py -=================================================================== ---- distributed-2021.2.0.orig/distributed/protocol/tests/test_pandas.py -+++ distributed-2021.2.0/distributed/protocol/tests/test_pandas.py -@@ -1,4 +1,4 @@ + serialize, + deserialize, +diff --git a/distributed/protocol/tests/test_pandas.py b/distributed/protocol/tests/test_pandas.py +index a8134d7e..6dd968eb 100644 +--- a/distributed/protocol/tests/test_pandas.py ++++ b/distributed/protocol/tests/test_pandas.py +@@ -1,6 +1,6 @@ -import numpy as np -+np = pytest.importorskip("numpy") - import pandas as pd +-import pandas as pd import pytest ++pd = pytest.importorskip("pandas") ++np = pytest.importorskip("numpy") -Index: distributed-2021.2.0/distributed/protocol/tests/test_serialize.py -=================================================================== ---- distributed-2021.2.0.orig/distributed/protocol/tests/test_serialize.py -+++ distributed-2021.2.0/distributed/protocol/tests/test_serialize.py -@@ -3,7 +3,7 @@ import copy + from dask.dataframe.utils import assert_eq + +diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py +index 735dffb5..6ff76eb6 100644 +--- a/distributed/protocol/tests/test_serialize.py ++++ b/distributed/protocol/tests/test_serialize.py +@@ -3,10 +3,11 @@ import copy import pickle import msgpack -import numpy as np -+np = pytest.importorskip("numpy") import pytest from tlz import identity -Index: distributed-2021.2.0/distributed/protocol/tests/test_sparse.py -=================================================================== ---- distributed-2021.2.0.orig/distributed/protocol/tests/test_sparse.py -+++ distributed-2021.2.0/distributed/protocol/tests/test_sparse.py -@@ -1,4 +1,4 @@ --import numpy as np +np = pytest.importorskip("numpy") - from numpy.testing import assert_allclose ++ + from dask.utils_test import inc + + from distributed import wait +diff --git a/distributed/protocol/tests/test_sparse.py b/distributed/protocol/tests/test_sparse.py +index 89f9da09..a08be59b 100644 +--- a/distributed/protocol/tests/test_sparse.py ++++ b/distributed/protocol/tests/test_sparse.py +@@ -1,9 +1,10 @@ +-import numpy as np +-from numpy.testing import assert_allclose import pytest -Index: distributed-2021.2.0/distributed/tests/test_actor.py -=================================================================== ---- distributed-2021.2.0.orig/distributed/tests/test_actor.py -+++ distributed-2021.2.0/distributed/tests/test_actor.py -@@ -426,7 +426,7 @@ async def test_load_balance_map(c, s, *w ++np = pytest.importorskip("numpy") + sparse = pytest.importorskip("sparse") + ++from numpy.testing import assert_allclose ++ + from distributed.protocol import deserialize, serialize + + +diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py +index 7b91b3da..ee393d49 100644 +--- a/distributed/tests/test_actor.py ++++ b/distributed/tests/test_actor.py +@@ -426,7 +426,7 @@ async def test_load_balance_map(c, s, *workers): @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4, Worker=Nanny) async def bench_param_server(c, s, *workers): import dask.array as da @@ -79,11 +151,11 @@ Index: distributed-2021.2.0/distributed/tests/test_actor.py x = da.random.random((500000, 1000), chunks=(1000, 1000)) x = x.persist() -Index: distributed-2021.2.0/distributed/tests/test_client.py -=================================================================== ---- distributed-2021.2.0.orig/distributed/tests/test_client.py -+++ distributed-2021.2.0/distributed/tests/test_client.py -@@ -5522,7 +5522,7 @@ async def test_client_active_bad_port(): +diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py +index 100b52ea..40716c40 100644 +--- a/distributed/tests/test_client.py ++++ b/distributed/tests/test_client.py +@@ -5526,7 +5526,7 @@ async def test_client_active_bad_port(): def test_turn_off_pickle(direct): @gen_cluster() async def test(s, a, b): @@ -92,7 +164,7 @@ Index: distributed-2021.2.0/distributed/tests/test_client.py async with Client( s.address, asynchronous=True, serializers=["dask", "msgpack"] -@@ -5560,7 +5560,7 @@ def test_turn_off_pickle(direct): +@@ -5564,7 +5564,7 @@ def test_turn_off_pickle(direct): @gen_cluster() async def test_de_serialization(s, a, b): @@ -101,7 +173,7 @@ Index: distributed-2021.2.0/distributed/tests/test_client.py c = await Client( s.address, -@@ -5581,7 +5581,7 @@ async def test_de_serialization(s, a, b) +@@ -5585,7 +5585,7 @@ async def test_de_serialization(s, a, b): @gen_cluster() async def test_de_serialization_none(s, a, b): @@ -110,7 +182,7 @@ Index: distributed-2021.2.0/distributed/tests/test_client.py c = await Client(s.address, asynchronous=True, deserializers=["msgpack"]) try: -@@ -6420,7 +6420,7 @@ async def test_annotations_retries(c, s, +@@ -6424,7 +6424,7 @@ async def test_annotations_retries(c, s, a, b): async def test_annotations_blockwise_unpack(c, s, a, b): da = pytest.importorskip("dask.array") from dask.array.utils import assert_eq @@ -119,23 +191,34 @@ Index: distributed-2021.2.0/distributed/tests/test_client.py # A flaky doubling function -- need extra args because it is called before # application to establish dtype/meta. -Index: distributed-2021.2.0/distributed/tests/test_collections.py -=================================================================== ---- distributed-2021.2.0.orig/distributed/tests/test_collections.py -+++ distributed-2021.2.0/distributed/tests/test_collections.py -@@ -11,7 +11,7 @@ import dask.bag as db +diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py +index 022db408..8b3e6cd5 100644 +--- a/distributed/tests/test_collections.py ++++ b/distributed/tests/test_collections.py +@@ -2,8 +2,8 @@ from distutils.version import LooseVersion + + import pytest + +-pytest.importorskip("numpy") +-pytest.importorskip("pandas") ++np = pytest.importorskip("numpy") ++pd = pytest.importorskip("pandas") + + import dask + import dask.dataframe as dd +@@ -11,8 +11,6 @@ import dask.bag as db from distributed.client import wait from distributed.utils_test import gen_cluster from distributed.utils_test import client, cluster_fixture, loop # noqa F401 -import numpy as np -+np = pytest.importorskip("numpy") - import pandas as pd +-import pandas as pd PANDAS_VERSION = LooseVersion(pd.__version__) -Index: distributed-2021.2.0/distributed/tests/test_nanny.py -=================================================================== ---- distributed-2021.2.0.orig/distributed/tests/test_nanny.py -+++ distributed-2021.2.0/distributed/tests/test_nanny.py + PANDAS_GT_100 = PANDAS_VERSION >= LooseVersion("1.0.0") +diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py +index 1993f11a..83c3de18 100644 +--- a/distributed/tests/test_nanny.py ++++ b/distributed/tests/test_nanny.py @@ -7,8 +7,6 @@ import random import sys import multiprocessing as mp @@ -145,7 +228,7 @@ Index: distributed-2021.2.0/distributed/tests/test_nanny.py import pytest from tlz import valmap, first from tornado.ioloop import IOLoop -@@ -201,6 +199,7 @@ async def test_random_seed(c, s, a, b): +@@ -202,6 +200,7 @@ async def test_random_seed(c, s, a, b): assert x != y await check_func(lambda a, b: random.randint(a, b)) @@ -153,10 +236,10 @@ Index: distributed-2021.2.0/distributed/tests/test_nanny.py await check_func(lambda a, b: np.random.randint(a, b)) -Index: distributed-2021.2.0/distributed/tests/test_pubsub.py -=================================================================== ---- distributed-2021.2.0.orig/distributed/tests/test_pubsub.py -+++ distributed-2021.2.0/distributed/tests/test_pubsub.py +diff --git a/distributed/tests/test_pubsub.py b/distributed/tests/test_pubsub.py +index 8f8a3b73..09ab4dcd 100644 +--- a/distributed/tests/test_pubsub.py ++++ b/distributed/tests/test_pubsub.py @@ -37,7 +37,7 @@ async def test_speed(c, s, a, b): # print(a, b, i) return n @@ -166,10 +249,10 @@ Index: distributed-2021.2.0/distributed/tests/test_pubsub.py x = np.random.random(1000) -Index: distributed-2021.2.0/distributed/tests/test_utils.py -=================================================================== ---- distributed-2021.2.0.orig/distributed/tests/test_utils.py -+++ distributed-2021.2.0/distributed/tests/test_utils.py +diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py +index 022e9925..ffda15a6 100644 +--- a/distributed/tests/test_utils.py ++++ b/distributed/tests/test_utils.py @@ -10,7 +10,6 @@ import sys from time import sleep import traceback diff --git a/python-distributed.changes b/python-distributed.changes index 0d39afd..9abeb1e 100644 --- a/python-distributed.changes +++ b/python-distributed.changes @@ -1,3 +1,45 @@ +------------------------------------------------------------------- +Sun Mar 7 16:46:23 UTC 2021 - Ben Greiner + +- Update to 2021.3.0 + * This is the first release with support for Python 3.9 and the + last release with support for Python 3.6 + * tcp.write(): cast memoryview to byte itemsize (#4555) + Mads R. B. Kristensen + * Refcount the thread_state.asynchronous flag (#4557) Mads + R. B. Kristensen + * Python 3.9 (#4460) crusaderky + * Better bokeh defaults for dashboard (#4554) Benjamin + Zaitlen + * Expose system monitor dashboard as individual plot for lab + extension (#4540) Jacob Tomlinson + * Pass on original temp dir from nanny to worker (#4549) + Martin Durant + * Serialize and split (#4541) Mads R. B. Kristensen + * Use the new HLG pack/unpack API in Dask (#4489) Mads R. + B. Kristensen + * Handle annotations for culled tasks (#4544) Tom + Augspurger + * Make sphinx autosummary and autoclass consistent (#4367) + Casey Clements + * Move _transition* to SchedulerState (#4545) jakirkham + * Migrate from travis to GitHub actions (#4504) crusaderky + * Move new_task to SchedulerState (#4527) jakirkham + * Batch more Scheduler sends (#4526) jakirkham + * transition_memory_released and get_nbytes() optimizations + (#4516) jakirkham + * Pin black pre-commit (#4533) James Bourbeau + * Read & write all frames in one pass (#4506) jakirkham + * Skip stream.write call for empty frames (#4507) jakirkham + * Prepend frame metadata header (#4505) jakirkham + * transition_processing_memory optimizations, etc. (#4487) + jakirkham + * Attempt to get client from worker in Queue and Variable + (#4490) James Bourbeau + * Use main branch for zict (#4499) jakirkham + * Use a callback to close TCP Comms, rather than check every + time (#4453) Matthew Rocklin + ------------------------------------------------------------------- Sun Feb 21 18:19:45 UTC 2021 - Ben Greiner diff --git a/python-distributed.spec b/python-distributed.spec index 83a3195..fd2fb47 100644 --- a/python-distributed.spec +++ b/python-distributed.spec @@ -27,7 +27,7 @@ %define skip_python2 1 %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-distributed%{psuffix} -Version: 2021.2.0 +Version: 2021.3.0 Release: 0 Summary: Library for distributed computing with Python License: BSD-3-Clause @@ -44,7 +44,7 @@ Requires: python-PyYAML Requires: python-certifi Requires: python-click >= 6.6 Requires: python-cloudpickle >= 1.5.0 -Requires: python-dask >= 2021.2.0 +Requires: python-dask >= %{version} Requires: python-msgpack Requires: python-psutil >= 5.0 Requires: python-sortedcontainers @@ -63,7 +63,7 @@ BuildRequires: %{python_module PyYAML} BuildRequires: %{python_module certifi} BuildRequires: %{python_module click >= 6.6} BuildRequires: %{python_module cloudpickle >= 1.5.0} -BuildRequires: %{python_module dask-all >= 2021.2.0} +BuildRequires: %{python_module dask-all >= %{version}} # need built extension BuildRequires: %{python_module contextvars if %python-base < 3.7} BuildRequires: %{python_module distributed = %{version}} @@ -73,6 +73,8 @@ BuildRequires: %{python_module jupyter_client} BuildRequires: %{python_module msgpack} BuildRequires: %{python_module psutil} BuildRequires: %{python_module pytest-asyncio} +BuildRequires: %{python_module pytest-rerunfailures} +BuildRequires: %{python_module pytest-timeout} BuildRequires: %{python_module pytest-xdist} BuildRequires: %{python_module pytest} BuildRequires: %{python_module requests} @@ -94,8 +96,6 @@ clusters. %prep %autosetup -p1 -n distributed-%{version} -# gh#dask/distributed#4467 -sed -i 's/raise pytest.skip(reason=/raise pytest.skip(/' distributed/tests/test_core.py %build %if ! %{with test} @@ -118,6 +118,7 @@ sed -i '/pytest/,/addopts/ s/-rsx/-rfEs/' setup.cfg # many tests from multiple files are broken by new pytest-asyncio # (see https://github.com/dask/distributed/pull/4212 and https://github.com/pytest-dev/pytest-asyncio/issues/168) # as a proof build it with old pytest-asyncio and see these tests pass +%if %{pkg_vcmp python3-pytest-asyncio >= 0.14} donttest+=" or (test_asyncprocess and test_child_main_thread)" donttest+=" or (test_asyncprocess and test_close)" donttest+=" or (test_asyncprocess and test_exit_callback)" @@ -142,8 +143,6 @@ donttest+=" or (test_client and test_reconnect)" donttest+=" or (test_client and test_secede_balances)" donttest+=" or (test_client and test_secede_simple)" donttest+=" or (test_client and test_serialize_collections)" -donttest+=" or (test_client and test_upload_file_exception_sync)" -donttest+=" or (test_client and test_upload_file_sync)" donttest+=" or (test_collections and test_sparse_arrays)" donttest+=" or (test_diskutils and test_workspace_concurrency_intense)" donttest+=" or (test_diskutils and test_workspace_concurrency)" @@ -172,6 +171,7 @@ donttest+=" or (test_resources and test_prefer_constrained)" donttest+=" or (test_scheduler and test_balance_many_workers_2)" donttest+=" or (test_scheduler and test_balance_many_workers)" donttest+=" or (test_scheduler and test_bandwidth_clear)" +donttest+=" or (test_scheduler and test_dont_recompute_if_persisted)" donttest+=" or (test_scheduler and test_file_descriptors)" donttest+=" or (test_scheduler and test_gather_allow_worker_reconnect)" donttest+=" or (test_scheduler and test_idle_timeout)" @@ -224,14 +224,23 @@ donttest+=" or (test_worker_client and test_scatter_singleton)" donttest+=" or (test_worker_client and test_secede_without_stealing_issue_1262)" donttest+=" or (test_worker_client and test_submit_different_names)" donttest+=" or (test_worker_client and test_submit_from_worker)" -donttest+=" or test_metrics" +%endif +# too slow for obs +donttest+=" or (test_metrics and time)" +donttest+=" or (test_client and test_upload_file_exception_sync)" +donttest+=" or (test_client and test_upload_file_sync)" # randomly fails donttest+=" or (test_worker and test_fail_write_to_disk)" # false version mismatch donttest+=" or test_version_warning_in_cluster" # ambiguous order in returned message donttest+=" or (test_client and test_as_completed_async_for_cancel)" -%pytest_arch distributed/tests/ -k "not (${donttest:4})" -m "not avoid_travis" -n auto +# indefinite hangs -- https://github.com/dask/distributed/issues/4564 +donttest+=" or (test_queues and test_queue_in_task)" +donttest+=" or (test_variable and test_variable_in_task)" +# too many open files +donttest+=" or (test_stress and test_stress_communication)" +%pytest_arch -ra -n auto distributed/tests/ -k "not (${donttest:4})" -m "not avoid_travis" --timeout 180 %endif %if ! %{with test}