Accepting request 907453 from devel:languages:python:numeric

OBS-URL: https://build.opensuse.org/request/show/907453
OBS-URL: https://build.opensuse.org/package/show/openSUSE:Factory/python-distributed?expand=0&rev=48

conftest.py

@@ -0,0 +1,37 @@
+# https://pytest.org/latest/example/simple.html#control-skipping-of-tests-according-to-command-line-option
+import pytest
+
+# Uncomment to enable more logging and checks
+# (https://docs.python.org/3/library/asyncio-dev.html)
+# Note this makes things slower and might consume much memory.
+# os.environ["PYTHONASYNCIODEBUG"] = "1"
+
+try:
+    import faulthandler
+except ImportError:
+    pass
+else:
+    try:
+        faulthandler.enable()
+    except Exception:
+        pass
+
+# Make all fixtures available
+from distributed.utils_test import *  # noqa
+
+
+def pytest_addoption(parser):
+    parser.addoption("--runslow", action="store_true", help="run slow tests")
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--runslow"):
+        # --runslow given in cli: do not skip slow tests
+        return
+    skip_slow = pytest.mark.skip(reason="need --runslow option to run")
+    for item in items:
+        if "slow" in item.keywords:
+            item.add_marker(skip_slow)
+
+
+pytest_plugins = ["distributed.pytest_resourceleaks"]

distributed-2021.5.0.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4da09db7972120db0a8e7fc2b2f91bb4d952ce21b185ccc1356603613060a0b3
-size 717781

distributed-2021.7.0.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6257f399ea564bfdcd80dcc9df6cdaf703dbadb94b7c068d103f8366dc7f8d1f
+size 1065464

python-distributed.changes

@@ -1,3 +1,117 @@
+-------------------------------------------------------------------
+Fri Jul 16 09:31:13 UTC 2021 - Ben Greiner <code@bnavigator.de>
+
+- Update to version 2021.7.0
+  * Fix Nbytes jitter - less expensive
+  * Use native GH actions cancel feature
+  * Don't require workers to report to scheduler if scheduler
+    shuts down
+  * Add pandas to the list of checked packages for client.
+    get_versions()
+  * Move worker preload before scheduler address is set
+  * Fix flaky test_oversubscribing_leases
+  * Update scheduling policy docs for #4967
+  * Add echo handler to Server class
+  * Also include pngs when bundling package
+  * Remove duplicated dashboard panes
+  * Fix worker memory dashboard flickering
+  * Tabs on bottom left corner on dashboard
+  * Rename nbytes widgets
+  * Co-assign root-ish tasks
+  * OSError tweaks
+  * Update imports to cudf.testing._utils
+  * Ensure shuffle split default durations uses proper prefix
+  * Follow up pyupgrade formatting
+  * Rename plot dropdown
+  * Pyupgrade
+  * Misc Sphinx tweaks
+  * No longer hold dependencies of erred tasks in memory
+  * Add maximum shard size to config
+  * Ensure shuffle split operations are blacklisted from work
+    stealing
+  * Add dropdown menu to access individual plots
+  * Edited the path to scheduler.py
+  * Task Group Graph Visualization
+  * Remove more internal references to deprecated utilities
+  * Restructure nbytes hover
+  * Except more errors in pynvml.nvmlInit()
+  * Add occupancy as individual plot
+  * Deprecate utilities which have moved to dask
+  * Ensure connectionpool does not leave comms if closed mid
+    connect
+  * Add support for registering scheduler plugins from Client
+  * Stealing dashboard fixes
+  * Allow requirements verification to be ignored when loading
+    backends from entrypoints
+  * Add Log and Logs to API docs
+  * Support fixtures and pytest.mark.parametrize with gen_cluster
+- Release 2021.06.2
+  * Revert refactor to utils.Log[s] and Cluster.get_logs
+  * Use deprecation utility from Dask
+  * Add transition counter to Scheduler
+  * Remove nbytes_in_memory
+- Release 2021.06.1
+  * Fix deadlock in handle_missing_dep if additional replicas are
+    available
+  * Add configuration to enable/disable NVML diagnostics
+  * Add scheduler log tab to performance reports
+  * Add HTML repr to scheduler_info and incorporate into client
+    and cluster reprs
+  * Fix error state typo
+  * Allow actor exceptions to propagate
+  * Remove importing apply from dask.compatibility
+  * Use more informative default name for WorkerPlugin s
+  * Removed unused utility functions
+  * Locally rerun successfully completed futures
+  * Forget erred tasks and fix deadlocks on worker
+  * Handle HTTPClientError in websocket connector
+  * Update dask_cuda usage in SSHCluster docstring
+  * Remove tests for process_time and thread_time
+  * Flake8 config cleanup
+  * Don't strip scheduler protocol when determining host
+  * Add more documentation on memory management
+  * Add range_query tests to NVML test suite
+  * No longer cancel result future in async process when using
+    timeouts
+- Release 2021.06.0
+  * Multiple worker executors
+  * Ensure PyNVML works correctly when installed with no GPUs
+  * Show more in test summary
+  * Move SystemMonitor s GPU initialization back to constructor
+  * Mark test_server_comms_mark_active_handlers with pytest.mark.asyncio
+  * Who has has what html reprs v2
+  * O(1) rebalance
+  * Ensure repr and eq for cluster always works
+- Release 2021.05.1
+  * Drop usage of WhoHas & WhatHas from Client
+  * Ensure adaptive scaling is properly awaited and closed
+  * Fix WhoHas/ HasWhat async usage
+  * Add HTML reprs for Client.who_has and Client.has_what
+  * Prevent accidentally starting multiple Worker s in the same
+    process
+  * Add system tab to performance reports
+  * Let servers close faster if there are no active handlers
+  * Fix UCX scrub config logging
+  * Ensure worker clients are closed
+  * Fix warning for attribute error when deleting a client
+  * Ensure exceptions are raised if workers are incorrectly started
+  * Update handling of UCX exceptions on endpoint closing
+  * Ensure busy workloads properly look up who_has
+  * Check distributed.scheduler.pickle in Scheduler.run_function
+  * Add performance_report to API docs
+  * Use dict _workers_dv in unordered use cases
+  * Bump pre-commit hook versions
+  * Do not mindlessly spawn workers when no memory limit is set
+  * test_memory to use gen_cluster
+  * Increase timeout of gen_test to 30s
+- Work on the very flaky testsuite:
+  * Add missing conftest.py not packaged on PyPI
+  * Add distributed-pr5022-improve_ci.patch in the hope for better
+    stability -- gh#dask/distributed#5022
+  * Do not use pytest-xdist
+- Add Cython as runtime dep because the scheduler checks the
+  presence
+
 -------------------------------------------------------------------
 Tue May 18 10:12:19 UTC 2021 - Ben Greiner <code@bnavigator.de>
 

python-distributed.spec

@@ -1,5 +1,5 @@
 #
-# spec file for package python-distributed-test
+# spec file
 #
 # Copyright (c) 2021 SUSE LLC
 #
@@ -24,23 +24,32 @@
 %define psuffix %{nil}
 %bcond_with test
 %endif
+%ifarch %{ix86} %{arm}
+# cython optimizations not supported on 32-bit: https://github.com/dask/dask/issues/7489
+%bcond_with cythonize
+%else
+%bcond_without cythonize
+%endif
+%if %{with cythonize}
+%define cythonize --with-cython
+%endif
 %{?!python_module:%define python_module() python3-%{**}}
 %define skip_python2 1
 %define skip_python36 1
-%ifnarch %{ix86} %{arm}
+%define ghversiontag 2021.07.0
-%define cythonize --with-cython
-# cython optimizations not supported on 32-bit: https://github.com/dask/dask/issues/7489
-%endif
 Name:           python-distributed%{psuffix}
 # Note: please always update together with python-dask
-Version:        2021.5.0
+Version:        2021.7.0
 Release:        0
 Summary:        Library for distributed computing with Python
 License:        BSD-3-Clause
 URL:            https://distributed.readthedocs.io/en/latest/
 Source:         https://files.pythonhosted.org/packages/source/d/distributed/distributed-%{version}.tar.gz
+# Missing in the PyPI package but needed for pytest fixtures. Note: One of the next releases will miss all of the tests. (gh#dask/distributed#5054)
+Source1:        https://github.com/dask/distributed/raw/%{ghversiontag}/conftest.py
 Source99:       python-distributed-rpmlintrc
-BuildRequires:  %{python_module Cython}
+# PATCH-FIX-UPSTREAM distributed-pr5022-improve_ci.patch -- gh#dask/distributed#5022
+Patch0:         distributed-pr5022-improve_ci.patch
 BuildRequires:  %{python_module base >= 3.7}
 BuildRequires:  %{python_module setuptools}
 BuildRequires:  fdupes
@@ -57,6 +66,11 @@ Requires:       python-tblib
 Requires:       python-toolz >= 0.8.2
 Requires:       python-tornado >= 6.0.3
 Requires:       python-zict >= 0.1.3
+%if %{with cythonize}
+BuildRequires:  %{python_module Cython}
+# the cythonized scheduler needs Cython also as runtime dep for some checks
+Requires:       python-Cython
+%endif
 %if %{with test}
 BuildRequires:  %{python_module PyYAML}
 BuildRequires:  %{python_module bokeh}
@@ -64,7 +78,6 @@ BuildRequires:  %{python_module certifi}
 BuildRequires:  %{python_module click >= 6.6}
 BuildRequires:  %{python_module cloudpickle >= 1.5.0}
 BuildRequires:  %{python_module dask-all = %{version}}
-# need built extension
 BuildRequires:  %{python_module distributed = %{version}}
 BuildRequires:  %{python_module ipykernel}
 BuildRequires:  %{python_module ipython}
@@ -74,7 +87,6 @@ BuildRequires:  %{python_module psutil}
 BuildRequires:  %{python_module pytest-asyncio}
 BuildRequires:  %{python_module pytest-rerunfailures}
 BuildRequires:  %{python_module pytest-timeout}
-BuildRequires:  %{python_module pytest-xdist}
 BuildRequires:  %{python_module pytest}
 BuildRequires:  %{python_module requests}
 BuildRequires:  %{python_module sortedcontainers}
@@ -93,6 +105,7 @@ clusters.
 
 %prep
 %autosetup -p1 -n distributed-%{version}
+cp %SOURCE1 .
 
 %build
 %if ! %{with test}
@@ -115,142 +128,25 @@ chmod -x %{buildroot}%{$python_sitearch}/distributed/tests/test_utils_test.py
 %check
 # many tests from multiple files are broken by new pytest-asyncio
 # (see https://github.com/dask/distributed/pull/4212 and https://github.com/pytest-dev/pytest-asyncio/issues/168)
-# as a proof build it with old pytest-asyncio and see these tests pass
 %if %{pkg_vcmp python3-pytest-asyncio >= 0.14}
-donttest+=" or (test_asyncprocess and test_child_main_thread)"
+donttest+=" or (test_client and test_get_client_functions_spawn_clusters)"
-donttest+=" or (test_asyncprocess and test_close)"
-donttest+=" or (test_asyncprocess and test_exitcode)"
-donttest+=" or (test_asyncprocess and test_num_fds)"
-donttest+=" or (test_asyncprocess and test_performance_report)"
-donttest+=" or (test_asyncprocess and test_signal)"
-donttest+=" or (test_client and test_add_worker_after_task)"
-donttest+=" or (test_client and test_bad_tasks_fail)"
-donttest+=" or (test_client and test_futures_in_subgraphs)"
-donttest+=" or (test_client and test_get_client)"
-donttest+=" or (test_client and test_logs)"
-donttest+=" or (test_client and test_lose_scattered_data)"
-donttest+=" or (test_client and test_performance_report)"
-donttest+=" or (test_client and test_quiet_client_close)"
-donttest+=" or (test_client and test_repr_async)"
-donttest+=" or (test_client and test_secede_balances)"
-donttest+=" or (test_client and test_secede_simple)"
-donttest+=" or (test_client and test_serialize_collections)"
-donttest+=" or (test_client_executor and test_cancellation)"
-donttest+=" or (test_client_loop and test_close_loop_sync)"
-donttest+=" or (test_collections and test_sparse_arrays)"
-donttest+=" or (test_events and test_event_on_workers)"
-donttest+=" or (test_events and test_set_not_set_many_events)"
-donttest+=" or (test_events and test_two_events_on_workers)"
-donttest+=" or (test_failed_workers and test_broken_worker_during_computation)"
-donttest+=" or (test_failed_workers and test_gather_then_submit_after_failed_workers)"
-donttest+=" or (test_failed_workers and test_restart)"
-donttest+=" or (test_failed_workers and test_worker_time_to_live)"
-donttest+=" or (test_failed_workers and test_worker_who_has_clears_after_failed_connection)"
-donttest+=" or (test_locks and test_lock)"
-donttest+=" or (test_locks and test_serializable)"
-donttest+=" or (test_nanny and test_mp_pool_worker_no_daemon)"
-donttest+=" or (test_nanny and test_mp_process_worker_no_daemon)"
-donttest+=" or (test_nanny and test_nanny)"
-donttest+=" or (test_nanny and test_num_fds)"
 donttest+=" or (test_preload and test_web_preload)"
-donttest+=" or (test_profile and test_watch)"
-donttest+=" or (test_publish and test_publish_simple)"
-donttest+=" or (test_queues and test_2220)"
-donttest+=" or (test_resources and test_prefer_constrained)"
-donttest+=" or (test_scheduler and test_balance_many_workers)"
-donttest+=" or (test_scheduler and test_bandwidth_clear)"
-donttest+=" or (test_scheduler and test_dashboard_address)"
-donttest+=" or (test_scheduler and test_dont_recompute_if_persisted)"
-donttest+=" or (test_scheduler and test_file_descriptors)"
-donttest+=" or (test_scheduler and test_gather_allow_worker_reconnect)"
-donttest+=" or (test_scheduler and test_idle_timeout)"
-donttest+=" or (test_scheduler and test_include_communication_in_occupancy)"
-donttest+=" or (test_scheduler and test_log_tasks_during_restart)"
-donttest+=" or (test_scheduler and test_restart)"
-donttest+=" or (test_scheduler and test_scheduler_init_pulls_blocked_handlers_from_config)"
-donttest+=" or (test_scheduler and test_service_hosts)"
-donttest+=" or (test_scheduler and test_steal_when_more_tasks)"
-donttest+=" or (test_scheduler and test_task_groups)"
-donttest+=" or (test_semaphor and test_getvalue)"
 donttest+=" or (test_semaphore and test_access_semaphore_by_name)"
 donttest+=" or (test_semaphore and test_close_async)"
 donttest+=" or (test_semaphore and test_oversubscribing_leases)"
-donttest+=" or (test_semaphore and test_release_failure)"
 donttest+=" or (test_semaphore and test_release_once_too_many_resilience)"
-donttest+=" or (test_semaphore and test_release_semaphore_after_timeout)"
 donttest+=" or (test_semaphore and test_release_simple)"
 donttest+=" or (test_semaphore and test_threadpoolworkers_pick_correct_ioloop)"
-donttest+=" or (test_sparse_arrays and concurrent)"
+donttest+=" or (test_worker and test_worker_client_closes_if_created_on_worker_last_worker_alive)"
-donttest+=" or (test_spec and test_address_default_none)"
+donttest+=" or (test_worker and test_worker_client_closes_if_created_on_worker_one_worker)"
-donttest+=" or (test_spec and test_child_address_persists)"
-donttest+=" or (test_steal and test_balance)"
-donttest+=" or (test_steal and test_dont_steal_already_released)"
-donttest+=" or (test_steal and test_dont_steal_few_saturated_tasks_many_workers)"
-donttest+=" or (test_steal and test_dont_steal_unknown_functions)"
-donttest+=" or (test_steal and test_eventually_steal_unknown_functions)"
-donttest+=" or (test_steal and test_restart)"
-donttest+=" or (test_steal and test_steal_more_attractive_tasks)"
-donttest+=" or (test_steal and test_steal_twice)"
-donttest+=" or (test_steal and test_steal_when_more_tasks)"
-donttest+=" or (test_steal and test_worksteal_many_thieves)"
-donttest+=" or (test_stress and test_cancel_stress)"
-donttest+=" or (test_tls_functional and test_retire_workers)"
-donttest+=" or (test_tls_functional and test_worker_client)"
-donttest+=" or (test_utils and test_sync_closed_loop)"
-donttest+=" or (test_worker and test_dont_overlap_communications_to_same_worker)"
-donttest+=" or (test_worker and test_gather_many_small)"
-donttest+=" or (test_worker and test_get_client)"
-donttest+=" or (test_worker and test_lifetime)"
-donttest+=" or (test_worker and test_robust_to_bad_sizeof_estimates)"
-donttest+=" or (test_worker and test_share_communication)"
-donttest+=" or (test_worker and test_statistical_profiling_2)"
-donttest+=" or (test_worker and test_stop_doing_unnecessary_work)"
-donttest+=" or (test_worker and test_wait_for_outgoing)"
-donttest+=" or (test_worker and test_workerstate_executing)"
-donttest+=" or (test_worker_client and test_async)"
-donttest+=" or (test_worker_client and test_client_executor)"
-donttest+=" or (test_worker_client and test_compute_within_worker_client)"
-donttest+=" or (test_worker_client and test_gather_multi_machine)"
-donttest+=" or (test_worker_client and test_local_client_warning)"
-donttest+=" or (test_worker_client and test_scatter_from_worker)"
-donttest+=" or (test_worker_client and test_scatter_singleton)"
-donttest+=" or (test_worker_client and test_secede_without_stealing_issue_1262)"
-donttest+=" or (test_worker_client and test_submit_different_names)"
-donttest+=" or (test_worker_client and test_submit_from_worker)"
 %endif
-# false version mismatch
-donttest+=" or test_version_warning_in_cluster"
-# ambiguous order in returned message
-donttest+=" or (test_client and test_as_completed_async_for_cancel)"
-# too many open files
-donttest+=" or (test_stress and test_stress_communication)"
 # randomly fail even with old asyncio -- too slow for obs (?)
 donttest+=" or (test_asyncprocess and test_exit_callback)"
-donttest+=" or (test_client and test_cleanup_after_broken_client_connection)"
-donttest+=" or (test_client and test_open_close_many_workers)"
-donttest+=" or (test_client and test_profile)"
-donttest+=" or (test_client and test_quiet_quit_when_cluster_leaves)"
-donttest+=" or (test_client and test_reconnect)"
-donttest+=" or (test_client and test_sub_submit_priority)"
-donttest+=" or (test_client and test_upload_file_exception_sync)"
-donttest+=" or (test_client and test_upload_file_sync)"
-donttest+=" or (test_diskutils and test_workspace_concurrency)"
-donttest+=" or (test_failed_workers and test_fast_kill)"
-donttest+=" or (test_metrics and time)"
-donttest+=" or (test_queues and test_race)"
-donttest+=" or (test_scheduler and test_gather_failing_cnn_recover)"
-donttest+=" or (test_steal and test_dont_steal_fast_tasks_compute_time)"
-donttest+=" or (test_stress and test_close_connections)"
 donttest+=" or (test_worker and test_fail_write_to_disk)"
-donttest+=" or test_queue_in_task or test_variable_in_task"
+# rebalance fails on the server, but not when building locally
-# https://github.com/dask/distributed/pull/4719: "This test is heavily influenced by hard-to-control factors such as memory management"
+donttest+=" or (test_scheduler and test_rebalance)"
-# probably influenced by OBS scheduling
+donttest+=" or (test_tls_functional and test_rebalance)"
-donttest+=" or (test_scheduler and test_memory)"
+%pytest_arch distributed/tests -r sfER -m "not avoid_ci" -k "not (${donttest:4})" --reruns 3 --reruns-delay 3
-# likely related to the above (https://github.com/dask/distributed/pull/4651)
-donttest+=" or (test_worker and test_spill_to_disk)"
-# flaky on i586
-donttest+=" or (test_client_executor and test_map)"
-%pytest_arch -rfE -n auto distributed/tests/ -k "not (${donttest:4})" -m "not avoid_travis" --timeout 180
 %endif
 
 %if ! %{with test}