forked from pool/python-Scrapy
Compare commits
10 Commits
Author | SHA256 | Date | |
---|---|---|---|
36b4bfaf0b | |||
dba2d7540a | |||
406916eda8 | |||
0d3dbc2801 | |||
a7924cbd08 | |||
f2ecbd1d47 | |||
d580aa635a | |||
f56239b802 | |||
91941765d1 | |||
eaffe32d22 |
@@ -1,3 +0,0 @@
|
|||||||
version https://git-lfs.github.com/spec/v1
|
|
||||||
oid sha256:733a039c7423e52b69bf2810b5332093d4e42a848460359c07b02ecff8f73ebe
|
|
||||||
size 1176726
|
|
18
no-dark-mode.patch
Normal file
18
no-dark-mode.patch
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
Index: scrapy-2.13.3/docs/conf.py
|
||||||
|
===================================================================
|
||||||
|
--- scrapy-2.13.3.orig/docs/conf.py
|
||||||
|
+++ scrapy-2.13.3/docs/conf.py
|
||||||
|
@@ -34,7 +34,7 @@ extensions = [
|
||||||
|
"sphinx.ext.coverage",
|
||||||
|
"sphinx.ext.intersphinx",
|
||||||
|
"sphinx.ext.viewcode",
|
||||||
|
- "sphinx_rtd_dark_mode",
|
||||||
|
+ "sphinx_rtd_theme",
|
||||||
|
]
|
||||||
|
|
||||||
|
templates_path = ["_templates"]
|
||||||
|
@@ -158,4 +158,3 @@ intersphinx_mapping = {
|
||||||
|
intersphinx_disabled_reftypes: Sequence[str] = []
|
||||||
|
|
||||||
|
# -- Other options ------------------------------------------------------------
|
||||||
|
-default_dark_mode = False
|
@@ -1,3 +1,108 @@
|
|||||||
|
-------------------------------------------------------------------
|
||||||
|
Thu Jul 31 05:18:40 UTC 2025 - Steve Kowalik <steven.kowalik@suse.com>
|
||||||
|
|
||||||
|
- Update to 2.13.3:
|
||||||
|
* Changed the values for DOWNLOAD_DELAY (from 0 to 1) and
|
||||||
|
CONCURRENT_REQUESTS_PER_DOMAIN (from 8 to 1) in the default project
|
||||||
|
template.
|
||||||
|
* Fixed several bugs in the engine initialization and exception handling
|
||||||
|
logic.
|
||||||
|
* Allowed running tests with Twisted 25.5.0+ again and fixed test failures
|
||||||
|
with lxml 6.0.0.
|
||||||
|
* Give callback requests precedence over start requests when priority
|
||||||
|
values are the same.
|
||||||
|
* The asyncio reactor is now enabled by default
|
||||||
|
* Replaced start_requests() (sync) with start() (async) and changed how it
|
||||||
|
is iterated.
|
||||||
|
* Added the allow_offsite request meta key
|
||||||
|
* Spider middlewares that don't support asynchronous spider output are
|
||||||
|
deprecated
|
||||||
|
* Added a base class for universal spider middlewares
|
||||||
|
- Add patch remove-hoverxref.patch:
|
||||||
|
* Do not use deprecated sphinx-hoverxref extension.
|
||||||
|
- Add patch no-dark-mode.patch:
|
||||||
|
* Do not use unavailable sphinx-rtd-dark-mode extension.
|
||||||
|
|
||||||
|
-------------------------------------------------------------------
|
||||||
|
Thu Mar 27 05:45:59 UTC 2025 - Steve Kowalik <steven.kowalik@suse.com>
|
||||||
|
|
||||||
|
- Normalize metadata directory name.
|
||||||
|
|
||||||
|
-------------------------------------------------------------------
|
||||||
|
Tue Dec 3 08:24:29 UTC 2024 - Steve Kowalik <steven.kowalik@suse.com>
|
||||||
|
|
||||||
|
- Update to 2.12.0:
|
||||||
|
* Dropped support for Python 3.8, added support for Python 3.13
|
||||||
|
* start_requests can now yield items
|
||||||
|
* Added scrapy.http.JsonResponse
|
||||||
|
* Added the CLOSESPIDER_PAGECOUNT_NO_ITEM setting
|
||||||
|
|
||||||
|
-------------------------------------------------------------------
|
||||||
|
Thu Jul 11 10:38:36 UTC 2024 - Dirk Müller <dmueller@suse.com>
|
||||||
|
|
||||||
|
- update to 2.11.2 (bsc#1224474, CVE-2024-1968):
|
||||||
|
* Redirects to non-HTTP protocols are no longer followed.
|
||||||
|
Please, see the 23j4-mw76-5v7h security advisory for more
|
||||||
|
information. (:issue:`457`)
|
||||||
|
* The Authorization header is now dropped on redirects to a
|
||||||
|
different scheme (http:// or https://) or port, even if the
|
||||||
|
domain is the same. Please, see the 4qqq-9vqf-3h3f security
|
||||||
|
advisory for more information.
|
||||||
|
* When using system proxy settings that are different for
|
||||||
|
http:// and https://, redirects to a different URL scheme
|
||||||
|
will now also trigger the corresponding change in proxy
|
||||||
|
settings for the redirected request. Please, see the
|
||||||
|
jm3v-qxmh-hxwv security advisory for more information.
|
||||||
|
(:issue:`767`)
|
||||||
|
* :attr:`Spider.allowed_domains
|
||||||
|
<scrapy.Spider.allowed_domains>` is now enforced for all
|
||||||
|
requests, and not only requests from spider callbacks.
|
||||||
|
* :func:`~scrapy.utils.iterators.xmliter_lxml` no longer
|
||||||
|
resolves XML entities.
|
||||||
|
* defusedxml is now used to make
|
||||||
|
:class:`scrapy.http.request.rpc.XmlRpcRequest` more secure.
|
||||||
|
* Restored support for brotlipy_, which had been dropped in
|
||||||
|
Scrapy 2.11.1 in favor of brotli. (:issue:`6261`) Note
|
||||||
|
brotlipy is deprecated, both in Scrapy and upstream. Use
|
||||||
|
brotli instead if you can.
|
||||||
|
* Make :setting:`METAREFRESH_IGNORE_TAGS` ["noscript"] by
|
||||||
|
default. This prevents :class:`~scrapy.downloadermiddlewares.
|
||||||
|
redirect.MetaRefreshMiddleware` from following redirects that
|
||||||
|
would not be followed by web browsers with JavaScript
|
||||||
|
enabled.
|
||||||
|
* During :ref:`feed export <topics-feed-exports>`, do not close
|
||||||
|
the underlying file from :ref:`built-in post-processing
|
||||||
|
plugins <builtin-plugins>`.
|
||||||
|
* :class:`LinkExtractor
|
||||||
|
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` now
|
||||||
|
properly applies the unique and canonicalize parameters.
|
||||||
|
* Do not initialize the scheduler disk queue if
|
||||||
|
:setting:`JOBDIR` is an empty string.
|
||||||
|
* Fix :attr:`Spider.logger <scrapy.Spider.logger>` not logging
|
||||||
|
custom extra information.
|
||||||
|
* robots.txt files with a non-UTF-8 encoding no longer prevent
|
||||||
|
parsing the UTF-8-compatible (e.g. ASCII) parts of the
|
||||||
|
document.
|
||||||
|
* :meth:`scrapy.http.cookies.WrappedRequest.get_header` no
|
||||||
|
longer raises an exception if default is None.
|
||||||
|
:func:`scrapy.utils.response.get_base_url` to determine the
|
||||||
|
base URL of a given :class:`~scrapy.http.Response`.
|
||||||
|
* :class:`~scrapy.selector.Selector` now uses
|
||||||
|
:func:`scrapy.utils.response.get_base_url` to determine the
|
||||||
|
base URL of a given :class:`~scrapy.http.Response`.
|
||||||
|
(:issue:`6265`)
|
||||||
|
* The :meth:`media_to_download` method of :ref:`media pipelines
|
||||||
|
<topics-media-pipeline>` now logs exceptions before stripping
|
||||||
|
them.
|
||||||
|
* When passing a callback to the :command:`parse` command,
|
||||||
|
build the callback callable with the right signature.
|
||||||
|
* Add a FAQ entry about :ref:`creating blank requests <faq-
|
||||||
|
blank-request>`.
|
||||||
|
* Document that :attr:`scrapy.selector.Selector.type` can be
|
||||||
|
"json".
|
||||||
|
* Make builds reproducible.
|
||||||
|
* Packaging and test fixes
|
||||||
|
|
||||||
-------------------------------------------------------------------
|
-------------------------------------------------------------------
|
||||||
Mon Mar 25 14:12:20 UTC 2024 - Dirk Müller <dmueller@suse.com>
|
Mon Mar 25 14:12:20 UTC 2024 - Dirk Müller <dmueller@suse.com>
|
||||||
|
|
||||||
|
@@ -1,7 +1,7 @@
|
|||||||
#
|
#
|
||||||
# spec file for package python-Scrapy
|
# spec file for package python-Scrapy
|
||||||
#
|
#
|
||||||
# Copyright (c) 2024 SUSE LLC
|
# Copyright (c) 2025 SUSE LLC
|
||||||
#
|
#
|
||||||
# All modifications and additions to the file contributed by third parties
|
# All modifications and additions to the file contributed by third parties
|
||||||
# remain the property of their copyright owners, unless otherwise agreed
|
# remain the property of their copyright owners, unless otherwise agreed
|
||||||
@@ -18,23 +18,29 @@
|
|||||||
|
|
||||||
%{?sle15_python_module_pythons}
|
%{?sle15_python_module_pythons}
|
||||||
Name: python-Scrapy
|
Name: python-Scrapy
|
||||||
Version: 2.11.1
|
Version: 2.13.3
|
||||||
Release: 0
|
Release: 0
|
||||||
Summary: A high-level Python Screen Scraping framework
|
Summary: A high-level Python Screen Scraping framework
|
||||||
License: BSD-3-Clause
|
License: BSD-3-Clause
|
||||||
Group: Development/Languages/Python
|
|
||||||
URL: https://scrapy.org
|
URL: https://scrapy.org
|
||||||
Source: https://files.pythonhosted.org/packages/source/S/Scrapy/Scrapy-%{version}.tar.gz
|
Source: https://files.pythonhosted.org/packages/source/s/scrapy/scrapy-%{version}.tar.gz
|
||||||
|
# PATCH-FIX-UPSTREAM gh#scrapy/scrapy#6922
|
||||||
|
Patch0: remove-hoverxref.patch
|
||||||
|
# PATCH-FIX-OPENSUSE No sphinx-rtd-dark-mode
|
||||||
|
Patch1: no-dark-mode.patch
|
||||||
|
BuildRequires: %{python_module Brotli}
|
||||||
BuildRequires: %{python_module Pillow}
|
BuildRequires: %{python_module Pillow}
|
||||||
BuildRequires: %{python_module Protego >= 0.1.15}
|
BuildRequires: %{python_module Protego}
|
||||||
BuildRequires: %{python_module PyDispatcher >= 2.0.5}
|
BuildRequires: %{python_module PyDispatcher >= 2.0.5}
|
||||||
BuildRequires: %{python_module Twisted >= 18.9.0}
|
BuildRequires: %{python_module Twisted >= 18.9.0}
|
||||||
BuildRequires: %{python_module attrs}
|
BuildRequires: %{python_module attrs}
|
||||||
BuildRequires: %{python_module base >= 3.8}
|
BuildRequires: %{python_module base >= 3.9}
|
||||||
BuildRequires: %{python_module botocore >= 1.4.87}
|
BuildRequires: %{python_module botocore >= 1.4.87}
|
||||||
BuildRequires: %{python_module cryptography >= 36.0.0}
|
BuildRequires: %{python_module cryptography >= 36.0.0}
|
||||||
BuildRequires: %{python_module cssselect >= 0.9.1}
|
BuildRequires: %{python_module cssselect >= 0.9.1}
|
||||||
BuildRequires: %{python_module dbm}
|
BuildRequires: %{python_module dbm}
|
||||||
|
BuildRequires: %{python_module defusedxml >= 0.7.1}
|
||||||
|
BuildRequires: %{python_module hatchling}
|
||||||
BuildRequires: %{python_module itemadapter >= 0.1.0}
|
BuildRequires: %{python_module itemadapter >= 0.1.0}
|
||||||
BuildRequires: %{python_module itemloaders >= 1.0.1}
|
BuildRequires: %{python_module itemloaders >= 1.0.1}
|
||||||
BuildRequires: %{python_module lxml >= 4.4.1}
|
BuildRequires: %{python_module lxml >= 4.4.1}
|
||||||
@@ -47,22 +53,23 @@ BuildRequires: %{python_module pytest-xdist}
|
|||||||
BuildRequires: %{python_module pytest}
|
BuildRequires: %{python_module pytest}
|
||||||
BuildRequires: %{python_module queuelib >= 1.4.2}
|
BuildRequires: %{python_module queuelib >= 1.4.2}
|
||||||
BuildRequires: %{python_module service_identity >= 18.1.0}
|
BuildRequires: %{python_module service_identity >= 18.1.0}
|
||||||
BuildRequires: %{python_module setuptools}
|
|
||||||
BuildRequires: %{python_module sybil}
|
BuildRequires: %{python_module sybil}
|
||||||
BuildRequires: %{python_module testfixtures}
|
BuildRequires: %{python_module testfixtures}
|
||||||
BuildRequires: %{python_module tldextract}
|
BuildRequires: %{python_module tldextract}
|
||||||
BuildRequires: %{python_module uvloop}
|
BuildRequires: %{python_module uvloop}
|
||||||
BuildRequires: %{python_module w3lib >= 1.17.0}
|
BuildRequires: %{python_module w3lib >= 1.17.0}
|
||||||
BuildRequires: %{python_module wheel}
|
|
||||||
BuildRequires: %{python_module zope.interface >= 5.1.0}
|
BuildRequires: %{python_module zope.interface >= 5.1.0}
|
||||||
BuildRequires: fdupes
|
BuildRequires: fdupes
|
||||||
BuildRequires: python-rpm-macros
|
BuildRequires: python-rpm-macros
|
||||||
BuildRequires: python3-Sphinx
|
BuildRequires: python3-Sphinx
|
||||||
|
BuildRequires: python3-sphinx-notfound-page
|
||||||
|
BuildRequires: python3-sphinx_rtd_theme
|
||||||
Requires: python-Protego >= 0.1.15
|
Requires: python-Protego >= 0.1.15
|
||||||
Requires: python-PyDispatcher >= 2.0.5
|
Requires: python-PyDispatcher >= 2.0.5
|
||||||
Requires: python-Twisted >= 18.9.0
|
Requires: python-Twisted >= 18.9.0
|
||||||
Requires: python-cryptography >= 36.0.0
|
Requires: python-cryptography >= 36.0.0
|
||||||
Requires: python-cssselect >= 0.9.1
|
Requires: python-cssselect >= 0.9.1
|
||||||
|
Requires: python-defusedxml >= 0.7.1
|
||||||
Requires: python-itemadapter >= 0.1.0
|
Requires: python-itemadapter >= 0.1.0
|
||||||
Requires: python-itemloaders >= 1.0.1
|
Requires: python-itemloaders >= 1.0.1
|
||||||
Requires: python-lxml >= 4.4.1
|
Requires: python-lxml >= 4.4.1
|
||||||
@@ -71,7 +78,6 @@ Requires: python-parsel >= 1.5.0
|
|||||||
Requires: python-pyOpenSSL >= 21.0.0
|
Requires: python-pyOpenSSL >= 21.0.0
|
||||||
Requires: python-queuelib >= 1.4.2
|
Requires: python-queuelib >= 1.4.2
|
||||||
Requires: python-service_identity >= 18.1.0
|
Requires: python-service_identity >= 18.1.0
|
||||||
Requires: python-setuptools
|
|
||||||
Requires: python-tldextract
|
Requires: python-tldextract
|
||||||
Requires: python-w3lib >= 1.17.2
|
Requires: python-w3lib >= 1.17.2
|
||||||
Requires: python-zope.interface >= 5.1.0
|
Requires: python-zope.interface >= 5.1.0
|
||||||
@@ -87,13 +93,12 @@ retrieval to monitoring or testing web sites.
|
|||||||
|
|
||||||
%package -n %{name}-doc
|
%package -n %{name}-doc
|
||||||
Summary: Documentation for %{name}
|
Summary: Documentation for %{name}
|
||||||
Group: Documentation/HTML
|
|
||||||
|
|
||||||
%description -n %{name}-doc
|
%description -n %{name}-doc
|
||||||
Provides documentation for %{name}.
|
Provides documentation for %{name}.
|
||||||
|
|
||||||
%prep
|
%prep
|
||||||
%autosetup -p1 -n Scrapy-%{version}
|
%autosetup -p1 -n scrapy-%{version}
|
||||||
|
|
||||||
sed -i -e 's:= python:= python3:g' docs/Makefile
|
sed -i -e 's:= python:= python3:g' docs/Makefile
|
||||||
|
|
||||||
@@ -112,9 +117,11 @@ popd
|
|||||||
# no color in obs chroot console
|
# no color in obs chroot console
|
||||||
skiplist="test_pformat"
|
skiplist="test_pformat"
|
||||||
# no online connection to toscrapy.com
|
# no online connection to toscrapy.com
|
||||||
skiplist="$skiplist or CheckCommandTest"
|
skiplist="$skiplist or CheckCommandTest or test_file_path"
|
||||||
# Flaky test gh#scrapy/scrapy#5703
|
# Flaky test gh#scrapy/scrapy#5703
|
||||||
skiplist="$skiplist or test_start_requests_laziness"
|
skiplist="$skiplist or test_start_requests_laziness"
|
||||||
|
# Fails on 32 bit arches
|
||||||
|
skiplist="$skiplist or test_queue_push_pop_priorities"
|
||||||
%{pytest -x \
|
%{pytest -x \
|
||||||
-k "not (${skiplist})" \
|
-k "not (${skiplist})" \
|
||||||
-W ignore::DeprecationWarning \
|
-W ignore::DeprecationWarning \
|
||||||
@@ -130,7 +137,7 @@ skiplist="$skiplist or test_start_requests_laziness"
|
|||||||
%license LICENSE
|
%license LICENSE
|
||||||
%doc AUTHORS README.rst
|
%doc AUTHORS README.rst
|
||||||
%{python_sitelib}/scrapy
|
%{python_sitelib}/scrapy
|
||||||
%{python_sitelib}/Scrapy-%{version}.dist-info
|
%{python_sitelib}/[Ss]crapy-%{version}.dist-info
|
||||||
%python_alternative %{_bindir}/scrapy
|
%python_alternative %{_bindir}/scrapy
|
||||||
|
|
||||||
%files -n %{name}-doc
|
%files -n %{name}-doc
|
||||||
|
56
remove-hoverxref.patch
Normal file
56
remove-hoverxref.patch
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
From 549730c23592479f200f3c1f941c59f68c510ff5 Mon Sep 17 00:00:00 2001
|
||||||
|
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
|
||||||
|
Date: Sat, 28 Jun 2025 12:32:55 +0200
|
||||||
|
Subject: [PATCH] Remove the deprecated sphinx-hoverxref
|
||||||
|
|
||||||
|
---
|
||||||
|
docs/conf.py | 20 +-------------------
|
||||||
|
docs/requirements.txt | 1 -
|
||||||
|
2 files changed, 1 insertion(+), 20 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/docs/conf.py b/docs/conf.py
|
||||||
|
index 493a6297624..0345ec69543 100644
|
||||||
|
--- a/docs/conf.py
|
||||||
|
+++ b/docs/conf.py
|
||||||
|
@@ -26,7 +26,6 @@
|
||||||
|
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
|
||||||
|
|
||||||
|
extensions = [
|
||||||
|
- "hoverxref.extension",
|
||||||
|
"notfound.extension",
|
||||||
|
"scrapydocs",
|
||||||
|
"sphinx.ext.autodoc",
|
||||||
|
@@ -157,22 +156,5 @@
|
||||||
|
}
|
||||||
|
intersphinx_disabled_reftypes: Sequence[str] = []
|
||||||
|
|
||||||
|
-
|
||||||
|
-# -- Options for sphinx-hoverxref extension ----------------------------------
|
||||||
|
-# https://sphinx-hoverxref.readthedocs.io/en/latest/configuration.html
|
||||||
|
-
|
||||||
|
-hoverxref_auto_ref = True
|
||||||
|
-hoverxref_role_types = {
|
||||||
|
- "class": "tooltip",
|
||||||
|
- "command": "tooltip",
|
||||||
|
- "confval": "tooltip",
|
||||||
|
- "hoverxref": "tooltip",
|
||||||
|
- "mod": "tooltip",
|
||||||
|
- "ref": "tooltip",
|
||||||
|
- "reqmeta": "tooltip",
|
||||||
|
- "setting": "tooltip",
|
||||||
|
- "signal": "tooltip",
|
||||||
|
-}
|
||||||
|
-hoverxref_roles = ["command", "reqmeta", "setting", "signal"]
|
||||||
|
-
|
||||||
|
+# -- Other options ------------------------------------------------------------
|
||||||
|
default_dark_mode = False
|
||||||
|
diff --git a/docs/requirements.txt b/docs/requirements.txt
|
||||||
|
index 103fb08d667..4b382b11eb9 100644
|
||||||
|
--- a/docs/requirements.txt
|
||||||
|
+++ b/docs/requirements.txt
|
||||||
|
@@ -1,5 +1,4 @@
|
||||||
|
sphinx==8.1.3
|
||||||
|
-sphinx-hoverxref==1.4.2
|
||||||
|
sphinx-notfound-page==1.0.4
|
||||||
|
sphinx-rtd-theme==3.0.2
|
||||||
|
sphinx-rtd-dark-mode==1.3.0
|
3
scrapy-2.13.3.tar.gz
Normal file
3
scrapy-2.13.3.tar.gz
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:bf17588c10e46a9d70c49a05380b749e3c7fba58204a367a5747ce6da2bd204d
|
||||||
|
size 1220051
|
Reference in New Issue
Block a user