From a61c1e43591d08245e8e3fe85a7ba412a10004d416ec8d1d8356c6dcbaaa637d Mon Sep 17 00:00:00 2001 From: Matej Cepl Date: Wed, 20 Jan 2021 21:48:31 +0000 Subject: [PATCH] - Update to 3.29.0: - Add better chapter error reporting, refactor proceed_question code. - Add plugin options to mark success/failed/chapter error individually. - Clear metadata cache after adapter metadata fetch. Cached metadata values may not be replace_metadata processed if fetched before their conditional dependencies. Revealed by AO3 one-shots using title for chapter name. - Re-enable ffnet and bump up sleep times for same. - ffnet fpcom, continue_on_chapter_error:true by default, increase sleep times. - First rough version of reporting continue_on_chapter_error chapters. - Report chapter_error in custom error column and marked (when configured). - Update to cloudscraper v1.2.52 - Increase times between retries on fetch error. - Tweek ffnet/fpcom sleep times again. - slow_down_sleep_time: randomize between 0.5 time and 1.5 time. - Report cloudscraper exceptions as such, plus hide the potentially misleading 'opensource' part. - adapter_fanficsme: do an extra fetch before login for cookie(?) Closes #633 - adapter_webnovelcom: Fixes for site changes. Closes #629 - quotev.com: site change in date parse, use utf8:ignore as first encoding choice. Closes #625 - Update to version 3.28.0+git.1610814197.37cdec2f: OBS-URL: https://build.opensuse.org/package/show/devel:languages:python/python-fanficfare?expand=0&rev=64 --- FanFicFare-3.28.0.tar.gz | 3 - FanFicFare-3.29.0.tar.gz | 3 + _service | 2 +- _servicedata | 2 +- no-cloudscraper.patch | 216 -------------------------------------- python-fanficfare.changes | 69 ++++++++++++ python-fanficfare.spec | 7 +- 7 files changed, 77 insertions(+), 225 deletions(-) delete mode 100644 FanFicFare-3.28.0.tar.gz create mode 100644 FanFicFare-3.29.0.tar.gz delete mode 100644 no-cloudscraper.patch diff --git a/FanFicFare-3.28.0.tar.gz b/FanFicFare-3.28.0.tar.gz deleted file mode 100644 index c2c1fc4..0000000 --- a/FanFicFare-3.28.0.tar.gz +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:03645a35aed9e521b6a0e2277bd6784de0c2b5bf6dece6d6028a871e14590d1e -size 1827324 diff --git a/FanFicFare-3.29.0.tar.gz b/FanFicFare-3.29.0.tar.gz new file mode 100644 index 0000000..2ace3ea --- /dev/null +++ b/FanFicFare-3.29.0.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7165364dfcb99bb84a23547252653d7dcfeba52b57dc167b3d6948dad8c67ad4 +size 1868454 diff --git a/_service b/_service index ed80b26..c855f6e 100644 --- a/_service +++ b/_service @@ -1,6 +1,6 @@ - 3.26.0+git + 3.28.0+git https://github.com/JimmXinu/FanFicFare.git git enable diff --git a/_servicedata b/_servicedata index b504334..8a78d85 100644 --- a/_servicedata +++ b/_servicedata @@ -1,4 +1,4 @@ https://github.com/JimmXinu/FanFicFare.git - fd11526da8a48d488df0ba019967ea41856a2f4c \ No newline at end of file + 37cdec2f27a3c54245c7db47c49602e480ab3d85 \ No newline at end of file diff --git a/no-cloudscraper.patch b/no-cloudscraper.patch deleted file mode 100644 index 57b541f..0000000 --- a/no-cloudscraper.patch +++ /dev/null @@ -1,216 +0,0 @@ ---- a/fanficfare/configurable.py -+++ b/fanficfare/configurable.py -@@ -44,7 +44,6 @@ import pickle - - ## isn't found in plugin when only imported down below inside - ## get_scraper() --import cloudscraper - - from . import exceptions - -@@ -210,7 +209,6 @@ def get_valid_set_options(): - 'titlepage_use_table':(None,None,boollist), - - 'use_ssl_unverified_context':(None,None,boollist), -- 'use_cloudscraper':(None,None,boollist), - 'continue_on_chapter_error':(None,None,boollist), - 'conditionals_use_lists':(None,None,boollist), - 'dedup_chapter_list':(None,None,boollist), -@@ -483,7 +481,6 @@ def get_valid_keywords(): - 'tweak_fg_sleep', - 'universe_as_series', - 'use_ssl_unverified_context', -- 'use_cloudscraper', - 'user_agent', - 'username', - 'website_encodings', -@@ -598,16 +595,11 @@ class Configuration(ConfigParser): - self.override_sleep = None - self.cookiejar = self.get_empty_cookiejar() - self.opener = build_opener(HTTPCookieProcessor(self.cookiejar),GZipProcessor()) -- self.scraper = None - - self.pagecache = self.get_empty_pagecache() - self.save_cache_file = None - self.save_cookiejar_file = None - -- def __del__(self): -- if self.scraper is not None: -- self.scraper.close() -- - def section_url_names(self,domain,section_url_f): - ## domain is passed as a method to limit the damage if/when an - ## adapter screws up _section_url -@@ -1073,24 +1065,6 @@ class Configuration(ConfigParser): - logger.warning("reduce_zalgo failed(%s), continuing."%e) - return data - -- def get_scraper(self): -- if not self.scraper: -- ## ffnet adapter can't parse mobile output, so we only -- ## want desktop browser. But cloudscraper then insists on -- ## a browser and platform, too. -- self.scraper = cloudscraper.CloudScraper(browser={ -- 'browser': 'chrome', -- 'platform': 'windows', -- 'mobile': False, -- 'desktop': True, -- }) -- ## CloudScraper is subclass of requests.Session. -- ## probably need import higher up if ever used. -- # import requests -- # self.scraper = requests.Session() -- self.scraper.cookies = self.cookiejar -- return self.scraper -- - # Assumes application/x-www-form-urlencoded. parameters, headers are dict()s - def _postUrl(self, url, - parameters={}, -@@ -1132,24 +1106,15 @@ class Configuration(ConfigParser): - # headers['Authorization']=b"Basic %s" % base64string - # logger.debug("http login for SB xf2test") - -- if self.getConfig('use_cloudscraper',False): -- logger.debug("Using cloudscraper for POST") -- resp = self.get_scraper().post(url, -- headers=dict(headers), -- data=parameters) -- logger.debug("response code:%s"%resp.status_code) -- resp.raise_for_status() # raises HTTPError if error code. -- data = resp.content -- else: -- req = Request(url, -- data=ensure_binary(urlencode(parameters)), -- headers=headers) -- -- ## Specific UA because too many sites are blocking the default python UA. -- self.opener.addheaders = [('User-Agent', self.getConfig('user_agent')), -- ('X-Clacks-Overhead','GNU Terry Pratchett')] -+ req = Request(url, -+ data=ensure_binary(urlencode(parameters)), -+ headers=headers) -+ -+ ## Specific UA because too many sites are blocking the default python UA. -+ self.opener.addheaders = [('User-Agent', self.getConfig('user_agent')), -+ ('X-Clacks-Overhead','GNU Terry Pratchett')] - -- data = self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read() -+ data = self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read() - data = self._do_reduce_zalgo(self._decode(data)) - self._progressbar() - ## postURL saves data to the pagecache *after* _decode() while -@@ -1227,37 +1192,16 @@ class Configuration(ConfigParser): - - self.opener.addheaders = headers - -- if self.getConfig('use_cloudscraper',False): -- ## requests / cloudscraper wants a dict() for headers, not -- ## list of tuples. -- headers = dict(headers) -- ## let cloudscraper do its thing with UA. -- if 'User-Agent' in headers: -- del headers['User-Agent'] -- if parameters != None: -- logger.debug("Using cloudscraper for fetch POST") -- resp = self.get_scraper().post(url, -- headers=headers, -- data=parameters) -- else: -- logger.debug("Using cloudscraper for GET") -- resp = self.get_scraper().get(url, -- headers=headers) -- logger.debug("response code:%s"%resp.status_code) -- resp.raise_for_status() # raises HTTPError if error code. -- data = resp.content -- opened = FakeOpened(data,resp.url) -+ ## opener.open() will to POST with params(data) and GET without. -+ if parameters != None: -+ opened = self.opener.open(url, -+ ensure_binary(urlencode(parameters)), -+ float(self.getConfig('connect_timeout',30.0))) - else: -- ## opener.open() will to POST with params(data) and GET without. -- if parameters != None: -- opened = self.opener.open(url, -- ensure_binary(urlencode(parameters)), -- float(self.getConfig('connect_timeout',30.0))) -- else: -- opened = self.opener.open(url, -- None, -- float(self.getConfig('connect_timeout',30.0))) -- data = opened.read() -+ opened = self.opener.open(url, -+ None, -+ float(self.getConfig('connect_timeout',30.0))) -+ data = opened.read() - self._progressbar() - ## postURL saves data to the pagecache *after* _decode() while - ## fetchRaw saves it *before* _decode()--because raw. ---- a/fanficfare/defaults.ini -+++ b/fanficfare/defaults.ini -@@ -2793,15 +2793,6 @@ type_label:Type of Couple - website_encodings:Windows-1252,utf8 - - [www.fanfiction.net] --## Using cloudscraper can satisfy the first couple levels of --## Cloudflare bot-proofing, but not all levels. Older versions of --## OpenSSL will also raise problems, so versions of Calibre older than --## v5 will probably fail. Only fanfiction.net and fictionpress.com --## are configured with use_cloudscraper:true by default, but it can be --## applied in other sites' ini sections. user_agent setting is --## ignored when use_cloudscraper:true --use_cloudscraper:true -- - ## fanfiction.net's 'cover' images are really just tiny thumbnails. - ## Set this to true to never use them. - #never_make_cover: false -@@ -2888,15 +2879,6 @@ website_encodings:Windows-1252,utf8 - slow_down_sleep_time:10 - - [www.fictionpress.com] --## Using cloudscraper can satisfy the first couple levels of --## Cloudflare bot-proofing, but not all levels. Older versions of --## OpenSSL will also raise problems, so versions of Calibre older than --## v5 will probably fail. Only fanfiction.net and fictionpress.com --## are configured with use_cloudscraper:true by default, but it can be --## applied in other sites' ini sections. user_agent setting is --## ignored when use_cloudscraper:true --use_cloudscraper:true -- - ## Clear FanFiction from defaults, fictionpress.com is original fiction. - extratags: - ---- a/makeplugin.py -+++ b/makeplugin.py -@@ -23,7 +23,7 @@ from makezip import createZipFile - if __name__=="__main__": - filename="FanFicFare.zip" - exclude=['*.pyc','*~','*.xcf','*[0-9].png','*.po','*.pot','*default.mo','*Thumbs.db'] -- -+ - os.chdir('calibre-plugin') - files=['plugin-defaults.ini','plugin-example.ini','about.html', - 'images','translations'] -@@ -35,8 +35,8 @@ if __name__=="__main__": - exclude=exclude) - - os.chdir('../included_dependencies') -- files=['bs4','chardet','html2text','soupsieve','backports', -- 'cloudscraper','requests','requests_toolbelt','urllib3', -+ files=['bs4', 'chardet', 'html2text', 'soupsieve', 'backports', -+ 'requests', 'requests_toolbelt', 'urllib3', - 'certifi','idna'] - ## Kept only for v2.85.1 support now. - createZipFile("../"+filename,"a", ---- a/setup.py -+++ b/setup.py -@@ -84,8 +84,7 @@ setup( - install_requires=['beautifulsoup4', - 'chardet', - 'html5lib', -- 'html2text', -- 'cloudscraper'], -+ 'html2text'], - # html5lib requires 'six', FFF includes it's own copy as fanficfare.six - - # List additional groups of dependencies here (e.g. development diff --git a/python-fanficfare.changes b/python-fanficfare.changes index 9721c49..92faeae 100644 --- a/python-fanficfare.changes +++ b/python-fanficfare.changes @@ -1,3 +1,72 @@ +------------------------------------------------------------------- +Wed Jan 20 21:46:43 UTC 2021 - Matej Cepl + +- Update to 3.29.0: + - Add better chapter error reporting, refactor proceed_question + code. + - Add plugin options to mark success/failed/chapter error + individually. + - Clear metadata cache after adapter metadata fetch. Cached + metadata values may not be replace_metadata processed if + fetched before their conditional dependencies. Revealed by + AO3 one-shots using title for chapter name. + - Re-enable ffnet and bump up sleep times for same. + - ffnet fpcom, continue_on_chapter_error:true by default, + increase sleep times. + - First rough version of reporting continue_on_chapter_error + chapters. + - Report chapter_error in custom error column and marked (when + configured). + - Update to cloudscraper v1.2.52 + - Increase times between retries on fetch error. + - Tweek ffnet/fpcom sleep times again. + - slow_down_sleep_time: randomize between 0.5 time and 1.5 + time. + - Report cloudscraper exceptions as such, plus hide the + potentially misleading 'opensource' part. + - adapter_fanficsme: do an extra fetch before login for + cookie(?) Closes #633 + - adapter_webnovelcom: Fixes for site changes. Closes #629 + - quotev.com: site change in date parse, use utf8:ignore as + first encoding choice. Closes #625 + + +------------------------------------------------------------------- +Sat Jan 16 20:39:24 UTC 2021 - mcepl@suse.com + +- Update to version 3.28.0+git.1610814197.37cdec2f: + * Bump Test Version 3.28.4 + * First rough version of reporting continue_on_chapter_error chapters. + * Tweak adapter_test1 chapter error case + * ffnet fpcom, continue_on_chapter_error:true by default, increase sleep times. + * Re-enable ffnet and bump up sleep times for same. + * Bump Test Version 3.28.3 + * adapter_fanficsme: do an extra fetch before login for cookie(?) Closes #633 + * Bump Test Version 3.28.2 + * adapter_webnovelcom: Fixes for site changes. Closes #629 + * Update translations. + * Bump Test Version 3.28.1 + * quotev.com: site change in date parse, use utf8:ignore as first encoding choice. Closes #625 + * Bump Release Version 3.28.0 + * Update translations + * Tweak disable adapter_fanfictionnet text. + * Bump Test Version 3.27.6 + * Catch exception from emails not decoding, skip & logger.error(). + * Disable adapter_fanfictionnet with warning about site blocking. + * Bump Test Version 3.27.5 + * Add a fake get_image_size() method for when no image processing available. Closes #621 + * Bump Test Version 3.27.4 + * Change adapter_twilightednet to https + * Bump Test Version 3.27.3 + * Change for adapter_fanfictionnetadapter_fanfictionnet to make skip_author_cover work again. + * Bump Test Version 3.27.2 + * Make included certifi and requests use same tmp file code and store under calibre tmp dir for cleanup. + * Bump Test Version 3.27.1 + * Add append_datepublished_to_storyurl option for storiesonline.net, finestories.com, scifistories.com only. + * Bump Release Version 3.27.0 + * Update translations. +- Add back dependency (both BR and R) on python-cloudscraper. + ------------------------------------------------------------------- Tue Jan 12 16:12:03 UTC 2021 - Matej Cepl diff --git a/python-fanficfare.spec b/python-fanficfare.spec index 44daf89..cd88d25 100644 --- a/python-fanficfare.spec +++ b/python-fanficfare.spec @@ -21,7 +21,7 @@ %define skip_python2 1 %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-fanficfare -Version: 3.28.0 +Version: 3.29.0 Release: 0 Summary: Tool for making eBooks from stories on fanfiction and other web sites License: GPL-3.0-only @@ -29,11 +29,9 @@ Group: Development/Languages/Python URL: https://github.com/JimmXinu/FanFicFare Source: https://github.com/JimmXinu/%{modname}/archive/v%{version}/%{modname}-%{version}.tar.gz # Source: %%{modname}-%%{version}.tar.gz -# PATCH-FEATURE-OPENSUSE no-cloudscraper.patch mcepl@suse.com -# don't use cloudscraper -Patch0: no-cloudscraper.patch BuildRequires: %{python_module beautifulsoup4} BuildRequires: %{python_module chardet} +BuildRequires: %{python_module cloudscraper} BuildRequires: %{python_module html2text} BuildRequires: %{python_module html5lib} BuildRequires: %{python_module setuptools >= 17.1} @@ -42,6 +40,7 @@ BuildRequires: fdupes BuildRequires: python-rpm-macros Requires: python-beautifulsoup4 Requires: python-chardet +Requires: python-cloudscraper Requires: python-html2text Requires: python-html5lib Requires: python-setuptools