Accepting request 862684 from devel:languages:python

- Add no-cloudscraper.patch to avoid need to use cloudscraper
- Update to 3.28.0:
  - **Disable adapter_fanfictionnet with warning about site
    blocking.**
  - Catch exception from emails not decoding, skip
    & logger.error().
  - Add a fake get_image_size() method for when no image
    processing available. Closes #621 (CLI only)
  - Change adapter_twilightednet to https
  - Change for adapter_fanfictionnetadapter_fanfictionnet to make
    skip_author_cover work again.
  - Make included certifi and requests use same tmp file code and
    store under calibre tmp dir for cleanup.
  - Add append_datepublished_to_storyurl option for
    storiesonline.net, finestories.com, scifistories.com only.

OBS-URL: https://build.opensuse.org/request/show/862684
OBS-URL: https://build.opensuse.org/package/show/openSUSE:Factory/python-fanficfare?expand=0&rev=28

no-cloudscraper.patch

@@ -0,0 +1,216 @@
+--- a/fanficfare/configurable.py
++++ b/fanficfare/configurable.py
+@@ -44,7 +44,6 @@ import pickle
+ 
+ ## isn't found in plugin when only imported down below inside
+ ## get_scraper()
+-import cloudscraper
+ 
+ from . import exceptions
+ 
+@@ -210,7 +209,6 @@ def get_valid_set_options():
+                'titlepage_use_table':(None,None,boollist),
+ 
+                'use_ssl_unverified_context':(None,None,boollist),
+-               'use_cloudscraper':(None,None,boollist),
+                'continue_on_chapter_error':(None,None,boollist),
+                'conditionals_use_lists':(None,None,boollist),
+                'dedup_chapter_list':(None,None,boollist),
+@@ -483,7 +481,6 @@ def get_valid_keywords():
+                  'tweak_fg_sleep',
+                  'universe_as_series',
+                  'use_ssl_unverified_context',
+-                 'use_cloudscraper',
+                  'user_agent',
+                  'username',
+                  'website_encodings',
+@@ -598,16 +595,11 @@ class Configuration(ConfigParser):
+         self.override_sleep = None
+         self.cookiejar = self.get_empty_cookiejar()
+         self.opener = build_opener(HTTPCookieProcessor(self.cookiejar),GZipProcessor())
+-        self.scraper = None
+ 
+         self.pagecache = self.get_empty_pagecache()
+         self.save_cache_file = None
+         self.save_cookiejar_file = None
+ 
+-    def __del__(self):
+-        if self.scraper is not None:
+-            self.scraper.close()
+-
+     def section_url_names(self,domain,section_url_f):
+         ## domain is passed as a method to limit the damage if/when an
+         ## adapter screws up _section_url
+@@ -1073,24 +1065,6 @@ class Configuration(ConfigParser):
+                 logger.warning("reduce_zalgo failed(%s), continuing."%e)
+         return data
+ 
+-    def get_scraper(self):
+-        if not self.scraper:
+-            ## ffnet adapter can't parse mobile output, so we only
+-            ## want desktop browser.  But cloudscraper then insists on
+-            ## a browser and platform, too.
+-            self.scraper = cloudscraper.CloudScraper(browser={
+-                    'browser': 'chrome',
+-                    'platform': 'windows',
+-                    'mobile': False,
+-                    'desktop': True,
+-                    })
+-            ## CloudScraper is subclass of requests.Session.
+-            ## probably need import higher up if ever used.
+-            # import requests
+-            # self.scraper = requests.Session()
+-            self.scraper.cookies = self.cookiejar
+-        return self.scraper
+-
+     # Assumes application/x-www-form-urlencoded.  parameters, headers are dict()s
+     def _postUrl(self, url,
+                  parameters={},
+@@ -1132,24 +1106,15 @@ class Configuration(ConfigParser):
+         #     headers['Authorization']=b"Basic %s" % base64string
+         #     logger.debug("http login for SB xf2test")
+ 
+-        if self.getConfig('use_cloudscraper',False):
+-            logger.debug("Using cloudscraper for POST")
+-            resp = self.get_scraper().post(url,
+-                                           headers=dict(headers),
+-                                           data=parameters)
+-            logger.debug("response code:%s"%resp.status_code)
+-            resp.raise_for_status() # raises HTTPError if error code.
+-            data = resp.content
+-        else:
+-            req = Request(url,
+-                          data=ensure_binary(urlencode(parameters)),
+-                          headers=headers)
+-
+-            ## Specific UA because too many sites are blocking the default python UA.
+-            self.opener.addheaders = [('User-Agent', self.getConfig('user_agent')),
+-                                      ('X-Clacks-Overhead','GNU Terry Pratchett')]
++        req = Request(url,
++                      data=ensure_binary(urlencode(parameters)),
++                      headers=headers)
++
++        ## Specific UA because too many sites are blocking the default python UA.
++        self.opener.addheaders = [('User-Agent', self.getConfig('user_agent')),
++                                  ('X-Clacks-Overhead','GNU Terry Pratchett')]
+ 
+-            data = self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read()
++        data = self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read()
+         data = self._do_reduce_zalgo(self._decode(data))
+         self._progressbar()
+         ## postURL saves data to the pagecache *after* _decode() while
+@@ -1227,37 +1192,16 @@ class Configuration(ConfigParser):
+ 
+         self.opener.addheaders = headers
+ 
+-        if self.getConfig('use_cloudscraper',False):
+-            ## requests / cloudscraper wants a dict() for headers, not
+-            ## list of tuples.
+-            headers = dict(headers)
+-            ## let cloudscraper do its thing with UA.
+-            if 'User-Agent' in headers:
+-                del headers['User-Agent']
+-            if parameters != None:
+-                logger.debug("Using cloudscraper for fetch POST")
+-                resp = self.get_scraper().post(url,
+-                                               headers=headers,
+-                                               data=parameters)
+-            else:
+-                logger.debug("Using cloudscraper for GET")
+-                resp = self.get_scraper().get(url,
+-                                              headers=headers)
+-            logger.debug("response code:%s"%resp.status_code)
+-            resp.raise_for_status() # raises HTTPError if error code.
+-            data = resp.content
+-            opened = FakeOpened(data,resp.url)
++        ## opener.open() will to POST with params(data) and GET without.
++        if parameters != None:
++            opened = self.opener.open(url,
++                                      ensure_binary(urlencode(parameters)),
++                                      float(self.getConfig('connect_timeout',30.0)))
+         else:
+-            ## opener.open() will to POST with params(data) and GET without.
+-            if parameters != None:
+-                opened = self.opener.open(url,
+-                                          ensure_binary(urlencode(parameters)),
+-                                          float(self.getConfig('connect_timeout',30.0)))
+-            else:
+-                opened = self.opener.open(url,
+-                                          None,
+-                                          float(self.getConfig('connect_timeout',30.0)))
+-            data = opened.read()
++            opened = self.opener.open(url,
++                                      None,
++                                      float(self.getConfig('connect_timeout',30.0)))
++        data = opened.read()
+         self._progressbar()
+         ## postURL saves data to the pagecache *after* _decode() while
+         ## fetchRaw saves it *before* _decode()--because raw.
+--- a/fanficfare/defaults.ini
++++ b/fanficfare/defaults.ini
+@@ -2793,15 +2793,6 @@ type_label:Type of Couple
+ website_encodings:Windows-1252,utf8
+ 
+ [www.fanfiction.net]
+-## Using cloudscraper can satisfy the first couple levels of
+-## Cloudflare bot-proofing, but not all levels.  Older versions of
+-## OpenSSL will also raise problems, so versions of Calibre older than
+-## v5 will probably fail.  Only fanfiction.net and fictionpress.com
+-## are configured with use_cloudscraper:true by default, but it can be
+-## applied in other sites' ini sections.  user_agent setting is
+-## ignored when use_cloudscraper:true
+-use_cloudscraper:true
+-
+ ## fanfiction.net's 'cover' images are really just tiny thumbnails.
+ ## Set this to true to never use them.
+ #never_make_cover: false
+@@ -2888,15 +2879,6 @@ website_encodings:Windows-1252,utf8
+ slow_down_sleep_time:10
+ 
+ [www.fictionpress.com]
+-## Using cloudscraper can satisfy the first couple levels of
+-## Cloudflare bot-proofing, but not all levels.  Older versions of
+-## OpenSSL will also raise problems, so versions of Calibre older than
+-## v5 will probably fail.  Only fanfiction.net and fictionpress.com
+-## are configured with use_cloudscraper:true by default, but it can be
+-## applied in other sites' ini sections.  user_agent setting is
+-## ignored when use_cloudscraper:true
+-use_cloudscraper:true
+-
+ ## Clear FanFiction from defaults, fictionpress.com is original fiction.
+ extratags:
+ 
+--- a/makeplugin.py
++++ b/makeplugin.py
+@@ -23,7 +23,7 @@ from makezip import createZipFile
+ if __name__=="__main__":
+     filename="FanFicFare.zip"
+     exclude=['*.pyc','*~','*.xcf','*[0-9].png','*.po','*.pot','*default.mo','*Thumbs.db']
+-    
++
+     os.chdir('calibre-plugin')
+     files=['plugin-defaults.ini','plugin-example.ini','about.html',
+            'images','translations']
+@@ -35,8 +35,8 @@ if __name__=="__main__":
+                   exclude=exclude)
+ 
+     os.chdir('../included_dependencies')
+-    files=['bs4','chardet','html2text','soupsieve','backports',
+-           'cloudscraper','requests','requests_toolbelt','urllib3',
++    files=['bs4', 'chardet', 'html2text', 'soupsieve', 'backports',
++           'requests', 'requests_toolbelt', 'urllib3',
+            'certifi','idna']
+     ## Kept only for v2.85.1 support now.
+     createZipFile("../"+filename,"a",
+--- a/setup.py
++++ b/setup.py
+@@ -84,8 +84,7 @@ setup(
+     install_requires=['beautifulsoup4',
+                       'chardet',
+                       'html5lib',
+-                      'html2text',
+-                      'cloudscraper'],
++                      'html2text'],
+     # html5lib requires 'six', FFF includes it's own copy as fanficfare.six
+ 
+     # List additional groups of dependencies here (e.g. development

python-fanficfare.changes

@@ -1,3 +1,26 @@
+-------------------------------------------------------------------
+Tue Jan 12 16:12:03 UTC 2021 - Matej Cepl <mcepl@suse.com>
+
+- Add no-cloudscraper.patch to avoid need to use cloudscraper
+
+-------------------------------------------------------------------
+Mon Jan 11 17:28:41 CET 2021 - Matej Cepl <mcepl@suse.com>
+
+- Update to 3.28.0:
+  - **Disable adapter_fanfictionnet with warning about site
+    blocking.**
+  - Catch exception from emails not decoding, skip
+    & logger.error().
+  - Add a fake get_image_size() method for when no image
+    processing available. Closes #621 (CLI only)
+  - Change adapter_twilightednet to https
+  - Change for adapter_fanfictionnetadapter_fanfictionnet to make
+    skip_author_cover work again.
+  - Make included certifi and requests use same tmp file code and
+    store under calibre tmp dir for cleanup.
+  - Add append_datepublished_to_storyurl option for
+    storiesonline.net, finestories.com, scifistories.com only.
+
 -------------------------------------------------------------------
 Thu Dec 24 16:17:00 UTC 2020 - Matej Cepl <mcepl@suse.com>
 

python-fanficfare.spec

@@ -1,7 +1,7 @@
 #
 # spec file for package python-fanficfare
 #
-# Copyright (c) 2020 SUSE LLC
+# Copyright (c) 2021 SUSE LLC
 #
 # All modifications and additions to the file contributed by third parties
 # remain the property of their copyright owners, unless otherwise agreed
@@ -21,7 +21,7 @@
 %define skip_python2 1
 %{?!python_module:%define python_module() python-%{**} python3-%{**}}
 Name:           python-fanficfare
-Version:        3.27.0
+Version:        3.28.0
 Release:        0
 Summary:        Tool for making eBooks from stories on fanfiction and other web sites
 License:        GPL-3.0-only
@@ -29,6 +29,9 @@ Group:          Development/Languages/Python
 URL:            https://github.com/JimmXinu/FanFicFare
 Source:         https://github.com/JimmXinu/%{modname}/archive/v%{version}/%{modname}-%{version}.tar.gz
 # Source:         %%{modname}-%%{version}.tar.gz
+# PATCH-FEATURE-OPENSUSE no-cloudscraper.patch mcepl@suse.com
+# don't use cloudscraper
+Patch0:         no-cloudscraper.patch
 BuildRequires:  %{python_module beautifulsoup4}
 BuildRequires:  %{python_module chardet}
 BuildRequires:  %{python_module html2text}
@@ -39,7 +42,6 @@ BuildRequires:  fdupes
 BuildRequires:  python-rpm-macros
 Requires:       python-beautifulsoup4
 Requires:       python-chardet
-Requires:       python-cloudscraper
 Requires:       python-html2text
 Requires:       python-html5lib
 Requires:       python-setuptools