217 lines
9.4 KiB
Diff
217 lines
9.4 KiB
Diff
|
--- a/fanficfare/configurable.py
|
||
|
|
+++ b/fanficfare/configurable.py
|
||
|
|
@@ -44,7 +44,6 @@ import pickle
|
||
|
|
|
||
|
|
## isn't found in plugin when only imported down below inside
|
||
|
|
## get_scraper()
|
||
|
|
-import cloudscraper
|
||
|
|
|
||
|
|
from . import exceptions
|
||
|
|
|
||
|
|
@@ -210,7 +209,6 @@ def get_valid_set_options():
|
||
|
|
'titlepage_use_table':(None,None,boollist),
|
||
|
|
|
||
|
|
'use_ssl_unverified_context':(None,None,boollist),
|
||
|
|
- 'use_cloudscraper':(None,None,boollist),
|
||
|
|
'continue_on_chapter_error':(None,None,boollist),
|
||
|
|
'conditionals_use_lists':(None,None,boollist),
|
||
|
|
'dedup_chapter_list':(None,None,boollist),
|
||
|
|
@@ -483,7 +481,6 @@ def get_valid_keywords():
|
||
|
|
'tweak_fg_sleep',
|
||
|
|
'universe_as_series',
|
||
|
|
'use_ssl_unverified_context',
|
||
|
|
- 'use_cloudscraper',
|
||
|
|
'user_agent',
|
||
|
|
'username',
|
||
|
|
'website_encodings',
|
||
|
|
@@ -598,16 +595,11 @@ class Configuration(ConfigParser):
|
||
|
|
self.override_sleep = None
|
||
|
|
self.cookiejar = self.get_empty_cookiejar()
|
||
|
|
self.opener = build_opener(HTTPCookieProcessor(self.cookiejar),GZipProcessor())
|
||
|
|
- self.scraper = None
|
||
|
|
|
||
|
|
self.pagecache = self.get_empty_pagecache()
|
||
|
|
self.save_cache_file = None
|
||
|
|
self.save_cookiejar_file = None
|
||
|
|
|
||
|
|
- def __del__(self):
|
||
|
|
- if self.scraper is not None:
|
||
|
|
- self.scraper.close()
|
||
|
|
-
|
||
|
|
def section_url_names(self,domain,section_url_f):
|
||
|
|
## domain is passed as a method to limit the damage if/when an
|
||
|
|
## adapter screws up _section_url
|
||
|
|
@@ -1073,24 +1065,6 @@ class Configuration(ConfigParser):
|
||
|
|
logger.warning("reduce_zalgo failed(%s), continuing."%e)
|
||
|
|
return data
|
||
|
|
|
||
|
|
- def get_scraper(self):
|
||
|
|
- if not self.scraper:
|
||
|
|
- ## ffnet adapter can't parse mobile output, so we only
|
||
|
|
- ## want desktop browser. But cloudscraper then insists on
|
||
|
|
- ## a browser and platform, too.
|
||
|
|
- self.scraper = cloudscraper.CloudScraper(browser={
|
||
|
|
- 'browser': 'chrome',
|
||
|
|
- 'platform': 'windows',
|
||
|
|
- 'mobile': False,
|
||
|
|
- 'desktop': True,
|
||
|
|
- })
|
||
|
|
- ## CloudScraper is subclass of requests.Session.
|
||
|
|
- ## probably need import higher up if ever used.
|
||
|
|
- # import requests
|
||
|
|
- # self.scraper = requests.Session()
|
||
|
|
- self.scraper.cookies = self.cookiejar
|
||
|
|
- return self.scraper
|
||
|
|
-
|
||
|
|
# Assumes application/x-www-form-urlencoded. parameters, headers are dict()s
|
||
|
|
def _postUrl(self, url,
|
||
|
|
parameters={},
|
||
|
|
@@ -1132,24 +1106,15 @@ class Configuration(ConfigParser):
|
||
|
|
# headers['Authorization']=b"Basic %s" % base64string
|
||
|
|
# logger.debug("http login for SB xf2test")
|
||
|
|
|
||
|
|
- if self.getConfig('use_cloudscraper',False):
|
||
|
|
- logger.debug("Using cloudscraper for POST")
|
||
|
|
- resp = self.get_scraper().post(url,
|
||
|
|
- headers=dict(headers),
|
||
|
|
- data=parameters)
|
||
|
|
- logger.debug("response code:%s"%resp.status_code)
|
||
|
|
- resp.raise_for_status() # raises HTTPError if error code.
|
||
|
|
- data = resp.content
|
||
|
|
- else:
|
||
|
|
- req = Request(url,
|
||
|
|
- data=ensure_binary(urlencode(parameters)),
|
||
|
|
- headers=headers)
|
||
|
|
-
|
||
|
|
- ## Specific UA because too many sites are blocking the default python UA.
|
||
|
|
- self.opener.addheaders = [('User-Agent', self.getConfig('user_agent')),
|
||
|
|
- ('X-Clacks-Overhead','GNU Terry Pratchett')]
|
||
|
|
+ req = Request(url,
|
||
|
|
+ data=ensure_binary(urlencode(parameters)),
|
||
|
|
+ headers=headers)
|
||
|
|
+
|
||
|
|
+ ## Specific UA because too many sites are blocking the default python UA.
|
||
|
|
+ self.opener.addheaders = [('User-Agent', self.getConfig('user_agent')),
|
||
|
|
+ ('X-Clacks-Overhead','GNU Terry Pratchett')]
|
||
|
|
|
||
|
|
- data = self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read()
|
||
|
|
+ data = self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read()
|
||
|
|
data = self._do_reduce_zalgo(self._decode(data))
|
||
|
|
self._progressbar()
|
||
|
|
## postURL saves data to the pagecache *after* _decode() while
|
||
|
|
@@ -1227,37 +1192,16 @@ class Configuration(ConfigParser):
|
||
|
|
|
||
|
|
self.opener.addheaders = headers
|
||
|
|
|
||
|
|
- if self.getConfig('use_cloudscraper',False):
|
||
|
|
- ## requests / cloudscraper wants a dict() for headers, not
|
||
|
|
- ## list of tuples.
|
||
|
|
- headers = dict(headers)
|
||
|
|
- ## let cloudscraper do its thing with UA.
|
||
|
|
- if 'User-Agent' in headers:
|
||
|
|
- del headers['User-Agent']
|
||
|
|
- if parameters != None:
|
||
|
|
- logger.debug("Using cloudscraper for fetch POST")
|
||
|
|
- resp = self.get_scraper().post(url,
|
||
|
|
- headers=headers,
|
||
|
|
- data=parameters)
|
||
|
|
- else:
|
||
|
|
- logger.debug("Using cloudscraper for GET")
|
||
|
|
- resp = self.get_scraper().get(url,
|
||
|
|
- headers=headers)
|
||
|
|
- logger.debug("response code:%s"%resp.status_code)
|
||
|
|
- resp.raise_for_status() # raises HTTPError if error code.
|
||
|
|
- data = resp.content
|
||
|
|
- opened = FakeOpened(data,resp.url)
|
||
|
|
+ ## opener.open() will to POST with params(data) and GET without.
|
||
|
|
+ if parameters != None:
|
||
|
|
+ opened = self.opener.open(url,
|
||
|
|
+ ensure_binary(urlencode(parameters)),
|
||
|
|
+ float(self.getConfig('connect_timeout',30.0)))
|
||
|
|
else:
|
||
|
|
- ## opener.open() will to POST with params(data) and GET without.
|
||
|
|
- if parameters != None:
|
||
|
|
- opened = self.opener.open(url,
|
||
|
|
- ensure_binary(urlencode(parameters)),
|
||
|
|
- float(self.getConfig('connect_timeout',30.0)))
|
||
|
|
- else:
|
||
|
|
- opened = self.opener.open(url,
|
||
|
|
- None,
|
||
|
|
- float(self.getConfig('connect_timeout',30.0)))
|
||
|
|
- data = opened.read()
|
||
|
|
+ opened = self.opener.open(url,
|
||
|
|
+ None,
|
||
|
|
+ float(self.getConfig('connect_timeout',30.0)))
|
||
|
|
+ data = opened.read()
|
||
|
|
self._progressbar()
|
||
|
|
## postURL saves data to the pagecache *after* _decode() while
|
||
|
|
## fetchRaw saves it *before* _decode()--because raw.
|
||
|
|
--- a/fanficfare/defaults.ini
|
||
|
|
+++ b/fanficfare/defaults.ini
|
||
|
|
@@ -2793,15 +2793,6 @@ type_label:Type of Couple
|
||
|
|
website_encodings:Windows-1252,utf8
|
||
|
|
|
||
|
|
[www.fanfiction.net]
|
||
|
|
-## Using cloudscraper can satisfy the first couple levels of
|
||
|
|
-## Cloudflare bot-proofing, but not all levels. Older versions of
|
||
|
|
-## OpenSSL will also raise problems, so versions of Calibre older than
|
||
|
|
-## v5 will probably fail. Only fanfiction.net and fictionpress.com
|
||
|
|
-## are configured with use_cloudscraper:true by default, but it can be
|
||
|
|
-## applied in other sites' ini sections. user_agent setting is
|
||
|
|
-## ignored when use_cloudscraper:true
|
||
|
|
-use_cloudscraper:true
|
||
|
|
-
|
||
|
|
## fanfiction.net's 'cover' images are really just tiny thumbnails.
|
||
|
|
## Set this to true to never use them.
|
||
|
|
#never_make_cover: false
|
||
|
|
@@ -2888,15 +2879,6 @@ website_encodings:Windows-1252,utf8
|
||
|
|
slow_down_sleep_time:10
|
||
|
|
|
||
|
|
[www.fictionpress.com]
|
||
|
|
-## Using cloudscraper can satisfy the first couple levels of
|
||
|
|
-## Cloudflare bot-proofing, but not all levels. Older versions of
|
||
|
|
-## OpenSSL will also raise problems, so versions of Calibre older than
|
||
|
|
-## v5 will probably fail. Only fanfiction.net and fictionpress.com
|
||
|
|
-## are configured with use_cloudscraper:true by default, but it can be
|
||
|
|
-## applied in other sites' ini sections. user_agent setting is
|
||
|
|
-## ignored when use_cloudscraper:true
|
||
|
|
-use_cloudscraper:true
|
||
|
|
-
|
||
|
|
## Clear FanFiction from defaults, fictionpress.com is original fiction.
|
||
|
|
extratags:
|
||
|
|
|
||
|
|
--- a/makeplugin.py
|
||
|
|
+++ b/makeplugin.py
|
||
|
|
@@ -23,7 +23,7 @@ from makezip import createZipFile
|
||
|
|
if __name__=="__main__":
|
||
|
|
filename="FanFicFare.zip"
|
||
|
|
exclude=['*.pyc','*~','*.xcf','*[0-9].png','*.po','*.pot','*default.mo','*Thumbs.db']
|
||
|
|
-
|
||
|
|
+
|
||
|
|
os.chdir('calibre-plugin')
|
||
|
|
files=['plugin-defaults.ini','plugin-example.ini','about.html',
|
||
|
|
'images','translations']
|
||
|
|
@@ -35,8 +35,8 @@ if __name__=="__main__":
|
||
|
|
exclude=exclude)
|
||
|
|
|
||
|
|
os.chdir('../included_dependencies')
|
||
|
|
- files=['bs4','chardet','html2text','soupsieve','backports',
|
||
|
|
- 'cloudscraper','requests','requests_toolbelt','urllib3',
|
||
|
|
+ files=['bs4', 'chardet', 'html2text', 'soupsieve', 'backports',
|
||
|
|
+ 'requests', 'requests_toolbelt', 'urllib3',
|
||
|
|
'certifi','idna']
|
||
|
|
## Kept only for v2.85.1 support now.
|
||
|
|
createZipFile("../"+filename,"a",
|
||
|
|
--- a/setup.py
|
||
|
|
+++ b/setup.py
|
||
|
|
@@ -84,8 +84,7 @@ setup(
|
||
|
|
install_requires=['beautifulsoup4',
|
||
|
|
'chardet',
|
||
|
|
'html5lib',
|
||
|
|
- 'html2text',
|
||
|
|
- 'cloudscraper'],
|
||
|
|
+ 'html2text'],
|
||
|
|
# html5lib requires 'six', FFF includes it's own copy as fanficfare.six
|
||
|
|
|
||
|
|
# List additional groups of dependencies here (e.g. development
|