mcepl/python-fanficfare
SHA256
1
0
Fork 0
forked from pool/python-fanficfare
Code Pull Requests Activity
aaf5253968
python-fanficfare/no-cloudscraper.patch

217 lines
9.4 KiB
Diff
Raw Blame History

--- a/fanficfare/configurable.py
+++ b/fanficfare/configurable.py
@@ -44,7 +44,6 @@ import pickle
## isn't found in plugin when only imported down below inside
## get_scraper()
-import cloudscraper
from . import exceptions
@@ -210,7 +209,6 @@ def get_valid_set_options():
'titlepage_use_table':(None,None,boollist),
'use_ssl_unverified_context':(None,None,boollist),
- 'use_cloudscraper':(None,None,boollist),
'continue_on_chapter_error':(None,None,boollist),
'conditionals_use_lists':(None,None,boollist),
'dedup_chapter_list':(None,None,boollist),
@@ -483,7 +481,6 @@ def get_valid_keywords():
'tweak_fg_sleep',
'universe_as_series',
'use_ssl_unverified_context',
- 'use_cloudscraper',
'user_agent',
'username',
'website_encodings',
@@ -598,16 +595,11 @@ class Configuration(ConfigParser):
self.override_sleep = None
self.cookiejar = self.get_empty_cookiejar()
self.opener = build_opener(HTTPCookieProcessor(self.cookiejar),GZipProcessor())
- self.scraper = None
self.pagecache = self.get_empty_pagecache()
self.save_cache_file = None
self.save_cookiejar_file = None
- def __del__(self):
- if self.scraper is not None:
- self.scraper.close()
-
def section_url_names(self,domain,section_url_f):
## domain is passed as a method to limit the damage if/when an
## adapter screws up _section_url
@@ -1073,24 +1065,6 @@ class Configuration(ConfigParser):
logger.warning("reduce_zalgo failed(%s), continuing."%e)
return data
- def get_scraper(self):
- if not self.scraper:
- ## ffnet adapter can't parse mobile output, so we only
- ## want desktop browser. But cloudscraper then insists on
- ## a browser and platform, too.
- self.scraper = cloudscraper.CloudScraper(browser={
- 'browser': 'chrome',
- 'platform': 'windows',
- 'mobile': False,
- 'desktop': True,
- })
- ## CloudScraper is subclass of requests.Session.
- ## probably need import higher up if ever used.
- # import requests
- # self.scraper = requests.Session()
- self.scraper.cookies = self.cookiejar
- return self.scraper
-
# Assumes application/x-www-form-urlencoded. parameters, headers are dict()s
def _postUrl(self, url,
parameters={},
@@ -1132,24 +1106,15 @@ class Configuration(ConfigParser):
# headers['Authorization']=b"Basic %s" % base64string
# logger.debug("http login for SB xf2test")
- if self.getConfig('use_cloudscraper',False):
- logger.debug("Using cloudscraper for POST")
- resp = self.get_scraper().post(url,
- headers=dict(headers),
- data=parameters)
- logger.debug("response code:%s"%resp.status_code)
- resp.raise_for_status() # raises HTTPError if error code.
- data = resp.content
- else:
- req = Request(url,
- data=ensure_binary(urlencode(parameters)),
- headers=headers)
-
- ## Specific UA because too many sites are blocking the default python UA.
- self.opener.addheaders = [('User-Agent', self.getConfig('user_agent')),
- ('X-Clacks-Overhead','GNU Terry Pratchett')]
+ req = Request(url,
+ data=ensure_binary(urlencode(parameters)),
+ headers=headers)
+
+ ## Specific UA because too many sites are blocking the default python UA.
+ self.opener.addheaders = [('User-Agent', self.getConfig('user_agent')),
+ ('X-Clacks-Overhead','GNU Terry Pratchett')]
- data = self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read()
+ data = self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read()
data = self._do_reduce_zalgo(self._decode(data))
self._progressbar()
## postURL saves data to the pagecache *after* _decode() while
@@ -1227,37 +1192,16 @@ class Configuration(ConfigParser):
self.opener.addheaders = headers
- if self.getConfig('use_cloudscraper',False):
- ## requests / cloudscraper wants a dict() for headers, not
- ## list of tuples.
- headers = dict(headers)
- ## let cloudscraper do its thing with UA.
- if 'User-Agent' in headers:
- del headers['User-Agent']
- if parameters != None:
- logger.debug("Using cloudscraper for fetch POST")
- resp = self.get_scraper().post(url,
- headers=headers,
- data=parameters)
- else:
- logger.debug("Using cloudscraper for GET")
- resp = self.get_scraper().get(url,
- headers=headers)
- logger.debug("response code:%s"%resp.status_code)
- resp.raise_for_status() # raises HTTPError if error code.
- data = resp.content
- opened = FakeOpened(data,resp.url)
+ ## opener.open() will to POST with params(data) and GET without.
+ if parameters != None:
+ opened = self.opener.open(url,
+ ensure_binary(urlencode(parameters)),
+ float(self.getConfig('connect_timeout',30.0)))
else:
- ## opener.open() will to POST with params(data) and GET without.
- if parameters != None:
- opened = self.opener.open(url,
- ensure_binary(urlencode(parameters)),
- float(self.getConfig('connect_timeout',30.0)))
- else:
- opened = self.opener.open(url,
- None,
- float(self.getConfig('connect_timeout',30.0)))
- data = opened.read()
+ opened = self.opener.open(url,
+ None,
+ float(self.getConfig('connect_timeout',30.0)))
+ data = opened.read()
self._progressbar()
## postURL saves data to the pagecache *after* _decode() while
## fetchRaw saves it *before* _decode()--because raw.
--- a/fanficfare/defaults.ini
+++ b/fanficfare/defaults.ini
@@ -2793,15 +2793,6 @@ type_label:Type of Couple
website_encodings:Windows-1252,utf8
[www.fanfiction.net]
-## Using cloudscraper can satisfy the first couple levels of
-## Cloudflare bot-proofing, but not all levels. Older versions of
-## OpenSSL will also raise problems, so versions of Calibre older than
-## v5 will probably fail. Only fanfiction.net and fictionpress.com
-## are configured with use_cloudscraper:true by default, but it can be
-## applied in other sites' ini sections. user_agent setting is
-## ignored when use_cloudscraper:true
-use_cloudscraper:true
-
## fanfiction.net's 'cover' images are really just tiny thumbnails.
## Set this to true to never use them.
#never_make_cover: false
@@ -2888,15 +2879,6 @@ website_encodings:Windows-1252,utf8
slow_down_sleep_time:10
[www.fictionpress.com]
-## Using cloudscraper can satisfy the first couple levels of
-## Cloudflare bot-proofing, but not all levels. Older versions of
-## OpenSSL will also raise problems, so versions of Calibre older than
-## v5 will probably fail. Only fanfiction.net and fictionpress.com
-## are configured with use_cloudscraper:true by default, but it can be
-## applied in other sites' ini sections. user_agent setting is
-## ignored when use_cloudscraper:true
-use_cloudscraper:true
-
## Clear FanFiction from defaults, fictionpress.com is original fiction.
extratags:
--- a/makeplugin.py
+++ b/makeplugin.py
@@ -23,7 +23,7 @@ from makezip import createZipFile
if __name__=="__main__":
filename="FanFicFare.zip"
exclude=['*.pyc','*~','*.xcf','*[0-9].png','*.po','*.pot','*default.mo','*Thumbs.db']
-
+
os.chdir('calibre-plugin')
files=['plugin-defaults.ini','plugin-example.ini','about.html',
'images','translations']
@@ -35,8 +35,8 @@ if __name__=="__main__":
exclude=exclude)
os.chdir('../included_dependencies')
- files=['bs4','chardet','html2text','soupsieve','backports',
- 'cloudscraper','requests','requests_toolbelt','urllib3',
+ files=['bs4', 'chardet', 'html2text', 'soupsieve', 'backports',
+ 'requests', 'requests_toolbelt', 'urllib3',
'certifi','idna']
## Kept only for v2.85.1 support now.
createZipFile("../"+filename,"a",
--- a/setup.py
+++ b/setup.py
@@ -84,8 +84,7 @@ setup(
install_requires=['beautifulsoup4',
'chardet',
'html5lib',
- 'html2text',
- 'cloudscraper'],
+ 'html2text'],
# html5lib requires 'six', FFF includes it's own copy as fanficfare.six
# List additional groups of dependencies here (e.g. development
View Git Blame Copy Permalink