importers/git-importer
3
0
Fork 2
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Load the proxy data for is_text as well

Browse Source 
Otherwise the text state changes over time
This commit is contained in:
Stephan Kulow 2022-11-04 20:04:46 +01:00
parent 7f88e0cc5c
commit 0bcc0183c9
2 changed files with 10 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/git_exporter.py
View File

@ -72,7 +72,7 @@ class GitExporter:
def commit_file(self, flat, file, size, md5): def commit_file(self, flat, file, size, md5):
# have such files been detected as text mimetype before? # have such files been detected as text mimetype before?
is_text = self.proxy_sha256.is_text(file.name) is_text = self.proxy_sha256.is_text(flat.commit.package, file.name)
if not is_text and is_binary_or_large(file.name, size): if not is_text and is_binary_or_large(file.name, size):
file_sha256 = self.proxy_sha256.get_or_put( file_sha256 = self.proxy_sha256.get_or_put(
flat.commit.project, flat.commit.project,

13
lib/proxy_sha256.py
View File

@ -31,18 +31,18 @@ class ProxySHA256:
self.url = url if url else "http://source.dyn.cloud.suse.de" self.url = url if url else "http://source.dyn.cloud.suse.de"
self.enabled = enabled self.enabled = enabled
self.hashes = None self.hashes = None
self.texts = set() self.texts = None
def load_package(self, package): def load_package(self, package):
# _project is unreachable for the proxy - due to being a fake package # _project is unreachable for the proxy - due to being a fake package
if package == "_project": if package == "_project":
self.enabled = False self.enabled = False
self.texts = set(["_config", "_service"]) self.texts = set(["_config", "_service", "_staging_workflow"])
self.hashes = dict() self.hashes = dict()
return return
logging.debug("Retrieve all previously defined SHA256") logging.debug("Retrieve all previously defined SHA256")
response = requests.get( response = requests.get(
f"http://source.dyn.cloud.suse.de/package/{package}", timeout=5 f"http://source.dyn.cloud.suse.de/package/{package}", timeout=50
) )
if response.status_code == 200: if response.status_code == 200:
json = response.json() json = response.json()
@ -94,7 +94,12 @@ class ProxySHA256:
return self._obs_put(project, package, name, revision, file_md5, size) return self._obs_put(project, package, name, revision, file_md5, size)
return self._proxy_put(project, package, name, revision, file_md5, size) return self._proxy_put(project, package, name, revision, file_md5, size)
def is_text(self, filename): def is_text(self, package, filename):
if self.texts is None:
if self.enabled:
self.load_package(package)
else:
self.texts = set()
return filename in self.texts return filename in self.texts
def get_or_put(self, project, package, name, revision, file_md5, size): def get_or_put(self, project, package, name, revision, file_md5, size):