forked from importers/git-importer
106 lines
3.3 KiB
Python
106 lines
3.3 KiB
Python
|
import logging
|
||
|
|
import requests
|
||
|
|
import urllib
|
||
|
|
import functools
|
||
|
|
import hashlib
|
||
|
|
|
||
|
|
|
||
|
|
def _hash(hash_alg, file_or_path):
|
||
|
|
h = hash_alg()
|
||
|
|
|
||
|
|
def __hash(f):
|
||
|
|
while chunk := f.read(1024 * 4):
|
||
|
|
h.update(chunk)
|
||
|
|
|
||
|
|
if hasattr(file_or_path, "read"):
|
||
|
|
__hash(file_or_path)
|
||
|
|
else:
|
||
|
|
with file_or_path.open("rb") as f:
|
||
|
|
__hash(f)
|
||
|
|
return h.hexdigest()
|
||
|
|
|
||
|
|
|
||
|
|
md5 = functools.partial(_hash, hashlib.md5)
|
||
|
|
sha256 = functools.partial(_hash, hashlib.sha256)
|
||
|
|
|
||
|
|
|
||
|
|
class ProxySHA256:
|
||
|
|
def __init__(self, obs, url=None, enabled=True):
|
||
|
|
self.obs = obs
|
||
|
|
self.url = url if url else "http://source.dyn.cloud.suse.de"
|
||
|
|
self.enabled = enabled
|
||
|
|
self.hashes = None
|
||
|
|
self.texts = set()
|
||
|
|
|
||
|
|
def load_package(self, package):
|
||
|
|
# _project is unreachable for the proxy - due to being a fake package
|
||
|
|
if package == "_project":
|
||
|
|
self.enabled = False
|
||
|
|
self.texts = set(["_config", "_service"])
|
||
|
|
self.hashes = dict()
|
||
|
|
return
|
||
|
|
logging.info("Retrieve all previously defined SHA256")
|
||
|
|
response = requests.get(f"http://source.dyn.cloud.suse.de/package/{package}")
|
||
|
|
if response.status_code == 200:
|
||
|
|
json = response.json()
|
||
|
|
self.hashes = json["shas"]
|
||
|
|
self.texts = set(json["texts"])
|
||
|
|
|
||
|
|
def get(self, package, name, file_md5):
|
||
|
|
key = f"{file_md5}-{name}"
|
||
|
|
if self.hashes is None:
|
||
|
|
if self.enabled:
|
||
|
|
self.load_package(package)
|
||
|
|
else:
|
||
|
|
self.hashes = {}
|
||
|
|
return self.hashes.get(key, None)
|
||
|
|
|
||
|
|
def _proxy_put(self, project, package, name, revision, file_md5, size):
|
||
|
|
quoted_name = urllib.parse.quote(name)
|
||
|
|
url = f"{self.obs.url}/public/source/{project}/{package}/{quoted_name}?rev={revision}"
|
||
|
|
response = requests.put(
|
||
|
|
self.url,
|
||
|
|
data={
|
||
|
|
"hash": file_md5,
|
||
|
|
"filename": name,
|
||
|
|
"url": url,
|
||
|
|
"package": package,
|
||
|
|
},
|
||
|
|
)
|
||
|
|
if response.status_code != 200:
|
||
|
|
raise Exception(f"Redirector error on {self.url} for {url}")
|
||
|
|
|
||
|
|
key = (file_md5, name)
|
||
|
|
self.hashes[key] = {
|
||
|
|
"sha256": response.content.decode("utf-8"),
|
||
|
|
"fsize": size,
|
||
|
|
}
|
||
|
|
return self.hashes[key]
|
||
|
|
|
||
|
|
def _obs_put(self, project, package, name, revision, file_md5, size):
|
||
|
|
key = (file_md5, name)
|
||
|
|
self.hashes[key] = {
|
||
|
|
"sha256": sha256(self.obs._download(project, package, name, revision)),
|
||
|
|
"fsize": size,
|
||
|
|
}
|
||
|
|
return self.hashes[key]
|
||
|
|
|
||
|
|
def put(self, project, package, name, revision, file_md5, size):
|
||
|
|
if not self.enabled:
|
||
|
|
return self._obs_put(project, package, name, revision, file_md5, size)
|
||
|
|
return self._proxy_put(project, package, name, revision, file_md5, size)
|
||
|
|
|
||
|
|
def is_text(self, filename):
|
||
|
|
return filename in self.texts
|
||
|
|
|
||
|
|
def get_or_put(self, project, package, name, revision, file_md5, size):
|
||
|
|
result = self.get(package, name, file_md5)
|
||
|
|
if not result:
|
||
|
|
result = self.put(project, package, name, revision, file_md5, size)
|
||
|
|
|
||
|
|
# Sanity check
|
||
|
|
if result["fsize"] != size:
|
||
|
|
raise Exception(f"Redirector has different size for {name}")
|
||
|
|
|
||
|
|
return result
|