import functools import hashlib import logging import urllib import requests def _hash(hash_alg, file_or_path): h = hash_alg() def __hash(f): while chunk := f.read(1024 * 4): h.update(chunk) if hasattr(file_or_path, "read"): __hash(file_or_path) else: with file_or_path.open("rb") as f: __hash(f) return h.hexdigest() md5 = functools.partial(_hash, hashlib.md5) sha256 = functools.partial(_hash, hashlib.sha256) class ProxySHA256: def __init__(self, obs, url=None, enabled=True): self.obs = obs self.url = url if url else "http://source.dyn.cloud.suse.de" self.enabled = enabled self.hashes = None self.texts = None def load_package(self, package): # _project is unreachable for the proxy - due to being a fake package if package == "_project": self.enabled = False self.texts = set(["_config", "_service", "_staging_workflow"]) self.hashes = dict() return logging.debug("Retrieve all previously defined SHA256") response = requests.get( f"http://source.dyn.cloud.suse.de/package/{package}", timeout=50 ) if response.status_code == 200: json = response.json() self.hashes = json["shas"] self.texts = set(json["texts"]) def get(self, package, name, file_md5): key = f"{file_md5}-{name}" if self.hashes is None: if self.enabled: self.load_package(package) else: self.hashes = {} return self.hashes.get(key, None) def _proxy_put(self, project, package, name, revision, file_md5, size): quoted_name = urllib.parse.quote(name) url = f"{self.obs.url}/public/source/{project}/{package}/{quoted_name}?rev={revision}" response = requests.put( self.url, data={ "hash": file_md5, "filename": name, "url": url, "package": package, }, timeout=10, ) if response.status_code != 200: raise Exception(f"Redirector error on {self.url} for {url}") key = (file_md5, name) self.hashes[key] = { "sha256": response.content.decode("utf-8"), "fsize": size, } return self.hashes[key] def _obs_put(self, project, package, name, revision, file_md5, size): key = (file_md5, name) self.hashes[key] = { "sha256": sha256(self.obs._download(project, package, name, revision)), "fsize": size, } return self.hashes[key] def put(self, project, package, name, revision, file_md5, size): if not self.enabled: return self._obs_put(project, package, name, revision, file_md5, size) return self._proxy_put(project, package, name, revision, file_md5, size) def is_text(self, package, filename): if self.texts is None: if self.enabled: self.load_package(package) else: self.texts = set() return filename in self.texts def get_or_put(self, project, package, name, revision, file_md5, size): result = self.get(package, name, file_md5) if not result: result = self.put(project, package, name, revision, file_md5, size) # Sanity check if result["fsize"] != size: raise Exception(f"Redirector has different size for {name}") return result