git-importer/lib/proxy_sha256.py

import functools
import hashlib
import logging
import urllib

import requests


def _hash(hash_alg, file_or_path):
    h = hash_alg()

    def __hash(f):
        while chunk := f.read(1024 * 4):
            h.update(chunk)

    if hasattr(file_or_path, "read"):
        __hash(file_or_path)
    else:
        with file_or_path.open("rb") as f:
            __hash(f)
    return h.hexdigest()


md5 = functools.partial(_hash, hashlib.md5)
sha256 = functools.partial(_hash, hashlib.sha256)


class ProxySHA256:
    def __init__(self, obs, url=None, enabled=True):
        self.obs = obs
        self.url = url if url else "http://source.dyn.cloud.suse.de"
        self.enabled = enabled
        self.hashes = None
        self.texts = set()

    def load_package(self, package):
        # _project is unreachable for the proxy - due to being a fake package
        if package == "_project":
            self.enabled = False
            self.texts = set(["_config", "_service"])
            self.hashes = dict()
            return
        logging.debug("Retrieve all previously defined SHA256")
        response = requests.get(f"http://source.dyn.cloud.suse.de/package/{package}")
        if response.status_code == 200:
            json = response.json()
            self.hashes = json["shas"]
            self.texts = set(json["texts"])

    def get(self, package, name, file_md5):
        key = f"{file_md5}-{name}"
        if self.hashes is None:
            if self.enabled:
                self.load_package(package)
            else:
                self.hashes = {}
        return self.hashes.get(key, None)

    def _proxy_put(self, project, package, name, revision, file_md5, size):
        quoted_name = urllib.parse.quote(name)
        url = f"{self.obs.url}/public/source/{project}/{package}/{quoted_name}?rev={revision}"
        response = requests.put(
            self.url,
            data={
                "hash": file_md5,
                "filename": name,
                "url": url,
                "package": package,
            },
        )
        if response.status_code != 200:
            raise Exception(f"Redirector error on {self.url} for {url}")

        key = (file_md5, name)
        self.hashes[key] = {
            "sha256": response.content.decode("utf-8"),
            "fsize": size,
        }
        return self.hashes[key]

    def _obs_put(self, project, package, name, revision, file_md5, size):
        key = (file_md5, name)
        self.hashes[key] = {
            "sha256": sha256(self.obs._download(project, package, name, revision)),
            "fsize": size,
        }
        return self.hashes[key]

    def put(self, project, package, name, revision, file_md5, size):
        if not self.enabled:
            return self._obs_put(project, package, name, revision, file_md5, size)
        return self._proxy_put(project, package, name, revision, file_md5, size)

    def is_text(self, filename):
        return filename in self.texts

    def get_or_put(self, project, package, name, revision, file_md5, size):
        result = self.get(package, name, file_md5)
        if not result:
            result = self.put(project, package, name, revision, file_md5, size)

        # Sanity check
        if result["fsize"] != size:
            raise Exception(f"Redirector has different size for {name}")

        return result