git-importer/lib/lfs_oid.py

from __future__ import annotations

import logging
import os
import sys

import requests

from lib.binary import is_text_mimetype
from lib.db import DB


# no need for this class yet, so just leave the migration code here
class LFSOid:
    def __init__(self, db: DB) -> None:
        self.db = db
        self.dbid = None
        self.project = None
        self.package = None
        self.filename = None
        self.revision = None
        self.sha = None
        self.size = None
        self.mimetype = None
        self.file_md5 = None

    @staticmethod
    def check_all(db, package):
        with db.cursor() as cur:
            cur.execute(
                "SELECT lfs_oid_id FROM lfs_oid_in_package WHERE package=%s ORDER BY lfs_oid_id DESC limit 10 ",
                (package,),
            )
            for row in cur.fetchall():
                oid = LFSOid(db).set_from_dbid(row[0])
                if not oid.check():
                    oid.register()

    def add(
        self,
        project: str,
        package: str,
        filename: str,
        revision: str,
        sha256: str,
        size: int,
        mimetype: str,
        file_md5: str,
    ) -> None:
        with self.db.cursor() as cur:
            # we UPDATE here so the return functions. conflicts are likely as we look for filename/md5 but conflict on sha256
            cur.execute(
                """INSERT INTO lfs_oids (project,package,filename,rev,sha256,size,mimetype,file_md5)
                          VALUES (%s,%s,%s,%s,%s,%s,%s,%s)
                          ON CONFLICT (sha256,size) DO UPDATE SET mimetype=EXCLUDED.mimetype
                          RETURNING id""",
                (
                    project,
                    package,
                    filename,
                    revision,
                    sha256,
                    size,
                    mimetype,
                    file_md5,
                ),
            )
            row = cur.fetchone()
            lfs_oid_id = row[0]
            cur.execute(
                """INSERT INTO lfs_oid_in_package (package,filename,lfs_oid_id) 
                        VALUES (%s,%s,%s)""",
                (package, filename, lfs_oid_id),
            )
            if is_text_mimetype(mimetype):
                cur.execute(
                    "INSERT INTO text_files (package,filename) VALUES (%s,%s)",
                    (package, filename),
                )
            self.db.conn.commit()
        self.set_from_dbid(lfs_oid_id)
        if not self.check():
            self.register()

    def check(self):
        url = f"http://localhost:9999/check/{self.sha256}/{self.size}"
        response = requests.get(
            url,
            timeout=10,
        )
        return response.status_code == 200

    def set_from_dbid(self, dbid: int) -> LFSOid:
        with self.db.cursor() as cur:
            cur.execute("SELECT * from lfs_oids where id=%s", (dbid,))
            row = cur.fetchone()
            self.set_from_row(row)
            assert self.dbid == dbid
        return self

    def set_from_row(self, row: list) -> LFSOid:
        (
            self.dbid,
            self.project,
            self.package,
            self.filename,
            self.revision,
            self.sha256,
            self.size,
            self.mimetype,
            self.file_md5,
        ) = row
        return self

    def register(self):
        if not os.getenv("GITEA_REGISTER_SECRET"):
            logging.info("Not registering LFS due to missing secret")
            return

        data = {
            "secret": os.getenv("GITEA_REGISTER_SECRET"),
            "project": self.project,
            "package": self.package,
            "filename": self.filename,
            "rev": self.revision,
            "sha256": self.sha256,
            "size": self.size,
        }

        url = "http://localhost:9999/register"
        response = requests.post(
            url,
            json=data,
            timeout=10,
        )
        logging.info(f"Register LFS returned {response.status_code}")


if __name__ == "__main__":
    """
    Import the old data - it only makes sense on a DB with previously scanned revisions
    curl -s  https://stephan.kulow.org/git_lfs.csv.xz | xz -cd | PYTHONPATH=$PWD /usr/bin/python3 lib/lfs_oid.py
    """
    db = DB()
    logging.basicConfig(level=logging.DEBUG)
    with db.cursor() as cur:
        while True:
            line = sys.stdin.readline()
            if not line:
                break
            (
                project,
                package,
                filename,
                rev,
                sha256,
                size,
                mimetype,
                md5,
            ) = line.strip().split("\t")
            cur.execute(
                """INSERT INTO lfs_oids (project,package,filename,rev,sha256,size,mimetype,file_md5)
                VALUES (%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING""",
                (project, package, filename, rev, sha256, size, mimetype, md5),
            )

        cur.execute(
            """
            CREATE TEMPORARY TABLE lfs_oid_in_revision (
                revision_id INTEGER, 
                lfs_oid_id  INTEGER NOT NULL,
                name        VARCHAR(255) NOT NULL
            )
            """
        )
        cur.execute(
            """INSERT INTO lfs_oid_in_revision (revision_id, lfs_oid_id, name)
                       SELECT revision_id,lfs_oids.id,files.name FROM lfs_oids JOIN files ON files.md5=lfs_oids.file_md5"""
        )
        cur.execute(
            """INSERT INTO text_files (package,filename)
                       SELECT DISTINCT r.package, lfs_oid_in_revision.name FROM lfs_oids
                          JOIN lfs_oid_in_revision on lfs_oid_in_revision.lfs_oid_id=lfs_oids.id
                          JOIN revisions r ON r.id=lfs_oid_in_revision.revision_id
                          WHERE lfs_oids.mimetype like 'text/%' ON CONFLICT DO NOTHING"""
        )
        cur.execute(
            """INSERT INTO lfs_oid_in_package (lfs_oid_id, package, filename)
                        SELECT DISTINCT lfs_oids.id,r.package, lfs_oid_in_revision.name FROM lfs_oids
                        JOIN lfs_oid_in_revision on lfs_oid_in_revision.lfs_oid_id=lfs_oids.id
                        JOIN revisions r ON r.id=lfs_oid_in_revision.revision_id"""
        )
    db.conn.commit()