git-importer/git-importer.py

#!/usr/bin/python3

import argparse
import logging
import pathlib
import sys

import osc.core

from lib.db import DB
from lib.db_revision import DBRevision
from lib.git_exporter import GitExporter
from lib.importer import Importer
from lib.test_exporter import TestExporter

URL_OBS = "https://api.opensuse.org"
URL_IBS = "https://api.suse.de"

# The order is relevant (from older to newer initial codebase)
# TODO: make something with these, for now we look purely at openSUSE:Factory
PROJECTS = [
    # ("SUSE:SLE-12:GA", "SLE_12", URL_IBS),
    # ("SUSE:SLE-12:Update", "SLE_12", URL_IBS),
    # ("SUSE:SLE-12-SP1:GA", "SLE_12_SP1", URL_IBS),
    # ("SUSE:SLE-12-SP1:Update", "SLE_12_SP1", URL_IBS),
    # ("SUSE:SLE-12-SP2:GA", "SLE_12_SP2", URL_IBS),
    # ("SUSE:SLE-12-SP2:Update", "SLE_12_SP2", URL_IBS),
    # ("SUSE:SLE-12-SP3:GA", "SLE_12_SP3", URL_IBS),
    # ("SUSE:SLE-12-SP3:Update", "SLE_12_SP3", URL_IBS),
    # ("SUSE:SLE-12-SP4:GA", "SLE_12_SP4", URL_IBS),
    # ("SUSE:SLE-12-SP4:Update", "SLE_12_SP4", URL_IBS),
    # ("SUSE:SLE-12-SP5:GA", "SLE_12_SP5", URL_IBS),
    # ("SUSE:SLE-12-SP5:Update", "SLE_12_SP5", URL_IBS),
    # ("SUSE:SLE-15:GA", "SLE_15", URL_IBS),
    # ("SUSE:SLE-15:Update", "SLE_15", URL_IBS),
    # ("SUSE:SLE-15-SP1:GA", "SLE_15_SP1", URL_IBS),
    # ("SUSE:SLE-15-SP1:Update", "SLE_15_SP1", URL_IBS),
    # ("SUSE:SLE-15-SP2:GA", "SLE_15_SP2", URL_IBS),
    # ("SUSE:SLE-15-SP2:Update", "SLE_15_SP2", URL_IBS),
    # ("SUSE:SLE-15-SP3:GA", "SLE_15_SP3", URL_IBS),
    # ("SUSE:SLE-15-SP3:Update", "SLE_15_SP3", URL_IBS),
    # ("SUSE:SLE-15-SP4:GA", "SLE_15_SP4", URL_IBS),
    # ("SUSE:SLE-15-SP4:Update", "SLE_15_SP4", URL_IBS),
]


def export_package(package, repodir, cachedir, gc):
    exporter = GitExporter(URL_OBS, "openSUSE:Factory", package, repodir, cachedir)
    exporter.set_gc_interval(gc)
    exporter.export_as_git()


def main():
    parser = argparse.ArgumentParser(description="OBS history importer into git")
    parser.add_argument("packages", help="OBS package names", nargs="*")
    parser.add_argument(
        "-r",
        "--repodir",
        required=False,
        default=pathlib.Path("repos"),
        type=pathlib.Path,
        help="Local git repository directory",
    )
    parser.add_argument(
        "-c",
        "--cachedir",
        required=False,
        type=pathlib.Path,
        help="Local cache directory",
    )
    parser.add_argument(
        "-g",
        "--gc",
        metavar="N",
        type=int,
        default=200,
        help="Garbage recollect and pack the git history each N commits",
    )
    parser.add_argument(
        "--level",
        "-l",
        default="INFO",
        help="logging level",
    )
    parser.add_argument(
        "--export",
        action="store_true",
        help="Export database fields for the given package as YAML",
    )

    args = parser.parse_args()

    if args.level:
        numeric_level = getattr(logging, args.level.upper(), None)
        if not isinstance(numeric_level, int):
            print(f"Invalid log level: {args.level}")
            sys.exit(-1)
        logging.basicConfig(level=numeric_level)
        if numeric_level == logging.DEBUG:
            osc.conf.config["debug"] = True
            requests_log = logging.getLogger("requests.packages.urllib3")
            requests_log.setLevel(logging.DEBUG)
            requests_log.propagate = True

    def check_old_package(db: DB, dir: pathlib.Path):
        md5file = dir / "MD5SUMS"
        print(md5file)
        valid_revisions = None
        with open(md5file, "rb") as f:
            for line in f.readlines():
                try:
                    md5, file = line.decode("utf-8").strip().split("  ")
                except UnicodeDecodeError:
                    logging.error(f"Corrupt MD5 file: {md5file}")
                    return
                if file == "ready":
                    continue
                if len(md5) != 32:
                    logging.error(f"Corrupt MD5 file: {md5file}")
                    return
                with db.cursor() as cur:
                    cur.execute(
                        "SELECT revision_id FROM files WHERE md5=%s AND name=%s",
                        (md5, file),
                    )
                    nrevs = set([row[0] for row in cur.fetchall()])
                    if valid_revisions is None:
                        valid_revisions = nrevs
                    else:
                        valid_revisions = valid_revisions & nrevs
                    if not valid_revisions:
                        break

        with db.cursor() as cur:
            cur.execute(
                "SELECT * FROM revisions WHERE id = ANY(%s) AND project=%s",
                (list(valid_revisions), "openSUSE:Factory"),
            )
            for row in cur.fetchall():
                r = DBRevision(db, row)
                print("Valid", r, r.files_hash)
                return True

    if False:
        import os

        db = DB()
        basedir = pathlib.Path(
            f"/mounts/work/SAVE/oldpackages/stable/{args.packages[0]}"
        )
        for subdir in sorted(os.listdir(basedir)):
            if check_old_package(db, basedir / subdir):
                break

    if args.export:
        if len(args.packages) != 1:
            print("Can only export one package")
            sys.exit(1)
        TestExporter(args.packages[0]).run()
        return

    if not args.cachedir:
        args.cachedir = pathlib.Path("~/.cache/git-import/").expanduser()

    importer = Importer(URL_OBS, "openSUSE:Factory", args.packages)
    importer.import_into_db()
    for package in args.packages:
        export_package(package, args.repodir, args.cachedir, args.gc)


if __name__ == "__main__":
    main()