#!/usr/bin/python3 import argparse import logging import pathlib import sys import osc.core from lib.db import DB from lib.db_revision import DBRevision from lib.git_exporter import GitExporter from lib.importer import Importer from lib.test_exporter import TestExporter URL_OBS = "https://api.opensuse.org" URL_IBS = "https://api.suse.de" # The order is relevant (from older to newer initial codebase) # TODO: make something with these, for now we look purely at openSUSE:Factory PROJECTS = [ # ("SUSE:SLE-12:GA", "SLE_12", URL_IBS), # ("SUSE:SLE-12:Update", "SLE_12", URL_IBS), # ("SUSE:SLE-12-SP1:GA", "SLE_12_SP1", URL_IBS), # ("SUSE:SLE-12-SP1:Update", "SLE_12_SP1", URL_IBS), # ("SUSE:SLE-12-SP2:GA", "SLE_12_SP2", URL_IBS), # ("SUSE:SLE-12-SP2:Update", "SLE_12_SP2", URL_IBS), # ("SUSE:SLE-12-SP3:GA", "SLE_12_SP3", URL_IBS), # ("SUSE:SLE-12-SP3:Update", "SLE_12_SP3", URL_IBS), # ("SUSE:SLE-12-SP4:GA", "SLE_12_SP4", URL_IBS), # ("SUSE:SLE-12-SP4:Update", "SLE_12_SP4", URL_IBS), # ("SUSE:SLE-12-SP5:GA", "SLE_12_SP5", URL_IBS), # ("SUSE:SLE-12-SP5:Update", "SLE_12_SP5", URL_IBS), # ("SUSE:SLE-15:GA", "SLE_15", URL_IBS), # ("SUSE:SLE-15:Update", "SLE_15", URL_IBS), # ("SUSE:SLE-15-SP1:GA", "SLE_15_SP1", URL_IBS), # ("SUSE:SLE-15-SP1:Update", "SLE_15_SP1", URL_IBS), # ("SUSE:SLE-15-SP2:GA", "SLE_15_SP2", URL_IBS), # ("SUSE:SLE-15-SP2:Update", "SLE_15_SP2", URL_IBS), # ("SUSE:SLE-15-SP3:GA", "SLE_15_SP3", URL_IBS), # ("SUSE:SLE-15-SP3:Update", "SLE_15_SP3", URL_IBS), # ("SUSE:SLE-15-SP4:GA", "SLE_15_SP4", URL_IBS), # ("SUSE:SLE-15-SP4:Update", "SLE_15_SP4", URL_IBS), ] def export_package(package, repodir, cachedir, gc): exporter = GitExporter(URL_OBS, "openSUSE:Factory", package, repodir, cachedir) exporter.set_gc_interval(gc) exporter.export_as_git() def main(): parser = argparse.ArgumentParser(description="OBS history importer into git") parser.add_argument("packages", help="OBS package names", nargs="*") parser.add_argument( "-r", "--repodir", required=False, default=pathlib.Path("repos"), type=pathlib.Path, help="Local git repository directory", ) parser.add_argument( "-c", "--cachedir", required=False, type=pathlib.Path, help="Local cache directory", ) parser.add_argument( "-g", "--gc", metavar="N", type=int, default=200, help="Garbage recollect and pack the git history each N commits", ) parser.add_argument( "--level", "-l", default="INFO", help="logging level", ) parser.add_argument( "--export", action="store_true", help="Export database fields for the given package as YAML", ) args = parser.parse_args() if args.level: numeric_level = getattr(logging, args.level.upper(), None) if not isinstance(numeric_level, int): print(f"Invalid log level: {args.level}") sys.exit(-1) logging.basicConfig(level=numeric_level) if numeric_level == logging.DEBUG: osc.conf.config["debug"] = True requests_log = logging.getLogger("requests.packages.urllib3") requests_log.setLevel(logging.DEBUG) requests_log.propagate = True def check_old_package(db: DB, dir: pathlib.Path): md5file = dir / "MD5SUMS" print(md5file) valid_revisions = None with open(md5file, "rb") as f: for line in f.readlines(): try: md5, file = line.decode("utf-8").strip().split(" ") except UnicodeDecodeError: logging.error(f"Corrupt MD5 file: {md5file}") return if file == "ready": continue if len(md5) != 32: logging.error(f"Corrupt MD5 file: {md5file}") return with db.cursor() as cur: cur.execute( "SELECT revision_id FROM files WHERE md5=%s AND name=%s", (md5, file), ) nrevs = set([row[0] for row in cur.fetchall()]) if valid_revisions is None: valid_revisions = nrevs else: valid_revisions = valid_revisions & nrevs if not valid_revisions: break with db.cursor() as cur: cur.execute( "SELECT * FROM revisions WHERE id = ANY(%s) AND project=%s", (list(valid_revisions), "openSUSE:Factory"), ) for row in cur.fetchall(): r = DBRevision(db, row) print("Valid", r, r.files_hash) return True if False: import os db = DB() basedir = pathlib.Path( f"/mounts/work/SAVE/oldpackages/stable/{args.packages[0]}" ) for subdir in sorted(os.listdir(basedir)): if check_old_package(db, basedir / subdir): break if args.export: if len(args.packages) != 1: print("Can only export one package") sys.exit(1) TestExporter(args.packages[0]).run() return if not args.cachedir: args.cachedir = pathlib.Path("~/.cache/git-import/").expanduser() importer = Importer(URL_OBS, "openSUSE:Factory", args.packages) importer.import_into_db() for package in args.packages: export_package(package, args.repodir, args.cachedir, args.gc) if __name__ == "__main__": main()