forked from importers/git-importer
a457a16a50
This is hard coding the limit, we may want to make this configurable but for now the machines supposed to run this code are very similiar
128 lines
4.0 KiB
Python
Executable File
128 lines
4.0 KiB
Python
Executable File
#!/usr/bin/python3
|
|
|
|
import argparse
|
|
import concurrent.futures
|
|
import logging
|
|
import pathlib
|
|
import sys
|
|
|
|
import osc.core
|
|
|
|
from lib.git_exporter import GitExporter
|
|
from lib.importer import Importer
|
|
from lib.test_exporter import TestExporter
|
|
|
|
URL_OBS = "https://api.opensuse.org"
|
|
URL_IBS = "https://api.suse.de"
|
|
|
|
# The order is relevant (from older to newer initial codebase)
|
|
# TODO: make something with these, for now we look purely at openSUSE:Factory
|
|
PROJECTS = [
|
|
# ("SUSE:SLE-12:GA", "SLE_12", URL_IBS),
|
|
# ("SUSE:SLE-12:Update", "SLE_12", URL_IBS),
|
|
# ("SUSE:SLE-12-SP1:GA", "SLE_12_SP1", URL_IBS),
|
|
# ("SUSE:SLE-12-SP1:Update", "SLE_12_SP1", URL_IBS),
|
|
# ("SUSE:SLE-12-SP2:GA", "SLE_12_SP2", URL_IBS),
|
|
# ("SUSE:SLE-12-SP2:Update", "SLE_12_SP2", URL_IBS),
|
|
# ("SUSE:SLE-12-SP3:GA", "SLE_12_SP3", URL_IBS),
|
|
# ("SUSE:SLE-12-SP3:Update", "SLE_12_SP3", URL_IBS),
|
|
# ("SUSE:SLE-12-SP4:GA", "SLE_12_SP4", URL_IBS),
|
|
# ("SUSE:SLE-12-SP4:Update", "SLE_12_SP4", URL_IBS),
|
|
# ("SUSE:SLE-12-SP5:GA", "SLE_12_SP5", URL_IBS),
|
|
# ("SUSE:SLE-12-SP5:Update", "SLE_12_SP5", URL_IBS),
|
|
# ("SUSE:SLE-15:GA", "SLE_15", URL_IBS),
|
|
# ("SUSE:SLE-15:Update", "SLE_15", URL_IBS),
|
|
# ("SUSE:SLE-15-SP1:GA", "SLE_15_SP1", URL_IBS),
|
|
# ("SUSE:SLE-15-SP1:Update", "SLE_15_SP1", URL_IBS),
|
|
# ("SUSE:SLE-15-SP2:GA", "SLE_15_SP2", URL_IBS),
|
|
# ("SUSE:SLE-15-SP2:Update", "SLE_15_SP2", URL_IBS),
|
|
# ("SUSE:SLE-15-SP3:GA", "SLE_15_SP3", URL_IBS),
|
|
# ("SUSE:SLE-15-SP3:Update", "SLE_15_SP3", URL_IBS),
|
|
# ("SUSE:SLE-15-SP4:GA", "SLE_15_SP4", URL_IBS),
|
|
# ("SUSE:SLE-15-SP4:Update", "SLE_15_SP4", URL_IBS),
|
|
]
|
|
|
|
|
|
def export_package(package, repodir, cachedir, gc):
|
|
exporter = GitExporter(URL_OBS, "openSUSE:Factory", package, repodir, cachedir)
|
|
exporter.set_gc_interval(gc)
|
|
exporter.export_as_git()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="OBS history importer into git")
|
|
parser.add_argument("packages", help="OBS package names", nargs="*")
|
|
parser.add_argument(
|
|
"-r",
|
|
"--repodir",
|
|
required=False,
|
|
default=pathlib.Path("repos"),
|
|
type=pathlib.Path,
|
|
help="Local git repository directory",
|
|
)
|
|
parser.add_argument(
|
|
"-c",
|
|
"--cachedir",
|
|
required=False,
|
|
type=pathlib.Path,
|
|
help="Local cache directory",
|
|
)
|
|
parser.add_argument(
|
|
"-g",
|
|
"--gc",
|
|
metavar="N",
|
|
type=int,
|
|
default=200,
|
|
help="Garbage recollect and pack the git history each N commits",
|
|
)
|
|
parser.add_argument(
|
|
"--level",
|
|
"-l",
|
|
default="INFO",
|
|
help="logging level",
|
|
)
|
|
parser.add_argument(
|
|
"--export",
|
|
action="store_true",
|
|
help="Export database fields for the given package as YAML",
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.level:
|
|
numeric_level = getattr(logging, args.level.upper(), None)
|
|
if not isinstance(numeric_level, int):
|
|
print(f"Invalid log level: {args.level}")
|
|
sys.exit(-1)
|
|
logging.basicConfig(level=numeric_level)
|
|
if numeric_level == logging.DEBUG:
|
|
osc.conf.config["debug"] = True
|
|
requests_log = logging.getLogger("requests.packages.urllib3")
|
|
requests_log.setLevel(logging.DEBUG)
|
|
requests_log.propagate = True
|
|
|
|
if args.export:
|
|
if len(args.packages) != 1:
|
|
print("Can only export one package")
|
|
sys.exit(1)
|
|
TestExporter(args.packages[0]).run()
|
|
return
|
|
|
|
if not args.cachedir:
|
|
args.cachedir = pathlib.Path("~/.cache/git-import/").expanduser()
|
|
|
|
importer = Importer(URL_OBS, "openSUSE:Factory", args.packages)
|
|
importer.import_into_db()
|
|
with concurrent.futures.ProcessPoolExecutor(max_workers=8) as executor:
|
|
fs = [
|
|
executor.submit(
|
|
export_package, package, args.repodir, args.cachedir, args.gc
|
|
)
|
|
for package in args.packages
|
|
]
|
|
concurrent.futures.wait(fs)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|