diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bee8a64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/README.md b/README.md index 9b49cf2..85772eb 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,3 @@ +sudo zypper in python3-psycopg2 sudo su - postgres # `createdb -O imported_git` diff --git a/git-importer.py b/git-importer.py index a6a3d56..8d9966d 100755 --- a/git-importer.py +++ b/git-importer.py @@ -1,15 +1,12 @@ -#!/usr/bin/env python3 +#!/usr/bin/python3 import argparse -import asyncio -import datetime import errno import fnmatch import functools import hashlib import itertools import logging -import os import pathlib import re import shutil @@ -24,8 +21,8 @@ import osc.core import pygit2 import requests -from osclib.cache import Cache - +from request import Request +from revision import Revision # Add a retry wrapper for some of the HTTP actions. def retry(func): @@ -638,140 +635,6 @@ class ProxySHA256: return result -class Request: - def parse(self, xml): - self.requestid = int(xml.get("id")) - self.creator = xml.get("creator") - - self.type_ = xml.find("action").get("type") - if self.type_ == "delete": - # not much to do - return self - - self.source = xml.find("action/source").get("project") - # expanded MD5 or commit revision - self.revisionid = xml.find("action/source").get("rev") - - self.target = xml.find("action/target").get("project") - - self.state = xml.find("state").get("name") - - # TODO: support muti-action requests - # TODO: parse review history - # TODO: add description - return self - - def type(self): - return self.type_ - - def __str__(self): - return f"Req {self.requestid} {self.creator} {self.type_} {self.source}->{self.target} {self.state}" - - def __repr__(self): - return f"[{self.__str__()}]" - - -class Revision: - def __init__(self, obs, history, project, package): - self.obs = obs - self.history = history - self.project = project - self.package = package - - self.commit = None - self.ignored = False - - def parse(self, xml): - self.rev = int(xml.get("rev")) - # Replaced in check_expanded - self.srcmd5 = xml.find("srcmd5").text - self.version = xml.find("version").text - - time = int(xml.find("time").text) - self.time = datetime.datetime.fromtimestamp(time) - - userid = xml.find("user") - if userid is not None: - self.userid = userid.text - else: - self.userid = "unknown" - - comment = xml.find("comment") - if comment is not None: - self.comment = comment.text or "" - else: - self.comment = "" - - # Populated by check_link - self.linkrev = None - - self.requestid = None - requestid = xml.find("requestid") - if requestid is not None: - self.requestid = int(requestid.text) - else: - # Sometimes requestid is missing, but can be extracted - # from "comment" - matched = re.match( - r"^Copy from .* based on submit request (\d+) from user .*$", - self.comment, - ) - if matched: - self.requestid = int(matched.group(1)) - - return self - - def __str__(self): - return f"Rev {self.project}/{self.rev} Md5 {self.srcmd5} {self.time} {self.userid} {self.requestid}" - - def __repr__(self): - return f"[{self.__str__()}]" - - def check_link(self): - """Add 'linkrev' attribute into the revision. Returns False if the link is invalid""" - try: - root = self.obs._xml( - f"source/{self.project}/{self.package}/_link", rev=self.srcmd5 - ) - except HTTPError as e: - if e.code == 404: - logging.debug("No _link for the revision") - return True - raise e - except ET.ParseError: - logging.error( - f"_link can't be parsed [{self.project}/{self.package} rev={self.srcmd5}]" - ) - return False - - target_project = root.get("project") - rev = self.history.find_last_rev_after_time(target_project, self.time) - if rev: - logging.debug(f"Linkrev found: {rev}") - self.linkrev = rev.srcmd5 - return True - - def check_expanded(self): - # Even if it's not a link we still need to check the expanded - # srcmd5 as it's possible used in submit requests - if not self.check_link(): - return False - - # If there is a "linkrev", "rev" is ignored - params = {"rev": self.srcmd5, "expand": "1"} - if self.linkrev: - params["linkrev"] = self.linkrev - - try: - root = self.obs._xml(f"source/{self.project}/{self.package}", **params) - except HTTPError as e: - if e.code == 400: - logging.error(f"Package [{self.project}/{self.package} {params}] can't be expanded: {e}") - return False - raise e - - self.srcmd5 = root.get("srcmd5") - return True class History: @@ -1277,8 +1140,6 @@ def main(): logging.info(f"Removing old repository {args.repodir}") shutil.rmtree(args.repodir) - Cache.init() - # TODO: use a CLI parameter to describe the projects importer = Importer( PROJECTS, args.package, args.repodir, args.search_ancestor, args.rebase_devel diff --git a/request.py b/request.py new file mode 100644 index 0000000..2b7baec --- /dev/null +++ b/request.py @@ -0,0 +1,31 @@ +class Request: + def parse(self, xml): + self.requestid = int(xml.get("id")) + self.creator = xml.get("creator") + + self.type_ = xml.find("action").get("type") + if self.type_ == "delete": + # not much to do + return self + + self.source = xml.find("action/source").get("project") + # expanded MD5 or commit revision + self.revisionid = xml.find("action/source").get("rev") + + self.target = xml.find("action/target").get("project") + + self.state = xml.find("state").get("name") + + # TODO: support muti-action requests + # TODO: parse review history + # TODO: add description + return self + + def type(self): + return self.type_ + + def __str__(self): + return f"Req {self.requestid} {self.creator} {self.type_} {self.source}->{self.target} {self.state}" + + def __repr__(self): + return f"[{self.__str__()}]" diff --git a/revision.py b/revision.py new file mode 100644 index 0000000..add95a1 --- /dev/null +++ b/revision.py @@ -0,0 +1,110 @@ +import datetime +import logging +import re +from urllib.error import HTTPError +import xml.etree.ElementTree as ET + + +class Revision: + def __init__(self, obs, history, project, package): + self.obs = obs + self.history = history + self.project = project + self.package = package + + self.commit = None + self.ignored = False + + def parse(self, xml): + self.rev = int(xml.get("rev")) + # Replaced in check_expanded + self.srcmd5 = xml.find("srcmd5").text + self.version = xml.find("version").text + + time = int(xml.find("time").text) + self.time = datetime.datetime.fromtimestamp(time) + + userid = xml.find("user") + if userid is not None: + self.userid = userid.text + else: + self.userid = "unknown" + + comment = xml.find("comment") + if comment is not None: + self.comment = comment.text or "" + else: + self.comment = "" + + # Populated by check_link + self.linkrev = None + + self.requestid = None + requestid = xml.find("requestid") + if requestid is not None: + self.requestid = int(requestid.text) + else: + # Sometimes requestid is missing, but can be extracted + # from "comment" + matched = re.match( + r"^Copy from .* based on submit request (\d+) from user .*$", + self.comment, + ) + if matched: + self.requestid = int(matched.group(1)) + + return self + + def __str__(self): + return f"Rev {self.project}/{self.rev} Md5 {self.srcmd5} {self.time} {self.userid} {self.requestid}" + + def __repr__(self): + return f"[{self.__str__()}]" + + def check_link(self): + """Add 'linkrev' attribute into the revision. Returns False if the link is invalid""" + try: + root = self.obs._xml( + f"source/{self.project}/{self.package}/_link", rev=self.srcmd5 + ) + except HTTPError as e: + if e.code == 404: + logging.debug("No _link for the revision") + return True + raise e + except ET.ParseError: + logging.error( + f"_link can't be parsed [{self.project}/{self.package} rev={self.srcmd5}]" + ) + return False + + target_project = root.get("project") + rev = self.history.find_last_rev_after_time(target_project, self.time) + if rev: + logging.debug(f"Linkrev found: {rev}") + self.linkrev = rev.srcmd5 + return True + + def check_expanded(self): + # Even if it's not a link we still need to check the expanded + # srcmd5 as it's possible used in submit requests + if not self.check_link(): + return False + + # If there is a "linkrev", "rev" is ignored + params = {"rev": self.srcmd5, "expand": "1"} + if self.linkrev: + params["linkrev"] = self.linkrev + + try: + root = self.obs._xml(f"source/{self.project}/{self.package}", **params) + except HTTPError as e: + if e.code == 400: + logging.error( + f"Package [{self.project}/{self.package} {params}] can't be expanded: {e}" + ) + return False + raise e + + self.srcmd5 = root.get("srcmd5") + return True