Split Request and Revision

.gitignore

@@ -0,0 +1 @@
+__pycache__

README.md

@@ -1,2 +1,3 @@
+sudo zypper in python3-psycopg2
 sudo su - postgres
 # `createdb -O <LOCAL_USER> imported_git`

git-importer.py

@@ -1,15 +1,12 @@
-#!/usr/bin/env python3
+#!/usr/bin/python3
 
 import argparse
-import asyncio
-import datetime
 import errno
 import fnmatch
 import functools
 import hashlib
 import itertools
 import logging
-import os
 import pathlib
 import re
 import shutil
@@ -24,8 +21,8 @@ import osc.core
 import pygit2
 import requests
 
-from osclib.cache import Cache
+from request import Request
-
+from revision import Revision
 
 # Add a retry wrapper for some of the HTTP actions.
 def retry(func):
@@ -638,140 +635,6 @@ class ProxySHA256:
         return result
 
 
-class Request:
-    def parse(self, xml):
-        self.requestid = int(xml.get("id"))
-        self.creator = xml.get("creator")
-
-        self.type_ = xml.find("action").get("type")
-        if self.type_ == "delete":
-            # not much to do
-            return self
-
-        self.source = xml.find("action/source").get("project")
-        # expanded MD5 or commit revision
-        self.revisionid = xml.find("action/source").get("rev")
-
-        self.target = xml.find("action/target").get("project")
-
-        self.state = xml.find("state").get("name")
-
-        # TODO: support muti-action requests
-        # TODO: parse review history
-        # TODO: add description
-        return self
-
-    def type(self):
-        return self.type_
-
-    def __str__(self):
-        return f"Req {self.requestid} {self.creator} {self.type_} {self.source}->{self.target} {self.state}"
-
-    def __repr__(self):
-        return f"[{self.__str__()}]"
-
-
-class Revision:
-    def __init__(self, obs, history, project, package):
-        self.obs = obs
-        self.history = history
-        self.project = project
-        self.package = package
-
-        self.commit = None
-        self.ignored = False
-
-    def parse(self, xml):
-        self.rev = int(xml.get("rev"))
-        # Replaced in check_expanded
-        self.srcmd5 = xml.find("srcmd5").text
-        self.version = xml.find("version").text
-
-        time = int(xml.find("time").text)
-        self.time = datetime.datetime.fromtimestamp(time)
-
-        userid = xml.find("user")
-        if userid is not None:
-            self.userid = userid.text
-        else:
-            self.userid = "unknown"
-
-        comment = xml.find("comment")
-        if comment is not None:
-            self.comment = comment.text or ""
-        else:
-            self.comment = ""
-
-        # Populated by check_link
-        self.linkrev = None
-
-        self.requestid = None
-        requestid = xml.find("requestid")
-        if requestid is not None:
-            self.requestid = int(requestid.text)
-        else:
-            # Sometimes requestid is missing, but can be extracted
-            # from "comment"
-            matched = re.match(
-                r"^Copy from .* based on submit request (\d+) from user .*$",
-                self.comment,
-            )
-            if matched:
-                self.requestid = int(matched.group(1))
-
-        return self
-
-    def __str__(self):
-        return f"Rev {self.project}/{self.rev} Md5 {self.srcmd5} {self.time} {self.userid} {self.requestid}"
-
-    def __repr__(self):
-        return f"[{self.__str__()}]"
-
-    def check_link(self):
-        """Add 'linkrev' attribute into the revision. Returns False if the link is invalid"""
-        try:
-            root = self.obs._xml(
-                f"source/{self.project}/{self.package}/_link", rev=self.srcmd5
-            )
-        except HTTPError as e:
-            if e.code == 404:
-                logging.debug("No _link for the revision")
-                return True
-            raise e
-        except ET.ParseError:
-            logging.error(
-                f"_link can't be parsed [{self.project}/{self.package} rev={self.srcmd5}]"
-            )
-            return False
-
-        target_project = root.get("project")
-        rev = self.history.find_last_rev_after_time(target_project, self.time)
-        if rev:
-            logging.debug(f"Linkrev found: {rev}")
-            self.linkrev = rev.srcmd5
-        return True
-
-    def check_expanded(self):
-        # Even if it's not a link we still need to check the expanded
-        # srcmd5 as it's possible used in submit requests
-        if not self.check_link():
-            return False
-
-        # If there is a "linkrev", "rev" is ignored
-        params = {"rev": self.srcmd5, "expand": "1"}
-        if self.linkrev:
-            params["linkrev"] = self.linkrev
-
-        try:
-            root = self.obs._xml(f"source/{self.project}/{self.package}", **params)
-        except HTTPError as e:
-            if e.code == 400:
-                logging.error(f"Package [{self.project}/{self.package} {params}] can't be expanded: {e}")
-                return False
-            raise e
-
-        self.srcmd5 = root.get("srcmd5")
-        return True
 
 
 class History:
@@ -1277,8 +1140,6 @@ def main():
         logging.info(f"Removing old repository {args.repodir}")
         shutil.rmtree(args.repodir)
 
-    Cache.init()
-
     # TODO: use a CLI parameter to describe the projects
     importer = Importer(
         PROJECTS, args.package, args.repodir, args.search_ancestor, args.rebase_devel

request.py

@@ -0,0 +1,31 @@
+class Request:
+    def parse(self, xml):
+        self.requestid = int(xml.get("id"))
+        self.creator = xml.get("creator")
+
+        self.type_ = xml.find("action").get("type")
+        if self.type_ == "delete":
+            # not much to do
+            return self
+
+        self.source = xml.find("action/source").get("project")
+        # expanded MD5 or commit revision
+        self.revisionid = xml.find("action/source").get("rev")
+
+        self.target = xml.find("action/target").get("project")
+
+        self.state = xml.find("state").get("name")
+
+        # TODO: support muti-action requests
+        # TODO: parse review history
+        # TODO: add description
+        return self
+
+    def type(self):
+        return self.type_
+
+    def __str__(self):
+        return f"Req {self.requestid} {self.creator} {self.type_} {self.source}->{self.target} {self.state}"
+
+    def __repr__(self):
+        return f"[{self.__str__()}]"

revision.py

@@ -0,0 +1,110 @@
+import datetime
+import logging
+import re
+from urllib.error import HTTPError
+import xml.etree.ElementTree as ET
+
+
+class Revision:
+    def __init__(self, obs, history, project, package):
+        self.obs = obs
+        self.history = history
+        self.project = project
+        self.package = package
+
+        self.commit = None
+        self.ignored = False
+
+    def parse(self, xml):
+        self.rev = int(xml.get("rev"))
+        # Replaced in check_expanded
+        self.srcmd5 = xml.find("srcmd5").text
+        self.version = xml.find("version").text
+
+        time = int(xml.find("time").text)
+        self.time = datetime.datetime.fromtimestamp(time)
+
+        userid = xml.find("user")
+        if userid is not None:
+            self.userid = userid.text
+        else:
+            self.userid = "unknown"
+
+        comment = xml.find("comment")
+        if comment is not None:
+            self.comment = comment.text or ""
+        else:
+            self.comment = ""
+
+        # Populated by check_link
+        self.linkrev = None
+
+        self.requestid = None
+        requestid = xml.find("requestid")
+        if requestid is not None:
+            self.requestid = int(requestid.text)
+        else:
+            # Sometimes requestid is missing, but can be extracted
+            # from "comment"
+            matched = re.match(
+                r"^Copy from .* based on submit request (\d+) from user .*$",
+                self.comment,
+            )
+            if matched:
+                self.requestid = int(matched.group(1))
+
+        return self
+
+    def __str__(self):
+        return f"Rev {self.project}/{self.rev} Md5 {self.srcmd5} {self.time} {self.userid} {self.requestid}"
+
+    def __repr__(self):
+        return f"[{self.__str__()}]"
+
+    def check_link(self):
+        """Add 'linkrev' attribute into the revision. Returns False if the link is invalid"""
+        try:
+            root = self.obs._xml(
+                f"source/{self.project}/{self.package}/_link", rev=self.srcmd5
+            )
+        except HTTPError as e:
+            if e.code == 404:
+                logging.debug("No _link for the revision")
+                return True
+            raise e
+        except ET.ParseError:
+            logging.error(
+                f"_link can't be parsed [{self.project}/{self.package} rev={self.srcmd5}]"
+            )
+            return False
+
+        target_project = root.get("project")
+        rev = self.history.find_last_rev_after_time(target_project, self.time)
+        if rev:
+            logging.debug(f"Linkrev found: {rev}")
+            self.linkrev = rev.srcmd5
+        return True
+
+    def check_expanded(self):
+        # Even if it's not a link we still need to check the expanded
+        # srcmd5 as it's possible used in submit requests
+        if not self.check_link():
+            return False
+
+        # If there is a "linkrev", "rev" is ignored
+        params = {"rev": self.srcmd5, "expand": "1"}
+        if self.linkrev:
+            params["linkrev"] = self.linkrev
+
+        try:
+            root = self.obs._xml(f"source/{self.project}/{self.package}", **params)
+        except HTTPError as e:
+            if e.code == 400:
+                logging.error(
+                    f"Package [{self.project}/{self.package} {params}] can't be expanded: {e}"
+                )
+                return False
+            raise e
+
+        self.srcmd5 = root.get("srcmd5")
+        return True