Split Revision into OBS and DB

lib/db.py

@@ -51,21 +51,25 @@ class DB:
         CREATE TABLE scheme (
             id SERIAL PRIMARY KEY,
             version SMALLINT NOT NULL
-        )
+            )
-        """,
+            """,
             "INSERT INTO scheme (version) VALUES(1)",
+            """ DROP TABLE IF EXISTS revisions""",
             """ CREATE TABLE revisions (
                 id                  SERIAL PRIMARY KEY,
                 project             VARCHAR(255) NOT NULL,
                 package             VARCHAR(255) NOT NULL,
-                rev                 VARCHAR(255) NOT NULL,
+                rev                 INTEGER      NOT NULL,
                 unexpanded_srcmd5   VARCHAR(255) NOT NULL,
                 commit_time         timestamp    NOT NULL,
                 userid              VARCHAR(255) NOT NULL,
                 comment             TEXT,
                 requestid           INTEGER
                 )
-        """,
+            """,
+            """
+            CREATE UNIQUE INDEX ppr ON revisions (project, package, rev);
+            """
         )
 
         if self.schema_version() > 0:
@@ -84,9 +88,11 @@ class DB:
             self.close()
             raise error
 
+    def cursor(self):
+        return self.conn.cursor()
+
     def import_rev(self, revision):
         cur = self.conn.cursor()
-        print(revision)
         cur.execute(
             """INSERT INTO revisions (project, package, rev, unexpanded_srcmd5, commit_time, userid, comment, requestid)  
                     VALUES(%s, %s, %s, %s, %s, %s, %s, %s)""",

lib/db_revision.py

@@ -0,0 +1,22 @@
+class DBRevision:
+
+    def __init__(self, row):
+        (self.dbid, self.project, self.package, self.rev, self.unexpanded_srcmd5, self.commit_time, self.userid, self.comment, self.requestid) = row
+
+    @classmethod
+    def fetch_revision(cls, db, project, package, rev):
+        cur = db.cursor()
+        cur.execute("SELECT * FROM revisions where project=%s and package=%s and rev=%s", (project, package, rev))
+        row = cur.fetchone()
+        cur.close()
+        return DBRevision(row)
+
+    @classmethod
+    def latest_revision(cls, db, project, package):
+        cur = db.cursor()
+        cur.execute("SELECT MAX(rev) FROM revisions where project=%s and package=%s", (project, package))
+        max = cur.fetchone()[0]
+        cur.close()
+        if max:
+            return DBRevision.fetch_revision(db, project, package, int(max))
+        return None

lib/history.py

@@ -2,7 +2,7 @@ import itertools
 import logging
 import re
 
-from lib.revision import Revision
+from lib.obs_revision import OBSRevision
 
 
 class History:
@@ -33,7 +33,7 @@ class History:
         root = self.obs._history(project, self.package, **params)
         if root is not None:
             return [
-                Revision(self.obs, self, project, self.package).parse(r)
+                OBSRevision(self.obs, self, project, self.package).parse(r)
                 for r in root.findall("revision")
             ]
 

lib/importer.py

@@ -6,6 +6,8 @@ from lib.db import DB
 from lib.git import Git
 from lib.history import History
 from lib.obs import OBS
+from lib.obs_revision import OBSRevision
+from lib.db_revision import DBRevision
 from lib.proxy_sha256 import ProxySHA256, md5, sha256
 
 
@@ -121,13 +123,22 @@ class Importer:
             print(f"Remove {name}")
             self.git.remove(name)
 
+    def update_db_package(self, db, project, package):
+        root = self.obs._history(project, package)
+        if root is None:
+            return
+        latest = DBRevision.latest_revision(db, project, self.package)
+        for r in root.findall("revision"):
+            rev = OBSRevision(self.obs, self, project, self.package).parse(r)
+            if not latest or rev.rev > latest.rev:
+                db.import_rev(rev)
+        db.conn.commit()
+
     def import_into_db(self):
         db = DB()
-        self.history.fetch_all_revisions(self.projects)
+        for project, _, api_url in self.projects:
-        for project, _, _ in self.projects:
+            self.obs.change_url(api_url)
-            for rev in self.history[project]:
+            self.update_db_package(db, project, self.package)
-                db.import_rev(rev)
-            db.conn.commit()
 
     def import_all_revisions(self, gc):
         # Fetch all the requests and sort them.  Ideally we should

lib/obs_revision.py

@@ -5,7 +5,7 @@ import xml.etree.ElementTree as ET
 from urllib.error import HTTPError
 
 
-class Revision:
+class OBSRevision:
     def __init__(self, obs, history, project, package):
         self.obs = obs
         self.history = history

tests/db_import_test.py

@@ -4,7 +4,7 @@ import xml.etree.ElementTree as ET
 from lib.db import DB
 from lib.history import History
 from lib.obs import OBS
-from lib.revision import Revision
+from lib.obs_revision import OBSRevision
 
 
 class TestDBMethods(unittest.TestCase):
@@ -14,7 +14,7 @@ class TestDBMethods(unittest.TestCase):
         self.history = History(self.obs, "xz")
 
     def test_import(self):
-        test_rev = Revision(self.obs, self.history, "openSUSE:Factory", "xz")
+        test_rev = OBSRevision(self.obs, self.history, "openSUSE:Factory", "xz")
         test_rev.parse(
             ET.fromstring(
                 """<revision rev="70" vrev="1">