Add experimental import into DB

git-importer.py

@@ -83,6 +83,11 @@ def main():
         default="INFO",
         help="logging level",
     )
+    parser.add_argument(
+        "--db",
+        action="store_true",
+        help="Import revisions into database only",
+    )
 
     args = parser.parse_args()
 
@@ -112,6 +117,9 @@ def main():
     importer = Importer(
         PROJECTS, args.package, args.repodir, args.search_ancestor, args.rebase_devel
     )
+    if args.db:
+        importer.import_into_db()
+        return
     importer.import_all_revisions(args.gc)
 
 

lib/db.py

@@ -1,10 +1,12 @@
 import psycopg2
-from config import config
+
+from lib.config import config
 
 
 class DB:
     def __init__(self):
         self.connect()
+        self.create_tables()
 
     def connect(self):
         try:
@@ -59,7 +61,8 @@ class DB:
                 unexpanded_srcmd5   VARCHAR(255) NOT NULL,
                 commit_time         timestamp    NOT NULL,
                 userid              VARCHAR(255) NOT NULL,
-                comment             VARCHAR(255)
+                comment             TEXT,
+                requestid           INTEGER
                 )
         """,
         )
@@ -80,6 +83,24 @@ class DB:
             self.close()
             raise error
 
+    def import_rev(self, revision):
+        cur = self.conn.cursor()
+        print(revision)
+        cur.execute(
+            """INSERT INTO revisions (project, package, rev, unexpanded_srcmd5, commit_time, userid, comment, requestid)  
+                    VALUES(%s, %s, %s, %s, %s, %s, %s, %s)""",
+            (
+                revision.project,
+                revision.package,
+                revision.rev,
+                revision.unexpanded_srcmd5,
+                revision.time,
+                revision.userid,
+                revision.comment,
+                revision.requestid,
+            ),
+        )
+
 
 if __name__ == "__main__":
     db = DB()

lib/importer.py

@@ -2,6 +2,7 @@ import functools
 import logging
 
 from lib.binary import is_binary_or_large
+from lib.db import DB
 from lib.git import Git
 from lib.history import History
 from lib.obs import OBS
@@ -120,6 +121,14 @@ class Importer:
             print(f"Remove {name}")
             self.git.remove(name)
 
+    def import_into_db(self):
+        db = DB()
+        self.history.fetch_all_revisions(self.projects)
+        for project, _, _ in self.projects:
+            for rev in self.history[project]:
+                db.import_rev(rev)
+            db.conn.commit()
+
     def import_all_revisions(self, gc):
         # Fetch all the requests and sort them.  Ideally we should
         # build the graph here, to avoid new commits before the merge.

lib/revision.py

@@ -17,8 +17,9 @@ class Revision:
 
     def parse(self, xml):
         self.rev = int(xml.get("rev"))
+        self.unexpanded_srcmd5 = xml.find("srcmd5").text
         # Replaced in check_expanded
-        self.srcmd5 = xml.find("srcmd5").text
+        self.srcmd5 = self.unexpanded_srcmd5
 
         time = int(xml.find("time").text)
         self.time = datetime.datetime.fromtimestamp(time)