Import file list of revisions

lib/db.py

@@ -77,14 +77,37 @@ class DB:
             """
             CREATE TABLE links (
                 id            SERIAL PRIMARY KEY,
-                revision_id   INTEGER,
+                revision_id   INTEGER NOT NULL,
                 project       VARCHAR(255) NOT NULL,
                 package       VARCHAR(255) NOT NULL
             )
             """,
+            "UPDATE scheme SET version=2",
+        )
+        schemes[3] = (
             """
-            UPDATE scheme SET version=2 
+            ALTER TABLE revisions ADD broken boolean NOT NULL DEFAULT(FALSE)
             """,
+            "UPDATE scheme SET version=3",
+        )
+        schemes[4] = (
+            """
+            ALTER TABLE revisions ADD expanded_srcmd5 VARCHAR(255)
+            """,
+            "UPDATE scheme SET version=4",
+        )
+        schemes[5] = (
+            """
+            CREATE TABLE files (
+                id              SERIAL PRIMARY KEY,
+                revision_id     INTEGER NOT NULL,
+                name            VARCHAR(255) NOT NULL,
+                md5             VARCHAR(255) NOT NULL,
+                size            INTEGER NOT NULL,
+                mtime           INTEGER NOT NULL
+            )
+            """,
+            "UPDATE scheme SET version=5",
         )
 
         schema_version = self.schema_version()

lib/db_revision.py

@@ -11,6 +11,8 @@ class DBRevision:
             self.userid,
             self.comment,
             self.requestid,
+            self.broken,
+            self.expanded_srcmd5,
         ) = row
 
     def __str__(self):
@@ -101,3 +103,28 @@ class DBRevision:
         if revisions:
             return revisions[0]
         return None
+
+    def set_broken(self, db):
+        cur = db.cursor()
+        cur.execute("UPDATE revisions SET broken=TRUE where id=%s", (self.dbid,))
+        cur.close()
+
+    def import_dir_list(self, db, xml):
+        cur = db.cursor()
+        cur.execute(
+            "UPDATE revisions SET expanded_srcmd5=%s where id=%s",
+            (xml.get("srcmd5"), self.dbid),
+        )
+        for entry in xml.findall("entry"):
+            cur.execute(
+                """INSERT INTO files (name, md5, size, mtime, revision_id) 
+                           VALUES (%s,%s,%s,%s,%s)""",
+                (
+                    entry.get("name"),
+                    entry.get("md5"),
+                    entry.get("size"),
+                    entry.get("mtime"),
+                    self.dbid,
+                ),
+            )
+        cur.close()

lib/importer.py

@@ -1,5 +1,6 @@
 import functools
 import logging
+import xml.etree.ElementTree as ET
 
 from lib.binary import is_binary_or_large
 from lib.db import DB
@@ -154,9 +155,22 @@ class Importer:
                 self.update_db_package(db, lproject, lpackage)
 
             for rev in DBRevision.all_revisions(db, project, self.package):
-                print(rev, rev.linked_rev(db))
+                if rev.broken or rev.expanded_srcmd5:
+                    continue
+                linked_rev = rev.linked_rev(db)
+                if linked_rev:
+                    linked_rev = linked_rev.unexpanded_srcmd5
+                list = self.obs.list(
+                    project, self.package, rev.unexpanded_srcmd5, linked_rev
+                )
+                if list:
+                    print(ET.tostring(list).decode("utf-8"))
+                    rev.import_dir_list(db, list)
+                else:
+                    rev.set_broken(db)
 
             cur.close()
+        db.conn.commit()
 
     def import_all_revisions(self, gc):
         # Fetch all the requests and sort them.  Ideally we should

lib/obs.py

@@ -140,3 +140,20 @@ class OBS:
     def download(self, project, package, name, revision, dirpath):
         with (dirpath / name).open("wb") as f:
             f.write(self._download(project, package, name, revision).read())
+
+    def list(self, project, package, srcmd5, linkrev):
+        params = {"rev": srcmd5, "expand": "1"}
+        if linkrev:
+            params["linkrev"] = linkrev
+
+        try:
+            root = self._xml(f"source/{project}/{package}", **params)
+        except HTTPError as e:
+            if e.code == 400:
+                logging.error(
+                    f"Package [{project}/{package} {params}] can't be expanded: {e}"
+                )
+                return None
+            raise e
+
+        return root