Import files of revisions more reliable and faster

lib/db.py

@@ -206,6 +206,10 @@ class DB:
             "ALTER TABLE linked_revs ADD COLUMN considered BOOLEAN DEFAULT FALSE",
             "UPDATE scheme SET version=18",
         )
+        schemes[19] = (
+            "CREATE INDEX ON files(revision_id);",
+            "UPDATE scheme SET version=19",
+        )
         schema_version = self.schema_version()
         if (schema_version + 1) not in schemes:
             return

lib/importer.py

@@ -267,6 +267,13 @@ class Importer:
                 (rev.dbid, linked.dbid),
             )
 
+    def revisions_without_files(self, db):
+        with db.cursor() as cur:
+            cur.execute(
+                "SELECT * FROM revisions WHERE broken=FALSE AND expanded_srcmd5 IS NULL"
+            )
+            return [DBRevision(row) for row in cur.fetchall()]
+
     def import_into_db(self):
         db = DB()
         for project, _, api_url in self.projects:
@@ -283,10 +290,7 @@ class Importer:
                 if missing_user:
                     missing_user.import_into_db(db)
 
-            for rev in DBRevision.all_revisions(db, project, self.package):
+            for rev in self.revisions_without_files(db):
-                # TODO move into SELECT
-                if rev.broken or rev.expanded_srcmd5:
-                    continue
                 with db.cursor() as cur:
                     cur.execute(
                         """SELECT unexpanded_srcmd5 from revisions WHERE
@@ -297,7 +301,7 @@ class Importer:
                 if linked_rev:
                     linked_rev = linked_rev[0]
                 list = self.obs.list(
-                    project, self.package, rev.unexpanded_srcmd5, linked_rev
+                    rev.project, rev.package, rev.unexpanded_srcmd5, linked_rev
                 )
                 if list:
                     rev.import_dir_list(db, list)