From 1c846e963c328d79c6a19c469a063113ba900306 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Wed, 26 Oct 2022 21:47:39 +0200 Subject: [PATCH] Import files of revisions more reliable and faster --- lib/db.py | 4 ++++ lib/importer.py | 14 +++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/db.py b/lib/db.py index da9c837..23e54bd 100644 --- a/lib/db.py +++ b/lib/db.py @@ -206,6 +206,10 @@ class DB: "ALTER TABLE linked_revs ADD COLUMN considered BOOLEAN DEFAULT FALSE", "UPDATE scheme SET version=18", ) + schemes[19] = ( + "CREATE INDEX ON files(revision_id);", + "UPDATE scheme SET version=19", + ) schema_version = self.schema_version() if (schema_version + 1) not in schemes: return diff --git a/lib/importer.py b/lib/importer.py index 14c3a9f..89e8aac 100644 --- a/lib/importer.py +++ b/lib/importer.py @@ -267,6 +267,13 @@ class Importer: (rev.dbid, linked.dbid), ) + def revisions_without_files(self, db): + with db.cursor() as cur: + cur.execute( + "SELECT * FROM revisions WHERE broken=FALSE AND expanded_srcmd5 IS NULL" + ) + return [DBRevision(row) for row in cur.fetchall()] + def import_into_db(self): db = DB() for project, _, api_url in self.projects: @@ -283,10 +290,7 @@ class Importer: if missing_user: missing_user.import_into_db(db) - for rev in DBRevision.all_revisions(db, project, self.package): - # TODO move into SELECT - if rev.broken or rev.expanded_srcmd5: - continue + for rev in self.revisions_without_files(db): with db.cursor() as cur: cur.execute( """SELECT unexpanded_srcmd5 from revisions WHERE @@ -297,7 +301,7 @@ class Importer: if linked_rev: linked_rev = linked_rev[0] list = self.obs.list( - project, self.package, rev.unexpanded_srcmd5, linked_rev + rev.project, rev.package, rev.unexpanded_srcmd5, linked_rev ) if list: rev.import_dir_list(db, list)