adamm/git-importer
1
0
Fork 1
forked from importers/git-importer
Code Pull Requests Activity

Import files of revisions more reliable and faster

Browse Source
This commit is contained in:
Stephan Kulow 2022-10-26 21:47:39 +02:00
parent 22b70d8bb7
commit 1c846e963c
2 changed files with 13 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lib/db.py
View File

@ -206,6 +206,10 @@ class DB:
"ALTER TABLE linked_revs ADD COLUMN considered BOOLEAN DEFAULT FALSE",
"UPDATE scheme SET version=18",
)
schemes[19] = (
"CREATE INDEX ON files(revision_id);",
"UPDATE scheme SET version=19",
)
schema_version = self.schema_version()
if (schema_version + 1) not in schemes:
return

14
lib/importer.py
View File

@ -267,6 +267,13 @@ class Importer:
(rev.dbid, linked.dbid),
)
def revisions_without_files(self, db):
with db.cursor() as cur:
cur.execute(
"SELECT * FROM revisions WHERE broken=FALSE AND expanded_srcmd5 IS NULL"
)
return [DBRevision(row) for row in cur.fetchall()]
def import_into_db(self):
db = DB()
for project, _, api_url in self.projects:
@ -283,10 +290,7 @@ class Importer:
if missing_user:
missing_user.import_into_db(db)
for rev in DBRevision.all_revisions(db, project, self.package):
# TODO move into SELECT
if rev.broken or rev.expanded_srcmd5:
continue
for rev in self.revisions_without_files(db):
with db.cursor() as cur:
cur.execute(
"""SELECT unexpanded_srcmd5 from revisions WHERE
@ -297,7 +301,7 @@ class Importer:
if linked_rev:
linked_rev = linked_rev[0]
list = self.obs.list(
project, self.package, rev.unexpanded_srcmd5, linked_rev
rev.project, rev.package, rev.unexpanded_srcmd5, linked_rev
)
if list:
rev.import_dir_list(db, list)