importers/git-importer
3
0
Fork 2
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make use of db.cursor() in with blocks

Browse Source
This commit is contained in:
Stephan Kulow 2022-10-26 15:49:14 +02:00
parent 217bbcadb5
commit 4dc7e972a2
3 changed files with 103 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
lib/db_revision.py
View File

@ -188,12 +188,14 @@ class DBRevision:
def files_list(self, db): def files_list(self, db):
if self._files: if self._files:
return self._files return self._files
cur = db.cursor() with db.cursor() as cur:
cur.execute("SELECT * from files where revision_id=%s", (self.dbid,)) cur.execute("SELECT * from files where revision_id=%s", (self.dbid,))
self._files = [] self._files = []
for row in cur.fetchall(): for row in cur.fetchall():
(_, _, name, md5, size, mtime) = row (_, _, name, md5, size, mtime) = row
self._files.append({"md5": md5, "size": size, "mtime": mtime, "name": name}) self._files.append(
{"md5": md5, "size": size, "mtime": mtime, "name": name}
)
self._files.sort(key=lambda x: x["name"]) self._files.sort(key=lambda x: x["name"])
return self._files return self._files

16
lib/exporter.py
View File

@ -12,13 +12,13 @@ class Exporter:
def run(self): def run(self):
db = DB() db = DB()
cur = db.cursor() with db.cursor() as cur:
cur.execute( cur.execute(
"SELECT * from revisions where package=%s ORDER BY project,rev", "SELECT * from revisions where package=%s ORDER BY project,rev",
(self.package,), (self.package,),
) )
data = {"revisions": []} data = {"revisions": []}
for row in cur.fetchall(): for row in cur.fetchall():
data["revisions"].append(DBRevision(row).as_dict(db)) data["revisions"].append(DBRevision(row).as_dict(db))
yaml.dump(data, sys.stdout, default_flow_style=False) yaml.dump(data, sys.stdout, default_flow_style=False)

168
lib/importer.py
View File

@ -149,72 +149,75 @@ class Importer:
db.conn.commit() db.conn.commit()
def find_linked_revs(self, db): def find_linked_revs(self, db):
cur = db.cursor() with db.cursor() as cur:
cur.execute(
"""SELECT * from revisions WHERE id in (SELECT l.revision_id FROM links l
LEFT JOIN linked_revs lrevs ON lrevs.revision_id=l.revision_id
WHERE lrevs.id IS NULL) and broken is FALSE;"""
)
for row in cur.fetchall():
rev = DBRevision(row)
linked_rev = rev.linked_rev(db)
if not linked_rev:
logging.debug(f"No link {rev}")
continue
cur.execute( cur.execute(
"""INSERT INTO linked_revs (revision_id, linked_id) """SELECT * from revisions WHERE id in (SELECT l.revision_id FROM links l
VALUES (%s,%s)""", LEFT JOIN linked_revs lrevs ON lrevs.revision_id=l.revision_id
(rev.dbid, linked_rev.dbid), WHERE lrevs.id IS NULL) and broken is FALSE;"""
) )
for row in cur.fetchall():
rev = DBRevision(row)
linked_rev = rev.linked_rev(db)
if not linked_rev:
logging.debug(f"No link {rev}")
continue
cur.execute(
"""INSERT INTO linked_revs (revision_id, linked_id)
VALUES (%s,%s)""",
(rev.dbid, linked_rev.dbid),
)
def calculate_file_hashes(self, db): def calculate_file_hashes(self, db):
cur = db.cursor() with db.cursor() as cur:
cur.execute(
"SELECT * from revisions where files_hash IS NULL AND broken is FALSE"
)
for row in cur.fetchall():
rev = DBRevision(row)
md5 = rev.calculate_files_hash(db)
cur.execute( cur.execute(
"UPDATE revisions SET files_hash=%s WHERE id=%s", (md5, rev.dbid) "SELECT * from revisions where files_hash IS NULL AND broken is FALSE"
) )
for row in cur.fetchall():
rev = DBRevision(row)
md5 = rev.calculate_files_hash(db)
cur.execute(
"UPDATE revisions SET files_hash=%s WHERE id=%s", (md5, rev.dbid)
)
def fetch_all_linked_packages(self, db, project, package): def fetch_all_linked_packages(self, db, project, package):
cur = db.cursor() with db.cursor() as cur:
cur.execute( cur.execute(
"""SELECT DISTINCT l.project, l.package from links l JOIN revisions r """SELECT DISTINCT l.project, l.package from links l JOIN revisions r
on r.id=l.revision_id WHERE r.project=%s AND r.package=%s""", on r.id=l.revision_id WHERE r.project=%s AND r.package=%s""",
(project, package), (project, package),
) )
for row in cur.fetchall(): for row in cur.fetchall():
(lproject, lpackage) = row (lproject, lpackage) = row
self.update_db_package(db, lproject, lpackage) self.update_db_package(db, lproject, lpackage)
def find_fake_revisions(self, db): def find_fake_revisions(self, db):
cur = db.cursor() with db.cursor() as cur:
cur.execute( cur.execute(
"""SELECT * from revisions WHERE """SELECT * from revisions WHERE
id in (SELECT revision_id from linked_revs WHERE considered=FALSE) AND id in (SELECT revision_id from linked_revs WHERE considered=FALSE) AND
id not in (SELECT revision_id FROM fake_revs) ORDER by project,package,rev""" id not in (SELECT revision_id FROM fake_revs) ORDER by project,package,rev"""
) )
for row in cur.fetchall(): for row in cur.fetchall():
rev = DBRevision(row) DBRevision(row)
prev = rev.previous_commit(db)
if not prev: def _find_fake_revision(self, db, rev):
prev = rev.previous_commit(db)
if not prev:
with db.cursor() as cur:
cur.execute( cur.execute(
"UPDATE linked_revs SET considered=TRUE where revision_id=%s", "UPDATE linked_revs SET considered=TRUE where revision_id=%s",
(rev.dbid,), (rev.dbid,),
) )
continue return
cur2 = db.cursor() with db.cursor() as cur:
cur2.execute( cur.execute(
"""SELECT * from revisions where id in """SELECT * from revisions where id in
(SELECT revision_id from linked_revs WHERE linked_id=%s) (SELECT revision_id from linked_revs WHERE linked_id=%s)
AND commit_time <= %s ORDER BY commit_time""", AND commit_time <= %s ORDER BY commit_time""",
(prev.dbid, rev.commit_time), (prev.dbid, rev.commit_time),
) )
last_linked = None last_linked = None
for linked in cur2.fetchall(): for linked in cur.fetchall():
linked = DBRevision(linked) linked = DBRevision(linked)
nextrev = linked.next_commit(db) nextrev = linked.next_commit(db)
if nextrev and nextrev.commit_time < rev.commit_time: if nextrev and nextrev.commit_time < rev.commit_time:
@ -224,42 +227,45 @@ class Importer:
"UPDATE linked_revs SET considered=TRUE where revision_id=%s", "UPDATE linked_revs SET considered=TRUE where revision_id=%s",
(rev.dbid,), (rev.dbid,),
) )
if last_linked: if not last_linked:
linked = last_linked return
cur2.execute(
"SELECT 1 FROM fake_revs where revision_id=%s AND linked_id=%s", with db.cursor() as cur:
(rev.dbid, linked.dbid), linked = last_linked
) cur.execute(
if cur2.fetchone(): "SELECT 1 FROM fake_revs where revision_id=%s AND linked_id=%s",
cur.execute( (rev.dbid, linked.dbid),
"UPDATE linked_revs SET considered=TRUE where revision_id=%s", )
(rev.dbid,), if cur.fetchone():
) cur.execute(
continue "UPDATE linked_revs SET considered=TRUE where revision_id=%s",
fake_rev = linked.rev + rev.rev / 1000.0 (rev.dbid,),
comment = f"Updating link to change in {rev.project}/{rev.package} revision {rev.rev}"
cur2.execute(
"""INSERT INTO revisions (project,package,rev,unexpanded_srcmd5,
commit_time, userid, comment) VALUES(%s,%s,%s,%s,%s,%s,%s) RETURNING id""",
(
linked.project,
linked.package,
fake_rev,
linked.unexpanded_srcmd5,
rev.commit_time,
"buildservice-autocommit",
comment,
),
)
new_id = cur2.fetchone()[0]
cur2.execute(
"""INSERT INTO linked_revs (revision_id, linked_id) VALUES (%s,%s)""",
(new_id, rev.dbid),
)
cur2.execute(
"""INSERT INTO fake_revs (revision_id, linked_id) VALUES (%s,%s)""",
(rev.dbid, linked.dbid),
) )
return
fake_rev = linked.rev + rev.rev / 1000.0
comment = f"Updating link to change in {rev.project}/{rev.package} revision {rev.rev}"
cur.execute(
"""INSERT INTO revisions (project,package,rev,unexpanded_srcmd5,
commit_time, userid, comment) VALUES(%s,%s,%s,%s,%s,%s,%s) RETURNING id""",
(
linked.project,
linked.package,
fake_rev,
linked.unexpanded_srcmd5,
rev.commit_time,
"buildservice-autocommit",
comment,
),
)
new_id = cur.fetchone()[0]
cur.execute(
"""INSERT INTO linked_revs (revision_id, linked_id) VALUES (%s,%s)""",
(new_id, rev.dbid),
)
cur.execute(
"""INSERT INTO fake_revs (revision_id, linked_id) VALUES (%s,%s)""",
(rev.dbid, linked.dbid),
)
def import_into_db(self): def import_into_db(self):
db = DB() db = DB()