diff --git a/lib/db_revision.py b/lib/db_revision.py index f4ee27d..06c3c98 100644 --- a/lib/db_revision.py +++ b/lib/db_revision.py @@ -188,12 +188,14 @@ class DBRevision: def files_list(self, db): if self._files: return self._files - cur = db.cursor() - cur.execute("SELECT * from files where revision_id=%s", (self.dbid,)) - self._files = [] - for row in cur.fetchall(): - (_, _, name, md5, size, mtime) = row - self._files.append({"md5": md5, "size": size, "mtime": mtime, "name": name}) + with db.cursor() as cur: + cur.execute("SELECT * from files where revision_id=%s", (self.dbid,)) + self._files = [] + for row in cur.fetchall(): + (_, _, name, md5, size, mtime) = row + self._files.append( + {"md5": md5, "size": size, "mtime": mtime, "name": name} + ) self._files.sort(key=lambda x: x["name"]) return self._files diff --git a/lib/exporter.py b/lib/exporter.py index c3a0cc5..e871b25 100644 --- a/lib/exporter.py +++ b/lib/exporter.py @@ -12,13 +12,13 @@ class Exporter: def run(self): db = DB() - cur = db.cursor() - cur.execute( - "SELECT * from revisions where package=%s ORDER BY project,rev", - (self.package,), - ) - data = {"revisions": []} - for row in cur.fetchall(): - data["revisions"].append(DBRevision(row).as_dict(db)) + with db.cursor() as cur: + cur.execute( + "SELECT * from revisions where package=%s ORDER BY project,rev", + (self.package,), + ) + data = {"revisions": []} + for row in cur.fetchall(): + data["revisions"].append(DBRevision(row).as_dict(db)) yaml.dump(data, sys.stdout, default_flow_style=False) diff --git a/lib/importer.py b/lib/importer.py index beb5fa5..14c3a9f 100644 --- a/lib/importer.py +++ b/lib/importer.py @@ -149,72 +149,75 @@ class Importer: db.conn.commit() def find_linked_revs(self, db): - cur = db.cursor() - cur.execute( - """SELECT * from revisions WHERE id in (SELECT l.revision_id FROM links l - LEFT JOIN linked_revs lrevs ON lrevs.revision_id=l.revision_id - WHERE lrevs.id IS NULL) and broken is FALSE;""" - ) - for row in cur.fetchall(): - rev = DBRevision(row) - linked_rev = rev.linked_rev(db) - if not linked_rev: - logging.debug(f"No link {rev}") - continue + with db.cursor() as cur: cur.execute( - """INSERT INTO linked_revs (revision_id, linked_id) - VALUES (%s,%s)""", - (rev.dbid, linked_rev.dbid), + """SELECT * from revisions WHERE id in (SELECT l.revision_id FROM links l + LEFT JOIN linked_revs lrevs ON lrevs.revision_id=l.revision_id + WHERE lrevs.id IS NULL) and broken is FALSE;""" ) + for row in cur.fetchall(): + rev = DBRevision(row) + linked_rev = rev.linked_rev(db) + if not linked_rev: + logging.debug(f"No link {rev}") + continue + cur.execute( + """INSERT INTO linked_revs (revision_id, linked_id) + VALUES (%s,%s)""", + (rev.dbid, linked_rev.dbid), + ) def calculate_file_hashes(self, db): - cur = db.cursor() - cur.execute( - "SELECT * from revisions where files_hash IS NULL AND broken is FALSE" - ) - for row in cur.fetchall(): - rev = DBRevision(row) - md5 = rev.calculate_files_hash(db) + with db.cursor() as cur: cur.execute( - "UPDATE revisions SET files_hash=%s WHERE id=%s", (md5, rev.dbid) + "SELECT * from revisions where files_hash IS NULL AND broken is FALSE" ) + for row in cur.fetchall(): + rev = DBRevision(row) + md5 = rev.calculate_files_hash(db) + cur.execute( + "UPDATE revisions SET files_hash=%s WHERE id=%s", (md5, rev.dbid) + ) def fetch_all_linked_packages(self, db, project, package): - cur = db.cursor() - cur.execute( - """SELECT DISTINCT l.project, l.package from links l JOIN revisions r - on r.id=l.revision_id WHERE r.project=%s AND r.package=%s""", - (project, package), - ) - for row in cur.fetchall(): - (lproject, lpackage) = row - self.update_db_package(db, lproject, lpackage) + with db.cursor() as cur: + cur.execute( + """SELECT DISTINCT l.project, l.package from links l JOIN revisions r + on r.id=l.revision_id WHERE r.project=%s AND r.package=%s""", + (project, package), + ) + for row in cur.fetchall(): + (lproject, lpackage) = row + self.update_db_package(db, lproject, lpackage) def find_fake_revisions(self, db): - cur = db.cursor() - cur.execute( - """SELECT * from revisions WHERE - id in (SELECT revision_id from linked_revs WHERE considered=FALSE) AND - id not in (SELECT revision_id FROM fake_revs) ORDER by project,package,rev""" - ) - for row in cur.fetchall(): - rev = DBRevision(row) - prev = rev.previous_commit(db) - if not prev: + with db.cursor() as cur: + cur.execute( + """SELECT * from revisions WHERE + id in (SELECT revision_id from linked_revs WHERE considered=FALSE) AND + id not in (SELECT revision_id FROM fake_revs) ORDER by project,package,rev""" + ) + for row in cur.fetchall(): + DBRevision(row) + + def _find_fake_revision(self, db, rev): + prev = rev.previous_commit(db) + if not prev: + with db.cursor() as cur: cur.execute( "UPDATE linked_revs SET considered=TRUE where revision_id=%s", (rev.dbid,), ) - continue - cur2 = db.cursor() - cur2.execute( + return + with db.cursor() as cur: + cur.execute( """SELECT * from revisions where id in (SELECT revision_id from linked_revs WHERE linked_id=%s) AND commit_time <= %s ORDER BY commit_time""", (prev.dbid, rev.commit_time), ) last_linked = None - for linked in cur2.fetchall(): + for linked in cur.fetchall(): linked = DBRevision(linked) nextrev = linked.next_commit(db) if nextrev and nextrev.commit_time < rev.commit_time: @@ -224,42 +227,45 @@ class Importer: "UPDATE linked_revs SET considered=TRUE where revision_id=%s", (rev.dbid,), ) - if last_linked: - linked = last_linked - cur2.execute( - "SELECT 1 FROM fake_revs where revision_id=%s AND linked_id=%s", - (rev.dbid, linked.dbid), - ) - if cur2.fetchone(): - cur.execute( - "UPDATE linked_revs SET considered=TRUE where revision_id=%s", - (rev.dbid,), - ) - continue - fake_rev = linked.rev + rev.rev / 1000.0 - comment = f"Updating link to change in {rev.project}/{rev.package} revision {rev.rev}" - cur2.execute( - """INSERT INTO revisions (project,package,rev,unexpanded_srcmd5, - commit_time, userid, comment) VALUES(%s,%s,%s,%s,%s,%s,%s) RETURNING id""", - ( - linked.project, - linked.package, - fake_rev, - linked.unexpanded_srcmd5, - rev.commit_time, - "buildservice-autocommit", - comment, - ), - ) - new_id = cur2.fetchone()[0] - cur2.execute( - """INSERT INTO linked_revs (revision_id, linked_id) VALUES (%s,%s)""", - (new_id, rev.dbid), - ) - cur2.execute( - """INSERT INTO fake_revs (revision_id, linked_id) VALUES (%s,%s)""", - (rev.dbid, linked.dbid), + if not last_linked: + return + + with db.cursor() as cur: + linked = last_linked + cur.execute( + "SELECT 1 FROM fake_revs where revision_id=%s AND linked_id=%s", + (rev.dbid, linked.dbid), + ) + if cur.fetchone(): + cur.execute( + "UPDATE linked_revs SET considered=TRUE where revision_id=%s", + (rev.dbid,), ) + return + fake_rev = linked.rev + rev.rev / 1000.0 + comment = f"Updating link to change in {rev.project}/{rev.package} revision {rev.rev}" + cur.execute( + """INSERT INTO revisions (project,package,rev,unexpanded_srcmd5, + commit_time, userid, comment) VALUES(%s,%s,%s,%s,%s,%s,%s) RETURNING id""", + ( + linked.project, + linked.package, + fake_rev, + linked.unexpanded_srcmd5, + rev.commit_time, + "buildservice-autocommit", + comment, + ), + ) + new_id = cur.fetchone()[0] + cur.execute( + """INSERT INTO linked_revs (revision_id, linked_id) VALUES (%s,%s)""", + (new_id, rev.dbid), + ) + cur.execute( + """INSERT INTO fake_revs (revision_id, linked_id) VALUES (%s,%s)""", + (rev.dbid, linked.dbid), + ) def import_into_db(self): db = DB()