forked from importers/git-importer
Make use of db.cursor() in with blocks
This commit is contained in:
Stephan Kulow
parent
217bbcadb5
commit
4dc7e972a2
@ -188,12 +188,14 @@ class DBRevision:
|
||||
def files_list(self, db):
|
||||
if self._files:
|
||||
return self._files
|
||||
cur = db.cursor()
|
||||
cur.execute("SELECT * from files where revision_id=%s", (self.dbid,))
|
||||
self._files = []
|
||||
for row in cur.fetchall():
|
||||
(_, _, name, md5, size, mtime) = row
|
||||
self._files.append({"md5": md5, "size": size, "mtime": mtime, "name": name})
|
||||
with db.cursor() as cur:
|
||||
cur.execute("SELECT * from files where revision_id=%s", (self.dbid,))
|
||||
self._files = []
|
||||
for row in cur.fetchall():
|
||||
(_, _, name, md5, size, mtime) = row
|
||||
self._files.append(
|
||||
{"md5": md5, "size": size, "mtime": mtime, "name": name}
|
||||
)
|
||||
self._files.sort(key=lambda x: x["name"])
|
||||
return self._files
|
||||
|
||||
|
||||
@ -12,13 +12,13 @@ class Exporter:
|
||||
|
||||
def run(self):
|
||||
db = DB()
|
||||
cur = db.cursor()
|
||||
cur.execute(
|
||||
"SELECT * from revisions where package=%s ORDER BY project,rev",
|
||||
(self.package,),
|
||||
)
|
||||
data = {"revisions": []}
|
||||
for row in cur.fetchall():
|
||||
data["revisions"].append(DBRevision(row).as_dict(db))
|
||||
with db.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT * from revisions where package=%s ORDER BY project,rev",
|
||||
(self.package,),
|
||||
)
|
||||
data = {"revisions": []}
|
||||
for row in cur.fetchall():
|
||||
data["revisions"].append(DBRevision(row).as_dict(db))
|
||||
|
||||
yaml.dump(data, sys.stdout, default_flow_style=False)
|
||||
|
||||
168
lib/importer.py
168
lib/importer.py
@ -149,72 +149,75 @@ class Importer:
|
||||
db.conn.commit()
|
||||
|
||||
def find_linked_revs(self, db):
|
||||
cur = db.cursor()
|
||||
cur.execute(
|
||||
"""SELECT * from revisions WHERE id in (SELECT l.revision_id FROM links l
|
||||
LEFT JOIN linked_revs lrevs ON lrevs.revision_id=l.revision_id
|
||||
WHERE lrevs.id IS NULL) and broken is FALSE;"""
|
||||
)
|
||||
for row in cur.fetchall():
|
||||
rev = DBRevision(row)
|
||||
linked_rev = rev.linked_rev(db)
|
||||
if not linked_rev:
|
||||
logging.debug(f"No link {rev}")
|
||||
continue
|
||||
with db.cursor() as cur:
|
||||
cur.execute(
|
||||
"""INSERT INTO linked_revs (revision_id, linked_id)
|
||||
VALUES (%s,%s)""",
|
||||
(rev.dbid, linked_rev.dbid),
|
||||
"""SELECT * from revisions WHERE id in (SELECT l.revision_id FROM links l
|
||||
LEFT JOIN linked_revs lrevs ON lrevs.revision_id=l.revision_id
|
||||
WHERE lrevs.id IS NULL) and broken is FALSE;"""
|
||||
)
|
||||
for row in cur.fetchall():
|
||||
rev = DBRevision(row)
|
||||
linked_rev = rev.linked_rev(db)
|
||||
if not linked_rev:
|
||||
logging.debug(f"No link {rev}")
|
||||
continue
|
||||
cur.execute(
|
||||
"""INSERT INTO linked_revs (revision_id, linked_id)
|
||||
VALUES (%s,%s)""",
|
||||
(rev.dbid, linked_rev.dbid),
|
||||
)
|
||||
|
||||
def calculate_file_hashes(self, db):
|
||||
cur = db.cursor()
|
||||
cur.execute(
|
||||
"SELECT * from revisions where files_hash IS NULL AND broken is FALSE"
|
||||
)
|
||||
for row in cur.fetchall():
|
||||
rev = DBRevision(row)
|
||||
md5 = rev.calculate_files_hash(db)
|
||||
with db.cursor() as cur:
|
||||
cur.execute(
|
||||
"UPDATE revisions SET files_hash=%s WHERE id=%s", (md5, rev.dbid)
|
||||
"SELECT * from revisions where files_hash IS NULL AND broken is FALSE"
|
||||
)
|
||||
for row in cur.fetchall():
|
||||
rev = DBRevision(row)
|
||||
md5 = rev.calculate_files_hash(db)
|
||||
cur.execute(
|
||||
"UPDATE revisions SET files_hash=%s WHERE id=%s", (md5, rev.dbid)
|
||||
)
|
||||
|
||||
def fetch_all_linked_packages(self, db, project, package):
|
||||
cur = db.cursor()
|
||||
cur.execute(
|
||||
"""SELECT DISTINCT l.project, l.package from links l JOIN revisions r
|
||||
on r.id=l.revision_id WHERE r.project=%s AND r.package=%s""",
|
||||
(project, package),
|
||||
)
|
||||
for row in cur.fetchall():
|
||||
(lproject, lpackage) = row
|
||||
self.update_db_package(db, lproject, lpackage)
|
||||
with db.cursor() as cur:
|
||||
cur.execute(
|
||||
"""SELECT DISTINCT l.project, l.package from links l JOIN revisions r
|
||||
on r.id=l.revision_id WHERE r.project=%s AND r.package=%s""",
|
||||
(project, package),
|
||||
)
|
||||
for row in cur.fetchall():
|
||||
(lproject, lpackage) = row
|
||||
self.update_db_package(db, lproject, lpackage)
|
||||
|
||||
def find_fake_revisions(self, db):
|
||||
cur = db.cursor()
|
||||
cur.execute(
|
||||
"""SELECT * from revisions WHERE
|
||||
id in (SELECT revision_id from linked_revs WHERE considered=FALSE) AND
|
||||
id not in (SELECT revision_id FROM fake_revs) ORDER by project,package,rev"""
|
||||
)
|
||||
for row in cur.fetchall():
|
||||
rev = DBRevision(row)
|
||||
prev = rev.previous_commit(db)
|
||||
if not prev:
|
||||
with db.cursor() as cur:
|
||||
cur.execute(
|
||||
"""SELECT * from revisions WHERE
|
||||
id in (SELECT revision_id from linked_revs WHERE considered=FALSE) AND
|
||||
id not in (SELECT revision_id FROM fake_revs) ORDER by project,package,rev"""
|
||||
)
|
||||
for row in cur.fetchall():
|
||||
DBRevision(row)
|
||||
|
||||
def _find_fake_revision(self, db, rev):
|
||||
prev = rev.previous_commit(db)
|
||||
if not prev:
|
||||
with db.cursor() as cur:
|
||||
cur.execute(
|
||||
"UPDATE linked_revs SET considered=TRUE where revision_id=%s",
|
||||
(rev.dbid,),
|
||||
)
|
||||
continue
|
||||
cur2 = db.cursor()
|
||||
cur2.execute(
|
||||
return
|
||||
with db.cursor() as cur:
|
||||
cur.execute(
|
||||
"""SELECT * from revisions where id in
|
||||
(SELECT revision_id from linked_revs WHERE linked_id=%s)
|
||||
AND commit_time <= %s ORDER BY commit_time""",
|
||||
(prev.dbid, rev.commit_time),
|
||||
)
|
||||
last_linked = None
|
||||
for linked in cur2.fetchall():
|
||||
for linked in cur.fetchall():
|
||||
linked = DBRevision(linked)
|
||||
nextrev = linked.next_commit(db)
|
||||
if nextrev and nextrev.commit_time < rev.commit_time:
|
||||
@ -224,42 +227,45 @@ class Importer:
|
||||
"UPDATE linked_revs SET considered=TRUE where revision_id=%s",
|
||||
(rev.dbid,),
|
||||
)
|
||||
if last_linked:
|
||||
linked = last_linked
|
||||
cur2.execute(
|
||||
"SELECT 1 FROM fake_revs where revision_id=%s AND linked_id=%s",
|
||||
(rev.dbid, linked.dbid),
|
||||
)
|
||||
if cur2.fetchone():
|
||||
cur.execute(
|
||||
"UPDATE linked_revs SET considered=TRUE where revision_id=%s",
|
||||
(rev.dbid,),
|
||||
)
|
||||
continue
|
||||
fake_rev = linked.rev + rev.rev / 1000.0
|
||||
comment = f"Updating link to change in {rev.project}/{rev.package} revision {rev.rev}"
|
||||
cur2.execute(
|
||||
"""INSERT INTO revisions (project,package,rev,unexpanded_srcmd5,
|
||||
commit_time, userid, comment) VALUES(%s,%s,%s,%s,%s,%s,%s) RETURNING id""",
|
||||
(
|
||||
linked.project,
|
||||
linked.package,
|
||||
fake_rev,
|
||||
linked.unexpanded_srcmd5,
|
||||
rev.commit_time,
|
||||
"buildservice-autocommit",
|
||||
comment,
|
||||
),
|
||||
)
|
||||
new_id = cur2.fetchone()[0]
|
||||
cur2.execute(
|
||||
"""INSERT INTO linked_revs (revision_id, linked_id) VALUES (%s,%s)""",
|
||||
(new_id, rev.dbid),
|
||||
)
|
||||
cur2.execute(
|
||||
"""INSERT INTO fake_revs (revision_id, linked_id) VALUES (%s,%s)""",
|
||||
(rev.dbid, linked.dbid),
|
||||
if not last_linked:
|
||||
return
|
||||
|
||||
with db.cursor() as cur:
|
||||
linked = last_linked
|
||||
cur.execute(
|
||||
"SELECT 1 FROM fake_revs where revision_id=%s AND linked_id=%s",
|
||||
(rev.dbid, linked.dbid),
|
||||
)
|
||||
if cur.fetchone():
|
||||
cur.execute(
|
||||
"UPDATE linked_revs SET considered=TRUE where revision_id=%s",
|
||||
(rev.dbid,),
|
||||
)
|
||||
return
|
||||
fake_rev = linked.rev + rev.rev / 1000.0
|
||||
comment = f"Updating link to change in {rev.project}/{rev.package} revision {rev.rev}"
|
||||
cur.execute(
|
||||
"""INSERT INTO revisions (project,package,rev,unexpanded_srcmd5,
|
||||
commit_time, userid, comment) VALUES(%s,%s,%s,%s,%s,%s,%s) RETURNING id""",
|
||||
(
|
||||
linked.project,
|
||||
linked.package,
|
||||
fake_rev,
|
||||
linked.unexpanded_srcmd5,
|
||||
rev.commit_time,
|
||||
"buildservice-autocommit",
|
||||
comment,
|
||||
),
|
||||
)
|
||||
new_id = cur.fetchone()[0]
|
||||
cur.execute(
|
||||
"""INSERT INTO linked_revs (revision_id, linked_id) VALUES (%s,%s)""",
|
||||
(new_id, rev.dbid),
|
||||
)
|
||||
cur.execute(
|
||||
"""INSERT INTO fake_revs (revision_id, linked_id) VALUES (%s,%s)""",
|
||||
(rev.dbid, linked.dbid),
|
||||
)
|
||||
|
||||
def import_into_db(self):
|
||||
db = DB()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user