From 7a9e9ce33465bbee015a02f41e84f1974ec7a393 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Thu, 27 Oct 2022 13:16:50 +0200 Subject: [PATCH 1/2] Add actual tree and a test case for zsh --- lib/db_revision.py | 3 + lib/importer.py | 2 +- lib/tree_builder.py | 121 +++++++++-- tests/fixtures/zsh-expected-tree.yaml | 300 ++++++++++++++++++++++++++ tests/tree_test.py | 13 +- 5 files changed, 419 insertions(+), 20 deletions(-) create mode 100644 tests/fixtures/zsh-expected-tree.yaml diff --git a/lib/db_revision.py b/lib/db_revision.py index eaebae6..a022108 100644 --- a/lib/db_revision.py +++ b/lib/db_revision.py @@ -24,6 +24,9 @@ class DBRevision: self.rev = float(self.rev) self._files = None + def short_string(self): + return f"Rev {self.project}/{self.package}/{self.rev}" + def __str__(self): return f"Rev {self.project}/{self.package}/{self.rev} Md5 {self.unexpanded_srcmd5} {self.commit_time} {self.userid} {self.request_number}" diff --git a/lib/importer.py b/lib/importer.py index 5d56824..03f2df2 100644 --- a/lib/importer.py +++ b/lib/importer.py @@ -304,7 +304,7 @@ class Importer: self.obs.request(number).import_into_db(db) db.conn.commit() - TreeBuilder(db).build(self.package) + # TreeBuilder(db).build(self.package) def import_all_revisions(self, gc): # Fetch all the requests and sort them. Ideally we should diff --git a/lib/tree_builder.py b/lib/tree_builder.py index 9779d57..c88732a 100644 --- a/lib/tree_builder.py +++ b/lib/tree_builder.py @@ -2,40 +2,125 @@ from lib.db_revision import DBRevision from lib.request import Request +class TreeNode: + """ + Nodes in this "tree" have either no parent (root), one parent (in a chain) + or two parents (in this case the merged revision wins in conflicts). + """ + + def __init__(self, rev): + self.parent = None + self.merged = None + self.revision = rev + self.merged_into = None + + def print(self): + node = self + while node: + print(node.revision, node.revision.files_hash) + if node.merged: + source_node = node.merged + while source_node: + print(" ", source_node.revision, source_node.revision.files_hash) + source_node = source_node.parent + if source_node and source_node.merged_into: + break + node = node.parent + + def as_list(self): + """Return a list for test cases""" + node = self + ret = [] + while node: + repr = {"commit": node.revision.short_string()} + if node.merged: + source_node = node.merged + repr["merged"] = [] + while source_node: + repr["merged"].append(source_node.revision.short_string()) + source_node = source_node.parent + if source_node and source_node.merged_into: + break + node = node.parent + ret.append(repr) + return ret + + class TreeBuilder: def __init__(self, db): self.db = db - def filtered_revisions(self, project, package): + def revisions_chain(self, project, package): revisions = DBRevision.all_revisions(self.db, project, package) revisions.sort() - ret = [] prev = None + tree = None for rev in revisions: if rev.broken: continue if prev and prev.files_hash == rev.files_hash: continue - ret.append(rev) prev = rev - return ret + new_tree = TreeNode(rev) + if tree: + new_tree.parent = tree + tree = new_tree - def build(self, package): - factory_revisions = self.filtered_revisions("openSUSE:Factory", package) + return tree + + def find_merge(self, revision, source_chain): + node = source_chain + while node: + # exclude reverts happening after the merge + if ( + node.revision.commit_time <= revision.commit_time + and node.revision.files_hash == revision.files_hash + ): + return node + node = node.parent + + def add_merge_points(self, factory_revisions): source_revisions = dict() - for rev in factory_revisions: - print(rev, rev.files_hash) - if rev.request_id: - req = Request.find(self.db, rev.request_id) - print(" ", req) + factory_node = factory_revisions + while factory_node: + if factory_node.revision.request_id: + req = Request.find(self.db, factory_node.revision.request_id) key = f"{req.source_project}/{req.source_package}" if key not in source_revisions: - source_revisions[key] = self.filtered_revisions( + source_revisions[key] = self.revisions_chain( req.source_project, req.source_package ) - for rev2 in source_revisions.get(key): - # this happened after the fact - possibly a revert - if rev2.commit_time > rev.commit_time: - continue - if rev2.files_hash == rev.files_hash: - print(" ", rev2, rev2.files_hash) + factory_node.merged = self.find_merge( + factory_node.revision, source_revisions[key] + ) + # add a reverse lookup + if factory_node.merged: + factory_node.merged.merged_into = factory_node + + factory_node = factory_node.parent + + def prune_loose_end(self, factory_node): + """Look for source revisions that end in a new root and prune them""" + last_merge = None + while factory_node: + if factory_node.merged: + source_node = factory_node.merged + ended_without_merge = False + while source_node: + source_node = source_node.parent + if source_node and source_node.merged_into: + ended_without_merge = True + break + if not ended_without_merge: + factory_node.merged = None + last_merge.parent = None + else: + last_merge = factory_node.merged + factory_node = factory_node.parent + + def build(self, package): + factory_revisions = self.revisions_chain("openSUSE:Factory", package) + self.add_merge_points(factory_revisions) + + self.prune_loose_end(factory_revisions) + return factory_revisions diff --git a/tests/fixtures/zsh-expected-tree.yaml b/tests/fixtures/zsh-expected-tree.yaml new file mode 100644 index 0000000..fedef37 --- /dev/null +++ b/tests/fixtures/zsh-expected-tree.yaml @@ -0,0 +1,300 @@ +- commit: Rev openSUSE:Factory/zsh/98.0 + merged: + - Rev shells/zsh/236.0 +- commit: Rev openSUSE:Factory/zsh/97.0 + merged: + - Rev shells/zsh/234.0 +- commit: Rev openSUSE:Factory/zsh/96.0 + merged: + - Rev shells/zsh/232.0 +- commit: Rev openSUSE:Factory/zsh/95.0 + merged: + - Rev shells/zsh/230.0 +- commit: Rev openSUSE:Factory/zsh/94.0 + merged: + - Rev shells/zsh/228.0 + - Rev shells/zsh/227.0 +- commit: Rev openSUSE:Factory/zsh/93.0 + merged: + - Rev shells/zsh/225.0 + - Rev shells/zsh/224.0 + - Rev shells/zsh/223.0 +- commit: Rev openSUSE:Factory/zsh/92.0 + merged: + - Rev shells/zsh/221.0 +- commit: Rev openSUSE:Factory/zsh/91.0 + merged: + - Rev shells/zsh/219.0 +- commit: Rev openSUSE:Factory/zsh/90.0 + merged: + - Rev shells/zsh/217.0 +- commit: Rev openSUSE:Factory/zsh/89.0 + merged: + - Rev shells/zsh/215.0 + - Rev shells/zsh/214.0 +- commit: Rev openSUSE:Factory/zsh/88.0 + merged: + - Rev shells/zsh/212.0 +- commit: Rev openSUSE:Factory/zsh/87.0 + merged: + - Rev shells/zsh/210.0 + - Rev shells/zsh/209.0 + - Rev shells/zsh/208.0 + - Rev shells/zsh/207.0 +- commit: Rev openSUSE:Factory/zsh/86.0 + merged: + - Rev shells/zsh/205.0 + - Rev shells/zsh/204.0 +- commit: Rev openSUSE:Factory/zsh/85.0 + merged: + - Rev shells/zsh/202.0 + - Rev shells/zsh/201.0 +- commit: Rev openSUSE:Factory/zsh/84.0 + merged: + - Rev shells/zsh/199.0 + - Rev shells/zsh/198.0 + - Rev shells/zsh/197.0 +- commit: Rev openSUSE:Factory/zsh/83.0 + merged: + - Rev shells/zsh/195.0 + - Rev shells/zsh/194.0 + - Rev shells/zsh/193.0 + - Rev shells/zsh/192.0 + - Rev shells/zsh/191.0 +- commit: Rev openSUSE:Factory/zsh/82.0 + merged: + - Rev shells/zsh/189.0 +- commit: Rev openSUSE:Factory/zsh/81.0 + merged: + - Rev shells/zsh/187.0 + - Rev shells/zsh/186.0 +- commit: Rev openSUSE:Factory/zsh/80.0 + merged: + - Rev shells/zsh/184.0 +- commit: Rev openSUSE:Factory/zsh/79.0 + merged: + - Rev shells/zsh/182.0 +- commit: Rev openSUSE:Factory/zsh/78.0 + merged: + - Rev shells/zsh/180.0 +- commit: Rev openSUSE:Factory/zsh/77.0 + merged: + - Rev shells/zsh/178.0 + - Rev shells/zsh/177.0 +- commit: Rev openSUSE:Factory/zsh/76.0 + merged: + - Rev shells/zsh/175.0 +- commit: Rev openSUSE:Factory/zsh/75.0 + merged: + - Rev shells/zsh/173.0 +- commit: Rev openSUSE:Factory/zsh/74.0 + merged: + - Rev shells/zsh/171.0 + - Rev shells/zsh/170.0 +- commit: Rev openSUSE:Factory/zsh/73.0 + merged: + - Rev shells/zsh/168.0 +- commit: Rev openSUSE:Factory/zsh/72.0 + merged: + - Rev shells/zsh/166.0 +- commit: Rev openSUSE:Factory/zsh/71.0 + merged: + - Rev shells/zsh/164.0 +- commit: Rev openSUSE:Factory/zsh/70.0 + merged: + - Rev shells/zsh/162.0 +- commit: Rev openSUSE:Factory/zsh/69.0 + merged: + - Rev shells/zsh/160.0 + - Rev shells/zsh/159.0 + - Rev shells/zsh/158.0 +- commit: Rev openSUSE:Factory/zsh/68.0 + merged: + - Rev shells/zsh/156.0 +- commit: Rev openSUSE:Factory/zsh/67.0 + merged: + - Rev shells/zsh/154.0 + - Rev shells/zsh/153.0 +- commit: Rev openSUSE:Factory/zsh/66.0 + merged: + - Rev shells/zsh/151.0 + - Rev shells/zsh/150.0 +- commit: Rev openSUSE:Factory/zsh/65.0 + merged: + - Rev shells/zsh/148.0 +- commit: Rev openSUSE:Factory/zsh/64.0 + merged: + - Rev shells/zsh/146.0 + - Rev shells/zsh/145.0 + - Rev shells/zsh/144.0 +- commit: Rev openSUSE:Factory/zsh/63.0 + merged: + - Rev shells/zsh/142.0 +- commit: Rev openSUSE:Factory/zsh/62.0 + merged: + - Rev shells/zsh/140.0 +- commit: Rev openSUSE:Factory/zsh/61.0 + merged: + - Rev shells/zsh/138.0 + - Rev shells/zsh/137.0 +- commit: Rev openSUSE:Factory/zsh/60.0 + merged: + - Rev shells/zsh/135.0 +- commit: Rev openSUSE:Factory/zsh/59.0 + merged: + - Rev shells/zsh/133.0 +- commit: Rev openSUSE:Factory/zsh/57.0 + merged: + - Rev shells/zsh/131.0 +- commit: Rev openSUSE:Factory/zsh/56.0 + merged: + - Rev shells/zsh/129.0 +- commit: Rev openSUSE:Factory/zsh/55.0 + merged: + - Rev shells/zsh/127.0 +- commit: Rev openSUSE:Factory/zsh/54.0 + merged: + - Rev shells/zsh/125.0 +- commit: Rev openSUSE:Factory/zsh/53.0 + merged: + - Rev shells/zsh/123.0 +- commit: Rev openSUSE:Factory/zsh/51.0 + merged: + - Rev shells/zsh/121.0 +- commit: Rev openSUSE:Factory/zsh/50.0 + merged: + - Rev shells/zsh/119.0 +- commit: Rev openSUSE:Factory/zsh/49.0 + merged: + - Rev shells/zsh/117.0 +- commit: Rev openSUSE:Factory/zsh/48.0 + merged: + - Rev shells/zsh/115.0 +- commit: Rev openSUSE:Factory/zsh/46.0 + merged: + - Rev shells/zsh/113.0 +- commit: Rev openSUSE:Factory/zsh/45.0 + merged: + - Rev shells/zsh/111.0 +- commit: Rev openSUSE:Factory/zsh/44.0 + merged: + - Rev shells/zsh/109.0 +- commit: Rev openSUSE:Factory/zsh/43.0 + merged: + - Rev shells/zsh/107.0 +- commit: Rev openSUSE:Factory/zsh/42.0 + merged: + - Rev shells/zsh/105.0 +- commit: Rev openSUSE:Factory/zsh/41.0 + merged: + - Rev shells/zsh/103.0 + - Rev shells/zsh/102.0 +- commit: Rev openSUSE:Factory/zsh/39.0 + merged: + - Rev shells/zsh/100.0 +- commit: Rev openSUSE:Factory/zsh/38.0 + merged: + - Rev shells/zsh/98.0 +- commit: Rev openSUSE:Factory/zsh/37.0 + merged: + - Rev shells/zsh/95.0 + - Rev shells/zsh/94.0 + - Rev shells/zsh/91.0 +- commit: Rev openSUSE:Factory/zsh/36.0 + merged: + - Rev shells/zsh/90.0 + - Rev shells/zsh/89.0 + - Rev shells/zsh/88.0 + - Rev shells/zsh/87.0 +- commit: Rev openSUSE:Factory/zsh/35.0 + merged: + - Rev shells/zsh/84.0 + - Rev shells/zsh/83.0 + - Rev shells/zsh/82.0 + - Rev shells/zsh/81.0 + - Rev shells/zsh/80.0 +- commit: Rev openSUSE:Factory/zsh/34.0 + merged: + - Rev shells/zsh/77.0 + - Rev shells/zsh/76.032 +- commit: Rev openSUSE:Factory/zsh/32.0 +- commit: Rev openSUSE:Factory/zsh/31.0 + merged: + - Rev shells/zsh/75.0 + - Rev shells/zsh/74.0 + - Rev shells/zsh/73.0 + - Rev shells/zsh/72.03 +- commit: Rev openSUSE:Factory/zsh/30.0 +- commit: Rev openSUSE:Factory/zsh/29.0 + merged: + - Rev shells/zsh/71.0 +- commit: Rev openSUSE:Factory/zsh/28.0 + merged: + - Rev shells/zsh/69.0 + - Rev shells/zsh/68.0 + - Rev shells/zsh/67.027 +- commit: Rev openSUSE:Factory/zsh/27.0 +- commit: Rev openSUSE:Factory/zsh/26.0 + merged: + - Rev shells/zsh/66.0 + - Rev shells/zsh/65.0 + - Rev shells/zsh/64.0 + - Rev shells/zsh/63.0 + - Rev shells/zsh/62.0 + - Rev shells/zsh/61.0 + - Rev shells/zsh/60.0 + - Rev shells/zsh/59.0 + - Rev shells/zsh/58.0 + - Rev shells/zsh/57.0 + - Rev shells/zsh/56.0 + - Rev shells/zsh/55.0 + - Rev shells/zsh/54.0 + - Rev shells/zsh/53.0 + - Rev shells/zsh/52.0 + - Rev shells/zsh/51.0 + - Rev shells/zsh/50.0 + - Rev shells/zsh/49.0 + - Rev shells/zsh/48.0 + - Rev shells/zsh/47.0 + - Rev shells/zsh/46.025 +- commit: Rev openSUSE:Factory/zsh/25.0 +- commit: Rev openSUSE:Factory/zsh/24.0 + merged: + - Rev shells/zsh/45.0 + - Rev shells/zsh/44.0 + - Rev shells/zsh/43.023 +- commit: Rev openSUSE:Factory/zsh/23.0 +- commit: Rev openSUSE:Factory/zsh/22.0 + merged: + - Rev shells/zsh/42.0 + - Rev shells/zsh/41.035 +- commit: Rev openSUSE:Factory/zsh/18.0 +- commit: Rev openSUSE:Factory/zsh/17.0 + merged: + - Rev shells/zsh/41.017 + - Rev shells/zsh/41.0 + - Rev shells/zsh/40.0 + - Rev shells/zsh/39.0 + - Rev shells/zsh/38.0 + - Rev shells/zsh/37.0 +- commit: Rev openSUSE:Factory/zsh/16.0 +- commit: Rev openSUSE:Factory/zsh/14.0 + merged: + - Rev shells/zsh/36.014 + - Rev shells/zsh/36.0 + - Rev shells/zsh/35.0 +- commit: Rev openSUSE:Factory/zsh/13.0 + merged: + - Rev shells/zsh/34.0 +- commit: Rev openSUSE:Factory/zsh/12.0 +- commit: Rev openSUSE:Factory/zsh/11.0 +- commit: Rev openSUSE:Factory/zsh/10.0 +- commit: Rev openSUSE:Factory/zsh/9.0 +- commit: Rev openSUSE:Factory/zsh/8.0 +- commit: Rev openSUSE:Factory/zsh/7.0 +- commit: Rev openSUSE:Factory/zsh/6.0 +- commit: Rev openSUSE:Factory/zsh/5.0 +- commit: Rev openSUSE:Factory/zsh/4.0 +- commit: Rev openSUSE:Factory/zsh/3.0 +- commit: Rev openSUSE:Factory/zsh/2.0 +- commit: Rev openSUSE:Factory/zsh/1.0 diff --git a/tests/tree_test.py b/tests/tree_test.py index 03cd73d..27febe5 100644 --- a/tests/tree_test.py +++ b/tests/tree_test.py @@ -18,7 +18,18 @@ class TestTreeMethods(unittest.TestCase): DBRevision.import_fixture_dict(self.db, rev) def test_create_tree(self): - TreeBuilder(self.db).build("zsh") + revisions = TreeBuilder(self.db).build("zsh") + + if False: + import sys + + yaml.dump(revisions.as_list(), sys.stdout) + path = os.path.join( + os.path.dirname(__file__), "fixtures/zsh-expected-tree.yaml" + ) + with open(path, "r") as f: + zsh_data = yaml.safe_load(f) + self.assertEqual(zsh_data, revisions.as_list()) if __name__ == "__main__": -- 2.45.2 From e636f3ed9d888726c26def44d02606b88173a1eb Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Thu, 27 Oct 2022 13:34:04 +0200 Subject: [PATCH 2/2] Print the calculated tree for now --- lib/importer.py | 3 +-- lib/tree_builder.py | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/importer.py b/lib/importer.py index 03f2df2..82e21ac 100644 --- a/lib/importer.py +++ b/lib/importer.py @@ -146,7 +146,6 @@ class Importer: tprj = root.get("project") or project tpkg = root.get("package") or package dbrev.links_to(db, tprj, tpkg) - db.conn.commit() def find_linked_revs(self, db): with db.cursor() as cur: @@ -304,7 +303,7 @@ class Importer: self.obs.request(number).import_into_db(db) db.conn.commit() - # TreeBuilder(db).build(self.package) + TreeBuilder(db).build(self.package).print() def import_all_revisions(self, gc): # Fetch all the requests and sort them. Ideally we should diff --git a/lib/tree_builder.py b/lib/tree_builder.py index c88732a..4cb6439 100644 --- a/lib/tree_builder.py +++ b/lib/tree_builder.py @@ -113,7 +113,8 @@ class TreeBuilder: break if not ended_without_merge: factory_node.merged = None - last_merge.parent = None + if last_merge: + last_merge.parent = None else: last_merge = factory_node.merged factory_node = factory_node.parent -- 2.45.2