importers/git-importer
3
0
Fork 2
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request 'Add actual tree and a test case for zsh' (#7) from add_tree into main

Browse Source 
Reviewed-on: https://gitea.opensuse.org/importers/git-importer/pulls/7
This commit is contained in:
coolo 2022-10-27 14:04:52 +02:00
commit 62511d15bc
5 changed files with 420 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lib/db_revision.py
View File

@ -24,6 +24,9 @@ class DBRevision:
self.rev = float(self.rev) self.rev = float(self.rev)
self._files = None self._files = None
def short_string(self):
return f"Rev {self.project}/{self.package}/{self.rev}"
def __str__(self): def __str__(self):
return f"Rev {self.project}/{self.package}/{self.rev} Md5 {self.unexpanded_srcmd5} {self.commit_time} {self.userid} {self.request_number}" return f"Rev {self.project}/{self.package}/{self.rev} Md5 {self.unexpanded_srcmd5} {self.commit_time} {self.userid} {self.request_number}"

3
lib/importer.py
View File

@ -146,7 +146,6 @@ class Importer:
tprj = root.get("project") or project tprj = root.get("project") or project
tpkg = root.get("package") or package tpkg = root.get("package") or package
dbrev.links_to(db, tprj, tpkg) dbrev.links_to(db, tprj, tpkg)
db.conn.commit()
def find_linked_revs(self, db): def find_linked_revs(self, db):
with db.cursor() as cur: with db.cursor() as cur:
@ -304,7 +303,7 @@ class Importer:
self.obs.request(number).import_into_db(db) self.obs.request(number).import_into_db(db)
db.conn.commit() db.conn.commit()
TreeBuilder(db).build(self.package) TreeBuilder(db).build(self.package).print()
def import_all_revisions(self, gc): def import_all_revisions(self, gc):
# Fetch all the requests and sort them. Ideally we should # Fetch all the requests and sort them. Ideally we should

122
lib/tree_builder.py
View File

@ -2,40 +2,126 @@ from lib.db_revision import DBRevision
from lib.request import Request from lib.request import Request
class TreeNode:
"""
Nodes in this "tree" have either no parent (root), one parent (in a chain)
or two parents (in this case the merged revision wins in conflicts).
"""
def __init__(self, rev):
self.parent = None
self.merged = None
self.revision = rev
self.merged_into = None
def print(self):
node = self
while node:
print(node.revision, node.revision.files_hash)
if node.merged:
source_node = node.merged
while source_node:
print(" ", source_node.revision, source_node.revision.files_hash)
source_node = source_node.parent
if source_node and source_node.merged_into:
break
node = node.parent
def as_list(self):
"""Return a list for test cases"""
node = self
ret = []
while node:
repr = {"commit": node.revision.short_string()}
if node.merged:
source_node = node.merged
repr["merged"] = []
while source_node:
repr["merged"].append(source_node.revision.short_string())
source_node = source_node.parent
if source_node and source_node.merged_into:
break
node = node.parent
ret.append(repr)
return ret
class TreeBuilder: class TreeBuilder:
def __init__(self, db): def __init__(self, db):
self.db = db self.db = db
def filtered_revisions(self, project, package): def revisions_chain(self, project, package):
revisions = DBRevision.all_revisions(self.db, project, package) revisions = DBRevision.all_revisions(self.db, project, package)
revisions.sort() revisions.sort()
ret = []
prev = None prev = None
tree = None
for rev in revisions: for rev in revisions:
if rev.broken: if rev.broken:
continue continue
if prev and prev.files_hash == rev.files_hash: if prev and prev.files_hash == rev.files_hash:
continue continue
ret.append(rev)
prev = rev prev = rev
return ret new_tree = TreeNode(rev)
if tree:
new_tree.parent = tree
tree = new_tree
def build(self, package): return tree
factory_revisions = self.filtered_revisions("openSUSE:Factory", package)
def find_merge(self, revision, source_chain):
node = source_chain
while node:
# exclude reverts happening after the merge
if (
node.revision.commit_time <= revision.commit_time
and node.revision.files_hash == revision.files_hash
):
return node
node = node.parent
def add_merge_points(self, factory_revisions):
source_revisions = dict() source_revisions = dict()
for rev in factory_revisions: factory_node = factory_revisions
print(rev, rev.files_hash) while factory_node:
if rev.request_id: if factory_node.revision.request_id:
req = Request.find(self.db, rev.request_id) req = Request.find(self.db, factory_node.revision.request_id)
print(" ", req)
key = f"{req.source_project}/{req.source_package}" key = f"{req.source_project}/{req.source_package}"
if key not in source_revisions: if key not in source_revisions:
source_revisions[key] = self.filtered_revisions( source_revisions[key] = self.revisions_chain(
req.source_project, req.source_package req.source_project, req.source_package
) )
for rev2 in source_revisions.get(key): factory_node.merged = self.find_merge(
# this happened after the fact - possibly a revert factory_node.revision, source_revisions[key]
if rev2.commit_time > rev.commit_time: )
continue # add a reverse lookup
if rev2.files_hash == rev.files_hash: if factory_node.merged:
print(" ", rev2, rev2.files_hash) factory_node.merged.merged_into = factory_node
factory_node = factory_node.parent
def prune_loose_end(self, factory_node):
"""Look for source revisions that end in a new root and prune them"""
last_merge = None
while factory_node:
if factory_node.merged:
source_node = factory_node.merged
ended_without_merge = False
while source_node:
source_node = source_node.parent
if source_node and source_node.merged_into:
ended_without_merge = True
break
if not ended_without_merge:
factory_node.merged = None
if last_merge:
last_merge.parent = None
else:
last_merge = factory_node.merged
factory_node = factory_node.parent
def build(self, package):
factory_revisions = self.revisions_chain("openSUSE:Factory", package)
self.add_merge_points(factory_revisions)
self.prune_loose_end(factory_revisions)
return factory_revisions

300
tests/fixtures/zsh-expected-tree.yaml vendored Normal file
View File

@ -0,0 +1,300 @@
- commit: Rev openSUSE:Factory/zsh/98.0
merged:
- Rev shells/zsh/236.0
- commit: Rev openSUSE:Factory/zsh/97.0
merged:
- Rev shells/zsh/234.0
- commit: Rev openSUSE:Factory/zsh/96.0
merged:
- Rev shells/zsh/232.0
- commit: Rev openSUSE:Factory/zsh/95.0
merged:
- Rev shells/zsh/230.0
- commit: Rev openSUSE:Factory/zsh/94.0
merged:
- Rev shells/zsh/228.0
- Rev shells/zsh/227.0
- commit: Rev openSUSE:Factory/zsh/93.0
merged:
- Rev shells/zsh/225.0
- Rev shells/zsh/224.0
- Rev shells/zsh/223.0
- commit: Rev openSUSE:Factory/zsh/92.0
merged:
- Rev shells/zsh/221.0
- commit: Rev openSUSE:Factory/zsh/91.0
merged:
- Rev shells/zsh/219.0
- commit: Rev openSUSE:Factory/zsh/90.0
merged:
- Rev shells/zsh/217.0
- commit: Rev openSUSE:Factory/zsh/89.0
merged:
- Rev shells/zsh/215.0
- Rev shells/zsh/214.0
- commit: Rev openSUSE:Factory/zsh/88.0
merged:
- Rev shells/zsh/212.0
- commit: Rev openSUSE:Factory/zsh/87.0
merged:
- Rev shells/zsh/210.0
- Rev shells/zsh/209.0
- Rev shells/zsh/208.0
- Rev shells/zsh/207.0
- commit: Rev openSUSE:Factory/zsh/86.0
merged:
- Rev shells/zsh/205.0
- Rev shells/zsh/204.0
- commit: Rev openSUSE:Factory/zsh/85.0
merged:
- Rev shells/zsh/202.0
- Rev shells/zsh/201.0
- commit: Rev openSUSE:Factory/zsh/84.0
merged:
- Rev shells/zsh/199.0
- Rev shells/zsh/198.0
- Rev shells/zsh/197.0
- commit: Rev openSUSE:Factory/zsh/83.0
merged:
- Rev shells/zsh/195.0
- Rev shells/zsh/194.0
- Rev shells/zsh/193.0
- Rev shells/zsh/192.0
- Rev shells/zsh/191.0
- commit: Rev openSUSE:Factory/zsh/82.0
merged:
- Rev shells/zsh/189.0
- commit: Rev openSUSE:Factory/zsh/81.0
merged:
- Rev shells/zsh/187.0
- Rev shells/zsh/186.0
- commit: Rev openSUSE:Factory/zsh/80.0
merged:
- Rev shells/zsh/184.0
- commit: Rev openSUSE:Factory/zsh/79.0
merged:
- Rev shells/zsh/182.0
- commit: Rev openSUSE:Factory/zsh/78.0
merged:
- Rev shells/zsh/180.0
- commit: Rev openSUSE:Factory/zsh/77.0
merged:
- Rev shells/zsh/178.0
- Rev shells/zsh/177.0
- commit: Rev openSUSE:Factory/zsh/76.0
merged:
- Rev shells/zsh/175.0
- commit: Rev openSUSE:Factory/zsh/75.0
merged:
- Rev shells/zsh/173.0
- commit: Rev openSUSE:Factory/zsh/74.0
merged:
- Rev shells/zsh/171.0
- Rev shells/zsh/170.0
- commit: Rev openSUSE:Factory/zsh/73.0
merged:
- Rev shells/zsh/168.0
- commit: Rev openSUSE:Factory/zsh/72.0
merged:
- Rev shells/zsh/166.0
- commit: Rev openSUSE:Factory/zsh/71.0
merged:
- Rev shells/zsh/164.0
- commit: Rev openSUSE:Factory/zsh/70.0
merged:
- Rev shells/zsh/162.0
- commit: Rev openSUSE:Factory/zsh/69.0
merged:
- Rev shells/zsh/160.0
- Rev shells/zsh/159.0
- Rev shells/zsh/158.0
- commit: Rev openSUSE:Factory/zsh/68.0
merged:
- Rev shells/zsh/156.0
- commit: Rev openSUSE:Factory/zsh/67.0
merged:
- Rev shells/zsh/154.0
- Rev shells/zsh/153.0
- commit: Rev openSUSE:Factory/zsh/66.0
merged:
- Rev shells/zsh/151.0
- Rev shells/zsh/150.0
- commit: Rev openSUSE:Factory/zsh/65.0
merged:
- Rev shells/zsh/148.0
- commit: Rev openSUSE:Factory/zsh/64.0
merged:
- Rev shells/zsh/146.0
- Rev shells/zsh/145.0
- Rev shells/zsh/144.0
- commit: Rev openSUSE:Factory/zsh/63.0
merged:
- Rev shells/zsh/142.0
- commit: Rev openSUSE:Factory/zsh/62.0
merged:
- Rev shells/zsh/140.0
- commit: Rev openSUSE:Factory/zsh/61.0
merged:
- Rev shells/zsh/138.0
- Rev shells/zsh/137.0
- commit: Rev openSUSE:Factory/zsh/60.0
merged:
- Rev shells/zsh/135.0
- commit: Rev openSUSE:Factory/zsh/59.0
merged:
- Rev shells/zsh/133.0
- commit: Rev openSUSE:Factory/zsh/57.0
merged:
- Rev shells/zsh/131.0
- commit: Rev openSUSE:Factory/zsh/56.0
merged:
- Rev shells/zsh/129.0
- commit: Rev openSUSE:Factory/zsh/55.0
merged:
- Rev shells/zsh/127.0
- commit: Rev openSUSE:Factory/zsh/54.0
merged:
- Rev shells/zsh/125.0
- commit: Rev openSUSE:Factory/zsh/53.0
merged:
- Rev shells/zsh/123.0
- commit: Rev openSUSE:Factory/zsh/51.0
merged:
- Rev shells/zsh/121.0
- commit: Rev openSUSE:Factory/zsh/50.0
merged:
- Rev shells/zsh/119.0
- commit: Rev openSUSE:Factory/zsh/49.0
merged:
- Rev shells/zsh/117.0
- commit: Rev openSUSE:Factory/zsh/48.0
merged:
- Rev shells/zsh/115.0
- commit: Rev openSUSE:Factory/zsh/46.0
merged:
- Rev shells/zsh/113.0
- commit: Rev openSUSE:Factory/zsh/45.0
merged:
- Rev shells/zsh/111.0
- commit: Rev openSUSE:Factory/zsh/44.0
merged:
- Rev shells/zsh/109.0
- commit: Rev openSUSE:Factory/zsh/43.0
merged:
- Rev shells/zsh/107.0
- commit: Rev openSUSE:Factory/zsh/42.0
merged:
- Rev shells/zsh/105.0
- commit: Rev openSUSE:Factory/zsh/41.0
merged:
- Rev shells/zsh/103.0
- Rev shells/zsh/102.0
- commit: Rev openSUSE:Factory/zsh/39.0
merged:
- Rev shells/zsh/100.0
- commit: Rev openSUSE:Factory/zsh/38.0
merged:
- Rev shells/zsh/98.0
- commit: Rev openSUSE:Factory/zsh/37.0
merged:
- Rev shells/zsh/95.0
- Rev shells/zsh/94.0
- Rev shells/zsh/91.0
- commit: Rev openSUSE:Factory/zsh/36.0
merged:
- Rev shells/zsh/90.0
- Rev shells/zsh/89.0
- Rev shells/zsh/88.0
- Rev shells/zsh/87.0
- commit: Rev openSUSE:Factory/zsh/35.0
merged:
- Rev shells/zsh/84.0
- Rev shells/zsh/83.0
- Rev shells/zsh/82.0
- Rev shells/zsh/81.0
- Rev shells/zsh/80.0
- commit: Rev openSUSE:Factory/zsh/34.0
merged:
- Rev shells/zsh/77.0
- Rev shells/zsh/76.032
- commit: Rev openSUSE:Factory/zsh/32.0
- commit: Rev openSUSE:Factory/zsh/31.0
merged:
- Rev shells/zsh/75.0
- Rev shells/zsh/74.0
- Rev shells/zsh/73.0
- Rev shells/zsh/72.03
- commit: Rev openSUSE:Factory/zsh/30.0
- commit: Rev openSUSE:Factory/zsh/29.0
merged:
- Rev shells/zsh/71.0
- commit: Rev openSUSE:Factory/zsh/28.0
merged:
- Rev shells/zsh/69.0
- Rev shells/zsh/68.0
- Rev shells/zsh/67.027
- commit: Rev openSUSE:Factory/zsh/27.0
- commit: Rev openSUSE:Factory/zsh/26.0
merged:
- Rev shells/zsh/66.0
- Rev shells/zsh/65.0
- Rev shells/zsh/64.0
- Rev shells/zsh/63.0
- Rev shells/zsh/62.0
- Rev shells/zsh/61.0
- Rev shells/zsh/60.0
- Rev shells/zsh/59.0
- Rev shells/zsh/58.0
- Rev shells/zsh/57.0
- Rev shells/zsh/56.0
- Rev shells/zsh/55.0
- Rev shells/zsh/54.0
- Rev shells/zsh/53.0
- Rev shells/zsh/52.0
- Rev shells/zsh/51.0
- Rev shells/zsh/50.0
- Rev shells/zsh/49.0
- Rev shells/zsh/48.0
- Rev shells/zsh/47.0
- Rev shells/zsh/46.025
- commit: Rev openSUSE:Factory/zsh/25.0
- commit: Rev openSUSE:Factory/zsh/24.0
merged:
- Rev shells/zsh/45.0
- Rev shells/zsh/44.0
- Rev shells/zsh/43.023
- commit: Rev openSUSE:Factory/zsh/23.0
- commit: Rev openSUSE:Factory/zsh/22.0
merged:
- Rev shells/zsh/42.0
- Rev shells/zsh/41.035
- commit: Rev openSUSE:Factory/zsh/18.0
- commit: Rev openSUSE:Factory/zsh/17.0
merged:
- Rev shells/zsh/41.017
- Rev shells/zsh/41.0
- Rev shells/zsh/40.0
- Rev shells/zsh/39.0
- Rev shells/zsh/38.0
- Rev shells/zsh/37.0
- commit: Rev openSUSE:Factory/zsh/16.0
- commit: Rev openSUSE:Factory/zsh/14.0
merged:
- Rev shells/zsh/36.014
- Rev shells/zsh/36.0
- Rev shells/zsh/35.0
- commit: Rev openSUSE:Factory/zsh/13.0
merged:
- Rev shells/zsh/34.0
- commit: Rev openSUSE:Factory/zsh/12.0
- commit: Rev openSUSE:Factory/zsh/11.0
- commit: Rev openSUSE:Factory/zsh/10.0
- commit: Rev openSUSE:Factory/zsh/9.0
- commit: Rev openSUSE:Factory/zsh/8.0
- commit: Rev openSUSE:Factory/zsh/7.0
- commit: Rev openSUSE:Factory/zsh/6.0
- commit: Rev openSUSE:Factory/zsh/5.0
- commit: Rev openSUSE:Factory/zsh/4.0
- commit: Rev openSUSE:Factory/zsh/3.0
- commit: Rev openSUSE:Factory/zsh/2.0
- commit: Rev openSUSE:Factory/zsh/1.0

13
tests/tree_test.py
View File

@ -18,7 +18,18 @@ class TestTreeMethods(unittest.TestCase):
DBRevision.import_fixture_dict(self.db, rev) DBRevision.import_fixture_dict(self.db, rev)
def test_create_tree(self): def test_create_tree(self):
TreeBuilder(self.db).build("zsh") revisions = TreeBuilder(self.db).build("zsh")
if False:
import sys
yaml.dump(revisions.as_list(), sys.stdout)
path = os.path.join(
os.path.dirname(__file__), "fixtures/zsh-expected-tree.yaml"
)
with open(path, "r") as f:
zsh_data = yaml.safe_load(f)
self.assertEqual(zsh_data, revisions.as_list())
if __name__ == "__main__": if __name__ == "__main__":