diff --git a/lib/db_revision.py b/lib/db_revision.py index 7c21727..4b23324 100644 --- a/lib/db_revision.py +++ b/lib/db_revision.py @@ -1,5 +1,6 @@ from hashlib import md5 +from lib.db import DB from lib.request import Request @@ -202,6 +203,24 @@ class DBRevision: self._files.sort(key=lambda x: x["name"]) return self._files + def calc_delta(self, db: DB, current_rev=None): + """Calculate the list of files to download and to delete""" + old_files = dict() + to_download = [] + to_delete = [] + if current_rev: + for entry in current_rev.files_list(db): + old_files[entry["name"]] = f"{entry['md5']}-{entry['size']}" + for entry in self.files_list(db): + if old_files.get(entry["name"], "") != f"{entry['md5']}-{entry['size']}": + print("Download", entry["name"]) + to_download.append(entry["name"]) + old_files.pop(entry["name"], None) + for entry in old_files.keys(): + print("Delete", entry) + to_delete.append(entry) + return to_download, to_delete + @staticmethod def requests_to_fetch(db): with db.cursor() as cur: diff --git a/lib/importer.py b/lib/importer.py index 726d1ce..e6bd17e 100644 --- a/lib/importer.py +++ b/lib/importer.py @@ -278,9 +278,11 @@ class Importer: return f"{self.branch} c:{self.commit.short_string()}{p1_str}{p2_str}" class FlatTreeWalker(AbstractWalker): - def __init__(self) -> None: + def __init__(self, rebase_devel) -> None: super().__init__() self.flats = [] + # the rebase_devel won't work as such as rebasing the branch needs an explicit action + self.rebase_devel = rebase_devel # remember the last merge point so we can know the parent of it for the root of the sources self.last_merge = None @@ -288,7 +290,7 @@ class Importer: self.flats.append(FlatNode(branch, commit, parent1, parent2)) def handle_source_node(self, node) -> None: - if node.parent and node.parent.merged_into and False: + if self.rebase_devel and node.parent and node.parent.merged_into: self.add("devel", node.revision, node.parent.merged_into.revision) return if node.parent: @@ -312,18 +314,40 @@ class Importer: self.last_merge = node - ftw = FlatTreeWalker() + ftw = FlatTreeWalker(self.rebase_devel) tree.walk(ftw) + branch_state = {"factory": None, "devel": None} for flat in reversed(ftw.flats): - self.commit_flat(flat) + self.commit_flat(db, flat, branch_state) - def commit_flat(self, flat): + def limit_download(self, file): + if file.endswith(".spec") or file.endswith(".changes"): + return True + return False + + def commit_flat(self, db, flat, branch_state): parents = [] self.git.checkout(flat.branch) if flat.parent1: parents.append(flat.parent1.git_commit) if flat.parent2: parents.append(flat.parent2.git_commit) + to_download, to_delete = flat.commit.calc_delta(db, branch_state[flat.branch]) + for file in to_delete: + if not self.limit_download(file): + continue + self.git.remove(file) + for file in to_download: + if not self.limit_download(file): + continue + self.obs.download( + flat.commit.project, + flat.commit.package, + file, + flat.commit.expanded_srcmd5, + self.git.path, + ) + self.git.add(file) commit = self.git.commit( f"OBS User {flat.commit.userid}", @@ -335,6 +359,7 @@ class Importer: parents=parents, ) flat.commit.git_commit = commit + branch_state[flat.branch] = flat.commit def import_into_db(self): db = DB() diff --git a/lib/obs.py b/lib/obs.py index 51e72e3..217e915 100644 --- a/lib/obs.py +++ b/lib/obs.py @@ -151,7 +151,9 @@ class OBS: ) return osc.core.http_GET(url) - def download(self, project, package, name, revision, dirpath): + def download( + self, project: str, package: str, name: str, revision: str, dirpath: str + ) -> None: with (dirpath / name).open("wb") as f: f.write(self._download(project, package, name, revision).read())