Download .spec and .changes file only for now

lib/db_revision.py

@@ -1,5 +1,6 @@
 from hashlib import md5
 
+from lib.db import DB
 from lib.request import Request
 
 
@@ -202,6 +203,24 @@ class DBRevision:
         self._files.sort(key=lambda x: x["name"])
         return self._files
 
+    def calc_delta(self, db: DB, current_rev=None):
+        """Calculate the list of files to download and to delete"""
+        old_files = dict()
+        to_download = []
+        to_delete = []
+        if current_rev:
+            for entry in current_rev.files_list(db):
+                old_files[entry["name"]] = f"{entry['md5']}-{entry['size']}"
+        for entry in self.files_list(db):
+            if old_files.get(entry["name"], "") != f"{entry['md5']}-{entry['size']}":
+                print("Download", entry["name"])
+                to_download.append(entry["name"])
+            old_files.pop(entry["name"], None)
+        for entry in old_files.keys():
+            print("Delete", entry)
+            to_delete.append(entry)
+        return to_download, to_delete
+
     @staticmethod
     def requests_to_fetch(db):
         with db.cursor() as cur:

lib/importer.py

@@ -278,9 +278,11 @@ class Importer:
                 return f"{self.branch} c:{self.commit.short_string()}{p1_str}{p2_str}"
 
         class FlatTreeWalker(AbstractWalker):
-            def __init__(self) -> None:
+            def __init__(self, rebase_devel) -> None:
                 super().__init__()
                 self.flats = []
+                # the rebase_devel won't work as such as rebasing the branch needs an explicit action
+                self.rebase_devel = rebase_devel
                 # remember the last merge point so we can know the parent of it for the root of the sources
                 self.last_merge = None
 
@@ -288,7 +290,7 @@ class Importer:
                 self.flats.append(FlatNode(branch, commit, parent1, parent2))
 
             def handle_source_node(self, node) -> None:
-                if node.parent and node.parent.merged_into and False:
+                if self.rebase_devel and node.parent and node.parent.merged_into:
                     self.add("devel", node.revision, node.parent.merged_into.revision)
                     return
                 if node.parent:
@@ -312,18 +314,40 @@ class Importer:
 
                 self.last_merge = node
 
-        ftw = FlatTreeWalker()
+        ftw = FlatTreeWalker(self.rebase_devel)
         tree.walk(ftw)
+        branch_state = {"factory": None, "devel": None}
         for flat in reversed(ftw.flats):
-            self.commit_flat(flat)
+            self.commit_flat(db, flat, branch_state)
 
-    def commit_flat(self, flat):
+    def limit_download(self, file):
+        if file.endswith(".spec") or file.endswith(".changes"):
+            return True
+        return False
+
+    def commit_flat(self, db, flat, branch_state):
         parents = []
         self.git.checkout(flat.branch)
         if flat.parent1:
             parents.append(flat.parent1.git_commit)
         if flat.parent2:
             parents.append(flat.parent2.git_commit)
+        to_download, to_delete = flat.commit.calc_delta(db, branch_state[flat.branch])
+        for file in to_delete:
+            if not self.limit_download(file):
+                continue
+            self.git.remove(file)
+        for file in to_download:
+            if not self.limit_download(file):
+                continue
+            self.obs.download(
+                flat.commit.project,
+                flat.commit.package,
+                file,
+                flat.commit.expanded_srcmd5,
+                self.git.path,
+            )
+            self.git.add(file)
 
         commit = self.git.commit(
             f"OBS User {flat.commit.userid}",
@@ -335,6 +359,7 @@ class Importer:
             parents=parents,
         )
         flat.commit.git_commit = commit
+        branch_state[flat.branch] = flat.commit
 
     def import_into_db(self):
         db = DB()

lib/obs.py

@@ -151,7 +151,9 @@ class OBS:
         )
         return osc.core.http_GET(url)
 
-    def download(self, project, package, name, revision, dirpath):
+    def download(
+        self, project: str, package: str, name: str, revision: str, dirpath: str
+    ) -> None:
         with (dirpath / name).open("wb") as f:
             f.write(self._download(project, package, name, revision).read())