Download the full revision

lib/db_revision.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import logging
 from hashlib import md5
-from pathlib import PurePath
+from pathlib import Path
 from typing import Optional
 
 from lib.db import DB
@@ -232,11 +232,11 @@ class DBRevision:
         for entry in self.files_list():
             if old_files.get(entry["name"]) != f"{entry['md5']}-{entry['size']}":
                 logging.debug(f"Download {entry['name']}")
-                to_download.append((PurePath(entry["name"]), entry["md5"]))
+                to_download.append((Path(entry["name"]), entry["size"], entry["md5"]))
             old_files.pop(entry["name"], None)
         for entry in old_files.keys():
             logging.debug(f"Delete {entry}")
-            to_delete.append(PurePath(entry))
+            to_delete.append(Path(entry))
         return to_download, to_delete
 
     @staticmethod

lib/git_exporter.py

@@ -1,6 +1,5 @@
 import logging
 import os
-from pathlib import Path
 
 import yaml
 
@@ -42,44 +41,10 @@ class GitExporter:
         # Download each file in OBS if it is not a binary (or large)
         # file
         for (name, size, file_md5) in obs_files:
-            # this file creates easily 100k commits and is just useless data :(
+            # Validate the MD5 of the downloaded file
-            # unfortunately it's stored in the same meta package as the project config
+            if md5(self.git.path / name) != file_md5:
-            if revision.package == "_project" and name == "_staging_workflow":
+                raise Exception(f"Download error in {name}")
-                continue
+            self.git.add(name)
-            # have such files been detected as text mimetype before?
-            is_text = self.proxy_sha256.is_text(name)
-            if not is_text and is_binary_or_large(name, size):
-                file_sha256 = self.proxy_sha256.get_or_put(
-                    revision.project,
-                    revision.package,
-                    name,
-                    revision.srcmd5,
-                    file_md5,
-                    size,
-                )
-                self.git.add_lfs(name, file_sha256["sha256"], size)
-            else:
-                if (name, size, file_md5) not in git_files:
-                    logging.debug(f"Download {name}")
-                    self.obs.download(
-                        revision.project,
-                        revision.package,
-                        name,
-                        revision.srcmd5,
-                        self.git.path,
-                        file_md5=file_md5,
-                    )
-                    # Validate the MD5 of the downloaded file
-                    if md5(self.git.path / name) != file_md5:
-                        raise Exception(f"Download error in {name}")
-                    self.git.add(name)
-
-        # Remove extra files
-        obs_names = {n for (n, _, _) in obs_files}
-        git_names = {n for (n, _, _) in git_files}
-        for name in git_names - obs_names:
-            logging.debug(f"Remove {name}")
-            self.git.remove(name)
 
     def set_gc_interval(self, gc):
         self.gc_interval = gc
@@ -122,36 +87,43 @@ class GitExporter:
             logging.debug(f"Committing {flat}")
             self.commit_flat(flat, branch_state)
 
-    def limit_download(self, file: Path):
-        return file.suffix in (".spec", ".changes")
-
     def commit_flat(self, flat, branch_state):
         parents = []
         self.git.checkout(flat.branch)
+
+        # Overwrite ".gitattributes" with the
+        self.git.add_default_lfs_gitattributes(force=True)
+
         if flat.parent1:
             parents.append(flat.parent1.git_commit)
         if flat.parent2:
             parents.append(flat.parent2.git_commit)
         to_download, to_delete = flat.commit.calc_delta(branch_state[flat.branch])
         for file in to_delete:
-            if not self.limit_download(file):
-                continue
             self.git.remove(file)
-        for file, md5 in to_download:
+        for file, size, md5 in to_download:
-            if not self.limit_download(file):
+            # have such files been detected as text mimetype before?
-                continue
+            is_text = self.proxy_sha256.is_text(file.name)
-            self.obs.download(
+            if not is_text and is_binary_or_large(file.name, size):
-                flat.commit.project,
+                file_sha256 = self.proxy_sha256.get_or_put(
-                flat.commit.package,
+                    flat.commit.project,
-                file.name,
+                    flat.commit.package,
-                flat.commit.expanded_srcmd5,
+                    file.name,
-                self.git.path,
+                    flat.commit.expanded_srcmd5,
-                file_md5=md5,
+                    md5,
-            )
+                    size,
-            self.git.add(file)
+                )
-
+                self.git.add_lfs(file.name, file_sha256["sha256"], size)
-        # Overwrite ".gitattributes" with the
+            else:
-        self.git.add_default_lfs_gitattributes(force=True)
+                self.obs.download(
+                    flat.commit.project,
+                    flat.commit.package,
+                    file.name,
+                    flat.commit.expanded_srcmd5,
+                    self.git.path,
+                    file_md5=md5,
+                )
+                self.git.add(file)
 
         commit = self.git.commit(
             f"OBS User {flat.commit.userid}",