Merge pull request 'No longer rely on external service for LFS tracking' (#18) from add_lfs into main

Reviewed-on: https://gitea.opensuse.org/importers/git-importer/pulls/18

lib/git_exporter.py

@@ -17,7 +17,8 @@ class GitExporter:
         self.obs = OBS(api_url)
         self.project = project
         self.package = package
-        self.proxy_sha256 = ProxySHA256(self.obs, enabled=True)
+        self.db = DB()
+        self.proxy_sha256 = ProxySHA256(self.obs, self.db)
         self.git = Git(
             repodir / package,
             committer="Git OBS Bridge",
@@ -59,8 +60,7 @@ class GitExporter:
         return left_to_commit
 
     def export_as_git(self):
-        db = DB()
-        tree = TreeBuilder(db).build(self.project, self.package)
+        tree = TreeBuilder(self.db).build(self.project, self.package)
         flats = tree.as_flat_list()
 
         branch_state = {"factory": None, "devel": None}
@@ -75,9 +75,8 @@ class GitExporter:
 
         for flat in left_to_commit:
             if flat.commit.userid not in users:
-                users[flat.commit.userid] = User(db, flat.commit.userid)
+                users[flat.commit.userid] = User.find(self.db, flat.commit.userid)
             flat.user = users[flat.commit.userid]
-            logging.debug(f"USER {flat.user}")
             self.gc_cnt -= 1
             if self.gc_cnt <= 0 and self.gc_interval:
                 self.run_gc()
@@ -90,10 +89,14 @@ class GitExporter:
         self.gc_cnt = self.gc_interval
         self.git.gc()
 
+    def is_lfs_file(self, package, filename, size):
+        if not is_binary_or_large(filename, size):
+            return False
+        return not self.proxy_sha256.is_text(package, filename)
+
     def commit_file(self, flat, file, size, md5):
         # have such files been detected as text mimetype before?
-        is_text = self.proxy_sha256.is_text(flat.commit.package, file.name)
-        if not is_text and is_binary_or_large(file.name, size):
+        if self.is_lfs_file(flat.commit.package, file.name, size):
             file_sha256 = self.proxy_sha256.get_or_put(
                 flat.commit.project,
                 flat.commit.package,
@@ -102,19 +105,24 @@ class GitExporter:
                 md5,
                 size,
             )
-            self.git.add_lfs(file.name, file_sha256["sha256"], size)
-        else:
-            self.obs.change_url(flat.commit.api_url)
-            self.obs.download(
-                flat.commit.project,
-                flat.commit.package,
-                file.name,
-                flat.commit.expanded_srcmd5,
-                self.git.path,
-                self.cachedir,
-                file_md5=md5,
-            )
-            self.git.add(file)
+            # as it's newly registered, it might be a text file now, so double check
+            if not self.proxy_sha256.is_text(flat.commit.package, file.name):
+                self.git.add_lfs(file.name, file_sha256, size)
+                return
+        self.commit_non_lfs_file(flat, file, md5)
+
+    def commit_non_lfs_file(self, flat, file, md5):
+        self.obs.change_url(flat.commit.api_url)
+        self.obs.download(
+            flat.commit.project,
+            flat.commit.package,
+            file.name,
+            flat.commit.expanded_srcmd5,
+            self.git.path,
+            self.cachedir,
+            file_md5=md5,
+        )
+        self.git.add(file)
 
     def branch_fits_parent1(self, flat, branch_state):
         if branch_state[flat.branch] is None: