Reuse the repository directory by storing a state yaml

Not using the database for that so that removing the repository directory will
automatically recreate it

git-importer.py

@@ -3,7 +3,6 @@
 import argparse
 import logging
 import pathlib
-import shutil
 import sys
 
 import osc.core
@@ -52,12 +51,6 @@ def main():
         type=pathlib.Path,
         help="Local git repository directory",
     )
-    parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        help="If the repository directory exists, remove it",
-    )
     parser.add_argument(
         "-a",
         "--search-ancestor",
@@ -116,13 +109,6 @@ def main():
     if not args.repodir:
         args.repodir = pathlib.Path(args.package)
 
-    if args.repodir.exists() and not args.force:
-        print(f"Repository {args.repodir} already present")
-        sys.exit(-1)
-    elif args.repodir.exists() and args.force:
-        logging.info(f"Removing old repository {args.repodir}")
-        shutil.rmtree(args.repodir)
-
     # TODO: use a CLI parameter to describe the projects
     importer = Importer(
         PROJECTS, args.package, args.repodir, args.search_ancestor, args.rebase_devel

lib/git.py

@@ -181,6 +181,9 @@ class Git:
         except:
             return None
 
+    def branch_head(self, branch):
+        return self.repo.references["refs/heads/" + branch].target
+
     def gc(self):
         logging.info(f"Garbage recollec and repackage {self.path}")
         subprocess.run(

lib/importer.py

@@ -1,7 +1,10 @@
 import functools
 import logging
+import os
 import xml.etree.ElementTree as ET
 
+import yaml
+
 from lib.binary import is_binary_or_large
 from lib.db import DB
 from lib.db_revision import DBRevision
@@ -55,6 +58,7 @@ class Importer:
             committer="Git OBS Bridge",
             committer_email="obsbridge@suse.de",
         ).create()
+        self.state_file = os.path.join(self.git.path, ".git", "_flat_state.yaml")
         self.proxy_sha256 = ProxySHA256(self.obs, enabled=True)
 
         self.history = History(self.obs, self.package)
@@ -278,6 +282,9 @@ class Importer:
                 return f"{self.branch} c:{self.commit.short_string()}{p1_str}{p2_str}"
 
         class FlatTreeWalker(AbstractWalker):
+            """While walking the tree, record the commits to do one after the other. These
+            FlatNodes are in the end in the flats array."""
+
             def __init__(self, rebase_devel) -> None:
                 super().__init__()
                 self.flats = []
@@ -317,7 +324,28 @@ class Importer:
         ftw = FlatTreeWalker(self.rebase_devel)
         tree.walk(ftw)
         branch_state = {"factory": None, "devel": None}
+        state_data = dict()
+        if os.path.exists(self.state_file):
+            with open(self.state_file, "r") as f:
+                state_data = yaml.safe_load(f)
+                if type(state_data) != dict:
+                    state_data = {}
+        left_to_commit = []
         for flat in reversed(ftw.flats):
+            found_state = False
+            for branch in ["factory", "devel"]:
+                if flat.commit.dbid == state_data.get(branch):
+                    branch_state[branch] = flat.commit
+                    flat.commit.git_commit = self.git.branch_head(branch)
+                    logging.debug(
+                        f"Found {self.git.path}'s {branch} branch in state {flat}"
+                    )
+                    left_to_commit = []
+                    found_state = True
+            if not found_state:
+                left_to_commit.append(flat)
+        for flat in left_to_commit:
+            logging.debug(f"Committing {flat}")
             self.commit_flat(db, flat, branch_state)
 
     def limit_download(self, file):
@@ -360,6 +388,13 @@ class Importer:
         )
         flat.commit.git_commit = commit
         branch_state[flat.branch] = flat.commit
+        with open(self.state_file, "w") as f:
+            data = {}
+            for branch in ["factory", "devel"]:
+                commit = branch_state[branch]
+                if commit:
+                    data[branch] = commit.dbid
+            yaml.dump(data, f)
 
     def import_into_db(self):
         db = DB()