Create the git repos in multiple processes

Threads appear to be too dangerous for this

Makefile

@@ -1,7 +1,7 @@
 all:
 	isort *.py lib/*py tests/*py
-	autoflake -r --in-place --remove-unused-variables .
-	black .
+	autoflake --in-place --remove-unused-variables *.py lib/*py tests/*py
+	black *.py lib/*py tests/*py
 
 test:
 	python3 -m unittest -v tests/*.py

git-importer.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 
 import argparse
+import concurrent.futures
 import logging
 import pathlib
 import sys
@@ -42,6 +43,12 @@ PROJECTS = [
 ]
 
 
+def export_package(package, repodir, cachedir, gc):
+    exporter = GitExporter(URL_OBS, "openSUSE:Factory", package, repodir, cachedir)
+    exporter.set_gc_interval(gc)
+    exporter.export_as_git()
+
+
 def main():
     parser = argparse.ArgumentParser(description="OBS history importer into git")
     parser.add_argument("packages", help="OBS package names", nargs="*")
@@ -106,12 +113,14 @@ def main():
 
     importer = Importer(URL_OBS, "openSUSE:Factory", args.packages)
     importer.import_into_db()
-    for package in args.packages:
-        exporter = GitExporter(
-            URL_OBS, "openSUSE:Factory", package, args.repodir, args.cachedir
-        )
-        exporter.set_gc_interval(args.gc)
-        exporter.export_as_git()
+    with concurrent.futures.ProcessPoolExecutor() as executor:
+        fs = [
+            executor.submit(
+                export_package, package, args.repodir, args.cachedir, args.gc
+            )
+            for package in args.packages
+        ]
+        concurrent.futures.wait(fs)
 
 
 if __name__ == "__main__":

lib/proxy_sha256.py

@@ -41,7 +41,9 @@ class ProxySHA256:
             self.hashes = dict()
             return
         logging.debug("Retrieve all previously defined SHA256")
-        response = requests.get(f"http://source.dyn.cloud.suse.de/package/{package}")
+        response = requests.get(
+            f"http://source.dyn.cloud.suse.de/package/{package}", timeout=5
+        )
         if response.status_code == 200:
             json = response.json()
             self.hashes = json["shas"]
@@ -67,6 +69,7 @@ class ProxySHA256:
                 "url": url,
                 "package": package,
             },
+            timeout=10,
         )
         if response.status_code != 200:
             raise Exception(f"Redirector error on {self.url} for {url}")