From 33a5733cb98eba090a8d231de9120a7ada4be745 Mon Sep 17 00:00:00 2001
From: Stephan Kulow <coolo@suse.de>
Date: Fri, 4 Nov 2022 07:48:17 +0100
Subject: [PATCH] Create the git repos in multiple processes

Threads appear to be too dangerous for this
---
 Makefile            |  4 ++--
 git-importer.py     | 21 +++++++++++++++------
 lib/proxy_sha256.py |  5 ++++-
 3 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/Makefile b/Makefile
index 3171f83..050b045 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 all:
 	isort *.py lib/*py tests/*py
-	autoflake -r --in-place --remove-unused-variables .
-	black .
+	autoflake --in-place --remove-unused-variables *.py lib/*py tests/*py
+	black *.py lib/*py tests/*py
 
 test:
 	python3 -m unittest -v tests/*.py
diff --git a/git-importer.py b/git-importer.py
index 0d7d98f..a40d04e 100755
--- a/git-importer.py
+++ b/git-importer.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 
 import argparse
+import concurrent.futures
 import logging
 import pathlib
 import sys
@@ -42,6 +43,12 @@ PROJECTS = [
 ]
 
 
+def export_package(package, repodir, cachedir, gc):
+    exporter = GitExporter(URL_OBS, "openSUSE:Factory", package, repodir, cachedir)
+    exporter.set_gc_interval(gc)
+    exporter.export_as_git()
+
+
 def main():
     parser = argparse.ArgumentParser(description="OBS history importer into git")
     parser.add_argument("packages", help="OBS package names", nargs="*")
@@ -106,12 +113,14 @@ def main():
 
     importer = Importer(URL_OBS, "openSUSE:Factory", args.packages)
     importer.import_into_db()
-    for package in args.packages:
-        exporter = GitExporter(
-            URL_OBS, "openSUSE:Factory", package, args.repodir, args.cachedir
-        )
-        exporter.set_gc_interval(args.gc)
-        exporter.export_as_git()
+    with concurrent.futures.ProcessPoolExecutor() as executor:
+        fs = [
+            executor.submit(
+                export_package, package, args.repodir, args.cachedir, args.gc
+            )
+            for package in args.packages
+        ]
+        concurrent.futures.wait(fs)
 
 
 if __name__ == "__main__":
diff --git a/lib/proxy_sha256.py b/lib/proxy_sha256.py
index 2864351..8513b04 100644
--- a/lib/proxy_sha256.py
+++ b/lib/proxy_sha256.py
@@ -41,7 +41,9 @@ class ProxySHA256:
             self.hashes = dict()
             return
         logging.debug("Retrieve all previously defined SHA256")
-        response = requests.get(f"http://source.dyn.cloud.suse.de/package/{package}")
+        response = requests.get(
+            f"http://source.dyn.cloud.suse.de/package/{package}", timeout=5
+        )
         if response.status_code == 200:
             json = response.json()
             self.hashes = json["shas"]
@@ -67,6 +69,7 @@ class ProxySHA256:
                 "url": url,
                 "package": package,
             },
+            timeout=10,
         )
         if response.status_code != 200:
             raise Exception(f"Redirector error on {self.url} for {url}")