adamm/git-importer
1
0
Fork 1
forked from importers/git-importer
Code Pull Requests Activity

Create the git repos in multiple processes

Browse Source 
Threads appear to be too dangerous for this
This commit is contained in:
Stephan Kulow 2022-11-04 07:48:17 +01:00
parent d21ce571f5
commit 33a5733cb9
3 changed files with 21 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -1,7 +1,7 @@
all: all:
isort *.py lib/*py tests/*py isort *.py lib/*py tests/*py
autoflake -r --in-place --remove-unused-variables . autoflake --in-place --remove-unused-variables *.py lib/*py tests/*py
black . black *.py lib/*py tests/*py
test: test:
python3 -m unittest -v tests/*.py python3 -m unittest -v tests/*.py

21
git-importer.py
View File

@ -1,6 +1,7 @@
#!/usr/bin/python3 #!/usr/bin/python3
import argparse import argparse
import concurrent.futures
import logging import logging
import pathlib import pathlib
import sys import sys
@ -42,6 +43,12 @@ PROJECTS = [
] ]
def export_package(package, repodir, cachedir, gc):
exporter = GitExporter(URL_OBS, "openSUSE:Factory", package, repodir, cachedir)
exporter.set_gc_interval(gc)
exporter.export_as_git()
def main(): def main():
parser = argparse.ArgumentParser(description="OBS history importer into git") parser = argparse.ArgumentParser(description="OBS history importer into git")
parser.add_argument("packages", help="OBS package names", nargs="*") parser.add_argument("packages", help="OBS package names", nargs="*")
@ -106,12 +113,14 @@ def main():
importer = Importer(URL_OBS, "openSUSE:Factory", args.packages) importer = Importer(URL_OBS, "openSUSE:Factory", args.packages)
importer.import_into_db() importer.import_into_db()
for package in args.packages: with concurrent.futures.ProcessPoolExecutor() as executor:
exporter = GitExporter( fs = [
URL_OBS, "openSUSE:Factory", package, args.repodir, args.cachedir executor.submit(
) export_package, package, args.repodir, args.cachedir, args.gc
exporter.set_gc_interval(args.gc) )
exporter.export_as_git() for package in args.packages
]
concurrent.futures.wait(fs)
if __name__ == "__main__": if __name__ == "__main__":

5
lib/proxy_sha256.py
View File

@ -41,7 +41,9 @@ class ProxySHA256:
self.hashes = dict() self.hashes = dict()
return return
logging.debug("Retrieve all previously defined SHA256") logging.debug("Retrieve all previously defined SHA256")
response = requests.get(f"http://source.dyn.cloud.suse.de/package/{package}") response = requests.get(
f"http://source.dyn.cloud.suse.de/package/{package}", timeout=5
)
if response.status_code == 200: if response.status_code == 200:
json = response.json() json = response.json()
self.hashes = json["shas"] self.hashes = json["shas"]
@ -67,6 +69,7 @@ class ProxySHA256:
"url": url, "url": url,
"package": package, "package": package,
}, },
timeout=10,
) )
if response.status_code != 200: if response.status_code != 200:
raise Exception(f"Redirector error on {self.url} for {url}") raise Exception(f"Redirector error on {self.url} for {url}")