forked from importers/git-importer
Merge pull request 'Also treat some more mimetypes as text' (#20) from add_further_mimetypes into main
Reviewed-on: https://gitea.opensuse.org/importers/git-importer/pulls/20
This commit is contained in:
commit
d311d54f26
@ -1,7 +1,6 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import logging
|
||||
import pathlib
|
||||
import sys
|
||||
@ -113,17 +112,8 @@ def main():
|
||||
|
||||
importer = Importer(URL_OBS, "openSUSE:Factory", args.packages)
|
||||
importer.import_into_db()
|
||||
if len(args.packages) != 1:
|
||||
with concurrent.futures.ProcessPoolExecutor(max_workers=8) as executor:
|
||||
fs = [
|
||||
executor.submit(
|
||||
export_package, package, args.repodir, args.cachedir, args.gc
|
||||
)
|
||||
for package in args.packages
|
||||
]
|
||||
concurrent.futures.wait(fs)
|
||||
else:
|
||||
export_package(args.packages[0], args.repodir, args.cachedir, args.gc)
|
||||
for package in args.packages:
|
||||
export_package(package, args.repodir, args.cachedir, args.gc)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
@ -25,18 +25,28 @@ BINARY = {
|
||||
".zst",
|
||||
}
|
||||
|
||||
TEXT_MIMETYPES = {
|
||||
"message/rfc822",
|
||||
"application/pgp-keys",
|
||||
"application/x-gnupg-keyring",
|
||||
}
|
||||
|
||||
|
||||
def is_text_mimetype(mimetype):
|
||||
if mimetype.startswith("text/"):
|
||||
return True
|
||||
|
||||
return mimetype.split(";")[0] in TEXT_MIMETYPES
|
||||
|
||||
|
||||
def is_binary_or_large(filename, size):
|
||||
"""Decide if is a binary file based on the extension or size"""
|
||||
binary_suffix = BINARY
|
||||
non_binary_suffix = {
|
||||
".1",
|
||||
".8",
|
||||
".SUSE",
|
||||
".asc",
|
||||
".c",
|
||||
".cabal",
|
||||
".cfg",
|
||||
".changes",
|
||||
".conf",
|
||||
".desktop",
|
||||
|
@ -6,6 +6,7 @@ import sys
|
||||
|
||||
import requests
|
||||
|
||||
from lib.binary import is_text_mimetype
|
||||
from lib.db import DB
|
||||
|
||||
|
||||
@ -71,13 +72,13 @@ class LFSOid:
|
||||
VALUES (%s,%s,%s)""",
|
||||
(package, filename, lfs_oid_id),
|
||||
)
|
||||
if mimetype.startswith("text/"):
|
||||
if is_text_mimetype(mimetype):
|
||||
cur.execute(
|
||||
"INSERT INTO text_files (package,filename) VALUES (%s,%s)",
|
||||
(package, filename),
|
||||
)
|
||||
self.db.conn.commit()
|
||||
self.from_db(lfs_oid_id)
|
||||
self.set_from_dbid(lfs_oid_id)
|
||||
if not self.check():
|
||||
self.register()
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user