Merge pull request 'Also treat some more mimetypes as text' (#20) from add_further_mimetypes into main

Reviewed-on: https://gitea.opensuse.org/importers/git-importer/pulls/20

git-importer.py

@@ -1,7 +1,6 @@
 #!/usr/bin/python3
 
 import argparse
-import concurrent.futures
 import logging
 import pathlib
 import sys
@@ -113,17 +112,8 @@ def main():
 
     importer = Importer(URL_OBS, "openSUSE:Factory", args.packages)
     importer.import_into_db()
-    if len(args.packages) != 1:
+    for package in args.packages:
-        with concurrent.futures.ProcessPoolExecutor(max_workers=8) as executor:
+        export_package(package, args.repodir, args.cachedir, args.gc)
-            fs = [
-                executor.submit(
-                    export_package, package, args.repodir, args.cachedir, args.gc
-                )
-                for package in args.packages
-            ]
-            concurrent.futures.wait(fs)
-    else:
-        export_package(args.packages[0], args.repodir, args.cachedir, args.gc)
 
 
 if __name__ == "__main__":

lib/binary.py

@@ -25,18 +25,28 @@ BINARY = {
     ".zst",
 }
 
+TEXT_MIMETYPES = {
+    "message/rfc822",
+    "application/pgp-keys",
+    "application/x-gnupg-keyring",
+}
+
+
+def is_text_mimetype(mimetype):
+    if mimetype.startswith("text/"):
+        return True
+
+    return mimetype.split(";")[0] in TEXT_MIMETYPES
+
 
 def is_binary_or_large(filename, size):
     """Decide if is a binary file based on the extension or size"""
     binary_suffix = BINARY
     non_binary_suffix = {
-        ".1",
-        ".8",
         ".SUSE",
         ".asc",
         ".c",
         ".cabal",
-        ".cfg",
         ".changes",
         ".conf",
         ".desktop",

lib/lfs_oid.py

@@ -6,6 +6,7 @@ import sys
 
 import requests
 
+from lib.binary import is_text_mimetype
 from lib.db import DB
 
 
@@ -71,13 +72,13 @@ class LFSOid:
                         VALUES (%s,%s,%s)""",
                 (package, filename, lfs_oid_id),
             )
-            if mimetype.startswith("text/"):
+            if is_text_mimetype(mimetype):
                 cur.execute(
                     "INSERT INTO text_files (package,filename) VALUES (%s,%s)",
                     (package, filename),
                 )
             self.db.conn.commit()
-        self.from_db(lfs_oid_id)
+        self.set_from_dbid(lfs_oid_id)
         if not self.check():
             self.register()