yudaike/openSUSE-release-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

factory-package-news.py: Compress data files with zstd

Browse Source 
It's less than 1/3rd of the uncompressed size.
This commit is contained in:
Fabian Vogt 2023-12-05 11:56:29 +01:00
parent 2925abf41a
commit 8516c1ad8f
2 changed files with 25 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dist/package/openSUSE-release-tools.spec vendored
View File

@ -57,6 +57,8 @@ Requires: python3-lxml
Requires: python3-pycurl
Requires: python3-python-dateutil
Requires: python3-pyxdg
# factory-package-news.py
Requires: python3-pyzstd
Requires: python3-requests
# typing extensions are needed on SLE & Leap
%if 0%{?suse_version} <= 1500

34
factory-package-news/factory-package-news.py
View File

@ -9,6 +9,7 @@ import rpm
import pickle
import cmdln
import re
import pyzstd
SRPM_RE = re.compile(
r'(?P<name>.+)-(?P<version>[^-]+)-(?P<release>[^-]+)\.(?P<suffix>(?:no)?src\.rpm)$')
@ -27,6 +28,17 @@ def utf8str(content):
return str(content, 'utf-8') if isinstance(content, bytes) else content
# Older versions wrote uncompressed files, support both for now.
def open_zstd_or_plain(path: str):
f = open(path, 'rb')
if f.read(4) == b'\x28\xb5\x2f\xfd':
f.close()
return pyzstd.open(path, 'rb')
f.seek(0, os.SEEK_SET)
return f
class ChangeLogger(cmdln.Cmdln):
def __init__(self, *args, **kwargs):
cmdln.Cmdln.__init__(self, args, kwargs)
@ -165,8 +177,8 @@ class ChangeLogger(cmdln.Cmdln):
if not opts.snapshot:
raise Exception("missing snapshot option")
f = open(os.path.join(opts.dir, opts.snapshot), 'wb')
pickle.dump([data_version, self.readChangeLogs(dirs)], f)
with pyzstd.open(os.path.join(opts.dir, opts.snapshot), 'wb') as f:
pickle.dump([data_version, self.readChangeLogs(dirs)], f)
def do_dump(self, subcmd, opts, *dirs):
"""${cmd_name}: pprint the package changelog information
@ -182,9 +194,9 @@ class ChangeLogger(cmdln.Cmdln):
${cmd_usage}
${cmd_option_list}
"""
f = open(filename, 'rb')
(v, (pkgs, changelogs)) = pickle.load(
f, encoding='utf-8', errors='backslashreplace')
with open_zstd_or_plain(filename) as f:
(v, (pkgs, changelogs)) = pickle.load(
f, encoding='utf-8', errors='backslashreplace')
pprint(pkgs[package])
pprint(changelogs[pkgs[package]['sourcerpm']])
@ -209,14 +221,14 @@ class ChangeLogger(cmdln.Cmdln):
if not os.path.isdir(opts.dir):
raise Exception("%s must be a directory" % opts.dir)
f = open(os.path.join(opts.dir, version1), 'rb')
(v, (v1pkgs, v1changelogs)) = pickle.load(f,
encoding='utf-8', errors='backslashreplace')
with open_zstd_or_plain(os.path.join(opts.dir, version1)) as f:
(v, (v1pkgs, v1changelogs)) = pickle.load(f,
encoding='utf-8', errors='backslashreplace')
if v != data_version:
raise Exception("not matching version %s in %s" % (v, version1))
f = open(os.path.join(opts.dir, version2), 'rb')
(v, (v2pkgs, v2changelogs)) = pickle.load(f,
encoding='utf-8', errors='backslashreplace')
with open_zstd_or_plain(os.path.join(opts.dir, version2)) as f:
(v, (v2pkgs, v2changelogs)) = pickle.load(f,
encoding='utf-8', errors='backslashreplace')
if v != data_version:
raise Exception("not matching version %s in %s" % (v, version2))