From c3a20e9a33b11665c3862a32e6b2abb0f7c059be Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Mon, 17 Jan 2022 17:20:54 +0100 Subject: [PATCH] Include compressed manpages correctly in the RPM package Compressed manpages have different extension than those listed in the RECORD file, so they were not recognized when %%pyproject_save_files '+auto' flag was provided. To enable the path recognition, if the manpage extension matches the one listed in brp-compres, the extension is removed, and an asterisk is now added to the manpages filenames. Source: https://docs.fedoraproject.org/en-US/packaging-guidelines/#_manpages Fixes: https://bugzilla.redhat.com/2033254 --- compare_mandata.py | 83 +++++++++++++++++++++++++++++ macros.pyproject | 1 + pyproject-rpm-macros.spec | 10 +++- pyproject_save_files.py | 81 ++++++++++++++++++++++++++-- pyproject_save_files_test_data.yaml | 6 +++ test_pyproject_save_files.py | 15 +++++- tests/python-getmac.spec | 50 +++++++++++++++++ tests/tests.yml | 3 ++ 8 files changed, 243 insertions(+), 6 deletions(-) create mode 100644 compare_mandata.py create mode 100644 tests/python-getmac.spec diff --git a/compare_mandata.py b/compare_mandata.py new file mode 100644 index 0000000..08d4c6b --- /dev/null +++ b/compare_mandata.py @@ -0,0 +1,83 @@ +'''Check whether the manpage extensions and directories list hardcoded in brp-compress +are the same as the lists stored in pyproject_save_files.py. +There is an open issue for RPM to provide them both as macros: +https://github.com/rpm-software-management/rpm/issues/1865 +Once that happens, this script can be removed. +''' + +import argparse +import re +import sys + +from pathlib import PosixPath + +from pyproject_buildrequires import print_err +from pyproject_save_files import prepend_mandirs, MANPAGE_EXTENSIONS + + + +def read_brp_compress(filename): + + contents = filename.read_text() + # To avoid duplicity of the manpage extensions which are listed a few times + # in the source file, they are stored in set and then retyped to a sorted list + manpage_exts = sorted( + set(re.findall(r'\(?(\w+)\\+\)?\$?', contents)) + ) + + # Get rid of ${PREFIX} when extracting the manpage directories + mandirs = [ + entry.replace('.${PREFIX}', '/PREFIX') + for entry in contents.split() + if entry.startswith('.${PREFIX}') + ] + + return manpage_exts, sorted(mandirs) + + +def compare_mandirs(brp_compress_mandirs): + ''' + Check whether each of brp-compress mandirs entry is present in the list + stored in pyproject_save_files.py + ''' + + pyp_save_files_mandirs = sorted(prepend_mandirs(prefix='/PREFIX')) + if brp_compress_mandirs == pyp_save_files_mandirs: + return True + else: + print_err('Mandir lists don\'t match, update the list in pyproject_save_files.py') + print_err('brp-compress list:', brp_compress_mandirs) + print_err('pyproject_save_files list:', pyp_save_files_mandirs) + return False + + +def compare_manpage_extensions(brp_compress_manpage_exts): + ''' + Check whether each of brp-compress manpage extension is present in the list + stored in pyproject_save_files.py + ''' + + if brp_compress_manpage_exts == sorted(MANPAGE_EXTENSIONS): + return True + else: + print_err('Manpage extension lists don\'t match, update the list in pyproject_save_files.py') + print_err('brp-compress list:', brp_compress_manpage_exts) + print_err('pyproject_save_files list:', sorted(MANPAGE_EXTENSIONS)) + return False + + +def main(args): + src_manpage_exts, src_mandirs = read_brp_compress(args.filename) + extension_check_successful = compare_manpage_extensions(src_manpage_exts) + mandir_check_successful = compare_mandirs(src_mandirs) + if extension_check_successful and mandir_check_successful: + sys.exit(0) + else: + sys.exit(1) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--filename', type=PosixPath, required=True, + help='Provide location of brp-compress file') + main(parser.parse_args()) diff --git a/macros.pyproject b/macros.pyproject index 57c6d48..0a34614 100644 --- a/macros.pyproject +++ b/macros.pyproject @@ -97,6 +97,7 @@ fi --sitearch "%{python3_sitearch}" \\ --python-version "%{python3_version}" \\ --pyproject-record "%{_pyproject_record}" \\ + --prefix "%{_prefix}" \\ %{*} } diff --git a/pyproject-rpm-macros.spec b/pyproject-rpm-macros.spec index 71cfe3a..16a24d1 100644 --- a/pyproject-rpm-macros.spec +++ b/pyproject-rpm-macros.spec @@ -6,7 +6,7 @@ License: MIT # Keep the version at zero and increment only release Version: 0 -Release: 53%{?dist} +Release: 54%{?dist} # Macro files Source001: macros.pyproject @@ -23,6 +23,7 @@ Source106: pyproject_requirements_txt.py Source201: test_pyproject_buildrequires.py Source202: test_pyproject_save_files.py Source203: test_pyproject_requirements_txt.py +Source204: compare_mandata.py # Test data Source301: pyproject_buildrequires_testcases.yaml @@ -100,6 +101,9 @@ install -m 644 pyproject_requirements_txt.py %{buildroot}%{_rpmconfigdir}/redhat %check export HOSTNAME="rpmbuild" # to speedup tox in network-less mock, see rhbz#1856356 %{python3} -m pytest -vv --doctest-modules + +# brp-compress is provided as an argument to get the right directory macro expansion +%{python3} compare_mandata.py -f %{_rpmconfigdir}/brp-compress %endif @@ -116,6 +120,10 @@ export HOSTNAME="rpmbuild" # to speedup tox in network-less mock, see rhbz#1856 %license LICENSE %changelog +* Wed Jan 19 2022 Karolina Surma - 0-54 +- Include compressed manpages to the package if flag '+auto' is provided to %%pyproject_save_files +- Fixes: rhbz#2033254 + * Fri Jan 14 2022 Miro Hrončok - 0-53 - %%pyproject_buildrequires: Make -r (include runtime) the default, use -R to opt-out diff --git a/pyproject_save_files.py b/pyproject_save_files.py index e3ffc9f..6cd53e7 100644 --- a/pyproject_save_files.py +++ b/pyproject_save_files.py @@ -12,6 +12,28 @@ from importlib.metadata import Distribution # From RPM's build/files.c strtokWithQuotes delim argument RPM_FILES_DELIMETERS = ' \n\t' +# RPM hardcodes the lists of manpage extensions and directories, +# so we have to maintain separate ones :( +# There is an issue for RPM to provide the lists as macros: +# https://github.com/rpm-software-management/rpm/issues/1865 +# The original lists can be found here: +# https://github.com/rpm-software-management/rpm/blob/master/scripts/brp-compress +MANPAGE_EXTENSIONS = ['gz', 'Z', 'bz2', 'xz', 'lzma', 'zst', 'zstd'] +MANDIRS = [ + '/man/man*', + '/man/*/man*', + '/info', + '/share/man/man*', + '/share/man/*/man*', + '/share/info', + '/kerberos/man', + '/X11R6/man/man*', + '/lib/perl5/man/man*', + '/share/doc/*/man/man*', + '/lib/*/man/man*', + '/share/fish/man/man*', +] + class BuildrootPath(PurePosixPath): """ @@ -145,6 +167,56 @@ def add_lang_to_module(paths, module_name, path): return True +def prepend_mandirs(prefix): + """ + Return the list of man page directories prepended with the given prefix. + """ + return [str(prefix) + mandir for mandir in MANDIRS] + + +def normalize_manpage_filename(prefix, path): + """ + If a path is processed by RPM's brp-compress script, strip it of the extension + (if the extension matches one of the listed by brp-compress), + append '*' to the filename and return it. If not, return the unchanged path. + Rationale: https://docs.fedoraproject.org/en-US/packaging-guidelines/#_manpages + + Examples: + + >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/share/man/de/man1/linkchecker.1')) + BuildrootPath('/usr/share/man/de/man1/linkchecker.1*') + + >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/share/doc/en/man/man1/getmac.1')) + BuildrootPath('/usr/share/doc/en/man/man1/getmac.1*') + + >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/share/man/man8/abc.8.zstd')) + BuildrootPath('/usr/share/man/man8/abc.8*') + + >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/kerberos/man/dir')) + BuildrootPath('/usr/kerberos/man/dir') + + >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/kerberos/man/dir.1')) + BuildrootPath('/usr/kerberos/man/dir.1*') + + >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/bin/getmac')) + BuildrootPath('/usr/bin/getmac') + """ + + prefixed_mandirs = prepend_mandirs(prefix) + for mandir in prefixed_mandirs: + # "dir" is explicitly excluded by RPM + # https://github.com/rpm-software-management/rpm/blob/rpm-4.17.0-release/scripts/brp-compress#L24 + if fnmatch.fnmatch(str(path.parent), mandir) and path.name != "dir": + # "abc.1.gz2" -> "abc.1*" + if path.suffix[1:] in MANPAGE_EXTENSIONS: + return BuildrootPath(path.parent / (path.stem + "*")) + # "abc.1 -> abc.1*" + else: + return BuildrootPath(path.parent / (path.name + "*")) + else: + return path + + def is_valid_module_name(s): """Return True if a string is considered a valid module name and False otherwise. @@ -215,7 +287,7 @@ def module_names_from_path(path): def classify_paths( - record_path, parsed_record_content, metadata, sitedirs, python_version + record_path, parsed_record_content, metadata, sitedirs, python_version, prefix ): """ For each BuildrootPath in parsed_record_content classify it to a dict structure @@ -301,6 +373,7 @@ def classify_paths( if path.suffix == ".mo": add_lang_to_module(paths, None, path) or paths["other"]["files"].append(path) else: + path = normalize_manpage_filename(prefix, path) paths["other"]["files"].append(path) return paths @@ -528,7 +601,7 @@ def dist_metadata(buildroot, record_path): return dist.metadata -def pyproject_save_files_and_modules(buildroot, sitelib, sitearch, python_version, pyproject_record, varargs): +def pyproject_save_files_and_modules(buildroot, sitelib, sitearch, python_version, pyproject_record, prefix, varargs): """ Takes arguments from the %{pyproject_save_files} macro @@ -548,7 +621,7 @@ def pyproject_save_files_and_modules(buildroot, sitelib, sitearch, python_versio for record_path, files in parsed_records.items(): metadata = dist_metadata(buildroot, record_path) paths_dict = classify_paths( - record_path, files, metadata, sitedirs, python_version + record_path, files, metadata, sitedirs, python_version, prefix ) final_file_list.extend( @@ -569,6 +642,7 @@ def main(cli_args): cli_args.sitearch, cli_args.python_version, cli_args.pyproject_record, + cli_args.prefix, cli_args.varargs, ) @@ -586,6 +660,7 @@ def argparser(): r.add_argument("--sitearch", type=BuildrootPath, required=True) r.add_argument("--python-version", type=str, required=True) r.add_argument("--pyproject-record", type=PosixPath, required=True) + r.add_argument("--prefix", type=PosixPath, required=True) parser.add_argument("varargs", nargs="+") return parser diff --git a/pyproject_save_files_test_data.yaml b/pyproject_save_files_test_data.yaml index 08dffd0..eeed5d3 100644 --- a/pyproject_save_files_test_data.yaml +++ b/pyproject_save_files_test_data.yaml @@ -217,6 +217,7 @@ classified: files: - /usr/bin/tldr - /usr/bin/tldr.py + - /usr/share/man/man1/tldr* ipykernel: metadata: dirs: @@ -376,6 +377,7 @@ classified: - /usr/share/jupyter/kernels/python3/logo-64x64.png - /usr/share/jupyter/kernels/python3/logo-32x32.png - /usr/share/jupyter/kernels/python3/kernel.json + - /usr/man/man5/ipykernel.5* zope: metadata: dirs: @@ -7517,6 +7519,7 @@ dumped: - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/WHEEL - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/top_level.txt - /usr/lib/python3.7/site-packages/tldr.py + - /usr/share/man/man1/tldr* - - tldr - - mistune - mistune @@ -7675,6 +7678,7 @@ dumped: - /usr/lib/python3.7/site-packages/ipykernel/tests/utils.py - /usr/lib/python3.7/site-packages/ipykernel/trio_runner.py - /usr/lib/python3.7/site-packages/ipykernel/zmqshell.py + - /usr/man/man5/ipykernel.5* - /usr/share/jupyter/kernels/python3/kernel.json - /usr/share/jupyter/kernels/python3/logo-32x32.png - /usr/share/jupyter/kernels/python3/logo-64x64.png @@ -15559,6 +15563,7 @@ records: ../../../bin/__pycache__/tldr.cpython-37.pyc,, ../../../bin/tldr,sha256=6MUiLCWhldmV8OelT2dvPgS7q5GFwuhvd6th0Bb-LH4,12766 ../../../bin/tldr.py,sha256=6MUiLCWhldmV8OelT2dvPgS7q5GFwuhvd6th0Bb-LH4,12766 + ../../../share/man/man1/tldr.bz2,sha256=xp_kqadh3PjDb4OaU8D8RQDcakrwl5AMmCnaOUV7ioo,10957 __pycache__/tldr.cpython-37.pyc,, tldr-0.5.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 tldr-0.5.dist-info/LICENSE,sha256=q7quAfjDWCYKC_WRk_uaP6d2wwVpOpVjUSkv8l6H7xI,1075 @@ -15587,6 +15592,7 @@ records: ../../../share/jupyter/kernels/python3/kernel.json,sha256=7o0-HNZRKjrk7Fqb71O3gptCssqWqfd_sxw5FNFeYO0,143 ../../../share/jupyter/kernels/python3/logo-32x32.png,sha256=4ytcKCBy1xeIe2DacxeP3TWmXcPK6sunoCblpCVcyZc,1084 ../../../share/jupyter/kernels/python3/logo-64x64.png,sha256=XJBjtDbO3wVnSA_kh-zg0UeeqVRfMQy6k_oYTMurKQ0,2180 + ../../../man/man5/ipykernel.5,sha256=xp_kqadh3PjDb4OaU8D8RQDcakrwl5AMmCnaOUV7ioo,10957 __pycache__/ipykernel_launcher.cpython-37.pyc,, ipykernel-5.2.1.dist-info/COPYING.md,sha256=YMWypaSJDUjGk7i5CKSWdbUkuErBWn7ByVY-Bea__ho,2835 ipykernel-5.2.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 diff --git a/test_pyproject_save_files.py b/test_pyproject_save_files.py index 98b880e..5dd0b27 100755 --- a/test_pyproject_save_files.py +++ b/test_pyproject_save_files.py @@ -11,6 +11,7 @@ from pyproject_save_files import main as save_files_main from pyproject_save_files import module_names_from_path DIR = Path(__file__).parent +PREFIX = Path("/usr") BINDIR = BuildrootPath("/usr/bin") DATADIR = BuildrootPath("/usr/share") SITELIB = BuildrootPath("/usr/lib/python3.7/site-packages") @@ -110,7 +111,15 @@ def test_parse_record_tensorflow(): def remove_others(expected): - return [p for p in expected if not (p.startswith(str(BINDIR)) or p.endswith(".pth") or p.rpartition(' ')[-1].startswith(str(DATADIR)))] + return [ + p for p in expected + if not ( + p.startswith(str(BINDIR)) or + p.endswith(".pth") or + p.endswith("*") or + p.rpartition(' ')[-1].startswith(str(DATADIR)) + ) + ] @pytest.mark.parametrize("include_auto", (True, False)) @@ -179,7 +188,9 @@ def default_options(output_files, output_modules, mock_root, pyproject_record): "--python-version", "3.7", # test data are for 3.7, "--pyproject-record", - str(pyproject_record) + str(pyproject_record), + "--prefix", + str(PREFIX), ] diff --git a/tests/python-getmac.spec b/tests/python-getmac.spec new file mode 100644 index 0000000..95bfaa0 --- /dev/null +++ b/tests/python-getmac.spec @@ -0,0 +1,50 @@ +Name: python-getmac +Version: 0.8.3 +Release: 0%{?dist} +Summary: Get MAC addresses of remote hosts and local interfaces +License: MIT +URL: https://github.com/GhostofGoes/getmac +Source0: %{pypi_source getmac} + +BuildArch: noarch +BuildRequires: python3-devel +BuildRequires: pyproject-rpm-macros + + +%global _description %{expand: +Test that manpages are correctly processed by %%pyproject_save_files '*' +auto.} + + +%description %_description + +%package -n python3-getmac +Summary: %{summary} + +%description -n python3-getmac %_description + + +%prep +%autosetup -p1 -n getmac-%{version} + + +%generate_buildrequires +%pyproject_buildrequires -r + + +%build +%pyproject_wheel + + +%install +%pyproject_install +%pyproject_save_files '*' +auto + + +%check +%pyproject_check_import +# Internal check for our macros, assert there is a manpage: +test -f %{buildroot}%{_mandir}/man1/getmac.1* + + +%files -n python3-getmac -f %{pyproject_files} + diff --git a/tests/tests.yml b/tests/tests.yml index 60e3576..a905377 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -76,6 +76,9 @@ - markupsafe: dir: . run: ./mocktest.sh python-markupsafe + - getmac: + dir: . + run: ./mocktest.sh python-getmac - double_install: dir: . run: ./mocktest.sh double-install