Include compressed manpages correctly in the RPM package

Compressed manpages have different extension than those listed in the RECORD file, so they were not recognized when %%pyproject_save_files '+auto' flag was provided. To enable the path recognition, if the manpage extension matches the one listed in brp-compres, the extension is removed, and an asterisk is now added to the manpages filenames. Source: https://docs.fedoraproject.org/en-US/packaging-guidelines/#_manpages Fixes: https://bugzilla.redhat.com/2033254
2022-01-17 17:20:54 +01:00 · 2022-01-17 17:20:54 +01:00 · c3a20e9a33
commit c3a20e9a33
parent 8c8afba774
8 changed files with 243 additions and 6 deletions
--- a/compare_mandata.py
+++ b/compare_mandata.py
@ -0,0 +1,83 @@
+'''Check whether the manpage extensions and directories list hardcoded in brp-compress
+are the same as the lists stored in pyproject_save_files.py.
+There is an open issue for RPM to provide them both as macros:
+https://github.com/rpm-software-management/rpm/issues/1865
+Once that happens, this script can be removed.
+'''
+
+import argparse
+import re
+import sys
+
+from pathlib import PosixPath
+
+from pyproject_buildrequires import print_err
+from pyproject_save_files import prepend_mandirs, MANPAGE_EXTENSIONS
+
+
+
+def read_brp_compress(filename):
+
+    contents = filename.read_text()
+    # To avoid duplicity of the manpage extensions which are listed a few times
+    # in the source file, they are stored in set and then retyped to a sorted list
+    manpage_exts = sorted(
+        set(re.findall(r'\(?(\w+)\\+\)?\$?', contents))
+    )
+
+    # Get rid of ${PREFIX} when extracting the manpage directories
+    mandirs = [
+        entry.replace('.${PREFIX}', '/PREFIX')
+        for entry in contents.split()
+        if entry.startswith('.${PREFIX}')
+    ]
+
+    return manpage_exts, sorted(mandirs)
+
+
+def compare_mandirs(brp_compress_mandirs):
+    '''
+    Check whether each of brp-compress mandirs entry is present in the list
+    stored in pyproject_save_files.py
+    '''
+
+    pyp_save_files_mandirs = sorted(prepend_mandirs(prefix='/PREFIX'))
+    if brp_compress_mandirs == pyp_save_files_mandirs:
+        return True
+    else:
+        print_err('Mandir lists don\'t match, update the list in pyproject_save_files.py')
+        print_err('brp-compress list:', brp_compress_mandirs)
+        print_err('pyproject_save_files list:', pyp_save_files_mandirs)
+        return False
+
+
+def compare_manpage_extensions(brp_compress_manpage_exts):
+    '''
+    Check whether each of brp-compress manpage extension is present in the list
+    stored in pyproject_save_files.py
+    '''
+
+    if brp_compress_manpage_exts == sorted(MANPAGE_EXTENSIONS):
+        return True
+    else:
+        print_err('Manpage extension lists don\'t match, update the list in pyproject_save_files.py')
+        print_err('brp-compress list:', brp_compress_manpage_exts)
+        print_err('pyproject_save_files list:', sorted(MANPAGE_EXTENSIONS))
+        return False
+
+
+def main(args):
+    src_manpage_exts, src_mandirs = read_brp_compress(args.filename)
+    extension_check_successful = compare_manpage_extensions(src_manpage_exts)
+    mandir_check_successful = compare_mandirs(src_mandirs)
+    if extension_check_successful and mandir_check_successful:
+        sys.exit(0)
+    else:
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-f', '--filename', type=PosixPath, required=True,
+                        help='Provide location of brp-compress file')
+    main(parser.parse_args())
--- a/macros.pyproject
+++ b/macros.pyproject
@ -97,6 +97,7 @@ fi
  --sitearch "%{python3_sitearch}" \\
  --python-version "%{python3_version}" \\
  --pyproject-record "%{_pyproject_record}" \\
+  --prefix "%{_prefix}" \\
  %{*}
 }

--- a/pyproject-rpm-macros.spec
+++ b/pyproject-rpm-macros.spec
@ -6,7 +6,7 @@ License:        MIT

 # Keep the version at zero and increment only release
 Version:        0
-Release:        53%{?dist}
+Release:        54%{?dist}

 # Macro files
 Source001:      macros.pyproject
@ -23,6 +23,7 @@ Source106:      pyproject_requirements_txt.py
 Source201:      test_pyproject_buildrequires.py
 Source202:      test_pyproject_save_files.py
 Source203:      test_pyproject_requirements_txt.py
+Source204:      compare_mandata.py

 # Test data
 Source301:      pyproject_buildrequires_testcases.yaml
@ -100,6 +101,9 @@ install -m 644 pyproject_requirements_txt.py %{buildroot}%{_rpmconfigdir}/redhat
 %check
 export HOSTNAME="rpmbuild"  # to speedup tox in network-less mock, see rhbz#1856356
 %{python3} -m pytest -vv --doctest-modules
+
+# brp-compress is provided as an argument to get the right directory macro expansion
+%{python3} compare_mandata.py -f %{_rpmconfigdir}/brp-compress
 %endif


@ -116,6 +120,10 @@ export HOSTNAME="rpmbuild"  # to speedup tox in network-less mock, see rhbz#1856
 %license LICENSE

 %changelog
+* Wed Jan 19 2022 Karolina Surma <ksurma@redhat.com> - 0-54
+- Include compressed manpages to the package if flag '+auto' is provided to %%pyproject_save_files
+- Fixes: rhbz#2033254
+
 * Fri Jan 14 2022 Miro Hrončok <mhroncok@redhat.com> - 0-53
 - %%pyproject_buildrequires: Make -r (include runtime) the default, use -R to opt-out

--- a/pyproject_save_files.py
+++ b/pyproject_save_files.py
@ -12,6 +12,28 @@ from importlib.metadata import Distribution
 # From RPM's build/files.c strtokWithQuotes delim argument
 RPM_FILES_DELIMETERS = ' \n\t'

+# RPM hardcodes the lists of manpage extensions and directories,
+# so we have to maintain separate ones :(
+# There is an issue for RPM to provide the lists as macros:
+# https://github.com/rpm-software-management/rpm/issues/1865
+# The original lists can be found here:
+# https://github.com/rpm-software-management/rpm/blob/master/scripts/brp-compress
+MANPAGE_EXTENSIONS = ['gz', 'Z', 'bz2', 'xz', 'lzma', 'zst', 'zstd']
+MANDIRS = [
+    '/man/man*',
+    '/man/*/man*',
+    '/info',
+    '/share/man/man*',
+    '/share/man/*/man*',
+    '/share/info',
+    '/kerberos/man',
+    '/X11R6/man/man*',
+    '/lib/perl5/man/man*',
+    '/share/doc/*/man/man*',
+    '/lib/*/man/man*',
+    '/share/fish/man/man*',
+]
+

 class BuildrootPath(PurePosixPath):
    """
@ -145,6 +167,56 @@ def add_lang_to_module(paths, module_name, path):
    return True


+def prepend_mandirs(prefix):
+    """
+    Return the list of man page directories prepended with the given prefix.
+    """
+    return [str(prefix) + mandir for mandir in MANDIRS]
+
+
+def normalize_manpage_filename(prefix, path):
+    """
+    If a path is processed by RPM's brp-compress script, strip it of the extension
+    (if the extension matches one of the listed by brp-compress),
+    append '*' to the filename and return it. If not, return the unchanged path.
+    Rationale: https://docs.fedoraproject.org/en-US/packaging-guidelines/#_manpages
+
+    Examples:
+
+        >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/share/man/de/man1/linkchecker.1'))
+        BuildrootPath('/usr/share/man/de/man1/linkchecker.1*')
+
+        >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/share/doc/en/man/man1/getmac.1'))
+        BuildrootPath('/usr/share/doc/en/man/man1/getmac.1*')
+
+        >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/share/man/man8/abc.8.zstd'))
+        BuildrootPath('/usr/share/man/man8/abc.8*')
+
+        >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/kerberos/man/dir'))
+        BuildrootPath('/usr/kerberos/man/dir')
+
+        >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/kerberos/man/dir.1'))
+        BuildrootPath('/usr/kerberos/man/dir.1*')
+
+        >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/bin/getmac'))
+        BuildrootPath('/usr/bin/getmac')
+    """
+
+    prefixed_mandirs = prepend_mandirs(prefix)
+    for mandir in prefixed_mandirs:
+        # "dir" is explicitly excluded by RPM
+        # https://github.com/rpm-software-management/rpm/blob/rpm-4.17.0-release/scripts/brp-compress#L24
+        if fnmatch.fnmatch(str(path.parent), mandir) and path.name != "dir":
+            # "abc.1.gz2" -> "abc.1*"
+            if path.suffix[1:] in MANPAGE_EXTENSIONS:
+                return BuildrootPath(path.parent / (path.stem + "*"))
+            # "abc.1 -> abc.1*"
+            else:
+                return BuildrootPath(path.parent / (path.name + "*"))
+    else:
+        return path
+
+
 def is_valid_module_name(s):
    """Return True if a string is considered a valid module name and False otherwise.

@ -215,7 +287,7 @@ def module_names_from_path(path):


 def classify_paths(
-    record_path, parsed_record_content, metadata, sitedirs, python_version
+    record_path, parsed_record_content, metadata, sitedirs, python_version, prefix
 ):
    """
    For each BuildrootPath in parsed_record_content classify it to a dict structure
@ -301,6 +373,7 @@ def classify_paths(
            if path.suffix == ".mo":
                add_lang_to_module(paths, None, path) or paths["other"]["files"].append(path)
            else:
+                path = normalize_manpage_filename(prefix, path)
                paths["other"]["files"].append(path)

    return paths
@ -528,7 +601,7 @@ def dist_metadata(buildroot, record_path):
    return dist.metadata


-def pyproject_save_files_and_modules(buildroot, sitelib, sitearch, python_version, pyproject_record, varargs):
+def pyproject_save_files_and_modules(buildroot, sitelib, sitearch, python_version, pyproject_record, prefix, varargs):
    """
    Takes arguments from the %{pyproject_save_files} macro

@ -548,7 +621,7 @@ def pyproject_save_files_and_modules(buildroot, sitelib, sitearch, python_versio
    for record_path, files in parsed_records.items():
        metadata = dist_metadata(buildroot, record_path)
        paths_dict = classify_paths(
-            record_path, files, metadata, sitedirs, python_version
+            record_path, files, metadata, sitedirs, python_version, prefix
        )

        final_file_list.extend(
@ -569,6 +642,7 @@ def main(cli_args):
        cli_args.sitearch,
        cli_args.python_version,
        cli_args.pyproject_record,
+        cli_args.prefix,
        cli_args.varargs,
    )

@ -586,6 +660,7 @@ def argparser():
    r.add_argument("--sitearch", type=BuildrootPath, required=True)
    r.add_argument("--python-version", type=str, required=True)
    r.add_argument("--pyproject-record", type=PosixPath, required=True)
+    r.add_argument("--prefix", type=PosixPath, required=True)
    parser.add_argument("varargs", nargs="+")
    return parser

--- a/pyproject_save_files_test_data.yaml
+++ b/pyproject_save_files_test_data.yaml
@ -217,6 +217,7 @@ classified:
      files:
      - /usr/bin/tldr
      - /usr/bin/tldr.py
+      - /usr/share/man/man1/tldr*
  ipykernel:
    metadata:
      dirs:
@ -376,6 +377,7 @@ classified:
      - /usr/share/jupyter/kernels/python3/logo-64x64.png
      - /usr/share/jupyter/kernels/python3/logo-32x32.png
      - /usr/share/jupyter/kernels/python3/kernel.json
+      - /usr/man/man5/ipykernel.5*
  zope:
    metadata:
      dirs:
@ -7517,6 +7519,7 @@ dumped:
    - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/WHEEL
    - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/top_level.txt
    - /usr/lib/python3.7/site-packages/tldr.py
+    - /usr/share/man/man1/tldr*
  - - tldr
 - - mistune
  - mistune
@ -7675,6 +7678,7 @@ dumped:
    - /usr/lib/python3.7/site-packages/ipykernel/tests/utils.py
    - /usr/lib/python3.7/site-packages/ipykernel/trio_runner.py
    - /usr/lib/python3.7/site-packages/ipykernel/zmqshell.py
+    - /usr/man/man5/ipykernel.5*
    - /usr/share/jupyter/kernels/python3/kernel.json
    - /usr/share/jupyter/kernels/python3/logo-32x32.png
    - /usr/share/jupyter/kernels/python3/logo-64x64.png
@ -15559,6 +15563,7 @@ records:
      ../../../bin/__pycache__/tldr.cpython-37.pyc,,
      ../../../bin/tldr,sha256=6MUiLCWhldmV8OelT2dvPgS7q5GFwuhvd6th0Bb-LH4,12766
      ../../../bin/tldr.py,sha256=6MUiLCWhldmV8OelT2dvPgS7q5GFwuhvd6th0Bb-LH4,12766
+      ../../../share/man/man1/tldr.bz2,sha256=xp_kqadh3PjDb4OaU8D8RQDcakrwl5AMmCnaOUV7ioo,10957
      __pycache__/tldr.cpython-37.pyc,,
      tldr-0.5.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
      tldr-0.5.dist-info/LICENSE,sha256=q7quAfjDWCYKC_WRk_uaP6d2wwVpOpVjUSkv8l6H7xI,1075
@ -15587,6 +15592,7 @@ records:
      ../../../share/jupyter/kernels/python3/kernel.json,sha256=7o0-HNZRKjrk7Fqb71O3gptCssqWqfd_sxw5FNFeYO0,143
      ../../../share/jupyter/kernels/python3/logo-32x32.png,sha256=4ytcKCBy1xeIe2DacxeP3TWmXcPK6sunoCblpCVcyZc,1084
      ../../../share/jupyter/kernels/python3/logo-64x64.png,sha256=XJBjtDbO3wVnSA_kh-zg0UeeqVRfMQy6k_oYTMurKQ0,2180
+      ../../../man/man5/ipykernel.5,sha256=xp_kqadh3PjDb4OaU8D8RQDcakrwl5AMmCnaOUV7ioo,10957
      __pycache__/ipykernel_launcher.cpython-37.pyc,,
      ipykernel-5.2.1.dist-info/COPYING.md,sha256=YMWypaSJDUjGk7i5CKSWdbUkuErBWn7ByVY-Bea__ho,2835
      ipykernel-5.2.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
--- a/test_pyproject_save_files.py
+++ b/test_pyproject_save_files.py
@ -11,6 +11,7 @@ from pyproject_save_files import main as save_files_main
 from pyproject_save_files import module_names_from_path

 DIR = Path(__file__).parent
+PREFIX = Path("/usr")
 BINDIR = BuildrootPath("/usr/bin")
 DATADIR = BuildrootPath("/usr/share")
 SITELIB = BuildrootPath("/usr/lib/python3.7/site-packages")
@ -110,7 +111,15 @@ def test_parse_record_tensorflow():


 def remove_others(expected):
-    return [p for p in expected if not (p.startswith(str(BINDIR)) or p.endswith(".pth") or p.rpartition(' ')[-1].startswith(str(DATADIR)))]
+    return [
+        p for p in expected
+        if not (
+            p.startswith(str(BINDIR)) or
+            p.endswith(".pth") or
+            p.endswith("*") or
+            p.rpartition(' ')[-1].startswith(str(DATADIR))
+        )
+    ]


@pytest.mark.parametrize("include_auto", (True, False))
@ -179,7 +188,9 @@ def default_options(output_files, output_modules, mock_root, pyproject_record):
        "--python-version",
        "3.7",  # test data are for 3.7,
        "--pyproject-record",
-        str(pyproject_record)
+        str(pyproject_record),
+        "--prefix",
+        str(PREFIX),
    ]


--- a/tests/python-getmac.spec
+++ b/tests/python-getmac.spec
@ -0,0 +1,50 @@
+Name:           python-getmac
+Version:        0.8.3
+Release:        0%{?dist}
+Summary:        Get MAC addresses of remote hosts and local interfaces
+License:        MIT
+URL:            https://github.com/GhostofGoes/getmac
+Source0:        %{pypi_source getmac}
+
+BuildArch:      noarch
+BuildRequires:  python3-devel
+BuildRequires:  pyproject-rpm-macros
+
+
+%global _description %{expand:
+Test that manpages are correctly processed by %%pyproject_save_files '*' +auto.}
+
+
+%description %_description
+
+%package -n     python3-getmac
+Summary:        %{summary}
+
+%description -n python3-getmac %_description
+
+
+%prep
+%autosetup -p1 -n getmac-%{version}
+
+
+%generate_buildrequires
+%pyproject_buildrequires -r
+
+
+%build
+%pyproject_wheel
+
+
+%install
+%pyproject_install
+%pyproject_save_files '*' +auto
+
+
+%check
+%pyproject_check_import
+# Internal check for our macros, assert there is a manpage:
+test -f %{buildroot}%{_mandir}/man1/getmac.1*
+
+
+%files -n python3-getmac -f %{pyproject_files}
+
--- a/tests/tests.yml
+++ b/tests/tests.yml
@ -76,6 +76,9 @@
    - markupsafe:
        dir: .
        run: ./mocktest.sh python-markupsafe
+    - getmac:
+        dir: .
+        run: ./mocktest.sh python-getmac
    - double_install:
        dir: .
        run: ./mocktest.sh double-install