From 50645e10a3fa2710edd0abf86af5f259986ffe7a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 17 Jul 2019 15:44:22 +0200 Subject: [PATCH] Refactor and add tests --- .gitignore | 1 + README.md | 4 + macros.pyproject | 2 +- pyproject-rpm-macros.spec | 15 ++ pyproject_buildrequires.py | 266 ++++++++++++++++++++++++-------- test_pyproject_buildrequires.py | 40 +++++ testcases.yaml | 119 ++++++++++++++ 7 files changed, 379 insertions(+), 68 deletions(-) create mode 100644 test_pyproject_buildrequires.py create mode 100644 testcases.yaml diff --git a/.gitignore b/.gitignore index e69de29..c18dd8d 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/README.md b/README.md index ede594e..3e14c00 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,10 @@ We plan to preserve existing Python flags in shebangs, but the work is not yet f The PEPs don't (yet) define a way to specify test dependencies and test runners. That means you still need to handle test dependencies and `%check` on your own. +Extras are currently ignored. + +Some valid Python version specifiers are not supported. + [PEP 517]: https://www.python.org/dev/peps/pep-0517/ [PEP 518]: https://www.python.org/dev/peps/pep-0518/ diff --git a/macros.pyproject b/macros.pyproject index ffc1916..6679b8e 100644 --- a/macros.pyproject +++ b/macros.pyproject @@ -23,6 +23,6 @@ echo 'python3dist(packaging)' echo 'python3dist(pip) >= 19' echo 'python3dist(pytoml)' if [ -f %{__python3} ]; then - %{__python3} -I %{_rpmconfigdir}/redhat/pyproject_buildrequires.py + %{__python3} -I %{_rpmconfigdir}/redhat/pyproject_buildrequires.py %{?*} fi } diff --git a/pyproject-rpm-macros.spec b/pyproject-rpm-macros.spec index 06811ac..7ca1813 100644 --- a/pyproject-rpm-macros.spec +++ b/pyproject-rpm-macros.spec @@ -12,6 +12,9 @@ Source1: pyproject_buildrequires.py Source8: README.md Source9: LICENSE +Source10: test_pyproject_buildrequires.py +Source11: testcases.yaml + URL: https://src.fedoraproject.org/rpms/pyproject-rpm-macros BuildArch: noarch @@ -23,6 +26,14 @@ BuildArch: noarch Requires: python3-pip >= 19 Requires: python3-devel +# Test dependencies +BuildRequires: python3dist(pytest) +BuildRequires: python3dist(pyyaml) +BuildRequires: python3dist(packaging) +BuildRequires: python3dist(pytoml) +BuildRequires: python3dist(pip) + + %description This is a provisional implementation of pyproject RPM macros for Fedora 30+. These macros are useful for packaging Python projects that use the PEP 517 @@ -45,6 +56,10 @@ mkdir -p %{buildroot}%{_rpmconfigdir}/redhat install -m 644 macros.pyproject %{buildroot}%{_rpmmacrodir}/ install -m 644 pyproject_buildrequires.py %{buildroot}%{_rpmconfigdir}/redhat/ +%check +%{__python3} -m pytest -vv + + %files %{_rpmmacrodir}/macros.pyproject %{_rpmconfigdir}/redhat/pyproject_buildrequires.py diff --git a/pyproject_buildrequires.py b/pyproject_buildrequires.py index 5e23411..53f2452 100644 --- a/pyproject_buildrequires.py +++ b/pyproject_buildrequires.py @@ -1,84 +1,216 @@ import sys import importlib +import argparse +import functools +import traceback +import contextlib +from io import StringIO +import subprocess +import pathlib +import re + +print_err = functools.partial(print, file=sys.stderr) + +# Some valid Python version specifiers are not supported. +# Whitelist characters we can handle. +VERSION_RE = re.compile('[a-zA-Z0-9.-]+') + +class EndPass(Exception): + """End current pass of generating requirements""" try: import pytoml from packaging.requirements import Requirement, InvalidRequirement + from packaging.version import Version from packaging.utils import canonicalize_name, canonicalize_version -except ImportError: + import pip +except ImportError as e: + print_err('Import error:', e) # already echoed by the %pyproject_buildrequires macro sys.exit(0) -try: - f = open("pyproject.toml") as f -except FileNotFoundError: - pyproject_data = {} -else: - with f: - pyproject_data = pytoml.load(f) +@contextlib.contextmanager +def hook_call(): + captured_out = StringIO() + with contextlib.redirect_stdout(captured_out): + yield + for line in captured_out.getvalue().splitlines(): + print_err('HOOK STDOUT:', line) + + +class Requirements: + """Requirement printer""" + def __init__(self, freeze_output): + self.installed_packages = {} + for line in freeze_output.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + name, version = line.split('==') + self.installed_packages[name.strip()] = Version(version) + + self.missing_requirements = False + + def add(self, requirement_str, *, source=None): + """Output a Python-style requirement string as RPM dep""" + print_err(f'Handling {requirement_str} from {source}') - try: - backend_name = pyproject_data["build-system"]["build-backend"] - except KeyError: try: - import setuptools.build_meta - except ImportError: - print("python3dist(setuptools) >= 40.8") - print("python3dist(wheel)") - sys.exit(0) - - backend = setuptools.build_meta - else: - try: - backend = importlib.import_module(backend_name) - except ImportError: - backend = None - - -requirements = set() -rpm_requirements = set() - - -def add_requirement(requirement): - try: - requirements.add(Requirement(requirement)) - except InvalidRequirement as e: - print( - f"WARNING: Skipping invalid requirement: {requirement}\n {e}", - file=sys.stderr, - ) - - -if "requires" in pyproject_data.get("build-system", {}): - for requirement in pyproject_data["build-system"]["requires"]: - add_requirement(requirement) - - -get_requires = getattr(backend, "get_requires_for_build_wheel", None) -if get_requires: - for requirement in get_requires(): - add_requirement(requirement) - -for requirement in requirements: - name = canonicalize_name(requirement.name) - if requirement.marker is not None and not requirement.marker.evaluate(): - continue - together = [] - for specifier in requirement.specifier: - version = canonicalize_version(specifier.version) - if specifier.operator == "!=": - together.append( - f"(python3dist({name}) < {version} or python3dist({name}) >= {version}.0)" + requirement = Requirement(requirement_str) + except InvalidRequirement as e: + print_err( + f'"WARNING: Skipping invalid requirement: {requirement_str}\n' + + f' {e}', ) + return + + name = canonicalize_name(requirement.name) + if requirement.marker is not None and not requirement.marker.evaluate(): + print_err(f'Ignoring alien requirement:', requirement_str) + return + + installed = self.installed_packages.get(requirement.name) + if installed and installed in requirement.specifier: + print_err(f'Requirement satisfied: {requirement_str}') + print_err(f' (installed: {requirement.name} {installed})') else: - together.append(f"python3dist({name}) {specifier.operator} {version}") - if len(together) == 0: - rpm_requirements.add(f"python3dist({name})") - if len(together) == 1: - rpm_requirements.add(together[0]) - elif len(together) > 1: - rpm_requirements.add(f"({' and '.join(together)})") + self.missing_requirements = True + + together = [] + for specifier in sorted( + requirement.specifier, + key=lambda s: (s.operator, s.version), + ): + version = canonicalize_version(specifier.version) + print_err(version) + if not VERSION_RE.fullmatch(str(specifier.version)): + raise ValueError( + f'Unknown character in version: {specifier.version}. ' + + '(This is probably a bug in pyproject-rpm-macros.)', + ) + if specifier.operator == "!=": + lower = python3dist(name, '<', version) + higher = python3dist(name, '>', f'{version}.0') + together.append( + f"({lower} or {higher})" + ) + else: + together.append(python3dist(name, specifier.operator, version)) + if len(together) == 0: + print(python3dist(name)) + elif len(together) == 1: + print(together[0]) + else: + print(f"({' and '.join(together)})") + + def check(self, *, source=None): + """End current pass if any unsatisfied dependencies were output""" + if self.missing_requirements: + print_err(f'Exiting dependency generation pass: {source}') + raise EndPass(source) + + def extend(self, requirement_strs, *, source=None): + """add() several requirements""" + for req_str in requirement_strs: + self.add(req_str, source=source) + +def get_backend(requirements): + try: + f = open('pyproject.toml') + except FileNotFoundError: + pyproject_data = {} + else: + with f: + pyproject_data = pytoml.load(f) + + buildsystem_data = pyproject_data.get("build-system", {}) + requirements.extend( + buildsystem_data.get("requires", ()), + source='build-system.requires', + ) + + backend_name = buildsystem_data.get('build-backend') + if not backend_name: + requirements.add("setuptools >= 40.8", source='default build backend') + requirements.add("wheel", source='default build backend') + + backend_name = 'setuptools.build_meta' + + requirements.check(source='build backend') + + backend_path = buildsystem_data.get('backend-path') + if backend_path: + sys.path.insert(0, backend_path) + + return importlib.import_module(backend_name) -print(*sorted(rpm_requirements), sep="\n") +def generate_build_requirements(backend, requirements): + get_requires = getattr(backend, "get_requires_for_build_wheel", None) + if get_requires: + with hook_call(): + new_reqs = get_requires() + requirements.extend(new_reqs, source='get_requires_for_build_wheel') + + +def python3dist(name, op=None, version=None): + if op is None: + if version is not None: + raise AssertionError('op and version go together') + return f'python3dist({name})' + else: + return f'python3dist({name}) {op} {version}' + + +def generate_requires(freeze_output): + requirements = Requirements(freeze_output) + + try: + backend = get_backend(requirements) + generate_build_requirements(backend, requirements) + except EndPass: + return 0 + + +def main(argv): + parser = argparse.ArgumentParser( + description='Generate BuildRequires for a Python project.' + ) + parser.add_argument( + '--runtime', action='store_true', + help='Generate run-time requirements (not implemented)', + ) + parser.add_argument( + '--toxenv', metavar='TOXENVS', + help='generate test tequirements from tox environment ' + + '(not implemented; implies --runtime)', + ) + parser.add_argument( + '--pyproject-file', default='pyproject.toml', + help='override project file (default: pyproject.toml)', + ) + + args = parser.parse_args(argv) + if args.toxenv: + args.runtime = True + if args.runtime: + print_err('--runtime is not implemented') + exit(1) + + freeze_output = subprocess.run( + ['pip', 'freeze', '--all'], + stdout=subprocess.PIPE, + check=True, + ).stdout + + try: + generate_requires(freeze_output) + except Exception as e: + # Log the traceback explicitly (it's useful debug info) + traceback.print_exc() + exit(1) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/test_pyproject_buildrequires.py b/test_pyproject_buildrequires.py new file mode 100644 index 0000000..edc17c5 --- /dev/null +++ b/test_pyproject_buildrequires.py @@ -0,0 +1,40 @@ +from pathlib import Path +import io + +import pytest +import yaml + +from pyproject_buildrequires import generate_requires + +testcases = {} +with Path(__file__).parent.joinpath('testcases.yaml').open() as f: + testcases = yaml.safe_load(f) + + +@pytest.mark.parametrize('case_name', testcases) +def test_data(case_name, capsys, tmp_path, monkeypatch): + case = testcases[case_name] + + cwd = tmp_path.joinpath('cwd') + cwd.mkdir() + monkeypatch.chdir(cwd) + + if 'pyproject.toml' in case: + cwd.joinpath('pyproject.toml').write_text(case['pyproject.toml']) + + if 'setup.py' in case: + cwd.joinpath('setup.py').write_text(case['setup.py']) + + try: + generate_requires( + case['freeze_output'], + ) + except SystemExit as e: + assert e.code == case['result'] + except Exception as e: + assert type(e).__name__ == case['except'] + else: + assert 0 == case['result'] + + captured = capsys.readouterr() + assert captured.out == case['expected'] diff --git a/testcases.yaml b/testcases.yaml new file mode 100644 index 0000000..6b6d5fc --- /dev/null +++ b/testcases.yaml @@ -0,0 +1,119 @@ +No pyproject.toml, nothing installed: + freeze_output: | + # empty + expected: | + python3dist(setuptools) >= 40.8 + python3dist(wheel) + result: 0 + +Nothing installed yet: + freeze_output: | + # empty + pyproject.toml: | + # empty + expected: | + python3dist(setuptools) >= 40.8 + python3dist(wheel) + result: 0 + +Insufficient version of setuptools: + freeze_output: | + setuptools==5 + wheel==1 + pyproject.toml: | + # empty + expected: | + python3dist(setuptools) >= 40.8 + python3dist(wheel) + result: 0 + +Empty pyproject.toml, empty setup.py: + freeze_output: | + setuptools==50 + wheel==1 + setup.py: | + expected: | + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + result: 0 + +Default build system, empty setup.py: + freeze_output: | + setuptools==50 + wheel==1 + pyproject.toml: | + # empty + setup.py: | + expected: | + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + result: 0 + +Erroring setup.py: + freeze_output: | + setuptools==50 + wheel==1 + setup.py: | + exit(77) + result: 77 + +Bad character in version: + freeze_output: | + pyproject.toml: | + [build-system] + requires = ["pkg == 0.$.^.*"] + except: ValueError + +Build system dependencies in pyproject.toml: + freeze_output: | + setuptools==50 + wheel==1 + pyproject.toml: | + [build-system] + requires = [ + "foo", + "ne!=1", + "ge>=1.2", + "le <= 1.2.3", + "lt < 1.2.3.4 ", + " gt > 1.2.3.4.5", + "combo >2, <5, != 3.0.0", + "invalid!!ignored", + "py2 ; python_version < '2.7'", + "py3 ; python_version > '3.0'", + "pkg [extra-currently-ignored]", + ] + expected: | + python3dist(foo) + (python3dist(ne) < 1 or python3dist(ne) > 1.0) + python3dist(ge) >= 1.2 + python3dist(le) <= 1.2.3 + python3dist(lt) < 1.2.3.4 + python3dist(gt) > 1.2.3.4.5 + ((python3dist(combo) < 3 or python3dist(combo) > 3.0) and python3dist(combo) < 5 and python3dist(combo) > 2) + python3dist(py3) + python3dist(pkg) + python3dist(setuptools) >= 40.8 + python3dist(wheel) + result: 0 + +Default build system, dependencies in setup.py: + freeze_output: | + setuptools==50 + wheel==1 + setup.py: | + from setuptools import setup + setup( + name='test', + version='0.1', + setup_requires=['foo', 'bar!=2'], + ) + expected: | + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + python3dist(foo) + (python3dist(bar) < 2 or python3dist(bar) > 2.0) + result: 0