Handle Python Extras in %pyproject_buildrequires on Fedora 33+

There is a slight problem when reporting that a dependency with extra is satisfied. In fact, we only check the "base" dependency. This can lead to a problem when a dependency is wrongly assumed as present and the script proceeds to the "next stage" without restarting -- if the next stage tries to use (import) the missing dependency, the script would crash. However, that might be a very unlikely set of events and if such case ever happens, we'll workaround it or fix it.
2020-08-14 14:57:40 +02:00 · 2020-08-14 14:57:40 +02:00 · a613e176e3
commit a613e176e3
parent 91acc88e2d
7 changed files with 154 additions and 35 deletions
--- a/macros.pyproject
+++ b/macros.pyproject
@ -79,10 +79,16 @@ echo 'python%{python3_pkgversion}dist(toml)'
 if [ ! -z "%{?python3_version_nodots}" ] && [ %{python3_version_nodots} -lt 38 ]; then
  echo 'python%{python3_pkgversion}dist(importlib-metadata)'
 fi
+# Check if we can generate dependencies on Python extras
+if [ "%{py_dist_name []}" == "[]" ]; then
+  extras_flag=%{?!_python_no_extras_requires:--generate-extras}
+else
+  extras_flag=
+fi
 # setuptools assumes no pre-existing dist-info
 rm -rfv *.dist-info/ >&2
 if [ -f %{__python3} ]; then
-  RPM_TOXENV="%{toxenv}" HOSTNAME="rpmbuild" %{__python3} -I %{_rpmconfigdir}/redhat/pyproject_buildrequires.py --python3_pkgversion %{python3_pkgversion} %{?**}
+  RPM_TOXENV="%{toxenv}" HOSTNAME="rpmbuild" %{__python3} -I %{_rpmconfigdir}/redhat/pyproject_buildrequires.py $extras_flag --python3_pkgversion %{python3_pkgversion} %{?**}
 fi
 }

--- a/pyproject-rpm-macros.spec
+++ b/pyproject-rpm-macros.spec
@ -6,7 +6,7 @@ License:        MIT

 # Keep the version at zero and increment only release
 Version:        0
-Release:        24%{?dist}
+Release:        25%{?dist}

 # Macro files
 Source001:      macros.pyproject
@ -88,6 +88,9 @@ export HOSTNAME="rpmbuild"  # to speedup tox in network-less mock, see rhbz#1856
 %license LICENSE

 %changelog
+* Fri Aug 14 2020 Miro Hrončok <mhroncok@redhat.com> - 0-25
+- Handle Python Extras in %%pyproject_buildrequires on Fedora 33+
+
 * Tue Aug 11 2020 Miro Hrončok <mhroncok@redhat.com> - 0-24
 - Allow multiple, comma-separated extras in %%pyproject_buildrequires -x

--- a/pyproject_buildrequires.py
+++ b/pyproject_buildrequires.py
@ -48,7 +48,7 @@ def hook_call():
 class Requirements:
    """Requirement printer"""
    def __init__(self, get_installed_version, extras='',
-                 python3_pkgversion='3'):
+                 generate_extras=False, python3_pkgversion='3'):
        self.get_installed_version = get_installed_version

        if extras:
@ -58,6 +58,7 @@ class Requirements:

        self.missing_requirements = False

+        self.generate_extras = generate_extras
        self.python3_pkgversion = python3_pkgversion

    def evaluate_all_environamnets(self, requirement):
@ -86,6 +87,7 @@ class Requirements:
            return

        try:
+            # TODO: check if requirements with extras are satisfied
            installed = self.get_installed_version(requirement.name)
        except importlib_metadata.PackageNotFoundError:
            print_err(f'Requirement not satisfied: {requirement_str}')
@ -93,38 +95,46 @@ class Requirements:
        if installed and installed in requirement.specifier:
            print_err(f'Requirement satisfied: {requirement_str}')
            print_err(f'   (installed: {requirement.name} {installed})')
+            if requirement.extras:
+                print_err(f'   (extras are currently not checked)')
        else:
            self.missing_requirements = True

-        together = []
-        for specifier in sorted(
-            requirement.specifier,
-            key=lambda s: (s.operator, s.version),
-        ):
-            version = canonicalize_version(specifier.version)
-            if not VERSION_RE.fullmatch(str(specifier.version)):
-                raise ValueError(
-                    f'Unknown character in version: {specifier.version}. '
-                    + '(This is probably a bug in pyproject-rpm-macros.)',
-                )
-            if specifier.operator == '!=':
-                lower = python3dist(name, '<', version,
-                                    self.python3_pkgversion)
-                higher = python3dist(name, '>', f'{version}.0',
-                                     self.python3_pkgversion)
-                together.append(
-                    f'({lower} or {higher})'
-                )
-            else:
-                together.append(python3dist(name, specifier.operator, version,
-                                            self.python3_pkgversion))
-        if len(together) == 0:
-            print(python3dist(name,
-                              python3_pkgversion=self.python3_pkgversion))
-        elif len(together) == 1:
-            print(together[0])
+        if self.generate_extras:
+            extra_names = [f'{name}[{extra}]' for extra in sorted(requirement.extras)]
        else:
-            print(f"({' and '.join(together)})")
+            extra_names = []
+
+        for name in [name] + extra_names:
+            together = []
+            for specifier in sorted(
+                requirement.specifier,
+                key=lambda s: (s.operator, s.version),
+            ):
+                version = canonicalize_version(specifier.version)
+                if not VERSION_RE.fullmatch(str(specifier.version)):
+                    raise ValueError(
+                        f'Unknown character in version: {specifier.version}. '
+                        + '(This is probably a bug in pyproject-rpm-macros.)',
+                    )
+                if specifier.operator == '!=':
+                    lower = python3dist(name, '<', version,
+                                        self.python3_pkgversion)
+                    higher = python3dist(name, '>', f'{version}.0',
+                                         self.python3_pkgversion)
+                    together.append(
+                        f'({lower} or {higher})'
+                    )
+                else:
+                    together.append(python3dist(name, specifier.operator, version,
+                                                self.python3_pkgversion))
+            if len(together) == 0:
+                print(python3dist(name,
+                                  python3_pkgversion=self.python3_pkgversion))
+            elif len(together) == 1:
+                print(together[0])
+            else:
+                print(f"({' and '.join(together)})")

    def check(self, *, source=None):
        """End current pass if any unsatisfied dependencies were output"""
@ -259,7 +269,7 @@ def python3dist(name, op=None, version=None, python3_pkgversion="3"):
 def generate_requires(
    *, include_runtime=False, toxenv=None, extras='',
    get_installed_version=importlib_metadata.version,  # for dep injection
-    python3_pkgversion="3",
+    generate_extras=False, python3_pkgversion="3",
 ):
    """Generate the BuildRequires for the project in the current directory

@ -267,6 +277,7 @@ def generate_requires(
    """
    requirements = Requirements(
        get_installed_version, extras=extras,
+        generate_extras=generate_extras,
        python3_pkgversion=python3_pkgversion
    )

@ -305,6 +316,10 @@ def main(argv):
        help='comma separated list of "extras" for runtime requirements '
             '(e.g. -x testing,feature-x) (implies --runtime)',
    )
+    parser.add_argument(
+        '--generate-extras', action='store_true',
+        help='Generate build requirements on Python Extras',
+    )
    parser.add_argument(
        '-p', '--python3_pkgversion', metavar='PYTHON3_PKGVERSION',
        default="3", help=('Python version for pythonXdist()'
@ -329,6 +344,7 @@ def main(argv):
            include_runtime=args.runtime,
            toxenv=args.toxenv,
            extras=args.extras,
+            generate_extras=args.generate_extras,
            python3_pkgversion=args.python3_pkgversion,
        )
    except Exception:
--- a/pyproject_buildrequires_testcases.yaml
+++ b/pyproject_buildrequires_testcases.yaml
@ -66,7 +66,8 @@ Bad character in version:
    requires = ["pkg == 0.$.^.*"]
  except: ValueError

-Build system dependencies in pyproject.toml:
+Build system dependencies in pyproject.toml with extras:
+  generate_extras: true
  installed:
    setuptools: 50
    wheel: 1
@ -74,27 +75,50 @@ Build system dependencies in pyproject.toml:
    [build-system]
    requires = [
        "foo",
+        "bar[baz] > 5",
        "ne!=1",
        "ge>=1.2",
        "le <= 1.2.3",
        "lt < 1.2.3.4      ",
        "    gt > 1.2.3.4.5",
+        "multi[extras1,extras2] == 6.0",
        "combo >2, <5, != 3.0.0",
        "invalid!!ignored",
        "py2 ; python_version < '2.7'",
        "py3 ; python_version > '3.0'",
-        "pkg [extra-currently-ignored]",
    ]
  expected: |
    python3dist(foo)
+    python3dist(bar) > 5
+    python3dist(bar[baz]) > 5
    (python3dist(ne) < 1 or python3dist(ne) > 1.0)
    python3dist(ge) >= 1.2
    python3dist(le) <= 1.2.3
    python3dist(lt) < 1.2.3.4
    python3dist(gt) > 1.2.3.4.5
+    python3dist(multi) == 6
+    python3dist(multi[extras1]) == 6
+    python3dist(multi[extras2]) == 6
    ((python3dist(combo) < 3 or python3dist(combo) > 3.0) and python3dist(combo) < 5 and python3dist(combo) > 2)
    python3dist(py3)
-    python3dist(pkg)
+    python3dist(setuptools) >= 40.8
+    python3dist(wheel)
+  result: 0
+
+Build system dependencies in pyproject.toml without extras:
+  generate_extras: false
+  installed:
+    setuptools: 50
+    wheel: 1
+  pyproject.toml: |
+    [build-system]
+    requires = [
+        "bar[baz] > 5",
+        "multi[extras1,extras2] == 6.0",
+    ]
+  expected: |
+    python3dist(bar) > 5
+    python3dist(multi) == 6
    python3dist(setuptools) >= 40.8
    python3dist(wheel)
  result: 0
--- a/test_pyproject_buildrequires.py
+++ b/test_pyproject_buildrequires.py
@ -45,6 +45,7 @@ def test_data(case_name, capsys, tmp_path, monkeypatch):
            include_runtime=case.get('include_runtime', False),
            extras=case.get('extras', ''),
            toxenv=case.get('toxenv', None),
+            generate_extras=case.get('generate_extras', False),
        )
    except SystemExit as e:
        assert e.code == case['result']
--- a/tests/python-httpbin.spec
+++ b/tests/python-httpbin.spec
@ -0,0 +1,66 @@
+Name:           python-httpbin
+Version:        0.7.0
+Release:        0%{?dist}
+Summary:        HTTP Request & Response Service, written in Python + Flask
+License:        MIT
+URL:            https://github.com/Runscope/httpbin
+Source0:        %{url}/archive/v%{version}/httpbin-%{version}.tar.gz
+BuildArch:      noarch
+
+BuildRequires:  python3-devel
+BuildRequires:  pyproject-rpm-macros
+
+%description
+This package buildrequires a package with extra: raven[flask].
+
+
+%package -n python3-httpbin
+Summary:            %{summary}
+
+%if 0%{?fedora} < 33 && 0%{?rhel} < 9
+# Old Fedoras don't understand Python extras yet
+# This package needs raven[flask]
+# So we add the transitive dependencies manually:
+BuildRequires:  %{py3_dist blinker flask}
+Requires:       %{py3_dist blinker flask}
+%endif
+
+%description -n python3-httpbin
+%{summary}.
+
+
+%prep
+%autosetup -n httpbin-%{version}
+
+# brotlipy wrapper is not packaged, httpbin works fine with brotli
+sed -i s/brotlipy/brotli/ setup.py
+
+# update test_httpbin.py to reflect new behavior of werkzeug
+sed -i /Content-Length/d test_httpbin.py
+
+
+%generate_buildrequires
+%pyproject_buildrequires -t
+
+
+%build
+%pyproject_wheel
+
+
+%install
+%pyproject_install
+%pyproject_save_files httpbin
+
+
+%check
+%tox
+
+# Internal check for our macros
+# The runtime dependencies contain raven[flask], we assert we got them.
+# The %%tox above also dies without it, but this makes it more explicit
+%{python3} -c 'import blinker, flask'  # transitive deps
+
+
+%files -n python3-httpbin -f %{pyproject_files}
+%doc README*
+%license LICENSE*
--- a/tests/tests.yml
+++ b/tests/tests.yml
@ -31,6 +31,9 @@
    - openqa_client:
        dir: .
        run: ./mocktest.sh python-openqa_client
+    - httpbin:
+        dir: .
+        run: ./mocktest.sh python-httpbin
    - ldap:
        dir: .
        run: ./mocktest.sh python-ldap