Redirect stdout to stderr via Shell

Dependencies are recorded to a text file that is catted at the end.

This should prevent subtle bugs like https://bugzilla.redhat.com/2183519 in the future.

macros.pyproject

@@ -19,6 +19,7 @@
 %_pyproject_modules %{_builddir}/%{_pyproject_files_prefix}-pyproject-modules
 %_pyproject_ghost_distinfo %{_builddir}/%{_pyproject_files_prefix}-pyproject-ghost-distinfo
 %_pyproject_record %{_builddir}/%{_pyproject_files_prefix}-pyproject-record
+%_pyproject_buildrequires %{_builddir}/%{_pyproject_files_prefix}-pyproject-buildrequires
 
 # Avoid leaking %%{_pyproject_builddir} to pytest collection
 # https://bugzilla.redhat.com/show_bug.cgi?id=1935212
@@ -169,8 +170,10 @@ fi}
 rm -rfv *.dist-info/ >&2
 if [ -f %{__python3} ]; then
   mkdir -p "%{_pyproject_builddir}"
+  echo -n > %{_pyproject_buildrequires}
   CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}" TMPDIR="%{_pyproject_builddir}" \\\
-  RPM_TOXENV="%{toxenv}" HOSTNAME="rpmbuild" %{__python3} -Bs %{_rpmconfigdir}/redhat/pyproject_buildrequires.py %{?!_python_no_extras_requires:--generate-extras} --python3_pkgversion %{python3_pkgversion} --wheeldir %{_pyproject_wheeldir} %{?**}
+  RPM_TOXENV="%{toxenv}" HOSTNAME="rpmbuild" %{__python3} -Bs %{_rpmconfigdir}/redhat/pyproject_buildrequires.py %{?!_python_no_extras_requires:--generate-extras} --python3_pkgversion %{python3_pkgversion} --wheeldir %{_pyproject_wheeldir} --output %{_pyproject_buildrequires} %{?**} >&2
+  cat %{_pyproject_buildrequires}
 fi
 # Incomplete .dist-info dir might confuse importlib.metadata
 rm -rfv *.dist-info/ >&2

pyproject-rpm-macros.spec

@@ -10,7 +10,7 @@ License:        MIT
 #   Increment Y and reset Z when new macros or features are added
 #   Increment Z when this is a bugfix or a cosmetic change
 # Dropping support for EOL Fedoras is *not* considered a breaking change
-Version:        1.6.3
+Version:        1.7.0
 Release:        1%{?dist}
 
 # Macro files
@@ -147,6 +147,11 @@ export HOSTNAME="rpmbuild"  # to speedup tox in network-less mock, see rhbz#1856
 
 
 %changelog
+* Fri Mar 31 2023 Miro Hrončok <mhroncok@redhat.com> - 1.7.0-1
+- %%pyproject_buildrequires: Redirect stdout to stderr via Shell
+- Dependencies are recorded to a text file that is catted at the end
+- Fixes: rhbz#2183519
+
 * Mon Feb 13 2023 Lumír Balhar <lbalhar@redhat.com> - 1.6.3-1
 - Remove .dist-info directory at the end of %%pyproject_buildrequires
 - An incomplete .dist-info directory in $PWD can confuse tests in %%check

pyproject_buildrequires.py

@@ -5,7 +5,6 @@ import sys
 import importlib.metadata
 import argparse
 import traceback
-import contextlib
 import json
 import subprocess
 import re
@@ -45,39 +44,6 @@ except ImportError as e:
 from pyproject_convert import convert
 
 
-@contextlib.contextmanager
-def hook_call():
-    """Context manager that records all stdout content (on FD level)
-    and prints it to stderr at the end, with a 'HOOK STDOUT: ' prefix."""
-    tmpfile = io.TextIOWrapper(
-        tempfile.TemporaryFile(buffering=0),
-        encoding='utf-8',
-        errors='replace',
-        write_through=True,
-    )
-
-    stdout_fd = 1
-    stdout_fd_dup = os.dup(stdout_fd)
-    stdout_orig = sys.stdout
-
-    # begin capture
-    sys.stdout = tmpfile
-    os.dup2(tmpfile.fileno(), stdout_fd)
-
-    try:
-        yield
-    finally:
-        # end capture
-        sys.stdout = stdout_orig
-        os.dup2(stdout_fd_dup, stdout_fd)
-
-        tmpfile.seek(0)  # rewind
-        for line in tmpfile:
-            print_err('HOOK STDOUT:', line, end='')
-
-        tmpfile.close()
-
-
 def guess_reason_for_invalid_requirement(requirement_str):
     if ':' in requirement_str:
         message = (
@@ -99,10 +65,11 @@ def guess_reason_for_invalid_requirement(requirement_str):
 
 
 class Requirements:
-    """Requirement printer"""
+    """Requirement gatherer. The macro will eventually print out output_lines."""
     def __init__(self, get_installed_version, extras=None,
                  generate_extras=False, python3_pkgversion='3'):
         self.get_installed_version = get_installed_version
+        self.output_lines = []
         self.extras = set()
 
         if extras:
@@ -191,12 +158,12 @@ class Requirements:
                 together.append(convert(python3dist(name, python3_pkgversion=self.python3_pkgversion),
                                         specifier.operator, specifier.version))
             if len(together) == 0:
-                print(python3dist(name,
+                dep = python3dist(name, python3_pkgversion=self.python3_pkgversion)
-                                  python3_pkgversion=self.python3_pkgversion))
+                self.output_lines.append(dep)
             elif len(together) == 1:
-                print(together[0])
+                self.output_lines.append(together[0])
             else:
-                print(f"({' with '.join(together)})")
+                self.output_lines.append(f"({' with '.join(together)})")
 
     def check(self, *, source=None):
         """End current pass if any unsatisfied dependencies were output"""
@@ -284,8 +251,7 @@ def get_backend(requirements):
 def generate_build_requirements(backend, requirements):
     get_requires = getattr(backend, 'get_requires_for_build_wheel', None)
     if get_requires:
-        with hook_call():
+        new_reqs = get_requires()
-            new_reqs = get_requires()
         requirements.extend(new_reqs, source='get_requires_for_build_wheel')
         requirements.check(source='get_requires_for_build_wheel')
 
@@ -305,8 +271,7 @@ def generate_run_requirements_hook(backend, requirements):
             'Use the provisional -w flag to build the wheel and parse the metadata from it, '
             'or use the -R flag not to generate runtime dependencies.'
         )
-    with hook_call():
+    dir_basename = prepare_metadata('.')
-        dir_basename = prepare_metadata('.')
     with open(dir_basename + '/METADATA') as metadata_file:
         for key, requires in requires_from_metadata_file(metadata_file).items():
             requirements.extend(requires, source=f'hook generated metadata: {key}')
@@ -411,10 +376,13 @@ def python3dist(name, op=None, version=None, python3_pkgversion="3"):
 def generate_requires(
     *, include_runtime=False, build_wheel=False, wheeldir=None, toxenv=None, extras=None,
     get_installed_version=importlib.metadata.version,  # for dep injection
-    generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True
+    generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True,
+    output,
 ):
     """Generate the BuildRequires for the project in the current directory
 
+    The generated BuildRequires are written to the provided output.
+
     This is the main Python entry point.
     """
     requirements = Requirements(
@@ -443,6 +411,8 @@ def generate_requires(
             generate_run_requirements(backend, requirements, build_wheel=build_wheel, wheeldir=wheeldir)
     except EndPass:
         return
+    finally:
+        output.write_text(os.linesep.join(requirements.output_lines) + os.linesep)
 
 
 def main(argv):
@@ -468,6 +438,9 @@ def main(argv):
         '-p', '--python3_pkgversion', metavar='PYTHON3_PKGVERSION',
         default="3", help=argparse.SUPPRESS,
     )
+    parser.add_argument(
+        '--output', type=pathlib.Path, required=True, help=argparse.SUPPRESS,
+    )
     parser.add_argument(
         '--wheeldir', metavar='PATH', default=None,
         help=argparse.SUPPRESS,
@@ -538,6 +511,7 @@ def main(argv):
             python3_pkgversion=args.python3_pkgversion,
             requirement_files=args.requirement_files,
             use_build_system=args.use_build_system,
+            output=args.output,
         )
     except Exception:
         # Log the traceback explicitly (it's useful debug info)

pyproject_buildrequires_testcases.yaml

@@ -820,7 +820,7 @@ Pre-releases are accepted:
   result: 0
 
 
-Wrapped subprocess prints to stdout from setup.py:
+Stdout from wrapped subprocess does not appear in output:
   installed:
     setuptools: 50
     wheel: 1
@@ -834,5 +834,4 @@ Wrapped subprocess prints to stdout from setup.py:
     python3dist(setuptools) >= 40.8
     python3dist(wheel)
     python3dist(wheel)
-  stderr_contains: "HOOK STDOUT: LEAK?"
   result: 0

test_pyproject_buildrequires.py

@@ -21,6 +21,7 @@ def test_data(case_name, capfd, tmp_path, monkeypatch):
     monkeypatch.chdir(cwd)
     wheeldir = cwd.joinpath('wheeldir')
     wheeldir.mkdir()
+    output = tmp_path.joinpath('output.txt')
 
     if case.get('xfail'):
         pytest.xfail(case.get('xfail'))
@@ -54,6 +55,7 @@ def test_data(case_name, capfd, tmp_path, monkeypatch):
             generate_extras=case.get('generate_extras', False),
             requirement_files=requirement_files,
             use_build_system=use_build_system,
+            output=output,
         )
     except SystemExit as e:
         assert e.code == case['result']
@@ -69,14 +71,15 @@ def test_data(case_name, capfd, tmp_path, monkeypatch):
         assert 'expected' in case or 'stderr_contains' in case
 
         out, err = capfd.readouterr()
+        dependencies = output.read_text()
 
         if 'expected' in case:
             expected = case['expected']
             if isinstance(expected, list):
                 # at least one of them needs to match
-                assert any(out == e for e in expected)
+                assert any(dependencies == e for e in expected)
             else:
-                assert out == expected
+                assert dependencies == expected
 
         # stderr_contains may be a string or list of strings
         stderr_contains = case.get('stderr_contains')

tests/python-pytest.spec

@@ -37,6 +37,9 @@ Summary:        %{summary}
 %autosetup -p1 -n %{pypi_name}-%{version}
 # remove optional test dependencies we don't like to pull in
 sed -E -i '/mock|nose/d' setup.cfg
+# internal check for our macros: insert a subprocess echo to setup.py
+# to ensure it's not generated as BuildRequires
+echo 'import os; os.system("echo if-this-is-generated-the-build-will-fail")' >> setup.py
 
 
 %generate_buildrequires