Include nested __pycache__ directories in %pyproject_save_files

pyproject-rpm-macros.spec

@@ -6,7 +6,7 @@ License:        MIT
 
 # Keep the version at zero and increment only release
 Version:        0
-Release:        37%{?dist}
+Release:        38%{?dist}
 
 # Macro files
 Source001:      macros.pyproject
@@ -104,6 +104,10 @@ export HOSTNAME="rpmbuild"  # to speedup tox in network-less mock, see rhbz#1856
 %license LICENSE
 
 %changelog
+* Sun Feb 07 2021 Miro Hrončok <mhroncok@redhat.com> - 0-38
+- Include nested __pycache__ directories in %%pyproject_save_files
+- Fixes: rhbz#1925963
+
 * Tue Feb 02 2021 Miro Hrončok <mhroncok@redhat.com> - 0-37
 - Remove support for Python 3.7 from %%pyproject_buildrequires
 - Generate python3dist(toml) BR with pyproject.toml earlier to avoid extra install round

pyproject_save_files.py

@@ -55,6 +55,21 @@ class BuildrootPath(PurePosixPath):
         return type(self)(os.path.normpath(self))
 
 
+def pycache_dir(script):
+    """
+    For a script BuildrootPath, return a BuildrootPath of its __pycache__ directory.
+
+    Example:
+
+        >>> pycache_dir(BuildrootPath('/whatever/bar.py'))
+        BuildrootPath('/whatever/__pycache__')
+
+        >>> pycache_dir(BuildrootPath('/opt/python3.10/foo.py'))
+        BuildrootPath('/opt/python3.10/__pycache__')
+    """
+    return script.parent / "__pycache__"
+
+
 def pycached(script, python_version):
     """
     For a script BuildrootPath, return a list with that path and its bytecode glob.
@@ -73,7 +88,7 @@ def pycached(script, python_version):
     assert script.suffix == ".py"
     pyver = "".join(python_version.split(".")[:2])
     pycname = f"{script.stem}.cpython-{pyver}{{,.opt-?}}.pyc"
-    pyc = script.parent / "__pycache__" / pycname
+    pyc = pycache_dir(script) / pycname
     return [script, pyc]
 
 
@@ -92,6 +107,18 @@ def add_file_to_module(paths, module_name, module_type, files_dirs, *files):
         )
 
 
+def add_py_file_to_module(paths, module_name, module_type, path, python_version,
+                          *, include_pycache_dir):
+    """
+    Helper procedure, adds given .py file to the module_name of a given module_type
+    Always also adds the bytecode cache.
+    If include_pycache_dir is set, also include the __pycache__ directory.
+    """
+    add_file_to_module(paths, module_name, module_type, "files", *pycached(path, python_version))
+    if include_pycache_dir:
+        add_file_to_module(paths, module_name, module_type, "dirs", pycache_dir(path))
+
+
 def add_lang_to_module(paths, module_name, path):
     """
     Helper procedure, divides lang files by language and adds them to the module_name
@@ -162,8 +189,10 @@ def classify_paths(
                         add_file_to_module(paths, name, "extension", "files", path)
                     elif path.suffix == ".py":
                         name = path.stem
-                        add_file_to_module(
-                            paths, name, "script", "files", *pycached(path, python_version)
+                        # we add the .pyc files, but not top-level __pycache__
+                        add_py_file_to_module(
+                            paths, name, "script", path, python_version,
+                            include_pycache_dir=False
                         )
                     else:
                         paths["other"]["files"].append(path)
@@ -177,8 +206,14 @@ def classify_paths(
                     if path.suffix == ".mo":
                         is_lang = add_lang_to_module(paths, module_dir.name, path)
                     if not is_lang:
-                        path = pycached(path, python_version) if path.suffix == ".py" else [path]
-                        add_file_to_module(paths, module_dir.name, "package", "files", *path)
+                        if path.suffix == ".py":
+                            # we add the .pyc files, and their __pycache__
+                            add_py_file_to_module(
+                                paths, module_dir.name, "package", path, python_version,
+                                include_pycache_dir=True
+                            )
+                        else:
+                            add_file_to_module(paths, module_dir.name, "package", "files", path)
                 break
         else:
             if path.suffix == ".mo":

tests/printrun.spec

@@ -46,6 +46,9 @@ grep '^%%lang' %{pyproject_files} | sort > tested.lang
 sort pronterface.lang plater.lang > expected.lang
 diff tested.lang expected.lang
 
+# Internal check that generated files contain nested __pycache__ directories
+grep -E '/printrun/__pycache__$' %{pyproject_files}
+
 
 %files -f %{pyproject_files}
 %doc README*

tests/python-isort.spec

@@ -47,7 +47,7 @@ test -d %{buildroot}%{python3_sitelib}/%{modname}/
 test -d %{buildroot}%{python3_sitelib}/%{modname}-%{version}.dist-info/
 
 # Internal check that executables are not present when +auto was not used with %%pyproject_save_files
-! grep -F %{buildroot}%{_bindir}/%{modname} %{pyproject_files}
+! grep -F %{_bindir}/%{modname} %{pyproject_files}
 
 
 %files -n python3-%{modname} -f %{pyproject_files}

tests/python-ldap.spec

@@ -70,8 +70,8 @@ test -f %{buildroot}%{python3_sitearch}/_ldap.cpython-*.so
 ! grep -F %{python3_sitearch}/slapdtest %{pyproject_files}
 
 # Internal check: Top level __pycache__ is never owned
-! grep -E '/__pycache__$' %{pyproject_files}
-! grep -E '/__pycache__/$' %{pyproject_files}
+! grep -E '/site-packages/__pycache__$' %{pyproject_files}
+! grep -E '/site-packages/__pycache__/$' %{pyproject_files}
 
 
 %files -n python3-ldap -f %{pyproject_files}