From f21150c420419447f4842ccf5e65e0bcdb4a9719ec15d30e680a16735c0b4271 Mon Sep 17 00:00:00 2001 From: Matej Cepl Date: Tue, 20 Jun 2023 21:41:03 +0000 Subject: [PATCH] - Add bpo-37596-make-set-marshalling.patch making marshalling of `set` and `frozenset` deterministic (bsc#1211765). OBS-URL: https://build.opensuse.org/package/show/devel:languages:python:Factory/python310?expand=0&rev=92 --- CVE-2007-4559-filter-tarfile_extractall.patch | 193 +++++++++--------- bpo-37596-make-set-marshalling.patch | 96 +++++++++ python310.changes | 6 + python310.spec | 5 +- 4 files changed, 199 insertions(+), 101 deletions(-) create mode 100644 bpo-37596-make-set-marshalling.patch diff --git a/CVE-2007-4559-filter-tarfile_extractall.patch b/CVE-2007-4559-filter-tarfile_extractall.patch index f0c85bf..1c00bca 100644 --- a/CVE-2007-4559-filter-tarfile_extractall.patch +++ b/CVE-2007-4559-filter-tarfile_extractall.patch @@ -1,8 +1,17 @@ -diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst -index 311aae414ae..3864e03898d 100644 +--- + Doc/library/shutil.rst | 25 + Doc/library/tarfile.rst | 458 ++++ + Doc/whatsnew/3.10.rst | 16 + Lib/shutil.py | 17 + Lib/tarfile.py | 351 +++ + Lib/test/test_shutil.py | 40 + Lib/test/test_tarfile.py | 967 +++++++++- + Misc/NEWS.d/next/Library/2023-03-23-15-24-38.gh-issue-102953.YR4KaK.rst | 4 + 8 files changed, 1782 insertions(+), 96 deletions(-) + --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst -@@ -620,7 +620,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. +@@ -620,7 +620,7 @@ provided. They rely on the :mod:`zipfil Remove the archive format *name* from the list of supported formats. @@ -11,7 +20,7 @@ index 311aae414ae..3864e03898d 100644 Unpack an archive. *filename* is the full path of the archive. -@@ -634,6 +634,15 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. +@@ -634,6 +634,15 @@ provided. They rely on the :mod:`zipfil registered for that extension. In case none is found, a :exc:`ValueError` is raised. @@ -27,7 +36,7 @@ index 311aae414ae..3864e03898d 100644 .. audit-event:: shutil.unpack_archive filename,extract_dir,format shutil.unpack_archive .. warning:: -@@ -646,6 +655,9 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. +@@ -646,6 +655,9 @@ provided. They rely on the :mod:`zipfil .. versionchanged:: 3.7 Accepts a :term:`path-like object` for *filename* and *extract_dir*. @@ -37,17 +46,16 @@ index 311aae414ae..3864e03898d 100644 .. function:: register_unpack_format(name, extensions, function[, extra_args[, description]]) Registers an unpack format. *name* is the name of the format and -@@ -653,11 +665,14 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. +@@ -653,11 +665,14 @@ provided. They rely on the :mod:`zipfil ``.zip`` for Zip files. *function* is the callable that will be used to unpack archives. The - callable will receive the path of the archive, followed by the directory - the archive must be extracted to. -- ++ callable will receive: + - When provided, *extra_args* is a sequence of ``(name, value)`` tuples that - will be passed as keywords arguments to the callable. -+ callable will receive: -+ + - the path of the archive, as a positional argument; + - the directory the archive must be extracted to, as a positional argument; + - possibly a *filter* keyword argument, if it was given to @@ -57,11 +65,9 @@ index 311aae414ae..3864e03898d 100644 *description* can be provided to describe the format, and will be returned by the :func:`get_unpack_formats` function. -diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst -index 226513f5fc1..836444ebb34 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst -@@ -206,6 +206,38 @@ The :mod:`tarfile` module defines the following exceptions: +@@ -206,6 +206,38 @@ The :mod:`tarfile` module defines the fo Is raised by :meth:`TarInfo.frombuf` if the buffer it gets is invalid. @@ -100,7 +106,7 @@ index 226513f5fc1..836444ebb34 100644 The following constants are available at the module level: .. data:: ENCODING -@@ -316,11 +348,8 @@ be finalized; only the internally used file object will be closed. See the +@@ -316,11 +348,8 @@ be finalized; only the internally used f *debug* can be set from ``0`` (no debug messages) up to ``3`` (all debug messages). The messages are written to ``sys.stderr``. @@ -114,7 +120,7 @@ index 226513f5fc1..836444ebb34 100644 The *encoding* and *errors* arguments define the character encoding to be used for reading or writing the archive and how conversion errors are going -@@ -387,7 +416,7 @@ be finalized; only the internally used file object will be closed. See the +@@ -387,7 +416,7 @@ be finalized; only the internally used f available. @@ -123,7 +129,7 @@ index 226513f5fc1..836444ebb34 100644 Extract all members from the archive to the current working directory or directory *path*. If optional *members* is given, it must be a subset of the -@@ -401,6 +430,12 @@ be finalized; only the internally used file object will be closed. See the +@@ -401,6 +430,12 @@ be finalized; only the internally used f are used to set the owner/group for the extracted files. Otherwise, the named values from the tarfile are used. @@ -136,7 +142,7 @@ index 226513f5fc1..836444ebb34 100644 .. warning:: Never extract archives from untrusted sources without prior inspection. -@@ -408,14 +443,20 @@ be finalized; only the internally used file object will be closed. See the +@@ -408,14 +443,20 @@ be finalized; only the internally used f that have absolute filenames starting with ``"/"`` or filenames with two dots ``".."``. @@ -158,7 +164,7 @@ index 226513f5fc1..836444ebb34 100644 Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately as possible. *member* -@@ -423,9 +464,8 @@ be finalized; only the internally used file object will be closed. See the +@@ -423,9 +464,8 @@ be finalized; only the internally used f directory using *path*. *path* may be a :term:`path-like object`. File attributes (owner, mtime, mode) are set unless *set_attrs* is false. @@ -170,7 +176,7 @@ index 226513f5fc1..836444ebb34 100644 .. note:: -@@ -436,6 +476,9 @@ be finalized; only the internally used file object will be closed. See the +@@ -436,6 +476,9 @@ be finalized; only the internally used f See the warning for :meth:`extractall`. @@ -180,7 +186,7 @@ index 226513f5fc1..836444ebb34 100644 .. versionchanged:: 3.2 Added the *set_attrs* parameter. -@@ -445,6 +488,9 @@ be finalized; only the internally used file object will be closed. See the +@@ -445,6 +488,9 @@ be finalized; only the internally used f .. versionchanged:: 3.6 The *path* parameter accepts a :term:`path-like object`. @@ -190,7 +196,7 @@ index 226513f5fc1..836444ebb34 100644 .. method:: TarFile.extractfile(member) -@@ -457,6 +503,57 @@ be finalized; only the internally used file object will be closed. See the +@@ -457,6 +503,57 @@ be finalized; only the internally used f .. versionchanged:: 3.3 Return an :class:`io.BufferedReader` object. @@ -248,7 +254,7 @@ index 226513f5fc1..836444ebb34 100644 .. method:: TarFile.add(name, arcname=None, recursive=True, *, filter=None) -@@ -532,7 +629,27 @@ permissions, owner etc.), it provides some useful methods to determine its type. +@@ -532,7 +629,27 @@ permissions, owner etc.), it provides so It does *not* contain the file's data itself. :class:`TarInfo` objects are returned by :class:`TarFile`'s methods @@ -277,7 +283,7 @@ index 226513f5fc1..836444ebb34 100644 .. class:: TarInfo(name="") -@@ -566,24 +683,39 @@ A ``TarInfo`` object has the following public data attributes: +@@ -566,24 +683,39 @@ A ``TarInfo`` object has the following p .. attribute:: TarInfo.name @@ -294,32 +300,32 @@ index 226513f5fc1..836444ebb34 100644 .. attribute:: TarInfo.mtime + :type: int | float -+ + +- Time of last modification. + Time of last modification in seconds since the :ref:`epoch `, + as in :attr:`os.stat_result.st_mtime`. -- Time of last modification. + .. versionchanged:: 3.11.4 - ++ + Can be set to ``None`` for :meth:`~TarFile.extract` and + :meth:`~TarFile.extractall`, causing extraction to skip applying this + attribute. .. attribute:: TarInfo.mode + :type: int - -- Permission bits. ++ + Permission bits, as for :func:`os.chmod`. +- Permission bits. + .. versionchanged:: 3.11.4 -+ + + Can be set to ``None`` for :meth:`~TarFile.extract` and + :meth:`~TarFile.extractall`, causing extraction to skip applying this + attribute. .. attribute:: TarInfo.type -@@ -595,35 +727,76 @@ A ``TarInfo`` object has the following public data attributes: +@@ -595,35 +727,76 @@ A ``TarInfo`` object has the following p .. attribute:: TarInfo.linkname @@ -396,7 +402,7 @@ index 226513f5fc1..836444ebb34 100644 A :class:`TarInfo` object also provides some convenient query methods: -@@ -673,9 +846,259 @@ A :class:`TarInfo` object also provides some convenient query methods: +@@ -673,9 +846,259 @@ A :class:`TarInfo` object also provides Return :const:`True` if it is one of character device, block device or FIFO. @@ -672,11 +678,9 @@ index 226513f5fc1..836444ebb34 100644 .. _tar-examples: Examples -diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst -index 47e38ae76ba..43da72aece9 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst -@@ -2332,3 +2332,19 @@ The deprecated :mod:`mailcap` module now refuses to inject unsafe text +@@ -2332,3 +2332,19 @@ The deprecated :mod:`mailcap` module now text, it will warn and act as if a match was not found (or for test commands, as if the test failed). (Contributed by Petr Viktorin in :gh:`98966`.) @@ -696,11 +700,9 @@ index 47e38ae76ba..43da72aece9 100644 + :exc:`DeprecationWarning`. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) -diff --git a/Lib/shutil.py b/Lib/shutil.py -index b7bffa3ea41..482ce95a7b2 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py -@@ -1222,7 +1222,7 @@ def _unpack_zipfile(filename, extract_dir): +@@ -1222,7 +1222,7 @@ def _unpack_zipfile(filename, extract_di finally: zip.close() @@ -709,7 +711,7 @@ index b7bffa3ea41..482ce95a7b2 100644 """Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir` """ import tarfile # late import for breaking circular dependency -@@ -1232,7 +1232,7 @@ def _unpack_tarfile(filename, extract_dir): +@@ -1232,7 +1232,7 @@ def _unpack_tarfile(filename, extract_di raise ReadError( "%s is not a compressed or uncompressed tar file" % filename) try: @@ -727,7 +729,7 @@ index b7bffa3ea41..482ce95a7b2 100644 """Unpack an archive. `filename` is the name of the archive. -@@ -1279,6 +1279,9 @@ def unpack_archive(filename, extract_dir=None, format=None): +@@ -1279,6 +1279,9 @@ def unpack_archive(filename, extract_dir was registered for that extension. In case none is found, a ValueError is raised. @@ -737,7 +739,7 @@ index b7bffa3ea41..482ce95a7b2 100644 """ sys.audit("shutil.unpack_archive", filename, extract_dir, format) -@@ -1288,6 +1291,10 @@ def unpack_archive(filename, extract_dir=None, format=None): +@@ -1288,6 +1291,10 @@ def unpack_archive(filename, extract_dir extract_dir = os.fspath(extract_dir) filename = os.fspath(filename) @@ -748,7 +750,7 @@ index b7bffa3ea41..482ce95a7b2 100644 if format is not None: try: format_info = _UNPACK_FORMATS[format] -@@ -1295,7 +1302,7 @@ def unpack_archive(filename, extract_dir=None, format=None): +@@ -1295,7 +1302,7 @@ def unpack_archive(filename, extract_dir raise ValueError("Unknown unpack format '{0}'".format(format)) from None func = format_info[1] @@ -757,7 +759,7 @@ index b7bffa3ea41..482ce95a7b2 100644 else: # we need to look at the registered unpackers supported extensions format = _find_unpack_format(filename) -@@ -1303,7 +1310,7 @@ def unpack_archive(filename, extract_dir=None, format=None): +@@ -1303,7 +1310,7 @@ def unpack_archive(filename, extract_dir raise ReadError("Unknown archive format '{0}'".format(filename)) func = _UNPACK_FORMATS[format][1] @@ -766,11 +768,9 @@ index b7bffa3ea41..482ce95a7b2 100644 func(filename, extract_dir, **kwargs) -diff --git a/Lib/tarfile.py b/Lib/tarfile.py -index dea150e8dbb..40599f27bce 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py -@@ -46,6 +46,7 @@ +@@ -45,6 +45,7 @@ import time import struct import copy import re @@ -778,7 +778,7 @@ index dea150e8dbb..40599f27bce 100755 try: import pwd -@@ -71,6 +72,7 @@ +@@ -70,6 +71,7 @@ __all__ = ["TarFile", "TarInfo", "is_tar "ENCODING", "USTAR_FORMAT", "GNU_FORMAT", "PAX_FORMAT", "DEFAULT_FORMAT", "open"] @@ -786,7 +786,7 @@ index dea150e8dbb..40599f27bce 100755 #--------------------------------------------------------- # tar constants #--------------------------------------------------------- -@@ -158,6 +160,8 @@ +@@ -157,6 +159,8 @@ else: def stn(s, length, encoding, errors): """Convert a string to a null-terminated bytes object. """ @@ -795,7 +795,7 @@ index dea150e8dbb..40599f27bce 100755 s = s.encode(encoding, errors) return s[:length] + (length - len(s)) * NUL -@@ -709,9 +713,127 @@ def __init__(self, tarfile, tarinfo): +@@ -708,9 +712,127 @@ class ExFileObject(io.BufferedReader): super().__init__(fileobj) #class ExFileObject @@ -923,7 +923,7 @@ index dea150e8dbb..40599f27bce 100755 class TarInfo(object): """Informational class which holds the details about an archive member given by a tar header block. -@@ -792,12 +914,44 @@ def linkpath(self, linkname): +@@ -791,12 +913,44 @@ class TarInfo(object): def __repr__(self): return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self)) @@ -969,7 +969,7 @@ index dea150e8dbb..40599f27bce 100755 "uid": self.uid, "gid": self.gid, "size": self.size, -@@ -820,6 +974,9 @@ def tobuf(self, format=DEFAULT_FORMAT, encoding=ENCODING, errors="surrogateescap +@@ -819,6 +973,9 @@ class TarInfo(object): """Return a tar header as a string of 512 byte blocks. """ info = self.get_info() @@ -979,7 +979,7 @@ index dea150e8dbb..40599f27bce 100755 if format == USTAR_FORMAT: return self.create_ustar_header(info, encoding, errors) -@@ -950,6 +1107,12 @@ def _create_header(info, format, encoding, errors): +@@ -949,6 +1106,12 @@ class TarInfo(object): devmajor = stn("", 8, encoding, errors) devminor = stn("", 8, encoding, errors) @@ -992,7 +992,7 @@ index dea150e8dbb..40599f27bce 100755 parts = [ stn(info.get("name", ""), 100, encoding, errors), itn(info.get("mode", 0) & 0o7777, 8, format), -@@ -958,7 +1121,7 @@ def _create_header(info, format, encoding, errors): +@@ -957,7 +1120,7 @@ class TarInfo(object): itn(info.get("size", 0), 12, format), itn(info.get("mtime", 0), 12, format), b" ", # checksum field @@ -1001,7 +1001,7 @@ index dea150e8dbb..40599f27bce 100755 stn(info.get("linkname", ""), 100, encoding, errors), info.get("magic", POSIX_MAGIC), stn(info.get("uname", ""), 32, encoding, errors), -@@ -1468,6 +1631,8 @@ class TarFile(object): +@@ -1467,6 +1630,8 @@ class TarFile(object): fileobject = ExFileObject # The file-object for extractfile(). @@ -1010,7 +1010,7 @@ index dea150e8dbb..40599f27bce 100755 def __init__(self, name=None, mode="r", fileobj=None, format=None, tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, errors="surrogateescape", pax_headers=None, debug=None, -@@ -1940,7 +2105,10 @@ def list(self, verbose=True, *, members=None): +@@ -1939,7 +2104,10 @@ class TarFile(object): members = self for tarinfo in members: if verbose: @@ -1022,7 +1022,7 @@ index dea150e8dbb..40599f27bce 100755 _safe_print("%s/%s" % (tarinfo.uname or tarinfo.uid, tarinfo.gname or tarinfo.gid)) if tarinfo.ischr() or tarinfo.isblk(): -@@ -1948,8 +2116,11 @@ def list(self, verbose=True, *, members=None): +@@ -1947,8 +2115,11 @@ class TarFile(object): ("%d,%d" % (tarinfo.devmajor, tarinfo.devminor))) else: _safe_print("%10d" % tarinfo.size) @@ -1036,7 +1036,7 @@ index dea150e8dbb..40599f27bce 100755 _safe_print(tarinfo.name + ("/" if tarinfo.isdir() else "")) -@@ -2036,32 +2207,58 @@ def addfile(self, tarinfo, fileobj=None): +@@ -2035,32 +2206,58 @@ class TarFile(object): self.members.append(tarinfo) @@ -1105,7 +1105,7 @@ index dea150e8dbb..40599f27bce 100755 # Set correct owner, mtime and filemode on directories. for tarinfo in directories: -@@ -2071,12 +2268,10 @@ def extractall(self, path=".", members=None, *, numeric_owner=False): +@@ -2070,12 +2267,10 @@ class TarFile(object): self.utime(tarinfo, dirpath) self.chmod(tarinfo, dirpath) except ExtractError as e: @@ -1121,7 +1121,7 @@ index dea150e8dbb..40599f27bce 100755 """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately as possible. `member' may be a filename or a TarInfo object. You can -@@ -2084,35 +2279,70 @@ def extract(self, member, path="", set_attrs=True, *, numeric_owner=False): +@@ -2083,35 +2278,70 @@ class TarFile(object): mtime, mode) are set unless `set_attrs' is False. If `numeric_owner` is True, only the numbers for user/group names are used and not the names. @@ -1203,7 +1203,7 @@ index dea150e8dbb..40599f27bce 100755 def extractfile(self, member): """Extract a member from the archive as a file object. `member' may be -@@ -2199,9 +2429,13 @@ def makedir(self, tarinfo, targetpath): +@@ -2198,9 +2428,13 @@ class TarFile(object): """Make a directory called targetpath. """ try: @@ -1220,7 +1220,7 @@ index dea150e8dbb..40599f27bce 100755 except FileExistsError: pass -@@ -2244,6 +2478,9 @@ def makedev(self, tarinfo, targetpath): +@@ -2243,6 +2477,9 @@ class TarFile(object): raise ExtractError("special devices not supported by system") mode = tarinfo.mode @@ -1230,7 +1230,7 @@ index dea150e8dbb..40599f27bce 100755 if tarinfo.isblk(): mode |= stat.S_IFBLK else: -@@ -2265,7 +2502,6 @@ def makelink(self, tarinfo, targetpath): +@@ -2264,7 +2501,6 @@ class TarFile(object): os.unlink(targetpath) os.symlink(tarinfo.linkname, targetpath) else: @@ -1238,7 +1238,7 @@ index dea150e8dbb..40599f27bce 100755 if os.path.exists(tarinfo._link_target): os.link(tarinfo._link_target, targetpath) else: -@@ -2290,15 +2526,19 @@ def chown(self, tarinfo, targetpath, numeric_owner): +@@ -2289,15 +2525,19 @@ class TarFile(object): u = tarinfo.uid if not numeric_owner: try: @@ -1260,7 +1260,7 @@ index dea150e8dbb..40599f27bce 100755 try: if tarinfo.issym() and hasattr(os, "lchown"): os.lchown(targetpath, u, g) -@@ -2310,6 +2550,8 @@ def chown(self, tarinfo, targetpath, numeric_owner): +@@ -2309,6 +2549,8 @@ class TarFile(object): def chmod(self, tarinfo, targetpath): """Set file permissions of targetpath according to tarinfo. """ @@ -1269,7 +1269,7 @@ index dea150e8dbb..40599f27bce 100755 try: os.chmod(targetpath, tarinfo.mode) except OSError as e: -@@ -2318,10 +2560,13 @@ def chmod(self, tarinfo, targetpath): +@@ -2317,10 +2559,13 @@ class TarFile(object): def utime(self, tarinfo, targetpath): """Set modification time of targetpath according to tarinfo. """ @@ -1284,7 +1284,7 @@ index dea150e8dbb..40599f27bce 100755 except OSError as e: raise ExtractError("could not change modification time") from e -@@ -2397,13 +2642,26 @@ def _getmember(self, name, tarinfo=None, normalize=False): +@@ -2396,13 +2641,26 @@ class TarFile(object): members = self.getmembers() # Limit the member search list up to tarinfo. @@ -1312,7 +1312,7 @@ index dea150e8dbb..40599f27bce 100755 if normalize: member_name = os.path.normpath(member.name) else: -@@ -2412,6 +2670,10 @@ def _getmember(self, name, tarinfo=None, normalize=False): +@@ -2411,6 +2669,10 @@ class TarFile(object): if name == member_name: return member @@ -1323,7 +1323,7 @@ index dea150e8dbb..40599f27bce 100755 def _load(self): """Read through the entire archive file and look for readable members. -@@ -2504,6 +2766,7 @@ def __exit__(self, type, value, traceback): +@@ -2503,6 +2765,7 @@ class TarFile(object): #-------------------- # exported functions #-------------------- @@ -1331,7 +1331,7 @@ index dea150e8dbb..40599f27bce 100755 def is_tarfile(name): """Return True if name points to a tar archive that we are able to handle, else return False. -@@ -2530,6 +2793,10 @@ def main(): +@@ -2529,6 +2792,10 @@ def main(): parser = argparse.ArgumentParser(description=description) parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose output') @@ -1342,7 +1342,7 @@ index dea150e8dbb..40599f27bce 100755 group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-l', '--list', metavar='', help='Show listing of a tarfile') -@@ -2541,8 +2808,12 @@ def main(): +@@ -2540,8 +2807,12 @@ def main(): help='Create tarfile from sources') group.add_argument('-t', '--test', metavar='', help='Test if a tarfile is valid') @@ -1355,7 +1355,7 @@ index dea150e8dbb..40599f27bce 100755 if args.test is not None: src = args.test if is_tarfile(src): -@@ -2573,7 +2844,7 @@ def main(): +@@ -2572,7 +2843,7 @@ def main(): if is_tarfile(src): with TarFile.open(src, 'r:*') as tf: @@ -1364,11 +1364,9 @@ index dea150e8dbb..40599f27bce 100755 if args.verbose: if curdir == '.': msg = '{!r} file is extracted.'.format(src) -diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py -index 0935b60d4c2..72fb3afcbef 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py -@@ -32,6 +32,7 @@ +@@ -32,6 +32,7 @@ except ImportError: from test import support from test.support import os_helper from test.support.os_helper import TESTFN, FakePath @@ -1376,7 +1374,7 @@ index 0935b60d4c2..72fb3afcbef 100644 TESTFN2 = TESTFN + "2" TESTFN_SRC = TESTFN + "_SRC" -@@ -1610,12 +1611,14 @@ def test_register_archive_format(self): +@@ -1610,12 +1611,14 @@ class TestArchives(BaseTest, unittest.Te ### shutil.unpack_archive @@ -1396,7 +1394,7 @@ index 0935b60d4c2..72fb3afcbef 100644 root_dir, base_dir = self._create_files() expected = rlistdir(root_dir) expected.remove('outer') -@@ -1625,36 +1628,47 @@ def check_unpack_archive_with_converter(self, format, converter): +@@ -1625,36 +1628,47 @@ class TestArchives(BaseTest, unittest.Te # let's try to unpack it now tmpdir2 = self.mkdtemp() @@ -1452,11 +1450,9 @@ index 0935b60d4c2..72fb3afcbef 100644 def test_unpack_registry(self): -diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py -index 89f5a561b4a..0d8d91b4d03 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py -@@ -5,6 +5,10 @@ +@@ -5,6 +5,10 @@ from hashlib import sha256 from contextlib import contextmanager from random import Random import pathlib @@ -1467,7 +1463,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 import unittest import unittest.mock -@@ -13,6 +17,7 @@ +@@ -13,6 +17,7 @@ import tarfile from test import support from test.support import os_helper from test.support import script_helper @@ -1475,7 +1471,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 # Check for our compression modules. try: -@@ -108,7 +113,7 @@ def test_fileobj_regular_file(self): +@@ -108,7 +113,7 @@ class UstarReadTest(ReadTest, unittest.T "regular file extraction failed") def test_fileobj_readlines(self): @@ -1484,7 +1480,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 tarinfo = self.tar.getmember("ustar/regtype") with open(os.path.join(TEMPDIR, "ustar/regtype"), "r") as fobj1: lines1 = fobj1.readlines() -@@ -126,7 +131,7 @@ def test_fileobj_readlines(self): +@@ -126,7 +131,7 @@ class UstarReadTest(ReadTest, unittest.T "fileobj.readlines() failed") def test_fileobj_iter(self): @@ -1493,7 +1489,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 tarinfo = self.tar.getmember("ustar/regtype") with open(os.path.join(TEMPDIR, "ustar/regtype"), "r") as fobj1: lines1 = fobj1.readlines() -@@ -136,7 +141,8 @@ def test_fileobj_iter(self): +@@ -136,7 +141,8 @@ class UstarReadTest(ReadTest, unittest.T "fileobj.__iter__() failed") def test_fileobj_seek(self): @@ -1503,7 +1499,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 with open(os.path.join(TEMPDIR, "ustar/regtype"), "rb") as fobj: data = fobj.read() -@@ -455,7 +461,7 @@ def test_premature_end_of_archive(self): +@@ -455,7 +461,7 @@ class CommonReadTest(ReadTest): t = tar.next() with self.assertRaisesRegex(tarfile.ReadError, "unexpected end of data"): @@ -1512,7 +1508,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 with self.assertRaisesRegex(tarfile.ReadError, "unexpected end of data"): tar.extractfile(t).read() -@@ -610,16 +616,16 @@ def test_find_members(self): +@@ -610,16 +616,16 @@ class MiscReadTestBase(CommonReadTest): def test_extract_hardlink(self): # Test hardlink extraction (e.g. bug #857297). with tarfile.open(tarname, errorlevel=1, encoding="iso8859-1") as tar: @@ -1532,7 +1528,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 self.addCleanup(os_helper.unlink, os.path.join(TEMPDIR, "ustar/symtype")) with open(os.path.join(TEMPDIR, "ustar/symtype"), "rb") as f: data = f.read() -@@ -633,13 +639,14 @@ def test_extractall(self): +@@ -633,13 +639,14 @@ class MiscReadTestBase(CommonReadTest): os.mkdir(DIR) try: directories = [t for t in tar if t.isdir()] @@ -1549,7 +1545,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 def format_mtime(mtime): if isinstance(mtime, float): return "{} ({})".format(mtime, mtime.hex()) -@@ -662,7 +669,7 @@ def test_extract_directory(self): +@@ -662,7 +669,7 @@ class MiscReadTestBase(CommonReadTest): try: with tarfile.open(tarname, encoding="iso8859-1") as tar: tarinfo = tar.getmember(dirtype) @@ -1558,7 +1554,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 extracted = os.path.join(DIR, dirtype) self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime) if sys.platform != "win32": -@@ -675,7 +682,7 @@ def test_extractall_pathlike_name(self): +@@ -675,7 +682,7 @@ class MiscReadTestBase(CommonReadTest): with os_helper.temp_dir(DIR), \ tarfile.open(tarname, encoding="iso8859-1") as tar: directories = [t for t in tar if t.isdir()] @@ -1567,7 +1563,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 for tarinfo in directories: path = DIR / tarinfo.name self.assertEqual(os.path.getmtime(path), tarinfo.mtime) -@@ -686,7 +693,7 @@ def test_extract_pathlike_name(self): +@@ -686,7 +693,7 @@ class MiscReadTestBase(CommonReadTest): with os_helper.temp_dir(DIR), \ tarfile.open(tarname, encoding="iso8859-1") as tar: tarinfo = tar.getmember(dirtype) @@ -1576,7 +1572,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 extracted = DIR / dirtype self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime) -@@ -1042,7 +1049,7 @@ class GNUReadTest(LongnameTest, ReadTest, unittest.TestCase): +@@ -1042,7 +1049,7 @@ class GNUReadTest(LongnameTest, ReadTest # an all platforms, and after that a test that will work only on # platforms/filesystems that prove to support sparse files. def _test_sparse_file(self, name): @@ -1585,7 +1581,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 filename = os.path.join(TEMPDIR, name) with open(filename, "rb") as fobj: data = fobj.read() -@@ -1409,7 +1416,8 @@ def test_extractall_symlinks(self): +@@ -1409,7 +1416,8 @@ class WriteTest(WriteTestBase, unittest. with tarfile.open(temparchive, errorlevel=2) as tar: # this should not raise OSError: [Errno 17] File exists try: @@ -1595,7 +1591,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 except OSError: self.fail("extractall failed with symlinked files") finally: -@@ -2406,7 +2414,12 @@ def test__all__(self): +@@ -2406,7 +2414,12 @@ class MiscTest(unittest.TestCase): 'PAX_NUMBER_FIELDS', 'stn', 'nts', 'nti', 'itn', 'calc_chksums', 'copyfileobj', 'filemode', 'EmptyHeaderError', 'TruncatedHeaderError', 'EOFHeaderError', 'InvalidHeaderError', @@ -1609,7 +1605,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 support.check__all__(self, tarfile, not_exported=not_exported) def test_useful_error_message_when_modules_missing(self): -@@ -2441,6 +2454,15 @@ def make_simple_tarfile(self, tar_name): +@@ -2441,6 +2454,15 @@ class CommandLineTest(unittest.TestCase) for tardata in files: tf.add(tardata, arcname=os.path.basename(tardata)) @@ -1625,7 +1621,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 def test_bad_use(self): rc, out, err = self.tarfilecmd_failure() self.assertEqual(out, b'') -@@ -2597,6 +2619,25 @@ def test_extract_command_verbose(self): +@@ -2597,6 +2619,25 @@ class CommandLineTest(unittest.TestCase) finally: os_helper.rmtree(tarextdir) @@ -1651,7 +1647,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 def test_extract_command_different_directory(self): self.make_simple_tarfile(tmpname) try: -@@ -2680,7 +2721,7 @@ class LinkEmulationTest(ReadTest, unittest.TestCase): +@@ -2680,7 +2721,7 @@ class LinkEmulationTest(ReadTest, unitte # symbolic or hard links tarfile tries to extract these types of members # as the regular files they point to. def _test_link_extraction(self, name): @@ -1660,7 +1656,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 with open(os.path.join(TEMPDIR, name), "rb") as f: data = f.read() self.assertEqual(sha256sum(data), sha256_regtype) -@@ -2812,8 +2853,10 @@ def test_extract_with_numeric_owner(self, mock_geteuid, mock_chmod, +@@ -2812,8 +2853,10 @@ class NumericOwnerTest(unittest.TestCase mock_chown): with self._setup_test(mock_geteuid) as (tarfl, filename_1, _, filename_2): @@ -1673,7 +1669,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 # convert to filesystem paths f_filename_1 = os.path.join(TEMPDIR, filename_1) -@@ -2831,7 +2874,8 @@ def test_extractall_with_numeric_owner(self, mock_geteuid, mock_chmod, +@@ -2831,7 +2874,8 @@ class NumericOwnerTest(unittest.TestCase mock_chown): with self._setup_test(mock_geteuid) as (tarfl, filename_1, dirname_1, filename_2): @@ -1683,7 +1679,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 # convert to filesystem paths f_filename_1 = os.path.join(TEMPDIR, filename_1) -@@ -2856,7 +2900,8 @@ def test_extractall_with_numeric_owner(self, mock_geteuid, mock_chmod, +@@ -2856,7 +2900,8 @@ class NumericOwnerTest(unittest.TestCase def test_extract_without_numeric_owner(self, mock_geteuid, mock_chmod, mock_chown): with self._setup_test(mock_geteuid) as (tarfl, filename_1, _, _): @@ -1693,7 +1689,7 @@ index 89f5a561b4a..0d8d91b4d03 100644 # convert to filesystem paths f_filename_1 = os.path.join(TEMPDIR, filename_1) -@@ -2870,6 +2915,888 @@ def test_keyword_only(self, mock_geteuid): +@@ -2870,6 +2915,888 @@ class NumericOwnerTest(unittest.TestCase tarfl.extract, filename_1, TEMPDIR, False, True) @@ -2582,9 +2578,6 @@ index 89f5a561b4a..0d8d91b4d03 100644 def setUpModule(): os_helper.unlink(TEMPDIR) os.makedirs(TEMPDIR) -diff --git a/Misc/NEWS.d/next/Library/2023-03-23-15-24-38.gh-issue-102953.YR4KaK.rst b/Misc/NEWS.d/next/Library/2023-03-23-15-24-38.gh-issue-102953.YR4KaK.rst -new file mode 100644 -index 00000000000..48a105a4a17 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-23-15-24-38.gh-issue-102953.YR4KaK.rst @@ -0,0 +1,4 @@ diff --git a/bpo-37596-make-set-marshalling.patch b/bpo-37596-make-set-marshalling.patch new file mode 100644 index 0000000..79c3f51 --- /dev/null +++ b/bpo-37596-make-set-marshalling.patch @@ -0,0 +1,96 @@ +From 33d95c6facdfda3c8c0feffa7a99184e4abc2f63 Mon Sep 17 00:00:00 2001 +From: Brandt Bucher +Date: Wed, 25 Aug 2021 04:14:34 -0700 +Subject: [PATCH] bpo-37596: Make `set` and `frozenset` marshalling + deterministic (GH-27926) + +--- + Lib/test/test_marshal.py | 25 +++++++ + Misc/NEWS.d/next/Library/2021-08-23-21-39-59.bpo-37596.ojRcwB.rst | 2 + Python/marshal.c | 32 ++++++++++ + 3 files changed, 59 insertions(+) + create mode 100644 Misc/NEWS.d/next/Library/2021-08-23-21-39-59.bpo-37596.ojRcwB.rst + +--- a/Lib/test/test_marshal.py ++++ b/Lib/test/test_marshal.py +@@ -318,6 +318,31 @@ class BugsTestCase(unittest.TestCase): + for i in range(len(data)): + self.assertRaises(EOFError, marshal.loads, data[0: i]) + ++ def test_deterministic_sets(self): ++ # bpo-37596: To support reproducible builds, sets and frozensets need to ++ # have their elements serialized in a consistent order (even when they ++ # have been scrambled by hash randomization): ++ for kind in ("set", "frozenset"): ++ for elements in ( ++ "float('nan'), b'a', b'b', b'c', 'x', 'y', 'z'", ++ # Also test for bad interactions with backreferencing: ++ "('string', 1), ('string', 2), ('string', 3)", ++ ): ++ s = f"{kind}([{elements}])" ++ with self.subTest(s): ++ # First, make sure that our test case still has different ++ # orders under hash seeds 0 and 1. If this check fails, we ++ # need to update this test with different elements: ++ args = ["-c", f"print({s})"] ++ _, repr_0, _ = assert_python_ok(*args, PYTHONHASHSEED="0") ++ _, repr_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1") ++ self.assertNotEqual(repr_0, repr_1) ++ # Then, perform the actual test: ++ args = ["-c", f"import marshal; print(marshal.dumps({s}))"] ++ _, dump_0, _ = assert_python_ok(*args, PYTHONHASHSEED="0") ++ _, dump_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1") ++ self.assertEqual(dump_0, dump_1) ++ + LARGE_SIZE = 2**31 + pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4 + +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2021-08-23-21-39-59.bpo-37596.ojRcwB.rst +@@ -0,0 +1,2 @@ ++Ensure that :class:`set` and :class:`frozenset` objects are always ++:mod:`marshalled ` reproducibly. +--- a/Python/marshal.c ++++ b/Python/marshal.c +@@ -502,9 +502,41 @@ w_complex_object(PyObject *v, char flag, + W_TYPE(TYPE_SET, p); + n = PySet_GET_SIZE(v); + W_SIZE(n, p); ++ // bpo-37596: To support reproducible builds, sets and frozensets need ++ // to have their elements serialized in a consistent order (even when ++ // they have been scrambled by hash randomization). To ensure this, we ++ // use an order equivalent to sorted(v, key=marshal.dumps): ++ PyObject *pairs = PyList_New(0); ++ if (pairs == NULL) { ++ p->error = WFERR_NOMEMORY; ++ return; ++ } + while (_PySet_NextEntry(v, &pos, &value, &hash)) { ++ PyObject *dump = PyMarshal_WriteObjectToString(value, p->version); ++ if (dump == NULL) { ++ p->error = WFERR_UNMARSHALLABLE; ++ goto anyset_done; ++ } ++ PyObject *pair = PyTuple_Pack(2, dump, value); ++ Py_DECREF(dump); ++ if (pair == NULL || PyList_Append(pairs, pair)) { ++ p->error = WFERR_NOMEMORY; ++ Py_XDECREF(pair); ++ goto anyset_done; ++ } ++ Py_DECREF(pair); ++ } ++ if (PyList_Sort(pairs)) { ++ p->error = WFERR_NOMEMORY; ++ goto anyset_done; ++ } ++ for (Py_ssize_t i = 0; i < n; i++) { ++ PyObject *pair = PyList_GET_ITEM(pairs, i); ++ value = PyTuple_GET_ITEM(pair, 1); + w_object(value, p); + } ++ anyset_done: ++ Py_DECREF(pairs); + } + else if (PyCode_Check(v)) { + PyCodeObject *co = (PyCodeObject *)v; diff --git a/python310.changes b/python310.changes index 1ff6218..35dd8b5 100644 --- a/python310.changes +++ b/python310.changes @@ -1,3 +1,9 @@ +------------------------------------------------------------------- +Tue Jun 20 21:39:58 UTC 2023 - Matej Cepl + +- Add bpo-37596-make-set-marshalling.patch making marshalling of + `set` and `frozenset` deterministic (bsc#1211765). + ------------------------------------------------------------------- Thu Apr 27 21:23:19 UTC 2023 - Matej Cepl diff --git a/python310.spec b/python310.spec index c9cc36a..a41651a 100644 --- a/python310.spec +++ b/python310.spec @@ -173,6 +173,9 @@ Patch37: CVE-2023-24329-blank-URL-bypass.patch # PATCH-FIX-UPSTREAM CVE-2007-4559-filter-tarfile_extractall.patch bsc#1203750 mcepl@suse.com # PEP 706 – Filter for tarfile.extractall Patch38: CVE-2007-4559-filter-tarfile_extractall.patch +# PATCH-FIX-UPSTREAM bpo-37596-make-set-marshalling.patch bsc#1211765 mcepl@suse.com +# Make `set` and `frozenset` marshalling deterministic +Patch39: bpo-37596-make-set-marshalling.patch BuildRequires: autoconf-archive BuildRequires: automake BuildRequires: fdupes @@ -432,7 +435,6 @@ other applications. %prep %setup -q -n %{tarname} %patch02 -p1 - %patch06 -p1 %patch07 -p1 %patch08 -p1 @@ -447,6 +449,7 @@ other applications. %patch36 -p1 %patch37 -p1 %patch38 -p1 +%patch39 -p1 # drop Autoconf version requirement sed -i 's/^AC_PREREQ/dnl AC_PREREQ/' configure.ac