From 8b8e68d3dc95f454f58fdd8aac10848facb1491d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 31 Oct 2025 15:49:51 +0200 Subject: [PATCH 1/2] [3.9] gh-136065: Fix quadratic complexity in os.path.expandvars() (GH-134952) (cherry picked from commit f029e8db626ddc6e3a3beea4eff511a71aaceb5c) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Serhiy Storchaka Co-authored-by: Łukasz Langa --- Lib/ntpath.py | 126 +++------- Lib/posixpath.py | 43 +-- Lib/test/test_genericpath.py | 19 + Lib/test/test_ntpath.py | 23 + Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst | 1 5 files changed, 96 insertions(+), 116 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst Index: Python-3.9.24/Lib/ntpath.py =================================================================== --- Python-3.9.24.orig/Lib/ntpath.py 2025-11-21 12:52:18.350673347 +0100 +++ Python-3.9.24/Lib/ntpath.py 2025-11-21 12:52:34.076133325 +0100 @@ -335,17 +335,23 @@ # XXX With COMMAND.COM you can use any characters in a variable name, # XXX except '^|<>='. +_varpattern = r"'[^']*'?|%(%|[^%]*%?)|\$(\$|[-\w]+|\{[^}]*\}?)" +_varsub = None +_varsubb = None + def expandvars(path): """Expand shell variables of the forms $var, ${var} and %var%. Unknown variables are left unchanged.""" path = os.fspath(path) + global _varsub, _varsubb if isinstance(path, bytes): if b'$' not in path and b'%' not in path: return path - import string - varchars = bytes(string.ascii_letters + string.digits + '_-', 'ascii') - quote = b'\'' + if not _varsubb: + import re + _varsubb = re.compile(_varpattern.encode(), re.ASCII).sub + sub = _varsubb percent = b'%' brace = b'{' rbrace = b'}' @@ -354,94 +360,44 @@ else: if '$' not in path and '%' not in path: return path - import string - varchars = string.ascii_letters + string.digits + '_-' - quote = '\'' + if not _varsub: + import re + _varsub = re.compile(_varpattern, re.ASCII).sub + sub = _varsub percent = '%' brace = '{' rbrace = '}' dollar = '$' environ = os.environ - res = path[:0] - index = 0 - pathlen = len(path) - while index < pathlen: - c = path[index:index+1] - if c == quote: # no expansion within single quotes - path = path[index + 1:] - pathlen = len(path) - try: - index = path.index(c) - res += c + path[:index + 1] - except ValueError: - res += c + path - index = pathlen - 1 - elif c == percent: # variable or '%' - if path[index + 1:index + 2] == percent: - res += c - index += 1 - else: - path = path[index+1:] - pathlen = len(path) - try: - index = path.index(percent) - except ValueError: - res += percent + path - index = pathlen - 1 - else: - var = path[:index] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = percent + var + percent - res += value - elif c == dollar: # variable or '$$' - if path[index + 1:index + 2] == dollar: - res += c - index += 1 - elif path[index + 1:index + 2] == brace: - path = path[index+2:] - pathlen = len(path) - try: - index = path.index(rbrace) - except ValueError: - res += dollar + brace + path - index = pathlen - 1 - else: - var = path[:index] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = dollar + brace + var + rbrace - res += value - else: - var = path[:0] - index += 1 - c = path[index:index + 1] - while c and c in varchars: - var += c - index += 1 - c = path[index:index + 1] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = dollar + var - res += value - if c: - index -= 1 + + def repl(m): + lastindex = m.lastindex + if lastindex is None: + return m[0] + name = m[lastindex] + if lastindex == 1: + if name == percent: + return name + if not name.endswith(percent): + return m[0] + name = name[:-1] else: - res += c - index += 1 - return res + if name == dollar: + return name + if name.startswith(brace): + if not name.endswith(rbrace): + return m[0] + name = name[1:-1] + + try: + if environ is None: + return os.fsencode(os.environ[os.fsdecode(name)]) + else: + return environ[name] + except KeyError: + return m[0] + + return sub(repl, path) # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B. Index: Python-3.9.24/Lib/posixpath.py =================================================================== --- Python-3.9.24.orig/Lib/posixpath.py 2025-11-21 12:52:18.388628236 +0100 +++ Python-3.9.24/Lib/posixpath.py 2025-11-21 12:52:34.076301225 +0100 @@ -275,42 +275,41 @@ # This expands the forms $variable and ${variable} only. # Non-existent variables are left unchanged. -_varprog = None -_varprogb = None +_varpattern = r'\$(\w+|\{[^}]*\}?)' +_varsub = None +_varsubb = None def expandvars(path): """Expand shell variables of form $var and ${var}. Unknown variables are left unchanged.""" path = os.fspath(path) - global _varprog, _varprogb + global _varsub, _varsubb if isinstance(path, bytes): if b'$' not in path: return path - if not _varprogb: + if not _varsubb: import re - _varprogb = re.compile(br'\$(\w+|\{[^}]*\})', re.ASCII) - search = _varprogb.search + _varsubb = re.compile(_varpattern.encode(), re.ASCII).sub + sub = _varsubb start = b'{' end = b'}' environ = getattr(os, 'environb', None) else: if '$' not in path: return path - if not _varprog: + if not _varsub: import re - _varprog = re.compile(r'\$(\w+|\{[^}]*\})', re.ASCII) - search = _varprog.search + _varsub = re.compile(_varpattern, re.ASCII).sub + sub = _varsub start = '{' end = '}' environ = os.environ - i = 0 - while True: - m = search(path, i) - if not m: - break - i, j = m.span(0) - name = m.group(1) - if name.startswith(start) and name.endswith(end): + + def repl(m): + name = m[1] + if name.startswith(start): + if not name.endswith(end): + return m[0] name = name[1:-1] try: if environ is None: @@ -318,13 +317,11 @@ else: value = environ[name] except KeyError: - i = j + return m[0] else: - tail = path[j:] - path = path[:i] + value - i = len(path) - path += tail - return path + return value + + return sub(repl, path) # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B. Index: Python-3.9.24/Lib/test/test_genericpath.py =================================================================== --- Python-3.9.24.orig/Lib/test/test_genericpath.py 2025-11-21 12:52:19.232406542 +0100 +++ Python-3.9.24/Lib/test/test_genericpath.py 2025-11-21 12:52:34.077309462 +0100 @@ -9,7 +9,7 @@ import warnings from test import support from test.support.script_helper import assert_python_ok -from test.support import FakePath +from test.support import FakePath, EnvironmentVarGuard def create_file(filename, data=b'foo'): @@ -374,7 +374,7 @@ def test_expandvars(self): expandvars = self.pathmodule.expandvars - with support.EnvironmentVarGuard() as env: + with EnvironmentVarGuard() as env: env.clear() env["foo"] = "bar" env["{foo"] = "baz1" @@ -408,7 +408,7 @@ expandvars = self.pathmodule.expandvars def check(value, expected): self.assertEqual(expandvars(value), expected) - with support.EnvironmentVarGuard() as env: + with EnvironmentVarGuard() as env: env.clear() nonascii = support.FS_NONASCII env['spam'] = nonascii @@ -429,6 +429,19 @@ os.fsencode('$bar%s bar' % nonascii)) check(b'$spam}bar', os.fsencode('%s}bar' % nonascii)) + @support.requires_resource('cpu') + def test_expandvars_large(self): + expandvars = self.pathmodule.expandvars + with EnvironmentVarGuard() as env: + env.clear() + env["A"] = "B" + n = 100_000 + self.assertEqual(expandvars('$A'*n), 'B'*n) + self.assertEqual(expandvars('${A}'*n), 'B'*n) + self.assertEqual(expandvars('$A!'*n), 'B!'*n) + self.assertEqual(expandvars('${A}A'*n), 'BA'*n) + self.assertEqual(expandvars('${'*10*n), '${'*10*n) + def test_abspath(self): self.assertIn("foo", self.pathmodule.abspath("foo")) with warnings.catch_warnings(): Index: Python-3.9.24/Lib/test/test_ntpath.py =================================================================== --- Python-3.9.24.orig/Lib/test/test_ntpath.py 2025-11-21 12:52:19.665352116 +0100 +++ Python-3.9.24/Lib/test/test_ntpath.py 2025-11-21 12:52:34.077441463 +0100 @@ -1,11 +1,10 @@ import ntpath import os -import subprocess import sys import unittest import warnings from ntpath import ALLOW_MISSING -from test.support import TestFailed, FakePath +from test.support import TestFailed, FakePath, EnvironmentVarGuard from test import support, test_genericpath from tempfile import TemporaryFile @@ -642,7 +641,7 @@ ntpath.realpath("file.txt", **kwargs)) def test_expandvars(self): - with support.EnvironmentVarGuard() as env: + with EnvironmentVarGuard() as env: env.clear() env["foo"] = "bar" env["{foo"] = "baz1" @@ -671,7 +670,7 @@ def test_expandvars_nonascii(self): def check(value, expected): tester('ntpath.expandvars(%r)' % value, expected) - with support.EnvironmentVarGuard() as env: + with EnvironmentVarGuard() as env: env.clear() nonascii = support.FS_NONASCII env['spam'] = nonascii @@ -687,10 +686,23 @@ check('%spam%bar', '%sbar' % nonascii) check('%{}%bar'.format(nonascii), 'ham%sbar' % nonascii) + @support.requires_resource('cpu') + def test_expandvars_large(self): + expandvars = ntpath.expandvars + with EnvironmentVarGuard() as env: + env.clear() + env["A"] = "B" + n = 100_000 + self.assertEqual(expandvars('%A%'*n), 'B'*n) + self.assertEqual(expandvars('%A%A'*n), 'BA'*n) + self.assertEqual(expandvars("''"*n + '%%'), "''"*n + '%') + self.assertEqual(expandvars("%%"*n), "%"*n) + self.assertEqual(expandvars("$$"*n), "$"*n) + def test_expanduser(self): tester('ntpath.expanduser("test")', 'test') - with support.EnvironmentVarGuard() as env: + with EnvironmentVarGuard() as env: env.clear() tester('ntpath.expanduser("~test")', '~test') @@ -908,6 +920,7 @@ self.assertIsInstance(b_final_path, bytes) self.assertGreater(len(b_final_path), 0) + class NtCommonTest(test_genericpath.CommonTest, unittest.TestCase): pathmodule = ntpath attributes = ['relpath'] Index: Python-3.9.24/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst =================================================================== --- /dev/null 1970-01-01 00:00:00.000000000 +0000 +++ Python-3.9.24/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst 2025-11-21 12:52:34.076771610 +0100 @@ -0,0 +1 @@ +Fix quadratic complexity in :func:`os.path.expandvars`.