pool/python-jedi
SHA256
4
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
python-jedi/python3.13-support.patch

524 lines
21 KiB
Diff
Raw Blame History

From 43902d83018c950c9ac1a97c58abc32838228867 Mon Sep 17 00:00:00 2001
From: Peter Law <PeterJCLaw@gmail.com>
Date: Thu, 4 Jul 2024 22:39:29 +0100
Subject: [PATCH 1/2] Merge branch 'ensure-unique-subprocess-reference-ids'
(cherry picked from commit e839683e91b78355f0363bcc6f74f762995344f8)
---
jedi/api/environment.py | 15 +-
.../inference/compiled/subprocess/__init__.py | 157 ++++++++++++++++--
2 files changed, 152 insertions(+), 20 deletions(-)
diff --git a/jedi/api/environment.py b/jedi/api/environment.py
index 771a9a83..cfe8cfe3 100644
--- a/jedi/api/environment.py
+++ b/jedi/api/environment.py
@@ -8,6 +8,7 @@ import hashlib
import filecmp
from collections import namedtuple
from shutil import which
+from typing import TYPE_CHECKING
from jedi.cache import memoize_method, time_cache
from jedi.inference.compiled.subprocess import CompiledSubprocess, \
@@ -15,6 +16,10 @@ from jedi.inference.compiled.subprocess import CompiledSubprocess, \
import parso
+if TYPE_CHECKING:
+ from jedi.inference import InferenceState
+
+
_VersionInfo = namedtuple('VersionInfo', 'major minor micro') # type: ignore[name-match]
_SUPPORTED_PYTHONS = ['3.12', '3.11', '3.10', '3.9', '3.8', '3.7', '3.6']
@@ -102,7 +107,10 @@ class Environment(_BaseEnvironment):
version = '.'.join(str(i) for i in self.version_info)
return '<%s: %s in %s>' % (self.__class__.__name__, version, self.path)
- def get_inference_state_subprocess(self, inference_state):
+ def get_inference_state_subprocess(
+ self,
+ inference_state: 'InferenceState',
+ ) -> InferenceStateSubprocess:
return InferenceStateSubprocess(inference_state, self._get_subprocess())
@memoize_method
@@ -134,7 +142,10 @@ class SameEnvironment(_SameEnvironmentMixin, Environment):
class InterpreterEnvironment(_SameEnvironmentMixin, _BaseEnvironment):
- def get_inference_state_subprocess(self, inference_state):
+ def get_inference_state_subprocess(
+ self,
+ inference_state: 'InferenceState',
+ ) -> InferenceStateSameProcess:
return InferenceStateSameProcess(inference_state)
def get_sys_path(self):
diff --git a/jedi/inference/compiled/subprocess/__init__.py b/jedi/inference/compiled/subprocess/__init__.py
index cd5fe74c..3a6039f7 100644
--- a/jedi/inference/compiled/subprocess/__init__.py
+++ b/jedi/inference/compiled/subprocess/__init__.py
@@ -5,6 +5,23 @@ goals:
1. Making it safer - Segfaults and RuntimeErrors as well as stdout/stderr can
be ignored and dealt with.
2. Make it possible to handle different Python versions as well as virtualenvs.
+
+The architecture here is briefly:
+ - For each Jedi `Environment` there is a corresponding subprocess which
+ operates within the target environment. If the subprocess dies it is replaced
+ at this level.
+ - `CompiledSubprocess` manages exactly one subprocess and handles communication
+ from the parent side.
+ - `Listener` runs within the subprocess, processing each request and yielding
+ results.
+ - `InterpreterEnvironment` provides an API which matches that of `Environment`,
+ but runs functionality inline rather than within a subprocess. It is thus
+ used both directly in places where a subprocess is unnecessary and/or
+ undesirable and also within subprocesses themselves.
+ - `InferenceStateSubprocess` (or `InferenceStateSameProcess`) provide high
+ level access to functionality within the subprocess from within the parent.
+ Each `InterpreterState` has an instance of one of these, provided by its
+ environment.
"""
import collections
@@ -16,6 +33,7 @@ import traceback
import weakref
from functools import partial
from threading import Thread
+from typing import Dict, TYPE_CHECKING
from jedi._compatibility import pickle_dump, pickle_load
from jedi import debug
@@ -25,6 +43,9 @@ from jedi.inference.compiled.access import DirectObjectAccess, AccessPath, \
SignatureParam
from jedi.api.exceptions import InternalError
+if TYPE_CHECKING:
+ from jedi.inference import InferenceState
+
_MAIN_PATH = os.path.join(os.path.dirname(__file__), '__main__.py')
PICKLE_PROTOCOL = 4
@@ -83,10 +104,9 @@ def _cleanup_process(process, thread):
class _InferenceStateProcess:
- def __init__(self, inference_state):
+ def __init__(self, inference_state: 'InferenceState') -> None:
self._inference_state_weakref = weakref.ref(inference_state)
- self._inference_state_id = id(inference_state)
- self._handles = {}
+ self._handles: Dict[int, AccessHandle] = {}
def get_or_create_access_handle(self, obj):
id_ = id(obj)
@@ -116,11 +136,49 @@ class InferenceStateSameProcess(_InferenceStateProcess):
class InferenceStateSubprocess(_InferenceStateProcess):
- def __init__(self, inference_state, compiled_subprocess):
+ """
+ API to functionality which will run in a subprocess.
+
+ This mediates the interaction between an `InferenceState` and the actual
+ execution of functionality running within a `CompiledSubprocess`. Available
+ functions are defined in `.functions`, though should be accessed via
+ attributes on this class of the same name.
+
+ This class is responsible for indicating that the `InferenceState` within
+ the subprocess can be removed once the corresponding instance in the parent
+ goes away.
+ """
+
+ def __init__(
+ self,
+ inference_state: 'InferenceState',
+ compiled_subprocess: 'CompiledSubprocess',
+ ) -> None:
super().__init__(inference_state)
self._used = False
self._compiled_subprocess = compiled_subprocess
+ # Opaque id we'll pass to the subprocess to identify the context (an
+ # `InferenceState`) which should be used for the request. This allows us
+ # to make subsequent requests which operate on results from previous
+ # ones, while keeping a single subprocess which can work with several
+ # contexts in the parent process. Once it is no longer needed(i.e: when
+ # this class goes away), we also use this id to indicate that the
+ # subprocess can discard the context.
+ #
+ # Note: this id is deliberately coupled to this class (and not to
+ # `InferenceState`) as this class manages access handle mappings which
+ # must correspond to those in the subprocess. This approach also avoids
+ # race conditions from successive `InferenceState`s with the same object
+ # id (as observed while adding support for Python 3.13).
+ #
+ # This value does not need to be the `id()` of this instance, we merely
+ # need to ensure that it enables the (visible) lifetime of the context
+ # within the subprocess to match that of this class. We therefore also
+ # depend on the semantics of `CompiledSubprocess.delete_inference_state`
+ # for correctness.
+ self._inference_state_id = id(self)
+
def __getattr__(self, name):
func = _get_function(name)
@@ -128,7 +186,7 @@ class InferenceStateSubprocess(_InferenceStateProcess):
self._used = True
result = self._compiled_subprocess.run(
- self._inference_state_weakref(),
+ self._inference_state_id,
func,
args=args,
kwargs=kwargs,
@@ -164,6 +222,17 @@ class InferenceStateSubprocess(_InferenceStateProcess):
class CompiledSubprocess:
+ """
+ A subprocess which runs inference within a target environment.
+
+ This class manages the interface to a single instance of such a process as
+ well as the lifecycle of the process itself. See `.__main__` and `Listener`
+ for the implementation of the subprocess and details of the protocol.
+
+ A single live instance of this is maintained by `jedi.api.environment.Environment`,
+ so that typically a single subprocess is used at a time.
+ """
+
is_crashed = False
def __init__(self, executable, env_vars=None):
@@ -213,18 +282,18 @@ class CompiledSubprocess:
t)
return process
- def run(self, inference_state, function, args=(), kwargs={}):
+ def run(self, inference_state_id, function, args=(), kwargs={}):
# Delete old inference_states.
while True:
try:
- inference_state_id = self._inference_state_deletion_queue.pop()
+ delete_id = self._inference_state_deletion_queue.pop()
except IndexError:
break
else:
- self._send(inference_state_id, None)
+ self._send(delete_id, None)
assert callable(function)
- return self._send(id(inference_state), function, args, kwargs)
+ return self._send(inference_state_id, function, args, kwargs)
def get_sys_path(self):
return self._send(None, functions.get_sys_path, (), {})
@@ -272,21 +341,65 @@ class CompiledSubprocess:
def delete_inference_state(self, inference_state_id):
"""
- Currently we are not deleting inference_state instantly. They only get
- deleted once the subprocess is used again. It would probably a better
- solution to move all of this into a thread. However, the memory usage
- of a single inference_state shouldn't be that high.
+ Indicate that an inference state (in the subprocess) is no longer
+ needed.
+
+ The state corresponding to the given id will become inaccessible and the
+ id may safely be re-used to refer to a different context.
+
+ Note: it is not guaranteed that the corresponding state will actually be
+ deleted immediately.
"""
- # With an argument - the inference_state gets deleted.
+ # Warning: if changing the semantics of context deletion see the comment
+ # in `InferenceStateSubprocess.__init__` regarding potential race
+ # conditions.
+
+ # Currently we are not deleting the related state instantly. They only
+ # get deleted once the subprocess is used again. It would probably a
+ # better solution to move all of this into a thread. However, the memory
+ # usage of a single inference_state shouldn't be that high.
self._inference_state_deletion_queue.append(inference_state_id)
class Listener:
+ """
+ Main loop for the subprocess which actually does the inference.
+
+ This class runs within the target environment. It listens to instructions
+ from the parent process, runs inference and returns the results.
+
+ The subprocess has a long lifetime and is expected to process several
+ requests, including for different `InferenceState` instances in the parent.
+ See `CompiledSubprocess` for the parent half of the system.
+
+ Communication is via pickled data sent serially over stdin and stdout.
+ Stderr is read only if the child process crashes.
+
+ The request protocol is a 4-tuple of:
+ * inference_state_id | None: an opaque identifier of the parent's
+ `InferenceState`. An `InferenceState` operating over an
+ `InterpreterEnvironment` is created within this process for each of
+ these, ensuring that each parent context has a corresponding context
+ here. This allows context to be persisted between requests. Unless
+ `None`, the local `InferenceState` will be passed to the given function
+ as the first positional argument.
+ * function | None: the function to run. This is expected to be a member of
+ `.functions`. `None` indicates that the corresponding inference state is
+ no longer needed and should be dropped.
+ * args: positional arguments to the `function`. If any of these are
+ `AccessHandle` instances they will be adapted to the local
+ `InferenceState` before being passed.
+ * kwargs: keyword arguments to the `function`. If any of these are
+ `AccessHandle` instances they will be adapted to the local
+ `InferenceState` before being passed.
+
+ The result protocol is a 3-tuple of either:
+ * (False, None, function result): if the function returns without error, or
+ * (True, traceback, exception): if the function raises an exception
+ """
+
def __init__(self):
self._inference_states = {}
- # TODO refactor so we don't need to process anymore just handle
- # controlling.
- self._process = _InferenceStateProcess(Listener)
def _get_inference_state(self, function, inference_state_id):
from jedi.inference import InferenceState
@@ -308,6 +421,9 @@ class Listener:
if inference_state_id is None:
return function(*args, **kwargs)
elif function is None:
+ # Warning: if changing the semantics of context deletion see the comment
+ # in `InferenceStateSubprocess.__init__` regarding potential race
+ # conditions.
del self._inference_states[inference_state_id]
else:
inference_state = self._get_inference_state(function, inference_state_id)
@@ -348,7 +464,12 @@ class Listener:
class AccessHandle:
- def __init__(self, subprocess, access, id_):
+ def __init__(
+ self,
+ subprocess: _InferenceStateProcess,
+ access: DirectObjectAccess,
+ id_: int,
+ ) -> None:
self.access = access
self._subprocess = subprocess
self.id = id_
--
2.45.2
From 2a638c784ff63ab869626d23ce95856dece31a9b Mon Sep 17 00:00:00 2001
From: Peter Law <PeterJCLaw@gmail.com>
Date: Sat, 6 Jul 2024 11:39:06 +0100
Subject: [PATCH 2/2] Merge branch 'python-3.13'
(cherry picked from commit 82d1902f382ddac5b0e6647646b72f28a3181ec3)
---
CHANGELOG.rst | 2 ++
jedi/_compatibility.py | 15 ++++++++++++++-
jedi/api/environment.py | 2 +-
jedi/inference/__init__.py | 2 +-
setup.py | 5 +++--
test/test_api/test_interpreter.py | 13 +++++++++----
test/test_inference/test_signature.py | 5 +++--
test/test_utils.py | 14 ++++++++++----
8 files changed, 43 insertions(+), 15 deletions(-)
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index fca94429..cf6810fa 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -6,6 +6,8 @@ Changelog
Unreleased
++++++++++
+- Python 3.13 support
+
0.19.1 (2023-10-02)
+++++++++++++++++++
diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py
index 13a74b7b..48563d00 100644
--- a/jedi/_compatibility.py
+++ b/jedi/_compatibility.py
@@ -5,11 +5,24 @@ different Python versions.
import errno
import sys
import pickle
+from typing import Any
+
+
+class Unpickler(pickle.Unpickler):
+ def find_class(self, module: str, name: str) -> Any:
+ # Python 3.13 moved pathlib implementation out of __init__.py as part of
+ # generalising its implementation. Ensure that we support loading
+ # pickles from 3.13 on older version of Python. Since 3.13 maintained a
+ # compatible API, pickles from older Python work natively on the newer
+ # version.
+ if module == 'pathlib._local':
+ module = 'pathlib'
+ return super().find_class(module, name)
def pickle_load(file):
try:
- return pickle.load(file)
+ return Unpickler(file).load()
# Python on Windows don't throw EOF errors for pipes. So reraise them with
# the correct type, which is caught upwards.
except OSError:
diff --git a/jedi/api/environment.py b/jedi/api/environment.py
index cfe8cfe3..a2134110 100644
--- a/jedi/api/environment.py
+++ b/jedi/api/environment.py
@@ -22,7 +22,7 @@ if TYPE_CHECKING:
_VersionInfo = namedtuple('VersionInfo', 'major minor micro') # type: ignore[name-match]
-_SUPPORTED_PYTHONS = ['3.12', '3.11', '3.10', '3.9', '3.8', '3.7', '3.6']
+_SUPPORTED_PYTHONS = ['3.13', '3.12', '3.11', '3.10', '3.9', '3.8', '3.7', '3.6']
_SAFE_PATHS = ['/usr/bin', '/usr/local/bin']
_CONDA_VAR = 'CONDA_PREFIX'
_CURRENT_VERSION = '%s.%s' % (sys.version_info.major, sys.version_info.minor)
diff --git a/jedi/inference/__init__.py b/jedi/inference/__init__.py
index aadfeba9..bd31cbd3 100644
--- a/jedi/inference/__init__.py
+++ b/jedi/inference/__init__.py
@@ -90,7 +90,7 @@ class InferenceState:
self.compiled_subprocess = environment.get_inference_state_subprocess(self)
self.grammar = environment.get_grammar()
- self.latest_grammar = parso.load_grammar(version='3.12')
+ self.latest_grammar = parso.load_grammar(version='3.13')
self.memoize_cache = {} # for memoize decorators
self.module_cache = imports.ModuleCache() # does the job of `sys.modules`.
self.stub_module_cache = {} # Dict[Tuple[str, ...], Optional[ModuleValue]]
diff --git a/setup.py b/setup.py
index 68210ef2..ed1e67a4 100755
--- a/setup.py
+++ b/setup.py
@@ -35,8 +35,8 @@ setup(name='jedi',
long_description=readme,
packages=find_packages(exclude=['test', 'test.*']),
python_requires='>=3.6',
- # Python 3.11 & 3.12 grammars are added to parso in 0.8.3
- install_requires=['parso>=0.8.3,<0.9.0'],
+ # Python 3.13 grammars are added to parso in 0.8.4
+ install_requires=['parso>=0.8.4,<0.9.0'],
extras_require={
'testing': [
'pytest<7.0.0',
@@ -101,6 +101,7 @@ setup(name='jedi',
'Programming Language :: Python :: 3.10',
'Programming Language :: Python :: 3.11',
'Programming Language :: Python :: 3.12',
+ 'Programming Language :: Python :: 3.13',
'Topic :: Software Development :: Libraries :: Python Modules',
'Topic :: Text Editors :: Integrated Development Environments (IDE)',
'Topic :: Utilities',
diff --git a/test/test_api/test_interpreter.py b/test/test_api/test_interpreter.py
index 74f066b8..efff7c5b 100644
--- a/test/test_api/test_interpreter.py
+++ b/test/test_api/test_interpreter.py
@@ -310,8 +310,9 @@ def test_completion_param_annotations():
# Need to define this function not directly in Python. Otherwise Jedi is too
# clever and uses the Python code instead of the signature object.
code = 'def foo(a: 1, b: str, c: int = 1.0) -> bytes: pass'
- exec(code, locals())
- script = jedi.Interpreter('foo', [locals()])
+ exec_locals = {}
+ exec(code, exec_locals)
+ script = jedi.Interpreter('foo', [exec_locals])
c, = script.complete()
sig, = c.get_signatures()
a, b, c = sig.params
@@ -323,7 +324,7 @@ def test_completion_param_annotations():
assert b.description == 'param b: str'
assert c.description == 'param c: int=1.0'
- d, = jedi.Interpreter('foo()', [locals()]).infer()
+ d, = jedi.Interpreter('foo()', [exec_locals]).infer()
assert d.name == 'bytes'
@@ -525,10 +526,14 @@ def test_partial_signatures(code, expected, index):
c = functools.partial(func, 1, c=2)
sig, = jedi.Interpreter(code, [locals()]).get_signatures()
- assert sig.name == 'partial'
assert [p.name for p in sig.params] == expected
assert index == sig.index
+ if sys.version_info < (3, 13):
+ # Python 3.13.0b3 makes functools.partial be a descriptor, which breaks
+ # Jedi's `py__name__` detection; see https://github.com/davidhalter/jedi/issues/2012
+ assert sig.name == 'partial'
+
def test_type_var():
"""This was an issue before, see Github #1369"""
diff --git a/test/test_inference/test_signature.py b/test/test_inference/test_signature.py
index f8f71581..4a1fcb62 100644
--- a/test/test_inference/test_signature.py
+++ b/test/test_inference/test_signature.py
@@ -1,5 +1,5 @@
from textwrap import dedent
-from operator import ge, lt
+from operator import eq, ge, lt
import re
import os
@@ -14,7 +14,8 @@ from ..helpers import get_example_dir
('import math; math.cos', 'cos(x, /)', ['x'], ge, (3, 6)),
('next', 'next(iterator, default=None, /)', ['iterator', 'default'], lt, (3, 12)),
- ('next', 'next()', [], ge, (3, 12)),
+ ('next', 'next()', [], eq, (3, 12)),
+ ('next', 'next(iterator, default=None, /)', ['iterator', 'default'], ge, (3, 13)),
('str', "str(object='', /) -> str", ['object'], ge, (3, 6)),
diff --git a/test/test_utils.py b/test/test_utils.py
index f17fc246..13786d38 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -73,15 +73,21 @@ class TestSetupReadline(unittest.TestCase):
import os
s = 'from os import '
goal = {s + el for el in dir(os)}
+
# There are minor differences, e.g. the dir doesn't include deleted
# items as well as items that are not only available on linux.
difference = set(self.complete(s)).symmetric_difference(goal)
+ ACCEPTED_DIFFERENCE_PREFIXES = [
+ '_', 'O_', 'EX_', 'EFD_', 'MFD_', 'TFD_',
+ 'SF_', 'ST_', 'CLD_', 'POSIX_SPAWN_', 'P_',
+ 'RWF_', 'CLONE_', 'SCHED_', 'SPLICE_',
+ ]
difference = {
x for x in difference
- if all(not x.startswith('from os import ' + s)
- for s in ['_', 'O_', 'EX_', 'MFD_', 'SF_', 'ST_',
- 'CLD_', 'POSIX_SPAWN_', 'P_', 'RWF_',
- 'CLONE_', 'SCHED_'])
+ if not any(
+ x.startswith('from os import ' + prefix)
+ for prefix in ACCEPTED_DIFFERENCE_PREFIXES
+ )
}
# There are quite a few differences, because both Windows and Linux
# (posix and nt) libraries are included.
--
2.45.2
Reference in New Issue View Git Blame Copy Permalink