From 389fbcfd675b914f53617a0de04f1184bd915fce Mon Sep 17 00:00:00 2001 From: Derek Gustafson Date: Wed, 17 Jul 2019 22:19:47 -0400 Subject: [PATCH] Partially remove dependency on imp. --- astroid/brain/brain_six.py | 2 +- astroid/interpreter/_import/spec.py | 89 +++++++++++-------- astroid/modutils.py | 27 ++++-- astroid/scoped_nodes.py | 1 + .../tests/testdata/python2/data/nonregr.py | 4 +- .../tests/testdata/python3/data/nonregr.py | 4 +- astroid/tests/unittest_brain.py | 6 +- astroid/tests/unittest_modutils.py | 17 ++++ 8 files changed, 97 insertions(+), 53 deletions(-) --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -12,17 +12,13 @@ import abc import collections import distutils import enum -import imp import os import sys import zipimport -try: - import importlib.machinery +import importlib.machinery - _HAS_MACHINERY = True -except ImportError: - _HAS_MACHINERY = False +_HAS_MACHINERY = True try: from functools import lru_cache @@ -37,22 +33,6 @@ ModuleType = enum.Enum( "PY_CODERESOURCE PY_COMPILED PY_FROZEN PY_RESOURCE " "PY_SOURCE PY_ZIPMODULE PY_NAMESPACE", ) -_ImpTypes = { - imp.C_BUILTIN: ModuleType.C_BUILTIN, - imp.C_EXTENSION: ModuleType.C_EXTENSION, - imp.PKG_DIRECTORY: ModuleType.PKG_DIRECTORY, - imp.PY_COMPILED: ModuleType.PY_COMPILED, - imp.PY_FROZEN: ModuleType.PY_FROZEN, - imp.PY_SOURCE: ModuleType.PY_SOURCE, -} -if hasattr(imp, "PY_RESOURCE"): - _ImpTypes[imp.PY_RESOURCE] = ModuleType.PY_RESOURCE -if hasattr(imp, "PY_CODERESOURCE"): - _ImpTypes[imp.PY_CODERESOURCE] = ModuleType.PY_CODERESOURCE - - -def _imp_type_to_module_type(imp_type): - return _ImpTypes[imp_type] _ModuleSpec = collections.namedtuple( @@ -118,23 +98,62 @@ class ImpFinder(Finder): """A finder based on the imp module.""" def find_module(self, modname, module_parts, processed, submodule_path): + if not isinstance(modname, str): + raise TypeError("'modname' must be a str, not {}".format(type(modname))) if submodule_path is not None: submodule_path = list(submodule_path) - try: - stream, mp_filename, mp_desc = imp.find_module(modname, submodule_path) - except ImportError: - return None - - # Close resources. - if stream: - stream.close() - - return ModuleSpec( - name=modname, - location=mp_filename, - module_type=_imp_type_to_module_type(mp_desc[2]), + else: + try: + loader = importlib.util.find_spec(modname) + if loader: + if loader.loader is importlib.machinery.BuiltinImporter: + return ModuleSpec( + name=modname, + location=None, + module_type=ModuleType.C_BUILTIN, + ) + if loader.loader is importlib.machinery.FrozenImporter: + return ModuleSpec( + name=modname, + location=None, + module_type=ModuleType.PY_FROZEN, + ) + except ValueError: + pass + submodule_path = sys.path + + suffixes = ( + [ + (s, ModuleType.C_EXTENSION) + for s in importlib.machinery.EXTENSION_SUFFIXES + ] + + [(s, ModuleType.PY_SOURCE) for s in importlib.machinery.SOURCE_SUFFIXES] + + [ + (s, ModuleType.PY_COMPILED) + for s in importlib.machinery.BYTECODE_SUFFIXES + ] ) + for entry in submodule_path: + package_directory = os.path.join(entry, modname) + for suffix in [".py", importlib.machinery.BYTECODE_SUFFIXES[0]]: + package_file_name = "__init__" + suffix + file_path = os.path.join(package_directory, package_file_name) + if os.path.isfile(file_path): + return ModuleSpec( + name=modname, + location=package_directory, + module_type=ModuleType.PKG_DIRECTORY, + ) + for suffix, type_ in suffixes: + file_name = modname + suffix + file_path = os.path.join(entry, file_name) + if os.path.isfile(file_path): + return ModuleSpec( + name=modname, location=file_path, module_type=type_ + ) + return None + def contribute_to_path(self, spec, processed): if spec.location is None: # Builtin. --- a/astroid/modutils.py +++ b/astroid/modutils.py @@ -32,6 +32,7 @@ :var BUILTIN_MODULES: dictionary with builtin module names has key """ import imp +import importlib.util import os import platform import sys @@ -418,7 +419,9 @@ def file_info_from_modpath(modpath, path elif modpath == ["os", "path"]: # FIXME: currently ignoring search_path... return spec.ModuleSpec( - name="os.path", location=os.path.__file__, module_type=imp.PY_SOURCE + name="os.path", + location=os.path.__file__, + module_type=spec.ModuleType.PY_SOURCE, ) return _spec_from_modpath(modpath, path, context) @@ -614,16 +617,22 @@ def is_relative(modname, from_file): from_file = os.path.dirname(from_file) if from_file in sys.path: return False - try: - stream, _, _ = imp.find_module(modname.split(".")[0], [from_file]) + name = os.path.basename(from_file) + file_path = os.path.dirname(from_file) + parent_spec = importlib.util.find_spec(name, from_file) + while parent_spec is None and len(file_path) > 0: + name = os.path.basename(file_path) + "." + name + file_path = os.path.dirname(file_path) + parent_spec = importlib.util.find_spec(name, from_file) - # Close the stream to avoid ResourceWarnings. - if stream: - stream.close() - return True - except ImportError: + if parent_spec is None: return False + submodule_spec = importlib.util.find_spec( + name + "." + modname.split(".")[0], parent_spec.submodule_search_locations + ) + return submodule_spec is not None + # internal only functions ##################################################### --- a/astroid/scoped_nodes.py +++ b/astroid/scoped_nodes.py @@ -1405,6 +1405,7 @@ class FunctionDef(mixins.MultiLineBlockM self.type_comment_returns = type_comment_returns self.type_comment_args = type_comment_args + # pylint: disable=invalid-overridden-method @decorators_mod.cachedproperty def extra_decorators(self): """The extra decorators that this function can have.