fixed subscription table
This commit is contained in:
@@ -0,0 +1,512 @@
|
||||
"""
|
||||
Makes it possible to do the compiled analysis in a subprocess. This has two
|
||||
goals:
|
||||
|
||||
1. Making it safer - Segfaults and RuntimeErrors as well as stdout/stderr can
|
||||
be ignored and dealt with.
|
||||
2. Make it possible to handle different Python versions as well as virtualenvs.
|
||||
|
||||
The architecture here is briefly:
|
||||
- For each Jedi `Environment` there is a corresponding subprocess which
|
||||
operates within the target environment. If the subprocess dies it is replaced
|
||||
at this level.
|
||||
- `CompiledSubprocess` manages exactly one subprocess and handles communication
|
||||
from the parent side.
|
||||
- `Listener` runs within the subprocess, processing each request and yielding
|
||||
results.
|
||||
- `InterpreterEnvironment` provides an API which matches that of `Environment`,
|
||||
but runs functionality inline rather than within a subprocess. It is thus
|
||||
used both directly in places where a subprocess is unnecessary and/or
|
||||
undesirable and also within subprocesses themselves.
|
||||
- `InferenceStateSubprocess` (or `InferenceStateSameProcess`) provide high
|
||||
level access to functionality within the subprocess from within the parent.
|
||||
Each `InterpreterState` has an instance of one of these, provided by its
|
||||
environment.
|
||||
"""
|
||||
|
||||
import collections
|
||||
import os
|
||||
import sys
|
||||
import queue
|
||||
import subprocess
|
||||
import traceback
|
||||
import weakref
|
||||
from functools import partial
|
||||
from threading import Thread
|
||||
from typing import Dict, TYPE_CHECKING
|
||||
|
||||
from jedi._compatibility import pickle_dump, pickle_load
|
||||
from jedi import debug
|
||||
from jedi.cache import memoize_method
|
||||
from jedi.inference.compiled.subprocess import functions
|
||||
from jedi.inference.compiled.access import DirectObjectAccess, AccessPath, \
|
||||
SignatureParam
|
||||
from jedi.api.exceptions import InternalError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from jedi.inference import InferenceState
|
||||
|
||||
|
||||
_MAIN_PATH = os.path.join(os.path.dirname(__file__), '__main__.py')
|
||||
PICKLE_PROTOCOL = 4
|
||||
|
||||
|
||||
def _GeneralizedPopen(*args, **kwargs):
|
||||
if os.name == 'nt':
|
||||
try:
|
||||
# Was introduced in Python 3.7.
|
||||
CREATE_NO_WINDOW = subprocess.CREATE_NO_WINDOW
|
||||
except AttributeError:
|
||||
CREATE_NO_WINDOW = 0x08000000
|
||||
kwargs['creationflags'] = CREATE_NO_WINDOW
|
||||
# The child process doesn't need file descriptors except 0, 1, 2.
|
||||
# This is unix only.
|
||||
kwargs['close_fds'] = 'posix' in sys.builtin_module_names
|
||||
|
||||
return subprocess.Popen(*args, **kwargs)
|
||||
|
||||
|
||||
def _enqueue_output(out, queue_):
|
||||
for line in iter(out.readline, b''):
|
||||
queue_.put(line)
|
||||
|
||||
|
||||
def _add_stderr_to_debug(stderr_queue):
|
||||
while True:
|
||||
# Try to do some error reporting from the subprocess and print its
|
||||
# stderr contents.
|
||||
try:
|
||||
line = stderr_queue.get_nowait()
|
||||
line = line.decode('utf-8', 'replace')
|
||||
debug.warning('stderr output: %s' % line.rstrip('\n'))
|
||||
except queue.Empty:
|
||||
break
|
||||
|
||||
|
||||
def _get_function(name):
|
||||
return getattr(functions, name)
|
||||
|
||||
|
||||
def _cleanup_process(process, thread):
|
||||
try:
|
||||
process.kill()
|
||||
process.wait()
|
||||
except OSError:
|
||||
# Raised if the process is already killed.
|
||||
pass
|
||||
thread.join()
|
||||
for stream in [process.stdin, process.stdout, process.stderr]:
|
||||
try:
|
||||
stream.close()
|
||||
except OSError:
|
||||
# Raised if the stream is broken.
|
||||
pass
|
||||
|
||||
|
||||
class _InferenceStateProcess:
|
||||
def __init__(self, inference_state: 'InferenceState') -> None:
|
||||
self._inference_state_weakref = weakref.ref(inference_state)
|
||||
self._handles: Dict[int, AccessHandle] = {}
|
||||
|
||||
def get_or_create_access_handle(self, obj):
|
||||
id_ = id(obj)
|
||||
try:
|
||||
return self.get_access_handle(id_)
|
||||
except KeyError:
|
||||
access = DirectObjectAccess(self._inference_state_weakref(), obj)
|
||||
handle = AccessHandle(self, access, id_)
|
||||
self.set_access_handle(handle)
|
||||
return handle
|
||||
|
||||
def get_access_handle(self, id_):
|
||||
return self._handles[id_]
|
||||
|
||||
def set_access_handle(self, handle):
|
||||
self._handles[handle.id] = handle
|
||||
|
||||
|
||||
class InferenceStateSameProcess(_InferenceStateProcess):
|
||||
"""
|
||||
Basically just an easy access to functions.py. It has the same API
|
||||
as InferenceStateSubprocess and does the same thing without using a subprocess.
|
||||
This is necessary for the Interpreter process.
|
||||
"""
|
||||
def __getattr__(self, name):
|
||||
return partial(_get_function(name), self._inference_state_weakref())
|
||||
|
||||
|
||||
class InferenceStateSubprocess(_InferenceStateProcess):
|
||||
"""
|
||||
API to functionality which will run in a subprocess.
|
||||
|
||||
This mediates the interaction between an `InferenceState` and the actual
|
||||
execution of functionality running within a `CompiledSubprocess`. Available
|
||||
functions are defined in `.functions`, though should be accessed via
|
||||
attributes on this class of the same name.
|
||||
|
||||
This class is responsible for indicating that the `InferenceState` within
|
||||
the subprocess can be removed once the corresponding instance in the parent
|
||||
goes away.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
inference_state: 'InferenceState',
|
||||
compiled_subprocess: 'CompiledSubprocess',
|
||||
) -> None:
|
||||
super().__init__(inference_state)
|
||||
self._used = False
|
||||
self._compiled_subprocess = compiled_subprocess
|
||||
|
||||
# Opaque id we'll pass to the subprocess to identify the context (an
|
||||
# `InferenceState`) which should be used for the request. This allows us
|
||||
# to make subsequent requests which operate on results from previous
|
||||
# ones, while keeping a single subprocess which can work with several
|
||||
# contexts in the parent process. Once it is no longer needed(i.e: when
|
||||
# this class goes away), we also use this id to indicate that the
|
||||
# subprocess can discard the context.
|
||||
#
|
||||
# Note: this id is deliberately coupled to this class (and not to
|
||||
# `InferenceState`) as this class manages access handle mappings which
|
||||
# must correspond to those in the subprocess. This approach also avoids
|
||||
# race conditions from successive `InferenceState`s with the same object
|
||||
# id (as observed while adding support for Python 3.13).
|
||||
#
|
||||
# This value does not need to be the `id()` of this instance, we merely
|
||||
# need to ensure that it enables the (visible) lifetime of the context
|
||||
# within the subprocess to match that of this class. We therefore also
|
||||
# depend on the semantics of `CompiledSubprocess.delete_inference_state`
|
||||
# for correctness.
|
||||
self._inference_state_id = id(self)
|
||||
|
||||
def __getattr__(self, name):
|
||||
func = _get_function(name)
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
self._used = True
|
||||
|
||||
result = self._compiled_subprocess.run(
|
||||
self._inference_state_id,
|
||||
func,
|
||||
args=args,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
# IMO it should be possible to create a hook in pickle.load to
|
||||
# mess with the loaded objects. However it's extremely complicated
|
||||
# to work around this so just do it with this call. ~ dave
|
||||
return self._convert_access_handles(result)
|
||||
|
||||
return wrapper
|
||||
|
||||
def _convert_access_handles(self, obj):
|
||||
if isinstance(obj, SignatureParam):
|
||||
return SignatureParam(*self._convert_access_handles(tuple(obj)))
|
||||
elif isinstance(obj, tuple):
|
||||
return tuple(self._convert_access_handles(o) for o in obj)
|
||||
elif isinstance(obj, list):
|
||||
return [self._convert_access_handles(o) for o in obj]
|
||||
elif isinstance(obj, AccessHandle):
|
||||
try:
|
||||
# Rewrite the access handle to one we're already having.
|
||||
obj = self.get_access_handle(obj.id)
|
||||
except KeyError:
|
||||
obj.add_subprocess(self)
|
||||
self.set_access_handle(obj)
|
||||
elif isinstance(obj, AccessPath):
|
||||
return AccessPath(self._convert_access_handles(obj.accesses))
|
||||
return obj
|
||||
|
||||
def __del__(self):
|
||||
if self._used and not self._compiled_subprocess.is_crashed:
|
||||
self._compiled_subprocess.delete_inference_state(self._inference_state_id)
|
||||
|
||||
|
||||
class CompiledSubprocess:
|
||||
"""
|
||||
A subprocess which runs inference within a target environment.
|
||||
|
||||
This class manages the interface to a single instance of such a process as
|
||||
well as the lifecycle of the process itself. See `.__main__` and `Listener`
|
||||
for the implementation of the subprocess and details of the protocol.
|
||||
|
||||
A single live instance of this is maintained by `jedi.api.environment.Environment`,
|
||||
so that typically a single subprocess is used at a time.
|
||||
"""
|
||||
|
||||
is_crashed = False
|
||||
|
||||
def __init__(self, executable, env_vars=None):
|
||||
self._executable = executable
|
||||
self._env_vars = env_vars
|
||||
self._inference_state_deletion_queue = collections.deque()
|
||||
self._cleanup_callable = lambda: None
|
||||
|
||||
def __repr__(self):
|
||||
pid = os.getpid()
|
||||
return '<%s _executable=%r, is_crashed=%r, pid=%r>' % (
|
||||
self.__class__.__name__,
|
||||
self._executable,
|
||||
self.is_crashed,
|
||||
pid,
|
||||
)
|
||||
|
||||
@memoize_method
|
||||
def _get_process(self):
|
||||
debug.dbg('Start environment subprocess %s', self._executable)
|
||||
parso_path = sys.modules['parso'].__file__
|
||||
args = (
|
||||
self._executable,
|
||||
_MAIN_PATH,
|
||||
os.path.dirname(os.path.dirname(parso_path)),
|
||||
'.'.join(str(x) for x in sys.version_info[:3]),
|
||||
)
|
||||
process = _GeneralizedPopen(
|
||||
args,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=self._env_vars
|
||||
)
|
||||
self._stderr_queue = queue.Queue()
|
||||
self._stderr_thread = t = Thread(
|
||||
target=_enqueue_output,
|
||||
args=(process.stderr, self._stderr_queue)
|
||||
)
|
||||
t.daemon = True
|
||||
t.start()
|
||||
# Ensure the subprocess is properly cleaned up when the object
|
||||
# is garbage collected.
|
||||
self._cleanup_callable = weakref.finalize(self,
|
||||
_cleanup_process,
|
||||
process,
|
||||
t)
|
||||
return process
|
||||
|
||||
def run(self, inference_state_id, function, args=(), kwargs={}):
|
||||
# Delete old inference_states.
|
||||
while True:
|
||||
try:
|
||||
delete_id = self._inference_state_deletion_queue.pop()
|
||||
except IndexError:
|
||||
break
|
||||
else:
|
||||
self._send(delete_id, None)
|
||||
|
||||
assert callable(function)
|
||||
return self._send(inference_state_id, function, args, kwargs)
|
||||
|
||||
def get_sys_path(self):
|
||||
return self._send(None, functions.get_sys_path, (), {})
|
||||
|
||||
def _kill(self):
|
||||
self.is_crashed = True
|
||||
self._cleanup_callable()
|
||||
|
||||
def _send(self, inference_state_id, function, args=(), kwargs={}):
|
||||
if self.is_crashed:
|
||||
raise InternalError("The subprocess %s has crashed." % self._executable)
|
||||
|
||||
data = inference_state_id, function, args, kwargs
|
||||
try:
|
||||
pickle_dump(data, self._get_process().stdin, PICKLE_PROTOCOL)
|
||||
except BrokenPipeError:
|
||||
self._kill()
|
||||
raise InternalError("The subprocess %s was killed. Maybe out of memory?"
|
||||
% self._executable)
|
||||
|
||||
try:
|
||||
is_exception, traceback, result = pickle_load(self._get_process().stdout)
|
||||
except EOFError as eof_error:
|
||||
try:
|
||||
stderr = self._get_process().stderr.read().decode('utf-8', 'replace')
|
||||
except Exception as exc:
|
||||
stderr = '<empty/not available (%r)>' % exc
|
||||
self._kill()
|
||||
_add_stderr_to_debug(self._stderr_queue)
|
||||
raise InternalError(
|
||||
"The subprocess %s has crashed (%r, stderr=%s)." % (
|
||||
self._executable,
|
||||
eof_error,
|
||||
stderr,
|
||||
))
|
||||
|
||||
_add_stderr_to_debug(self._stderr_queue)
|
||||
|
||||
if is_exception:
|
||||
# Replace the attribute error message with a the traceback. It's
|
||||
# way more informative.
|
||||
result.args = (traceback,)
|
||||
raise result
|
||||
return result
|
||||
|
||||
def delete_inference_state(self, inference_state_id):
|
||||
"""
|
||||
Indicate that an inference state (in the subprocess) is no longer
|
||||
needed.
|
||||
|
||||
The state corresponding to the given id will become inaccessible and the
|
||||
id may safely be re-used to refer to a different context.
|
||||
|
||||
Note: it is not guaranteed that the corresponding state will actually be
|
||||
deleted immediately.
|
||||
"""
|
||||
# Warning: if changing the semantics of context deletion see the comment
|
||||
# in `InferenceStateSubprocess.__init__` regarding potential race
|
||||
# conditions.
|
||||
|
||||
# Currently we are not deleting the related state instantly. They only
|
||||
# get deleted once the subprocess is used again. It would probably a
|
||||
# better solution to move all of this into a thread. However, the memory
|
||||
# usage of a single inference_state shouldn't be that high.
|
||||
self._inference_state_deletion_queue.append(inference_state_id)
|
||||
|
||||
|
||||
class Listener:
|
||||
"""
|
||||
Main loop for the subprocess which actually does the inference.
|
||||
|
||||
This class runs within the target environment. It listens to instructions
|
||||
from the parent process, runs inference and returns the results.
|
||||
|
||||
The subprocess has a long lifetime and is expected to process several
|
||||
requests, including for different `InferenceState` instances in the parent.
|
||||
See `CompiledSubprocess` for the parent half of the system.
|
||||
|
||||
Communication is via pickled data sent serially over stdin and stdout.
|
||||
Stderr is read only if the child process crashes.
|
||||
|
||||
The request protocol is a 4-tuple of:
|
||||
* inference_state_id | None: an opaque identifier of the parent's
|
||||
`InferenceState`. An `InferenceState` operating over an
|
||||
`InterpreterEnvironment` is created within this process for each of
|
||||
these, ensuring that each parent context has a corresponding context
|
||||
here. This allows context to be persisted between requests. Unless
|
||||
`None`, the local `InferenceState` will be passed to the given function
|
||||
as the first positional argument.
|
||||
* function | None: the function to run. This is expected to be a member of
|
||||
`.functions`. `None` indicates that the corresponding inference state is
|
||||
no longer needed and should be dropped.
|
||||
* args: positional arguments to the `function`. If any of these are
|
||||
`AccessHandle` instances they will be adapted to the local
|
||||
`InferenceState` before being passed.
|
||||
* kwargs: keyword arguments to the `function`. If any of these are
|
||||
`AccessHandle` instances they will be adapted to the local
|
||||
`InferenceState` before being passed.
|
||||
|
||||
The result protocol is a 3-tuple of either:
|
||||
* (False, None, function result): if the function returns without error, or
|
||||
* (True, traceback, exception): if the function raises an exception
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._inference_states = {}
|
||||
|
||||
def _get_inference_state(self, function, inference_state_id):
|
||||
from jedi.inference import InferenceState
|
||||
|
||||
try:
|
||||
inference_state = self._inference_states[inference_state_id]
|
||||
except KeyError:
|
||||
from jedi import InterpreterEnvironment
|
||||
inference_state = InferenceState(
|
||||
# The project is not actually needed. Nothing should need to
|
||||
# access it.
|
||||
project=None,
|
||||
environment=InterpreterEnvironment()
|
||||
)
|
||||
self._inference_states[inference_state_id] = inference_state
|
||||
return inference_state
|
||||
|
||||
def _run(self, inference_state_id, function, args, kwargs):
|
||||
if inference_state_id is None:
|
||||
return function(*args, **kwargs)
|
||||
elif function is None:
|
||||
# Warning: if changing the semantics of context deletion see the comment
|
||||
# in `InferenceStateSubprocess.__init__` regarding potential race
|
||||
# conditions.
|
||||
del self._inference_states[inference_state_id]
|
||||
else:
|
||||
inference_state = self._get_inference_state(function, inference_state_id)
|
||||
|
||||
# Exchange all handles
|
||||
args = list(args)
|
||||
for i, arg in enumerate(args):
|
||||
if isinstance(arg, AccessHandle):
|
||||
args[i] = inference_state.compiled_subprocess.get_access_handle(arg.id)
|
||||
for key, value in kwargs.items():
|
||||
if isinstance(value, AccessHandle):
|
||||
kwargs[key] = inference_state.compiled_subprocess.get_access_handle(value.id)
|
||||
|
||||
return function(inference_state, *args, **kwargs)
|
||||
|
||||
def listen(self):
|
||||
stdout = sys.stdout
|
||||
# Mute stdout. Nobody should actually be able to write to it,
|
||||
# because stdout is used for IPC.
|
||||
sys.stdout = open(os.devnull, 'w')
|
||||
stdin = sys.stdin
|
||||
stdout = stdout.buffer
|
||||
stdin = stdin.buffer
|
||||
|
||||
while True:
|
||||
try:
|
||||
payload = pickle_load(stdin)
|
||||
except EOFError:
|
||||
# It looks like the parent process closed.
|
||||
# Don't make a big fuss here and just exit.
|
||||
exit(0)
|
||||
try:
|
||||
result = False, None, self._run(*payload)
|
||||
except Exception as e:
|
||||
result = True, traceback.format_exc(), e
|
||||
|
||||
pickle_dump(result, stdout, PICKLE_PROTOCOL)
|
||||
|
||||
|
||||
class AccessHandle:
|
||||
def __init__(
|
||||
self,
|
||||
subprocess: _InferenceStateProcess,
|
||||
access: DirectObjectAccess,
|
||||
id_: int,
|
||||
) -> None:
|
||||
self.access = access
|
||||
self._subprocess = subprocess
|
||||
self.id = id_
|
||||
|
||||
def add_subprocess(self, subprocess):
|
||||
self._subprocess = subprocess
|
||||
|
||||
def __repr__(self):
|
||||
try:
|
||||
detail = self.access
|
||||
except AttributeError:
|
||||
detail = '#' + str(self.id)
|
||||
return '<%s of %s>' % (self.__class__.__name__, detail)
|
||||
|
||||
def __getstate__(self):
|
||||
return self.id
|
||||
|
||||
def __setstate__(self, state):
|
||||
self.id = state
|
||||
|
||||
def __getattr__(self, name):
|
||||
if name in ('id', 'access') or name.startswith('_'):
|
||||
raise AttributeError("Something went wrong with unpickling")
|
||||
|
||||
# print('getattr', name, file=sys.stderr)
|
||||
return partial(self._workaround, name)
|
||||
|
||||
def _workaround(self, name, *args, **kwargs):
|
||||
"""
|
||||
TODO Currently we're passing slice objects around. This should not
|
||||
happen. They are also the only unhashable objects that we're passing
|
||||
around.
|
||||
"""
|
||||
if args and isinstance(args[0], slice):
|
||||
return self._subprocess.get_compiled_method_return(self.id, name, *args, **kwargs)
|
||||
return self._cached_results(name, *args, **kwargs)
|
||||
|
||||
@memoize_method
|
||||
def _cached_results(self, name, *args, **kwargs):
|
||||
return self._subprocess.get_compiled_method_return(self.id, name, *args, **kwargs)
|
||||
@@ -0,0 +1,40 @@
|
||||
import os
|
||||
import sys
|
||||
from importlib.abc import MetaPathFinder
|
||||
from importlib.machinery import PathFinder
|
||||
|
||||
# Remove the first entry, because it's simply a directory entry that equals
|
||||
# this directory.
|
||||
del sys.path[0]
|
||||
|
||||
|
||||
def _get_paths():
|
||||
# Get the path to jedi.
|
||||
_d = os.path.dirname
|
||||
_jedi_path = _d(_d(_d(_d(_d(__file__)))))
|
||||
_parso_path = sys.argv[1]
|
||||
# The paths are the directory that jedi and parso lie in.
|
||||
return {'jedi': _jedi_path, 'parso': _parso_path}
|
||||
|
||||
|
||||
class _ExactImporter(MetaPathFinder):
|
||||
def __init__(self, path_dct):
|
||||
self._path_dct = path_dct
|
||||
|
||||
def find_spec(self, fullname, path=None, target=None):
|
||||
if path is None and fullname in self._path_dct:
|
||||
p = self._path_dct[fullname]
|
||||
spec = PathFinder.find_spec(fullname, path=[p], target=target)
|
||||
return spec
|
||||
return None
|
||||
|
||||
|
||||
# Try to import jedi/parso.
|
||||
sys.meta_path.insert(0, _ExactImporter(_get_paths()))
|
||||
from jedi.inference.compiled import subprocess # noqa: E402
|
||||
sys.meta_path.pop(0)
|
||||
|
||||
# Retrieve the pickle protocol.
|
||||
host_sys_version = [int(x) for x in sys.argv[2].split('.')]
|
||||
# And finally start the client.
|
||||
subprocess.Listener().listen()
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,257 @@
|
||||
import sys
|
||||
import os
|
||||
import inspect
|
||||
import importlib
|
||||
from pathlib import Path
|
||||
from zipfile import ZipFile
|
||||
from zipimport import zipimporter, ZipImportError
|
||||
from importlib.machinery import all_suffixes
|
||||
|
||||
from jedi.inference.compiled import access
|
||||
from jedi import debug
|
||||
from jedi import parser_utils
|
||||
from jedi.file_io import KnownContentFileIO, ZipFileIO
|
||||
|
||||
|
||||
def get_sys_path():
|
||||
return sys.path
|
||||
|
||||
|
||||
def load_module(inference_state, **kwargs):
|
||||
return access.load_module(inference_state, **kwargs)
|
||||
|
||||
|
||||
def get_compiled_method_return(inference_state, id, attribute, *args, **kwargs):
|
||||
handle = inference_state.compiled_subprocess.get_access_handle(id)
|
||||
return getattr(handle.access, attribute)(*args, **kwargs)
|
||||
|
||||
|
||||
def create_simple_object(inference_state, obj):
|
||||
return access.create_access_path(inference_state, obj)
|
||||
|
||||
|
||||
def get_module_info(inference_state, sys_path=None, full_name=None, **kwargs):
|
||||
"""
|
||||
Returns Tuple[Union[NamespaceInfo, FileIO, None], Optional[bool]]
|
||||
"""
|
||||
if sys_path is not None:
|
||||
sys.path, temp = sys_path, sys.path
|
||||
try:
|
||||
return _find_module(full_name=full_name, **kwargs)
|
||||
except ImportError:
|
||||
return None, None
|
||||
finally:
|
||||
if sys_path is not None:
|
||||
sys.path = temp
|
||||
|
||||
|
||||
def get_builtin_module_names(inference_state):
|
||||
return sys.builtin_module_names
|
||||
|
||||
|
||||
def _test_raise_error(inference_state, exception_type):
|
||||
"""
|
||||
Raise an error to simulate certain problems for unit tests.
|
||||
"""
|
||||
raise exception_type
|
||||
|
||||
|
||||
def _test_print(inference_state, stderr=None, stdout=None):
|
||||
"""
|
||||
Force some prints in the subprocesses. This exists for unit tests.
|
||||
"""
|
||||
if stderr is not None:
|
||||
print(stderr, file=sys.stderr)
|
||||
sys.stderr.flush()
|
||||
if stdout is not None:
|
||||
print(stdout)
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def _get_init_path(directory_path):
|
||||
"""
|
||||
The __init__ file can be searched in a directory. If found return it, else
|
||||
None.
|
||||
"""
|
||||
for suffix in all_suffixes():
|
||||
path = os.path.join(directory_path, '__init__' + suffix)
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
def safe_literal_eval(inference_state, value):
|
||||
return parser_utils.safe_literal_eval(value)
|
||||
|
||||
|
||||
def iter_module_names(*args, **kwargs):
|
||||
return list(_iter_module_names(*args, **kwargs))
|
||||
|
||||
|
||||
def _iter_module_names(inference_state, paths):
|
||||
# Python modules/packages
|
||||
for path in paths:
|
||||
try:
|
||||
dir_entries = ((entry.name, entry.is_dir()) for entry in os.scandir(path))
|
||||
except OSError:
|
||||
try:
|
||||
zip_import_info = zipimporter(path)
|
||||
# Unfortunately, there is no public way to access zipimporter's
|
||||
# private _files member. We therefore have to use a
|
||||
# custom function to iterate over the files.
|
||||
dir_entries = _zip_list_subdirectory(
|
||||
zip_import_info.archive, zip_import_info.prefix)
|
||||
except ZipImportError:
|
||||
# The file might not exist or reading it might lead to an error.
|
||||
debug.warning("Not possible to list directory: %s", path)
|
||||
continue
|
||||
for name, is_dir in dir_entries:
|
||||
# First Namespaces then modules/stubs
|
||||
if is_dir:
|
||||
# pycache is obviously not an interesting namespace. Also the
|
||||
# name must be a valid identifier.
|
||||
if name != '__pycache__' and name.isidentifier():
|
||||
yield name
|
||||
else:
|
||||
if name.endswith('.pyi'): # Stub files
|
||||
modname = name[:-4]
|
||||
else:
|
||||
modname = inspect.getmodulename(name)
|
||||
|
||||
if modname and '.' not in modname:
|
||||
if modname != '__init__':
|
||||
yield modname
|
||||
|
||||
|
||||
def _find_module(string, path=None, full_name=None, is_global_search=True):
|
||||
"""
|
||||
Provides information about a module.
|
||||
|
||||
This function isolates the differences in importing libraries introduced with
|
||||
python 3.3 on; it gets a module name and optionally a path. It will return a
|
||||
tuple containin an open file for the module (if not builtin), the filename
|
||||
or the name of the module if it is a builtin one and a boolean indicating
|
||||
if the module is contained in a package.
|
||||
"""
|
||||
spec = None
|
||||
loader = None
|
||||
|
||||
for finder in sys.meta_path:
|
||||
if is_global_search and finder != importlib.machinery.PathFinder:
|
||||
p = None
|
||||
else:
|
||||
p = path
|
||||
try:
|
||||
find_spec = finder.find_spec
|
||||
except AttributeError:
|
||||
# These are old-school clases that still have a different API, just
|
||||
# ignore those.
|
||||
continue
|
||||
|
||||
spec = find_spec(string, p)
|
||||
if spec is not None:
|
||||
if spec.origin == "frozen":
|
||||
continue
|
||||
|
||||
loader = spec.loader
|
||||
|
||||
if loader is None and not spec.has_location:
|
||||
# This is a namespace package.
|
||||
full_name = string if not path else full_name
|
||||
implicit_ns_info = ImplicitNSInfo(full_name, spec.submodule_search_locations._path)
|
||||
return implicit_ns_info, True
|
||||
break
|
||||
|
||||
return _find_module_py33(string, path, loader)
|
||||
|
||||
|
||||
def _find_module_py33(string, path=None, loader=None, full_name=None, is_global_search=True):
|
||||
if not loader:
|
||||
spec = importlib.machinery.PathFinder.find_spec(string, path)
|
||||
if spec is not None:
|
||||
loader = spec.loader
|
||||
|
||||
if loader is None and path is None: # Fallback to find builtins
|
||||
try:
|
||||
spec = importlib.util.find_spec(string)
|
||||
if spec is not None:
|
||||
loader = spec.loader
|
||||
except ValueError as e:
|
||||
# See #491. Importlib might raise a ValueError, to avoid this, we
|
||||
# just raise an ImportError to fix the issue.
|
||||
raise ImportError("Originally " + repr(e))
|
||||
|
||||
if loader is None:
|
||||
raise ImportError("Couldn't find a loader for {}".format(string))
|
||||
|
||||
return _from_loader(loader, string)
|
||||
|
||||
|
||||
def _from_loader(loader, string):
|
||||
try:
|
||||
is_package_method = loader.is_package
|
||||
except AttributeError:
|
||||
is_package = False
|
||||
else:
|
||||
is_package = is_package_method(string)
|
||||
try:
|
||||
get_filename = loader.get_filename
|
||||
except AttributeError:
|
||||
return None, is_package
|
||||
else:
|
||||
module_path = get_filename(string)
|
||||
|
||||
# To avoid unicode and read bytes, "overwrite" loader.get_source if
|
||||
# possible.
|
||||
try:
|
||||
f = type(loader).get_source
|
||||
except AttributeError:
|
||||
raise ImportError("get_source was not defined on loader")
|
||||
|
||||
if f is not importlib.machinery.SourceFileLoader.get_source:
|
||||
# Unfortunately we are reading unicode here, not bytes.
|
||||
# It seems hard to get bytes, because the zip importer
|
||||
# logic just unpacks the zip file and returns a file descriptor
|
||||
# that we cannot as easily access. Therefore we just read it as
|
||||
# a string in the cases where get_source was overwritten.
|
||||
code = loader.get_source(string)
|
||||
else:
|
||||
code = _get_source(loader, string)
|
||||
|
||||
if code is None:
|
||||
return None, is_package
|
||||
if isinstance(loader, zipimporter):
|
||||
return ZipFileIO(module_path, code, Path(loader.archive)), is_package
|
||||
|
||||
return KnownContentFileIO(module_path, code), is_package
|
||||
|
||||
|
||||
def _get_source(loader, fullname):
|
||||
"""
|
||||
This method is here as a replacement for SourceLoader.get_source. That
|
||||
method returns unicode, but we prefer bytes.
|
||||
"""
|
||||
path = loader.get_filename(fullname)
|
||||
try:
|
||||
return loader.get_data(path)
|
||||
except OSError:
|
||||
raise ImportError('source not available through get_data()',
|
||||
name=fullname)
|
||||
|
||||
|
||||
def _zip_list_subdirectory(zip_path, zip_subdir_path):
|
||||
zip_file = ZipFile(zip_path)
|
||||
zip_subdir_path = Path(zip_subdir_path)
|
||||
zip_content_file_paths = zip_file.namelist()
|
||||
for raw_file_name in zip_content_file_paths:
|
||||
file_path = Path(raw_file_name)
|
||||
if file_path.parent == zip_subdir_path:
|
||||
file_path = file_path.relative_to(zip_subdir_path)
|
||||
yield file_path.name, raw_file_name.endswith("/")
|
||||
|
||||
|
||||
class ImplicitNSInfo:
|
||||
"""Stores information returned from an implicit namespace spec"""
|
||||
def __init__(self, name, paths):
|
||||
self.name = name
|
||||
self.paths = paths
|
||||
Reference in New Issue
Block a user