337 lines
9.8 KiB
Python
337 lines
9.8 KiB
Python
"""
|
|
This caching is very important for speed and memory optimizations. There's
|
|
nothing really spectacular, just some decorators. The following cache types are
|
|
available:
|
|
|
|
- module caching (`load_parser` and `save_parser`), which uses pickle and is
|
|
really important to assure low load times of modules like ``numpy``.
|
|
- ``time_cache`` can be used to cache something for just a limited time span,
|
|
which can be useful if there's user interaction and the user cannot react
|
|
faster than a certain time.
|
|
|
|
This module is one of the reasons why |jedi| is not thread-safe. As you can see
|
|
there are global variables, which are holding the cache information. Some of
|
|
these variables are being cleaned after every API usage.
|
|
"""
|
|
import time
|
|
import os
|
|
import sys
|
|
import json
|
|
import hashlib
|
|
import gc
|
|
import inspect
|
|
import shutil
|
|
import re
|
|
try:
|
|
import cPickle as pickle
|
|
except ImportError:
|
|
import pickle
|
|
|
|
from jedi import settings
|
|
from jedi import common
|
|
from jedi import debug
|
|
|
|
_time_caches = {}
|
|
|
|
# for fast_parser, should not be deleted
|
|
parser_cache = {}
|
|
|
|
|
|
class ParserCacheItem(object):
|
|
def __init__(self, parser, change_time=None):
|
|
self.parser = parser
|
|
if change_time is None:
|
|
change_time = time.time()
|
|
self.change_time = change_time
|
|
|
|
|
|
def clear_time_caches(delete_all=False):
|
|
""" Jedi caches many things, that should be completed after each completion
|
|
finishes.
|
|
|
|
:param delete_all: Deletes also the cache that is normally not deleted,
|
|
like parser cache, which is important for faster parsing.
|
|
"""
|
|
global _time_caches
|
|
|
|
if delete_all:
|
|
for cache in _time_caches.values():
|
|
cache.clear()
|
|
parser_cache.clear()
|
|
else:
|
|
# normally just kill the expired entries, not all
|
|
for tc in _time_caches.values():
|
|
# check time_cache for expired entries
|
|
for key, (t, value) in list(tc.items()):
|
|
if t < time.time():
|
|
# delete expired entries
|
|
del tc[key]
|
|
|
|
|
|
def time_cache(time_add_setting):
|
|
"""
|
|
s
|
|
This decorator works as follows: Call it with a setting and after that
|
|
use the function with a callable that returns the key.
|
|
But: This function is only called if the key is not available. After a
|
|
certain amount of time (`time_add_setting`) the cache is invalid.
|
|
"""
|
|
def _temp(key_func):
|
|
dct = {}
|
|
_time_caches[time_add_setting] = dct
|
|
|
|
def wrapper(*args, **kwargs):
|
|
generator = key_func(*args, **kwargs)
|
|
key = next(generator)
|
|
try:
|
|
expiry, value = dct[key]
|
|
if expiry > time.time():
|
|
return value
|
|
except KeyError:
|
|
pass
|
|
|
|
value = next(generator)
|
|
time_add = getattr(settings, time_add_setting)
|
|
if key is not None:
|
|
dct[key] = time.time() + time_add, value
|
|
return value
|
|
return wrapper
|
|
return _temp
|
|
|
|
|
|
@time_cache("call_signatures_validity")
|
|
def cache_call_signatures(evaluator, call, source, user_pos):
|
|
"""This function calculates the cache key."""
|
|
index = user_pos[0] - 1
|
|
lines = common.splitlines(source)
|
|
|
|
before_cursor = lines[index][:user_pos[1]]
|
|
other_lines = lines[call.start_pos[0]:index]
|
|
whole = '\n'.join(other_lines + [before_cursor])
|
|
before_bracket = re.match(r'.*\(', whole, re.DOTALL)
|
|
|
|
module_path = call.get_parent_until().path
|
|
yield None if module_path is None else (module_path, before_bracket, call.start_pos)
|
|
yield evaluator.eval_element(call)
|
|
|
|
|
|
def underscore_memoization(func):
|
|
"""
|
|
Decorator for methods::
|
|
|
|
class A(object):
|
|
def x(self):
|
|
if self._x:
|
|
self._x = 10
|
|
return self._x
|
|
|
|
Becomes::
|
|
|
|
class A(object):
|
|
@underscore_memoization
|
|
def x(self):
|
|
return 10
|
|
|
|
A now has an attribute ``_x`` written by this decorator.
|
|
"""
|
|
name = '_' + func.__name__
|
|
|
|
def wrapper(self):
|
|
try:
|
|
return getattr(self, name)
|
|
except AttributeError:
|
|
result = func(self)
|
|
if inspect.isgenerator(result):
|
|
result = list(result)
|
|
setattr(self, name, result)
|
|
return result
|
|
|
|
return wrapper
|
|
|
|
|
|
def memoize_method(method):
|
|
"""A normal memoize function."""
|
|
def wrapper(self, *args, **kwargs):
|
|
dct = self.__dict__.setdefault('_memoize_method_dct', {})
|
|
key = (args, frozenset(kwargs.items()))
|
|
try:
|
|
return dct[key]
|
|
except KeyError:
|
|
result = method(self, *args, **kwargs)
|
|
dct[key] = result
|
|
return result
|
|
return wrapper
|
|
|
|
|
|
def cache_star_import(func):
|
|
@time_cache("star_import_cache_validity")
|
|
def wrapper(self):
|
|
yield self.base # The cache key
|
|
yield func(self)
|
|
return wrapper
|
|
|
|
|
|
def _invalidate_star_import_cache_module(module, only_main=False):
|
|
""" Important if some new modules are being reparsed """
|
|
try:
|
|
t, modules = _time_caches['star_import_cache_validity'][module]
|
|
except KeyError:
|
|
pass
|
|
else:
|
|
del _time_caches['star_import_cache_validity'][module]
|
|
|
|
|
|
def invalidate_star_import_cache(path):
|
|
"""On success returns True."""
|
|
try:
|
|
parser_cache_item = parser_cache[path]
|
|
except KeyError:
|
|
pass
|
|
else:
|
|
_invalidate_star_import_cache_module(parser_cache_item.parser.module)
|
|
|
|
|
|
def load_parser(path):
|
|
"""
|
|
Returns the module or None, if it fails.
|
|
"""
|
|
p_time = os.path.getmtime(path) if path else None
|
|
try:
|
|
parser_cache_item = parser_cache[path]
|
|
if not path or p_time <= parser_cache_item.change_time:
|
|
return parser_cache_item.parser
|
|
else:
|
|
# In case there is already a module cached and this module
|
|
# has to be reparsed, we also need to invalidate the import
|
|
# caches.
|
|
_invalidate_star_import_cache_module(parser_cache_item.parser.module)
|
|
except KeyError:
|
|
if settings.use_filesystem_cache:
|
|
return ParserPickling.load_parser(path, p_time)
|
|
|
|
|
|
def save_parser(path, parser, pickling=True):
|
|
try:
|
|
p_time = None if path is None else os.path.getmtime(path)
|
|
except OSError:
|
|
p_time = None
|
|
pickling = False
|
|
|
|
item = ParserCacheItem(parser, p_time)
|
|
parser_cache[path] = item
|
|
if settings.use_filesystem_cache and pickling:
|
|
ParserPickling.save_parser(path, item)
|
|
|
|
|
|
class ParserPickling(object):
|
|
|
|
version = 24
|
|
"""
|
|
Version number (integer) for file system cache.
|
|
|
|
Increment this number when there are any incompatible changes in
|
|
parser representation classes. For example, the following changes
|
|
are regarded as incompatible.
|
|
|
|
- Class name is changed.
|
|
- Class is moved to another module.
|
|
- Defined slot of the class is changed.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.__index = None
|
|
self.py_tag = 'cpython-%s%s' % sys.version_info[:2]
|
|
"""
|
|
Short name for distinguish Python implementations and versions.
|
|
|
|
It's like `sys.implementation.cache_tag` but for Python < 3.3
|
|
we generate something similar. See:
|
|
http://docs.python.org/3/library/sys.html#sys.implementation
|
|
|
|
.. todo:: Detect interpreter (e.g., PyPy).
|
|
"""
|
|
|
|
def load_parser(self, path, original_changed_time):
|
|
try:
|
|
pickle_changed_time = self._index[path]
|
|
except KeyError:
|
|
return None
|
|
if original_changed_time is not None \
|
|
and pickle_changed_time < original_changed_time:
|
|
# the pickle file is outdated
|
|
return None
|
|
|
|
with open(self._get_hashed_path(path), 'rb') as f:
|
|
try:
|
|
gc.disable()
|
|
parser_cache_item = pickle.load(f)
|
|
finally:
|
|
gc.enable()
|
|
|
|
debug.dbg('pickle loaded: %s', path)
|
|
parser_cache[path] = parser_cache_item
|
|
return parser_cache_item.parser
|
|
|
|
def save_parser(self, path, parser_cache_item):
|
|
self.__index = None
|
|
try:
|
|
files = self._index
|
|
except KeyError:
|
|
files = {}
|
|
self._index = files
|
|
|
|
with open(self._get_hashed_path(path), 'wb') as f:
|
|
pickle.dump(parser_cache_item, f, pickle.HIGHEST_PROTOCOL)
|
|
files[path] = parser_cache_item.change_time
|
|
|
|
self._flush_index()
|
|
|
|
@property
|
|
def _index(self):
|
|
if self.__index is None:
|
|
try:
|
|
with open(self._get_path('index.json')) as f:
|
|
data = json.load(f)
|
|
except (IOError, ValueError):
|
|
self.__index = {}
|
|
else:
|
|
# 0 means version is not defined (= always delete cache):
|
|
if data.get('version', 0) != self.version:
|
|
self.clear_cache()
|
|
self.__index = {}
|
|
else:
|
|
self.__index = data['index']
|
|
return self.__index
|
|
|
|
def _remove_old_modules(self):
|
|
# TODO use
|
|
change = False
|
|
if change:
|
|
self._flush_index(self)
|
|
self._index # reload index
|
|
|
|
def _flush_index(self):
|
|
data = {'version': self.version, 'index': self._index}
|
|
with open(self._get_path('index.json'), 'w') as f:
|
|
json.dump(data, f)
|
|
self.__index = None
|
|
|
|
def clear_cache(self):
|
|
shutil.rmtree(self._cache_directory())
|
|
|
|
def _get_hashed_path(self, path):
|
|
return self._get_path('%s.pkl' % hashlib.md5(path.encode("utf-8")).hexdigest())
|
|
|
|
def _get_path(self, file):
|
|
dir = self._cache_directory()
|
|
if not os.path.exists(dir):
|
|
os.makedirs(dir)
|
|
return os.path.join(dir, file)
|
|
|
|
def _cache_directory(self):
|
|
return os.path.join(settings.cache_directory, self.py_tag)
|
|
|
|
|
|
# is a singleton
|
|
ParserPickling = ParserPickling()
|