diff --git a/jedi/builtin.py b/jedi/builtin.py index 42e94fc28..bd6de005b 100644 --- a/jedi/builtin.py +++ b/jedi/builtin.py @@ -12,6 +12,7 @@ import debug import parsing +import imports def get_sys_path(): @@ -50,6 +51,10 @@ def parser(self): if not self.path or os.path.getmtime(self.path) <= timestamp: self._parser = parser else: + # In case there is already a module cached and this module + # has to be reparsed, we also need to invalidate the import + # caches. + imports.invalidate_star_import_cache(self._parser.module) raise KeyError() except KeyError: self._load_module() diff --git a/jedi/imports.py b/jedi/imports.py index 5ac6cbe24..f51bdd0f9 100644 --- a/jedi/imports.py +++ b/jedi/imports.py @@ -5,6 +5,7 @@ import imp import sys import weakref +import time import builtin import modules @@ -12,10 +13,13 @@ import parsing import evaluate import itertools +import settings # for debugging purposes only imports_processed = 0 +star_import_cache = {} + class ModuleNotFound(Exception): pass @@ -264,12 +268,42 @@ def strip_imports(scopes): return result -def star_import_cache(func): +def cache_star_import(func): def wrapper(scope, *args, **kwargs): - return func(scope, *args, **kwargs) + try: + mods = star_import_cache[scope] + if mods[0] + settings.star_import_cache_validity > time.time(): + return mods[1] + except KeyError: + pass + # cache is too old and therefore invalid or not available + invalidate_star_import_cache(scope) + mods = func(scope, *args, **kwargs) + star_import_cache[scope] = time.time(), mods + + return mods return wrapper -@star_import_cache + +def invalidate_star_import_cache(module, only_main=False): + """ Important if some new modules are being reparsed """ + try: + mods = star_import_cache[module] + + for t, m in mods: + invalidate_star_import_cache(m, only_main=True) + + del star_import_cache[module] + except KeyError: + pass + + if not only_main: + for key, (t, mods) in star_import_cache.items(): + if module in mods: + invalidate_star_import_cache(key) + + +@cache_star_import def remove_star_imports(scope, ignored_modules=[]): """ Check a module for star imports: @@ -285,6 +319,4 @@ def remove_star_imports(scope, ignored_modules=[]): modules += new # Filter duplicate modules. - if len(modules) > 10: - print scope, len(modules) return set(modules) diff --git a/jedi/settings.py b/jedi/settings.py index 6a223d099..23f366dce 100644 --- a/jedi/settings.py +++ b/jedi/settings.py @@ -75,3 +75,13 @@ # Size of the current code part, which is used to speed up parsing. part_line_length = 20 + +# ---------------- +# star import caching +# ---------------- + +# In huge packages like numpy, checking all star imports on every completion +# might be slow, therefore we do a star import caching, that lasts a certain +# time span (in seconds). + +star_import_cache_validity = 60.0 diff --git a/test/regression.py b/test/regression.py index a4f513f41..88949eaf0 100755 --- a/test/regression.py +++ b/test/regression.py @@ -249,7 +249,7 @@ def wrapper(self): for i in range(number): func(self) single_time = (time.time() - first) / number - print(func, single_time) + print('speed', func, single_time) assert single_time < time_per_run return wrapper return decorated @@ -259,15 +259,12 @@ def test_os_path_join(self): s = "from posixpath import join; join('', '')." assert len(self.complete(s)) > 10 # is a str completion - @_check_speed(0.2, number=1) + @_check_speed(0.1) def test_scipy_speed(self): s = 'import scipy.weave; scipy.weave.inline(' - #api.set_debug_function(api.debug.print_to_stdout) script = api.Script(s, 1, len(s), '') script.get_in_function_call() - # self.get_in_function_call(s) - #api.set_debug_function(None) - print(api.imports.imports_processed) + #print(api.imports.imports_processed) if __name__ == '__main__':