diff --git a/news/pkgrsrc.rst b/news/pkgrsrc.rst index 0421c9621..7e84b997b 100644 --- a/news/pkgrsrc.rst +++ b/news/pkgrsrc.rst @@ -1,7 +1,7 @@ **Added:** -* New ``xonsh.bg_pkg_resources`` module for loading the ``pkg_resources`` - module in a background thread. +* New tools in ``xonsh.lazyasd`` module for loading modules in background + threads. **Changed:** @@ -9,6 +9,8 @@ background. * Sped up loading of prompt-toolkit by ~2x-3x by loading ``pkg_resources`` in background. +* ``setup.py`` will no longer git checkout to replace the version number. + Now it simply stores and reuses the original version line. **Deprecated:** None diff --git a/setup.py b/setup.py index ea37b6621..d38b7390c 100755 --- a/setup.py +++ b/setup.py @@ -114,40 +114,41 @@ def dirty_version(): _version = _version.decode('ascii') except (subprocess.CalledProcessError, FileNotFoundError): return False - try: base, N, sha = _version.strip().split('-') except ValueError: #on base release open('xonsh/dev.githash', 'w').close() return False - replace_version(base, N) with open('xonsh/dev.githash', 'w') as f: f.write(sha) - return True +ORIGINAL_VERSION_LINE = None + def replace_version(base, N): """Replace version in `__init__.py` with devN suffix""" + global ORIGINAL_VERSION_LINE with open('xonsh/__init__.py', 'r') as f: raw = f.read() lines = raw.splitlines() + ORIGINAL_VERSION_LINE = lines[0] lines[0] = "__version__ = '{}.dev{}'".format(base, N) upd = '\n'.join(lines) + '\n' with open('xonsh/__init__.py', 'w') as f: f.write(upd) -def discard_changes(): - """If we touch ``__init__.py``, discard changes after install""" - try: - _ = subprocess.check_output(['git', - 'checkout', - '--', - 'xonsh/__init__.py']) - except subprocess.CalledProcessError: - pass +def restore_version(): + """If we touch the version in __init__.py discard changes after install.""" + with open('xonsh/__init__.py', 'r') as f: + raw = f.read() + lines = raw.splitlines() + lines[0] = ORIGINAL_VERSION_LINE + upd = '\n'.join(lines) + '\n' + with open('xonsh/__init__.py', 'w') as f: + f.write(upd) class xinstall(install): @@ -168,7 +169,7 @@ class xinstall(install): print('Installing Jupyter hook failed.') install.run(self) if dirty: - discard_changes() + restore_version() @@ -180,7 +181,7 @@ class xsdist(sdist): dirty = dirty_version() sdist.make_release_tree(self, basedir, files) if dirty: - discard_changes() + restore_version() #----------------------------------------------------------------------------- @@ -215,7 +216,7 @@ if HAVE_SETUPTOOLS: dirty = dirty_version() develop.run(self) if dirty: - discard_changes() + restore_version() def main(): diff --git a/xonsh/__init__.py b/xonsh/__init__.py index 45b0d34fd..8d5fdb949 100644 --- a/xonsh/__init__.py +++ b/xonsh/__init__.py @@ -9,8 +9,6 @@ else: import sys as _sys try: from xonsh import __amalgam__ - bg_pkg_resources = __amalgam__ - _sys.modules['xonsh.bg_pkg_resources'] = __amalgam__ completer = __amalgam__ _sys.modules['xonsh.completer'] = __amalgam__ lazyasd = __amalgam__ diff --git a/xonsh/bg_pkg_resources.py b/xonsh/bg_pkg_resources.py deleted file mode 100644 index 93d0a5c3b..000000000 --- a/xonsh/bg_pkg_resources.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Background thread loader for pkg_resources.""" -import os -import sys -import time -import builtins -import threading -import importlib - - -class PkgResourcesProxy(object): - """Proxy object for pkg_resources module that throws an ImportError - whenever an attribut is accessed. - """ - - def __getattr__(self, name): - raise ImportError('cannot access ' + name + 'on PkgResourcesProxy, ' - 'please wait for pkg_resources module to be fully ' - 'loaded.') - - -class PkgResourcesLoader(threading.Thread): - """Thread to load the pkg_resources module.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.daemon = True - self.start() - - def run(self): - # wait for other modules to stop being imported - i = 0 - last = -6 - hist = [-5, -4, -3, -2, -1] - while not all(last == x for x in hist): - time.sleep(0.001) - last = hist[i%5] = len(sys.modules) - i += 1 - # now import pkg_resources properly - if isinstance(sys.modules['pkg_resources'], PkgResourcesProxy): - del sys.modules['pkg_resources'] - pr = importlib.import_module('pkg_resources') - if 'pygments.plugin' in sys.modules: - sys.modules['pygments.plugin'].pkg_resources = pr - - -def load_pkg_resources_in_background(): - """Entry point for loading pkg_resources module in background.""" - if 'pkg_resources' in sys.modules: - return - env = getattr(builtins, '__xonsh_env__', os.environ) - if env.get('XONSH_DEBUG', None): - import pkg_resources - return - sys.modules['pkg_resources'] = PkgResourcesProxy() - PkgResourcesLoader() diff --git a/xonsh/lazyasd.py b/xonsh/lazyasd.py index 3af53cc92..bad114578 100644 --- a/xonsh/lazyasd.py +++ b/xonsh/lazyasd.py @@ -1,4 +1,12 @@ """Lazy and self destrctive containers for speeding up module import.""" +import os +import sys +import time +import types +import builtins +import threading +import importlib +import importlib.util import collections.abc as abc class LazyObject(object): @@ -166,3 +174,106 @@ class LazyBool(object): else: res = self._result return res + +# +# Background module loaders +# + +class BackgroundModuleProxy(types.ModuleType): + """Proxy object for modules loaded in the background that block attribute + access until the module is loaded.. + """ + + def __init__(self, modname): + self.__dct__ = { + 'loaded': False, + 'modname': modname, + } + + def __getattribute__(self, name): + passthrough = frozenset({'__dct__','__class__', '__spec__'}) + if name in passthrough: + return super().__getattribute__(name) + dct = self.__dct__ + modname = dct['modname'] + if dct['loaded']: + mod = sys.modules[modname] + else: + delay_types = (BackgroundModuleProxy, type(None)) + while isinstance(sys.modules.get(modname, None), delay_types): + time.sleep(0.001) + mod = sys.modules[modname] + dct['loaded'] = True + return getattr(mod, name) + + +class BackgroundModuleLoader(threading.Thread): + """Thread to load modules in the background.""" + + def __init__(self, name, package, replacements, *args, **kwargs): + super().__init__(*args, **kwargs) + self.daemon = True + self.name = name + self.package = package + self.replacements = replacements + self.start() + + def run(self): + # wait for other modules to stop being imported + i = 0 + last = -6 + hist = [-5, -4, -3, -2, -1] + while not all(last == x for x in hist): + time.sleep(0.001) + last = hist[i%5] = len(sys.modules) + i += 1 + # now import pkg_resources properly + modname = importlib.util.resolve_name(self.name, self.package) + if isinstance(sys.modules[modname], BackgroundModuleProxy): + del sys.modules[modname] + mod = importlib.import_module(self.name, package=self.package) + for targname, varname in self.replacements.items(): + if targname in sys.modules: + targmod = sys.modules[targname] + setattr(targmod, varname, mod) + + +def load_module_in_background(name, package=None, debug='DEBUG', env=None, + replacements=None): + """Entry point for loading modules in background thread. + + Parameters + ---------- + name : str + Module name to load in background thread. + package : str or None, optional + Package name, has the same meaning as in importlib.import_module(). + debug : str, optional + Debugging symbol name to look up in the environment. + env : Mapping or None, optional + Environment this will default to __xonsh_env__, if available, and + os.environ otherwise. + replacements : Mapping or None, optional + Dictionary mapping fully qualified module names (eg foo.bar.baz) that + import the lazily loaded moudle, with the variable name in that + module. For example, suppose that foo.bar imports module a as b, + this dict is then {'foo.bar': 'b'}. + + Returns + ------- + module : ModuleType + This is either the original module that is found in sys.modules or + a proxy module that will block until delay attribute access until the + module is fully loaded. + """ + modname = importlib.util.resolve_name(name, package) + if modname in sys.modules: + return sys.modules[modname] + if env is None: + env = getattr(builtins, '__xonsh_env__', os.environ) + if env.get(debug, None): + mod = importlib.import_module(name, package=package) + return mod + proxy = sys.modules[modname] = BackgroundModuleProxy(modname) + BackgroundModuleLoader(name, package, replacements or {}) + return proxy diff --git a/xonsh/ptk/__init__.py b/xonsh/ptk/__init__.py index 112c3b322..4bde5426d 100644 --- a/xonsh/ptk/__init__.py +++ b/xonsh/ptk/__init__.py @@ -1,3 +1,4 @@ # must come before ptk / pygments imports -from xonsh.bg_pkg_resources import load_pkg_resources_in_background -load_pkg_resources_in_background() +from xonsh.lazyasd import load_module_in_background +load_module_in_background('pkg_resources', debug='XONSH_DEBUG', + replacements={'pygments.plugin': 'pkg_resources'}) diff --git a/xonsh/pyghooks.py b/xonsh/pyghooks.py index e05976ad0..c5513a3c2 100644 --- a/xonsh/pyghooks.py +++ b/xonsh/pyghooks.py @@ -10,8 +10,9 @@ from collections import ChainMap from collections.abc import MutableMapping # must come before pygments imports -from xonsh.bg_pkg_resources import load_pkg_resources_in_background -load_pkg_resources_in_background() +from xonsh.lazyasd import load_module_in_background +load_module_in_background('pkg_resources', debug='XONSH_DEBUG', + replacements={'pygments.plugin': 'pkg_resources'}) from pygments.lexer import inherit, bygroups, using, this from pygments.lexers.shell import BashLexer