Merge pull request #1359 from xonsh/bgmod

Background thread module loading

news/pkgrsrc.rst

@@ -1,7 +1,7 @@
 **Added:**
 
-* New ``xonsh.bg_pkg_resources`` module for loading the ``pkg_resources``
+* New tools in ``xonsh.lazyasd`` module for loading modules in background
-  module in a background thread.
+  threads.
 
 **Changed:**
 
@@ -9,6 +9,8 @@
   background.
 * Sped up loading of prompt-toolkit by ~2x-3x by loading ``pkg_resources``
   in background.
+* ``setup.py`` will no longer git checkout to replace the version number.
+  Now it simply stores and reuses the original version line.
 
 **Deprecated:** None
 

setup.py

@@ -114,40 +114,41 @@ def dirty_version():
         _version = _version.decode('ascii')
     except (subprocess.CalledProcessError, FileNotFoundError):
         return False
-
     try:
         base, N, sha = _version.strip().split('-')
     except ValueError: #on base release
         open('xonsh/dev.githash', 'w').close()
         return False
-
     replace_version(base, N)
     with open('xonsh/dev.githash', 'w') as f:
         f.write(sha)
-
     return True
 
 
+ORIGINAL_VERSION_LINE = None
+
 def replace_version(base, N):
     """Replace version in `__init__.py` with devN suffix"""
+    global ORIGINAL_VERSION_LINE
     with open('xonsh/__init__.py', 'r') as f:
         raw = f.read()
     lines = raw.splitlines()
+    ORIGINAL_VERSION_LINE = lines[0]
     lines[0] = "__version__ = '{}.dev{}'".format(base, N)
     upd = '\n'.join(lines) + '\n'
     with open('xonsh/__init__.py', 'w') as f:
         f.write(upd)
 
 
-def discard_changes():
+def restore_version():
-    """If we touch ``__init__.py``, discard changes after install"""
+    """If we touch the version in __init__.py discard changes after install."""
-    try:
+    with open('xonsh/__init__.py', 'r') as f:
-        _ = subprocess.check_output(['git',
+        raw = f.read()
-                                     'checkout',
+    lines = raw.splitlines()
-                                     '--',
+    lines[0] = ORIGINAL_VERSION_LINE
-                                     'xonsh/__init__.py'])
+    upd = '\n'.join(lines) + '\n'
-    except subprocess.CalledProcessError:
+    with open('xonsh/__init__.py', 'w') as f:
-        pass
+        f.write(upd)
 
 
 class xinstall(install):
@@ -168,7 +169,7 @@ class xinstall(install):
             print('Installing Jupyter hook failed.')
         install.run(self)
         if dirty:
-            discard_changes()
+            restore_version()
 
 
 
@@ -180,7 +181,7 @@ class xsdist(sdist):
         dirty = dirty_version()
         sdist.make_release_tree(self, basedir, files)
         if dirty:
-            discard_changes()
+            restore_version()
 
 
 #-----------------------------------------------------------------------------
@@ -215,7 +216,7 @@ if HAVE_SETUPTOOLS:
             dirty = dirty_version()
             develop.run(self)
             if dirty:
-                discard_changes()
+                restore_version()
 
 
 def main():

xonsh/__init__.py

@@ -9,8 +9,6 @@ else:
     import sys as _sys
     try:
         from xonsh import __amalgam__
-        bg_pkg_resources = __amalgam__
-        _sys.modules['xonsh.bg_pkg_resources'] = __amalgam__
         completer = __amalgam__
         _sys.modules['xonsh.completer'] = __amalgam__
         lazyasd = __amalgam__

xonsh/bg_pkg_resources.py

@@ -1,55 +0,0 @@
-"""Background thread loader for pkg_resources."""
-import os
-import sys
-import time
-import builtins
-import threading
-import importlib
-
-
-class PkgResourcesProxy(object):
-    """Proxy object for pkg_resources module that throws an ImportError
-    whenever an attribut is accessed.
-    """
-
-    def __getattr__(self, name):
-        raise ImportError('cannot access ' + name + 'on PkgResourcesProxy, '
-                          'please wait for pkg_resources module to be fully '
-                          'loaded.')
-
-
-class PkgResourcesLoader(threading.Thread):
-    """Thread to load the pkg_resources module."""
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.daemon = True
-        self.start()
-
-    def run(self):
-        # wait for other modules to stop being imported
-        i = 0
-        last = -6
-        hist = [-5, -4, -3, -2, -1]
-        while not all(last == x for x in hist):
-            time.sleep(0.001)
-            last = hist[i%5] = len(sys.modules)
-            i += 1
-        # now import pkg_resources properly
-        if isinstance(sys.modules['pkg_resources'], PkgResourcesProxy):
-            del sys.modules['pkg_resources']
-        pr = importlib.import_module('pkg_resources')
-        if 'pygments.plugin' in sys.modules:
-            sys.modules['pygments.plugin'].pkg_resources = pr
-
-
-def load_pkg_resources_in_background():
-    """Entry point for loading pkg_resources module in background."""
-    if 'pkg_resources' in sys.modules:
-        return
-    env = getattr(builtins, '__xonsh_env__', os.environ)
-    if env.get('XONSH_DEBUG', None):
-        import pkg_resources
-        return
-    sys.modules['pkg_resources'] = PkgResourcesProxy()
-    PkgResourcesLoader()

xonsh/lazyasd.py

@@ -1,4 +1,12 @@
 """Lazy and self destrctive containers for speeding up module import."""
+import os
+import sys
+import time
+import types
+import builtins
+import threading
+import importlib
+import importlib.util
 import collections.abc as abc
 
 class LazyObject(object):
@@ -166,3 +174,106 @@ class LazyBool(object):
         else:
             res = self._result
         return res
+
+#
+# Background module loaders
+#
+
+class BackgroundModuleProxy(types.ModuleType):
+    """Proxy object for modules loaded in the background that block attribute
+    access until the module is loaded..
+    """
+
+    def __init__(self, modname):
+        self.__dct__ = {
+            'loaded': False,
+            'modname': modname,
+            }
+
+    def __getattribute__(self, name):
+        passthrough = frozenset({'__dct__','__class__', '__spec__'})
+        if name in passthrough:
+            return super().__getattribute__(name)
+        dct = self.__dct__
+        modname = dct['modname']
+        if dct['loaded']:
+            mod = sys.modules[modname]
+        else:
+            delay_types = (BackgroundModuleProxy, type(None))
+            while isinstance(sys.modules.get(modname, None), delay_types):
+                time.sleep(0.001)
+            mod = sys.modules[modname]
+            dct['loaded'] = True
+        return getattr(mod, name)
+
+
+class BackgroundModuleLoader(threading.Thread):
+    """Thread to load modules in the background."""
+
+    def __init__(self, name, package, replacements, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.daemon = True
+        self.name = name
+        self.package = package
+        self.replacements = replacements
+        self.start()
+
+    def run(self):
+        # wait for other modules to stop being imported
+        i = 0
+        last = -6
+        hist = [-5, -4, -3, -2, -1]
+        while not all(last == x for x in hist):
+            time.sleep(0.001)
+            last = hist[i%5] = len(sys.modules)
+            i += 1
+        # now import pkg_resources properly
+        modname = importlib.util.resolve_name(self.name, self.package)
+        if isinstance(sys.modules[modname], BackgroundModuleProxy):
+            del sys.modules[modname]
+        mod = importlib.import_module(self.name, package=self.package)
+        for targname, varname in self.replacements.items():
+            if targname in sys.modules:
+                targmod = sys.modules[targname]
+                setattr(targmod, varname, mod)
+
+
+def load_module_in_background(name, package=None, debug='DEBUG', env=None,
+                              replacements=None):
+    """Entry point for loading modules in background thread.
+
+    Parameters
+    ----------
+    name : str
+        Module name to load in background thread.
+    package : str or None, optional
+        Package name, has the same meaning as in importlib.import_module().
+    debug : str, optional
+        Debugging symbol name to look up in the environment.
+    env : Mapping or None, optional
+        Environment this will default to __xonsh_env__, if available, and
+        os.environ otherwise.
+    replacements : Mapping or None, optional
+        Dictionary mapping fully qualified module names (eg foo.bar.baz) that
+        import the lazily loaded moudle, with the variable name in that
+        module. For example, suppose that foo.bar imports module a as b,
+        this dict is then {'foo.bar': 'b'}.
+
+    Returns
+    -------
+    module : ModuleType
+        This is either the original module that is found in sys.modules or
+        a proxy module that will block until delay attribute access until the
+        module is fully loaded.
+    """
+    modname = importlib.util.resolve_name(name, package)
+    if modname in sys.modules:
+        return sys.modules[modname]
+    if env is None:
+        env = getattr(builtins, '__xonsh_env__', os.environ)
+    if env.get(debug, None):
+        mod = importlib.import_module(name, package=package)
+        return mod
+    proxy = sys.modules[modname] = BackgroundModuleProxy(modname)
+    BackgroundModuleLoader(name, package, replacements or {})
+    return proxy

xonsh/ptk/__init__.py

@@ -1,3 +1,4 @@
 # must come before ptk / pygments imports
-from xonsh.bg_pkg_resources import load_pkg_resources_in_background
+from xonsh.lazyasd import load_module_in_background
-load_pkg_resources_in_background()
+load_module_in_background('pkg_resources', debug='XONSH_DEBUG',
+    replacements={'pygments.plugin': 'pkg_resources'})

xonsh/pyghooks.py

@@ -10,8 +10,9 @@ from collections import ChainMap
 from collections.abc import MutableMapping
 
 # must come before pygments imports
-from xonsh.bg_pkg_resources import load_pkg_resources_in_background
+from xonsh.lazyasd import load_module_in_background
-load_pkg_resources_in_background()
+load_module_in_background('pkg_resources', debug='XONSH_DEBUG',
+    replacements={'pygments.plugin': 'pkg_resources'})
 
 from pygments.lexer import inherit, bygroups, using, this
 from pygments.lexers.shell import BashLexer