Merge pull request #2187 from xonsh/lr

Xonsh startup loading refactor
This commit is contained in:
Morten Enemark Lund 2017-02-15 04:54:53 +01:00 committed by GitHub
commit 79a7fc49fa
13 changed files with 201 additions and 131 deletions

1
.gitignore vendored
View file

@ -13,6 +13,7 @@ tests/lexer_table.py
tests/parser_table.py
tests/lexer_test_table.py
tests/parser_test_table.py
tests/testfile
build/
dist/
xonsh.egg-info/

29
news/lr.rst Normal file
View file

@ -0,0 +1,29 @@
**Added:**
* New ``--rc`` command line option allows users to specify paths to run control
files from the command line. This includes both xonsh-based and JSON-based
configuration.
**Changed:**
* ``$XONSHRC`` and related configuration variables now accept JSON-based
static configuration file names as elements. This unifies the two methods
of run control to a single entry point and loading system.
* The ``xonsh.shell.Shell()`` class now requires that an Execer instance
be explicitly provided to its init method. This class is no longer
responsible for creating an execer an its deprendencies.
**Deprecated:**
* The ``--config-path`` command line option is now deprecated in favor of
``--rc``.
**Removed:**
* ``xonsh.environ.DEFAULT_XONSHRC`` has been removed due to deprecation.
For this value, please check the environment instead, or call
``xonsh.environ.default_xonshrc(env)``.
**Fixed:** None
**Security:** None

View file

@ -4,8 +4,6 @@ import os
import pytest
import xonsh.built_ins
from xonsh.built_ins import ensure_list_of_strs, enter_macro
from xonsh.execer import Execer
from xonsh.jobs import tasks
@ -25,9 +23,9 @@ def source_path():
@pytest.fixture
def xonsh_execer(monkeypatch):
"""Initiate the Execer with a mocked nop `load_builtins`"""
monkeypatch.setattr(xonsh.built_ins, 'load_builtins',
lambda *args, **kwargs: None)
execer = Execer(login=False, unload=False)
monkeypatch.setattr('xonsh.built_ins.load_builtins.__code__',
(lambda *args, **kwargs: None).__code__)
execer = Execer(unload=False)
builtins.__xonsh_execer__ = execer
return execer
@ -74,23 +72,34 @@ def xonsh_builtins(xonsh_events):
# be firing events on the global instance.
builtins.events = xonsh_events
yield builtins
del builtins.__xonsh_env__
del builtins.__xonsh_ctx__
if hasattr(builtins, '__xonsh_env__'):
del builtins.__xonsh_env__
if hasattr(builtins, '__xonsh_ctx__'):
del builtins.__xonsh_ctx__
del builtins.__xonsh_shell__
del builtins.__xonsh_help__
del builtins.__xonsh_glob__
del builtins.__xonsh_exit__
del builtins.__xonsh_superhelp__
if hasattr(builtins, '__xonsh_help__'):
del builtins.__xonsh_help__
if hasattr(builtins, '__xonsh_glob__'):
del builtins.__xonsh_glob__
if hasattr(builtins, '__xonsh_exit__'):
del builtins.__xonsh_exit__
if hasattr(builtins, '__xonsh_superhelp__'):
del builtins.__xonsh_superhelp__
del builtins.__xonsh_regexpath__
del builtins.__xonsh_expand_path__
del builtins.__xonsh_stdout_uncaptured__
del builtins.__xonsh_stderr_uncaptured__
if hasattr(builtins, '__xonsh_expand_path__'):
del builtins.__xonsh_expand_path__
if hasattr(builtins, '__xonsh_stdout_uncaptured__'):
del builtins.__xonsh_stdout_uncaptured__
if hasattr(builtins, '__xonsh_stderr_uncaptured__'):
del builtins.__xonsh_stderr_uncaptured__
del builtins.__xonsh_subproc_captured__
del builtins.__xonsh_subproc_uncaptured__
if hasattr(builtins, '__xonsh_subproc_uncaptured__'):
del builtins.__xonsh_subproc_uncaptured__
del builtins.__xonsh_ensure_list_of_strs__
del builtins.__xonsh_commands_cache__
del builtins.__xonsh_all_jobs__
del builtins.__xonsh_history__
if hasattr(builtins, '__xonsh_history__'):
del builtins.__xonsh_history__
del builtins.__xonsh_enter_macro__
del builtins.evalx
del builtins.execx

View file

@ -6,16 +6,16 @@ import builtins
import pytest
from xonsh import imphooks
from xonsh.execer import Execer
from xonsh.environ import Env
from xonsh.built_ins import load_builtins, unload_builtins
from xonsh.built_ins import unload_builtins
imphooks.install_hook()
@pytest.yield_fixture(autouse=True)
def imp_env(xonsh_execer):
"""Call `load_builtins` with `xonsh_execer`"""
load_builtins(execer=xonsh_execer)
def imp_env():
execer = Execer(unload=False)
builtins.__xonsh_env__ = Env({'PATH': [], 'PATHEXT': []})
yield
unload_builtins()

View file

@ -17,7 +17,7 @@ def Shell(*args, **kwargs):
@pytest.fixture
def shell(xonsh_builtins, monkeypatch):
def shell(xonsh_builtins, xonsh_execer, monkeypatch):
"""Xonsh Shell Mock"""
monkeypatch.setattr(xonsh.main, 'Shell', Shell)
@ -62,7 +62,7 @@ def test_premain_interactive__with_file_argument(shell):
@pytest.mark.parametrize('case', ['----', '--hep', '-TT', '--TTTT'])
def test_premain_invalid_arguments(case, shell, capsys):
def test_premain_invalid_arguments(shell, case, capsys):
with pytest.raises(SystemExit):
xonsh.main.premain([case])
assert 'unrecognized argument' in capsys.readouterr()[1]

View file

@ -7,6 +7,7 @@ import builtins
import pytest
from xonsh.shell import Shell
from xonsh.execer import Execer
from xonsh.replay import Replayer
from tools import skip_if_on_darwin
@ -18,7 +19,9 @@ HISTDIR = os.path.join(os.path.dirname(__file__), 'histories')
@pytest.yield_fixture(scope='module', autouse=True)
def ctx():
"""Create a global Shell instance to use in all the test."""
builtins.__xonsh_shell__ = Shell({'PATH': []})
ctx = {'PATH': []}
execer = Execer(xonsh_ctx=ctx)
builtins.__xonsh_shell__ = Shell(execer=execer, ctx=ctx)
yield
del builtins.__xonsh_shell__

View file

@ -23,7 +23,7 @@ VER_MAJOR_MINOR = sys.version_info[:2]
VER_FULL = sys.version_info[:3]
ON_DARWIN = (platform.system() == 'Darwin')
ON_WINDOWS = (platform.system() == 'Windows')
ON_CONDA = True in [conda in pytest.__file__ for conda
ON_CONDA = True in [conda in pytest.__file__.lower() for conda
in ['anaconda', 'miniconda']]
ON_TRAVIS = 'TRAVIS' in os.environ and 'CI' in os.environ
TEST_DIR = os.path.dirname(__file__)

View file

@ -60,14 +60,20 @@ else:
_sys.modules['xonsh.dirstack'] = __amalgam__
inspectors = __amalgam__
_sys.modules['xonsh.inspectors'] = __amalgam__
shell = __amalgam__
_sys.modules['xonsh.shell'] = __amalgam__
timings = __amalgam__
_sys.modules['xonsh.timings'] = __amalgam__
xonfig = __amalgam__
_sys.modules['xonsh.xonfig'] = __amalgam__
base_shell = __amalgam__
_sys.modules['xonsh.base_shell'] = __amalgam__
environ = __amalgam__
_sys.modules['xonsh.environ'] = __amalgam__
tracer = __amalgam__
_sys.modules['xonsh.tracer'] = __amalgam__
readline_shell = __amalgam__
_sys.modules['xonsh.readline_shell'] = __amalgam__
replay = __amalgam__
_sys.modules['xonsh.replay'] = __amalgam__
aliases = __amalgam__
@ -78,14 +84,8 @@ else:
_sys.modules['xonsh.execer'] = __amalgam__
imphooks = __amalgam__
_sys.modules['xonsh.imphooks'] = __amalgam__
shell = __amalgam__
_sys.modules['xonsh.shell'] = __amalgam__
base_shell = __amalgam__
_sys.modules['xonsh.base_shell'] = __amalgam__
main = __amalgam__
_sys.modules['xonsh.main'] = __amalgam__
readline_shell = __amalgam__
_sys.modules['xonsh.readline_shell'] = __amalgam__
del __amalgam__
except ImportError:
pass

View file

@ -26,7 +26,6 @@ from xonsh.lazyasd import LazyObject, lazyobject
from xonsh.inspectors import Inspector
from xonsh.aliases import Aliases, make_default_aliases
from xonsh.environ import Env, default_env, locate_binary
from xonsh.foreign_shells import load_foreign_aliases
from xonsh.jobs import add_job
from xonsh.platform import ON_POSIX, ON_WINDOWS
from xonsh.proc import (
@ -1124,14 +1123,14 @@ def enter_macro(obj, raw_block, glbs, locs):
return obj
def load_builtins(execer=None, config=None, login=False, ctx=None):
def load_builtins(execer=None, ctx=None):
"""Loads the xonsh builtins into the Python builtins. Sets the
BUILTINS_LOADED variable to True.
"""
global BUILTINS_LOADED
# private built-ins
builtins.__xonsh_config__ = {}
builtins.__xonsh_env__ = Env(default_env(config=config, login=login))
builtins.__xonsh_env__ = Env(default_env())
builtins.__xonsh_help__ = helper
builtins.__xonsh_superhelp__ = superhelper
builtins.__xonsh_pathsearch__ = pathsearch
@ -1174,8 +1173,6 @@ def load_builtins(execer=None, config=None, login=False, ctx=None):
# Need this inline/lazy import here since we use locate_binary that
# relies on __xonsh_env__ in default aliases
builtins.default_aliases = builtins.aliases = Aliases(make_default_aliases())
if login:
builtins.aliases.update(load_foreign_aliases(issue_warning=False))
builtins.__xonsh_history__ = None
atexit.register(_lastflush)
for sig in AT_EXIT_SIGNALS:

View file

@ -9,7 +9,6 @@ import textwrap
import locale
import builtins
import warnings
import traceback
import contextlib
import collections
import collections.abc as cabc
@ -19,7 +18,8 @@ from xonsh.lazyasd import LazyObject, lazyobject
from xonsh.codecache import run_script_with_cache
from xonsh.dirstack import _get_cwd
from xonsh.events import events
from xonsh.foreign_shells import load_foreign_envs
from xonsh.foreign_shells import load_foreign_envs, load_foreign_aliases
from xonsh.xontribs import update_context, prompt_xontrib_install
from xonsh.platform import (
BASH_COMPLETIONS_DEFAULT, DEFAULT_ENCODING, PATH_DEFAULT,
ON_WINDOWS, ON_LINUX
@ -39,6 +39,7 @@ from xonsh.tools import (
is_logfile_opt, to_logfile_opt, logfile_opt_to_str, executables_in,
is_nonstring_seq_of_strings, pathsep_to_upper_seq,
seq_to_upper_pathsep, print_color, is_history_backend, to_itself,
swap_values,
)
import xonsh.prompt.base as prompt
@ -234,20 +235,19 @@ def xonshconfig(env):
return xc
def default_xonshrc():
@default_value
def default_xonshrc(env):
"""Creates a new instance of the default xonshrc tuple."""
if ON_WINDOWS:
dxrc = (os.path.join(os.environ['ALLUSERSPROFILE'],
dxrc = (xonshconfig(env),
os.path.join(os.environ['ALLUSERSPROFILE'],
'xonsh', 'xonshrc'),
os.path.expanduser('~/.xonshrc'))
else:
dxrc = ('/etc/xonshrc', os.path.expanduser('~/.xonshrc'))
dxrc = (xonshconfig(env), '/etc/xonshrc', os.path.expanduser('~/.xonshrc'))
return dxrc
DEFAULT_XONSHRC = LazyObject(default_xonshrc, globals(), 'DEFAULT_XONSHRC')
# Default values should generally be immutable, that way if a user wants
# to set them they have to do a copy and write them to the environment.
# try to keep this sorted.
@ -317,7 +317,7 @@ def DEFAULT_VALUES():
'XDG_DATA_HOME': os.path.expanduser(os.path.join('~', '.local',
'share')),
'XONSHCONFIG': xonshconfig,
'XONSHRC': default_xonshrc(),
'XONSHRC': default_xonshrc,
'XONSH_AUTOPAIR': False,
'XONSH_CACHE_SCRIPTS': True,
'XONSH_CACHE_EVERYTHING': False,
@ -1077,36 +1077,26 @@ def load_static_config(ctx, config=None):
return conf
def xonshrc_context(rcfiles=None, execer=None, initial=None):
"""Attempts to read in xonshrc file, and return the contents."""
loaded = builtins.__xonsh_env__['LOADED_RC_FILES'] = []
if initial is None:
env = {}
else:
env = initial
if rcfiles is None or execer is None:
def xonshrc_context(rcfiles=None, execer=None, ctx=None, env=None, login=True):
"""Attempts to read in all xonshrc files and return the context."""
loaded = env['LOADED_RC_FILES'] = []
ctx = {} if ctx is None else ctx
if rcfiles is None:
return env
env['XONSHRC'] = tuple(rcfiles)
for rcfile in rcfiles:
if not os.path.isfile(rcfile):
loaded.append(False)
continue
try:
run_script_with_cache(rcfile, execer, env)
loaded.append(True)
except SyntaxError as err:
loaded.append(False)
exc = traceback.format_exc()
msg = '{0}\nsyntax error in xonsh run control file {1!r}: {2!s}'
warnings.warn(msg.format(exc, rcfile, err), RuntimeWarning)
continue
except Exception as err:
loaded.append(False)
exc = traceback.format_exc()
msg = '{0}\nerror running xonsh run control file {1!r}: {2!s}'
warnings.warn(msg.format(exc, rcfile, err), RuntimeWarning)
continue
return env
_, ext = os.path.splitext(rcfile)
if ext == '.json':
status = static_config_run_control(rcfile, ctx, env, execer=execer,
login=login)
else:
status = xonsh_script_run_control(rcfile, ctx, env, execer=execer,
login=login)
loaded.append(status)
return ctx
def windows_foreign_env_fixes(ctx):
@ -1132,7 +1122,54 @@ def foreign_env_fixes(ctx):
del ctx['PROMPT']
def default_env(env=None, config=None, login=True):
def static_config_run_control(filename, ctx, env, execer=None, login=True):
"""Loads a static config file and applies it as a run control."""
if not login:
return
conf = load_static_config(env, config=filename)
# load foreign shells
foreign_env = load_foreign_envs(shells=conf.get('foreign_shells', ()),
issue_warning=False)
if ON_WINDOWS:
windows_foreign_env_fixes(foreign_env)
foreign_env_fixes(foreign_env)
env.update(foreign_env)
foreign_aliases = load_foreign_aliases(config=filename, issue_warning=True)
builtins.aliases.update(foreign_aliases)
# load xontribs
names = conf.get('xontribs', ())
for name in names:
update_context(name, ctx=ctx)
if getattr(update_context, 'bad_imports', None):
prompt_xontrib_install(update_context.bad_imports)
del update_context.bad_imports
# Do static config environment last, to allow user to override any of
# our environment choices
env.update(conf.get('env', ()))
return True
def xonsh_script_run_control(filename, ctx, env, execer=None, login=True):
"""Loads a xonsh file and applies it as a run control."""
if execer is None:
return False
updates = {'__file__': filename, '__name__': os.path.abspath(filename)}
try:
with swap_values(ctx, updates):
run_script_with_cache(filename, execer, ctx)
loaded = True
except SyntaxError as err:
msg = 'syntax error in xonsh run control file {0!r}: {1!s}'
print_exception(msg.format(filename, err))
loaded = False
except Exception as err:
msg = 'error running xonsh run control file {0!r}: {1!s}'
print_exception(msg.format(filename, err))
loaded = False
return loaded
def default_env(env=None):
"""Constructs a default xonsh environment."""
# in order of increasing precedence
ctx = dict(BASE_ENV)
@ -1143,17 +1180,6 @@ def default_env(env=None, config=None, login=True):
del ctx['PROMPT']
except KeyError:
pass
if login:
conf = load_static_config(ctx, config=config)
foreign_env = load_foreign_envs(shells=conf.get('foreign_shells', ()),
issue_warning=False)
if ON_WINDOWS:
windows_foreign_env_fixes(foreign_env)
foreign_env_fixes(foreign_env)
ctx.update(foreign_env)
# Do static config environment last, to allow user to override any of
# our environment choices
ctx.update(conf.get('env', ()))
# finalize env
if env is not None:
ctx.update(env)

View file

@ -16,7 +16,7 @@ class Execer(object):
"""Executes xonsh code in a context."""
def __init__(self, filename='<xonsh-code>', debug_level=0, parser_args=None,
unload=True, config=None, login=True, xonsh_ctx=None):
unload=True, xonsh_ctx=None, scriptcache=True, cacheall=False):
"""Parameters
----------
filename : str, optional
@ -27,18 +27,24 @@ class Execer(object):
Arguments to pass down to the parser.
unload : bool, optional
Whether or not to unload xonsh builtins upon deletion.
config : str, optional
Path to configuration file.
xonsh_ctx : dict or None, optional
Xonsh xontext to load as builtins.__xonsh_ctx__
scriptcache : bool, optional
Whether or not to use a precompiled bytecode cache when execing
code, default: True.
cacheall : bool, optional
Whether or not to cache all xonsh code, and not just files. If this
is set to true, it will cache command line input too, default: False.
"""
parser_args = parser_args or {}
self.parser = Parser(**parser_args)
self.filename = filename
self.debug_level = debug_level
self.unload = unload
self.scriptcache = scriptcache
self.cacheall = cacheall
self.ctxtransformer = CtxAwareTransformer(self.parser)
load_builtins(execer=self, config=config, login=login, ctx=xonsh_ctx)
load_builtins(execer=self, ctx=xonsh_ctx)
def __del__(self):
if self.unload:

View file

@ -13,15 +13,17 @@ from xonsh.timings import setup_timings
from xonsh.lazyasd import lazyobject
from xonsh.shell import Shell
from xonsh.pretty import pretty
from xonsh.execer import Execer
from xonsh.proc import HiddenCommandPipeline
from xonsh.jobs import ignore_sigtstp
from xonsh.tools import setup_win_unicode_console, print_color
from xonsh.tools import setup_win_unicode_console, print_color, to_bool_or_int
from xonsh.platform import HAS_PYGMENTS, ON_WINDOWS
from xonsh.codecache import run_script_with_cache, run_code_with_cache
from xonsh.xonfig import xonfig_main
from xonsh.lazyimps import pygments, pyghooks
from xonsh.imphooks import install_hook
from xonsh.events import events
from xonsh.environ import xonshrc_context
events.transmogrify('on_post_init', 'LoadEvent')
@ -125,10 +127,18 @@ def parser():
action='store_true',
default=False)
p.add_argument('--config-path',
help='specify a custom static configuration file',
help='DEPRECATED: static configuration files may now be used '
'in the XONSHRC file list, see the --rc option.',
dest='config_path',
default=None,
type=path_argument)
p.add_argument('--rc',
help="The xonshrc files to load, these may be either xonsh "
"files or JSON-based static configuration files.",
dest='rc',
nargs='+',
type=path_argument,
default=None)
p.add_argument('--no-rc',
help="Do not load the .xonshrc files",
dest='norc',
@ -207,6 +217,33 @@ class XonshMode(enum.Enum):
interactive = 3
def start_services(shell_kwargs):
"""Starts up the essential services in the proper order.
This returns the envrionment instance as a convenience.
"""
install_hook()
# create execer, which loads builtins
ctx = shell_kwargs.get('ctx', {})
debug = to_bool_or_int(os.getenv('XONSH_DEBUG', '0'))
events.on_timingprobe.fire(name='pre_execer_init')
execer = Execer(xonsh_ctx=ctx, debug_level=debug,
scriptcache=shell_kwargs.get('scriptcache', True),
cacheall=shell_kwargs.get('cacheall', False))
events.on_timingprobe.fire(name='post_execer_init')
# load rc files
login = shell_kwargs.get('login', True)
env = builtins.__xonsh_env__
rc = shell_kwargs.get('rc', None)
rc = env.get('XONSHRC') if rc is None else rc
events.on_pre_rc.fire()
xonshrc_context(rcfiles=rc, execer=execer, ctx=ctx, env=env, login=login)
events.on_post_rc.fire()
# create shell
builtins.__xonsh_shell__ = Shell(execer=execer, **shell_kwargs)
ctx['__name__'] = '__main__'
return env
def premain(argv=None):
"""Setup for main xonsh entry point, returns parsed arguments."""
if argv is None:
@ -232,8 +269,6 @@ def premain(argv=None):
'ctx': builtins.__xonsh_ctx__}
if args.login:
shell_kwargs['login'] = True
if args.config_path is not None:
shell_kwargs['config'] = args.config_path
if args.norc:
shell_kwargs['rc'] = ()
setattr(sys, 'displayhook', _pprint_displayhook)
@ -250,9 +285,7 @@ def premain(argv=None):
args.mode = XonshMode.interactive
shell_kwargs['completer'] = True
shell_kwargs['login'] = True
install_hook()
builtins.__xonsh_shell__ = Shell(**shell_kwargs)
env = builtins.__xonsh_env__
env = start_services(shell_kwargs)
env['XONSH_LOGIN'] = shell_kwargs['login']
if args.defines is not None:
env.update([x.split('=', 1) for x in args.defines])

View file

@ -1,6 +1,5 @@
# -*- coding: utf-8 -*-
"""The xonsh shell"""
import os
import sys
import random
import time
@ -8,12 +7,9 @@ import difflib
import builtins
import warnings
from xonsh.xontribs import update_context, prompt_xontrib_install
from xonsh.environ import xonshrc_context
from xonsh.execer import Execer
from xonsh.platform import (best_shell_type, has_prompt_toolkit,
ptk_version_is_supported)
from xonsh.tools import XonshError, to_bool_or_int, print_exception
from xonsh.tools import XonshError, print_exception
from xonsh.events import events
import xonsh.history.main as xhm
@ -90,11 +86,12 @@ class Shell(object):
readline version of shell should be used.
"""
def __init__(self, ctx=None, shell_type=None, config=None, rc=None,
**kwargs):
def __init__(self, execer, ctx=None, shell_type=None, **kwargs):
"""
Parameters
----------
execer : Execer
An execer instance capable of running xonsh code.
ctx : Mapping, optional
The execution context for the shell (e.g. the globals namespace).
If none, this is computed by loading the rc files. If not None,
@ -103,16 +100,10 @@ class Shell(object):
shell_type : str, optional
The shell type to start, such as 'readline', 'prompt_toolkit',
or 'random'.
config : str, optional
Path to configuration file.
rc : list of str, optional
Sequence of paths to run control files.
"""
self.login = kwargs.get('login', True)
self.execer = execer
self.ctx = {} if ctx is None else ctx
self.stype = shell_type
self._init_environ(ctx, config, rc,
kwargs.get('scriptcache', True),
kwargs.get('cacheall', False))
env = builtins.__xonsh_env__
# build history backend before creating shell
builtins.__xonsh_history__ = hist = xhm.construct_history(
@ -160,28 +151,3 @@ class Shell(object):
def __getattr__(self, attr):
"""Delegates calls to appropriate shell instance."""
return getattr(self.shell, attr)
def _init_environ(self, ctx, config, rc, scriptcache, cacheall):
self.ctx = {} if ctx is None else ctx
debug = to_bool_or_int(os.getenv('XONSH_DEBUG', '0'))
events.on_timingprobe.fire(name='pre_execer_init')
self.execer = Execer(config=config, login=self.login, xonsh_ctx=self.ctx,
debug_level=debug)
events.on_timingprobe.fire(name='post_execer_init')
self.execer.scriptcache = scriptcache
self.execer.cacheall = cacheall
if self.stype != 'none' or self.login:
# load xontribs from config file
names = builtins.__xonsh_config__.get('xontribs', ())
for name in names:
update_context(name, ctx=self.ctx)
if getattr(update_context, 'bad_imports', None):
prompt_xontrib_install(update_context.bad_imports)
del update_context.bad_imports
# load run control files
env = builtins.__xonsh_env__
rc = env.get('XONSHRC') if rc is None else rc
events.on_pre_rc.fire()
self.ctx.update(xonshrc_context(rcfiles=rc, execer=self.execer, initial=self.ctx))
events.on_post_rc.fire()
self.ctx['__name__'] = '__main__'