Merge master and update setup syntax in execer test

.appveyor.yml

@@ -1,12 +1,9 @@
-version: 0.2.7.{build}
+version: 0.3.4.{build}
 os: Windows Server 2012 R2
 install:
-- C:\Python34\Scripts\pip install ply pyreadline nose pygments prompt_toolkit
 - C:\Python35\Scripts\pip install ply pyreadline nose pygments prompt_toolkit
 build_script:
-- C:\Python34\python setup.py install
 - C:\Python35\python setup.py install
 test_script:
-- C:\Python34\Scripts\nosetests -q
 - C:\Python35\Scripts\nosetests -q
 

.gitattributes

@@ -1,4 +1,32 @@
+# Set the default behavior, in case people don't have core.autocrlf set.
+# This should be first, because the git documentation says "When more
+# than one pattern matches the path, a later line overrides an earlier line."
+* text=auto
+
+# Text files
+*.bat text
+*.css_t text
+*.in text
+*.json text
+*.py text
+*.rst text
+*.sh text
+*.txt text
+*.xsh text
+*.yaml text
+*.yml text
+CONTRIBUTING text
+license text
+LICENSE text
+Makefile text
+README text
+
+# Files in the lazyjson format require LF line endings
 tests/histories/*.json text eol=lf
 
-# Set the default behavior, in case people don't have core.autocrlf set.
-* text=auto
+# Binary files
+*.ico binary
+*.gif binary
+*.gz binary
+*.png binary
+*.webm binary

.gitignore

@@ -3,6 +3,7 @@
 *.xcf
 *.egg
 .eggs/
+__amalgam__.py
 lexer_table.py
 parser_table.py
 parser_test_table.py

.travis.yml

@@ -2,7 +2,10 @@ language: python
 python:
   - 3.4
   - 3.5
+  - "nightly"
 install:
   - pip install -r requirements-tests.txt
 script:
-  - nosetests -q
+  - nosetests -q --with-coverage --cover-package=xonsh
+after_success:
+  - codecov

CHANGELOG.rst

@@ -1,8 +1,151 @@
-================
+====================
 Xonsh Change Log
-================
+====================
 
-Current Developments
+.. current developments
+
+v0.3.4
+====================
+
+**Changed:**
+
+* ``$PROMPT`` from foreign shells is now ignored.
+* ``$RC_FILES`` environment variable now stores the run control files we
+  attempted to load.
+* Only show the prompt for the wizard if we did not attempt to load any run
+  control files (as opposed to if none were successfully loaded).
+* Git and mercurial branch and dirty function refactor to imporve run times.
+
+
+**Fixed:**
+
+* Fixed an issue whereby attempting to delete a literal value (e.g., ``del 7``)
+  in the prompt_toolkit shell would cause xonsh to crash.
+* Fixed broken behavior when using ``cd ..`` to move into a nonexistent
+  directory.
+* Partial workaround for Cygwin where ``pthread_sigmask`` appears to be missing
+  from the ``signal`` module.
+* Fixed crash resulting from malformed ``$PROMPT``.
+* Fixed regression on Windows with the locate_binary() function.
+  The bug prevented `source-cmd` from working correctly and broke the
+  ``activate``/``deactivate`` aliases for the conda environements.
+* Fixed crash resulting from errors other than syntax errors in run control
+  file.
+* On Windows if bash is not on the path look in the registry for the defaults
+  install directory for GitForWindows.
+
+
+v0.3.3
+====================
+**Added:**
+
+* Question mark literals, ``?``, are now allowed as part of
+  subprocess argument names.
+* IPython style visual pointer to show where syntax error was detected
+* Pretty printing of output and syntax highlighting of input and output can now
+  be controlled via new environment variables ``$COLOR_INPUT``,
+  ``$COLOR_RESULTS``, and ``$PRETTY_PRINT_RESULTS``.
+
+* In interactive mode, if there are stopped or background jobs, Xonsh prompts
+  for confirmations rather than just killing all jobs and exiting.
+
+**Changed:**
+
+* ``which`` now gives a better verbose report of where the executables are
+  found.
+* Tab completion now uses a different interface, which allows new completers
+  to be implemented in Python.
+* Most functions in the ``Execer`` now take an extra argument
+  ``transform``, indicating whether the syntax tree transformations should
+  be applied.
+* ``prompt_toolkit`` is now loaded lazily, decreasing load times when using
+  the ``readline`` shell.
+* RC files are now executed directly in the appropriate context.
+* ``_`` is now updated by ``![]``, to contain the appropriate
+  ``CompletedCommand`` object.
+
+
+
+**Removed:**
+
+* Fixed bug on Windows where ``which`` did not include current directory
+
+**Fixed:**
+
+* Fixed crashed bash-completer when bash is not avaiable on Windows
+* Fixed bug on Windows where tab-completion for executables would return all files.
+* Fixed bug on Windows which caused the bash $PROMPT variable to be used when no
+  no $PROMPT variable was set in .xonshrc
+* Improved start-up times by caching information about bash completion
+  functions
+* The --shell-type CLI flag now takes precedence over $SHELL_TYPE specified in
+  .xonshrc
+* Fixed an issue about ``os.killpg()`` on OS X which caused xonsh crash with
+  occasionality
+
+
+
+v0.3.2
+====================
+**Fixed:**
+
+* Fixed PermissionError when tab completions tries to lookup executables in
+  directories without read permissions.
+* Fixed incorrect parsing of command line flags
+
+
+
+v0.3.1
+====================
+**Added:**
+
+* When a subprocess exits with a signal (e.g. SIGSEGV), a message is printed,
+  similar to Bash.
+* Added comma literals to subproc mode.
+* ``@$(cmd)`` has been added as a subprocess-mode operator, which replaces in
+  the subprocess command itself with the result of running ``cmd``.
+* New ``showcmd`` alias for displaying how xonsh interprets subprocess mode
+  commands and arguments.
+* Added ``$DYNAMIC_CWD_WIDTH`` to allow the adjusting of the current working
+  directory width in the prompt.
+* Added ``$XONSH_DEBUG`` environment variable to help with debuging.
+* The ``${...}`` shortcut for ``__xonsh_env__`` now returns appropriate
+  completion options
+
+**Changed:**
+
+* On Windows the default bash completions files ``$BASH_COMPLETIONS`` now points
+  to the default location of the completions files used by 'Git for Windows'
+* On Cygwin, some tweaks are applied to foreign shell subprocess calls and the
+  readline import, in order to avoid hangs on launch.
+
+
+**Removed:**
+
+* Special cased code for handling version of prompt_toolkit < v1.0.0
+
+
+**Fixed:**
+
+* Show sorted bash completions suggestions.
+* Fix bash completions (e.g git etc.) on windows when completions files have
+  spaces in their path names
+* Fixed a bug preventing ``source-bash`` from working on Windows
+* Numerous improvements to job control via a nearly-complete rewrite.
+* Addressed issue with finding the next break in subproc mode in context
+  sensitive parsing.
+* Fixed issue with loading readline init files (inputrc) that seems to be
+  triggered by libedit.
+* ``$MULTILINE_PROMPT`` now allows colors, as originally intended.
+* Rectified install issue with Jupyter hook when installing with pyenv,
+  Jupyter install hook now repects ``--prefix`` argument.
+* Fixed issue with the xonsh.ply subpackage not being installed.
+* Fixed a parsing bug whereby a trailing ``&`` on a line was being ignored
+  (processes were unable to be started in the background)
+
+
+
+v0.3.0
 ====================
 **Added:**
 
@@ -59,10 +202,13 @@ Current Developments
   recommended.
 * Provide better user feedback when running ``which`` in a platform that doesn't
   provide it (e.g. Windows).
+* The lexer now uses a custom tokenizer that handles regex globs in the proper
+  way.
+
+
+
 
-**Deprecated:** None
 
-**Removed:** None
 
 **Fixed:**
 
@@ -86,7 +232,7 @@ Current Developments
 * Fixed xonsh.exe launcher on Windows, when Python install directory has a space in it
 * Fixed `$CDPATH` to support `~` and environments variables in its items
 
-**Security:** None
+
 
 
 v0.2.7

CONTRIBUTING

@@ -1 +0,0 @@
-docs/devguide.rst

CONTRIBUTING.rst

@@ -0,0 +1,424 @@
+.. _devguide:
+
+=================
+Developer's Guide
+=================
+.. image:: _static/knight-vs-snail.jpg
+   :width: 80 %
+   :alt: knight-vs-snail
+   :align: center
+
+Welcome to the xonsh developer's guide!  This is a place for developers to
+place information that does not belong in the user's guide or the library
+reference but is useful or necessary for the next people that come along to
+develop xonsh.
+
+.. note:: All code changes must go through the pull request review procedure.
+
+Changelog
+=========
+Pull requests will often have CHANGELOG entries associated with. However,
+to avoid excessive merge conflicts, please follow the following procedure:
+
+1. Go into the ``news/`` directory,
+2. Copy the ``TEMPLATE.rst`` file to another file in the ``news/`` directory.
+   We suggest using the branchname::
+
+        $ cp TEMPLATE.rst branch.rst
+
+3. Add your entries as a bullet pointed lists in your ``branch.rst`` file in
+   the appropriate category. It is OK to leave the ``None`` entries for later
+   use.
+4. Commit your ``branch.rst``.
+
+Feel free to update this file whenever you want! Please don't use someone
+else's file name. All of the files in this ``news/`` directory will be merged
+automatically at release time.  The ``None`` entries will be automatically
+filtered out too!
+
+
+Style Guide
+===========
+xonsh is a pure Python project, and so we use PEP8 (with some additions) to
+ensure consistency throughout the code base.
+
+----------------------------------
+Rules to Write By
+----------------------------------
+It is important to refer to things and concepts by their most specific name.
+When writing xonsh code or documentation please use technical terms
+appropriately. The following rules help provide needed clarity.
+
+**********
+Interfaces
+**********
+* User-facing APIs should be as generic and robust as possible.
+* Tests belong in the top-level ``tests`` directory.
+* Documentation belongs in the top-level ``docs`` directory.
+
+************
+Expectations
+************
+* Code must have associated tests and adequate documentation.
+* User-interaction code (such as the Shell class) is hard to test.
+  Mechanism to test such constructs should be developed over time.
+* Have *extreme* empathy for your users.
+* Be selfish. Since you will be writing tests you will be your first user.
+
+-------------------
+Python Style Guide
+-------------------
+xonsh uses `PEP8`_ for all Python code. The following rules apply where `PEP8`_
+is open to interpretation.
+
+* Use absolute imports (``import xonsh.tools``) rather than explicit
+  relative imports (``import .tools``). Implicit relative imports
+  (``import tools``) are never allowed.
+* Use ``'single quotes'`` for string literals, and
+  ``"""triple double quotes"""`` for docstrings. Double quotes are allowed to
+  prevent single quote escaping, e.g. ``"Y'all c'mon o'er here!"``
+* We use sphinx with the numpydoc extension to autogenerate API documentation. Follow
+  the `numpydoc`_ standard for docstrings.
+* Simple functions should have simple docstrings.
+* Lines should be at most 80 characters long. The 72 and 79 character
+  recommendations from PEP8 are not required here.
+* All Python code should be compliant with Python 3.4+.  At some
+  unforeseen date in the future, Python 2.7 support *may* be supported.
+* Tests should be written with nose using a procedural style. Do not use
+  unittest directly or write tests in an object-oriented style.
+* Test generators make more dots and the dots must flow!
+
+You can easily check for style issues, including some outright bugs such
+as mispelled variable names, using pylint. If you're using Anaconda you'll
+need to run "conda install pylint" once. You can easily run pylint on
+the edited files in your uncommited git change::
+
+    $ pylint $(git status -s | awk '/\.py$$/ { print $$2 }' | sort)
+
+If you want to lint the entire code base run::
+
+    $ pylint $(find tests xonsh -name \*.py | sort)
+
+**********
+Imports
+**********
+Xonsh source code may be amalgamated into a single file (``__amalgam__.py``)
+to speed up imports. The way the code amalgamater works is that other modules
+that are in the same package (and amalgamated) should be imported with::
+
+    from pkg.x import a, c, d
+
+This is because the amalgamater puts all such modules in the same globals(),
+which is effectively what the from-imports do. For example, ``xonsh.ast`` and
+``xonsh.execer`` are both in the same package (``xonsh``). Thus they should use
+the above from from-import syntax.
+
+Alternatively, for modules outside of the current package (or modules that are
+not amalgamated) the import statement should be either ``import pkg.x`` or
+``import pkg.x as name``. This is because these are the only cases where the
+amalgamater is able to automatically insert lazy imports in way that is guarantted
+to be safe. This is due to the ambiguity that ``from pkg.x import name`` may
+import a variable that cannot be lazily constructed or may import a module.
+So the simple rules to follow are that:
+
+1. Import objects from modules in the same package directly in using from-import,
+2. Import objects from moudules outside of the package via a direct import
+   or import-as statement.
+
+How to Test
+================
+
+----------------------------------
+Docker
+----------------------------------
+
+If you want to run your "work in progress version" without installing
+and in a fresh environment you can use Docker. If Docker is installed
+you just have to run this::
+
+  $ python xonsh-in-docker.py
+
+This will build and run the current state of the repository in an isolated
+container (it may take a while the first time you run it). There are two
+additionals arguments you can pass this script.
+
+* The version of python
+* the version of ``prompt_toolkit``
+
+Example::
+
+  $ python docker.py 3.4 0.57
+
+Ensure your cwd is the root directory of the project (i.e., the one containing the
+.git directory).
+
+----------------------------------
+Dependencies
+----------------------------------
+
+Prep your environment for running the tests::
+
+    $ pip install requirements-tests.txt
+
+
+----------------------------------
+Running the Tests - Basic
+----------------------------------
+
+Run all the tests using Nose::
+
+    $ nosetests -q
+
+Use "-q" to keep nose from outputing a dot for every test.  There are A LOT of tests
+and you will waste time waiting for all the dots to get pushed through stdout.
+
+----------------------------------
+Running the Tests - Advanced
+----------------------------------
+
+To perform all unit tests::
+
+    $ scripts/run_tests.xsh all
+
+If you're working on a change and haven't yet committed it you can run the
+tests associated with the change. This does not require that the change
+include the unit test module. This will execute any unit tests that are
+part of the change as well as the unit tests for xonsh source modules in
+the change::
+
+    $ scripts/run_tests.xsh
+
+If you want to run specific tests you can specify the test names to
+execute. For example to run test_aliases::
+
+    $ scripts/run_tests.xsh aliases
+
+The test name can be the bare test name (e.g., ``aliases``), include
+the ``test_`` prefix and ``.py`` suffix without the directory
+(e.g., ``test_aliases.py``), or the complete relative path (e.g.,
+``tests/test_aliases.py``). For example:
+
+Note that you can pass multiple test names in the above examples::
+
+    $ scripts/run_tests.xsh aliases environ
+
+As before, if you want to test the xonsh code that is installed on your
+system first cd into the `tests` directory then run the tests::
+
+    $ cd tests
+    $ env XONSHRC='' nosetests test_aliases.py test_environ.py
+
+Happy testing!
+
+
+How to Document
+====================
+Documentation takes many forms. This will guide you through the steps of
+successful documentation.
+
+----------
+Docstrings
+----------
+No matter what language you are writing in, you should always have
+documentation strings along with you code. This is so important that it is
+part of the style guide.  When writing in Python, your docstrings should be
+in reStructured Text using the `numpydoc`_ format.
+
+------------------------
+Auto-Documentation Hooks
+------------------------
+The docstrings that you have written will automatically be connected to the
+website, once the appropriate hooks have been setup.  At this stage, all
+documentation lives within xonsh's top-level ``docs`` directory.
+We uses the sphinx tool to manage and generate the documentation, which
+you can learn about from `the sphinx website <http://sphinx-doc.org/>`_.
+If you want to generate the documentation, first xonsh itself must be installed
+and then you may run the following command from the ``docs`` dir:
+
+.. code-block:: bash
+
+    ~/xonsh/docs $ make html
+
+For each new
+module, you will have to supply the appropriate hooks. This should be done the
+first time that the module appears in a pull request.  From here, call the
+new module ``mymod``.  The following explains how to add hooks.
+
+------------------------
+Python Hooks
+------------------------
+Python documentation lives in the ``docs/api`` directory.
+First, create a file in this directory that represents the new module called
+``mymod.rst``.
+The ``docs/api`` directory matches the structure of the ``xonsh/`` directory.
+So if your module is in a sub-package, you'll need to go into the sub-package's
+directory before creating ``mymod.rst``.
+The contents of this file should be as follows:
+
+**mymod.rst:**
+
+.. code-block:: rst
+
+    .. _xonsh_mymod:
+
+    =======================================
+    My Awesome Module -- :mod:`xonsh.mymod`
+    =======================================
+
+    .. currentmodule:: xonsh.mymod
+
+    .. automodule:: xonsh.mymod
+        :members:
+
+This will discover all of the docstrings in ``mymod`` and create the
+appropriate webpage. Now, you need to hook this page up to the rest of the
+website.
+
+Go into the ``index.rst`` file in ``docs/xonsh`` or other subdirectory and add
+``mymod`` to the appropriate ``toctree`` (which stands for table-of-contents
+tree). Note that every sub-package has its own ``index.rst`` file.
+
+
+Building the Website
+===========================
+
+Building the website/documentation requires the following dependencies:
+
+#. `Sphinx <http://sphinx-doc.org/>`_
+#. `Cloud Sphinx Theme <https://pythonhosted.org/cloud_sptheme/cloud_theme.html>`_
+
+-----------------------------------
+Procedure for modifying the website
+-----------------------------------
+The xonsh website source files are located in the ``docs`` directory.
+A developer first makes necessary changes, then rebuilds the website locally
+by executing the command::
+
+    $ make html
+
+This will generate html files for the website in the ``_build/html/`` folder.
+The developer may view the local changes by opening these files with their
+favorite browser, e.g.::
+
+    $ google-chrome _build/html/index.html
+
+Once the developer is satisfied with the changes, the changes should be
+committed and pull-requested per usual. Once the pull request is accepted, the
+developer can push their local changes directly to the website by::
+
+    $ make push-root
+
+Branches and Releases
+=============================
+Mainline xonsh development occurs on the ``master`` branch. Other branches
+may be used for feature development (topical branches) or to represent
+past and upcoming releases.
+
+All releases should have a release candidate ('-rc1') that comes out 2 - 5 days
+prior to the scheduled release.  During this time, no changes should occur to
+a special release branch ('vX.X.X-release').
+
+The release branch is there so that development can continue on the
+develop branch while the release candidates (rc) are out and under review.
+This is because otherwise any new developments would have to wait until
+post-release to be merged into develop to prevent them from accidentally
+getting released early.
+
+As such, the 'vX.X.X-release' branch should only exist while there are
+release candidates out.  They are akin to a temporary second level of staging,
+and so everything that is in this branch should also be part of master.
+
+Every time a new release candidate comes out the vX.X.X-release should be
+tagged with the name 'X.X.X-rcX'.  There should be a 2 - 5 day period of time
+in between release candidates.  When the full and final release happens, the
+'vX.X.X-release' branch is merged into master and then deleted.
+
+If you have a new fix that needs to be in the next release candidate, you
+should make a topical branch and then pull request it into the release branch.
+After this has been accepted, the topical branch should be merged with
+master as well.
+
+The release branch must be quiet and untouched for 2 - 5 days prior to the
+full release.
+
+The release candidate procedure here only applies to major and minor releases.
+Micro releases may be pushed and released directly without having a release
+candidate.
+
+------------------
+Checklist
+------------------
+When releasing xonsh, make sure to do the following items in order:
+
+1. Review **ALL** issues in the issue tracker, reassigning or closing them as
+   needed.
+2. Ensure that all issues in this release's milestone have been closed. Moving issues
+   to the next release's milestone is a perfectly valid strategy for
+   completing this milestone.
+3. Perform maintenance tasks for this project, see below.
+4. Write and commit the release notes.
+5. Review the current state of documentation and make appropriate updates.
+6. Bump the version (in code, documentation, etc.) and commit the change.
+7. If this is a release candidate, tag the release branch with a name that
+   matches that of the release:
+
+   * If this is the first release candidate, create a release branch called
+     'vX.X.X-release' off of develop.  Tag this branch with the name
+     'X.X.X-rc1'.
+   * If this is the second or later release candidate, tag the release branch
+     with the name 'X.X.X-rcX'.
+
+8. If this is the full and final release (and not a release candidate),
+   merge the release branch into the master branch.  Next, tag the master
+   branch with the name 'X.X.X'. Finally, delete the release branch.
+9. Push the tags upstream
+10. Update release information on the website.
+
+--------------------
+Maintenance Tasks
+--------------------
+You can cleanup your local repository of transient files such as \*.pyc files
+created by unit testing by running::
+
+    $ rm -f xonsh/lexer_table.py xonsh/parser_table.py
+    $ rm -f xonsh/lexer_test_table.py xonsh/parser_test_table.py
+    $ rm -f xonsh/*.pyc tests/*.pyc
+    $ rm -f xonsh/*.rej tests/*.rej
+    $rm -fr build
+
+-----------------------
+Performing the Release
+-----------------------
+To perform the release, run these commands for the following tasks:
+
+**pip upload:**
+
+.. code-block:: bash
+
+    $ ./setup.py sdist upload
+
+
+**conda upload:**
+
+.. code-block:: bash
+
+    $ rm -f /path/to/conda/conda-bld/src_cache/xonsh.tar.gz
+    $ conda build --no-test recipe
+    $ conda convert -p all -o /path/to/conda/conda-bld /path/to/conda/conda-bld/linux-64/xonsh-X.X.X-0.tar.bz2
+    $ binstar upload /path/to/conda/conda-bld/*/xonsh-X.X.X*.tar.bz2
+
+**website:**
+
+.. code-block:: bash
+
+    $ cd docs
+    $ make clean html push-root
+
+
+Document History
+===================
+Portions of this page have been forked from the PyNE documentation,
+Copyright 2011-2015, the PyNE Development Team. All rights reserved.
+
+.. _PEP8: https://www.python.org/dev/peps/pep-0008/
+.. _numpydoc: https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt

README.rst

@@ -1,18 +1,18 @@
 xonsh
 =====
 
-.. image:: https://badges.gitter.im/scopatz/xonsh.svg
-   :alt: Join the chat at https://gitter.im/scopatz/xonsh
-   :target: https://gitter.im/scopatz/xonsh?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+.. image:: https://badges.gitter.im/xonsh/xonsh.svg
+   :alt: Join the chat at https://gitter.im/xonsh/xonsh
+   :target: https://gitter.im/xonsh/xonsh?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
 
-.. image:: https://travis-ci.org/scopatz/xonsh.svg?branch=master
-    :target: https://travis-ci.org/scopatz/xonsh
+.. image:: https://travis-ci.org/xonsh/xonsh.svg?branch=master
+    :target: https://travis-ci.org/xonsh/xonsh
 
 .. image:: https://ci.appveyor.com/api/projects/status/ufqtigii8ma3rctt/branch/master?svg=true
     :target: https://ci.appveyor.com/project/rbrewer123/xonsh-unq93
 
-.. image:: https://landscape.io/github/scopatz/xonsh/master/landscape.svg?style=flat
-    :target: https://landscape.io/github/scopatz/xonsh/master
+.. image:: https://landscape.io/github/xonsh/xonsh/master/landscape.svg?style=flat
+    :target: https://landscape.io/github/xonsh/xonsh/master
     :alt: Code Health
 
 xonsh is a Python-ish, BASHwards-looking shell language and command prompt.

amalgamate.py

@@ -0,0 +1,378 @@
+#!/usr/bin/env python
+"""A package-based, source code amalgamater."""
+import os
+import sys
+import pprint
+from itertools import repeat
+from collections import namedtuple
+from collections.abc import Mapping
+from ast import parse, walk, literal_eval, Import, ImportFrom
+
+ModNode = namedtuple('ModNode', ['name', 'pkgdeps', 'extdeps'])
+ModNode.__doc__ = """Module node for dependency graph.
+
+Attributes
+----------
+name : str
+    Module name.
+pkgdeps : frozenset of str
+    Module dependencies in the same package.
+extdeps : frozenset of str
+    External module dependencies from outside of the package.
+"""
+
+
+class SourceCache(Mapping):
+    """Stores / loads source code for files based on package and module names."""
+
+    def __init__(self, *args, **kwargs):
+        self._d = dict(*args, **kwargs)
+
+    def __getitem__(self, key):
+        d = self._d
+        if key in d:
+            return d[key]
+        pkg, name = key
+        pkgdir = pkg.replace('.', os.sep)
+        fname = pkgdir + os.sep + name + '.py'
+        with open(fname, encoding='utf-8', errors='surrogateescape') as f:
+            raw = f.read()
+        d[key] = raw
+        return raw
+
+    def __iter__(self):
+        yield from self._d
+
+    def __len__(self):
+        return len(self._d)
+
+
+SOURCES = SourceCache()
+
+def make_node(name, pkg, allowed):
+    """Makes a node by parsing a file and traversing its AST."""
+    raw = SOURCES[pkg, name]
+    tree = parse(raw, filename=name)
+    # we only want to deal with global import statements
+    pkgdot = pkg + '.'
+    pkgdeps = set()
+    extdeps = set()
+    for a in tree.body:
+        if isinstance(a, Import):
+            for n in a.names:
+                p, dot, m = n.name.rpartition('.')
+                if p == pkg and m in allowed:
+                    pkgdeps.add(m)
+                else:
+                    extdeps.add(n.name)
+        elif isinstance(a, ImportFrom):
+            if a.module == pkg:
+                pkgdeps.update(n.name for n in a.names if n.name in allowed)
+            elif a.module.startswith(pkgdot):
+                p, dot, m = a.module.rpartition('.')
+                if p == pkg and m in allowed:
+                    pkgdeps.add(m)
+                else:
+                    extdeps.add(a.module)
+    return ModNode(name, frozenset(pkgdeps), frozenset(extdeps))
+
+
+def make_graph(pkg, exclude=None):
+    """Create a graph (dict) of module dependencies."""
+    graph = {}
+    pkgdir = pkg.replace('.', os.sep)
+    allowed = set()
+    files = os.listdir(pkgdir)
+    for fname in files:
+        base, ext = os.path.splitext(fname)
+        if base.startswith('__') or ext != '.py':
+            continue
+        allowed.add(base)
+    if exclude:
+        allowed -= exclude
+    for base in allowed:
+        graph[base] = make_node(base, pkg, allowed)
+    return graph
+
+
+def depsort(graph):
+    """Sort modules by dependency."""
+    remaining = set(graph.keys())
+    seder = []
+    solved = set()
+    while 0 < len(remaining):
+        nodeps = {m for m in remaining if len(graph[m].pkgdeps - solved) == 0}
+        if len(nodeps) == 0:
+            msg = ('\nsolved order = {0}\nremaining = {1}\nCycle detected in '
+                   'module graph!').format(pprint.pformat(seder),
+                                           pprint.pformat(remaining))
+            raise RuntimeError(msg)
+        solved |= nodeps
+        remaining -= nodeps
+        seder += sorted(nodeps)
+    return seder
+
+
+LAZY_IMPORTS = """
+from sys import modules as _modules
+from types import ModuleType as _ModuleType
+from importlib import import_module as _import_module
+
+
+class _LazyModule(_ModuleType):
+
+    def __init__(self, pkg, mod, asname=None):
+        '''Lazy module 'pkg.mod' in package 'pkg'.'''
+        self.__dct__ = {
+            'loaded': False,
+            'pkg': pkg,  # pkg
+            'mod': mod,  # pkg.mod
+            'asname': asname,  # alias
+            }
+
+    @classmethod
+    def load(cls, pkg, mod, asname=None):
+        if mod in _modules:
+            return _modules[pkg]
+        else:
+            return cls(pkg, mod, asname)
+
+    def __getattribute__(self, name):
+        if name == '__dct__':
+            return super().__getattribute__(name)
+        dct = self.__dct__
+        mod = dct['mod']
+        if dct['loaded']:
+            m = _modules[mod]
+        else:
+            m = _import_module(mod)
+            glbs = globals()
+            pkg = dct['pkg']
+            asname = dct['asname']
+            if asname is None:
+                glbs[pkg] = _modules[pkg]
+            else:
+                glbs[asname] = m
+            dct['loaded'] = True
+        return getattr(m, name)
+
+"""
+
+def get_lineno(node, default=0):
+    """Gets the lineno of a node or returns the default."""
+    return getattr(node, 'lineno', default)
+
+
+def min_line(node):
+    """Computes the minimum lineno."""
+    node_line = get_lineno(node)
+    return min(map(get_lineno, walk(node), repeat(node_line)))
+
+
+def format_import(names):
+    """Format an import line"""
+    parts = []
+    for _, name, asname in names:
+        if asname is None:
+            parts.append(name)
+        else:
+            parts.append(name + ' as ' + asname)
+    line = 'import ' + ', '.join(parts) + '\n'
+    return line
+
+
+def format_lazy_import(names):
+    """Formats lazy import lines"""
+    lines = ''
+    for _, name, asname in names:
+        pkg, _, _ = name.partition('.')
+        target = asname or pkg
+        if asname is None:
+            line = '{pkg} = _LazyModule.load({pkg!r}, {mod!r})\n'
+        else:
+            line = '{asname} = _LazyModule.load({pkg!r}, {mod!r}, {asname!r})\n'
+        lines += line.format(pkg=pkg, mod=name, asname=asname)
+    return lines
+
+
+def format_from_import(names):
+    """Format a from import line"""
+    parts = []
+    for _, module, name, asname in names:
+        if asname is None:
+            parts.append(name)
+        else:
+            parts.append(name + ' as ' + asname)
+    line = 'from ' + module
+    line += ' import ' + ', '.join(parts) + '\n'
+    return line
+
+
+def rewrite_imports(name, pkg, order, imps):
+    """Rewrite the global imports in the file given the amalgamation."""
+    pkgdot = pkg + '.'
+    raw = SOURCES[pkg, name]
+    tree = parse(raw, filename=name)
+    replacements = []  # list of (startline, stopline, str) tuples
+    # collect replacements in forward direction
+    for a, b in zip(tree.body, tree.body[1:] + [None]):
+        if not isinstance(a, (Import, ImportFrom)):
+            continue
+        start = min_line(a) - 1
+        stop = len(tree.body) if b is None else min_line(b) - 1
+        if isinstance(a, Import):
+            keep = []
+            for n in a.names:
+                p, dot, m = n.name.rpartition('.')
+                if p == pkg and m in order:
+                    msg = ('Cannot amalgamate almagate import of '
+                           'amalgamated module:\n\n  import {0}.{1}\n'
+                           '\nin {0}/{2}.py').format(pkg, n.name, name)
+                    raise RuntimeError(msg)
+                imp = (Import, n.name, n.asname)
+                if imp not in imps:
+                    imps.add(imp)
+                    keep.append(imp)
+            if len(keep) == 0:
+                s = ', '.join(n.name for n in  a.names)
+                s = '# amalgamated ' + s + '\n'
+            else:
+                s = format_lazy_import(keep)
+            replacements.append((start, stop, s))
+        elif isinstance(a, ImportFrom):
+            p, dot, m = a.module.rpartition('.')
+            if a.module == pkg:
+                for n in a.names:
+                    if n.name in order:
+                        msg = ('Cannot amalgamate import of '
+                               'amalgamated module:\n\n  from {0} import {1}\n'
+                               '\nin {0}/{2}.py').format(pkg, n.name, name)
+                        raise RuntimeError(msg)
+            elif a.module.startswith(pkgdot) and p == pkg and m in order:
+                replacements.append((start, stop,
+                                     '# amalgamated ' + a.module + '\n'))
+            else:
+                keep = []
+                for n in a.names:
+                    imp = (ImportFrom, a.module, n.name, n.asname)
+                    if imp not in imps:
+                        imps.add(imp)
+                        keep.append(imp)
+                if len(keep) == len(a.names):
+                    continue  # all new imports
+                elif len(keep) == 0:
+                    s = ', '.join(n.name for n in a.names)
+                    s = '# amalgamated from ' + a.module + ' import ' + s + '\n'
+                else:
+                    s = format_from_import(keep)
+                replacements.append((start, stop, s))
+    # apply replacements in reverse
+    lines = raw.splitlines(keepends=True)
+    for start, stop, s in replacements[::-1]:
+        lines[start] = s
+        for i in range(stop - start - 1):
+            del lines[start+1]
+    return ''.join(lines)
+
+
+def amalgamate(order, graph, pkg):
+    """Create amalgamated source."""
+    src = ('\"\"\"Amalgamation of {0} package, made up of the following '
+           'modules, in order:\n\n* ').format(pkg)
+    src += '\n* '.join(order)
+    src += '\n\n\"\"\"\n'
+    src += LAZY_IMPORTS
+    imps = set()
+    for name in order:
+        lines = rewrite_imports(name, pkg, order, imps)
+        src += '#\n# ' + name + '\n#\n' + lines + '\n'
+    return src
+
+
+def write_amalgam(src, pkg):
+    """Write out __amalgam__.py file"""
+    pkgdir = pkg.replace('.', os.sep)
+    fname = os.path.join(pkgdir, '__amalgam__.py')
+    with open(fname, 'w', encoding='utf-8', errors='surrogateescape') as f:
+        f.write(src)
+
+
+def _init_name_lines(pkg):
+    pkgdir = pkg.replace('.', os.sep)
+    fname = os.path.join(pkgdir, '__init__.py')
+    with open(fname, encoding='utf-8', errors='surrogateescape') as f:
+        raw = f.read()
+    lines = raw.splitlines()
+    return fname, lines
+
+
+def read_exclude(pkg):
+    """reads in modules to exclude from __init__.py"""
+    _, lines = _init_name_lines(pkg)
+    exclude = set()
+    for line in lines:
+        if line.startswith('# amalgamate exclude'):
+            exclude.update(line.split()[3:])
+    return exclude
+
+
+FAKE_LOAD = """
+import os as _os
+if _os.getenv('{debug}', ''):
+    pass
+else:
+    import sys as _sys
+    try:
+        from {pkg} import __amalgam__
+        {load}
+        del __amalgam__
+    except ImportError:
+        pass
+    del _sys
+del _os
+""".strip()
+
+
+def rewrite_init(pkg, order, debug='DEBUG'):
+    """Rewrites the init file to insert modules."""
+    fname, lines = _init_name_lines(pkg)
+    for i, line in enumerate(lines):
+        if line.startswith('# amalgamate end'):
+            stop = i
+        elif line.startswith('# amalgamate'):
+            start = i
+    t = ("{1} = __amalgam__\n        "
+         "_sys.modules['{0}.{1}'] = __amalgam__")
+    load = '\n        '.join(t.format(pkg, m) for m in order)
+    s = FAKE_LOAD.format(pkg=pkg, load=load, debug=debug)
+    if start + 1 == stop:
+        lines.insert(stop, s)
+    else:
+        lines[start+1] = s
+        lines = lines[:start+2] + lines[stop:]
+    init = '\n'.join(lines)
+    with open(fname, 'w', encoding='utf-8', errors='surrogateescape') as f:
+        f.write(init)
+
+
+def main(args=None):
+    if args is None:
+        args = sys.argv
+    debug = 'DEBUG'
+    for pkg in args[1:]:
+        if pkg.startswith('--debug='):
+            debug = pkg[8:]
+            continue
+        print('Amalgamating ' + pkg)
+        exclude = read_exclude(pkg)
+        print('  excluding {}'.format(pprint.pformat(exclude)))
+        graph = make_graph(pkg, exclude=exclude)
+        order = depsort(graph)
+        src = amalgamate(order, graph, pkg)
+        write_amalgam(src, pkg)
+        rewrite_init(pkg, order, debug=debug)
+        print('  collapsed {} modules'.format(len(order)))
+
+
+if __name__ == '__main__':
+    main()

codecov.yml

@@ -0,0 +1,9 @@
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 10
+    patch:
+      default:
+        target: 0%

docs/add_to_shell.rst

@@ -1,7 +1,7 @@
 Additional Setup
 ================
-
-If you want to use xonsh as your default shell, you will first have to add xonsh to `/etc/shells`.
+If you want to use xonsh as your default shell, you will first have
+to add xonsh to `/etc/shells`.
 
 First ensure that xonsh is on your ``$PATH``
 
@@ -21,4 +21,4 @@ To change shells, run
 
    $ chsh -s $(which xonsh)
 
-You will have to log out and log back in before the changes take effect.   
+You will have to log out and log back in before the changes take effect.

docs/aliases.rst

@@ -96,7 +96,7 @@ It will pick up the environment and any aliases.
 Tools for dealing with xonsh history. See `the history tutorial <tutorial_hist.html>`_
 for more information all the history command and all of its sub-commands.
 
-.. command-help:: xonsh.history.main
+.. command-help:: xonsh.history.history_main
 
 
 ``replay``
@@ -104,7 +104,7 @@ for more information all the history command and all of its sub-commands.
 Replays a xonsh history file.  See `the replay section of the history tutorial
 <tutorial_hist.html#replay-action>`_ for more information about this command.
 
-.. command-help:: xonsh.replay.main
+.. command-help:: xonsh.replay.replay_main
 
 
 ``!n``
@@ -128,6 +128,10 @@ Runs timing study on arguments. Similar to IPython's ``%timeit`` magic.
 =================
 Simple alias defined as ``['rsync', '--partial', '-h', '--progress', '--rsh=ssh']``.
 
+``showcmd``
+============
+Displays how comands and arguments are evaluated.
+
 
 ``ipynb``
 =================
@@ -138,14 +142,14 @@ Simple alias defined as ``['ipython', 'notebook', '--no-browser']``.
 =================
 Provides an interface to printing lines of source code prior to their execution.
 
-.. command-help:: xonsh.tracer.main
+.. command-help:: xonsh.tracer.tracermain
 
 
 ``xonfig``
 =================
 Manages xonsh configuration information.
 
-.. command-help:: xonsh.xonfig.main
+.. command-help:: xonsh.xonfig.xonfig_main
 
 
 Windows cmd Aliases
@@ -175,11 +179,11 @@ The following aliases on Windows are expanded to ``['cmd', '/c', alias]``:
 
 
 ``activate``/``deactivate`` on Windows with Anaconda
-=====================
+=========================================================
 On Windows with an Anaconda Python distribution, ``activate`` and
 ``deactivate`` are aliased to ``['source-bat activate']`` and ``['source-bat deactivate']``.
-This makes it possible to use the same commands to activate/deactivate conda environments as 
-in cmd.exe.  
+This makes it possible to use the same commands to activate/deactivate conda environments as
+in cmd.exe.
 
 
 ``sudo`` on Windows

docs/api/blank.rst

@@ -0,0 +1,11 @@
+.. _xonsh_mod:
+
+********************************************************************************
+ (``xonsh.mod``)
+********************************************************************************
+
+.. automodule:: xonsh.mod
+    :members:
+    :undoc-members:
+    :inherited-members:
+

docs/api/completers/base.rst

@@ -0,0 +1,11 @@
+.. _xonsh_completers_base:
+
+***********************************************
+Base Completer (``xonsh.completers.base``)
+***********************************************
+
+.. automodule:: xonsh.completers.base
+    :members:
+    :undoc-members:
+    :inherited-members:
+

docs/api/completers/bash.rst

@@ -0,0 +1,11 @@
+.. _xonsh_completers_bash:
+
+***********************************************
+Bash Completers (``xonsh.completers.bash``)
+***********************************************
+
+.. automodule:: xonsh.completers.bash
+    :members:
+    :undoc-members:
+    :inherited-members:
+

docs/api/completers/commands.rst

@@ -0,0 +1,11 @@
+.. _xonsh_completers_commands:
+
+*******************************************************
+First Command Completer (``xonsh.completers.commands``)
+*******************************************************
+
+.. automodule:: xonsh.completers.commands
+    :members:
+    :undoc-members:
+    :inherited-members:
+

docs/api/completers/dirs.rst

@@ -0,0 +1,11 @@
+.. _xonsh_completers_dirs:
+
+************************************************
+Directory Completers (``xonsh.completers.dirs``)
+************************************************
+
+.. automodule:: xonsh.completers.dirs
+    :members:
+    :undoc-members:
+    :inherited-members:
+

docs/api/completers/index.rst

@@ -0,0 +1,20 @@
+.. _api_completers:
+
+=================
+Completers API
+=================
+All of the ways that xonsh completes you.
+
+**Stock Xonsh COmpleters:**
+
+.. toctree::
+    :maxdepth: 1
+
+    base
+    bash
+    commands
+    dirs
+    man
+    path
+    python
+    tools

docs/api/completers/man.rst

@@ -0,0 +1,11 @@
+.. _xonsh_completers_man:
+
+***********************************************************
+Manual Page Based Completers (``xonsh.completers.man``)
+***********************************************************
+
+.. automodule:: xonsh.completers.man
+    :members:
+    :undoc-members:
+    :inherited-members:
+

docs/api/completers/path.rst

@@ -0,0 +1,11 @@
+.. _xonsh_completers_path:
+
+**********************************************************
+File System Path Completer (``xonsh.completers.path``)
+**********************************************************
+
+.. automodule:: xonsh.completers.base
+    :members:
+    :undoc-members:
+    :inherited-members:
+

docs/api/completers/python.rst

@@ -0,0 +1,11 @@
+.. _xonsh_completers_python:
+
+***********************************************
+Python Completer (``xonsh.completers.python``)
+***********************************************
+
+.. automodule:: xonsh.completers.python
+    :members:
+    :undoc-members:
+    :inherited-members:
+

docs/api/completers/tools.rst

@@ -0,0 +1,11 @@
+.. _xonsh_completers_tools:
+
+***********************************************
+Completion Tools (``xonsh.completers.tools``)
+***********************************************
+
+.. automodule:: xonsh.completers.tools
+    :members:
+    :undoc-members:
+    :inherited-members:
+

docs/api/contexts.rst

@@ -0,0 +1,11 @@
+.. _xonsh_contexts:
+
+********************************************************************************
+Context Managers for Xonsh (``xonsh.contexts``)
+********************************************************************************
+
+.. automodule:: xonsh.contexts
+    :members:
+    :undoc-members:
+    :inherited-members:
+

docs/api/index.rst

@@ -31,6 +31,7 @@ For those of you who want the gritty details.
     inspectors
     history
     completer
+    completers/index
     shell
     base_shell
     readline_shell
@@ -49,6 +50,7 @@ For those of you who want the gritty details.
     :maxdepth: 1
 
     tools
+    platform
     lazyjson
     teepty
     openpy
@@ -60,6 +62,7 @@ For those of you who want the gritty details.
     wizard
     xonfig
     codecache
+    contexts
     vox
 
 

docs/api/platform.rst

@@ -0,0 +1,21 @@
+.. _xonsh_platform:
+
+Platform-specific constants and implementations (``xonsh.platform``)
+====================================================================
+
+.. automodule:: xonsh.platform
+    :members:
+    :undoc-members:
+
+
+.. py:function:: scandir
+
+   This is either `os.scandir` on Python 3.5+ or a function providing a
+   compatibility layer for it.
+   It is recommended for iterations over directory entries at a significantly
+   higher speed than `os.listdir` on Python 3.5+. It also caches properties
+   that are commonly used for filtering.
+
+   :param str path: The path to scan for entries.
+   :return: A generator yielding `DirEntry` instances.
+

docs/bash_to_xsh.rst

@@ -14,16 +14,21 @@ will help you put a finger on how to do the equivelent task in xonsh.
     * - ``$NAME`` or ``${NAME}``
       - ``$NAME``
       - Look up an environment variable by name.
-    * - ``${${VAR}}``
+    * - ``echo "$HOME/hello"``
+      - ``echo "$HOME/hello"``
+      - Construct an argument using an environment variable.
+    * - ``something/$SOME_VAR/$(some_command)``
+      - ``@('something/' + $SOME_VAR + $(some_command).strip())``
+      - Concatenate a variable or text with the result of running a command.
+    * - ``${!VAR}``
       - ``${var or expr}``
       - Look up an environment variable via another variable name. In xonsh,
         this may be any valid expression.
     * - ``$(cmd args)`` or ```cmd args```
-      - ``$(cmd args)``
-      - Use the ``$()`` operator to capture subprocesses as strings. Bash's
-        version of (now-deprecated) backticks is not supported. Note that
-        Bash will automatically tokenize the string, while xonsh just returns
-        a str of stdout.
+      - ``@$(cmd args)``
+      - Command substitution (allow the output of a command to replace the
+        command itself).  Tokenizes and executes the output of a subprocess
+        command as another subprocess.
     * - ``set -e``
       - ``$RAISE_SUBPROC_ERROR = True``
       - Cause a failure after a non-zero return code. Xonsh will raise a
@@ -38,5 +43,10 @@ will help you put a finger on how to do the equivelent task in xonsh.
       - ``or`` as well as ``||``
       - Logical-or operator for subprocesses.
     * - ``$?``
-      - ``__xonsh_history__.rtns[-1]``
+      - ``_.rtn``
       - Returns the exit code, or status, of the previous command.
+    * - ``N=V command``
+      - ``with ${...}.swap(N=V): command``
+      - Set temporary environment variable(s) and execute for command.
+        Use an indented block to execute many commands in the same context.
+

docs/cmdhelp.py

@@ -1,18 +1,18 @@
-"""This module adds a reST directive to sphinx that generates alias 
+"""This module adds a reST directive to sphinx that generates alias
 documentation. For example::
 
     .. command-help:: xonsh.aliases.source_foreign
 
-    .. command-help:: xonsh.aliases.source_foreign -h 
+    .. command-help:: xonsh.aliases.source_foreign -h
 
-will create help for aliases. 
+will create help for aliases.
 """
 import io
 import textwrap
 import importlib
 from docutils import nodes, statemachine, utils
 try:
-    from docutils.utils.error_reporting import ErrorString  # the new way 
+    from docutils.utils.error_reporting import ErrorString  # the new way
 except ImportError:
     from docutils.error_reporting import ErrorString        # the old way
 from docutils.parsers.rst import Directive, convert_directive_function
@@ -27,7 +27,7 @@ from xonsh.tools import redirect_stdout, redirect_stderr
 
 
 class CommandHelp(Directive):
-    """The command-help directive, which is based on constructing a list of 
+    """The command-help directive, which is based on constructing a list of
     of string lines of restructured text and then parsing it into its own node.
     Note that this will add the '--help' flag automatically.
     """

docs/conf.py

@@ -11,6 +11,8 @@ import os
 import sys
 import builtins
 
+os.environ['XONSH_DEBUG'] = '1'
+
 from xonsh import __version__ as XONSH_VERSION
 from xonsh.environ import DEFAULT_DOCS, Env
 from xonsh.xontribs import xontrib_metadata
@@ -25,7 +27,7 @@ on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.pngmath',
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.imgmath',
               'sphinx.ext.inheritance_diagram', 'sphinx.ext.viewcode',
               #'sphinx.ext.autosummary',
               'numpydoc', 'cmdhelp',
@@ -68,7 +70,7 @@ release = XONSH_VERSION
 #today_fmt = '%B %d, %Y'
 
 # List of documents that shouldn't be included in the build.
-#unused_docs = []
+exclude_patterns = ['api/blank.rst']
 
 # List of directories, relative to source directory, that shouldn't be searched
 # for source files.
@@ -78,7 +80,7 @@ exclude_trees = []
 #default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
@@ -98,7 +100,7 @@ pygments_style = 'sphinx'
 #pygments_style = 'pastie'
 
 # A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+modindex_common_prefix = ['xonsh.']
 
 
 # -- Options for HTML output ---------------------------------------------------
@@ -262,6 +264,7 @@ def make_envvars():
            '{docstr}\n\n'
            '**configurable:** {configurable}\n\n'
            '**default:** {default}\n\n'
+           '**store_as_str:** {store_as_str}\n\n'
            '-------\n\n')
     for var in vars:
         title = '$' + var
@@ -269,7 +272,7 @@ def make_envvars():
         vd = env.get_docs(var)
         s += sec.format(low=var.lower(), title=title, under=under,
                         docstr=vd.docstr, configurable=vd.configurable,
-                        default=vd.default)
+                        default=vd.default, store_as_str=vd.store_as_str)
     s = s[:-9]
     fname = os.path.join(os.path.dirname(__file__), 'envvarsbody')
     with open(fname, 'w') as f:
@@ -340,4 +343,5 @@ def make_xontribs():
 make_envvars()
 make_xontribs()
 
-builtins.__xonsh_history__= None
+builtins.__xonsh_history__ = None
+builtins.__xonsh_env__ = {}

docs/dependencies.rst

@@ -6,9 +6,11 @@ Xonsh currently has the following external dependencies,
 
     #. Python v3.4+
     #. PLY (optional, included with xonsh)
-    #. prompt-toolkit (optional)
-    #. Jupyter (optional)
-    #. setproctitle (optional)
+    #. prompt-toolkit (optional, requires `pygments`): 
+       *advanced readline library, syntax-highlighting, line-editing*
+    #. Jupyter (optional): *in-browser REPL, run xonsh in jupyter notebook*
+    #. setproctitle (optional): *change the title of terminal to reflect the current subprocess*
+    #. distro (optional): *linux specific platform information*
 
 *Documentation:*
 

docs/devguide.rst

@@ -1,350 +0,0 @@
-.. _devguide:
-
-=================
-Developer's Guide
-=================
-Welcome to the xonsh developer's guide!  This is a place for developers to
-place information that does not belong in the user's guide or the library
-reference but is useful or necessary for the next people that come along to
-develop xonsh.
-
-.. note:: All code changes must go through the pull request review procedure.
-
-Style Guide
-===========
-xonsh is a pure Python project, and so we use PEP8 (with some additions) to
-ensure consistency throughout the code base.
-
-----------------------------------
-Rules to Write By
-----------------------------------
-It is important to refer to things and concepts by their most specific name.
-When writing xonsh code or documentation please use technical terms
-appropriately. The following rules help provide needed clarity.
-
-**********
-Interfaces
-**********
-* User-facing APIs should be as generic and robust as possible.
-* Tests belong in the top-level ``tests`` directory.
-* Documentation belongs in the top-level ``docs`` directory.
-
-************
-Expectations
-************
-* Code must have associated tests and adequate documentation.
-* User-interaction code (such as the Shell class) is hard to test.
-  Mechanism to test such constructs should be developed over time.
-* Have *extreme* empathy for your users.
-* Be selfish. Since you will be writing tests you will be your first user.
-
--------------------
-Python Style Guide
--------------------
-xonsh uses `PEP8`_ for all Python code. The following rules apply where `PEP8`_
-is open to interpretation.
-
-* Use absolute imports (``import xonsh.tools``) rather than explicit
-  relative imports (``import .tools``). Implicit relative imports
-  (``import tools``) are never allowed.
-* Use ``'single quotes'`` for string literals, and
-  ``"""triple double quotes"""`` for docstrings. Double quotes are allowed to
-  prevent single quote escaping, e.g. ``"Y'all c'mon o'er here!"``
-* We use sphinx with the numpydoc extension to autogenerate API documentation. Follow
-  the `numpydoc`_ standard for docstrings.
-* Simple functions should have simple docstrings.
-* Lines should be at most 80 characters long. The 72 and 79 character
-  recommendations from PEP8 are not required here.
-* All Python code should be compliant with Python 3.4+.  At some
-  unforeseen date in the future, Python 2.7 support *may* be supported.
-* Tests should be written with nose using a procedural style. Do not use
-  unittest directly or write tests in an object-oriented style.
-* Test generators make more dots and the dots must flow!
-
-You can easily check for style issues, including some outright bugs such
-as mispelled variable names, using pylint. If you're using Anaconda you'll
-need to run "conda install pylint" once. You can easily run pylint on
-the edited files in your uncommited git change::
-
-    $ pylint $(git status -s | awk '/\.py$$/ { print $$2 }' | sort)
-
-If you want to lint the entire code base run::
-
-    $ pylint $(find tests xonsh -name \*.py | sort)
-
-How to Test
-================
-
-Ensure your cwd is the root directory of the project (i.e., the one containing the
-.git directory).
-
-----------------------------------
-Dependencies
-----------------------------------
-
-Prep your environment for running the tests::
-
-    $ pip install requirements-tests.txt
-
-
-----------------------------------
-Running the Tests - Basic
-----------------------------------
-
-Run all the tests using Nose::
-
-    $ nosetests -q
-
-Use "-q" to keep nose from outputing a dot for every test.  There are A LOT of tests
-and you will waste time waiting for all the dots to get pushed through stdout.
-
-----------------------------------
-Running the Tests - Advanced
-----------------------------------
-
-To perform all unit tests::
-
-    $ scripts/run_tests.xsh all
-
-If you're working on a change and haven't yet committed it you can run the
-tests associated with the change. This does not require that the change
-include the unit test module. This will execute any unit tests that are
-part of the change as well as the unit tests for xonsh source modules in
-the change::
-
-    $ scripts/run_tests.xsh
-
-If you want to run specific tests you can specify the test names to
-execute. For example to run test_aliases::
-
-    $ scripts/run_tests.xsh aliases
-
-The test name can be the bare test name (e.g., ``aliases``), include
-the ``test_`` prefix and ``.py`` suffix without the directory
-(e.g., ``test_aliases.py``), or the complete relative path (e.g.,
-``tests/test_aliases.py``). For example:
-
-Note that you can pass multiple test names in the above examples::
-
-    $ scripts/run_tests.xsh aliases environ
-
-As before, if you want to test the xonsh code that is installed on your
-system first cd into the `tests` directory then run the tests::
-
-    $ cd tests
-    $ env XONSHRC='' nosetests test_aliases.py test_environ.py
-
-Happy testing!
-
-
-How to Document
-====================
-Documentation takes many forms. This will guide you through the steps of
-successful documentation.
-
-----------
-Docstrings
-----------
-No matter what language you are writing in, you should always have
-documentation strings along with you code. This is so important that it is
-part of the style guide.  When writing in Python, your docstrings should be
-in reStructured Text using the `numpydoc`_ format.
-
-------------------------
-Auto-Documentation Hooks
-------------------------
-The docstrings that you have written will automatically be connected to the
-website, once the appropriate hooks have been setup.  At this stage, all
-documentation lives within xonsh's top-level ``docs`` directory.
-We uses the sphinx tool to manage and generate the documentation, which
-you can learn about from `the sphinx website <http://sphinx-doc.org/>`_.
-If you want to generate the documentation, first xonsh itself must be installed
-and then you may run the following command from the ``docs`` dir:
-
-.. code-block:: bash
-
-    ~/xonsh/docs $ make html
-
-For each new
-module, you will have to supply the appropriate hooks. This should be done the
-first time that the module appears in a pull request.  From here, call the
-new module ``mymod``.  The following explains how to add hooks.
-
-------------------------
-Python Hooks
-------------------------
-Python documentation lives in the ``docs/api`` directory.
-First, create a file in this directory that represents the new module called
-``mymod.rst``.
-The ``docs/api`` directory matches the structure of the ``xonsh/`` directory.
-So if your module is in a sub-package, you'll need to go into the sub-package's
-directory before creating ``mymod.rst``.
-The contents of this file should be as follows:
-
-**mymod.rst:**
-
-.. code-block:: rst
-
-    .. _xonsh_mymod:
-
-    =======================================
-    My Awesome Module -- :mod:`xonsh.mymod`
-    =======================================
-
-    .. currentmodule:: xonsh.mymod
-
-    .. automodule:: xonsh.mymod
-        :members:
-
-This will discover all of the docstrings in ``mymod`` and create the
-appropriate webpage. Now, you need to hook this page up to the rest of the
-website.
-
-Go into the ``index.rst`` file in ``docs/xonsh`` or other subdirectory and add
-``mymod`` to the appropriate ``toctree`` (which stands for table-of-contents
-tree). Note that every sub-package has its own ``index.rst`` file.
-
-
-Building the Website
-===========================
-
-Building the website/documentation requires the following dependencies:
-
-#. `Sphinx <http://sphinx-doc.org/>`_
-#. `Cloud Sphinx Theme <https://pythonhosted.org/cloud_sptheme/cloud_theme.html>`_
-
------------------------------------
-Procedure for modifying the website
------------------------------------
-The xonsh website source files are located in the ``docs`` directory.
-A developer first makes necessary changes, then rebuilds the website locally
-by executing the command::
-
-    $ make html
-
-This will generate html files for the website in the ``_build/html/`` folder.
-The developer may view the local changes by opening these files with their
-favorite browser, e.g.::
-
-    $ google-chrome _build/html/index.html
-
-Once the developer is satisfied with the changes, the changes should be
-committed and pull-requested per usual. Once the pull request is accepted, the
-developer can push their local changes directly to the website by::
-
-    $ make push-root
-
-Branches and Releases
-=============================
-Mainline xonsh development occurs on the ``master`` branch. Other branches
-may be used for feature development (topical branches) or to represent
-past and upcoming releases.
-
-All releases should have a release candidate ('-rc1') that comes out 2 - 5 days
-prior to the scheduled release.  During this time, no changes should occur to
-a special release branch ('vX.X.X-release').
-
-The release branch is there so that development can continue on the
-develop branch while the release candidates (rc) are out and under review.
-This is because otherwise any new developments would have to wait until
-post-release to be merged into develop to prevent them from accidentally
-getting released early.
-
-As such, the 'vX.X.X-release' branch should only exist while there are
-release candidates out.  They are akin to a temporary second level of staging,
-and so everything that is in this branch should also be part of master.
-
-Every time a new release candidate comes out the vX.X.X-release should be
-tagged with the name 'X.X.X-rcX'.  There should be a 2 - 5 day period of time
-in between release candidates.  When the full and final release happens, the
-'vX.X.X-release' branch is merged into master and then deleted.
-
-If you have a new fix that needs to be in the next release candidate, you
-should make a topical branch and then pull request it into the release branch.
-After this has been accepted, the topical branch should be merged with
-master as well.
-
-The release branch must be quiet and untouched for 2 - 5 days prior to the
-full release.
-
-The release candidate procedure here only applies to major and minor releases.
-Micro releases may be pushed and released directly without having a release
-candidate.
-
-------------------
-Checklist
-------------------
-When releasing xonsh, make sure to do the following items in order:
-
-1. Review **ALL** issues in the issue tracker, reassigning or closing them as
-   needed.
-2. Ensure that all issues in this release's milestone have been closed. Moving issues
-   to the next release's milestone is a perfectly valid strategy for
-   completing this milestone.
-3. Perform maintenance tasks for this project, see below.
-4. Write and commit the release notes.
-5. Review the current state of documentation and make appropriate updates.
-6. Bump the version (in code, documentation, etc.) and commit the change.
-7. If this is a release candidate, tag the release branch with a name that
-   matches that of the release:
-
-   * If this is the first release candidate, create a release branch called
-     'vX.X.X-release' off of develop.  Tag this branch with the name
-     'X.X.X-rc1'.
-   * If this is the second or later release candidate, tag the release branch
-     with the name 'X.X.X-rcX'.
-
-8. If this is the full and final release (and not a release candidate),
-   merge the release branch into the master branch.  Next, tag the master
-   branch with the name 'X.X.X'. Finally, delete the release branch.
-9. Push the tags upstream
-10. Update release information on the website.
-
---------------------
-Maintenance Tasks
---------------------
-You can cleanup your local repository of transient files such as \*.pyc files
-created by unit testing by running::
-
-    $ rm -f xonsh/lexer_table.py xonsh/parser_table.py
-    $ rm -f xonsh/lexer_test_table.py xonsh/parser_test_table.py
-    $ rm -f xonsh/*.pyc tests/*.pyc
-    $ rm -f xonsh/*.rej tests/*.rej
-    $rm -fr build
-
------------------------
-Performing the Release
------------------------
-To perform the release, run these commands for the following tasks:
-
-**pip upload:**
-
-.. code-block:: bash
-
-    $ ./setup.py sdist upload
-
-
-**conda upload:**
-
-.. code-block:: bash
-
-    $ rm -f /path/to/conda/conda-bld/src_cache/xonsh.tar.gz
-    $ conda build --no-test recipe
-    $ conda convert -p all -o /path/to/conda/conda-bld /path/to/conda/conda-bld/linux-64/xonsh-X.X.X-0.tar.bz2
-    $ binstar upload /path/to/conda/conda-bld/*/xonsh-X.X.X*.tar.bz2
-
-**website:**
-
-.. code-block:: bash
-
-    $ cd docs
-    $ make clean html push-root
-
-
-Document History
-===================
-Portions of this page have been forked from the PyNE documentation,
-Copyright 2011-2015, the PyNE Development Team. All rights reserved.
-
-.. _PEP8: https://www.python.org/dev/peps/pep-0008/
-.. _numpydoc: https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt

docs/devguide.rst

@@ -0,0 +1 @@
+../CONTRIBUTING.rst

docs/faq.rst

@@ -5,12 +5,12 @@ Ok, so, maybe no one actually asked them.
 
 1. Why xonsh?
 -------------
-The idea for xonsh first struck while I was reviewing the BASH chapter
+The idea for xonsh first struck while I was reviewing the Bash chapter
 (written by my co-author `Katy Huff <http://katyhuff.github.io/>`_)
 of `Effective Computation in Physics <http://physics.codes/>`_. In the book,
 we spend a bunch of time describing important, but complex ideas, such
-as piping. However, we don't even touch on more 'basic' aspects of the BASH
-language, such as if-statements or loops. Even though I have been using BASH
+as piping. However, we don't even touch on more 'basic' aspects of the Bash
+language, such as if-statements or loops. Even though I have been using Bash
 for well over a decade, I am not even sure I *know how*
 to add two numbers together in it or consistently create an array. This is
 normal.
@@ -93,13 +93,35 @@ manually use the ``![]``, ``!()``, ``$[]`` or ``$()`` operators on your code.
 
 5. Context-sensitive parsing is gross
 --------------------------------------
-Yes, context-sensitive parsing is gross. But the point of xonsh is that it uses xontext-sensitive parsing and
-is ultimately a lot less gross than other shell languages, such as BASH.
+Yes, context-sensitive parsing is gross. But the point of xonsh is that it uses
+xontext-sensitive parsing and
+is ultimately a lot less gross than other shell languages, such as Bash.
 Furthermore, its use is heavily limited here.
 
 
+6. My Branches are Timing Out?!
+-------------------------------
+Depending on you system, setup, and repository sizes, computing branch names
+and colors (i.e. if the branch is dirty or not), can be a pretty slow operation.
+This is bad news because xonsh can try to compute these each time it formats
+the ``$PROMPT``.
 
-6. Gotchas
+In order to keep xonsh snappy, we have implemented branch computation timeouts.
+This is set to a nominal value (usually 0.1 sec) via the ``$VC_BRANCH_TIMEOUT``
+environment variable.
+
+Feel free to set this to any limit that you feel comfortable with. So if you
+don't mind a potentially slow prompt, set it to 1, 5, 20, 100 seconds! However,
+if you never want to deal with a slow prompt or seeing this timeout message,
+you can remove the ``{curr_branch}``, ``{branch_color}`` and ``{branch_bg_color}``
+portions of your ``$PROMPT``, and these values will never be computed.
+
+It is also worth noting that ``{branch_color}`` is usually the slow poke.
+Just removing the color lookup from the ``$PROMPT`` can still provide the branch
+name while being fast enough.
+
+
+7. Gotchas
 ----------
 
 There are a few gotchas when using xonsh across multiple versions of Python,

docs/index.rst

@@ -36,14 +36,20 @@ the xonsh shell
         "This is Major Tom to Ground Xonshtrol",
         "Sally sells csh and keeps xonsh to herself",
         "Nice indeed. Everything's accounted for, except your old shell.",
-        "I wanna thank you for putting me back in my snail shell.",
+        "I wanna thank you for putting me back in my snail shell",
         "Crustaceanly Yours",
         "With great shell comes great reproducibility",
         "None shell pass",
+        "You shell not pass!",
         "The x-on shell",
+        "Ever wonder why there isn't a Taco Shell? Because it is a corny idea.",
         "It is pronounced <i>コンチ</i>",
         "It is pronounced <i>コンッチ</i>",
         "It is pronounced <i>コンシュ</i>",
+        "The carcolh will catch you!",
+        "People xonshtantly mispronounce these things",
+        "WHAT...is your favorite shell?",
+        "Exploiting the workers and hanging on to outdated imperialist dogma since 2015."
         ];
     document.write(taglines[Math.floor(Math.random() * taglines.length)]);
     </script>
@@ -75,12 +81,14 @@ Contents
 **Installation:**
 
 .. toctree::
-   :titlesonly:
-   :maxdepth: 1
+    :titlesonly:
+    :maxdepth: 1
 
-   linux
-   osx
-   windows
+    dependencies
+    linux
+    osx
+    windows
+    add_to_shell
 
 **Guides:**
 
@@ -91,6 +99,7 @@ Contents
     tutorial
     tutorial_hist
     tutorial_xontrib
+    tutorial_completers
     bash_to_xsh
     python_virtual_environments
 
@@ -106,6 +115,16 @@ Contents
     aliases
     xontribs
 
+**News & Media:**
+
+.. toctree::
+    :titlesonly:
+    :maxdepth: 1
+
+    talks_and_articles
+    quotes
+
+
 **Development Spiral:**
 
 .. toctree::
@@ -215,24 +234,9 @@ table lists features and capabilities that various tools may or may not share.
       - ✓
 
 
-============
-Dependencies
-============
-Xonsh currently has the following external dependencies,
 
-*Run Time:*
+.. include:: dependencies.rst
 
-    #. Python v3.4+
-    #. PLY (optional, included with xonsh)
-    #. prompt-toolkit (optional)
-    #. Jupyter (optional)
-    #. setproctitle (optional)
-
-*Documentation:*
-
-    #. `Sphinx <http://sphinx-doc.org/>`_ (which uses  `reStructuredText <http://sphinx-doc.org/rest.html>`_)
-    #. Numpydoc
-    #. Cloud Sphinx Theme
 
 ============
 Contributing
@@ -255,9 +259,10 @@ Helpful Links
 =============
 
 * `Documentation <http://xon.sh>`_
+* `Gitter <https://gitter.im/scopatz/xonsh>`_
 * `Mailing list <https://groups.google.com/forum/#!forum/xonsh>`_
-* `GitHub Repository <https://github.com/scopatz/xonsh>`_
 * `IRC: channel #xonsh on OFTC <http://www.oftc.net/>`_
+* `GitHub Repository <https://github.com/scopatz/xonsh>`_
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`

docs/linux.rst

@@ -11,7 +11,8 @@ You can install xonsh using ``conda``, ``pip``, or from source.
 
 .. code-block:: bash
 
-    $ conda install -c conda-forge xonsh
+    $ conda config --add channels conda-forge
+    $ conda install xonsh
 
 .. note:: For the bleeding edge development version use ``conda install -c xonsh/channel/dev xonsh``
 
@@ -86,3 +87,30 @@ lines to your ``~/.bashrc file``:
 
     unset module
     unset scl
+
+
+
+Default Ubuntu .bashrc breaks Foreign Shell Functions
+=====================================================
+
+Xonsh supports importing functions from foreign shells using the
+`ForeignShellFunctionAlias` class, which calls functions as if they were
+aliases. This is implemented by executing a command that sources the file
+containing the function definition and then immediately calls the function with
+any necessary arguments.
+
+The default user `~/.bashrc` file in Ubuntu 15.10 has the following snippet at
+the top, which causes the script to exit immediately if not run interactively.
+
+.. code-block:: bash
+                
+    # If not running interactively, don't do anything
+    case $- in
+        *i*) ;;
+          *) return;;
+    esac
+
+This means that any function you have added to the file after this point will be
+registered as a xonsh alias but will fail on execution. Previous versions of
+Ubuntu have a different test for interactivity at the top of the file that
+yields the same problem.

docs/osx.rst

@@ -5,13 +5,21 @@ OSX Guide
 Installation
 ============
 
-You can install xonsh using conda, pip, or from source.
+You can install xonsh using homebrew, conda, pip, or from source.
+
+**homebrew:**
+
+.. code-block:: bash
+
+   $ brew install xonsh
+
 
 **conda:**
 
 .. code-block:: bash
 
-    $ conda install -c conda-forge xonsh
+    $ conda config --add channels conda-forge
+    $ conda install xonsh
 
 .. note:: For the bleeding edge development version use ``conda install -c xonsh/channel/dev xonsh``
     
@@ -35,3 +43,13 @@ the following from the source directory,
 .. include:: add_to_shell.rst
 
 .. include:: dependencies.rst
+
+
+GNU Readline
+============
+
+On Mac OSX, it is *strongly* recommended to install the ``gnureadline`` library if using the readline shell.  ``gnureadline`` can be installed via pip:
+
+.. code-block:: bash
+
+    $ pip install gnureadline

docs/quotes.rst

@@ -0,0 +1,25 @@
+==========
+Quotes
+==========
+`@gilforsyth <https://twitter.com/gilforsyth>`_ **says,**
+
+.. epigraph::
+
+    “Just stumbled across xonsh by @scopatz -- holy cow it's amazing.  I've never
+    been so happy to rewrite a .rc file”
+
+
+`@wbuthod <https://twitter.com/wbuthod>`_ **says,**
+
+.. epigraph::
+
+    “I've tweeted about Xonsh before, and finally spent a day using it
+    exclusively. I must have it on ALL PLATFORMS now.”
+
+`@biochemistries <https://twitter.com/biochemistries>`_ **says,**
+
+.. epigraph::
+
+    “@pathogenomenick @btnaughton @lexnederbragt the dark wizardry of @scopatz :-)
+    check out xonsh.org”
+

docs/talks_and_articles.rst

@@ -0,0 +1,25 @@
+==========================
+Talks & Articles
+==========================
+Here are some talks, articles, and other sundry about your favorite shell.
+
+Talks
+============
+**PyCon 2016:** presented by Anthony Scopatz
+
+.. raw:: html
+
+    <div style="text-align:center;">
+    <iframe width="560" height="315" src="https://www.youtube.com/embed/uaje5I22kgE" frameborder="0" allowfullscreen></iframe>
+    </div>
+
+
+Articles
+=========
+
+* `First Impressions of Xonsh <https://www.fusionbox.com/blog/detail/thoughts-on-pycon-2016/606/>`_,
+  fussionbox, June 20, 2016.
+* `InfoWorld on xonsh <http://www.infoworld.com/article/3078017/application-development/new-shell-packs-power-of-python-and-bash.html>`_,
+  Paul Krill, June 2nd, 2016.
+* `Slashdot on xonsh <https://developers.slashdot.org/story/16/06/04/0039245/pythonunix-hybrid-demoed-at-pycon>`_,
+  June 2016.

docs/tutorial.rst

@@ -9,10 +9,10 @@ commands, manipulating the environment, and dealing with the file system
 easy.  The xonsh command prompt gives users interactive access to the xonsh
 language.
 
-While all Python code is also xonsh, not all BASH code can be used in xonsh.
+While all Python code is also xonsh, not all Bash code can be used in xonsh.
 That would defeat the purpose, and Python is better anyway! Still, xonsh is
-BASH-wards compatible in the ways that matter, such as for running commands,
-reading in the BASH environment, and utilizing BASH tab completion.
+Bash-wards compatible in the ways that matter, such as for running commands,
+reading in the Bash environment, and utilizing Bash tab completion.
 
 The purpose of this tutorial is to teach you xonsh. There are many excellent
 guides out there for learning Python, and this will not join their ranks.
@@ -136,19 +136,25 @@ variable in Python.  The same is true for deleting them too.
 
 Very nice.
 
-__xonsh_env__
---------------
+The Environment Itself ``${...}``
+---------------------------------
 
-All environment variables live in the built-in ``__xonsh_env__`` mapping. You can access this
-mapping directly, but in most situations, you shouldn't need to.
+All environment variables live in the built-in ``${...}`` (aka ``__xonsh_env__``) mapping.
+You can access this mapping directly, but in most situations, you shouldn’t need to.
 
-One helpful method on the __xonsh_env__ is :func:`~xonsh.environ.Env.swap`.  It can be used to temporarily set an
-environment variable:
+If you want for example to check if an environment variable is present in your current
+session (say, in your awesome new ``xonsh`` script) you can use the membership operator:
+.. code-block:: xonshcon
 
+   >>> 'HOME' in ${...}
+   True
+
+One helpful method on the ``${...}`` is :func:`~xonsh.environ.Env.swap`.
+It can be used to temporarily set an environment variable:
 
 .. code-block:: xonshcon
 
-    >>> with __xonsh_env__.swap(SOMEVAR='foo'):
+    >>> with ${...}.swap(SOMEVAR='foo'):
     ...     echo $SOMEVAR
     ...
     ...
@@ -157,6 +163,31 @@ environment variable:
 
     >>>
 
+Environment Lookup with ``${<expr>}``
+-------------------------------------
+
+The ``$NAME`` is great as long as you know the name of the environment
+variable you want to look up.  But what if you want to construct the name
+programmatically, or read it from another variable?  Enter the ``${}``
+operator.
+
+.. warning:: In Bash, ``$NAME`` and ``${NAME}`` are syntactically equivalent.
+             In xonsh, they have separate meanings.
+
+We can place any valid Python expression inside of the curly braces in
+``${<expr>}``. This result of this expression will then be used to look up a
+value in the environment. Here are a couple of examples in action:
+
+.. code-block:: xonshcon
+
+    >>> x = 'USER'
+    >>> ${x}
+    'snail'
+    >>> ${'HO' + 'ME'}
+    '/home/snail'
+
+Not bad, xonsh, not bad.
+
 Environment Types
 -----------------
 
@@ -193,41 +224,6 @@ They can be seen on the `Environment Variables page <envvars.html>`_.
           ``KeyError`` will be raised if the variable does not exist in the
           environment.
 
-Environment Lookup with ``${}``
-================================
-The ``$NAME`` is great as long as you know the name of the environment
-variable you want to look up.  But what if you want to construct the name
-programmatically, or read it from another variable?  Enter the ``${}``
-operator.
-
-.. warning:: In BASH, ``$NAME`` and ``${NAME}`` are syntactically equivalent.
-             In xonsh, they have separate meanings.
-
-We can place any valid Python expression inside of the curly braces in
-``${<expr>}``. This result of this expression will then be used to look up a
-value in the environment.  In fact, ``${<expr>}`` is the same as doing
-``__xonsh_env__[<expr>]``, but much nicer to look at. Here are a couple of
-examples in action:
-
-.. code-block:: xonshcon
-
-    >>> x = 'USER'
-    >>> ${x}
-    'snail'
-    >>> ${'HO' + 'ME'}
-    '/home/snail'
-
-Not bad, xonsh, not bad.
-
-If you want to check if an environment variable is present in your current
-session (say, in your awesome new ``xonsh`` script) you can pass an Ellipsis to
-the ``${}`` operator:
-
-.. code-block:: xonshcon
-
-   >>> 'HOME' in ${...}
-   True
-
 Running Commands
 ==============================
 As a shell, xonsh is meant to make running commands easy and fun.
@@ -316,7 +312,7 @@ The ``$(<expr>)`` operator in xonsh executes a subprocess command and
 *captures* some information about that command.
 
 The ``$()`` syntax captures and returns the standard output stream of the
-command as a Python string.  This is similar to how ``$()`` performs in BASH.
+command as a Python string.  This is similar to how ``$()`` performs in Bash.
 For example,
 
 .. code-block:: xonshcon
@@ -327,7 +323,7 @@ For example,
 The ``!()`` syntax captured more information about the command, as an instance
 of a class called ``CompletedCommand``.  This object contains more information
 about the result of the given command, including the return code, the process
-id, the stdanard output and standard error streams, and information about how
+id, the standard output and standard error streams, and information about how
 input and output were redirected.  For example:
 
 .. code-block:: xonshcon
@@ -356,6 +352,43 @@ example:
             sleep 1
 
 
+If you iterate over the ``CompletedCommand`` object, it will yield lines of its
+output.  Using this, you can quickly and cleanly process output from commands.
+Additionally, these objects expose a method ``itercheck``, which behaves the same
+as the built-in iterator but raises ``XonshCalledProcessError`` if the process
+had a nonzero return code.
+
+.. code-block:: xonshcon
+
+    def get_wireless_interface():
+        """Returns devicename of first connected wifi, None otherwise"""
+        for line in !(nmcli device):
+            dev, typ, state, conn_name = line.split(None, 3)
+            if typ == 'wifi' and state == 'connected':
+                return dev
+
+    def grep_path(path, regexp):
+        """Recursively greps `path` for perl `regexp`
+
+        Returns a dict of 'matches' and 'failures'.
+        Matches are files that contain the given regexp.
+        Failures are files that couldn't be scanned.
+        """
+        matches = []
+        failures = []
+
+        try:
+            for match in !(grep -RPl @(regexp) @(str(path))).itercheck():
+                matches.append(match)
+        except XonshCalledProcessError as error:
+            for line in error.stderr.split('\n'):
+                if not line.strip():
+                    continue
+                filename = line.split('grep: ', 1)[1].rsplit(':', 1)[0]
+                failures.append(filename)
+        return {'matches': matches, 'failures': failures}
+
+
 The ``$()`` and ``!()`` operators are expressions themselves. This means that
 we can assign the results to a variable or perform any other manipulations we
 want.
@@ -422,12 +455,14 @@ terminal, and the resulting object is not displayed.  For example
 Python Evaluation with ``@()``
 ===============================
 
-The ``@(<expr>)`` operator from will evaluate arbitrary Python code in
-subprocess mode and the result will be appended to the subprocess command
-list. If the result is a string, it is appended to the argument list.
-If the result is a list or other non-string sequence, the contents are
-converted to strings and appended to the argument list in order. Otherwise, the
-result is automatically converted to a string.  For example,
+The ``@(<expr>)`` operator form works in subprocess mode, and will evaluate
+arbitrary Python code. The result is appended to the subprocess command list.
+If the result is a string, it is appended to the argument list. If the result
+is a list or other non-string sequence, the contents are converted to strings
+and appended to the argument list in order. If the result in the first position
+is a function, it is treated as an alias (see the section on `Aliases`_ below),
+even if it was not explicitly added to the ``aliases`` mapping.  Otherwise, the
+result is automatically converted to a string. For example,
 
 .. code-block:: xonshcon
 
@@ -439,6 +474,8 @@ result is automatically converted to a string.  For example,
     4
     >>> echo @([42, 'yo'])
     42 yo
+    >>> echo "hello" | @(lambda a, s=None: s.strip + " world")
+    hello world
 
 This syntax can be used inside of a captured or uncaptured subprocess, and can
 be used to generate any of the tokens in the subprocess command list.
@@ -459,17 +496,46 @@ feed them to a subprocess as needed.  For example:
     for i in range(20):
         $[touch @('file%02d' % i)]
 
+Command Substitution with ``@$()``
+==================================
+
+A common use of the ``@()`` and ``$()`` operators is allowing the output of a
+command to replace the command itself (command substitution):
+``@([i.strip() for i in $(cmd).split()])``.  Xonsh offers a
+short-hand syntax for this operation: ``@$(cmd)``.
+
+Consider the following example:
+
+.. code-block:: xonshcon
+
+    >>> # this returns a string representing stdout
+    >>> $(which ls)
+    'ls --color=auto\n'
+
+    >>> # this attempts to run the command, but as one argument
+    >>> # (looks for 'ls --color=auto\n' with spaces and newline)
+    >>> @($(which ls).strip())
+    xonsh: subprocess mode: command not found: ls --color=auto
+
+    >>> # this actually executes the intended command
+    >>> @([i.strip() for i in $(which ls).split()])
+    some_file  some_other_file
+
+    >>> # this does the same thing, but is much more concise
+    >>> @$(which ls)
+    some_file  some_other_file
+
 
 Nesting Subprocesses
 =====================================
 Though I am begging you not to abuse this, it is possible to nest the
 subprocess operators that we have seen so far (``$()``, ``$[]``, ``${}``,
-``@()``).  An instance of ``ls -l`` that is on the wrong side of the border of
-the absurd is shown below:
+``@()``, ``@$()``).  An instance of ``ls -l`` that is on the wrong side of the
+border of the absurd is shown below:
 
-.. code-block:: xonshcon
+.. code-block:: console
 
-    >>> $[$(echo ls) @('-' + $(echo l).strip())]
+    >>> $[@$(which @($(echo ls).strip())) @('-' + $(printf 'l'))]
     total 0
     -rw-rw-r-- 1 snail snail 0 Mar  8 15:46 xonsh
 
@@ -707,7 +773,8 @@ Each job has a unique identifier (starting with 1 and counting upward).  By
 default, the ``fg`` and ``bg`` commands operate on the job that was started
 most recently.  You can bring older jobs to the foreground or background by
 specifying the appropriate ID; for example, ``fg 1`` brings the job with ID 1
-to the foreground.
+to the foreground. Additionally, specify "+" for the most recent job and "-"
+for the second most recent job.
 
 String Literals in Subprocess-mode
 ====================================
@@ -772,8 +839,15 @@ This is not available in Python-mode because multiplication is pretty
 important.
 
 
-Regular Expression Filename Globbing with Backticks
-=====================================================
+Advanced Path Search with Backticks
+===================================
+
+xonsh offers additional ways to find path names beyond regular globbing, both
+in Python mode and in subprocess mode.
+
+Regular Expression Globbing
+---------------------------
+
 If you have ever felt that normal globbing could use some more octane,
 then regex globbing is the tool for you! Any string that uses backticks
 (`````) instead of quotes (``'``, ``"``) is interpreted as a regular
@@ -795,13 +869,58 @@ Let's see a demonstration with some simple filenames:
     >>> len(`a(a+|b+)a`)
     3
 
-Other than the regex matching, this functions in the same way as normal
-globbing.
-For more information, please see the documentation for the ``re`` module in
-the Python standard library.
+This same kind of search is performed if the backticks are prefaced with ``r``.
+So the following expresions are equivalent: ```test``` and ``r`test```.
 
-.. warning:: This backtick syntax has very different from that of BASH.  In
-             BASH, backticks mean to run a captured subprocess ``$()``.
+Other than the regex matching, this functions in the same way as normal
+globbing.  For more information, please see the documentation for the ``re``
+module in the Python standard library.
+
+.. warning:: This backtick syntax has very different from that of Bash.  In
+             Bash, backticks mean to run a captured subprocess ``$()``.
+
+
+Normal Globbing
+---------------
+
+In subprocess mode, normal globbing happens without any special syntax.
+However, the backtick syntax has an additional feature: it is available inside
+of Python mode as well as subprocess mode.
+
+Similarly to regex globbing, normal globbing can be performed (either in Python
+mode or subprocess mode) by using the ``g````:
+
+.. code-block:: xonshcon
+
+    >>> touch a aa aaa aba abba aab aabb abcba
+    >>> ls a*b*
+    aab  aabb  aba  abba  abcba
+    >>> ls g`a*b*`
+    aab  aabb  aba  abba  abcba
+    >>> print(g`a*b*`)
+    ['aab', 'aabb', 'abba', 'abcba', 'aba']
+    >>> len(g`a*b*`)
+    5
+
+
+Custom Path Searches
+--------------------
+
+In addition, if normal globbing and regular expression globbing are not enough,
+xonsh allows you to specify your own search functions.
+
+A search function is defined as a function of a single argument (a string) that
+returns a list of possible matches to that string.  Search functions can then
+be used with backticks with the following syntax: ``@<name>`test```
+
+The following example shows the form of these functions:
+
+.. code-block:: xonshcon
+
+    >>> def foo(s):
+    ...     return [i for i in os.listdir('.') if i.startswith(s)]
+    >>> @foo`aa`
+    ['aa', 'aaa', 'aab', 'aabb']
 
 
 Help & Superhelp with ``?`` & ``??``
@@ -887,7 +1006,7 @@ Of course, for subprocess commands, you still want to use the ``man`` command.
 
 Compile, Evaluate, & Execute
 ================================
-Like Python and BASH, xonsh provides built-in hooks to compile, evaluate,
+Like Python and Bash, xonsh provides built-in hooks to compile, evaluate,
 and execute strings of xonsh code.  To prevent this functionality from having
 serious name collisions with the Python built-in ``compile()``, ``eval()``,
 and ``exec()`` functions, the xonsh equivalents all append an 'x'.  So for
@@ -899,7 +1018,7 @@ Aliases
 ==============================
 Another important xonsh built-in is the ``aliases`` mapping.  This is
 like a dictionary that affects how subprocess commands are run.  If you are
-familiar with the BASH ``alias`` built-in, this is similar.  Alias command
+familiar with the Bash ``alias`` built-in, this is similar.  Alias command
 matching only occurs for the first element of a subprocess command.
 
 The keys of ``aliases`` are strings that act as commands in subprocess-mode.
@@ -921,13 +1040,16 @@ If you were to run ``gco feature-fabulous`` with the above aliases in effect,
 the command would reduce to ``['git', 'checkout', 'feature-fabulous']`` before
 being executed.
 
+
+Callable Aliases
+----------------
 Lastly, if an alias value is a function (or other callable), then this
 function is called *instead* of going to a subprocess command. Such functions
-must have the following signature:
+must have one of the following two signatures
 
 .. code-block:: python
 
-    def mycmd(args, stdin=None):
+    def _mycmd(args, stdin=None):
         """args will be a list of strings representing the arguments to this
         command. stdin will be a string, if present. This is used to pipe
         the output of the previous command into this one.
@@ -962,29 +1084,77 @@ must have the following signature:
         # examples the return code would be 0/success.
         return (None, "I failed", 2)
 
+
+.. code-block:: python
+
+    def _mycmd2(args, stdin, stdout, stderr):
+        """args will be a list of strings representing the arguments to this
+        command.  stdin is a read-only file-like object, and stdout and stderr
+        are write-only file-like objects
+        """
+        # This form allows "streaming" data to stdout and stderr
+        import time
+        for i in range(5):
+            time.sleep(i)
+            print(i, file=stdout)
+
+        # In this form, the return value should be a single integer
+        # representing the "return code" of the alias (zero if successful,
+        # non-zero otherwise)
+        return 0
+
+
+Adding and Removing Aliases
+---------------------------
 We can dynamically alter the aliases present simply by modifying the
 built-in mapping.  Here is an example using a function value:
 
 .. code-block:: xonshcon
 
-    >>> aliases['banana'] = lambda args, stdin=None: ('My spoon is tooo big!', None)
+    >>> def _banana(args, stdin=None):
+    ...     return ('My spoon is tooo big!', None)
+    >>> aliases['banana'] = _banana
     >>> banana
     'My spoon is tooo big!'
 
-Usually, callable alias commands will be run in a separate thread so that
-users may background them interactively. However, some aliases may need to be
-executed on the thread that they were called from. This is mostly useful for debuggers
-and profilers. To make an alias run in the foreground, decorate its function
-with the ``xonsh.proc.foreground`` decorator.
+.. note::
+
+   Alias functions should generally be defined with a leading underscore.
+   Otherwise, they may shadow the alias itself, as Python variables take
+   precedence over aliases when xonsh executes commands.
+
+
+Anonymous Aliases
+-----------------
+As mentioned above, it is also possible to treat functions outside this mapping
+as aliases, by wrapping them in ``@()``.  For example:
+
+.. code-block:: xonshcon
+
+    >>> @(_banana)
+    'My spoon is tooo big!'
+    >>> echo "hello" | @(lambda args, stdin=None: stdin.strip() + args[0]) world
+    hello world
+
+
+Foreground-only Aliases
+-----------------------
+Usually, callable alias commands will be run in a separate thread so that users
+they may be run in the background.  However, some aliases may need to be
+executed on the thread that they were called from. This is mostly useful for
+debuggers and profilers. To make an alias run in the foreground, decorate its
+function with the ``xonsh.proc.foreground`` decorator.
 
 .. code-block:: python
 
     from xonsh.proc import foreground
 
     @foreground
-    def mycmd(args, stdin=None):
+    def _mycmd(args, stdin=None):
         return 'In your face!'
 
+    aliases['mycmd'] = _mycmd
+
 Aliasing is a powerful way that xonsh allows you to seamlessly interact to
 with Python and subprocess.
 
@@ -993,19 +1163,28 @@ Up, Down, Tab
 The up and down keys search history matching from the start of the line,
 much like they do in the IPython shell.
 
-Tab completion is present as well. In Python-mode you are able to complete
-based on the variable names in the current builtins, globals, and locals,
-as well as xonsh languages keywords & operator, files & directories, and
-environment variable names. In subprocess-mode, you additionally complete
-on any file names on your ``$PATH``, alias keys, and full BASH completion
-for the commands themselves.
+Tab completion is present as well. By default, in Python-mode you are able to
+complete based on the variable names in the current builtins, globals, and
+locals, as well as xonsh languages keywords & operator, files & directories,
+and environment variable names. In subprocess-mode, you additionally complete
+on the names of executable files on your ``$PATH``, alias keys, and full Bash
+completion for the commands themselves.
+
+xonsh also provides a means of modifying the behavior of the tab completer.  More
+detail is available on the `Tab Completion page <tutorial_completers.html>`_.
 
 Customizing the Prompt
 ======================
-Customizing the prompt is probably the most common reason for altering an
-environment variable.  The ``PROMPT`` variable can be a string, or it can be a
-function (of no arguments) that returns a string.  The result can contain
-keyword arguments, which will be replaced automatically:
+Customizing the prompt by modifying ``$PROMPT`` is probably the most common
+reason for altering an environment variable.
+
+.. note:: Note that the ``$PROMPT`` variable will never be inherited from a
+          parent process (regardless of whether that parent is a foreign shell
+          or an instance of xonsh).
+
+The ``$PROMPT`` variable can be a string, or it can be a function (of no
+arguments) that returns a string.  The result can contain keyword arguments,
+which will be replaced automatically:
 
 .. code-block:: xonshcon
 
@@ -1018,7 +1197,8 @@ By default, the following variables are available for use:
 
   * ``user``: The username of the current user
   * ``hostname``: The name of the host computer
-  * ``cwd``: The current working directory
+  * ``cwd``: The current working directory, you may use ``$DYNAMIC_CWD_WIDTH`` to
+    set a maximum width for this variable.
   * ``short_cwd``: A shortened form of the current working directory; e.g.,
     ``/path/to/xonsh`` becomes ``/p/t/xonsh``
   * ``cwd_dir``: The dirname of the current working directory, e.g. ``/path/to`` in
@@ -1028,7 +1208,10 @@ By default, the following variables are available for use:
   * ``curr_branch``: The name of the current git branch (preceded by space),
     if any.
   * ``branch_color``: ``{BOLD_GREEN}`` if the current git branch is clean,
-    otherwise ``{BOLD_RED}``
+    otherwise ``{BOLD_RED}``. This is yellow if the branch color could not be
+    determined.
+  * ``branch_bg_color``: Like, ``{branch_color}``, but sets a background color
+    instead.
   * ``prompt_end``: `#` if the user has root/admin permissions `$` otherwise
   * ``current_job``: The name of the command currently running in the
     foreground, if any.
@@ -1073,7 +1256,7 @@ functions of no arguments (which will be called each time the prompt is
 generated, and the results of those calls will be inserted into the prompt).
 For example:
 
-.. code-block:: xonshcon
+.. code-block:: console
 
     snail@home ~ $ $FORMATTER_DICT['test'] = "hey"
     snail@home ~ $ $PROMPT = "{test} {cwd} $ "
@@ -1091,7 +1274,7 @@ interpreted as an empty string.
 Environment variables and functions are also available with the ``$``
 prefix.  For example:
 
-.. code-block:: xonshcon
+.. code-block:: console
 
     snail@home ~ $ $PROMPT = "{$LANG} >"
     en_US.utf8 >
@@ -1124,7 +1307,7 @@ script, stored in ``test.xsh``:
     print('adding files')
     # This is a comment
     for i, x in enumerate("xonsh"):
-        echo @(x) > @("file%d.txt" % i)
+        echo @(x) > @("file{0}.txt".format(i))
 
     print($(ls).replace('\n', ' '))
 
@@ -1178,7 +1361,7 @@ operates on a given argument, rather than on the string ``'xonsh'`` (notice how
     print('adding files')
     # This is a comment
     for i, x in enumerate($ARG1):
-        echo @(x) > @("file%d.txt" % i)
+        echo @(x) > @("file{0}.txt".format(i))
 
     print($(ls).replace('\n', ' '))
     print()

docs/tutorial_completers.rst

@@ -0,0 +1,132 @@
+.. _tutorial_completers:
+
+*************************************
+Tutorial: Programmable Tab-Completion
+*************************************
+
+Overview
+================================
+
+As with many other shells, xonsh ships with the ability to complete
+partially-specified arguments upon hitting the "tab" key.
+
+In Python-mode, pressing the "tab" key will complete based on the variable
+names in the current builtins, globals, and locals, as well as xonsh language
+keywords & operators, files & directories, and environment variable names. In
+subprocess-mode, xonsh additionally completes based on the names of any
+executable files on your $PATH, alias keys, and full Bash completion for the
+commands themselves.
+
+xonsh also provides a mechanism by which the results of a tab completion can be
+customized (i.e., new completions can be generated, or a subset of the built-in
+completions can be ignored).
+
+This page details the internal structure of xonsh's completion system and
+includes instructions for implementing new tab completion functions.
+
+
+Structure
+==========
+
+xonsh's built-in completers live in the ``xonsh.completers`` package, and they
+are managed through an instance of ``OrderedDict`` (``__xonsh_completers__``)
+that maps unique identifiers to completion functions.
+
+When the "tab" key is pressed, xonsh loops over the completion functions in
+order, calling each one in turn until it reaches one that returns a non-empty
+set of completion for the current line.  This set is then displayed to the
+user.
+
+
+Listing Active Completers
+=========================
+
+A list of the active completers can be viewed by running the
+``completer list`` command.  This command will display names and descriptions
+of the currently-active completers, in the order in which they will be
+checked.
+
+
+Writing a New Completer
+=======================
+
+Completers are implemented as Python functions that take five arguments:
+
+* ``prefix``: the string to be matched (the last whitespace-separated token in the current line)
+* ``line``: a string representing the entire current line
+* ``begidx``: the index at which ``prefix`` starts in ``line``
+* ``endidx``: the index at which ``prefix`` ends in ``line``
+* ``ctx``: the current Python environment, as a dictionary mapping names to values
+
+This function should return a Python set of possible completions for ``prefix``
+in the current context.  If the completer should not be used in this case, it
+should return ``None`` or an empty set, which will cause xonsh to move on and
+try to use the next completer.
+
+Occasionally, completers will need to return a match that does not actually
+start with ``prefix``.  In this case, a completer should instead return a tuple
+``(completions, prefixlength)``, where ``completions`` is the set of
+appropriate completions, and ``prefixlength`` is the number of characters in
+``line`` that should be treated as part of the completion.
+
+The docstring of a completer should contain a brief description of its
+functionality, which will be displayed by ``completer list``.
+
+Three examples follow.  For more examples, see the source code of the completers
+xonsh actually uses, in the ``xonsh.completers`` module.
+
+.. code-block:: python
+
+    def dummy_completer(prefix, line, begidx, endidx, ctx):
+        '''
+        Completes everything with options "lou" and "carcolh",
+        regardless of the value of prefix.
+        '''
+        return {"lou", "carcolh"}
+    
+    def python_context_completer(prefix, line, begidx, endidx, ctx):
+        '''
+        Completes based on the names in the current Python environment
+        '''
+        return {i for i in ctx if i.startswith(prefix)}
+
+    def unbeliever_completer(prefix, line, begidx, endidx, ctx):
+        '''
+        Replaces "lou carcolh" with "snail" if tab is pressed after typing
+        "lou" and when typing "carcolh"
+        '''
+        if 'carcolh'.startswith(prefix) and line[:begidx].split()[-1] == 'lou':
+            return ({'snail'}, len('lou ') + len(prefix))
+
+
+Registering a Completer
+=======================
+
+Once you have created a completion function, you can add it to the list of
+active completers via the ``completer add`` command::
+
+    Usage:
+        completer add NAME FUNC [POS]
+
+``NAME`` is a unique name to use in the listing
+
+``FUNC`` is the name of a completer function to use.
+
+``POS`` (optional) is a position into the list of completers at which the new completer should be added.  It can be one of the following values:
+
+* ``"start"`` indicates that the completer should be added to the start of the list of completers (it should be run before all others)
+* ``"end"`` indicates that the completer should be added to the end of the list of completers (it should be run after all others)
+* ``">KEY"``, where ``KEY`` is a pre-existing name, indicates that this should be added after the completer named ``KEY``
+* ``"<KEY"``, where ``KEY`` is a pre-existing name, indicates that this should be added before the completer named ``KEY``
+
+If ``POS`` is not provided, it defaults to ``"start"``.
+
+.. note:: It is also possible to manipulate ``__xonsh_completers__`` directly,
+          but this is the preferred method.
+
+Removing a Completer
+====================
+
+To remove a completer from the list of active completers, run
+``completer remove NAME``, where ``NAME`` is the unique identifier associated
+with the completer you wish to remove.

docs/tutorial_hist.rst

@@ -115,6 +115,33 @@ only the even indices from above, you could write:
      0  1 + 1
      2  history
 
+The ``xonsh`` action displays the past inputs along with the index from all
+valid json files found in ``XONSH_DATA_DIR``. As such, this operates on all
+past and present xonsh sessions.
+
+The ``all`` action is an alias for ``xonsh``.
+
+The ``xonsh`` action accepts the same arguments as ``show``.
+
+The ``zsh`` action will display all history from the history file specified
+by the ``HISTFILE`` environmental variable in zsh.
+By default this is ``~/.zsh_history``. However, they can also be respectively
+specified in both ``~/.zshrc`` and ``~/.zprofile``. Xonsh will parse these files
+(rc file first) to check if ``HISTFILE`` has been set.
+
+
+The ``bash`` action will display all history from the history file specified
+by the ``HISTFILE`` environmental variable in bash.
+By default this is ``~/.bash_history``. However, they can also be respectively
+specified in both ``~/.bashrc`` and ``~/.bash_profile``. Xonsh will parse these
+files (rc file first) to check if ``HISTFILE`` has been set.
+
+The ``__xonsh_history__.show(action)`` method can be returns history a list
+with each item in the format: (name, start_time, index). The action parameter
+can be a string of the following types: ``session``, ``show``, ``all``,
+``xonsh``,``zsh``, ``bash``, where ``session`` is an alias of ``show`` and
+``all`` is an alias of ``xonsh``.
+
 In the future, ``show`` may also be used to display outputs, return values, and time stamps.
 But the default behavior will remain as shown here.
 

docs/tutorial_xontrib.rst

@@ -1,7 +1,7 @@
 .. _tutorial_xontrib:
 
 ************************************
-Tutorial: Extensions
+Tutorial: Extensions (Xontribs)
 ************************************
 Take a deep breath and prepare for some serious Show & Tell; it's time to
 learn about xonsh extensions!
@@ -222,4 +222,4 @@ Note that you can have as many entries in the ``"install"`` dict as you
 want. Also, the keys are arbitrary labels, so feel free to pick whatever
 you want.
 
-Go forth!
+Go forth!

docs/windows.rst

@@ -20,7 +20,8 @@ Install xonsh with the following command:
 
 .. code-block:: bat
 
-   > conda install xonsh --channel conda-forge
+   > conda config --add channels conda-forge
+   > conda install xonsh
 
 .. note:: For the bleeding edge development version use ``conda install -c xonsh/channel/dev xonsh``
 

docs/xonshconfig.rst

@@ -50,7 +50,7 @@ dictionaries have the following structure:
     ``default=false``
 :envcmd: *str, optional* - The command to generate environment output with.
     ``default="env"``
-:aliascmd: *str, optional* - The command to generate alais output with.
+:aliascmd: *str, optional* - The command to generate alias output with.
     ``default="alias"``
 :extra_args: *list of str, optional* - Addtional command line options to pass
     into the shell. ``default=[]``

docs/xontribs.rst

@@ -2,13 +2,13 @@ Xontribs
 ========
 The following lists known xonsh contributions (xontribs), a description of the
 xontrib, and how to get your hands on it. Once installed, these xontribs
-can be loaded into you session by adding them to the ``xontribs`` list in the
+can be loaded into your session by adding them to the ``xontribs`` list in the
 config file, or dynamically in your xonshrc file or on the command line using
 the ``xontrib`` command:
 
 .. code-block:: xonshcon
 
-    $ xontrib mpl xomg ...
+    >>> xontrib mpl xo ...
 
 .. See the xontrib tutorial for more information.
 

news/035.rst

@@ -0,0 +1,68 @@
+**Added:**
+
+* Tab completers can now raise ``StopIteration`` to prevent consideration of
+  remaining completers.
+* Added tab completer for the ``completer`` alias.
+* New ``Block`` and ``Functor`` context managers are now available as
+  part of the ``xonsh.contexts`` module.
+* ``Block`` provides support for turning a context body into a non-executing
+  list of string lines. This is implmement via a syntax tree transformation.
+  This is useful for creating remote execution tools that seek to prevent
+  local execution.
+* ``Functor`` is a subclass of the ``Block`` context manager that turns the
+  block into a callable object.  The function object is available via the
+  ``func()`` attribute.  However, the ``Functor`` instance is itself callable
+  and will dispatch to ``func()``.
+* New ``$VC_BRANCH_TIMEOUT`` environment variable is the time (in seconds)
+  of how long to spend attempting each individual version control branch
+  information command during ``$PROMPT`` formatting.  This allows for faster
+  prompt resolution and faster startup times.
+* New lazy methods added to CommandsCache allowing for testing and inspection
+  without the possibility of recomputing the cache.
+* ``!(command)`` is now usefully iterable, yielding lines of stdout
+* Added XonshCalledProcessError, which includes the relevant CompletedCommand.
+  Also handles differences between Py3.4 and 3.5 in CalledProcessError
+* Tab completion of paths now includes zsh-style path expansion (subsequence
+  matching), toggleable with ``$SUBSEQUENCE_PATH_COMPLETION``
+* Tab completion of paths now includes "fuzzy" matches that are accurate to
+  within a few characters, toggleable with ``$FUZZY_PATH_COMPLETION``
+* Provide ``$XONSH_SOURCE`` for scripts in the environment variables pointing to
+  the currently running script's path
+* Arguments '+' and '-' for the ``fg`` command (job control)
+* Provide ``$XONSH_SOURCE`` for scripts in the environment variables pointing to
+  the currently running script's path
+* ``!(command)`` is now usefully iterable, yielding lines of stdout
+* Added XonshCalledProcessError, which includes the relevant CompletedCommand.
+  Also handles differences between Py3.4 and 3.5 in CalledProcessError
+* XonshError and XonshCalledProcessError are now in builtins
+
+**Changed:**
+
+* Functions in ``Execer`` now take ``transform`` kwarg instead of
+  ``wrap_subproc``.
+* Provide ``$XONSH_SOURCE`` for scripts in the environment variables pointing to
+  the currently running script's path
+* XonshError and XonshCalledProcessError are now in builtins
+
+**Deprecated:** None
+
+
+**Removed:**
+
+* ``ensure_git()`` and ``ensure_hg()`` decorators removed.
+* ``call_hg_command()`` function removed.
+
+
+**Fixed:**
+
+* Strip leading space in commands passed using the "-c" switch
+* Fixed xonfig wizard failing on Windows due to colon in created filename.
+* Ensured that the prompt_toolkit shell functions, even without a ``completer``
+  attribute.
+* Fixed crash resulting from malformed ``$PROMPT`` or ``$TITLE``.
+* xonsh no longer backgrounds itself after every command on Cygwin.
+* Fixed an issue about ``os.killpg()`` on Cygwin which caused xonsh to crash
+  occasionally
+* Fix crash on startup when Bash Windows Subsystem for Linux is on the Path.
+
+**Security:** None

news/TEMPLATE.rst

@@ -0,0 +1,11 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

news/adqm-anon_aliases.rst

@@ -0,0 +1,15 @@
+**Added:** None
+
+**Changed:**
+
+* ``@()`` now passes through functions as well as strings, which allows for the
+  use of anonymous aliases and aliases not explicitly added to the ``aliases``
+  mapping.
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

news/adqm-more_search.rst

@@ -0,0 +1,15 @@
+**Added:**
+
+* Normal globbing is now available in Python mode via ``g````
+* Backticks were expanded to allow searching using arbitrary functions, via
+  ``@<func>````
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

news/amal.rst

@@ -0,0 +1,43 @@
+**Added:**
+
+* New amalgamate tool collapses modules inside of a package into a single
+  ``__amalgam__.py`` module. This tool glues together all of the code from the
+  modules in a package, finds and removes intra-package imports, makes all
+  non-package imports lazy, and adds hooks into the ``__init__.py``.
+  This helps makes initial imports of modules fast and decreases startup time.
+  Packages and sub-packages must be amalgamated separately.
+* New lazy and self-destructive module ``xonsh.lazyasd`` adds a suite of
+  classes for delayed creation of objects.
+
+    - A ``LazyObject`` won't be created until it has an attribute accessed.
+    - A ``LazyDict`` will load each value only when a key is accessed.
+    - A ``LazyBool`` will only be created when ``__bool__()`` is called.
+
+  Additionally, when fully loaded, the above objects will replace themselves
+  by name in the context that they were handed, thus derefenceing themselves.
+  This is useful for global variables that may be expensive to create,
+  should only be created once, and may not be used in any particular session.
+* New ``xon.sh`` script added for launching xonsh from a sh environment.
+  This should be used if the normal ``xonsh`` script does not work for
+  some reason.
+
+**Changed:**
+
+* ``$XONSH_DEBUG`` will now supress amalgamted imports. This usually needs to be
+  set in the calling environment or prior to *any* xonsh imports.
+* Restuctured ``xonsh.platform`` to be fully lazy.
+* Restuctured ``xonsh.ansi_colors`` to be fully lazy.
+* Ensured the ``pygments`` and ``xonsh.pyghooks`` are not imported until
+  actually needed.
+* Yacc parser is now loaded in a background thread.
+
+**Deprecated:** None
+
+**Removed:** None
+
+* The ``'console_scripts'`` option to setuptools has been removed. It was found
+  to cause slowdowns of over 150 ms on every startup.
+
+**Fixed:** None
+
+**Security:** None

news/clean-up-premain.rst

@@ -0,0 +1,14 @@
+**Added:** None
+
+**Changed:**
+
+* Cleaned up argument parsing in ``xonsh.main.premain`` by removing the
+  ``undo_args`` hack.
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

news/completer_list_bugs.rst

@@ -0,0 +1,13 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+* Fixed ``_list_completers`` such that it does not throw a ValueError if no completer is registered.
+* Fixed ``_list_completers`` such that it does not throw an AttributeError if a completer has no docstring.
+
+**Security:** None

news/fix-main-config-file.rst

@@ -0,0 +1,13 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+* Bug that caused command line argument ``--config-path`` to be ignored.
+
+**Security:** None

news/fix_alias.rst

@@ -0,0 +1,13 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:**
+
+* Fixed a problem with aliases not always beeing found.  
+
+**Security:** None

news/fix_pipeline.rst

@@ -0,0 +1,14 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:**
+
+* Fixed issue where input was directed to the last process in a pipeline,
+  rather than the first.
+
+**Security:** None

news/fix_xonfig.rst

@@ -0,0 +1,13 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:**
+
+* Bug where xonfig wizard can't find ENV docs
+
+**Security:** None

news/history-all.rst

@@ -0,0 +1,18 @@
+**Added:**
+
+- ``history session``
+- ``history xonsh``
+  ``history all``
+- ``history zsh``
+- ``history bash``
+- ``__xonsh_history__.show()``
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

news/issue_1233.rst

@@ -0,0 +1,13 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+* Bug that caused xonsh to break on startup when prompt-toolkit < 1.0.0.
+
+**Security:** None

news/path_expand.rst

@@ -0,0 +1,25 @@
+**Added:**
+
+* A new class, ``xonsh.tools.EnvPath`` has been added. This class implements a
+  ``MutableSequence`` object and overrides the ``__getitem__`` method so that
+  when its entries are requested (either explicitly or implicitly), variable
+  and user expansion is performed, and relative paths are resolved.
+  ``EnvPath`` accepts objects (or lists of objects) of ``str``, ``bytes`` or
+  ``pathlib.Path`` types.
+
+**Changed:**
+
+* All ``PATH``-like environment variables are now stored in an ``EnvPath``
+  object, so that non-absolute paths or paths containing environment variables
+  can be resolved properly.
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** 
+
+* Issue where ``xonsh`` did not expand user and environment variables in
+  ``$PATH``, forcing the user to add absolute paths.
+
+**Security:** None

news/path_tuple.rst

@@ -0,0 +1,13 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:**
+
+* Fixed ``xonsh.environ.locate_binary()`` to handle PATH variable are given as a tuple. 
+
+**Security:** None

news/pid.rst

@@ -0,0 +1,14 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:**
+
+* Fixed issue with setting and signaling process groups on Linux when the first
+  process is a function alias and has no pid.
+
+**Security:** None

news/quoted_path_completions.rst

@@ -0,0 +1,14 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:**
+
+* Fixed missing completions for ``cd`` and ```rmdir`` when directories had spaces 
+  in their names.
+
+**Security:** None

news/replace_prompt.rst

@@ -0,0 +1,15 @@
+**Added:** None
+
+**Changed:** 
+
+* On Windows the ``PROMPT`` environment variable is reset to `$P$G` before starting
+  subprocesses. This prevents the unformatted xonsh ``PROMPT`` tempalte from showing up 
+  when running batch files with ``ECHO ON```  
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

news/rmbash.rst

@@ -0,0 +1,22 @@
+**Added:**
+
+* ``xonsh.platform`` now has a new ``PATH_DEFAULT`` variable.
+
+**Changed:**
+
+* ``Env`` now guarantees that the ``$PATH`` is available and mutable when
+  initialized.
+
+**Deprecated:** None
+
+**Removed:**
+
+* Bash is no longer loaded by default as a foreign shell for initial
+  configuration. This was done to increase stock startup times. This
+  behaviour can be recovered by adding ``{"shell": "bash"}`` to your
+  ``"foreign_shells"`` in your config.json file. For more details,
+  see http://xon.sh/xonshconfig.html#foreign-shells
+
+**Fixed:** None
+
+**Security:** None

news/simplify_env_repr.rst

@@ -0,0 +1,14 @@
+**Added:** None
+
+**Changed:** 
+
+* ``__repr__`` on the environment only shows a short representation of the 
+  object instead of printing the whole environment dictionary
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

news/sorted_comp.rst

@@ -0,0 +1,13 @@
+**Added:** None
+
+**Changed:** 
+
+* Ignore case and leading a quotes when sorting completions 
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

news/source_cmd_prompt.rst

@@ -0,0 +1,14 @@
+**Added:** None
+
+**Changed:**
+
+* On Windows the ``PROMPT`` environment variable is reset to `$P$G` before
+  sourcing *.bat files.
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

news/vi_mode_change.rst

@@ -0,0 +1,14 @@
+**Added:** None
+
+**Changed:**
+
+* In ``VI_MODE``, the ``v`` key will enter character selection mode, not open
+  the editor.  ``Ctrl-X Ctrl-E`` will still open an editor in any mode
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

news/wizard_foreign_shell_prompt.rst

@@ -0,0 +1,13 @@
+**Added:** None
+
+**Changed:**
+
+* More informative prompt when configuring foreign shells in the wizard.
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

news/xonsh_darwin.rst

@@ -0,0 +1,13 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+* Bug preventing `xonsh` executable being installed on macOS.
+
+**Security:** None

release.xsh

@@ -1,4 +1,4 @@
-#!/usr/bin/env xonsh 
+#!/usr/bin/env xonsh
 """Release helper script for xonsh."""
 import os
 import re
@@ -18,30 +18,52 @@ def replace_in_file(pattern, new, fname):
     with open(fname, 'w') as f:
         f.write(upd)
 
-NEW_DEV = """
-Current Developments
-====================
-**Added:** None
 
-**Changed:** None
+NEWS = [os.path.join('news', f) for f in os.listdir('news')
+        if f != 'TEMPLATE.rst']
+NEWS_CATEGORIES = ['Added', 'Changed', 'Deprecated', 'Removed', 'Fixed',
+                   'Security']
+NEWS_RE = re.compile('\*\*({0}):\*\*'.format('|'.join(NEWS_CATEGORIES)),
+                     flags=re.DOTALL)
 
-**Deprecated:** None
-
-**Removed:** None
-
-**Fixed:** None
-
-**Security:** None
-""".strip()
+def merge_news():
+    """Reads news files and merges them."""
+    cats = {c: '' for c in NEWS_CATEGORIES}
+    for news in NEWS:
+        with open(news) as f:
+            raw = f.read()
+        raw = raw.strip()
+        parts = NEWS_RE.split(raw)
+        while len(parts) > 0 and parts[0] not in NEWS_CATEGORIES:
+            parts = parts[1:]
+        for key, val in zip(parts[::2], parts[1::2]):
+            val = val.strip()
+            if val == 'None':
+                continue
+            cats[key] += val + '\n'
+    for news in NEWS:
+        os.remove(news)
+    s = ''
+    for c in NEWS_CATEGORIES:
+        val = cats[c]
+        if len(val) == 0:
+            continue
+        s += '**' + c + ':**\n\n' + val + '\n\n'
+    return s
 
 def version_update(ver):
     """Updates version strings in relevant files."""
+    fnews = ('.. current developments\n\n'
+             'v{0}\n'
+             '====================\n\n'
+             '{1}')
+    news = merge_news()
+    news = fnews.format(ver, news)
     pnfs = [
-        ('__version__\s*=.*', "__version__ = '{0}'".format(ver), 
+        ('__version__\s*=.*', "__version__ = '{0}'".format(ver),
          ['xonsh', '__init__.py']),
         ('version:\s*', 'version: {0}.{{build}}'.format(ver), ['.appveyor.yml']),
-        ('\*\*\w+:\*\* None', '', ['CHANGELOG.rst']),
-        ('Current Developments', NEW_DEV + '\n\nv' + ver, ['CHANGELOG.rst']),
+        ('.. current developments', news, ['CHANGELOG.rst']),
       ]
     for p, n, f in pnfs:
         replace_in_file(p, n, os.path.join(*f))
@@ -97,10 +119,10 @@ class OnlyAction(Action):
 
 def main(args=None):
     parser = ArgumentParser('release')
-    parser.add_argument('--upstream', 
-                        default='git@github.com:scopatz/xonsh.git', 
+    parser.add_argument('--upstream',
+                        default='git@github.com:scopatz/xonsh.git',
                         help='upstream repo')
-    parser.add_argument('-b', '--branch', default='master', 
+    parser.add_argument('-b', '--branch', default='master',
                          help='branch to commit / push to.')
     for doer in DOERS:
         base = doer[3:].replace('_', '-')

requirements-tests.txt

@@ -2,3 +2,5 @@ ply
 nose
 prompt-toolkit
 pygments
+coverage
+codecov

requirements.txt

@@ -1 +0,0 @@
-numpydoc==0.5

scripts/xon.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+/usr/bin/env PYTHONUNBUFFERED=1 python3 -u -m xonsh $@

setup.py

@@ -33,9 +33,8 @@ try:
 except ImportError:
     HAVE_JUPYTER = False
 
-from xonsh import __version__ as XONSH_VERSION
 
-TABLES = ['xonsh/lexer_table.py', 'xonsh/parser_table.py']
+TABLES = ['xonsh/lexer_table.py', 'xonsh/parser_table.py', 'xonsh/__amalgam__.py']
 
 
 def clean_tables():
@@ -43,9 +42,12 @@ def clean_tables():
     for f in TABLES:
         if os.path.isfile(f):
             os.remove(f)
-            print('Remove ' + f)
+            print('Removed ' + f)
 
 
+os.environ['XONSH_DEBUG'] = '1'
+from xonsh import __version__ as XONSH_VERSION
+
 def build_tables():
     """Build the lexer/parser modules."""
     print('Building lexer and parser tables.')
@@ -53,10 +55,12 @@ def build_tables():
     from xonsh.parser import Parser
     Parser(lexer_table='lexer_table', yacc_table='parser_table',
            outputdir='xonsh')
+    import amalgamate
+    amalgamate.main(['amalgamate', '--debug=XONSH_DEBUG', 'xonsh'])
     sys.path.pop(0)
 
 
-def install_jupyter_hook(root=None):
+def install_jupyter_hook(prefix=None, root=None):
     """Make xonsh available as a Jupyter kernel."""
     if not HAVE_JUPYTER:
         print('Could not install Jupyter kernel spec, please install '
@@ -76,13 +80,16 @@ def install_jupyter_hook(root=None):
         with open(os.path.join(d, 'kernel.json'), 'w') as f:
             json.dump(spec, f, sort_keys=True)
         if 'CONDA_BUILD' in os.environ:
-            root = sys.prefix
+            prefix = sys.prefix
             if sys.platform == 'win32':
-                root = root.replace(os.sep, os.altsep)
-        print('Installing Jupyter kernel spec...')
+                prefix = prefix.replace(os.sep, os.altsep)
+        user = ('--user' in sys.argv)
+        print('Installing Jupyter kernel spec:')
+        print('  root: {0!r}'.format(root))
+        print('  prefix: {0!r}'.format(prefix))
+        print('  as user: {0}'.format(user))
         KernelSpecManager().install_kernel_spec(
-            d, 'xonsh', user=('--user' in sys.argv), replace=True,
-            prefix=root)
+            d, 'xonsh', user=user, replace=True, prefix=prefix)
 
 
 class xinstall(install):
@@ -90,7 +97,15 @@ class xinstall(install):
     def run(self):
         clean_tables()
         build_tables()
-        install_jupyter_hook(self.root if self.root else None)
+        # install Jupyter hook
+        root = self.root if self.root else None
+        prefix = self.prefix if self.prefix else None
+        try:
+            install_jupyter_hook(prefix=prefix, root=root)
+        except Exception:
+            import traceback
+            traceback.print_exc()
+            print('Installing Jupyter hook failed.')
         install.run(self)
 
 
@@ -136,7 +151,7 @@ if HAVE_SETUPTOOLS:
 
 def main():
     """The main entry point."""
-    if sys.version_info[0] < 3:
+    if sys.version_info[:2] < (3, 4):
         sys.exit('xonsh currently requires Python 3.4+')
     try:
         if '--name' not in sys.argv:
@@ -148,6 +163,11 @@ def main():
         pass
     with open(os.path.join(os.path.dirname(__file__), 'README.rst'), 'r') as f:
         readme = f.read()
+    scripts = ['scripts/xon.sh']
+    if sys.platform == 'win32':
+        scripts.append('scripts/xonsh.bat')
+    else:
+        scripts.append('scripts/xonsh')
     skw = dict(
         name='xonsh',
         description='A general purpose, Python-ish shell',
@@ -160,22 +180,29 @@ def main():
         url='https://github.com/scopatz/xonsh',
         platforms='Cross Platform',
         classifiers=['Programming Language :: Python :: 3'],
-        packages=['xonsh', 'xonsh.ptk', 'xonsh.parsers', 'xonsh.xoreutils', 'xontrib'],
+        packages=['xonsh', 'xonsh.ply', 'xonsh.ptk', 'xonsh.parsers',
+                  'xonsh.xoreutils', 'xontrib', 'xonsh.completers'],
         package_dir={'xonsh': 'xonsh', 'xontrib': 'xontrib'},
         package_data={'xonsh': ['*.json'], 'xontrib': ['*.xsh']},
-        cmdclass=cmdclass
+        cmdclass=cmdclass,
+        scripts=scripts,
         )
     if HAVE_SETUPTOOLS:
+        # WARNING!!! Do not use setuptools 'console_scripts'
+        # It validates the depenendcies (of which we have none) everytime the
+        # 'xonsh' command is run. This validation adds ~0.2 sec. to the startup
+        # time of xonsh - for every single xonsh run.  This prevents us from
+        # reaching the goal of a startup time of < 0.1 sec.  So never ever write
+        # the following:
+        #
+        #     'console_scripts': ['xonsh = xonsh.main:main'],
+        #
+        # END WARNING
         skw['entry_points'] = {
             'pygments.lexers': ['xonsh = xonsh.pyghooks:XonshLexer',
-                                'xonshcon = xonsh.pyghooks:XonshConsoleLexer',
-                               ],
-            'console_scripts': ['xonsh = xonsh.main:main'],
+                                'xonshcon = xonsh.pyghooks:XonshConsoleLexer'],
             }
         skw['cmdclass']['develop'] = xdevelop
-    else:
-        skw['scripts'] = ['scripts/xonsh'] if 'win' not in sys.platform else ['scripts/xonsh.bat'],
-
     setup(**skw)
 
 

tests/test_aliases.py

@@ -85,7 +85,7 @@ class TestWhich:
         arg = 'whichtestapp1'
         matches = list(_which.whichgen(arg, path=[testdir]))
         assert len(matches) == 1
-        assert self._file_match(matches[0], os.path.join(testdir, arg))
+        assert self._file_match(matches[0][0], os.path.join(testdir, arg))
 
     def test_whichgen_failure(self):
         testdir = self.testdirs[0].name
@@ -108,8 +108,8 @@ class TestWhich:
         arg = 'whichtestapp1'
         matches = list(_which.whichgen(arg, path=[testdir0, testdir1]))
         assert len(matches) == 2
-        assert self._file_match(matches[0], os.path.join(testdir0, arg))
-        assert self._file_match(matches[1], os.path.join(testdir1, arg))
+        assert self._file_match(matches[0][0], os.path.join(testdir0, arg))
+        assert self._file_match(matches[1][0], os.path.join(testdir1, arg))
 
     if ON_WINDOWS:
         def test_whichgen_ext_failure(self):
@@ -123,7 +123,7 @@ class TestWhich:
                 arg = 'whichtestapp2'
                 matches = list(_which.whichgen(arg, path=[testdir], exts = ['.wta']))
                 assert len(matches) == 1
-                assert self._file_match(matches[0], os.path.join(testdir, arg))
+                assert self._file_match(matches[0][0], os.path.join(testdir, arg))
 
     def _file_match(self, path1, path2):
         if ON_WINDOWS:

tests/test_ast.py

@@ -0,0 +1,34 @@
+"""Xonsh AST tests."""
+from nose.tools import assert_equal
+
+from xonsh import ast
+from xonsh.ast import Tuple, Name, Store, min_line
+
+from tools import execer_setup, check_parse
+
+
+def setup():
+    execer_setup()
+
+
+def test_gather_names_name():
+    node = Name(id='y', ctx=Store())
+    exp = {'y'}
+    obs = ast.gather_names(node)
+    assert_equal(exp, obs)
+
+
+def test_gather_names_tuple():
+    node = Tuple(elts=[Name(id='y', ctx=Store()),
+                       Name(id='z', ctx=Store())])
+    exp = {'y', 'z'}
+    obs = ast.gather_names(node)
+    assert_equal(exp, obs)
+
+
+def test_multilline_num():
+    code = ('x = 1\n'
+            'ls -l\n')  # this second line wil be transformed
+    tree = check_parse(code)
+    lsnode = tree.body[1]
+    assert_equal(2, min_line(lsnode))

tests/test_builtins.py

@@ -9,8 +9,9 @@ from nose.plugins.skip import SkipTest
 from nose.tools import assert_equal, assert_true, assert_not_in
 
 from xonsh import built_ins
-from xonsh.built_ins import reglob, regexpath, helper, superhelper, \
-    ensure_list_of_strs, expand_case_matching
+from xonsh.built_ins import reglob, pathsearch, helper, superhelper, \
+    ensure_list_of_strs, list_of_strs_or_callables, regexsearch, \
+    globsearch
 from xonsh.environ import Env
 from xonsh.tools import ON_WINDOWS
 
@@ -32,7 +33,7 @@ def test_repath_backslash():
         exp = os.listdir(home)
         exp = {p for p in exp if re.match(r'\w\w.*', p)}
         exp = {os.path.join(home, p) for p in exp}
-        obs = set(regexpath(r'~/\w\w.*'))
+        obs = set(pathsearch(regexsearch, r'~/\w\w.*'))
         assert_equal(exp, obs)
 
 def test_repath_home_itself():
@@ -41,7 +42,7 @@ def test_repath_home_itself():
     exp = os.path.expanduser('~')
     built_ins.ENV = Env(HOME=exp)
     with mock_xonsh_env(built_ins.ENV):
-        obs = regexpath('~')
+        obs = pathsearch(regexsearch, '~')
         assert_equal(1, len(obs))
         assert_equal(exp, obs[0])
 
@@ -53,7 +54,7 @@ def test_repath_home_contents():
     with mock_xonsh_env(built_ins.ENV):
         exp = os.listdir(home)
         exp = {os.path.join(home, p) for p in exp}
-        obs = set(regexpath('~/.*'))
+        obs = set(pathsearch(regexsearch, '~/.*'))
         assert_equal(exp, obs)
 
 def test_repath_home_var():
@@ -62,7 +63,7 @@ def test_repath_home_var():
     exp = os.path.expanduser('~')
     built_ins.ENV = Env(HOME=exp)
     with mock_xonsh_env(built_ins.ENV):
-        obs = regexpath('$HOME')
+        obs = pathsearch(regexsearch, '$HOME')
         assert_equal(1, len(obs))
         assert_equal(exp, obs[0])
 
@@ -72,7 +73,7 @@ def test_repath_home_var_brace():
     exp = os.path.expanduser('~')
     built_ins.ENV = Env(HOME=exp)
     with mock_xonsh_env(built_ins.ENV):
-        obs = regexpath('${"HOME"}')
+        obs = pathsearch(regexsearch, '${"HOME"}')
         assert_equal(1, len(obs))
         assert_equal(exp, obs[0])
 
@@ -106,18 +107,13 @@ def test_ensure_list_of_strs():
         obs = ensure_list_of_strs(inp)
         yield assert_equal, exp, obs
 
-def test_expand_case_matching():
-    cases = {
-        'yo': '[Yy][Oo]',
-        '[a-f]123e': '[a-f]123[Ee]',
-        '${HOME}/yo': '${HOME}/[Yy][Oo]',
-        './yo/mom': './[Yy][Oo]/[Mm][Oo][Mm]',
-        'Eßen': '[Ee][Ss]?[Ssß][Ee][Nn]',
-        }
-    for inp, exp in cases.items():
-        obs = expand_case_matching(inp)
+def test_list_of_strs_or_callables():
+    f = lambda x: 20
+    cases = [(['yo'], 'yo'), (['yo'], ['yo']), (['42'], 42), (['42'], [42]),
+             ([f], f), ([f], [f])]
+    for exp, inp in cases:
+        obs = list_of_strs_or_callables(inp)
         yield assert_equal, exp, obs
 
-
 if __name__ == '__main__':
     nose.runmodule()

tests/test_contexts.py

@@ -0,0 +1,340 @@
+"""Tests xonsh contexts."""
+from nose.tools import assert_equal, assert_is, assert_is_not
+
+from tools import (mock_xonsh_env, execer_setup, check_exec, check_eval,
+    check_parse, skip_if)
+
+from xonsh.contexts import Block, Functor
+
+#
+# helpers
+#
+
+def setup():
+    execer_setup()
+
+
+X1_WITH = ('x = 1\n'
+           'with Block() as b:\n')
+SIMPLE_WITH = 'with Block() as b:\n'
+FUNC_WITH = ('x = 1\n'
+             'def func():\n'
+             '    y = 1\n'
+             '    with Block() as b:\n'
+             '{body}'
+             '    y += 1\n'
+             '    return b\n'
+             'x += 1\n'
+             'rtn = func()\n'
+             'x += 1\n')
+
+FUNC_OBSG = {'x': 3}
+FUNC_OBSL = {'y': 1}
+
+def block_checks_glb(name, glbs, body, obs=None):
+    block = glbs[name]
+    obs = obs or {}
+    for k, v in obs.items():
+        yield assert_equal, v, glbs[k]
+    if isinstance(body, str):
+        body = body.splitlines()
+    yield assert_equal, body, block.lines
+    yield assert_is, glbs, block.glbs
+    yield assert_is, None, block.locs
+
+
+def block_checks_func(name, glbs, body, obsg=None, obsl=None):
+    block = glbs[name]
+    obsg = obsg or {}
+    for k, v in obsg.items():
+        yield assert_equal, v, glbs[k]
+    if isinstance(body, str):
+        body = body.splitlines()
+    yield assert_equal, body, block.lines
+    yield assert_is, glbs, block.glbs
+    # local context tests
+    locs = block.locs
+    yield assert_is_not, None, locs
+    obsl = obsl or {}
+    for k, v in obsl.items():
+        yield assert_equal, v, locs[k]
+
+
+#
+# Block tests
+#
+
+def test_block_noexec():
+    s = ('x = 1\n'
+         'with Block():\n'
+         '    x += 42\n')
+    glbs = {'Block': Block}
+    check_exec(s, glbs=glbs, locs=None)
+    assert_equal(1, glbs['x'])
+
+
+def test_block_oneline():
+    body = '    x += 42\n'
+    s = X1_WITH + body
+    glbs = {'Block': Block}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_glb('b', glbs, body, {'x': 1})
+
+
+def test_block_manylines():
+    body = ('    ![echo wow mom]\n'
+            '# bad place for a comment\n'
+            '    x += 42')
+    s = X1_WITH + body
+    glbs = {'Block': Block}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_glb('b', glbs, body, {'x': 1})
+
+
+def test_block_leading_comment():
+    # leading comments do not show up in block lines
+    body = ('    # I am a leading comment\n'
+            '    x += 42\n')
+    s = X1_WITH + body
+    glbs = {'Block': Block}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_glb('b', glbs, ['    x += 42'], {'x': 1})
+
+
+def test_block_trailing_comment():
+    # trailing comments do not show up in block lines
+    body = ('    x += 42\n'
+            '    # I am a trailing comment\n')
+    s = X1_WITH + body
+    glbs = {'Block': Block}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_glb('b', glbs, ['    x += 42'], {'x': 1})
+
+
+def test_block_trailing_line_continuation():
+    body = ('    x += \\\n'
+            '         42\n')
+    s = X1_WITH + body
+    glbs = {'Block': Block}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_glb('b', glbs, body, {'x': 1})
+
+
+def test_block_trailing_close_paren():
+    body = ('    x += int("42"\n'
+            '             )\n')
+    s = X1_WITH + body
+    glbs = {'Block': Block}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_glb('b', glbs, body, {'x': 1})
+
+
+def test_block_trailing_close_many():
+    body = ('    x = {None: [int("42"\n'
+            '                    )\n'
+            '                ]\n'
+            '         }\n')
+    s = SIMPLE_WITH + body
+    glbs = {'Block': Block}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_glb('b', glbs, body)
+
+
+def test_block_trailing_triple_string():
+    body = ('    x = """This\n'
+            'is\n'
+            '"probably"\n'
+            '\'not\' what I meant.\n'
+            '"""\n')
+    s = SIMPLE_WITH + body
+    glbs = {'Block': Block}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_glb('b', glbs, body)
+
+
+def test_block_func_oneline():
+    body = '        x += 42\n'
+    s = FUNC_WITH.format(body=body)
+    glbs = {'Block': Block}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_func('rtn', glbs, body, FUNC_OBSG, FUNC_OBSL)
+
+
+def test_block_func_manylines():
+    body = ('        ![echo wow mom]\n'
+            '# bad place for a comment\n'
+            '        x += 42\n')
+    s = FUNC_WITH.format(body=body)
+    glbs = {'Block': Block}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_func('rtn', glbs, body, FUNC_OBSG, FUNC_OBSL)
+
+
+def test_block_func_leading_comment():
+    # leading comments do not show up in block lines
+    body = ('        # I am a leading comment\n'
+            '        x += 42\n')
+    s = FUNC_WITH.format(body=body)
+    glbs = {'Block': Block}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_func('rtn', glbs, '        x += 42\n',
+                                 FUNC_OBSG, FUNC_OBSL)
+
+
+def test_block_func_trailing_comment():
+    # trailing comments do not show up in block lines
+    body = ('        x += 42\n'
+            '        # I am a trailing comment\n')
+    s = FUNC_WITH.format(body=body)
+    glbs = {'Block': Block}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_func('rtn', glbs, '        x += 42\n',
+                                 FUNC_OBSG, FUNC_OBSL)
+
+
+def test_blockfunc__trailing_line_continuation():
+    body = ('        x += \\\n'
+            '             42\n')
+    s = FUNC_WITH.format(body=body)
+    glbs = {'Block': Block}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_func('rtn', glbs, body, FUNC_OBSG, FUNC_OBSL)
+
+
+def test_block_func_trailing_close_paren():
+    body = ('        x += int("42"\n'
+            '                 )\n')
+    s = FUNC_WITH.format(body=body)
+    glbs = {'Block': Block}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_func('rtn', glbs, body, FUNC_OBSG, FUNC_OBSL)
+
+
+def test_block_func_trailing_close_many():
+    body = ('        x = {None: [int("42"\n'
+            '                        )\n'
+            '                    ]\n'
+            '             }\n')
+    s = FUNC_WITH.format(body=body)
+    glbs = {'Block': Block}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_func('rtn', glbs, body, FUNC_OBSG, FUNC_OBSL)
+
+
+def test_block_func_trailing_triple_string():
+    body = ('        x = """This\n'
+            'is\n'
+            '"probably"\n'
+            '\'not\' what I meant.\n'
+            '"""\n')
+    s = FUNC_WITH.format(body=body)
+    glbs = {'Block': Block}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_func('rtn', glbs, body, FUNC_OBSG, FUNC_OBSL)
+
+
+#
+# Functor tests
+#
+
+X2_WITH = ('x = 1\n'
+           'with Functor() as f:\n'
+           '{body}'
+           'x += 1\n'
+           '{calls}\n'
+           )
+
+def test_functor_oneline_onecall_class():
+    body = ('    global x\n'
+            '    x += 42\n')
+    calls = 'f()'
+    s = X2_WITH.format(body=body, calls=calls)
+    glbs = {'Functor': Functor}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_glb('f', glbs, body, {'x': 44})
+
+
+def test_functor_oneline_onecall_func():
+    body = ('    global x\n'
+            '    x += 42\n')
+    calls = 'f.func()'
+    s = X2_WITH.format(body=body, calls=calls)
+    glbs = {'Functor': Functor}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_glb('f', glbs, body, {'x': 44})
+
+
+def test_functor_oneline_onecall_both():
+    body = ('    global x\n'
+            '    x += 42\n')
+    calls = 'f()\nf.func()'
+    s = X2_WITH.format(body=body, calls=calls)
+    glbs = {'Functor': Functor}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_glb('f', glbs, body, {'x': 86})
+
+
+XA_WITH = ('x = [1]\n'
+           'with Functor() as f:\n'
+           '{body}'
+           'x.append(2)\n'
+           '{calls}\n'
+           )
+
+def test_functor_oneline_append():
+    body = '    x.append(3)\n'
+    calls = 'f()\n'
+    s = XA_WITH.format(body=body, calls=calls)
+    glbs = {'Functor': Functor}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_glb('f', glbs, body, {'x': [1, 2, 3]})
+
+
+def test_functor_return():
+    body = '    x = 42\n'
+    t = ('res = 0\n'
+         'with Functor(rtn="x") as f:\n'
+         '{body}\n'
+         'res = f()\n')
+    s = t.format(body=body)
+    glbs = {'Functor': Functor}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_glb('f', glbs, body, {'res': 42})
+
+
+def test_functor_args():
+    body = '    x = 42 + a\n'
+    t = ('res = 0\n'
+         'with Functor(args=("a",), rtn="x") as f:\n'
+         '{body}\n'
+         'res = f(2)\n')
+    s = t.format(body=body)
+    glbs = {'Functor': Functor}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_glb('f', glbs, body, {'res': 44})
+
+
+def test_functor_kwargs():
+    body = '    x = 42 + a + b\n'
+    t = ('res = 0\n'
+         'with Functor(kwargs={{"a": 1, "b": 12}}, rtn="x") as f:\n'
+         '{body}\n'
+         'res = f(b=6)\n')
+    s = t.format(body=body)
+    glbs = {'Functor': Functor}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_glb('f', glbs, body, {'res': 49})
+
+
+def test_functor_fullsig():
+    body = '    x = 42 + a + b + c\n'
+    t = ('res = 0\n'
+         'with Functor(args=("c",), kwargs={{"a": 1, "b": 12}}, rtn="x") as f:\n'
+         '{body}\n'
+         'res = f(55)\n')
+    s = t.format(body=body)
+    glbs = {'Functor': Functor}
+    check_exec(s, glbs=glbs, locs=None)
+    yield from block_checks_glb('f', glbs, body, {'res': 110})
+
+

tests/test_dirstack.py

@@ -35,20 +35,21 @@ def xonsh_env(env):
 
 def test_simple():
     load_builtins()
-    with chdir(PARENT):
-        assert_not_equal(os.getcwd(), HERE)
-        dirstack.cd(["tests"])
-        assert_equal(os.getcwd(), HERE)
+    with xonsh_env(Env(CDPATH=PARENT, PWD=PARENT)):
+        with chdir(PARENT):
+            assert_not_equal(os.getcwd(), HERE)
+            dirstack.cd(["tests"])
+            assert_equal(os.getcwd(), HERE)
 
 def test_cdpath_simple():
-    with xonsh_env(Env(CDPATH=PARENT)):
+    with xonsh_env(Env(CDPATH=PARENT, PWD=HERE)):
         with chdir(os.path.normpath("/")):
             assert_not_equal(os.getcwd(), HERE)
             dirstack.cd(["tests"])
             assert_equal(os.getcwd(), HERE)
 
 def test_cdpath_collision():
-    with xonsh_env(Env(CDPATH=PARENT)):
+    with xonsh_env(Env(CDPATH=PARENT, PWD=HERE)):
         sub_tests = os.path.join(HERE, "tests")
         if not os.path.exists(sub_tests):
             os.mkdir(sub_tests)

tests/test_environ.py

@@ -4,12 +4,16 @@ from __future__ import unicode_literals, print_function
 import os
 import tempfile
 import builtins
+from tempfile import TemporaryDirectory
+from xonsh.tools import ON_WINDOWS
+
 
 import nose
 from nose.tools import (assert_equal, assert_true, assert_not_in,
     assert_is_instance, assert_in, assert_raises)
 
-from xonsh.environ import Env, format_prompt, load_static_config
+from xonsh.environ import (Env, format_prompt, load_static_config,
+    locate_binary, partial_format_prompt)
 
 from tools import mock_xonsh_env
 
@@ -18,23 +22,38 @@ def test_env_normal():
     assert_equal('wakka', env['VAR'])
 
 def test_env_path_list():
+    env = Env(MYPATH=['/home/wakka'])
+    assert_equal(['/home/wakka'], env['MYPATH'].paths)
     env = Env(MYPATH=['wakka'])
-    assert_equal(['wakka'], env['MYPATH'])
+    assert_equal(['wakka'], env['MYPATH'].paths)
 
 def test_env_path_str():
+    env = Env(MYPATH='/home/wakka' + os.pathsep + '/home/jawaka')
+    assert_equal(['/home/wakka', '/home/jawaka'], env['MYPATH'].paths)
     env = Env(MYPATH='wakka' + os.pathsep + 'jawaka')
-    assert_equal(['wakka', 'jawaka'], env['MYPATH'])
+    assert_equal(['wakka', 'jawaka'],
+                 env['MYPATH'].paths)
 
 def test_env_detype():
     env = Env(MYPATH=['wakka', 'jawaka'])
-    assert_equal({'MYPATH': 'wakka' + os.pathsep + 'jawaka'}, env.detype())
+    assert_equal('wakka' + os.pathsep + 'jawaka',
+                 env.detype()['MYPATH'])
 
 def test_env_detype_mutable_access_clear():
+    env = Env(MYPATH=['/home/wakka', '/home/jawaka'])
+    assert_equal('/home/wakka' + os.pathsep + '/home/jawaka',
+                 env.detype()['MYPATH'])
+    env['MYPATH'][0] = '/home/woah'
+    assert_equal(None, env._detyped)
+    assert_equal('/home/woah' + os.pathsep + '/home/jawaka',
+                 env.detype()['MYPATH'])
     env = Env(MYPATH=['wakka', 'jawaka'])
-    assert_equal({'MYPATH': 'wakka' + os.pathsep + 'jawaka'}, env.detype())
+    assert_equal('wakka' + os.pathsep + 'jawaka',
+                 env.detype()['MYPATH'])
     env['MYPATH'][0] = 'woah'
     assert_equal(None, env._detyped)
-    assert_equal({'MYPATH': 'woah' + os.pathsep + 'jawaka'}, env.detype())
+    assert_equal('woah' + os.pathsep + 'jawaka',
+                 env.detype()['MYPATH'])
 
 def test_env_detype_no_dict():
     env = Env(YO={'hey': 42})
@@ -55,6 +74,37 @@ def test_format_prompt():
     for p, exp in cases.items():
         obs = format_prompt(template=p, formatter_dict=formatter_dict)
         yield assert_equal, exp, obs
+    for p, exp in cases.items():
+        obs = partial_format_prompt(template=p, formatter_dict=formatter_dict)
+        yield assert_equal, exp, obs
+
+def test_format_prompt_with_broken_template():
+    for p in ('{user', '{user}{hostname'):
+        assert_equal(partial_format_prompt(p), p)
+        assert_equal(format_prompt(p), p)
+
+    # '{{user' will be parsed to '{user'
+    for p in ('{{user}', '{{user'):
+        assert_in('user', partial_format_prompt(p))
+        assert_in('user', format_prompt(p))
+
+def test_format_prompt_with_broken_template_in_func():
+    for p in (
+        lambda: '{user',
+        lambda: '{{user',
+        lambda: '{{user}',
+        lambda: '{user}{hostname',
+    ):
+        # '{{user' will be parsed to '{user'
+        assert_in('user', partial_format_prompt(p))
+        assert_in('user', format_prompt(p))
+
+def test_format_prompt_with_invalid_func():
+    def p():
+        foo = bar  # raises exception
+        return '{user}'
+    assert_is_instance(partial_format_prompt(p), str)
+    assert_is_instance(format_prompt(p), str)
 
 def test_HISTCONTROL():
     env = Env(HISTCONTROL=None)
@@ -105,11 +155,13 @@ def test_swap():
     assert 'VAR3' not in env
 
 def check_load_static_config(s, exp, loaded):
-    env = {'XONSH_SHOW_TRACEBACK': False}
-    with tempfile.NamedTemporaryFile() as f, mock_xonsh_env(env):
+    env = Env({'XONSH_SHOW_TRACEBACK': False})
+    f = tempfile.NamedTemporaryFile(delete=False)
+    with mock_xonsh_env(env):
         f.write(s)
-        f.flush()
+        f.close()
         conf = load_static_config(env, f.name)
+    os.unlink(f.name)
     assert_equal(exp, conf)
     assert_equal(env['LOADED_CONFIG'], loaded)
 
@@ -125,6 +177,23 @@ def test_load_static_config_json_fail():
     s = b'{"best": "awash"'
     check_load_static_config(s, {}, False)
 
+if ON_WINDOWS:
+    def test_locate_binary_on_windows():
+        files = ('file1.exe', 'FILE2.BAT', 'file3.txt')
+        with TemporaryDirectory() as tmpdir:
+            for fname in files:
+                fpath = os.path.join(tmpdir, fname)
+                with open(fpath, 'w') as f:
+                    f.write(fpath)
+            env = Env({'PATH': [tmpdir], 'PATHEXT': ['.COM', '.EXE', '.BAT']})
+            with mock_xonsh_env(env):
+                assert_equal( locate_binary('file1'), os.path.join(tmpdir,'file1.exe'))
+                assert_equal( locate_binary('file1.exe'), os.path.join(tmpdir,'file1.exe'))
+                assert_equal( locate_binary('file2'), os.path.join(tmpdir,'FILE2.BAT'))
+                assert_equal( locate_binary('file2.bat'), os.path.join(tmpdir,'FILE2.BAT'))
+                assert_equal( locate_binary('file3'), None)
+
+
 
 if __name__ == '__main__':
     nose.runmodule()

tests/test_execer.py

@@ -10,53 +10,24 @@ from nose.tools import assert_raises
 from xonsh.execer import Execer
 from xonsh.tools import ON_WINDOWS
 
-from tools import mock_xonsh_env
-
-DEBUG_LEVEL = 0
-EXECER = None
-
-#
-# Helpers
-#
+from tools import (mock_xonsh_env, execer_setup, check_exec, check_eval,
+    check_parse, skip_if)
 
 def setup_module():
-    # only setup one parser
-    global EXECER
-    EXECER = Execer(debug_level=DEBUG_LEVEL)
+    execer_setup()
 
-def check_exec(input):
-    with mock_xonsh_env(None):
-        if not input.endswith('\n'):
-            input += '\n'
-        EXECER.debug_level = DEBUG_LEVEL
-        EXECER.exec(input)
+@skip_if(not ON_WINDOWS)
+def test_win_ipconfig():
+    yield (check_eval,
+           os.environ['SYSTEMROOT'] + '\\System32\\ipconfig.exe /all')
 
-def check_eval(input):
-    with mock_xonsh_env({'AUTO_CD': False, 'XONSH_ENCODING' :'utf-8',
-                         'XONSH_ENCODING_ERRORS': 'strict', 'PATH': []}):
-        EXECER.debug_level = DEBUG_LEVEL
-        EXECER.eval(input)
+@skip_if(not ON_WINDOWS)
+def test_ipconfig():
+    yield check_eval, 'ipconfig /all'
 
-def check_parse(input):
-    with mock_xonsh_env(None):
-        EXECER.debug_level = DEBUG_LEVEL
-        EXECER.parse(input, ctx=None)
-
-#
-# Tests
-#
-
-if ON_WINDOWS:
-    def test_win_ipconfig():
-        yield (check_eval,
-               os.environ['SYSTEMROOT'] + '\\System32\\ipconfig.exe /all')
-
-    def test_ipconfig():
-        yield check_eval, 'ipconfig /all'
-
-else:
-    def test_bin_ls():
-        yield check_eval, '/bin/ls -l'
+@skip_if(ON_WINDOWS)
+def test_bin_ls():
+    yield check_eval, '/bin/ls -l'
 
 def test_ls_dashl():
     yield check_parse, 'ls -l'
@@ -72,6 +43,17 @@ def test_simple_func():
             "    return '{user}'.format(user='me')\n")
     yield check_parse, code
 
+def test_lookup_alias():
+    code = (
+        'def foo(a,  s=None):\n'
+        '    return "bar"\n'
+        '@(foo)\n')
+    yield check_parse, code
+
+def test_lookup_anon_alias():
+    code = ('echo "hi" | @(lambda a, s=None: a[0]) foo bar baz\n')
+    yield check_parse, code
+
 def test_simple_func_broken():
     code = ('def prompt():\n'
             "    return '{user}'.format(\n"
@@ -83,6 +65,16 @@ def test_bad_indent():
             'x = 1\n')
     assert_raises(SyntaxError, check_parse, code)
 
+def test_good_rhs_subproc():
+    # nonsense but parsebale
+    code = 'str().split() | ![grep exit]\n'
+    check_parse(code)
+
+def test_bad_rhs_subproc():
+    # nonsense but unparsebale
+    code = 'str().split() | grep exit\n'
+    assert_raises(SyntaxError, check_parse, code)
+
 def test_indent_with_empty_line():
     code = ('if True:\n'
             '\n'
@@ -99,6 +91,22 @@ def test_command_in_func_with_comment():
             '    echo hello # comment\n')
     yield check_parse, code
 
+def test_pyeval_redirect():
+    code = 'echo @("foo") > bar\n'
+    yield check_parse, code
+
+def test_echo_comma():
+    code = 'echo ,\n'
+    yield check_parse, code
+
+def test_echo_comma_val():
+    code = 'echo ,1\n'
+    yield check_parse, code
+
+def test_echo_comma_2val():
+    code = 'echo 1,2\n'
+    yield check_parse, code
+
 
 
 if __name__ == '__main__':

tests/test_history.py

@@ -94,13 +94,13 @@ def test_show_cmd():
 
     def format_hist_line(idx, cmd):
         """Construct a history output line."""
-        return ' {:d}  {:s}\n'.format(idx, cmd)
+        return ' {:d}: {:s}\n'.format(idx, cmd)
 
     def run_show_cmd(hist_args, commands, base_idx=0, step=1):
         """Run and evaluate the output of the given show command."""
         stdout.seek(0, io.SEEK_SET)
         stdout.truncate()
-        history._main(hist, hist_args)
+        history._hist_main(hist, hist_args)
         stdout.seek(0, io.SEEK_SET)
         hist_lines = stdout.readlines()
         yield assert_equal, len(commands), len(hist_lines)
@@ -114,33 +114,35 @@ def test_show_cmd():
     sys.stdout = stdout
 
     with mock_xonsh_env({'HISTCONTROL': set()}):
-        for cmd in cmds:  # populate the shell history
-            hist.append({'inp': cmd, 'rtn': 0})
+        for ts,cmd in enumerate(cmds):  # populate the shell history
+            hist.append({'inp': cmd, 'rtn': 0, 'ts':(ts+1, ts+1.5)})
 
-        # Verify an implicit "show" emits the entire history.
+        # Verify an implicit "show" emits show history
         for x in run_show_cmd([], cmds):
             yield x
 
-        # Verify an explicit "show" with no qualifiers emits the entire history.
+        # Verify an explicit "show" with no qualifiers emits
+        # show history.
         for x in run_show_cmd(['show'], cmds):
             yield x
 
-        # Verify an explicit "show" with a reversed qualifier emits the entire
-        # history in reverse order.
+        # Verify an explicit "show" with a reversed qualifier
+        # emits show history in reverse order.
         for x in run_show_cmd(['show', '-r'], list(reversed(cmds)),
-                              len(cmds) - 1, -1):
+                                 len(cmds) - 1, -1):
             yield x
 
-        # Verify that showing a specific history entry relative to the start of the
-        # history works.
+        # Verify that showing a specific history entry relative to
+        # the start of the history works.
         for x in run_show_cmd(['show', '0'], [cmds[0]], 0):
             yield x
         for x in run_show_cmd(['show', '1'], [cmds[1]], 1):
             yield x
 
-        # Verify that showing a specific history entry relative to the end of the
-        # history works.
-        for x in run_show_cmd(['show', '-2'], [cmds[-2]], len(cmds) - 2):
+        # Verify that showing a specific history entry relative to
+        # the end of the history works.
+        for x in run_show_cmd(['show', '-2'], [cmds[-2]],
+                               len(cmds) - 2):
             yield x
 
         # Verify that showing a history range relative to the start of the
@@ -152,9 +154,11 @@ def test_show_cmd():
 
         # Verify that showing a history range relative to the end of the
         # history works.
-        for x in run_show_cmd(['show', '-2:'], cmds[-2:], len(cmds) - 2):
+        for x in run_show_cmd(['show', '-2:'], 
+                               cmds[-2:], len(cmds) - 2):
             yield x
-        for x in run_show_cmd(['show', '-4:-2'], cmds[-4:-2], len(cmds) - 4):
+        for x in run_show_cmd(['show', '-4:-2'], 
+                               cmds[-4:-2], len(cmds) - 4):
             yield x
 
     sys.stdout = saved_stdout

tests/test_imphooks.py

@@ -25,23 +25,23 @@ def teardown():
         unload_builtins()
 
 def test_import():
-    with mock_xonsh_env({}):
+    with mock_xonsh_env({'PATH': []}):
         import sample
         assert_equal('hello mom jawaka\n', sample.x)
 
 def test_absolute_import():
-    with mock_xonsh_env({}):
+    with mock_xonsh_env({'PATH': []}):
         from xpack import sample
         assert_equal('hello mom jawaka\n', sample.x)
 
 def test_relative_import():
-    with mock_xonsh_env({}):
+    with mock_xonsh_env({'PATH': []}):
         from xpack import relimp
         assert_equal('hello mom jawaka\n', relimp.sample.x)
         assert_equal('hello mom jawaka\ndark chest of wonders', relimp.y)
 
 def test_sub_import():
-    with mock_xonsh_env({}):
+    with mock_xonsh_env({'PATH': []}):
         from xpack.sub import sample
         assert_equal('hello mom jawaka\n', sample.x)
 

tests/test_lazyjson.py

@@ -7,7 +7,7 @@ import nose
 from nose.tools import assert_equal, assert_is_instance
 assert_equal.__self__.maxDiff = None
 
-from xonsh.lazyjson import index, dump, LazyJSON, Node
+from xonsh.lazyjson import index, ljdump, LazyJSON, LJNode
 
 def test_index_int():
     exp = {'offsets': 0, 'sizes': 2}
@@ -63,7 +63,7 @@ def test_index_dict_dict_int():
 
 def test_lazy_load_index():
     f = StringIO()
-    dump({'wakka': 42}, f)
+    ljdump({'wakka': 42}, f)
     f.seek(0)
     lj = LazyJSON(f)
     assert_equal({'wakka': 10, '__total__': 0}, lj.offsets)
@@ -71,14 +71,14 @@ def test_lazy_load_index():
 
 def test_lazy_int():
     f = StringIO()
-    dump(42, f)
+    ljdump(42, f)
     f.seek(0)
     lj = LazyJSON(f)
     assert_equal(42, lj.load())
 
 def test_lazy_str():
     f = StringIO()
-    dump('wakka', f)
+    ljdump('wakka', f)
     f.seek(0)
     lj = LazyJSON(f)
     assert_equal('wakka', lj.load())
@@ -86,7 +86,7 @@ def test_lazy_str():
 def test_lazy_list_empty():
     x = []
     f = StringIO()
-    dump(x, f)
+    ljdump(x, f)
     f.seek(0)
     lj = LazyJSON(f)
     assert_equal(0, len(lj))
@@ -95,7 +95,7 @@ def test_lazy_list_empty():
 def test_lazy_list_ints():
     x = [0, 1, 6, 28, 496, 8128]
     f = StringIO()
-    dump(x, f)
+    ljdump(x, f)
     f.seek(0)
     lj = LazyJSON(f)
     assert_equal(28, lj[3])
@@ -106,7 +106,7 @@ def test_lazy_list_ints():
 def test_lazy_list_ints():
     x = [0, 1, 6, 28, 496, 8128]
     f = StringIO()
-    dump(x, f)
+    ljdump(x, f)
     f.seek(0)
     lj = LazyJSON(f)
     assert_equal(28, lj[3])
@@ -117,7 +117,7 @@ def test_lazy_list_ints():
 def test_lazy_list_str():
     x = ['I', 'have', 'seen', 'the', 'wind', 'blow']
     f = StringIO()
-    dump(x, f)
+    ljdump(x, f)
     f.seek(0)
     lj = LazyJSON(f)
     assert_equal('the', lj[3])
@@ -128,7 +128,7 @@ def test_lazy_list_str():
 def test_lazy_list_ints():
     x = [0, 1, 6, 28, 496, 8128]
     f = StringIO()
-    dump(x, f)
+    ljdump(x, f)
     f.seek(0)
     lj = LazyJSON(f)
     assert_equal(28, lj[3])
@@ -139,10 +139,10 @@ def test_lazy_list_ints():
 def test_lazy_list_list_ints():
     x = [[0, 1], [6, 28], [496, 8128]]
     f = StringIO()
-    dump(x, f)
+    ljdump(x, f)
     f.seek(0)
     lj = LazyJSON(f)
-    assert_is_instance(lj[1], Node)
+    assert_is_instance(lj[1], LJNode)
     assert_equal(28, lj[1][1])
     assert_equal([6, 28], lj[1].load())
     assert_equal(x, lj.load())
@@ -150,7 +150,7 @@ def test_lazy_list_list_ints():
 def test_lazy_dict_empty():
     x = {}
     f = StringIO()
-    dump(x, f)
+    ljdump(x, f)
     f.seek(0)
     lj = LazyJSON(f)
     assert_equal(0, len(lj))
@@ -158,7 +158,7 @@ def test_lazy_dict_empty():
 
 def test_lazy_dict():
     f = StringIO()
-    dump({'wakka': 42}, f)
+    ljdump({'wakka': 42}, f)
     f.seek(0)
     lj = LazyJSON(f)
     assert_equal(['wakka'], list(lj.keys()))
@@ -169,11 +169,11 @@ def test_lazy_dict():
 def test_lazy_dict_dict_int():
     x = {'wakka': {'jawaka': 42}}
     f = StringIO()
-    dump(x, f)
+    ljdump(x, f)
     f.seek(0)
     lj = LazyJSON(f)
     assert_equal(['wakka'], list(lj.keys()))
-    assert_is_instance(lj['wakka'], Node)
+    assert_is_instance(lj['wakka'], LJNode)
     assert_equal(42, lj['wakka']['jawaka'])
     assert_equal(1, len(lj))
     assert_equal(x, lj.load())

tests/test_lexer.py

@@ -134,12 +134,24 @@ def test_multiline():
            ('NAME', 'y', 0),]
     yield check_tokens, inp, exp
 
+def test_atdollar_expression():
+    inp = '@$(which python)'
+    exp = [('ATDOLLAR_LPAREN', '@$(', 0),
+           ('NAME', 'which', 3),
+           ('WS', ' ', 8),
+           ('NAME', 'python', 9),
+           ('RPAREN', ')', 15)]
+    yield check_tokens, inp, exp
+
 def test_and():
     yield check_token, 'and', ['AND', 'and', 0]
 
 def test_ampersand():
     yield check_token, '&', ['AMPERSAND', '&', 0]
 
+def test_atdollar():
+    yield check_token, '@$', ['ATDOLLAR', '@$', 0]
+
 def test_doubleamp():
     yield check_token, '&&', ['AND', 'and', 0]
 
@@ -176,6 +188,12 @@ def test_double_unicode_literal():
 def test_single_bytes_literal():
     yield check_token, "b'yo'", ['STRING', "b'yo'", 0]
 
+def test_regex_globs():
+    for i in ('.*', r'\d*', '.*#{1,2}'):
+        for p in ('', 'r', 'g', '@somethingelse'):
+            c = '{}`{}`'.format(p,i)
+            yield check_token, c, ['SEARCHPATH', c, 0]
+
 def test_float_literals():
     cases = ['0.0', '.0', '0.', '1e10', '1.e42', '0.1e42', '0.5e-42',
              '5E10', '5e+42']

tests/test_man.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 import os
+import tempfile
 
 import nose
 from nose.tools import assert_true
 from nose.plugins.skip import SkipTest
 
 from xonsh.tools import ON_WINDOWS
-from xonsh.completer import ManCompleter
+from xonsh.completers.man import complete_from_man
 
 from tools import mock_xonsh_env
 
@@ -29,11 +30,11 @@ def teardown():
 def test_man_completion():
     if ON_WINDOWS:
         raise SkipTest
-    with mock_xonsh_env({}):
-        man_completer = ManCompleter()
-        completions = man_completer.option_complete('--', 'yes')
-    assert_true('--version' in completions)
-    assert_true('--help' in completions)
+    with tempfile.TemporaryDirectory() as tempdir:
+        with mock_xonsh_env({'XONSH_DATA_DIR': tempdir}):
+            completions = complete_from_man('--', 'yes --', 4, 6, __xonsh_env__)
+        assert_true('--version' in completions)
+        assert_true('--help' in completions)
 
 
 if __name__ == '__main__':

tests/test_parser.py

@@ -1553,6 +1553,12 @@ def test_dollar_sub_space():
 def test_ls_dot():
     yield check_xonsh_ast, {}, '$(ls .)', False
 
+def test_lambda_in_atparens():
+    yield check_xonsh_ast, {}, '$(echo hello | @(lambda a, s=None: "hey!") foo bar baz)', False
+
+def test_nested_madness():
+    yield check_xonsh_ast, {}, '$(@$(which echo) ls | @(lambda a, s=None: $(@(s.strip()) @(a[1]))) foo -la baz)', False
+
 def test_ls_dot_nesting():
     yield check_xonsh_ast, {}, '$(ls @(None or "."))', False
 
@@ -1619,6 +1625,45 @@ def test_ls_regex():
 def test_backtick():
     yield check_xonsh_ast, {}, 'print(`.*`)', False
 
+def test_ls_regex_octothorpe():
+    yield check_xonsh_ast, {}, '$(ls `#[Ff]+i*LE` -l)', False
+
+def test_ls_explicitregex():
+    yield check_xonsh_ast, {}, '$(ls r`[Ff]+i*LE` -l)', False
+
+def test_rbacktick():
+    yield check_xonsh_ast, {}, 'print(r`.*`)', False
+
+def test_ls_explicitregex_octothorpe():
+    yield check_xonsh_ast, {}, '$(ls r`#[Ff]+i*LE` -l)', False
+
+def test_ls_glob():
+    yield check_xonsh_ast, {}, '$(ls g`[Ff]+i*LE` -l)', False
+
+def test_gbacktick():
+    yield check_xonsh_ast, {}, 'print(g`.*`)', False
+
+def test_ls_glob_octothorpe():
+    yield check_xonsh_ast, {}, '$(ls g`#[Ff]+i*LE` -l)', False
+
+def test_ls_customsearch():
+    yield check_xonsh_ast, {}, '$(ls @foo`[Ff]+i*LE` -l)', False
+
+def test_custombacktick():
+    yield check_xonsh_ast, {}, 'print(@foo`.*`)', False
+
+def test_ls_customsearch_octothorpe():
+    yield check_xonsh_ast, {}, '$(ls @foo`#[Ff]+i*LE` -l)', False
+
+def test_injection():
+    yield check_xonsh_ast, {}, '$[@$(which python)]', False
+
+def test_rhs_nested_injection():
+    yield check_xonsh_ast, {}, '$[ls @$(dirname @$(which python))]', False
+
+def test_backtick_octothorpe():
+    yield check_xonsh_ast, {}, 'print(`#.*`)', False
+
 def test_uncaptured_sub():
     yield check_xonsh_ast, {}, '$[ls]', False
 
@@ -1731,9 +1776,19 @@ def test_git_two_quotes_space_space():
 def test_ls_quotes_3_space():
     yield check_xonsh_ast, {}, '$[ls "wakka jawaka baraka"]', False
 
+def test_echo_comma():
+    yield check_xonsh_ast, {}, '![echo ,]', False
+
+def test_echo_internal_comma():
+    yield check_xonsh_ast, {}, '![echo 1,2]', False
+
 def test_comment_only():
     yield check_xonsh_ast, {}, '# hello'
 
+def test_echo_slash_question():
+    yield check_xonsh_ast, {}, '![echo /?]', False
+
+
 _error_names = {'e', 'err', '2'}
 _output_names = {'', 'o', 'out', '1'}
 _all_names = {'a', 'all', '&'}

tests/test_tools.py

@@ -1,51 +1,67 @@
 # -*- coding: utf-8 -*-
 """Tests the xonsh lexer."""
-from __future__ import unicode_literals, print_function
 import os
+import pathlib
+from tempfile import TemporaryDirectory
+import stat
 
 import nose
 from nose.tools import assert_equal, assert_true, assert_false
 
+from xonsh.platform import ON_WINDOWS
 from xonsh.lexer import Lexer
-from xonsh.tools import (subproc_toks, subexpr_from_unbalanced, is_int,
-    always_true, always_false, ensure_string, is_env_path, str_to_env_path,
-    env_path_to_str, escape_windows_cmd_string, is_bool, to_bool, bool_to_str,
-    ensure_int_or_slice, is_float, is_string, check_for_partial_string,
-    argvquote)
+
+from xonsh.tools import (
+    CommandsCache, EnvPath, always_false, always_true, argvquote,
+    bool_or_int_to_str, bool_to_str, check_for_partial_string,
+    dynamic_cwd_tuple_to_str, ensure_int_or_slice, ensure_string,
+    env_path_to_str, escape_windows_cmd_string, executables_in,
+    expand_case_matching, find_next_break, is_bool, is_bool_or_int,
+    is_callable, is_dynamic_cwd_width, is_env_path, is_float, is_int,
+    is_int_as_str, is_logfile_opt, is_slice_as_str, is_string,
+    is_string_or_callable, logfile_opt_to_str, str_to_env_path,
+    subexpr_from_unbalanced, subproc_toks, to_bool, to_bool_or_int,
+    to_dynamic_cwd_tuple, to_logfile_opt)
 
 LEXER = Lexer()
 LEXER.build()
 
 INDENT = '    '
 
+
 def test_subproc_toks_x():
     exp = '![x]'
     obs = subproc_toks('x', lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_ls_l():
     exp = '![ls -l]'
     obs = subproc_toks('ls -l', lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_git():
     s = 'git commit -am "hello doc"'
     exp = '![{0}]'.format(s)
     obs = subproc_toks(s, lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_git_semi():
     s = 'git commit -am "hello doc"'
     exp = '![{0}];'.format(s)
     obs = subproc_toks(s + ';', lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_git_nl():
     s = 'git commit -am "hello doc"'
     exp = '![{0}]\n'.format(s)
     obs = subproc_toks(s + '\n', lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_indent_ls():
     s = 'ls -l'
     exp = INDENT + '![{0}]'.format(s)
@@ -53,6 +69,7 @@ def test_subproc_toks_indent_ls():
                        returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_indent_ls_nl():
     s = 'ls -l'
     exp = INDENT + '![{0}]\n'.format(s)
@@ -60,30 +77,35 @@ def test_subproc_toks_indent_ls_nl():
                        returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_indent_ls_no_min():
     s = 'ls -l'
     exp = INDENT + '![{0}]'.format(s)
     obs = subproc_toks(INDENT + s, lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_indent_ls_no_min_nl():
     s = 'ls -l'
     exp = INDENT + '![{0}]\n'.format(s)
     obs = subproc_toks(INDENT + s + '\n', lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_indent_ls_no_min_semi():
     s = 'ls'
     exp = INDENT + '![{0}];'.format(s)
     obs = subproc_toks(INDENT + s + ';', lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_indent_ls_no_min_semi_nl():
     s = 'ls'
     exp = INDENT + '![{0}];\n'.format(s)
     obs = subproc_toks(INDENT + s + ';\n', lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_ls_comment():
     s = 'ls -l'
     com = '  # lets list'
@@ -91,6 +113,7 @@ def test_subproc_toks_ls_comment():
     obs = subproc_toks(s + com, lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_ls_42_comment():
     s = 'ls 42'
     com = '  # lets list'
@@ -98,6 +121,7 @@ def test_subproc_toks_ls_42_comment():
     obs = subproc_toks(s + com, lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_ls_str_comment():
     s = 'ls "wakka"'
     com = '  # lets list'
@@ -105,6 +129,7 @@ def test_subproc_toks_ls_str_comment():
     obs = subproc_toks(s + com, lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_indent_ls_comment():
     ind = '    '
     s = 'ls -l'
@@ -113,6 +138,7 @@ def test_subproc_toks_indent_ls_comment():
     obs = subproc_toks(ind + s + com, lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_indent_ls_str():
     ind = '    '
     s = 'ls "wakka"'
@@ -121,6 +147,7 @@ def test_subproc_toks_indent_ls_str():
     obs = subproc_toks(ind + s + com, lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_ls_l_semi_ls_first():
     lsdl = 'ls -l'
     ls = 'ls'
@@ -129,6 +156,7 @@ def test_subproc_toks_ls_l_semi_ls_first():
     obs = subproc_toks(s, lexer=LEXER, maxcol=6, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_ls_l_semi_ls_second():
     lsdl = 'ls -l'
     ls = 'ls'
@@ -137,6 +165,7 @@ def test_subproc_toks_ls_l_semi_ls_second():
     obs = subproc_toks(s, lexer=LEXER, mincol=7, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_hello_mom_first():
     fst = "echo 'hello'"
     sec = "echo 'mom'"
@@ -145,6 +174,7 @@ def test_subproc_toks_hello_mom_first():
     obs = subproc_toks(s, lexer=LEXER, maxcol=len(fst)+1, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_hello_mom_second():
     fst = "echo 'hello'"
     sec = "echo 'mom'"
@@ -153,58 +183,69 @@ def test_subproc_toks_hello_mom_second():
     obs = subproc_toks(s, lexer=LEXER, mincol=len(fst), returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_comment():
     exp = None
     obs = subproc_toks('# I am a comment', lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_not():
     exp = 'not ![echo mom]'
     obs = subproc_toks('not echo mom', lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_paren():
     exp = '(![echo mom])'
     obs = subproc_toks('(echo mom)', lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_paren_ws():
     exp = '(![echo mom])  '
     obs = subproc_toks('(echo mom)  ', lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_not_paren():
     exp = 'not (![echo mom])'
     obs = subproc_toks('not (echo mom)', lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_and_paren():
     exp = 'True and (![echo mom])'
     obs = subproc_toks('True and (echo mom)', lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_paren_and_paren():
     exp = '(![echo a]) and (echo b)'
     obs = subproc_toks('(echo a) and (echo b)', maxcol=9, lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_semicolon_only():
     exp = None
     obs = subproc_toks(';', lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_pyeval():
     s = 'echo @(1+1)'
     exp = '![{0}]'.format(s)
     obs = subproc_toks(s, lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_twopyeval():
     s = 'echo @(1+1) @(40 + 2)'
     exp = '![{0}]'.format(s)
     obs = subproc_toks(s, lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_pyeval_parens():
     s = 'echo @(1+1)'
     inp = '({0})'.format(s)
@@ -212,6 +253,7 @@ def test_subproc_toks_pyeval_parens():
     obs = subproc_toks(inp, lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_twopyeval_parens():
     s = 'echo @(1+1) @(40+2)'
     inp = '({0})'.format(s)
@@ -219,12 +261,14 @@ def test_subproc_toks_twopyeval_parens():
     obs = subproc_toks(inp, lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_pyeval_nested():
     s = 'echo @(min(1, 42))'
     exp = '![{0}]'.format(s)
     obs = subproc_toks(s, lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_pyeval_nested_parens():
     s = 'echo @(min(1, 42))'
     inp = '({0})'.format(s)
@@ -232,12 +276,14 @@ def test_subproc_toks_pyeval_nested_parens():
     obs = subproc_toks(inp, lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_capstdout():
     s = 'echo $(echo bat)'
     exp = '![{0}]'.format(s)
     obs = subproc_toks(s, lexer=LEXER, returnline=True)
     assert_equal(exp, obs)
 
+
 def test_subproc_toks_capproc():
     s = 'echo !(echo bat)'
     exp = '![{0}]'.format(s)
@@ -245,6 +291,14 @@ def test_subproc_toks_capproc():
     assert_equal(exp, obs)
 
 
+def test_subproc_toks_pyeval_redirect():
+    s = 'echo @("foo") > bar'
+    inp = '{0}'.format(s)
+    exp = '![{0}]'.format(s)
+    obs = subproc_toks(inp, lexer=LEXER, returnline=True)
+    assert_equal(exp, obs)
+
+
 def test_subexpr_from_unbalanced_parens():
     cases = [
         ('f(x.', 'x.'),
@@ -255,26 +309,122 @@ def test_subexpr_from_unbalanced_parens():
         obs = subexpr_from_unbalanced(expr, '(', ')')
         yield assert_equal, exp, obs
 
+def test_find_next_break():
+    cases = [
+        ('ls && echo a', 0, 4),
+        ('ls && echo a', 6, None),
+        ('ls && echo a || echo b', 6, 14),
+        ('(ls) && echo a', 1, 4),
+        ('not ls && echo a', 0, 8),
+        ('not (ls) && echo a', 0, 8),
+        ]
+    for line, mincol, exp in cases:
+        obs = find_next_break(line, mincol=mincol, lexer=LEXER)
+        yield assert_equal, exp, obs
+
+
 def test_is_int():
-    yield assert_true, is_int(42)
-    yield assert_false, is_int('42')
+    cases = [
+        (42, True),
+        (42.0, False),
+        ('42', False),
+        ('42.0', False),
+        ([42], False),
+        ([], False),
+        (None, False),
+        ('', False)
+        ]
+    for inp, exp in cases:
+        obs = is_int(inp)
+        yield assert_equal, exp, obs
+
+
+def test_is_int_as_str():
+    cases = [
+        ('42', True),
+        ('42.0', False),
+        (42, False),
+        ([42], False),
+        ([], False),
+        (None, False),
+        ('', False),
+        (False, False),
+        (True, False),
+        ]
+    for inp, exp in cases:
+        obs = is_int_as_str(inp)
+        yield assert_equal, exp, obs
+
 
 def test_is_float():
-    yield assert_true, is_float(42.0)
-    yield assert_false, is_float('42.0')
+    cases = [
+        (42.0, True),
+        (42.000101010010101010101001010101010001011100001101101011100, True),
+        (42, False),
+        ('42', False),
+        ('42.0', False),
+        ([42], False),
+        ([], False),
+        (None, False),
+        ('', False),
+        (False, False),
+        (True, False),
+        ]
+    for inp, exp in cases:
+        obs = is_float(inp)
+        yield assert_equal, exp, obs
+
+
+def test_is_slice_as_str():
+    cases = [
+        (42, False),
+        (None, False),
+        ('42', False),
+        ('-42', False),
+        (slice(1,2,3), False),
+        ([], False),
+        (False, False),
+        (True, False),
+        ('1:2:3', True),
+        ('1::3', True),
+        ('1:', True),
+        (':', True),
+        ('[1:2:3]', True),
+        ('(1:2:3)', True),
+        ('r', False),
+        ('r:11', False),
+        ]
+    for inp, exp in cases:
+        obs = is_slice_as_str(inp)
+        yield assert_equal, exp, obs
+
 
 def test_is_string():
     yield assert_true, is_string('42.0')
     yield assert_false, is_string(42.0)
 
+
+def test_is_callable():
+    yield assert_true, is_callable(lambda: 42.0)
+    yield assert_false, is_callable(42.0)
+
+
+def test_is_string_or_callable():
+    yield assert_true, is_string_or_callable('42.0')
+    yield assert_true, is_string_or_callable(lambda: 42.0)
+    yield assert_false, is_string(42.0)
+
+
 def test_always_true():
     yield assert_true, always_true(42)
     yield assert_true, always_true('42')
 
+
 def test_always_false():
     yield assert_false, always_false(42)
     yield assert_false, always_false('42')
 
+
 def test_ensure_string():
     cases = [
         (42, '42'),
@@ -284,24 +434,31 @@ def test_ensure_string():
         obs = ensure_string(inp)
         yield assert_equal, exp, obs
 
+
 def test_is_env_path():
     cases = [
         ('/home/wakka', False),
-        (['/home/jawaka'], True),
+        (['/home/jawaka'], False),
+        (EnvPath(['/home/jawaka']), True),
+        (EnvPath(['jawaka']), True),
+        (EnvPath(b'jawaka:wakka'), True),
         ]
     for inp, exp in cases:
         obs = is_env_path(inp)
         yield assert_equal, exp, obs
 
+
 def test_str_to_env_path():
     cases = [
         ('/home/wakka', ['/home/wakka']),
         ('/home/wakka' + os.pathsep + '/home/jawaka',
          ['/home/wakka', '/home/jawaka']),
+        (b'/home/wakka', ['/home/wakka']),
         ]
     for inp, exp in cases:
         obs = str_to_env_path(inp)
-        yield assert_equal, exp, obs
+        yield assert_equal, exp, obs.paths
+
 
 def test_env_path_to_str():
     cases = [
@@ -314,6 +471,110 @@ def test_env_path_to_str():
         yield assert_equal, exp, obs
 
 
+def test_env_path():
+    # lambda to expand the expected paths
+    expand = lambda path: os.path.expanduser(os.path.expandvars(path))
+    getitem_cases = [
+        ('xonsh_dir', 'xonsh_dir'),
+        ('.', '.'),
+        ('../', '../'),
+        ('~/', '~/'),
+        (b'~/../', '~/../'),
+    ]
+    for inp, exp in getitem_cases:
+        obs = EnvPath(inp)[0]  # call to __getitem__
+        yield assert_equal, expand(exp), obs
+
+    # cases that involve path-separated strings
+    multipath_cases = [
+        (os.pathsep.join(['xonsh_dir', '../', '.', '~/']),
+         ['xonsh_dir', '../', '.', '~/']),
+        ('/home/wakka' + os.pathsep + '/home/jakka' + os.pathsep + '~/',
+         ['/home/wakka', '/home/jakka', '~/'])
+    ]
+    for inp, exp in multipath_cases:
+        obs = [i for i in EnvPath(inp)]
+        yield assert_equal, [expand(i) for i in exp], obs
+
+    # cases that involve pathlib.Path objects
+    pathlib_cases = [
+        (pathlib.Path('/home/wakka'), ['/home/wakka'.replace('/',os.sep)]),
+        (pathlib.Path('~/'), ['~']),
+        (pathlib.Path('.'), ['.']),
+        (['/home/wakka', pathlib.Path('/home/jakka'), '~/'],
+         ['/home/wakka', '/home/jakka'.replace('/',os.sep), '~/']),
+        (['/home/wakka', pathlib.Path('../'), '../'],
+         ['/home/wakka', '..', '../']),
+        (['/home/wakka', pathlib.Path('~/'), '~/'],
+         ['/home/wakka', '~', '~/']),
+    ]
+
+    for inp, exp in pathlib_cases:
+        # iterate over EnvPath to acquire all expanded paths
+        obs = [i for i in EnvPath(inp)]
+        yield assert_equal, [expand(i) for i in exp], obs
+
+
+def test_env_path_slices():
+    # build os-dependent paths properly
+    mkpath = lambda *paths: os.sep + os.sep.join(paths)
+
+    # get all except the last element in a slice
+    slice_last = [
+        ([mkpath('home', 'wakka'),
+          mkpath('home', 'jakka'),
+          mkpath('home', 'yakka')],
+         [mkpath('home', 'wakka'),
+          mkpath('home', 'jakka')])]
+
+    for inp, exp in slice_last:
+        obs = EnvPath(inp)[:-1]
+        yield assert_equal, exp, obs
+
+    # get all except the first element in a slice
+    slice_first = [
+        ([mkpath('home', 'wakka'),
+          mkpath('home', 'jakka'),
+          mkpath('home', 'yakka')],
+         [mkpath('home', 'jakka'),
+          mkpath('home', 'yakka')])]
+
+    for inp, exp in slice_first:
+        obs = EnvPath(inp)[1:]
+        yield assert_equal, exp, obs
+
+    # slice paths with a step
+    slice_step = [
+        ([mkpath('home', 'wakka'),
+          mkpath('home', 'jakka'),
+          mkpath('home', 'yakka'),
+          mkpath('home', 'takka')],
+         [mkpath('home', 'wakka'),
+          mkpath('home', 'yakka')],
+         [mkpath('home', 'jakka'),
+          mkpath('home', 'takka')])]
+
+    for inp, exp_a, exp_b in slice_step:
+        obs_a = EnvPath(inp)[0::2]
+        yield assert_equal, exp_a, obs_a
+        obs_b = EnvPath(inp)[1::2]
+        yield assert_equal, exp_b, obs_b
+
+    # keep only non-home paths
+    slice_normal = [
+        ([mkpath('home', 'wakka'),
+          mkpath('home', 'xakka'),
+          mkpath('other', 'zakka'),
+          mkpath('another', 'akka'),
+          mkpath('home', 'bakka')],
+         [mkpath('other', 'zakka'),
+          mkpath('another', 'akka')])]
+
+    for inp, exp in slice_normal:
+        obs = EnvPath(inp)[2:4]
+        yield assert_equal, exp, obs
+
+
 def test_is_bool():
     yield assert_equal, True, is_bool(True)
     yield assert_equal, True, is_bool(False)
@@ -345,6 +606,51 @@ def test_bool_to_str():
     yield assert_equal, '', bool_to_str(False)
 
 
+def test_is_bool_or_int():
+    cases = [
+        (True, True),
+        (False, True),
+        (1, True),
+        (0, True),
+        ('Yolo', False),
+        (1.0, False),
+        ]
+    for inp, exp in cases:
+        obs = is_bool_or_int(inp)
+        yield assert_equal, exp, obs
+
+
+def test_to_bool_or_int():
+    cases = [
+        (True, True),
+        (False, False),
+        (1, 1),
+        (0, 0),
+        ('', False),
+        (0.0, False),
+        (1.0, True),
+        ('T', True),
+        ('f', False),
+        ('0', 0),
+        ('10', 10),
+        ]
+    for inp, exp in cases:
+        obs = to_bool_or_int(inp)
+        yield assert_equal, exp, obs
+
+
+def test_bool_or_int_to_str():
+    cases = [
+        (True, '1'),
+        (False, ''),
+        (1, '1'),
+        (0, '0'),
+        ]
+    for inp, exp in cases:
+        obs = bool_or_int_to_str(inp)
+        yield assert_equal, exp, obs
+
+
 def test_ensure_int_or_slice():
     cases = [
         (42, 42),
@@ -353,15 +659,101 @@ def test_ensure_int_or_slice():
         ('-42', -42),
         ('1:2:3', slice(1, 2, 3)),
         ('1::3', slice(1, None, 3)),
+        (':', slice(None, None, None)),
         ('1:', slice(1, None, None)),
         ('[1:2:3]', slice(1, 2, 3)),
         ('(1:2:3)', slice(1, 2, 3)),
+        ('r', False),
+        ('r:11', False),
         ]
     for inp, exp in cases:
         obs = ensure_int_or_slice(inp)
         yield assert_equal, exp, obs
 
 
+def test_is_dynamic_cwd_width():
+    cases = [
+        ('20', False),
+        ('20%', False),
+        ((20, 'c'), False),
+        ((20.0, 'm'), False),
+        ((20.0, 'c'), True),
+        ((20.0, '%'), True),
+        ]
+    for inp, exp in cases:
+        obs = is_dynamic_cwd_width(inp)
+        yield assert_equal, exp, obs
+
+def test_is_logfile_opt():
+    cases = [
+        ('throwback.log', True),
+        ('', True),
+        (None, True),
+        (True, False),
+        (False, False),
+        (42, False),
+        ([1, 2, 3], False),
+        ((1, 2), False),
+        (("wrong", "parameter"), False)
+    ]
+    if not ON_WINDOWS:
+        cases.append(('/dev/null', True))
+    for inp, exp in cases:
+        obs = is_logfile_opt(inp)
+        yield assert_equal, exp, obs
+
+def test_to_logfile_opt():
+    cases = [
+        (True, None),
+        (False, None),
+        (1, None),
+        (None, None),
+        ('throwback.log', 'throwback.log'),
+    ]
+    if not ON_WINDOWS:
+        cases.append(('/dev/null', '/dev/null'))
+        cases.append(('/dev/nonexistent_dev', None))
+    for inp, exp in cases:
+        obs = to_logfile_opt(inp)
+        yield assert_equal, exp, obs
+
+def test_logfile_opt_to_str():
+    cases = [
+        (None, ''),
+        ('', ''),
+        ('throwback.log', 'throwback.log'),
+        ('/dev/null', '/dev/null')
+    ]
+    for inp, exp in cases:
+        obs = logfile_opt_to_str(inp)
+        yield assert_equal, exp, obs
+
+def test_to_dynamic_cwd_tuple():
+    cases = [
+        ('20', (20.0, 'c')),
+        ('20%', (20.0, '%')),
+        ((20, 'c'), (20.0, 'c')),
+        ((20, '%'), (20.0, '%')),
+        ((20.0, 'c'), (20.0, 'c')),
+        ((20.0, '%'), (20.0, '%')),
+        ('inf', (float('inf'), 'c')),
+        ]
+    for inp, exp in cases:
+        obs = to_dynamic_cwd_tuple(inp)
+        yield assert_equal, exp, obs
+
+
+def test_dynamic_cwd_tuple_to_str():
+    cases = [
+        ((20.0, 'c'), '20.0'),
+        ((20.0, '%'), '20.0%'),
+        ((float('inf'), 'c'), 'inf'),
+        ]
+    for inp, exp in cases:
+        obs = dynamic_cwd_tuple_to_str(inp)
+        yield assert_equal, exp, obs
+
+
 def test_escape_windows_cmd_string():
     cases = [
         ('', ''),
@@ -407,11 +799,12 @@ _startend = {c+s: s for c in _chars for s in _squote}
 
 inners = "this is a string"
 
+
 def test_partial_string():
     # single string at start
     yield assert_equal, check_for_partial_string('no strings here'), (None, None, None)
     yield assert_equal, check_for_partial_string(''), (None, None, None)
-    for s,e in _startend.items():
+    for s, e in _startend.items():
         _test = s + inners + e
         for l in _leaders:
             for f in _leaders:
@@ -433,5 +826,61 @@ def test_partial_string():
                             yield assert_equal, _res, (len(l+_test+f+l2), None, s2)
 
 
+def test_executables_in():
+    expected = set()
+    types = ('file', 'directory', 'brokensymlink')
+    if ON_WINDOWS:
+        # Don't test symlinks on windows since it requires admin
+        types = ('file', 'directory')
+    executables = (True, False)
+    with TemporaryDirectory() as test_path:
+        for _type in types:
+            for executable in executables:
+                fname = '%s_%s' % (_type, executable)
+                if _type == 'none':
+                    continue
+                if _type == 'file' and executable:
+                    ext = '.exe' if ON_WINDOWS else ''
+                    expected.add(fname + ext)
+                else:
+                    ext = ''
+                path = os.path.join(test_path, fname + ext)
+                if _type == 'file':
+                    with open(path, 'w') as f:
+                        f.write(fname)
+                elif _type == 'directory':
+                    os.mkdir(path)
+                elif _type == 'brokensymlink':
+                    tmp_path = os.path.join(test_path, 'i_wont_exist')
+                    with open(tmp_path, 'w') as f:
+                        f.write('deleteme')
+                        os.symlink(tmp_path, path)
+                    os.remove(tmp_path)
+                if executable and not _type == 'brokensymlink':
+                    os.chmod(path, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR)
+            result = set(executables_in(test_path))
+    assert_equal(expected, result)
+
+
+def test_expand_case_matching():
+    cases = {
+        'yo': '[Yy][Oo]',
+        '[a-f]123e': '[a-f]123[Ee]',
+        '${HOME}/yo': '${HOME}/[Yy][Oo]',
+        './yo/mom': './[Yy][Oo]/[Mm][Oo][Mm]',
+        'Eßen': '[Ee][Ss]?[Ssß][Ee][Nn]',
+        }
+    for inp, exp in cases.items():
+        obs = expand_case_matching(inp)
+        yield assert_equal, exp, obs
+
+
+def test_commands_cache_lazy():
+    cc = CommandsCache()
+    yield assert_false, cc.lazyin('xonsh')
+    yield assert_equal, 0, len(list(cc.lazyiter()))
+    yield assert_equal, 0, cc.lazylen()
+
+
 if __name__ == '__main__':
     nose.runmodule()

tests/test_xontribs.py

@@ -0,0 +1,6 @@
+"""xontrib tests, such as they are"""
+from xonsh.xontribs import xontrib_metadata
+
+def test_load_xontrib_metadata():
+    # Simply tests that the xontribs JSON files isn't malformed.
+    xontrib_metadata()

tests/tools.py

@@ -12,7 +12,10 @@ from contextlib import contextmanager
 from nose.plugins.skip import SkipTest
 
 from xonsh.built_ins import ensure_list_of_strs
+builtins.__xonsh_env__ = {}
 from xonsh.base_shell import BaseShell
+from xonsh.execer import Execer
+from xonsh.tools import XonshBlockError
 
 
 VER_3_4 = (3, 4)
@@ -60,6 +63,7 @@ def mock_xonsh_env(xenv):
     builtins.__xonsh_subproc_captured__ = sp
     builtins.__xonsh_subproc_uncaptured__ = sp
     builtins.__xonsh_ensure_list_of_strs__ = ensure_list_of_strs
+    builtins.XonshBlockError = XonshBlockError
     builtins.evalx = eval
     builtins.execx = None
     builtins.compilex = None
@@ -77,6 +81,7 @@ def mock_xonsh_env(xenv):
     del builtins.__xonsh_subproc_captured__
     del builtins.__xonsh_subproc_uncaptured__
     del builtins.__xonsh_ensure_list_of_strs__
+    del builtins.XonshBlockError
     del builtins.evalx
     del builtins.execx
     del builtins.compilex
@@ -95,3 +100,36 @@ def skip_if(cond):
         else:
             return f
     return dec
+
+#
+# Execer tools
+#
+
+DEBUG_LEVEL = 0
+EXECER = None
+
+def execer_setup():
+    # only setup one parser
+    global EXECER
+    if EXECER is None:
+        EXECER = Execer(debug_level=DEBUG_LEVEL, login=False)
+
+def check_exec(input, **kwargs):
+    with mock_xonsh_env(None):
+        if not input.endswith('\n'):
+            input += '\n'
+        EXECER.debug_level = DEBUG_LEVEL
+        EXECER.exec(input, **kwargs)
+
+def check_eval(input):
+    with mock_xonsh_env({'AUTO_CD': False, 'XONSH_ENCODING' :'utf-8',
+                         'XONSH_ENCODING_ERRORS': 'strict', 'PATH': []}):
+        EXECER.debug_level = DEBUG_LEVEL
+        EXECER.eval(input)
+
+def check_parse(input):
+    with mock_xonsh_env(None):
+        EXECER.debug_level = DEBUG_LEVEL
+        tree = EXECER.parse(input, ctx=None)
+    return tree
+

xonsh-in-docker.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+import sys 
+import subprocess
+import os
+import argparse
+
+program_description = """Build and run Xonsh in a fresh, controlled 
+    environment using docker """
+
+parser = argparse.ArgumentParser(description=program_description)
+
+parser.add_argument('env', nargs='*', default=[], metavar='ENV=value')
+parser.add_argument('--python', '-p', default='3.4', metavar='python_version')
+parser.add_argument('--ptk', '-t', default='1.00', metavar='ptk_version')
+parser.add_argument('--keep', action='store_true')
+parser.add_argument('--build', action='store_true')
+parser.add_argument('--command', '-c', default='xonsh', 
+                    metavar='command')
+
+args = parser.parse_args()
+
+docker_script = """
+from python:{python_version}
+RUN pip install --upgrade pip && pip install \\
+  ply \\
+  prompt-toolkit=={ptk_version} \\
+  pygments
+RUN mkdir /xonsh
+WORKDIR /xonsh
+ADD ./ ./
+RUN python setup.py install
+""".format(
+        python_version = args.python,
+        ptk_version = args.ptk)
+
+print('Building and running Xonsh')
+print('Using python ', args.python)
+print('Using prompt-toolkit ', args.ptk)
+
+with open('./Dockerfile', 'w+') as f:
+    f.write(docker_script)
+
+env_string = ' '.join(args.env)
+
+subprocess.call(['docker', 'build', '-t' , 'xonsh', '.'])
+os.remove('./Dockerfile')
+
+if not args.build:
+    run_args = ['docker', 'run', '-ti']
+    for e in args.env:
+        run_args += ['-e', e]
+    if not args.keep:
+        run_args.append('--rm')
+    run_args += ['xonsh', args.command]
+    subprocess.call(run_args)

xonsh/__init__.py

@@ -1 +1,91 @@
-__version__ = '0.2.7'
+__version__ = '0.3.4'
+
+# amalgamate exclude jupyter_kernel parser_table parser_test_table pyghooks
+# amalgamate exclude winutils wizard
+import os as _os
+if _os.getenv('XONSH_DEBUG', ''):
+    pass
+else:
+    import sys as _sys
+    try:
+        from xonsh import __amalgam__
+        completer = __amalgam__
+        _sys.modules['xonsh.completer'] = __amalgam__
+        lazyasd = __amalgam__
+        _sys.modules['xonsh.lazyasd'] = __amalgam__
+        lazyjson = __amalgam__
+        _sys.modules['xonsh.lazyjson'] = __amalgam__
+        pretty = __amalgam__
+        _sys.modules['xonsh.pretty'] = __amalgam__
+        timings = __amalgam__
+        _sys.modules['xonsh.timings'] = __amalgam__
+        ansi_colors = __amalgam__
+        _sys.modules['xonsh.ansi_colors'] = __amalgam__
+        codecache = __amalgam__
+        _sys.modules['xonsh.codecache'] = __amalgam__
+        openpy = __amalgam__
+        _sys.modules['xonsh.openpy'] = __amalgam__
+        platform = __amalgam__
+        _sys.modules['xonsh.platform'] = __amalgam__
+        teepty = __amalgam__
+        _sys.modules['xonsh.teepty'] = __amalgam__
+        jobs = __amalgam__
+        _sys.modules['xonsh.jobs'] = __amalgam__
+        parser = __amalgam__
+        _sys.modules['xonsh.parser'] = __amalgam__
+        tokenize = __amalgam__
+        _sys.modules['xonsh.tokenize'] = __amalgam__
+        tools = __amalgam__
+        _sys.modules['xonsh.tools'] = __amalgam__
+        vox = __amalgam__
+        _sys.modules['xonsh.vox'] = __amalgam__
+        ast = __amalgam__
+        _sys.modules['xonsh.ast'] = __amalgam__
+        contexts = __amalgam__
+        _sys.modules['xonsh.contexts'] = __amalgam__
+        diff_history = __amalgam__
+        _sys.modules['xonsh.diff_history'] = __amalgam__
+        dirstack = __amalgam__
+        _sys.modules['xonsh.dirstack'] = __amalgam__
+        foreign_shells = __amalgam__
+        _sys.modules['xonsh.foreign_shells'] = __amalgam__
+        inspectors = __amalgam__
+        _sys.modules['xonsh.inspectors'] = __amalgam__
+        lexer = __amalgam__
+        _sys.modules['xonsh.lexer'] = __amalgam__
+        proc = __amalgam__
+        _sys.modules['xonsh.proc'] = __amalgam__
+        xontribs = __amalgam__
+        _sys.modules['xonsh.xontribs'] = __amalgam__
+        environ = __amalgam__
+        _sys.modules['xonsh.environ'] = __amalgam__
+        history = __amalgam__
+        _sys.modules['xonsh.history'] = __amalgam__
+        base_shell = __amalgam__
+        _sys.modules['xonsh.base_shell'] = __amalgam__
+        replay = __amalgam__
+        _sys.modules['xonsh.replay'] = __amalgam__
+        tracer = __amalgam__
+        _sys.modules['xonsh.tracer'] = __amalgam__
+        xonfig = __amalgam__
+        _sys.modules['xonsh.xonfig'] = __amalgam__
+        aliases = __amalgam__
+        _sys.modules['xonsh.aliases'] = __amalgam__
+        readline_shell = __amalgam__
+        _sys.modules['xonsh.readline_shell'] = __amalgam__
+        built_ins = __amalgam__
+        _sys.modules['xonsh.built_ins'] = __amalgam__
+        execer = __amalgam__
+        _sys.modules['xonsh.execer'] = __amalgam__
+        imphooks = __amalgam__
+        _sys.modules['xonsh.imphooks'] = __amalgam__
+        shell = __amalgam__
+        _sys.modules['xonsh.shell'] = __amalgam__
+        main = __amalgam__
+        _sys.modules['xonsh.main'] = __amalgam__
+        del __amalgam__
+    except ImportError:
+        pass
+    del _sys
+del _os
+# amalgamate end

xonsh/__main__.py

@@ -0,0 +1,2 @@
+from xonsh.main import main
+main()

xonsh/aliases.py

@@ -1,29 +1,28 @@
 # -*- coding: utf-8 -*-
 """Aliases for the xonsh shell."""
 
-import os
-import shlex
-import builtins
-import sys
-import subprocess
-from functools import lru_cache
 from argparse import ArgumentParser, Action
+import builtins
 from collections.abc import MutableMapping, Iterable, Sequence
+import os
+import sys
+import shlex
 
 from xonsh.dirstack import cd, pushd, popd, dirs, _get_cwd
-from xonsh.jobs import jobs, fg, bg, kill_all_jobs
-from xonsh.proc import foreground
-from xonsh.timings import timeit_alias
-from xonsh.tools import (ON_MAC, ON_WINDOWS, ON_ANACONDA,
-    XonshError, to_bool, string_types)
-from xonsh.history import main as history_alias
-from xonsh.replay import main as replay_main
-from xonsh.xontribs import main as xontribs_main
 from xonsh.environ import locate_binary
 from xonsh.foreign_shells import foreign_shell_data
+from xonsh.jobs import jobs, fg, bg, clean_jobs
+from xonsh.history import history_main
+from xonsh.platform import ON_ANACONDA, ON_DARWIN, ON_WINDOWS, scandir
+from xonsh.proc import foreground
+from xonsh.replay import replay_main
+from xonsh.timings import timeit_alias
+from xonsh.tools import (XonshError, argvquote, escape_windows_cmd_string,
+                         to_bool)
 from xonsh.vox import Vox
-from xonsh.tools import argvquote, escape_windows_cmd_string
+from xonsh.xontribs import xontribs_main
 from xonsh.xoreutils import _which
+from xonsh.completers._aliases import completer_alias
 
 
 class Aliases(MutableMapping):
@@ -106,7 +105,7 @@ class Aliases(MutableMapping):
         return self._raw[key]
 
     def __setitem__(self, key, val):
-        if isinstance(val, string_types):
+        if isinstance(val, str):
             self._raw[key] = shlex.split(val)
         else:
             self._raw[key] = val
@@ -143,10 +142,12 @@ class Aliases(MutableMapping):
 
 
 
-def exit(args, stdin=None):  # pylint:disable=redefined-builtin,W0622
+def xonsh_exit(args, stdin=None):
     """Sends signal to exit shell."""
+    if not clean_jobs():
+        # Do not exit if jobs not cleaned up
+        return None, None
     builtins.__xonsh_exit__ = True
-    kill_all_jobs()
     print()  # gimme a newline
     return None, None
 
@@ -280,7 +281,8 @@ def source_cmd(args, stdin=None):
     args.append('--envcmd=set')
     args.append('--seterrpostcmd=if errorlevel 1 exit 1')
     args.append('--use-tmpfile=1')
-    return source_foreign(args, stdin=stdin)
+    with builtins.__xonsh_env__.swap(PROMPT='$P$G'):
+        return source_foreign(args, stdin=stdin)
 
 
 def xexec(args, stdin=None):
@@ -330,7 +332,6 @@ def bang_bang(args, stdin=None):
 
 class AWitchAWitch(Action):
     SUPPRESS = '==SUPPRESS=='
-
     def __init__(self, option_strings, version=None, dest=SUPPRESS,
                  default=SUPPRESS, **kwargs):
         super().__init__(option_strings=option_strings, dest=dest,
@@ -353,15 +354,22 @@ def which(args, stdin=None, stdout=None, stderr=None):
     parser = ArgumentParser('which', description=desc)
     parser.add_argument('args', type=str, nargs='+',
                         help='The executables or aliases to search for')
-    parser.add_argument('-a', action='store_true', dest='all',
+    parser.add_argument('-a','--all', action='store_true', dest='all',
                         help='Show all matches in $PATH and xonsh.aliases')
     parser.add_argument('-s', '--skip-alias', action='store_true',
                         help='Do not search in xonsh.aliases', dest='skip')
     parser.add_argument('-V', '--version', action='version',
-                        version='{}'.format(_which.__version__))
-    parser.add_argument('-v', '--verbose', action='store_true', dest='verbose')
+                        version='{}'.format(_which.__version__),
+                        help='Display the version of the python which module '
+                        'used by xonsh')
+    parser.add_argument('-v', '--verbose', action='store_true', dest='verbose',
+                        help='Print out how matches were located and show '
+                        'near misses on stderr')
     parser.add_argument('-p', '--plain', action='store_true', dest='plain',
-                        help='Do not display alias expansions')
+                        help='Do not display alias expansions or location of '
+                             'where binaries are found. This is the '
+                             'default behavior, but the option can be used to '
+                             'override the --verbose option')
     parser.add_argument('--very-small-rocks', action=AWitchAWitch)
     if ON_WINDOWS:
         parser.add_argument('-e', '--exts', nargs='*', type=str,
@@ -377,7 +385,10 @@ def which(args, stdin=None, stdout=None, stderr=None):
         parser.print_usage(file=stderr)
         return -1
     pargs = parser.parse_args(args)
-    
+
+    if pargs.all:
+        pargs.verbose = True
+
     if ON_WINDOWS:
         if pargs.exts:
             exts = pargs.exts
@@ -391,31 +402,39 @@ def which(args, stdin=None, stdout=None, stderr=None):
         nmatches = 0
         # skip alias check if user asks to skip
         if (arg in builtins.aliases and not pargs.skip):
-            if pargs.plain:
-                print(arg, file=stdout)
+            if pargs.plain or not pargs.verbose:
+                if isinstance(builtins.aliases[arg], list):
+                    print(' '.join(builtins.aliases[arg]), file=stdout)
+                else:
+                    print(arg, file=stdout)
             else:
-                print('{} -> {}'.format(arg, builtins.aliases[arg]), file=stdout)
+                print("aliases['{}'] = {}".format(arg, builtins.aliases[arg]), file=stdout)
             nmatches += 1
             if not pargs.all:
                 continue
-        for match in _which.whichgen(arg, path=builtins.__xonsh_env__['PATH'],
-                                     exts=exts, verbose=pargs.verbose):
-            abs_name, from_where = match if pargs.verbose else (match, '')
+        # which.whichgen gives the nicest 'verbose' output if PATH is taken
+        # from os.environ so we temporarily override it with
+        # __xosnh_env__['PATH']
+        original_os_path = os.environ['PATH']
+        os.environ['PATH'] = builtins.__xonsh_env__.detype()['PATH']
+        matches = _which.whichgen(arg, exts=exts, verbose=pargs.verbose)
+        for abs_name, from_where in matches:
             if ON_WINDOWS:
                 # Use list dir to get correct case for the filename
-                # i.e. windows is case insesitive but case preserving
+                # i.e. windows is case insensitive but case preserving
                 p, f = os.path.split(abs_name)
-                f = next(s for s in os.listdir(p) if s.lower() == f.lower())
+                f = next(s.name for s in scandir(p) if s.name.lower() == f.lower())
                 abs_name = os.path.join(p, f)
                 if builtins.__xonsh_env__.get('FORCE_POSIX_PATHS', False):
                     abs_name.replace(os.sep, os.altsep)
-            if pargs.verbose:
-                print('{} ({})'.format(abs_name, from_where), file=stdout)
-            else:
+            if pargs.plain or not pargs.verbose:
                 print(abs_name, file=stdout)
+            else:
+                print('{} ({})'.format(abs_name, from_where), file=stdout)
             nmatches += 1
             if not pargs.all:
                 break
+        os.environ['PATH'] = original_os_path
         if not nmatches:
             failures.append(arg)
     if len(failures) == 0:
@@ -430,15 +449,18 @@ def which(args, stdin=None, stdout=None, stderr=None):
 
 def xonfig(args, stdin=None):
     """Runs the xonsh configuration utility."""
-    from xonsh.xonfig import main  # lazy import
-    return main(args)
+    from xonsh.xonfig import xonfig_main  # lazy import
+    return xonfig_main(args)
 
 
 @foreground
 def trace(args, stdin=None):
     """Runs the xonsh tracer utility."""
-    from xonsh.tracer import main  # lazy import
-    return main(args)
+    from xonsh.tracer import tracermain  # lazy import
+    try:
+        return tracermain(args)
+    except SystemExit:
+        pass
 
 
 def vox(args, stdin=None):
@@ -447,6 +469,26 @@ def vox(args, stdin=None):
     return vox(args, stdin=stdin)
 
 
+def showcmd(args, stdin=None):
+    """usage: showcmd [-h|--help|cmd args]
+
+    Displays the command and arguments as a list of strings that xonsh would
+    run in subprocess mode. This is useful for determining how xonsh evaluates
+    your commands and arguments prior to running these commands.
+
+    optional arguments:
+      -h, --help            show this help message and exit
+
+    example:
+      >>> showcmd echo $USER can't hear "the sea"
+      ['echo', 'I', "can't", 'hear', 'the sea']
+    """
+    if len(args) == 0 or (len(args) == 1 and args[0] in {'-h', '--help'}):
+        print(showcmd.__doc__.rstrip().replace('\n    ', '\n'))
+    else:
+        sys.displayhook(args)
+
+
 def make_default_aliases():
     """Creates a new default aliases dictionary."""
     default_aliases = {
@@ -457,16 +499,16 @@ def make_default_aliases():
         'jobs': jobs,
         'fg': fg,
         'bg': bg,
-        'EOF': exit,
-        'exit': exit,
-        'quit': exit,
+        'EOF': xonsh_exit,
+        'exit': xonsh_exit,
+        'quit': xonsh_exit,
         'xexec': xexec,
         'source': source_alias,
         'source-zsh': ['source-foreign', 'zsh', '--sourcer=source'],
         'source-bash':  ['source-foreign', 'bash', '--sourcer=source'],
         'source-cmd': source_cmd,
         'source-foreign': source_foreign,
-        'history': history_alias,
+        'history': history_main,
         'replay': replay_main,
         '!!': bang_bang,
         '!n': bang_n,
@@ -474,10 +516,12 @@ def make_default_aliases():
         'timeit': timeit_alias,
         'xonfig': xonfig,
         'scp-resume': ['rsync', '--partial', '-h', '--progress', '--rsh=ssh'],
+        'showcmd': showcmd,
         'ipynb': ['jupyter', 'notebook', '--no-browser'],
         'vox': vox,
         'which': which,
         'xontrib': xontribs_main,
+        'completer': completer_alias
     }
     if ON_WINDOWS:
         # Borrow builtin commands from cmd.exe.
@@ -503,17 +547,11 @@ def make_default_aliases():
             default_aliases[alias] = ['cmd', '/c', alias]
         default_aliases['call'] = ['source-cmd']
         default_aliases['source-bat'] = ['source-cmd']
-        # Add aliases specific to the Anaconda python distribution.
+        default_aliases['clear'] = 'cls'
         if ON_ANACONDA:
-            def source_cmd_keep_prompt(args, stdin=None):
-                p = builtins.__xonsh_env__.get('PROMPT')
-                source_cmd(args, stdin=stdin)
-                builtins.__xonsh_env__['PROMPT'] = p
-            default_aliases['source-cmd-keep-promt'] = source_cmd_keep_prompt
-            default_aliases['activate'] = ['source-cmd-keep-promt',
-                                           'activate.bat']
-            default_aliases['deactivate'] = ['source-cmd-keep-promt',
-                                             'deactivate.bat']
+            # Add aliases specific to the Anaconda python distribution.
+            default_aliases['activate'] = ['source-cmd', 'activate.bat']
+            default_aliases['deactivate'] = ['source-cmd', 'deactivate.bat']
         if not locate_binary('sudo'):
             import xonsh.winutils as winutils
 
@@ -533,10 +571,11 @@ def make_default_aliases():
                     print(msg.format(cmd))
 
             default_aliases['sudo'] = sudo
-    elif ON_MAC:
+    elif ON_DARWIN:
         default_aliases['ls'] = ['ls', '-G']
     else:
         default_aliases['grep'] = ['grep', '--color=auto']
+        default_aliases['egrep'] = ['egrep', '--color=auto']
+        default_aliases['fgrep'] = ['fgrep', '--color=auto']
         default_aliases['ls'] = ['ls', '--color=auto', '-v']
     return default_aliases
-

xonsh/ast.py

@@ -12,15 +12,16 @@ from ast import Module, Num, Expr, Str, Bytes, UnaryOp, UAdd, USub, Invert, \
     YieldFrom, Return, IfExp, Lambda, arguments, arg, Call, keyword, \
     Attribute, Global, Nonlocal, If, While, For, withitem, With, Try, \
     ExceptHandler, FunctionDef, ClassDef, Starred, NodeTransformer, \
-    Interactive, Expression, Index, literal_eval, dump, walk
-from ast import Ellipsis  # pylint: disable=redefined-builtin
+    Interactive, Expression, Index, literal_eval, dump, walk, increment_lineno
+from ast import Ellipsis as EllipsisNode
 # pylint: enable=unused-import
 import textwrap
-from itertools import repeat
+import itertools
 
-from xonsh.tools import subproc_toks, VER_3_5, VER_MAJOR_MINOR
+from xonsh.tools import subproc_toks, find_next_break
+from xonsh.platform import PYTHON_VERSION_INFO
 
-if VER_3_5 <= VER_MAJOR_MINOR:
+if PYTHON_VERSION_INFO >= (3, 5, 0):
     # pylint: disable=unused-import
     # pylint: disable=no-name-in-module
     from ast import MatMult, AsyncFunctionDef, AsyncWith, AsyncFor, Await
@@ -51,11 +52,30 @@ def leftmostname(node):
     elif isinstance(node, (Str, Bytes)):
         # handles case of "./my executable"
         rtn = leftmostname(node.s)
+    elif isinstance(node, Tuple) and len(node.elts) > 0:
+        # handles case of echo ,1,2,3
+        rtn = leftmostname(node.elts[0])
     else:
         rtn = None
     return rtn
 
 
+def get_lineno(node, default=0):
+    """Gets the lineno of a node or returns the default."""
+    return getattr(node, 'lineno', default)
+
+
+def min_line(node):
+    """Computes the minimum lineno."""
+    node_line = get_lineno(node)
+    return min(map(get_lineno, walk(node), itertools.repeat(node_line)))
+
+
+def max_line(node):
+    """Computes the maximum lineno."""
+    return max(map(get_lineno, walk(node)))
+
+
 def get_col(node, default=-1):
     """Gets the col_offset of a node, or returns the default"""
     return getattr(node, 'col_offset', default)
@@ -63,7 +83,7 @@ def get_col(node, default=-1):
 
 def min_col(node):
     """Computes the minimum col_offset."""
-    return min(map(get_col, walk(node), repeat(node.col_offset)))
+    return min(map(get_col, walk(node), itertools.repeat(node.col_offset)))
 
 
 def max_col(node):
@@ -74,6 +94,30 @@ def max_col(node):
     return col
 
 
+def get_id(node, default=None):
+    """Gets the id attribute of a node, or returns a default."""
+    return getattr(node, 'id', default)
+
+
+def gather_names(node):
+    """Returns the set of all names present in the node's tree."""
+    rtn = set(map(get_id, walk(node)))
+    rtn.discard(None)
+    return rtn
+
+
+def has_elts(x):
+    """Tests if x is an AST node with elements."""
+    return isinstance(x, AST) and hasattr(x, 'elts')
+
+
+def xonsh_call(name, args, lineno=None, col=None):
+    """Creates the AST node for calling a function of a given name."""
+    return Call(func=Name(id=name, ctx=Load(), lineno=lineno, col_offset=col),
+                args=args, keywords=[], starargs=None, kwargs=None,
+                lineno=lineno, col_offset=col)
+
+
 def isdescendable(node):
     """Deteremines whether or not a node is worth visiting. Currently only
     UnaryOp and BoolOp nodes are visited.
@@ -100,6 +144,7 @@ class CtxAwareTransformer(NodeTransformer):
         self.contexts = []
         self.lines = None
         self.mode = None
+        self._nwith = 0
 
     def ctxvisit(self, node, inp, ctx, mode='exec'):
         """Transforms the node in a context-dependent way.
@@ -121,8 +166,10 @@ class CtxAwareTransformer(NodeTransformer):
         self.lines = inp.splitlines()
         self.contexts = [ctx, set()]
         self.mode = mode
+        self._nwith = 0
         node = self.visit(node)
         del self.lines, self.contexts, self.mode
+        self._nwith = 0
         return node
 
     def ctxupdate(self, iterable):
@@ -148,7 +195,12 @@ class CtxAwareTransformer(NodeTransformer):
             maxcol = None
         else:
             mincol = min_col(node)
-            maxcol = max_col(node) + 1
+            maxcol = max_col(node)
+            if mincol == maxcol:
+                maxcol = find_next_break(line, mincol=mincol,
+                                         lexer=self.parser.lexer)
+            else:
+                maxcol += 1
         spline = subproc_toks(line,
                               mincol=mincol,
                               maxcol=maxcol,
@@ -162,7 +214,7 @@ class CtxAwareTransformer(NodeTransformer):
             if not isinstance(newnode, AST):
                 # take the first (and only) Expr
                 newnode = newnode[0]
-            newnode.lineno = node.lineno
+            increment_lineno(newnode, n=node.lineno - 1)
             newnode.col_offset = node.col_offset
         except SyntaxError:
             newnode = node
@@ -182,6 +234,85 @@ class CtxAwareTransformer(NodeTransformer):
                 break
         return inscope
 
+    #
+    # With Transformers
+    #
+    def insert_with_block_check(self, node):
+        """Modifies a with statement node in-place to add an initial check
+        for whether or not the block should be executed. If the block is
+        not executed it will raise a XonshBlockError containing the required
+        information.
+        """
+        nwith = self._nwith  # the nesting level of the current with-statement
+        lineno = get_lineno(node)
+        col = get_col(node, 0)
+        # Add or discover target names
+        targets = set()
+        i = 0  # index of unassigned items
+
+        def make_next_target():
+            nonlocal i
+            targ = '__xonsh_with_target_{}_{}__'.format(nwith, i)
+            n = Name(id=targ, ctx=Store(), lineno=lineno, col_offset=col)
+            targets.add(targ)
+            i += 1
+            return n
+        for item in node.items:
+            if item.optional_vars is None:
+                if has_elts(item.context_expr):
+                    targs = [make_next_target() for _ in item.context_expr.elts]
+                    optvars = Tuple(elts=targs, ctx=Store(), lineno=lineno,
+                                    col_offset=col)
+                else:
+                    optvars = make_next_target()
+                item.optional_vars = optvars
+            else:
+                targets.update(gather_names(item.optional_vars))
+        # Ok, now that targets have been found / created, make the actual check
+        # to see if we are in a non-executing block. This is equivalent to
+        # writing the following condition:
+        #
+        #     if getattr(targ0, '__xonsh_block__', False) or \
+        #        getattr(targ1, '__xonsh_block__', False) or ...:
+        #         raise XonshBlockError(lines, globals(), locals())
+        tests = [_getblockattr(t, lineno, col) for t in sorted(targets)]
+        if len(tests) == 1:
+            test = tests[0]
+        else:
+            test = BoolOp(op=Or(), values=tests, lineno=lineno, col_offset=col)
+        ldx, udx = self._find_with_block_line_idx(node)
+        lines = [Str(s=s, lineno=lineno, col_offset=col)
+                 for s in self.lines[ldx:udx]]
+        check = If(test=test, body=[
+            Raise(exc=xonsh_call('XonshBlockError',
+                                 args=[List(elts=lines, ctx=Load(),
+                                            lineno=lineno, col_offset=col),
+                                       xonsh_call('globals', args=[],
+                                                  lineno=lineno, col=col),
+                                       xonsh_call('locals', args=[],
+                                                  lineno=lineno, col=col)],
+                                 lineno=lineno, col=col),
+                  cause=None, lineno=lineno, col_offset=col)],
+                orelse=[], lineno=lineno, col_offset=col)
+        node.body.insert(0, check)
+
+    def _find_with_block_line_idx(self, node):
+        ldx = min_line(node.body[0]) - 1
+        udx = max_line(node.body[-1])
+        # now check if parsable, or add lines until it is or we run out of lines
+        nlines = len(self.lines)
+        lines = 'with __xonsh_dummy__:\n' + '\n'.join(self.lines[ldx:udx])
+        lines += '\n'
+        parsable = False
+        while not parsable and udx < nlines:
+            try:
+                self.parser.parse(lines, mode=self.mode)
+                parsable = True
+            except SyntaxError:
+                lines += self.lines[udx] + '\n'
+                udx += 1
+        return ldx, udx
+
     #
     # Replacement visitors
     #
@@ -277,8 +408,11 @@ class CtxAwareTransformer(NodeTransformer):
         """Handle visiting a with statement."""
         for item in node.items:
             if item.optional_vars is not None:
-                self.ctxadd(leftmostname(item.optional_vars))
+                self.ctxupdate(gather_names(item.optional_vars))
+        self._nwith += 1
         self.generic_visit(node)
+        self._nwith -= 1
+        self.insert_with_block_check(node)
         return node
 
     def visit_For(self, node):
@@ -356,4 +490,19 @@ def pdump(s, **kwargs):
     return pre + mid + post
 
 
+def pprint(s, *, sep=None, end=None, file=None, flush=False, **kwargs):
+    """Performs a pretty print of the AST nodes."""
+    print(pdump(s, **kwargs), sep=sep, end=end, file=file, flush=flush)
 
+
+#
+# Private helpers
+#
+
+def _getblockattr(name, lineno, col):
+    """calls getattr(name, '__xonsh_block__', False)."""
+    return xonsh_call('getattr', args=[
+        Name(id=name, ctx=Load(), lineno=lineno, col_offset=col),
+        Str(s='__xonsh_block__', lineno=lineno, col_offset=col),
+        NameConstant(value=False, lineno=lineno, col_offset=col)],
+        lineno=lineno, col=col)

xonsh/base_shell.py

@@ -6,15 +6,14 @@ import sys
 import time
 import builtins
 
-from xonsh.tools import XonshError, escape_windows_cmd_string, ON_WINDOWS, \
-    print_exception, HAVE_PYGMENTS
+from xonsh.tools import (XonshError, escape_windows_cmd_string, print_exception,
+    DefaultNotGiven)
+from xonsh.platform import HAS_PYGMENTS, ON_WINDOWS
 from xonsh.codecache import (should_use_cache, code_cache_name,
                              code_cache_check, get_cache_filename,
                              update_cache, run_compiled_code)
 from xonsh.completer import Completer
 from xonsh.environ import multiline_prompt, format_prompt, partial_format_prompt
-if HAVE_PYGMENTS:
-    from xonsh.pyghooks import XonshStyle
 
 
 class _TeeOut(object):
@@ -46,8 +45,7 @@ class _TeeOut(object):
 
     def fileno(self):
         """Tunnel fileno() calls."""
-        _ = self
-        return sys.stdout.fileno()
+        return self.stdout.fileno()
 
 
 class _TeeErr(object):
@@ -79,8 +77,7 @@ class _TeeErr(object):
 
     def fileno(self):
         """Tunnel fileno() calls."""
-        _ = self
-        return sys.stderr.fileno()
+        return self.stderr.fileno()
 
 
 class Tee(io.StringIO):
@@ -114,16 +111,30 @@ class BaseShell(object):
         super().__init__()
         self.execer = execer
         self.ctx = ctx
-        if kwargs.get('completer', True):
-            self.completer = Completer()
+        self.completer = Completer() if kwargs.get('completer', True) else None
         self.buffer = []
         self.need_more_lines = False
         self.mlprompt = None
-        if HAVE_PYGMENTS:
-            env = builtins.__xonsh_env__
-            self.styler = XonshStyle(env.get('XONSH_COLOR_STYLE'))
-        else:
-            self.styler = None
+        self._styler = DefaultNotGiven
+
+    @property
+    def styler(self):
+        if self._styler is DefaultNotGiven:
+            if HAS_PYGMENTS:
+                from xonsh.pyghooks import XonshStyle
+                env = builtins.__xonsh_env__
+                self._styler = XonshStyle(env.get('XONSH_COLOR_STYLE'))
+            else:
+                self._styler = None
+        return self._styler
+
+    @styler.setter
+    def styler(self, value):
+        self._styler = value
+
+    @styler.deleter
+    def styler(self):
+        self._styler = DefaultNotGiven
 
     def emptyline(self):
         """Called when an empty line has been entered."""

xonsh/built_ins.py

@@ -4,43 +4,60 @@
 Note that this module is named 'built_ins' so as not to be confused with the
 special Python builtins module.
 """
+import atexit
+import builtins
+from collections import Sequence
+from contextlib import contextmanager
+import inspect
+from glob import iglob
 import os
 import re
-import sys
-import time
 import shlex
-import atexit
 import signal
-import inspect
-import builtins
-import tempfile
-from glob import glob, iglob
 from subprocess import Popen, PIPE, STDOUT, CalledProcessError
-from contextlib import contextmanager
-from collections import Sequence, Iterable
+import sys
+import tempfile
+import time
 
-from xonsh.tools import (
-    suggest_commands, XonshError, ON_POSIX, ON_WINDOWS, string_types,
-    expandvars, CommandsCache
-)
+from xonsh.lazyasd import LazyObject
+from xonsh.history import History
+from xonsh.tokenize import SearchPath
 from xonsh.inspectors import Inspector
+from xonsh.aliases import Aliases, make_default_aliases
 from xonsh.environ import Env, default_env, locate_binary
+from xonsh.foreign_shells import load_foreign_aliases
 from xonsh.jobs import add_job, wait_for_active_job
+from xonsh.platform import ON_POSIX, ON_WINDOWS
 from xonsh.proc import (ProcProxy, SimpleProcProxy, ForegroundProcProxy,
                         SimpleForegroundProcProxy, TeePTYProc,
                         CompletedCommand, HiddenCompletedCommand)
-from xonsh.aliases import Aliases, make_default_aliases
-from xonsh.history import History
-from xonsh.foreign_shells import load_foreign_aliases
+from xonsh.tools import (
+    suggest_commands, expandvars, CommandsCache, globpath, XonshError,
+    XonshCalledProcessError, XonshBlockError
+)
+
 
 ENV = None
 BUILTINS_LOADED = False
-INSPECTOR = Inspector()
+INSPECTOR = LazyObject(Inspector, globals(), 'INSPECTOR')
 AT_EXIT_SIGNALS = (signal.SIGABRT, signal.SIGFPE, signal.SIGILL, signal.SIGSEGV,
                    signal.SIGTERM)
+
+SIGNAL_MESSAGES = {
+    signal.SIGABRT: 'Aborted',
+    signal.SIGFPE: 'Floating point exception',
+    signal.SIGILL: 'Illegal instructions',
+    signal.SIGTERM: 'Terminated',
+    signal.SIGSEGV: 'Segmentation fault'
+}
+
 if ON_POSIX:
     AT_EXIT_SIGNALS += (signal.SIGTSTP, signal.SIGQUIT, signal.SIGHUP)
-
+    SIGNAL_MESSAGES.update({
+        signal.SIGQUIT: 'Quit',
+        signal.SIGHUP: 'Hangup',
+        signal.SIGKILL: 'Killed'
+    })
 
 def resetting_signal_handle(sig, f):
     """Sets a new signal handle that will automatically restore the old value
@@ -76,46 +93,6 @@ def expand_path(s):
     return os.path.expanduser(s)
 
 
-WINDOWS_DRIVE_MATCHER = re.compile(r'^\w:')
-
-
-def expand_case_matching(s):
-    """Expands a string to a case insenstive globable string."""
-    t = []
-    openers = {'[', '{'}
-    closers = {']', '}'}
-    nesting = 0
-
-    drive_part = WINDOWS_DRIVE_MATCHER.match(s) if ON_WINDOWS else None
-
-    if drive_part:
-        drive_part = drive_part.group(0)
-        t.append(drive_part)
-        s = s[len(drive_part):]
-
-    for c in s:
-        if c in openers:
-            nesting += 1
-        elif c in closers:
-            nesting -= 1
-        elif nesting > 0:
-            pass
-        elif c.isalpha():
-            folded = c.casefold()
-            if len(folded) == 1:
-                c = '[{0}{1}]'.format(c.upper(), c.lower())
-            else:
-                newc = ['[{0}{1}]?'.format(f.upper(), f.lower())
-                        for f in folded[:-1]]
-                newc = ''.join(newc)
-                newc += '[{0}{1}{2}]'.format(folded[-1].upper(),
-                                             folded[-1].lower(),
-                                             c)
-                c = newc
-        t.append(c)
-    return ''.join(t)
-
-
 def reglob(path, parts=None, i=None):
     """Regular expression-based globbing."""
     if parts is None:
@@ -155,40 +132,29 @@ def reglob(path, parts=None, i=None):
     return paths
 
 
-def regexpath(s, pymode=False):
-    """Takes a regular expression string and returns a list of file
-    paths that match the regex.
-    """
+def regexsearch(s):
     s = expand_path(s)
-    o = reglob(s)
+    return reglob(s)
+
+
+def globsearch(s):
+    csc = builtins.__xonsh_env__.get('CASE_SENSITIVE_COMPLETIONS')
+    return globpath(s, ignore_case=(not csc), return_empty=True)
+
+
+def pathsearch(func, s, pymode=False):
+    """
+    Takes a string and returns a list of file paths that match (regex, glob,
+    or arbitrary search function).
+    """
+    if (not callable(func) or
+            len(inspect.signature(func).parameters) != 1):
+        error = "%r is not a known path search function"
+        raise XonshError(error % searchfunc)
+    o = func(s)
     no_match = [] if pymode else [s]
     return o if len(o) != 0 else no_match
 
-
-def globpath(s, ignore_case=False):
-    """Simple wrapper around glob that also expands home and env vars."""
-    o, s = _iglobpath(s, ignore_case=ignore_case)
-    o = list(o)
-    return o if len(o) != 0 else [s]
-
-
-def _iglobpath(s, ignore_case=False):
-    s = expand_path(s)
-    if ignore_case:
-        s = expand_case_matching(s)
-    if sys.version_info > (3, 5):
-        if '**' in s and '**/*' not in s:
-            s = s.replace('**', '**/*')
-        # `recursive` is only a 3.5+ kwarg.
-        return iglob(s, recursive=True), s
-    else:
-        return iglob(s), s
-
-def iglobpath(s, ignore_case=False):
-    """Simple wrapper around iglob that also expands home and env vars."""
-    return _iglobpath(s, ignore_case)[0]
-
-
 RE_SHEBANG = re.compile(r'#![ \t]*(.+?)$')
 
 
@@ -401,7 +367,7 @@ def run_subproc(cmds, captured=False):
         procinfo['args'] = list(cmd)
         stdin = None
         stderr = None
-        if isinstance(cmd, string_types):
+        if isinstance(cmd, str):
             continue
         streams = {}
         while True:
@@ -451,13 +417,19 @@ def run_subproc(cmds, captured=False):
         elif builtins.__xonsh_stderr_uncaptured__ is not None:
             stderr = builtins.__xonsh_stderr_uncaptured__
         uninew = (ix == last_cmd) and (not _capture_streams)
-        alias = builtins.aliases.get(cmd[0], None)
+
+        if callable(cmd[0]):
+            alias = cmd[0]
+        else:
+            alias = builtins.aliases.get(cmd[0], None)
+            binary_loc = locate_binary(cmd[0])
+
         procinfo['alias'] = alias
         if (alias is None and
                 builtins.__xonsh_env__.get('AUTO_CD') and
                 len(cmd) == 1 and
                 os.path.isdir(cmd[0]) and
-                locate_binary(cmd[0]) is None):
+                binary_loc is None):
             cmd.insert(0, 'cd')
             alias = builtins.aliases.get('cd', None)
 
@@ -465,13 +437,13 @@ def run_subproc(cmds, captured=False):
             aliased_cmd = alias
         else:
             if alias is not None:
-                cmd = alias + cmd[1:]
-            n = locate_binary(cmd[0])
-            if n is None:
-                aliased_cmd = cmd
+                aliased_cmd = alias + cmd[1:]
             else:
+                aliased_cmd = cmd
+            if binary_loc is not None:
                 try:
-                    aliased_cmd = get_script_subproc_command(n, cmd[1:])
+                    aliased_cmd = get_script_subproc_command(binary_loc,
+                                                             aliased_cmd[1:])
                 except PermissionError:
                     e = 'xonsh: subprocess mode: permission denied: {0}'
                     raise XonshError(e.format(cmd[0]))
@@ -512,10 +484,15 @@ def run_subproc(cmds, captured=False):
             subproc_kwargs = {}
             if ON_POSIX and cls is Popen:
                 subproc_kwargs['preexec_fn'] = _subproc_pre
+            env = ENV.detype()
+            if ON_WINDOWS:
+                # Over write prompt variable as xonsh's $PROMPT does
+                # not make much sense for other subprocs
+                env['PROMPT'] = '$P$G'
             try:
                 proc = cls(aliased_cmd,
                            universal_newlines=uninew,
-                           env=ENV.detype(),
+                           env=env,
                            stdin=stdin,
                            stdout=stdout,
                            stderr=stderr,
@@ -532,11 +509,6 @@ def run_subproc(cmds, captured=False):
                 raise XonshError(e)
         procs.append(proc)
         prev_proc = proc
-    for proc in procs[:-1]:
-        try:
-            proc.stdout.close()
-        except OSError:
-            pass
     if not prev_is_proxy:
         add_job({
             'cmds': cmds,
@@ -557,6 +529,11 @@ def run_subproc(cmds, captured=False):
     if prev_is_proxy:
         prev_proc.wait()
     wait_for_active_job()
+    for proc in procs[:-1]:
+        try:
+            proc.stdout.close()
+        except OSError:
+            pass
     hist = builtins.__xonsh_history__
     hist.last_cmd_rtn = prev_proc.returncode
     # get output
@@ -599,6 +576,13 @@ def run_subproc(cmds, captured=False):
                 errout = errout.replace('\r\n', '\n')
                 procinfo['stderr'] = errout
 
+    if getattr(prev_proc, 'signal', None):
+        sig, core = prev_proc.signal
+        sig_str = SIGNAL_MESSAGES.get(sig)
+        if sig_str:
+            if core:
+                sig_str += ' (core dumped)'
+            print(sig_str, file=sys.stderr)
     if (not prev_is_proxy and
             hist.last_cmd_rtn is not None and
             hist.last_cmd_rtn > 0 and
@@ -607,6 +591,7 @@ def run_subproc(cmds, captured=False):
     if captured == 'stdout':
         return output
     elif captured is not False:
+        procinfo['executed_cmd'] = aliased_cmd
         procinfo['pid'] = prev_proc.pid
         procinfo['returncode'] = prev_proc.returncode
         procinfo['timestamp'] = (starttime, time.time())
@@ -628,6 +613,12 @@ def subproc_captured_stdout(*cmds):
     return run_subproc(cmds, captured='stdout')
 
 
+def subproc_captured_inject(*cmds):
+    """Runs a subprocess, capturing the output. Returns a list of
+    whitespace-separated strings in the stdout that was produced."""
+    return [i.strip() for i in run_subproc(cmds, captured='stdout').split()]
+
+
 def subproc_captured_object(*cmds):
     """
     Runs a subprocess, capturing the output. Returns an instance of
@@ -653,10 +644,21 @@ def subproc_uncaptured(*cmds):
 
 def ensure_list_of_strs(x):
     """Ensures that x is a list of strings."""
-    if isinstance(x, string_types):
+    if isinstance(x, str):
         rtn = [x]
     elif isinstance(x, Sequence):
-        rtn = [i if isinstance(i, string_types) else str(i) for i in x]
+        rtn = [i if isinstance(i, str) else str(i) for i in x]
+    else:
+        rtn = [str(x)]
+    return rtn
+
+
+def list_of_strs_or_callables(x):
+    """Ensures that x is a list of strings or functions"""
+    if isinstance(x, str) or callable(x):
+        rtn = [x]
+    elif isinstance(x, Sequence):
+        rtn = [i if isinstance(i, str) or callable(i) else str(i) for i in x]
     else:
         rtn = [str(x)]
     return rtn
@@ -670,10 +672,11 @@ def load_builtins(execer=None, config=None, login=False, ctx=None):
     # private built-ins
     builtins.__xonsh_config__ = {}
     builtins.__xonsh_env__ = ENV = Env(default_env(config=config, login=login))
-    builtins.__xonsh_ctx__ = {} if ctx is None else ctx
     builtins.__xonsh_help__ = helper
     builtins.__xonsh_superhelp__ = superhelper
-    builtins.__xonsh_regexpath__ = regexpath
+    builtins.__xonsh_pathsearch__ = pathsearch
+    builtins.__xonsh_globsearch__ = globsearch
+    builtins.__xonsh_regexsearch__ = regexsearch
     builtins.__xonsh_glob__ = globpath
     builtins.__xonsh_expand_path__ = expand_path
     builtins.__xonsh_exit__ = False
@@ -686,19 +689,24 @@ def load_builtins(execer=None, config=None, login=False, ctx=None):
         builtins.__xonsh_pyquit__ = builtins.quit
         del builtins.quit
     builtins.__xonsh_subproc_captured_stdout__ = subproc_captured_stdout
+    builtins.__xonsh_subproc_captured_inject__ = subproc_captured_inject
     builtins.__xonsh_subproc_captured_object__ = subproc_captured_object
     builtins.__xonsh_subproc_captured_hiddenobject__ = subproc_captured_hiddenobject
     builtins.__xonsh_subproc_uncaptured__ = subproc_uncaptured
     builtins.__xonsh_execer__ = execer
     builtins.__xonsh_commands_cache__ = CommandsCache()
     builtins.__xonsh_all_jobs__ = {}
-    builtins.__xonsh_active_job__ = None
     builtins.__xonsh_ensure_list_of_strs__ = ensure_list_of_strs
+    builtins.__xonsh_list_of_strs_or_callables__ = list_of_strs_or_callables
     # public built-ins
+    builtins.XonshError = XonshError
+    builtins.XonshBlockError = XonshBlockError
+    builtins.XonshCalledProcessError = XonshCalledProcessError
     builtins.evalx = None if execer is None else execer.eval
     builtins.execx = None if execer is None else execer.exec
     builtins.compilex = None if execer is None else execer.compile
 
+    # sneak the path search functions into the aliases
     # Need this inline/lazy import here since we use locate_binary that relies on __xonsh_env__ in default aliases
     builtins.default_aliases = builtins.aliases = Aliases(make_default_aliases())
     if login:
@@ -714,7 +722,8 @@ def load_builtins(execer=None, config=None, login=False, ctx=None):
 
 
 def _lastflush(s=None, f=None):
-    builtins.__xonsh_history__.flush(at_exit=True)
+    if hasattr(builtins, '__xonsh_history__'):
+        builtins.__xonsh_history__.flush(at_exit=True)
 
 
 def unload_builtins():
@@ -736,7 +745,9 @@ def unload_builtins():
              '__xonsh_ctx__',
              '__xonsh_help__',
              '__xonsh_superhelp__',
-             '__xonsh_regexpath__',
+             '__xonsh_pathsearch__',
+             '__xonsh_globsearch__',
+             '__xonsh_regexsearch__',
              '__xonsh_glob__',
              '__xonsh_expand_path__',
              '__xonsh_exit__',
@@ -745,18 +756,22 @@ def unload_builtins():
              '__xonsh_pyexit__',
              '__xonsh_pyquit__',
              '__xonsh_subproc_captured_stdout__',
+             '__xonsh_subproc_captured_inject__',
              '__xonsh_subproc_captured_object__',
              '__xonsh_subproc_captured_hiddenobject__',
              '__xonsh_subproc_uncaptured__',
              '__xonsh_execer__',
              '__xonsh_commands_cache__',
+             'XonshError',
+             'XonshBlockError',
+             'XonshCalledProcessError',
              'evalx',
              'execx',
              'compilex',
              'default_aliases',
              '__xonsh_all_jobs__',
-             '__xonsh_active_job__',
              '__xonsh_ensure_list_of_strs__',
+             '__xonsh_list_of_strs_or_callables__',
              '__xonsh_history__',
              ]
     for name in names:

xonsh/codecache.py

@@ -4,6 +4,8 @@ import hashlib
 import marshal
 import builtins
 
+from xonsh.lazyasd import LazyObject
+
 def _splitpath(path, sofar=[]):
     folder, path = os.path.split(path)
     if path == "":
@@ -13,8 +15,15 @@ def _splitpath(path, sofar=[]):
     else:
         return _splitpath(folder, sofar + [path])
 
-_CHARACTER_MAP = {chr(o): '_%s' % chr(o+32) for o in range(65, 91)}
-_CHARACTER_MAP.update({'.': '_.', '_': '__'})
+
+def _character_map():
+    cmap = {chr(o): '_%s' % chr(o+32) for o in range(65, 91)}
+    cmap.update({'.': '_.', '_': '__'})
+    return cmap
+
+
+_CHARACTER_MAP = LazyObject(_character_map, globals(), '_CHARACTER_MAP')
+del _character_map
 
 
 def _cache_renamer(path, code=False):
@@ -133,7 +142,7 @@ def run_script_with_cache(filename, execer, glb=None, loc=None, mode='exec'):
         with open(filename, 'r') as f:
             code = f.read()
         ccode = compile_code(filename, code, execer, glb, loc, mode)
-        update_cache(ccode, cachefname) 
+        update_cache(ccode, cachefname)
     run_compiled_code(ccode, glb, loc, mode)
 
 
@@ -162,7 +171,7 @@ def code_cache_check(cachefname):
         with open(cachefname, 'rb') as cfile:
             ccode = marshal.load(cfile)
             run_cached = True
-    return run_cached, ccode 
+    return run_cached, ccode
 
 
 def run_code_with_cache(code, execer, glb=None, loc=None, mode='exec'):
@@ -178,5 +187,5 @@ def run_code_with_cache(code, execer, glb=None, loc=None, mode='exec'):
         run_cached, ccode = code_cache_check(cachefname)
     if not run_cached:
         ccode = compile_code(filename, code, execer, glb, loc, mode)
-        update_cache(ccode, cachefname) 
+        update_cache(ccode, cachefname)
     run_compiled_code(ccode, glb, loc, mode)