From 394b95bbc6a5642b9ef2bb8475f47216ac76a1b9 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Thu, 28 Jul 2016 00:30:32 -0400
Subject: [PATCH 001/190] Basic event system.

---
 xonsh/events.py | 110 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 110 insertions(+)
 create mode 100644 xonsh/events.py

diff --git a/xonsh/events.py b/xonsh/events.py
new file mode 100644
index 000000000..119c6f8ed
--- /dev/null
+++ b/xonsh/events.py
@@ -0,0 +1,110 @@
+"""
+Events for xonsh.
+
+In all likelihood, you want builtins.__xonsh_events__
+
+The best way to "declare" an event is something like::
+
+    __xonsh_events__.on_spam.__doc__ = "Comes with eggs"
+"""
+
+class Event(set):
+    """
+    A given event that handlers can register against.
+
+    Acts as a ``set`` for registered handlers.
+    """
+    def __init__(self, doc=None):
+        self.__doc__ = doc
+        self._pman = parent
+
+    def handler(self, callable):
+        """
+        Registers a handler. It's suggested to use this as a decorator.
+        """
+        self.add(callable)
+        return callable
+
+    def calleach(self, *pargs, **kwargs):
+        """
+        The core handler caller that all others build on.
+
+        This works as a generator. Each handler is called in turn and its
+        results are yielded.
+
+        If the generator is interupted, no further handlers are called.
+
+        The caller may send() new positional arguments (eg, to implement
+        modifying semantics). Keyword arguments cannot be modified this way.
+        """
+        for handler in self:
+            newargs = yield handler(*pargs, **kwargs)
+            if newargs is not None:
+                pargs = newargs
+
+    def __call__(self, *pargs, **kwargs):
+        """
+        The simplest use case: Calls each handler in turn with the provided
+        arguments and ignore the return values.
+        """
+        for _ in self.calleach(*pargs, **kwargs):
+            pass
+
+    def untilTrue(self, *pargs, **kwargs):
+        """
+        Calls each handler until one returns something truthy.
+
+        Returns that truthy value.
+        """
+        for rv in self.calleach(*pargs, **kwargs):
+            if rv: 
+                return rv
+
+    def untilFalse(self, *pargs, **kwargs):
+        """
+        Calls each handler until one returns something falsey.
+        """
+        for rv in self.calleach(*pargs, **kwargs):
+            if not rv:
+                return
+
+    def loopback(self, *pargs, **kwargs):
+        """
+        Calls each handler in turn. If it returns a value, the arguments are modified.
+
+        The final result is returned.
+
+        NOTE: Each handler must return the same number of values it was
+        passed, or nothing at all.
+        """
+        calls = self.calleach(*pargs, **kwargs)
+        newvals = calls.next()
+        while True:
+            if newvals is not None:
+                if len(pargs) == 1:
+                    pargs = newvals,
+                else:
+                    pargs = newvals
+            try:
+                newvals = calls.send(pargs)
+            except StopIteration:
+                break
+
+        if newvals is not None:
+            return newvals
+        else:
+            return pargs
+
+def Events:
+    """
+    Container for all events in a system.
+
+    Meant to be a singleton, but doesn't enforce that itself.
+
+    Each event is just an attribute. They're created dynamically on first use.
+    """
+
+    def __getattr__(self, name):
+        e = Event()
+        setattr(self, name, e)
+        return e

From eddd11ae863a7206ec6cc2c6b3f821d4e4d64bd7 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Thu, 28 Jul 2016 00:42:00 -0400
Subject: [PATCH 002/190] Add tests for the events system.

---
 tests/test_events.py | 77 ++++++++++++++++++++++++++++++++++++++++++++
 xonsh/events.py      |  7 ++--
 2 files changed, 81 insertions(+), 3 deletions(-)
 create mode 100644 tests/test_events.py

diff --git a/tests/test_events.py b/tests/test_events.py
new file mode 100644
index 000000000..4e14a10e6
--- /dev/null
+++ b/tests/test_events.py
@@ -0,0 +1,77 @@
+"""Event tests"""
+from xonsh.events import Events
+
+def test_calling():
+    e = Events()
+    e.test.__doc__ = "Test event"
+
+    called = False
+    @e.test.handler
+    def _(spam):
+        nonlocal called
+        called = spam
+
+    e.test("eggs")
+
+    assert called == "eggs"
+
+def test_untilTrue():
+    e = Events()
+    e.test.__doc__ = "Test event"
+
+    called = 0
+
+    @e.test.handler
+    def first():
+        nonlocal called
+        called += 1
+        return True
+
+    @e.test.handler
+    def second():
+        nonlocal called
+        called += 1
+        return True
+
+    e.test.untilTrue()
+
+    assert called == 1
+
+def test_untilFalse():
+    e = Events()
+    e.test.__doc__ = "Test event"
+
+    called = 0
+
+    @e.test.handler
+    def first():
+        nonlocal called
+        called += 1
+        return False
+
+    @e.test.handler
+    def second():
+        nonlocal called
+        called += 1
+        return False
+
+    e.test.untilFalse()
+
+    assert called == 1
+
+def test_loopback():
+    e = Events()
+    e.test.__doc__ = "Test event"
+
+    @e.test.handler
+    def first(num):
+        return num + 1
+
+    @e.test.handler
+    def second(num):
+        return num + 1
+
+    rv = e.test.loopback(0)
+
+    assert rv == 2
+
diff --git a/xonsh/events.py b/xonsh/events.py
index 119c6f8ed..a0e32788e 100644
--- a/xonsh/events.py
+++ b/xonsh/events.py
@@ -13,10 +13,11 @@ class Event(set):
     A given event that handlers can register against.
 
     Acts as a ``set`` for registered handlers.
+
+    Note that ordering is never guarenteed.
     """
     def __init__(self, doc=None):
         self.__doc__ = doc
-        self._pman = parent
 
     def handler(self, callable):
         """
@@ -78,7 +79,7 @@ class Event(set):
         passed, or nothing at all.
         """
         calls = self.calleach(*pargs, **kwargs)
-        newvals = calls.next()
+        newvals = next(calls)
         while True:
             if newvals is not None:
                 if len(pargs) == 1:
@@ -95,7 +96,7 @@ class Event(set):
         else:
             return pargs
 
-def Events:
+class Events:
     """
     Container for all events in a system.
 

From 57488e2b4ddb561bb5dc7de11f6ba35f5488ae04 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Thu, 28 Jul 2016 00:43:21 -0400
Subject: [PATCH 003/190] flake8

---
 xonsh/events.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/xonsh/events.py b/xonsh/events.py
index a0e32788e..7498bcb5f 100644
--- a/xonsh/events.py
+++ b/xonsh/events.py
@@ -8,6 +8,7 @@ The best way to "declare" an event is something like::
     __xonsh_events__.on_spam.__doc__ = "Comes with eggs"
 """
 
+
 class Event(set):
     """
     A given event that handlers can register against.
@@ -58,7 +59,7 @@ class Event(set):
         Returns that truthy value.
         """
         for rv in self.calleach(*pargs, **kwargs):
-            if rv: 
+            if rv:
                 return rv
 
     def untilFalse(self, *pargs, **kwargs):
@@ -96,6 +97,7 @@ class Event(set):
         else:
             return pargs
 
+
 class Events:
     """
     Container for all events in a system.

From 3bb099b05b27208d755b66c3732497a599d86077 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Thu, 28 Jul 2016 00:45:34 -0400
Subject: [PATCH 004/190] Create the standard, global event bus

---
 xonsh/built_ins.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py
index 6da0421d3..64393681e 100644
--- a/xonsh/built_ins.py
+++ b/xonsh/built_ins.py
@@ -35,6 +35,7 @@ from xonsh.tools import (
     XonshCalledProcessError, XonshBlockError
 )
 from xonsh.commands_cache import CommandsCache
+from xonsh.events import Events
 
 import xonsh.completers.init
 
@@ -714,6 +715,7 @@ def load_builtins(execer=None, config=None, login=False, ctx=None):
     builtins.__xonsh_ensure_list_of_strs__ = ensure_list_of_strs
     builtins.__xonsh_list_of_strs_or_callables__ = list_of_strs_or_callables
     builtins.__xonsh_completers__ = xonsh.completers.init.default_completers()
+    builtins.__xonsh_events__ = Events()
     # public built-ins
     builtins.XonshError = XonshError
     builtins.XonshBlockError = XonshBlockError

From acce6ed89a2a04390d3830d460c85534d71f06ba Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sun, 14 Aug 2016 12:14:50 -0400
Subject: [PATCH 005/190] initial m

---
 xonsh/parsers/base.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index a56004538..c6f4c60f1 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -1815,6 +1815,10 @@ class BaseParser(object):
         """trailer : LPAREN arglist_opt RPAREN"""
         p[0] = [p[2] or dict(args=[], keywords=[], starargs=None, kwargs=None)]
 
+    def p_trailer_bang_lparen(self, p):
+        """trailer : BANG_LPAREN macroarglist_opt RPAREN"""
+        p[0] = [p[2] or dict(args=[], keywords=[], starargs=None, kwargs=None)]
+
     def p_trailer_p3(self, p):
         """trailer : LBRACKET subscriptlist RBRACKET
                    | PERIOD NAME

From 0430524ee65a5635101fd3ae7dad811c89a13dbc Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sun, 14 Aug 2016 18:34:48 -0400
Subject: [PATCH 006/190] raw attempt

---
 xonsh/lexer.py        |  3 ++-
 xonsh/parsers/base.py | 19 +++++++++++++++++--
 xonsh/tokenize.py     |  9 +++++++--
 3 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/xonsh/lexer.py b/xonsh/lexer.py
index 6c491876e..72948d940 100644
--- a/xonsh/lexer.py
+++ b/xonsh/lexer.py
@@ -15,7 +15,7 @@ from xonsh.lazyasd import lazyobject
 from xonsh.platform import PYTHON_VERSION_INFO
 from xonsh.tokenize import (OP, IOREDIRECT, STRING, DOLLARNAME, NUMBER,
                             SEARCHPATH, NEWLINE, INDENT, DEDENT, NL, COMMENT, ENCODING,
-                            ENDMARKER, NAME, ERRORTOKEN, tokenize, TokenError)
+                            ENDMARKER, NAME, ERRORTOKEN, tokenize, TokenError, NOCOMMA)
 
 
 @lazyobject
@@ -62,6 +62,7 @@ def token_map():
         from xonsh.tokenize import ASYNC, AWAIT
         tm[ASYNC] = 'ASYNC'
         tm[AWAIT] = 'AWAIT'
+    tm[NOCOMMA] = 'NOCOMMA'
     return tm
 
 
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index c6f4c60f1..b8a3375ea 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -233,7 +233,7 @@ class BaseParser(object):
             'op_factor_list', 'trailer_list', 'testlist_comp',
             'yield_expr_or_testlist_comp', 'dictorsetmaker',
             'comma_subscript_list', 'test', 'sliceop', 'comp_iter',
-            'yield_arg', 'test_comma_list']
+            'yield_arg', 'test_comma_list', 'comma_nocomma_list', 'macroarglist']
         for rule in opt_rules:
             self._opt_rule(rule)
 
@@ -247,7 +247,7 @@ class BaseParser(object):
             'pm_term', 'op_factor', 'trailer', 'comma_subscript',
             'comma_expr_or_star_expr', 'comma_test', 'comma_argument',
             'comma_item', 'attr_period_name', 'test_comma',
-            'equals_yield_expr_or_testlist']
+            'equals_yield_expr_or_testlist', 'comma_nocomma']
         for rule in list_rules:
             self._list_rule(rule)
 
@@ -1831,6 +1831,21 @@ class BaseParser(object):
         """
         p[0] = [p[1]]
 
+    def p_comma_nocomma(self, p):
+        """comma_nocomma : COMMA NOCOMMA"""
+        p[0] = [p[2]]
+
+    def p_macroarglist(self, p):
+        """macroarglist : NOCOMMA comma_nocomma_list_opt comma_opt"""
+        p1, p2 = p[1], p[2]
+        if p2 is None:
+            elts = [p1]
+        else:
+            elts = [p1] + p2
+        p0 = ast.Tuple(elts=elts, ctx.load(), lineno=p1.lineno,
+                       col_offset=p1.col_offset)
+        p[0] = p0
+
     def p_subscriptlist(self, p):
         """subscriptlist : subscript comma_subscript_list_opt comma_opt"""
         p1, p2 = p[1], p[2]
diff --git a/xonsh/tokenize.py b/xonsh/tokenize.py
index 8ab583c79..e3f50610c 100644
--- a/xonsh/tokenize.py
+++ b/xonsh/tokenize.py
@@ -51,7 +51,7 @@ import token
 __all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding",
                            "NL", "untokenize", "ENCODING", "TokenInfo",
                            "TokenError", 'SEARCHPATH', 'ATDOLLAR', 'ATEQUAL',
-                           'DOLLARNAME', 'IOREDIRECT']
+                           'DOLLARNAME', 'IOREDIRECT', 'NOCOMMA']
 PY35 = PYTHON_VERSION_INFO >= (3, 5, 0)
 if PY35:
     ASYNC = token.ASYNC
@@ -85,6 +85,9 @@ N_TOKENS += 1
 ATEQUAL = N_TOKENS
 tok_name[N_TOKENS] = 'ATEQUAL'
 N_TOKENS += 1
+NOCOMMA = N_TOKENS
+tok_name[N_TOKENS] = 'NOCOMMA'
+N_TOKENS += 1
 _xonsh_tokens = {
     '?': 'QUESTION',
     '@=': 'ATEQUAL',
@@ -241,8 +244,10 @@ Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=", r"//=?", r"->",
 Bracket = '[][(){}]'
 Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]')
 Funny = group(Operator, Bracket, Special)
+NoComma = r"('.*'|\".*\"|'''.*'''|\"\"\".*\"\"\"|\(.*\)|\[.*\]|{.*}|[^,]*)*"
 
-PlainToken = group(IORedirect, Number, Funny, String, Name_RE, SearchPath)
+PlainToken = group(IORedirect, Number, Funny, String, Name_RE, SearchPath,
+                   NoComma)
 Token = Ignore + PlainToken
 
 # First (or only) line of ' or " string.

From 6f1fe6093039831bf6044a57e025f85681855dc2 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sun, 14 Aug 2016 18:36:48 -0400
Subject: [PATCH 007/190] newb error

---
 xonsh/parsers/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index b8a3375ea..fe9525923 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -1842,7 +1842,7 @@ class BaseParser(object):
             elts = [p1]
         else:
             elts = [p1] + p2
-        p0 = ast.Tuple(elts=elts, ctx.load(), lineno=p1.lineno,
+        p0 = ast.Tuple(elts=elts, ctx=ast.Load(), lineno=p1.lineno,
                        col_offset=p1.col_offset)
         p[0] = p0
 

From 49b2f8d7d8f2f570e6e887b8f3872627d40e2c4b Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sun, 14 Aug 2016 18:52:39 -0400
Subject: [PATCH 008/190] newb error

---
 tests/test_parser.py | 10 ++++++++--
 xonsh/tokenize.py    |  2 ++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/tests/test_parser.py b/tests/test_parser.py
index 56c13f051..8aa289cbb 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -58,10 +58,10 @@ def check_stmts(inp, run=True, mode='exec'):
         inp += '\n'
     check_ast(inp, run=run, mode=mode)
 
-def check_xonsh_ast(xenv, inp, run=True, mode='eval'):
+def check_xonsh_ast(xenv, inp, run=True, mode='eval', debug_level=0):
     __tracebackhide__ = True
     builtins.__xonsh_env__ = xenv
-    obs = PARSER.parse(inp)
+    obs = PARSER.parse(inp, debug_level=debug_level)
     if obs is None:
         return  # comment only
     bytecode = compile(obs, '<test-xonsh-ast>', mode)
@@ -1798,3 +1798,9 @@ def test_redirect_error_to_output(r, o):
     assert check_xonsh_ast({}, '$[echo "test" {} {}> test.txt]'.format(r, o), False)
     assert check_xonsh_ast({}, '$[< input.txt echo "test" {} {}> test.txt]'.format(r, o), False)
     assert check_xonsh_ast({}, '$[echo "test" {} {}> test.txt < input.txt]'.format(r, o), False)
+
+def test_macro_call_empty():
+    check_xonsh_ast({}, 'f!()', False)
+
+def test_macro_call_one_arg():
+    check_xonsh_ast({}, 'f!(x)', False, debug_level=100)
diff --git a/xonsh/tokenize.py b/xonsh/tokenize.py
index e3f50610c..23e6f1490 100644
--- a/xonsh/tokenize.py
+++ b/xonsh/tokenize.py
@@ -698,6 +698,8 @@ def _tokenize(readline, encoding):
                         yield stashed
                         stashed = None
                     yield TokenInfo(COMMENT, token, spos, epos, line)
+                elif re.match(NoComma, token):
+                    yield TokenInfo(NOCOMMA, token, spos, epos, line)
                 # Xonsh-specific Regex Globbing
                 elif re.match(SearchPath, token):
                     yield TokenInfo(SEARCHPATH, token, spos, epos, line)

From 356397d6982c5d07008334e606b5825bf6b82ad6 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Mon, 15 Aug 2016 12:02:15 -0400
Subject: [PATCH 009/190] tokenize edits

---
 xonsh/tokenize.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/xonsh/tokenize.py b/xonsh/tokenize.py
index 23e6f1490..e3f50610c 100644
--- a/xonsh/tokenize.py
+++ b/xonsh/tokenize.py
@@ -698,8 +698,6 @@ def _tokenize(readline, encoding):
                         yield stashed
                         stashed = None
                     yield TokenInfo(COMMENT, token, spos, epos, line)
-                elif re.match(NoComma, token):
-                    yield TokenInfo(NOCOMMA, token, spos, epos, line)
                 # Xonsh-specific Regex Globbing
                 elif re.match(SearchPath, token):
                     yield TokenInfo(SEARCHPATH, token, spos, epos, line)

From 9dcd0f9fbf9e6bf2c6b879afbaa1f4dfd1f7575d Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Mon, 15 Aug 2016 22:59:30 -0400
Subject: [PATCH 010/190] Add validators, plus some clarity and PEP8 fixes

---
 tests/test_events.py | 37 +++++++++++++++++++++++++++++++++----
 xonsh/events.py      | 34 ++++++++++++++++++++++++++--------
 2 files changed, 59 insertions(+), 12 deletions(-)

diff --git a/tests/test_events.py b/tests/test_events.py
index 4e14a10e6..a6001fa90 100644
--- a/tests/test_events.py
+++ b/tests/test_events.py
@@ -15,7 +15,7 @@ def test_calling():
 
     assert called == "eggs"
 
-def test_untilTrue():
+def test_until_true():
     e = Events()
     e.test.__doc__ = "Test event"
 
@@ -33,11 +33,11 @@ def test_untilTrue():
         called += 1
         return True
 
-    e.test.untilTrue()
+    e.test.until_true()
 
     assert called == 1
 
-def test_untilFalse():
+def test_until_false():
     e = Events()
     e.test.__doc__ = "Test event"
 
@@ -55,7 +55,7 @@ def test_untilFalse():
         called += 1
         return False
 
-    e.test.untilFalse()
+    e.test.until_false()
 
     assert called == 1
 
@@ -75,3 +75,32 @@ def test_loopback():
 
     assert rv == 2
 
+
+def test_validator():
+    e = Events()
+    e.test.__doc__ = "Test event"
+
+    called = 0
+
+    @e.test.handler
+    def first(n):
+        nonlocal called
+        called += 1
+        return False
+
+    @first.validator
+    def v(n):
+        return n == 'spam'
+
+    @e.test.handler
+    def second(n):
+        nonlocal called
+        called += 1
+        return False
+
+    e.test('egg')
+    assert called == 1
+
+    called = 0
+    e.test('spam')
+    assert called == 2
diff --git a/xonsh/events.py b/xonsh/events.py
index 7498bcb5f..da4b18c53 100644
--- a/xonsh/events.py
+++ b/xonsh/events.py
@@ -15,17 +15,33 @@ class Event(set):
 
     Acts as a ``set`` for registered handlers.
 
-    Note that ordering is never guarenteed.
+    Note that ordering is never guaranteed.
     """
     def __init__(self, doc=None):
         self.__doc__ = doc
 
-    def handler(self, callable):
+    def handler(self, func):
         """
         Registers a handler. It's suggested to use this as a decorator.
+
+        A decorator method is added to the handler, validator(). If a validator
+        function is added, it can filter if the handler will be considered. The
+        validator takes the same arguments as the handler. If it returns False,
+        the handler will not called or considered, as if it was not registered
+        at all.
         """
-        self.add(callable)
-        return callable
+        #  Using Pythons "private" munging to minimize hypothetical collisions
+        func.__validator = None
+        self.add(func)
+
+        def validator(vfunc):
+            """
+            Adds a validator function to a handler to limit when it is considered.
+            """
+            func.__validator = vfunc
+        func.validator = validator
+
+        return func
 
     def calleach(self, *pargs, **kwargs):
         """
@@ -40,6 +56,8 @@ class Event(set):
         modifying semantics). Keyword arguments cannot be modified this way.
         """
         for handler in self:
+            if handler.__validator is not None and not handler.__validator(*pargs, **kwargs):
+                continue
             newargs = yield handler(*pargs, **kwargs)
             if newargs is not None:
                 pargs = newargs
@@ -52,7 +70,7 @@ class Event(set):
         for _ in self.calleach(*pargs, **kwargs):
             pass
 
-    def untilTrue(self, *pargs, **kwargs):
+    def until_true(self, *pargs, **kwargs):
         """
         Calls each handler until one returns something truthy.
 
@@ -62,13 +80,13 @@ class Event(set):
             if rv:
                 return rv
 
-    def untilFalse(self, *pargs, **kwargs):
+    def until_false(self, *pargs, **kwargs):
         """
         Calls each handler until one returns something falsey.
         """
         for rv in self.calleach(*pargs, **kwargs):
             if not rv:
-                return
+                return rv
 
     def loopback(self, *pargs, **kwargs):
         """
@@ -84,7 +102,7 @@ class Event(set):
         while True:
             if newvals is not None:
                 if len(pargs) == 1:
-                    pargs = newvals,
+                    pargs = (newvals,)
                 else:
                     pargs = newvals
             try:

From 44333fdcd525629143f67aab195280ad733b8c1d Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Mon, 15 Aug 2016 23:03:32 -0400
Subject: [PATCH 011/190] Remove problematic "loopback" event style.

This has limited use because you have to do it in such a way that different
handlers don't step on each other or blow up on input they don't understand.
The worst case is the first use-case: command preprocessing.
---
 tests/test_events.py | 17 -----------------
 xonsh/events.py      | 32 +-------------------------------
 2 files changed, 1 insertion(+), 48 deletions(-)

diff --git a/tests/test_events.py b/tests/test_events.py
index a6001fa90..6d0eec88c 100644
--- a/tests/test_events.py
+++ b/tests/test_events.py
@@ -59,23 +59,6 @@ def test_until_false():
 
     assert called == 1
 
-def test_loopback():
-    e = Events()
-    e.test.__doc__ = "Test event"
-
-    @e.test.handler
-    def first(num):
-        return num + 1
-
-    @e.test.handler
-    def second(num):
-        return num + 1
-
-    rv = e.test.loopback(0)
-
-    assert rv == 2
-
-
 def test_validator():
     e = Events()
     e.test.__doc__ = "Test event"
diff --git a/xonsh/events.py b/xonsh/events.py
index da4b18c53..3e6064740 100644
--- a/xonsh/events.py
+++ b/xonsh/events.py
@@ -58,9 +58,7 @@ class Event(set):
         for handler in self:
             if handler.__validator is not None and not handler.__validator(*pargs, **kwargs):
                 continue
-            newargs = yield handler(*pargs, **kwargs)
-            if newargs is not None:
-                pargs = newargs
+            yield handler(*pargs, **kwargs)
 
     def __call__(self, *pargs, **kwargs):
         """
@@ -88,34 +86,6 @@ class Event(set):
             if not rv:
                 return rv
 
-    def loopback(self, *pargs, **kwargs):
-        """
-        Calls each handler in turn. If it returns a value, the arguments are modified.
-
-        The final result is returned.
-
-        NOTE: Each handler must return the same number of values it was
-        passed, or nothing at all.
-        """
-        calls = self.calleach(*pargs, **kwargs)
-        newvals = next(calls)
-        while True:
-            if newvals is not None:
-                if len(pargs) == 1:
-                    pargs = (newvals,)
-                else:
-                    pargs = newvals
-            try:
-                newvals = calls.send(pargs)
-            except StopIteration:
-                break
-
-        if newvals is not None:
-            return newvals
-        else:
-            return pargs
-
-
 class Events:
     """
     Container for all events in a system.

From 81d066a341c1739017a42dcc2e2432b02b5221ee Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Mon, 15 Aug 2016 23:13:44 -0400
Subject: [PATCH 012/190] Rename so that __call__() is to register

```
@events.on_spam
def myhandler(eggs):
    ...
```
---
 tests/test_events.py | 34 +++++++++++++++++-----------------
 xonsh/events.py      |  5 +++--
 2 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/tests/test_events.py b/tests/test_events.py
index 6d0eec88c..24b8c23d1 100644
--- a/tests/test_events.py
+++ b/tests/test_events.py
@@ -3,69 +3,69 @@ from xonsh.events import Events
 
 def test_calling():
     e = Events()
-    e.test.__doc__ = "Test event"
+    e.on_test.__doc__ = "Test event"
 
     called = False
-    @e.test.handler
+    @e.on_test
     def _(spam):
         nonlocal called
         called = spam
 
-    e.test("eggs")
+    e.on_test.fire("eggs")
 
     assert called == "eggs"
 
 def test_until_true():
     e = Events()
-    e.test.__doc__ = "Test event"
+    e.on_test.__doc__ = "Test event"
 
     called = 0
 
-    @e.test.handler
-    def first():
+    @e.test
+    def on_test():
         nonlocal called
         called += 1
         return True
 
-    @e.test.handler
+    @e.on_test
     def second():
         nonlocal called
         called += 1
         return True
 
-    e.test.until_true()
+    e.on_test.until_true()
 
     assert called == 1
 
 def test_until_false():
     e = Events()
-    e.test.__doc__ = "Test event"
+    e.on_test.__doc__ = "Test event"
 
     called = 0
 
-    @e.test.handler
+    @e.on_test
     def first():
         nonlocal called
         called += 1
         return False
 
-    @e.test.handler
+    @e.on_test
     def second():
         nonlocal called
         called += 1
         return False
 
-    e.test.until_false()
+    e.on_test.until_false()
 
     assert called == 1
 
 def test_validator():
     e = Events()
-    e.test.__doc__ = "Test event"
+    e.on_test.__doc__ = "Test event"
 
     called = 0
 
-    @e.test.handler
+    @e.on_test
     def first(n):
         nonlocal called
         called += 1
@@ -75,15 +75,15 @@ def test_validator():
     def v(n):
         return n == 'spam'
 
-    @e.test.handler
+    @e.on_test
     def second(n):
         nonlocal called
         called += 1
         return False
 
-    e.test('egg')
+    e.on_test.fire('egg')
     assert called == 1
 
     called = 0
-    e.test('spam')
+    e.on_test.fire('spam')
     assert called == 2
diff --git a/xonsh/events.py b/xonsh/events.py
index 3e6064740..389bd4a4b 100644
--- a/xonsh/events.py
+++ b/xonsh/events.py
@@ -20,7 +20,7 @@ class Event(set):
     def __init__(self, doc=None):
         self.__doc__ = doc
 
-    def handler(self, func):
+    def __call__(self, func):
         """
         Registers a handler. It's suggested to use this as a decorator.
 
@@ -60,7 +60,7 @@ class Event(set):
                 continue
             yield handler(*pargs, **kwargs)
 
-    def __call__(self, *pargs, **kwargs):
+    def fire(self, *pargs, **kwargs):
         """
         The simplest use case: Calls each handler in turn with the provided
         arguments and ignore the return values.
@@ -86,6 +86,7 @@ class Event(set):
             if not rv:
                 return rv
 
+
 class Events:
     """
     Container for all events in a system.

From 5141fadfb4c3d85c2904f1c6b8bf257bf860f8ef Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Mon, 15 Aug 2016 23:22:19 -0400
Subject: [PATCH 013/190] Do some magic to make event docstrings work

---
 tests/test_events.py | 16 ++++++++++++----
 xonsh/events.py      | 10 ++++++----
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/tests/test_events.py b/tests/test_events.py
index 24b8c23d1..0b388fe53 100644
--- a/tests/test_events.py
+++ b/tests/test_events.py
@@ -3,7 +3,7 @@ from xonsh.events import Events
 
 def test_calling():
     e = Events()
-    e.on_test.__doc__ = "Test event"
+    e.on_test.doc("Test event")
 
     called = False
     @e.on_test
@@ -17,7 +17,7 @@ def test_calling():
 
 def test_until_true():
     e = Events()
-    e.on_test.__doc__ = "Test event"
+    e.on_test.doc("Test event")
 
     called = 0
 
@@ -39,7 +39,7 @@ def test_until_true():
 
 def test_until_false():
     e = Events()
-    e.on_test.__doc__ = "Test event"
+    e.on_test.doc("Test event")
 
     called = 0
 
@@ -61,7 +61,7 @@ def test_until_false():
 
 def test_validator():
     e = Events()
-    e.on_test.__doc__ = "Test event"
+    e.on_test.doc("Test event")
 
     called = 0
 
@@ -87,3 +87,11 @@ def test_validator():
     called = 0
     e.on_test.fire('spam')
     assert called == 2
+
+def test_eventdoc():
+    docstring = "Test event"
+    e = Events()
+    e.on_test.doc(docstring)
+
+    import inspect
+    assert inspect.getdoc(e.on_test) == docstring
\ No newline at end of file
diff --git a/xonsh/events.py b/xonsh/events.py
index 389bd4a4b..76184483e 100644
--- a/xonsh/events.py
+++ b/xonsh/events.py
@@ -5,7 +5,7 @@ In all likelihood, you want builtins.__xonsh_events__
 
 The best way to "declare" an event is something like::
 
-    __xonsh_events__.on_spam.__doc__ = "Comes with eggs"
+    __xonsh_events__.on_spam.doc("Comes with eggs")
 """
 
 
@@ -17,8 +17,10 @@ class Event(set):
 
     Note that ordering is never guaranteed.
     """
-    def __init__(self, doc=None):
-        self.__doc__ = doc
+
+    @classmethod
+    def doc(cls, text):
+        cls.__doc__ = text
 
     def __call__(self, func):
         """
@@ -97,6 +99,6 @@ class Events:
     """
 
     def __getattr__(self, name):
-        e = Event()
+        e = type(name, (Event,), {'__doc__': None})()
         setattr(self, name, e)
         return e

From 13d1dcdaad41fd7c2d1ea2b4ec391804d80eb88e Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Tue, 16 Aug 2016 00:06:28 -0400
Subject: [PATCH 014/190] Outdated docstring

---
 xonsh/events.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/xonsh/events.py b/xonsh/events.py
index 76184483e..15a435cb7 100644
--- a/xonsh/events.py
+++ b/xonsh/events.py
@@ -53,9 +53,6 @@ class Event(set):
         results are yielded.
 
         If the generator is interupted, no further handlers are called.
-
-        The caller may send() new positional arguments (eg, to implement
-        modifying semantics). Keyword arguments cannot be modified this way.
         """
         for handler in self:
             if handler.__validator is not None and not handler.__validator(*pargs, **kwargs):

From 4ff14f50946e02671034948ad346d313bb59ea47 Mon Sep 17 00:00:00 2001
From: adam j hartz <hartz@mit.edu>
Date: Fri, 19 Aug 2016 15:11:34 -0400
Subject: [PATCH 015/190] first go at new resolution for additional syntax
 errors

---
 xonsh/parsers/base.py          |  3 ++
 xonsh/parsers/context_check.py | 81 ++++++++++++++++++++++++++++++++++
 2 files changed, 84 insertions(+)
 create mode 100644 xonsh/parsers/context_check.py

diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index a2e037430..b56e2ae1b 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -17,6 +17,7 @@ from xonsh.lexer import Lexer, LexToken
 from xonsh.platform import PYTHON_VERSION_INFO
 from xonsh.tokenize import SearchPath
 from xonsh.lazyasd import LazyObject
+from xonsh.parsers.context_check import check_contexts
 
 RE_SEARCHPATH = LazyObject(lambda: re.compile(SearchPath), globals(),
                            'RE_SEARCHPATH')
@@ -309,6 +310,8 @@ class BaseParser(object):
         while self.parser is None:
             time.sleep(0.01)  # block until the parser is ready
         tree = self.parser.parse(input=s, lexer=self.lexer, debug=debug_level)
+        if tree is not None:
+            check_contexts(tree)
         # hack for getting modes right
         if mode == 'single':
             if isinstance(tree, ast.Expression):
diff --git a/xonsh/parsers/context_check.py b/xonsh/parsers/context_check.py
new file mode 100644
index 000000000..74b6ed1a3
--- /dev/null
+++ b/xonsh/parsers/context_check.py
@@ -0,0 +1,81 @@
+import ast
+import keyword
+import collections
+
+_all_keywords = frozenset(keyword.kwlist)
+
+def _not_assignable(x, augassign=False):
+    """
+    If ``x`` represents a value that can be assigned to, return ``None``.
+    Otherwise, return a string describing the object.  For use in generating
+    meaningful syntax errors.
+    """
+    if augassign and isinstance(x, (ast.Tuple, ast.List)):
+        return 'literal'
+    elif isinstance(x, (ast.Tuple, ast.List)):
+        if len(x.elts) == 0:
+            return '()'
+        for i in x.elts:
+            res = _not_assignable(i)
+            if res is not None:
+                return res
+    elif isinstance(x, (ast.Set, ast.Dict, ast.Num, ast.Str, ast.Bytes)):
+        return 'literal'
+    elif isinstance(x, ast.Call):
+        return 'function call'
+    elif isinstance(x, ast.Lambda):
+        return 'lambda'
+    elif isinstance(x, (ast.BoolOp, ast.BinOp, ast.UnaryOp)):
+        return 'operator'
+    elif isinstance(x, ast.IfExp):
+        return 'conditional expression'
+    elif isinstance(x, ast.ListComp):
+        return 'list comprehension'
+    elif isinstance(x, ast.DictComp):
+        return 'dictionary comprehension'
+    elif isinstance(x, ast.SetComp):
+        return 'set comprehension'
+    elif isinstance(x, ast.GeneratorExp):
+        return 'generator expression'
+    elif isinstance(x, ast.Compare):
+        return 'comparison'
+    elif isinstance(x, ast.Name) and x.id in _all_keywords:
+        return 'keyword'
+    elif isinstance(x, ast.NameConstant):
+        return 'keyword'
+
+_loc = collections.namedtuple('_loc', ['lineno', 'column'])
+
+def check_contexts(tree):
+    c = ContextCheckingVisitor()
+    c.visit(tree)
+    if c.error is not None:
+        e = SyntaxError(c.error[0])
+        e.loc = _loc(c.error[1], c.error[2])
+        raise e
+
+class ContextCheckingVisitor(ast.NodeVisitor):
+    def __init__(self):
+        self.error = None
+
+    def visit_Delete(self, node):
+        for i in node.targets:
+            err = _not_assignable(i)
+            if err is not None:
+                msg = "can't delete {}".format(err)
+                self.error = msg, i.lineno, i.col_offset
+                break
+
+    def visit_Assign(self, node):
+        for i in node.targets:
+            err = _not_assignable(i)
+            if err is not None:
+                msg = "can't assign to {}".format(err)
+                self.error = msg, i.lineno, i.col_offset
+                break
+    
+    def visit_AugAssign(self, node):
+       err = _not_assignable(node.target, True)
+       if err is not None:
+           msg = "illegal target for augmented assignment: {}".format(err)
+           self.error = msg, node.target.lineno, node.target.col_offset

From 199f889cf6b95d8fee744ffdbae365398eaf97cf Mon Sep 17 00:00:00 2001
From: adam j hartz <hartz@mit.edu>
Date: Fri, 19 Aug 2016 15:15:11 -0400
Subject: [PATCH 016/190] slightly better error message reporting for syntax
 errors

---
 xonsh/execer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xonsh/execer.py b/xonsh/execer.py
index 5910bb20e..f35a0a894 100644
--- a/xonsh/execer.py
+++ b/xonsh/execer.py
@@ -163,7 +163,7 @@ class Execer(object):
                 if (e.loc is None) or (last_error_line == e.loc.lineno and
                                        last_error_col in (e.loc.column + 1,
                                                           e.loc.column)):
-                    raise original_error
+                    raise original_error from None
                 last_error_col = e.loc.column
                 last_error_line = e.loc.lineno
                 idx = last_error_line - 1

From bab5895171319ab6c9dfa2938c27806ea180e5e8 Mon Sep 17 00:00:00 2001
From: adam j hartz <hartz@mit.edu>
Date: Fri, 19 Aug 2016 15:16:22 -0400
Subject: [PATCH 017/190] news entry

---
 news/more_syntax_errors.rst | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 news/more_syntax_errors.rst

diff --git a/news/more_syntax_errors.rst b/news/more_syntax_errors.rst
new file mode 100644
index 000000000..3f04ce0e7
--- /dev/null
+++ b/news/more_syntax_errors.rst
@@ -0,0 +1,13 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:**
+
+* xonsh now properly handles syntax error messages arising from using values in inappropriate contexts (e.g., ``del 7``).
+
+**Security:** None

From 959270c0d2d143177629ca42178e152312f28c5d Mon Sep 17 00:00:00 2001
From: adam j hartz <hartz@mit.edu>
Date: Fri, 19 Aug 2016 15:43:26 -0400
Subject: [PATCH 018/190] add tests

---
 tests/test_parser.py | 150 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 150 insertions(+)

diff --git a/tests/test_parser.py b/tests/test_parser.py
index 757a74c3f..7dc1c6bf2 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -1782,3 +1782,153 @@ def test_redirect_error_to_output(r, o):
     assert check_xonsh_ast({}, '$[echo "test" {} {}> test.txt]'.format(r, o), False)
     assert check_xonsh_ast({}, '$[< input.txt echo "test" {} {}> test.txt]'.format(r, o), False)
     assert check_xonsh_ast({}, '$[echo "test" {} {}> test.txt < input.txt]'.format(r, o), False)
+
+# test invalid expressions
+
+def test_syntax_error_del_literal():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('del 7')
+
+def test_syntax_error_del_constant():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('del True')
+
+def test_syntax_error_del_emptytuple():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('del ()')
+
+def test_syntax_error_del_call():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('del foo()')
+
+def test_syntax_error_del_lambda():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('del lambda x: "yay"')
+
+def test_syntax_error_del_ifexp():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('del x if y else z')
+
+def test_syntax_error_del_comps():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('del [i for i in foo]')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('del {i for i in foo}')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('del {k:v for k,v in d.items()}')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('del (i for i in foo)')
+
+def test_syntax_error_del_ops():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('del x + y')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('del x and y')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('del -y')
+
+def test_syntax_error_del_cmp():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('del x > y')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('del x > y > z')
+
+def test_syntax_error_lonely_del():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('del')
+
+def test_syntax_error_assign_literal():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('7 = x')
+
+def test_syntax_error_assign_constant():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('True = 8')
+
+def test_syntax_error_assign_emptytuple():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('() = x')
+
+def test_syntax_error_assign_call():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('foo() = x')
+
+def test_syntax_error_assign_lambda():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('lambda x: "yay" = y')
+
+def test_syntax_error_assign_ifexp():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('x if y else z = 8')
+
+def test_syntax_error_assign_comps():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('[i for i in foo] = y')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('{i for i in foo} = y')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('{k:v for k,v in d.items()} = y')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('(k for k in d) = y')
+
+def test_syntax_error_assign_ops():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('x + y = z')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('x and y = z')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('-y = z')
+
+def test_syntax_error_assign_cmp():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('x > y = z')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('x > y > z = a')
+
+def test_syntax_error_augassign_literal():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('7 += x')
+
+def test_syntax_error_augassign_constant():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('True += 8')
+
+def test_syntax_error_augassign_emptytuple():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('() += x')
+
+def test_syntax_error_augassign_call():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('foo() += x')
+
+def test_syntax_error_augassign_lambda():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('lambda x: "yay" += y')
+
+def test_syntax_error_augassign_ifexp():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('x if y else z += 8')
+
+def test_syntax_error_augassign_comps():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('[i for i in foo] += y')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('{i for i in foo} += y')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('{k:v for k,v in d.items()} += y')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('(k for k in d) += y')
+
+def test_syntax_error_augassign_ops():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('x + y += z')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('x and y += z')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('-y += z')
+
+def test_syntax_error_augassign_cmp():
+    with pytest.raises(SyntaxError):
+        PARSER.parse('x > y += z')
+    with pytest.raises(SyntaxError):
+        PARSER.parse('x > y > z += a')

From 6be5388ddd7f034011b6df71ce5929744f711863 Mon Sep 17 00:00:00 2001
From: adam j hartz <hartz@mit.edu>
Date: Fri, 19 Aug 2016 15:53:27 -0400
Subject: [PATCH 019/190] pep8 formatting

---
 xonsh/parsers/context_check.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/xonsh/parsers/context_check.py b/xonsh/parsers/context_check.py
index 74b6ed1a3..5e6453f93 100644
--- a/xonsh/parsers/context_check.py
+++ b/xonsh/parsers/context_check.py
@@ -4,6 +4,7 @@ import collections
 
 _all_keywords = frozenset(keyword.kwlist)
 
+
 def _not_assignable(x, augassign=False):
     """
     If ``x`` represents a value that can be assigned to, return ``None``.
@@ -46,6 +47,7 @@ def _not_assignable(x, augassign=False):
 
 _loc = collections.namedtuple('_loc', ['lineno', 'column'])
 
+
 def check_contexts(tree):
     c = ContextCheckingVisitor()
     c.visit(tree)
@@ -54,6 +56,7 @@ def check_contexts(tree):
         e.loc = _loc(c.error[1], c.error[2])
         raise e
 
+
 class ContextCheckingVisitor(ast.NodeVisitor):
     def __init__(self):
         self.error = None
@@ -73,9 +76,9 @@ class ContextCheckingVisitor(ast.NodeVisitor):
                 msg = "can't assign to {}".format(err)
                 self.error = msg, i.lineno, i.col_offset
                 break
-    
+
     def visit_AugAssign(self, node):
-       err = _not_assignable(node.target, True)
-       if err is not None:
-           msg = "illegal target for augmented assignment: {}".format(err)
-           self.error = msg, node.target.lineno, node.target.col_offset
+        err = _not_assignable(node.target, True)
+        if err is not None:
+            msg = "illegal target for augmented assignment: {}".format(err)
+            self.error = msg, node.target.lineno, node.target.col_offset

From f9d4c9d3e77a37e53e8a378815785674c9247c9d Mon Sep 17 00:00:00 2001
From: adam j hartz <hartz@mit.edu>
Date: Fri, 19 Aug 2016 16:07:08 -0400
Subject: [PATCH 020/190] better test organization

---
 tests/test_parser.py | 109 ++++++++++++++++++++++---------------------
 1 file changed, 55 insertions(+), 54 deletions(-)

diff --git a/tests/test_parser.py b/tests/test_parser.py
index 7dc1c6bf2..414bde94a 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -1809,29 +1809,29 @@ def test_syntax_error_del_ifexp():
     with pytest.raises(SyntaxError):
         PARSER.parse('del x if y else z')
 
-def test_syntax_error_del_comps():
-    with pytest.raises(SyntaxError):
-        PARSER.parse('del [i for i in foo]')
-    with pytest.raises(SyntaxError):
-        PARSER.parse('del {i for i in foo}')
-    with pytest.raises(SyntaxError):
-        PARSER.parse('del {k:v for k,v in d.items()}')
-    with pytest.raises(SyntaxError):
-        PARSER.parse('del (i for i in foo)')
 
-def test_syntax_error_del_ops():
+@pytest.mark.parametrize('exp', ['[i for i in foo]',
+                                 '{i for i in foo}',
+                                 '(i for i in foo)',
+                                 '{k:v for k,v in d.items()}'])
+def test_syntax_error_del_comps(exp):
     with pytest.raises(SyntaxError):
-        PARSER.parse('del x + y')
-    with pytest.raises(SyntaxError):
-        PARSER.parse('del x and y')
-    with pytest.raises(SyntaxError):
-        PARSER.parse('del -y')
+        PARSER.parse('del {}'.format(exp))
 
-def test_syntax_error_del_cmp():
+
+@pytest.mark.parametrize('exp', ['x + y',
+                                 'x and y',
+                                 '-x'])
+def test_syntax_error_del_ops(exp):
     with pytest.raises(SyntaxError):
-        PARSER.parse('del x > y')
+        PARSER.parse('del {}'.format(exp))
+
+
+@pytest.mark.parametrize('exp', ['x > y',
+                                 'x > y == z'])
+def test_syntax_error_del_cmp(exp):
     with pytest.raises(SyntaxError):
-        PARSER.parse('del x > y > z')
+        PARSER.parse('del {}'.format(exp))
 
 def test_syntax_error_lonely_del():
     with pytest.raises(SyntaxError):
@@ -1861,29 +1861,30 @@ def test_syntax_error_assign_ifexp():
     with pytest.raises(SyntaxError):
         PARSER.parse('x if y else z = 8')
 
-def test_syntax_error_assign_comps():
-    with pytest.raises(SyntaxError):
-        PARSER.parse('[i for i in foo] = y')
-    with pytest.raises(SyntaxError):
-        PARSER.parse('{i for i in foo} = y')
-    with pytest.raises(SyntaxError):
-        PARSER.parse('{k:v for k,v in d.items()} = y')
-    with pytest.raises(SyntaxError):
-        PARSER.parse('(k for k in d) = y')
 
-def test_syntax_error_assign_ops():
+@pytest.mark.parametrize('exp', ['[i for i in foo]',
+                                 '{i for i in foo}',
+                                 '(i for i in foo)',
+                                 '{k:v for k,v in d.items()}'])
+def test_syntax_error_assign_comps(exp):
     with pytest.raises(SyntaxError):
-        PARSER.parse('x + y = z')
-    with pytest.raises(SyntaxError):
-        PARSER.parse('x and y = z')
-    with pytest.raises(SyntaxError):
-        PARSER.parse('-y = z')
+        PARSER.parse('{} = z'.format(exp))
 
-def test_syntax_error_assign_cmp():
+
+@pytest.mark.parametrize('exp', ['x + y',
+                                 'x and y',
+                                 '-x'])
+def test_syntax_error_assign_ops(exp):
     with pytest.raises(SyntaxError):
-        PARSER.parse('x > y = z')
+        PARSER.parse('{} = z'.format(exp))
+
+
+@pytest.mark.parametrize('exp', ['x > y',
+                                 'x > y == z'])
+def test_syntax_error_assign_cmp(exp):
     with pytest.raises(SyntaxError):
-        PARSER.parse('x > y > z = a')
+        PARSER.parse('{} = a'.format(exp))
+
 
 def test_syntax_error_augassign_literal():
     with pytest.raises(SyntaxError):
@@ -1909,26 +1910,26 @@ def test_syntax_error_augassign_ifexp():
     with pytest.raises(SyntaxError):
         PARSER.parse('x if y else z += 8')
 
-def test_syntax_error_augassign_comps():
-    with pytest.raises(SyntaxError):
-        PARSER.parse('[i for i in foo] += y')
-    with pytest.raises(SyntaxError):
-        PARSER.parse('{i for i in foo} += y')
-    with pytest.raises(SyntaxError):
-        PARSER.parse('{k:v for k,v in d.items()} += y')
-    with pytest.raises(SyntaxError):
-        PARSER.parse('(k for k in d) += y')
 
-def test_syntax_error_augassign_ops():
+@pytest.mark.parametrize('exp', ['[i for i in foo]',
+                                 '{i for i in foo}',
+                                 '(i for i in foo)',
+                                 '{k:v for k,v in d.items()}'])
+def test_syntax_error_augassign_comps(exp):
     with pytest.raises(SyntaxError):
-        PARSER.parse('x + y += z')
-    with pytest.raises(SyntaxError):
-        PARSER.parse('x and y += z')
-    with pytest.raises(SyntaxError):
-        PARSER.parse('-y += z')
+        PARSER.parse('{} += z'.format(exp))
 
-def test_syntax_error_augassign_cmp():
+
+@pytest.mark.parametrize('exp', ['x + y',
+                                 'x and y',
+                                 '-x'])
+def test_syntax_error_augassign_ops(exp):
     with pytest.raises(SyntaxError):
-        PARSER.parse('x > y += z')
+        PARSER.parse('{} += z'.format(exp))
+
+
+@pytest.mark.parametrize('exp', ['x > y',
+                                 'x > y +=+= z'])
+def test_syntax_error_augassign_cmp(exp):
     with pytest.raises(SyntaxError):
-        PARSER.parse('x > y > z += a')
+        PARSER.parse('{} += a'.format(exp))

From 3d1e2deae7a631c2b844d5f59a93fc18007bc11a Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 04:40:54 -0400
Subject: [PATCH 021/190] remove bad pathway

---
 xonsh/lexer.py    | 3 +--
 xonsh/tokenize.py | 9 ++-------
 2 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/xonsh/lexer.py b/xonsh/lexer.py
index 72948d940..6c491876e 100644
--- a/xonsh/lexer.py
+++ b/xonsh/lexer.py
@@ -15,7 +15,7 @@ from xonsh.lazyasd import lazyobject
 from xonsh.platform import PYTHON_VERSION_INFO
 from xonsh.tokenize import (OP, IOREDIRECT, STRING, DOLLARNAME, NUMBER,
                             SEARCHPATH, NEWLINE, INDENT, DEDENT, NL, COMMENT, ENCODING,
-                            ENDMARKER, NAME, ERRORTOKEN, tokenize, TokenError, NOCOMMA)
+                            ENDMARKER, NAME, ERRORTOKEN, tokenize, TokenError)
 
 
 @lazyobject
@@ -62,7 +62,6 @@ def token_map():
         from xonsh.tokenize import ASYNC, AWAIT
         tm[ASYNC] = 'ASYNC'
         tm[AWAIT] = 'AWAIT'
-    tm[NOCOMMA] = 'NOCOMMA'
     return tm
 
 
diff --git a/xonsh/tokenize.py b/xonsh/tokenize.py
index e3f50610c..8ab583c79 100644
--- a/xonsh/tokenize.py
+++ b/xonsh/tokenize.py
@@ -51,7 +51,7 @@ import token
 __all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding",
                            "NL", "untokenize", "ENCODING", "TokenInfo",
                            "TokenError", 'SEARCHPATH', 'ATDOLLAR', 'ATEQUAL',
-                           'DOLLARNAME', 'IOREDIRECT', 'NOCOMMA']
+                           'DOLLARNAME', 'IOREDIRECT']
 PY35 = PYTHON_VERSION_INFO >= (3, 5, 0)
 if PY35:
     ASYNC = token.ASYNC
@@ -85,9 +85,6 @@ N_TOKENS += 1
 ATEQUAL = N_TOKENS
 tok_name[N_TOKENS] = 'ATEQUAL'
 N_TOKENS += 1
-NOCOMMA = N_TOKENS
-tok_name[N_TOKENS] = 'NOCOMMA'
-N_TOKENS += 1
 _xonsh_tokens = {
     '?': 'QUESTION',
     '@=': 'ATEQUAL',
@@ -244,10 +241,8 @@ Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=", r"//=?", r"->",
 Bracket = '[][(){}]'
 Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]')
 Funny = group(Operator, Bracket, Special)
-NoComma = r"('.*'|\".*\"|'''.*'''|\"\"\".*\"\"\"|\(.*\)|\[.*\]|{.*}|[^,]*)*"
 
-PlainToken = group(IORedirect, Number, Funny, String, Name_RE, SearchPath,
-                   NoComma)
+PlainToken = group(IORedirect, Number, Funny, String, Name_RE, SearchPath)
 Token = Ignore + PlainToken
 
 # First (or only) line of ' or " string.

From 2b829d05570cb648defb12d6a49d6d70c886c3c4 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 06:12:35 -0400
Subject: [PATCH 022/190] "a little further""

---
 xonsh/parsers/base.py | 56 +++++++++++++++++++++++++++++++++++++++----
 1 file changed, 52 insertions(+), 4 deletions(-)

diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index 753d88b18..f97844318 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -219,6 +219,8 @@ class BaseParser(object):
         self.lexer = lexer = Lexer()
         self.tokens = lexer.tokens
 
+        self._attach_nocomma_tok_rules()
+
         opt_rules = [
             'newlines', 'arglist', 'func_call', 'rarrow_test', 'typedargslist',
             'equals_test', 'colon_test', 'tfpdef', 'comma_tfpdef_list',
@@ -1836,19 +1838,65 @@ class BaseParser(object):
         """
         p[0] = [p[1]]
 
+    def _attach_nocomma_tok_rules(self):
+        toks = sorted(self.tokens)
+        toks.remove('COMMA')
+        toks.remove('RPAREN')
+        ts = '\n             | '.join(toks)
+        doc = 'nocomma_tok : ' + ts + '\n'
+        self.p_nocomma_tok.__func__.__doc__ = doc
+
+    def p_nocomma_tok(self, p):
+        # see attachement function above for docstring
+        p[0] = p[1]
+
+    def p_any_raw_tok(self, p):
+        """any_raw_tok : nocomma
+                       | COMMA
+                       | RPAREN
+        """
+        p[0] = p[1]
+
+    def p_any_raw_toks_one(self, p):
+        """any_raw_toks : any_raw_tok"""
+        p[0] = p[1]
+
+    def p_any_raw_toks_many(self, p):
+        """any_raw_toks : any_raw_toks any_raw_tok"""
+        p[0] = p[1] + p[2]
+
+    def p_nocomma_part_tok(self, p):
+        """nocomma_part : nocomma_tok"""
+        p[0] = p[1]
+
+    def p_nocomma_part_any(self, p):
+        """nocomma_part : LPAREN any_raw_toks RPAREN
+                        | AT_LPAREN any_raw_toks RPAREN
+                        | BANG_LPAREN any_raw_toks RPAREN
+        """
+        p[0] = p[1] + p[2]
+
+    def p_nocomma_base(self, p):
+        """nocomma : nocomma_part"""
+        p[0] = p[1]
+
+    def p_nocomma_append(self, p):
+        """nocomma : nocomma nocomma_part"""
+        p[0] = p[1] + p[2]
+
     def p_comma_nocomma(self, p):
-        """comma_nocomma : COMMA NOCOMMA"""
+        """comma_nocomma : COMMA nocomma"""
         p[0] = [p[2]]
 
     def p_macroarglist(self, p):
-        """macroarglist : NOCOMMA comma_nocomma_list_opt comma_opt"""
+        """macroarglist : nocomma comma_nocomma_list_opt comma_opt"""
         p1, p2 = p[1], p[2]
         if p2 is None:
             elts = [p1]
         else:
             elts = [p1] + p2
-        p0 = ast.Tuple(elts=elts, ctx=ast.Load(), lineno=p1.lineno,
-                       col_offset=p1.col_offset)
+        p0 = ast.Tuple(elts=elts, ctx=ast.Load(), lineno=self.lineno,
+                       col_offset=self.col)
         p[0] = p0
 
     def p_subscriptlist(self, p):

From b0825b611d8510012a5ecc78653e57a4b099e851 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 07:22:53 -0400
Subject: [PATCH 023/190] can parse simple things

---
 xonsh/built_ins.py    | 22 ++++++++++++++++++++++
 xonsh/parsers/base.py | 17 ++++++++++++++---
 2 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py
index 6da0421d3..a9ba12691 100644
--- a/xonsh/built_ins.py
+++ b/xonsh/built_ins.py
@@ -679,6 +679,26 @@ def list_of_strs_or_callables(x):
     return rtn
 
 
+def call_macro(f, args, glbs, locs):
+    """Calls a function as a macro, returning its result.
+
+    Parameters
+    ----------
+    f : callable object
+        The function that is called as f(*args).
+    args : tuple of str
+        The str reprensetaion of arguments of that were passed into the
+        macro.  These strings will be parsed, compiled, evaled, or left as
+        a string dependending on the annotations of f.
+    glbs : Mapping
+        The globals from the call site.
+    locs : Mapping or None
+        The locals from the call site.
+    """
+    args = [eval(a, glbs, locs) for a in args]  # punt for the moment
+    return f(*args)
+
+
 def load_builtins(execer=None, config=None, login=False, ctx=None):
     """Loads the xonsh builtins into the Python builtins. Sets the
     BUILTINS_LOADED variable to True.
@@ -714,6 +734,7 @@ def load_builtins(execer=None, config=None, login=False, ctx=None):
     builtins.__xonsh_ensure_list_of_strs__ = ensure_list_of_strs
     builtins.__xonsh_list_of_strs_or_callables__ = list_of_strs_or_callables
     builtins.__xonsh_completers__ = xonsh.completers.init.default_completers()
+    builtins.__xonsh_call_macro__ = call_macro
     # public built-ins
     builtins.XonshError = XonshError
     builtins.XonshBlockError = XonshBlockError
@@ -779,6 +800,7 @@ def unload_builtins():
              '__xonsh_execer__',
              '__xonsh_commands_cache__',
              '__xonsh_completers__',
+             '__xonsh_call_macro__',
              'XonshError',
              'XonshBlockError',
              'XonshCalledProcessError',
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index f97844318..5903bac24 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -476,7 +476,8 @@ class BaseParser(object):
     def p_eval_input(self, p):
         """eval_input : testlist newlines_opt
         """
-        p[0] = ast.Expression(body=p[1])
+        p1 = p[1]
+        p[0] = ast.Expression(body=p1, lineno=p1.lineno, col_offset=p1.col_offset)
 
     def p_func_call(self, p):
         """func_call : LPAREN arglist_opt RPAREN"""
@@ -1640,9 +1641,17 @@ class BaseParser(object):
                                    lineno=leader.lineno,
                                    col_offset=leader.col_offset)
             elif isinstance(trailer, Mapping):
+                # call normal functions
                 p0 = ast.Call(func=leader,
                               lineno=leader.lineno,
                               col_offset=leader.col_offset, **trailer)
+            elif isinstance(trailer, ast.Tuple):
+                # call macro functions
+                l, c = leader.lineno, leader.col_offset
+                gblcall = xonsh_call('globals', [], lineno=l, col=c)
+                loccall = xonsh_call('locals', [], lineno=l, col=c)
+                margs = [leader, trailer, gblcall, loccall]
+                p0 = xonsh_call('__xonsh_call_macro__', margs, lineno=l, col=c)
             elif isinstance(trailer, str):
                 if trailer == '?':
                     p0 = xonsh_help(leader, lineno=leader.lineno,
@@ -1895,8 +1904,10 @@ class BaseParser(object):
             elts = [p1]
         else:
             elts = [p1] + p2
-        p0 = ast.Tuple(elts=elts, ctx=ast.Load(), lineno=self.lineno,
-                       col_offset=self.col)
+        l = self.lineno
+        c = self.col
+        elts = [ast.Str(s=elt, lineno=l, col_offset=c) for elt in elts]
+        p0 = ast.Tuple(elts=elts, ctx=ast.Load(), lineno=l, col_offset=c)
         p[0] = p0
 
     def p_subscriptlist(self, p):

From 15e23a9060b9f25d53e888cddabd03d12537c464 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 07:46:03 -0400
Subject: [PATCH 024/190] more fixes

---
 tests/test_parser.py  | 29 +++++++++++++++++++++++------
 xonsh/parsers/base.py |  5 +++--
 2 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/tests/test_parser.py b/tests/test_parser.py
index 5b714f145..a9dd32cd1 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -7,7 +7,7 @@ import builtins
 
 import pytest
 
-from xonsh.ast import pdump
+from xonsh.ast import pdump, AST
 from xonsh.parser import Parser
 
 from tools import VER_FULL, skip_if_py34, nodes_equal
@@ -42,7 +42,8 @@ def check_stmts(inp, run=True, mode='exec'):
         inp += '\n'
     check_ast(inp, run=run, mode=mode)
 
-def check_xonsh_ast(xenv, inp, run=True, mode='eval', debug_level=0):
+def check_xonsh_ast(xenv, inp, run=True, mode='eval', debug_level=0,
+                    return_obs=False):
     __tracebackhide__ = True
     builtins.__xonsh_env__ = xenv
     obs = PARSER.parse(inp, debug_level=debug_level)
@@ -51,7 +52,7 @@ def check_xonsh_ast(xenv, inp, run=True, mode='eval', debug_level=0):
     bytecode = compile(obs, '<test-xonsh-ast>', mode)
     if run:
         exec(bytecode)
-    return True
+    return obs if return_obs else True
 
 def check_xonsh(xenv, inp, run=True, mode='exec'):
     __tracebackhide__ = True
@@ -1784,7 +1785,23 @@ def test_redirect_error_to_output(r, o):
     assert check_xonsh_ast({}, '$[echo "test" {} {}> test.txt < input.txt]'.format(r, o), False)
 
 def test_macro_call_empty():
-    check_xonsh_ast({}, 'f!()', False)
+    assert check_xonsh_ast({}, 'f!()', False)
+
+@pytest.mark.parametrize('s', [
+    'f!(x)',
+    'f!(True)',
+    'f!(None)',
+    'f!(import os)',
+    'f!(x=10)',
+    'f!("oh no, mom")',
+    'f!(if True:\n  pass)',
+    #'f!({x: y})',
+    #'f!((x, y))',
+])
+def test_macro_call_one_arg(s):
+    tree = check_xonsh_ast({}, s, False, return_obs=True)#, debug_level=100)
+    assert isinstance(tree, AST)
+    args = tree.body.args[1].elts
+    assert len(args) == 1
+    assert args[0].s == s[3:-1].strip()
 
-def test_macro_call_one_arg():
-    check_xonsh_ast({}, 'f!(x)', False, debug_level=100)
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index 5903bac24..e496ed262 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -1882,8 +1882,9 @@ class BaseParser(object):
         """nocomma_part : LPAREN any_raw_toks RPAREN
                         | AT_LPAREN any_raw_toks RPAREN
                         | BANG_LPAREN any_raw_toks RPAREN
+                        | LBRACE any_raw_toks RBRACE
         """
-        p[0] = p[1] + p[2]
+        p[0] = p[1] + p[2] + p[3]
 
     def p_nocomma_base(self, p):
         """nocomma : nocomma_part"""
@@ -1906,7 +1907,7 @@ class BaseParser(object):
             elts = [p1] + p2
         l = self.lineno
         c = self.col
-        elts = [ast.Str(s=elt, lineno=l, col_offset=c) for elt in elts]
+        elts = [ast.Str(s=elt.strip(), lineno=l, col_offset=c) for elt in elts]
         p0 = ast.Tuple(elts=elts, ctx=ast.Load(), lineno=l, col_offset=c)
         p[0] = p0
 

From 4dd6e9621dde6c54ade862712cbedcd7d2c904b0 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 08:33:01 -0400
Subject: [PATCH 025/190] have tuples, missing whitespace

---
 tests/test_parser.py  |  7 +++++--
 xonsh/parsers/base.py | 12 ++++++------
 xonsh/tokenize.py     |  1 +
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/tests/test_parser.py b/tests/test_parser.py
index a9dd32cd1..1ec55d77f 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -1794,12 +1794,15 @@ def test_macro_call_empty():
     'f!(import os)',
     'f!(x=10)',
     'f!("oh no, mom")',
-    'f!(if True:\n  pass)',
+    'f!(...)',
+    'f!( ... )',
+    #'f!(if True:\n  pass)',
     #'f!({x: y})',
+    'f!((x,y))',
     #'f!((x, y))',
 ])
 def test_macro_call_one_arg(s):
-    tree = check_xonsh_ast({}, s, False, return_obs=True)#, debug_level=100)
+    tree = check_xonsh_ast({}, s, False, return_obs=True, debug_level=100)
     assert isinstance(tree, AST)
     args = tree.body.args[1].elts
     assert len(args) == 1
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index e496ed262..3916b545a 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -425,7 +425,8 @@ class BaseParser(object):
                   ('left', 'EQ', 'NE'), ('left', 'GT', 'GE', 'LT', 'LE'),
                   ('left', 'RSHIFT', 'LSHIFT'), ('left', 'PLUS', 'MINUS'),
                   ('left', 'TIMES', 'DIVIDE', 'DOUBLEDIV', 'MOD'),
-                  ('left', 'POW'), )
+                  ('left', 'POW'),
+                  )
 
     #
     # Grammar as defined by BNF
@@ -1848,10 +1849,10 @@ class BaseParser(object):
         p[0] = [p[1]]
 
     def _attach_nocomma_tok_rules(self):
-        toks = sorted(self.tokens)
-        toks.remove('COMMA')
-        toks.remove('RPAREN')
-        ts = '\n             | '.join(toks)
+        toks = set(self.tokens)
+        toks -= {'COMMA', 'LPAREN', 'RPAREN', 'LBRACE', 'RBRACE', 'LBRACKET',
+                 'RBRACKET', 'AT_LPAREN', 'BANG_LPAREN'}
+        ts = '\n            | '.join(sorted(toks))
         doc = 'nocomma_tok : ' + ts + '\n'
         self.p_nocomma_tok.__func__.__doc__ = doc
 
@@ -1862,7 +1863,6 @@ class BaseParser(object):
     def p_any_raw_tok(self, p):
         """any_raw_tok : nocomma
                        | COMMA
-                       | RPAREN
         """
         p[0] = p[1]
 
diff --git a/xonsh/tokenize.py b/xonsh/tokenize.py
index 8ab583c79..774571101 100644
--- a/xonsh/tokenize.py
+++ b/xonsh/tokenize.py
@@ -683,6 +683,7 @@ def _tokenize(readline, encoding):
                     if parenlev > 0:
                         yield TokenInfo(NL, token, spos, epos, line)
                     else:
+                        print(token)
                         yield TokenInfo(NEWLINE, token, spos, epos, line)
                         if async_def:
                             async_def_nl = True

From 698e16c3b57ba4c47a3a74b185793fe3c2402b2e Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 10:30:23 -0400
Subject: [PATCH 026/190] some fixes

---
 tests/test_parser.py  | 10 +++--
 xonsh/parsers/base.py | 94 +++++++++++++++++++++++++++++++------------
 2 files changed, 74 insertions(+), 30 deletions(-)

diff --git a/tests/test_parser.py b/tests/test_parser.py
index 1ec55d77f..6f26417f6 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -1796,13 +1796,15 @@ def test_macro_call_empty():
     'f!("oh no, mom")',
     'f!(...)',
     'f!( ... )',
-    #'f!(if True:\n  pass)',
-    #'f!({x: y})',
+    'f!(if True:\n  pass)',
+    'f!({x: y})',
     'f!((x,y))',
-    #'f!((x, y))',
+    'f!((x, y))',
 ])
 def test_macro_call_one_arg(s):
-    tree = check_xonsh_ast({}, s, False, return_obs=True, debug_level=100)
+    debug = 100
+    debug = 0
+    tree = check_xonsh_ast({}, s, False, return_obs=True, debug_level=debug)
     assert isinstance(tree, AST)
     args = tree.body.args[1].elts
     assert len(args) == 1
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index 3916b545a..bcc4420e3 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -219,6 +219,8 @@ class BaseParser(object):
         self.lexer = lexer = Lexer()
         self.tokens = lexer.tokens
 
+        self._lines = None
+        self.xonsh_code = None
         self._attach_nocomma_tok_rules()
 
         opt_rules = [
@@ -235,7 +237,8 @@ class BaseParser(object):
             'op_factor_list', 'trailer_list', 'testlist_comp',
             'yield_expr_or_testlist_comp', 'dictorsetmaker',
             'comma_subscript_list', 'test', 'sliceop', 'comp_iter',
-            'yield_arg', 'test_comma_list', 'comma_nocomma_list', 'macroarglist']
+            'yield_arg', 'test_comma_list', 'comma_nocomma_list',
+            'macroarglist', 'comma_tok']
         for rule in opt_rules:
             self._opt_rule(rule)
 
@@ -261,7 +264,7 @@ class BaseParser(object):
                      'for', 'colon', 'import', 'except', 'nonlocal', 'global',
                      'yield', 'from', 'raise', 'with', 'dollar_lparen',
                      'dollar_lbrace', 'dollar_lbracket', 'try',
-                     'bang_lparen', 'bang_lbracket']
+                     'bang_lparen', 'bang_lbracket', 'comma', 'rparen']
         for rule in tok_rules:
             self._tok_rule(rule)
 
@@ -286,6 +289,8 @@ class BaseParser(object):
         """Resets for clean parsing."""
         self.lexer.reset()
         self._last_yielded_token = None
+        self._lines = None
+        self.xonsh_code = None
 
     def parse(self, s, filename='<code>', mode='exec', debug_level=0):
         """Returns an abstract syntax tree of xonsh code.
@@ -320,7 +325,7 @@ class BaseParser(object):
         return tree
 
     def _lexer_errfunc(self, msg, line, column):
-        self._parse_error(msg, self.currloc(line, column), self.xonsh_code)
+        self._parse_error(msg, self.currloc(line, column))
 
     def _yacc_lookahead_token(self):
         """Gets the next-to-last and last token seen by the lexer."""
@@ -402,15 +407,33 @@ class BaseParser(object):
             return self.token_col(t)
         return 0
 
-    def _parse_error(self, msg, loc, line=None):
-        if line is None:
+    @property
+    def lines(self):
+        if self._lines is None and self.xonsh_code is not None:
+            self._lines = self.xonsh_code.splitlines(keepends=True)
+        return self._lines
+
+    def source_slice(self, start, stop):
+        """Gets the original source code from two (line, col) tuples in
+        source-space (i.e. lineno start at 1).
+        """
+        bline, bcol = start
+        eline, ecol = stop
+        bline -= 1
+        lines = self.lines[bline:eline]
+        lines[-1] = lines[-1][:ecol]
+        lines[0] = lines[0][bcol:]
+        return ''.join(lines)
+
+    def _parse_error(self, msg, loc):
+        if self.xonsh_code is None:
             err_line_pointer = ''
         else:
             col = loc.column + 1
-            lines = line.splitlines()
+            lines = self.lines
             i = loc.lineno - 1
             if 0 <= i < len(lines):
-                err_line = lines[i]
+                err_line = lines[i].rstrip()
                 err_line_pointer = '\n{}\n{: >{}}'.format(err_line, '^', col)
             else:
                 err_line_pointer = ''
@@ -1833,8 +1856,33 @@ class BaseParser(object):
         p[0] = [p[2] or dict(args=[], keywords=[], starargs=None, kwargs=None)]
 
     def p_trailer_bang_lparen(self, p):
-        """trailer : BANG_LPAREN macroarglist_opt RPAREN"""
-        p[0] = [p[2] or dict(args=[], keywords=[], starargs=None, kwargs=None)]
+        """trailer : bang_lparen_tok macroarglist_opt rparen_tok"""
+        p1, p2, p3 = p[1], p[2], p[3]
+        begins = [(p1.lineno, p1.lexpos + 2, None)]
+        ends = [(p3.lineno, p3.lexpos, None)]
+        if p2:
+            if p2[-1][-1] == 'trailing':  # handle trailing comma
+                begins.extend(p2[:-1])
+                ends = p2
+            else:
+                begins.extend(p2)
+                ends = p2 + ends
+        elts = []
+        for beg, end in zip(begins, ends):
+            beg = beg[:2]
+            end = end[:2]
+            s = self.source_slice(beg, end).strip()
+            if not s:
+                if len(begins) == 1:
+                    break
+                else:
+                    msg = 'empty macro arguments not allowed'
+                    self._parse_error(msg, self.currloc(*beg))
+            node = ast.Str(s=s, lineno=beg[0], col_offset=beg[1])
+            elts.append(node)
+        p0 = ast.Tuple(elts=elts, ctx=ast.Load(), lineno=p1.lineno,
+                       col_offset=p1.lexpos)
+        p[0] = [p0]
 
     def p_trailer_p3(self, p):
         """trailer : LBRACKET subscriptlist RBRACKET
@@ -1895,21 +1943,17 @@ class BaseParser(object):
         p[0] = p[1] + p[2]
 
     def p_comma_nocomma(self, p):
-        """comma_nocomma : COMMA nocomma"""
-        p[0] = [p[2]]
+        """comma_nocomma : comma_tok nocomma"""
+        p1 = p[1]
+        p[0] = [(p1.lineno, p1.lexpos, None)]
 
     def p_macroarglist(self, p):
-        """macroarglist : nocomma comma_nocomma_list_opt comma_opt"""
-        p1, p2 = p[1], p[2]
-        if p2 is None:
-            elts = [p1]
-        else:
-            elts = [p1] + p2
-        l = self.lineno
-        c = self.col
-        elts = [ast.Str(s=elt.strip(), lineno=l, col_offset=c) for elt in elts]
-        p0 = ast.Tuple(elts=elts, ctx=ast.Load(), lineno=l, col_offset=c)
-        p[0] = p0
+        """macroarglist : nocomma comma_nocomma_list_opt comma_tok_opt"""
+        p2, p3 = p[2], p[3]
+        pos = [] if p2 is None else p2
+        if p3 is not None:
+            pos.append((p3.lineno, p3.lexpos, 'trailing'))
+        p[0] = pos
 
     def p_subscriptlist(self, p):
         """subscriptlist : subscript comma_subscript_list_opt comma_opt"""
@@ -2418,11 +2462,9 @@ class BaseParser(object):
             else:
                 self._parse_error(p.value,
                                   self.currloc(lineno=p.lineno,
-                                               column=p.lexpos),
-                                  self.xonsh_code)
+                                               column=p.lexpos))
         else:
             msg = 'code: {0}'.format(p.value),
             self._parse_error(msg,
                               self.currloc(lineno=p.lineno,
-                                           column=p.lexpos),
-                              self.xonsh_code)
+                                           column=p.lexpos))

From cc02ea1f1be21b99cc49038adda38a50da0e9861 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 10:39:00 -0400
Subject: [PATCH 027/190] no-oped some code

---
 xonsh/parsers/base.py | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index bcc4420e3..75d6a58e3 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -1904,27 +1904,35 @@ class BaseParser(object):
         doc = 'nocomma_tok : ' + ts + '\n'
         self.p_nocomma_tok.__func__.__doc__ = doc
 
+    # The following grammar rules are no-ops because we don't need to glue the
+    # source code back together piece-by-piece. Instead, we simply look for
+    # top-level commas and record their positions. With these positions and
+    # the bounding parantheses !() positions we can use the source_slice()
+    # method. This does a much better job of capturing exactly the source code
+    # that was provided. The tokenizer & lexer can be a little lossy, especially
+    # with respect to whitespace.
+
     def p_nocomma_tok(self, p):
         # see attachement function above for docstring
-        p[0] = p[1]
+        pass
 
     def p_any_raw_tok(self, p):
         """any_raw_tok : nocomma
                        | COMMA
         """
-        p[0] = p[1]
+        pass
 
     def p_any_raw_toks_one(self, p):
         """any_raw_toks : any_raw_tok"""
-        p[0] = p[1]
+        pass
 
     def p_any_raw_toks_many(self, p):
         """any_raw_toks : any_raw_toks any_raw_tok"""
-        p[0] = p[1] + p[2]
+        pass
 
     def p_nocomma_part_tok(self, p):
         """nocomma_part : nocomma_tok"""
-        p[0] = p[1]
+        pass
 
     def p_nocomma_part_any(self, p):
         """nocomma_part : LPAREN any_raw_toks RPAREN
@@ -1932,15 +1940,15 @@ class BaseParser(object):
                         | BANG_LPAREN any_raw_toks RPAREN
                         | LBRACE any_raw_toks RBRACE
         """
-        p[0] = p[1] + p[2] + p[3]
+        pass
 
     def p_nocomma_base(self, p):
         """nocomma : nocomma_part"""
-        p[0] = p[1]
+        pass
 
     def p_nocomma_append(self, p):
         """nocomma : nocomma nocomma_part"""
-        p[0] = p[1] + p[2]
+        pass
 
     def p_comma_nocomma(self, p):
         """comma_nocomma : comma_tok nocomma"""

From e0d0d42f557a8a3034eba4b3d535acd23019a0fe Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 12:39:12 -0400
Subject: [PATCH 028/190] more macros

---
 tests/test_parser.py  | 17 +++++++++++++++++
 xonsh/parsers/base.py | 20 ++++++++++++++------
 2 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/tests/test_parser.py b/tests/test_parser.py
index 6f26417f6..2ae063056 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -1798,8 +1798,25 @@ def test_macro_call_empty():
     'f!( ... )',
     'f!(if True:\n  pass)',
     'f!({x: y})',
+    'f!({x: y, 42: 5})',
+    'f!({1, 2, 3,})',
     'f!((x,y))',
     'f!((x, y))',
+    'f!(((x, y), z))',
+    'f!(g())',
+    'f!(range(10))',
+    'f!(range(1, 10, 2))',
+    'f!(())',
+    'f!({})',
+    'f!([])',
+    'f!([1, 2])',
+    'f!(@(x))',
+    'f!(!(ls -l))',
+    'f!(![ls -l])',
+    'f!($(ls -l))',
+    'f!(${x + y})',
+    'f!($[ls -l])',
+    'f!(@$(which xonsh))',
 ])
 def test_macro_call_one_arg(s):
     debug = 100
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index 75d6a58e3..1c450507b 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -238,7 +238,7 @@ class BaseParser(object):
             'yield_expr_or_testlist_comp', 'dictorsetmaker',
             'comma_subscript_list', 'test', 'sliceop', 'comp_iter',
             'yield_arg', 'test_comma_list', 'comma_nocomma_list',
-            'macroarglist', 'comma_tok']
+            'macroarglist', 'comma_tok', 'any_raw_toks']
         for rule in opt_rules:
             self._opt_rule(rule)
 
@@ -1899,7 +1899,9 @@ class BaseParser(object):
     def _attach_nocomma_tok_rules(self):
         toks = set(self.tokens)
         toks -= {'COMMA', 'LPAREN', 'RPAREN', 'LBRACE', 'RBRACE', 'LBRACKET',
-                 'RBRACKET', 'AT_LPAREN', 'BANG_LPAREN'}
+                 'RBRACKET', 'AT_LPAREN', 'BANG_LPAREN', 'BANG_LBRACKET',
+                 'DOLLAR_LPAREN', 'DOLLAR_LBRACE', 'DOLLAR_LBRACKET',
+                 'ATDOLLAR_LPAREN'}
         ts = '\n            | '.join(sorted(toks))
         doc = 'nocomma_tok : ' + ts + '\n'
         self.p_nocomma_tok.__func__.__doc__ = doc
@@ -1935,10 +1937,16 @@ class BaseParser(object):
         pass
 
     def p_nocomma_part_any(self, p):
-        """nocomma_part : LPAREN any_raw_toks RPAREN
-                        | AT_LPAREN any_raw_toks RPAREN
-                        | BANG_LPAREN any_raw_toks RPAREN
-                        | LBRACE any_raw_toks RBRACE
+        """nocomma_part : LPAREN any_raw_toks_opt RPAREN
+                        | LBRACE any_raw_toks_opt RBRACE
+                        | LBRACKET any_raw_toks_opt RBRACKET
+                        | AT_LPAREN any_raw_toks_opt RPAREN
+                        | BANG_LPAREN any_raw_toks_opt RPAREN
+                        | BANG_LBRACKET any_raw_toks_opt RBRACKET
+                        | DOLLAR_LPAREN any_raw_toks_opt RPAREN
+                        | DOLLAR_LBRACE any_raw_toks_opt RBRACE
+                        | DOLLAR_LBRACKET any_raw_toks_opt RBRACKET
+                        | ATDOLLAR_LPAREN any_raw_toks_opt RPAREN
         """
         pass
 

From 57726327ddef96b2bfb7efca651dcebbd1ef94a9 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 13:08:56 -0400
Subject: [PATCH 029/190] even more tests

---
 tests/test_parser.py  | 37 ++++++++++++++++++++++++++++++++++---
 xonsh/parsers/base.py | 14 ++++++--------
 2 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/tests/test_parser.py b/tests/test_parser.py
index 2ae063056..c38168d4f 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -4,6 +4,7 @@ import os
 import sys
 import ast
 import builtins
+import itertools
 
 import pytest
 
@@ -1819,11 +1820,41 @@ def test_macro_call_empty():
     'f!(@$(which xonsh))',
 ])
 def test_macro_call_one_arg(s):
-    debug = 100
-    debug = 0
-    tree = check_xonsh_ast({}, s, False, return_obs=True, debug_level=debug)
+    tree = check_xonsh_ast({}, s, False, return_obs=True)
     assert isinstance(tree, AST)
     args = tree.body.args[1].elts
     assert len(args) == 1
     assert args[0].s == s[3:-1].strip()
 
+
+MACRO_ARGS = [
+    'x', 'True', 'None', 'import os', 'x=10', '"oh no, mom"', '...', ' ... ',
+    'if True:\n  pass', '{x: y}', '{x: y, 42: 5}', '{1, 2, 3,}', '(x,y)',
+    '(x, y)', '((x, y), z)', 'g()', 'range(10)', 'range(1, 10, 2)', '()', '{}',
+    '[]', '[1, 2]', '@(x)', '!(ls -l)', '![ls -l]', '$(ls -l)', '${x + y}',
+    '$[ls -l]', '@$(which xonsh)',
+]
+
+@pytest.mark.parametrize('s,t', itertools.product(MACRO_ARGS[::2],
+                                                  MACRO_ARGS[1::2]))
+def test_macro_call_two_args(s, t):
+    f = 'f!({}, {})'.format(s, t)
+    tree = check_xonsh_ast({}, f, False, return_obs=True)
+    assert isinstance(tree, AST)
+    args = tree.body.args[1].elts
+    assert len(args) == 2
+    assert args[0].s == s.strip()
+    assert args[1].s == t.strip()
+
+@pytest.mark.parametrize('s,t,u', itertools.product(MACRO_ARGS[::3],
+                                                    MACRO_ARGS[1::3],
+                                                    MACRO_ARGS[2::3]))
+def test_macro_call_three_args(s, t, u):
+    f = 'f!({}, {}, {})'.format(s, t, u)
+    tree = check_xonsh_ast({}, f, False, return_obs=True)
+    assert isinstance(tree, AST)
+    args = tree.body.args[1].elts
+    assert len(args) == 3
+    assert args[0].s == s.strip()
+    assert args[1].s == t.strip()
+    assert args[2].s == u.strip()
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index 1c450507b..3e16116e1 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -1858,19 +1858,17 @@ class BaseParser(object):
     def p_trailer_bang_lparen(self, p):
         """trailer : bang_lparen_tok macroarglist_opt rparen_tok"""
         p1, p2, p3 = p[1], p[2], p[3]
-        begins = [(p1.lineno, p1.lexpos + 2, None)]
-        ends = [(p3.lineno, p3.lexpos, None)]
+        begins = [(p1.lineno, p1.lexpos + 2)]
+        ends = [(p3.lineno, p3.lexpos)]
         if p2:
             if p2[-1][-1] == 'trailing':  # handle trailing comma
-                begins.extend(p2[:-1])
-                ends = p2
+                begins.extend([(x[0], x[1] + 1) for x in p2[:-1]])
+                ends = [x[:2] for x in p2]
             else:
-                begins.extend(p2)
-                ends = p2 + ends
+                begins.extend([(x[0], x[1] + 1) for x in p2])
+                ends = [x[:2] for x in p2] + ends
         elts = []
         for beg, end in zip(begins, ends):
-            beg = beg[:2]
-            end = end[:2]
             s = self.source_slice(beg, end).strip()
             if not s:
                 if len(begins) == 1:

From b2da4ca2e7814767db6f5f5f733aa330956f273b Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 13:11:48 -0400
Subject: [PATCH 030/190] test refactor

---
 tests/test_parser.py | 49 ++++++++++----------------------------------
 1 file changed, 11 insertions(+), 38 deletions(-)

diff --git a/tests/test_parser.py b/tests/test_parser.py
index c38168d4f..023af58c4 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -1788,44 +1788,6 @@ def test_redirect_error_to_output(r, o):
 def test_macro_call_empty():
     assert check_xonsh_ast({}, 'f!()', False)
 
-@pytest.mark.parametrize('s', [
-    'f!(x)',
-    'f!(True)',
-    'f!(None)',
-    'f!(import os)',
-    'f!(x=10)',
-    'f!("oh no, mom")',
-    'f!(...)',
-    'f!( ... )',
-    'f!(if True:\n  pass)',
-    'f!({x: y})',
-    'f!({x: y, 42: 5})',
-    'f!({1, 2, 3,})',
-    'f!((x,y))',
-    'f!((x, y))',
-    'f!(((x, y), z))',
-    'f!(g())',
-    'f!(range(10))',
-    'f!(range(1, 10, 2))',
-    'f!(())',
-    'f!({})',
-    'f!([])',
-    'f!([1, 2])',
-    'f!(@(x))',
-    'f!(!(ls -l))',
-    'f!(![ls -l])',
-    'f!($(ls -l))',
-    'f!(${x + y})',
-    'f!($[ls -l])',
-    'f!(@$(which xonsh))',
-])
-def test_macro_call_one_arg(s):
-    tree = check_xonsh_ast({}, s, False, return_obs=True)
-    assert isinstance(tree, AST)
-    args = tree.body.args[1].elts
-    assert len(args) == 1
-    assert args[0].s == s[3:-1].strip()
-
 
 MACRO_ARGS = [
     'x', 'True', 'None', 'import os', 'x=10', '"oh no, mom"', '...', ' ... ',
@@ -1835,6 +1797,16 @@ MACRO_ARGS = [
     '$[ls -l]', '@$(which xonsh)',
 ]
 
+@pytest.mark.parametrize('s', MACRO_ARGS)
+def test_macro_call_one_arg(s):
+    f = 'f!({})'.format(s)
+    tree = check_xonsh_ast({}, f, False, return_obs=True)
+    assert isinstance(tree, AST)
+    args = tree.body.args[1].elts
+    assert len(args) == 1
+    assert args[0].s == s.strip()
+
+
 @pytest.mark.parametrize('s,t', itertools.product(MACRO_ARGS[::2],
                                                   MACRO_ARGS[1::2]))
 def test_macro_call_two_args(s, t):
@@ -1846,6 +1818,7 @@ def test_macro_call_two_args(s, t):
     assert args[0].s == s.strip()
     assert args[1].s == t.strip()
 
+
 @pytest.mark.parametrize('s,t,u', itertools.product(MACRO_ARGS[::3],
                                                     MACRO_ARGS[1::3],
                                                     MACRO_ARGS[2::3]))

From b602e238fec9539a339d889bc075b7754d91d4da Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 15:05:15 -0400
Subject: [PATCH 031/190] macro arg conversion

---
 xonsh/built_ins.py | 125 +++++++++++++++++++++++++++++++++++++++++++--
 xonsh/execer.py    |  27 +++++++---
 2 files changed, 141 insertions(+), 11 deletions(-)

diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py
index a9ba12691..b27dba01d 100644
--- a/xonsh/built_ins.py
+++ b/xonsh/built_ins.py
@@ -8,6 +8,7 @@ import os
 import re
 import sys
 import time
+import types
 import shlex
 import signal
 import atexit
@@ -18,6 +19,7 @@ import subprocess
 import contextlib
 import collections.abc as abc
 
+from xonsh.ast import AST
 from xonsh.lazyasd import LazyObject, lazyobject
 from xonsh.history import History
 from xonsh.inspectors import Inspector
@@ -679,6 +681,110 @@ def list_of_strs_or_callables(x):
     return rtn
 
 
+@lazyobject
+def MACRO_FLAG_KINDS():
+    return {
+        's': str,
+        'str': str,
+        'string': str,
+        'a': AST,
+        'ast': AST,
+        'c': types.CodeType,
+        'code': types.CodeType,
+        'compile': types.CodeType,
+        'v': eval,
+        'eval': eval,
+        'x': exec,
+        'exec': exec,
+        }
+
+def _convert_kind_flag(x):
+    """Puts a kind flag (string) a canonical form."""
+    x = x.lower()
+    kind = MACRO_FLAG_KINDS.get(x, None)
+    if kind is None:
+        raise TypeError('{0!r} not a recognized macro type.'.format(x))
+    return kind
+
+
+def convert_macro_arg(raw_arg, kind, glbs, locs, *, name='<arg>',
+                      macroname='<macro>'):
+    """Converts a string macro argument based on the requested kind.
+
+    Parameters
+    ----------
+    raw_arg : str
+        The str reprensetaion of the macro argument.
+    kind : object
+        A flag or type representing how to convert the argument.
+    glbs : Mapping
+        The globals from the call site.
+    locs : Mapping or None
+        The locals from the call site.
+    name : str, optional
+        The macro argument name.
+    macroname : str, optional
+        The name of the macro itself.
+
+    Returns
+    -------
+    The converted argument.
+    """
+    # munge kind and mode to start
+    mode = None
+    if isinstance(kind, abc.Sequence) and not isinstance(kind, str):
+        # have (kind, mode) tuple
+        kind, mode = kind
+    if isinstance(kind, str):
+        kind = _convert_kind_flag(kind)
+    if kind is str:
+        return raw_arg  # short circut since there is nothing else to do
+    # select from kind and convert
+    execer = builtins.__xonsh_execer__
+    filename = macroname + '(' + name + ')'
+    if kind is AST:
+        ctx = set(dir(builtins)) | set(glbs.keys())
+        if locs is not None:
+            ctx |= set(locs.keys())
+        mode = mode or 'eval'
+        arg = execer.parse(raw_arg, ctx, mode=mode, filename=filename)
+    elif kind is types.CodeType or kind is compile or kind is execer.compile:
+        mode = mode or 'eval'
+        arg = execer.compile(raw_arg, mode=mode, glbs=glbs, locs=locs,
+                             filename=filename)
+    elif kind is eval or kind is execer.eval:
+        arg = execer.eval(raw_arg, glbs=glbs, locs=locs, filename=filename)
+    elif kind is exec or kind is execer.exec:
+        mode = mode or 'exec'
+        arg = execer.exec(raw_arg, mode=mode, glbs=glbs, locs=locs,
+                          filename=filename)
+    else:
+        msg = ('kind={0!r} and mode={1!r} was not recongnized for macro '
+               'argument {2!r}')
+        raise TypeError(msg.format(kind, mode, name))
+    return arg
+
+
+@contextlib.contextmanager
+def macro_context(f, glbs, locs):
+    """Attaches macro globals and locals temporarily to function as a
+    context manager.
+
+    Parameters
+    ----------
+    f : callable object
+        The function that is called as f(*args).
+    glbs : Mapping
+        The globals from the call site.
+    locs : Mapping or None
+        The locals from the call site.
+    """
+    f.macro_globals = glbs
+    f.macro_locals = locs
+    yield
+    del f.macro_globals, f.macro_locals
+
+
 def call_macro(f, args, glbs, locs):
     """Calls a function as a macro, returning its result.
 
@@ -686,9 +792,9 @@ def call_macro(f, args, glbs, locs):
     ----------
     f : callable object
         The function that is called as f(*args).
-    args : tuple of str
+    raw_args : tuple of str
         The str reprensetaion of arguments of that were passed into the
-        macro.  These strings will be parsed, compiled, evaled, or left as
+        macro. These strings will be parsed, compiled, evaled, or left as
         a string dependending on the annotations of f.
     glbs : Mapping
         The globals from the call site.
@@ -696,7 +802,20 @@ def call_macro(f, args, glbs, locs):
         The locals from the call site.
     """
     args = [eval(a, glbs, locs) for a in args]  # punt for the moment
-    return f(*args)
+    sig = inspect.signature(f)
+    empty = inspect.Parameter.empty
+    macroname = f.__name__
+    args = []
+    for (key, param), raw_arg in zip(sig.items(), raw_args):
+        kind = param.annotation
+        if kind is empty or kind is None:
+            kind = eval
+        arg = convert_macro_arg(raw_arg, kind, glbs, locs, name=key,
+                                macroname=macroname)
+        args.append(arg)
+    with macro_context(f, glbs, locs):
+        rtn = f(*args)
+    return rtn
 
 
 def load_builtins(execer=None, config=None, login=False, ctx=None):
diff --git a/xonsh/execer.py b/xonsh/execer.py
index 5910bb20e..93f4c5573 100644
--- a/xonsh/execer.py
+++ b/xonsh/execer.py
@@ -45,13 +45,15 @@ class Execer(object):
         if self.unload:
             unload_builtins()
 
-    def parse(self, input, ctx, mode='exec', transform=True):
+    def parse(self, input, ctx, mode='exec', filename=None, transform=True):
         """Parses xonsh code in a context-aware fashion. For context-free
         parsing, please use the Parser class directly or pass in
         transform=False.
         """
+        if filename is None:
+            filename = self.filename
         if not transform:
-            return self.parser.parse(input, filename=self.filename, mode=mode,
+            return self.parser.parse(input, filename=filename, mode=mode,
                                      debug_level=(self.debug_level > 1))
 
         # Parsing actually happens in a couple of phases. The first is a
@@ -68,7 +70,7 @@ class Execer(object):
         # tokens for all of the Python rules. The lazy way implemented here
         # is to parse a line a second time with a $() wrapper if it fails
         # the first time. This is a context-free phase.
-        tree, input = self._parse_ctx_free(input, mode=mode)
+        tree, input = self._parse_ctx_free(input, mode=mode, filename=filename)
         if tree is None:
             return None
 
@@ -97,7 +99,8 @@ class Execer(object):
             glbs = frame.f_globals if glbs is None else glbs
             locs = frame.f_locals if locs is None else locs
         ctx = set(dir(builtins)) | set(glbs.keys()) | set(locs.keys())
-        tree = self.parse(input, ctx, mode=mode, transform=transform)
+        tree = self.parse(input, ctx, mode=mode, filename=filename,
+                          transform=transform)
         if tree is None:
             return None  # handles comment only input
         if transform:
@@ -110,45 +113,53 @@ class Execer(object):
         return code
 
     def eval(self, input, glbs=None, locs=None, stacklevel=2,
-             transform=True):
+             filename=None, transform=True):
         """Evaluates (and returns) xonsh code."""
         if isinstance(input, types.CodeType):
             code = input
         else:
+            if filename is None:
+                filename = self.filename
             code = self.compile(input=input,
                                 glbs=glbs,
                                 locs=locs,
                                 mode='eval',
                                 stacklevel=stacklevel,
+                                filename=filename,
                                 transform=transform)
         if code is None:
             return None  # handles comment only input
         return eval(code, glbs, locs)
 
     def exec(self, input, mode='exec', glbs=None, locs=None, stacklevel=2,
-             transform=True):
+             filename=None, transform=True):
         """Execute xonsh code."""
         if isinstance(input, types.CodeType):
             code = input
         else:
+            if filename is None:
+                filename = self.filename
             code = self.compile(input=input,
                                 glbs=glbs,
                                 locs=locs,
                                 mode=mode,
                                 stacklevel=stacklevel,
+                                filename=filname,
                                 transform=transform)
         if code is None:
             return None  # handles comment only input
         return exec(code, glbs, locs)
 
-    def _parse_ctx_free(self, input, mode='exec'):
+    def _parse_ctx_free(self, input, mode='exec', filename=None):
         last_error_line = last_error_col = -1
         parsed = False
         original_error = None
+        if filename is None:
+            filename = self.filename
         while not parsed:
             try:
                 tree = self.parser.parse(input,
-                                         filename=self.filename,
+                                         filename=filename,
                                          mode=mode,
                                          debug_level=(self.debug_level > 1))
                 parsed = True

From 3f2a168d47d5b0235cd30ed4f2a4e65bdc303a71 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 15:39:02 -0400
Subject: [PATCH 032/190] some tests

---
 tests/test_builtins.py | 65 +++++++++++++++++++++++++++++++++++++++++-
 xonsh/built_ins.py     |  8 ++++--
 xonsh/execer.py        |  2 +-
 3 files changed, 70 insertions(+), 5 deletions(-)

diff --git a/tests/test_builtins.py b/tests/test_builtins.py
index 838c592a2..f122c793e 100644
--- a/tests/test_builtins.py
+++ b/tests/test_builtins.py
@@ -3,13 +3,16 @@
 from __future__ import unicode_literals, print_function
 import os
 import re
+import builtins
+import types
+from ast import AST
 
 import pytest
 
 from xonsh import built_ins
 from xonsh.built_ins import reglob, pathsearch, helper, superhelper, \
     ensure_list_of_strs, list_of_strs_or_callables, regexsearch, \
-    globsearch
+    globsearch, convert_macro_arg
 from xonsh.environ import Env
 
 from tools import skip_if_on_windows
@@ -18,6 +21,10 @@ from tools import skip_if_on_windows
 HOME_PATH = os.path.expanduser('~')
 
 
+@pytest.fixture(autouse=True)
+def xonsh_execer_autouse(xonsh_execer):
+    return xonsh_execer
+
 @pytest.mark.parametrize('testfile', reglob('test_.*'))
 def test_reglob_tests(testfile):
     assert (testfile.startswith('test_'))
@@ -113,3 +120,59 @@ f = lambda x: 20
 def test_list_of_strs_or_callables(exp, inp):
     obs = list_of_strs_or_callables(inp)
     assert exp == obs
+
+
+@pytest.mark.parametrize('kind', [str, 's', 'S', 'str', 'string'])
+def test_convert_macro_arg_str(kind):
+    raw_arg = 'value'
+    arg = convert_macro_arg(raw_arg, kind, None, None)
+    assert arg is raw_arg
+
+
+@pytest.mark.parametrize('kind', [AST, 'a', 'Ast'])
+def test_convert_macro_arg_ast(kind):
+    raw_arg = '42'
+    arg = convert_macro_arg(raw_arg, kind, {}, None)
+    assert isinstance(arg, AST)
+
+
+@pytest.mark.parametrize('kind', [types.CodeType, compile, 'c', 'code',
+                                  'compile'])
+def test_convert_macro_arg_code(kind):
+    raw_arg = '42'
+    arg = convert_macro_arg(raw_arg, kind, {}, None)
+    assert isinstance(arg, types.CodeType)
+
+
+@pytest.mark.parametrize('kind', [eval, 'v', 'eval'])
+def test_convert_macro_arg_eval(kind):
+    # literals
+    raw_arg = '42'
+    arg = convert_macro_arg(raw_arg, kind, {}, None)
+    assert arg == 42
+    # exprs
+    raw_arg = 'x + 41'
+    arg = convert_macro_arg(raw_arg, kind, {}, {'x': 1})
+    assert arg == 42
+
+
+@pytest.mark.parametrize('kind', [exec, 'x', 'exec'])
+def test_convert_macro_arg_eval(kind):
+    # at global scope
+    raw_arg = 'def f(x, y):\n    return x + y'
+    glbs = {}
+    arg = convert_macro_arg(raw_arg, kind, glbs, None)
+    assert arg is None
+    assert 'f' in glbs
+    assert glbs['f'](1, 41) == 42
+    # at local scope
+    raw_arg = 'def g(z):\n    return x + z\ny += 42'
+    glbs = {'x': 40}
+    locs = {'y': 1}
+    arg = convert_macro_arg(raw_arg, kind, glbs, locs)
+    assert arg is None
+    assert 'g' in locs
+    assert locs['g'](1) == 41
+    assert 'y' in locs
+    assert locs['y'] == 43
+
diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py
index b27dba01d..1af6b464c 100644
--- a/xonsh/built_ins.py
+++ b/xonsh/built_ins.py
@@ -748,14 +748,16 @@ def convert_macro_arg(raw_arg, kind, glbs, locs, *, name='<arg>',
             ctx |= set(locs.keys())
         mode = mode or 'eval'
         arg = execer.parse(raw_arg, ctx, mode=mode, filename=filename)
-    elif kind is types.CodeType or kind is compile or kind is execer.compile:
+    elif kind is types.CodeType or kind is compile:
         mode = mode or 'eval'
         arg = execer.compile(raw_arg, mode=mode, glbs=glbs, locs=locs,
                              filename=filename)
-    elif kind is eval or kind is execer.eval:
+    elif kind is eval:
         arg = execer.eval(raw_arg, glbs=glbs, locs=locs, filename=filename)
-    elif kind is exec or kind is execer.exec:
+    elif kind is exec:
         mode = mode or 'exec'
+        if not raw_arg.endswith('\n'):
+            raw_arg += '\n'
         arg = execer.exec(raw_arg, mode=mode, glbs=glbs, locs=locs,
                           filename=filename)
     else:
diff --git a/xonsh/execer.py b/xonsh/execer.py
index 93f4c5573..2e99b6ff9 100644
--- a/xonsh/execer.py
+++ b/xonsh/execer.py
@@ -144,7 +144,7 @@ class Execer(object):
                                 locs=locs,
                                 mode=mode,
                                 stacklevel=stacklevel,
-                                filename=filname,
+                                filename=filename,
                                 transform=transform)
         if code is None:
             return None  # handles comment only input

From 37e458109ce70a6c69b9255598a3749e714695f7 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 15:50:36 -0400
Subject: [PATCH 033/190] more call macro tests

---
 tests/test_builtins.py | 28 +++++++++++++++++++++++++++-
 xonsh/built_ins.py     |  5 ++---
 2 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/tests/test_builtins.py b/tests/test_builtins.py
index f122c793e..ab9e98a42 100644
--- a/tests/test_builtins.py
+++ b/tests/test_builtins.py
@@ -12,7 +12,7 @@ import pytest
 from xonsh import built_ins
 from xonsh.built_ins import reglob, pathsearch, helper, superhelper, \
     ensure_list_of_strs, list_of_strs_or_callables, regexsearch, \
-    globsearch, convert_macro_arg
+    globsearch, convert_macro_arg, macro_context, call_macro
 from xonsh.environ import Env
 
 from tools import skip_if_on_windows
@@ -176,3 +176,29 @@ def test_convert_macro_arg_eval(kind):
     assert 'y' in locs
     assert locs['y'] == 43
 
+
+def test_macro_context():
+    def f():
+        pass
+    with macro_context(f, True, True):
+        assert f.macro_globals
+        assert f.macro_locals
+    assert not hasattr(f, 'macro_globals')
+    assert not hasattr(f, 'macro_locals')
+
+
+@pytest.mark.parametrize('arg', ['x', '42', 'x + y'])
+def test_call_macro_str(arg):
+    def f(x : str):
+        return x
+    rtn = call_macro(f, [arg], None, None)
+    assert rtn is arg
+
+
+@pytest.mark.parametrize('arg', ['x', '42', 'x + y'])
+def test_call_macro_code(arg):
+    def f(x : compile):
+        return x
+    rtn = call_macro(f, [arg], {}, None)
+    assert isinstance(rtn, types.CodeType)
+
diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py
index 1af6b464c..aaad01299 100644
--- a/xonsh/built_ins.py
+++ b/xonsh/built_ins.py
@@ -787,7 +787,7 @@ def macro_context(f, glbs, locs):
     del f.macro_globals, f.macro_locals
 
 
-def call_macro(f, args, glbs, locs):
+def call_macro(f, raw_args, glbs, locs):
     """Calls a function as a macro, returning its result.
 
     Parameters
@@ -803,12 +803,11 @@ def call_macro(f, args, glbs, locs):
     locs : Mapping or None
         The locals from the call site.
     """
-    args = [eval(a, glbs, locs) for a in args]  # punt for the moment
     sig = inspect.signature(f)
     empty = inspect.Parameter.empty
     macroname = f.__name__
     args = []
-    for (key, param), raw_arg in zip(sig.items(), raw_args):
+    for (key, param), raw_arg in zip(sig.parameters.items(), raw_args):
         kind = param.annotation
         if kind is empty or kind is None:
             kind = eval

From d5be829598277f83c29ec58f777eebd1e09e5832 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 16:00:30 -0400
Subject: [PATCH 034/190] even more macro tests

---
 tests/test_builtins.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/tests/test_builtins.py b/tests/test_builtins.py
index ab9e98a42..48de5ab15 100644
--- a/tests/test_builtins.py
+++ b/tests/test_builtins.py
@@ -195,6 +195,14 @@ def test_call_macro_str(arg):
     assert rtn is arg
 
 
+@pytest.mark.parametrize('arg', ['x', '42', 'x + y'])
+def test_call_macro_ast(arg):
+    def f(x : AST):
+        return x
+    rtn = call_macro(f, [arg], {}, None)
+    assert isinstance(rtn, AST)
+
+
 @pytest.mark.parametrize('arg', ['x', '42', 'x + y'])
 def test_call_macro_code(arg):
     def f(x : compile):
@@ -202,3 +210,22 @@ def test_call_macro_code(arg):
     rtn = call_macro(f, [arg], {}, None)
     assert isinstance(rtn, types.CodeType)
 
+
+@pytest.mark.parametrize('arg', ['x', '42', 'x + y'])
+def test_call_macro_eval(arg):
+    def f(x : eval):
+        return x
+    rtn = call_macro(f, [arg], {'x': 42, 'y': 0}, None)
+    assert rtn == 42
+
+
+@pytest.mark.parametrize('arg', ['if y:\n    pass',
+                                 'if 42:\n    pass',
+                                 'if x + y:\n    pass'])
+def test_call_macro_exec(arg):
+    def f(x : exec):
+        return x
+    rtn = call_macro(f, [arg], {'x': 42, 'y': 0}, None)
+    assert rtn is None
+
+

From 93e144f5b11a5eb3dcd0918a4eaf44723982e6dc Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 17:00:10 -0400
Subject: [PATCH 035/190] started docs

---
 docs/index.rst           |  1 +
 docs/tutorial_macros.rst | 52 ++++++++++++++++++++++++++++++++++++++++
 tests/test_builtins.py   |  2 --
 3 files changed, 53 insertions(+), 2 deletions(-)
 create mode 100644 docs/tutorial_macros.rst

diff --git a/docs/index.rst b/docs/index.rst
index c603cea6b..73200ed54 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -98,6 +98,7 @@ Contents
 
     tutorial
     tutorial_hist
+    tutorial_macros
     tutorial_xontrib
     tutorial_completers
     bash_to_xsh
diff --git a/docs/tutorial_macros.rst b/docs/tutorial_macros.rst
new file mode 100644
index 000000000..5cdf7ed11
--- /dev/null
+++ b/docs/tutorial_macros.rst
@@ -0,0 +1,52 @@
+.. _tutorial_macros:
+
+************************************
+Tutorial: Macros
+************************************
+Bust out your DSLRs, people. It is time to closely examine macros!
+
+What are macro instructions?
+============================
+In generic terms, a programming macro is a special kind of syntax that
+replaces a smaller amount of code with a larger expression, syntax tree,
+code object, etc after the macro has been evaluated.
+In practice, macros pause the normal parsing and evaluation of the code
+that they contain. This is so that they can perform their expansion with
+a complete inputs. Roughly, the algorithm executing a macro follows is:
+
+1. Macro start, pause or skip normal parsing
+2. Gather macro inputs as strings
+3. Evaluate macro with inputs
+4. Resume normal parsing and execution.
+
+Is this metaprogramming? You betcha!
+
+When and where are macros used?
+===============================
+Macros are a practicality-beats-purity feature of many programing
+languages. Because they allow you break out of the normal parsing
+cycle, depending on the language, you acn do some truly wild things with
+them. However, macros are really there to reduce the amount of boiler plate
+code that users and developers have to write.
+
+In C and C++ (and Fortran), the C Preprocessor ``cpp`` is a macro evaluation
+engine. For example, every time you see an ``#include`` or ``#ifdef``, this is
+the ``cpp`` macro system in action.
+In these languages, the macros are technically outside of the definition
+of the language at hand. Furthermore, because ``cpp`` must function with only
+a single pass through the code, the sorts of macros that can be written with
+``cpp`` are relatively simple.
+
+Rust, on the other hand, has a first-class notion of macros that look and
+feel a lot like normal functions. Macros in Rust are capable of pulling off
+type information from their arguments and preventing their return values
+from being consumed.
+
+Other languages like Lisp, Forth, and Julia also provide thier macro systems.
+Even restructured text (rST) directives could be considered macros.
+Haskell and other more purely functional languages do not need macros (since
+evaluation is lazy anyway), and so do not have them.
+
+Function Macros
+===============
+Xonsh supports Rust-like macros that are based on Python callables.
diff --git a/tests/test_builtins.py b/tests/test_builtins.py
index 48de5ab15..e888dc878 100644
--- a/tests/test_builtins.py
+++ b/tests/test_builtins.py
@@ -227,5 +227,3 @@ def test_call_macro_exec(arg):
         return x
     rtn = call_macro(f, [arg], {'x': 42, 'y': 0}, None)
     assert rtn is None
-
-

From 0b67ae18650368c5698c23a207ec1853f54d4905 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 17:43:37 -0400
Subject: [PATCH 036/190] more docs

---
 docs/tutorial_macros.rst | 90 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 89 insertions(+), 1 deletion(-)

diff --git a/docs/tutorial_macros.rst b/docs/tutorial_macros.rst
index 5cdf7ed11..4a27bcad7 100644
--- a/docs/tutorial_macros.rst
+++ b/docs/tutorial_macros.rst
@@ -47,6 +47,94 @@ Even restructured text (rST) directives could be considered macros.
 Haskell and other more purely functional languages do not need macros (since
 evaluation is lazy anyway), and so do not have them.
 
+If these seem unfamiliar to the Python world, note that Jupyter and IPython
+magics ``%`` and ``%%`` are macros!
+
 Function Macros
 ===============
-Xonsh supports Rust-like macros that are based on Python callables.
+Xonsh supports Rust-like macros that are based on normal Python callables.
+Macros do not require a special definition in xonsh. However, like in Rust,
+they must be called with an exclamation point ``!`` between the callable
+and the opening parentheses ``(``. Macro arguments are split on the top-level
+commas ``,``, like normal Python functions.  For example, say we have the
+functions ``f`` and ``g``. We could perform a macro call on these functions
+with the following:
+
+.. code-block:: xonsh
+
+    # No macro args
+    f!()
+
+    # Single arg
+    f!(x)
+    g!([y, 43, 44])
+
+    # Two args
+    f!(x, x + 42)
+    g!([y, 43, 44], f!(z))
+
+Not so bad, right?  So what actually happens when to the arguments when used
+in a macro call?  Well, that depends onthe defintion of the function. In
+particular, each argument in the macro call is matched up with the cooresponding
+parameter annotation in the callable's signature.  For example, say we have
+an ``identity()`` function that is annotates its sole argument as a string:
+
+.. code-block:: xonsh
+
+    def identity(x : str):
+        return x
+
+If we call this normally, we'll just get whatever object we put in back out,
+even if that object is not a string:
+
+.. code-block:: xonshcon
+
+    >>> identity('me')
+    'me'
+
+    >>> identity(42)
+    42
+
+    >>> identity(identity)
+    <function __main__.identity>
+
+However, if we perform macro calls instead we are now gauranteed to get a
+the string of the source code that is in the macro call:
+
+.. code-block:: xonshcon
+
+    >>> identity!('me')
+    "'me'"
+
+    >>> identity!(42)
+    '42'
+
+    >>> identity!(identity)
+    'identity'
+
+Also note that each macro argument is stripped prior to passing it to the
+macro itself. This is done for consistency.
+
+.. code-block:: xonshcon
+
+    >>> identity!(42)
+    '42'
+
+    >>> identity!(  42 )
+    '42'
+
+Importantly, because we are capturing and not evaluating the source code,
+a macro call can contain input that is beyond the usual syntax. In fact, that
+is sort of the whole point. Here are some cases to start your gears turning:
+
+.. code-block:: xonshcon
+
+    >>> identity!(import os)
+    'import os'
+
+    >>> identity!(if True:
+    >>>     pass)
+    'if True:\n    pass'
+
+
+

From 92e055975f817d65350eca13c284d38945d38962 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 18:13:36 -0400
Subject: [PATCH 037/190] more

---
 docs/tutorial_macros.rst | 42 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/docs/tutorial_macros.rst b/docs/tutorial_macros.rst
index 4a27bcad7..899017f6f 100644
--- a/docs/tutorial_macros.rst
+++ b/docs/tutorial_macros.rst
@@ -136,5 +136,47 @@ is sort of the whole point. Here are some cases to start your gears turning:
     >>>     pass)
     'if True:\n    pass'
 
+    >>> identity!(std::vector<std::string> x = {"yoo", "hoo"})
+    'std::vector<std::string> x = {"yoo", "hoo"}'
+
+You do you, ``identity()``.
+
+Calling Function Macros
+=======================
+There are a couple of points to consider when calling macros. The first is
+that passing in arguments by name will not behave as expected. This is because
+the ``<name>=`` is captured by the macro itself. Using the ``identity()``
+function from above:
+
+.. code-block:: xonshcon
+
+    >>> identity!(x=42)
+    'x=42'
+
+Performing a macro call uses only argument order to pass in values.
+
+Additionally, macro calls split arguments only on the top-level commas.
+The top-level commas are not included in any argument.
+This behaves analogously to normal Python function calls. For instance,
+say we have the following ``g()`` function that accepts two arguments:
+
+.. code-block:: xonsh
+
+    def g(x : str, y : str):
+        print('x = ' + repr(x))
+        print('y = ' + repr(y))
+
+Then you can see the splitting and stripping behaviour on each macro
+argument:
+
+.. code-block:: xonshcon
+
+    >>> g!(42, 65)
+    x = '42'
+    y = '65'
 
 
+Writing Function Macros
+=======================
+Though any function (or callable) can be used as a macro, this functionality
+is probably most useful if the function was designed to be used as a macro.

From 92466763a1feeafbf82c7139a1a4f09de88f2656 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 18:48:55 -0400
Subject: [PATCH 038/190] fixed trailing macro comma issue

---
 tests/test_parser.py  | 10 ++++++++++
 xonsh/parsers/base.py | 13 ++++++++-----
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/tests/test_parser.py b/tests/test_parser.py
index 023af58c4..6ee917b21 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -1831,3 +1831,13 @@ def test_macro_call_three_args(s, t, u):
     assert args[0].s == s.strip()
     assert args[1].s == t.strip()
     assert args[2].s == u.strip()
+
+
+@pytest.mark.parametrize('s', MACRO_ARGS)
+def test_macro_call_one_trailing(s):
+    f = 'f!({0},)'.format(s)
+    tree = check_xonsh_ast({}, f, False, return_obs=True)
+    assert isinstance(tree, AST)
+    args = tree.body.args[1].elts
+    assert len(args) == 1
+    assert args[0].s == s.strip()
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index 3e16116e1..c04ef51ea 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -238,7 +238,7 @@ class BaseParser(object):
             'yield_expr_or_testlist_comp', 'dictorsetmaker',
             'comma_subscript_list', 'test', 'sliceop', 'comp_iter',
             'yield_arg', 'test_comma_list', 'comma_nocomma_list',
-            'macroarglist', 'comma_tok', 'any_raw_toks']
+            'macroarglist', 'any_raw_toks']
         for rule in opt_rules:
             self._opt_rule(rule)
 
@@ -1961,12 +1961,15 @@ class BaseParser(object):
         p1 = p[1]
         p[0] = [(p1.lineno, p1.lexpos, None)]
 
+    def p_comma_trailing_nocomma(self, p):
+        """comma_nocomma : comma_tok"""
+        p1 = p[1]
+        p[0] = [(p1.lineno, p1.lexpos, 'trailing')]
+
     def p_macroarglist(self, p):
-        """macroarglist : nocomma comma_nocomma_list_opt comma_tok_opt"""
-        p2, p3 = p[2], p[3]
+        """macroarglist : nocomma comma_nocomma_list_opt"""
+        p2 = p[2]
         pos = [] if p2 is None else p2
-        if p3 is not None:
-            pos.append((p3.lineno, p3.lexpos, 'trailing'))
         p[0] = pos
 
     def p_subscriptlist(self, p):

From ea58a8bdd032b2056750d600dcf66595ac44bdec Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 19:00:41 -0400
Subject: [PATCH 039/190] added ws after comma

---
 docs/tutorial_macros.rst | 4 ++++
 tests/test_parser.py     | 9 +++++++++
 xonsh/parsers/base.py    | 6 ++++--
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/docs/tutorial_macros.rst b/docs/tutorial_macros.rst
index 899017f6f..89a95339b 100644
--- a/docs/tutorial_macros.rst
+++ b/docs/tutorial_macros.rst
@@ -175,6 +175,10 @@ argument:
     x = '42'
     y = '65'
 
+    >>> g!(42, 65,)
+    x = '42'
+    y = '65'
+
 
 Writing Function Macros
 =======================
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 6ee917b21..56f7f476e 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -1841,3 +1841,12 @@ def test_macro_call_one_trailing(s):
     args = tree.body.args[1].elts
     assert len(args) == 1
     assert args[0].s == s.strip()
+
+@pytest.mark.parametrize('s', MACRO_ARGS)
+def test_macro_call_one_trailing_space(s):
+    f = 'f!( {0}, )'.format(s)
+    tree = check_xonsh_ast({}, f, False, return_obs=True)
+    assert isinstance(tree, AST)
+    args = tree.body.args[1].elts
+    assert len(args) == 1
+    assert args[0].s == s.strip()
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index c04ef51ea..22bddd013 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -238,7 +238,7 @@ class BaseParser(object):
             'yield_expr_or_testlist_comp', 'dictorsetmaker',
             'comma_subscript_list', 'test', 'sliceop', 'comp_iter',
             'yield_arg', 'test_comma_list', 'comma_nocomma_list',
-            'macroarglist', 'any_raw_toks']
+            'macroarglist', 'any_raw_toks', 'comma_tok']
         for rule in opt_rules:
             self._opt_rule(rule)
 
@@ -1962,7 +1962,9 @@ class BaseParser(object):
         p[0] = [(p1.lineno, p1.lexpos, None)]
 
     def p_comma_trailing_nocomma(self, p):
-        """comma_nocomma : comma_tok"""
+        """comma_nocomma : comma_tok
+                         | comma_tok WS
+        """
         p1 = p[1]
         p[0] = [(p1.lineno, p1.lexpos, 'trailing')]
 

From bed66a0ea548bef41582b6f7ee3410ed21ec81de Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 19:05:41 -0400
Subject: [PATCH 040/190] minor docs

---
 docs/tutorial_macros.rst | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docs/tutorial_macros.rst b/docs/tutorial_macros.rst
index 89a95339b..3029cb333 100644
--- a/docs/tutorial_macros.rst
+++ b/docs/tutorial_macros.rst
@@ -179,6 +179,15 @@ argument:
     x = '42'
     y = '65'
 
+    >>> g!( 42, 65, )
+    x = '42'
+    y = '65'
+
+    >>> g!(['x', 'y'], {1: 1, 2: 3})
+    x = "['x', 'y']"
+    y = '{1: 1, 2: 3}'
+
+Hopefully now you see the big picture.
 
 Writing Function Macros
 =======================

From 398e11b388866f46fed1668fa1473da800839551 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 19:43:12 -0400
Subject: [PATCH 041/190] fixed macro context bug

---
 docs/tutorial_macros.rst |  7 +++++--
 xonsh/built_ins.py       | 11 ++++++++++-
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/docs/tutorial_macros.rst b/docs/tutorial_macros.rst
index 3029cb333..7c38703e3 100644
--- a/docs/tutorial_macros.rst
+++ b/docs/tutorial_macros.rst
@@ -187,9 +187,12 @@ argument:
     x = "['x', 'y']"
     y = '{1: 1, 2: 3}'
 
-Hopefully now you see the big picture.
+Hopefully, now you see the big picture.
 
 Writing Function Macros
 =======================
 Though any function (or callable) can be used as a macro, this functionality
-is probably most useful if the function was designed to be used as a macro.
+is probably most useful if the function was *designed* as a macro. There
+are two aspects
+
+globals, locals
diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py
index aaad01299..8e01c529f 100644
--- a/xonsh/built_ins.py
+++ b/xonsh/built_ins.py
@@ -781,10 +781,19 @@ def macro_context(f, glbs, locs):
     locs : Mapping or None
         The locals from the call site.
     """
+    prev_glbs = getattr(f, 'macro_globals', None)
+    prev_locs = getattr(f, 'macro_locals', None)
     f.macro_globals = glbs
     f.macro_locals = locs
     yield
-    del f.macro_globals, f.macro_locals
+    if prev_glbs is None:
+        del f.macro_globals
+    else:
+        f.macro_globals = prev_glbs
+    if prev_locs is None:
+        del f.macro_locals
+    else:
+        f.macro_locals = prev_locs
 
 
 def call_macro(f, raw_args, glbs, locs):

From dcc17effd10d1d37d3c3b0110ce9cc07b4c85809 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 20:32:30 -0400
Subject: [PATCH 042/190] more stuff

---
 docs/tutorial_macros.rst | 87 +++++++++++++++++++++++++++++++++++++++-
 tests/test_builtins.py   |  2 +-
 xonsh/built_ins.py       |  2 +-
 3 files changed, 88 insertions(+), 3 deletions(-)

diff --git a/docs/tutorial_macros.rst b/docs/tutorial_macros.rst
index 7c38703e3..fae187ebc 100644
--- a/docs/tutorial_macros.rst
+++ b/docs/tutorial_macros.rst
@@ -193,6 +193,91 @@ Writing Function Macros
 =======================
 Though any function (or callable) can be used as a macro, this functionality
 is probably most useful if the function was *designed* as a macro. There
-are two aspects
+are two main aspects of macro design to consider: argument annotations and
+call site execution context.
 
+Macro Function Argument Annotations
+-----------------------------------
+There are five kinds of annotations that macros are able to interpret:
+
+.. list-table:: Kinds of Annotation
+   :header-rows: 1
+
+   * - Category
+     - Object
+     - Flags
+     - Modes
+     - Returns
+   * - String
+     - ``str``
+     - ``'s'``, ``'str'``, or ``'string'``
+     -
+     - Source code of argument as string.
+   * - AST
+     - ``ast.AST``
+     - ``'a'`` or ``'ast'``
+     - ``'eval'`` (default), ``'exec'``, or ``'single'``
+     - Abstract syntax tree of argument.
+   * - Code
+     - ``types.CodeType`` or ``compile``
+     - ``'c'``, ``'code'``, or ``'compile'``
+     - ``'eval'`` (default), ``'exec'``, or ``'single'``
+     - Compiled code object of argument.
+   * - Eval
+     - ``eval`` or ``None``
+     - ``'v'`` or ``'eval'``
+     -
+     - Evaluation of the argument, *default*.
+   * - Exec
+     - ``exec``
+     - ``'x'`` or ``'exec'``
+     - ``'exec'`` (default) or ``'single'``
+     - Execs the argument and returns None.
+
+These annotations allow you to hook into whichever stage of the compilation
+that you desire. It is important note that the string form of the arguments
+is split and stripped (as described above) prior to conversion to the
+annotation type.
+
+Each argument may be annotated with its own indivdual type. Annotations
+may be provided as either objects or as the string flags seen in the above
+table. String flags are case-insensitive.
+If an argument does not have an annotation, ``eval`` is selected.
+This makes the macro call behave like a normal function call for
+arguments whose annotations are unspecified.  For example,
+
+.. code-block:: xonsh
+
+    def func(a, b : 'AST', c : compile):
+        pass
+
+In a macro call of ``func!()``,
+
+* ``a`` will be evaluated with ``eval`` since no annotation was provided,
+* ``b`` will be parsed into a syntax tree node, and
+* ``c`` will be compiled into code object since the builtin ``compile()``
+  function was used as the annotation.
+
+Additionally, certain kinds of annotations have different modes that
+affect the parsing, compilation, and execution of its argument.  While a
+sensible default is provided, you may also supply your own. This is
+done by annotating with a (kind, mode) tuple.  The first element can
+be any valid object or flag. The sencond element must be a cooresponding
+mode as a string.  For instance,
+
+.. code-block:: xonsh
+
+    def gunc(d : (exec, 'single'), e : ('c', 'exec')):
+        pass
+
+Thus in a macro call of ``gunc!()``,
+
+* ``d`` will be exec'd in single-mode (rather than exec-mode), and
+* ``e`` will be compiled in exec-mode (rather than eval-mode).
+
+For more information on the differences between the exec, eval, and single
+modes please see the Python documentation.
+
+Macro Function Execution Context
+--------------------------------
 globals, locals
diff --git a/tests/test_builtins.py b/tests/test_builtins.py
index e888dc878..8d4cac395 100644
--- a/tests/test_builtins.py
+++ b/tests/test_builtins.py
@@ -144,7 +144,7 @@ def test_convert_macro_arg_code(kind):
     assert isinstance(arg, types.CodeType)
 
 
-@pytest.mark.parametrize('kind', [eval, 'v', 'eval'])
+@pytest.mark.parametrize('kind', [eval, None, 'v', 'eval'])
 def test_convert_macro_arg_eval(kind):
     # literals
     raw_arg = '42'
diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py
index 8e01c529f..380b49a06 100644
--- a/xonsh/built_ins.py
+++ b/xonsh/built_ins.py
@@ -752,7 +752,7 @@ def convert_macro_arg(raw_arg, kind, glbs, locs, *, name='<arg>',
         mode = mode or 'eval'
         arg = execer.compile(raw_arg, mode=mode, glbs=glbs, locs=locs,
                              filename=filename)
-    elif kind is eval:
+    elif kind is eval or kind is None:
         arg = execer.eval(raw_arg, glbs=glbs, locs=locs, filename=filename)
     elif kind is exec:
         mode = mode or 'exec'

From 9f8035ee49855b35776221c1edbe171f9dbd1e6c Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 21:00:00 -0400
Subject: [PATCH 043/190] more macros

---
 docs/tutorial_macros.rst | 37 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)

diff --git a/docs/tutorial_macros.rst b/docs/tutorial_macros.rst
index fae187ebc..bc5f64ed2 100644
--- a/docs/tutorial_macros.rst
+++ b/docs/tutorial_macros.rst
@@ -196,6 +196,7 @@ is probably most useful if the function was *designed* as a macro. There
 are two main aspects of macro design to consider: argument annotations and
 call site execution context.
 
+
 Macro Function Argument Annotations
 -----------------------------------
 There are five kinds of annotations that macros are able to interpret:
@@ -278,6 +279,40 @@ Thus in a macro call of ``gunc!()``,
 For more information on the differences between the exec, eval, and single
 modes please see the Python documentation.
 
+
 Macro Function Execution Context
 --------------------------------
-globals, locals
+Equally important as having the macro arguments is knowing the execution
+context of the macro call itself. Rather than mucking around with frames,
+macros provide both the globals and locals of the call site.  These are
+accessible as the ``macro_globals`` and ``macro_locals`` attributes of
+the macro function itself while the macro is being executed.
+
+For example, consider a macro which replaces all literal ``1`` digits
+with the literal ``2``, evaluates the modification, and returns the results.
+To eval, the macro will need to pull off its globals and locals:
+
+.. code-block:: xonsh
+
+    def one_to_two(x : str):
+        s = x.replace('1', '2')
+        glbs = one_to_two.macro_globals
+        locs = one_to_two.macro_locals
+        return eval(s, glbs, locs)
+
+Running this with a few of different inputs, we see:
+
+.. code-block:: xonshcon
+
+    >>> one_to_two!(1 + 1)
+    4
+
+    >>> one_to_two!(11)
+    22
+
+    >>> x = 1
+    >>> one_to_two!(x + 1)
+    3
+
+Of course, many other more sophisticated options are available depending on the
+use case.
\ No newline at end of file

From 1d595e9ad9cb129383456f5a8923744f2cf68426 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 21:11:07 -0400
Subject: [PATCH 044/190] added types to macros

---
 docs/tutorial_macros.rst |  7 ++++++-
 tests/test_builtins.py   | 14 +++++++++++++-
 xonsh/built_ins.py       |  5 +++++
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/docs/tutorial_macros.rst b/docs/tutorial_macros.rst
index bc5f64ed2..bde403e7a 100644
--- a/docs/tutorial_macros.rst
+++ b/docs/tutorial_macros.rst
@@ -199,7 +199,7 @@ call site execution context.
 
 Macro Function Argument Annotations
 -----------------------------------
-There are five kinds of annotations that macros are able to interpret:
+There are six kinds of annotations that macros are able to interpret:
 
 .. list-table:: Kinds of Annotation
    :header-rows: 1
@@ -234,6 +234,11 @@ There are five kinds of annotations that macros are able to interpret:
      - ``'x'`` or ``'exec'``
      - ``'exec'`` (default) or ``'single'``
      - Execs the argument and returns None.
+   * - Type
+     - ``type``
+     - ``'t'`` or ``'type'``
+     -
+     - The type of the argument after ithas been evaluated.
 
 These annotations allow you to hook into whichever stage of the compilation
 that you desire. It is important note that the string form of the arguments
diff --git a/tests/test_builtins.py b/tests/test_builtins.py
index 8d4cac395..559e458e8 100644
--- a/tests/test_builtins.py
+++ b/tests/test_builtins.py
@@ -157,7 +157,7 @@ def test_convert_macro_arg_eval(kind):
 
 
 @pytest.mark.parametrize('kind', [exec, 'x', 'exec'])
-def test_convert_macro_arg_eval(kind):
+def test_convert_macro_arg_exec(kind):
     # at global scope
     raw_arg = 'def f(x, y):\n    return x + y'
     glbs = {}
@@ -177,6 +177,18 @@ def test_convert_macro_arg_eval(kind):
     assert locs['y'] == 43
 
 
+@pytest.mark.parametrize('kind', [type, 't', 'type'])
+def test_convert_macro_arg_eval(kind):
+    # literals
+    raw_arg = '42'
+    arg = convert_macro_arg(raw_arg, kind, {}, None)
+    assert arg is int
+    # exprs
+    raw_arg = 'x + 41'
+    arg = convert_macro_arg(raw_arg, kind, {}, {'x': 1})
+    assert arg is int
+
+
 def test_macro_context():
     def f():
         pass
diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py
index 380b49a06..d16d2b486 100644
--- a/xonsh/built_ins.py
+++ b/xonsh/built_ins.py
@@ -696,6 +696,8 @@ def MACRO_FLAG_KINDS():
         'eval': eval,
         'x': exec,
         'exec': exec,
+        't': type,
+        'type': type,
         }
 
 def _convert_kind_flag(x):
@@ -760,6 +762,9 @@ def convert_macro_arg(raw_arg, kind, glbs, locs, *, name='<arg>',
             raw_arg += '\n'
         arg = execer.exec(raw_arg, mode=mode, glbs=glbs, locs=locs,
                           filename=filename)
+    elif kind is type:
+        arg = type(execer.eval(raw_arg, glbs=glbs, locs=locs,
+                               filename=filename))
     else:
         msg = ('kind={0!r} and mode={1!r} was not recongnized for macro '
                'argument {2!r}')

From 0e7ed93c24e2d59a7ec26b5ec0802ac908898895 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 21:16:45 -0400
Subject: [PATCH 045/190] doc spell check

---
 docs/tutorial_macros.rst | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/docs/tutorial_macros.rst b/docs/tutorial_macros.rst
index bde403e7a..4d55ec23d 100644
--- a/docs/tutorial_macros.rst
+++ b/docs/tutorial_macros.rst
@@ -19,13 +19,13 @@ a complete inputs. Roughly, the algorithm executing a macro follows is:
 3. Evaluate macro with inputs
 4. Resume normal parsing and execution.
 
-Is this metaprogramming? You betcha!
+Is this meta-programming? You betcha!
 
 When and where are macros used?
 ===============================
 Macros are a practicality-beats-purity feature of many programing
 languages. Because they allow you break out of the normal parsing
-cycle, depending on the language, you acn do some truly wild things with
+cycle, depending on the language, you can do some truly wild things with
 them. However, macros are really there to reduce the amount of boiler plate
 code that users and developers have to write.
 
@@ -42,7 +42,7 @@ feel a lot like normal functions. Macros in Rust are capable of pulling off
 type information from their arguments and preventing their return values
 from being consumed.
 
-Other languages like Lisp, Forth, and Julia also provide thier macro systems.
+Other languages like Lisp, Forth, and Julia also provide their macro systems.
 Even restructured text (rST) directives could be considered macros.
 Haskell and other more purely functional languages do not need macros (since
 evaluation is lazy anyway), and so do not have them.
@@ -74,8 +74,8 @@ with the following:
     g!([y, 43, 44], f!(z))
 
 Not so bad, right?  So what actually happens when to the arguments when used
-in a macro call?  Well, that depends onthe defintion of the function. In
-particular, each argument in the macro call is matched up with the cooresponding
+in a macro call?  Well, that depends on the definition of the function. In
+particular, each argument in the macro call is matched up with the corresponding
 parameter annotation in the callable's signature.  For example, say we have
 an ``identity()`` function that is annotates its sole argument as a string:
 
@@ -98,7 +98,7 @@ even if that object is not a string:
     >>> identity(identity)
     <function __main__.identity>
 
-However, if we perform macro calls instead we are now gauranteed to get a
+However, if we perform macro calls instead we are now guaranteed to get a
 the string of the source code that is in the macro call:
 
 .. code-block:: xonshcon
@@ -166,7 +166,7 @@ say we have the following ``g()`` function that accepts two arguments:
         print('x = ' + repr(x))
         print('y = ' + repr(y))
 
-Then you can see the splitting and stripping behaviour on each macro
+Then you can see the splitting and stripping behavior on each macro
 argument:
 
 .. code-block:: xonshcon
@@ -238,14 +238,14 @@ There are six kinds of annotations that macros are able to interpret:
      - ``type``
      - ``'t'`` or ``'type'``
      -
-     - The type of the argument after ithas been evaluated.
+     - The type of the argument after it has been evaluated.
 
 These annotations allow you to hook into whichever stage of the compilation
 that you desire. It is important note that the string form of the arguments
 is split and stripped (as described above) prior to conversion to the
 annotation type.
 
-Each argument may be annotated with its own indivdual type. Annotations
+Each argument may be annotated with its own individual type. Annotations
 may be provided as either objects or as the string flags seen in the above
 table. String flags are case-insensitive.
 If an argument does not have an annotation, ``eval`` is selected.
@@ -268,7 +268,7 @@ Additionally, certain kinds of annotations have different modes that
 affect the parsing, compilation, and execution of its argument.  While a
 sensible default is provided, you may also supply your own. This is
 done by annotating with a (kind, mode) tuple.  The first element can
-be any valid object or flag. The sencond element must be a cooresponding
+be any valid object or flag. The second element must be a corresponding
 mode as a string.  For instance,
 
 .. code-block:: xonsh
@@ -320,4 +320,10 @@ Running this with a few of different inputs, we see:
     3
 
 Of course, many other more sophisticated options are available depending on the
-use case.
\ No newline at end of file
+use case.
+
+
+Take Away
+=========
+Hopefully, at this point, you see that a few well placed macros can be extremely
+convenient and valuable to any project.
\ No newline at end of file

From 841e916217b6e2a2e4b7df8f4a76f9f8b32a3aff Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 20 Aug 2016 21:20:29 -0400
Subject: [PATCH 046/190] macro news

---
 news/m.rst | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 news/m.rst

diff --git a/news/m.rst b/news/m.rst
new file mode 100644
index 000000000..99be46fe9
--- /dev/null
+++ b/news/m.rst
@@ -0,0 +1,14 @@
+**Added:**
+
+* Macro function calls are now available. These use a Rust-like
+  ``f!(arg)`` syntax.
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

From c2a56b5bc75537b14f285b30c4ca92e8e0cab2d7 Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Sun, 21 Aug 2016 10:07:18 +0800
Subject: [PATCH 047/190] fix parsing for bare tuple of tuples

---
 tests/test_parser.py  |  9 +++++++++
 xonsh/parsers/base.py | 13 ++++++++-----
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/tests/test_parser.py b/tests/test_parser.py
index 757a74c3f..0e6a1c223 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -437,6 +437,15 @@ def test_tuple_three():
 def test_tuple_three_comma():
     check_ast('(1, 42, 65,)')
 
+def test_bare_tuple_of_tuples():
+    check_ast('(),')
+    check_ast('((),),(1,)')
+    check_ast('(),(),')
+    check_ast('[],')
+    check_ast('[],()')
+    check_ast('(),[],')
+    check_ast('((),[()],)')
+
 def test_set_one():
     check_ast('{42}')
 
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index a2e037430..424061cb3 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -1675,6 +1675,7 @@ class BaseParser(object):
             # empty container atom
             p0 = ast.Tuple(elts=[], ctx=ast.Load(), lineno=self.lineno,
                            col_offset=self.col)
+            p0._real_tuple = True
         elif isinstance(p2, ast.AST):
             p0 = p2
             p0._lopen_lineno, p0._lopen_col = p1_tok.lineno, p1_tok.lexpos
@@ -1894,15 +1895,16 @@ class BaseParser(object):
         """testlist : test"""
         p1 = p[1]
         if isinstance(p1, ast.Tuple) and (hasattr(p1, '_real_tuple') and
-                                          p1._real_tuple):
+                                          p1._real_tuple and p1.elts):
             p1.lineno, p1.col_offset = lopen_loc(p1.elts[0])
         p[0] = p1
 
     def p_testlist_single(self, p):
         """testlist : test COMMA"""
         p1 = p[1]
-        if isinstance(p1, ast.Tuple) and (hasattr(p1, '_real_tuple') and
-                                          p1._real_tuple):
+        if isinstance(p1, ast.List) or (isinstance(p1, ast.Tuple) and
+                                        hasattr(p1, '_real_tuple') and
+                                        p1._real_tuple):
             lineno, col = lopen_loc(p1)
             p[0] = ast.Tuple(elts=[p1], ctx=ast.Load(),
                              lineno=p1.lineno, col_offset=p1.col_offset)
@@ -1914,8 +1916,9 @@ class BaseParser(object):
                     | test comma_test_list
         """
         p1 = p[1]
-        if isinstance(p1, ast.Tuple) and (hasattr(p1, '_real_tuple') and
-                                          p1._real_tuple):
+        if isinstance(p1, ast.List) or (isinstance(p1, ast.Tuple) and
+                                        hasattr(p1, '_real_tuple') and
+                                        p1._real_tuple):
             lineno, col = lopen_loc(p1)
             p1 = ast.Tuple(elts=[p1], ctx=ast.Load(),
                            lineno=p1.lineno, col_offset=p1.col_offset)

From 48e7506ca0fe0c4ad8271f4d04c49af41bc96bfc Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sun, 21 Aug 2016 06:14:55 -0400
Subject: [PATCH 048/190] NetBSD support

---
 news/netbsd.rst   | 13 +++++++++++++
 xonsh/aliases.py  |  4 ++--
 xonsh/platform.py |  6 ++++++
 3 files changed, 21 insertions(+), 2 deletions(-)
 create mode 100644 news/netbsd.rst

diff --git a/news/netbsd.rst b/news/netbsd.rst
new file mode 100644
index 000000000..0bc886fe7
--- /dev/null
+++ b/news/netbsd.rst
@@ -0,0 +1,13 @@
+**Added:**
+
+* NetBSD is now supported.
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None
diff --git a/xonsh/aliases.py b/xonsh/aliases.py
index 2024b713b..c1eff7934 100644
--- a/xonsh/aliases.py
+++ b/xonsh/aliases.py
@@ -14,7 +14,7 @@ from xonsh.environ import locate_binary
 from xonsh.foreign_shells import foreign_shell_data
 from xonsh.jobs import jobs, fg, bg, clean_jobs
 from xonsh.history import history_main
-from xonsh.platform import (ON_ANACONDA, ON_DARWIN, ON_WINDOWS, ON_FREEBSD,
+from xonsh.platform import (ON_ANACONDA, ON_DARWIN, ON_WINDOWS, ON_BSD,
                             scandir)
 from xonsh.proc import foreground
 from xonsh.replay import replay_main
@@ -588,7 +588,7 @@ def make_default_aliases():
             default_aliases['sudo'] = sudo
     elif ON_DARWIN:
         default_aliases['ls'] = ['ls', '-G']
-    elif ON_FREEBSD:
+    elif ON_BSD:
         default_aliases['grep'] = ['grep', '--color=auto']
         default_aliases['egrep'] = ['egrep', '--color=auto']
         default_aliases['fgrep'] = ['fgrep', '--color=auto']
diff --git a/xonsh/platform.py b/xonsh/platform.py
index f2f2e1fcc..f86383bb9 100644
--- a/xonsh/platform.py
+++ b/xonsh/platform.py
@@ -46,6 +46,12 @@ ON_POSIX = LazyBool(lambda: (os.name == 'posix'), globals(), 'ON_POSIX')
 ON_FREEBSD = LazyBool(lambda: (sys.platform.startswith('freebsd')),
                       globals(), 'ON_FREEBSD')
 """``True`` if on a FreeBSD operating system, else ``False``."""
+ON_NETBSD = LazyBool(lambda: (sys.platform.startswith('netbsd')),
+                      globals(), 'ON_NETBSD')
+"""``True`` if on a NetBSD operating system, else ``False``."""
+ON_BSD = LazyBool(lambda: (ON_FREEBSD or ON_NETBSD)),
+                  globals(), 'ON_BSD')
+"""``True`` if on a BSD operating system, else ``False``."""
 
 
 #

From 6880e74bae96d31efcf77d7c55d182699f1edd8e Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sun, 21 Aug 2016 06:52:20 -0400
Subject: [PATCH 049/190] some updates

---
 xonsh/aliases.py  | 10 +++++++---
 xonsh/platform.py |  2 +-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/xonsh/aliases.py b/xonsh/aliases.py
index c1eff7934..d185404f9 100644
--- a/xonsh/aliases.py
+++ b/xonsh/aliases.py
@@ -14,8 +14,8 @@ from xonsh.environ import locate_binary
 from xonsh.foreign_shells import foreign_shell_data
 from xonsh.jobs import jobs, fg, bg, clean_jobs
 from xonsh.history import history_main
-from xonsh.platform import (ON_ANACONDA, ON_DARWIN, ON_WINDOWS, ON_BSD,
-                            scandir)
+from xonsh.platform import (ON_ANACONDA, ON_DARWIN, ON_WINDOWS, ON_FREEBSD,
+                            ON_NETBSD, scandir)
 from xonsh.proc import foreground
 from xonsh.replay import replay_main
 from xonsh.timings import timeit_alias
@@ -588,11 +588,15 @@ def make_default_aliases():
             default_aliases['sudo'] = sudo
     elif ON_DARWIN:
         default_aliases['ls'] = ['ls', '-G']
-    elif ON_BSD:
+    elif ON_FREEBSD:
         default_aliases['grep'] = ['grep', '--color=auto']
         default_aliases['egrep'] = ['egrep', '--color=auto']
         default_aliases['fgrep'] = ['fgrep', '--color=auto']
         default_aliases['ls'] = ['ls', '-G']
+    elif ON_NETBSD:
+        default_aliases['grep'] = ['grep', '--color=auto']
+        default_aliases['egrep'] = ['egrep', '--color=auto']
+        default_aliases['fgrep'] = ['fgrep', '--color=auto']
     else:
         default_aliases['grep'] = ['grep', '--color=auto']
         default_aliases['egrep'] = ['egrep', '--color=auto']
diff --git a/xonsh/platform.py b/xonsh/platform.py
index f86383bb9..11921cc38 100644
--- a/xonsh/platform.py
+++ b/xonsh/platform.py
@@ -49,7 +49,7 @@ ON_FREEBSD = LazyBool(lambda: (sys.platform.startswith('freebsd')),
 ON_NETBSD = LazyBool(lambda: (sys.platform.startswith('netbsd')),
                       globals(), 'ON_NETBSD')
 """``True`` if on a NetBSD operating system, else ``False``."""
-ON_BSD = LazyBool(lambda: (ON_FREEBSD or ON_NETBSD)),
+ON_BSD = LazyBool(lambda: ON_FREEBSD or ON_NETBSD,
                   globals(), 'ON_BSD')
 """``True`` if on a BSD operating system, else ``False``."""
 

From 577f5661dd59bca11759a3a222bd57768f04cdc9 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sun, 21 Aug 2016 07:02:16 -0400
Subject: [PATCH 050/190] silly formatting

---
 xonsh/platform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xonsh/platform.py b/xonsh/platform.py
index 11921cc38..0430d66c7 100644
--- a/xonsh/platform.py
+++ b/xonsh/platform.py
@@ -47,7 +47,7 @@ ON_FREEBSD = LazyBool(lambda: (sys.platform.startswith('freebsd')),
                       globals(), 'ON_FREEBSD')
 """``True`` if on a FreeBSD operating system, else ``False``."""
 ON_NETBSD = LazyBool(lambda: (sys.platform.startswith('netbsd')),
-                      globals(), 'ON_NETBSD')
+                     globals(), 'ON_NETBSD')
 """``True`` if on a NetBSD operating system, else ``False``."""
 ON_BSD = LazyBool(lambda: ON_FREEBSD or ON_NETBSD,
                   globals(), 'ON_BSD')

From bf5935c3ae97459cd9946604ee08021ea51b176a Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Sun, 21 Aug 2016 20:47:13 +0800
Subject: [PATCH 051/190] add more more test case

---
 tests/test_parser.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_parser.py b/tests/test_parser.py
index 0e6a1c223..044a54d1b 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -442,6 +442,7 @@ def test_bare_tuple_of_tuples():
     check_ast('((),),(1,)')
     check_ast('(),(),')
     check_ast('[],')
+    check_ast('[],[]')
     check_ast('[],()')
     check_ast('(),[],')
     check_ast('((),[()],)')

From 24801ac3ae46224006efad2554072ae6634cbd53 Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Sun, 21 Aug 2016 20:48:08 +0800
Subject: [PATCH 052/190] add changelog

---
 news/bare-tuple-of-tuples.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 news/bare-tuple-of-tuples.rst

diff --git a/news/bare-tuple-of-tuples.rst b/news/bare-tuple-of-tuples.rst
new file mode 100644
index 000000000..328b3e444
--- /dev/null
+++ b/news/bare-tuple-of-tuples.rst
@@ -0,0 +1,11 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** fix parsing for tuple of tuples (like `(),()`)
+
+**Security:** None

From da62cf2fa4c734ea389e9808588bc9e70a561260 Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Sun, 21 Aug 2016 20:52:13 +0800
Subject: [PATCH 053/190] fix last commit: format of changelog

---
 news/bare-tuple-of-tuples.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/news/bare-tuple-of-tuples.rst b/news/bare-tuple-of-tuples.rst
index 328b3e444..d09109b3e 100644
--- a/news/bare-tuple-of-tuples.rst
+++ b/news/bare-tuple-of-tuples.rst
@@ -6,6 +6,8 @@
 
 **Removed:** None
 
-**Fixed:** fix parsing for tuple of tuples (like `(),()`)
+**Fixed:**
+
+* fix parsing for tuple of tuples (like `(),()`)
 
 **Security:** None

From ef291361330e672f3b762281e0fba1be1059dd9f Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sun, 21 Aug 2016 09:59:20 -0400
Subject: [PATCH 054/190] some flake8 fixes

---
 setup.cfg          | 2 +-
 xonsh/built_ins.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 4c0ce9b9c..c6528a4b9 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -5,7 +5,7 @@ flake8-ignore =
     *.py E402
     tests/tools.py E128
     xonsh/ast.py F401
-    xonsh/built_ins.py F821
+    xonsh/built_ins.py F821 E721
     xonsh/commands_cache.py F841
     xonsh/history.py F821
     xonsh/pyghooks.py F821
diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py
index d16d2b486..0b12ab7e1 100644
--- a/xonsh/built_ins.py
+++ b/xonsh/built_ins.py
@@ -700,6 +700,7 @@ def MACRO_FLAG_KINDS():
         'type': type,
         }
 
+
 def _convert_kind_flag(x):
     """Puts a kind flag (string) a canonical form."""
     x = x.lower()

From f344231e21934530f056c3629af815727efe4fd7 Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Sun, 21 Aug 2016 22:52:55 +0800
Subject: [PATCH 055/190] source `bash_completion` to provide helper funcs

---
 xonsh/completers/bash.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/xonsh/completers/bash.py b/xonsh/completers/bash.py
index 071242ddc..56d006e9d 100644
--- a/xonsh/completers/bash.py
+++ b/xonsh/completers/bash.py
@@ -25,6 +25,10 @@ CACHED_HASH = None
 CACHED_FUNCS = None
 CACHED_FILES = None
 
+BASH_COMPLETE_SCRIPT_PRE = """BASH_COMPLETION_DIR="nonexist"
+BASH_COMPLETION_COMPAT_DIR="nonexist"
+"""
+
 BASH_COMPLETE_SCRIPT = """source "{filename}"
 COMP_WORDS=({line})
 COMP_LINE={comp_line}
@@ -108,6 +112,9 @@ def complete_from_bash(prefix, line, begidx, endidx, ctx):
         filename=fnme, line=' '.join(shlex.quote(p) for p in splt),
         comp_line=shlex.quote(line), n=n, func=func, cmd=cmd,
         end=endidx + 1, prefix=prefix, prev=shlex.quote(prev))
+    script = BASH_COMPLETE_SCRIPT_PRE + '\n' + \
+             '\n'.join(_collect_completions_sources()) + '\n' + \
+             script
     try:
         out = subprocess.check_output(
             [xp.bash_command()], input=script, universal_newlines=True,

From c9d18ffa83f401baffaf53934398e5724745ddc2 Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Sun, 21 Aug 2016 23:19:31 +0800
Subject: [PATCH 056/190] flake8 fix

---
 xonsh/completers/bash.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/xonsh/completers/bash.py b/xonsh/completers/bash.py
index 56d006e9d..8105b6285 100644
--- a/xonsh/completers/bash.py
+++ b/xonsh/completers/bash.py
@@ -113,8 +113,7 @@ def complete_from_bash(prefix, line, begidx, endidx, ctx):
         comp_line=shlex.quote(line), n=n, func=func, cmd=cmd,
         end=endidx + 1, prefix=prefix, prev=shlex.quote(prev))
     script = BASH_COMPLETE_SCRIPT_PRE + '\n' + \
-             '\n'.join(_collect_completions_sources()) + '\n' + \
-             script
+        '\n'.join(_collect_completions_sources()) + '\n' + script
     try:
         out = subprocess.check_output(
             [xp.bash_command()], input=script, universal_newlines=True,

From 1baa1d3de6c36d8f461321f0d3c9ef5c7ba603e4 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sun, 21 Aug 2016 12:09:51 -0400
Subject: [PATCH 057/190] yacc_debug loads parser on same thread.

---
 news/syncp.rst        | 16 ++++++++++++++++
 setup.py              |  2 +-
 xonsh/parsers/base.py |  9 ++++++---
 3 files changed, 23 insertions(+), 4 deletions(-)
 create mode 100644 news/syncp.rst

diff --git a/news/syncp.rst b/news/syncp.rst
new file mode 100644
index 000000000..0c44a4679
--- /dev/null
+++ b/news/syncp.rst
@@ -0,0 +1,16 @@
+**Added:** None
+
+**Changed:**
+
+* ``yacc_debug=True`` now load the parser on the same thread that the
+  Parser instance is created. ``setup.py`` now uses this synchronous
+  form as it was causing the parser table to be missed by some package
+  managers.
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None
diff --git a/setup.py b/setup.py
index 7771b1211..b5823e4c1 100755
--- a/setup.py
+++ b/setup.py
@@ -70,7 +70,7 @@ def build_tables():
     sys.path.insert(0, os.path.dirname(__file__))
     from xonsh.parser import Parser
     Parser(lexer_table='lexer_table', yacc_table='parser_table',
-           outputdir='xonsh')
+           outputdir='xonsh', yacc_debug=True)
     sys.path.pop(0)
 
 
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index a2e037430..fc99a3d45 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -273,9 +273,12 @@ class BaseParser(object):
         if outputdir is None:
             outputdir = os.path.dirname(os.path.dirname(__file__))
         yacc_kwargs['outputdir'] = outputdir
-        self.parser = None
-        YaccLoader(self, yacc_kwargs)
-        # self.parser = yacc.yacc(**yacc_kwargs)
+        if yacc_debug:
+            # create parser on main thread
+            self.parser = yacc.yacc(**yacc_kwargs)
+        else:
+            self.parser = None
+            YaccLoader(self, yacc_kwargs)
 
         # Keeps track of the last token given to yacc (the lookahead token)
         self._last_yielded_token = None

From 7837563baf6d3aa271257ec5c3eccccf9f60e4f8 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sun, 21 Aug 2016 14:58:07 -0400
Subject: [PATCH 058/190] check all names

---
 docs/faq.rst      |  7 ++++---
 docs/tutorial.rst | 14 +++++++-------
 news/allnames.rst | 16 ++++++++++++++++
 tests/test_ast.py | 27 +++++++++++++++++++++++----
 xonsh/ast.py      |  9 +++++----
 5 files changed, 55 insertions(+), 18 deletions(-)
 create mode 100644 news/allnames.rst

diff --git a/docs/faq.rst b/docs/faq.rst
index bc599f9c2..b3e6045af 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -74,9 +74,10 @@ Python variables, this could be transformed to the equivalent (Python)
 expressions ``ls - l`` or ``ls-l``.  Neither of which are valid listing
 commands.
 
-What xonsh does to overcome such ambiguity is to check if the left-most
-name (``ls`` above) is in the present Python context. If it is, then it takes
-the line to be valid xonsh as written. If the left-most name cannot be found,
+What xonsh does to overcome such ambiguity is to check if the names in the
+expression (``ls`` and ``l`` above) are in the present Python context. If they are,
+then it takes
+the line to be valid xonsh as written. If one of the names cannot be found,
 then xonsh assumes that the left-most name is an external command. It thus
 attempts to parse the line after wrapping it in an uncaptured subprocess
 call ``![]``.  If wrapped version successfully parses, the ``![]`` version
diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index c8ec0a306..9a67916a0 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -267,11 +267,11 @@ have also been written as ``ls - l`` or ``ls-l``.  So how does xonsh know
 that ``ls -l`` is meant to be run in subprocess-mode?
 
 For any given line that only contains an expression statement (expr-stmt,
-see the Python AST docs for more information), if the left-most name cannot
-be found as a current variable name xonsh will try to parse the line as a
-subprocess command instead.  In the above, if ``ls`` is not a variable,
-then subprocess mode will be attempted. If parsing in subprocess mode fails,
-then the line is left in Python-mode.
+see the Python AST docs for more information), if all the names cannot
+be found as current variables name xonsh will try to parse the line as a
+subprocess command instead.  In the above, if ``ls`` ans ``l`` are not a
+variables, then subprocess mode will be attempted. If parsing in subprocess
+mode fails, then the line is left in Python-mode.
 
 In the following example, we will list the contents of the directory
 with ``ls -l``. Then we'll make new variable names ``ls`` and ``l`` and then
@@ -284,7 +284,7 @@ the directories again.
     >>> ls -l
     total 0
     -rw-rw-r-- 1 snail snail 0 Mar  8 15:46 xonsh
-    >>> # set an ls variable to force python-mode
+    >>> # set ls and l variables to force python-mode
     >>> ls = 44
     >>> l = 2
     >>> ls -l
@@ -1132,7 +1132,7 @@ with keyword arguments:
 Removing an alias is as easy as deleting the key from the alias dictionary:
 
 .. code-block:: xonshcon
-    
+
     >>> del aliases['banana']
 
 .. note::
diff --git a/news/allnames.rst b/news/allnames.rst
new file mode 100644
index 000000000..cfd99b2ac
--- /dev/null
+++ b/news/allnames.rst
@@ -0,0 +1,16 @@
+**Added:** None
+
+**Changed:**
+
+* Context sensitive AST transformation now checks that all names in an
+  expression are in scope. If they are, then Python mode is retained. However,
+  if even one is missing, subprocess wrapping is attempted. Previously, only the
+  left-most name was examined for being within scope.
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None
diff --git a/tests/test_ast.py b/tests/test_ast.py
index 30245e8e5..f7c8a842c 100644
--- a/tests/test_ast.py
+++ b/tests/test_ast.py
@@ -2,7 +2,7 @@
 import ast as pyast
 
 from xonsh import ast
-from xonsh.ast import Tuple, Name, Store, min_line
+from xonsh.ast import Tuple, Name, Store, min_line, Call, BinOp
 
 import pytest
 
@@ -27,12 +27,31 @@ def test_gather_names_tuple():
     obs = ast.gather_names(node)
     assert exp == obs
 
-def test_multilline_num():
-    code = ('x = 1\n'
-            'ls -l\n')  # this second line wil be transformed
+
+@pytest.mark.parametrize('line1', [
+    # this second line wil be transformed into a subprocess call
+    'x = 1',
+    # this second line wil be transformed into a subprocess call even though
+    # ls is defined.
+    'ls = 1',
+    # the second line wil be transformed still even though l exists.
+    'l = 1',
+])
+def test_multilline_num(line1):
+    code = line1 + '\nls -l\n'
     tree = check_parse(code)
     lsnode = tree.body[1]
     assert 2 == min_line(lsnode)
+    assert isinstance(lsnode.value, Call)
+
+
+def test_multilline_no_transform():
+    # no subprocess transformations happen here since all variables are known
+    code = 'ls = 1\nl = 1\nls -l\n'
+    tree = check_parse(code)
+    lsnode = tree.body[2]
+    assert 3 == min_line(lsnode)
+    assert isinstance(lsnode.value, BinOp)
 
 
 @pytest.mark.parametrize('inp', [
diff --git a/xonsh/ast.py b/xonsh/ast.py
index 115c04ded..75be53ce7 100644
--- a/xonsh/ast.py
+++ b/xonsh/ast.py
@@ -226,12 +226,13 @@ class CtxAwareTransformer(NodeTransformer):
 
     def is_in_scope(self, node):
         """Determines whether or not the current node is in scope."""
-        lname = leftmostname(node)
-        if lname is None:
-            return node
+        names = gather_names(node)
+        if not names:
+            return True
         inscope = False
         for ctx in reversed(self.contexts):
-            if lname in ctx:
+            names -= ctx
+            if not names:
                 inscope = True
                 break
         return inscope

From 12b6340700a65fbc25554e955e92dbd910e63c8b Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sun, 21 Aug 2016 17:23:53 -0400
Subject: [PATCH 059/190] foreign shell function empty filename fix

---
 news/emptysh.rst        | 14 ++++++++++++++
 xonsh/foreign_shells.py |  4 ++--
 2 files changed, 16 insertions(+), 2 deletions(-)
 create mode 100644 news/emptysh.rst

diff --git a/news/emptysh.rst b/news/emptysh.rst
new file mode 100644
index 000000000..abcfdce76
--- /dev/null
+++ b/news/emptysh.rst
@@ -0,0 +1,14 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:**
+
+* Foreign shell functions that are mapped to empty filenames no longer
+  receive alaises since they can't be found to source later.
+
+**Security:** None
diff --git a/xonsh/foreign_shells.py b/xonsh/foreign_shells.py
index 77bf5d96d..83236212b 100644
--- a/xonsh/foreign_shells.py
+++ b/xonsh/foreign_shells.py
@@ -343,8 +343,8 @@ def parse_funcs(s, shell, sourcer=None):
         else sourcer
     funcs = {}
     for funcname, filename in namefiles.items():
-        if funcname.startswith('_'):
-            continue  # skip private functions
+        if funcname.startswith('_') or not filename:
+            continue  # skip private functions and invalid files
         if not os.path.isabs(filename):
             filename = os.path.abspath(filename)
         wrapper = ForeignShellFunctionAlias(name=funcname, shell=shell,

From 6e254f9180824ea81d47b64be656510d548a34ec Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sun, 21 Aug 2016 17:56:09 -0400
Subject: [PATCH 060/190] rm'd errant print

---
 xonsh/tokenize.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/xonsh/tokenize.py b/xonsh/tokenize.py
index 774571101..8ab583c79 100644
--- a/xonsh/tokenize.py
+++ b/xonsh/tokenize.py
@@ -683,7 +683,6 @@ def _tokenize(readline, encoding):
                     if parenlev > 0:
                         yield TokenInfo(NL, token, spos, epos, line)
                     else:
-                        print(token)
                         yield TokenInfo(NEWLINE, token, spos, epos, line)
                         if async_def:
                             async_def_nl = True

From 6513af02b31090061ec17a911804f60de30aa09f Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Mon, 22 Aug 2016 09:53:11 +0800
Subject: [PATCH 061/190] do not use BASH_COMPLETE_SCRIPT_PRE

---
 xonsh/completers/bash.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/xonsh/completers/bash.py b/xonsh/completers/bash.py
index 8105b6285..998e2d93e 100644
--- a/xonsh/completers/bash.py
+++ b/xonsh/completers/bash.py
@@ -25,11 +25,11 @@ CACHED_HASH = None
 CACHED_FUNCS = None
 CACHED_FILES = None
 
-BASH_COMPLETE_SCRIPT_PRE = """BASH_COMPLETION_DIR="nonexist"
+BASH_COMPLETE_SCRIPT = """
+BASH_COMPLETION_DIR="nonexist"
 BASH_COMPLETION_COMPAT_DIR="nonexist"
-"""
-
-BASH_COMPLETE_SCRIPT = """source "{filename}"
+{completions_sources}
+source "{filename}"
 COMP_WORDS=({line})
 COMP_LINE={comp_line}
 COMP_POINT=${{#COMP_LINE}}
@@ -111,9 +111,8 @@ def complete_from_bash(prefix, line, begidx, endidx, ctx):
     script = BASH_COMPLETE_SCRIPT.format(
         filename=fnme, line=' '.join(shlex.quote(p) for p in splt),
         comp_line=shlex.quote(line), n=n, func=func, cmd=cmd,
-        end=endidx + 1, prefix=prefix, prev=shlex.quote(prev))
-    script = BASH_COMPLETE_SCRIPT_PRE + '\n' + \
-        '\n'.join(_collect_completions_sources()) + '\n' + script
+        end=endidx + 1, prefix=prefix, prev=shlex.quote(prev),
+        completions_sources='\n'.join(_collect_completions_sources()))
     try:
         out = subprocess.check_output(
             [xp.bash_command()], input=script, universal_newlines=True,

From 10093a9f862182c1618d4623b522d8875fbf1abd Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Mon, 22 Aug 2016 10:00:11 +0800
Subject: [PATCH 062/190] add changelog

---
 news/fix-bash-completions.rst | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 news/fix-bash-completions.rst

diff --git a/news/fix-bash-completions.rst b/news/fix-bash-completions.rst
new file mode 100644
index 000000000..52b276062
--- /dev/null
+++ b/news/fix-bash-completions.rst
@@ -0,0 +1,13 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:**
+
+* Source `bash_completion` at first on bash completion
+
+**Security:** None

From c374787f047d445c96f2b0a4919a88d5c49cac1c Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Tue, 23 Aug 2016 08:33:47 -0400
Subject: [PATCH 063/190] Conches for the xonsh god!

---
 docs/index.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/index.rst b/docs/index.rst
index c603cea6b..6c9ecac0a 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -49,6 +49,7 @@ the xonsh shell
         "The carcolh will catch you!",
         "People xonshtantly mispronounce these things",
         "WHAT...is your favorite shell?",
+        "Conches for the xonsh god!",
         "Exploiting the workers and hanging on to outdated imperialist dogma since 2015."
         ];
     document.write(taglines[Math.floor(Math.random() * taglines.length)]);

From eb6b1dcf11d49dbbde773948c347268c3270a0d3 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Tue, 23 Aug 2016 08:45:51 -0400
Subject: [PATCH 064/190] small typo fix

---
 docs/tutorial.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index 9a67916a0..3448032fd 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -268,8 +268,8 @@ that ``ls -l`` is meant to be run in subprocess-mode?
 
 For any given line that only contains an expression statement (expr-stmt,
 see the Python AST docs for more information), if all the names cannot
-be found as current variables name xonsh will try to parse the line as a
-subprocess command instead.  In the above, if ``ls`` ans ``l`` are not a
+be found as current variables xonsh will try to parse the line as a
+subprocess command instead.  In the above, if ``ls`` and ``l`` are not 
 variables, then subprocess mode will be attempted. If parsing in subprocess
 mode fails, then the line is left in Python-mode.
 

From 188fee34dd13dc44f4122c5067108d208b8c6b42 Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Tue, 23 Aug 2016 22:00:27 +0800
Subject: [PATCH 065/190] rewrite bash completer

---
 news/fix-bash-completions.rst |   8 +-
 xonsh/completer.py            |   5 --
 xonsh/completers/bash.py      | 160 +++++-----------------------------
 xonsh/environ.py              |  17 ++--
 xonsh/platform.py             |  17 ++--
 5 files changed, 39 insertions(+), 168 deletions(-)

diff --git a/news/fix-bash-completions.rst b/news/fix-bash-completions.rst
index 52b276062..8cc70c124 100644
--- a/news/fix-bash-completions.rst
+++ b/news/fix-bash-completions.rst
@@ -1,13 +1,13 @@
 **Added:** None
 
-**Changed:** None
+**Changed:**
+
+* New implementation of bash completer with better performance and compatibility.
 
 **Deprecated:** None
 
 **Removed:** None
 
-**Fixed:**
-
-* Source `bash_completion` at first on bash completion
+**Fixed:** None
 
 **Security:** None
diff --git a/xonsh/completer.py b/xonsh/completer.py
index 4c80c1e23..aac3559bf 100644
--- a/xonsh/completer.py
+++ b/xonsh/completer.py
@@ -3,14 +3,9 @@
 import builtins
 import collections.abc as abc
 
-import xonsh.completers.bash as compbash
-
 
 class Completer(object):
     """This provides a list of optional completions for the xonsh shell."""
-    def __init__(self):
-        compbash.update_bash_completion()
-
     def complete(self, prefix, line, begidx, endidx, ctx=None):
         """Complete the string, given a possible execution context.
 
diff --git a/xonsh/completers/bash.py b/xonsh/completers/bash.py
index 998e2d93e..da59662a6 100644
--- a/xonsh/completers/bash.py
+++ b/xonsh/completers/bash.py
@@ -1,101 +1,38 @@
-import os
-import re
 import shlex
-import pickle
-import hashlib
 import pathlib
 import builtins
 import subprocess
 
-import xonsh.lazyasd as xl
 import xonsh.platform as xp
 
 from xonsh.completers.path import _quote_paths
 
-RE_DASHF = xl.LazyObject(lambda: re.compile(r'-F\s+(\w+)'),
-                         globals(), 'RE_DASHF')
-
-INITED = False
-
-BASH_COMPLETE_HASH = None
-BASH_COMPLETE_FUNCS = {}
-BASH_COMPLETE_FILES = {}
-
-CACHED_HASH = None
-CACHED_FUNCS = None
-CACHED_FILES = None
-
-BASH_COMPLETE_SCRIPT = """
-BASH_COMPLETION_DIR="nonexist"
-BASH_COMPLETION_COMPAT_DIR="nonexist"
-{completions_sources}
-source "{filename}"
+BASH_COMPLETE_SCRIPT = r'''
+{sources}
+if (complete -p "{cmd}" 2> /dev/null || echo _minimal) | grep --quiet -e "_minimal"
+then
+    declare -f _completion_loader > /dev/null && _completion_loader "{cmd}"
+fi
+_func=$(complete -p {cmd} | grep -o -e '-F \w\+' | cut -d ' ' -f 2)
 COMP_WORDS=({line})
 COMP_LINE={comp_line}
 COMP_POINT=${{#COMP_LINE}}
 COMP_COUNT={end}
 COMP_CWORD={n}
-{func} {cmd} {prefix} {prev}
+$_func {cmd} {prefix} {prev}
 for ((i=0;i<${{#COMPREPLY[*]}};i++)) do echo ${{COMPREPLY[i]}}; done
-"""
-
-
-def update_bash_completion():
-    global BASH_COMPLETE_FUNCS, BASH_COMPLETE_FILES, BASH_COMPLETE_HASH
-    global CACHED_FUNCS, CACHED_FILES, CACHED_HASH, INITED
-
-    completers = builtins.__xonsh_env__.get('BASH_COMPLETIONS', ())
-    BASH_COMPLETE_HASH = hashlib.md5(repr(completers).encode()).hexdigest()
-
-    datadir = builtins.__xonsh_env__['XONSH_DATA_DIR']
-    cachefname = os.path.join(datadir, 'bash_completion_cache')
-
-    if not INITED:
-        if os.path.isfile(cachefname):
-            # load from cache
-            with open(cachefname, 'rb') as cache:
-                CACHED_HASH, CACHED_FUNCS, CACHED_FILES = pickle.load(cache)
-                BASH_COMPLETE_HASH = CACHED_HASH
-                BASH_COMPLETE_FUNCS = CACHED_FUNCS
-                BASH_COMPLETE_FILES = CACHED_FILES
-        else:
-            # create initial cache
-            _load_bash_complete_funcs()
-            _load_bash_complete_files()
-            CACHED_HASH = BASH_COMPLETE_HASH
-            CACHED_FUNCS = BASH_COMPLETE_FUNCS
-            CACHED_FILES = BASH_COMPLETE_FILES
-            with open(cachefname, 'wb') as cache:
-                val = (CACHED_HASH, CACHED_FUNCS, CACHED_FILES)
-                pickle.dump(val, cache)
-        INITED = True
-
-    invalid = ((not os.path.isfile(cachefname)) or
-               BASH_COMPLETE_HASH != CACHED_HASH or
-               _completions_time() > os.stat(cachefname).st_mtime)
-
-    if invalid:
-        # update the cache
-        _load_bash_complete_funcs()
-        _load_bash_complete_files()
-        CACHED_HASH = BASH_COMPLETE_HASH
-        CACHED_FUNCS = BASH_COMPLETE_FUNCS
-        CACHED_FILES = BASH_COMPLETE_FILES
-        with open(cachefname, 'wb') as cache:
-            val = (CACHED_HASH, BASH_COMPLETE_FUNCS, BASH_COMPLETE_FILES)
-            pickle.dump(val, cache)
-
+'''
 
 def complete_from_bash(prefix, line, begidx, endidx, ctx):
     """Completes based on results from BASH completion."""
-    update_bash_completion()
+    sources = _collect_completions_sources()
+    if not sources:
+        return set()
+
     splt = line.split()
     cmd = splt[0]
-    func = BASH_COMPLETE_FUNCS.get(cmd, None)
-    fnme = BASH_COMPLETE_FILES.get(cmd, None)
-    if func is None or fnme is None:
-        return set()
     idx = n = 0
+    prev = ''
     for n, tok in enumerate(splt):
         if tok == prefix:
             idx = line.find(prefix, idx)
@@ -109,10 +46,14 @@ def complete_from_bash(prefix, line, begidx, endidx, ctx):
         prefix = shlex.quote(prefix)
 
     script = BASH_COMPLETE_SCRIPT.format(
-        filename=fnme, line=' '.join(shlex.quote(p) for p in splt),
-        comp_line=shlex.quote(line), n=n, func=func, cmd=cmd,
+        sources='\n'.join(sources), line=' '.join(shlex.quote(p) for p in splt),
+        comp_line=shlex.quote(line), n=n, cmd=cmd,
         end=endidx + 1, prefix=prefix, prev=shlex.quote(prev),
-        completions_sources='\n'.join(_collect_completions_sources()))
+    )
+
+    with open('/tmp/tmp.log', 'w') as f:
+        f.write(script)
+
     try:
         out = subprocess.check_output(
             [xp.bash_command()], input=script, universal_newlines=True,
@@ -124,59 +65,6 @@ def complete_from_bash(prefix, line, begidx, endidx, ctx):
     return rtn
 
 
-def _load_bash_complete_funcs():
-    global BASH_COMPLETE_FUNCS
-    BASH_COMPLETE_FUNCS = bcf = {}
-    inp = _collect_completions_sources()
-    if not inp:
-        return
-    inp.append('complete -p\n')
-    out = _source_completions(inp)
-    for line in out.splitlines():
-        head, _, cmd = line.rpartition(' ')
-        if len(cmd) == 0 or cmd == 'cd':
-            continue
-        m = RE_DASHF.search(head)
-        if m is None:
-            continue
-        bcf[cmd] = m.group(1)
-
-
-def _load_bash_complete_files():
-    global BASH_COMPLETE_FILES
-    inp = _collect_completions_sources()
-    if not inp:
-        BASH_COMPLETE_FILES = {}
-        return
-    if BASH_COMPLETE_FUNCS:
-        inp.append('shopt -s extdebug')
-        bash_funcs = set(BASH_COMPLETE_FUNCS.values())
-        inp.append('declare -F ' + ' '.join([f for f in bash_funcs]))
-        inp.append('shopt -u extdebug\n')
-    out = _source_completions(inp)
-    func_files = {}
-    for line in out.splitlines():
-        parts = line.split()
-        if xp.ON_WINDOWS:
-            parts = [parts[0], ' '.join(parts[2:])]
-        func_files[parts[0]] = parts[-1]
-    BASH_COMPLETE_FILES = {
-        cmd: func_files[func]
-        for cmd, func in BASH_COMPLETE_FUNCS.items()
-        if func in func_files
-    }
-
-
-def _source_completions(source):
-    try:
-        return subprocess.check_output(
-            [xp.bash_command()], input='\n'.join(source),
-            universal_newlines=True, env=builtins.__xonsh_env__.detype(),
-            stderr=subprocess.DEVNULL)
-    except FileNotFoundError:
-        return ''
-
-
 def _collect_completions_sources():
     sources = []
     completers = builtins.__xonsh_env__.get('BASH_COMPLETIONS', ())
@@ -188,9 +76,3 @@ def _collect_completions_sources():
             for _file in (x for x in path.glob('*') if x.is_file()):
                 sources.append('source "{}"'.format(_file.as_posix()))
     return sources
-
-
-def _completions_time():
-    compfiles = builtins.__xonsh_env__.get('BASH_COMPLETIONS', ())
-    compfiles = [os.stat(x).st_mtime for x in compfiles if os.path.exists(x)]
-    return max(compfiles) if compfiles else 0
diff --git a/xonsh/environ.py b/xonsh/environ.py
index 73b280d5a..40adaeea4 100644
--- a/xonsh/environ.py
+++ b/xonsh/environ.py
@@ -355,19 +355,20 @@ def DEFAULT_DOCS():
         'shell.\n\nPressing the right arrow key inserts the currently '
         'displayed suggestion. Only usable with $SHELL_TYPE=prompt_toolkit.'),
     'BASH_COMPLETIONS': VarDocs(
-        'This is a list (or tuple) of strings that specifies where the BASH '
-        'completion files may be found. The default values are platform '
+        'This is a list (or tuple) of strings that specifies where the '
+        '`bash_completion` script may be found. For better performance, '
+        'base-completion v2.x is recommended since it lazy-loads individual '
+        'completion scripts. Paths or directories of individual completion '
+        'scripts (like `.../completes/ssh`) do not need to be included here. '
+        'The default values are platform '
         'dependent, but sane. To specify an alternate list, do so in the run '
         'control file.', default=(
             "Normally this is:\n\n"
-            "    ('/etc/bash_completion',\n"
-            "     '/usr/share/bash-completion/completions/git')\n\n"
+            "    ('/etc/bash_completion', )\n\n"
             "But, on Mac it is:\n\n"
-            "    ('/usr/local/etc/bash_completion',\n"
-            "     '/opt/local/etc/profile.d/bash_completion.sh')\n\n"
+            "    ('/usr/local/etc/bash_completion', )\n\n"
             "And on Arch Linux it is:\n\n"
-            "    ('/usr/share/bash-completion/bash_completion',\n"
-            "     '/usr/share/bash-completion/completions/git')\n\n"
+            "    ('/usr/share/bash-completion/bash_completion', )\n\n"
             "Other OS-specific defaults may be added in the future.")),
     'CASE_SENSITIVE_COMPLETIONS': VarDocs(
         'Sets whether completions should be case sensitive or case '
diff --git a/xonsh/platform.py b/xonsh/platform.py
index 0430d66c7..83111d484 100644
--- a/xonsh/platform.py
+++ b/xonsh/platform.py
@@ -271,22 +271,15 @@ def BASH_COMPLETIONS_DEFAULT():
     """
     if ON_LINUX or ON_CYGWIN:
         if linux_distro() == 'arch':
-            bcd = (
-                '/usr/share/bash-completion/bash_completion',
-                '/usr/share/bash-completion/completions')
+            bcd = ('/usr/share/bash-completion/bash_completion', )
         else:
-            bcd = ('/usr/share/bash-completion',
-                   '/usr/share/bash-completion/completions')
+            bcd = ('/usr/share/bash-completion', )
     elif ON_DARWIN:
-        bcd = ('/usr/local/etc/bash_completion',
-               '/opt/local/etc/profile.d/bash_completion.sh')
+        bcd = ('/usr/local/share/bash-completion/bash_completion', # v2.x
+               '/usr/local/etc/bash_completion')  # v1.x
     elif ON_WINDOWS and git_for_windows_path():
         bcd = (os.path.join(git_for_windows_path(),
-                            'usr\\share\\bash-completion'),
-               os.path.join(git_for_windows_path(),
-                            'usr\\share\\bash-completion\\completions'),
-               os.path.join(git_for_windows_path(),
-                            'mingw64\\share\\git\\completion\\git-completion.bash'))
+                            'usr\\share\\bash-completion'), )
     else:
         bcd = ()
     return bcd

From 4fda4da1ccc452e4acc1d154cfff570ce3283add Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Tue, 23 Aug 2016 22:05:38 +0800
Subject: [PATCH 066/190] remove debug code

---
 xonsh/completers/bash.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/xonsh/completers/bash.py b/xonsh/completers/bash.py
index da59662a6..3af5e4682 100644
--- a/xonsh/completers/bash.py
+++ b/xonsh/completers/bash.py
@@ -51,9 +51,6 @@ def complete_from_bash(prefix, line, begidx, endidx, ctx):
         end=endidx + 1, prefix=prefix, prev=shlex.quote(prev),
     )
 
-    with open('/tmp/tmp.log', 'w') as f:
-        f.write(script)
-
     try:
         out = subprocess.check_output(
             [xp.bash_command()], input=script, universal_newlines=True,

From e1a3be7a1cf1e9ba16a3a5d885b7a617ba1eeac4 Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Tue, 23 Aug 2016 22:12:32 +0800
Subject: [PATCH 067/190] flake8 fix

---
 xonsh/completers/bash.py | 1 +
 xonsh/platform.py        | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/xonsh/completers/bash.py b/xonsh/completers/bash.py
index 3af5e4682..6eb5eb468 100644
--- a/xonsh/completers/bash.py
+++ b/xonsh/completers/bash.py
@@ -23,6 +23,7 @@ $_func {cmd} {prefix} {prev}
 for ((i=0;i<${{#COMPREPLY[*]}};i++)) do echo ${{COMPREPLY[i]}}; done
 '''
 
+
 def complete_from_bash(prefix, line, begidx, endidx, ctx):
     """Completes based on results from BASH completion."""
     sources = _collect_completions_sources()
diff --git a/xonsh/platform.py b/xonsh/platform.py
index 83111d484..dc44390fe 100644
--- a/xonsh/platform.py
+++ b/xonsh/platform.py
@@ -275,7 +275,7 @@ def BASH_COMPLETIONS_DEFAULT():
         else:
             bcd = ('/usr/share/bash-completion', )
     elif ON_DARWIN:
-        bcd = ('/usr/local/share/bash-completion/bash_completion', # v2.x
+        bcd = ('/usr/local/share/bash-completion/bash_completion',  # v2.x
                '/usr/local/etc/bash_completion')  # v1.x
     elif ON_WINDOWS and git_for_windows_path():
         bcd = (os.path.join(git_for_windows_path(),

From a3989c271e1ef3c9c878c1c57048c6bfe9983755 Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Tue, 23 Aug 2016 22:42:17 +0800
Subject: [PATCH 068/190] do not complete env variables

---
 xonsh/completers/bash.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/xonsh/completers/bash.py b/xonsh/completers/bash.py
index 6eb5eb468..72b827929 100644
--- a/xonsh/completers/bash.py
+++ b/xonsh/completers/bash.py
@@ -30,6 +30,9 @@ def complete_from_bash(prefix, line, begidx, endidx, ctx):
     if not sources:
         return set()
 
+    if prefix.startswith('$'):  # do not complete env variables
+        return set()
+
     splt = line.split()
     cmd = splt[0]
     idx = n = 0

From f27c858391126db78f711aff1d50ffb1b2f6fb65 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Tue, 23 Aug 2016 12:44:27 -0400
Subject: [PATCH 069/190] disable bkts for sys.stdin and add envvar to
 configure

Add `$COMPLETION_BRACKETS` (default True) to determine if the python
completer should add opening parens and square brackets to its
completions.

Whether or not `$COMPLETION_BRACKETS` is True, `sys.stdin`, `sys.stdout`
and `sys.stderr` do not have brackets in their completions.
---
 xonsh/completers/python.py | 12 ++++++++----
 xonsh/environ.py           |  2 ++
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/xonsh/completers/python.py b/xonsh/completers/python.py
index ed6a5ec1a..0cd9fdf4f 100644
--- a/xonsh/completers/python.py
+++ b/xonsh/completers/python.py
@@ -95,10 +95,14 @@ def attr_complete(prefix, ctx, filter_func):
         except:  # pylint:disable=bare-except
             continue
         a = getattr(val, opt)
-        if callable(a):
-            rpl = opt + '('
-        elif isinstance(a, abc.Iterable):
-            rpl = opt + '['
+        if (builtins.__xonsh_env__['COMPLETIONS_BRACKETS'] and
+                        opt not in ['stdin', 'stdout', 'stderr']):
+            if callable(a):
+                rpl = opt + '('
+            elif isinstance(a, abc.Iterable):
+                rpl = opt + '['
+            else:
+                rpl = opt
         else:
             rpl = opt
         # note that prefix[:prelen-len(attr)] != prefix[:-len(attr)]
diff --git a/xonsh/environ.py b/xonsh/environ.py
index 73b280d5a..6cd232c2c 100644
--- a/xonsh/environ.py
+++ b/xonsh/environ.py
@@ -101,6 +101,7 @@ def DEFAULT_ENSURERS():
     re.compile('\w*DIRS$'): (is_env_path, str_to_env_path, env_path_to_str),
     'COLOR_INPUT': (is_bool, to_bool, bool_to_str),
     'COLOR_RESULTS': (is_bool, to_bool, bool_to_str),
+    'COMPLETIONS_BRACKETS': (is_bool, to_bool, bool_to_str),
     'COMPLETIONS_DISPLAY': (is_completions_display_value,
                             to_completions_display_value, str),
     'COMPLETIONS_MENU_ROWS': (is_int, int, str),
@@ -244,6 +245,7 @@ def DEFAULT_VALUES():
         'CDPATH': (),
         'COLOR_INPUT': True,
         'COLOR_RESULTS': True,
+        'COMPLETIONS_BRACKETS': True,
         'COMPLETIONS_DISPLAY': 'multi',
         'COMPLETIONS_MENU_ROWS': 5,
         'DIRSTACK_SIZE': 20,

From 2f96365257b9f8ec8e66767e4d2934a25cb66167 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Tue, 23 Aug 2016 12:48:27 -0400
Subject: [PATCH 070/190] add changelog

---
 news/completer_brackets.rst | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 news/completer_brackets.rst

diff --git a/news/completer_brackets.rst b/news/completer_brackets.rst
new file mode 100644
index 000000000..f6037cdd1
--- /dev/null
+++ b/news/completer_brackets.rst
@@ -0,0 +1,17 @@
+**Added:** None
+
+**Changed:** 
+
+* ``$COMPLETIONS_BRACKETS`` is now available to determine whether or not to 
+  include opening brackets in Python completions
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** 
+
+* ``sys.stdin``, ``sys.stdout``, ``sys.stderr`` no longer complete with 
+  opening square brackets
+
+**Security:** None

From 9c40c230e04f33d664531259ecd6b4bde9646310 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Tue, 23 Aug 2016 19:29:32 -0400
Subject: [PATCH 071/190] add VarDocs for `COMPLETIONS_BRACKETS`

---
 xonsh/environ.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/xonsh/environ.py b/xonsh/environ.py
index 6cd232c2c..f664069ad 100644
--- a/xonsh/environ.py
+++ b/xonsh/environ.py
@@ -379,6 +379,9 @@ def DEFAULT_DOCS():
         'with Bash, xonsh always prefer an existing relative path.'),
     'COLOR_INPUT': VarDocs('Flag for syntax highlighting interactive input.'),
     'COLOR_RESULTS': VarDocs('Flag for syntax highlighting return values.'),
+    'COMPLETIONS_BRACKETS': VarDocs(
+        'Flag to enable/disable inclusion of square brackets and parentheses '
+        'in Python attribute completions.', default='True'),
     'COMPLETIONS_DISPLAY': VarDocs(
         'Configure if and how Python completions are displayed by the '
         'prompt_toolkit shell.\n\nThis option does not affect Bash '

From a5c11fdeaf0bd1b0f6040122168b68b881df6b36 Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Wed, 24 Aug 2016 04:19:27 +0300
Subject: [PATCH 072/190] ignore foreign aliases name

---
 xonsh/foreign_shells.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/xonsh/foreign_shells.py b/xonsh/foreign_shells.py
index 83236212b..fa6690aa1 100644
--- a/xonsh/foreign_shells.py
+++ b/xonsh/foreign_shells.py
@@ -540,9 +540,20 @@ def load_foreign_aliases(shells=None, config=None, issue_warning=True):
     """
     shells = _get_shells(shells=shells, config=config, issue_warning=issue_warning)
     aliases = {}
+    xonsh_aliases = builtins.aliases
     for shell in shells:
         shell = ensure_shell(shell)
         _, shaliases = foreign_shell_data(**shell)
-        if shaliases:
-            aliases.update(shaliases)
+        if not shaliases:
+            continue
+        for alias in list(shaliases.keys()):
+            if alias not in xonsh_aliases:
+                continue
+            else:
+                del shaliases[alias]
+                print('aliases: error: alias {!r} of shell {!r} '
+                      'tries to override xonsh builtin alias, '
+                      'xonsh wins!'.format(alias, shell['shell']),
+                      file=sys.stderr)
+        aliases.update(shaliases)
     return aliases

From 783eef7eb75914182926580b7275b92080007877 Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Wed, 24 Aug 2016 04:21:52 +0300
Subject: [PATCH 073/190] news entry

---
 news/foreign-aliases.rst | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 news/foreign-aliases.rst

diff --git a/news/foreign-aliases.rst b/news/foreign-aliases.rst
new file mode 100644
index 000000000..bffdc8d09
--- /dev/null
+++ b/news/foreign-aliases.rst
@@ -0,0 +1,13 @@
+**Added:** None
+
+**Changed:**
+
+* Foreign aliases that match xonsh builtin aliases are now ignored with a warning.
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

From 05cc6c39e85b85662e41f87b62074ebbd38af527 Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Wed, 24 Aug 2016 04:30:29 +0300
Subject: [PATCH 074/190] import sys

---
 xonsh/foreign_shells.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/xonsh/foreign_shells.py b/xonsh/foreign_shells.py
index fa6690aa1..6cd71d7c5 100644
--- a/xonsh/foreign_shells.py
+++ b/xonsh/foreign_shells.py
@@ -4,6 +4,7 @@ import os
 import re
 import json
 import shlex
+import sys
 import tempfile
 import builtins
 import subprocess
@@ -551,7 +552,7 @@ def load_foreign_aliases(shells=None, config=None, issue_warning=True):
                 continue
             else:
                 del shaliases[alias]
-                print('aliases: error: alias {!r} of shell {!r} '
+                print('aliases: alias {!r} of shell {!r} '
                       'tries to override xonsh builtin alias, '
                       'xonsh wins!'.format(alias, shell['shell']),
                       file=sys.stderr)

From f59619c90b8ec0b49e6e6cbefd86a0d3de7da6e7 Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Wed, 24 Aug 2016 04:46:40 +0300
Subject: [PATCH 075/190] typo

---
 xonsh/foreign_shells.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xonsh/foreign_shells.py b/xonsh/foreign_shells.py
index 6cd71d7c5..396a87386 100644
--- a/xonsh/foreign_shells.py
+++ b/xonsh/foreign_shells.py
@@ -553,7 +553,7 @@ def load_foreign_aliases(shells=None, config=None, issue_warning=True):
             else:
                 del shaliases[alias]
                 print('aliases: alias {!r} of shell {!r} '
-                      'tries to override xonsh builtin alias, '
+                      'tries to override xonsh alias, '
                       'xonsh wins!'.format(alias, shell['shell']),
                       file=sys.stderr)
         aliases.update(shaliases)

From 8a95037d384205a1b03bfed0ccbaa7c32048e8e4 Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Wed, 24 Aug 2016 04:52:58 +0300
Subject: [PATCH 076/190] intersection

---
 xonsh/foreign_shells.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/xonsh/foreign_shells.py b/xonsh/foreign_shells.py
index 396a87386..02801143e 100644
--- a/xonsh/foreign_shells.py
+++ b/xonsh/foreign_shells.py
@@ -547,14 +547,11 @@ def load_foreign_aliases(shells=None, config=None, issue_warning=True):
         _, shaliases = foreign_shell_data(**shell)
         if not shaliases:
             continue
-        for alias in list(shaliases.keys()):
-            if alias not in xonsh_aliases:
-                continue
-            else:
-                del shaliases[alias]
-                print('aliases: alias {!r} of shell {!r} '
-                      'tries to override xonsh alias, '
-                      'xonsh wins!'.format(alias, shell['shell']),
-                      file=sys.stderr)
+        for alias in set(shaliases) & set(xonsh_aliases):
+            del shaliases[alias]
+            print('aliases: alias {!r} of shell {!r} '
+                  'tries to override xonsh alias, '
+                  'xonsh wins!'.format(alias, shell['shell']),
+                  file=sys.stderr)
         aliases.update(shaliases)
     return aliases

From c15277f8ff6de2923cc3cfbe9f324f996bb44f50 Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Wed, 24 Aug 2016 10:19:38 +0800
Subject: [PATCH 077/190] new option: COMPLETION_CONFIRM

---
 xonsh/environ.py          |  5 +++++
 xonsh/ptk/key_bindings.py | 19 +++++++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/xonsh/environ.py b/xonsh/environ.py
index 73b280d5a..c88218372 100644
--- a/xonsh/environ.py
+++ b/xonsh/environ.py
@@ -244,6 +244,7 @@ def DEFAULT_VALUES():
         'CDPATH': (),
         'COLOR_INPUT': True,
         'COLOR_RESULTS': True,
+        'COMPLETION_CONFIRM': False,
         'COMPLETIONS_DISPLAY': 'multi',
         'COMPLETIONS_MENU_ROWS': 5,
         'DIRSTACK_SIZE': 20,
@@ -393,6 +394,10 @@ def DEFAULT_DOCS():
         "writing \"$COMPLETIONS_DISPLAY = None\" and \"$COMPLETIONS_DISPLAY "
         "= 'none'\" are equivalent. Only usable with "
         "$SHELL_TYPE=prompt_toolkit"),
+    'COMPLETION_CONFIRM': VarDocs(
+        'While tab-completions menu is displayed, press <Enter> to confirm '
+        'completion instead of running command. This only affects the '
+        'prompt-toolkit shell.'),
     'COMPLETIONS_MENU_ROWS': VarDocs(
         'Number of rows to reserve for tab-completions menu if '
         "$COMPLETIONS_DISPLAY is 'single' or 'multi'. This only affects the "
diff --git a/xonsh/ptk/key_bindings.py b/xonsh/ptk/key_bindings.py
index 4e545e402..dad0302f2 100644
--- a/xonsh/ptk/key_bindings.py
+++ b/xonsh/ptk/key_bindings.py
@@ -107,6 +107,15 @@ class EndOfLine(Filter):
         return bool(at_end and not last_line)
 
 
+class ShouldConfirmCompletion(Filter):
+    """
+    Check if completion needs confirmation
+    """
+    def __call__(self, cli):
+        return (builtins.__xonsh_env__.get('COMPLETION_CONFIRM', True)
+                and bool(cli.current_buffer.complete_state))
+
+
 # Copied from prompt-toolkit's key_binding/bindings/basic.py
 @Condition
 def ctrl_d_condition(cli):
@@ -173,6 +182,16 @@ def load_xonsh_bindings(key_bindings_manager):
         b = event.cli.current_buffer
         carriage_return(b, event.cli)
 
+    @handle(Keys.ControlJ, filter=ShouldConfirmCompletion())
+    def enter_confirm_completion(event):
+        """Ignore <enter> (confirm completion)"""
+        event.current_buffer.complete_state = None
+
+    @handle(Keys.Escape, filter=ShouldConfirmCompletion())
+    def esc_cancel_completion(event):
+        """Use <ESC> to cancel completion"""
+        event.cli.current_buffer.cancel_completion()
+
     @handle(Keys.Left, filter=BeginningOfLine())
     def wrap_cursor_back(event):
         """Move cursor to end of previous line unless at beginning of document"""

From 05d8e4ffc39ab9d1ad3a0e61f62a17aee1bbffc8 Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Wed, 24 Aug 2016 10:24:36 +0800
Subject: [PATCH 078/190] COMPLETION_CONFIRM defaults to False

---
 xonsh/ptk/key_bindings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xonsh/ptk/key_bindings.py b/xonsh/ptk/key_bindings.py
index dad0302f2..b3c9c7a9b 100644
--- a/xonsh/ptk/key_bindings.py
+++ b/xonsh/ptk/key_bindings.py
@@ -112,7 +112,7 @@ class ShouldConfirmCompletion(Filter):
     Check if completion needs confirmation
     """
     def __call__(self, cli):
-        return (builtins.__xonsh_env__.get('COMPLETION_CONFIRM', True)
+        return (builtins.__xonsh_env__.get('COMPLETION_CONFIRM', False)
                 and bool(cli.current_buffer.complete_state))
 
 

From e5c913ac428a21c83ad9b0eba86acc7a4a7f149a Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Wed, 24 Aug 2016 10:27:20 +0800
Subject: [PATCH 079/190] add changelog

---
 news/completion-confirm.rst | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 news/completion-confirm.rst

diff --git a/news/completion-confirm.rst b/news/completion-confirm.rst
new file mode 100644
index 000000000..14f6d4fa3
--- /dev/null
+++ b/news/completion-confirm.rst
@@ -0,0 +1,14 @@
+**Added:**
+
+* New option ``COMPLETION_CONFIRM``. When set, ``<Enter>`` is used to confirm
+  completion instead of running command while completion menu is displayed.
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

From 1c1122b7423309ccf82d887dc31c5bffb6a7b239 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Tue, 23 Aug 2016 23:44:05 -0400
Subject: [PATCH 080/190] Refactor everything.

(Maybe I should have broken this up better?)
---
 tests/test_events.py |  79 +++++++++--------------
 xonsh/built_ins.py   |   4 +-
 xonsh/events.py      | 146 ++++++++++++++++++++++++++++---------------
 3 files changed, 126 insertions(+), 103 deletions(-)

diff --git a/tests/test_events.py b/tests/test_events.py
index 0b388fe53..aa813f83d 100644
--- a/tests/test_events.py
+++ b/tests/test_events.py
@@ -1,71 +1,47 @@
 """Event tests"""
-from xonsh.events import Events
+import pytest
+from xonsh.events import EventManager
 
-def test_calling():
-    e = Events()
-    e.on_test.doc("Test event")
+@pytest.fixture
+def events():
+    return EventManager()
 
+def test_event_calling(events):
     called = False
-    @e.on_test
+
+    @events.on_test
     def _(spam):
         nonlocal called
         called = spam
 
-    e.on_test.fire("eggs")
+    events.on_test.fire("eggs")
 
     assert called == "eggs"
 
-def test_until_true():
-    e = Events()
-    e.on_test.doc("Test event")
-
+def test_event_returns(events):
     called = 0
 
-    @e.test
+    @events.on_test
     def on_test():
         nonlocal called
         called += 1
-        return True
+        return 1
 
-    @e.on_test
+    @events.on_test
     def second():
         nonlocal called
         called += 1
-        return True
+        return 2
 
-    e.on_test.until_true()
+    vals = events.on_test.fire()
 
-    assert called == 1
+    assert called == 2
+    assert set(vals) == {1, 2}
 
-def test_until_false():
-    e = Events()
-    e.on_test.doc("Test event")
+def test_validator(events):
+    called = None
 
-    called = 0
-
-    @e.on_test
-    def first():
-        nonlocal called
-        called += 1
-        return False
-
-    @e.on_test
-    def second():
-        nonlocal called
-        called += 1
-        return False
-
-    e.on_test.until_false()
-
-    assert called == 1
-
-def test_validator():
-    e = Events()
-    e.on_test.doc("Test event")
-
-    called = 0
-
-    @e.on_test
+    @events.on_test
     def first(n):
         nonlocal called
         called += 1
@@ -75,23 +51,24 @@ def test_validator():
     def v(n):
         return n == 'spam'
 
-    @e.on_test
+    @events.on_test
     def second(n):
         nonlocal called
         called += 1
         return False
 
-    e.on_test.fire('egg')
+    called = 0
+    events.on_test.fire('egg')
     assert called == 1
 
     called = 0
-    e.on_test.fire('spam')
+    events.on_test.fire('spam')
     assert called == 2
 
-def test_eventdoc():
+
+def test_eventdoc(events):
     docstring = "Test event"
-    e = Events()
-    e.on_test.doc(docstring)
+    events.doc('on_test', docstring)
 
     import inspect
-    assert inspect.getdoc(e.on_test) == docstring
\ No newline at end of file
+    assert inspect.getdoc(events.on_test) == docstring
diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py
index 64393681e..6aa07e44c 100644
--- a/xonsh/built_ins.py
+++ b/xonsh/built_ins.py
@@ -35,7 +35,7 @@ from xonsh.tools import (
     XonshCalledProcessError, XonshBlockError
 )
 from xonsh.commands_cache import CommandsCache
-from xonsh.events import Events
+from xonsh.events import events
 
 import xonsh.completers.init
 
@@ -715,7 +715,7 @@ def load_builtins(execer=None, config=None, login=False, ctx=None):
     builtins.__xonsh_ensure_list_of_strs__ = ensure_list_of_strs
     builtins.__xonsh_list_of_strs_or_callables__ = list_of_strs_or_callables
     builtins.__xonsh_completers__ = xonsh.completers.init.default_completers()
-    builtins.__xonsh_events__ = Events()
+    builtins.__xonsh_events__ = events
     # public built-ins
     builtins.XonshError = XonshError
     builtins.XonshBlockError = XonshBlockError
diff --git a/xonsh/events.py b/xonsh/events.py
index 15a435cb7..2c2f9970e 100644
--- a/xonsh/events.py
+++ b/xonsh/events.py
@@ -7,22 +7,14 @@ The best way to "declare" an event is something like::
 
     __xonsh_events__.on_spam.doc("Comes with eggs")
 """
+import abc
+import collections.abc
+import traceback
+import sys
 
 
-class Event(set):
-    """
-    A given event that handlers can register against.
-
-    Acts as a ``set`` for registered handlers.
-
-    Note that ordering is never guaranteed.
-    """
-
-    @classmethod
-    def doc(cls, text):
-        cls.__doc__ = text
-
-    def __call__(self, func):
+class AbstractEvent(collections.abc.MutableSet, abc.ABC):
+    def __call__(self, handler):
         """
         Registers a handler. It's suggested to use this as a decorator.
 
@@ -33,60 +25,86 @@ class Event(set):
         at all.
         """
         #  Using Pythons "private" munging to minimize hypothetical collisions
-        func.__validator = None
-        self.add(func)
+        handler.__validator = None
+        self.add(handler)
 
         def validator(vfunc):
             """
             Adds a validator function to a handler to limit when it is considered.
             """
-            func.__validator = vfunc
-        func.validator = validator
+            handler.__validator = vfunc
+        handler.validator = validator
 
-        return func
+        return handler
 
-    def calleach(self, *pargs, **kwargs):
+    def _filterhandlers(self, *pargs, **kwargs):
         """
-        The core handler caller that all others build on.
-
-        This works as a generator. Each handler is called in turn and its
-        results are yielded.
-
-        If the generator is interupted, no further handlers are called.
+        Helper method for implementing classes. Generates the handlers that pass validation.
         """
         for handler in self:
             if handler.__validator is not None and not handler.__validator(*pargs, **kwargs):
                 continue
-            yield handler(*pargs, **kwargs)
+            yield handler
+
+    @abc.abstractmethod
+    def fire(self, *pargs, **kwargs):
+        """
+        Fires an event, calling registered handlers with the given arguments.
+        """
+
+
+class Event(AbstractEvent):
+    """
+    A given event that handlers can register against.
+
+    Acts as a ``set`` for registered handlers.
+
+    Note that ordering is never guaranteed.
+    """
+    # Wish I could just pull from set...
+    def __init__(self):
+        self._handlers = set()
+
+    def __len__(self):
+        return len(self._handlers)
+
+    def __contains__(self, item):
+        return item in self._handlers
+
+    def __iter__(self):
+        yield from self._handlers
+
+    def add(self, item):
+        return self._handlers.add(item)
+
+    def discard(self, item):
+        return self._handlers.discard(item)
 
     def fire(self, *pargs, **kwargs):
         """
-        The simplest use case: Calls each handler in turn with the provided
-        arguments and ignore the return values.
+        Fires each event, returning a non-unique iterable of the results.
         """
-        for _ in self.calleach(*pargs, **kwargs):
-            pass
-
-    def until_true(self, *pargs, **kwargs):
-        """
-        Calls each handler until one returns something truthy.
-
-        Returns that truthy value.
-        """
-        for rv in self.calleach(*pargs, **kwargs):
-            if rv:
-                return rv
-
-    def until_false(self, *pargs, **kwargs):
-        """
-        Calls each handler until one returns something falsey.
-        """
-        for rv in self.calleach(*pargs, **kwargs):
-            if not rv:
-                return rv
+        vals = []
+        for handler in self._filterhandlers(*pargs, **kwargs):
+            try:
+                rv = handler(*pargs, **kwargs)
+            except Exception:
+                print("Exception raised in event handler; ignored.", file=sys.stderr)
+                traceback.print_exc()
+            else:
+                vals.append(rv)
+        return vals
 
 
-class Events:
+class LoadEvent(Event):
+    """
+    A kind of event in which each handler is called exactly once.
+    """
+    def __call__(self, *pargs, **kwargs):
+        raise NotImplementedError("See #1550")
+
+
+class EventManager:
     """
     Container for all events in a system.
 
@@ -95,7 +113,35 @@ class Events:
     Each event is just an attribute. They're created dynamically on first use.
     """
 
+    def doc(self, name, docstring):
+        """
+        Applies a docstring to an event.
+        """
+        type(getattr(self, name)).__doc__ = docstring
+
+    def transmogrify(self, name, klass):
+        """
+        Converts an event from one species to another.
+
+        Please note: Some species may do special things with handlers. This is lost.
+        """
+        if isinstance(klass, str):
+            klass = globals()[klass]
+
+        if not issubclass(klass, AbstractEvent):
+            raise ValueError("Invalid event class; must be a subclass of AbstractEvent")
+
+        oldevent = getattr(self, name)
+        newevent = type(name, (klass,), {'__doc__': type(oldevent).__doc__})()
+        setattr(self, name, newevent)
+
+        for handler in oldevent:
+            newevent.add(handler)
+
     def __getattr__(self, name):
         e = type(name, (Event,), {'__doc__': None})()
         setattr(self, name, e)
         return e
+
+
+events = EventManager()

From 2d2f8bdf4b408b7eb803dc3c8343b59a8d431b2e Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Wed, 24 Aug 2016 11:44:47 +0800
Subject: [PATCH 081/190] clean up code

---
 xonsh/ptk/key_bindings.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/xonsh/ptk/key_bindings.py b/xonsh/ptk/key_bindings.py
index b3c9c7a9b..19969b847 100644
--- a/xonsh/ptk/key_bindings.py
+++ b/xonsh/ptk/key_bindings.py
@@ -112,8 +112,8 @@ class ShouldConfirmCompletion(Filter):
     Check if completion needs confirmation
     """
     def __call__(self, cli):
-        return (builtins.__xonsh_env__.get('COMPLETION_CONFIRM', False)
-                and bool(cli.current_buffer.complete_state))
+        return (builtins.__xonsh_env__.get('COMPLETION_CONFIRM')
+                and cli.current_buffer.complete_state)
 
 
 # Copied from prompt-toolkit's key_binding/bindings/basic.py

From 4ba21c9b1607399172393941b6528d11e7f4bcce Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Tue, 23 Aug 2016 23:54:29 -0400
Subject: [PATCH 082/190] Actually test event transmogrification

---
 tests/test_events.py | 23 +++++++++++++++++++++--
 xonsh/events.py      | 22 ++++++++++++++++++++--
 2 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/tests/test_events.py b/tests/test_events.py
index aa813f83d..ba9090df2 100644
--- a/tests/test_events.py
+++ b/tests/test_events.py
@@ -1,6 +1,7 @@
 """Event tests"""
+import inspect
 import pytest
-from xonsh.events import EventManager
+from xonsh.events import EventManager, Event, LoadEvent
 
 @pytest.fixture
 def events():
@@ -70,5 +71,23 @@ def test_eventdoc(events):
     docstring = "Test event"
     events.doc('on_test', docstring)
 
-    import inspect
+    assert inspect.getdoc(events.on_test) == docstring
+
+
+def test_transmogrify(events):
+    docstring = "Test event"
+    events.doc('on_test', docstring)
+
+    @events.on_test
+    def func():
+        pass
+
+    assert isinstance(events.on_test, Event)
+    assert len(events.on_test) == 1
+    assert inspect.getdoc(events.on_test) == docstring
+
+    events.transmogrify('on_test', LoadEvent)
+
+    assert isinstance(events.on_test, LoadEvent)
+    assert len(events.on_test) == 1
     assert inspect.getdoc(events.on_test) == docstring
diff --git a/xonsh/events.py b/xonsh/events.py
index 2c2f9970e..08987712f 100644
--- a/xonsh/events.py
+++ b/xonsh/events.py
@@ -96,11 +96,29 @@ class Event(AbstractEvent):
         return vals
 
 
-class LoadEvent(Event):
+class LoadEvent(AbstractEvent):
     """
     A kind of event in which each handler is called exactly once.
     """
-    def __call__(self, *pargs, **kwargs):
+    def __init__(self):
+        self._handlers = set()
+
+    def __len__(self):
+        return len(self._handlers)
+
+    def __contains__(self, item):
+        return item in self._handlers
+
+    def __iter__(self):
+        yield from self._handlers
+
+    def add(self, item):
+        return self._handlers.add(item)
+
+    def discard(self, item):
+        return self._handlers.discard(item)
+
+    def fire(self, *pargs, **kwargs):
         raise NotImplementedError("See #1550")
 
 

From 4be3bf89502481b863f3b3b7d289c874aaba11a5 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Tue, 23 Aug 2016 23:58:22 -0400
Subject: [PATCH 083/190] Test for naming classes by string, too

---
 tests/test_events.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/tests/test_events.py b/tests/test_events.py
index ba9090df2..984d4db83 100644
--- a/tests/test_events.py
+++ b/tests/test_events.py
@@ -91,3 +91,21 @@ def test_transmogrify(events):
     assert isinstance(events.on_test, LoadEvent)
     assert len(events.on_test) == 1
     assert inspect.getdoc(events.on_test) == docstring
+
+def test_transmogrify_by_string(events):
+    docstring = "Test event"
+    events.doc('on_test', docstring)
+
+    @events.on_test
+    def func():
+        pass
+
+    assert isinstance(events.on_test, Event)
+    assert len(events.on_test) == 1
+    assert inspect.getdoc(events.on_test) == docstring
+
+    events.transmogrify('on_test', 'LoadEvent')
+
+    assert isinstance(events.on_test, LoadEvent)
+    assert len(events.on_test) == 1
+    assert inspect.getdoc(events.on_test) == docstring

From 79353def69b2c54d97d71c5431ab55fd01e1a419 Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Wed, 24 Aug 2016 11:47:12 +0800
Subject: [PATCH 084/190] pep8 fix

---
 xonsh/completers/bash.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/xonsh/completers/bash.py b/xonsh/completers/bash.py
index 72b827929..09f5e2e54 100644
--- a/xonsh/completers/bash.py
+++ b/xonsh/completers/bash.py
@@ -7,7 +7,7 @@ import xonsh.platform as xp
 
 from xonsh.completers.path import _quote_paths
 
-BASH_COMPLETE_SCRIPT = r'''
+BASH_COMPLETE_SCRIPT = r"""
 {sources}
 if (complete -p "{cmd}" 2> /dev/null || echo _minimal) | grep --quiet -e "_minimal"
 then
@@ -21,7 +21,7 @@ COMP_COUNT={end}
 COMP_CWORD={n}
 $_func {cmd} {prefix} {prev}
 for ((i=0;i<${{#COMPREPLY[*]}};i++)) do echo ${{COMPREPLY[i]}}; done
-'''
+"""
 
 
 def complete_from_bash(prefix, line, begidx, endidx, ctx):

From 4f03c4d22131667f711ae1b729a9c1d172919b2f Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Wed, 24 Aug 2016 00:04:33 -0400
Subject: [PATCH 085/190] Implement what I figure the data model should be for
 LoadEvent()

---
 xonsh/events.py | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/xonsh/events.py b/xonsh/events.py
index 08987712f..3e79dbd85 100644
--- a/xonsh/events.py
+++ b/xonsh/events.py
@@ -37,11 +37,11 @@ class AbstractEvent(collections.abc.MutableSet, abc.ABC):
 
         return handler
 
-    def _filterhandlers(self, *pargs, **kwargs):
+    def _filterhandlers(self, handlers, *pargs, **kwargs):
         """
         Helper method for implementing classes. Generates the handlers that pass validation.
         """
-        for handler in self:
+        for handler in handlers:
             if handler.__validator is not None and not handler.__validator(*pargs, **kwargs):
                 continue
             yield handler
@@ -75,17 +75,17 @@ class Event(AbstractEvent):
         yield from self._handlers
 
     def add(self, item):
-        return self._handlers.add(item)
+        self._handlers.add(item)
 
     def discard(self, item):
-        return self._handlers.discard(item)
+        self._handlers.discard(item)
 
     def fire(self, *pargs, **kwargs):
         """
         Fires each event, returning a non-unique iterable of the results.
         """
         vals = []
-        for handler in self._filterhandlers(*pargs, **kwargs):
+        for handler in self._filterhandlers(self._handlers, *pargs, **kwargs):
             try:
                 rv = handler(*pargs, **kwargs)
             except Exception:
@@ -101,22 +101,26 @@ class LoadEvent(AbstractEvent):
     A kind of event in which each handler is called exactly once.
     """
     def __init__(self):
-        self._handlers = set()
+        self._fired = set()
+        self._unfired = set()
+        self._hasfired = False
 
     def __len__(self):
-        return len(self._handlers)
+        return len(self._fired) + len(self._unfired)
 
     def __contains__(self, item):
-        return item in self._handlers
+        return item in self._fired or item in self._unfired
 
     def __iter__(self):
-        yield from self._handlers
+        yield from self._fired
+        yield from self._unfired
 
     def add(self, item):
-        return self._handlers.add(item)
+        self._fired.add(item)
 
     def discard(self, item):
-        return self._handlers.discard(item)
+        self._fired.discard(item)
+        self._unfired.discard(item)
 
     def fire(self, *pargs, **kwargs):
         raise NotImplementedError("See #1550")

From 78783c71b5d1949a2967f65c3a65d49fed139a34 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Wed, 24 Aug 2016 01:08:02 -0400
Subject: [PATCH 086/190] Change aliased `collections.abc` to `cabc` to avoid
 conflicts with the `abc` module.

---
 xonsh/aliases.py           |  8 ++++----
 xonsh/built_ins.py         |  6 +++---
 xonsh/commands_cache.py    |  4 ++--
 xonsh/completer.py         |  4 ++--
 xonsh/completers/python.py |  4 ++--
 xonsh/environ.py           | 12 ++++++------
 xonsh/execer.py            |  4 ++--
 xonsh/foreign_shells.py    |  8 ++++----
 xonsh/history.py           |  4 ++--
 xonsh/lazyasd.py           |  4 ++--
 xonsh/lazyjson.py          | 18 +++++++++---------
 xonsh/proc.py              |  4 ++--
 xonsh/replay.py            |  4 ++--
 xonsh/tools.py             | 14 +++++++-------
 xonsh/xoreutils/_which.py  |  4 ++--
 15 files changed, 51 insertions(+), 51 deletions(-)

diff --git a/xonsh/aliases.py b/xonsh/aliases.py
index d185404f9..6403ee1c4 100644
--- a/xonsh/aliases.py
+++ b/xonsh/aliases.py
@@ -6,7 +6,7 @@ import shlex
 import inspect
 import argparse
 import builtins
-import collections.abc as abc
+import collections.abc as cabc
 
 from xonsh.lazyasd import lazyobject
 from xonsh.dirstack import cd, pushd, popd, dirs, _get_cwd
@@ -27,7 +27,7 @@ from xonsh.xoreutils import _which
 import xonsh.completers._aliases as xca
 
 
-class Aliases(abc.MutableMapping):
+class Aliases(cabc.MutableMapping):
     """Represents a location to hold and look up aliases."""
 
     def __init__(self, *args, **kwargs):
@@ -45,7 +45,7 @@ class Aliases(abc.MutableMapping):
         val = self._raw.get(key)
         if val is None:
             return default
-        elif isinstance(val, abc.Iterable) or callable(val):
+        elif isinstance(val, cabc.Iterable) or callable(val):
             return self.eval_alias(val, seen_tokens={key})
         else:
             msg = 'alias of {!r} has an inappropriate type: {!r}'
@@ -94,7 +94,7 @@ class Aliases(abc.MutableMapping):
         """
         word = line.split(' ', 1)[0]
         if word in builtins.aliases and isinstance(self.get(word),
-                                                   abc.Sequence):
+                                                   cabc.Sequence):
             word_idx = line.find(word)
             expansion = ' '.join(self.get(word))
             line = line[:word_idx] + expansion + line[word_idx+len(word):]
diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py
index 6aa07e44c..082554250 100644
--- a/xonsh/built_ins.py
+++ b/xonsh/built_ins.py
@@ -16,7 +16,7 @@ import tempfile
 import builtins
 import subprocess
 import contextlib
-import collections.abc as abc
+import collections.abc as cabc
 
 from xonsh.lazyasd import LazyObject, lazyobject
 from xonsh.history import History
@@ -662,7 +662,7 @@ def ensure_list_of_strs(x):
     """Ensures that x is a list of strings."""
     if isinstance(x, str):
         rtn = [x]
-    elif isinstance(x, abc.Sequence):
+    elif isinstance(x, cabc.Sequence):
         rtn = [i if isinstance(i, str) else str(i) for i in x]
     else:
         rtn = [str(x)]
@@ -673,7 +673,7 @@ def list_of_strs_or_callables(x):
     """Ensures that x is a list of strings or functions"""
     if isinstance(x, str) or callable(x):
         rtn = [x]
-    elif isinstance(x, abc.Sequence):
+    elif isinstance(x, cabc.Sequence):
         rtn = [i if isinstance(i, str) or callable(i) else str(i) for i in x]
     else:
         rtn = [str(x)]
diff --git a/xonsh/commands_cache.py b/xonsh/commands_cache.py
index 2c954980e..fcd01b6a7 100644
--- a/xonsh/commands_cache.py
+++ b/xonsh/commands_cache.py
@@ -1,14 +1,14 @@
 # -*- coding: utf-8 -*-
 import os
 import builtins
-import collections.abc as abc
+import collections.abc as cabc
 
 from xonsh.dirstack import _get_cwd
 from xonsh.platform import ON_WINDOWS
 from xonsh.tools import executables_in
 
 
-class CommandsCache(abc.Mapping):
+class CommandsCache(cabc.Mapping):
     """A lazy cache representing the commands available on the file system.
     The keys are the command names and the values a tuple of (loc, has_alias)
     where loc is either a str pointing to the executable on the file system or
diff --git a/xonsh/completer.py b/xonsh/completer.py
index 4c80c1e23..4b465f2da 100644
--- a/xonsh/completer.py
+++ b/xonsh/completer.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """A (tab-)completer for xonsh."""
 import builtins
-import collections.abc as abc
+import collections.abc as cabc
 
 import xonsh.completers.bash as compbash
 
@@ -41,7 +41,7 @@ class Completer(object):
                 out = func(prefix, line, begidx, endidx, ctx)
             except StopIteration:
                 return set(), len(prefix)
-            if isinstance(out, abc.Sequence):
+            if isinstance(out, cabc.Sequence):
                 res, lprefix = out
             else:
                 res = out
diff --git a/xonsh/completers/python.py b/xonsh/completers/python.py
index ed6a5ec1a..52c061b5a 100644
--- a/xonsh/completers/python.py
+++ b/xonsh/completers/python.py
@@ -4,7 +4,7 @@ import sys
 import inspect
 import builtins
 import importlib
-import collections.abc as abc
+import collections.abc as cabc
 
 import xonsh.tools as xt
 import xonsh.lazyasd as xl
@@ -97,7 +97,7 @@ def attr_complete(prefix, ctx, filter_func):
         a = getattr(val, opt)
         if callable(a):
             rpl = opt + '('
-        elif isinstance(a, abc.Iterable):
+        elif isinstance(a, cabc.Iterable):
             rpl = opt + '['
         else:
             rpl = opt
diff --git a/xonsh/environ.py b/xonsh/environ.py
index 73b280d5a..e226d21f3 100644
--- a/xonsh/environ.py
+++ b/xonsh/environ.py
@@ -17,7 +17,7 @@ import itertools
 import contextlib
 import subprocess
 import collections
-import collections.abc as abc
+import collections.abc as cabc
 
 from xonsh import __version__ as XONSH_VERSION
 from xonsh.jobs import get_next_task
@@ -654,7 +654,7 @@ def DEFAULT_DOCS():
 # actual environment
 #
 
-class Env(abc.MutableMapping):
+class Env(cabc.MutableMapping):
     """A xonsh environment, whose variables have limited typing
     (unlike BASH). Most variables are, by default, strings (like BASH).
     However, the following rules also apply based on variable-name:
@@ -695,7 +695,7 @@ class Env(abc.MutableMapping):
 
     @staticmethod
     def detypeable(val):
-        return not (callable(val) or isinstance(val, abc.MutableMapping))
+        return not (callable(val) or isinstance(val, cabc.MutableMapping))
 
     def detype(self):
         if self._detyped is not None:
@@ -811,8 +811,8 @@ class Env(abc.MutableMapping):
         else:
             e = "Unknown environment variable: ${}"
             raise KeyError(e.format(key))
-        if isinstance(val, (abc.MutableSet, abc.MutableSequence,
-                            abc.MutableMapping)):
+        if isinstance(val, (cabc.MutableSet, cabc.MutableSequence,
+                            cabc.MutableMapping)):
             self._detyped = None
         return val
 
@@ -1416,7 +1416,7 @@ def load_static_config(ctx, config=None):
         with open(config, 'r', encoding=encoding, errors=errors) as f:
             try:
                 conf = json.load(f)
-                assert isinstance(conf, abc.Mapping)
+                assert isinstance(conf, cabc.Mapping)
                 ctx['LOADED_CONFIG'] = True
             except Exception as e:
                 conf = {}
diff --git a/xonsh/execer.py b/xonsh/execer.py
index 5910bb20e..253dee670 100644
--- a/xonsh/execer.py
+++ b/xonsh/execer.py
@@ -5,7 +5,7 @@ import types
 import inspect
 import builtins
 import warnings
-import collections.abc as abc
+import collections.abc as cabc
 
 from xonsh.ast import CtxAwareTransformer
 from xonsh.parser import Parser
@@ -80,7 +80,7 @@ class Execer(object):
         # it also is valid as a subprocess line.
         if ctx is None:
             ctx = set()
-        elif isinstance(ctx, abc.Mapping):
+        elif isinstance(ctx, cabc.Mapping):
             ctx = set(ctx.keys())
         tree = self.ctxtransformer.ctxvisit(tree, input, ctx, mode=mode)
         return tree
diff --git a/xonsh/foreign_shells.py b/xonsh/foreign_shells.py
index 83236212b..e5e7ed2c6 100644
--- a/xonsh/foreign_shells.py
+++ b/xonsh/foreign_shells.py
@@ -9,7 +9,7 @@ import builtins
 import subprocess
 import warnings
 import functools
-import collections.abc as abc
+import collections.abc as cabc
 
 from xonsh.lazyasd import LazyObject
 from xonsh.tools import to_bool, ensure_string
@@ -423,7 +423,7 @@ VALID_SHELL_PARAMS = LazyObject(lambda: frozenset([
 
 def ensure_shell(shell):
     """Ensures that a mapping follows the shell specification."""
-    if not isinstance(shell, abc.MutableMapping):
+    if not isinstance(shell, cabc.MutableMapping):
         shell = dict(shell)
     shell_keys = set(shell.keys())
     if not (shell_keys <= VALID_SHELL_PARAMS):
@@ -444,9 +444,9 @@ def ensure_shell(shell):
         shell['extra_args'] = tuple(map(ensure_string, shell['extra_args']))
     if 'currenv' in shell_keys and not isinstance(shell['currenv'], tuple):
         ce = shell['currenv']
-        if isinstance(ce, abc.Mapping):
+        if isinstance(ce, cabc.Mapping):
             ce = tuple([(ensure_string(k), v) for k, v in ce.items()])
-        elif isinstance(ce, abc.Sequence):
+        elif isinstance(ce, cabc.Sequence):
             ce = tuple([(ensure_string(k), v) for k, v in ce])
         else:
             raise RuntimeError('unrecognized type for currenv')
diff --git a/xonsh/history.py b/xonsh/history.py
index 640c6abbe..2ef65856d 100644
--- a/xonsh/history.py
+++ b/xonsh/history.py
@@ -13,7 +13,7 @@ import functools
 import itertools
 import threading
 import collections
-import collections.abc as abc
+import collections.abc as cabc
 
 from xonsh.lazyasd import lazyobject
 from xonsh.lazyjson import LazyJSON, ljdump, LJNode
@@ -178,7 +178,7 @@ class HistoryFlusher(threading.Thread):
             ljdump(hist, f, sort_keys=True)
 
 
-class CommandField(abc.Sequence):
+class CommandField(cabc.Sequence):
     """A field in the 'cmds' portion of history."""
 
     def __init__(self, field, hist, default=None):
diff --git a/xonsh/lazyasd.py b/xonsh/lazyasd.py
index a32946c97..fd6121a53 100644
--- a/xonsh/lazyasd.py
+++ b/xonsh/lazyasd.py
@@ -8,7 +8,7 @@ import builtins
 import threading
 import importlib
 import importlib.util
-import collections.abc as abc
+import collections.abc as cabc
 
 __version__ = '0.1.1'
 
@@ -128,7 +128,7 @@ def lazyobject(f):
     return LazyObject(f, f.__globals__, f.__name__)
 
 
-class LazyDict(abc.MutableMapping):
+class LazyDict(cabc.MutableMapping):
 
     def __init__(self, loaders, ctx, name):
         """Dictionary like object that lazily loads its values from an initial
diff --git a/xonsh/lazyjson.py b/xonsh/lazyjson.py
index 09944054f..7962e8019 100644
--- a/xonsh/lazyjson.py
+++ b/xonsh/lazyjson.py
@@ -4,7 +4,7 @@ import io
 import json
 import weakref
 import contextlib
-import collections.abc as abc
+import collections.abc as cabc
 
 
 def _to_json_with_size(obj, offset=0, sort_keys=False):
@@ -12,7 +12,7 @@ def _to_json_with_size(obj, offset=0, sort_keys=False):
         s = json.dumps(obj)
         o = offset
         n = size = len(s.encode())  # size in bytes
-    elif isinstance(obj, abc.Mapping):
+    elif isinstance(obj, cabc.Mapping):
         s = '{'
         j = offset + 1
         o = {}
@@ -35,7 +35,7 @@ def _to_json_with_size(obj, offset=0, sort_keys=False):
         n = len(s)
         o['__total__'] = offset
         size['__total__'] = n
-    elif isinstance(obj, abc.Sequence):
+    elif isinstance(obj, cabc.Sequence):
         s = '['
         j = offset + 1
         o = []
@@ -94,7 +94,7 @@ def ljdump(obj, fp, sort_keys=False):
     fp.write(s)
 
 
-class LJNode(abc.Mapping, abc.Sequence):
+class LJNode(cabc.Mapping, cabc.Sequence):
     """A proxy node for JSON nodes. Acts as both sequence and mapping."""
 
     def __init__(self, offsets, sizes, root):
@@ -110,8 +110,8 @@ class LJNode(abc.Mapping, abc.Sequence):
         self.offsets = offsets
         self.sizes = sizes
         self.root = root
-        self.is_mapping = isinstance(self.offsets, abc.Mapping)
-        self.is_sequence = isinstance(self.offsets, abc.Sequence)
+        self.is_mapping = isinstance(self.offsets, cabc.Mapping)
+        self.is_sequence = isinstance(self.offsets, cabc.Sequence)
 
     def __len__(self):
         # recall that for maps, the '__total__' key is added and for
@@ -137,7 +137,7 @@ class LJNode(abc.Mapping, abc.Sequence):
                 f.seek(self.root.dloc + offset)
                 s = f.read(size)
             val = json.loads(s)
-        elif isinstance(offset, (abc.Mapping, abc.Sequence)):
+        elif isinstance(offset, (cabc.Mapping, cabc.Sequence)):
             val = LJNode(offset, size, self.root)
         else:
             raise TypeError('incorrect types for offset node')
@@ -204,8 +204,8 @@ class LazyJSON(LJNode):
             self._f = open(f, 'r', newline='\n')
         self._load_index()
         self.root = weakref.proxy(self)
-        self.is_mapping = isinstance(self.offsets, abc.Mapping)
-        self.is_sequence = isinstance(self.offsets, abc.Sequence)
+        self.is_mapping = isinstance(self.offsets, cabc.Mapping)
+        self.is_sequence = isinstance(self.offsets, cabc.Sequence)
 
     def __del__(self):
         self.close()
diff --git a/xonsh/proc.py b/xonsh/proc.py
index adc1b7ef2..8a2ebce5f 100644
--- a/xonsh/proc.py
+++ b/xonsh/proc.py
@@ -17,7 +17,7 @@ import functools
 import threading
 import subprocess
 import collections
-import collections.abc as abc
+import collections.abc as cabc
 
 from xonsh.platform import ON_WINDOWS, ON_LINUX, ON_POSIX
 from xonsh.tools import (redirect_stdout, redirect_stderr, fallback,
@@ -358,7 +358,7 @@ def wrap_simple_command(f, args, stdin, stdout, stderr):
             cmd_result = 0
             if isinstance(r, str):
                 stdout.write(r)
-            elif isinstance(r, abc.Sequence):
+            elif isinstance(r, cabc.Sequence):
                 if r[0] is not None:
                     stdout.write(r[0])
                 if r[1] is not None:
diff --git a/xonsh/replay.py b/xonsh/replay.py
index 03ac19895..4674f8138 100644
--- a/xonsh/replay.py
+++ b/xonsh/replay.py
@@ -2,7 +2,7 @@
 """Tools to replay xonsh history files."""
 import time
 import builtins
-import collections.abc as abc
+import collections.abc as cabc
 
 from xonsh.tools import swap
 from xonsh.lazyjson import LazyJSON
@@ -68,7 +68,7 @@ class Replayer(object):
                 new_env.update(re_env)
             elif e == 'native':
                 new_env.update(builtins.__xonsh_env__)
-            elif isinstance(e, abc.Mapping):
+            elif isinstance(e, cabc.Mapping):
                 new_env.update(e)
             else:
                 raise TypeError('Type of env not understood: {0!r}'.format(e))
diff --git a/xonsh/tools.py b/xonsh/tools.py
index 89ffda1c8..126291508 100644
--- a/xonsh/tools.py
+++ b/xonsh/tools.py
@@ -19,7 +19,7 @@ Implementations:
 """
 import builtins
 import collections
-import collections.abc as abc
+import collections.abc as cabc
 import contextlib
 import ctypes
 import datetime
@@ -980,7 +980,7 @@ def is_int_as_str(x):
 
 def is_string_set(x):
     """Tests if something is a set of strings"""
-    return (isinstance(x, abc.Set) and
+    return (isinstance(x, cabc.Set) and
             all(isinstance(a, str) for a in x))
 
 
@@ -1016,7 +1016,7 @@ def set_to_pathsep(x, sort=False):
 
 def is_string_seq(x):
     """Tests if something is a sequence of strings"""
-    return (isinstance(x, abc.Sequence) and
+    return (isinstance(x, cabc.Sequence) and
             all(isinstance(a, str) for a in x))
 
 
@@ -1024,7 +1024,7 @@ def is_nonstring_seq_of_strings(x):
     """Tests if something is a sequence of strings, where the top-level
     sequence is not a string itself.
     """
-    return (isinstance(x, abc.Sequence) and not isinstance(x, str) and
+    return (isinstance(x, cabc.Sequence) and not isinstance(x, str) and
             all(isinstance(a, str) for a in x))
 
 
@@ -1058,7 +1058,7 @@ def seq_to_upper_pathsep(x):
 
 def is_bool_seq(x):
     """Tests if an object is a sequence of bools."""
-    return isinstance(x, abc.Sequence) and all(isinstance(y, bool) for y in x)
+    return isinstance(x, cabc.Sequence) and all(isinstance(y, bool) for y in x)
 
 
 def csv_to_bool_seq(x):
@@ -1177,7 +1177,7 @@ HISTORY_UNITS = LazyObject(lambda: {
 
 def is_history_tuple(x):
     """Tests if something is a proper history value, units tuple."""
-    if (isinstance(x, abc.Sequence) and
+    if (isinstance(x, cabc.Sequence) and
             len(x) == 2 and
             isinstance(x[0], (int, float)) and
             x[1].lower() in CANON_HISTORY_UNITS):
@@ -1224,7 +1224,7 @@ RE_HISTORY_TUPLE = LazyObject(
 
 def to_history_tuple(x):
     """Converts to a canonincal history tuple."""
-    if not isinstance(x, (abc.Sequence, float, int)):
+    if not isinstance(x, (cabc.Sequence, float, int)):
         raise ValueError('history size must be given as a sequence or number')
     if isinstance(x, str):
         m = RE_HISTORY_TUPLE.match(x.strip().lower())
diff --git a/xonsh/xoreutils/_which.py b/xonsh/xoreutils/_which.py
index f86105c61..eeaa86a58 100644
--- a/xonsh/xoreutils/_which.py
+++ b/xonsh/xoreutils/_which.py
@@ -29,7 +29,7 @@ import sys
 import stat
 import getopt
 import builtins
-import collections.abc as abc
+import collections.abc as cabc
 
 r"""Find the full path to commands.
 
@@ -199,7 +199,7 @@ def whichgen(command, path=None, verbose=0, exts=None):
                     break
             else:
                 exts = ['.COM', '.EXE', '.BAT', '.CMD']
-        elif not isinstance(exts, abc.Sequence):
+        elif not isinstance(exts, cabc.Sequence):
             raise TypeError("'exts' argument must be a sequence or None")
     else:
         if exts is not None:

From e0f5fa66cdf415f7b6d8e9ba277b2940866d85da Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Wed, 24 Aug 2016 13:38:40 +0300
Subject: [PATCH 087/190] cleanup

---
 xonsh/foreign_shells.py | 100 +++++++++++++++++++++++++---------------
 1 file changed, 63 insertions(+), 37 deletions(-)

diff --git a/xonsh/foreign_shells.py b/xonsh/foreign_shells.py
index 02801143e..99815acea 100644
--- a/xonsh/foreign_shells.py
+++ b/xonsh/foreign_shells.py
@@ -12,7 +12,7 @@ import warnings
 import functools
 import collections.abc as abc
 
-from xonsh.lazyasd import LazyObject
+from xonsh.lazyasd import lazyobject
 from xonsh.tools import to_bool, ensure_string
 from xonsh.platform import ON_WINDOWS, ON_CYGWIN
 
@@ -80,7 +80,9 @@ fi
 echo ${namefile}"""
 
 # mapping of shell name alises to keys in other lookup dictionaries.
-CANON_SHELL_NAMES = LazyObject(lambda: {
+@lazyobject
+def CANON_SHELL_NAMES():
+    return {
     'bash': 'bash',
     '/bin/bash': 'bash',
     'zsh': 'zsh',
@@ -88,55 +90,72 @@ CANON_SHELL_NAMES = LazyObject(lambda: {
     '/usr/bin/zsh': 'zsh',
     'cmd': 'cmd',
     'cmd.exe': 'cmd',
-}, globals(), 'CANON_SHELL_NAMES')
+    }
 
-DEFAULT_ENVCMDS = LazyObject(lambda: {
+@lazyobject
+def DEFAULT_ENVCMDS():
+    return {
     'bash': 'env',
     'zsh': 'env',
     'cmd': 'set',
-}, globals(), 'DEFAULT_ENVCMDS')
+    }
 
-DEFAULT_ALIASCMDS = LazyObject(lambda: {
+@lazyobject
+def DEFAULT_ALIASCMDS():
+    return {
     'bash': 'alias',
     'zsh': 'alias -L',
     'cmd': '',
-}, globals(), 'DEFAULT_ALIASCMDS')
+    }
 
-DEFAULT_FUNCSCMDS = LazyObject(lambda: {
+@lazyobject
+def DEFAULT_FUNCSCMDS():
+    return {
     'bash': DEFAULT_BASH_FUNCSCMD,
     'zsh': DEFAULT_ZSH_FUNCSCMD,
     'cmd': '',
-}, globals(), 'DEFAULT_FUNCSCMDS')
+    }
 
-DEFAULT_SOURCERS = LazyObject(lambda: {
+
+@lazyobject
+def DEFAULT_SOURCERS():
+    return {
     'bash': 'source',
     'zsh': 'source',
     'cmd': 'call',
-}, globals(), 'DEFAULT_SOURCERS')
+    }
 
-DEFAULT_TMPFILE_EXT = LazyObject(lambda: {
+@lazyobject
+def DEFAULT_TMPFILE_EXT():
+    return {
     'bash': '.sh',
     'zsh': '.zsh',
     'cmd': '.bat',
-}, globals(), 'DEFAULT_TMPFILE_EXT')
+    }
 
-DEFAULT_RUNCMD = LazyObject(lambda: {
+@lazyobject
+def DEFAULT_RUNCMD():
+    return {
     'bash': '-c',
     'zsh': '-c',
     'cmd': '/C',
-}, globals(), 'DEFAULT_RUNCMD')
+    }
 
-DEFAULT_SETERRPREVCMD = LazyObject(lambda: {
+@lazyobject
+def DEFAULT_SETERRPREVCMD():
+    return {
     'bash': 'set -e',
     'zsh': 'set -e',
     'cmd': '@echo off',
-}, globals(), 'DEFAULT_SETERRPREVCMD')
+    }
 
-DEFAULT_SETERRPOSTCMD = LazyObject(lambda: {
+@lazyobjecet
+def DEFAULT_SETERRPOSTCMD():
+    return {
     'bash': '',
     'zsh': '',
     'cmd': 'if errorlevel 1 exit 1',
-}, globals(), 'DEFAULT_SETERRPOSTCMD')
+    }
 
 
 @functools.lru_cache()
@@ -262,12 +281,15 @@ def foreign_shell_data(shell, interactive=True, login=False, envcmd=None,
     return env, aliases
 
 
-ENV_RE = LazyObject(lambda: re.compile('__XONSH_ENV_BEG__\n(.*)'
-                                       '__XONSH_ENV_END__', flags=re.DOTALL),
-                    globals(), 'ENV_RE')
-ENV_SPLIT_RE = LazyObject(lambda: re.compile('^([^=]+)=([^=]*|[^\n]*)$',
-                                             flags=re.DOTALL | re.MULTILINE),
-                          globals(), 'ENV_SPLIT_RE')
+@lazyobject
+def ENV_RE():
+    return re.compile('__XONSH_ENV_BEG__\n(.*)'
+                      '__XONSH_ENV_END__', flags=re.DOTALL)
+
+@lazyobject
+def ENV_SPLIT_RE():
+    return re.compile('^([^=]+)=([^=]*|[^\n]*)$',
+               flags=re.DOTALL | re.MULTILINE)
 
 
 def parse_env(s):
@@ -281,10 +303,11 @@ def parse_env(s):
     return env
 
 
-ALIAS_RE = LazyObject(lambda: re.compile('__XONSH_ALIAS_BEG__\n(.*)'
-                                         '__XONSH_ALIAS_END__',
-                                         flags=re.DOTALL),
-                      globals(), 'ALIAS_RE')
+@lazyobject
+def ALIAS_RE():
+    return re.compile('__XONSH_ALIAS_BEG__\n(.*)'
+                      '__XONSH_ALIAS_END__',
+                      flags=re.DOTALL)
 
 
 def parse_aliases(s):
@@ -313,10 +336,12 @@ def parse_aliases(s):
     return aliases
 
 
-FUNCS_RE = LazyObject(lambda: re.compile('__XONSH_FUNCS_BEG__\n(.+)\n'
-                                         '__XONSH_FUNCS_END__',
-                                         flags=re.DOTALL),
-                      globals(), 'FUNCS_RE')
+
+@lazyobject
+def FUNCS_RE():
+   return re.compile('__XONSH_FUNCS_BEG__\n(.+)\n'
+                     '__XONSH_FUNCS_END__',
+                     flags=re.DOTALL)
 
 
 def parse_funcs(s, shell, sourcer=None):
@@ -415,11 +440,14 @@ class ForeignShellFunctionAlias(object):
         return args, True
 
 
-VALID_SHELL_PARAMS = LazyObject(lambda: frozenset([
+
+@lazyobject
+def VALID_SHELL_PARAMS():
+    return frozenset([
     'shell', 'interactive', 'login', 'envcmd',
     'aliascmd', 'extra_args', 'currenv', 'safe',
     'prevcmd', 'postcmd', 'funcscmd', 'sourcer',
-]), globals(), 'VALID_SHELL_PARAMS')
+    ])
 
 
 def ensure_shell(shell):
@@ -545,8 +573,6 @@ def load_foreign_aliases(shells=None, config=None, issue_warning=True):
     for shell in shells:
         shell = ensure_shell(shell)
         _, shaliases = foreign_shell_data(**shell)
-        if not shaliases:
-            continue
         for alias in set(shaliases) & set(xonsh_aliases):
             del shaliases[alias]
             print('aliases: alias {!r} of shell {!r} '

From 6b1b2d66f7e9aa9ac111db7f84ac80e3e9aa4fe8 Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Wed, 24 Aug 2016 13:42:44 +0300
Subject: [PATCH 088/190] typo

---
 xonsh/foreign_shells.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xonsh/foreign_shells.py b/xonsh/foreign_shells.py
index 99815acea..8f7460bca 100644
--- a/xonsh/foreign_shells.py
+++ b/xonsh/foreign_shells.py
@@ -149,7 +149,7 @@ def DEFAULT_SETERRPREVCMD():
     'cmd': '@echo off',
     }
 
-@lazyobjecet
+@lazyobject
 def DEFAULT_SETERRPOSTCMD():
     return {
     'bash': '',

From 54ac7d108c16b29b68dce4558bf78ce52ba6e36a Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Wed, 24 Aug 2016 14:00:10 +0300
Subject: [PATCH 089/190] pep8tify

---
 xonsh/foreign_shells.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/xonsh/foreign_shells.py b/xonsh/foreign_shells.py
index 8f7460bca..71919d3d6 100644
--- a/xonsh/foreign_shells.py
+++ b/xonsh/foreign_shells.py
@@ -79,6 +79,7 @@ else
 fi
 echo ${namefile}"""
 
+
 # mapping of shell name alises to keys in other lookup dictionaries.
 @lazyobject
 def CANON_SHELL_NAMES():
@@ -92,6 +93,7 @@ def CANON_SHELL_NAMES():
     'cmd.exe': 'cmd',
     }
 
+
 @lazyobject
 def DEFAULT_ENVCMDS():
     return {
@@ -100,6 +102,7 @@ def DEFAULT_ENVCMDS():
     'cmd': 'set',
     }
 
+
 @lazyobject
 def DEFAULT_ALIASCMDS():
     return {
@@ -108,6 +111,7 @@ def DEFAULT_ALIASCMDS():
     'cmd': '',
     }
 
+
 @lazyobject
 def DEFAULT_FUNCSCMDS():
     return {
@@ -125,6 +129,7 @@ def DEFAULT_SOURCERS():
     'cmd': 'call',
     }
 
+
 @lazyobject
 def DEFAULT_TMPFILE_EXT():
     return {
@@ -133,6 +138,7 @@ def DEFAULT_TMPFILE_EXT():
     'cmd': '.bat',
     }
 
+
 @lazyobject
 def DEFAULT_RUNCMD():
     return {
@@ -141,6 +147,7 @@ def DEFAULT_RUNCMD():
     'cmd': '/C',
     }
 
+
 @lazyobject
 def DEFAULT_SETERRPREVCMD():
     return {
@@ -149,6 +156,7 @@ def DEFAULT_SETERRPREVCMD():
     'cmd': '@echo off',
     }
 
+
 @lazyobject
 def DEFAULT_SETERRPOSTCMD():
     return {
@@ -286,10 +294,11 @@ def ENV_RE():
     return re.compile('__XONSH_ENV_BEG__\n(.*)'
                       '__XONSH_ENV_END__', flags=re.DOTALL)
 
+
 @lazyobject
 def ENV_SPLIT_RE():
     return re.compile('^([^=]+)=([^=]*|[^\n]*)$',
-               flags=re.DOTALL | re.MULTILINE)
+                      flags=re.DOTALL | re.MULTILINE)
 
 
 def parse_env(s):
@@ -336,12 +345,11 @@ def parse_aliases(s):
     return aliases
 
 
-
 @lazyobject
 def FUNCS_RE():
-   return re.compile('__XONSH_FUNCS_BEG__\n(.+)\n'
-                     '__XONSH_FUNCS_END__',
-                     flags=re.DOTALL)
+    return re.compile('__XONSH_FUNCS_BEG__\n(.+)\n'
+                      '__XONSH_FUNCS_END__',
+                      flags=re.DOTALL)
 
 
 def parse_funcs(s, shell, sourcer=None):
@@ -440,7 +448,6 @@ class ForeignShellFunctionAlias(object):
         return args, True
 
 
-
 @lazyobject
 def VALID_SHELL_PARAMS():
     return frozenset([

From 4e9142d58aeabfd40e45ea55e929756da448f744 Mon Sep 17 00:00:00 2001
From: Andrei <fizicars@gmail.com>
Date: Wed, 24 Aug 2016 13:11:42 +0200
Subject: [PATCH 090/190] improved #1170 warning message "no readline"

https://github.com/xonsh/xonsh/issues/1170#issuecomment-240922451
---
 xonsh/readline_shell.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/xonsh/readline_shell.py b/xonsh/readline_shell.py
index 1198ae0d5..bc3798c7f 100644
--- a/xonsh/readline_shell.py
+++ b/xonsh/readline_shell.py
@@ -57,9 +57,14 @@ def setup_readline():
             pass
         else:
             break
+
     if readline is None:
-        print("No readline implementation available.  Skipping setup.")
+        print("""Skipping setup. Because no `readline` implementation available.
+            Please install a backend (`readline`, `prompt-toolkit`, etc) to use
+            `xonsh` interactively.
+            See https://github.com/xonsh/xonsh/issues/1170""")
         return
+
     import ctypes
     import ctypes.util
     uses_libedit = readline.__doc__ and 'libedit' in readline.__doc__

From 5156bac3c2462472ac2a682309e2f035f2f78ac3 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Wed, 24 Aug 2016 07:46:10 -0400
Subject: [PATCH 091/190] use `[` only when attribute has `__get_item__`

---
 xonsh/completers/python.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/xonsh/completers/python.py b/xonsh/completers/python.py
index 0cd9fdf4f..7c1dfc161 100644
--- a/xonsh/completers/python.py
+++ b/xonsh/completers/python.py
@@ -4,7 +4,7 @@ import sys
 import inspect
 import builtins
 import importlib
-import collections.abc as abc
+import collections.abc as cabc
 
 import xonsh.tools as xt
 import xonsh.lazyasd as xl
@@ -95,11 +95,10 @@ def attr_complete(prefix, ctx, filter_func):
         except:  # pylint:disable=bare-except
             continue
         a = getattr(val, opt)
-        if (builtins.__xonsh_env__['COMPLETIONS_BRACKETS'] and
-                        opt not in ['stdin', 'stdout', 'stderr']):
+        if builtins.__xonsh_env__['COMPLETIONS_BRACKETS']:
             if callable(a):
                 rpl = opt + '('
-            elif isinstance(a, abc.Iterable):
+            elif isinstance(a, (cabc.Sequence, cabc.Mapping)):
                 rpl = opt + '['
             else:
                 rpl = opt

From 271ec614d628934101f4e85abce54b9963c26ee7 Mon Sep 17 00:00:00 2001
From: Guillaume Leclerc <guillaume.leclerc.work@gmail.com>
Date: Wed, 24 Aug 2016 15:44:55 +0200
Subject: [PATCH 092/190] Add information about utf8 support on linux

This commit explains how to fix the problem described in #912

Closes #912
---
 docs/linux.rst | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/docs/linux.rst b/docs/linux.rst
index 5c584636d..877e4d795 100644
--- a/docs/linux.rst
+++ b/docs/linux.rst
@@ -139,3 +139,20 @@ you can try to replace a command for this action by the following:
 
 In order to do this, go to ``Edit > Configure custom actions...``,
 then choose ``Open Terminal Here`` and click on ``Edit currently selected action`` button.
+
+Unable to use utf-8 characters inside xonsh
+===========================================
+
+If you are unable to use utf-8 (ie. non-ascii) characters in xonsh. For example if you get the following output
+
+.. code-block:: xonsh
+
+    echo "ßðđ"
+    
+    xonsh: For full traceback set: $XONSH_SHOW_TRACEBACK = True
+    UnicodeEncodeError: 'ascii' codec can't encode characters in position 0-2: ordinal not in range(128)
+    
+The problem might be: 
+
+- Your locale is not set to utf-8, to check this you can set the content of the environment variable ``LC_TYPE``
+- Your locale is correctly set but **after** xonsh started. This is typically the case if you set your ``LC_TYPE`` inside your ``.xonshrc`` and xonsh is your default/login shell. To fix this you should see the documentation of your operating system to know how to correctly setup environment variables before the shell start (``~/.pam_environment`` for example)

From b3c0b695bd2c2c359a91261d64617351c5a39048 Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Thu, 25 Aug 2016 00:58:42 +0800
Subject: [PATCH 093/190] add to DEFAULT_ENSURES

---
 xonsh/environ.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/xonsh/environ.py b/xonsh/environ.py
index c88218372..174da02c6 100644
--- a/xonsh/environ.py
+++ b/xonsh/environ.py
@@ -101,6 +101,7 @@ def DEFAULT_ENSURERS():
     re.compile('\w*DIRS$'): (is_env_path, str_to_env_path, env_path_to_str),
     'COLOR_INPUT': (is_bool, to_bool, bool_to_str),
     'COLOR_RESULTS': (is_bool, to_bool, bool_to_str),
+    'COMPLETION_CONFIRM': (is_bool, to_bool, bool_to_str),
     'COMPLETIONS_DISPLAY': (is_completions_display_value,
                             to_completions_display_value, str),
     'COMPLETIONS_MENU_ROWS': (is_int, int, str),

From 1461b41a299396622e49070ea37819c9213cc1d3 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Wed, 24 Aug 2016 13:20:17 -0400
Subject: [PATCH 094/190] add circleci yml file

---
 .circle.yml | 19 +++++++++++++++++++
 circle.yml  |  1 +
 2 files changed, 20 insertions(+)
 create mode 100644 .circle.yml
 create mode 120000 circle.yml

diff --git a/.circle.yml b/.circle.yml
new file mode 100644
index 000000000..c912de39b
--- /dev/null
+++ b/.circle.yml
@@ -0,0 +1,19 @@
+machine:
+    environment:
+        PATH: /home/ubuntu/miniconda/bin:$PATH
+
+dependencies:
+    pre:
+        - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+        - bash miniconda.sh -b -p $HOME/miniconda
+        - hash -r
+        - conda config --set always_yes yes --set changeps1 no
+        - conda update -q conda
+        - conda info -a
+        - conda create -q -n test_env python=3.5 pygments prompt_toolkit ply pytest pytest-timeout psutil
+        - source activate test_env
+        - python setup.py install
+
+test:
+    override:
+        - py.test --timeout=10
diff --git a/circle.yml b/circle.yml
new file mode 120000
index 000000000..c1ea182b2
--- /dev/null
+++ b/circle.yml
@@ -0,0 +1 @@
+.circle.yml
\ No newline at end of file

From 091ce80a94b1e0cbd0ce1a770cd62bbabd8a5c8c Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Wed, 24 Aug 2016 13:22:37 -0400
Subject: [PATCH 095/190] use hardcoded path to environment folder

circle ci launches a new terminal for every command so source activate won't work correctly
---
 .circle.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circle.yml b/.circle.yml
index c912de39b..ee6fa2fa0 100644
--- a/.circle.yml
+++ b/.circle.yml
@@ -1,6 +1,6 @@
 machine:
     environment:
-        PATH: /home/ubuntu/miniconda/bin:$PATH
+        PATH: /home/ubuntu/miniconda/envs/test_env/bin:/home/ubuntu/miniconda/bin:$PATH
 
 dependencies:
     pre:

From 14194bc5e09f48ec1d85e60acc5c2649ffdfd443 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Wed, 24 Aug 2016 13:36:16 -0400
Subject: [PATCH 096/190] move setup to post

---
 .circle.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.circle.yml b/.circle.yml
index ee6fa2fa0..2322edffa 100644
--- a/.circle.yml
+++ b/.circle.yml
@@ -11,7 +11,8 @@ dependencies:
         - conda update -q conda
         - conda info -a
         - conda create -q -n test_env python=3.5 pygments prompt_toolkit ply pytest pytest-timeout psutil
-        - source activate test_env
+        - echo $(which python)
+    post:
         - python setup.py install
 
 test:

From a9a7858d411e5ebdc3d03e5847d0549dc7a1a4a6 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Wed, 24 Aug 2016 13:45:30 -0400
Subject: [PATCH 097/190] remove virtualenv from build

---
 .circle.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.circle.yml b/.circle.yml
index 2322edffa..4989c38d4 100644
--- a/.circle.yml
+++ b/.circle.yml
@@ -6,12 +6,11 @@ dependencies:
     pre:
         - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
         - bash miniconda.sh -b -p $HOME/miniconda
-        - hash -r
         - conda config --set always_yes yes --set changeps1 no
         - conda update -q conda
-        - conda info -a
         - conda create -q -n test_env python=3.5 pygments prompt_toolkit ply pytest pytest-timeout psutil
-        - echo $(which python)
+        - rm -rf ~/.pyenv
+        - rm -rf ~/virtualenvs
     post:
         - python setup.py install
 

From 416da7900b6f31f9c866f40f20e5ca20cc8eb754 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Wed, 24 Aug 2016 13:50:15 -0400
Subject: [PATCH 098/190] add multiple environments

---
 .circle.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.circle.yml b/.circle.yml
index 4989c38d4..5ebdc6f6c 100644
--- a/.circle.yml
+++ b/.circle.yml
@@ -1,6 +1,8 @@
 machine:
     environment:
         PATH: /home/ubuntu/miniconda/envs/test_env/bin:/home/ubuntu/miniconda/bin:$PATH
+    post:
+        - pyenv global 3.4 3.5
 
 dependencies:
     pre:

From c03122aafae9982aa5a67ecc88ad1ec95bdecfca Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Wed, 24 Aug 2016 13:52:49 -0400
Subject: [PATCH 099/190] fix version numbers

---
 .circle.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circle.yml b/.circle.yml
index 5ebdc6f6c..b1eb6626d 100644
--- a/.circle.yml
+++ b/.circle.yml
@@ -2,7 +2,7 @@ machine:
     environment:
         PATH: /home/ubuntu/miniconda/envs/test_env/bin:/home/ubuntu/miniconda/bin:$PATH
     post:
-        - pyenv global 3.4 3.5
+        - pyenv global 3.4.4 3.5.1
 
 dependencies:
     pre:

From 0d916c426c954c85c100d689671c546d9d465cbf Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Wed, 24 Aug 2016 14:12:56 -0400
Subject: [PATCH 100/190] hack in separate 3.4 and 3.5 tests in conda

---
 .circle.yml          |  2 +-
 .circle_miniconda.sh | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)
 create mode 100644 .circle_miniconda.sh

diff --git a/.circle.yml b/.circle.yml
index b1eb6626d..59108b022 100644
--- a/.circle.yml
+++ b/.circle.yml
@@ -10,7 +10,7 @@ dependencies:
         - bash miniconda.sh -b -p $HOME/miniconda
         - conda config --set always_yes yes --set changeps1 no
         - conda update -q conda
-        - conda create -q -n test_env python=3.5 pygments prompt_toolkit ply pytest pytest-timeout psutil
+        - ./.circle_miniconda.sh
         - rm -rf ~/.pyenv
         - rm -rf ~/virtualenvs
     post:
diff --git a/.circle_miniconda.sh b/.circle_miniconda.sh
new file mode 100644
index 000000000..35e8906d1
--- /dev/null
+++ b/.circle_miniconda.sh
@@ -0,0 +1,10 @@
+if [[ $CIRCLE_NODE_INDEX == 0 ]]
+then
+    conda create -q -n test_env python=3.4 pygments prompt_toolkit ply pytest pytest-timeout psutil
+fi
+
+
+if [[ $CIRCLE_NODE_INDEX == 1 ]]
+then
+    conda create -q -n test_env python=3.5 pygments prompt_toolkit ply pytest pytest-timeout psutil
+fi

From 34fe72cec3f3b84051cd239f9ed3e1fce8beba5c Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Wed, 24 Aug 2016 14:15:21 -0400
Subject: [PATCH 101/190] use bash explicitly to run script

---
 .circle.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circle.yml b/.circle.yml
index 59108b022..1da5af45d 100644
--- a/.circle.yml
+++ b/.circle.yml
@@ -10,7 +10,7 @@ dependencies:
         - bash miniconda.sh -b -p $HOME/miniconda
         - conda config --set always_yes yes --set changeps1 no
         - conda update -q conda
-        - ./.circle_miniconda.sh
+        - bash .circle_miniconda.sh
         - rm -rf ~/.pyenv
         - rm -rf ~/virtualenvs
     post:

From fbe773ff3ddaf616f17e2e45fbe6ff228a439b19 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Wed, 24 Aug 2016 14:21:37 -0400
Subject: [PATCH 102/190] parallelize runs

---
 .circle.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.circle.yml b/.circle.yml
index 1da5af45d..49cd576ff 100644
--- a/.circle.yml
+++ b/.circle.yml
@@ -14,8 +14,10 @@ dependencies:
         - rm -rf ~/.pyenv
         - rm -rf ~/virtualenvs
     post:
-        - python setup.py install
+        - case $CIRCLE_NODE_INDEX in 0) python setup.py install ;; 1) python setup.py install ;; esac:
+            parallel: true
 
 test:
     override:
-        - py.test --timeout=10
+        - case $CIRCLE_NODE_INDEX in 0) py.test --timeout=10 ;; 1) py.test --timeout=10 ;; esac:
+            parallel: true

From 679026212d0cb0355906c1927649d2ff72fc6cba Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Wed, 24 Aug 2016 14:26:46 -0400
Subject: [PATCH 103/190] remove py34 and py35 from travis build

leaving nightly in for now since that isn't available on circleci (i
don't think)
---
 .travis.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 0ca14a9f7..95b99b823 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,12 +4,6 @@ matrix:
         - os: linux
           python: "nightly"
           env: RUN_COVERAGE=true
-        - os: linux
-          language: generic
-          env: PYTHON="3.4" MINICONDA_OS="Linux"
-        - os: linux
-          language: generic
-          env: PYTHON="3.5" MINICONDA_OS="Linux"
         - os: osx
           language: generic
           env: PYTHON="3.4" MINICONDA_OS="MacOSX"

From 8c23bde001279ad794ca6b3afb503305b66e4906 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Wed, 24 Aug 2016 19:04:02 -0400
Subject: [PATCH 104/190] Rename __xonsh_events__ to events

---
 xonsh/built_ins.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py
index 082554250..0b5bc6468 100644
--- a/xonsh/built_ins.py
+++ b/xonsh/built_ins.py
@@ -715,7 +715,7 @@ def load_builtins(execer=None, config=None, login=False, ctx=None):
     builtins.__xonsh_ensure_list_of_strs__ = ensure_list_of_strs
     builtins.__xonsh_list_of_strs_or_callables__ = list_of_strs_or_callables
     builtins.__xonsh_completers__ = xonsh.completers.init.default_completers()
-    builtins.__xonsh_events__ = events
+    builtins.events = events
     # public built-ins
     builtins.XonshError = XonshError
     builtins.XonshBlockError = XonshBlockError

From a81e2d6bd7fafc58e5978d1c8b933801e33cc3e5 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Wed, 24 Aug 2016 19:04:53 -0400
Subject: [PATCH 105/190] Make a really good attempt at method documentation.

---
 xonsh/events.py | 98 ++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 88 insertions(+), 10 deletions(-)

diff --git a/xonsh/events.py b/xonsh/events.py
index 3e79dbd85..4ae81c316 100644
--- a/xonsh/events.py
+++ b/xonsh/events.py
@@ -14,6 +14,13 @@ import sys
 
 
 class AbstractEvent(collections.abc.MutableSet, abc.ABC):
+    """
+    A given event that handlers can register against.
+
+    Acts as a ``MutableSet`` for registered handlers.
+
+    Note that ordering is never guaranteed.
+    """
     def __call__(self, handler):
         """
         Registers a handler. It's suggested to use this as a decorator.
@@ -23,6 +30,16 @@ class AbstractEvent(collections.abc.MutableSet, abc.ABC):
         validator takes the same arguments as the handler. If it returns False,
         the handler will not called or considered, as if it was not registered
         at all.
+
+        Parameters
+        ----------
+        handler : callable
+            The handler to register
+
+        Returns
+        -------
+        rtn : callable
+            The handler
         """
         #  Using Pythons "private" munging to minimize hypothetical collisions
         handler.__validator = None
@@ -50,16 +67,19 @@ class AbstractEvent(collections.abc.MutableSet, abc.ABC):
     def fire(self, *pargs, **kwargs):
         """
         Fires an event, calling registered handlers with the given arguments.
+
+        Parameters
+        ----------
+        *pargs :
+            Positional arguments to pass to each handler
+        **kwargs :
+            Keyword arguments to pass to each handler
         """
 
 
 class Event(AbstractEvent):
     """
-    A given event that handlers can register against.
-
-    Acts as a ``set`` for registered handlers.
-
-    Note that ordering is never guaranteed.
+    An event species for notify and scatter-gather events.
     """
     # Wish I could just pull from set...
     def __init__(self):
@@ -75,14 +95,40 @@ class Event(AbstractEvent):
         yield from self._handlers
 
     def add(self, item):
+        """
+        Add an element to a set.
+
+        This has no effect if the element is already present.
+        """
         self._handlers.add(item)
 
     def discard(self, item):
+        """
+        Remove an element from a set if it is a member.
+
+        If the element is not a member, do nothing.
+        """
         self._handlers.discard(item)
 
     def fire(self, *pargs, **kwargs):
         """
-        Fires each event, returning a non-unique iterable of the results.
+        Fires an event, calling registered handlers with the given arguments. A non-unique iterable
+        of the results is returned.
+
+        Each handler is called immediately. Exceptions are turned in to warnings.
+
+        Parameters
+        ----------
+        *pargs :
+            Positional arguments to pass to each handler
+        **kwargs :
+            Keyword arguments to pass to each handler
+
+        Returns
+        -------
+        vals : iterable
+            Return values of each handler. If multiple handlers return the same value, it will 
+            appear multiple times.
         """
         vals = []
         for handler in self._filterhandlers(self._handlers, *pargs, **kwargs):
@@ -91,6 +137,7 @@ class Event(AbstractEvent):
             except Exception:
                 print("Exception raised in event handler; ignored.", file=sys.stderr)
                 traceback.print_exc()
+                # FIXME: Actually warn
             else:
                 vals.append(rv)
         return vals
@@ -98,7 +145,8 @@ class Event(AbstractEvent):
 
 class LoadEvent(AbstractEvent):
     """
-    A kind of event in which each handler is called exactly once.
+    An event species where each handler is called exactly once, shortly after either the event is
+    fired or the handler is registered (whichever is later).
     """
     def __init__(self):
         self._fired = set()
@@ -116,9 +164,19 @@ class LoadEvent(AbstractEvent):
         yield from self._unfired
 
     def add(self, item):
+        """
+        Add an element to a set.
+
+        This has no effect if the element is already present.
+        """
         self._fired.add(item)
 
     def discard(self, item):
+        """
+        Remove an element from a set if it is a member.
+
+        If the element is not a member, do nothing.
+        """
         self._fired.discard(item)
         self._unfired.discard(item)
 
@@ -138,14 +196,28 @@ class EventManager:
     def doc(self, name, docstring):
         """
         Applies a docstring to an event.
+
+        Parameters
+        ----------
+        name : str
+            The name of the event, eg "on_precommand"
+        docstring : str
+            The docstring to apply to the event
         """
         type(getattr(self, name)).__doc__ = docstring
 
     def transmogrify(self, name, klass):
         """
-        Converts an event from one species to another.
+        Converts an event from one species to another, preserving handlers and docstring.
 
-        Please note: Some species may do special things with handlers. This is lost.
+        Please note: Some species maintain specialized state. This is lost on transmogrification.
+
+        Parameters
+        ----------
+        name : str
+            The name of the event, eg "on_precommand"
+        klass : sublcass of AbstractEvent
+            The type to turn the event in to.
         """
         if isinstance(klass, str):
             klass = globals()[klass]
@@ -161,8 +233,14 @@ class EventManager:
             newevent.add(handler)
 
     def __getattr__(self, name):
-        e = type(name, (Event,), {'__doc__': None})()
+        """
+        Get an event, if it doesn't already exist.
+        """
+        # This is only called if the attribute doesn't exist, so create the Event...
+        e = type(name, (Event,), {'__doc__': None})()  # (A little bit of magic to enable docstrings to work right)
+        # ... and save it.
         setattr(self, name, e)
+        # Now it exists, and we won't be called again.
         return e
 
 

From 3c38e4b3f7afbd519674780c1379ca36fbb978fd Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Wed, 24 Aug 2016 21:46:51 -0400
Subject: [PATCH 106/190] Start in on event documentation

---
 docs/events.rst          |  9 +++++++++
 docs/tutorial_events.rst | 30 ++++++++++++++++++++++++++++++
 2 files changed, 39 insertions(+)
 create mode 100644 docs/events.rst
 create mode 100644 docs/tutorial_events.rst

diff --git a/docs/events.rst b/docs/events.rst
new file mode 100644
index 000000000..0de19e3af
--- /dev/null
+++ b/docs/events.rst
@@ -0,0 +1,9 @@
+.. _events:
+
+********************
+Advanced Events
+********************
+
+If you havent, go read the `events tutorial`_ first.
+
+{This is where we document events for core developers and advanced users.}
\ No newline at end of file
diff --git a/docs/tutorial_events.rst b/docs/tutorial_events.rst
new file mode 100644
index 000000000..044972c32
--- /dev/null
+++ b/docs/tutorial_events.rst
@@ -0,0 +1,30 @@
+.. _tutorial_events:
+
+************************************
+Tutorial: Events
+************************************
+What's the best way to keep informed in xonsh? Subscribe to an event!
+
+Overview
+========
+Simply, events are a way for xonsh to .... do something.
+
+
+Show me the code!
+=================
+Fine, fine!
+
+This will add a line to a file every time the current directory changes (due to ``cd``, ``pushd``,
+or several other commands).
+
+    @events.on_chdir
+    def add_to_file(newdir):
+        with open(g`~/.dirhist`[0], 'a') as dh:
+            print(newdir, file=dh)
+
+Core Events
+===========
+
+* ``on_precommand``
+* ``on_postcommand``
+* ``on_chdir``

From 9a958f2ebacc909e8765e7ceb497ddc53d2e3913 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Thu, 25 Aug 2016 00:35:02 -0400
Subject: [PATCH 107/190] more expand vars

---
 news/expandvar.rst    | 14 ++++++++++++++
 tests/test_parser.py  |  3 +++
 xonsh/parsers/base.py | 10 +---------
 3 files changed, 18 insertions(+), 9 deletions(-)
 create mode 100644 news/expandvar.rst

diff --git a/news/expandvar.rst b/news/expandvar.rst
new file mode 100644
index 000000000..b6180ad44
--- /dev/null
+++ b/news/expandvar.rst
@@ -0,0 +1,14 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:**
+
+* Environment variables in subprocess mode were not being expanded
+  unless they were in a sting. They are now expanded properly.
+
+**Security:** None
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 044a54d1b..5b40d6aea 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -1641,6 +1641,9 @@ def test_uncaptured_sub():
 def test_hiddenobj_sub():
     check_xonsh_ast({}, '![ls]', False)
 
+def test_slash_envarv_echo():
+    check_xonsh_ast({}, '![echo $HOME/place]', False)
+
 def test_bang_two_cmds_one_pipe():
     check_xonsh_ast({}, '!(ls | grep wakka)', False)
 
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index bac4e07c3..1b81e8061 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -2253,15 +2253,6 @@ class BaseParser(object):
         p0._cliarg_action = 'append'
         p[0] = p0
 
-    def p_subproc_atom_dollar_name(self, p):
-        """subproc_atom : DOLLAR_NAME"""
-        p0 = self._envvar_getter_by_name(p[1][1:], lineno=self.lineno,
-                                         col=self.col)
-        p0 = xonsh_call('__xonsh_ensure_list_of_strs__', [p0],
-                        lineno=self.lineno, col=self.col)
-        p0._cliarg_action = 'extend'
-        p[0] = p0
-
     def p_subproc_atom_re(self, p):
         """subproc_atom : SEARCHPATH"""
         p0 = xonsh_pathsearch(p[1], pymode=False, lineno=self.lineno,
@@ -2323,6 +2314,7 @@ class BaseParser(object):
                             | STRING
                             | COMMA
                             | QUESTION
+                            | DOLLAR_NAME
         """
         # Many tokens cannot be part of this list, such as $, ', ", ()
         # Use a string atom instead.

From 50e2f80e102dc574cfc2cdf03d7bc675504f7382 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Thu, 25 Aug 2016 01:04:08 -0400
Subject: [PATCH 108/190] more tilde expansions

---
 news/tilde.rst         | 14 ++++++++++++++
 tests/test_builtins.py | 15 ++++++++++++++-
 xonsh/built_ins.py     |  9 ++++++++-
 3 files changed, 36 insertions(+), 2 deletions(-)
 create mode 100644 news/tilde.rst

diff --git a/news/tilde.rst b/news/tilde.rst
new file mode 100644
index 000000000..75bf3003e
--- /dev/null
+++ b/news/tilde.rst
@@ -0,0 +1,14 @@
+**Added:** None
+
+**Changed:**
+
+* Tilde expansion dor the home directory now has the same semantics as Bash.
+  Previously it only matched leading tildes.
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None
diff --git a/tests/test_builtins.py b/tests/test_builtins.py
index 838c592a2..6fe171a5e 100644
--- a/tests/test_builtins.py
+++ b/tests/test_builtins.py
@@ -9,7 +9,7 @@ import pytest
 from xonsh import built_ins
 from xonsh.built_ins import reglob, pathsearch, helper, superhelper, \
     ensure_list_of_strs, list_of_strs_or_callables, regexsearch, \
-    globsearch
+    globsearch, expand_path
 from xonsh.environ import Env
 
 from tools import skip_if_on_windows
@@ -113,3 +113,16 @@ f = lambda x: 20
 def test_list_of_strs_or_callables(exp, inp):
     obs = list_of_strs_or_callables(inp)
     assert exp == obs
+
+
+@pytest.mark.parametrize('s', [
+    '~',
+    '~/',
+    'x=~/place',
+    'one:~/place',
+    'one:~/place:~/yo',
+    '~/one:~/place:~/yo',
+    ])
+def test_expand_path(s, home_env):
+    assert expand_path(s) == s.replace('~', HOME_PATH)
+
diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py
index 6da0421d3..672cbaaed 100644
--- a/xonsh/built_ins.py
+++ b/xonsh/built_ins.py
@@ -99,7 +99,14 @@ def expand_path(s):
     """Takes a string path and expands ~ to home and environment vars."""
     if builtins.__xonsh_env__.get('EXPAND_ENV_VARS'):
         s = expandvars(s)
-    return os.path.expanduser(s)
+    # expand ~ according to Bash unquoted rules "Each variable assignment is
+    # checked for unquoted tilde-prefixes immediately following a ':' or the
+    # first '='". See the following for more details.
+    # https://www.gnu.org/software/bash/manual/html_node/Tilde-Expansion.html
+    pre, char, post = s.partition('=')
+    s = pre + char + os.path.expanduser(post) if char else s
+    s = os.pathsep.join(map(os.path.expanduser, s.split(os.pathsep)))
+    return s
 
 
 def reglob(path, parts=None, i=None):

From d02b05c89d8a4733ca683f5ecc25d64cabf3ff8f Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Thu, 25 Aug 2016 01:15:32 -0400
Subject: [PATCH 109/190] make test windows friendly

---
 tests/test_builtins.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_builtins.py b/tests/test_builtins.py
index 6fe171a5e..ad90bdc3b 100644
--- a/tests/test_builtins.py
+++ b/tests/test_builtins.py
@@ -124,5 +124,7 @@ def test_list_of_strs_or_callables(exp, inp):
     '~/one:~/place:~/yo',
     ])
 def test_expand_path(s, home_env):
+    if os.sep != '/':
+        s = s.replace('/', os.sep)
     assert expand_path(s) == s.replace('~', HOME_PATH)
 

From d0804650f39f6c6488cf0349810d6c1e7e79d69d Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Thu, 25 Aug 2016 01:21:17 -0400
Subject: [PATCH 110/190] more make test windows friendly

---
 tests/test_builtins.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_builtins.py b/tests/test_builtins.py
index ad90bdc3b..829e7eb31 100644
--- a/tests/test_builtins.py
+++ b/tests/test_builtins.py
@@ -126,5 +126,7 @@ def test_list_of_strs_or_callables(exp, inp):
 def test_expand_path(s, home_env):
     if os.sep != '/':
         s = s.replace('/', os.sep)
+    if os.pathsep != ':':
+        s = s.replace(':', os.pathsep)
     assert expand_path(s) == s.replace('~', HOME_PATH)
 

From d47ac71e6b93a3e902bb19ec4b88e9965fc8b977 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Thu, 25 Aug 2016 09:20:16 -0400
Subject: [PATCH 111/190] only show rightmost completion in nested attrs

only prompt_toolkit so far
---
 xonsh/completers/python.py | 3 ++-
 xonsh/ptk/completer.py     | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/xonsh/completers/python.py b/xonsh/completers/python.py
index 7c1dfc161..16a8698a3 100644
--- a/xonsh/completers/python.py
+++ b/xonsh/completers/python.py
@@ -106,7 +106,8 @@ def attr_complete(prefix, ctx, filter_func):
             rpl = opt
         # note that prefix[:prelen-len(attr)] != prefix[:-len(attr)]
         # when len(attr) == 0.
-        comp = prefix[:prelen - len(attr)] + rpl
+    #    comp = prefix[:prelen - len(attr)] + rpl
+        comp = rpl
         attrs.add(comp)
     return attrs
 
diff --git a/xonsh/ptk/completer.py b/xonsh/ptk/completer.py
index da135baa7..266984d66 100644
--- a/xonsh/ptk/completer.py
+++ b/xonsh/ptk/completer.py
@@ -41,6 +41,8 @@ class PromptToolkitCompleter(Completer):
                 elif len(os.path.commonprefix(completions)) <= len(prefix):
                     self.reserve_space()
                 for comp in completions:
+                    prefix = prefix.rsplit('.', 1)[-1]
+                    l = len(prefix) if prefix in comp else 0
                     yield Completion(comp, -l)
 
     def reserve_space(self):

From fbe2f1c60f5f324dd348d3fc8a9995896f8348b4 Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Thu, 25 Aug 2016 22:24:43 +0800
Subject: [PATCH 112/190] rename `COMPLETION_CONFIRM` -> `COMPLETIONS_CONFIRM`

---
 news/completion-confirm.rst | 2 +-
 xonsh/environ.py            | 6 +++---
 xonsh/ptk/key_bindings.py   | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/news/completion-confirm.rst b/news/completion-confirm.rst
index 14f6d4fa3..cb69a36ca 100644
--- a/news/completion-confirm.rst
+++ b/news/completion-confirm.rst
@@ -1,6 +1,6 @@
 **Added:**
 
-* New option ``COMPLETION_CONFIRM``. When set, ``<Enter>`` is used to confirm
+* New option ``COMPLETIONS_CONFIRM``. When set, ``<Enter>`` is used to confirm
   completion instead of running command while completion menu is displayed.
 
 **Changed:** None
diff --git a/xonsh/environ.py b/xonsh/environ.py
index e44c5d354..f5a17285f 100644
--- a/xonsh/environ.py
+++ b/xonsh/environ.py
@@ -102,7 +102,7 @@ def DEFAULT_ENSURERS():
     'COLOR_INPUT': (is_bool, to_bool, bool_to_str),
     'COLOR_RESULTS': (is_bool, to_bool, bool_to_str),
     'COMPLETIONS_BRACKETS': (is_bool, to_bool, bool_to_str),
-    'COMPLETION_CONFIRM': (is_bool, to_bool, bool_to_str),
+    'COMPLETIONS_CONFIRM': (is_bool, to_bool, bool_to_str),
     'COMPLETIONS_DISPLAY': (is_completions_display_value,
                             to_completions_display_value, str),
     'COMPLETIONS_MENU_ROWS': (is_int, int, str),
@@ -247,7 +247,7 @@ def DEFAULT_VALUES():
         'COLOR_INPUT': True,
         'COLOR_RESULTS': True,
         'COMPLETIONS_BRACKETS': True,
-        'COMPLETION_CONFIRM': False,
+        'COMPLETIONS_CONFIRM': False,
         'COMPLETIONS_DISPLAY': 'multi',
         'COMPLETIONS_MENU_ROWS': 5,
         'DIRSTACK_SIZE': 20,
@@ -401,7 +401,7 @@ def DEFAULT_DOCS():
         "writing \"$COMPLETIONS_DISPLAY = None\" and \"$COMPLETIONS_DISPLAY "
         "= 'none'\" are equivalent. Only usable with "
         "$SHELL_TYPE=prompt_toolkit"),
-    'COMPLETION_CONFIRM': VarDocs(
+    'COMPLETIONS_CONFIRM': VarDocs(
         'While tab-completions menu is displayed, press <Enter> to confirm '
         'completion instead of running command. This only affects the '
         'prompt-toolkit shell.'),
diff --git a/xonsh/ptk/key_bindings.py b/xonsh/ptk/key_bindings.py
index 19969b847..b692ea5ea 100644
--- a/xonsh/ptk/key_bindings.py
+++ b/xonsh/ptk/key_bindings.py
@@ -112,7 +112,7 @@ class ShouldConfirmCompletion(Filter):
     Check if completion needs confirmation
     """
     def __call__(self, cli):
-        return (builtins.__xonsh_env__.get('COMPLETION_CONFIRM')
+        return (builtins.__xonsh_env__.get('COMPLETIONS_CONFIRM')
                 and cli.current_buffer.complete_state)
 
 

From 4672438eed411947273237b45f4363537a11226b Mon Sep 17 00:00:00 2001
From: Cody Scott <cody.j.b.scott@gmail.com>
Date: Thu, 25 Aug 2016 12:32:15 -0400
Subject: [PATCH 113/190] Adding prompt_ret_code to xontribs

---
 xonsh/xontribs.json         |  6 +++---
 xontrib/prompt_ret_code.xsh | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 3 deletions(-)
 create mode 100644 xontrib/prompt_ret_code.xsh

diff --git a/xonsh/xontribs.json b/xonsh/xontribs.json
index f7a6db42c..758d1e1b0 100644
--- a/xonsh/xontribs.json
+++ b/xonsh/xontribs.json
@@ -27,9 +27,9 @@
   "url": "http://xon.sh",
   "description": ["Python virtual environment manager for xonsh."]
   },
- {"name": "prompt_ret_code",
-  "package": "xontrib-prompt-ret-code",
-  "url": "https://github.com/Siecje/xontrib-prompt-ret-code",
+{"name": "prompt_ret_code",
+  "package": "xonsh",
+  "url": "http://xon.sh",
   "description": ["Adds return code info to the prompt"]
   },
  {"name": "xo",
diff --git a/xontrib/prompt_ret_code.xsh b/xontrib/prompt_ret_code.xsh
new file mode 100644
index 000000000..039f7d25a
--- /dev/null
+++ b/xontrib/prompt_ret_code.xsh
@@ -0,0 +1,35 @@
+from xonsh.tools import ON_WINDOWS as _ON_WINDOWS
+
+
+def _ret_code_color():
+    if __xonsh_history__.rtns:
+        color = 'blue' if __xonsh_history__.rtns[-1] == 0 else 'red'
+    else:
+        color = 'blue'
+    if _ON_WINDOWS:
+        if color == 'blue':
+            return '{BOLD_INTENSE_CYAN}'
+        elif color == 'red':
+            return '{BOLD_INTENSE_RED}'
+    else:
+        if color == 'blue':
+            return '{BOLD_BLUE}'
+        elif color == 'red':
+            return '{BOLD_RED}'
+
+
+def _ret_code():
+    if __xonsh_history__.rtns:
+        return_code = __xonsh_history__.rtns[-1]
+        if return_code != 0:
+            return '[{}]'.format(return_code)
+    return ''
+
+
+$PROMPT = $PROMPT.replace('{prompt_end}{NO_COLOR}',
+                          '{ret_code_color}{ret_code}{prompt_end}{NO_COLOR}')
+
+
+$FORMATTER_DICT['ret_code_color'] = _ret_code_color
+$FORMATTER_DICT['ret_code'] = _ret_code
+

From 7854fb0526c31bfe4242712747adb40299b31760 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Thu, 25 Aug 2016 16:05:53 -0400
Subject: [PATCH 114/190] use rpartition to split up prefix

---
 xonsh/completers/python.py | 3 +--
 xonsh/ptk/completer.py     | 6 ++++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/xonsh/completers/python.py b/xonsh/completers/python.py
index 16a8698a3..7c1dfc161 100644
--- a/xonsh/completers/python.py
+++ b/xonsh/completers/python.py
@@ -106,8 +106,7 @@ def attr_complete(prefix, ctx, filter_func):
             rpl = opt
         # note that prefix[:prelen-len(attr)] != prefix[:-len(attr)]
         # when len(attr) == 0.
-    #    comp = prefix[:prelen - len(attr)] + rpl
-        comp = rpl
+        comp = prefix[:prelen - len(attr)] + rpl
         attrs.add(comp)
     return attrs
 
diff --git a/xonsh/ptk/completer.py b/xonsh/ptk/completer.py
index 266984d66..1fb2e4b38 100644
--- a/xonsh/ptk/completer.py
+++ b/xonsh/ptk/completer.py
@@ -40,9 +40,11 @@ class PromptToolkitCompleter(Completer):
                     pass
                 elif len(os.path.commonprefix(completions)) <= len(prefix):
                     self.reserve_space()
+                prefix, _, compprefix = prefix.rpartition('.')
                 for comp in completions:
-                    prefix = prefix.rsplit('.', 1)[-1]
-                    l = len(prefix) if prefix in comp else 0
+                    if comp.rsplit('.', 1)[0] in prefix:
+                        comp = comp.rsplit('.', 1)[-1]
+                    l = len(compprefix) if compprefix in comp else 0
                     yield Completion(comp, -l)
 
     def reserve_space(self):

From 7aef01eea0d03e419e28c3e4fd24f7c3f7cb0741 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Fri, 26 Aug 2016 09:36:17 -0400
Subject: [PATCH 115/190] remove pacman completion xontrib

This xontrib is no longer needed thanks to improvements in
bash-completion imports.  Yay!  One less thing to maintain!
---
 xonsh/xontribs.json | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/xonsh/xontribs.json b/xonsh/xontribs.json
index 758d1e1b0..f2d1bb4e7 100644
--- a/xonsh/xontribs.json
+++ b/xonsh/xontribs.json
@@ -50,11 +50,6 @@
   "url": "https://github.com/xsteadfastx/xonsh-docker-tabcomplete",
   "description": ["Adds tabcomplete functionality to docker inside of xonsh."]
  },
- {"name": "pacman_tabcomplete",
-  "package": "xonsh-pacman-tabcomplete",
-  "url": "https://github.com/gforsyth/xonsh-pacman-tabcomplete",
-  "description": ["Adds tabcomplete functionality to pacman inside of xonsh."]
- },
  {"name": "scrapy_tabcomplete",
   "package": "xonsh-scrapy-tabcomplete",
   "url": "https://github.com/Granitas/xonsh-scrapy-tabcomplete",
@@ -125,13 +120,6 @@
     "pip": "pip install xonsh-docker-tabcomplete"
    }
   },
-  "xonsh-pacman-tabcomplete": {
-   "license": "MIT",
-   "url": "https://github.com/gforsyth/xonsh-pacman-tabcomplete",
-   "install": {
-    "pip": "pip install xonsh-pacman-tabcomplete"
-   }
-   },
   "xonsh-scrapy-tabcomplete": {
    "license": "GPLv3",
    "url": "https://github.com/Granitas/xonsh-scrapy-tabcomplete",

From e49c37e67c22c1eea81bbff2e584d1b1fe3306b4 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Fri, 26 Aug 2016 17:46:48 -0400
Subject: [PATCH 116/190] flake8

---
 xonsh/events.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xonsh/events.py b/xonsh/events.py
index 4ae81c316..d8d527f98 100644
--- a/xonsh/events.py
+++ b/xonsh/events.py
@@ -127,7 +127,7 @@ class Event(AbstractEvent):
         Returns
         -------
         vals : iterable
-            Return values of each handler. If multiple handlers return the same value, it will 
+            Return values of each handler. If multiple handlers return the same value, it will
             appear multiple times.
         """
         vals = []

From 3bf460687a340975ed9f5394a702bcdbc03d6bfe Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Sat, 27 Aug 2016 03:01:18 +0300
Subject: [PATCH 117/190] __file__

---
 news/module-file-attr.rst | 15 +++++++++++++++
 xonsh/imphooks.py         | 16 +++++++++++++++-
 2 files changed, 30 insertions(+), 1 deletion(-)
 create mode 100644 news/module-file-attr.rst

diff --git a/news/module-file-attr.rst b/news/module-file-attr.rst
new file mode 100644
index 000000000..93a49c147
--- /dev/null
+++ b/news/module-file-attr.rst
@@ -0,0 +1,15 @@
+**Added:** None
+
+**Changed:**
+
+* ``create_module`` implementation on XonshImportHook
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:**
+
+* xonsh modules imported now have the __file__ attribute
+
+**Security:** None
diff --git a/xonsh/imphooks.py b/xonsh/imphooks.py
index 4277ebd9f..52b65c7c3 100644
--- a/xonsh/imphooks.py
+++ b/xonsh/imphooks.py
@@ -3,9 +3,10 @@
 
 This module registers the hooks it defines when it is imported.
 """
+import builtins
 import os
 import sys
-import builtins
+import types
 from importlib.machinery import ModuleSpec
 from importlib.abc import MetaPathFinder, SourceLoader
 
@@ -13,6 +14,9 @@ from xonsh.execer import Execer
 from xonsh.platform import scandir
 
 
+class XonshModule(types.ModuleType):
+    pass
+
 class XonshImportHook(MetaPathFinder, SourceLoader):
     """Implements the import hook for xonsh source files."""
 
@@ -60,6 +64,16 @@ class XonshImportHook(MetaPathFinder, SourceLoader):
     #
     # SourceLoader methods
     #
+    def create_module(self, spec):
+        """Create a xonsh module with the appropriate attributes."""
+        name = spec.name
+        parent = spec.parent
+        mod = XonshModule(name)
+        mod.__file__ = self.get_filename(name)
+        mod.__loader__ = self
+        mod.__package__ = parent or ''
+        return mod
+
     def get_filename(self, fullname):
         """Returns the filename for a module's fullname."""
         return self._filenames[fullname]

From b11a25a6a6bcddc4a30662003819c926215f9633 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 02:55:40 -0400
Subject: [PATCH 118/190] Use print_exception()

---
 xonsh/events.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/xonsh/events.py b/xonsh/events.py
index d8d527f98..ced0f531c 100644
--- a/xonsh/events.py
+++ b/xonsh/events.py
@@ -9,8 +9,8 @@ The best way to "declare" an event is something like::
 """
 import abc
 import collections.abc
-import traceback
-import sys
+
+from xonsh.tools import print_exception
 
 
 class AbstractEvent(collections.abc.MutableSet, abc.ABC):
@@ -135,9 +135,7 @@ class Event(AbstractEvent):
             try:
                 rv = handler(*pargs, **kwargs)
             except Exception:
-                print("Exception raised in event handler; ignored.", file=sys.stderr)
-                traceback.print_exc()
-                # FIXME: Actually warn
+                print_exception("Exception raised in event handler; ignored.")
             else:
                 vals.append(rv)
         return vals
@@ -244,4 +242,7 @@ class EventManager:
         return e
 
 
+# Not lazy because:
+# 1. Initialization of EventManager can't be much cheaper
+# 2. It's expected to be used at load time, negating any benefits of using lazy object
 events = EventManager()

From bbc965346b8744843cd6746692477290baf5c999 Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Sat, 27 Aug 2016 13:51:45 +0300
Subject: [PATCH 119/190] cleanup/tests

---
 tests/test_imphooks.py | 16 ++++++++++++++++
 xonsh/imphooks.py      | 19 ++++++++-----------
 2 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/tests/test_imphooks.py b/tests/test_imphooks.py
index eb519db7d..f8be37fae 100644
--- a/tests/test_imphooks.py
+++ b/tests/test_imphooks.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 """Testing xonsh import hooks"""
+import os
 import pytest
 
 from xonsh import imphooks
@@ -38,3 +39,18 @@ def test_relative_import():
 def test_sub_import():
     from xpack.sub import sample
     assert ('hello mom jawaka\n' == sample.x)
+
+
+TEST_DIR = os.path.dirname(__file__)
+
+
+def test_module_dunder_file_attribute():
+    import sample
+    exp = os.path.join(TEST_DIR, 'sample.xsh')
+    assert sample.__file__ == exp
+
+
+def test_module_dunder_file_attribute_sub():
+    from xpack.sub import sample
+    exp = os.path.join(TEST_DIR, 'xpack', 'sub', 'sample.xsh')
+    assert sample.__file__ == exp
diff --git a/xonsh/imphooks.py b/xonsh/imphooks.py
index 52b65c7c3..1099b7f40 100644
--- a/xonsh/imphooks.py
+++ b/xonsh/imphooks.py
@@ -14,9 +14,6 @@ from xonsh.execer import Execer
 from xonsh.platform import scandir
 
 
-class XonshModule(types.ModuleType):
-    pass
-
 class XonshImportHook(MetaPathFinder, SourceLoader):
     """Implements the import hook for xonsh source files."""
 
@@ -66,12 +63,10 @@ class XonshImportHook(MetaPathFinder, SourceLoader):
     #
     def create_module(self, spec):
         """Create a xonsh module with the appropriate attributes."""
-        name = spec.name
-        parent = spec.parent
-        mod = XonshModule(name)
-        mod.__file__ = self.get_filename(name)
+        mod = types.ModuleType(spec.name)
+        mod.__file__ = self.get_filename(spec.name)
         mod.__loader__ = self
-        mod.__package__ = parent or ''
+        mod.__package__ = spec.parent or ''
         return mod
 
     def get_filename(self, fullname):
@@ -84,7 +79,7 @@ class XonshImportHook(MetaPathFinder, SourceLoader):
 
     def get_code(self, fullname):
         """Gets the code object for a xonsh file."""
-        filename = self._filenames.get(fullname, None)
+        filename = self.get_filename(fullname)
         if filename is None:
             msg = "xonsh file {0!r} could not be found".format(fullname)
             raise ImportError(msg)
@@ -106,6 +101,8 @@ def install_hook():
     Can safely be called many times, will be no-op if a xonsh import hook is
     already present.
     """
-
-    if XonshImportHook not in {type(hook) for hook in sys.meta_path}:
+    for hook in sys.meta_path:
+        if isinstance(hook, XonshImportHook):
+            break
+    else:
         sys.meta_path.append(XonshImportHook())

From 614687aad9c05ed0b78706e941e15a1404b660c8 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Sat, 27 Aug 2016 08:30:48 -0400
Subject: [PATCH 120/190] fix for envvar expansion completion in ptk

---
 xonsh/ptk/completer.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/xonsh/ptk/completer.py b/xonsh/ptk/completer.py
index 1fb2e4b38..f4012e812 100644
--- a/xonsh/ptk/completer.py
+++ b/xonsh/ptk/completer.py
@@ -40,12 +40,17 @@ class PromptToolkitCompleter(Completer):
                     pass
                 elif len(os.path.commonprefix(completions)) <= len(prefix):
                     self.reserve_space()
-                prefix, _, compprefix = prefix.rpartition('.')
-                for comp in completions:
-                    if comp.rsplit('.', 1)[0] in prefix:
-                        comp = comp.rsplit('.', 1)[-1]
-                    l = len(compprefix) if compprefix in comp else 0
-                    yield Completion(comp, -l)
+                # don't mess with envvar and path expansion comps
+                if any(x in prefix for x in ['$', '/']):
+                    for comp in completions:
+                        yield Completion(comp, -l)
+                else:  # don't show common prefixes in attr completions
+                    prefix, _, compprefix = prefix.rpartition('.')
+                    for comp in completions:
+                        if comp.rsplit('.', 1)[0] in prefix:
+                            comp = comp.rsplit('.', 1)[-1]
+                        l = len(compprefix) if compprefix in comp else 0
+                        yield Completion(comp, -l)
 
     def reserve_space(self):
         cli = builtins.__xonsh_shell__.shell.prompter.cli

From aea5740ed63cd545d86d0fb4d77c39f768db3a9e Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 27 Aug 2016 10:18:07 -0400
Subject: [PATCH 121/190] amalgam events

---
 xonsh/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/xonsh/__init__.py b/xonsh/__init__.py
index a1cdb871f..ccb123b7f 100644
--- a/xonsh/__init__.py
+++ b/xonsh/__init__.py
@@ -45,6 +45,8 @@ else:
         _sys.modules['xonsh.diff_history'] = __amalgam__
         dirstack = __amalgam__
         _sys.modules['xonsh.dirstack'] = __amalgam__
+        events = __amalgam__
+        _sys.modules['xonsh.events'] = __amalgam__
         foreign_shells = __amalgam__
         _sys.modules['xonsh.foreign_shells'] = __amalgam__
         lexer = __amalgam__

From e7540488f7caf0a890363bf233182c5251172af4 Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Sat, 27 Aug 2016 17:30:53 +0300
Subject: [PATCH 122/190] history magic

---
 news/history-api.rst  | 17 +++++++++++
 tests/test_history.py | 36 +++++++++++++++++-------
 xonsh/history.py      | 65 +++++++++++++++++++++++++++++++++----------
 3 files changed, 94 insertions(+), 24 deletions(-)
 create mode 100644 news/history-api.rst

diff --git a/news/history-api.rst b/news/history-api.rst
new file mode 100644
index 000000000..849d72a55
--- /dev/null
+++ b/news/history-api.rst
@@ -0,0 +1,17 @@
+**Added:**
+
+* ``History`` methods ``_get`` and ``__getitem__``
+
+**Changed:**
+
+* ``_curr_session_parser`` now uses ``History_.get``
+
+**Deprecated:** None
+
+**Removed:**
+
+* ``History`` method ``show``
+
+**Fixed:** None
+
+**Security:** None
diff --git a/tests/test_history.py b/tests/test_history.py
index 60f93b51e..bc9bd1a81 100644
--- a/tests/test_history.py
+++ b/tests/test_history.py
@@ -68,24 +68,24 @@ def test_cmd_field(hist, xonsh_builtins):
     assert None == hist.outs[-1]
 
 
-cmds = ['ls', 'cat hello kitty', 'abc', 'def', 'touch me', 'grep from me']
+CMDS = ['ls', 'cat hello kitty', 'abc', 'def', 'touch me', 'grep from me']
 
 @pytest.mark.parametrize('inp, commands, offset', [
-    ('', cmds, (0, 1)),
-    ('-r', list(reversed(cmds)), (len(cmds)- 1, -1)),
-    ('0', cmds[0:1], (0, 1)),
-    ('1', cmds[1:2], (1, 1)),
-    ('-2', cmds[-2:-1], (len(cmds) -2 , 1)),
-    ('1:3', cmds[1:3], (1, 1)),
-    ('1::2', cmds[1::2], (1, 2)),
-    ('-4:-2', cmds[-4:-2], (len(cmds) - 4, 1))
+    ('', CMDS, (0, 1)),
+    ('-r', list(reversed(CMDS)), (len(CMDS)- 1, -1)),
+    ('0', CMDS[0:1], (0, 1)),
+    ('1', CMDS[1:2], (1, 1)),
+    ('-2', CMDS[-2:-1], (len(CMDS) -2 , 1)),
+    ('1:3', CMDS[1:3], (1, 1)),
+    ('1::2', CMDS[1::2], (1, 2)),
+    ('-4:-2', CMDS[-4:-2], (len(CMDS) - 4, 1))
     ])
 def test_show_cmd_numerate(inp, commands, offset, hist, xonsh_builtins, capsys):
     """Verify that CLI history commands work."""
     base_idx, step = offset
     xonsh_builtins.__xonsh_history__ = hist
     xonsh_builtins.__xonsh_env__['HISTCONTROL'] = set()
-    for ts,cmd in enumerate(cmds):  # populate the shell history
+    for ts,cmd in enumerate(CMDS):  # populate the shell history
         hist.append({'inp': cmd, 'rtn': 0, 'ts':(ts+1, ts+1.5)})
 
     exp = ('{}: {}'.format(base_idx + idx * step, cmd)
@@ -185,3 +185,19 @@ def test_parser_show(args, exp):
               'timestamp': False}
     ns = _hist_parse_args(shlex.split(args))
     assert ns.__dict__ == exp_ns
+
+
+# CMDS = ['ls', 'cat hello kitty', 'abc', 'def', 'touch me', 'grep from me']
+
+def test_history_getitem(hist, xonsh_builtins):
+    xonsh_builtins.__xonsh_env__['HISTCONTROL'] = set()
+    for ts,cmd in enumerate(CMDS):  # populate the shell history
+        hist.append({'inp': cmd, 'rtn': 0, 'ts':(ts+1, ts+1.5)})
+
+    # indexing
+    assert hist[-1] == 'grep from me'
+    assert hist['hello'] == 'cat hello kitty'
+
+    # word parts
+    assert hist[-1, -1] == 'me'
+    assert hist['hello', 1] == 'hello'
diff --git a/xonsh/history.py b/xonsh/history.py
index 640c6abbe..46b5671d5 100644
--- a/xonsh/history.py
+++ b/xonsh/history.py
@@ -8,6 +8,7 @@ import time
 import uuid
 import argparse
 import builtins
+import collections
 import datetime
 import functools
 import itertools
@@ -262,7 +263,7 @@ def _all_xonsh_parser(**kwargs):
     """
     Returns all history as found in XONSH_DATA_DIR.
 
-    return format: (name, start_time, index)
+    return format: (cmd, start_time, index)
     """
     data_dir = builtins.__xonsh_env__.get('XONSH_DATA_DIR')
     data_dir = expanduser_abs_path(data_dir)
@@ -285,14 +286,11 @@ def _all_xonsh_parser(**kwargs):
 def _curr_session_parser(hist=None, **kwargs):
     """
     Take in History object and return command list tuple with
-    format: (name, start_time, index)
+    format: (cmd, start_time, index)
     """
     if hist is None:
         hist = builtins.__xonsh_history__
-    start_times = (start for start, end in hist.tss)
-    names = (name.rstrip() for name in hist.inps)
-    for ind, (c, t) in enumerate(zip(names, start_times)):
-        yield (c, t, ind)
+    return hist._get()
 
 
 def _zsh_hist_parser(location=None, **kwargs):
@@ -488,6 +486,18 @@ def _hist_show(ns, *args, **kwargs):
 class History(object):
     """Xonsh session history.
 
+    History object supports indexing with some rules for extra functionality:
+
+        - index must be one of string, int or tuple of length two
+        - if the index is an int the appropriate command in order is returned
+        - if the index is a string the last command that contains
+          the string is returned
+        - if the index is a tuple:
+
+            - the first item follows the previous
+              two rules.
+            - the second item is the slice of the arguments to be returned
+
     Attributes
     ----------
     rtns : sequence of ints
@@ -605,16 +615,43 @@ class History(object):
         self.buffer.clear()
         return hf
 
-    def show(self, *args, **kwargs):
-        """Return shell history as a list
+    def _get(self):
+        """Get current session history.
 
-        Valid options:
-            `session` - returns xonsh history from current session
-            `xonsh`   - returns xonsh history from all sessions
-            `zsh`     - returns all zsh history
-            `bash`    - returns all bash history
+        Yields
+        ------
+        tuple
+            ``tuple`` of the form (cmd, start_time, index).
         """
-        return list(_hist_get(*args, **kwargs))
+        start_times = (start for start, end in self.tss)
+        names = (name.rstrip() for name in self.inps)
+        for ind, (c, t) in enumerate(zip(names, start_times)):
+            yield (c, t, ind)
+
+    def __getitem__(self, item):
+        # accept only one of str, int, tuple of length two
+        if isinstance(item, tuple):
+            pattern, part = item
+        else:
+            pattern, part = item, None
+        # find command
+        hist = [c for c, *_ in self._get()]
+        command = None
+        if isinstance(pattern, str):
+            for command in reversed(hist):
+                if pattern in command:
+                    break
+        elif isinstance(pattern, int):
+            # catch index error?
+            command = hist[pattern]
+        else:
+            raise TypeError('history index must be of type '
+                            'str, int, tuple of length two')
+        # get command part
+        if command and part:
+            part = ensure_slice(part)
+            command = ' '.join(command.split()[part])
+        return command
 
 
 def _hist_info(ns, hist):

From 699d6ae7143bf7c9725f8ebcd1ea94faab9a13f0 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 27 Aug 2016 11:05:21 -0400
Subject: [PATCH 123/190] some clean up

---
 xonsh/built_ins.py    |  3 ++-
 xonsh/parsers/base.py | 57 ++++++++++++++++++++++++++-----------------
 2 files changed, 37 insertions(+), 23 deletions(-)

diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py
index ca11db4ef..6b827a134 100644
--- a/xonsh/built_ins.py
+++ b/xonsh/built_ins.py
@@ -736,7 +736,7 @@ def convert_macro_arg(raw_arg, kind, glbs, locs, *, name='<arg>',
     """
     # munge kind and mode to start
     mode = None
-    if isinstance(kind, abc.Sequence) and not isinstance(kind, str):
+    if isinstance(kind, cabc.Sequence) and not isinstance(kind, str):
         # have (kind, mode) tuple
         kind, mode = kind
     if isinstance(kind, str):
@@ -823,6 +823,7 @@ def call_macro(f, raw_args, glbs, locs):
     empty = inspect.Parameter.empty
     macroname = f.__name__
     args = []
+    print(raw_args)
     for (key, param), raw_arg in zip(sig.parameters.items(), raw_args):
         kind = param.annotation
         if kind is empty or kind is None:
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index 01e13dbc7..780a7b78f 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -429,7 +429,7 @@ class BaseParser(object):
         return ''.join(lines)
 
     def _parse_error(self, msg, loc):
-        if self.xonsh_code is None:
+        if self.xonsh_code is None or loc is None:
             err_line_pointer = ''
         else:
             col = loc.column + 1
@@ -1672,11 +1672,13 @@ class BaseParser(object):
                 p0 = ast.Call(func=leader,
                               lineno=leader.lineno,
                               col_offset=leader.col_offset, **trailer)
-            elif isinstance(trailer, ast.Tuple):
+            elif isinstance(trailer, (ast.Tuple, tuple)):
                 # call macro functions
                 l, c = leader.lineno, leader.col_offset
                 gblcall = xonsh_call('globals', [], lineno=l, col=c)
                 loccall = xonsh_call('locals', [], lineno=l, col=c)
+                if isinstance(trailer, tuple):
+                    trailer, arglist = trailer
                 margs = [leader, trailer, gblcall, loccall]
                 p0 = xonsh_call('__xonsh_call_macro__', margs, lineno=l, col=c)
             elif isinstance(trailer, str):
@@ -1860,17 +1862,18 @@ class BaseParser(object):
         p[0] = [p[2] or dict(args=[], keywords=[], starargs=None, kwargs=None)]
 
     def p_trailer_bang_lparen(self, p):
-        """trailer : bang_lparen_tok macroarglist_opt rparen_tok"""
+        """trailer : bang_lparen_tok macroarglist_opt rparen_tok
+                   | bang_lparen_tok nocomma comma_tok rparen_tok
+                   | bang_lparen_tok nocomma comma_tok WS rparen_tok
+                   | bang_lparen_tok macroarglist comma_tok rparen_tok
+                   | bang_lparen_tok macroarglist comma_tok WS rparen_tok
+        """
         p1, p2, p3 = p[1], p[2], p[3]
         begins = [(p1.lineno, p1.lexpos + 2)]
         ends = [(p3.lineno, p3.lexpos)]
         if p2:
-            if p2[-1][-1] == 'trailing':  # handle trailing comma
-                begins.extend([(x[0], x[1] + 1) for x in p2[:-1]])
-                ends = [x[:2] for x in p2]
-            else:
-                begins.extend([(x[0], x[1] + 1) for x in p2])
-                ends = [x[:2] for x in p2] + ends
+            begins.extend([(x[0], x[1] + 1) for x in p2])
+            ends = p2 + ends
         elts = []
         for beg, end in zip(begins, ends):
             s = self.source_slice(beg, end).strip()
@@ -1886,6 +1889,15 @@ class BaseParser(object):
                        col_offset=p1.lexpos)
         p[0] = [p0]
 
+    #def p_trailer_bang_lparen_star(self, p):
+    #    """trailer : bang_lparen_tok macroarglist comma_tok TIMES COMMA arglist rparen_tok"""
+    #    self.p_trailer_bang_lparen(p)
+    #    assert False
+        #if len(p) == 7:
+        #    p[0] = [(p0, p[6])]
+        #else:
+
+
     def p_trailer_p3(self, p):
         """trailer : LBRACKET subscriptlist RBRACKET
                    | PERIOD NAME
@@ -1903,7 +1915,7 @@ class BaseParser(object):
         toks -= {'COMMA', 'LPAREN', 'RPAREN', 'LBRACE', 'RBRACE', 'LBRACKET',
                  'RBRACKET', 'AT_LPAREN', 'BANG_LPAREN', 'BANG_LBRACKET',
                  'DOLLAR_LPAREN', 'DOLLAR_LBRACE', 'DOLLAR_LBRACKET',
-                 'ATDOLLAR_LPAREN'}
+                 'ATDOLLAR_LPAREN', 'TIMES'}
         ts = '\n            | '.join(sorted(toks))
         doc = 'nocomma_tok : ' + ts + '\n'
         self.p_nocomma_tok.__func__.__doc__ = doc
@@ -1938,6 +1950,12 @@ class BaseParser(object):
         """nocomma_part : nocomma_tok"""
         pass
 
+    def p_nocomma_part_times(self, p):
+        """nocomma_part : TIMES nocomma_part
+                        | nocomma_part TIMES
+        """
+        pass
+
     def p_nocomma_part_any(self, p):
         """nocomma_part : LPAREN any_raw_toks_opt RPAREN
                         | LBRACE any_raw_toks_opt RBRACE
@@ -1963,20 +1981,15 @@ class BaseParser(object):
     def p_comma_nocomma(self, p):
         """comma_nocomma : comma_tok nocomma"""
         p1 = p[1]
-        p[0] = [(p1.lineno, p1.lexpos, None)]
+        p[0] = [(p1.lineno, p1.lexpos)]
 
-    def p_comma_trailing_nocomma(self, p):
-        """comma_nocomma : comma_tok
-                         | comma_tok WS
-        """
-        p1 = p[1]
-        p[0] = [(p1.lineno, p1.lexpos, 'trailing')]
+    def p_macroarglist_single(self, p):
+        """macroarglist : nocomma"""
+        p[0] = []
 
-    def p_macroarglist(self, p):
-        """macroarglist : nocomma comma_nocomma_list_opt"""
-        p2 = p[2]
-        pos = [] if p2 is None else p2
-        p[0] = pos
+    def p_macroarglist_many(self, p):
+        """macroarglist : nocomma comma_nocomma_list"""
+        p[0] = p[2]
 
     def p_subscriptlist(self, p):
         """subscriptlist : subscript comma_subscript_list_opt comma_opt"""

From 6990d29ba58ea7c598f23c4dc5efe657f455fa31 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 11:13:56 -0400
Subject: [PATCH 124/190] Basic events newsitem

---
 news/events.rst | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 news/events.rst

diff --git a/news/events.rst b/news/events.rst
new file mode 100644
index 000000000..e8402f878
--- /dev/null
+++ b/news/events.rst
@@ -0,0 +1,13 @@
+**Added:** 
+
+* A new `event subsystem <>`_ has been added.
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

From 866725dd2ba47ac450df9d2fc765f4885c2fd9cc Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 11:24:43 -0400
Subject: [PATCH 125/190] Speculative url

---
 news/events.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/news/events.rst b/news/events.rst
index e8402f878..9fcdf7000 100644
--- a/news/events.rst
+++ b/news/events.rst
@@ -1,6 +1,6 @@
 **Added:** 
 
-* A new `event subsystem <>`_ has been added.
+* A new `event subsystem <http://xon.sh/tutorial_events.html>`_ has been added.
 
 **Changed:** None
 

From a4471a0201bfcbc493d65893c38549ea0851a6d9 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 27 Aug 2016 11:40:52 -0400
Subject: [PATCH 126/190] some macro arg cleanup

---
 tests/test_parser.py  |  1 +
 xonsh/parsers/base.py | 17 +----------------
 2 files changed, 2 insertions(+), 16 deletions(-)

diff --git a/tests/test_parser.py b/tests/test_parser.py
index 08b891fd6..ede47bf95 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -1855,6 +1855,7 @@ def test_macro_call_one_trailing(s):
     assert len(args) == 1
     assert args[0].s == s.strip()
 
+
 @pytest.mark.parametrize('s', MACRO_ARGS)
 def test_macro_call_one_trailing_space(s):
     f = 'f!( {0}, )'.format(s)
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index 780a7b78f..cd8ea272e 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -1889,15 +1889,6 @@ class BaseParser(object):
                        col_offset=p1.lexpos)
         p[0] = [p0]
 
-    #def p_trailer_bang_lparen_star(self, p):
-    #    """trailer : bang_lparen_tok macroarglist comma_tok TIMES COMMA arglist rparen_tok"""
-    #    self.p_trailer_bang_lparen(p)
-    #    assert False
-        #if len(p) == 7:
-        #    p[0] = [(p0, p[6])]
-        #else:
-
-
     def p_trailer_p3(self, p):
         """trailer : LBRACKET subscriptlist RBRACKET
                    | PERIOD NAME
@@ -1915,7 +1906,7 @@ class BaseParser(object):
         toks -= {'COMMA', 'LPAREN', 'RPAREN', 'LBRACE', 'RBRACE', 'LBRACKET',
                  'RBRACKET', 'AT_LPAREN', 'BANG_LPAREN', 'BANG_LBRACKET',
                  'DOLLAR_LPAREN', 'DOLLAR_LBRACE', 'DOLLAR_LBRACKET',
-                 'ATDOLLAR_LPAREN', 'TIMES'}
+                 'ATDOLLAR_LPAREN'}
         ts = '\n            | '.join(sorted(toks))
         doc = 'nocomma_tok : ' + ts + '\n'
         self.p_nocomma_tok.__func__.__doc__ = doc
@@ -1950,12 +1941,6 @@ class BaseParser(object):
         """nocomma_part : nocomma_tok"""
         pass
 
-    def p_nocomma_part_times(self, p):
-        """nocomma_part : TIMES nocomma_part
-                        | nocomma_part TIMES
-        """
-        pass
-
     def p_nocomma_part_any(self, p):
         """nocomma_part : LPAREN any_raw_toks_opt RPAREN
                         | LBRACE any_raw_toks_opt RBRACE

From c2fe714701a35f6a5dc6de591cf481ed34d0d2b0 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 27 Aug 2016 11:42:49 -0400
Subject: [PATCH 127/190] removed unused rules

---
 xonsh/parsers/base.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index cd8ea272e..42715b329 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -237,8 +237,8 @@ class BaseParser(object):
             'op_factor_list', 'trailer_list', 'testlist_comp',
             'yield_expr_or_testlist_comp', 'dictorsetmaker',
             'comma_subscript_list', 'test', 'sliceop', 'comp_iter',
-            'yield_arg', 'test_comma_list', 'comma_nocomma_list',
-            'macroarglist', 'any_raw_toks', 'comma_tok']
+            'yield_arg', 'test_comma_list',
+            'macroarglist', 'any_raw_toks']
         for rule in opt_rules:
             self._opt_rule(rule)
 

From 9ceb8f12925354a0d761d415fd16a8a6d127a4f1 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 11:47:24 -0400
Subject: [PATCH 128/190] Refactor _change_working_directory() slightly to
 prepare for events

---
 xonsh/dirstack.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/xonsh/dirstack.py b/xonsh/dirstack.py
index 8d5ab8a50..060d834ac 100644
--- a/xonsh/dirstack.py
+++ b/xonsh/dirstack.py
@@ -23,18 +23,19 @@ def _change_working_directory(newdir):
     env = builtins.__xonsh_env__
     old = env['PWD']
     new = os.path.join(old, newdir)
+    absnew = os.path.abspath(new)
     try:
-        os.chdir(os.path.abspath(new))
+        os.chdir(absnew)
     except (OSError, FileNotFoundError):
         if new.endswith(get_sep()):
             new = new[:-1]
         if os.path.basename(new) == '..':
             env['PWD'] = new
-        return
-    if old is not None:
-        env['OLDPWD'] = old
-    if new is not None:
-        env['PWD'] = os.path.abspath(new)
+    else:
+        if old is not None:
+            env['OLDPWD'] = old
+        if new is not None:
+            env['PWD'] = absnew
 
 
 def _try_cdpath(apath):

From 573e78372f0ec4676d9088e7fbb352f93489a343 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 12:06:16 -0400
Subject: [PATCH 129/190] Add `on_chdir` event and test.

Test is, of course, failing
---
 tests/conftest.py      |  3 +++
 tests/test_dirstack.py | 21 +++++++++++++++++++++
 xonsh/dirstack.py      | 12 ++++++++++++
 3 files changed, 36 insertions(+)

diff --git a/tests/conftest.py b/tests/conftest.py
index 297690ecd..f15e4eccc 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -5,6 +5,7 @@ import xonsh.built_ins
 from xonsh.built_ins import ensure_list_of_strs
 from xonsh.execer import Execer
 from xonsh.tools import XonshBlockError
+from xonsh.events import EventManager
 import glob
 
 
@@ -38,6 +39,7 @@ def xonsh_builtins():
     builtins.execx = None
     builtins.compilex = None
     builtins.aliases = {}
+    builtins.events = EventManager()
     yield builtins
     del builtins.__xonsh_env__
     del builtins.__xonsh_ctx__
@@ -56,3 +58,4 @@ def xonsh_builtins():
     del builtins.execx
     del builtins.compilex
     del builtins.aliases
+    del builtins.events
diff --git a/tests/test_dirstack.py b/tests/test_dirstack.py
index 7e6482b24..306be03b1 100644
--- a/tests/test_dirstack.py
+++ b/tests/test_dirstack.py
@@ -66,3 +66,24 @@ def test_cdpath_expansion(xonsh_builtins):
     except Exception as e:
         tuple(os.rmdir(_) for _ in test_dirs if os.path.exists(_))
         raise e
+
+def test_cdpath_events(xonsh_builtins, tmpdir):
+    xonsh_builtins.__xonsh_env__ = Env(CDPATH=PARENT, PWD=HERE)
+    target = str(tmpdir)
+
+    ev = None
+    @xonsh_builtins.events.on_chdir
+    def handler(old, new):
+        nonlocal ev
+        ev = old, new
+
+    old_dir = os.getcwd()
+    try:
+        dirstack.cd([target])
+    except:
+        raise
+    else:
+        assert (old_dir, target) == ev
+    finally:
+        # Use os.chdir() here so dirstack.cd() doesn't fire events (or fail again)
+        os.chdir(old_dir)
diff --git a/xonsh/dirstack.py b/xonsh/dirstack.py
index 060d834ac..cc5685b35 100644
--- a/xonsh/dirstack.py
+++ b/xonsh/dirstack.py
@@ -7,11 +7,19 @@ import builtins
 
 from xonsh.lazyasd import lazyobject
 from xonsh.tools import get_sep
+from xonsh.events import events
 
 DIRSTACK = []
 """A list containing the currently remembered directories."""
 
 
+events.doc('on_chdir', """
+on_chdir(olddir: str, newdir: str) -> None
+
+Fires when the current directory is changed for any reason.
+""")
+
+
 def _get_cwd():
     try:
         return os.getcwd()
@@ -37,6 +45,10 @@ def _change_working_directory(newdir):
         if new is not None:
             env['PWD'] = absnew
 
+    # Fire event if the path actually changed
+    if old != env['PWD']:
+        events.on_chdir.fire(old, env['PWD'])
+
 
 def _try_cdpath(apath):
     # NOTE: this CDPATH implementation differs from the bash one.

From ab6501748d0bc5296082c3d5e7aa42194a1c76bc Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 12:21:15 -0400
Subject: [PATCH 130/190] Use singleton instance of EventManager

---
 tests/conftest.py      | 5 +++--
 tests/test_dirstack.py | 3 ++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index f15e4eccc..5c55bdc47 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -5,7 +5,7 @@ import xonsh.built_ins
 from xonsh.built_ins import ensure_list_of_strs
 from xonsh.execer import Execer
 from xonsh.tools import XonshBlockError
-from xonsh.events import EventManager
+from xonsh.events import events
 import glob
 
 
@@ -39,7 +39,8 @@ def xonsh_builtins():
     builtins.execx = None
     builtins.compilex = None
     builtins.aliases = {}
-    builtins.events = EventManager()
+    # Unlike all the other stuff, this has to refer to the "real" one because all modules that would be firing events on the global instance.
+    builtins.events = events
     yield builtins
     del builtins.__xonsh_env__
     del builtins.__xonsh_ctx__
diff --git a/tests/test_dirstack.py b/tests/test_dirstack.py
index 306be03b1..20ded0768 100644
--- a/tests/test_dirstack.py
+++ b/tests/test_dirstack.py
@@ -68,7 +68,7 @@ def test_cdpath_expansion(xonsh_builtins):
         raise e
 
 def test_cdpath_events(xonsh_builtins, tmpdir):
-    xonsh_builtins.__xonsh_env__ = Env(CDPATH=PARENT, PWD=HERE)
+    xonsh_builtins.__xonsh_env__ = Env(CDPATH=PARENT, PWD=os.getcwd())
     target = str(tmpdir)
 
     ev = None
@@ -77,6 +77,7 @@ def test_cdpath_events(xonsh_builtins, tmpdir):
         nonlocal ev
         ev = old, new
 
+
     old_dir = os.getcwd()
     try:
         dirstack.cd([target])

From 314269f7ce4b7c84864c15936052fc7e8062b948 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 27 Aug 2016 12:55:19 -0400
Subject: [PATCH 131/190] added * breaking

---
 tests/test_imphooks.py |  4 ++--
 xonsh/built_ins.py     | 42 ++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/tests/test_imphooks.py b/tests/test_imphooks.py
index f8be37fae..b53a0dd9b 100644
--- a/tests/test_imphooks.py
+++ b/tests/test_imphooks.py
@@ -47,10 +47,10 @@ TEST_DIR = os.path.dirname(__file__)
 def test_module_dunder_file_attribute():
     import sample
     exp = os.path.join(TEST_DIR, 'sample.xsh')
-    assert sample.__file__ == exp
+    assert os.path.abspath(sample.__file__) == exp
 
 
 def test_module_dunder_file_attribute_sub():
     from xpack.sub import sample
     exp = os.path.join(TEST_DIR, 'xpack', 'sub', 'sample.xsh')
-    assert sample.__file__ == exp
+    assert os.path.abspath(sample.__file__) == exp
diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py
index 6b827a134..a1cd58250 100644
--- a/xonsh/built_ins.py
+++ b/xonsh/built_ins.py
@@ -15,6 +15,7 @@ import atexit
 import inspect
 import tempfile
 import builtins
+import itertools
 import subprocess
 import contextlib
 import collections.abc as cabc
@@ -822,20 +823,57 @@ def call_macro(f, raw_args, glbs, locs):
     sig = inspect.signature(f)
     empty = inspect.Parameter.empty
     macroname = f.__name__
+    i = 0
     args = []
-    print(raw_args)
     for (key, param), raw_arg in zip(sig.parameters.items(), raw_args):
+        i += 1
+        if raw_arg == '*':
+            break
         kind = param.annotation
         if kind is empty or kind is None:
             kind = eval
         arg = convert_macro_arg(raw_arg, kind, glbs, locs, name=key,
                                 macroname=macroname)
         args.append(arg)
+    reg_args, kwargs = _eval_regular_args(raw_args[i:], glbs, locs)
+    args += reg_args
     with macro_context(f, glbs, locs):
-        rtn = f(*args)
+        rtn = f(*args, **kwargs)
     return rtn
 
 
+@lazyobject
+def KWARG_RE():
+    return re.compile('[A-Za-z_]\w*=')
+
+
+def _starts_as_arg(s):
+    """Tests if a string starts as a non-kwarg string would."""
+    return KWARG_RE.match(s) is None
+
+
+def _eval_regular_args(raw_args, glbs, locs):
+    if not raw_args:
+        return [], {}
+    arglist = list(itertools.takewhile(_starts_as_arg, raw_args))
+    kwarglist = raw_args[len(arglist):]
+    execer = builtins.__xonsh_execer__
+    if not arglist:
+        args = arglist
+        kwargstr = 'dict({})'.format(', '.join(kwarglist))
+        kwargs = execer.eval(kwargstr, glbs=glbs, locs=locs)
+    elif not kwarglist:
+        argstr = '({},)'.format(', '.join(arglist))
+        args = execer.eval(argstr, glbs=glbs, locs=locs)
+        kwargs = {}
+    else:
+        argstr = '({},)'.format(', '.join(arglist))
+        kwargstr = 'dict({})'.format(', '.join(kwarglist))
+        both = '({}, {})'.format(argstr, kwargstr)
+        args, kwargs = execer.eval(both, glbs=glbs, locs=locs)
+    return args, kwargs
+
+
 def load_builtins(execer=None, config=None, login=False, ctx=None):
     """Loads the xonsh builtins into the Python builtins. Sets the
     BUILTINS_LOADED variable to True.

From 8ddb15861f1a1181c400b7fd4210009e40fc0b1d Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 27 Aug 2016 13:06:08 -0400
Subject: [PATCH 132/190] some raw tests

---
 tests/test_builtins.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/tests/test_builtins.py b/tests/test_builtins.py
index 559e458e8..402e2cee3 100644
--- a/tests/test_builtins.py
+++ b/tests/test_builtins.py
@@ -239,3 +239,19 @@ def test_call_macro_exec(arg):
         return x
     rtn = call_macro(f, [arg], {'x': 42, 'y': 0}, None)
     assert rtn is None
+
+
+@pytest.mark.parametrize('arg', ['x', '42', 'x + y'])
+def test_call_macro_raw_arg(arg):
+    def f(x : str):
+        return x
+    rtn = call_macro(f, ['*', arg], {'x': 42, 'y': 0}, None)
+    assert rtn == 42
+
+
+@pytest.mark.parametrize('arg', ['x', '42', 'x + y'])
+def test_call_macro_raw_kwarg(arg):
+    def f(x : str):
+        return x
+    rtn = call_macro(f, ['*', 'x=' + arg], {'x': 42, 'y': 0}, None)
+    assert rtn == 42

From a3c03b0b074eb3b246abccbc9e86f8dd3c63ad1c Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 27 Aug 2016 13:27:15 -0400
Subject: [PATCH 133/190] macro escape

---
 docs/tutorial_macros.rst | 21 +++++++++++++++++++++
 tests/test_builtins.py   |  8 ++++++++
 xonsh/built_ins.py       |  2 +-
 3 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/docs/tutorial_macros.rst b/docs/tutorial_macros.rst
index 4d55ec23d..145b41af9 100644
--- a/docs/tutorial_macros.rst
+++ b/docs/tutorial_macros.rst
@@ -187,6 +187,27 @@ argument:
     x = "['x', 'y']"
     y = '{1: 1, 2: 3}'
 
+Sometimes you may only want to pass in the first few arguments as macro
+arguments and you want the rest to be treated as normal Python arguments.
+By convention, xonsh's macro caller will look for a lone ``*`` argument
+in order to split the macro arguments and the regular arguments. So for
+example:
+
+.. code-block:: xonshcon
+
+    >>> g!(42, *, 65)
+    x = '42'
+    y = 65
+
+    >>> g!(42, *, y=65)
+    x = '42'
+    y = 65
+
+In the above, note that ``x`` is still captured as a macro argument. However,
+everything after the ``*``, namely ``y``, is evaluated is if it were passed
+in to a normal function call.  This can be useful for large interfaces where
+only a handful of args are expected as macro arguments.
+
 Hopefully, now you see the big picture.
 
 Writing Function Macros
diff --git a/tests/test_builtins.py b/tests/test_builtins.py
index 402e2cee3..b899eae9f 100644
--- a/tests/test_builtins.py
+++ b/tests/test_builtins.py
@@ -255,3 +255,11 @@ def test_call_macro_raw_kwarg(arg):
         return x
     rtn = call_macro(f, ['*', 'x=' + arg], {'x': 42, 'y': 0}, None)
     assert rtn == 42
+
+
+@pytest.mark.parametrize('arg', ['x', '42', 'x + y'])
+def test_call_macro_raw_kwargs(arg):
+    def f(x : str):
+        return x
+    rtn = call_macro(f, ['*', '**{"x" :' + arg + '}'], {'x': 42, 'y': 0}, None)
+    assert rtn == 42
diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py
index a1cd58250..07d1eb900 100644
--- a/xonsh/built_ins.py
+++ b/xonsh/built_ins.py
@@ -844,7 +844,7 @@ def call_macro(f, raw_args, glbs, locs):
 
 @lazyobject
 def KWARG_RE():
-    return re.compile('[A-Za-z_]\w*=')
+    return re.compile('([A-Za-z_]\w*=|\*\*)')
 
 
 def _starts_as_arg(s):

From 33db3acc02f056d3a6092586f4427e64a65c371e Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 27 Aug 2016 16:07:24 -0400
Subject: [PATCH 134/190] something

---
 xonsh/lexer.py        |  4 ++-
 xonsh/parsers/base.py | 60 +++++++++++++++++++++++++++++++++----------
 2 files changed, 50 insertions(+), 14 deletions(-)

diff --git a/xonsh/lexer.py b/xonsh/lexer.py
index 6c491876e..47a4c23d3 100644
--- a/xonsh/lexer.py
+++ b/xonsh/lexer.py
@@ -146,7 +146,8 @@ def handle_error_token(state, token):
     """
     state['last'] = token
     if not state['pymode'][-1][0]:
-        typ = 'NAME'
+        #typ = 'NAME'
+        typ = 'NAME_BANG' if token.string.endswith('!') else 'NAME'
     else:
         typ = 'ERRORTOKEN'
     yield _new_token(typ, token.string, token.start)
@@ -340,6 +341,7 @@ class Lexer(object):
         if self._tokens is None:
             t = tuple(token_map.values()) + (
                 'NAME',                  # name tokens
+                'NAME_BANG',             # name! tokens
                 'WS',                    # whitespace in subprocess mode
                 'LPAREN', 'RPAREN',      # ( )
                 'LBRACKET', 'RBRACKET',  # [ ]
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index 42715b329..fe5f552dd 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -222,6 +222,7 @@ class BaseParser(object):
         self._lines = None
         self.xonsh_code = None
         self._attach_nocomma_tok_rules()
+        self._attach_nonamebang_base_rules()
 
         opt_rules = [
             'newlines', 'arglist', 'func_call', 'rarrow_test', 'typedargslist',
@@ -256,7 +257,7 @@ class BaseParser(object):
         for rule in list_rules:
             self._list_rule(rule)
 
-        tok_rules = ['def', 'class', 'return', 'number', 'name',
+        tok_rules = ['def', 'class', 'return', 'number', 'name', 'name_bang',
                      'none', 'true', 'false', 'ellipsis', 'if', 'del',
                      'assert', 'lparen', 'lbrace', 'lbracket', 'string',
                      'times', 'plus', 'minus', 'divide', 'doublediv', 'mod',
@@ -264,7 +265,8 @@ class BaseParser(object):
                      'for', 'colon', 'import', 'except', 'nonlocal', 'global',
                      'yield', 'from', 'raise', 'with', 'dollar_lparen',
                      'dollar_lbrace', 'dollar_lbracket', 'try',
-                     'bang_lparen', 'bang_lbracket', 'comma', 'rparen']
+                     'bang_lparen', 'bang_lbracket', 'comma', 'rparen',
+                     'rbracket']
         for rule in tok_rules:
             self._tok_rule(rule)
 
@@ -1815,6 +1817,34 @@ class BaseParser(object):
         """
         p[0] = self._dollar_rules(p)
 
+    def p_atom_bang_fistful_of_dollars(self, p):
+        """atom : bang_lbracket_tok name_bang_tok nonamebang rbracket_tok
+        """
+        assert False
+        p[0] = self._dollar_rules(p)
+
+    def _attach_nonamebang_base_rules(self):
+        toks = set(self.tokens)
+        toks -= {'NAME_BANG', 'LPAREN', 'RPAREN', 'LBRACE', 'RBRACE',
+                 'LBRACKET', 'RBRACKET', 'AT_LPAREN', 'BANG_LPAREN',
+                 'BANG_LBRACKET', 'DOLLAR_LPAREN', 'DOLLAR_LBRACE',
+                 'DOLLAR_LBRACKET', 'ATDOLLAR_LPAREN'}
+        ts = '\n           | '.join(sorted(toks))
+        doc = 'nonamebang : ' + ts + '\n'
+        self.p_nonamebang_base.__func__.__doc__ = doc
+
+    def p_nonamebang_base(self, p):
+        # see above attachament function
+        pass
+
+    def p_nonamebang_nocomma(self, p):
+        """nonamebang : any_nested_raw"""
+        pass
+
+    def p_nonamebang_many(self, p):
+        """nonamebang : nonamebang nonamebang"""
+        pass
+
     def p_string_literal(self, p):
         """string_literal : string_tok"""
         p1 = p[1]
@@ -1941,20 +1971,24 @@ class BaseParser(object):
         """nocomma_part : nocomma_tok"""
         pass
 
-    def p_nocomma_part_any(self, p):
-        """nocomma_part : LPAREN any_raw_toks_opt RPAREN
-                        | LBRACE any_raw_toks_opt RBRACE
-                        | LBRACKET any_raw_toks_opt RBRACKET
-                        | AT_LPAREN any_raw_toks_opt RPAREN
-                        | BANG_LPAREN any_raw_toks_opt RPAREN
-                        | BANG_LBRACKET any_raw_toks_opt RBRACKET
-                        | DOLLAR_LPAREN any_raw_toks_opt RPAREN
-                        | DOLLAR_LBRACE any_raw_toks_opt RBRACE
-                        | DOLLAR_LBRACKET any_raw_toks_opt RBRACKET
-                        | ATDOLLAR_LPAREN any_raw_toks_opt RPAREN
+    def p_any_nested_raw(self, p):
+        """any_nested_raw : LPAREN any_raw_toks_opt RPAREN
+                          | LBRACE any_raw_toks_opt RBRACE
+                          | LBRACKET any_raw_toks_opt RBRACKET
+                          | AT_LPAREN any_raw_toks_opt RPAREN
+                          | BANG_LPAREN any_raw_toks_opt RPAREN
+                          | BANG_LBRACKET any_raw_toks_opt RBRACKET
+                          | DOLLAR_LPAREN any_raw_toks_opt RPAREN
+                          | DOLLAR_LBRACE any_raw_toks_opt RBRACE
+                          | DOLLAR_LBRACKET any_raw_toks_opt RBRACKET
+                          | ATDOLLAR_LPAREN any_raw_toks_opt RPAREN
         """
         pass
 
+    def p_nocomma_part_any(self, p):
+        """nocomma_part : any_nested_raw"""
+        pass
+
     def p_nocomma_base(self, p):
         """nocomma : nocomma_part"""
         pass

From 14ba018c8d20236dad8132aae821ce0a8e4434eb Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 27 Aug 2016 16:46:32 -0400
Subject: [PATCH 135/190] have no-ops

---
 tests/test_parser.py  | 13 +++++++++++++
 xonsh/lexer.py        |  4 ++--
 xonsh/parsers/base.py | 38 ++++++++++++++++++++++----------------
 3 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/tests/test_parser.py b/tests/test_parser.py
index ede47bf95..b516cdeaf 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -1864,3 +1864,16 @@ def test_macro_call_one_trailing_space(s):
     args = tree.body.args[1].elts
     assert len(args) == 1
     assert args[0].s == s.strip()
+
+
+@pytest.mark.parametrize('opener, closer', [
+    ('!(', ')'),
+    ('$(', ')'),
+    ('![', ']'),
+    ('$[', ']'),
+    ])
+def test_simple_subprocbang(opener, closer):
+    assert check_xonsh_ast({}, opener + 'echo!' + closer, False)
+    assert check_xonsh_ast({}, opener + 'echo !' + closer, False)
+    assert check_xonsh_ast({}, opener + 'echo ! ' + closer, False)
+
diff --git a/xonsh/lexer.py b/xonsh/lexer.py
index 47a4c23d3..c5ccdf996 100644
--- a/xonsh/lexer.py
+++ b/xonsh/lexer.py
@@ -147,7 +147,7 @@ def handle_error_token(state, token):
     state['last'] = token
     if not state['pymode'][-1][0]:
         #typ = 'NAME'
-        typ = 'NAME_BANG' if token.string.endswith('!') else 'NAME'
+        typ = 'BANG' if token.string == '!' else 'NAME'
     else:
         typ = 'ERRORTOKEN'
     yield _new_token(typ, token.string, token.start)
@@ -341,7 +341,7 @@ class Lexer(object):
         if self._tokens is None:
             t = tuple(token_map.values()) + (
                 'NAME',                  # name tokens
-                'NAME_BANG',             # name! tokens
+                'BANG',                  # ! tokens
                 'WS',                    # whitespace in subprocess mode
                 'LPAREN', 'RPAREN',      # ( )
                 'LBRACKET', 'RBRACKET',  # [ ]
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index fe5f552dd..e651975f5 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -222,7 +222,7 @@ class BaseParser(object):
         self._lines = None
         self.xonsh_code = None
         self._attach_nocomma_tok_rules()
-        self._attach_nonamebang_base_rules()
+        self._attach_nocloser_base_rules()
 
         opt_rules = [
             'newlines', 'arglist', 'func_call', 'rarrow_test', 'typedargslist',
@@ -257,7 +257,7 @@ class BaseParser(object):
         for rule in list_rules:
             self._list_rule(rule)
 
-        tok_rules = ['def', 'class', 'return', 'number', 'name', 'name_bang',
+        tok_rules = ['def', 'class', 'return', 'number', 'name', 'bang',
                      'none', 'true', 'false', 'ellipsis', 'if', 'del',
                      'assert', 'lparen', 'lbrace', 'lbracket', 'string',
                      'times', 'plus', 'minus', 'divide', 'doublediv', 'mod',
@@ -1810,39 +1810,45 @@ class BaseParser(object):
 
     def p_atom_fistful_of_dollars(self, p):
         """atom : dollar_lbrace_tok test RBRACE
-                | dollar_lparen_tok subproc RPAREN
                 | bang_lparen_tok subproc RPAREN
+                | dollar_lparen_tok subproc RPAREN
                 | bang_lbracket_tok subproc RBRACKET
                 | dollar_lbracket_tok subproc RBRACKET
         """
         p[0] = self._dollar_rules(p)
 
-    def p_atom_bang_fistful_of_dollars(self, p):
-        """atom : bang_lbracket_tok name_bang_tok nonamebang rbracket_tok
+    def p_atom_bang_empty_fistful_of_dollars(self, p):
+        """atom : bang_lparen_tok subproc BANG RPAREN
+                | bang_lparen_tok subproc BANG WS RPAREN
+                | dollar_lparen_tok subproc BANG RPAREN
+                | dollar_lparen_tok subproc BANG WS RPAREN
+                | bang_lbracket_tok subproc BANG RBRACKET
+                | bang_lbracket_tok subproc BANG WS RBRACKET
+                | dollar_lbracket_tok subproc BANG RBRACKET
+                | dollar_lbracket_tok subproc BANG WS RBRACKET
         """
-        assert False
         p[0] = self._dollar_rules(p)
 
-    def _attach_nonamebang_base_rules(self):
+    def _attach_nocloser_base_rules(self):
         toks = set(self.tokens)
-        toks -= {'NAME_BANG', 'LPAREN', 'RPAREN', 'LBRACE', 'RBRACE',
+        toks -= {'LPAREN', 'RPAREN', 'LBRACE', 'RBRACE',
                  'LBRACKET', 'RBRACKET', 'AT_LPAREN', 'BANG_LPAREN',
                  'BANG_LBRACKET', 'DOLLAR_LPAREN', 'DOLLAR_LBRACE',
                  'DOLLAR_LBRACKET', 'ATDOLLAR_LPAREN'}
-        ts = '\n           | '.join(sorted(toks))
-        doc = 'nonamebang : ' + ts + '\n'
-        self.p_nonamebang_base.__func__.__doc__ = doc
+        ts = '\n       | '.join(sorted(toks))
+        doc = 'nocloser : ' + ts + '\n'
+        self.p_nocloser_base.__func__.__doc__ = doc
 
-    def p_nonamebang_base(self, p):
+    def p_nocloser_base(self, p):
         # see above attachament function
         pass
 
-    def p_nonamebang_nocomma(self, p):
-        """nonamebang : any_nested_raw"""
+    def p_nocloser_any(self, p):
+        """nocloser : any_nested_raw"""
         pass
 
-    def p_nonamebang_many(self, p):
-        """nonamebang : nonamebang nonamebang"""
+    def p_nocloser_many(self, p):
+        """nocloser : nocloser nocloser"""
         pass
 
     def p_string_literal(self, p):

From fd2d37ff5020ddd7377527db4c7752a7253cbb92 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Sat, 27 Aug 2016 16:56:41 -0400
Subject: [PATCH 136/190] fix up completion display in ptk

---
 xonsh/ptk/completer.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/xonsh/ptk/completer.py b/xonsh/ptk/completer.py
index f4012e812..bc4c58c4d 100644
--- a/xonsh/ptk/completer.py
+++ b/xonsh/ptk/completer.py
@@ -4,7 +4,7 @@ import os
 import builtins
 
 from prompt_toolkit.layout.dimension import LayoutDimension
-from prompt_toolkit.completion import Completer, Completion
+from prompt_toolkit.completion import Completer, Completion, _commonprefix
 
 
 class PromptToolkitCompleter(Completer):
@@ -41,16 +41,17 @@ class PromptToolkitCompleter(Completer):
                 elif len(os.path.commonprefix(completions)) <= len(prefix):
                     self.reserve_space()
                 # don't mess with envvar and path expansion comps
-                if any(x in prefix for x in ['$', '/']):
-                    for comp in completions:
-                        yield Completion(comp, -l)
-                else:  # don't show common prefixes in attr completions
-                    prefix, _, compprefix = prefix.rpartition('.')
-                    for comp in completions:
-                        if comp.rsplit('.', 1)[0] in prefix:
-                            comp = comp.rsplit('.', 1)[-1]
-                        l = len(compprefix) if compprefix in comp else 0
-                        yield Completion(comp, -l)
+#                if any(x in prefix for x in ['$', '/']):
+                common_prefix = _commonprefix([a.strip('\'') for a in completions])
+                for comp in completions:
+                    yield Completion(comp, -l, display=comp[len(common_prefix):])
+#                else:  # don't show common prefixes in attr completions
+#                    prefix, _, compprefix = prefix.rpartition('.')
+#                    for comp in completions:
+#                        if comp.rsplit('.', 1)[0] in prefix:
+#                            comp = comp.rsplit('.', 1)[-1]
+#                        l = len(compprefix) if compprefix in comp else 0
+#                        yield Completion(comp, -l)
 
     def reserve_space(self):
         cli = builtins.__xonsh_shell__.shell.prompter.cli

From 2e70e6cee16e3788b7da719f5a452fcac56a0578 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Sat, 27 Aug 2016 17:09:51 -0400
Subject: [PATCH 137/190] make sure path completions show full name in dir

`cd git/xon` would show `sh/` as the completion which is not wrong, but
less helpful
---
 xonsh/ptk/completer.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/xonsh/ptk/completer.py b/xonsh/ptk/completer.py
index bc4c58c4d..4878a2e3d 100644
--- a/xonsh/ptk/completer.py
+++ b/xonsh/ptk/completer.py
@@ -42,9 +42,10 @@ class PromptToolkitCompleter(Completer):
                     self.reserve_space()
                 # don't mess with envvar and path expansion comps
 #                if any(x in prefix for x in ['$', '/']):
-                common_prefix = _commonprefix([a.strip('\'') for a in completions])
+                c_prefix = _commonprefix([a.strip('\'/').rsplit('/', 1)[0] for a in completions])
                 for comp in completions:
-                    yield Completion(comp, -l, display=comp[len(common_prefix):])
+                    display = comp[len(c_prefix):].lstrip('/')
+                    yield Completion(comp, -l, display=display)
 #                else:  # don't show common prefixes in attr completions
 #                    prefix, _, compprefix = prefix.rpartition('.')
 #                    for comp in completions:

From fea5c10d5f767308b3af29f5769be04f07c0c743 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Sat, 27 Aug 2016 17:12:57 -0400
Subject: [PATCH 138/190] remove comments and fix long line

---
 xonsh/ptk/completer.py | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/xonsh/ptk/completer.py b/xonsh/ptk/completer.py
index 4878a2e3d..5773f7aee 100644
--- a/xonsh/ptk/completer.py
+++ b/xonsh/ptk/completer.py
@@ -40,19 +40,11 @@ class PromptToolkitCompleter(Completer):
                     pass
                 elif len(os.path.commonprefix(completions)) <= len(prefix):
                     self.reserve_space()
-                # don't mess with envvar and path expansion comps
-#                if any(x in prefix for x in ['$', '/']):
-                c_prefix = _commonprefix([a.strip('\'/').rsplit('/', 1)[0] for a in completions])
+                c_prefix = _commonprefix([a.strip('\'/').rsplit('/', 1)[0]
+                                          for a in completions])
                 for comp in completions:
                     display = comp[len(c_prefix):].lstrip('/')
                     yield Completion(comp, -l, display=display)
-#                else:  # don't show common prefixes in attr completions
-#                    prefix, _, compprefix = prefix.rpartition('.')
-#                    for comp in completions:
-#                        if comp.rsplit('.', 1)[0] in prefix:
-#                            comp = comp.rsplit('.', 1)[-1]
-#                        l = len(compprefix) if compprefix in comp else 0
-#                        yield Completion(comp, -l)
 
     def reserve_space(self):
         cli = builtins.__xonsh_shell__.shell.prompter.cli

From 2c45da2dafd07ebcf800af945c146eb5b1609ebf Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Sat, 27 Aug 2016 17:22:10 -0400
Subject: [PATCH 139/190] fix for same-level directory completion display

`cd xon` should display `xonsh/` not `sh/`
---
 xonsh/ptk/completer.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/xonsh/ptk/completer.py b/xonsh/ptk/completer.py
index 5773f7aee..d769b91c0 100644
--- a/xonsh/ptk/completer.py
+++ b/xonsh/ptk/completer.py
@@ -43,6 +43,8 @@ class PromptToolkitCompleter(Completer):
                 c_prefix = _commonprefix([a.strip('\'/').rsplit('/', 1)[0]
                                           for a in completions])
                 for comp in completions:
+                    if comp.endswith('/') and not c_prefix.startswith('/'):
+                        c_prefix = ''
                     display = comp[len(c_prefix):].lstrip('/')
                     yield Completion(comp, -l, display=display)
 

From fdddaba88417d4f6334ca68fdb839f3b3beae527 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Sat, 27 Aug 2016 17:24:21 -0400
Subject: [PATCH 140/190] add news entry

---
 news/ptk-completion-display.rst | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 news/ptk-completion-display.rst

diff --git a/news/ptk-completion-display.rst b/news/ptk-completion-display.rst
new file mode 100644
index 000000000..ff24c6484
--- /dev/null
+++ b/news/ptk-completion-display.rst
@@ -0,0 +1,14 @@
+**Added:** None
+
+**Changed:** 
+
+* `prompt_toolkit` completions now only show the rightmost portion
+  of a given completion in the dropdown
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

From d4fc7ef5ad20bb47a6c8eef8cf0b17ecbc84e2f7 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 27 Aug 2016 17:28:45 -0400
Subject: [PATCH 141/190] mostly there

---
 tests/test_parser.py  | 16 +++++++++++++---
 xonsh/parsers/base.py | 39 ++++++++++++++++++++++++++++++---------
 2 files changed, 43 insertions(+), 12 deletions(-)

diff --git a/tests/test_parser.py b/tests/test_parser.py
index b516cdeaf..61763d145 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -1866,14 +1866,24 @@ def test_macro_call_one_trailing_space(s):
     assert args[0].s == s.strip()
 
 
-@pytest.mark.parametrize('opener, closer', [
+SUBPROC_MACRO_OC = [
     ('!(', ')'),
     ('$(', ')'),
     ('![', ']'),
     ('$[', ']'),
-    ])
-def test_simple_subprocbang(opener, closer):
+    ]
+
+@pytest.mark.parametrize('opener, closer', SUBPROC_MACRO_OC)
+def test_empty_subprocbang(opener, closer):
     assert check_xonsh_ast({}, opener + 'echo!' + closer, False)
     assert check_xonsh_ast({}, opener + 'echo !' + closer, False)
     assert check_xonsh_ast({}, opener + 'echo ! ' + closer, False)
 
+
+@pytest.mark.parametrize('opener, closer', SUBPROC_MACRO_OC)
+def test_single_subprocbang(opener, closer):
+    assert check_xonsh_ast({}, opener + 'echo!x' + closer, False)
+    assert check_xonsh_ast({}, opener + 'echo !x' + closer, False)
+    assert check_xonsh_ast({}, opener + 'echo! x' + closer, False)
+    assert check_xonsh_ast({}, opener + 'echo ! x' + closer, False)
+
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index e651975f5..e46f87179 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -1818,15 +1818,28 @@ class BaseParser(object):
         p[0] = self._dollar_rules(p)
 
     def p_atom_bang_empty_fistful_of_dollars(self, p):
-        """atom : bang_lparen_tok subproc BANG RPAREN
-                | bang_lparen_tok subproc BANG WS RPAREN
-                | dollar_lparen_tok subproc BANG RPAREN
-                | dollar_lparen_tok subproc BANG WS RPAREN
-                | bang_lbracket_tok subproc BANG RBRACKET
-                | bang_lbracket_tok subproc BANG WS RBRACKET
-                | dollar_lbracket_tok subproc BANG RBRACKET
-                | dollar_lbracket_tok subproc BANG WS RBRACKET
+        """atom : bang_lparen_tok subproc bang_tok RPAREN
+                | dollar_lparen_tok subproc bang_tok RPAREN
+                | bang_lbracket_tok subproc bang_tok RBRACKET
+                | dollar_lbracket_tok subproc bang_tok RBRACKET
         """
+        p3 = p[3]
+        node = ast.Str(s='', lineno=p3.lineno, col_offset=p3.lexpos)
+        p[2][-1].elts.append(node)
+        p[0] = self._dollar_rules(p)
+
+    def p_atom_bang_fistful_of_dollars(self, p):
+        """atom : bang_lparen_tok subproc bang_tok nocloser rparen_tok
+                | dollar_lparen_tok subproc bang_tok nocloser rparen_tok
+                | bang_lbracket_tok subproc bang_tok nocloser rbracket_tok
+                | dollar_lbracket_tok subproc bang_tok nocloser rbracket_tok
+        """
+        p3, p5 = p[3], p[5]
+        beg = (p3.lineno, p3.lexpos)
+        end = (p5.lineno, p5.lexpos)
+        s = self.source_slice(beg, end).strip()
+        node = ast.Str(s=s, lineno=beg[0], col_offset=beg[1])
+        p[2][-1].elts.append(node)
         p[0] = self._dollar_rules(p)
 
     def _attach_nocloser_base_rules(self):
@@ -1842,14 +1855,22 @@ class BaseParser(object):
     def p_nocloser_base(self, p):
         # see above attachament function
         pass
+        print('base', repr(p[1]))
 
     def p_nocloser_any(self, p):
         """nocloser : any_nested_raw"""
         pass
+        print('any', repr(p[1]))
 
     def p_nocloser_many(self, p):
-        """nocloser : nocloser nocloser"""
+        """nocloser : nocloser nocloser
+        """
+                    #| nocloser WS
+                    #| nocloser WS nocloser
+                    #| WS nocloser
+                    #| WS nocloser WS
         pass
+        print('many', repr(p[1]), repr(p[2]))
 
     def p_string_literal(self, p):
         """string_literal : string_tok"""

From 0edb58f1b87c39739be6031934986d2c9ceef0a3 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 27 Aug 2016 17:38:57 -0400
Subject: [PATCH 142/190] some test fixes

---
 tests/test_parser.py  | 23 ++++++++++++++---------
 xonsh/parsers/base.py |  4 ++--
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/tests/test_parser.py b/tests/test_parser.py
index 61763d145..156f91b99 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -1874,16 +1874,21 @@ SUBPROC_MACRO_OC = [
     ]
 
 @pytest.mark.parametrize('opener, closer', SUBPROC_MACRO_OC)
-def test_empty_subprocbang(opener, closer):
-    assert check_xonsh_ast({}, opener + 'echo!' + closer, False)
-    assert check_xonsh_ast({}, opener + 'echo !' + closer, False)
-    assert check_xonsh_ast({}, opener + 'echo ! ' + closer, False)
+@pytest.mark.parametrize('body', ['echo!', 'echo !', 'echo ! '])
+def test_empty_subprocbang(opener, closer, body):
+    tree = check_xonsh_ast({}, opener + body + closer, False, return_obs=True)
+    assert isinstance(tree, AST)
+    cmd = tree.body.args[0].elts
+    assert len(cmd) == 2
+    assert cmd[1].s == ''
 
 
 @pytest.mark.parametrize('opener, closer', SUBPROC_MACRO_OC)
-def test_single_subprocbang(opener, closer):
-    assert check_xonsh_ast({}, opener + 'echo!x' + closer, False)
-    assert check_xonsh_ast({}, opener + 'echo !x' + closer, False)
-    assert check_xonsh_ast({}, opener + 'echo! x' + closer, False)
-    assert check_xonsh_ast({}, opener + 'echo ! x' + closer, False)
+@pytest.mark.parametrize('body', ['echo!x', 'echo !x', 'echo !x', 'echo ! x'])
+def test_single_subprocbang(opener, closer, body):
+    tree = check_xonsh_ast({}, opener + body + closer, False, return_obs=True)
+    assert isinstance(tree, AST)
+    cmd = tree.body.args[0].elts
+    assert len(cmd) == 2
+    assert cmd[1].s == 'x'
 
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index e46f87179..9ec99422a 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -1824,7 +1824,7 @@ class BaseParser(object):
                 | dollar_lbracket_tok subproc bang_tok RBRACKET
         """
         p3 = p[3]
-        node = ast.Str(s='', lineno=p3.lineno, col_offset=p3.lexpos)
+        node = ast.Str(s='', lineno=p3.lineno, col_offset=p3.lexpos + 1)
         p[2][-1].elts.append(node)
         p[0] = self._dollar_rules(p)
 
@@ -1835,7 +1835,7 @@ class BaseParser(object):
                 | dollar_lbracket_tok subproc bang_tok nocloser rbracket_tok
         """
         p3, p5 = p[3], p[5]
-        beg = (p3.lineno, p3.lexpos)
+        beg = (p3.lineno, p3.lexpos + 1)
         end = (p5.lineno, p5.lexpos)
         s = self.source_slice(beg, end).strip()
         node = ast.Str(s=s, lineno=beg[0], col_offset=beg[1])

From 1a62cfe248f09b1eb9309c533e56f44b59450fa9 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 27 Aug 2016 18:12:09 -0400
Subject: [PATCH 143/190] tests indicate that it works

---
 tests/test_parser.py  | 41 +++++++++++++++++++++++++++++++++++++++++
 xonsh/lexer.py        |  8 +++++---
 xonsh/parsers/base.py | 11 ++---------
 3 files changed, 48 insertions(+), 12 deletions(-)

diff --git a/tests/test_parser.py b/tests/test_parser.py
index 156f91b99..ccab1e4e7 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -1892,3 +1892,44 @@ def test_single_subprocbang(opener, closer, body):
     assert len(cmd) == 2
     assert cmd[1].s == 'x'
 
+
+@pytest.mark.parametrize('opener, closer', SUBPROC_MACRO_OC)
+@pytest.mark.parametrize('body', ['echo -n!x', 'echo -n!x', 'echo -n !x',
+                                  'echo -n ! x'])
+def test_arg_single_subprocbang(opener, closer, body):
+    tree = check_xonsh_ast({}, opener + body + closer, False, return_obs=True)
+    assert isinstance(tree, AST)
+    cmd = tree.body.args[0].elts
+    assert len(cmd) == 3
+    assert cmd[2].s == 'x'
+
+
+@pytest.mark.parametrize('opener, closer', SUBPROC_MACRO_OC)
+@pytest.mark.parametrize('body', [
+    'echo!x + y',
+    'echo !x + y',
+    'echo !x + y',
+    'echo ! x + y',
+    'timeit! bang! and more',
+    'timeit! recurse() and more',
+    'timeit! recurse[] and more',
+    'timeit! recurse!() and more',
+    'timeit! recurse![] and more',
+    'timeit! recurse$() and more',
+    'timeit! recurse$[] and more',
+    'timeit! recurse!() and more',
+    'timeit!!!!',
+    'timeit! (!)',
+    'timeit! [!]',
+    'timeit!!(ls)',
+    'timeit!"!)"',
+    ])
+def test_many_subprocbang(opener, closer, body):
+    tree = check_xonsh_ast({}, opener + body + closer, False, return_obs=True,
+                           debug_level=100,
+                           )
+    assert isinstance(tree, AST)
+    cmd = tree.body.args[0].elts
+    assert len(cmd) == 2
+    assert cmd[1].s == body.partition('!')[-1].strip()
+
diff --git a/xonsh/lexer.py b/xonsh/lexer.py
index c5ccdf996..382f03fcd 100644
--- a/xonsh/lexer.py
+++ b/xonsh/lexer.py
@@ -145,9 +145,11 @@ def handle_error_token(state, token):
     Function for handling error tokens
     """
     state['last'] = token
-    if not state['pymode'][-1][0]:
-        #typ = 'NAME'
-        typ = 'BANG' if token.string == '!' else 'NAME'
+    if token.string == '!':
+        typ = 'BANG'
+    elif not state['pymode'][-1][0]:
+        typ = 'NAME'
+        #typ = 'BANG' if token.string == '!' else 'NAME'
     else:
         typ = 'ERRORTOKEN'
     yield _new_token(typ, token.string, token.start)
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index 9ec99422a..632c85e7d 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -1848,6 +1848,7 @@ class BaseParser(object):
                  'LBRACKET', 'RBRACKET', 'AT_LPAREN', 'BANG_LPAREN',
                  'BANG_LBRACKET', 'DOLLAR_LPAREN', 'DOLLAR_LBRACE',
                  'DOLLAR_LBRACKET', 'ATDOLLAR_LPAREN'}
+        #toks.add('ERRORTOKEN')
         ts = '\n       | '.join(sorted(toks))
         doc = 'nocloser : ' + ts + '\n'
         self.p_nocloser_base.__func__.__doc__ = doc
@@ -1855,22 +1856,14 @@ class BaseParser(object):
     def p_nocloser_base(self, p):
         # see above attachament function
         pass
-        print('base', repr(p[1]))
 
     def p_nocloser_any(self, p):
         """nocloser : any_nested_raw"""
         pass
-        print('any', repr(p[1]))
 
     def p_nocloser_many(self, p):
-        """nocloser : nocloser nocloser
-        """
-                    #| nocloser WS
-                    #| nocloser WS nocloser
-                    #| WS nocloser
-                    #| WS nocloser WS
+        """nocloser : nocloser nocloser"""
         pass
-        print('many', repr(p[1]), repr(p[2]))
 
     def p_string_literal(self, p):
         """string_literal : string_tok"""

From 71bbe17a66355f7470249c8b31d2ec8ec869a167 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 27 Aug 2016 18:16:19 -0400
Subject: [PATCH 144/190] some comment removal

---
 xonsh/lexer.py        | 1 -
 xonsh/parsers/base.py | 1 -
 2 files changed, 2 deletions(-)

diff --git a/xonsh/lexer.py b/xonsh/lexer.py
index 382f03fcd..365f39461 100644
--- a/xonsh/lexer.py
+++ b/xonsh/lexer.py
@@ -149,7 +149,6 @@ def handle_error_token(state, token):
         typ = 'BANG'
     elif not state['pymode'][-1][0]:
         typ = 'NAME'
-        #typ = 'BANG' if token.string == '!' else 'NAME'
     else:
         typ = 'ERRORTOKEN'
     yield _new_token(typ, token.string, token.start)
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index 632c85e7d..b96ddae5a 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -1848,7 +1848,6 @@ class BaseParser(object):
                  'LBRACKET', 'RBRACKET', 'AT_LPAREN', 'BANG_LPAREN',
                  'BANG_LBRACKET', 'DOLLAR_LPAREN', 'DOLLAR_LBRACE',
                  'DOLLAR_LBRACKET', 'ATDOLLAR_LPAREN'}
-        #toks.add('ERRORTOKEN')
         ts = '\n       | '.join(sorted(toks))
         doc = 'nocloser : ' + ts + '\n'
         self.p_nocloser_base.__func__.__doc__ = doc

From 3c979c2fce862601b5a518570705bab202304488 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 27 Aug 2016 19:05:42 -0400
Subject: [PATCH 145/190] more docs

---
 docs/tutorial_macros.rst | 65 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)

diff --git a/docs/tutorial_macros.rst b/docs/tutorial_macros.rst
index 145b41af9..cbd4fd3f2 100644
--- a/docs/tutorial_macros.rst
+++ b/docs/tutorial_macros.rst
@@ -344,6 +344,71 @@ Of course, many other more sophisticated options are available depending on the
 use case.
 
 
+Subprocess Macros
+=================
+Like with function macros above, subprocess macros allow you to pause the parser
+for until you are ready to exit subprocess mode. Unlike function macros, there
+is only a single macro argument and its macro type is always a string.  This
+is because it (usually) doesn't make sense to pass non-string arguments to a
+command. And when it does, there is the ``@()`` syntax!
+
+In the simplest case, subprocess macros look like the equivalent of their
+function macro counterparts:
+
+.. code-block:: xonshcon
+
+    >>> echo! I'm Mr. Meeseeks.
+    I'm Mr. Meeseeks.
+
+Again, note that everything to the right of the ``!`` is passed down to the
+``echo`` command as the final, single argument. This is space preserving,
+like wrapping with quotes:
+
+.. code-block:: xonshcon
+
+    # normally, xonsh will split on whitespace,
+    # so each argument is passed in separately
+    >>> echo x  y       z
+    x y z
+
+    # usually space can be preserved with quotes
+    >>> echo "x  y       z"
+    x  y       z
+
+    # however, subproces macros will pause and then strip
+    # all input after the exclamation point
+    >>> echo! x  y       z
+    x  y       z
+
+However, the macro will pause everything, including path and environment variable
+expansion, that might be present even with quotes.  For example:
+
+.. code-block:: xonshcon
+
+    # without macros, envrioment variable are expanded
+    >>> echo $USER
+    lou
+
+    # inside of a macro, all additional munging is turned off.
+    >>> echo! $USER
+    $USER
+
+Everything to the right of the exclamation point, except the leading and trailing
+whitespace, is passed into the command directly as written. This allows certain
+commands to function in cases where quoting or piping might be more burdensome.
+The ``timeit`` command is a great example where simple syntax will often fail,
+but will be easily executable as a macro:
+
+.. code-block:: xonshcon
+
+    # fails normally
+    >>> timeit "hello mom " + "and dad"
+    xonsh: subprocess mode: command not found: hello
+
+    # macro success!
+    >>> timeit! "hello mom " + "and dad"
+    100000000 loops, best of 3: 8.24 ns per loop
+
 Take Away
 =========
 Hopefully, at this point, you see that a few well placed macros can be extremely

From 82ae566fff426485077deb8dd05b38807b87ba80 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 27 Aug 2016 19:35:48 -0400
Subject: [PATCH 146/190] fixed up subproc warpper for macros

---
 tests/test_tools.py | 14 ++++++++++++++
 xonsh/tools.py      |  8 ++++++++
 2 files changed, 22 insertions(+)

diff --git a/tests/test_tools.py b/tests/test_tools.py
index 34ffcd662..a55913d95 100644
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -74,6 +74,20 @@ def test_subproc_toks_git_nl():
     assert (exp == obs)
 
 
+def test_bash_macro():
+    s = 'bash -c ! export var=42; echo $var'
+    exp = '![{0}]\n'.format(s)
+    obs = subproc_toks(s + '\n', lexer=LEXER, returnline=True)
+    assert (exp == obs)
+
+
+def test_python_macro():
+    s = 'python -c ! import os; print(os.path.abspath("/"))'
+    exp = '![{0}]\n'.format(s)
+    obs = subproc_toks(s + '\n', lexer=LEXER, returnline=True)
+    assert (exp == obs)
+
+
 def test_subproc_toks_indent_ls():
     s = 'ls -l'
     exp = INDENT + '![{0}]'.format(s)
diff --git a/xonsh/tools.py b/xonsh/tools.py
index 126291508..f57086389 100644
--- a/xonsh/tools.py
+++ b/xonsh/tools.py
@@ -259,11 +259,17 @@ def subproc_toks(line, mincol=-1, maxcol=None, lexer=None, returnline=False):
     lexer.input(line)
     toks = []
     lparens = []
+    saw_macro = False
     end_offset = 0
     for tok in lexer:
         pos = tok.lexpos
         if tok.type not in END_TOK_TYPES and pos >= maxcol:
             break
+        if tok.type == 'BANG':
+            saw_macro = True
+        if saw_macro and tok.type not in ('NEWLINE', 'DEDENT'):
+            toks.append(tok)
+            continue
         if tok.type in LPARENS:
             lparens.append(tok.type)
         if len(toks) == 0 and tok.type in BEG_TOK_SKIPS:
@@ -313,6 +319,8 @@ def subproc_toks(line, mincol=-1, maxcol=None, lexer=None, returnline=False):
             end_offset = len(el)
     if len(toks) == 0:
         return  # handle comment lines
+    elif saw_macro:
+        end_offset = len(toks[-1].value.rstrip())
     beg, end = toks[0].lexpos, (toks[-1].lexpos + end_offset)
     end = len(line[:end].rstrip())
     rtn = '![' + line[beg:end] + ']'

From 5a66fd05b5b635676c942c23b157cfcdd5b438ec Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 27 Aug 2016 19:55:58 -0400
Subject: [PATCH 147/190] more auto-wrapping fixes

---
 tests/test_tools.py | 2 ++
 xonsh/tools.py      | 5 ++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/test_tools.py b/tests/test_tools.py
index a55913d95..68d36194a 100644
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -342,6 +342,8 @@ def test_subexpr_from_unbalanced_parens(inp, exp):
     ('(ls) && echo a', 1, 4),
     ('not ls && echo a', 0, 8),
     ('not (ls) && echo a', 0, 8),
+    ('bash -c ! export var=42; echo $var', 0, 35),
+    ('python -c ! import os; print(os.path.abspath("/"))', 0, 51),
 ])
 def test_find_next_break(line, mincol, exp):
     obs = find_next_break(line, mincol=mincol, lexer=LEXER)
diff --git a/xonsh/tools.py b/xonsh/tools.py
index f57086389..42e09d118 100644
--- a/xonsh/tools.py
+++ b/xonsh/tools.py
@@ -243,6 +243,9 @@ def find_next_break(line, mincol=0, lexer=None):
         elif tok.type == 'ERRORTOKEN' and ')' in tok.value:
             maxcol = tok.lexpos + mincol + 1
             break
+        elif tok.type == 'BANG':
+            maxcol = mincol + len(line) + 1
+            break
     return maxcol
 
 
@@ -320,7 +323,7 @@ def subproc_toks(line, mincol=-1, maxcol=None, lexer=None, returnline=False):
     if len(toks) == 0:
         return  # handle comment lines
     elif saw_macro:
-        end_offset = len(toks[-1].value.rstrip())
+        end_offset = len(toks[-1].value.rstrip()) + 1
     beg, end = toks[0].lexpos, (toks[-1].lexpos + end_offset)
     end = len(line[:end].rstrip())
     rtn = '![' + line[beg:end] + ']'

From d30bcf97cea9d3a75c52bbefdab3c1e2ee2677df Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sat, 27 Aug 2016 20:02:45 -0400
Subject: [PATCH 148/190] a few more

---
 docs/tutorial_macros.rst | 20 +++++++++++++++++++-
 news/m.rst               |  2 ++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/docs/tutorial_macros.rst b/docs/tutorial_macros.rst
index cbd4fd3f2..5ba089368 100644
--- a/docs/tutorial_macros.rst
+++ b/docs/tutorial_macros.rst
@@ -409,7 +409,25 @@ but will be easily executable as a macro:
     >>> timeit! "hello mom " + "and dad"
     100000000 loops, best of 3: 8.24 ns per loop
 
+All expressions to the left of the exclamation point are passed in normally and
+are not treated as the special macro argument. This allows the mixing of
+simple and complex command line arguments. For example, sometimes you might
+really want to write some code in another language:
+
+.. code-block:: xonshcon
+
+    # don't worry, it is temporary!
+    >>> bash -c ! export var=42; echo $var
+    42
+
+    # that's better!
+    >>> python -c ! import os; print(os.path.abspath("/"))
+    /
+
+Compared to function macros, subprocess macros are relatively simple.
+However, they can still be very expressive!
+
 Take Away
 =========
 Hopefully, at this point, you see that a few well placed macros can be extremely
-convenient and valuable to any project.
\ No newline at end of file
+convenient and valuable to any project.
diff --git a/news/m.rst b/news/m.rst
index 99be46fe9..e5928ecd5 100644
--- a/news/m.rst
+++ b/news/m.rst
@@ -2,6 +2,8 @@
 
 * Macro function calls are now available. These use a Rust-like
   ``f!(arg)`` syntax.
+* Macro subprocess call now avalaible with the ``echo! x y z``
+  syntax.
 
 **Changed:** None
 

From ab13740e136aca3d488395707f61704736055a06 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 21:24:27 -0400
Subject: [PATCH 149/190] PEP8

---
 tests/conftest.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index 5c55bdc47..06d4ca443 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -39,7 +39,8 @@ def xonsh_builtins():
     builtins.execx = None
     builtins.compilex = None
     builtins.aliases = {}
-    # Unlike all the other stuff, this has to refer to the "real" one because all modules that would be firing events on the global instance.
+    # Unlike all the other stuff, this has to refer to the "real" one because all modules that would
+    # be firing events on the global instance.
     builtins.events = events
     yield builtins
     del builtins.__xonsh_env__

From 0223905752988425e305a0606442597d07beb0c9 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gilforsyth@gmail.com>
Date: Sat, 27 Aug 2016 21:32:09 -0400
Subject: [PATCH 150/190] fix news entry

---
 news/ptk-completion-display.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/news/ptk-completion-display.rst b/news/ptk-completion-display.rst
index ff24c6484..7288ec572 100644
--- a/news/ptk-completion-display.rst
+++ b/news/ptk-completion-display.rst
@@ -2,7 +2,7 @@
 
 **Changed:** 
 
-* `prompt_toolkit` completions now only show the rightmost portion
+* ``prompt_toolkit`` completions now only show the rightmost portion
   of a given completion in the dropdown
 
 **Deprecated:** None

From 0ad9a2e2ab9b0334d9ecd9ff3eb30e168ba45172 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 21:44:26 -0400
Subject: [PATCH 151/190] Add test to detect event typos

---
 tests/test_events.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/test_events.py b/tests/test_events.py
index 984d4db83..842a59db3 100644
--- a/tests/test_events.py
+++ b/tests/test_events.py
@@ -109,3 +109,7 @@ def test_transmogrify_by_string(events):
     assert isinstance(events.on_test, LoadEvent)
     assert len(events.on_test) == 1
     assert inspect.getdoc(events.on_test) == docstring
+
+def test_typos(xonsh_builtins):
+    for ev in vars(xonsh_builtins.events).values():
+        assert inspect.getdoc(ev)
\ No newline at end of file

From 3721204304e046842347c095a2b68ce7ea3ed780 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 22:25:29 -0400
Subject: [PATCH 152/190] Add an autogenerated events listing

---
 .gitignore               |  1 +
 docs/advanced_events.rst |  9 +++++++++
 docs/conf.py             | 39 +++++++++++++++++++++++++++++++++++++++
 docs/events.rst          | 12 ++++--------
 4 files changed, 53 insertions(+), 8 deletions(-)
 create mode 100644 docs/advanced_events.rst

diff --git a/.gitignore b/.gitignore
index 98dc83897..0c972c4c5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,6 +19,7 @@ xonsh.egg-info/
 docs/_build/
 docs/envvarsbody
 docs/xontribsbody
+docs/eventsbody
 xonsh/dev.githash
 
 # temporary files from vim and emacs
diff --git a/docs/advanced_events.rst b/docs/advanced_events.rst
new file mode 100644
index 000000000..0de19e3af
--- /dev/null
+++ b/docs/advanced_events.rst
@@ -0,0 +1,9 @@
+.. _events:
+
+********************
+Advanced Events
+********************
+
+If you havent, go read the `events tutorial`_ first.
+
+{This is where we document events for core developers and advanced users.}
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
index bd810c497..1aa72738e 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -10,12 +10,14 @@
 import os
 import sys
 import builtins
+import inspect
 
 os.environ['XONSH_DEBUG'] = '1'
 
 from xonsh import __version__ as XONSH_VERSION
 from xonsh.environ import DEFAULT_DOCS, Env
 from xonsh.xontribs import xontrib_metadata
+from xonsh.events import events
 
 sys.path.insert(0, os.path.dirname(__file__))
 
@@ -340,8 +342,45 @@ def make_xontribs():
         f.write(s)
 
 
+def make_events():
+    names = sorted(vars(events).keys())
+    s = ('.. list-table::\n'
+         '    :header-rows: 0\n\n')
+    table = []
+    ncol = 3
+    row = '    {0} - :ref:`{1} <{2}>`'
+    for i, var in enumerate(names):
+        star = '*' if i%ncol == 0 else ' '
+        table.append(row.format(star, var, var.lower()))
+    table.extend(['      -']*((ncol - len(names)%ncol)%ncol))
+    s += '\n'.join(table) + '\n\n'
+    s += ('Listing\n'
+          '-------\n\n')
+    sec = ('.. _{low}:\n\n'
+           '{title}\n'
+           '{under}\n'
+           '{docstr}\n\n'
+           '-------\n\n')
+    for name in names:
+        event = getattr(events, name)
+        title = name
+        docstr = inspect.getdoc(event)
+        if docstr.startswith(name):
+          # Assume the first line is a signature
+          title, docstr = docstr.split('\n', 1)
+          docstr = docstr.strip()
+        under = '.' * len(title)
+        s += sec.format(low=var.lower(), title=title, under=under,
+                        docstr=docstr)
+    s = s[:-9]
+    fname = os.path.join(os.path.dirname(__file__), 'eventsbody')
+    with open(fname, 'w') as f:
+        f.write(s)
+
+
 make_envvars()
 make_xontribs()
+make_events()
 
 builtins.__xonsh_history__ = None
 builtins.__xonsh_env__ = {}
diff --git a/docs/events.rst b/docs/events.rst
index 0de19e3af..207e55640 100644
--- a/docs/events.rst
+++ b/docs/events.rst
@@ -1,9 +1,5 @@
-.. _events:
+Core Events
+===========
+The following events are defined by xonsh itself.
 
-********************
-Advanced Events
-********************
-
-If you havent, go read the `events tutorial`_ first.
-
-{This is where we document events for core developers and advanced users.}
\ No newline at end of file
+.. include:: eventsbody

From ce61c59d4b7f26d00f82ba523d069975e4fda704 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 22:29:11 -0400
Subject: [PATCH 153/190] Do some fixed-width formatting

---
 docs/conf.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 1aa72738e..c4ef016ee 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -357,7 +357,7 @@ def make_events():
     s += ('Listing\n'
           '-------\n\n')
     sec = ('.. _{low}:\n\n'
-           '{title}\n'
+           '``{title}``\n'
            '{under}\n'
            '{docstr}\n\n'
            '-------\n\n')
@@ -369,7 +369,7 @@ def make_events():
           # Assume the first line is a signature
           title, docstr = docstr.split('\n', 1)
           docstr = docstr.strip()
-        under = '.' * len(title)
+        under = '.' * (len(title) + 4)
         s += sec.format(low=var.lower(), title=title, under=under,
                         docstr=docstr)
     s = s[:-9]

From c9ad375f8dee489470f008a3a2169f71a9e8ab4c Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 22:38:12 -0400
Subject: [PATCH 154/190] Some interlinking

---
 docs/advanced_events.rst |  2 +-
 docs/index.rst           |  3 +++
 docs/tutorial_events.rst | 10 +++++-----
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/docs/advanced_events.rst b/docs/advanced_events.rst
index 0de19e3af..626133bc6 100644
--- a/docs/advanced_events.rst
+++ b/docs/advanced_events.rst
@@ -4,6 +4,6 @@
 Advanced Events
 ********************
 
-If you havent, go read the `events tutorial`_ first.
+If you havent, go read the `events tutorial <tutorial_events.rst>`_ first.
 
 {This is where we document events for core developers and advanced users.}
\ No newline at end of file
diff --git a/docs/index.rst b/docs/index.rst
index 6c9ecac0a..7740f7cdb 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -100,6 +100,7 @@ Contents
     tutorial
     tutorial_hist
     tutorial_xontrib
+    tutorial_events
     tutorial_completers
     bash_to_xsh
     python_virtual_environments
@@ -115,6 +116,7 @@ Contents
     envvars
     aliases
     xontribs
+    events
 
 **News & Media:**
 
@@ -133,6 +135,7 @@ Contents
     :maxdepth: 1
 
     api/index
+    advanced_events
     devguide/
     previous/index
     faq
diff --git a/docs/tutorial_events.rst b/docs/tutorial_events.rst
index 044972c32..61d89b71a 100644
--- a/docs/tutorial_events.rst
+++ b/docs/tutorial_events.rst
@@ -22,9 +22,9 @@ or several other commands).
         with open(g`~/.dirhist`[0], 'a') as dh:
             print(newdir, file=dh)
 
-Core Events
-===========
 
-* ``on_precommand``
-* ``on_postcommand``
-* ``on_chdir``
+
+Under the Hood
+==============
+
+`Advanced Events <advanced_events.html>`_
\ No newline at end of file

From 0313026036a49d385e96917d46e0dee6e9646268 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 23:00:38 -0400
Subject: [PATCH 155/190] Update module docstring

---
 xonsh/events.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/xonsh/events.py b/xonsh/events.py
index ced0f531c..dd1cf773d 100644
--- a/xonsh/events.py
+++ b/xonsh/events.py
@@ -1,11 +1,11 @@
 """
 Events for xonsh.
 
-In all likelihood, you want builtins.__xonsh_events__
+In all likelihood, you want builtins.events
 
 The best way to "declare" an event is something like::
 
-    __xonsh_events__.on_spam.doc("Comes with eggs")
+    events.doc('on_spam', "Comes with eggs")
 """
 import abc
 import collections.abc

From d14a4bf93883e4a023de9abcc315f198be2cd88a Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 23:00:50 -0400
Subject: [PATCH 156/190] Add module API docs

---
 docs/api/events.rst | 11 +++++++++++
 docs/api/index.rst  |  1 +
 2 files changed, 12 insertions(+)
 create mode 100644 docs/api/events.rst

diff --git a/docs/api/events.rst b/docs/api/events.rst
new file mode 100644
index 000000000..01457b59f
--- /dev/null
+++ b/docs/api/events.rst
@@ -0,0 +1,11 @@
+.. _xonsh_events:
+
+********************************************************************************
+ Events (``xonsh.events`)
+********************************************************************************
+
+.. automodule:: xonsh.events
+    :members:
+    :undoc-members:
+    :inherited-members:
+
diff --git a/docs/api/index.rst b/docs/api/index.rst
index 6332a2467..8d72e5bff 100644
--- a/docs/api/index.rst
+++ b/docs/api/index.rst
@@ -49,6 +49,7 @@ For those of you who want the gritty details.
 .. toctree::
     :maxdepth: 1
 
+    events
     tools
     platform
     lazyjson

From e01b6a0919788eee72d1c008e52fe80ce2ceaa75 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 23:10:24 -0400
Subject: [PATCH 157/190] Attempted advanced events

---
 docs/advanced_events.rst | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/docs/advanced_events.rst b/docs/advanced_events.rst
index 626133bc6..c9f44521f 100644
--- a/docs/advanced_events.rst
+++ b/docs/advanced_events.rst
@@ -4,6 +4,30 @@
 Advanced Events
 ********************
 
-If you havent, go read the `events tutorial <tutorial_events.rst>`_ first.
+If you havent, go read the `events tutorial <tutorial_events.rst>`_ first. This documents the messy
+details of the event system.
 
-{This is where we document events for core developers and advanced users.}
\ No newline at end of file
+You may also find the `events API reference <api/events.html>`_ useful.
+
+What are Species?
+=================
+In xonsh, events come in species. Each one may look like an event and quack like an event, but they
+behave differently.
+
+This was done because load hooks look like events and quack like events, but they have different
+semantics. See `LoadEvents <api/events.html#xonsh.events.LoadEvent>`_ for details.
+
+Why Unordered?
+==============
+Yes, handler call order is not guarenteed. Please don't file bugs about this.
+
+This was chosen because the order of handler registration is dependant on load order, which is 
+stable in a release but not something generally reasoned about. In addition, xontribs mean that we
+don't know what handlers could be registered.
+
+Because of this, the event system is not ordered, and order-dependant semantics are not encouraged.
+
+So how do I handle results?
+===========================
+``Event.fire()`` returns a list of the returns from the handlers. You should merge this list in an 
+appropriate way.

From 8e537aaa050a93fa0a234a2583cbb3747c7eb9bc Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 23:16:11 -0400
Subject: [PATCH 158/190] Typo

---
 docs/api/events.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/api/events.rst b/docs/api/events.rst
index 01457b59f..d2e06302e 100644
--- a/docs/api/events.rst
+++ b/docs/api/events.rst
@@ -1,7 +1,7 @@
 .. _xonsh_events:
 
 ********************************************************************************
- Events (``xonsh.events`)
+ Events (``xonsh.events``)
 ********************************************************************************
 
 .. automodule:: xonsh.events

From e446c63e60478cebfdc0673d10e0ee67f48355d2 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 23:20:44 -0400
Subject: [PATCH 159/190] Got something usable.

---
 docs/tutorial_events.rst | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/docs/tutorial_events.rst b/docs/tutorial_events.rst
index 61d89b71a..37a29c3a9 100644
--- a/docs/tutorial_events.rst
+++ b/docs/tutorial_events.rst
@@ -7,24 +7,42 @@ What's the best way to keep informed in xonsh? Subscribe to an event!
 
 Overview
 ========
-Simply, events are a way for xonsh to .... do something.
+Simply, events are a way for various pieces of xonsh to tell each other what's going on. They're 
+fired when something of note happens, eg the current directory changes or just before a command is
+executed.
 
+While xonsh has its own event system, it is not dissimilar to other event systems.
 
 Show me the code!
 =================
 Fine, fine!
 
 This will add a line to a file every time the current directory changes (due to ``cd``, ``pushd``,
-or several other commands).
+or several other commands)::
 
     @events.on_chdir
     def add_to_file(newdir):
         with open(g`~/.dirhist`[0], 'a') as dh:
             print(newdir, file=dh)
 
+The exact arguments passed and returns expected vary from event to event; see the 
+`event list <events.html>`_ for the details.
 
+Can I use this, too?
+====================
+
+Yes! It's even easy! In your xontrib, you just have to do something like::
+
+    events.doc('myxontrib_on_spam', """
+    myxontrib_on_spam(can: Spam) -> bool?
+
+    Fired in case of spam. Return ``True`` if it's been eaten.
+    """)
+
+This will enable users to call ``help(events.myxontrib_on_spam)`` and get useful output.
 
 Under the Hood
 ==============
 
-`Advanced Events <advanced_events.html>`_
\ No newline at end of file
+If you want to know more about the gory details of what makes events tick, see 
+`Advanced Events <advanced_events.html>`_.

From 74c423f2152d772e3e9bb8f71ca8e90f70bb857c Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 23:27:52 -0400
Subject: [PATCH 160/190] Expand on events under the hood some.

---
 docs/advanced_events.rst | 33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/docs/advanced_events.rst b/docs/advanced_events.rst
index c9f44521f..0893adf7a 100644
--- a/docs/advanced_events.rst
+++ b/docs/advanced_events.rst
@@ -9,6 +9,23 @@ details of the event system.
 
 You may also find the `events API reference <api/events.html>`_ useful.
 
+Why Unordered?
+==============
+Yes, handler call order is not guarenteed. Please don't file bugs about this.
+
+This was chosen because the order of handler registration is dependant on load order, which is 
+stable in a release but not something generally reasoned about. In addition, xontribs mean that we
+don't know what handlers could be registered. So even an "ordered" event system would be unable to
+make guarentees about ordering because of the larger system.
+
+Because of this, the event system is not ordered; this is a form of abstraction. Order-dependant 
+semantics are not encouraged by the built-in methods.
+
+So how do I handle results?
+===========================
+``Event.fire()`` returns a list of the returns from the handlers. You should merge this list in an 
+appropriate way.
+
 What are Species?
 =================
 In xonsh, events come in species. Each one may look like an event and quack like an event, but they
@@ -17,17 +34,9 @@ behave differently.
 This was done because load hooks look like events and quack like events, but they have different
 semantics. See `LoadEvents <api/events.html#xonsh.events.LoadEvent>`_ for details.
 
-Why Unordered?
-==============
-Yes, handler call order is not guarenteed. Please don't file bugs about this.
+In order to turn an event from the default ``Event``, you must transmogrify it, using 
+``events.transmogrify()``. The class the event is turned in to must be a subclass of ``AbstractEvent``.
 
-This was chosen because the order of handler registration is dependant on load order, which is 
-stable in a release but not something generally reasoned about. In addition, xontribs mean that we
-don't know what handlers could be registered.
+(Under the hood, transmogrify creates a new instance and copies the handlers and docstring from the
+old instance to the new one.)
 
-Because of this, the event system is not ordered, and order-dependant semantics are not encouraged.
-
-So how do I handle results?
-===========================
-``Event.fire()`` returns a list of the returns from the handlers. You should merge this list in an 
-appropriate way.

From 0fcf737e082a4920221e4fedbefadfb4abd7b422 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sat, 27 Aug 2016 23:29:23 -0400
Subject: [PATCH 161/190] Actually finish that paragraph.

---
 docs/tutorial_events.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/tutorial_events.rst b/docs/tutorial_events.rst
index 37a29c3a9..c8ca7db46 100644
--- a/docs/tutorial_events.rst
+++ b/docs/tutorial_events.rst
@@ -11,7 +11,8 @@ Simply, events are a way for various pieces of xonsh to tell each other what's g
 fired when something of note happens, eg the current directory changes or just before a command is
 executed.
 
-While xonsh has its own event system, it is not dissimilar to other event systems.
+While xonsh has its own event system, it is not dissimilar to other event systems. If you do know
+events, this should be easy to understand. If not, then this document is extra for you.
 
 Show me the code!
 =================

From 78a87d03041995dfd0ca86a7c422b73d9fd89c9a Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sun, 28 Aug 2016 09:57:31 -0400
Subject: [PATCH 162/190] some emergency fixup

---
 tests/test_events.py   |  2 +-
 tests/test_imphooks.py |  4 ++--
 xonsh/__init__.py      | 12 ++++++------
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/tests/test_events.py b/tests/test_events.py
index 842a59db3..91b59c4f8 100644
--- a/tests/test_events.py
+++ b/tests/test_events.py
@@ -112,4 +112,4 @@ def test_transmogrify_by_string(events):
 
 def test_typos(xonsh_builtins):
     for ev in vars(xonsh_builtins.events).values():
-        assert inspect.getdoc(ev)
\ No newline at end of file
+        assert inspect.getdoc(ev)
diff --git a/tests/test_imphooks.py b/tests/test_imphooks.py
index f8be37fae..b53a0dd9b 100644
--- a/tests/test_imphooks.py
+++ b/tests/test_imphooks.py
@@ -47,10 +47,10 @@ TEST_DIR = os.path.dirname(__file__)
 def test_module_dunder_file_attribute():
     import sample
     exp = os.path.join(TEST_DIR, 'sample.xsh')
-    assert sample.__file__ == exp
+    assert os.path.abspath(sample.__file__) == exp
 
 
 def test_module_dunder_file_attribute_sub():
     from xpack.sub import sample
     exp = os.path.join(TEST_DIR, 'xpack', 'sub', 'sample.xsh')
-    assert sample.__file__ == exp
+    assert os.path.abspath(sample.__file__) == exp
diff --git a/xonsh/__init__.py b/xonsh/__init__.py
index ccb123b7f..17c00e74d 100644
--- a/xonsh/__init__.py
+++ b/xonsh/__init__.py
@@ -43,8 +43,6 @@ else:
         _sys.modules['xonsh.contexts'] = __amalgam__
         diff_history = __amalgam__
         _sys.modules['xonsh.diff_history'] = __amalgam__
-        dirstack = __amalgam__
-        _sys.modules['xonsh.dirstack'] = __amalgam__
         events = __amalgam__
         _sys.modules['xonsh.events'] = __amalgam__
         foreign_shells = __amalgam__
@@ -57,14 +55,16 @@ else:
         _sys.modules['xonsh.proc'] = __amalgam__
         xontribs = __amalgam__
         _sys.modules['xonsh.xontribs'] = __amalgam__
-        commands_cache = __amalgam__
-        _sys.modules['xonsh.commands_cache'] = __amalgam__
-        environ = __amalgam__
-        _sys.modules['xonsh.environ'] = __amalgam__
+        dirstack = __amalgam__
+        _sys.modules['xonsh.dirstack'] = __amalgam__
         history = __amalgam__
         _sys.modules['xonsh.history'] = __amalgam__
         inspectors = __amalgam__
         _sys.modules['xonsh.inspectors'] = __amalgam__
+        commands_cache = __amalgam__
+        _sys.modules['xonsh.commands_cache'] = __amalgam__
+        environ = __amalgam__
+        _sys.modules['xonsh.environ'] = __amalgam__
         base_shell = __amalgam__
         _sys.modules['xonsh.base_shell'] = __amalgam__
         replay = __amalgam__

From 5985c3d33bc173b9b55256e318c27176c9a545cb Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sun, 28 Aug 2016 10:32:10 -0400
Subject: [PATCH 163/190] hot fix for pytest events

---
 tests/test_events.py | 4 +++-
 xonsh/events.py      | 7 +++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/tests/test_events.py b/tests/test_events.py
index 91b59c4f8..d425c4aa3 100644
--- a/tests/test_events.py
+++ b/tests/test_events.py
@@ -111,5 +111,7 @@ def test_transmogrify_by_string(events):
     assert inspect.getdoc(events.on_test) == docstring
 
 def test_typos(xonsh_builtins):
-    for ev in vars(xonsh_builtins.events).values():
+    for name, ev in vars(xonsh_builtins.events).items():
+        if 'pytest' in name:
+            continue
         assert inspect.getdoc(ev)
diff --git a/xonsh/events.py b/xonsh/events.py
index dd1cf773d..6601ee4c7 100644
--- a/xonsh/events.py
+++ b/xonsh/events.py
@@ -231,11 +231,10 @@ class EventManager:
             newevent.add(handler)
 
     def __getattr__(self, name):
-        """
-        Get an event, if it doesn't already exist.
-        """
+        """Get an event, if it doesn't already exist."""
         # This is only called if the attribute doesn't exist, so create the Event...
-        e = type(name, (Event,), {'__doc__': None})()  # (A little bit of magic to enable docstrings to work right)
+        # (A little bit of magic to enable docstrings to work right)
+        e = type(name, (Event,), {'__doc__': None})()
         # ... and save it.
         setattr(self, name, e)
         # Now it exists, and we won't be called again.

From e19a0aa1e59c9338e5120a688a35fc7473a33ca7 Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Sun, 28 Aug 2016 13:11:36 -0400
Subject: [PATCH 164/190] some copy edits from @gforsyth, I shouldn't write
 docs while jet lagged

---
 docs/tutorial_macros.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/tutorial_macros.rst b/docs/tutorial_macros.rst
index 145b41af9..5449eb81a 100644
--- a/docs/tutorial_macros.rst
+++ b/docs/tutorial_macros.rst
@@ -73,11 +73,11 @@ with the following:
     f!(x, x + 42)
     g!([y, 43, 44], f!(z))
 
-Not so bad, right?  So what actually happens when to the arguments when used
+Not so bad, right?  So what actually happens to the arguments when used
 in a macro call?  Well, that depends on the definition of the function. In
 particular, each argument in the macro call is matched up with the corresponding
 parameter annotation in the callable's signature.  For example, say we have
-an ``identity()`` function that is annotates its sole argument as a string:
+an ``identity()`` function that annotates its sole argument as a string:
 
 .. code-block:: xonsh
 
@@ -98,7 +98,7 @@ even if that object is not a string:
     >>> identity(identity)
     <function __main__.identity>
 
-However, if we perform macro calls instead we are now guaranteed to get a
+However, if we perform macro calls instead we are now guaranteed to get
 the string of the source code that is in the macro call:
 
 .. code-block:: xonshcon
@@ -262,7 +262,7 @@ There are six kinds of annotations that macros are able to interpret:
      - The type of the argument after it has been evaluated.
 
 These annotations allow you to hook into whichever stage of the compilation
-that you desire. It is important note that the string form of the arguments
+that you desire. It is important to note that the string form of the arguments
 is split and stripped (as described above) prior to conversion to the
 annotation type.
 

From 0d791d1bea78498713e6d298fd24f74169aa8516 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sun, 28 Aug 2016 13:37:23 -0400
Subject: [PATCH 165/190] Add events to vox.

---
 tests/test_vox.py | 30 ++++++++++++++++++++++++++++++
 xontrib/voxapi.py | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)

diff --git a/tests/test_vox.py b/tests/test_vox.py
index 18d7dc4f8..b5d38fc88 100644
--- a/tests/test_vox.py
+++ b/tests/test_vox.py
@@ -14,9 +14,23 @@ def test_crud(xonsh_builtins, tmpdir):
     Creates a virtual environment, gets it, enumerates it, and then deletes it.
     """
     xonsh_builtins.__xonsh_env__['VIRTUALENV_HOME'] = str(tmpdir)
+
+    last_event = None
+
+    @xonsh_builtins.events.vox_on_create
+    def create(name):
+        nonlocal last_event
+        last_event = 'create', name
+
+    @xonsh_builtins.events.vox_on_delete
+    def delete(name):
+        nonlocal last_event
+        last_event = 'delete', name
+
     vox = Vox()
     vox.create('spam')
     assert stat.S_ISDIR(tmpdir.join('spam').stat().mode)
+    assert last_event == ('create', 'spam')
 
     env, bin = vox['spam']
     assert env == str(tmpdir.join('spam'))
@@ -27,6 +41,7 @@ def test_crud(xonsh_builtins, tmpdir):
     del vox['spam']
 
     assert not tmpdir.join('spam').check()
+    assert last_event == ('delete', 'spam')
 
 
 @skip_if_on_conda
@@ -37,12 +52,27 @@ def test_activate(xonsh_builtins, tmpdir):
     xonsh_builtins.__xonsh_env__['VIRTUALENV_HOME'] = str(tmpdir)
     # I consider the case that the user doesn't have a PATH set to be unreasonable
     xonsh_builtins.__xonsh_env__.setdefault('PATH', [])
+
+    last_event = None
+
+    @xonsh_builtins.events.vox_on_activate
+    def activate(name):
+        nonlocal last_event
+        last_event = 'activate', name
+
+    @xonsh_builtins.events.vox_on_deactivate
+    def deactivate(name):
+        nonlocal last_event
+        last_event = 'deactivate', name
+
     vox = Vox()
     vox.create('spam')
     vox.activate('spam')
     assert xonsh_builtins.__xonsh_env__['VIRTUAL_ENV'] == vox['spam'].env
+    assert last_event == ('activate', 'spam')
     vox.deactivate()
     assert 'VIRTUAL_ENV' not in xonsh_builtins.__xonsh_env__
+    assert last_event == ('deactivate', 'spam')
 
 
 @skip_if_on_conda
diff --git a/xontrib/voxapi.py b/xontrib/voxapi.py
index 919ac6f87..1adf7eb80 100644
--- a/xontrib/voxapi.py
+++ b/xontrib/voxapi.py
@@ -7,6 +7,35 @@ import collections.abc
 
 from xonsh.platform import ON_POSIX, ON_WINDOWS, scandir
 
+# This is because builtins aren't globally created during testing.
+# FIXME: Is there a better way?
+from xonsh.events import events
+
+
+events.doc('vox_on_create', """
+vox_on_create(env: str) -> None
+
+Fired after an environment is created.
+""")
+
+events.doc('vox_on_activate', """
+vox_on_activate(env: str) -> None
+
+Fired after an environment is activated.
+""")
+
+events.doc('vox_on_deactivate', """
+vox_on_deactivate(env: str) -> None
+
+Fired after an environment is deactivated.
+""")
+
+events.doc('vox_on_delete', """
+vox_on_delete(env: str) -> None
+
+Fired after an environment is deleted (through vox).
+""")
+
 
 VirtualEnvironment = collections.namedtuple('VirtualEnvironment', ['env', 'bin'])
 
@@ -60,6 +89,7 @@ class Vox(collections.abc.Mapping):
             env_path,
             system_site_packages=system_site_packages, symlinks=symlinks,
             with_pip=with_pip)
+        events.vox_on_create.fire(name)
 
     def upgrade(self, name, *, symlinks=False, with_pip=True):
         """Create a virtual environment in $VIRTUALENV_HOME with python3's ``venv``.
@@ -185,6 +215,8 @@ class Vox(collections.abc.Mapping):
         if 'PYTHONHOME' in env:
             type(self).oldvars['PYTHONHOME'] = env.pop('PYTHONHOME')
 
+        events.vox_on_activate.fire(name)
+
     def deactivate(self):
         """
         Deactive the active virtual environment. Returns the name of it.
@@ -203,6 +235,7 @@ class Vox(collections.abc.Mapping):
 
         env.pop('VIRTUAL_ENV')
 
+        events.vox_on_deactivate.fire(env_name)
         return env_name
 
     def __delitem__(self, name):
@@ -222,3 +255,5 @@ class Vox(collections.abc.Mapping):
             # No current venv, ... fails
             pass
         shutil.rmtree(env_path)
+
+        events.vox_on_delete.fire(name)

From d8bc83cd517f7af3d6d3ba985e8ca2da9d509824 Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sun, 28 Aug 2016 13:43:08 -0400
Subject: [PATCH 166/190] Add some documentations

---
 xontrib/voxapi.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/xontrib/voxapi.py b/xontrib/voxapi.py
index 1adf7eb80..d748bd5ac 100644
--- a/xontrib/voxapi.py
+++ b/xontrib/voxapi.py
@@ -1,4 +1,13 @@
-"""API for Vox, the Python virtual environment manager for xonsh."""
+"""
+API for Vox, the Python virtual environment manager for xonsh.
+
+Vox defines several events related to the life cycle of virtual environments:
+
+* ``vox_on_create(env: str) -> None``
+* ``vox_on_activate(env: str) -> None``
+* ``vox_on_deactivate(env: str) -> None``
+* ``vox_on_delete(env: str) -> None``
+"""
 import os
 import venv
 import shutil
@@ -41,11 +50,11 @@ VirtualEnvironment = collections.namedtuple('VirtualEnvironment', ['env', 'bin']
 
 
 class EnvironmentInUse(Exception):
-    pass
+    """The given environment is currently activated, and the operation cannot be performed."""
 
 
 class NoEnvironmentActive(Exception):
-    pass
+    """No environment is currently activated, and the operation cannot be performed."""
 
 
 class Vox(collections.abc.Mapping):

From ffb4e5e0359379f353e2dc63bff640b996bac60d Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Sun, 28 Aug 2016 22:17:26 +0300
Subject: [PATCH 167/190] docs/tests

---
 tests/test_history.py | 19 ++++++++-------
 xonsh/history.py      | 55 ++++++++++++++++++++++++++++++++-----------
 2 files changed, 52 insertions(+), 22 deletions(-)

diff --git a/tests/test_history.py b/tests/test_history.py
index bc9bd1a81..f908c7188 100644
--- a/tests/test_history.py
+++ b/tests/test_history.py
@@ -189,15 +189,18 @@ def test_parser_show(args, exp):
 
 # CMDS = ['ls', 'cat hello kitty', 'abc', 'def', 'touch me', 'grep from me']
 
-def test_history_getitem(hist, xonsh_builtins):
+
+@pytest.mark.parametrize('index, exp',[
+    (-1, 'grep from me'),
+    ('hello', 'cat hello kitty'),
+    ((-1, -1), 'me'),
+    (('hello', 0), 'cat'),
+    ((-1, 0:2), 'grep from'),
+    (('kitty', 1:), 'hello kitty')
+])
+def test_history_getitem(index, exp, hist, xonsh_builtins):
     xonsh_builtins.__xonsh_env__['HISTCONTROL'] = set()
     for ts,cmd in enumerate(CMDS):  # populate the shell history
         hist.append({'inp': cmd, 'rtn': 0, 'ts':(ts+1, ts+1.5)})
 
-    # indexing
-    assert hist[-1] == 'grep from me'
-    assert hist['hello'] == 'cat hello kitty'
-
-    # word parts
-    assert hist[-1, -1] == 'me'
-    assert hist['hello', 1] == 'hello'
+    assert hist[index] == exp
diff --git a/xonsh/history.py b/xonsh/history.py
index d3a1c97ee..709bbb3e8 100644
--- a/xonsh/history.py
+++ b/xonsh/history.py
@@ -290,7 +290,7 @@ def _curr_session_parser(hist=None, **kwargs):
     """
     if hist is None:
         hist = builtins.__xonsh_history__
-    return hist._get()
+    return iter(hist)
 
 
 def _zsh_hist_parser(location=None, **kwargs):
@@ -486,17 +486,36 @@ def _hist_show(ns, *args, **kwargs):
 class History(object):
     """Xonsh session history.
 
-    History object supports indexing with some rules for extra functionality:
+    Indexing
+    --------
+    History object acts like a sequence that can be indexed in a special way
+    that adds extra functionality. At the moment only history from the
+    current session can be retrieved. Note that the most recent command
+    is the last item in history.
 
-        - index must be one of string, int or tuple of length two
-        - if the index is an int the appropriate command in order is returned
-        - if the index is a string the last command that contains
-          the string is returned
-        - if the index is a tuple:
+    The index acts as a filter with two parts, command and argument,
+    separated by comma.  Based on the type of each part different 
+    filtering can be achieved,
 
-            - the first item follows the previous
-              two rules.
-            - the second item is the slice of the arguments to be returned
+        for the command part:
+
+            - an int returns the command in that position.
+            - a slice returns a list of commands.
+            - a string returns the most recent command containing the string.
+
+        for the argument part:
+
+            - an int returns the argument of the command in that position.
+            - a slice returns a part of the command based on the argument
+              position.
+
+    The argument part of the filter can be omitted but the command part is
+    required.
+
+    Command arguments are separated by white space.
+
+    If the command filter produces a list then the argument filter
+    will be applied to each element of that list.
 
     Attributes
     ----------
@@ -615,7 +634,7 @@ class History(object):
         self.buffer.clear()
         return hf
 
-    def _get(self):
+    def __iter__(self):
         """Get current session history.
 
         Yields
@@ -629,13 +648,15 @@ class History(object):
             yield (c, t, ind)
 
     def __getitem__(self, item):
+        """Retrieve history parts based on filtering rules,
+        look at ``History`` docs for more info."""
         # accept only one of str, int, tuple of length two
         if isinstance(item, tuple):
             pattern, part = item
         else:
             pattern, part = item, None
         # find command
-        hist = [c for c, *_ in self._get()]
+        hist = [c for c, *_ in self]
         command = None
         if isinstance(pattern, str):
             for command in reversed(hist):
@@ -649,10 +670,16 @@ class History(object):
                             'str, int, tuple of length two')
         # get command part
         if command and part:
-            part = ensure_slice(part)
-            command = ' '.join(command.split()[part])
+            s = ensure_slice(part)
+            print('SLICE:', s)
+            command = ' '.join(command.split()[s])
         return command
 
+    def __setitem__(self, *args):
+        raise PermissionError('You cannot change history! '
+                              'you can create new though.')
+
+
 
 def _hist_info(ns, hist):
     """Display information about the shell history."""

From db8f5bc0d52c2b8c61c8a705058b38030f403023 Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Sun, 28 Aug 2016 22:26:50 +0300
Subject: [PATCH 168/190] ensure_slice bug for 0

---
 tests/test_tools.py | 1 +
 xonsh/tools.py      | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_tools.py b/tests/test_tools.py
index 34ffcd662..03d072ad1 100644
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -816,6 +816,7 @@ def test_bool_or_int_to_str(inp, exp):
 
 @pytest.mark.parametrize('inp, exp', [
         (42, slice(42, 43)),
+        (0, slice(0,1)),
         (None, slice(None, None, None)),
         (slice(1,2), slice(1,2)),
         ('-1', slice(-1, None, None)),
diff --git a/xonsh/tools.py b/xonsh/tools.py
index 126291508..bd78b82e9 100644
--- a/xonsh/tools.py
+++ b/xonsh/tools.py
@@ -928,7 +928,7 @@ def SLICE_REG():
 
 def ensure_slice(x):
     """Try to convert an object into a slice, complain on failure"""
-    if not x:
+    if not x and x != 0:
         return slice(None)
     elif isinstance(x, slice):
         return x

From a3c1dfd28e6015bac41e0e63b52fdba4fc333427 Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Mon, 29 Aug 2016 02:45:36 +0300
Subject: [PATCH 169/190] generators everywhere

---
 tests/test_history.py |  6 ++---
 tests/test_tools.py   |  2 +-
 xonsh/history.py      | 62 ++++++++++++++++++++++++++++---------------
 3 files changed, 44 insertions(+), 26 deletions(-)

diff --git a/tests/test_history.py b/tests/test_history.py
index f908c7188..e20dd705d 100644
--- a/tests/test_history.py
+++ b/tests/test_history.py
@@ -190,13 +190,13 @@ def test_parser_show(args, exp):
 # CMDS = ['ls', 'cat hello kitty', 'abc', 'def', 'touch me', 'grep from me']
 
 
-@pytest.mark.parametrize('index, exp',[
+@pytest.mark.parametrize('index, exp', [
     (-1, 'grep from me'),
     ('hello', 'cat hello kitty'),
     ((-1, -1), 'me'),
     (('hello', 0), 'cat'),
-    ((-1, 0:2), 'grep from'),
-    (('kitty', 1:), 'hello kitty')
+    ((-1, slice(0,2)), 'grep from'),
+    (('kitty', slice(1,3)), 'hello kitty')
 ])
 def test_history_getitem(index, exp, hist, xonsh_builtins):
     xonsh_builtins.__xonsh_env__['HISTCONTROL'] = set()
diff --git a/tests/test_tools.py b/tests/test_tools.py
index 03d072ad1..387a3c525 100644
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -816,7 +816,7 @@ def test_bool_or_int_to_str(inp, exp):
 
 @pytest.mark.parametrize('inp, exp', [
         (42, slice(42, 43)),
-        (0, slice(0,1)),
+        (0, slice(0, 1)),
         (None, slice(None, None, None)),
         (slice(1,2), slice(1,2)),
         ('-1', slice(-1, None, None)),
diff --git a/xonsh/history.py b/xonsh/history.py
index 709bbb3e8..733c2da65 100644
--- a/xonsh/history.py
+++ b/xonsh/history.py
@@ -649,31 +649,49 @@ class History(object):
 
     def __getitem__(self, item):
         """Retrieve history parts based on filtering rules,
-        look at ``History`` docs for more info."""
-        # accept only one of str, int, tuple of length two
+        see ``History`` docs for more info. Accepts one of
+        int, string, slice or tuple of length two.
+        """
         if isinstance(item, tuple):
-            pattern, part = item
+            cmd_pat, arg_pat = item
         else:
-            pattern, part = item, None
-        # find command
-        hist = [c for c, *_ in self]
-        command = None
-        if isinstance(pattern, str):
-            for command in reversed(hist):
-                if pattern in command:
-                    break
-        elif isinstance(pattern, int):
-            # catch index error?
-            command = hist[pattern]
+            cmd_pat, arg_pat = item, None
+        cmds = (c for c, *_ in self)
+        cmds = self._cmd_filter(cmds, cmd_pat)
+        if arg_pat is not None:
+            cmds = self._args_filter(cmds, arg_pat)
+        cmds = list(cmds)
+        if len(cmds) == 1:
+            return cmds[0]
         else:
-            raise TypeError('history index must be of type '
-                            'str, int, tuple of length two')
-        # get command part
-        if command and part:
-            s = ensure_slice(part)
-            print('SLICE:', s)
-            command = ' '.join(command.split()[s])
-        return command
+            return cmds
+
+    def _cmd_filter(self, cmds, pat):
+        if isinstance(pat, (int, slice)):
+            s = [ensure_slice(pat)]
+            yield from _hist_get_portion(cmds, s)
+        elif isinstance(pat, str):
+            for command in reversed(list(cmds)):
+                if pat in command:
+                   yield command
+        else:
+            raise TypeError('Command filter must be '
+                            'string, int or slice')
+
+    def _args_filter(self, cmds, pat):
+        args = None
+        if isinstance(pat, (int, slice)):
+            s = ensure_slice(pat)
+            for command in cmds:
+                yield ' '.join(command.split()[s])
+        else:
+            raise TypeError('Argument filter must be of '
+                            'int or slice')
+        return args
+
+
+
+
 
     def __setitem__(self, *args):
         raise PermissionError('You cannot change history! '

From 2e24b9a69878f1b45cf7cc72157b6d26f3899acd Mon Sep 17 00:00:00 2001
From: Jamie Bliss <astronouth7303@gmail.com>
Date: Sun, 28 Aug 2016 23:34:11 -0400
Subject: [PATCH 170/190] Deal with leading backslash, too

---
 xontrib/voxapi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xontrib/voxapi.py b/xontrib/voxapi.py
index d748bd5ac..c3a9069fb 100644
--- a/xontrib/voxapi.py
+++ b/xontrib/voxapi.py
@@ -198,7 +198,7 @@ class Vox(collections.abc.Mapping):
         env_path = builtins.__xonsh_env__['VIRTUAL_ENV']
         if env_path.startswith(self.venvdir):
             name = env_path[len(self.venvdir):]
-            if name[0] == '/':
+            if name[0] in '/\\':
                 name = name[1:]
             return name
         else:

From b3b38eee611a8e5456540e41caabedbc64a7584b Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Mon, 29 Aug 2016 14:07:54 +0800
Subject: [PATCH 171/190] fix path completion for empty directories

Before this commit, `ls /tmp/empty_dir/<TAB>` would results `ls //`,
this commit fixes that bug
---
 xonsh/completers/path.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/xonsh/completers/path.py b/xonsh/completers/path.py
index b137441fc..f005776f3 100644
--- a/xonsh/completers/path.py
+++ b/xonsh/completers/path.py
@@ -186,12 +186,11 @@ def _splitpath(path):
 
 def _splitpath_helper(path, sofar=()):
     folder, path = os.path.split(path)
-    if path == "":
+    if path:
+        sofar = sofar + (path, )
+    if not folder or folder == xt.get_sep():
         return sofar[::-1]
-    elif folder == "":
-        return (sofar + (path, ))[::-1]
-    else:
-        return _splitpath_helper(folder, sofar + (path, ))
+    return _splitpath_helper(folder, sofar)
 
 
 def subsequence_match(ref, typed, csc):

From 07f9c50e7e5695ae3d33a289e5f170f8067d40da Mon Sep 17 00:00:00 2001
From: Morten Enemark Lund <melund@gmail.com>
Date: Tue, 5 Jul 2016 09:10:22 +0200
Subject: [PATCH 172/190] Added framework for collecting and running *.xsh
 files with py.test

---
 tests/conftest.py | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/tests/conftest.py b/tests/conftest.py
index 06d4ca443..848df9ed5 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,4 +1,7 @@
+import glob
 import builtins
+from traceback import format_list, extract_tb
+
 import pytest
 from tools import DummyShell, sp
 import xonsh.built_ins
@@ -9,6 +12,44 @@ from xonsh.events import events
 import glob
 
 
+
+def _limited_traceback(excinfo):
+    """ Return a formatted traceback with all the stack
+        from this frame (i.e __file__) up removed
+    """
+    tb = extract_tb(excinfo.tb)
+    try:
+        idx = [__file__ in e for e in tb].index(True)
+        return format_list(tb[idx+1:])
+    except ValueError:
+        return format_list(tb)
+
+def pytest_collect_file(parent, path):
+    if path.ext == ".xsh" and path.basename.startswith("test"):
+        return XshFile(path, parent)
+
+class XshFile(pytest.File):
+    def collect(self):
+        name = self.fspath.basename
+        yield XshItem(name, self)
+
+class XshItem(pytest.Item):
+    def __init__(self, name, parent):
+        super().__init__(name, parent)
+
+    def runtest(self):
+        xonsh_main([str(self.parent.fspath), '--no-script-cache', '--no-rc'])
+
+    def repr_failure(self, excinfo):
+        """ called when self.runtest() raises an exception. """
+        formatted_tb = _limited_traceback(excinfo)
+        formatted_tb.insert(0, "Xonsh execution failed\n")
+        return "".join(formatted_tb)
+
+    def reportinfo(self):
+        return self.fspath, 0, "usecase: %s" % self.name
+
+
 @pytest.fixture
 def xonsh_execer(monkeypatch):
     """Initiate the Execer with a mocked nop `load_builtins`"""

From eb79dd119ac64a3aa23cfbea599701e134175178 Mon Sep 17 00:00:00 2001
From: Morten Enemark Lund <melund@gmail.com>
Date: Tue, 5 Jul 2016 09:17:35 +0200
Subject: [PATCH 173/190] Added a changelog entry

---
 news/test_xsh.rst | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 news/test_xsh.rst

diff --git a/news/test_xsh.rst b/news/test_xsh.rst
new file mode 100644
index 000000000..3da88ef1f
--- /dev/null
+++ b/news/test_xsh.rst
@@ -0,0 +1,12 @@
+**Added:** 
+* Added a local py.test framwork to collect and run `test_*.xsh` files. 
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

From 7ff95bf6d7ab77da7e2f2dc24ac5629e199c1838 Mon Sep 17 00:00:00 2001
From: Morten Enemark Lund <melund@gmail.com>
Date: Tue, 5 Jul 2016 13:37:09 +0200
Subject: [PATCH 174/190] Add an example test file

---
 tests/test_xonsh.xsh | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 tests/test_xonsh.xsh

diff --git a/tests/test_xonsh.xsh b/tests/test_xonsh.xsh
new file mode 100644
index 000000000..9688f2456
--- /dev/null
+++ b/tests/test_xonsh.xsh
@@ -0,0 +1,19 @@
+
+
+assert 1 + 1 == 2
+
+
+$USER = 'snail'
+
+x = 'USER'
+assert x in ${...}
+assert ${'U' + 'SER'} == 'snail'
+
+
+echo "Yoo hoo"
+$(echo $USER)
+
+x = 'xonsh'
+y = 'party'
+out = $(echo @(x + ' ' + y))
+assert out == 'xonsh party\n'

From fe4e7af4bbc27544946ca802501cfe16bd837618 Mon Sep 17 00:00:00 2001
From: Morten Enemark Lund <melund@gmail.com>
Date: Thu, 18 Aug 2016 21:49:18 +0200
Subject: [PATCH 175/190] Moved plugin from conftest to its own module

---
 tests/conftest.py      | 42 +++----------------------------
 xonsh/pytest_plugin.py | 57 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+), 38 deletions(-)
 create mode 100644 xonsh/pytest_plugin.py

diff --git a/tests/conftest.py b/tests/conftest.py
index 848df9ed5..61412deb3 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,53 +1,19 @@
 import glob
 import builtins
-from traceback import format_list, extract_tb
 
 import pytest
-from tools import DummyShell, sp
+
 import xonsh.built_ins
+
 from xonsh.built_ins import ensure_list_of_strs
 from xonsh.execer import Execer
 from xonsh.tools import XonshBlockError
 from xonsh.events import events
 import glob
 
+from tools import DummyShell, sp
 
-
-def _limited_traceback(excinfo):
-    """ Return a formatted traceback with all the stack
-        from this frame (i.e __file__) up removed
-    """
-    tb = extract_tb(excinfo.tb)
-    try:
-        idx = [__file__ in e for e in tb].index(True)
-        return format_list(tb[idx+1:])
-    except ValueError:
-        return format_list(tb)
-
-def pytest_collect_file(parent, path):
-    if path.ext == ".xsh" and path.basename.startswith("test"):
-        return XshFile(path, parent)
-
-class XshFile(pytest.File):
-    def collect(self):
-        name = self.fspath.basename
-        yield XshItem(name, self)
-
-class XshItem(pytest.Item):
-    def __init__(self, name, parent):
-        super().__init__(name, parent)
-
-    def runtest(self):
-        xonsh_main([str(self.parent.fspath), '--no-script-cache', '--no-rc'])
-
-    def repr_failure(self, excinfo):
-        """ called when self.runtest() raises an exception. """
-        formatted_tb = _limited_traceback(excinfo)
-        formatted_tb.insert(0, "Xonsh execution failed\n")
-        return "".join(formatted_tb)
-
-    def reportinfo(self):
-        return self.fspath, 0, "usecase: %s" % self.name
+pytest_plugins = ("xonsh.pytest_plugin",)
 
 
 @pytest.fixture
diff --git a/xonsh/pytest_plugin.py b/xonsh/pytest_plugin.py
new file mode 100644
index 000000000..23c4bc9b2
--- /dev/null
+++ b/xonsh/pytest_plugin.py
@@ -0,0 +1,57 @@
+import sys
+import importlib
+from traceback import format_list, extract_tb
+
+import pytest
+import xonsh.imphooks
+
+
+def pytest_configure(config):
+    xonsh.imphooks.install_hook()
+
+
+def _limited_traceback(excinfo):
+    """ Return a formatted traceback with all the stack
+        from this frame (i.e __file__) up removed
+    """
+    tb = extract_tb(excinfo.tb)
+    try:
+        idx = [__file__ in e for e in tb].index(True)
+        return format_list(tb[idx+1:])
+    except ValueError:
+        return format_list(tb)
+
+
+def pytest_collect_file(parent, path):
+    if path.ext.lower() == ".xsh" and path.basename.startswith("test_"):
+        return XshFile(path, parent)
+
+
+class XshFile(pytest.File):
+    def collect(self):
+        sys.path.append(self.fspath.dirname)
+        mod = importlib.import_module(self.fspath.purebasename)
+        sys.path.pop(0)
+        tests = [t for t in dir(mod) if t.startswith('test_')]
+        for test_name in tests:
+            obj = getattr(mod, test_name)
+            if hasattr(obj, '__call__'):
+                yield XshItem(test_name, self, obj)
+
+
+class XshItem(pytest.Item):
+    def __init__(self, name, parent, test_func):
+        super().__init__(name, parent)
+        self._test_func = test_func
+
+    def runtest(self):
+        self._test_func()
+
+    def repr_failure(self, excinfo):
+        """ called when self.runtest() raises an exception. """
+        formatted_tb = _limited_traceback(excinfo)
+        formatted_tb.insert(0, "xonsh execution failed\n")
+        return "".join(formatted_tb)
+
+    def reportinfo(self):
+        return self.fspath, 0, "xonsh test: {}".format(self.name)

From 4673a57acee2d73c35a8f5aa4198001fe9207d6f Mon Sep 17 00:00:00 2001
From: Morten Enemark Lund <melund@gmail.com>
Date: Thu, 18 Aug 2016 21:49:44 +0200
Subject: [PATCH 176/190] Add pytest entry point to setup.py

---
 setup.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index b5823e4c1..b59209759 100755
--- a/setup.py
+++ b/setup.py
@@ -310,10 +310,12 @@ def main():
         skw['entry_points'] = {
             'pygments.lexers': ['xonsh = xonsh.pyghooks:XonshLexer',
                                 'xonshcon = xonsh.pyghooks:XonshConsoleLexer'],
-            }
+            'pytest11': ['xonsh = xonsh.pytest_plugin'] 
+        }
         skw['cmdclass']['develop'] = xdevelop
     setup(**skw)
 
 
 if __name__ == '__main__':
     main()
+

From 8e61cb540209b6368580ef35d5f7da8974074734 Mon Sep 17 00:00:00 2001
From: Morten Enemark Lund <melund@gmail.com>
Date: Thu, 18 Aug 2016 21:50:06 +0200
Subject: [PATCH 177/190] Add a simple xsh test file

---
 tests/test_xonsh.xsh | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/tests/test_xonsh.xsh b/tests/test_xonsh.xsh
index 9688f2456..f68666ff1 100644
--- a/tests/test_xonsh.xsh
+++ b/tests/test_xonsh.xsh
@@ -1,19 +1,18 @@
 
 
-assert 1 + 1 == 2
+def test_simple():
+  assert 1 + 1 == 2
 
+  
+def test_envionment():
+  $USER = 'snail'
+  x = 'USER'
+  assert x in ${...}
+  assert ${'U' + 'SER'} == 'snail'
 
-$USER = 'snail'
-
-x = 'USER'
-assert x in ${...}
-assert ${'U' + 'SER'} == 'snail'
-
-
-echo "Yoo hoo"
-$(echo $USER)
-
-x = 'xonsh'
-y = 'party'
-out = $(echo @(x + ' ' + y))
-assert out == 'xonsh party\n'
+  
+def test_xonsh_party():
+  x = 'xonsh'
+  y = 'party'
+  out = $(echo @(x + ' ' + y))
+  assert out == 'xonsh party\n'

From 06ea1dac73ac39800fc8260e5047cfecec016314 Mon Sep 17 00:00:00 2001
From: Morten Enemark Lund <melund@gmail.com>
Date: Thu, 18 Aug 2016 21:52:41 +0200
Subject: [PATCH 178/190] Update changelog

---
 news/test_xsh.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/news/test_xsh.rst b/news/test_xsh.rst
index 3da88ef1f..7b527b477 100644
--- a/news/test_xsh.rst
+++ b/news/test_xsh.rst
@@ -1,5 +1,6 @@
 **Added:** 
-* Added a local py.test framwork to collect and run `test_*.xsh` files. 
+
+* Added a py.test plugin to collect `test_*.xsh` files and run `test_*()` functions. 
 
 **Changed:** None
 

From 500963ae8b30d9652683f4843cd43bc2d3062e81 Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Mon, 29 Aug 2016 13:42:40 +0300
Subject: [PATCH 179/190] get_portions

---
 news/history-api.rst |  4 ++++
 tests/test_tools.py  | 19 +++++++++++++++++--
 xonsh/history.py     | 43 +++++++++++--------------------------------
 xonsh/tools.py       | 27 +++++++++++++++++++++++++--
 4 files changed, 57 insertions(+), 36 deletions(-)

diff --git a/news/history-api.rst b/news/history-api.rst
index 849d72a55..e3ee77161 100644
--- a/news/history-api.rst
+++ b/news/history-api.rst
@@ -2,6 +2,8 @@
 
 * ``History`` methods ``_get`` and ``__getitem__``
 
+* ``tools.get_portions`` that yields parts of an iterable
+
 **Changed:**
 
 * ``_curr_session_parser`` now uses ``History_.get``
@@ -12,6 +14,8 @@
 
 * ``History`` method ``show``
 
+* ``_hist_get_portion`` in favor of ``tools.get_portions``
+
 **Fixed:** None
 
 **Security:** None
diff --git a/tests/test_tools.py b/tests/test_tools.py
index 387a3c525..f9b610973 100644
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -24,7 +24,7 @@ from xonsh.tools import (
     to_dynamic_cwd_tuple, to_logfile_opt, pathsep_to_set, set_to_pathsep,
     is_string_seq, pathsep_to_seq, seq_to_pathsep, is_nonstring_seq_of_strings,
     pathsep_to_upper_seq, seq_to_upper_pathsep, expandvars, is_int_as_str, is_slice_as_str,
-    ensure_timestamp,
+    ensure_timestamp, get_portions
     )
 from xonsh.commands_cache import CommandsCache
 from xonsh.built_ins import expand_path
@@ -836,7 +836,22 @@ def test_ensure_slice(inp, exp):
     assert exp == obs
 
 
-@pytest.mark.parametrize('inp', [
+@pytest.mark.parametrize('inp, exp', [
+    ((range(50), slice(25, 40)),
+     list(i for i in range(25,40))),
+
+    (([1,2,3,4,5,6,7,8,9,10], [slice(1,4), slice(6, None)]),
+     [2, 3, 4, 7, 8, 9, 10]),
+
+    (([1,2,3,4,5], [slice(-2, None), slice(-5, -3)]),
+     [4, 5, 1, 2]),
+])
+def test_get_portions(inp, exp):
+    obs = get_portions(*inp)
+    assert list(obs) == exp
+
+
+pytest.mark.parametrize('inp', [
     '42.3',
     '3:asd5:1',
     'test' ,
diff --git a/xonsh/history.py b/xonsh/history.py
index 733c2da65..00a80069c 100644
--- a/xonsh/history.py
+++ b/xonsh/history.py
@@ -11,15 +11,13 @@ import builtins
 import collections
 import datetime
 import functools
-import itertools
 import threading
-import collections
 import collections.abc as cabc
 
 from xonsh.lazyasd import lazyobject
 from xonsh.lazyjson import LazyJSON, ljdump, LJNode
-from xonsh.tools import (ensure_slice, to_history_tuple,
-                         expanduser_abs_path, ensure_timestamp)
+from xonsh.tools import (ensure_slice, to_history_tuple, is_string,
+                         get_portions, expanduser_abs_path, ensure_timestamp)
 from xonsh.diff_history import _dh_create_parser, _dh_main_action
 
 
@@ -392,20 +390,6 @@ def _hist_create_parser():
     return p
 
 
-def _hist_get_portion(commands, slices):
-    """Yield from portions of history commands."""
-    if len(slices) == 1:
-        s = slices[0]
-        try:
-            yield from itertools.islice(commands, s.start, s.stop, s.step)
-            return
-        except ValueError:  # islice failed
-            pass
-    commands = list(commands)
-    for s in slices:
-        yield from commands[s]
-
-
 def _hist_filter_ts(commands, start_time, end_time):
     """Yield only the commands between start and end time."""
     for cmd in commands:
@@ -437,7 +421,7 @@ def _hist_get(session='session', *, slices=None, datetime_format=None,
     if slices:
         # transform/check all slices
         slices = [ensure_slice(s) for s in slices]
-        cmds = _hist_get_portion(cmds, slices)
+        cmds = get_portions(cmds, slices)
     if start_time or end_time:
         if start_time is None:
             start_time = 0.0
@@ -494,7 +478,7 @@ class History(object):
     is the last item in history.
 
     The index acts as a filter with two parts, command and argument,
-    separated by comma.  Based on the type of each part different 
+    separated by comma.  Based on the type of each part different
     filtering can be achieved,
 
         for the command part:
@@ -514,8 +498,8 @@ class History(object):
 
     Command arguments are separated by white space.
 
-    If the command filter produces a list then the argument filter
-    will be applied to each element of that list.
+    If the filtering produces only one result it is
+    returned as a string else a list of strings is returned.
 
     Attributes
     ----------
@@ -668,12 +652,12 @@ class History(object):
 
     def _cmd_filter(self, cmds, pat):
         if isinstance(pat, (int, slice)):
-            s = [ensure_slice(pat)]
-            yield from _hist_get_portion(cmds, s)
-        elif isinstance(pat, str):
+            s = ensure_slice(pat)
+            yield from get_portions(cmds, s)
+        elif is_string(pat):
             for command in reversed(list(cmds)):
                 if pat in command:
-                   yield command
+                    yield command
         else:
             raise TypeError('Command filter must be '
                             'string, int or slice')
@@ -685,20 +669,15 @@ class History(object):
             for command in cmds:
                 yield ' '.join(command.split()[s])
         else:
-            raise TypeError('Argument filter must be of '
+            raise TypeError('Argument filter must be '
                             'int or slice')
         return args
 
-
-
-
-
     def __setitem__(self, *args):
         raise PermissionError('You cannot change history! '
                               'you can create new though.')
 
 
-
 def _hist_info(ns, hist):
     """Display information about the shell history."""
     data = collections.OrderedDict()
diff --git a/xonsh/tools.py b/xonsh/tools.py
index bd78b82e9..23e487af9 100644
--- a/xonsh/tools.py
+++ b/xonsh/tools.py
@@ -25,6 +25,7 @@ import ctypes
 import datetime
 import functools
 import glob
+import itertools
 import os
 import pathlib
 import re
@@ -930,12 +931,12 @@ def ensure_slice(x):
     """Try to convert an object into a slice, complain on failure"""
     if not x and x != 0:
         return slice(None)
-    elif isinstance(x, slice):
+    elif is_slice(x):
         return x
     try:
         x = int(x)
         if x != -1:
-            s = slice(x, x+1)
+            s = slice(x, x + 1)
         else:
             s = slice(-1, None, None)
     except ValueError:
@@ -954,6 +955,28 @@ def ensure_slice(x):
     return s
 
 
+def get_portions(it, slices):
+    """Yield from portions of an iterable.
+
+    Parameters
+    ----------
+    it: iterable
+    slices: a slice or a list of slice objects
+    """
+    if is_slice(slices):
+        slices = [slices]
+    if len(slices) == 1:
+        s = slices[0]
+        try:
+            yield from itertools.islice(it, s.start, s.stop, s.step)
+            return
+        except ValueError:  # islice failed
+            pass
+    it = list(it)
+    for s in slices:
+        yield from it[s]
+
+
 def is_slice_as_str(x):
     """
     Test if string x is a slice. If not a string return False.

From 295e04a7c48174833aa8c346a98b8a9a82686050 Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Mon, 29 Aug 2016 13:46:22 +0300
Subject: [PATCH 180/190] typo

---
 tests/test_history.py | 3 ---
 tests/test_tools.py   | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/tests/test_history.py b/tests/test_history.py
index e20dd705d..30fb3ede8 100644
--- a/tests/test_history.py
+++ b/tests/test_history.py
@@ -187,9 +187,6 @@ def test_parser_show(args, exp):
     assert ns.__dict__ == exp_ns
 
 
-# CMDS = ['ls', 'cat hello kitty', 'abc', 'def', 'touch me', 'grep from me']
-
-
 @pytest.mark.parametrize('index, exp', [
     (-1, 'grep from me'),
     ('hello', 'cat hello kitty'),
diff --git a/tests/test_tools.py b/tests/test_tools.py
index f9b610973..ae24dd9bc 100644
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -851,7 +851,7 @@ def test_get_portions(inp, exp):
     assert list(obs) == exp
 
 
-pytest.mark.parametrize('inp', [
+@pytest.mark.parametrize('inp', [
     '42.3',
     '3:asd5:1',
     'test' ,

From 70b802da338b6f16fc0759f6c4213dde72b31462 Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Mon, 29 Aug 2016 15:56:43 +0300
Subject: [PATCH 181/190] linux guide update

---
 docs/linux.rst       | 26 ++++++++++++++++++++++----
 news/linux-guide.rst | 13 +++++++++++++
 2 files changed, 35 insertions(+), 4 deletions(-)
 create mode 100644 news/linux-guide.rst

diff --git a/docs/linux.rst b/docs/linux.rst
index 877e4d795..52e22a222 100644
--- a/docs/linux.rst
+++ b/docs/linux.rst
@@ -31,20 +31,37 @@ the following from the source directory,
     $ python setup.py install
 
 
-Arch Linux users can install xonsh from the Arch User Repository with e.g.
-``yaourt``, ``aura``, ``pacaur``, ``PKGBUILD``, etc...:
+Debian/Ubuntu users can install xonsh from the repository with:
+
+**apt:**
+
+.. code-block:: console
+
+    $ apt install xonsh
+
+
+Fedora users can install xonsh from the repository with:
+
+**dnf:**
+
+.. code-block:: console
+
+    $ dnf install xonsh
+
+
+Arch Linux users can install xonsh from the Arch User Repository with:
 
 **yaourt:**
 
 .. code-block:: console
 
-    $ yaourt -Sa xonsh      # yaourt will call sudo when needed
+    $ yaourt -Sa xonsh
 
 **aura:**
 
 .. code-block:: console
 
-    $ sudo aura -A xonsh
+    $ aura -A xonsh
 
 **pacaur:**
 
@@ -52,6 +69,7 @@ Arch Linux users can install xonsh from the Arch User Repository with e.g.
 
     $ pacaur -S xonsh
 
+Note that some of these may require ``sudo``.
 If you run into any problems, please let us know!
 
 .. include:: add_to_shell.rst
diff --git a/news/linux-guide.rst b/news/linux-guide.rst
new file mode 100644
index 000000000..5b0b22f23
--- /dev/null
+++ b/news/linux-guide.rst
@@ -0,0 +1,13 @@
+**Added:**
+
+* howto install sections for Debian/Ubuntu and Fedora.
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

From 0dd798071d06d0c935d919989714550c5d0accef Mon Sep 17 00:00:00 2001
From: Morten Enemark Lund <melund@gmail.com>
Date: Mon, 29 Aug 2016 15:28:52 +0200
Subject: [PATCH 182/190] Update pytest plugin to always run xsh tests first

---
 xonsh/pytest_plugin.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/xonsh/pytest_plugin.py b/xonsh/pytest_plugin.py
index 23c4bc9b2..4b35c1b8f 100644
--- a/xonsh/pytest_plugin.py
+++ b/xonsh/pytest_plugin.py
@@ -1,6 +1,6 @@
 import sys
 import importlib
-from traceback import format_list, extract_tb
+from traceback import format_list, extract_tb, format_exception
 
 import pytest
 import xonsh.imphooks
@@ -10,6 +10,10 @@ def pytest_configure(config):
     xonsh.imphooks.install_hook()
 
 
+def pytest_collection_modifyitems(items):
+    items.sort(key=lambda x: 0 if isinstance(x, XshFunction) else 1)
+
+
 def _limited_traceback(excinfo):
     """ Return a formatted traceback with all the stack
         from this frame (i.e __file__) up removed
@@ -36,13 +40,15 @@ class XshFile(pytest.File):
         for test_name in tests:
             obj = getattr(mod, test_name)
             if hasattr(obj, '__call__'):
-                yield XshItem(test_name, self, obj)
+                yield XshFunction(name=test_name, parent=self,
+                                  test_func=obj, test_module=mod)
 
 
-class XshItem(pytest.Item):
-    def __init__(self, name, parent, test_func):
+class XshFunction(pytest.Item):
+    def __init__(self, name, parent, test_func, test_module):
         super().__init__(name, parent)
         self._test_func = test_func
+        self._test_module = test_module
 
     def runtest(self):
         self._test_func()
@@ -51,6 +57,7 @@ class XshItem(pytest.Item):
         """ called when self.runtest() raises an exception. """
         formatted_tb = _limited_traceback(excinfo)
         formatted_tb.insert(0, "xonsh execution failed\n")
+        formatted_tb.append(excinfo.type.__name__ +": " + str(excinfo.value))
         return "".join(formatted_tb)
 
     def reportinfo(self):

From 40dcc4d7d5e3f65bcdd62012b8573505929ddc9f Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Mon, 29 Aug 2016 16:38:29 +0300
Subject: [PATCH 183/190] news fix

---
 news/history-api.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/news/history-api.rst b/news/history-api.rst
index e3ee77161..0fbdbfb99 100644
--- a/news/history-api.rst
+++ b/news/history-api.rst
@@ -1,12 +1,12 @@
 **Added:**
 
-* ``History`` methods ``_get`` and ``__getitem__``
+* ``History`` methods ``__iter__`` and ``__getitem__``
 
 * ``tools.get_portions`` that yields parts of an iterable
 
 **Changed:**
 
-* ``_curr_session_parser`` now uses ``History_.get``
+* ``_curr_session_parser`` now iterates over ``History``
 
 **Deprecated:** None
 

From d4fa6a095214ec179603eb4a85d3d2d4a031006a Mon Sep 17 00:00:00 2001
From: laerus <laerusk@gmail.com>
Date: Mon, 29 Aug 2016 19:30:47 +0300
Subject: [PATCH 184/190] staticmethod

---
 xonsh/history.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/xonsh/history.py b/xonsh/history.py
index 00a80069c..556d4496a 100644
--- a/xonsh/history.py
+++ b/xonsh/history.py
@@ -650,7 +650,8 @@ class History(object):
         else:
             return cmds
 
-    def _cmd_filter(self, cmds, pat):
+    @staticmethod
+    def _cmd_filter(cmds, pat):
         if isinstance(pat, (int, slice)):
             s = ensure_slice(pat)
             yield from get_portions(cmds, s)
@@ -662,7 +663,8 @@ class History(object):
             raise TypeError('Command filter must be '
                             'string, int or slice')
 
-    def _args_filter(self, cmds, pat):
+    @staticmethod
+    def _args_filter(cmds, pat):
         args = None
         if isinstance(pat, (int, slice)):
             s = ensure_slice(pat)

From af232e47aafb42c225f0bab0f0a23f7cef071cf0 Mon Sep 17 00:00:00 2001
From: Morten Enemark Lund <melund@gmail.com>
Date: Mon, 29 Aug 2016 21:24:14 +0200
Subject: [PATCH 185/190] Flake 8 fixes

---
 setup.py               | 3 +--
 tests/conftest.py      | 1 -
 xonsh/pytest_plugin.py | 8 ++++----
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/setup.py b/setup.py
index b59209759..862fbd01a 100755
--- a/setup.py
+++ b/setup.py
@@ -310,7 +310,7 @@ def main():
         skw['entry_points'] = {
             'pygments.lexers': ['xonsh = xonsh.pyghooks:XonshLexer',
                                 'xonshcon = xonsh.pyghooks:XonshConsoleLexer'],
-            'pytest11': ['xonsh = xonsh.pytest_plugin'] 
+            'pytest11': ['xonsh = xonsh.pytest_plugin']
         }
         skw['cmdclass']['develop'] = xdevelop
     setup(**skw)
@@ -318,4 +318,3 @@ def main():
 
 if __name__ == '__main__':
     main()
-
diff --git a/tests/conftest.py b/tests/conftest.py
index 61412deb3..f1c08a722 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -9,7 +9,6 @@ from xonsh.built_ins import ensure_list_of_strs
 from xonsh.execer import Execer
 from xonsh.tools import XonshBlockError
 from xonsh.events import events
-import glob
 
 from tools import DummyShell, sp
 
diff --git a/xonsh/pytest_plugin.py b/xonsh/pytest_plugin.py
index 4b35c1b8f..6edf567d5 100644
--- a/xonsh/pytest_plugin.py
+++ b/xonsh/pytest_plugin.py
@@ -1,13 +1,13 @@
 import sys
 import importlib
-from traceback import format_list, extract_tb, format_exception
+from traceback import format_list, extract_tb
 
 import pytest
-import xonsh.imphooks
+from xonsh import imphooks
 
 
 def pytest_configure(config):
-    xonsh.imphooks.install_hook()
+    imphooks.install_hook()
 
 
 def pytest_collection_modifyitems(items):
@@ -57,7 +57,7 @@ class XshFunction(pytest.Item):
         """ called when self.runtest() raises an exception. """
         formatted_tb = _limited_traceback(excinfo)
         formatted_tb.insert(0, "xonsh execution failed\n")
-        formatted_tb.append(excinfo.type.__name__ +": " + str(excinfo.value))
+        formatted_tb.append('{}: {}'.format(excinfo.type.__name__, excinfo.value))
         return "".join(formatted_tb)
 
     def reportinfo(self):

From 8a7ecc6f3397d2f36bda05acaf6f5100de5d6d4d Mon Sep 17 00:00:00 2001
From: Morten Enemark Lund <melund@gmail.com>
Date: Tue, 30 Aug 2016 00:02:45 +0200
Subject: [PATCH 186/190] Exclude pytest_plugin from amalgamate

---
 xonsh/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xonsh/__init__.py b/xonsh/__init__.py
index 17c00e74d..0c801e0b5 100644
--- a/xonsh/__init__.py
+++ b/xonsh/__init__.py
@@ -1,7 +1,7 @@
 __version__ = '0.4.5'
 
 # amalgamate exclude jupyter_kernel parser_table parser_test_table pyghooks
-# amalgamate exclude winutils wizard
+# amalgamate exclude winutils wizard pytest_plugin
 import os as _os
 if _os.getenv('XONSH_DEBUG', ''):
     pass

From 770a6f3864012d58d2e04b935429f2541ac8d666 Mon Sep 17 00:00:00 2001
From: Morten Enemark Lund <melund@gmail.com>
Date: Tue, 30 Aug 2016 00:08:58 +0200
Subject: [PATCH 187/190] Remove pytest plugin registration from conftest file.
 It is not necessary when registered in setup.py

---
 tests/conftest.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index f1c08a722..d4a5846c8 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -12,8 +12,6 @@ from xonsh.events import events
 
 from tools import DummyShell, sp
 
-pytest_plugins = ("xonsh.pytest_plugin",)
-
 
 @pytest.fixture
 def xonsh_execer(monkeypatch):

From 37dc751478997d338ff42d82429c0fcb8f44f708 Mon Sep 17 00:00:00 2001
From: Morten Enemark Lund <melund@gmail.com>
Date: Tue, 30 Aug 2016 00:09:33 +0200
Subject: [PATCH 188/190] Add header to pytest_plugin.py file

---
 xonsh/pytest_plugin.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/xonsh/pytest_plugin.py b/xonsh/pytest_plugin.py
index 6edf567d5..b69a99d81 100644
--- a/xonsh/pytest_plugin.py
+++ b/xonsh/pytest_plugin.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+"""Pytest plugin for testing xsh files."""
 import sys
 import importlib
 from traceback import format_list, extract_tb

From c1f08ad66dcadc2d50dfb15a804c8b8232841aaf Mon Sep 17 00:00:00 2001
From: Morten Enemark Lund <melund@gmail.com>
Date: Tue, 30 Aug 2016 00:20:51 +0200
Subject: [PATCH 189/190] Change import form back to `import xonsh.imphooks`

---
 xonsh/pytest_plugin.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/xonsh/pytest_plugin.py b/xonsh/pytest_plugin.py
index b69a99d81..d10a077a4 100644
--- a/xonsh/pytest_plugin.py
+++ b/xonsh/pytest_plugin.py
@@ -5,11 +5,11 @@ import importlib
 from traceback import format_list, extract_tb
 
 import pytest
-from xonsh import imphooks
+from xonsh.imphooks import install_hook
 
 
 def pytest_configure(config):
-    imphooks.install_hook()
+    install_hook()
 
 
 def pytest_collection_modifyitems(items):

From c9b6667aec54fc95344592b2799e3a5a988d168a Mon Sep 17 00:00:00 2001
From: Anthony Scopatz <scopatz@gmail.com>
Date: Mon, 29 Aug 2016 19:35:09 -0400
Subject: [PATCH 190/190] some typos

---
 docs/tutorial_macros.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/tutorial_macros.rst b/docs/tutorial_macros.rst
index f0cee883d..a78e7a102 100644
--- a/docs/tutorial_macros.rst
+++ b/docs/tutorial_macros.rst
@@ -375,7 +375,7 @@ like wrapping with quotes:
     >>> echo "x  y       z"
     x  y       z
 
-    # however, subproces macros will pause and then strip
+    # however, subprocess macros will pause and then strip
     # all input after the exclamation point
     >>> echo! x  y       z
     x  y       z
@@ -385,7 +385,7 @@ expansion, that might be present even with quotes.  For example:
 
 .. code-block:: xonshcon
 
-    # without macros, envrioment variable are expanded
+    # without macros, environment variable are expanded
     >>> echo $USER
     lou