so close...

xonsh/ast.py

@@ -12,7 +12,7 @@ from ast import Module, Num, Expr, Str, Bytes, UnaryOp, UAdd, USub, Invert, \
     ExceptHandler, FunctionDef, ClassDef, Starred, NodeTransformer, \
     Interactive, Expression, dump
 
-from xonsh.tools import subproc_line
+from xonsh.tools import subproc_line, subproc_toks
 
 STATEMENTS = (FunctionDef, ClassDef, Return, Delete, Assign, AugAssign, For,
               While, If, With, Raise, Try, Assert, Import, ImportFrom, Global, 
@@ -98,7 +98,15 @@ class CtxAwareTransformer(NodeTransformer):
 
     def try_subproc_line(self, node):
         """Tries to parse the line of the node as a subprocess."""
-        spline = subproc_line(self.lines[node.lineno - 1]).lstrip()
+        #spline = subproc_line(self.lines[node.lineno - 1]).lstrip()
+        line = self.lines[node.lineno - 1]
+        mincol = line.rfind(';', 0, node.col_offset-1)
+        line = line[mincol+1:].lstrip()
+        mincol = -1
+        spline = subproc_toks(line, 
+                    mincol=mincol, 
+                    returnline=False, 
+                    lexer=self.parser.lexer).lstrip()
         try:
             newnode = self.parser.parse(spline, mode=self.mode)
             newnode = newnode.body

xonsh/execer.py

@@ -9,7 +9,7 @@ from collections import Iterable, Sequence, Mapping
 
 from xonsh import ast
 from xonsh.parser import Parser
-from xonsh.tools import subproc_line
+from xonsh.tools import subproc_line, subproc_toks
 from xonsh.built_ins import load_builtins, unload_builtins
 
 class Execer(object):
@@ -110,19 +110,31 @@ class Execer(object):
         return exec(code, glbs, locs)
 
     def _parse_ctx_free(self, input, mode='exec'):
-        last_error_line = -1
+        last_error_line = last_error_col = -1
         parsed = False
         while not parsed:
             try:
                 tree = self.parser.parse(input, filename=self.filename,
                             mode=mode, debug_level=self.debug_level)
                 parsed = True
+            #except SyntaxError as e:
+            #    if (e.loc is None) or (last_error_line == e.loc.lineno):
+            #        raise
+            #    last_error_line = e.loc.lineno
+            #    idx = last_error_line - 1
+            #    lines = input.splitlines()
+            #    lines[idx] = subproc_line(lines[idx])
+            #    input = '\n'.join(lines)
             except SyntaxError as e:
-                if (e.loc is None) or (last_error_line == e.loc.lineno):
+                if (e.loc is None) or (last_error_line == e.loc.lineno and 
+                                       last_error_col == e.loc.column):
                     raise
+                last_error_col = e.loc.column
                 last_error_line = e.loc.lineno
                 idx = last_error_line - 1
                 lines = input.splitlines()
-                lines[idx] = subproc_line(lines[idx])
+                lines[idx] = subproc_toks(lines[idx], mincol=last_error_col+1, 
+                                    returnline=True, lexer=self.parser.lexer)
                 input = '\n'.join(lines)
+                print(repr(input))
         return tree

xonsh/tools.py

@@ -67,6 +67,44 @@ def subproc_toks(line, mincol=-1, lexer=None, returnline=False):
         rtn = line[:beg] + rtn + line[end:]
     return rtn
 
+def subproc_toks(line, mincol=-1, maxcol=9000, lexer=None, returnline=False):
+    """Excapsulates tokens in a source code line in a uncaptured 
+    subprocess $[] starting at a minimum column.
+    """
+    if lexer is None:
+        lexer = builtins.__xonsh_execer__.parser.lexer
+    lexer.reset()
+    lexer.input(line)
+    poses = []
+    last_was_semi = False
+    for tok in lexer:
+        if last_was_semi:
+            poses.clear()
+            last_was_semi = False
+        pos = tok.lexpos
+        if pos < mincol:
+            continue
+        if pos >= maxcol:
+            break
+        poses.append(pos)
+        if tok.type == 'SEMI':
+            poses.pop()
+            last_was_semi = True
+        elif tok.type == 'NEWLINE':
+            break
+    else:
+        pos = poses[-1]
+        if isinstance(tok.value, string_types):
+            poses.append(pos + len(tok.value))
+        else:
+            el = line[pos:].split('#')[0].rstrip()
+            poses.append(pos + len(el))
+    beg, end = poses[0], poses[-1]
+    rtn = '$[' + line[beg:end] + ']'
+    if returnline:
+        rtn = line[:beg] + rtn + line[end:]
+    return rtn
+
 def decode(s, encoding=None):
     encoding = encoding or DEFAULT_ENCODING
     return s.decode(encoding, "replace")