Fix the handling of colons when mixed with comments (#4332)

* Add ends_with_colon_token tool

* Add execer test for comment ending with a colon

* Use ends_with_colon_token() to catch non-indented blocks in execer

* Use ends_with_colon_token() for automatic indentation in prompt_toolkit and readline

* Add news

news/fix-colon-comment-handling.rst

@@ -0,0 +1,25 @@
+**Added:**
+
+* <news item>
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* Fixed the incorrect SyntaxError that was thrown when a subprocess command was preceded by a comment ending with a colon
+* Fixed the missing auto-indentation in readline and prompt_toolkit when a statement ending with a colon was followed by a comment
+* Fixed the incorrect auto-indentation in prompt_toolkit when a comment ended with a colon
+
+**Security:**
+
+* <news item>

tests/test_execer.py

@@ -75,6 +75,11 @@ def test_bad_indent():
         check_parse(code)
 
 
+def test_comment_colon_ending():
+    code = "# this is a comment:\necho hello"
+    assert check_parse(code)
+
+
 def test_good_rhs_subproc():
     # nonsense but parsable
     code = "str().split() | ![grep exit]\n"

tests/test_tools.py

@@ -79,6 +79,7 @@ from xonsh.tools import (
     deprecated,
     is_writable_file,
     balanced_parens,
+    ends_with_colon_token,
     iglobpath,
     all_permutations,
     register_custom_style,
@@ -633,6 +634,29 @@ def test_balanced_parens(line, exp):
         assert not obs
 
 
+@pytest.mark.parametrize(
+    "line, exp",
+    [
+        ("if 1:", True),
+        ("elif 2: #comment", True),
+        ("elif 3: #colon comment:", True),
+        ("else: ", True),
+        ("for s in '#not-a-comment':", True),
+        ("", False),
+        ("#comment", False),
+        ("#colon comment:", False),
+        ("print('hello')", False),
+        ("print('hello') #colon comment:", False),
+    ],
+)
+def test_ends_with_colon_token(line, exp):
+    obs = ends_with_colon_token(line, lexer=LEXER)
+    if exp:
+        assert obs
+    else:
+        assert not obs
+
+
 @pytest.mark.parametrize(
     "line, mincol, exp",
     [

xonsh/execer.py

@@ -15,6 +15,7 @@ from xonsh.tools import (
     replace_logical_line,
     balanced_parens,
     starting_whitespace,
+    ends_with_colon_token,
 )
 from xonsh.built_ins import XSH
 
@@ -268,7 +269,7 @@ class Execer(object):
                     input = "\n".join(lines)
                     continue
 
-                if last_error_line > 1 and lines[idx - 1].rstrip()[-1:] == ":":
+                if last_error_line > 1 and ends_with_colon_token(lines[idx - 1]):
                     # catch non-indented blocks and raise error.
                     prev_indent = len(lines[idx - 1]) - len(lines[idx - 1].lstrip())
                     curr_indent = len(lines[idx]) - len(lines[idx].lstrip())

xonsh/ptk_shell/key_bindings.py

@@ -17,7 +17,11 @@ from prompt_toolkit.key_binding.key_bindings import KeyBindings, KeyBindingsBase
 from prompt_toolkit.key_binding.bindings.named_commands import get_by_name
 
 from xonsh.aliases import xonsh_exit
-from xonsh.tools import check_for_partial_string, get_line_continuation
+from xonsh.tools import (
+    check_for_partial_string,
+    get_line_continuation,
+    ends_with_colon_token,
+)
 from xonsh.built_ins import XSH
 from xonsh.shell import transform_command
 
@@ -49,7 +53,7 @@ def carriage_return(b, cli, *, autoindent=True):
     )
 
     # indent after a colon
-    if doc.current_line_before_cursor.strip().endswith(":") and at_end_of_line:
+    if ends_with_colon_token(doc.current_line_before_cursor) and at_end_of_line:
         b.newline(copy_margin=autoindent)
         b.insert_text(indent, fire_event=False)
     # if current line isn't blank, check dedent tokens

xonsh/readline_shell.py

@@ -29,7 +29,13 @@ from xonsh.ansi_colors import (
     ansi_color_style,
 )
 from xonsh.prompt.base import multiline_prompt
-from xonsh.tools import print_exception, to_bool, columnize, carriage_return
+from xonsh.tools import (
+    print_exception,
+    to_bool,
+    columnize,
+    carriage_return,
+    ends_with_colon_token,
+)
 from xonsh.platform import (
     ON_WINDOWS,
     ON_CYGWIN,
@@ -491,7 +497,7 @@ class ReadlineShell(BaseShell, cmd.Cmd):
             if len(line.strip()) == 0:
                 readline.set_pre_input_hook(None)
                 self._current_indent = ""
-            elif line.rstrip()[-1] == ":":
+            elif ends_with_colon_token(line):
                 ind = line[: len(line) - len(line.lstrip())]
                 ind += XSH.env.get("INDENT")
                 readline.set_pre_input_hook(_insert_text_func(ind, readline))

xonsh/tools.py

@@ -351,6 +351,15 @@ def balanced_parens(line, mincol=0, maxcol=None, lexer=None):
     return cnt == 0
 
 
+def ends_with_colon_token(line, lexer=None):
+    """Determines whether a line ends with a colon token, ignoring comments."""
+    if lexer is None:
+        lexer = xsh.execer.parser.lexer
+    lexer.input(line)
+    toks = list(lexer)
+    return len(toks) > 0 and toks[-1].type == "COLON"
+
+
 def find_next_break(line, mincol=0, lexer=None):
     """Returns the column number of the next logical break in subproc mode.
     This function may be useful in finding the maxcol argument of