diff --git a/tests/test_lexer.py b/tests/test_lexer.py index 965cb7624..ea0b8297d 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -70,6 +70,15 @@ def test_indent(): assert_tokens_equal(exp, obs) +def test_post_whitespace(): + l = Lexer() + l.build() + l.input('42 \t ') + obs = list(l) + exp = [('INT_LITERAL', '42', 1, 0)] + assert_tokens_equal(exp, obs) + + if __name__ == '__main__': diff --git a/xonsh/lexer.py b/xonsh/lexer.py index 1d6fc2818..622bf545c 100644 --- a/xonsh/lexer.py +++ b/xonsh/lexer.py @@ -87,7 +87,7 @@ class Lexer(object): # tokens = pykeywords + ( # Misc - 'ID', 'INDENT', 'NEWLINE', + 'ID', 'INDENT', 'NEWLINE', 'WHITESPACE', # literals 'INT_LITERAL', 'HEX_LITERAL', 'OCT_LITERAL', 'BIN_LITERAL', @@ -145,8 +145,8 @@ class Lexer(object): # # Rules # - t_INDENT = '^[ \t]+' - t_ignore = ' \t' + t_INDENT = r'^[ \t]+' + t_ignore_WHITESPACE = r' \t' # Newlines def t_NEWLINE(self, t):