more cleanup

xonsh/lexer.py

@@ -362,47 +362,15 @@ class Lexer(object):
             yield t
             t = self.token()
 
-    #
-    # Python keywords
-    #
-    pykeywords = ('AND', 'AS', 'ASSERT', 'BREAK', 'CLASS', 'CONTINUE', 'DEF',
-                  'DEL', 'ELIF', 'ELSE', 'EXCEPT', 'FINALLY', 'FOR', 'FROM',
-                  'GLOBAL', 'IMPORT', 'IF', 'IN', 'IS', 'LAMBDA', 'NONLOCAL',
-                  'NOT', 'OR', 'PASS', 'RAISE', 'RETURN', 'TRY', 'WHILE',
-                  'WITH', 'YIELD',)
-
-    pykeyword_map = {k.lower(): k for k in pykeywords}
-
     #
     # All the tokens recognized by the lexer
     #
-    tokens = pykeywords + (
-        # Misc
-        'NAME', 'INDENT', 'DEDENT', 'NEWLINE', 'ENDMARKER',
-        'NONE', 'TRUE', 'FALSE', 'WS',
-
-        # literals
-        'NUMBER', 'STRING',
-
-        # Basic Operators
-        'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'DOUBLEDIV', 'MOD', 'POW',
-        'PIPE', 'AMPERSAND', 'TILDE', 'XOR', 'LSHIFT', 'RSHIFT',
-        'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', 'RARROW',
-
-        # Assignment Operators
-        'EQUALS', 'PLUSEQUAL', 'MINUSEQUAL', 'TIMESEQUAL', 'DIVEQUAL',
-        'MODEQUAL', 'POWEQUAL', 'LSHIFTEQUAL', 'RSHIFTEQUAL', 'AMPERSANDEQUAL',
-        'XOREQUAL', 'PIPEEQUAL', 'DOUBLEDIVEQUAL',
-
-        # Command line
-        'REGEXPATH',
-
-        # Delimeters
+    tokens = tuple(token_map.values()) + (
+        'WS',                    # whitespace in subprocess mode
+        'REGEXPATH',             # regex escaped with backticks
         'LPAREN', 'RPAREN',      # ( )
         'LBRACKET', 'RBRACKET',  # [ ]
         'LBRACE', 'RBRACE',      # { }
-        'COMMA', 'PERIOD',       # . ,
-        'SEMI', 'COLON',         # ; :
         'AT',                    # @
         'QUESTION',              # ?
         'DOUBLE_QUESTION',       # ??
@@ -411,7 +379,4 @@ class Lexer(object):
         'DOLLAR_LPAREN',         # $(
         'DOLLAR_LBRACE',         # ${
         'DOLLAR_LBRACKET',       # $[
-
-        # Ellipsis (...)
-        'ELLIPSIS',
         )

xonsh/parser.py

@@ -373,7 +373,6 @@ class Parser(object):
                          | file_input
                          | eval_input
                          | empty
-                         | ENDMARKER
         """
         p[0] = p[1]
 
@@ -415,7 +414,6 @@ class Parser(object):
 
     def p_eval_input(self, p):
         """eval_input : testlist newlines_opt
-                      | testlist newlines_opt ENDMARKER
         """
         p[0] = ast.Expression(body=p[1])
 
@@ -1998,11 +1996,9 @@ class Parser(object):
     def p_subproc(self, p):
         """subproc : subproc_atoms
                    | subproc_atoms WS
-                   | subproc_atoms ENDMARKER
                    | subproc AMPERSAND
                    | subproc subproc_special subproc_atoms
                    | subproc subproc_special subproc_atoms WS
-                   | subproc subproc_special subproc_atoms ENDMARKER
         """
         lineno = self.lineno
         col = self.col