mirrors/xonsh
1
0
Fork 0
mirror of https://github.com/xonsh/xonsh.git synced 2025-03-04 08:24:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

more lexer changes

Browse source
This commit is contained in:
Anthony Scopatz 2015-01-21 17:28:46 -05:00
parent 5757edb15e
commit e5c61daf43
1 changed files with 16 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
xonsh/lexer.py
View file

@ -74,44 +74,26 @@ class Lexer(object):
## ##
## Reserved keywords ## Reserved keywords
## ##
keywords = ( pykeywords = ('AND', 'AS', 'ASSERT', 'BREAK', 'CLASS', 'CONTINUE', 'DEF',
'_BOOL', '_COMPLEX', 'AUTO', 'BREAK', 'CASE', 'CHAR', 'CONST', 'DEL', 'ELIF', 'ELSE', 'EXCEPT', 'FINALLY', 'FOR', 'FROM', 'GLOBAL',
'CONTINUE', 'DEFAULT', 'DO', 'DOUBLE', 'ELSE', 'ENUM', 'EXTERN', 'IMPORT', 'IF', 'IN', 'IS', 'LAMBDA', 'NONLOCAL', 'NOT', 'OR', 'PASS',
'FLOAT', 'FOR', 'GOTO', 'IF', 'INLINE', 'INT', 'LONG', 'REGISTER', 'RAISE', 'TRY', 'WHILE', 'WITH', 'YIELD',)
'RESTRICT', 'RETURN', 'SHORT', 'SIGNED', 'SIZEOF', 'STATIC', 'STRUCT',
'SWITCH', 'TYPEDEF', 'UNION', 'UNSIGNED', 'VOID',
'VOLATILE', 'WHILE',
)
keyword_map = {} pykeyword_map = {k.lower(): k for k in pykeywords}
for keyword in keywords:
if keyword == '_BOOL':
keyword_map['_Bool'] = keyword
elif keyword == '_COMPLEX':
keyword_map['_Complex'] = keyword
else:
keyword_map[keyword.lower()] = keyword
## ##
## All the tokens recognized by the lexer ## All the tokens recognized by the lexer
## ##
tokens = keywords + ( tokens = pykeywords + (
# Identifiers # Identifiers
'ID', 'ID',
# Type identifiers (identifiers previously defined as
# types with typedef)
'TYPEID',
# constants # constants
'INT_CONST_DEC', 'INT_CONST_OCT', 'INT_CONST_HEX', 'INT_CONST_DEC', 'INT_CONST_OCT', 'INT_CONST_HEX',
'FLOAT_CONST', 'HEX_FLOAT_CONST', 'FLOAT_CONST', 'HEX_FLOAT_CONST',
'CHAR_CONST',
'WCHAR_CONST',
# String literals # String literals
'STRING_LITERAL', 'STRING_LITERAL',
'WSTRING_LITERAL',
# Operators # Operators
'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD', 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD',
@ -125,28 +107,21 @@ class Lexer(object):
'LSHIFTEQUAL','RSHIFTEQUAL', 'ANDEQUAL', 'XOREQUAL', 'LSHIFTEQUAL','RSHIFTEQUAL', 'ANDEQUAL', 'XOREQUAL',
'OREQUAL', 'OREQUAL',
# Increment/decrement # Command line
'PLUSPLUS', 'MINUSMINUS', 'OPTION',
# Structure dereference (->)
'ARROW',
# Conditional operator (?)
'CONDOP',
# Delimeters # Delimeters
'LPAREN', 'RPAREN', # ( ) 'LPAREN', 'RPAREN', # ( )
'LBRACKET', 'RBRACKET', # [ ] 'LBRACKET', 'RBRACKET', # [ ]
'LBRACE', 'RBRACE', # { } 'LBRACE', 'RBRACE', # { }
'COMMA', 'PERIOD', # . , 'COMMA', 'PERIOD', # . ,
'SEMI', 'COLON', # ; : 'SEMI', 'COLON', # ; :
'AT', 'DOLLAR', # @ $
'COMMENT', # #
# Ellipsis (...) # Ellipsis (...)
'ELLIPSIS', 'ELLIPSIS',
)
# pre-processor
'PPHASH', # '#'
)
## ##
## Regexes for use in tokens ## Regexes for use in tokens