mirrors/xonsh
1
0
Fork 0
mirror of https://github.com/xonsh/xonsh.git synced 2025-03-05 17:00:58 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

major lexer refactor

Browse source
This commit is contained in:
adam j hartz 2015-05-19 17:10:54 -04:00
parent 44348dfa19
commit af36590ea0
1 changed files with 47 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
xonsh/lexer.py
View file

@ -30,7 +30,7 @@ _op_map = {
# basic operators
'+': 'PLUS', '-': 'MINUS', '*': 'TIMES', '/': 'DIVIDE',
'//': 'DOUBLEDIV', '%': 'MOD', '**': 'POW', '|': 'PIPE',
'&': 'AMPERSAND', '~': 'TILDE', '^': 'XOR', '<<': 'LSHIFT',
'~': 'TILDE', '^': 'XOR', '<<': 'LSHIFT',
'>>': 'RSHIFT', '<': 'LT', '<=': 'LE', '>': 'GT', '>=': 'GE',
'==': 'EQ', '!=': 'NE', '->': 'RARROW',
# assignment operators
@ -65,20 +65,13 @@ def handle_name(state, token, stream):
# subprocess mode
n = next(stream, None)
string = token.string
if n is None or \
n.start != token.end or \
token.string not in _REDIRECT_NAMES:
# we want to treat this as a normal name if:
# * there is no next token, or
# * the next token isn't adjacent, or
# * this isn't one of our special redirect names
yield _new_token('NAME', token.string, token.start)
if n is not None:
yield from handle_token(state, n, stream)
elif n is not None and n.string in {'<', '>', '>>'}:
if n is not None and n.string in {'<', '>', '>>'} and n.start == token.end and token.string in _REDIRECT_NAME:
# looks like a redirect to me!
string += n.string
n2 = next(stream, None)
if n2 is not None and n2.string == '&' and n2.start == n.end:
string += n2.string
n2 = next(stream, None)
if n2 is not None:
if (n2.start == n.end and
(n2.type == tokenize.NUMBER or
@ -99,25 +92,21 @@ def handle_name(state, token, stream):
yield from handle_token(state, n, stream)
def handle_number(state, token, stream):
"""
Function for handling number tokens
"""
def _make_special_handler(token_type):
def inner_handler(state, token, stream):
state['last'] = token
if state['pymode'][-1][0]:
yield _new_token('NUMBER', token.string, token.start)
yield _new_token(token_type, token.string, token.start)
else:
# subprocess mode
n = next(stream, None)
string = token.string
if n is None or \
n.start != token.end:
yield _new_token('NUMBER', token.string, token.start)
if n is not None:
yield from handle_token(state, n, stream)
elif n is not None and n.string in {'<', '>', '>>'}:
if n is not None and n.string in {'<', '>', '>>'} and n.start == token.end:
string += n.string
n2 = next(stream, None)
if n2 is not None and n2.string == '&' and n2.start == n.end:
string += n2.string
n2 = next(stream, None)
if n2 is not None:
if (n2.start == n.end and
(n2.type == tokenize.NUMBER or
@ -133,9 +122,17 @@ def handle_number(state, token, stream):
state['last'] = n
yield _new_token('IOREDIRECT', string, token.start)
else:
yield _new_token('NUMBER', token.string, token.start)
yield _new_token(token_type, token.string, token.start)
if n is not None:
yield from handle_token(state, n, stream)
return inner_handler
handle_number = _make_special_handler('NUMBER')
"""Function for handling number tokens"""
handle_ampersand = _make_special_handler('AMPERSAND')
"""Function for handling ampersand tokens"""
def handle_dollar(state, token, stream):
@ -349,6 +346,7 @@ special_handlers = {
tokenize.NAME: handle_name,
tokenize.NUMBER: handle_number,
tokenize.ERRORTOKEN: handle_error_token,
(tokenize.OP, '&'): handle_ampersand,
(tokenize.OP, '@'): handle_at,
(tokenize.OP, '('): handle_lparen,
(tokenize.OP, ')'): handle_rparen,
@ -494,6 +492,7 @@ class Lexer(object):
'NAME', # name tokens
'NUMBER', # numbers
'WS', # whitespace in subprocess mode
'AMPERSAND', # &
'REGEXPATH', # regex escaped with backticks
'IOREDIRECT', # subprocess io redirection token
'LPAREN', 'RPAREN', # ( )