mirror of
https://github.com/xonsh/xonsh.git
synced 2025-03-05 17:00:58 +01:00
parser working except subprocess mode
This commit is contained in:
adam j hartz
parent
37e61cb96a
commit
ed897c813a
4 changed files with 65 additions and 48 deletions
|
|
@ -98,7 +98,6 @@ class CtxAwareTransformer(NodeTransformer):
|
|||
|
||||
def try_subproc_toks(self, node):
|
||||
"""Tries to parse the line of the node as a subprocess."""
|
||||
print('trying to run as subprocess')
|
||||
line = self.lines[node.lineno - 1]
|
||||
mincol = len(line) - len(line.lstrip())
|
||||
maxcol = None# if self.mode == 'eval' else node.col_offset
|
||||
|
|
|
|||
|
|
@ -49,11 +49,20 @@ def handle_indent(state, token, stream):
|
|||
level = len(token.string)
|
||||
if token.type == tokenize.DEDENT:
|
||||
state['indents'].pop()
|
||||
yield _new_token(state, 'DEDENT', ' '*state['indents'][-1], token.start[0], token.start[1])
|
||||
yield _new_token(state, 'DEDENT', ' '*state['indents'][-1], token.start)
|
||||
elif token.type == tokenize.INDENT:
|
||||
#moving forward
|
||||
state['indents'].append(level)
|
||||
yield _new_token(state, 'INDENT', token.string, token.start[0], token.start[1])
|
||||
yield _new_token(state, 'INDENT', token.string, token.start)
|
||||
|
||||
try:
|
||||
n = next(stream)
|
||||
except:
|
||||
n = None
|
||||
if n is not None:
|
||||
if n.type != tokenize.ENDMARKER:
|
||||
for i in handle_token(state, n, stream):
|
||||
yield i
|
||||
|
||||
def handle_dollar(state, token, stream):
|
||||
try:
|
||||
|
|
@ -65,13 +74,13 @@ def handle_dollar(state, token, stream):
|
|||
raise Exception("unexpected whitespace after $")
|
||||
|
||||
if n.type == tokenize.NAME:
|
||||
yield _new_token(state, 'DOLLAR_NAME', '$' + n.string, token.start[0], token.start[1])
|
||||
yield _new_token(state, 'DOLLAR_NAME', '$' + n.string, token.start)
|
||||
elif n.type == tokenize.OP and n.string == '(':
|
||||
yield _new_token(state, 'DOLLAR_LPAREN', '$(', token.start[0], token.start[1])
|
||||
yield _new_token(state, 'DOLLAR_LPAREN', '$(', token.start)
|
||||
elif n.type == tokenize.OP and n.string == '[':
|
||||
yield _new_token(state, 'DOLLAR_LBRACKET', '$[', token.start[0], token.start[1])
|
||||
yield _new_token(state, 'DOLLAR_LBRACKET', '$[', token.start)
|
||||
elif n.type == tokenize.OP and n.string == '{':
|
||||
yield _new_token(state, 'DOLLAR_LBRACE', '${', token.start[0], token.start[1])
|
||||
yield _new_token(state, 'DOLLAR_LBRACE', '${', token.start)
|
||||
else:
|
||||
e = 'expected NAME, (, [, or {{ after $, but got {0}'
|
||||
raise Exception(e.format(n))
|
||||
|
|
@ -84,9 +93,9 @@ def handle_at(state, token, stream):
|
|||
|
||||
if n.type == tokenize.OP and n.string == '(' and \
|
||||
n.start == token.end:
|
||||
yield _new_token(state, 'AT_LPAREN', '@(', token.start[0], token.start[1])
|
||||
yield _new_token(state, 'AT_LPAREN', '@(', token.start)
|
||||
else:
|
||||
yield _new_token(state, 'AT', '@', token.start[0], token.start[1])
|
||||
yield _new_token(state, 'AT', '@', token.start)
|
||||
for i in handle_token(state, n, stream):
|
||||
yield i
|
||||
|
||||
|
|
@ -98,9 +107,9 @@ def handle_question(state, token, stream):
|
|||
|
||||
if n.type == tokenize.ERRORTOKEN and n.string == '?' and \
|
||||
n.start == token.end:
|
||||
yield _new_token(state, 'DOUBLE_QUESTION', '??', token.start[0], token.start[1])
|
||||
yield _new_token(state, 'DOUBLE_QUESTION', '??', token.start)
|
||||
else:
|
||||
yield _new_token(state, 'QUESTION', '?', token.start[0], token.start[1])
|
||||
yield _new_token(state, 'QUESTION', '?', token.start)
|
||||
for i in handle_token(state, n, stream):
|
||||
yield i
|
||||
|
||||
|
|
@ -123,7 +132,7 @@ def handle_backtick(state, token, stream):
|
|||
except:
|
||||
n = None
|
||||
if found_match:
|
||||
yield _new_token(state, 'REGEXPATH', sofar, token.start[0], token.start[1])
|
||||
yield _new_token(state, 'REGEXPATH', sofar, token.start)
|
||||
else:
|
||||
e = "Could not find matching backtick for regex on line {0}"
|
||||
raise Exception(e.format(token.start[0]))
|
||||
|
|
@ -134,7 +143,7 @@ def handle_newline(state, token, stream):
|
|||
except:
|
||||
n = None
|
||||
|
||||
yield _new_token(state, 'NEWLINE', '\n', token.start[0], token.start[1])
|
||||
yield _new_token(state, 'NEWLINE', '\n', token.start)
|
||||
|
||||
if n is not None:
|
||||
if n.type != tokenize.ENDMARKER:
|
||||
|
|
@ -156,11 +165,12 @@ special_handlers = {
|
|||
def handle_token(state, token, stream):
|
||||
typ = token.type
|
||||
st = token.string
|
||||
print('trying', typ, st)
|
||||
#print('state',state)
|
||||
#print('handling', typ, st)
|
||||
if (typ, st) in token_map:
|
||||
yield _new_token(state, token_map[(typ, st)], st, token.start[0], token.start[1])
|
||||
yield _new_token(state, token_map[(typ, st)], st, token.start)
|
||||
elif typ in token_map:
|
||||
yield _new_token(state, token_map[typ], st, token.start[0], token.start[1])
|
||||
yield _new_token(state, token_map[typ], st, token.start)
|
||||
elif (typ, st) in special_handlers:
|
||||
for i in special_handlers[(typ, st)](state, token, stream):
|
||||
yield i
|
||||
|
|
@ -171,8 +181,8 @@ def handle_token(state, token, stream):
|
|||
raise Exception('Unexpected token: {0}'.format(token))
|
||||
|
||||
def preprocess_tokens(tokstream):
|
||||
#tokstream = clear_NL(tokstream)
|
||||
state = {'indents': [0], 'lexpos': 0}
|
||||
tokstream = clear_NL(tokstream)
|
||||
state = {'indents': [0]}
|
||||
for token in tokstream:
|
||||
for i in handle_token(state, token, tokstream):
|
||||
yield i
|
||||
|
|
@ -188,15 +198,11 @@ def tok(s):
|
|||
|
||||
|
||||
#synthesize a new PLY token
|
||||
def _new_token(state, type, value, lineno, col):
|
||||
def _new_token(state, type, value, pos):
|
||||
o = LexToken()
|
||||
o.type = type
|
||||
o.value = value
|
||||
o.lineno = lineno
|
||||
o.lexpos = state['lexpos']
|
||||
o.col = col
|
||||
print('col',col)
|
||||
state['lexpos'] += 1
|
||||
o.lineno, o.lexpos = pos
|
||||
return o
|
||||
|
||||
def anyof(*regexes):
|
||||
|
|
@ -237,7 +243,7 @@ class Lexer(object):
|
|||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.lexer.lineno = 1
|
||||
#self.lexer.lineno = 1
|
||||
self.indent = ''
|
||||
self.last = None
|
||||
self.in_py_mode = [True]
|
||||
|
|
@ -255,15 +261,15 @@ class Lexer(object):
|
|||
|
||||
def input(self, s):
|
||||
"""Calls the lexer on the string s."""
|
||||
print('code:\n',repr(s))
|
||||
#print('code:\n',repr(s))
|
||||
self.token_stream = preprocess_tokens(tok(s))
|
||||
|
||||
def token(self):
|
||||
"""Retrieves the next token."""
|
||||
try:
|
||||
o = next(self.token_stream)
|
||||
print(o)
|
||||
return o
|
||||
self.last = next(self.token_stream)
|
||||
#print(self.last)
|
||||
return self.last
|
||||
except:
|
||||
return None
|
||||
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ class Parser(object):
|
|||
lexer_kwargs = dict(optimize=lexer_optimize, lextab=lexer_table)
|
||||
if outputdir is not None:
|
||||
lexer_kwargs['outputdir'] = outputdir
|
||||
lexer.build(**lexer_kwargs)
|
||||
#lexer.build(**lexer_kwargs)
|
||||
self.tokens = lexer.tokens
|
||||
|
||||
opt_rules = (
|
||||
|
|
@ -333,15 +333,21 @@ class Parser(object):
|
|||
|
||||
def token_col(self, t):
|
||||
"""Gets ths token column"""
|
||||
return t.lexpos
|
||||
return self.lexer.token_col(t)
|
||||
|
||||
@property
|
||||
def lineno(self):
|
||||
try:
|
||||
return self.lexer.last.lineno
|
||||
except:
|
||||
return 0
|
||||
return self.lexer.lineno
|
||||
|
||||
@lineno.setter
|
||||
def lineno(self, value):
|
||||
self.lexer.lineno = value
|
||||
pass
|
||||
#self.lexer.lineno = value
|
||||
|
||||
@property
|
||||
def col(self):
|
||||
|
|
@ -1531,6 +1537,8 @@ class Parser(object):
|
|||
p1 = ast.Str(s=p1.strip(bt), lineno=self.lineno,
|
||||
col_offset=self.col)
|
||||
p1 = xonsh_regexpath(p1, lineno=self.lineno, col=self.col)
|
||||
elif p1.startswith('$'):
|
||||
p1 = self._envvar_by_name(p1[1:], lineno=self.lineno, col=self.col)
|
||||
else:
|
||||
p1 = ast.Name(id=p1, ctx=ast.Load(), lineno=self.lineno,
|
||||
col_offset=self.col)
|
||||
|
|
@ -1956,8 +1964,18 @@ class Parser(object):
|
|||
return ast.Subscript(value=xenv, slice=idx, ctx=ast.Load(),
|
||||
lineno=lineno, col_offset=col)
|
||||
|
||||
def _hz_printer(self, x):
|
||||
if hasattr(x, 'elts'):
|
||||
return [self._hz_printer(i) for i in x.elts]
|
||||
elif hasattr(x, 's'):
|
||||
return x.s
|
||||
elif hasattr(x, 'n'):
|
||||
return x.n
|
||||
|
||||
def _subproc_cliargs(self, args, lineno=None, col=None):
|
||||
"""Creates an expression for subprocess CLI arguments."""
|
||||
print('cliargs enter',lineno,col)
|
||||
print([self._hz_printer(i) for i in args])
|
||||
cliargs = currlist = empty_list(lineno=lineno, col=col)
|
||||
for arg in args:
|
||||
action = arg._cliarg_action
|
||||
|
|
@ -1982,6 +2000,7 @@ class Parser(object):
|
|||
else:
|
||||
raise ValueError("action not understood: " + action)
|
||||
del arg._cliarg_action
|
||||
print(self._hz_printer(cliargs))
|
||||
return cliargs
|
||||
|
||||
def p_subproc_special_atom(self, p):
|
||||
|
|
@ -2028,6 +2047,7 @@ class Parser(object):
|
|||
cliargs = self._subproc_cliargs(p[3], lineno=lineno, col=col)
|
||||
p0 = p1 + [p[2], cliargs]
|
||||
# return arguments list
|
||||
print('subproc_return',p0)
|
||||
p[0] = p0
|
||||
|
||||
def p_subproc_atoms(self, p):
|
||||
|
|
@ -2150,4 +2170,4 @@ class Parser(object):
|
|||
else:
|
||||
msg = 'code: {0}'.format(p.value),
|
||||
self._parse_error(msg, self.currloc(lineno=p.lineno,
|
||||
column=self.lexer.token_col(p)))
|
||||
column=p.lexpos))
|
||||
|
|
|
|||
|
|
@ -35,33 +35,27 @@ def subproc_toks(line, mincol=-1, maxcol=None, lexer=None, returnline=False):
|
|||
subprocess $[] starting at a minimum column. If there are no tokens
|
||||
(ie in a comment line) this returns None.
|
||||
"""
|
||||
line = line if line.endswith('\n') else (line+'\n')
|
||||
if not line.endswith('\n'):
|
||||
line = line + '\n'
|
||||
if lexer is None:
|
||||
lexer = builtins.__xonsh_execer__.parser.lexer
|
||||
if maxcol is None:
|
||||
print(len(line))
|
||||
maxcol = len(line) + 1
|
||||
print(maxcol)
|
||||
print(len(line), repr(line), maxcol)
|
||||
lexer.reset()
|
||||
lexer.input(line)
|
||||
toks = []
|
||||
end_offset = 0
|
||||
for tok in lexer:
|
||||
print('TOKEN',tok)
|
||||
pos = tok.col
|
||||
pos = tok.lexpos
|
||||
if pos >= maxcol:
|
||||
print(pos,maxcol)
|
||||
print('too far')
|
||||
print('too big',pos,maxcol)
|
||||
break
|
||||
if len(toks) > 0 and toks[-1].type == 'SEMI':
|
||||
print('semi')
|
||||
toks.clear()
|
||||
if pos < mincol:
|
||||
print('minicol')
|
||||
continue
|
||||
toks.append(tok)
|
||||
if tok.type in ('NEWLINE', 'ENDMARKER'):
|
||||
if tok.type =='NEWLINE':
|
||||
break
|
||||
else:
|
||||
if len(toks) == 0:
|
||||
|
|
@ -69,7 +63,7 @@ def subproc_toks(line, mincol=-1, maxcol=None, lexer=None, returnline=False):
|
|||
if toks[-1].type == 'SEMI':
|
||||
toks.pop()
|
||||
tok = toks[-1]
|
||||
pos = tok.col
|
||||
pos = tok.lexpos
|
||||
if isinstance(tok.value, string_types):
|
||||
end_offset = len(tok.value)
|
||||
else:
|
||||
|
|
@ -77,10 +71,8 @@ def subproc_toks(line, mincol=-1, maxcol=None, lexer=None, returnline=False):
|
|||
end_offset = len(el)
|
||||
if len(toks) == 0:
|
||||
return # handle comment lines
|
||||
print(toks)
|
||||
beg, end = toks[0].col, (toks[-1].col + end_offset)
|
||||
print('LINE,BEG,END', (line,beg,end))
|
||||
print('LINE:',line[beg:end])
|
||||
beg, end = toks[0].lexpos, (toks[-1].lexpos + end_offset)
|
||||
print(repr(line), toks, beg, end)
|
||||
rtn = '$[' + line[beg:end] + ']'
|
||||
if returnline:
|
||||
rtn = line[:beg] + rtn + line[end:]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue