From 84a5b05a36463115b332d87c25b7b5d4f5de6e02 Mon Sep 17 00:00:00 2001 From: Anthony Scopatz Date: Wed, 7 Nov 2018 17:32:53 -0500 Subject: [PATCH 1/5] partial andadh fix --- tests/test_lexer.py | 5 ++++- tests/test_tools.py | 21 +++++++++++++++++++++ xonsh/lexer.py | 23 ++++++++++++++++++----- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/tests/test_lexer.py b/tests/test_lexer.py index ca9a13598..9b84e0986 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -165,7 +165,10 @@ def test_atdollar_expression(): def test_and(): - assert check_token("and", ["AND", "and", 0]) + # no preceding whitespace or other tokens, so this + # resolves to NAME, since it doesn't make sense for + # Python code to start with "and" + assert check_token("and", ["NAME", "and", 0]) def test_ampersand(): diff --git a/tests/test_tools.py b/tests/test_tools.py index 09dd1b231..30b866065 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -374,6 +374,27 @@ def test_subproc_toks_pyeval_nested(): assert exp == obs +@pytest.mark.parametrize('phrase', [ + 'xandy', + 'xory', + 'xand', + 'andy', + 'xor', + 'ory', + 'x-and', + 'x-or', + 'and-y', + 'or-y', + 'x-and-y', + 'x-or-y', +]) +def test_subproc_toks_and_or(phrase): + s = "echo " + phrase + exp = "![{0}]".format(s) + obs = subproc_toks(s, lexer=LEXER, returnline=True) + assert exp == obs + + def test_subproc_toks_pyeval_nested_parens(): s = "echo @(min(1, 42))" inp = "({0})".format(s) diff --git a/xonsh/lexer.py b/xonsh/lexer.py index 9eff6d6f0..85356511a 100644 --- a/xonsh/lexer.py +++ b/xonsh/lexer.py @@ -116,24 +116,37 @@ def token_map(): return tm +NEED_WHITESPACE = frozenset(["and", "or"]) + + def handle_name(state, token): """Function for handling name tokens""" typ = "NAME" + prev = state["last"] + state["last"] = token + next_tok = next(state["stream"]) + has_whitespace = (prev is not None and + prev.end != token.start + and token.end != next_tok.start) + #) + print(state) if state["pymode"][-1][0]: - if token.string in kwmod.kwlist: + if not has_whitespace and token.string in NEED_WHITESPACE: + pass + elif token.string in kwmod.kwlist: typ = token.string.upper() - state["last"] = token + print("transformed to ", typ) yield _new_token(typ, token.string, token.start) else: - prev = state["last"] - state["last"] = token - has_whitespace = prev.end != token.start if token.string == "and" and has_whitespace: yield _new_token("AND", token.string, token.start) elif token.string == "or" and has_whitespace: yield _new_token("OR", token.string, token.start) else: yield _new_token("NAME", token.string, token.start) + print('\n\n') + #yield from handle_token(state, next_tok) + yield _new_token(next_tok.type, next_tok.string, next_tok.start) def _end_delimiter(state, token): From 351fa27d7ccae8d1727c566d5e28e7a2779058e7 Mon Sep 17 00:00:00 2001 From: Anthony Scopatz Date: Thu, 8 Nov 2018 13:16:44 -0500 Subject: [PATCH 2/5] fixed up dashing --- news/andash.rst | 28 ++++++++++++++++++++++++++++ tests/sample.xsh | 2 +- xonsh/lexer.py | 27 ++++++++++++++------------- 3 files changed, 43 insertions(+), 14 deletions(-) create mode 100644 news/andash.rst diff --git a/news/andash.rst b/news/andash.rst new file mode 100644 index 000000000..f1ea2680a --- /dev/null +++ b/news/andash.rst @@ -0,0 +1,28 @@ +**Added:** + +* + +**Changed:** + +* + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* Fixed issue with ``and`` & ``or`` being incoreectly tokenized in implicit + subprocesses. Auto-wrapping of certain subprocesses will now correctly work. + For example:: + + $ echo x-and-y + x-and-y + +**Security:** + +* diff --git a/tests/sample.xsh b/tests/sample.xsh index 9f6b92249..0795e25ac 100644 --- a/tests/sample.xsh +++ b/tests/sample.xsh @@ -2,4 +2,4 @@ aliases['echo'] = lambda args, stdin=None: print(' '.join(args)) $WAKKA = "jawaka" -x = $(echo "hello mom" $WAKKA) \ No newline at end of file +x = $(echo "hello mom" $WAKKA) diff --git a/xonsh/lexer.py b/xonsh/lexer.py index 85356511a..4368196de 100644 --- a/xonsh/lexer.py +++ b/xonsh/lexer.py @@ -4,6 +4,7 @@ Written using a hybrid of ``tokenize`` and PLY. """ import io +import re # 'keyword' interferes with ast.keyword import keyword as kwmod @@ -119,34 +120,34 @@ def token_map(): NEED_WHITESPACE = frozenset(["and", "or"]) +@lazyobject +def RE_NEED_WHITESPACE(): + pattern = "\s?(" + "|".join(NEED_WHITESPACE) + r")(\s|[\\]$)" + return re.compile(pattern) + + def handle_name(state, token): """Function for handling name tokens""" typ = "NAME" prev = state["last"] state["last"] = token - next_tok = next(state["stream"]) - has_whitespace = (prev is not None and - prev.end != token.start - and token.end != next_tok.start) - #) - print(state) + needs_whitespace = token.string in NEED_WHITESPACE + has_whitespace = needs_whitespace and RE_NEED_WHITESPACE.match( + token.line[max(0, token.start[1] - 1) :] + ) if state["pymode"][-1][0]: - if not has_whitespace and token.string in NEED_WHITESPACE: + if needs_whitespace and not has_whitespace: pass elif token.string in kwmod.kwlist: typ = token.string.upper() - print("transformed to ", typ) yield _new_token(typ, token.string, token.start) else: - if token.string == "and" and has_whitespace: + if has_whitespace and token.string == "and": yield _new_token("AND", token.string, token.start) - elif token.string == "or" and has_whitespace: + elif has_whitespace and token.string == "or": yield _new_token("OR", token.string, token.start) else: yield _new_token("NAME", token.string, token.start) - print('\n\n') - #yield from handle_token(state, next_tok) - yield _new_token(next_tok.type, next_tok.string, next_tok.start) def _end_delimiter(state, token): From 3e72fb06c7971c54039b2b1665f48c94fd13b599 Mon Sep 17 00:00:00 2001 From: Anthony Scopatz Date: Thu, 8 Nov 2018 13:19:21 -0500 Subject: [PATCH 3/5] path tests --- tests/test_tools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_tools.py b/tests/test_tools.py index 30b866065..74ff7e834 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -387,6 +387,8 @@ def test_subproc_toks_pyeval_nested(): 'or-y', 'x-and-y', 'x-or-y', + 'in/and/path', + 'in/or/path', ]) def test_subproc_toks_and_or(phrase): s = "echo " + phrase From f0b78c223a7963a4b3533036385e5cf7c82107c3 Mon Sep 17 00:00:00 2001 From: Anthony Scopatz Date: Thu, 8 Nov 2018 13:26:09 -0500 Subject: [PATCH 4/5] fix deprecation warning --- xonsh/lexer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xonsh/lexer.py b/xonsh/lexer.py index 4368196de..1884c49f3 100644 --- a/xonsh/lexer.py +++ b/xonsh/lexer.py @@ -122,7 +122,7 @@ NEED_WHITESPACE = frozenset(["and", "or"]) @lazyobject def RE_NEED_WHITESPACE(): - pattern = "\s?(" + "|".join(NEED_WHITESPACE) + r")(\s|[\\]$)" + pattern = r"\s?(" + "|".join(NEED_WHITESPACE) + r")(\s|[\\]$)" return re.compile(pattern) From 5b54e3cb04641e5c4ce95e5f0512fc3560a25190 Mon Sep 17 00:00:00 2001 From: Anthony Scopatz Date: Thu, 8 Nov 2018 13:41:26 -0500 Subject: [PATCH 5/5] rm unused variable --- xonsh/lexer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xonsh/lexer.py b/xonsh/lexer.py index 1884c49f3..e51cdd82a 100644 --- a/xonsh/lexer.py +++ b/xonsh/lexer.py @@ -129,7 +129,6 @@ def RE_NEED_WHITESPACE(): def handle_name(state, token): """Function for handling name tokens""" typ = "NAME" - prev = state["last"] state["last"] = token needs_whitespace = token.string in NEED_WHITESPACE has_whitespace = needs_whitespace and RE_NEED_WHITESPACE.match(