diff --git a/tests/test_lexer.py b/tests/test_lexer.py index 90b678826..e44d0829d 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -192,6 +192,11 @@ def test_single_bytes_literal(): def test_path_string_literal(): assert check_token("p'/foo'", ['STRING', "p'/foo'", 0]) assert check_token('p"/foo"', ['STRING', 'p"/foo"', 0]) + assert check_token("pr'/foo'", ['STRING', "pr'/foo'", 0]) + assert check_token('pr"/foo"', ['STRING', 'pr"/foo"', 0]) + assert check_token("rp'/foo'", ['STRING', "rp'/foo'", 0]) + assert check_token('rp"/foo"', ['STRING', 'rp"/foo"', 0]) + def test_regex_globs(): for i in ('.*', r'\d*', '.*#{1,2}'): diff --git a/tests/test_parser.py b/tests/test_parser.py index 6f69e22ef..81b9f8743 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1492,6 +1492,10 @@ def test_async_await(): def test_path_literal(): check_xonsh_ast({}, 'p"/foo"', False) + check_xonsh_ast({}, 'pr"/foo"', False) + check_xonsh_ast({}, 'rp"/foo"', False) + check_xonsh_ast({}, 'pR"/foo"', False) + check_xonsh_ast({}, 'Rp"/foo"', False) def test_dollar_name(): check_xonsh_ast({'WAKKA': 42}, '$WAKKA') diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py index 2c95e6d0d..085f8b8f3 100644 --- a/xonsh/parsers/base.py +++ b/xonsh/parsers/base.py @@ -16,12 +16,14 @@ from xonsh import ast from xonsh.ast import has_elts, xonsh_call from xonsh.lexer import Lexer, LexToken from xonsh.platform import PYTHON_VERSION_INFO -from xonsh.tokenize import SearchPath +from xonsh.tokenize import SearchPath, StringPrefix from xonsh.lazyasd import LazyObject from xonsh.parsers.context_check import check_contexts RE_SEARCHPATH = LazyObject(lambda: re.compile(SearchPath), globals(), 'RE_SEARCHPATH') +RE_STRINGPREFIX = LazyObject(lambda: re.compile(StringPrefix), globals(), + 'RE_STRINGPREFIX') class Location(object): @@ -1977,14 +1979,16 @@ class BaseParser(object): def p_string_literal(self, p): """string_literal : string_tok""" p1 = p[1] - if p1.value.startswith('p'): - s = ast.Str(s=ast.literal_eval(p1.value[1:]), lineno=p1.lineno, + prefix = RE_STRINGPREFIX.match(p1.value).group() + if 'p' in prefix: + value_without_p = prefix.replace('p', '') + p1.value[len(prefix):] + s = ast.Str(s=ast.literal_eval(value_without_p), lineno=p1.lineno, col_offset=p1.lexpos) p[0] = xonsh_call('__xonsh_path_literal__', [s], lineno=p1.lineno, col=p1.lexpos) else: s = ast.literal_eval(p1.value) - cls = ast.Bytes if p1.value.startswith('b') else ast.Str + cls = ast.Bytes if 'b' in prefix else ast.Str p[0] = cls(s=s, lineno=p1.lineno, col_offset=p1.lexpos) def p_string_literal_list(self, p): diff --git a/xonsh/tokenize.py b/xonsh/tokenize.py index 4b1837522..3d4b5d92e 100644 --- a/xonsh/tokenize.py +++ b/xonsh/tokenize.py @@ -205,7 +205,7 @@ Floatnumber = group(Pointfloat, Expfloat) Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) -StringPrefix = r'(?:[bB][rR]?|[rR][bB]?|[uU]|[p])?' +StringPrefix = r'(?:[bBp][rR]?|[rR][bBp]?|[uU])?' # Tail end of ' string. Single = r"[^'\\]*(?:\\.[^'\\]*)*'" @@ -276,6 +276,10 @@ endpats = {"'": Single, '"': Double, "u'''": Single3, 'u"""': Double3, "U'''": Single3, 'U"""': Double3, "p'''": Single3, 'p"""': Double3, + "pr'''": Single3, 'pr"""': Double3, + "pR'''": Single3, 'pR"""': Double3, + "rp'''": Single3, 'rp"""': Double3, + "Rp'''": Single3, 'Rp"""': Double3, 'r': None, 'R': None, 'b': None, 'B': None, 'u': None, 'U': None, 'p': None} @@ -288,7 +292,9 @@ for t in ("'''", '"""', "rb'''", 'rb"""', "rB'''", 'rB"""', "Rb'''", 'Rb"""', "RB'''", 'RB"""', "u'''", 'u"""', "U'''", 'U"""', - "p'''", 'p""""', + "p'''", 'p""""', "pr'''", 'pr""""', + "pR'''", 'pR""""', "rp'''", 'rp""""', + "Rp'''", 'Rp""""', ): triple_quoted[t] = t single_quoted = {} @@ -300,7 +306,9 @@ for t in ("'", '"', "rb'", 'rb"', "rB'", 'rB"', "Rb'", 'Rb"', "RB'", 'RB"', "u'", 'u"', "U'", 'U"', - "p'", 'p"', + "p'", 'p"', "pr'", 'pr"', + "pR'", 'pR"', "rp'", 'rp"', + "Rp'", 'Rp"', ): single_quoted[t] = t