mirrors/xonsh
1
0
Fork 0
mirror of https://github.com/xonsh/xonsh.git synced 2025-03-04 16:34:47 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Support raw-string path literals

Browse source
This commit is contained in:
Gordon Ball 2016-12-08 13:56:45 +01:00
parent ee97ae8bd0
commit dbc86bc955
4 changed files with 28 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
tests/test_lexer.py
View file

@ -192,6 +192,11 @@ def test_single_bytes_literal():
def test_path_string_literal():
assert check_token("p'/foo'", ['STRING', "p'/foo'", 0])
assert check_token('p"/foo"', ['STRING', 'p"/foo"', 0])
assert check_token("pr'/foo'", ['STRING', "pr'/foo'", 0])
assert check_token('pr"/foo"', ['STRING', 'pr"/foo"', 0])
assert check_token("rp'/foo'", ['STRING', "rp'/foo'", 0])
assert check_token('rp"/foo"', ['STRING', 'rp"/foo"', 0])
def test_regex_globs():
for i in ('.*', r'\d*', '.*#{1,2}'):

4
tests/test_parser.py
View file

@ -1492,6 +1492,10 @@ def test_async_await():
def test_path_literal():
check_xonsh_ast({}, 'p"/foo"', False)
check_xonsh_ast({}, 'pr"/foo"', False)
check_xonsh_ast({}, 'rp"/foo"', False)
check_xonsh_ast({}, 'pR"/foo"', False)
check_xonsh_ast({}, 'Rp"/foo"', False)
def test_dollar_name():
check_xonsh_ast({'WAKKA': 42}, '$WAKKA')

12
xonsh/parsers/base.py
View file

@ -16,12 +16,14 @@ from xonsh import ast
from xonsh.ast import has_elts, xonsh_call
from xonsh.lexer import Lexer, LexToken
from xonsh.platform import PYTHON_VERSION_INFO
from xonsh.tokenize import SearchPath
from xonsh.tokenize import SearchPath, StringPrefix
from xonsh.lazyasd import LazyObject
from xonsh.parsers.context_check import check_contexts
RE_SEARCHPATH = LazyObject(lambda: re.compile(SearchPath), globals(),
'RE_SEARCHPATH')
RE_STRINGPREFIX = LazyObject(lambda: re.compile(StringPrefix), globals(),
'RE_STRINGPREFIX')
class Location(object):
@ -1977,14 +1979,16 @@ class BaseParser(object):
def p_string_literal(self, p):
"""string_literal : string_tok"""
p1 = p[1]
if p1.value.startswith('p'):
s = ast.Str(s=ast.literal_eval(p1.value[1:]), lineno=p1.lineno,
prefix = RE_STRINGPREFIX.match(p1.value).group()
if 'p' in prefix:
value_without_p = prefix.replace('p', '') + p1.value[len(prefix):]
s = ast.Str(s=ast.literal_eval(value_without_p), lineno=p1.lineno,
col_offset=p1.lexpos)
p[0] = xonsh_call('__xonsh_path_literal__', [s],
lineno=p1.lineno, col=p1.lexpos)
else:
s = ast.literal_eval(p1.value)
cls = ast.Bytes if p1.value.startswith('b') else ast.Str
cls = ast.Bytes if 'b' in prefix else ast.Str
p[0] = cls(s=s, lineno=p1.lineno, col_offset=p1.lexpos)
def p_string_literal_list(self, p):

14
xonsh/tokenize.py
View file

@ -205,7 +205,7 @@ Floatnumber = group(Pointfloat, Expfloat)
Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
Number = group(Imagnumber, Floatnumber, Intnumber)
StringPrefix = r'(?:[bB][rR]?|[rR][bB]?|[uU]|[p])?'
StringPrefix = r'(?:[bBp][rR]?|[rR][bBp]?|[uU])?'
# Tail end of ' string.
Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
@ -276,6 +276,10 @@ endpats = {"'": Single, '"': Double,
"u'''": Single3, 'u"""': Double3,
"U'''": Single3, 'U"""': Double3,
"p'''": Single3, 'p"""': Double3,
"pr'''": Single3, 'pr"""': Double3,
"pR'''": Single3, 'pR"""': Double3,
"rp'''": Single3, 'rp"""': Double3,
"Rp'''": Single3, 'Rp"""': Double3,
'r': None, 'R': None, 'b': None, 'B': None,
'u': None, 'U': None, 'p': None}
@ -288,7 +292,9 @@ for t in ("'''", '"""',
"rb'''", 'rb"""', "rB'''", 'rB"""',
"Rb'''", 'Rb"""', "RB'''", 'RB"""',
"u'''", 'u"""', "U'''", 'U"""',
"p'''", 'p""""',
"p'''", 'p""""', "pr'''", 'pr""""',
"pR'''", 'pR""""', "rp'''", 'rp""""',
"Rp'''", 'Rp""""',
):
triple_quoted[t] = t
single_quoted = {}
@ -300,7 +306,9 @@ for t in ("'", '"',
"rb'", 'rb"', "rB'", 'rB"',
"Rb'", 'Rb"', "RB'", 'RB"',
"u'", 'u"', "U'", 'U"',
"p'", 'p"',
"p'", 'p"', "pr'", 'pr"',
"pR'", 'pR"', "rp'", 'rp"',
"Rp'", 'Rp"',
):
single_quoted[t] = t