feat: f-glob strings (#4835)

* feat: add support for f-glob strings

Move xonsh_pathsearch() into the BaseParser class because it needs to use self._set_error()

Parametrize 6 backtick tests in test_parser.py into test_backtick() (and add cases for f-glob strings)

* add news

* docs: update tutorial

* fix news file

docs/tutorial.rst

@@ -1054,6 +1054,23 @@ mode or subprocess mode) by using the ``g````:
     5
 
 
+Formatted Glob Literals
+-----------------------
+
+Using the ``f`` modifier with either regex or normal globbing makes
+the glob pattern behave like a formatted string literal. This can be used to
+substitute variables and other expressions into the glob pattern:
+
+.. code-block:: xonshcon
+
+    >>> touch a aa aaa aba abba aab aabb abcba
+    >>> mypattern = 'ab'
+    >>> print(f`{mypattern[0]}+`)
+    ['a', 'aa', 'aaa']
+    >>> print(gf`{mypattern}*`)
+    ['aba', 'abba', 'abcba']
+
+
 Custom Path Searches
 --------------------
 

news/feat-f-glob-strings.rst

@@ -0,0 +1,23 @@
+**Added:**
+
+* Support for f-glob strings (e.g. ``fg`{prefix}*```)
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

tests/test_parser.py

@@ -2376,8 +2376,11 @@ def test_ls_regex(check_xonsh_ast):
     check_xonsh_ast({}, "$(ls `[Ff]+i*LE` -l)", False)
 
 
-def test_backtick(check_xonsh_ast):
-    check_xonsh_ast({}, "print(`.*`)", False)
+@pytest.mark.parametrize("p", ["", "p"])
+@pytest.mark.parametrize("f", ["", "f"])
+@pytest.mark.parametrize("glob_type", ["", "r", "g"])
+def test_backtick(p, f, glob_type, check_xonsh_ast):
+    check_xonsh_ast({}, f"print({p}{f}{glob_type}`.*`)", False)
 
 
 def test_ls_regex_octothorpe(check_xonsh_ast):
@@ -2388,10 +2391,6 @@ def test_ls_explicitregex(check_xonsh_ast):
     check_xonsh_ast({}, "$(ls r`[Ff]+i*LE` -l)", False)
 
 
-def test_rbacktick(check_xonsh_ast):
-    check_xonsh_ast({}, "print(r`.*`)", False)
-
-
 def test_ls_explicitregex_octothorpe(check_xonsh_ast):
     check_xonsh_ast({}, "$(ls r`#[Ff]+i*LE` -l)", False)
 
@@ -2400,22 +2399,6 @@ def test_ls_glob(check_xonsh_ast):
     check_xonsh_ast({}, "$(ls g`[Ff]+i*LE` -l)", False)
 
 
-def test_gbacktick(check_xonsh_ast):
-    check_xonsh_ast({}, "print(g`.*`)", False)
-
-
-def test_pbacktrick(check_xonsh_ast):
-    check_xonsh_ast({}, "print(p`.*`)", False)
-
-
-def test_pgbacktick(check_xonsh_ast):
-    check_xonsh_ast({}, "print(pg`.*`)", False)
-
-
-def test_prbacktick(check_xonsh_ast):
-    check_xonsh_ast({}, "print(pr`.*`)", False)
-
-
 def test_ls_glob_octothorpe(check_xonsh_ast):
     check_xonsh_ast({}, "$(ls g`#[Ff]+i*LE` -l)", False)
 

xonsh/parsers/base.py

@@ -124,31 +124,6 @@ def xonsh_superhelp(x, lineno=None, col=None):
     return xonsh_call("__xonsh__.superhelp", [x], lineno=lineno, col=col)
 
 
-def xonsh_pathsearch(pattern, pymode=False, lineno=None, col=None):
-    """Creates the AST node for calling the __xonsh__.pathsearch() function.
-    The pymode argument indicate if it is called from subproc or python mode"""
-    pymode = ast.NameConstant(value=pymode, lineno=lineno, col_offset=col)
-    searchfunc, pattern = RE_SEARCHPATH.match(pattern).groups()
-    pattern = ast.Str(s=pattern, lineno=lineno, col_offset=col)
-    pathobj = False
-    if searchfunc.startswith("@"):
-        func = searchfunc[1:]
-    elif "g" in searchfunc:
-        func = "__xonsh__.globsearch"
-        pathobj = "p" in searchfunc
-    else:
-        func = "__xonsh__.regexsearch"
-        pathobj = "p" in searchfunc
-    func = load_attribute_chain(func, lineno=lineno, col=col)
-    pathobj = ast.NameConstant(value=pathobj, lineno=lineno, col_offset=col)
-    return xonsh_call(
-        "__xonsh__.pathsearch",
-        args=[func, pattern, pymode, pathobj],
-        lineno=lineno,
-        col=col,
-    )
-
-
 def load_ctx(x):
     """Recursively sets ctx to ast.Load()"""
     if not hasattr(x, "ctx"):
@@ -658,6 +633,44 @@ class BaseParser:
     def _parse_error(self, msg, loc):
         raise_parse_error(msg, loc, self._source, self.lines)
 
+    def xonsh_pathsearch(self, pattern, pymode=False, lineno=None, col=None):
+        """Creates the AST node for calling the __xonsh__.pathsearch() function.
+        The pymode argument indicate if it is called from subproc or python mode"""
+        pymode = ast.NameConstant(value=pymode, lineno=lineno, col_offset=col)
+        searchfunc, pattern = RE_SEARCHPATH.match(pattern).groups()
+        if not searchfunc.startswith("@") and "f" in searchfunc:
+            pattern_as_str = f"f'''{pattern}'''"
+            try:
+                pattern = pyparse(pattern_as_str).body[0].value
+            except SyntaxError:
+                pattern = None
+            if pattern is None:
+                try:
+                    pattern = FStringAdaptor(
+                        pattern_as_str, "f", filename=self.lexer.fname
+                    ).run()
+                except SyntaxError as e:
+                    self._set_error(str(e), self.currloc(lineno=lineno, column=col))
+        else:
+            pattern = ast.Str(s=pattern, lineno=lineno, col_offset=col)
+        pathobj = False
+        if searchfunc.startswith("@"):
+            func = searchfunc[1:]
+        elif "g" in searchfunc:
+            func = "__xonsh__.globsearch"
+            pathobj = "p" in searchfunc
+        else:
+            func = "__xonsh__.regexsearch"
+            pathobj = "p" in searchfunc
+        func = load_attribute_chain(func, lineno=lineno, col=col)
+        pathobj = ast.NameConstant(value=pathobj, lineno=lineno, col_offset=col)
+        return xonsh_call(
+            "__xonsh__.pathsearch",
+            args=[func, pattern, pymode, pathobj],
+            lineno=lineno,
+            col=col,
+        )
+
     #
     # Precedence of operators
     #
@@ -2413,7 +2426,9 @@ class BaseParser:
 
     def p_atom_pathsearch(self, p):
         """atom : SEARCHPATH"""
-        p[0] = xonsh_pathsearch(p[1], pymode=True, lineno=self.lineno, col=self.col)
+        p[0] = self.xonsh_pathsearch(
+            p[1], pymode=True, lineno=self.lineno, col=self.col
+        )
 
     # introduce seemingly superfluous symbol 'atom_dname' to enable reuse it in other places
     def p_atom_dname_indirection(self, p):
@@ -3352,7 +3367,7 @@ class BaseParser:
 
     def p_subproc_atom_re(self, p):
         """subproc_atom : SEARCHPATH"""
-        p0 = xonsh_pathsearch(p[1], pymode=False, lineno=self.lineno, col=self.col)
+        p0 = self.xonsh_pathsearch(p[1], pymode=False, lineno=self.lineno, col=self.col)
         p0._cliarg_action = "extend"
         p[0] = p0
 

xonsh/tokenize.py

@@ -305,7 +305,7 @@ String = group(
 )
 
 # Xonsh-specific Syntax
-SearchPath = r"((?:[rgp]+|@\w*)?)`([^\n`\\]*(?:\\.[^\n`\\]*)*)`"
+SearchPath = r"((?:[rgpf]+|@\w*)?)`([^\n`\\]*(?:\\.[^\n`\\]*)*)`"
 
 # Because of leftmost-then-longest match semantics, be sure to put the
 # longest operators first (e.g., if = came before ==, == would get