Xompletions (#4521)

* feat: add command completers logic

* feat: implement xompleter logic

* fix: handle callable object in exception

* chore: add xompletions package to setup.py

* fix: update tests for changes to command completer logic

* docs:

* fix: qa errors

fixes #4514

* feat: add xonsh completions

* refactor: split module matcher to separate class

* feat: add django-admin completions

* fix: failing tests

* feat: add more properties to completion-context

* refactor: cleanup code

* todo item add

news/feat-xompletions.rst

@@ -0,0 +1,24 @@
+**Added:**
+
+* Python files with command completions can be put inside ``xompletions`` namespace package,
+  they will get loaded lazily.
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

setup.py

@@ -323,10 +323,12 @@ def main():
             "xonsh.prompt",
             "xonsh.lib",
             "xonsh.webconfig",
+            "xompletions",
         ],
         package_dir={
             "xonsh": "xonsh",
             "xontrib": "xontrib",
+            "xompletions": "xompletions",
             "xonsh.lib": "xonsh/lib",
             "xonsh.webconfig": "xonsh/webconfig",
         },

tests/completers/test_dir_completers.py

@@ -1,21 +1,10 @@
-import pytest
 import tempfile
 from os import sep
 
-from xonsh.completers.tools import RichCompletion
-from xonsh.completers.dirs import complete_cd, complete_rmdir
-from xonsh.parsers.completion_context import (
-    CompletionContext,
-    CommandContext,
-    CommandArg,
-)
+import pytest
 
 from tests.tools import ON_WINDOWS
-
-COMPLETERS = {
-    "cd": complete_cd,
-    "rmdir": complete_rmdir,
-}
+from xonsh.completers.tools import RichCompletion
 
 CUR_DIR = "." if ON_WINDOWS else "./"
 PARENT_DIR = ".." if ON_WINDOWS else "../"
@@ -28,52 +17,14 @@ def setup(xession, xonsh_execer):
         xession.env["CDPATH"] = set()
 
 
-@pytest.fixture(params=list(COMPLETERS))
+@pytest.fixture(params=["cd", "rmdir"])
 def cmd(request):
     return request.param
 
 
-def test_not_cmd(cmd):
-    """Ensure the cd completer doesn't complete other commands"""
-    assert not COMPLETERS[cmd](
-        CompletionContext(
-            CommandContext(
-                args=(CommandArg(f"not-{cmd}"),),
-                arg_index=1,
-            )
-        )
-    )
-
-
-def complete_cmd(cmd, prefix, opening_quote="", closing_quote=""):
-    result = COMPLETERS[cmd](
-        CompletionContext(
-            CommandContext(
-                args=(CommandArg(cmd),),
-                arg_index=1,
-                prefix=prefix,
-                opening_quote=opening_quote,
-                closing_quote=closing_quote,
-                is_after_closing_quote=bool(closing_quote),
-            )
-        )
-    )
-    assert result and len(result) == 2
-    completions, lprefix = result
-    assert lprefix == len(opening_quote) + len(prefix) + len(
-        closing_quote
-    )  # should override the quotes
-    return completions
-
-
-def complete_cmd_dirs(*a, **kw):
-    return [r.value for r in complete_cmd(*a, **kw)]
-
-
-def test_non_dir(cmd):
+def test_non_dir(cmd, check_completer):
     with tempfile.NamedTemporaryFile(dir=".", suffix="_dummySuffix") as tmp:
-        with pytest.raises(StopIteration):  # tmp is a file
-            complete_cmd(cmd, tmp.name[:-2])
+        assert not check_completer(cmd, prefix=tmp.name[:-2])
 
 
 @pytest.fixture(scope="module")
@@ -82,16 +33,16 @@ def dir_path():
         yield tmp_path
 
 
-def test_dirs_only(cmd, dir_path):
-    completions = complete_cmd(cmd, dir_path[:-2])
+def test_dirs_only(cmd, dir_path, check_completer):
+    completions = check_completer(cmd, dir_path[:-2])
     assert completions == {dir_path + sep}
 
 
-def test_opening_quotes(cmd, dir_path):
-    assert complete_cmd(cmd, dir_path, opening_quote="r'") == {f"r'{dir_path}{sep}'"}
+def test_opening_quotes(cmd, dir_path, check_completer):
+    assert check_completer(cmd, "r'" + dir_path) == {f"r'{dir_path}{sep}'"}
 
 
-def test_closing_quotes(cmd, dir_path):
+def test_closing_quotes(cmd, dir_path, check_completer):
     prefix = dir_path
     exp = f"'''{dir_path}{sep}'''"
     if ON_WINDOWS:
@@ -99,33 +50,35 @@ def test_closing_quotes(cmd, dir_path):
         # the path completer converts to a raw string if there's a backslash
         exp = "r" + exp
 
-    completions = complete_cmd(cmd, prefix, opening_quote="'''", closing_quote="'''")
+    values, completions = check_completer(
+        cmd, "'''" + prefix + "'''", send_original=True
+    )
 
-    assert completions == {exp}
+    assert values == {exp}
 
-    completion = completions.pop()
+    completion = list(completions).pop()
     assert isinstance(completion, RichCompletion)
     assert completion.append_closing_quote is False
 
 
-def test_complete_dots(xession):
+def test_complete_dots(xession, check_completer):
     with xession.env.swap(COMPLETE_DOTS="never"):
-        dirs = complete_cmd_dirs("cd", "")
+        dirs = check_completer("cd")
         assert CUR_DIR not in dirs and PARENT_DIR not in dirs
 
-        dirs = complete_cmd_dirs("cd", ".")
+        dirs = check_completer("cd", ".")
         assert CUR_DIR not in dirs and PARENT_DIR not in dirs
 
     with xession.env.swap(COMPLETE_DOTS="matching"):
-        dirs = complete_cmd_dirs("cd", "")
+        dirs = check_completer("cd", "")
         assert CUR_DIR not in dirs and PARENT_DIR not in dirs
 
-        dirs = complete_cmd_dirs("cd", ".")
+        dirs = check_completer("cd", ".")
         assert CUR_DIR in dirs and PARENT_DIR in dirs
 
     with xession.env.swap(COMPLETE_DOTS="always"):
-        dirs = complete_cmd_dirs("cd", "")
+        dirs = check_completer("cd", "")
         assert CUR_DIR in dirs and PARENT_DIR in dirs
 
-        dirs = complete_cmd_dirs("cd", ".")
+        dirs = check_completer("cd", ".")
         assert CUR_DIR in dirs and PARENT_DIR in dirs

tests/completers/test_pip_completer.py

@@ -1,23 +1,22 @@
 import pytest
 
-from tests.tools import ON_WINDOWS
-from xonsh.completers.pip import PIP_RE
+from xonsh.completers.commands import complete_xompletions
+
+
+regex_cases = [
+    "pip",
+    "pip.exe",
+    "pip3.6.exe",
+    "xpip",
+]
 
 
 @pytest.mark.parametrize(
     "line",
-    [
-        "pip",
-        "pip.exe",
-        "pip3.6.exe",
-        "xpip",
-        "/usr/bin/pip3",
-        r"C:\Python\Scripts\pip",
-        r"C:\Python\Scripts\pip.exe",
-    ],
+    regex_cases,
 )
 def test_pip_re(line):
-    assert PIP_RE.search(line)
+    assert complete_xompletions.matcher.search_completer(line)
 
 
 @pytest.mark.parametrize(
@@ -35,7 +34,7 @@ def test_pip_re(line):
     ],
 )
 def test_pip_list_re1(line):
-    assert PIP_RE.search(line) is None
+    assert complete_xompletions.matcher.search_completer(line) is None
 
 
 @pytest.mark.parametrize(
@@ -49,8 +48,6 @@ def test_completions(line, prefix, exp, check_completer, xession, os_env, monkey
     # use the actual PATH from os. Otherwise subproc will fail on windows. `unintialized python...`
     monkeypatch.setattr(xession, "env", os_env)
 
-    if ON_WINDOWS:
-        line = line.replace("pip", "pip.exe")
     comps = check_completer(line, prefix=prefix)
 
     assert comps.intersection(exp)

tests/conftest.py

@@ -232,10 +232,15 @@ def completion_context_parse():
     return CompletionContextParser().parse
 
 
+@pytest.fixture(scope="session")
+def completer_obj():
+    return Completer()
+
+
 @pytest.fixture
-def check_completer():
+def check_completer(completer_obj):
     """Helper function to run completer and parse the results as set of strings"""
-    completer = Completer()
+    completer = completer_obj
 
     def _factory(line: str, prefix="", send_original=False):
         completions, _ = completer.complete_line(line, prefix=prefix)

xompletions/cd.py

@@ -0,0 +1,12 @@
+from xonsh.completers.path import complete_dir
+from xonsh.parsers.completion_context import CommandContext
+
+
+def xonsh_complete(command: CommandContext):
+    """
+    Completion for "cd", includes only valid directory names.
+    """
+    results, lprefix = complete_dir(command)
+    if len(results) == 0:
+        raise StopIteration
+    return results, lprefix

xompletions/django-admin.py

@@ -0,0 +1,16 @@
+"""Completers for pip."""
+
+from xonsh.completers.tools import comp_based_completer
+
+from xonsh.parsers.completion_context import CommandContext
+
+
+def xonsh_complete(ctx: CommandContext):
+    """Completes python's package manager pip."""
+    # adapted from https://github.com/django/django/blob/main/extras/django_bash_completion
+
+    # todo: find a way to get description for the completions like here
+    #   1. https://github.com/apie/fish-django-completions/blob/master/fish_django_completions.py
+    #   2. complete python manage.py invocations
+    # https://github.com/django/django/blob/main/extras/django_bash_completion
+    return comp_based_completer(ctx, DJANGO_AUTO_COMPLETE="1")

xompletions/pip.py

@@ -0,0 +1,10 @@
+"""Completers for pip."""
+
+from xonsh.completers.tools import comp_based_completer
+from xonsh.parsers.completion_context import CommandContext
+
+
+def xonsh_complete(ctx: CommandContext):
+    """Completes python's package manager pip."""
+
+    return comp_based_completer(ctx, PIP_AUTO_COMPLETE="1")

xompletions/rmdir.py

@@ -0,0 +1,14 @@
+from xonsh.completers.man import complete_from_man
+from xonsh.completers.path import complete_dir
+from xonsh.parsers.completion_context import CompletionContext, CommandContext
+
+
+def xonsh_complete(command: CommandContext):
+    """
+    Completion for "rmdir", includes only valid directory names.
+    """
+    opts = complete_from_man(CompletionContext(command))
+    comps, lp = complete_dir(command)
+    if len(comps) == 0 and len(opts) == 0:
+        raise StopIteration
+    return comps | opts, lp

xompletions/xonsh.py

@@ -0,0 +1,17 @@
+from xonsh.cli_utils import ArgparseCompleter
+
+from xonsh.completers.tools import get_filter_function
+from xonsh.parsers.completion_context import CommandContext
+
+
+def xonsh_complete(command: CommandContext):
+    """Completer for ``xonsh`` command using its ``argparser``"""
+
+    from xonsh.main import parser
+
+    completer = ArgparseCompleter(parser, command=command)
+    fltr = get_filter_function()
+    for comp in completer.complete():
+        if fltr(comp, command.prefix):
+            yield comp
+    # todo: part of return value will have unfiltered=False/True. based on that we can use fuzzy to rank the results

xonsh/completer.py

@@ -154,8 +154,9 @@ class Completer:
                 # completer requested to stop collecting completions
                 break
             except Exception as e:
+                name = func.__name__ if hasattr(func, "__name__") else str(func)
                 print_exception(
-                    f"Completer {func.__name__} raises exception when gets "
+                    f"Completer {name} raises exception when gets "
                     f"old_args={old_completer_args[:-1]} / completion_context={completion_context!r}:\n"
                     f"{type(e)} - {e}"
                 )

xonsh/completers/commands.py

@@ -1,4 +1,8 @@
+import functools
+import importlib.util
 import os
+import re
+import typing as tp
 
 import xonsh.tools as xt
 import xonsh.platform as xp
@@ -96,3 +100,91 @@ def complete_end_proc_keywords(command_context: CommandContext):
     if prefix in END_PROC_KEYWORDS:
         return {RichCompletion(prefix, append_space=True)}
     return None
+
+
+class ModuleMatcher:
+    """Reusable module matcher. Can be used by other completers like Python to find matching script completions"""
+
+    def __init__(self, base: str):
+        # list of pre-defined patterns. More can be added using the public method ``.wrap``
+        self._patterns: tp.Dict[str, str] = {
+            r"\bx?pip(?:\d|\.)*(exe)?$": "pip",
+        }
+        self._compiled: tp.Dict[str, tp.Pattern] = {}
+        self.contextual = True
+        self.base = base
+
+    def wrap(self, pattern: str, module: str):
+        """For any commands matching the pattern complete from the ``module``"""
+        self._patterns[pattern] = module
+
+    def get_module(self, name):
+        try:
+            # todo: not just namespace package,
+            #  add an environment variable to get list of paths to check for completions like fish
+            #  - https://fishshell.com/docs/current/completions.html#where-to-put-completions
+            return importlib.import_module(f"{self.base}.{name}")
+        except ModuleNotFoundError:
+            pass
+
+    def search_completer(self, cmd: str, cleaned=False):
+        if not cleaned:
+            cmd = CommandCompleter.clean_cmd_name(cmd)
+        # try any pattern match
+        for pattern, mod_name in self._patterns.items():
+            # lazy compile regex
+            if pattern not in self._compiled:
+                self._compiled[pattern] = re.compile(pattern, re.IGNORECASE)
+            regex = self._compiled[pattern]
+            if regex.match(cmd):
+                return self.get_module(mod_name)
+
+
+class CommandCompleter:
+    """Lazily complete commands from `xompletions` package
+
+    The base-name (case-insensitive) of the executable is used to find the matching completer module
+    or the regex patterns.
+    """
+
+    def __init__(self):
+        self.contextual = True
+        self.matcher = ModuleMatcher("xompletions")
+
+    @staticmethod
+    @functools.lru_cache(10)
+    def clean_cmd_name(cmd: str):
+        cmd_name = os.path.basename(cmd).lower()
+        exts = XSH.env.get("PATHEXT", [])
+        for ex in exts:
+            if cmd_name.endswith(ex.lower()):
+                # windows handling
+                cmd_name = cmd_name.rstrip(ex.lower())
+                break
+        return cmd_name
+
+    def __call__(self, full_ctx: CompletionContext):
+        """For the given command load completions lazily"""
+
+        # completion for commands only
+        ctx = full_ctx.command
+        if not ctx:
+            return
+
+        if ctx.arg_index == 0:
+            return
+
+        cmd_name = self.clean_cmd_name(ctx.command)
+        module = self.matcher.get_module(cmd_name) or self.matcher.search_completer(
+            cmd_name, cleaned=True
+        )
+
+        if not module:
+            return
+
+        if hasattr(module, "xonsh_complete"):
+            func = module.xonsh_complete
+            return func(ctx)
+
+
+complete_xompletions = CommandCompleter()

xonsh/completers/dirs.py

@@ -1,30 +0,0 @@
-from xonsh.completers.man import complete_from_man
-from xonsh.completers.path import complete_dir
-from xonsh.completers.tools import contextual_command_completer_for
-from xonsh.parsers.completion_context import (
-    CompletionContext,
-    CommandContext,
-)
-
-
-@contextual_command_completer_for("cd")
-def complete_cd(command: CommandContext):
-    """
-    Completion for "cd", includes only valid directory names.
-    """
-    results, lprefix = complete_dir(command)
-    if len(results) == 0:
-        raise StopIteration
-    return results, lprefix
-
-
-@contextual_command_completer_for("rmdir")
-def complete_rmdir(command: CommandContext):
-    """
-    Completion for "rmdir", includes only valid directory names.
-    """
-    opts = complete_from_man(CompletionContext(command))
-    comps, lp = complete_dir(command)
-    if len(comps) == 0 and len(opts) == 0:
-        raise StopIteration
-    return comps | opts, lp

xonsh/completers/init.py

@@ -1,12 +1,10 @@
 """Constructor for xonsh completer objects."""
 import collections
 
-from xonsh.completers.pip import complete_pip
 from xonsh.completers.man import complete_from_man
 from xonsh.completers.bash import complete_from_bash
 from xonsh.completers.base import complete_base
 from xonsh.completers.path import complete_path
-from xonsh.completers.dirs import complete_cd, complete_rmdir
 from xonsh.completers.python import (
     complete_python,
 )
@@ -15,6 +13,7 @@ from xonsh.completers.commands import (
     complete_skipper,
     complete_end_proc_tokens,
     complete_end_proc_keywords,
+    complete_xompletions,
 )
 from xonsh.completers._aliases import complete_aliases
 from xonsh.completers.environment import complete_environment_vars
@@ -32,9 +31,7 @@ def default_completers():
             ("base", complete_base),
             ("skip", complete_skipper),
             ("alias", complete_aliases),
-            ("pip", complete_pip),
-            ("cd", complete_cd),
-            ("rmdir", complete_rmdir),
+            ("xompleter", complete_xompletions),
             ("import", complete_import),
             ("bash", complete_from_bash),
             ("man", complete_from_man),

xonsh/completers/pip.py

@@ -1,56 +0,0 @@
-"""Completers for pip."""
-import re
-import shlex
-import subprocess
-
-import xonsh.lazyasd as xl
-from xonsh.built_ins import XSH
-from xonsh.completers.tools import (
-    contextual_command_completer,
-    get_filter_function,
-    RichCompletion,
-)
-from xonsh.parsers.completion_context import CommandContext
-
-
-@xl.lazyobject
-def PIP_RE():
-    return re.compile(r"\bx?pip(?:\d|\.)*(exe)?$")
-
-
-@contextual_command_completer
-def complete_pip(context: CommandContext):
-    """Completes python's package manager pip."""
-    prefix = context.prefix
-
-    if context.arg_index == 0 or (not PIP_RE.search(context.args[0].value.lower())):
-        return None
-    filter_func = get_filter_function()
-
-    args = [arg.raw_value for arg in context.args]
-    env = XSH.env.detype()
-    env.update(
-        {
-            "PIP_AUTO_COMPLETE": "1",
-            "COMP_WORDS": " ".join(args),
-            "COMP_CWORD": str(len(context.args)),
-        }
-    )
-
-    try:
-        proc = subprocess.run(
-            [args[0]],
-            stderr=subprocess.DEVNULL,
-            stdout=subprocess.PIPE,
-            env=env,
-        )
-    except FileNotFoundError:
-        return None
-
-    if proc.stdout:
-        out = shlex.split(proc.stdout.decode())
-        for cmp in out:
-            if filter_func(cmp, prefix):
-                yield RichCompletion(cmp, append_space=True)
-
-    return None

xonsh/completers/tools.py

@@ -1,5 +1,7 @@
 """Xonsh completer tools."""
 import inspect
+import shlex
+import subprocess
 import textwrap
 import typing as tp
 from functools import wraps
@@ -201,3 +203,35 @@ def apply_lprefix(comps, lprefix):
                 yield comp
         else:
             yield RichCompletion(comp, prefix_len=lprefix)
+
+
+def comp_based_completer(ctx: CommandContext, **env: str):
+    """Helper function to complete commands such as ``pip``,``django-admin``,... that use bash's ``complete``"""
+    prefix = ctx.prefix
+
+    filter_func = get_filter_function()
+
+    args = [arg.raw_value for arg in ctx.args]
+    env.update(
+        {
+            "COMP_WORDS": " ".join(args),
+            "COMP_CWORD": str(ctx.arg_index),
+        }
+    )
+    env.update(XSH.env.detype())
+
+    try:
+        proc = subprocess.run(
+            [args[0]],
+            stderr=subprocess.DEVNULL,
+            stdout=subprocess.PIPE,
+            env=env,
+        )
+    except FileNotFoundError:
+        return None
+
+    if proc.stdout:
+        out = shlex.split(proc.stdout.decode())
+        for cmp in out:
+            if filter_func(cmp, prefix):
+                yield RichCompletion(cmp, append_space=True)

xonsh/parsers/completion_context.py

@@ -71,7 +71,7 @@ class CommandContext(NamedTuple):
 
     def completing_command(self, command: str) -> bool:
         """Return whether this context is completing args for a command"""
-        return self.arg_index > 0 and self.args[0].value == command
+        return self.arg_index > 0 and self.command == command
 
     @property
     def raw_prefix(self):