diff --git a/news/completions-improve.rst b/news/completions-improve.rst new file mode 100644 index 000000000..ee88fbab4 --- /dev/null +++ b/news/completions-improve.rst @@ -0,0 +1,23 @@ +**Added:** + +* completions from man page will now show the description for the options if available. + +**Changed:** + +* + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* ``pip`` completer now handles path completions correctly + +**Security:** + +* diff --git a/tests/completers/test_gh.py b/tests/completers/test_gh.py new file mode 100644 index 000000000..769aacd34 --- /dev/null +++ b/tests/completers/test_gh.py @@ -0,0 +1,21 @@ +import pytest + +from tests.tools import skip_if_not_has + +pytestmark = skip_if_not_has("gh") + + +@pytest.mark.parametrize( + "line, exp", + [ + ["gh rep", {"repo"}], + ["gh repo ", {"archive", "clone", "create", "delete", "edit", "fork"}], + ], +) +def test_completions(line, exp, check_completer, xsh_with_env): + # use the actual PATH from os. Otherwise subproc will fail on windows. `unintialized python...` + comps = check_completer(line, prefix=None) + + if callable(exp): + exp = exp() + assert comps.intersection(exp) diff --git a/tests/completers/test_pip_completer.py b/tests/completers/test_pip_completer.py index ed1f69578..c46ce019d 100644 --- a/tests/completers/test_pip_completer.py +++ b/tests/completers/test_pip_completer.py @@ -1,3 +1,6 @@ +import json +import subprocess + import pytest from xonsh.completers.commands import complete_xompletions @@ -37,17 +40,23 @@ def test_pip_list_re1(line): assert complete_xompletions.matcher.search_completer(line) is None +def pip_installed(): + out = subprocess.check_output(["pip", "list", "--format=json"]).decode() + pkgs = json.loads(out) + return {p["name"] for p in pkgs} + + @pytest.mark.parametrize( "line, prefix, exp", [ ["pip", "c", {"cache", "check", "config"}], - ["pip show", "", {"setuptools", "wheel", "pip"}], + ["pip show", "", pip_installed], ], ) -def test_completions(line, prefix, exp, check_completer, xession, os_env, monkeypatch): +def test_completions(line, prefix, exp, check_completer, xsh_with_env): # use the actual PATH from os. Otherwise subproc will fail on windows. `unintialized python...` - monkeypatch.setattr(xession, "env", os_env) - comps = check_completer(line, prefix=prefix) + if callable(exp): + exp = exp() assert comps.intersection(exp) diff --git a/tests/conftest.py b/tests/conftest.py index a567ea730..e9a908d5a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -224,9 +224,16 @@ def xession(mock_xonsh_session) -> XonshSession: @pytest.fixture def xsh_with_aliases(mock_xonsh_session) -> XonshSession: + """Xonsh mock-session with default set of aliases""" return mock_xonsh_session("aliases") +@pytest.fixture +def xsh_with_env(mock_xonsh_session) -> XonshSession: + """Xonsh mock-session with os.environ""" + return mock_xonsh_session("env") + + @pytest.fixture(scope="session") def completion_context_parse(): return CompletionContextParser().parse @@ -242,8 +249,35 @@ def check_completer(completer_obj): """Helper function to run completer and parse the results as set of strings""" completer = completer_obj - def _factory(line: str, prefix="", send_original=False): - completions, _ = completer.complete_line(line, prefix=prefix) + def _factory( + line: str, prefix: "None|str" = "", send_original=False, complete_fn=None + ): + """ + + Parameters + ---------- + line + prefix + send_original + if True, return the original result from the completer (e.g. RichCompletion instances ...) + complete_fn + if given, use that to get the completions + + Returns + ------- + completions as set of string if not send + """ + if prefix is not None: + line += " " + prefix + if complete_fn is None: + completions, _ = completer.complete_line(line) + else: + ctx = completer_obj.parse(line) + out = complete_fn(ctx) + if isinstance(out, tuple): + completions = out[0] + else: + completions = out values = {getattr(i, "value", i).strip() for i in completions} if send_original: diff --git a/tests/man1/man.1.gz b/tests/man1/man.1.gz new file mode 100644 index 000000000..c474dea55 Binary files /dev/null and b/tests/man1/man.1.gz differ diff --git a/tests/test_man.py b/tests/test_man.py index eb3767205..dcd597ec4 100644 --- a/tests/test_man.py +++ b/tests/test_man.py @@ -1,27 +1,107 @@ import os +import subprocess + import pytest # noqa F401 + +from tools import skip_if_on_windows, skip_if_not_on_darwin from xonsh.completers.man import complete_from_man -from tools import skip_if_on_windows - -from xonsh.parsers.completion_context import ( - CompletionContext, - CommandContext, - CommandArg, -) - @skip_if_on_windows -def test_man_completion(monkeypatch, tmpdir, xession): - tempdir = tmpdir.mkdir("test_man") - monkeypatch.setitem( - os.environ, "MANPATH", os.path.dirname(os.path.abspath(__file__)) - ) - xession.env.update({"XONSH_DATA_DIR": str(tempdir)}) - completions = complete_from_man( - CompletionContext( - CommandContext(args=(CommandArg("yes"),), arg_index=1, prefix="--") - ) - ) - assert "--version" in completions - assert "--help" in completions +@pytest.mark.parametrize( + "cmd,exp", + [ + [ + "yes", + {"--version", "--help"}, + ], + [ + "man", + { + "--all", + "--apropos", + "--ascii", + "--catman", + "--config-file", + "--debug", + "--default", + "--ditroff", + "--encoding", + "--extension", + "--global-apropos", + "--gxditview", + "--help", + "--html", + "--ignore-case", + "--local-file", + "--locale", + "--location", + "--location-cat", + "--manpath", + "--match-case", + "--names-only", + "--nh", + "--nj", + "--no-subpages", + "--pager", + "--preprocessor", + "--prompt", + "--recode", + "--regex", + "--sections", + "--systems", + "--troff", + "--troff-device", + "--update", + "--usage", + "--version", + "--warnings", + "--whatis", + "--wildcard", + }, + ], + ], +) +def test_man_completion(xession, check_completer, cmd, exp): + xession.env["MANPATH"] = os.path.dirname(os.path.abspath(__file__)) + completions = check_completer(cmd, complete_fn=complete_from_man, prefix="-") + assert completions == exp + + +@skip_if_not_on_darwin +@pytest.mark.parametrize( + "cmd,exp", + [ + [ + "ar", + { + "-L", + "-S", + "-T", + "-a", + "-b", + "-c", + "-d", + "-i", + "-m", + "-o", + "-p", + "-q", + "-r", + "-s", + "-t", + "-u", + "-x", + }, + ], + ], +) +def test_bsd_man_page_completions(xession, check_completer, cmd, exp): + proc = subprocess.run([cmd, "--version"], stderr=subprocess.PIPE) + if (cmd == "ar" and proc.returncode != 1) or ( + cmd == "man" and proc.stderr.strip() not in {b"man, version 1.6g"} + ): + pytest.skip("A different man page version is installed") + # BSD & Linux have different man page version + completions = check_completer(cmd, complete_fn=complete_from_man, prefix="-") + assert completions == exp diff --git a/tests/test_ptk_shell.py b/tests/test_ptk_shell.py index faac16266..4541c1ff1 100644 --- a/tests/test_ptk_shell.py +++ b/tests/test_ptk_shell.py @@ -9,6 +9,7 @@ import pyte from xonsh.ptk_shell.shell import tokenize_ansi from xonsh.shell import Shell +from tests.tools import ON_WINDOWS @pytest.mark.parametrize( @@ -106,6 +107,10 @@ def test_tokenize_ansi(prompt_tokens, ansi_string_parts): ["2 * 3", "6"], ], ) +@pytest.mark.xfail( + ON_WINDOWS, + reason="Recent versions use Proactor event loop. This may need some handling", +) def test_ptk_prompt(line, exp, ptk_shell, capsys): inp, out, shell = ptk_shell inp.send_text(f"{line}\nexit\n") # note: terminate with '\n' diff --git a/tests/tools.py b/tests/tools.py index 4f1560e98..c1dcf5a03 100644 --- a/tests/tools.py +++ b/tests/tools.py @@ -1,6 +1,7 @@ """Tests the xonsh lexer.""" import copy import os +import shutil import sys import ast import platform @@ -39,11 +40,19 @@ skip_if_on_unix = pytest.mark.skipif(not ON_WINDOWS, reason="Windows stuff") skip_if_on_darwin = pytest.mark.skipif(ON_DARWIN, reason="not Mac friendly") +skip_if_not_on_darwin = pytest.mark.skipif(not ON_DARWIN, reason="Mac only") + skip_if_on_travis = pytest.mark.skipif(ON_TRAVIS, reason="not Travis CI friendly") skip_if_pre_3_8 = pytest.mark.skipif(VER_FULL < (3, 8), reason="Python >= 3.8 feature") +def skip_if_not_has(exe: str): + has_exe = shutil.which(exe) + + return pytest.mark.skipif(not has_exe, reason=f"{exe} is not available.") + + def sp(cmd): return subprocess.check_output(cmd, universal_newlines=True) diff --git a/xompletions/xonsh.py b/xompletions/_xonsh.py similarity index 50% rename from xompletions/xonsh.py rename to xompletions/_xonsh.py index 3977643d5..0964552ae 100644 --- a/xompletions/xonsh.py +++ b/xompletions/_xonsh.py @@ -1,6 +1,5 @@ from xonsh.cli_utils import ArgparseCompleter -from xonsh.completers.tools import get_filter_function from xonsh.parsers.completion_context import CommandContext @@ -10,8 +9,4 @@ def xonsh_complete(command: CommandContext): from xonsh.main import parser completer = ArgparseCompleter(parser, command=command) - fltr = get_filter_function() - for comp in completer.complete(): - if fltr(comp, command.prefix): - yield comp - # todo: part of return value will have unfiltered=False/True. based on that we can use fuzzy to rank the results + return completer.complete(), False diff --git a/xompletions/gh.py b/xompletions/gh.py new file mode 100644 index 000000000..7160a8d89 --- /dev/null +++ b/xompletions/gh.py @@ -0,0 +1,25 @@ +"""Completers for gh CLI""" + +from xonsh.completers.tools import sub_proc_get_output, completion_from_cmd_output +from xonsh.parsers.completion_context import CommandContext + + +def _complete(cmd, *args): + out, _ = sub_proc_get_output(cmd, "__complete", *args) + if out: + # directives + # shellCompDirectiveError 1 + # shellCompDirectiveNoSpace 2 + # shellCompDirectiveNoFileComp 4 + # shellCompDirectiveFilterFileExt 8 + # shellCompDirectiveFilterDirs 16 + # todo: implement directive-numbers above + *lines, dir_num = out.decode().splitlines() + for ln in lines: + yield completion_from_cmd_output(ln) + + +def xonsh_complete(ctx: CommandContext): + cmd, *args = [arg.value for arg in ctx.args] + [ctx.prefix] + + return _complete(cmd, *args) diff --git a/xompletions/rmdir.py b/xompletions/rmdir.py index 0740685a2..e560d76dc 100644 --- a/xompletions/rmdir.py +++ b/xompletions/rmdir.py @@ -1,14 +1,14 @@ -from xonsh.completers.man import complete_from_man from xonsh.completers.path import complete_dir -from xonsh.parsers.completion_context import CompletionContext, CommandContext +from xonsh.parsers.completion_context import CommandContext -def xonsh_complete(command: CommandContext): +def xonsh_complete(ctx: CommandContext): """ Completion for "rmdir", includes only valid directory names. """ - opts = complete_from_man(CompletionContext(command)) - comps, lp = complete_dir(command) - if len(comps) == 0 and len(opts) == 0: - raise StopIteration - return comps | opts, lp + # if starts with the given prefix then it will get completions from man page + if not ctx.prefix.startswith("-") and ctx.arg_index > 0: + comps, lprefix = complete_dir(ctx) + if not comps: + raise StopIteration # no further file completions + return comps, lprefix diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py index d552c68b2..5f03552ba 100644 --- a/xonsh/built_ins.py +++ b/xonsh/built_ins.py @@ -610,7 +610,7 @@ class XonshSession: self.modules_cache = {} self.all_jobs = {} - self.completers = default_completers() + self.completers = default_completers(self.commands_cache) self.builtins = get_default_builtins(execer) self._initial_builtin_names = frozenset(vars(self.builtins)) diff --git a/xonsh/completer.py b/xonsh/completer.py index a350452d5..ff369de2c 100644 --- a/xonsh/completer.py +++ b/xonsh/completer.py @@ -9,6 +9,7 @@ from xonsh.completers.tools import ( RichCompletion, apply_lprefix, is_exclusive_completer, + get_filter_function, ) from xonsh.built_ins import XSH from xonsh.parsers.completion_context import CompletionContext, CompletionContextParser @@ -21,18 +22,48 @@ class Completer: def __init__(self): self.context_parser = CompletionContextParser() - def complete_line(self, line: str, prefix: str = None): - """Handy wrapper to build completion-context when cursor is at the end""" - line = line.strip() - if prefix: - begidx = len(line) + 1 - endidx = begidx + len(prefix) - line = " ".join([line, prefix]) - else: - line += " " - begidx = endidx = len(line) + def parse( + self, text: str, cursor_index: "None|int" = None, ctx=None + ) -> "CompletionContext": + """Parse the given text + + Parameters + ---------- + text + multi-line text + cursor_index + position of the cursor. If not given, then it is considered to be at the end. + ctx + Execution context + """ + cursor_index = len(text) if cursor_index is None else cursor_index + return self.context_parser.parse(text, cursor_index, ctx) + + def complete_line(self, text: str): + """Handy wrapper to build command-completion-context when cursor is at the end. + + Notes + ----- + suffix is not supported; text after last space is parsed as prefix. + """ + ctx = self.parse(text) + cmd_ctx = ctx.command + if not cmd_ctx: + raise RuntimeError("Only Command context is empty") + prefix = cmd_ctx.prefix + + line = text + begidx = text.rfind(prefix) + endidx = begidx + len(prefix) + return self.complete( - prefix, line, begidx, endidx, cursor_index=len(line), multiline_text=line + prefix, + line, + begidx, + endidx, + cursor_index=len(line), + multiline_text=line, + completion_context=ctx, ) def complete( @@ -44,6 +75,7 @@ class Completer: ctx=None, multiline_text=None, cursor_index=None, + completion_context=None, ): """Complete the string, given a possible execution context. @@ -74,16 +106,16 @@ class Completer: Length of the prefix to be replaced in the completion. """ - if multiline_text is not None and cursor_index is not None: - completion_context: tp.Optional[ - CompletionContext - ] = self.context_parser.parse( + if ( + (multiline_text is not None) + and (cursor_index is not None) + and (completion_context is None) + ): + completion_context: tp.Optional[CompletionContext] = self.parse( multiline_text, cursor_index, ctx, ) - else: - completion_context = None ctx = ctx or {} return self.complete_from_context( @@ -140,6 +172,8 @@ class Completer: def generate_completions( completion_context, old_completer_args, trace: bool ) -> tp.Iterator[tp.Tuple[Completion, int]]: + filter_func = get_filter_function() + for name, func in XSH.completers.items(): try: if is_contextual_completer(func): @@ -167,24 +201,38 @@ class Completer: and completion_context is not None and completion_context.command is not None ) + + # -- set comp-defaults -- + + # the default is that the completer function filters out as necessary + # we can change that once fuzzy/substring matches are added + is_filtered = True + custom_lprefix = False + prefix = "" + if completing_contextual_command: + prefix = completion_context.command.prefix + elif old_completer_args is not None: + prefix = old_completer_args[0] + lprefix = len(prefix) + if isinstance(out, cabc.Sequence): - res, lprefix = out - custom_lprefix = True + # update comp-defaults from + res, lprefix_filtered = out + if isinstance(lprefix_filtered, bool): + is_filtered = lprefix_filtered + else: + lprefix = lprefix_filtered + custom_lprefix = True else: res = out - custom_lprefix = False - if completing_contextual_command: - lprefix = len(completion_context.command.prefix) - elif old_completer_args is not None: - lprefix = len(old_completer_args[0]) - else: - lprefix = 0 if res is None: continue items = [] for comp in res: + if (not is_filtered) and (not filter_func(comp, prefix)): + continue comp = Completer._format_completion( comp, completion_context, @@ -215,7 +263,10 @@ class Completer: print("\nTRACE COMPLETIONS: Getting completions with context:") sys.displayhook(completion_context) lprefix = 0 - completions = set() + + # using dict to keep order py3.6+ + completions = {} + query_limit = XSH.env.get("COMPLETION_QUERY_LIMIT") for comp in self.generate_completions( @@ -224,7 +275,7 @@ class Completer: trace, ): completion, lprefix = comp - completions.add(completion) + completions[completion] = None if query_limit and len(completions) >= query_limit: if trace: print( @@ -233,6 +284,7 @@ class Completer: break def sortkey(s): + # todo: should sort with prefix > substring > fuzzy return s.lstrip(''''"''').lower() # the last completer's lprefix is returned. other lprefix values are inside the RichCompletions. diff --git a/xonsh/completers/_aliases.py b/xonsh/completers/_aliases.py index caedc3a5e..437720f2e 100644 --- a/xonsh/completers/_aliases.py +++ b/xonsh/completers/_aliases.py @@ -5,10 +5,7 @@ from xonsh.completers.completer import ( remove_completer, add_one_completer, ) -from xonsh.completers.tools import ( - contextual_command_completer, - get_filter_function, -) +from xonsh.completers.tools import contextual_command_completer from xonsh.parsers.completion_context import CommandContext # for backward compatibility @@ -105,8 +102,7 @@ class CompleterAlias(xcli.ArgParserAlias): def complete( self, - line: xcli.Annotated["list[str]", xcli.Arg(nargs="...")], - prefix: "str | None" = None, + line: str, ): """Output the completions to stdout @@ -119,16 +115,18 @@ class CompleterAlias(xcli.ArgParserAlias): Examples -------- - To get completions such as `git checkout` + To get completions such as ``pip install`` - $ completer complete --prefix=check git + $ completer complete 'pip in' + + To get ``pip`` sub-commands, pass the command with a space at the end + + $ completer complete 'pip ' """ from xonsh.completer import Completer completer = Completer() - completions, prefix_length = completer.complete_line( - " ".join(line), prefix=prefix - ) + completions, prefix_length = completer.complete_line(line) self.out(f"Prefix Length: {prefix_length}") for comp in completions: @@ -172,7 +170,4 @@ def complete_aliases(command: CommandContext): return possible = completer(command=command, alias=alias) - fltr = get_filter_function() - for comp in possible: - if fltr(comp, command.prefix): - yield comp + return possible, False diff --git a/xonsh/completers/commands.py b/xonsh/completers/commands.py index e5a3f8c71..445557c00 100644 --- a/xonsh/completers/commands.py +++ b/xonsh/completers/commands.py @@ -1,3 +1,4 @@ +import contextlib import functools import importlib import importlib.util as im_util @@ -121,9 +122,7 @@ class ModuleMatcher: extra search paths to use if finding module on namespace package fails """ # list of pre-defined patterns. More can be added using the public method ``.wrap`` - self._patterns: tp.Dict[str, str] = { - r"\bx?pip(?:\d|\.)*(exe)?$": "pip", - } + self._patterns: tp.Dict[str, str] = {} self._compiled: tp.Dict[str, tp.Pattern] = {} self.contextual = True self.base = base @@ -182,13 +181,16 @@ class ModuleMatcher: return module @functools.lru_cache(maxsize=10) - def get_module(self, name): - try: - return importlib.import_module(f"{self.base}.{name}") - except ModuleNotFoundError: - file = self._find_file_path(name) - if file: - return self.import_module(file, self.base, name) + def get_module(self, module: str): + for name in [ + module, + f"_{module}", # naming convention to not clash with actual python package + ]: + with contextlib.suppress(ModuleNotFoundError): + return importlib.import_module(f"{self.base}.{name}") + file = self._find_file_path(module) + if file: + return self.import_module(file, self.base, module) def search_completer(self, cmd: str, cleaned=False): if not cleaned: @@ -221,6 +223,7 @@ class CommandCompleter: "xompletions", extra_paths=XSH.env.get("XONSH_COMPLETER_DIRS", []), ) + self._matcher.wrap(r"\bx?pip(?:\d|\.)*(exe)?$", "pip") return self._matcher @staticmethod diff --git a/xonsh/completers/init.py b/xonsh/completers/init.py index 7c7d69988..00fd650b1 100644 --- a/xonsh/completers/init.py +++ b/xonsh/completers/init.py @@ -19,23 +19,32 @@ from xonsh.completers._aliases import complete_aliases from xonsh.completers.environment import complete_environment_vars -def default_completers(): +def default_completers(cmd_cache): """Creates a copy of the default completers.""" - return collections.OrderedDict( + defaults = [ + # non-exclusive completers: + ("end_proc_tokens", complete_end_proc_tokens), + ("end_proc_keywords", complete_end_proc_keywords), + ("environment_vars", complete_environment_vars), + # exclusive completers: + ("base", complete_base), + ("skip", complete_skipper), + ("alias", complete_aliases), + ("xompleter", complete_xompletions), + ("import", complete_import), + ] + + for cmd, func in [ + ("bash", complete_from_bash), + ("man", complete_from_man), + ]: + if cmd in cmd_cache: + defaults.append((cmd, func)) + + defaults.extend( [ - # non-exclusive completers: - ("end_proc_tokens", complete_end_proc_tokens), - ("end_proc_keywords", complete_end_proc_keywords), - ("environment_vars", complete_environment_vars), - # exclusive completers: - ("base", complete_base), - ("skip", complete_skipper), - ("alias", complete_aliases), - ("xompleter", complete_xompletions), - ("import", complete_import), - ("bash", complete_from_bash), - ("man", complete_from_man), ("python", complete_python), ("path", complete_path), ] ) + return collections.OrderedDict(defaults) diff --git a/xonsh/completers/man.py b/xonsh/completers/man.py index cb440d100..db0838a61 100644 --- a/xonsh/completers/man.py +++ b/xonsh/completers/man.py @@ -1,27 +1,128 @@ -import os +import functools +import json import re -import pickle +import shutil import subprocess -import typing as tp +import textwrap +from pathlib import Path -from xonsh.parsers.completion_context import CommandContext from xonsh.built_ins import XSH -import xonsh.lazyasd as xl - -from xonsh.completers.tools import get_filter_function, contextual_command_completer - -OPTIONS: tp.Optional[tp.Dict[str, tp.Any]] = None -OPTIONS_PATH: tp.Optional[str] = None +from xonsh.completers.tools import ( + contextual_command_completer, + RichCompletion, +) +from xonsh.parsers.completion_context import CommandContext -@xl.lazyobject -def SCRAPE_RE(): - return re.compile(r"^(?:\s*(?:-\w|--[a-z0-9-]+)[\s,])+", re.M) +@functools.lru_cache(maxsize=None) +def get_man_completions_path() -> Path: + env = XSH.env or {} + datadir = Path(env["XONSH_DATA_DIR"]) / "generated_completions" / "man" + if datadir.exists() and (not datadir.is_dir()): + shutil.move(datadir, datadir.with_suffix(".bkp")) + if not datadir.exists(): + datadir.mkdir(exist_ok=True, parents=True) + return datadir -@xl.lazyobject -def INNER_OPTIONS_RE(): - return re.compile(r"-\w|--[a-z0-9-]+") +def _get_man_page(cmd: str): + """without control characters""" + env = XSH.env.detype() + manpage = subprocess.Popen( + ["man", cmd], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env + ) + # This is a trick to get rid of reverse line feeds + return subprocess.check_output(["col", "-b"], stdin=manpage.stdout, env=env) + + +@functools.lru_cache(maxsize=None) +def _man_option_string_regex(): + return re.compile( + r"(?:(,\s?)|^|(\sor\s))(?P