From 059fc301e7577ebe975ae017fa3fcd0275f965a6 Mon Sep 17 00:00:00 2001 From: Andy Kipp Date: Tue, 9 Jul 2024 09:44:03 +0200 Subject: [PATCH] Introduce new resolver for executables to replace commands_cache usages and speed up everything (#5544) * remove commands_cache from pyghooks to avoid cc.update_cache on every key press * create executables.py * replace cc.locate_binary to locate_executable * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * vc: replace locate_binary * pyghooks: remove commands_cache * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * remove unused func _yield_executables * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Move `executables_in` from tools to commands_cache to avoid circular imports. * First steps to `procs.executables` that is based on `commands_cache` and `tools`. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * test_executables * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add not recommended notes * tests * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add `get_paths` with test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix source test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * vc: remove tests because commands cache is not used * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * specs: fix exception for recursive call * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * specs: fix exception for recursive call * improve test_locate_executable * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix test * beautify pathext * tests * docs * tests * update locators * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * locate_file * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * comments * return environ locate bin test * comments * Update xonsh/procs/executables.py Co-authored-by: Jason R. Coombs * Update xonsh/procs/executables.py Co-authored-by: Jason R. Coombs * Update xonsh/procs/executables.py Co-authored-by: Jason R. Coombs * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add itertools * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * moving is_executable * doc * optimization is_executable_in_windows * micro improvements * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * news * news * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * bump test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: a <1@1.1> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Jason R. Coombs --- .../commands_cache_new_resolve_executable.rst | 23 ++++ tests/aliases/test_source.py | 8 +- tests/bin/bar | 1 + tests/bin/foo | 1 + tests/procs/test_executables.py | 46 ++++++++ tests/prompt/test_vc.py | 22 ---- tests/test_commands_cache.py | 46 ++++++++ tests/test_integrations.py | 5 +- tests/test_tools.py | 44 -------- xonsh/aliases.py | 3 +- xonsh/commands_cache.py | 86 +++++++++++---- xonsh/completers/commands.py | 6 +- xonsh/environ.py | 22 +--- xonsh/lib/itertools.py | 26 +++++ xonsh/procs/executables.py | 103 ++++++++++++++++++ xonsh/procs/specs.py | 8 +- xonsh/prompt/vc.py | 7 +- xonsh/pyghooks.py | 9 +- xonsh/pytest/plugin.py | 4 +- xonsh/tools.py | 56 ---------- 20 files changed, 341 insertions(+), 185 deletions(-) create mode 100644 news/commands_cache_new_resolve_executable.rst create mode 100755 tests/bin/bar create mode 100755 tests/bin/foo create mode 100644 tests/procs/test_executables.py create mode 100644 xonsh/procs/executables.py diff --git a/news/commands_cache_new_resolve_executable.rst b/news/commands_cache_new_resolve_executable.rst new file mode 100644 index 000000000..4eb803d9f --- /dev/null +++ b/news/commands_cache_new_resolve_executable.rst @@ -0,0 +1,23 @@ +**Added:** + +* + +**Changed:** + +* New executable resolving method was introduced and the commands_cache usages were replaced in the key places. As result we expect speed up in xonsh startup, reducing lagging during typing in prompt and speed ups during the commands execution (#5544 by @anki-code). + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* + +**Security:** + +* diff --git a/tests/aliases/test_source.py b/tests/aliases/test_source.py index 07401b6c1..6a39f96f1 100644 --- a/tests/aliases/test_source.py +++ b/tests/aliases/test_source.py @@ -1,6 +1,7 @@ import builtins import os.path from contextlib import contextmanager +from pathlib import Path from unittest.mock import MagicMock import pytest @@ -34,10 +35,9 @@ def test_source_current_dir(mockopen, monkeypatch, mocked_execx_checker): assert mocked_execx_checker == ["foo", "bar"] -def test_source_path(mockopen, mocked_execx_checker, patch_locate_binary, xession): - patch_locate_binary(xession.commands_cache) - - source_alias(["foo", "bar"]) +def test_source_path(mockopen, mocked_execx_checker, xession): + with xession.env.swap(PATH=[Path(__file__).parent.parent / "bin"]): + source_alias(["foo", "bar"]) path_foo = os.path.join("bin", "foo") path_bar = os.path.join("bin", "bar") assert mocked_execx_checker[0].endswith(path_foo) diff --git a/tests/bin/bar b/tests/bin/bar new file mode 100755 index 000000000..2f1be35ad --- /dev/null +++ b/tests/bin/bar @@ -0,0 +1 @@ +# Test source_alias \ No newline at end of file diff --git a/tests/bin/foo b/tests/bin/foo new file mode 100755 index 000000000..2f1be35ad --- /dev/null +++ b/tests/bin/foo @@ -0,0 +1 @@ +# Test source_alias \ No newline at end of file diff --git a/tests/procs/test_executables.py b/tests/procs/test_executables.py new file mode 100644 index 000000000..e552e548f --- /dev/null +++ b/tests/procs/test_executables.py @@ -0,0 +1,46 @@ +import os + +from xonsh.environ import Env +from xonsh.platform import ON_WINDOWS +from xonsh.procs.executables import get_paths, get_possible_names, locate_executable + + +def test_get_possible_names(): + env = Env(PATHEXT=[".EXE", ".COM"]) + assert get_possible_names("file", env) == ["file", "file.exe", "file.com"] + assert get_possible_names("FILE", env) == ["FILE", "FILE.EXE", "FILE.COM"] + + +def test_get_paths(tmpdir): + bindir1 = str(tmpdir.mkdir("bindir1")) + bindir2 = str(tmpdir.mkdir("bindir2")) + env = Env(PATH=[bindir1, bindir2, bindir1, "nodir"]) + assert get_paths(env) == (bindir2, bindir1) + + +def test_locate_executable(tmpdir, xession): + bindir = tmpdir.mkdir("bindir") + bindir.mkdir("subdir") + executables = ["file1.EXE", "file2.COM", "file2.EXE", "file3"] + not_executables = ["file4.EXE", "file5"] + for exefile in executables + not_executables: + f = bindir / exefile + f.write_text("binary", encoding="utf8") + if exefile in executables: + os.chmod(f, 0o777) + + pathext = [".EXE", ".COM"] if ON_WINDOWS else [] + with xession.env.swap(PATH=str(bindir), PATHEXT=pathext): + assert locate_executable("file1.EXE") + assert locate_executable("nofile") is None + assert locate_executable("file5") is None + assert locate_executable("subdir") is None + if ON_WINDOWS: + assert locate_executable("file1") + assert locate_executable("file4") + assert locate_executable("file2").endswith("file2.exe") + else: + assert locate_executable("file3") + assert locate_executable("file1") is None + assert locate_executable("file4") is None + assert locate_executable("file2") is None diff --git a/tests/prompt/test_vc.py b/tests/prompt/test_vc.py index c33ee679b..93fc6ae7f 100644 --- a/tests/prompt/test_vc.py +++ b/tests/prompt/test_vc.py @@ -2,7 +2,6 @@ import os import subprocess as sp import textwrap from pathlib import Path -from unittest.mock import Mock import pytest @@ -119,27 +118,6 @@ current = yellow reverse assert not branch.startswith("\u001b[") -def test_current_branch_calls_locate_binary_for_empty_cmds_cache(xession, monkeypatch): - cache = xession.commands_cache - monkeypatch.setattr(cache, "is_empty", Mock(return_value=True)) - monkeypatch.setattr(cache, "locate_binary", Mock(return_value="")) - vc.current_branch() - assert cache.locate_binary.called - - -def test_current_branch_does_not_call_locate_binary_for_non_empty_cmds_cache( - xession, - monkeypatch, -): - cache = xession.commands_cache - monkeypatch.setattr(cache, "is_empty", Mock(return_value=False)) - monkeypatch.setattr(cache, "locate_binary", Mock(return_value="")) - # make lazy locate return nothing to avoid running vc binaries - monkeypatch.setattr(cache, "lazy_locate_binary", Mock(return_value="")) - vc.current_branch() - assert not cache.locate_binary.called - - def test_dirty_working_directory(repo, set_xenv): get_dwd = "{}_dirty_working_directory".format(repo["vc"]) set_xenv(repo["dir"]) diff --git a/tests/test_commands_cache.py b/tests/test_commands_cache.py index bf3012dbd..b8825ba1b 100644 --- a/tests/test_commands_cache.py +++ b/tests/test_commands_cache.py @@ -1,6 +1,8 @@ import os import pickle +import stat import time +from tempfile import TemporaryDirectory import pytest @@ -8,12 +10,16 @@ from xonsh.commands_cache import ( SHELL_PREDICTOR_PARSER, CommandsCache, _Commands, + executables_in, predict_false, predict_shell, predict_true, ) +from xonsh.platform import ON_WINDOWS from xonsh.pytest.tools import skip_if_on_windows +PATHEXT_ENV = {"PATHEXT": [".COM", ".EXE", ".BAT"]} + def test_commands_cache_lazy(xession): cc = xession.commands_cache @@ -260,3 +266,43 @@ def test_nixos_coreutils(tmp_path): assert cache.predict_threadable(["echo", "1"]) is True assert cache.predict_threadable(["cat", "file"]) is False + + +def test_executables_in(xession): + expected = set() + types = ("file", "directory", "brokensymlink") + if ON_WINDOWS: + # Don't test symlinks on windows since it requires admin + types = ("file", "directory") + executables = (True, False) + with TemporaryDirectory() as test_path: + for _type in types: + for executable in executables: + fname = f"{_type}_{executable}" + if _type == "none": + continue + if _type == "file" and executable: + ext = ".exe" if ON_WINDOWS else "" + expected.add(fname + ext) + else: + ext = "" + path = os.path.join(test_path, fname + ext) + if _type == "file": + with open(path, "w") as f: + f.write(fname) + elif _type == "directory": + os.mkdir(path) + elif _type == "brokensymlink": + tmp_path = os.path.join(test_path, "i_wont_exist") + with open(tmp_path, "w") as f: + f.write("deleteme") + os.symlink(tmp_path, path) + os.remove(tmp_path) + if executable and not _type == "brokensymlink": + os.chmod(path, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + if ON_WINDOWS: + xession.env = PATHEXT_ENV + result = set(executables_in(test_path)) + else: + result = set(executables_in(test_path)) + assert expected == result diff --git a/tests/test_integrations.py b/tests/test_integrations.py index 5970d845d..191831dbc 100644 --- a/tests/test_integrations.py +++ b/tests/test_integrations.py @@ -729,8 +729,11 @@ def test_script(case): if ON_DARWIN: script = script.replace("tests/bin", str(Path(__file__).parent / "bin")) out, err, rtn = run_xonsh(script) + out = out.replace("bash: no job control in this shell\n", "") if callable(exp_out): - assert exp_out(out) + assert exp_out( + out + ), f"CASE:\nscript=***\n{script}\n***,\nExpected: {exp_out!r},\nActual: {out!r}" else: assert exp_out == out assert exp_rtn == rtn diff --git a/tests/test_tools.py b/tests/test_tools.py index 725bbad4c..49fcc6211 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -4,10 +4,8 @@ import datetime as dt import os import pathlib import re -import stat import subprocess import warnings -from tempfile import TemporaryDirectory import pytest @@ -35,7 +33,6 @@ from xonsh.tools import ( ensure_timestamp, env_path_to_str, escape_windows_cmd_string, - executables_in, expand_case_matching, expand_path, expandvars, @@ -101,7 +98,6 @@ INDENT = " " TOOLS_ENV = {"EXPAND_ENV_VARS": True, "XONSH_ENCODING_ERRORS": "strict"} ENCODE_ENV_ONLY = {"XONSH_ENCODING_ERRORS": "strict"} -PATHEXT_ENV = {"PATHEXT": [".COM", ".EXE", ".BAT"]} def test_random_choice(): @@ -1616,46 +1612,6 @@ def test_partial_string(leaders, prefix, quote): assert obs == exp -def test_executables_in(xession): - expected = set() - types = ("file", "directory", "brokensymlink") - if ON_WINDOWS: - # Don't test symlinks on windows since it requires admin - types = ("file", "directory") - executables = (True, False) - with TemporaryDirectory() as test_path: - for _type in types: - for executable in executables: - fname = f"{_type}_{executable}" - if _type == "none": - continue - if _type == "file" and executable: - ext = ".exe" if ON_WINDOWS else "" - expected.add(fname + ext) - else: - ext = "" - path = os.path.join(test_path, fname + ext) - if _type == "file": - with open(path, "w") as f: - f.write(fname) - elif _type == "directory": - os.mkdir(path) - elif _type == "brokensymlink": - tmp_path = os.path.join(test_path, "i_wont_exist") - with open(tmp_path, "w") as f: - f.write("deleteme") - os.symlink(tmp_path, path) - os.remove(tmp_path) - if executable and not _type == "brokensymlink": - os.chmod(path, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) - if ON_WINDOWS: - xession.env = PATHEXT_ENV - result = set(executables_in(test_path)) - else: - result = set(executables_in(test_path)) - assert expected == result - - @pytest.mark.parametrize( "inp, exp", [ diff --git a/xonsh/aliases.py b/xonsh/aliases.py index 2ae5cc1ca..bc0e4b070 100644 --- a/xonsh/aliases.py +++ b/xonsh/aliases.py @@ -31,6 +31,7 @@ from xonsh.platform import ( ON_OPENBSD, ON_WINDOWS, ) +from xonsh.procs.executables import locate_file from xonsh.procs.jobs import bg, clean_jobs, disown, fg, jobs from xonsh.procs.specs import SpecAttrModifierAlias, SpecModifierAlias from xonsh.timings import timeit_alias @@ -637,7 +638,7 @@ def source_alias(args, stdin=None): for i, fname in enumerate(args): fpath = fname if not os.path.isfile(fpath): - fpath = locate_binary(fname) + fpath = locate_file(fname) if fpath is None: if env.get("XONSH_DEBUG"): print(f"source: {fname}: No such file", file=sys.stderr) diff --git a/xonsh/commands_cache.py b/xonsh/commands_cache.py index cfa32d16e..0e8191584 100644 --- a/xonsh/commands_cache.py +++ b/xonsh/commands_cache.py @@ -16,7 +16,12 @@ from pathlib import Path from xonsh.lib.lazyasd import lazyobject from xonsh.platform import ON_POSIX, ON_WINDOWS, pathbasename -from xonsh.tools import executables_in +from xonsh.procs.executables import ( + get_paths, + get_possible_names, + is_executable_in_posix, + is_executable_in_windows, +) if ON_WINDOWS: from case_insensitive_dict import CaseInsensitiveDict as CacheDict @@ -29,12 +34,58 @@ class _Commands(tp.NamedTuple): cmds: "tuple[str, ...]" +def _yield_accessible_unix_file_names(path): + """yield file names of executable files in path.""" + if not os.path.exists(path): + return + for file_ in os.scandir(path): + if is_executable_in_posix(file_): + yield file_.name + + +def _executables_in_posix(path): + if not os.path.exists(path): + return + else: + yield from _yield_accessible_unix_file_names(path) + + +def _executables_in_windows(path): + if not os.path.isdir(path): + return + try: + for x in os.scandir(path): + if is_executable_in_windows(x): + yield x.name + except FileNotFoundError: + # On Windows, there's no guarantee for the directory to really + # exist even if isdir returns True. This may happen for instance + # if the path contains trailing spaces. + return + + +def executables_in(path) -> tp.Iterable[str]: + """Returns a generator of files in path that the user could execute.""" + if ON_WINDOWS: + func = _executables_in_windows + else: + func = _executables_in_posix + try: + yield from func(path) + except PermissionError: + return + + class CommandsCache(cabc.Mapping): """A lazy cache representing the commands available on the file system. The keys are the command names and the values a tuple of (loc, has_alias) where loc is either a str pointing to the executable on the file system or None (if no executable exists) and has_alias is a boolean flag for whether the command has an alias. + + Note! There is ``xonsh.procs.executables`` module with resolving executables. + Usage ``executables`` is preferred instead of commands_cache for cases + where you just need to locate executable command. """ CACHE_FILE = "path-commands-cache.pickle" @@ -99,25 +150,7 @@ class CommandsCache(cabc.Mapping): return len(self._cmds_cache) == 0 def get_possible_names(self, name): - """Expand name to all possible variants based on `PATHEXT`. - - PATHEXT is a Windows convention containing extensions to be - considered when searching for an executable file. - - Conserves order of any extensions found and gives precedence - to the bare name. - """ - extensions = [""] + self.env.get("PATHEXT", []) - return [name + ext for ext in extensions] - - @staticmethod - def remove_dups(paths): - cont = set() - for p in map(os.path.realpath, paths): - if p not in cont: - cont.add(p) - if os.path.isdir(p): - yield p + return get_possible_names(name, self.env) def _update_aliases_cache(self): """Update aliases checksum and return result: updated or not.""" @@ -162,10 +195,15 @@ class CommandsCache(cabc.Mapping): return current_path def update_cache(self): + """The main function to update commands cache. + Note! There is ``xonsh.procs.executables`` module with resolving executables. + Usage ``executables`` is preferred instead of commands_cache for cases + where you just need to locate executable command. + """ env = self.env # iterate backwards so that entries at the front of PATH overwrite # entries at the back. - paths = tuple(reversed(tuple(self.remove_dups(env.get("PATH") or [])))) + paths = get_paths(env) if self._update_and_check_changes(paths): all_cmds = CacheDict() for cmd, path in self._iter_binaries(paths): @@ -256,6 +294,9 @@ class CommandsCache(cabc.Mapping): def locate_binary(self, name, ignore_alias=False): """Locates an executable on the file system using the cache. + NOT RECOMMENDED. Take a look into `xonsh.procs.executables.locate_executable` + before using this function. + Parameters ---------- name : str @@ -270,6 +311,9 @@ class CommandsCache(cabc.Mapping): def lazy_locate_binary(self, name, ignore_alias=False): """Locates an executable in the cache, without checking its validity. + NOT RECOMMENDED. Take a look into `xonsh.procs.executables.locate_executable` + before using this function. + Parameters ---------- name : str diff --git a/xonsh/completers/commands.py b/xonsh/completers/commands.py index 59e624edc..f6a67e491 100644 --- a/xonsh/completers/commands.py +++ b/xonsh/completers/commands.py @@ -4,8 +4,8 @@ import re import typing as tp import xonsh.platform as xp -import xonsh.tools as xt from xonsh.built_ins import XSH +from xonsh.commands_cache import executables_in from xonsh.completer import Completer from xonsh.completers.tools import ( RichCompletion, @@ -35,12 +35,12 @@ def complete_command(command: CommandContext): kwargs["description"] = "Alias" if is_alias else path yield RichCompletion(s, append_space=True, **kwargs) if xp.ON_WINDOWS: - for i in xt.executables_in("."): + for i in executables_in("."): if i.startswith(cmd): yield RichCompletion(i, append_space=True) base = os.path.basename(cmd) if os.path.isdir(base): - for i in xt.executables_in(base): + for i in executables_in(base): if i.startswith(cmd): yield RichCompletion(os.path.join(base, i)) diff --git a/xonsh/environ.py b/xonsh/environ.py index 87c29e8a5..9ff7121d8 100644 --- a/xonsh/environ.py +++ b/xonsh/environ.py @@ -54,7 +54,6 @@ from xonsh.tools import ( dynamic_cwd_tuple_to_str, ensure_string, env_path_to_str, - executables_in, history_tuple_to_str, intensify_colors_on_win_setter, is_bool, @@ -2547,22 +2546,13 @@ class InternalEnvironDict(ChainMap): local.update(new_local) -def _yield_executables(directory, name): - if ON_WINDOWS: - base_name, ext = os.path.splitext(name.lower()) - for fname in executables_in(directory): - fbase, fext = os.path.splitext(fname.lower()) - if base_name == fbase and (len(ext) == 0 or ext == fext): - yield os.path.join(directory, fname) - else: - for x in executables_in(directory): - if x == name: - yield os.path.join(directory, name) - return - - def locate_binary(name): - """Locates an executable on the file system.""" + """Locates an executable on the file system. + + NOT RECOMMENDED because ``commands_cache.locate_binary`` contains ``update_cache`` + with scanning all files in ``$PATH``. First of all take a look into ``xonsh.specs.executables`` + for more fast implementation the locate operation. + """ return XSH.commands_cache.locate_binary(name) diff --git a/xonsh/lib/itertools.py b/xonsh/lib/itertools.py index 1b64fc201..1f07920d8 100644 --- a/xonsh/lib/itertools.py +++ b/xonsh/lib/itertools.py @@ -1,3 +1,6 @@ +from itertools import filterfalse + + def as_iterable(iterable_or_scalar): """Utility for converting an object to an iterable. Parameters @@ -34,3 +37,26 @@ def as_iterable(iterable_or_scalar): return iterable_or_scalar else: return (iterable_or_scalar,) + + +def unique_everseen(iterable, key=None): + """Yield unique elements, preserving order. Remember all elements ever seen. + + ``` + unique_everseen('AAAABBBCCDAABBB') → A B C D + unique_everseen('ABBcCAD', str.casefold) → A B c D + ``` + + Source code: https://docs.python.org/3/library/itertools.html#itertools-recipes + """ + seen = set() + if key is None: + for element in filterfalse(seen.__contains__, iterable): + seen.add(element) + yield element + else: + for element in iterable: + k = key(element) + if k not in seen: + seen.add(k) + yield element diff --git a/xonsh/procs/executables.py b/xonsh/procs/executables.py new file mode 100644 index 000000000..a6c81989b --- /dev/null +++ b/xonsh/procs/executables.py @@ -0,0 +1,103 @@ +"""Interfaces to locate executable files on file system.""" + +import itertools +import os +from pathlib import Path + +from xonsh.built_ins import XSH +from xonsh.lib.itertools import unique_everseen +from xonsh.platform import ON_WINDOWS + + +def get_possible_names(name, env=None): + """Expand name to all possible variants based on `PATHEXT`. + + PATHEXT is a Windows convention containing extensions to be + considered when searching for an executable file. + + Conserves order of any extensions found and gives precedence + to the bare name. + """ + env = env if env is not None else XSH.env + env_pathext = env.get("PATHEXT", []) + if not env_pathext: + return [name] + upper = name.upper() == name + extensions = [""] + env_pathext + return [name + (ext.upper() if upper else ext.lower()) for ext in extensions] + + +def clear_paths(paths): + """Remove duplicates and nonexistent directories from paths.""" + return filter(os.path.isdir, unique_everseen(map(os.path.realpath, paths))) + + +def get_paths(env=None): + """Return tuple with deduplicated and existent paths from ``$PATH``.""" + env = env if env is not None else XSH.env + return tuple(reversed(tuple(clear_paths(env.get("PATH") or [])))) + + +def is_executable_in_windows(filepath, env=None): + """Check the file is executable in Windows.""" + filepath = Path(filepath) + try: + try: + if not filepath.is_file(): + return False + except OSError: + return False + + env = env if env is not None else XSH.env + return any(s.lower() == filepath.suffix.lower() for s in env.get("PATHEXT", [])) + except FileNotFoundError: + # On Windows, there's no guarantee for the directory to really + # exist even if isdir returns True. This may happen for instance + # if the path contains trailing spaces. + return False + + +def is_executable_in_posix(filepath): + """Check the file is executable in POSIX.""" + try: + return filepath.is_file() and os.access(filepath, os.X_OK) + except OSError: + # broken Symlink are neither dir not files + pass + return False + + +is_executable = is_executable_in_windows if ON_WINDOWS else is_executable_in_posix + + +def locate_executable(name, env=None): + """Search executable binary name in ``$PATH`` and return full path.""" + return locate_file(name, env=env, check_executable=True, use_pathext=True) + + +def locate_file(name, env=None, check_executable=False, use_pathext=False): + """Search file name in ``$PATH`` and return full path. + + Compromise. There is no way to get case sensitive file name without listing all files. + If the file name is ``CaMeL.exe`` and we found that ``camel.EXE`` exists there is no way + to get back the case sensitive name. We don't want to read the list of files in all ``$PATH`` + directories because of performance reasons. So we're ok to get existent + but case insensitive (or different) result from resolver. + May be in the future file systems as well as Python Path will be smarter to get the case sensitive name. + The task for reading and returning case sensitive filename we give to completer in interactive mode + with ``commands_cache``. + """ + env = env if env is not None else XSH.env + env_path = env.get("PATH", []) + paths = tuple(reversed(tuple(clear_paths(env_path)))) + possible_names = get_possible_names(name, env) if use_pathext else [name] + + for path, possible_name in itertools.product(paths, possible_names): + filepath = Path(path) / possible_name + + try: + if check_executable and not is_executable(filepath): + continue + return str(filepath) + except PermissionError: + return diff --git a/xonsh/procs/specs.py b/xonsh/procs/specs.py index d16c6d3ce..6df5c96d6 100644 --- a/xonsh/procs/specs.py +++ b/xonsh/procs/specs.py @@ -12,13 +12,13 @@ import stat import subprocess import sys -import xonsh.environ as xenv import xonsh.lib.lazyasd as xl import xonsh.lib.lazyimps as xli import xonsh.platform as xp import xonsh.procs.jobs as xj import xonsh.tools as xt from xonsh.built_ins import XSH +from xonsh.procs.executables import locate_executable from xonsh.procs.pipelines import ( STDOUT_CAPTURE_KINDS, CommandPipeline, @@ -734,13 +734,13 @@ class SubprocSpec: alias = self.alias if alias is None: cmd0 = self.cmd[0] - binary_loc = xenv.locate_binary(cmd0) - if binary_loc == cmd0 and cmd0 in self.alias_stack: + binary_loc = locate_executable(cmd0) + if binary_loc is None and cmd0 and cmd0 in self.alias_stack: raise Exception(f'Recursive calls to "{cmd0}" alias.') elif callable(alias): binary_loc = None else: - binary_loc = xenv.locate_binary(alias[0]) + binary_loc = locate_executable(alias[0]) self.binary_loc = binary_loc def resolve_auto_cd(self): diff --git a/xonsh/prompt/vc.py b/xonsh/prompt/vc.py index 7bf145d29..c4d67c30e 100644 --- a/xonsh/prompt/vc.py +++ b/xonsh/prompt/vc.py @@ -13,6 +13,7 @@ import threading import xonsh.tools as xt from xonsh.built_ins import XSH from xonsh.lib.lazyasd import LazyObject +from xonsh.procs.executables import locate_executable RE_REMOVE_ANSI = LazyObject( lambda: re.compile(r"(?:\x1B[@-_]|[\x80-\x9F])[0-?]*[ -/]*[@-~]"), @@ -169,11 +170,7 @@ def _first_branch_timeout_message(): def _vc_has(binary): """This allows us to locate binaries after git only if necessary""" - cmds = XSH.commands_cache - if cmds.is_empty(): - return bool(cmds.locate_binary(binary, ignore_alias=True)) - else: - return bool(cmds.lazy_locate_binary(binary, ignore_alias=True)) + return bool(locate_executable(binary)) def current_branch(): diff --git a/xonsh/pyghooks.py b/xonsh/pyghooks.py index 13e3882e3..28d41ae77 100644 --- a/xonsh/pyghooks.py +++ b/xonsh/pyghooks.py @@ -38,7 +38,6 @@ from xonsh.color_tools import ( make_palette, warn_deprecated_no_color, ) -from xonsh.commands_cache import CommandsCache from xonsh.events import events from xonsh.lib.lazyasd import LazyDict, LazyObject, lazyobject from xonsh.lib.lazyimps import html, os_listxattr, terminal256 @@ -48,6 +47,7 @@ from xonsh.platform import ( pygments_version_info, win_ansi_support, ) +from xonsh.procs.executables import locate_executable from xonsh.pygments_cache import add_custom_style, get_style_by_name from xonsh.style_tools import DEFAULT_STYLE_DICT, norm_name from xonsh.tools import ( @@ -1600,7 +1600,7 @@ def _command_is_valid(cmd): cmd_abspath = os.path.abspath(os.path.expanduser(cmd)) except OSError: return False - return (cmd in XSH.commands_cache and not iskeyword(cmd)) or ( + return ((cmd in XSH.aliases or locate_executable(cmd)) and not iskeyword(cmd)) or ( os.path.isfile(cmd_abspath) and os.access(cmd_abspath, os.X_OK) ) @@ -1649,15 +1649,12 @@ class XonshLexer(Python3Lexer): def __init__(self, *args, **kwargs): # If the lexer is loaded as a pygment plugin, we have to mock - # __xonsh__.env and __xonsh__.commands_cache + # __xonsh__.env if getattr(XSH, "env", None) is None: XSH.env = {} if ON_WINDOWS: pathext = os_environ.get("PATHEXT", [".EXE", ".BAT", ".CMD"]) XSH.env["PATHEXT"] = pathext.split(os.pathsep) - if getattr(XSH, "commands_cache", None) is None: - XSH.commands_cache = CommandsCache(XSH.env) - _ = XSH.commands_cache.all_commands # NOQA super().__init__(*args, **kwargs) tokens = { diff --git a/xonsh/pytest/plugin.py b/xonsh/pytest/plugin.py index 91be2de2f..26ac5c4d7 100644 --- a/xonsh/pytest/plugin.py +++ b/xonsh/pytest/plugin.py @@ -132,8 +132,8 @@ def mock_executables_in(xession, tmp_path, monkeypatch): @pytest.fixture def patch_locate_binary(monkeypatch): - def locate_binary(self, name): - return os.path.join(os.path.dirname(__file__), "bin", name) + def locate_binary(self, name, *args): + return str(Path(__file__).parent.parent.parent / "tests" / "bin" / name) def factory(cc: commands_cache.CommandsCache): monkeypatch.setattr(cc, "locate_binary", types.MethodType(locate_binary, cc)) diff --git a/xonsh/tools.py b/xonsh/tools.py index 159ee52a1..ebb7b7c62 100644 --- a/xonsh/tools.py +++ b/xonsh/tools.py @@ -822,62 +822,6 @@ class redirect_stderr(_RedirectStream): _stream = "stderr" -def _yield_accessible_unix_file_names(path): - """yield file names of executable files in path.""" - if not os.path.exists(path): - return - for file_ in os.scandir(path): - try: - if file_.is_file() and os.access(file_.path, os.X_OK): - yield file_.name - except OSError: - # broken Symlink are neither dir not files - pass - - -def _executables_in_posix(path): - if not os.path.exists(path): - return - else: - yield from _yield_accessible_unix_file_names(path) - - -def _executables_in_windows(path): - if not os.path.isdir(path): - return - extensions = xsh.env["PATHEXT"] - try: - for x in os.scandir(path): - try: - is_file = x.is_file() - except OSError: - continue - if is_file: - fname = x.name - else: - continue - base_name, ext = os.path.splitext(fname) - if ext.upper() in extensions: - yield fname - except FileNotFoundError: - # On Windows, there's no guarantee for the directory to really - # exist even if isdir returns True. This may happen for instance - # if the path contains trailing spaces. - return - - -def executables_in(path) -> tp.Iterable[str]: - """Returns a generator of files in path that the user could execute.""" - if ON_WINDOWS: - func = _executables_in_windows - else: - func = _executables_in_posix - try: - yield from func(path) - except PermissionError: - return - - def debian_command_not_found(cmd): """Uses the debian/ubuntu command-not-found utility to suggest packages for a command that cannot currently be found.