mirrors/xonsh
1
0
Fork 0
mirror of https://github.com/xonsh/xonsh.git synced 2025-03-04 16:34:47 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1329 from xonsh/cc-windows

Browse source 
Make CommandsCache play well with Windows
This commit is contained in:
Anthony Scopatz 2016-06-24 14:09:03 -04:00 • committed by GitHub
commit 60c9e27b8d
7 changed files with 197 additions and 154 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
docs/api/commands_cache.rst Normal file
View file

@ -0,0 +1,10 @@
.. _xonsh_commands_cache:
******************************************************
CommandsCache (``xonsh.commands_cache``)
******************************************************
.. automodule:: xonsh.commands_cache
:members:
:undoc-members:
:inherited-members:

5
tests/test_tools.py
View file

@ -11,7 +11,7 @@ from xonsh.platform import ON_WINDOWS
from xonsh.lexer import Lexer
from xonsh.tools import (
CommandsCache, EnvPath, always_false, always_true, argvquote,
EnvPath, always_false, always_true, argvquote,
bool_or_int_to_str, bool_to_str, check_for_partial_string,
dynamic_cwd_tuple_to_str, ensure_int_or_slice, ensure_string,
env_path_to_str, escape_windows_cmd_string, executables_in,
@ -24,8 +24,7 @@ from xonsh.tools import (
is_string_seq, pathsep_to_seq, seq_to_pathsep, is_nonstring_seq_of_strings,
pathsep_to_upper_seq, seq_to_upper_pathsep,
)
from tools import mock_xonsh_env
from xonsh.commands_cache import CommandsCache
from tools import mock_xonsh_env

2
xonsh/__init__.py
View file

@ -57,6 +57,8 @@ else:
_sys.modules['xonsh.proc'] = __amalgam__
xontribs = __amalgam__
_sys.modules['xonsh.xontribs'] = __amalgam__
commands_cache = __amalgam__
_sys.modules['xonsh.commands_cache'] = __amalgam__
environ = __amalgam__
_sys.modules['xonsh.environ'] = __amalgam__
history = __amalgam__

9
xonsh/built_ins.py
View file

@ -9,7 +9,6 @@ import builtins
from collections import Sequence
from contextlib import contextmanager
import inspect
from glob import iglob
import os
import re
import shlex
@ -21,7 +20,6 @@ import time
from xonsh.lazyasd import LazyObject
from xonsh.history import History
from xonsh.tokenize import SearchPath
from xonsh.inspectors import Inspector
from xonsh.aliases import Aliases, make_default_aliases
from xonsh.environ import Env, default_env, locate_binary
@ -32,9 +30,10 @@ from xonsh.proc import (ProcProxy, SimpleProcProxy, ForegroundProcProxy,
SimpleForegroundProcProxy, TeePTYProc,
CompletedCommand, HiddenCompletedCommand)
from xonsh.tools import (
suggest_commands, expandvars, CommandsCache, globpath, XonshError,
suggest_commands, expandvars, globpath, XonshError,
XonshCalledProcessError, XonshBlockError
)
from xonsh.commands_cache import CommandsCache
ENV = None
@ -455,8 +454,8 @@ def run_subproc(cmds, captured=False):
if (stdin is not None and
ENV.get('XONSH_STORE_STDIN') and
captured == 'object' and
'cat' in __xonsh_commands_cache__ and
'tee' in __xonsh_commands_cache__):
__xonsh_commands_cache__.lazy_locate_binary('cat') and
__xonsh_commands_cache__.lazy_locate_binary('tee')):
_stdin_file = tempfile.NamedTemporaryFile()
cproc = Popen(['cat'],
stdin=stdin,

142
xonsh/commands_cache.py Normal file
View file

@ -0,0 +1,142 @@
# -*- coding: utf-8 -*-
import builtins
import os
import collections.abc as abc
from xonsh.dirstack import _get_cwd
from xonsh.platform import ON_WINDOWS
from xonsh.tools import executables_in
class CommandsCache(abc.Mapping):
"""A lazy cache representing the commands available on the file system.
The keys are the command names and the values a tuple of (loc, has_alias)
where loc is either a str pointing to the executable on the file system or
None (if no executable exists) and has_alias is a boolean flag for whether
the command has an alias.
"""
def __init__(self):
self._cmds_cache = {}
self._path_checksum = None
self._alias_checksum = None
self._path_mtime = -1
def __contains__(self, key):
return key in self.all_commands
def __iter__(self):
return iter(self.all_commands)
def __len__(self):
return len(self.all_commands)
def __getitem__(self, key):
return self.all_commands[key]
def is_empty(self):
"""Returns whether the cache is populated or not."""
return len(self._cmds_cache) == 0
@staticmethod
def get_possible_names(name):
"""Generates the possible `PATHEXT` extension variants of a given executable
name on Windows as a list, conserving the ordering in `PATHEXT`.
Returns a list as `name` being the only item in it on other platforms."""
if ON_WINDOWS:
name = name.upper()
return [
name + ext
for ext in ([''] + builtins.__xonsh_env__['PATHEXT'])
]
else:
return [name]
@property
def all_commands(self):
paths = builtins.__xonsh_env__.get('PATH', [])
pathset = frozenset(x for x in paths if os.path.isdir(x))
# did PATH change?
path_hash = hash(pathset)
cache_valid = path_hash == self._path_checksum
self._path_checksum = path_hash
# did aliases change?
alss = getattr(builtins, 'aliases', set())
al_hash = hash(frozenset(alss))
cache_valid = cache_valid and al_hash == self._alias_checksum
self._alias_checksum = al_hash
# did the contents of any directory in PATH change?
max_mtime = 0
for path in pathset:
mtime = os.stat(path).st_mtime
if mtime > max_mtime:
max_mtime = mtime
cache_valid = cache_valid and (max_mtime <= self._path_mtime)
self._path_mtime = max_mtime
if cache_valid:
return self._cmds_cache
allcmds = {}
for path in reversed(paths):
# iterate backwards so that entries at the front of PATH overwrite
# entries at the back.
for cmd in executables_in(path):
key = cmd.upper() if ON_WINDOWS else cmd
allcmds[key] = (os.path.join(path, cmd), cmd in alss)
only_alias = (None, True)
for cmd in alss:
if cmd not in allcmds:
allcmds[cmd] = only_alias
self._cmds_cache = allcmds
return allcmds
def lazyin(self, key):
"""Checks if the value is in the current cache without the potential to
update the cache. It just says whether the value is known *now*. This
may not reflect precisely what is on the $PATH.
"""
return key in self._cmds_cache
def lazyiter(self):
"""Returns an iterator over the current cache contents without the
potential to update the cache. This may not reflect what is on the
$PATH.
"""
return iter(self._cmds_cache)
def lazylen(self):
"""Returns the length of the current cache contents without the
potential to update the cache. This may not reflect precisely
what is on the $PATH.
"""
return len(self._cmds_cache)
def lazyget(self, key, default=None):
"""A lazy value getter."""
return self._cmds_cache.get(key, default)
def locate_binary(self, name):
"""Locates an executable on the file system using the cache."""
# make sure the cache is up to date by accessing the property
_ = self.all_commands
return self.lazy_locate_binary(name)
def lazy_locate_binary(self, name):
"""Locates an executable in the cache, without checking its validity."""
possibilities = self.get_possible_names(name)
if ON_WINDOWS:
# Windows users expect to be able to execute files in the same
# directory without `./`
cwd = _get_cwd()
local_bin = next((
full_name for full_name in possibilities
if os.path.isfile(full_name)
), None)
if local_bin:
return os.path.abspath(os.path.relpath(local_bin, cwd))
cached = next((cmd for cmd in possibilities if cmd in self._cmds_cache), None)
if cached:
return self._cmds_cache[cached][0]
elif os.path.isfile(name) and name != os.path.basename(name):
return name

58
xonsh/environ.py
View file

@ -7,7 +7,6 @@ import json
import locale
import builtins
from contextlib import contextmanager
from functools import wraps
from itertools import chain
from pprint import pformat
import re
@ -16,30 +15,33 @@ import string
import subprocess
import shutil
from warnings import warn
from collections import (Mapping, MutableMapping, MutableSequence, MutableSet,
namedtuple)
from collections import (
Mapping, MutableMapping, MutableSequence, MutableSet,
namedtuple
)
from xonsh import __version__ as XONSH_VERSION
from xonsh.jobs import get_next_task
from xonsh.codecache import run_script_with_cache
from xonsh.dirstack import _get_cwd
from xonsh.foreign_shells import load_foreign_envs
from xonsh.platform import (BASH_COMPLETIONS_DEFAULT, ON_ANACONDA, ON_LINUX,
ON_WINDOWS, DEFAULT_ENCODING, ON_CYGWIN, PATH_DEFAULT)
from xonsh.platform import (
BASH_COMPLETIONS_DEFAULT, DEFAULT_ENCODING, PATH_DEFAULT,
ON_WINDOWS, ON_ANACONDA, ON_LINUX, ON_CYGWIN,
)
from xonsh.tools import (
is_superuser, always_true, always_false, ensure_string, is_env_path,
str_to_env_path, env_path_to_str, is_bool, to_bool, bool_to_str,
is_history_tuple, to_history_tuple, history_tuple_to_str, is_float,
is_string, is_callable, is_string_or_callable,
is_string, is_string_or_callable,
is_completions_display_value, to_completions_display_value,
is_string_set, csv_to_set, set_to_csv, get_sep, is_int, is_bool_seq,
is_bool_or_int, to_bool_or_int, bool_or_int_to_str,
to_bool_or_int, bool_or_int_to_str,
csv_to_bool_seq, bool_seq_to_csv, DefaultNotGiven, print_exception,
setup_win_unicode_console, intensify_colors_on_win_setter, format_color,
is_dynamic_cwd_width, to_dynamic_cwd_tuple, dynamic_cwd_tuple_to_str,
is_logfile_opt, to_logfile_opt, logfile_opt_to_str, executables_in,
pathsep_to_set, set_to_pathsep, pathsep_to_seq, seq_to_pathsep,
is_string_seq, is_nonstring_seq_of_strings, pathsep_to_upper_seq,
is_nonstring_seq_of_strings, pathsep_to_upper_seq,
seq_to_upper_pathsep,
)
@ -823,31 +825,7 @@ def _yield_executables(directory, name):
def locate_binary(name):
"""Locates an executable on the file system."""
if ON_WINDOWS:
# Windows users expect to be able to execute files in the same
# directory without `./`
cwd = _get_cwd()
if os.path.isfile(name):
return os.path.abspath(os.path.relpath(name, cwd))
exts = builtins.__xonsh_env__['PATHEXT']
for ext in exts:
namext = name + ext
if os.path.isfile(namext):
return os.path.abspath(os.path.relpath(namext, cwd))
elif os.path.isfile(name) and name != os.path.basename(name):
return name
cc = builtins.__xonsh_commands_cache__
if ON_WINDOWS:
upname = name.upper()
if upname in cc:
return cc.lazyget(upname)[0]
for ext in exts:
upnamext = upname + ext
if cc.lazyin(upnamext):
return cc.lazyget(upnamext)[0]
elif name in cc:
# can be lazy here since we know name is already available
return cc.lazyget(name)[0]
return builtins.__xonsh_commands_cache__.locate_binary(name)
def get_git_branch():
@ -957,14 +935,14 @@ def _first_branch_timeout_message():
def current_branch(pad=True):
"""Gets the branch for a current working directory. Returns an empty string
if the cwd is not a repository. This currently only works for git and hg
and should be extended in the future. If a timeout occured, the string
and should be extended in the future. If a timeout occurred, the string
'<branch-timeout>' is returned.
"""
branch = ''
cmds = builtins.__xonsh_commands_cache__
if cmds.lazyin('git') or cmds.lazylen() == 0:
if cmds.lazy_locate_binary('git') or cmds.is_empty():
branch = get_git_branch()
if (cmds.lazyin('hg') or cmds.lazylen() == 0) and not branch:
if (cmds.lazy_locate_binary('hg') or cmds.is_empty()) and not branch:
branch = get_hg_branch()
if isinstance(branch, subprocess.TimeoutExpired):
branch = '<branch-timeout>'
@ -1028,16 +1006,16 @@ def dirty_working_directory(cwd=None):
"""
dwd = None
cmds = builtins.__xonsh_commands_cache__
if cmds.lazyin('git') or cmds.lazylen() == 0:
if cmds.lazy_locate_binary('git') or cmds.is_empty():
dwd = git_dirty_working_directory()
if (cmds.lazyin('hg') or cmds.lazylen() == 0) and (dwd is None):
if (cmds.lazy_locate_binary('hg') or cmds.is_empty()) and (dwd is None):
dwd = hg_dirty_working_directory()
return dwd
def branch_color():
"""Return red if the current branch is dirty, yellow if the dirtiness can
not be determined, and green if it clean. Thes are bold, intesnse colors
not be determined, and green if it clean. These are bold, intense colors
for the foreground.
"""
dwd = dirty_working_directory()

125
xonsh/tools.py
View file

@ -17,30 +17,32 @@ Implementations:
* indent()
"""
import os
import re
import sys
import ast
import glob
import string
import ctypes
import builtins
import pathlib
import warnings
import functools
import threading
import traceback
import subprocess
import collections
import collections.abc as abc
import ctypes
import functools
import glob
import os
import pathlib
import re
import string
import subprocess
import sys
import threading
import traceback
import warnings
from contextlib import contextmanager
from subprocess import CalledProcessError
# adding further imports from xonsh modules is discouraged to avoid cirular
# adding further imports from xonsh modules is discouraged to avoid circular
# dependencies
from xonsh.lazyasd import LazyObject, LazyDict
from xonsh.platform import (has_prompt_toolkit, scandir,
DEFAULT_ENCODING, ON_LINUX, ON_WINDOWS, PYTHON_VERSION_INFO)
from xonsh.platform import (
has_prompt_toolkit, scandir,
DEFAULT_ENCODING, ON_LINUX, ON_WINDOWS, PYTHON_VERSION_INFO,
)
@functools.lru_cache(1)
def is_superuser():
@ -1550,101 +1552,12 @@ def expanduser_abs_path(inp):
return os.path.abspath(os.path.expanduser(inp))
class CommandsCache(abc.Mapping):
"""A lazy cache representing the commands available on the file system.
The keys are the command names and the values a tuple of (loc, has_alias)
where loc is either a str pointing to the executable on the file system or
None (if no executable exists) and has_alias is a boolean flag for whether
the command has an alias.
"""
def __init__(self):
self._cmds_cache = {}
self._path_checksum = None
self._alias_checksum = None
self._path_mtime = -1
def __contains__(self, key):
return key in self.all_commands
def __iter__(self):
return iter(self.all_commands)
def __len__(self):
return len(self.all_commands)
def __getitem__(self, key):
return self.all_commands[key]
@property
def all_commands(self):
paths = builtins.__xonsh_env__.get('PATH', [])
pathset = frozenset(x for x in paths if os.path.isdir(x))
# did PATH change?
path_hash = hash(pathset)
cache_valid = path_hash == self._path_checksum
self._path_checksum = path_hash
# did aliases change?
alss = getattr(builtins, 'aliases', set())
al_hash = hash(frozenset(alss))
cache_valid = cache_valid and al_hash == self._alias_checksum
self._alias_checksum = al_hash
# did the contents of any directory in PATH change?
max_mtime = 0
for path in pathset:
mtime = os.stat(path).st_mtime
if mtime > max_mtime:
max_mtime = mtime
cache_valid = cache_valid and (max_mtime <= self._path_mtime)
self._path_mtime = max_mtime
if cache_valid:
return self._cmds_cache
allcmds = {}
for path in reversed(paths):
# iterate backwards so that entries at the front of PATH overwrite
# entries at the back.
for cmd in executables_in(path):
key = cmd.upper() if ON_WINDOWS else cmd
allcmds[key] = (os.path.join(path, cmd), cmd in alss)
only_alias = (None, True)
for cmd in alss:
if cmd not in allcmds:
allcmds[cmd] = only_alias
self._cmds_cache = allcmds
return allcmds
def lazyin(self, value):
"""Checks if the value is in the current cache without the potential to
update the cache. It just says whether the value is known *now*. This
may not reflect precisely what is on the $PATH.
"""
return value in self._cmds_cache
def lazyiter(self):
"""Returns an iterator over the current cache contents without the
potential to update the cache. This may not reflect what is on the
$PATH.
"""
return iter(self._cmds_cache)
def lazylen(self):
"""Returns the length of the current cache contents without the
potential to update the cache. This may not reflect precicesly
what is on the $PATH.
"""
return len(self._cmds_cache)
def lazyget(self, key, default=None):
"""A lazy value getter."""
return self._cmds_cache.get(key, default)
WINDOWS_DRIVE_MATCHER = LazyObject(lambda: re.compile(r'^\w:'),
globals(), 'WINDOWS_DRIVE_MATCHER')
def expand_case_matching(s):
"""Expands a string to a case insenstive globable string."""
"""Expands a string to a case insensitive globable string."""
t = []
openers = {'[', '{'}
closers = {']', '}'}