Merge remote-tracking branch 'origin/main' into fix_callias_capturing

docs/api/index.rst

@@ -58,7 +58,6 @@ For those of you who want the gritty details.
     xonsh.lib
     xonsh.tools
     xonsh.platform
-    xonsh.jsonutils
     xonsh.lazyjson
     xonsh.lazyasd
     xonsh.openpy

news/commands_cache_fix_update.rst

@@ -0,0 +1,23 @@
+**Added:**
+
+* <news item>
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* Commands Cache: Fixed cache update logic that lead to lagging during typing.
+
+**Security:**
+
+* <news item>

news/prompt_switch_to_ptk.rst

@@ -0,0 +1,23 @@
+**Added:**
+
+* <news item>
+
+**Changed:**
+
+* prompt: Switching to prompt_toolkit in edge case of sending stdin to interactive mode (#5462 #5517).
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

news/sqlite-history-gc.rst

@@ -0,0 +1,23 @@
+**Added:**
+
+* <news item>
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* Fixed ``history gc`` invocation failing when ``sqlite`` history backend is used.
+
+**Security:**
+
+* <news item>

pyproject.toml

@@ -29,6 +29,7 @@ packages = [
     "xonsh.ply.ply",
     "xonsh.ptk_shell",
     "xonsh.procs",
+    "xonsh.platform",
     "xonsh.parsers",
     "xonsh.xoreutils",
     "xontrib",
@@ -223,7 +224,7 @@ convention = "numpy"
 "xonsh/history.py" = ["F821"]
 "xonsh/lexer.py" = ["E741"]
 "xonsh/parsers/completion_context.py" = ["B018"]
-"xonsh/tokenize.py" = [
+"xonsh/lib/tokenize.py" = [
     "F821",
     "F841",
     "B904" # Within an `except` clause, raise exceptions with `raise ... from err`
@@ -252,7 +253,7 @@ convention = "numpy"
 "xonsh/xonfig.py" = ["E731"]
 "xontrib/vox.py" = ["F821"]
 "xonsh/inspectors.py" = ["E722"]
-"xonsh/platform.py" = ["F401"]
+"xonsh/platform/__init__.py" = ["F401"]
 "xonsh/parsers/*.py" = [
     "E741", #  E741 Ambiguous variable name
 ]

tests/completers/test_command_completers.py

@@ -56,3 +56,27 @@ def test_skipper_arg(completion_context_parse, xession, monkeypatch):
     assert context.command == CommandContext(
         args=(CommandArg("grep"),), arg_index=1, prefix="--coun"
     )
+
+
+def test_argparse_completer(check_completer, monkeypatch):
+    assert check_completer("xonsh", prefix="-").issuperset(
+        {
+            "--cache-everything",
+            "--help",
+            "--interactive",
+            "--login",
+            "--no-env",
+            "--no-rc",
+            "--no-script-cache",
+            "--rc",
+            "--shell-type",
+            "--timings",
+            "--version",
+        }
+    )
+
+
+def test_argparse_completer_after_option(check_completer, tmp_path):
+    prefix = str(tmp_path)[:-1]
+    # has one or more completions including the above tmp_path
+    assert check_completer("xonsh --no-rc", prefix)

tests/lib/test_jsonutils.py

@@ -4,7 +4,7 @@ import json
 
 import pytest
 
-from xonsh.jsonutils import serialize_xonsh_json
+from xonsh.lib.jsonutils import serialize_xonsh_json
 from xonsh.tools import EnvPath
 
 

tests/lib/test_pretty.py

@@ -2,7 +2,7 @@ import re
 
 import pytest
 
-from xonsh import pretty
+from xonsh.lib import pretty
 
 long_list = ["str"] * 30
 long_list_exp = "[" + (",\n ".join(["'str'"] * 30) + "]")

tests/test_commands_cache.py

@@ -187,9 +187,16 @@ def test_update_cache(xession, tmp_path):
     file1.touch()
     file1.chmod(0o755)
 
-    cache = CommandsCache({"PATH": [subdir2, subdir1]})
+    paths = [subdir2, subdir1]
+    cache = CommandsCache({"PATH": paths})
     cached = cache.update_cache()
 
+    # Check there are no changes after update cache.
+    c1 = cache._update_and_check_changes(paths)
+    c2 = cache._update_and_check_changes(paths)
+    c3 = cache._update_and_check_changes(paths)
+    assert [c1, c2, c3] == [True, False, False]
+
     assert file1.samefile(cached[basename][0])
 
     # give the os enough time to update the mtime field of the parent directory

xonsh/aliases.py

@@ -980,7 +980,7 @@ def make_default_aliases():
             default_aliases["deactivate"] = ["source-cmd", "deactivate.bat"]
         if shutil.which("sudo", path=XSH.env.get_detyped("PATH")):
             # XSH.commands_cache is not available during setup
-            import xonsh.winutils as winutils
+            import xonsh.platform.winutils as winutils
 
             def sudo(args):
                 if len(args) < 1:

xonsh/built_ins.py

@@ -19,8 +19,8 @@ import types
 import warnings
 from ast import AST
 
-from xonsh.inspectors import Inspector
 from xonsh.lazyasd import lazyobject
+from xonsh.lib.inspectors import Inspector
 from xonsh.platform import ON_POSIX
 from xonsh.tools import (
     XonshCalledProcessError,

xonsh/cli_utils.py

@@ -545,7 +545,7 @@ class ArgparseCompleter:
                 break
             # it is a valid option and advance
             self.remaining_args = self.remaining_args[1:]
-            act, _, value = act_res
+            act, *_, value = act_res
 
             # remove the found option
             # todo: not remove if append/extend

xonsh/commands_cache.py

@@ -119,14 +119,21 @@ class CommandsCache(cabc.Mapping):
                 if os.path.isdir(p):
                     yield p
 
-    def _check_changes(self, paths: tuple[str, ...]):
+    def _update_aliases_cache(self):
-        # did PATH change?
+        """Update aliases checksum and return result: updated or not."""
-        yield self._update_paths_cache(paths)
+        prev_hash = self._alias_checksum
+        self._alias_checksum = hash(frozenset(self.aliases))
+        return prev_hash != self._alias_checksum
 
-        # did aliases change?
+    def _update_and_check_changes(self, paths: tuple[str, ...]):
-        al_hash = hash(frozenset(self.aliases))
+        """Update cache and return the result: updated or still the same.
-        yield al_hash != self._alias_checksum
+
-        self._alias_checksum = al_hash
+        Be careful in this place. Both `_update_*` functions must be called
+        because they are changing state after update.
+        """
+        is_aliases_change = self._update_aliases_cache()
+        is_paths_change = self._update_paths_cache(paths)
+        return is_aliases_change or is_paths_change
 
     @property
     def all_commands(self):
@@ -159,7 +166,7 @@ class CommandsCache(cabc.Mapping):
         # iterate backwards so that entries at the front of PATH overwrite
         # entries at the back.
         paths = tuple(reversed(tuple(self.remove_dups(env.get("PATH") or []))))
-        if any(self._check_changes(paths)):
+        if self._update_and_check_changes(paths):
             all_cmds = CacheDict()
             for cmd, path in self._iter_binaries(paths):
                 # None     -> not in aliases

xonsh/environ.py

@@ -859,28 +859,6 @@ class Xettings:
 class GeneralSetting(Xettings):
     """General"""
 
-    AUTO_CONTINUE = Var.with_default(
-        False,
-        "If ``True``, automatically resume stopped jobs when they are disowned. "
-        "When stopped jobs are disowned and this option is ``False``, a warning "
-        "will print information about how to continue the stopped process.",
-    )
-
-    COMMANDS_CACHE_SAVE_INTERMEDIATE = Var.with_default(
-        False,
-        "If enabled, the CommandsCache is saved between runs and can reduce the startup time.",
-    )
-
-    ENABLE_COMMANDS_CACHE = Var(
-        default=True,
-        doc="command names in a directory are cached when enabled. "
-        "On some platforms it may not be accurate enough"
-        "(e.g. Windows, Linux save mtime in seconds). "
-        "Setting it to False would disable the caching mechanism "
-        "and may slow down the shell",
-        doc_default="True",
-    )
-
     HOSTNAME = Var.with_default(
         default=default_value(lambda env: platform.node()),
         doc="Automatically set to the name of the current host.",
@@ -891,18 +869,6 @@ class GeneralSetting(Xettings):
         doc="Automatically set to a string that fully describes the system type on which xonsh is executing.",
         type_str="str",
     )
-    LANG = Var.with_default(
-        default="C.UTF-8",
-        doc="Fallback locale setting for systems where it matters",
-        type_str="str",
-    )
-    LC_COLLATE = Var.for_locale("LC_COLLATE")
-    LC_CTYPE = Var.for_locale("LC_CTYPE")
-    LC_MONETARY = Var.for_locale("LC_MONETARY")
-    LC_NUMERIC = Var.for_locale("LC_NUMERIC")
-    LC_TIME = Var.for_locale("LC_TIME")
-    if hasattr(locale, "LC_MESSAGES"):
-        LC_MESSAGES = Var.for_locale("LC_MESSAGES")
 
     PWD = Var.with_default(
         _get_cwd() or ".",
@@ -939,18 +905,6 @@ class GeneralSetting(Xettings):
         "filtering valid executables by. Each element must be "
         "uppercase.",
     )
-    RAISE_SUBPROC_ERROR = Var.with_default(
-        False,
-        "Whether or not to raise an error if a subprocess (captured or "
-        "uncaptured) returns a non-zero exit status, which indicates failure. "
-        "This is most useful in xonsh scripts or modules where failures "
-        "should cause an end to execution. This is less useful at a terminal. "
-        "The error that is raised is a ``subprocess.CalledProcessError``.",
-    )
-    LAST_RETURN_CODE = Var.with_default(
-        0,
-        "Integer return code of the last command. Only updated during interactive use, i.e. not during execution of scripts.",
-    )
 
     SHLVL = Var(
         is_valid_shlvl,
@@ -960,10 +914,7 @@ class GeneralSetting(Xettings):
         "Shell nesting level typed as integer, mirrors bash's $SHLVL.",
         is_configurable=False,
     )
-    XONSH_SUBPROC_CAPTURED_PRINT_STDERR = Var.with_default(
+
-        False,
-        "If ``True`` the stderr from captured subproc will be printed automatically.",
-    )
     TERM = Var.no_default(
         "str",
         "TERM is sometimes set by the terminal emulator. This is used (when "
@@ -982,47 +933,7 @@ class GeneralSetting(Xettings):
         "not always happen.",
         is_configurable=False,
     )
-    XONSH_SUBPROC_OUTPUT_FORMAT = Var.with_default(
+
-        "stream_lines",
-        "Set output format for subprocess e.g. ``du $(ls)``. "
-        "By default (``stream_lines``) subprocess operator returns text output. "
-        "Set ``list_lines`` to have list of lines.",
-    )
-    XONSH_CAPTURE_ALWAYS = Var.with_default(
-        False,
-        "Try to capture output of commands run without explicit capturing.\n"
-        "If True, xonsh will capture the output of commands run directly or in ``![]``"
-        "to the session history.\n"
-        "Setting to True has the following disadvantages:\n"
-        "* Some interactive commands won't work properly (like when ``git`` invokes an interactive editor).\n"
-        "  For more information see discussion at https://github.com/xonsh/xonsh/issues/3672.\n"
-        "* Stopping these commands with ^Z (i.e. ``SIGTSTP``)\n"
-        "  is disabled as it causes deadlocked terminals.\n"
-        "  ``SIGTSTP`` may still be issued and only the physical pressing\n"
-        "  of ``Ctrl+Z`` is ignored.\n\n"
-        "Regardless of this value, commands run in ``$()``, ``!()`` or with an IO redirection (``>`` or ``|``) "
-        "will always be captured.\n"
-        "Setting this to True depends on ``$THREAD_SUBPROCS`` being True.",
-    )
-    THREAD_SUBPROCS = Var(
-        is_bool_or_none,
-        to_bool_or_none,
-        bool_or_none_to_str,
-        not ON_CYGWIN,
-        "Note: The ``$XONSH_CAPTURE_ALWAYS`` variable introduces finer control "
-        "and you should probably use that instead.\n\n"
-        "Whether or not to try to run subrocess mode in a Python thread, "
-        "when trying to capture its output. There are various trade-offs.\n\n"
-        "If True, xonsh is able capture & store the stdin, stdout, and stderr"
-        " of threadable subprocesses.\n"
-        "The disadvantages are listed in ``$XONSH_CAPTURE_ALWAYS``.\n"
-        "The desired effect is often up to the command, user, or use case.\n\n"
-        "None values are for internal use only and are used to turn off "
-        "threading when loading xonshrc files. This is done because Bash "
-        "was automatically placing new xonsh instances in the background "
-        "at startup when threadable subprocs were used. Please see "
-        "https://github.com/xonsh/xonsh/pull/3705 for more information.\n",
-    )
     UPDATE_OS_ENVIRON = Var.with_default(
         False,
         "If True ``os_environ`` will always be updated "
@@ -1070,33 +981,7 @@ class GeneralSetting(Xettings):
         "are loaded after any files in XONSHRC.",
         type_str="env_path",
     )
-    XONSH_COMPLETER_DIRS = Var.with_default(
+
-        default_completer_dirs,
-        """\
-A list of paths where Xonsh searches for command completions.
-Any completions defined are lazy loaded when needed.
-The name of the completer file should match that of the completing command.
-The file should contain a function with the signature
-``xonsh_complete(ctx: CommandContext) -> Iterator[RichCompletion|str]``.
-""",
-        type_str="env_path",
-    )
-    XONSH_APPEND_NEWLINE = Var.with_default(
-        xonsh_append_newline,
-        "Append new line when a partial line is preserved in output.",
-        doc_default="``$XONSH_INTERACTIVE``",
-        type_str="bool",
-    )
-    XONSH_CACHE_SCRIPTS = Var.with_default(
-        True,
-        "Controls whether the code for scripts run from xonsh will be cached"
-        " (``True``) or re-compiled each time (``False``).",
-    )
-    XONSH_CACHE_EVERYTHING = Var.with_default(
-        False,
-        "Controls whether all code (including code entered at the interactive"
-        " prompt) will be cached.",
-    )
     XONSH_CONFIG_DIR = Var.with_default(
         xonsh_config_dir,
         "This is the location where xonsh user-level configuration information is stored.",
@@ -1115,12 +1000,7 @@ The file should contain a function with the signature
         "a color map. Run ``xonfig styles`` to see the available styles.",
         type_str="str",
     )
-    XONSH_DATETIME_FORMAT = Var.with_default(
+
-        "%Y-%m-%d %H:%M",
-        "The format that is used for ``datetime.strptime()`` in various places, "
-        "i.e the history timestamp option.",
-        type_str="str",
-    )
     XONSH_DEBUG = Var(
         always_false,
         to_debug,
@@ -1138,12 +1018,7 @@ The file should contain a function with the signature
         doc_default="``$XDG_DATA_HOME/xonsh``",
         type_str="str",
     )
-    XONSH_CACHE_DIR = Var.with_default(
+
-        xonsh_cache_dir,
-        "This is the location where cache files used by xonsh are stored, such as commands-cache...",
-        doc_default="``$XDG_CACHE_HOME/xonsh``",
-        type_str="str",
-    )
     XONSH_ENCODING = Var.with_default(
         DEFAULT_ENCODING,
         "This is the encoding that xonsh should use for subprocess operations.",
@@ -1172,19 +1047,7 @@ The file should contain a function with the signature
         "``True`` if xonsh is running as a login shell, and ``False`` otherwise.",
         is_configurable=False,
     )
-    XONSH_PROC_FREQUENCY = Var.with_default(
+
-        1e-4,
-        "The process frequency is the time that "
-        "xonsh process threads sleep for while running command pipelines. "
-        "The value has units of seconds [s].",
-    )
-    XONSH_SHOW_TRACEBACK = Var.with_default(
-        False,
-        "Controls if a traceback is shown if exceptions occur in the shell. "
-        "Set to ``True`` to always show traceback or ``False`` to always hide. "
-        "If undefined then the traceback is hidden but a notice is shown on how "
-        "to enable the full traceback.",
-    )
     XONSH_SOURCE = Var.with_default(
         "",
         "When running a xonsh script, this variable contains the absolute path "
@@ -1208,6 +1071,84 @@ The file should contain a function with the signature
         "    - ptk style name (string) - ``$XONSH_STYLE_OVERRIDES['pygments.keyword'] = '#ff0000'``\n\n"
         "(The rules above are all have the same effect.)",
     )
+
+    STAR_PATH = Var.no_default("env_path", pattern=re.compile(r"\w*PATH$"))
+    STAR_DIRS = Var.no_default("env_path", pattern=re.compile(r"\w*DIRS$"))
+
+
+class SubprocessSetting(Xettings):
+    """Subprocess Settings"""
+
+    RAISE_SUBPROC_ERROR = Var.with_default(
+        False,
+        "Whether or not to raise an error if a subprocess (captured or "
+        "uncaptured) returns a non-zero exit status, which indicates failure. "
+        "This is most useful in xonsh scripts or modules where failures "
+        "should cause an end to execution. This is less useful at a terminal. "
+        "The error that is raised is a ``subprocess.CalledProcessError``.",
+    )
+    LAST_RETURN_CODE = Var.with_default(
+        0,
+        "Integer return code of the last command. Only updated during interactive use, i.e. not during execution of scripts.",
+    )
+    XONSH_SUBPROC_CAPTURED_PRINT_STDERR = Var.with_default(
+        False,
+        "If ``True`` the stderr from captured subproc will be printed automatically.",
+    )
+    XONSH_SUBPROC_OUTPUT_FORMAT = Var.with_default(
+        "stream_lines",
+        "Set output format for subprocess e.g. ``du $(ls)``. "
+        "By default (``stream_lines``) subprocess operator returns text output. "
+        "Set ``list_lines`` to have list of lines.",
+    )
+
+    XONSH_CAPTURE_ALWAYS = Var.with_default(
+        False,
+        "Try to capture output of commands run without explicit capturing.\n"
+        "If True, xonsh will capture the output of commands run directly or in ``![]``"
+        "to the session history.\n"
+        "Setting to True has the following disadvantages:\n"
+        "* Some interactive commands won't work properly (like when ``git`` invokes an interactive editor).\n"
+        "  For more information see discussion at https://github.com/xonsh/xonsh/issues/3672.\n"
+        "* Stopping these commands with ^Z (i.e. ``SIGTSTP``)\n"
+        "  is disabled as it causes deadlocked terminals.\n"
+        "  ``SIGTSTP`` may still be issued and only the physical pressing\n"
+        "  of ``Ctrl+Z`` is ignored.\n\n"
+        "Regardless of this value, commands run in ``$()``, ``!()`` or with an IO redirection (``>`` or ``|``) "
+        "will always be captured.\n"
+        "Setting this to True depends on ``$THREAD_SUBPROCS`` being True.",
+    )
+    THREAD_SUBPROCS = Var(
+        is_bool_or_none,
+        to_bool_or_none,
+        bool_or_none_to_str,
+        not ON_CYGWIN,
+        "Note: The ``$XONSH_CAPTURE_ALWAYS`` variable introduces finer control "
+        "and you should probably use that instead.\n\n"
+        "Whether or not to try to run subrocess mode in a Python thread, "
+        "when trying to capture its output. There are various trade-offs.\n\n"
+        "If True, xonsh is able capture & store the stdin, stdout, and stderr"
+        " of threadable subprocesses.\n"
+        "The disadvantages are listed in ``$XONSH_CAPTURE_ALWAYS``.\n"
+        "The desired effect is often up to the command, user, or use case.\n\n"
+        "None values are for internal use only and are used to turn off "
+        "threading when loading xonshrc files. This is done because Bash "
+        "was automatically placing new xonsh instances in the background "
+        "at startup when threadable subprocs were used. Please see "
+        "https://github.com/xonsh/xonsh/pull/3705 for more information.\n",
+    )
+    XONSH_APPEND_NEWLINE = Var.with_default(
+        xonsh_append_newline,
+        "Append new line when a partial line is preserved in output.",
+        doc_default="``$XONSH_INTERACTIVE``",
+        type_str="bool",
+    )
+    XONSH_PROC_FREQUENCY = Var.with_default(
+        1e-4,
+        "The process frequency is the time that "
+        "xonsh process threads sleep for while running command pipelines. "
+        "The value has units of seconds [s].",
+    )
     XONSH_TRACE_SUBPROC = Var(
         default=False,
         validate=is_bool_or_int,
@@ -1215,23 +1156,31 @@ The file should contain a function with the signature
         doc="Set to ``True`` or ``1`` to show arguments list of every executed subprocess command. "
         "Use ``2`` to have a specification. Use ``3`` to have full specification.",
     )
-    XONSH_TRACE_COMPLETIONS = Var.with_default(
-        False,
-        "Set to ``True`` to show completers invoked and their return values.",
-    )
     XONSH_TRACE_SUBPROC_FUNC = Var.with_default(
         None,
         doc=(
             "A callback function used to format the trace output shown when $XONSH_TRACE_SUBPROC=True."
         ),
         doc_default="""\
-By default it just prints ``cmds`` like below.
+    By default it just prints ``cmds`` like below.
 
-.. code-block:: python
+    .. code-block:: python
 
-    def tracer(cmds: list, captured: Union[bool, str]):
+        def tracer(cmds: list, captured: Union[bool, str]):
-        print(f"TRACE SUBPROC: {cmds}, captured={captured}", file=sys.stderr)
+            print(f"TRACE SUBPROC: {cmds}, captured={captured}", file=sys.stderr)
-""",
+    """,
+    )
+
+
+class ErrorHandlingSetting(Xettings):
+    """Error Handling Settings"""
+
+    XONSH_SHOW_TRACEBACK = Var.with_default(
+        False,
+        "Controls if a traceback is shown if exceptions occur in the shell. "
+        "Set to ``True`` to always show traceback or ``False`` to always hide. "
+        "If undefined then the traceback is hidden but a notice is shown on how "
+        "to enable the full traceback.",
     )
     XONSH_TRACEBACK_LOGFILE = Var(
         is_logfile_opt,
@@ -1243,19 +1192,79 @@ By default it just prints ``cmds`` like below.
         "or None / the empty string if traceback logging is not desired. "
         "Logging to a file is not enabled by default.",
     )
-    XONTRIBS_AUTOLOAD_DISABLED = Var(
+
-        default=False,
+
-        doc="""\
+class JobsSetting(Xettings):
-Controls auto-loading behaviour of xontrib packages at the startup.
+    """Jobs Settings"""
-* Set this to ``True`` to disable autoloading completely.
+
-* Setting this to a list of xontrib names will block loading those specifically.
+    AUTO_CONTINUE = Var.with_default(
-""",
+        False,
-        doc_default="""\
+        "If ``True``, automatically resume stopped jobs when they are disowned. "
-Xontribs with ``xonsh.xontrib`` entrypoint will be loaded automatically by default.
+        "When stopped jobs are disowned and this option is ``False``, a warning "
-""",
+        "will print information about how to continue the stopped process.",
+    )
+
+
+class LangSetting(Xettings):
+    """Language and locale settings."""
+
+    LANG = Var.with_default(
+        default="C.UTF-8",
+        doc="Fallback locale setting for systems where it matters",
+        type_str="str",
+    )
+    LC_COLLATE = Var.for_locale("LC_COLLATE")
+    LC_CTYPE = Var.for_locale("LC_CTYPE")
+    LC_MONETARY = Var.for_locale("LC_MONETARY")
+    LC_NUMERIC = Var.for_locale("LC_NUMERIC")
+    LC_TIME = Var.for_locale("LC_TIME")
+    if hasattr(locale, "LC_MESSAGES"):
+        LC_MESSAGES = Var.for_locale("LC_MESSAGES")
+
+    XONSH_DATETIME_FORMAT = Var.with_default(
+        "%Y-%m-%d %H:%M",
+        "The format that is used for ``datetime.strptime()`` in various places, "
+        "i.e the history timestamp option.",
+        type_str="str",
+    )
+
+
+class CacheSetting(Xettings):
+    """Cache Settings"""
+
+    XONSH_CACHE_SCRIPTS = Var.with_default(
+        True,
+        "Controls whether the code for scripts run from xonsh will be cached"
+        " (``True``) or re-compiled each time (``False``).",
+    )
+
+    XONSH_CACHE_EVERYTHING = Var.with_default(
+        False,
+        "Controls whether all code (including code entered at the interactive"
+        " prompt) will be cached.",
+    )
+
+    XONSH_CACHE_DIR = Var.with_default(
+        xonsh_cache_dir,
+        "This is the location where cache files used by xonsh are stored, such as commands-cache...",
+        doc_default="``$XDG_CACHE_HOME/xonsh``",
+        type_str="str",
+    )
+
+    ENABLE_COMMANDS_CACHE = Var(
+        default=True,
+        doc="command names in a directory are cached when enabled. "
+        "On some platforms it may not be accurate enough"
+        "(e.g. Windows, Linux save mtime in seconds). "
+        "Setting it to False would disable the caching mechanism "
+        "and may slow down the shell",
+        doc_default="True",
+    )
+
+    COMMANDS_CACHE_SAVE_INTERMEDIATE = Var.with_default(
+        False,
+        "If enabled, the CommandsCache is saved between runs and can reduce the startup time.",
     )
-    STAR_PATH = Var.no_default("env_path", pattern=re.compile(r"\w*PATH$"))
-    STAR_DIRS = Var.no_default("env_path", pattern=re.compile(r"\w*DIRS$"))
 
 
 class ChangeDirSetting(Xettings):
@@ -1336,6 +1345,22 @@ class InterpreterSetting(Xettings):
     )
 
 
+class XontribSetting(Xettings):
+    """Xontrib Settings"""
+
+    XONTRIBS_AUTOLOAD_DISABLED = Var(
+        default=False,
+        doc="""\
+    Controls auto-loading behaviour of xontrib packages at the startup.
+    * Set this to ``True`` to disable autoloading completely.
+    * Setting this to a list of xontrib names will block loading those specifically.
+    """,
+        doc_default="""\
+    Xontribs with ``xonsh.xontrib`` entrypoint will be loaded automatically by default.
+    """,
+    )
+
+
 class PromptSetting(Xettings):
     """Interactive Prompt"""
 
@@ -1851,6 +1876,21 @@ This is to reduce the noise in generated completions.""",
         "``xonsh.platform.bash_command``.",
         doc_default="None",
     )
+    XONSH_COMPLETER_DIRS = Var.with_default(
+        default_completer_dirs,
+        """\
+A list of paths where Xonsh searches for command completions.
+Any completions defined are lazy loaded when needed.
+The name of the completer file should match that of the completing command.
+The file should contain a function with the signature
+``xonsh_complete(ctx: CommandContext) -> Iterator[RichCompletion|str]``.
+""",
+        type_str="env_path",
+    )
+    XONSH_TRACE_COMPLETIONS = Var.with_default(
+        False,
+        "Set to ``True`` to show completers invoked and their return values.",
+    )
 
 
 class PTKCompletionSetting(AutoCompletionSetting):

xonsh/history/base.py

@@ -156,7 +156,7 @@ class History:
         """
         raise NotImplementedError
 
-    def run_gc(self, size=None, blocking=True):
+    def run_gc(self, size=None, blocking=True, **_):
         """Run the garbage collector.
 
         Parameters

xonsh/history/json.py

@@ -585,7 +585,7 @@ class JsonHistory(History):
         data["gc_last_size"] = f"{(self.hist_size, self.hist_units)}"
         return data
 
-    def run_gc(self, size=None, blocking=True, force=False):
+    def run_gc(self, size=None, blocking=True, force=False, **_):
         self.gc = JsonHistoryGC(wait_for_shell=False, size=size, force=force)
         if blocking:
             while self.gc.is_alive():  # while waiting for gc.

xonsh/history/main.py

@@ -9,7 +9,7 @@ import threading
 import typing as tp
 
 import xonsh.cli_utils as xcli
-import xonsh.diff_history as xdh
+import xonsh.history.diff_history as xdh
 import xonsh.tools as xt
 from xonsh.built_ins import XSH
 from xonsh.history.base import History

xonsh/history/sqlite.py

@@ -382,7 +382,7 @@ class SqliteHistory(History):
         self.last_pull_time = time.time()
         return cnt
 
-    def run_gc(self, size=None, blocking=True):
+    def run_gc(self, size=None, blocking=True, **_):
         self.gc = SqliteHistoryGC(wait_for_shell=False, size=size)
         if blocking:
             while self.gc.is_alive():

xonsh/lazyimps.py

@@ -67,7 +67,7 @@ def msvcrt():
 @lazyobject
 def winutils():
     if ON_WINDOWS:
-        import xonsh.winutils as m
+        import xonsh.platform.winutils as m
     else:
         m = None
     return m
@@ -76,7 +76,7 @@ def winutils():
 @lazyobject
 def macutils():
     if ON_DARWIN:
-        import xonsh.macutils as m
+        import xonsh.platform.macutils as m
     else:
         m = None
     return m

xonsh/lexer.py

@@ -11,9 +11,7 @@ import re
 import typing as tp
 
 from xonsh.lazyasd import lazyobject
-from xonsh.platform import PYTHON_VERSION_INFO
+from xonsh.lib.tokenize import (
-from xonsh.ply.ply.lex import LexToken
-from xonsh.tokenize import (
     CASE,
     COMMENT,
     DEDENT,
@@ -38,6 +36,8 @@ from xonsh.tokenize import (
     TokenError,
     tokenize,
 )
+from xonsh.platform import PYTHON_VERSION_INFO
+from xonsh.ply.ply.lex import LexToken
 
 
 @lazyobject

xonsh/lib/inspectors.py

@@ -17,10 +17,10 @@ import types
 
 from xonsh.lazyasd import LazyObject
 from xonsh.lazyimps import pyghooks, pygments
-from xonsh.openpy import read_py_file
+from xonsh.lib.openpy import read_py_file
+from xonsh.lib.tokenize import detect_encoding
 from xonsh.platform import HAS_PYGMENTS
 from xonsh.style_tools import partial_color_tokenize
-from xonsh.tokenize import detect_encoding
 from xonsh.tools import cast_unicode, format_color, indent, print_color, safe_hasattr
 
 # builtin docstrings to ignore

xonsh/lib/openpy.py

@@ -16,7 +16,7 @@ import io
 import re
 
 from xonsh.lazyasd import LazyObject
-from xonsh.tokenize import detect_encoding, tokopen
+from xonsh.lib.tokenize import detect_encoding, tokopen
 
 cookie_comment_re = LazyObject(
     lambda: re.compile(r"^\s*#.*coding[:=]\s*([-\w.]+)", re.UNICODE),

xonsh/main.py

@@ -20,8 +20,8 @@ from xonsh.imphooks import install_import_hooks
 from xonsh.jobs import ignore_sigtstp
 from xonsh.lazyasd import lazyobject
 from xonsh.lazyimps import pyghooks, pygments
+from xonsh.lib.pretty import pretty
 from xonsh.platform import HAS_PYGMENTS, ON_WINDOWS
-from xonsh.pretty import pretty
 from xonsh.shell import Shell
 from xonsh.timings import setup_timings
 from xonsh.tools import (
@@ -488,7 +488,8 @@ def _failback_to_other_shells(args, err):
 
     if foreign_shell:
         traceback.print_exc()
-        print("Xonsh encountered an issue during launch", file=sys.stderr)
+        print("Xonsh encountered an issue during launch.", file=sys.stderr)
+        print("Please report to https://github.com/xonsh/xonsh/issues", file=sys.stderr)
         print(f"Failback to {foreign_shell}", file=sys.stderr)
         os.execlp(foreign_shell, foreign_shell)
     else:

xonsh/parsers/base.py

@@ -14,11 +14,11 @@ from xonsh import ast
 from xonsh.ast import has_elts, load_attribute_chain, xonsh_call
 from xonsh.lazyasd import LazyObject
 from xonsh.lexer import Lexer, LexToken
+from xonsh.lib.tokenize import SearchPath, StringPrefix
 from xonsh.parsers.context_check import check_contexts
 from xonsh.parsers.fstring_adaptor import FStringAdaptor
 from xonsh.platform import PYTHON_VERSION_INFO
 from xonsh.ply.ply import yacc
-from xonsh.tokenize import SearchPath, StringPrefix
 
 RE_SEARCHPATH = LazyObject(lambda: re.compile(SearchPath), globals(), "RE_SEARCHPATH")
 RE_STRINGPREFIX = LazyObject(

xonsh/shell.py

@@ -202,9 +202,19 @@ class Shell:
         if is_class(backend):
             cls = backend
         else:
+            """
+            There is an edge case that we're using mostly in integration tests:
+            `echo 'echo 1' | xonsh -i` and it's not working with `TERM=dumb` (#5462 #5517)
+            because `dumb` is readline where stdin is not supported yet. PR is very welcome!
+            So in this case we need to force using prompt_toolkit.
+            """
+            is_stdin_to_interactive = (
+                XSH.env.get("XONSH_INTERACTIVE", False) and not sys.stdin.isatty()
+            )
+
             if backend == "none":
                 from xonsh.base_shell import BaseShell as cls
-            elif backend == "prompt_toolkit":
+            elif backend == "prompt_toolkit" or is_stdin_to_interactive:
                 from xonsh.ptk_shell.shell import PromptToolkitShell as cls
             elif backend == "readline":
                 from xonsh.readline_shell import ReadlineShell as cls

xonsh/tracer.py

@@ -11,9 +11,9 @@ import typing as tp
 import xonsh.procs.pipelines as xpp
 import xonsh.prompt.cwd as prompt
 from xonsh.cli_utils import Annotated, Arg, ArgParserAlias
-from xonsh.inspectors import find_file
 from xonsh.lazyasd import LazyObject
 from xonsh.lazyimps import pyghooks, pygments
+from xonsh.lib.inspectors import find_file
 from xonsh.platform import HAS_PYGMENTS
 from xonsh.tools import DefaultNotGiven, normabspath, print_color, to_bool
 

xonsh/wizard.py

@@ -11,7 +11,7 @@ import textwrap
 import typing as tp
 
 from xonsh.built_ins import XSH
-from xonsh.jsonutils import serialize_xonsh_json
+from xonsh.lib.jsonutils import serialize_xonsh_json
 from xonsh.tools import backup_file, print_color, to_bool, to_bool_or_break