Merge branch 'commands_cache_path_read_fix' of https://github.com/anki-code/xonsh into commands_cache_path_read_fix

.github/workflows/genbuilds.xsh

@@ -13,7 +13,9 @@ OS_IMAGES = {
     "macos": "macOS-latest",
     "windows": "windows-latest",
 }
-PYTHON_VERSIONS = ["3.6", "3.7", "3.8"]
+PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"]
+
+ALLOWED_FAILURES = ["3.9"]
 
 CURR_DIR = os.path.dirname(__file__)
 template_path = os.path.join(CURR_DIR, "pytest.tmpl")
@@ -23,5 +25,7 @@ for os_name, python_version in product(OS_NAMES, PYTHON_VERSIONS):
     s = template.replace("OS_NAME", os_name)
     s = s.replace("OS_IMAGE", OS_IMAGES[os_name])
     s = s.replace("PYTHON_VERSION", python_version)
+    if python_version in ALLOWED_FAILURES:
+        s = "\n".join((s, "        continue-on-error: true\n"))
     fname = os.path.join(CURR_DIR, f"pytest-{os_name}-{python_version}.yml")
     ![echo @(s) > @(fname)]

.github/workflows/pytest-linux-3.6.yml

@@ -33,7 +33,8 @@ jobs:
           update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
-      - shell: bash -l {0}
+      - name: Install Xonsh and run tests
+        shell: bash -l {0}
         run: |
           python -m pip --version
           python -m pip install -r requirements/tests.txt

.github/workflows/pytest-linux-3.7.yml

@@ -33,7 +33,8 @@ jobs:
           update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
-      - shell: bash -l {0}
+      - name: Install Xonsh and run tests
+        shell: bash -l {0}
         run: |
           python -m pip --version
           python -m pip install -r requirements/tests.txt

.github/workflows/pytest-linux-3.8.yml

@@ -33,7 +33,8 @@ jobs:
           update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
-      - shell: bash -l {0}
+      - name: Install Xonsh and run tests
+        shell: bash -l {0}
         run: |
           python -m pip --version
           python -m pip install -r requirements/tests.txt

.github/workflows/pytest-linux-3.9.yml

@@ -0,0 +1,45 @@
+name: pytest linux 3.9
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: [3.9]
+    name: Python ${{ matrix.python-version }} ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ~/miniconda*/envs/
+          key: ${{ runner.os }}-${{ matrix.python-version }}-env-${{ hashFiles('requirements/tests.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-env-
+      - name: Setup conda
+        uses: conda-incubator/setup-miniconda@v1
+        with:
+          activate-environment: xonsh-test
+          update-conda: true
+          python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
+          condarc-file: ci/condarc.yml
+      - name: Install Xonsh and run tests
+        shell: bash -l {0}
+        run: |
+          python -m pip --version
+          python -m pip install -r requirements/tests.txt
+          python -m pip install . --no-deps
+          python -m xonsh run-tests.xsh test -- --timeout=240
+
+        continue-on-error: true
+

.github/workflows/pytest-macos-3.6.yml

@@ -33,7 +33,8 @@ jobs:
           update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
-      - shell: bash -l {0}
+      - name: Install Xonsh and run tests
+        shell: bash -l {0}
         run: |
           python -m pip --version
           python -m pip install -r requirements/tests.txt

.github/workflows/pytest-macos-3.7.yml

@@ -33,7 +33,8 @@ jobs:
           update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
-      - shell: bash -l {0}
+      - name: Install Xonsh and run tests
+        shell: bash -l {0}
         run: |
           python -m pip --version
           python -m pip install -r requirements/tests.txt

.github/workflows/pytest-macos-3.8.yml

@@ -33,7 +33,8 @@ jobs:
           update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
-      - shell: bash -l {0}
+      - name: Install Xonsh and run tests
+        shell: bash -l {0}
         run: |
           python -m pip --version
           python -m pip install -r requirements/tests.txt

.github/workflows/pytest-macos-3.9.yml

@@ -0,0 +1,45 @@
+name: pytest macos 3.9
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macOS-latest]
+        python-version: [3.9]
+    name: Python ${{ matrix.python-version }} ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ~/miniconda*/envs/
+          key: ${{ runner.os }}-${{ matrix.python-version }}-env-${{ hashFiles('requirements/tests.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-env-
+      - name: Setup conda
+        uses: conda-incubator/setup-miniconda@v1
+        with:
+          activate-environment: xonsh-test
+          update-conda: true
+          python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
+          condarc-file: ci/condarc.yml
+      - name: Install Xonsh and run tests
+        shell: bash -l {0}
+        run: |
+          python -m pip --version
+          python -m pip install -r requirements/tests.txt
+          python -m pip install . --no-deps
+          python -m xonsh run-tests.xsh test -- --timeout=240
+
+        continue-on-error: true
+

.github/workflows/pytest-windows-3.6.yml

@@ -33,7 +33,8 @@ jobs:
           update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
-      - shell: bash -l {0}
+      - name: Install Xonsh and run tests
+        shell: bash -l {0}
         run: |
           python -m pip --version
           python -m pip install -r requirements/tests.txt

.github/workflows/pytest-windows-3.7.yml

@@ -33,7 +33,8 @@ jobs:
           update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
-      - shell: bash -l {0}
+      - name: Install Xonsh and run tests
+        shell: bash -l {0}
         run: |
           python -m pip --version
           python -m pip install -r requirements/tests.txt

.github/workflows/pytest-windows-3.8.yml

@@ -33,7 +33,8 @@ jobs:
           update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
-      - shell: bash -l {0}
+      - name: Install Xonsh and run tests
+        shell: bash -l {0}
         run: |
           python -m pip --version
           python -m pip install -r requirements/tests.txt

.github/workflows/pytest-windows-3.9.yml

@@ -0,0 +1,45 @@
+name: pytest windows 3.9
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest]
+        python-version: [3.9]
+    name: Python ${{ matrix.python-version }} ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ~/miniconda*/envs/
+          key: ${{ runner.os }}-${{ matrix.python-version }}-env-${{ hashFiles('requirements/tests.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-env-
+      - name: Setup conda
+        uses: conda-incubator/setup-miniconda@v1
+        with:
+          activate-environment: xonsh-test
+          update-conda: true
+          python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
+          condarc-file: ci/condarc.yml
+      - name: Install Xonsh and run tests
+        shell: bash -l {0}
+        run: |
+          python -m pip --version
+          python -m pip install -r requirements/tests.txt
+          python -m pip install . --no-deps
+          python -m xonsh run-tests.xsh test -- --timeout=240
+
+        continue-on-error: true
+

.github/workflows/pytest.tmpl

@@ -33,7 +33,8 @@ jobs:
           update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
-      - shell: bash -l {0}
+      - name: Install Xonsh and run tests
+        shell: bash -l {0}
         run: |
           python -m pip --version
           python -m pip install -r requirements/tests.txt

.gitignore

@@ -83,3 +83,4 @@ venv/
 
 # mypy
 .dmypy.json
+.mypy_cache

news/custom_prompt_tokens_formatter.rst

@@ -0,0 +1,28 @@
+**Added:**
+
+* Added new environment variable ``$PROMPT_TOKENS_FORMATTER``.
+    That can be used to set a callable that receives all tokens in the prompt template.
+    It gives option to format the prompt with different prefix based on other tokens values.
+    Enables users to implement something like [powerline](https://github.com/vaaaaanquish/xontrib-powerline2)
+    without resorting to separate $PROMPT_FIELDS. Works with ``ASYNC_PROMPT`` as well.
+    Check the `PR <https://github.com/xonsh/xonsh/pull/3922>`_ for a snippet implementing powerline
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

news/custom_theme_ptk_support.rst

@@ -0,0 +1,31 @@
+**Added:**
+
+* PTK style rules can be defined in custom styles using the ``Token.PTK`` token prefix.
+  For example ``custom_style["Token.PTK.CompletionMenu.Completion.Current"] = "bg:#ff0000 #fff"`` sets the ``completion-menu.completion.current`` PTK style to white on red.
+* Added new environment variable ``XONSH_STYLE_OVERRIDES``. It's a dictionary containing pygments/ptk style definitions that overrides the styles defined by ``XONSH_COLOR_STYLE``.
+  For example::
+
+    $XONSH_STYLE_OVERRIDES["Token.Literal.String.Single"] = "#00ff00"  # green 'strings' (pygments)
+    $XONSH_STYLE_OVERRIDES["completion-menu"] = "bg:#ffff00 #000"  # black on yellow completion (ptk)
+    $XONSH_STYLE_OVERRIDES["Token.PTK.CompletionMenu.Completion.Current"] = "bg:#ff0000 #fff" # current completion is white on red (ptk via pygments)
+
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* ``PTK_STYLE_OVERRIDES`` has been deprecated, its function replaced by ``XONSH_STYLE_OVERRIDES``
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

news/github_workflow.rst

@@ -0,0 +1,23 @@
+**Added:**
+
+* Added Python 3.9 to continuous integration.
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

news/xontrib-cmd-durations.rst

@@ -0,0 +1,23 @@
+**Added:**
+
+* added `xontrib-long-cmd-durations <https://github.com/jnoortheen/xontrib-cmd-durations>`_
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

news/zoxide_gitinfo.rst

@@ -0,0 +1,24 @@
+**Added:**
+
+* Added ``xontrib-zoxide`` to the list of xontribs.
+* Added ``xontrib-gitinfo`` to the list of xontribs.
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

scent.py

@@ -26,11 +26,9 @@ def python(*_):
     group = int(time.time())  # unique per run
 
     for count, (command, title) in enumerate((
-            (('dmypy', 'run', "--", "xonsh"), "Lint"),
-            (('flake8', '--count'), "Lint"),
-            (('pytest', 'tests/test_main.py'), "Test main"),
-            (('pytest', 'tests/test_ptk_highlight.py'), "Test ptk highlight"),
-            (('pytest', '--ignore', 'tests/test_main.py', 'tests/test_ptk_highlight.py'), "Test Rest"),
+            (('dmypy', 'run', "--", "xonsh"), "type-check"),
+            (('flake8', '.'), "Lint"),
+            (('xonsh', 'run-tests.xsh', 'test'), "test"),
     ), start=1):
 
         print(f"\n$ {' '.join(command)}")

tests/test_ansi_colors.py

@@ -147,9 +147,10 @@ def test_ansi_color_name_to_escape_code_for_all_styles(color, style):
     [
         ("test1", {}, {}),
         ("test2", {"Color.RED": "#ff0000"}, {"RED": "38;5;196"}),
-        ("test3", {"BOLD_RED": "bold #ff0000"}, {"BOLD_RED": "1;38;5;196"}),
+        ("test3", {"Token.Color.RED": "#ff0000"}, {"RED": "38;5;196"}),
+        ("test4", {"BOLD_RED": "bold #ff0000"}, {"BOLD_RED": "1;38;5;196"}),
         (
-            "test4",
+            "test5",
             {"INTENSE_RED": "italic underline bg:#ff0000 #ff0000"},
             {"INTENSE_RED": "3;4;48;5;196;38;5;196"},
         ),

tests/test_pyghooks.py

@@ -361,6 +361,11 @@ def test_colorize_file_ca(xonsh_builtins_LS_COLORS, monkeypatch):
             {"Literal.String.Single": "#ff0000"},
             {Token.Literal.String.Single: "#ff0000"},
         ),  # short str key
+        (
+            "test5",
+            {"completion-menu.completion.current": "#00ff00"},
+            {Token.PTK.CompletionMenu.Completion.Current: "#00ff00"},
+        ),  # ptk style
     ],
 )
 def test_register_custom_pygments_style(name, styles, refrules):

xonsh/ansi_colors.py

@@ -146,6 +146,9 @@ def ansi_partial_color_format(template, style="default", cmap=None, hide=False):
 
 def _ansi_partial_color_format_main(template, style="default", cmap=None, hide=False):
     cmap = _ensure_color_map(style=style, cmap=cmap)
+    overrides = builtins.__xonsh__.env["XONSH_STYLE_OVERRIDES"]
+    if overrides:
+        cmap.update(_style_dict_to_ansi(overrides))
     esc = ("\001" if hide else "") + "\033["
     m = "m" + ("\002" if hide else "")
     bopen = "{"
@@ -1104,6 +1107,18 @@ def _pygments_to_ansi_style(style):
     return ";".join(ansi_style_list)
 
 
+def _style_dict_to_ansi(styles):
+    """Converts pygments like style dict to ANSI rules"""
+    ansi_style = {}
+    for token, style in styles.items():
+        token = str(token)  # convert pygments token to str
+        parts = token.split(".")
+        if len(parts) == 1 or parts[-2] == "Color":
+            ansi_style[parts[-1]] = _pygments_to_ansi_style(style)
+
+    return ansi_style
+
+
 def register_custom_ansi_style(name, styles, base="default"):
     """Register custom ANSI style.
 
@@ -1118,11 +1133,7 @@ def register_custom_ansi_style(name, styles, base="default"):
     """
     base_style = ANSI_STYLES[base].copy()
 
-    for token, style in styles.items():
-        token = str(token)  # convert pygments token to str
-        parts = token.split(".")
-        if len(parts) == 1 or parts[-2] == "Color":
-            base_style[parts[-1]] = _pygments_to_ansi_style(style)
+    base_style.update(_style_dict_to_ansi(styles))
 
     ANSI_STYLES[name] = base_style
 

xonsh/environ.py

@@ -30,8 +30,6 @@ from xonsh.platform import (
     os_environ,
 )
 
-from xonsh.style_tools import PTK2_STYLE
-
 from xonsh.tools import (
     always_true,
     always_false,
@@ -1194,6 +1192,16 @@ def DEFAULT_VARS():
             "NOTE: ``$UPDATE_PROMPT_ON_KEYPRESS`` must be set to ``True`` for this "
             "variable to take effect.",
         ),
+        "PROMPT_TOKENS_FORMATTER": Var(
+            validate=callable,
+            convert=None,
+            detype=None,
+            default=prompt.prompt_tokens_formatter_default,
+            doc="Final processor that receives all tokens in the prompt template. "
+            "It gives option to format the prompt with different prefix based on other tokens values. "
+            "Highly useful for implementing something like powerline theme.",
+            doc_default="``xonsh.prompt.base.prompt_tokens_formatter_default``",
+        ),
         "PROMPT_TOOLKIT_COLOR_DEPTH": Var(
             always_false,
             ptk2_color_depth_setter,
@@ -1207,8 +1215,8 @@ def DEFAULT_VARS():
             is_str_str_dict,
             to_str_str_dict,
             dict_to_str,
-            dict(PTK2_STYLE),
-            "A dictionary containing custom prompt_toolkit style definitions.",
+            {},
+            "A dictionary containing custom prompt_toolkit style definitions. (deprecated)",
         ),
         "PUSHD_MINUS": Var(
             is_bool,
@@ -1708,6 +1716,18 @@ def DEFAULT_VARS():
             "Whether or not to store the ``stdout`` and ``stderr`` streams in the "
             "history files.",
         ),
+        "XONSH_STYLE_OVERRIDES": Var(
+            is_str_str_dict,
+            to_str_str_dict,
+            dict_to_str,
+            {},
+            "A dictionary containing custom prompt_toolkit/pygments style definitions.\n"
+            "The following style definitions are supported:\n\n"
+            "    - ``pygments.token.Token`` - ``$XONSH_STYLE_OVERRIDES[Token.Keyword] = '#ff0000'``\n"
+            "    - pygments token name (string) - ``$XONSH_STYLE_OVERRIDES['Token.Keyword'] = '#ff0000'``\n"
+            "    - ptk style name (string) - ``$XONSH_STYLE_OVERRIDES['pygments.keyword'] = '#ff0000'``\n\n"
+            "(The rules above are all have the same effect.)",
+        ),
         "XONSH_TRACE_SUBPROC": Var(
             is_bool,
             to_bool,

xonsh/prompt/base.py

@@ -7,6 +7,7 @@ import os
 import re
 import socket
 import sys
+import typing as tp
 
 import xonsh.lazyasd as xl
 import xonsh.tools as xt
@@ -29,6 +30,54 @@ def DEFAULT_PROMPT():
     return default_prompt()
 
 
+class _ParsedToken(tp.NamedTuple):
+    """It can either be a literal value alone or a field and its resultant value"""
+
+    value: str
+    field: tp.Optional[str] = None
+
+
+class ParsedTokens(tp.NamedTuple):
+    tokens: tp.List[_ParsedToken]
+    template: tp.Union[str, tp.Callable]
+
+    def process(self) -> str:
+        """Wrapper that gets formatter-function from environment and returns final prompt."""
+        processor = builtins.__xonsh__.env.get(  # type: ignore
+            "PROMPT_TOKENS_FORMATTER", prompt_tokens_formatter_default
+        )
+        return processor(self)
+
+    def update(
+        self,
+        idx: int,
+        val: tp.Optional[str],
+        spec: tp.Optional[str],
+        conv: tp.Optional[str],
+    ) -> None:
+        """Update tokens list in-place"""
+        if idx < len(self.tokens):
+            tok = self.tokens[idx]
+            self.tokens[idx] = _ParsedToken(_format_value(val, spec, conv), tok.field)
+
+
+def prompt_tokens_formatter_default(container: ParsedTokens) -> str:
+    """
+        Join the tokens
+
+    Parameters
+    ----------
+    container: ParsedTokens
+        parsed tokens holder
+
+    Returns
+    -------
+    str
+        process the tokens and finally return the prompt string
+    """
+    return "".join([tok.value for tok in container.tokens])
+
+
 class PromptFormatter:
     """Class that holds all the related prompt formatting methods,
     uses the ``PROMPT_FIELDS`` envvar (no color formatting).
@@ -37,36 +86,41 @@ class PromptFormatter:
     def __init__(self):
         self.cache = {}
 
-    def __call__(self, template=DEFAULT_PROMPT, fields=None, **kwargs):
+    def __call__(self, template=DEFAULT_PROMPT, fields=None, **kwargs) -> str:
         """Formats a xonsh prompt template string."""
 
         # keep cache only during building prompt
         self.cache.clear()
 
         if fields is None:
-            self.fields = builtins.__xonsh__.env.get("PROMPT_FIELDS", PROMPT_FIELDS)
+            self.fields = builtins.__xonsh__.env.get("PROMPT_FIELDS", PROMPT_FIELDS)  # type: ignore
         else:
             self.fields = fields
         try:
-            prompt = self._format_prompt(template=template, **kwargs)
-        except Exception:
+            toks = self._format_prompt(template=template, **kwargs)
+            prompt = toks.process()
+        except Exception as ex:
+            # make it obvious why it has failed
+            print(
+                f"Failed to format prompt `{template}`-> {type(ex)}:{ex}",
+                file=sys.stderr,
+            )
             return _failover_template_format(template)
         return prompt
 
-    def _format_prompt(self, template=DEFAULT_PROMPT, **kwargs):
-        return "".join(self._get_tokens(template, **kwargs))
-
-    def _get_tokens(self, template, **kwargs):
-        template = template() if callable(template) else template
+    def _format_prompt(self, template=DEFAULT_PROMPT, **kwargs) -> ParsedTokens:
+        tmpl = template() if callable(template) else template
         toks = []
-        for literal, field, spec, conv in xt.FORMATTER.parse(template):
-            toks.append(literal)
+        for literal, field, spec, conv in xt.FORMATTER.parse(tmpl):
+            if literal:
+                toks.append(_ParsedToken(literal))
             entry = self._format_field(field, spec, conv, idx=len(toks), **kwargs)
             if entry is not None:
-                toks.append(entry)
-        return toks
+                toks.append(_ParsedToken(entry, field))
 
-    def _format_field(self, field, spec, conv, **kwargs):
+        return ParsedTokens(toks, template)
+
+    def _format_field(self, field, spec="", conv=None, **kwargs):
         if field is None:
             return
         elif field.startswith("$"):

xonsh/ptk_shell/formatter.py

@@ -1,10 +1,11 @@
 """PTK specific PromptFormatter class."""
 
 import functools
+import typing as tp
 
 from prompt_toolkit import PromptSession
 from xonsh.prompt.base import PromptFormatter, DEFAULT_PROMPT
-from xonsh.ptk_shell.updator import PromptUpdator
+from xonsh.ptk_shell.updator import PromptUpdator, AsyncPrompt
 
 
 class PTKPromptFormatter(PromptFormatter):
@@ -35,17 +36,21 @@ class PTKPromptFormatter(PromptFormatter):
             kwargs["async_prompt"] = self.updator.add(prompt_name)
 
         # in case of failure it returns a fail-over template. otherwise it returns list of tokens
-        prompt_or_tokens = super().__call__(template, fields, **kwargs)
+        return super().__call__(template, fields, **kwargs)
 
-        if isinstance(prompt_or_tokens, list):
-            if threaded:
-                self.updator.set_tokens(prompt_name, prompt_or_tokens)
-            return "".join(prompt_or_tokens)
-
-        return prompt_or_tokens
-
-    def _format_prompt(self, template=DEFAULT_PROMPT, **kwargs):
-        return self._get_tokens(template, **kwargs)
+    def _format_prompt(
+        self,
+        template=DEFAULT_PROMPT,
+        async_prompt: tp.Optional[AsyncPrompt] = None,
+        **kwargs
+    ):
+        toks = super()._format_prompt(
+            template=template, async_prompt=async_prompt, **kwargs
+        )
+        if async_prompt is not None:
+            # late binding of values
+            async_prompt.tokens = toks
+        return toks
 
     def _no_cache_field_value(
         self, field, field_value, async_prompt=None, idx=None, spec=None, conv=None, **_

xonsh/ptk_shell/shell.py

@@ -10,7 +10,7 @@ from xonsh.events import events
 from xonsh.base_shell import BaseShell
 from xonsh.ptk_shell.formatter import PTKPromptFormatter
 from xonsh.shell import transform_command
-from xonsh.tools import print_exception, carriage_return
+from xonsh.tools import print_exception, print_warning, carriage_return
 from xonsh.platform import HAS_PYGMENTS, ON_WINDOWS, ON_POSIX
 from xonsh.style_tools import partial_color_tokenize, _TokenType, DEFAULT_STYLE_DICT
 from xonsh.lazyimps import pygments, pyghooks, winutils
@@ -33,13 +33,11 @@ from prompt_toolkit.shortcuts import CompleteStyle
 from prompt_toolkit.shortcuts.prompt import PromptSession
 from prompt_toolkit.formatted_text import PygmentsTokens, to_formatted_text
 from prompt_toolkit.styles import merge_styles, Style
-from prompt_toolkit.styles.pygments import (
-    style_from_pygments_cls,
-    style_from_pygments_dict,
-)
+from prompt_toolkit.styles.pygments import pygments_token_to_classname
 
 
 ANSI_OSC_PATTERN = re.compile("\x1b].*?\007")
+CAPITAL_PATTERN = re.compile(r"([a-z])([A-Z])")
 Token = _TokenType()
 
 events.transmogrify("on_ptk_create", "LoadEvent")
@@ -92,6 +90,46 @@ def remove_ansi_osc(prompt):
     return prompt, osc_tokens
 
 
+def _pygments_token_to_classname(token):
+    """Converts pygments Tokens, token names (strings) to PTK style names."""
+    if token and isinstance(token, str):
+        # if starts with non capital letter => leave it as it is
+        if token[0].islower():
+            return token
+        # if starts with capital letter => pygments token name
+        if token.startswith("Token."):
+            token = token[6:]
+        # short colors - all caps
+        if token == token.upper():
+            token = "color." + token
+        return "pygments." + token.lower()
+
+    return pygments_token_to_classname(token)
+
+
+def _style_from_pygments_dict(pygments_dict):
+    """Custom implementation of ``style_from_pygments_dict`` that supports PTK specific
+    (``Token.PTK``) styles.
+    """
+    pygments_style = []
+
+    for token, style in pygments_dict.items():
+        # if ``Token.PTK`` then add it as "native" PTK style too
+        if str(token).startswith("Token.PTK"):
+            key = CAPITAL_PATTERN.sub(r"\1-\2", str(token)[10:]).lower()
+            pygments_style.append((key, style))
+        pygments_style.append((_pygments_token_to_classname(token), style))
+
+    return Style(pygments_style)
+
+
+def _style_from_pygments_cls(pygments_cls):
+    """Custom implementation of ``style_from_pygments_cls`` that supports PTK specific
+    (``Token.PTK``) styles.
+    """
+    return _style_from_pygments_dict(pygments_cls.styles)
+
+
 class PromptToolkitShell(BaseShell):
     """The xonsh shell for prompt_toolkit v2 and later."""
 
@@ -112,6 +150,7 @@ class PromptToolkitShell(BaseShell):
         self.prompt_formatter = PTKPromptFormatter(self.prompter)
         self.pt_completer = PromptToolkitCompleter(self.completer, self.ctx, self)
         self.key_bindings = load_xonsh_bindings()
+        self._overrides_deprecation_warning_shown = False
 
         # Store original `_history_matches` in case we need to restore it
         self._history_matches_orig = self.prompter.default_buffer._history_matches
@@ -202,21 +241,36 @@ class PromptToolkitShell(BaseShell):
 
         if env.get("COLOR_INPUT"):
             events.on_timingprobe.fire(name="on_pre_prompt_style")
+            style_overrides_env = env.get("PTK_STYLE_OVERRIDES", {}).copy()
+            if (
+                len(style_overrides_env) > 0
+                and not self._overrides_deprecation_warning_shown
+            ):
+                print_warning(
+                    "$PTK_STYLE_OVERRIDES is deprecated, use $XONSH_STYLE_OVERRIDES instead!"
+                )
+                self._overrides_deprecation_warning_shown = True
+            style_overrides_env.update(env.get("XONSH_STYLE_OVERRIDES", {}))
             if HAS_PYGMENTS:
                 prompt_args["lexer"] = PygmentsLexer(pyghooks.XonshLexer)
-                style = style_from_pygments_cls(pyghooks.xonsh_style_proxy(self.styler))
+                self.styler.override(style_overrides_env)
+                style = _style_from_pygments_cls(
+                    pyghooks.xonsh_style_proxy(self.styler)
+                )
             else:
-                style = style_from_pygments_dict(DEFAULT_STYLE_DICT)
-            prompt_args["style"] = style
-            events.on_timingprobe.fire(name="on_post_prompt_style")
-
-            style_overrides_env = env.get("PTK_STYLE_OVERRIDES")
-            if style_overrides_env:
                 try:
-                    style_overrides = Style.from_dict(style_overrides_env)
-                    prompt_args["style"] = merge_styles([style, style_overrides])
+                    style = merge_styles(
+                        [
+                            _style_from_pygments_dict(DEFAULT_STYLE_DICT),
+                            _style_from_pygments_dict(style_overrides_env),
+                        ]
+                    )
                 except (AttributeError, TypeError, ValueError):
                     print_exception()
+                    style = _style_from_pygments_dict(DEFAULT_STYLE_DICT)
+
+            prompt_args["style"] = style
+            events.on_timingprobe.fire(name="on_post_prompt_style")
 
         if env["ENABLE_ASYNC_PROMPT"]:
             # once the prompt is done, update it in background as each future is completed
@@ -363,7 +417,9 @@ class PromptToolkitShell(BaseShell):
         tokens = partial_color_tokenize(string)
         if force_string and HAS_PYGMENTS:
             env = builtins.__xonsh__.env
+            style_overrides_env = env.get("XONSH_STYLE_OVERRIDES", {})
             self.styler.style_name = env.get("XONSH_COLOR_STYLE")
+            self.styler.override(style_overrides_env)
             proxy_style = pyghooks.xonsh_style_proxy(self.styler)
             formatter = pyghooks.XonshTerminal256Formatter(style=proxy_style)
             s = pygments.format(tokens, formatter)
@@ -382,14 +438,21 @@ class PromptToolkitShell(BaseShell):
             # assume this is a list of (Token, str) tuples and just print
             tokens = string
         tokens = PygmentsTokens(tokens)
+        env = builtins.__xonsh__.env
+        style_overrides_env = env.get("XONSH_STYLE_OVERRIDES", {})
         if HAS_PYGMENTS:
-            env = builtins.__xonsh__.env
             self.styler.style_name = env.get("XONSH_COLOR_STYLE")
-            proxy_style = style_from_pygments_cls(
+            self.styler.override(style_overrides_env)
+            proxy_style = _style_from_pygments_cls(
                 pyghooks.xonsh_style_proxy(self.styler)
             )
         else:
-            proxy_style = style_from_pygments_dict(DEFAULT_STYLE_DICT)
+            proxy_style = merge_styles(
+                [
+                    _style_from_pygments_dict(DEFAULT_STYLE_DICT),
+                    _style_from_pygments_dict(style_overrides_env),
+                ]
+            )
         ptk_print(
             tokens, style=proxy_style, end=end, include_default_pygments_style=False
         )

xonsh/ptk_shell/updator.py

@@ -8,7 +8,7 @@ import typing as tp
 from prompt_toolkit import PromptSession
 from prompt_toolkit.formatted_text import PygmentsTokens
 
-from xonsh.prompt.base import _format_value
+from xonsh.prompt.base import ParsedTokens
 from xonsh.style_tools import partial_color_tokenize, style_as_faded
 
 
@@ -64,7 +64,7 @@ class AsyncPrompt:
         self.name = name
 
         # list of tokens in that prompt. It could either be resolved or not resolved.
-        self.tokens: tp.List[str] = []
+        self.tokens: tp.Optional[ParsedTokens] = None
         self.timer = None
         self.session = session
         self.executor = executor
@@ -85,6 +85,9 @@ class AsyncPrompt:
         on_complete:
             callback to notify after all the futures are completed
         """
+        if not self.tokens:
+            print(f"Warn: AsyncPrompt is created without tokens - {self.name}")
+            return
         for fut in concurrent.futures.as_completed(self.futures):
             val = fut.result()
 
@@ -97,29 +100,18 @@ class AsyncPrompt:
             # example: placeholder="{field}", idx=10, spec="env: {}"
 
             if isinstance(idx, int):
-                self.update_token(idx, val, spec, conv)
+                self.tokens.update(idx, val, spec, conv)
             else:  # when the function is called outside shell.
-                for idx, sect in enumerate(self.tokens):
-                    if placeholder in sect:
-                        val = sect.replace(placeholder, val)
-                        self.update_token(idx, val, spec, conv)
+                for idx, ptok in enumerate(self.tokens.tokens):
+                    if placeholder in ptok.value:
+                        val = ptok.value.replace(placeholder, val)
+                        self.tokens.update(idx, val, spec, conv)
 
             # calling invalidate in less period is inefficient
             self.invalidate()
 
         on_complete(self.name)
 
-    def update_token(
-        self,
-        idx: int,
-        val: tp.Optional[str],
-        spec: tp.Optional[str],
-        conv: tp.Optional[str],
-    ) -> None:
-        """Update tokens list in-place"""
-        if idx < len(self.tokens):
-            self.tokens[idx] = _format_value(val, spec, conv)
-
     def invalidate(self):
         """Create a timer to update the prompt. The timing can be configured through env variables.
         threading.Timer is used to stop calling invalidate frequently.
@@ -130,7 +122,7 @@ class AsyncPrompt:
             self.timer.cancel()
 
         def _invalidate():
-            new_prompt = "".join(self.tokens)
+            new_prompt = self.tokens.process()
             formatted_tokens = tokenize_ansi(
                 PygmentsTokens(partial_color_tokenize(new_prompt))
             )
@@ -169,10 +161,10 @@ class PromptUpdator:
         self.prompter = session
         self.executor = Executor()
 
-    def add(self, prompt_name: tp.Optional[str]):
+    def add(self, prompt_name: tp.Optional[str]) -> tp.Optional[AsyncPrompt]:
         # clear out old futures from the same prompt
         if prompt_name is None:
-            return
+            return None
 
         if prompt_name in self.prompts:
             self.stop(prompt_name)
@@ -198,7 +190,3 @@ class PromptUpdator:
 
     def on_complete(self, prompt_name):
         self.prompts.pop(prompt_name, None)
-
-    def set_tokens(self, prompt_name, tokens: tp.List[str]):
-        if prompt_name in self.prompts:
-            self.prompts[prompt_name].tokens = tokens

xonsh/pyghooks.py

@@ -52,7 +52,7 @@ from xonsh.color_tools import (
     iscolor,
     warn_deprecated_no_color,
 )
-from xonsh.style_tools import norm_name
+from xonsh.style_tools import norm_name, DEFAULT_STYLE_DICT
 from xonsh.lazyimps import terminal256, html
 from xonsh.platform import (
     os_environ,
@@ -234,7 +234,7 @@ def color_token_by_name(xc: tuple, styles=None) -> _TokenType:
         tokName += "__" + xc[1]
 
     token = getattr(Color, norm_name(tokName))
-    if token not in styles:
+    if token not in styles or not styles[token]:
         styles[token] = pc
     return token
 
@@ -382,8 +382,13 @@ class XonshStyle(Style):
                 self.background_color = style_obj.background_color
             except (ImportError, pygments.util.ClassNotFound):
                 self._smap = XONSH_BASE_STYLE.copy()
-        compound = CompoundColorMap(ChainMap(self.trap, cmap, PTK_STYLE, self._smap))
-        self.styles = ChainMap(self.trap, cmap, PTK_STYLE, self._smap, compound)
+
+        compound = CompoundColorMap(
+            ChainMap(self.trap, cmap, self._smap, DEFAULT_STYLE_DICT)
+        )
+        self.styles = ChainMap(
+            self.trap, cmap, self._smap, DEFAULT_STYLE_DICT, compound
+        )
         self._style_name = value
 
         for file_type, xonsh_color in builtins.__xonsh__.env.get(
@@ -399,6 +404,9 @@ class XonshStyle(Style):
     def style_name(self):
         self._style_name = ""
 
+    def override(self, style_dict):
+        self.trap.update(_tokenize_style_dict(style_dict))
+
     def enhance_colors_for_cmd_exe(self):
         """ Enhance colors when using cmd.exe on windows.
             When using the default style all blue and dark red colors
@@ -438,24 +446,43 @@ def xonsh_style_proxy(styler):
     return XonshStyleProxy
 
 
+def _format_ptk_style_name(name):
+    """Format PTK style name to be able to include it in a pygments style"""
+    parts = name.split("-")
+    return "".join(part.capitalize() for part in parts)
+
+
 def _get_token_by_name(name):
     """Get pygments token object by its string representation."""
+    if not isinstance(name, str):
+        return name
+
     token = Token
     parts = name.split(".")
 
+    # PTK - all lowercase
+    if parts[0] == parts[0].lower():
+        parts = ["PTK"] + [_format_ptk_style_name(part) for part in parts]
+
+    # color name
     if len(parts) == 1:
-        parts = ["Color"] + parts
+        return color_token_by_name((name,))
 
     if parts[0] == "Token":
         parts = parts[1:]
 
-    while len(parts):
+    while len(parts) > 0:
         token = getattr(token, parts[0])
         parts = parts[1:]
 
     return token
 
 
+def _tokenize_style_dict(styles):
+    """Converts possible string keys in style dicts to Tokens"""
+    return {_get_token_by_name(token): value for token, value in styles.items()}
+
+
 def register_custom_pygments_style(
     name, styles, highlight_color=None, background_color=None, base="default"
 ):
@@ -481,9 +508,7 @@ def register_custom_pygments_style(
     base_style = get_style_by_name(base)
     custom_styles = base_style.styles.copy()
 
-    for token, value in styles.items():
-        if isinstance(token, str):
-            token = _get_token_by_name(token)
+    for token, value in _tokenize_style_dict(styles).items():
         custom_styles[token] = value
 
     style = type(
@@ -514,22 +539,6 @@ def register_custom_pygments_style(
     return style
 
 
-PTK_STYLE = LazyObject(
-    lambda: {
-        Token.Menu.Completions: "bg:ansigray ansiblack",
-        Token.Menu.Completions.Completion: "",
-        Token.Menu.Completions.Completion.Current: "bg:ansibrightblack ansiwhite",
-        Token.Scrollbar: "bg:ansibrightblack",
-        Token.Scrollbar.Button: "bg:ansiblack",
-        Token.Scrollbar.Arrow: "bg:ansiblack ansiwhite bold",
-        Token.AutoSuggestion: "ansibrightblack",
-        Token.Aborted: "ansibrightblack",
-    },
-    globals(),
-    "PTK_STYLE",
-)
-
-
 XONSH_BASE_STYLE = LazyObject(
     lambda: {
         Whitespace: "ansigray",

xonsh/readline_shell.py

@@ -633,7 +633,9 @@ class ReadlineShell(BaseShell, cmd.Cmd):
         else:
             # assume this is a list of (Token, str) tuples and format it
             env = builtins.__xonsh__.env
+            style_overrides_env = env.get("XONSH_STYLE_OVERRIDES", {})
             self.styler.style_name = env.get("XONSH_COLOR_STYLE")
+            self.styler.override(style_overrides_env)
             style_proxy = pyghooks.xonsh_style_proxy(self.styler)
             formatter = pyghooks.XonshTerminal256Formatter(style=style_proxy)
             s = pygments.format(string, formatter).rstrip()

xonsh/style_tools.py

@@ -181,8 +181,6 @@ DEFAULT_STYLE_DICT = LazyObject(
         lambda: "",
         {
             Token: "",
-            Token.Aborted: "ansibrightblack",
-            Token.AutoSuggestion: "ansibrightblack",
             Token.Color.BACKGROUND_BLACK: "bg:ansiblack",
             Token.Color.BACKGROUND_BLUE: "bg:ansiblue",
             Token.Color.BACKGROUND_CYAN: "bg:ansicyan",
@@ -314,9 +312,6 @@ DEFAULT_STYLE_DICT = LazyObject(
             Token.Literal.String.Regex: "ansimagenta",
             Token.Literal.String.Single: "",
             Token.Literal.String.Symbol: "ansiyellow",
-            Token.Menu.Completions: "bg:ansigray ansiblack",
-            Token.Menu.Completions.Completion: "",
-            Token.Menu.Completions.Completion.Current: "bg:ansibrightblack ansiwhite",
             Token.Name: "",
             Token.Name.Attribute: "ansibrightyellow",
             Token.Name.Builtin: "ansigreen",
@@ -342,24 +337,18 @@ DEFAULT_STYLE_DICT = LazyObject(
             Token.Operator.Word: "bold ansimagenta",
             Token.Other: "",
             Token.Punctuation: "",
-            Token.Scrollbar: "bg:ansibrightblack",
-            Token.Scrollbar.Arrow: "bg:ansiblack ansiwhite bold",
-            Token.Scrollbar.Button: "bg:ansiblack",
             Token.Text: "",
             Token.Text.Whitespace: "ansigray",
+            Token.PTK.Aborting: "ansibrightblack",
+            Token.PTK.AutoSuggestion: "ansibrightblack",
+            Token.PTK.CompletionMenu: "bg:ansigray ansiblack",
+            Token.PTK.CompletionMenu.Completion: "",
+            Token.PTK.CompletionMenu.Completion.Current: "bg:ansibrightblack ansiwhite",
+            Token.PTK.Scrollbar.Arrow: "bg:ansiblack ansiwhite bold",
+            Token.PTK.Scrollbar.Background: "bg:ansibrightblack",
+            Token.PTK.Scrollbar.Button: "bg:ansiblack",
         },
     ),
     globals(),
     "DEFAULT_STYLE_DICT",
 )
-
-PTK2_STYLE = {
-    "completion-menu": "bg:ansigray ansiblack",
-    "completion-menu.completion": "",
-    "completion-menu.completion.current": "bg:ansibrightblack ansiwhite",
-    "scrollbar.background": "bg:ansibrightblack",
-    "scrollbar.arrow": "bg:ansiblack ansiwhite bold",
-    "scrollbar.button": "bg:ansiblack",
-    "auto-suggestion": "ansibrightblack",
-    "aborting": "ansibrightblack",
-}

xonsh/xontribs.json

@@ -90,6 +90,16 @@
    "tools are cross-platform."
   ]
  },
+ {
+  "name": "cmd_done",
+  "package": "xontrib-cmd-durations",
+  "url": "https://github.com/jnoortheen/xontrib-cmd-durations",
+  "description": [
+   "send notification once long-running command is finished.",
+   " Adds `long_cmd_duration` field to $PROMPT_FIELDS.",
+   " Note: It needs `xdotool` installed to detect current window."
+  ]
+ },
  {
   "name": "direnv",
   "package": "xonsh-direnv",
@@ -148,6 +158,11 @@
   "url": "https://github.com/laloch/xontrib-fzf-widgets",
   "description": ["Adds some fzf widgets to your xonsh shell."]
  },
+ {"name": "gitinfo",
+  "package": "xontrib-gitinfo",
+  "url": "https://github.com/dyuri/xontrib-gitinfo",
+  "description": ["Displays git information on entering a repository folder. Uses ``onefetch`` if available."]
+ },
  {"name": "histcpy",
   "package": "xontrib-histcpy",
   "url": "https://github.com/con-f-use/xontrib-histcpy",
@@ -290,6 +305,11 @@
   "package": "xontrib-z",
   "url": "https://github.com/AstraLuma/xontrib-z",
   "description": ["Tracks your most used directories, based on 'frecency'."]
+ },
+ {"name": "zoxide",
+  "package": "xontrib-zoxide",
+  "url": "https://github.com/dyuri/xontrib-zoxide",
+  "description": ["Zoxide integration for xonsh."]
  }
  ],
  "packages": {
@@ -358,6 +378,13 @@
     "pip": "xpip install xonsh-docker-tabcomplete"
    }
   },
+  "xontrib-hist-navigator": {
+   "license": "MIT",
+   "url": "https://github.com/jnoortheen/xontrib-hist-navigator",
+   "install": {
+    "pip": "xpip install xontrib-hist-navigator"
+   }
+  },
   "xonsh-scrapy-tabcomplete": {
    "license": "GPLv3",
    "url": "https://github.com/Granitas/xonsh-scrapy-tabcomplete",
@@ -386,6 +413,13 @@
     "pip": "xpip install xontrib-avox"
    }
   },
+  "xontrib-cmd-durations": {
+    "license": "MIT",    
+    "url": "https://github.com/jnoortheen/xontrib-cmd-durations",
+    "install": {
+      "pip": "xpip install xontrib-cmd-durations"
+    }
+  },
   "xontrib-fzf-widgets": {
    "license": "GPLv3",
    "url": "https://github.com/laloch/xontrib-fzf-widgets",
@@ -393,6 +427,13 @@
     "pip": "xpip install xontrib-fzf-widgets"
    }
   },
+  "xontrib-gitinfo": {
+   "license": "MIT",
+   "url": "https://github.com/dyuri/xontrib-gitinfo",
+   "install": {
+    "pip": "xpip install xontrib-gitinfo"
+   }
+  },
   "xontrib-histcpy": {
    "license": "GPLv3",
    "url": "https://github.com/con-f-use/xontrib-histcpy",
@@ -519,6 +560,13 @@
    "install": {
     "pip": "xpip install xontrib-z"
    }
+  },
+  "xontrib-zoxide": {
+   "license": "MIT",
+   "url": "https://github.com/dyuri/xontrib-zoxide",
+   "install": {
+    "pip": "xpip install xontrib-zoxide"
+   }
   }
  }
 }