Merge branch 'master' of https://github.com/xonsh/xonsh into rever_appimage

.gitignore

@@ -62,6 +62,10 @@ rever/
 # Allow the lib
 !/xonsh/lib
 
+
+# Allow tests/bin
+!/tests/bin
+
 # elm
 xonsh/webconfig/elm-stuff/
 xonsh/webconfig/js/app.js

.travis.yml

@@ -49,5 +49,5 @@ script:
       cd ..;
       doctr deploy --deploy-repo xonsh/xonsh-docs .;
     else
-      xonsh run-tests.xsh --timeout=10;
+      xonsh run-tests.xsh;
     fi

CONTRIBUTING.rst

@@ -20,7 +20,10 @@ Making Your First Change
 ========================
 
 First, install xonsh from source and open a xonsh shell in your favorite
-terminal application. See installation instructions for details.
+terminal application. See installation instructions for details, but it
+is recommended to do an 'editable' install via `pip'
+
+        $ pip install -e .
         
 Next, make a trivial change (e.g. ``print("hello!")`` in ``main.py``).
 
@@ -108,8 +111,7 @@ is open to interpretation.
 * Simple functions should have simple docstrings.
 * Lines should be at most 80 characters long. The 72 and 79 character
   recommendations from PEP8 are not required here.
-* All Python code should be compliant with Python 3.5+.  At some
-  unforeseen date in the future, Python 2.7 support *may* be supported.
+* All Python code should be compliant with Python 3.5+.  
 * Tests should be written with `pytest <https://docs.pytest.org/>`_ using a procedural style. Do not use
   unittest directly or write tests in an object-oriented style.
 * Test generators make more dots and the dots must flow!

docs/linux.rst

@@ -28,7 +28,7 @@ the following from the source directory,
 
 .. code-block:: console
 
-    $ python setup.py install
+    $ pip install .
 
 
 Debian/Ubuntu users can install xonsh from the repository with:

docs/osx.rst

@@ -41,7 +41,7 @@ the following from the source directory,
 
 .. code-block:: console
 
-    $ python3 setup.py install
+    $ pip3 install xonsh
 
 Extras for macOS
 ==================

docs/windows.rst

@@ -50,7 +50,7 @@ Now install xonsh:
 .. code-block:: bat
 
    > cd xonsh-master
-   > python setup.py install
+   > pip install .
 
 Next, run xonsh:
 

news/is_3661.rst

@@ -0,0 +1,29 @@
+**Added:**
+
+* <news item>
+
+**Changed:**
+
+* custom startup scripts replaced by setup.py -generated (console) entrypoint scripts for both xonsh and xonsh-cat.
+  This means xonsh.bat and xonsh-cat.bat are replaced on Windows by xonsh.exe and xonsh-cat.exe, respectively.
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* Avoid startup error on Windows when py.exe chooses wrong python interpreter to run xonsh.
+  When multiple interpreters are in PATH, 'py' will choose the first one (usually in the virtual environment),
+  but 'py -3' finds the system-wide one, apparently by design.
+
+* For xonsh-cat, avoid parsing and processing first (0'th) argument when invoked directly from OS shell.  
+
+
+**Security:**
+
+* <news item>

news/vend_prompt_toolkit.rst

@@ -0,0 +1,23 @@
+**Added:**
+
+* xonsh now comes with a bulitin version of prompt-toolkit (3.0.5) which will be used as fall back if prompt_toolkit is not installed.
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

pyproject.toml

@@ -29,6 +29,8 @@ exclude = '''
      |  \.vscode
      |  \.pytest_cache
      |  ply
+     |  vended_ptk/prompt_toolkit
+     |  vended_ptk/wcwidth
      )/
 )
 '''

scripts/xonsh

@@ -1,4 +0,0 @@
-#!/usr/bin/env -S python3 -u
-
-from xonsh.main import main
-main()

scripts/xonsh-cat

@@ -1,4 +0,0 @@
-#!/usr/bin/env -S python3 -u
-
-from xonsh.xoreutils.cat import cat_main as main
-main()

scripts/xonsh-cat.bat

@@ -1,14 +0,0 @@
-@echo off
-call :s_which py.exe
-if not "%_path%" == "" (
-  py -3 -m xonsh.xoreutils.cat %*
-) else (
-  python -m xonsh.xoreutils.cat %*
-)
-
-goto :eof
-
-:s_which
-  setlocal
-  endlocal & set _path=%~$PATH:1
-  goto :eof

scripts/xonsh.bat

@@ -1,14 +0,0 @@
-@echo off
-call :s_which py.exe
-if not "%_path%" == "" (
-  py -3 -m xonsh %*
-) else (
-  python -m xonsh %*
-)
-
-goto :eof
-
-:s_which
-  setlocal
-  endlocal & set _path=%~$PATH:1
-  goto :eof

scripts/xonsh.ps1

@@ -0,0 +1 @@
+python -m xonsh $args

setup.cfg

@@ -8,6 +8,8 @@ exclude =
     __amalgam__.py,
     tests/,
     docs/,
+    */vended_ptk/prompt_toolkit/,
+    */vended_ptk/wcwidth/,
     */ply/,
     parser*_table.py,
     build/,
@@ -16,6 +18,7 @@ exclude =
     .vscode/,
     feedstock,
     rever,
+    .venv,
     # remove later
     pygments_cache.py
 # lint nits that are acceptable in Xonsh project:

setup.py

@@ -196,7 +196,10 @@ def restore_version():
 
 
 class xinstall(install):
-    """Xonsh specialization of setuptools install class."""
+    """Xonsh specialization of setuptools install class.
+    For production, let setuptools generate the 
+    startup script, e.g: `pip installl .' rather than
+    relying on 'python setup.py install'."""
 
     def run(self):
         clean_tables()
@@ -323,12 +326,6 @@ def main():
     with open(os.path.join(os.path.dirname(__file__), "README.rst"), "r") as f:
         readme = f.read()
     scripts = ["scripts/xon.sh"]
-    if sys.platform == "win32":
-        scripts.append("scripts/xonsh.bat")
-        scripts.append("scripts/xonsh-cat.bat")
-    else:
-        scripts.append("scripts/xonsh")
-        scripts.append("scripts/xonsh-cat")
     skw = dict(
         name="xonsh",
         description="Python-powered, cross-platform, Unix-gazing shell",
@@ -344,6 +341,8 @@ def main():
         packages=[
             "xonsh",
             "xonsh.ply.ply",
+            "xonsh.vended_ptk.prompt_toolkit",
+            "xonsh.vended_ptk.wcwidth",
             "xonsh.ptk_shell",
             "xonsh.ptk2",
             "xonsh.parsers",
@@ -363,6 +362,7 @@ def main():
         },
         package_data={
             "xonsh": ["*.json", "*.githash"],
+            "xonsh.vended_ptk": ["LICENSE-prompt-toolkit","LICENSE-wcwidth"],
             "xontrib": ["*.xsh"],
             "xonsh.lib": ["*.xsh"],
             "xonsh.webconfig": [
@@ -370,27 +370,24 @@ def main():
                 "js/app.min.js",
                 "js/bootstrap.min.css",
                 "js/LICENSE-bootstrap",
-            ]
+            ],
         },
         cmdclass=cmdclass,
         scripts=scripts,
     )
-    # WARNING!!! Do not use setuptools 'console_scripts'
-    # It validates the dependencies (of which we have none) every time the
-    # 'xonsh' command is run. This validation adds ~0.2 sec. to the startup
-    # time of xonsh - for every single xonsh run.  This prevents us from
-    # reaching the goal of a startup time of < 0.1 sec.  So never ever write
-    # the following:
-    #
-    #     'console_scripts': ['xonsh = xonsh.main:main'],
-    #
-    # END WARNING
+    # We used to avoid setuptools 'console_scripts' due to startup performance
+    # concerns which have since been resolved, so long as install is done
+    # via `pip install .` and not `python setup.py install`.
     skw["entry_points"] = {
         "pygments.lexers": [
             "xonsh = xonsh.pyghooks:XonshLexer",
             "xonshcon = xonsh.pyghooks:XonshConsoleLexer",
         ],
         "pytest11": ["xonsh = xonsh.pytest_plugin"],
+        "console_scripts": [
+            "xonsh = xonsh.main:main",
+            "xonsh-cat = xonsh.xoreutils.cat:cat_main",
+        ],
     }
     skw["cmdclass"]["develop"] = xdevelop
     skw["extras_require"] = {

tests/bin/cat.bat

@@ -1,15 +1 @@
-@echo on
-call :s_which py.exe
-rem note that %~dp0 is dir of this batch script
-if not "%_path%" == "" (
-  py -3 %~dp0cat %*
-) else (
-  python %~dp0cat %*
-)
-
-goto :eof
-
-:s_which
-  setlocal
-  endlocal & set _path=%~$PATH:1
-  goto :eof
+@python %~dp0cat %*

tests/bin/pwd.bat

@@ -1,15 +1 @@
-@echo on
-call :s_which py.exe
-rem note that %~dp0 is dir of this batch script
-if not "%_path%" == "" (
-  py -3 %~dp0pwd %*
-) else (
-  python %~dp0pwd %*
-)
-
-goto :eof
-
-:s_which
-  setlocal
-  endlocal & set _path=%~$PATH:1
-  goto :eof
+@python %~dp0pwd %*

tests/bin/wc.bat

@@ -1,15 +1 @@
-@echo on
-call :s_which py.exe
-rem note that %~dp0 is dir of this batch script
-if not "%_path%" == "" (
-  py -3 %~dp0wc %*
-) else (
-  python %~dp0wc %*
-)
-
-goto :eof
-
-:s_which
-  setlocal
-  endlocal & set _path=%~$PATH:1
-  goto :eof
+@python %~dp0wc %*

tests/test_integrations.py

@@ -1,12 +1,13 @@
+"""Tests involving running Xonsh in subproc.
+This requires Xonsh installed in venv or otherwise available on PATH
+"""
 import os
-import sys
 import shutil
 import tempfile
 import subprocess as sp
 
 import pytest
 
-import xonsh
 from xonsh.lib.os import indir
 
 from tools import (
@@ -19,50 +20,34 @@ from tools import (
 )
 
 
-XONSH_PREFIX = xonsh.__file__
-if "site-packages" in XONSH_PREFIX:
-    # must be installed version of xonsh
-    num_up = 5
-else:
-    # must be in source dir
-    num_up = 2
-for i in range(num_up):
-    XONSH_PREFIX = os.path.dirname(XONSH_PREFIX)
 PATH = (
-    os.path.join(os.path.dirname(__file__), "bin")
-    + os.pathsep
-    + os.path.join(XONSH_PREFIX, "bin")
-    + os.pathsep
-    + os.path.join(XONSH_PREFIX, "Scripts")
-    + os.pathsep
-    + os.path.join(XONSH_PREFIX, "scripts")
-    + os.pathsep
-    + os.path.dirname(sys.executable)
+    os.path.join(os.path.abspath(os.path.dirname(__file__)), "bin")
     + os.pathsep
     + os.environ["PATH"]
 )
 
 
 skip_if_no_xonsh = pytest.mark.skipif(
-    shutil.which("xonsh", path=PATH) is None, reason="xonsh not on PATH"
+    shutil.which("xonsh") is None, reason="xonsh not on PATH"
 )
 skip_if_no_make = pytest.mark.skipif(
-    shutil.which("make", path=PATH) is None, reason="make command not on PATH"
+    shutil.which("make") is None, reason="make command not on PATH"
 )
 skip_if_no_sleep = pytest.mark.skipif(
-    shutil.which("sleep", path=PATH) is None, reason="sleep command not on PATH"
+    shutil.which("sleep") is None, reason="sleep command not on PATH"
 )
 
 
-def run_xonsh(cmd, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.STDOUT, single_command=False):
+def run_xonsh(
+    cmd, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.STDOUT, single_command=False
+):
     env = dict(os.environ)
     env["PATH"] = PATH
     env["XONSH_DEBUG"] = "0"  # was "1"
     env["XONSH_SHOW_TRACEBACK"] = "1"
     env["RAISE_SUBPROC_ERROR"] = "0"
     env["PROMPT"] = ""
-    xonsh = "xonsh.bat" if ON_WINDOWS else "xon.sh"
-    xonsh = shutil.which(xonsh, path=PATH)
+    xonsh = shutil.which("xonsh", path=PATH)
     if single_command:
         args = [xonsh, "--no-rc", "-c", cmd]
         input = None
@@ -80,7 +65,7 @@ def run_xonsh(cmd, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.STDOUT, single_comma
     )
 
     try:
-        out, err = proc.communicate(input=input, timeout=10)
+        out, err = proc.communicate(input=input, timeout=20)
     except sp.TimeoutExpired:
         proc.kill()
         raise
@@ -227,9 +212,9 @@ g
 with open('tttt', 'w') as fp:
     fp.write("Wow mom!\\n")
 
-![cat tttt | wc]
+![python tests/bin/cat tttt | python tests/bin/wc]
 """,
-        "      1       2      10\n" if ON_WINDOWS else " 1  2 9 <stdin>\n",
+        " 1  2 10 <stdin>\n" if ON_WINDOWS else " 1  2 9 <stdin>\n",
         0,
     ),
     # test double  piping 'real' command
@@ -238,9 +223,9 @@ with open('tttt', 'w') as fp:
 with open('tttt', 'w') as fp:
     fp.write("Wow mom!\\n")
 
-![cat tttt | wc | wc]
+![python tests/bin/cat tttt | python tests/bin/wc | python tests/bin/wc]
 """,
-        "      1       3      24\n" if ON_WINDOWS else " 1  4 16 <stdin>\n",
+        " 1  4 18 <stdin>\n" if ON_WINDOWS else " 1  4 16 <stdin>\n",
         0,
     ),
     # test unthreadable alias (which should trigger a ProcPoxy call)
@@ -352,7 +337,7 @@ echo foo_@$(echo spam sausage)_bar
     (
         """
 echo Just the place for a snark. >tttt
-cat tttt
+python tests/bin/cat tttt
 """,
         "Just the place for a snark.\n",
         0,
@@ -472,6 +457,7 @@ a
 ]
 
 
+@skip_if_no_xonsh
 @pytest.mark.parametrize("case", ALL_PLATFORMS)
 def test_script(case):
     script, exp_out, exp_rtn = case
@@ -496,6 +482,7 @@ f o>e
 ]
 
 
+@skip_if_no_xonsh
 @pytest.mark.parametrize("case", ALL_PLATFORMS_STDERR)
 def test_script_stderr(case):
     script, exp_err, exp_rtn = case
@@ -504,6 +491,7 @@ def test_script_stderr(case):
     assert exp_rtn == rtn
 
 
+@skip_if_no_xonsh
 @skip_if_on_windows
 @pytest.mark.parametrize(
     "cmd, fmt, exp",
@@ -517,6 +505,7 @@ def test_single_command_no_windows(cmd, fmt, exp):
     check_run_xonsh(cmd, fmt, exp)
 
 
+@skip_if_no_xonsh
 def test_eof_syntax_error():
     """Ensures syntax errors for EOF appear on last line."""
     script = "x = 1\na = (1, 0\n"
@@ -525,6 +514,7 @@ def test_eof_syntax_error():
     assert ":2:0: EOF in multi-line statement" in err
 
 
+@skip_if_no_xonsh
 def test_open_quote_syntax_error():
     script = (
         "#!/usr/bin/env xonsh\n\n"
@@ -544,21 +534,25 @@ _bad_case = pytest.mark.skipif(
 )
 
 
+@skip_if_no_xonsh
 @_bad_case
 def test_printfile():
     check_run_xonsh("printfile.xsh", None, "printfile.xsh\n")
 
 
+@skip_if_no_xonsh
 @_bad_case
 def test_printname():
     check_run_xonsh("printfile.xsh", None, "printfile.xsh\n")
 
 
+@skip_if_no_xonsh
 @_bad_case
 def test_sourcefile():
     check_run_xonsh("printfile.xsh", None, "printfile.xsh\n")
 
 
+@skip_if_no_xonsh
 @_bad_case
 @pytest.mark.parametrize(
     "cmd, fmt, exp",
@@ -591,6 +585,7 @@ def test_subshells(cmd, fmt, exp):
     check_run_xonsh(cmd, fmt, exp)
 
 
+@skip_if_no_xonsh
 @skip_if_on_windows
 @pytest.mark.parametrize("cmd, exp", [("pwd", lambda: os.getcwd() + "\n")])
 def test_redirect_out_to_file(cmd, exp, tmpdir):
@@ -628,15 +623,17 @@ def test_xonsh_no_close_fds():
         assert "warning" not in out
 
 
+@skip_if_no_xonsh
 @pytest.mark.parametrize(
     "cmd, fmt, exp",
-    [("ls | wc", lambda x: x > "", True),],  # noqa E231 (black removes space)
+    [("cat tttt | wc", lambda x: x > "", True),],  # noqa E231 (black removes space)
 )
 def test_pipe_between_subprocs(cmd, fmt, exp):
     "verify pipe between subprocesses doesn't throw an exception"
     check_run_xonsh(cmd, fmt, exp)
 
 
+@skip_if_no_xonsh
 @skip_if_on_windows
 def test_negative_exit_codes_fail():
     # see issue 3309
@@ -646,6 +643,7 @@ def test_negative_exit_codes_fail():
     assert "OK" != err
 
 
+@skip_if_no_xonsh
 @pytest.mark.parametrize(
     "cmd, exp",
     [
@@ -670,6 +668,7 @@ aliases['echo'] = _echo
 
 
 # issue 3402
+@skip_if_no_xonsh
 @skip_if_on_windows
 @pytest.mark.parametrize(
     "cmd, exp_rtn",
@@ -685,6 +684,7 @@ def test_single_command_return_code(cmd, exp_rtn):
     assert rtn == exp_rtn
 
 
+@skip_if_no_xonsh
 @skip_if_on_msys
 @skip_if_on_windows
 @skip_if_on_darwin

tests/test_ptk_shell.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 """Test initialization of prompt_toolkit shell"""
 
+import sys
 import pytest
 
 from xonsh.platform import minimum_required_ptk_version
@@ -11,21 +12,21 @@ from xonsh.shell import Shell
 
 
 @pytest.mark.parametrize(
-    "ptk_ver, ini_shell_type, exp_shell_type, warn_snip",
+    "ptk_ver, ini_shell_type, exp_shell_type, warn_snip, using_vended_ptk",
     [
-        (None, "prompt_toolkit", "readline", "prompt_toolkit is not available"),
-        ((0, 5, 7), "prompt_toolkit", "readline", "is not supported"),
-        ((1, 0, 0), "prompt_toolkit", "readline", "is not supported"),
-        ((2, 0, 0), "prompt_toolkit", "prompt_toolkit", None),
-        ((2, 0, 0), "best", "prompt_toolkit", None),
-        ((2, 0, 0), "readline", "readline", None),
-        ((3, 0, 0), "prompt_toolkit", "prompt_toolkit", None),
-        ((3, 0, 0), "best", "prompt_toolkit", None),
-        ((3, 0, 0), "readline", "readline", None),
-        ((4, 0, 0), "prompt_toolkit", "prompt_toolkit", None),
+        (None, "prompt_toolkit", "prompt_toolkit", None, True),
+        ((0, 5, 7), "prompt_toolkit", "prompt_toolkit", "is not supported", True),
+        ((1, 0, 0), "prompt_toolkit", "prompt_toolkit", "is not supported", True),
+        ((2, 0, 0), "prompt_toolkit", "prompt_toolkit", None, False),
+        ((2, 0, 0), "best", "prompt_toolkit", None, False),
+        ((2, 0, 0), "readline", "readline", None, False),
+        ((3, 0, 0), "prompt_toolkit", "prompt_toolkit", None, False),
+        ((3, 0, 0), "best", "prompt_toolkit", None, False),
+        ((3, 0, 0), "readline", "readline", None, False),
+        ((4, 0, 0), "prompt_toolkit", "prompt_toolkit", None, False),
     ],
 )
-def test_prompt_toolkit_version_checks(ptk_ver, ini_shell_type, exp_shell_type, warn_snip, monkeypatch, xonsh_builtins):
+def test_prompt_toolkit_version_checks(ptk_ver, ini_shell_type, exp_shell_type, warn_snip, using_vended_ptk, monkeypatch, xonsh_builtins):
 
     mocked_warn = ""
 
@@ -36,7 +37,7 @@ def test_prompt_toolkit_version_checks(ptk_ver, ini_shell_type, exp_shell_type,
 
     def mock_ptk_above_min_supported():
         nonlocal ptk_ver
-        return ptk_ver and (ptk_ver[:2] >= minimum_required_ptk_version)
+        return ptk_ver and (ptk_ver[:3] >= minimum_required_ptk_version)
 
     def mock_has_prompt_toolkit():
         nonlocal ptk_ver
@@ -45,11 +46,21 @@ def test_prompt_toolkit_version_checks(ptk_ver, ini_shell_type, exp_shell_type,
     monkeypatch.setattr("xonsh.shell.warnings.warn", mock_warning)      # hardwon: patch the caller!
     monkeypatch.setattr("xonsh.shell.ptk_above_min_supported", mock_ptk_above_min_supported)    # have to patch both callers
     monkeypatch.setattr("xonsh.platform.ptk_above_min_supported", mock_ptk_above_min_supported)
-    monkeypatch.setattr("xonsh.shell.has_prompt_toolkit", mock_has_prompt_toolkit)
     monkeypatch.setattr("xonsh.platform.has_prompt_toolkit", mock_has_prompt_toolkit)
     
+    old_syspath = sys.path.copy()
+
     act_shell_type = Shell.choose_shell_type(ini_shell_type, {})
 
+    if using_vended_ptk:
+        # ensure PTK has been unloaded and the vended version added to sys.path
+        assert len(old_syspath) < len(sys.path)
+    else:
+        assert len(old_syspath) == len(sys.path)
+
+    sys.path = old_syspath
+
+
     assert act_shell_type == exp_shell_type
 
     if warn_snip:

xonsh/platform.py

@@ -143,6 +143,13 @@ def pygments_version_info():
         return None
 
 
+def use_vended_prompt_toolkit():
+    """ Unload any prompt_toolkit libraries and add vended version to sys.path """
+    for mod in (mod for mod in list(sys.modules) if mod.startswith("prompt_toolkit")):
+        del sys.modules[mod]
+    sys.path.insert(0, str(pathlib.Path(__file__).with_name("vended_ptk").resolve()))
+
+
 @functools.lru_cache(1)
 def has_prompt_toolkit():
     """Tests if the `prompt_toolkit` is available."""
@@ -170,7 +177,7 @@ def ptk_version_info():
         return None
 
 
-minimum_required_ptk_version = (2, 0)
+minimum_required_ptk_version = (2, 0, 0)
 """Minimum version of prompt-toolkit supported by Xonsh"""
 
 
@@ -201,10 +208,8 @@ def ptk_below_max_supported():
 def best_shell_type():
     if builtins.__xonsh__.env.get("TERM", "") == "dumb":
         return "dumb"
-    elif ON_WINDOWS or has_prompt_toolkit():
-        return "prompt_toolkit"
     else:
-        return "readline"
+        return "prompt_toolkit"
 
 
 @functools.lru_cache(1)

xonsh/shell.py

@@ -9,8 +9,9 @@ import warnings
 
 from xonsh.platform import (
     best_shell_type,
-    has_prompt_toolkit,
     ptk_above_min_supported,
+    use_vended_prompt_toolkit,
+    has_prompt_toolkit,
     minimum_required_ptk_version,
 )
 from xonsh.tools import XonshError, print_exception
@@ -160,19 +161,15 @@ class Shell(object):
             shell_type = random.choice(("readline", "prompt_toolkit"))
         if shell_type == "prompt_toolkit":
             if not has_prompt_toolkit():
-                warnings.warn(
-                    "prompt_toolkit is not available, using " "readline instead."
-                )
-                shell_type = "readline"
+                use_vended_prompt_toolkit()
             elif not ptk_above_min_supported():
                 warnings.warn(
-                    "prompt-toolkit version < v{}.{}.0 is not ".format(
+                    "Installed prompt-toolkit version < v{}.{}.{} is not ".format(
                         *minimum_required_ptk_version
                     )
-                    + "supported. Please update prompt-toolkit. Using "
-                    + "readline instead."
+                    + "supported. Falling back to the builtin prompt-toolkit."
                 )
-                shell_type = "readline"
+                use_vended_prompt_toolkit()
             if init_shell_type in ("ptk1", "prompt_toolkit1"):
                 warnings.warn(
                     "$SHELL_TYPE='{}' now deprecated, please update your run control file'".format(

xonsh/vended_ptk/LICENSE-prompt-toolkit

@@ -0,0 +1,27 @@
+Copyright (c) 2014, Jonathan Slenders
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+* Neither the name of the {organization} nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

xonsh/vended_ptk/LICENSE-wcwidth

@@ -0,0 +1,27 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Jeff Quast <contact@jeffquast.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+Markus Kuhn -- 2007-05-26 (Unicode 5.0)
+
+Permission to use, copy, modify, and distribute this software
+for any purpose and without fee is hereby granted. The author
+disclaims all warranties with regard to this software.

xonsh/vended_ptk/prompt_toolkit/__init__.py

@@ -0,0 +1,37 @@
+"""
+prompt_toolkit
+==============
+
+Author: Jonathan Slenders
+
+Description: prompt_toolkit is a Library for building powerful interactive
+             command lines in Python.  It can be a replacement for GNU
+             Readline, but it can be much more than that.
+
+See the examples directory to learn about the usage.
+
+Probably, to get started, you might also want to have a look at
+`prompt_toolkit.shortcuts.prompt`.
+"""
+from .application import Application
+from .formatted_text import ANSI, HTML
+from .shortcuts import PromptSession, print_formatted_text, prompt
+
+# Don't forget to update in `docs/conf.py`!
+__version__ = "3.0.5"
+
+# Version tuple.
+VERSION = tuple(__version__.split("."))
+
+
+__all__ = [
+    # Application.
+    "Application",
+    # Shortcuts.
+    "prompt",
+    "PromptSession",
+    "print_formatted_text",
+    # Formatted text.
+    "HTML",
+    "ANSI",
+]

xonsh/vended_ptk/prompt_toolkit/application/__init__.py

@@ -0,0 +1,26 @@
+from .application import Application
+from .current import (
+    create_app_session,
+    get_app,
+    get_app_or_none,
+    get_app_session,
+    set_app,
+)
+from .dummy import DummyApplication
+from .run_in_terminal import in_terminal, run_in_terminal
+
+__all__ = [
+    # Application.
+    "Application",
+    # Current.
+    "get_app_session",
+    "create_app_session",
+    "get_app",
+    "get_app_or_none",
+    "set_app",
+    # Dummy.
+    "DummyApplication",
+    # Run_in_terminal
+    "in_terminal",
+    "run_in_terminal",
+]

xonsh/vended_ptk/prompt_toolkit/application/current.py

@@ -0,0 +1,165 @@
+import sys
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Any, Generator, Optional
+
+try:
+    from contextvars import ContextVar
+except ImportError:
+    from prompt_toolkit.eventloop.dummy_contextvars import ContextVar  # type: ignore
+
+if TYPE_CHECKING:
+    from .application import Application
+    from prompt_toolkit.input.defaults import Input
+    from prompt_toolkit.output.defaults import Output
+
+__all__ = [
+    "get_app_session",
+    "get_app",
+    "get_app_or_none",
+    "set_app",
+    "create_app_session",
+]
+
+
+class AppSession:
+    """
+    An AppSession is an interactive session, usually connected to one terminal.
+    Within one such session, interaction with many applications can happen, one
+    after the other.
+
+    The input/output device is not supposed to change during one session.
+
+    :param input: Use this as a default input for all applications
+        running in this session, unless an input is passed to the `Application`
+        explicitely.
+    :param output: Use this as a default output.
+    """
+
+    def __init__(
+        self, input: Optional["Input"] = None, output: Optional["Output"] = None
+    ) -> None:
+
+        self._input = input
+        self._output = output
+
+        # The application will be set dynamically by the `set_app` context
+        # manager. This is called in the application itself.
+        self.app: Optional["Application[Any]"] = None
+
+    def __repr__(self) -> str:
+        return "AppSession(app=%r)" % (self.app,)
+
+    @property
+    def input(self) -> "Input":
+        if self._input is None:
+            from prompt_toolkit.input.defaults import create_input
+
+            self._input = create_input()
+        return self._input
+
+    @property
+    def output(self) -> "Output":
+        if self._output is None:
+            from prompt_toolkit.output.defaults import create_output
+
+            self._output = create_output()
+        return self._output
+
+
+_current_app_session: ContextVar["AppSession"] = ContextVar(
+    "_current_app_session", default=AppSession()
+)
+
+
+def get_app_session() -> AppSession:
+    return _current_app_session.get()
+
+
+def get_app() -> "Application[Any]":
+    """
+    Get the current active (running) Application.
+    An :class:`.Application` is active during the
+    :meth:`.Application.run_async` call.
+
+    We assume that there can only be one :class:`.Application` active at the
+    same time. There is only one terminal window, with only one stdin and
+    stdout. This makes the code significantly easier than passing around the
+    :class:`.Application` everywhere.
+
+    If no :class:`.Application` is running, then return by default a
+    :class:`.DummyApplication`. For practical reasons, we prefer to not raise
+    an exception. This way, we don't have to check all over the place whether
+    an actual `Application` was returned.
+
+    (For applications like pymux where we can have more than one `Application`,
+    we'll use a work-around to handle that.)
+    """
+    session = _current_app_session.get()
+    if session.app is not None:
+        return session.app
+
+    from .dummy import DummyApplication
+
+    return DummyApplication()
+
+
+def get_app_or_none() -> Optional["Application[Any]"]:
+    """
+    Get the current active (running) Application, or return `None` if no
+    application is running.
+    """
+    session = _current_app_session.get()
+    return session.app
+
+
+@contextmanager
+def set_app(app: "Application[Any]") -> Generator[None, None, None]:
+    """
+    Context manager that sets the given :class:`.Application` active in an
+    `AppSession`.
+
+    This should only be called by the `Application` itself.
+    The application will automatically be active while its running. If you want
+    the application to be active in other threads/coroutines, where that's not
+    the case, use `contextvars.copy_context()`, or use `Application.context` to
+    run it in the appropriate context.
+    """
+    session = _current_app_session.get()
+
+    previous_app = session.app
+    session.app = app
+    try:
+        yield
+    finally:
+        session.app = previous_app
+
+
+@contextmanager
+def create_app_session(
+    input: Optional["Input"] = None, output: Optional["Output"] = None
+) -> Generator[AppSession, None, None]:
+    """
+    Create a separate AppSession.
+
+    This is useful if there can be multiple individual `AppSession`s going on.
+    Like in the case of an Telnet/SSH server. This functionality uses
+    contextvars and requires at least Python 3.7.
+    """
+    if sys.version_info <= (3, 6):
+        raise RuntimeError("Application sessions require Python 3.7.")
+
+    # If no input/output is specified, fall back to the current input/output,
+    # whatever that is.
+    if input is None:
+        input = get_app_session().input
+    if output is None:
+        output = get_app_session().output
+
+    # Create new `AppSession` and activate.
+    session = AppSession(input=input, output=output)
+
+    token = _current_app_session.set(session)
+    try:
+        yield session
+    finally:
+        _current_app_session.reset(token)

xonsh/vended_ptk/prompt_toolkit/application/dummy.py

@@ -0,0 +1,47 @@
+from typing import Callable, Optional
+
+from prompt_toolkit.formatted_text import AnyFormattedText
+from prompt_toolkit.input import DummyInput
+from prompt_toolkit.output import DummyOutput
+
+from .application import Application
+
+__all__ = [
+    "DummyApplication",
+]
+
+
+class DummyApplication(Application[None]):
+    """
+    When no :class:`.Application` is running,
+    :func:`.get_app` will run an instance of this :class:`.DummyApplication` instead.
+    """
+
+    def __init__(self) -> None:
+        super().__init__(output=DummyOutput(), input=DummyInput())
+
+    def run(
+        self,
+        pre_run: Optional[Callable[[], None]] = None,
+        set_exception_handler: bool = True,
+    ) -> None:
+        raise NotImplementedError("A DummyApplication is not supposed to run.")
+
+    async def run_async(
+        self,
+        pre_run: Optional[Callable[[], None]] = None,
+        set_exception_handler: bool = True,
+    ) -> None:
+        raise NotImplementedError("A DummyApplication is not supposed to run.")
+
+    async def run_system_command(
+        self,
+        command: str,
+        wait_for_enter: bool = True,
+        display_before_text: AnyFormattedText = "",
+        wait_text: str = "",
+    ) -> None:
+        raise NotImplementedError
+
+    def suspend_to_background(self, suspend_group: bool = True) -> None:
+        raise NotImplementedError

xonsh/vended_ptk/prompt_toolkit/application/run_in_terminal.py

@@ -0,0 +1,116 @@
+"""
+Tools for running functions on the terminal above the current application or prompt.
+"""
+from asyncio import Future, ensure_future
+from typing import AsyncGenerator, Awaitable, Callable, TypeVar
+
+from prompt_toolkit.eventloop import run_in_executor_with_context
+
+from .current import get_app_or_none
+
+try:
+    from contextlib import asynccontextmanager  # type: ignore
+except ImportError:
+    from prompt_toolkit.eventloop.async_context_manager import asynccontextmanager
+
+
+__all__ = [
+    "run_in_terminal",
+    "in_terminal",
+]
+
+_T = TypeVar("_T")
+
+
+def run_in_terminal(
+    func: Callable[[], _T], render_cli_done: bool = False, in_executor: bool = False
+) -> Awaitable[_T]:
+    """
+    Run function on the terminal above the current application or prompt.
+
+    What this does is first hiding the prompt, then running this callable
+    (which can safely output to the terminal), and then again rendering the
+    prompt which causes the output of this function to scroll above the
+    prompt.
+
+    ``func`` is supposed to be a synchronous function. If you need an
+    asynchronous version of this function, use the ``in_terminal`` context
+    manager directly.
+
+    :param func: The callable to execute.
+    :param render_cli_done: When True, render the interface in the
+            'Done' state first, then execute the function. If False,
+            erase the interface first.
+    :param in_executor: When True, run in executor. (Use this for long
+        blocking functions, when you don't want to block the event loop.)
+
+    :returns: A `Future`.
+    """
+
+    async def run() -> _T:
+        async with in_terminal(render_cli_done=render_cli_done):
+            if in_executor:
+                return await run_in_executor_with_context(func)
+            else:
+                return func()
+
+    return ensure_future(run())
+
+
+@asynccontextmanager
+async def in_terminal(render_cli_done: bool = False) -> AsyncGenerator[None, None]:
+    """
+    Asynchronous context manager that suspends the current application and runs
+    the body in the terminal.
+
+    .. code::
+
+        async def f():
+            async with in_terminal():
+                call_some_function()
+                await call_some_async_function()
+    """
+    app = get_app_or_none()
+    if app is None or not app._is_running:
+        yield
+        return
+
+    # When a previous `run_in_terminal` call was in progress. Wait for that
+    # to finish, before starting this one. Chain to previous call.
+    previous_run_in_terminal_f = app._running_in_terminal_f
+    new_run_in_terminal_f: Future[None] = Future()
+    app._running_in_terminal_f = new_run_in_terminal_f
+
+    # Wait for the previous `run_in_terminal` to finish.
+    if previous_run_in_terminal_f is not None:
+        await previous_run_in_terminal_f
+
+    # Wait for all CPRs to arrive. We don't want to detach the input until
+    # all cursor position responses have been arrived. Otherwise, the tty
+    # will echo its input and can show stuff like ^[[39;1R.
+    if app.input.responds_to_cpr:
+        await app.renderer.wait_for_cpr_responses()
+
+    # Draw interface in 'done' state, or erase.
+    if render_cli_done:
+        app._redraw(render_as_done=True)
+    else:
+        app.renderer.erase()
+
+    # Disable rendering.
+    app._running_in_terminal = True
+
+    # Detach input.
+    try:
+        with app.input.detach():
+            with app.input.cooked_mode():
+                yield
+    finally:
+        # Redraw interface again.
+        try:
+            app._running_in_terminal = False
+            app.renderer.reset()
+            app._request_absolute_cursor_position()
+            app._redraw()
+        finally:
+            new_run_in_terminal_f.set_result(None)

xonsh/vended_ptk/prompt_toolkit/auto_suggest.py

@@ -0,0 +1,187 @@
+"""
+`Fish-style <http://fishshell.com/>`_  like auto-suggestion.
+
+While a user types input in a certain buffer, suggestions are generated
+(asynchronously.) Usually, they are displayed after the input. When the cursor
+presses the right arrow and the cursor is at the end of the input, the
+suggestion will be inserted.
+
+If you want the auto suggestions to be asynchronous (in a background thread),
+because they take too much time, and could potentially block the event loop,
+then wrap the :class:`.AutoSuggest` instance into a
+:class:`.ThreadedAutoSuggest`.
+"""
+from abc import ABCMeta, abstractmethod
+from typing import TYPE_CHECKING, Callable, Optional, Union
+
+from prompt_toolkit.eventloop import run_in_executor_with_context
+
+from .document import Document
+from .filters import Filter, to_filter
+
+if TYPE_CHECKING:
+    from .buffer import Buffer
+
+__all__ = [
+    "Suggestion",
+    "AutoSuggest",
+    "ThreadedAutoSuggest",
+    "DummyAutoSuggest",
+    "AutoSuggestFromHistory",
+    "ConditionalAutoSuggest",
+    "DynamicAutoSuggest",
+]
+
+
+class Suggestion:
+    """
+    Suggestion returned by an auto-suggest algorithm.
+
+    :param text: The suggestion text.
+    """
+
+    def __init__(self, text: str) -> None:
+        self.text = text
+
+    def __repr__(self) -> str:
+        return "Suggestion(%s)" % self.text
+
+
+class AutoSuggest(metaclass=ABCMeta):
+    """
+    Base class for auto suggestion implementations.
+    """
+
+    @abstractmethod
+    def get_suggestion(
+        self, buffer: "Buffer", document: Document
+    ) -> Optional[Suggestion]:
+        """
+        Return `None` or a :class:`.Suggestion` instance.
+
+        We receive both :class:`~prompt_toolkit.buffer.Buffer` and
+        :class:`~prompt_toolkit.document.Document`. The reason is that auto
+        suggestions are retrieved asynchronously. (Like completions.) The
+        buffer text could be changed in the meantime, but ``document`` contains
+        the buffer document like it was at the start of the auto suggestion
+        call. So, from here, don't access ``buffer.text``, but use
+        ``document.text`` instead.
+
+        :param buffer: The :class:`~prompt_toolkit.buffer.Buffer` instance.
+        :param document: The :class:`~prompt_toolkit.document.Document` instance.
+        """
+
+    async def get_suggestion_async(
+        self, buff: "Buffer", document: Document
+    ) -> Optional[Suggestion]:
+        """
+        Return a :class:`.Future` which is set when the suggestions are ready.
+        This function can be overloaded in order to provide an asynchronous
+        implementation.
+        """
+        return self.get_suggestion(buff, document)
+
+
+class ThreadedAutoSuggest(AutoSuggest):
+    """
+    Wrapper that runs auto suggestions in a thread.
+    (Use this to prevent the user interface from becoming unresponsive if the
+    generation of suggestions takes too much time.)
+    """
+
+    def __init__(self, auto_suggest: AutoSuggest) -> None:
+        self.auto_suggest = auto_suggest
+
+    def get_suggestion(
+        self, buff: "Buffer", document: Document
+    ) -> Optional[Suggestion]:
+        return self.auto_suggest.get_suggestion(buff, document)
+
+    async def get_suggestion_async(
+        self, buff: "Buffer", document: Document
+    ) -> Optional[Suggestion]:
+        """
+        Run the `get_suggestion` function in a thread.
+        """
+
+        def run_get_suggestion_thread() -> Optional[Suggestion]:
+            return self.get_suggestion(buff, document)
+
+        return await run_in_executor_with_context(run_get_suggestion_thread)
+
+
+class DummyAutoSuggest(AutoSuggest):
+    """
+    AutoSuggest class that doesn't return any suggestion.
+    """
+
+    def get_suggestion(
+        self, buffer: "Buffer", document: Document
+    ) -> Optional[Suggestion]:
+        return None  # No suggestion
+
+
+class AutoSuggestFromHistory(AutoSuggest):
+    """
+    Give suggestions based on the lines in the history.
+    """
+
+    def get_suggestion(
+        self, buffer: "Buffer", document: Document
+    ) -> Optional[Suggestion]:
+        history = buffer.history
+
+        # Consider only the last line for the suggestion.
+        text = document.text.rsplit("\n", 1)[-1]
+
+        # Only create a suggestion when this is not an empty line.
+        if text.strip():
+            # Find first matching line in history.
+            for string in reversed(list(history.get_strings())):
+                for line in reversed(string.splitlines()):
+                    if line.startswith(text):
+                        return Suggestion(line[len(text) :])
+
+        return None
+
+
+class ConditionalAutoSuggest(AutoSuggest):
+    """
+    Auto suggest that can be turned on and of according to a certain condition.
+    """
+
+    def __init__(self, auto_suggest: AutoSuggest, filter: Union[bool, Filter]) -> None:
+
+        self.auto_suggest = auto_suggest
+        self.filter = to_filter(filter)
+
+    def get_suggestion(
+        self, buffer: "Buffer", document: Document
+    ) -> Optional[Suggestion]:
+        if self.filter():
+            return self.auto_suggest.get_suggestion(buffer, document)
+
+        return None
+
+
+class DynamicAutoSuggest(AutoSuggest):
+    """
+    Validator class that can dynamically returns any Validator.
+
+    :param get_validator: Callable that returns a :class:`.Validator` instance.
+    """
+
+    def __init__(self, get_auto_suggest: Callable[[], Optional[AutoSuggest]]) -> None:
+        self.get_auto_suggest = get_auto_suggest
+
+    def get_suggestion(
+        self, buff: "Buffer", document: Document
+    ) -> Optional[Suggestion]:
+        auto_suggest = self.get_auto_suggest() or DummyAutoSuggest()
+        return auto_suggest.get_suggestion(buff, document)
+
+    async def get_suggestion_async(
+        self, buff: "Buffer", document: Document
+    ) -> Optional[Suggestion]:
+        auto_suggest = self.get_auto_suggest() or DummyAutoSuggest()
+        return await auto_suggest.get_suggestion_async(buff, document)

xonsh/vended_ptk/prompt_toolkit/cache.py

@@ -0,0 +1,125 @@
+from collections import deque
+from functools import wraps
+from typing import Any, Callable, Deque, Dict, Generic, Hashable, Tuple, TypeVar, cast
+
+__all__ = [
+    "SimpleCache",
+    "FastDictCache",
+    "memoized",
+]
+
+_T = TypeVar("_T", bound=Hashable)
+_U = TypeVar("_U")
+
+
+class SimpleCache(Generic[_T, _U]):
+    """
+    Very simple cache that discards the oldest item when the cache size is
+    exceeded.
+
+    :param maxsize: Maximum size of the cache. (Don't make it too big.)
+    """
+
+    def __init__(self, maxsize: int = 8) -> None:
+        assert maxsize > 0
+
+        self._data: Dict[_T, _U] = {}
+        self._keys: Deque[_T] = deque()
+        self.maxsize: int = maxsize
+
+    def get(self, key: _T, getter_func: Callable[[], _U]) -> _U:
+        """
+        Get object from the cache.
+        If not found, call `getter_func` to resolve it, and put that on the top
+        of the cache instead.
+        """
+        # Look in cache first.
+        try:
+            return self._data[key]
+        except KeyError:
+            # Not found? Get it.
+            value = getter_func()
+            self._data[key] = value
+            self._keys.append(key)
+
+            # Remove the oldest key when the size is exceeded.
+            if len(self._data) > self.maxsize:
+                key_to_remove = self._keys.popleft()
+                if key_to_remove in self._data:
+                    del self._data[key_to_remove]
+
+            return value
+
+    def clear(self) -> None:
+        " Clear cache. "
+        self._data = {}
+        self._keys = deque()
+
+
+_K = TypeVar("_K", bound=Tuple)
+_V = TypeVar("_V")
+
+
+class FastDictCache(Dict[_K, _V]):
+    """
+    Fast, lightweight cache which keeps at most `size` items.
+    It will discard the oldest items in the cache first.
+
+    The cache is a dictionary, which doesn't keep track of access counts.
+    It is perfect to cache little immutable objects which are not expensive to
+    create, but where a dictionary lookup is still much faster than an object
+    instantiation.
+
+    :param get_value: Callable that's called in case of a missing key.
+    """
+
+    # NOTE: This cache is used to cache `prompt_toolkit.layout.screen.Char` and
+    #       `prompt_toolkit.Document`. Make sure to keep this really lightweight.
+    #       Accessing the cache should stay faster than instantiating new
+    #       objects.
+    #       (Dictionary lookups are really fast.)
+    #       SimpleCache is still required for cases where the cache key is not
+    #       the same as the arguments given to the function that creates the
+    #       value.)
+    def __init__(self, get_value: Callable[..., _V], size: int = 1000000) -> None:
+        assert size > 0
+
+        self._keys: Deque[_K] = deque()
+        self.get_value = get_value
+        self.size = size
+
+    def __missing__(self, key: _K) -> _V:
+        # Remove the oldest key when the size is exceeded.
+        if len(self) > self.size:
+            key_to_remove = self._keys.popleft()
+            if key_to_remove in self:
+                del self[key_to_remove]
+
+        result = self.get_value(*key)
+        self[key] = result
+        self._keys.append(key)
+        return result
+
+
+_F = TypeVar("_F", bound=Callable)
+
+
+def memoized(maxsize: int = 1024) -> Callable[[_F], _F]:
+    """
+    Memoization decorator for immutable classes and pure functions.
+    """
+
+    def decorator(obj: _F) -> _F:
+        cache: SimpleCache[Hashable, Any] = SimpleCache(maxsize=maxsize)
+
+        @wraps(obj)
+        def new_callable(*a: Any, **kw: Any) -> Any:
+            def create_new() -> Any:
+                return obj(*a, **kw)
+
+            key = (a, tuple(sorted(kw.items())))
+            return cache.get(key, create_new)
+
+        return cast(_F, new_callable)
+
+    return decorator

xonsh/vended_ptk/prompt_toolkit/clipboard/__init__.py

@@ -0,0 +1,15 @@
+from .base import Clipboard, ClipboardData, DummyClipboard, DynamicClipboard
+from .in_memory import InMemoryClipboard
+
+# We are not importing `PyperclipClipboard` here, because it would require the
+# `pyperclip` module to be present.
+
+# from .pyperclip import PyperclipClipboard
+
+__all__ = [
+    "Clipboard",
+    "ClipboardData",
+    "DummyClipboard",
+    "DynamicClipboard",
+    "InMemoryClipboard",
+]

xonsh/vended_ptk/prompt_toolkit/clipboard/base.py

@@ -0,0 +1,107 @@
+"""
+Clipboard for command line interface.
+"""
+from abc import ABCMeta, abstractmethod
+from typing import Callable, Optional
+
+from prompt_toolkit.selection import SelectionType
+
+__all__ = [
+    "Clipboard",
+    "ClipboardData",
+    "DummyClipboard",
+    "DynamicClipboard",
+]
+
+
+class ClipboardData:
+    """
+    Text on the clipboard.
+
+    :param text: string
+    :param type: :class:`~prompt_toolkit.selection.SelectionType`
+    """
+
+    def __init__(
+        self, text: str = "", type: SelectionType = SelectionType.CHARACTERS
+    ) -> None:
+
+        self.text = text
+        self.type = type
+
+
+class Clipboard(metaclass=ABCMeta):
+    """
+    Abstract baseclass for clipboards.
+    (An implementation can be in memory, it can share the X11 or Windows
+    keyboard, or can be persistent.)
+    """
+
+    @abstractmethod
+    def set_data(self, data: ClipboardData) -> None:
+        """
+        Set data to the clipboard.
+
+        :param data: :class:`~.ClipboardData` instance.
+        """
+
+    def set_text(self, text: str) -> None:  # Not abstract.
+        """
+        Shortcut for setting plain text on clipboard.
+        """
+        self.set_data(ClipboardData(text))
+
+    def rotate(self) -> None:
+        """
+        For Emacs mode, rotate the kill ring.
+        """
+
+    @abstractmethod
+    def get_data(self) -> ClipboardData:
+        """
+        Return clipboard data.
+        """
+
+
+class DummyClipboard(Clipboard):
+    """
+    Clipboard implementation that doesn't remember anything.
+    """
+
+    def set_data(self, data: ClipboardData) -> None:
+        pass
+
+    def set_text(self, text: str) -> None:
+        pass
+
+    def rotate(self) -> None:
+        pass
+
+    def get_data(self) -> ClipboardData:
+        return ClipboardData()
+
+
+class DynamicClipboard(Clipboard):
+    """
+    Clipboard class that can dynamically returns any Clipboard.
+
+    :param get_clipboard: Callable that returns a :class:`.Clipboard` instance.
+    """
+
+    def __init__(self, get_clipboard: Callable[[], Optional[Clipboard]]) -> None:
+        self.get_clipboard = get_clipboard
+
+    def _clipboard(self) -> Clipboard:
+        return self.get_clipboard() or DummyClipboard()
+
+    def set_data(self, data: ClipboardData) -> None:
+        self._clipboard().set_data(data)
+
+    def set_text(self, text: str) -> None:
+        self._clipboard().set_text(text)
+
+    def rotate(self) -> None:
+        self._clipboard().rotate()
+
+    def get_data(self) -> ClipboardData:
+        return self._clipboard().get_data()

xonsh/vended_ptk/prompt_toolkit/clipboard/in_memory.py

@@ -0,0 +1,46 @@
+from collections import deque
+from typing import Deque, Optional
+
+from .base import Clipboard, ClipboardData
+
+__all__ = [
+    "InMemoryClipboard",
+]
+
+
+class InMemoryClipboard(Clipboard):
+    """
+    Default clipboard implementation.
+    Just keep the data in memory.
+
+    This implements a kill-ring, for Emacs mode.
+    """
+
+    def __init__(
+        self, data: Optional[ClipboardData] = None, max_size: int = 60
+    ) -> None:
+
+        assert max_size >= 1
+
+        self.max_size = max_size
+        self._ring: Deque[ClipboardData] = deque()
+
+        if data is not None:
+            self.set_data(data)
+
+    def set_data(self, data: ClipboardData) -> None:
+        self._ring.appendleft(data)
+
+        while len(self._ring) > self.max_size:
+            self._ring.pop()
+
+    def get_data(self) -> ClipboardData:
+        if self._ring:
+            return self._ring[0]
+        else:
+            return ClipboardData()
+
+    def rotate(self) -> None:
+        if self._ring:
+            # Add the very first item at the end.
+            self._ring.append(self._ring.popleft())

xonsh/vended_ptk/prompt_toolkit/clipboard/pyperclip.py

@@ -0,0 +1,41 @@
+from typing import Optional
+
+import pyperclip
+from prompt_toolkit.selection import SelectionType
+
+from .base import Clipboard, ClipboardData
+
+__all__ = [
+    "PyperclipClipboard",
+]
+
+
+class PyperclipClipboard(Clipboard):
+    """
+    Clipboard that synchronizes with the Windows/Mac/Linux system clipboard,
+    using the pyperclip module.
+    """
+
+    def __init__(self) -> None:
+        self._data: Optional[ClipboardData] = None
+
+    def set_data(self, data: ClipboardData) -> None:
+        self._data = data
+        pyperclip.copy(data.text)
+
+    def get_data(self) -> ClipboardData:
+        text = pyperclip.paste()
+
+        # When the clipboard data is equal to what we copied last time, reuse
+        # the `ClipboardData` instance. That way we're sure to keep the same
+        # `SelectionType`.
+        if self._data and self._data.text == text:
+            return self._data
+
+        # Pyperclip returned something else. Create a new `ClipboardData`
+        # instance.
+        else:
+            return ClipboardData(
+                text=text,
+                type=SelectionType.LINES if "\n" in text else SelectionType.LINES,
+            )

xonsh/vended_ptk/prompt_toolkit/completion/__init__.py

@@ -0,0 +1,36 @@
+from .base import (
+    CompleteEvent,
+    Completer,
+    Completion,
+    DummyCompleter,
+    DynamicCompleter,
+    ThreadedCompleter,
+    get_common_complete_suffix,
+    merge_completers,
+)
+from .filesystem import ExecutableCompleter, PathCompleter
+from .fuzzy_completer import FuzzyCompleter, FuzzyWordCompleter
+from .nested import NestedCompleter
+from .word_completer import WordCompleter
+
+__all__ = [
+    # Base.
+    "Completion",
+    "Completer",
+    "ThreadedCompleter",
+    "DummyCompleter",
+    "DynamicCompleter",
+    "CompleteEvent",
+    "merge_completers",
+    "get_common_complete_suffix",
+    # Filesystem.
+    "PathCompleter",
+    "ExecutableCompleter",
+    # Fuzzy
+    "FuzzyCompleter",
+    "FuzzyWordCompleter",
+    # Nested.
+    "NestedCompleter",
+    # Word completer.
+    "WordCompleter",
+]

xonsh/vended_ptk/prompt_toolkit/completion/base.py

@@ -0,0 +1,349 @@
+"""
+"""
+from abc import ABCMeta, abstractmethod
+from typing import AsyncGenerator, Callable, Iterable, Optional, Sequence
+
+from prompt_toolkit.document import Document
+from prompt_toolkit.eventloop import generator_to_async_generator
+from prompt_toolkit.formatted_text import AnyFormattedText, StyleAndTextTuples
+
+__all__ = [
+    "Completion",
+    "Completer",
+    "ThreadedCompleter",
+    "DummyCompleter",
+    "DynamicCompleter",
+    "CompleteEvent",
+    "merge_completers",
+    "get_common_complete_suffix",
+]
+
+
+class Completion:
+    """
+    :param text: The new string that will be inserted into the document.
+    :param start_position: Position relative to the cursor_position where the
+        new text will start. The text will be inserted between the
+        start_position and the original cursor position.
+    :param display: (optional string or formatted text) If the completion has
+        to be displayed differently in the completion menu.
+    :param display_meta: (Optional string or formatted text) Meta information
+        about the completion, e.g. the path or source where it's coming from.
+        This can also be a callable that returns a string.
+    :param style: Style string.
+    :param selected_style: Style string, used for a selected completion.
+        This can override the `style` parameter.
+    """
+
+    def __init__(
+        self,
+        text: str,
+        start_position: int = 0,
+        display: Optional[AnyFormattedText] = None,
+        display_meta: Optional[AnyFormattedText] = None,
+        style: str = "",
+        selected_style: str = "",
+    ) -> None:
+
+        from prompt_toolkit.formatted_text import to_formatted_text
+
+        self.text = text
+        self.start_position = start_position
+        self._display_meta = display_meta
+
+        if display is None:
+            display = text
+
+        self.display = to_formatted_text(display)
+
+        self.style = style
+        self.selected_style = selected_style
+
+        assert self.start_position <= 0
+
+    def __repr__(self) -> str:
+        if isinstance(self.display, str) and self.display == self.text:
+            return "%s(text=%r, start_position=%r)" % (
+                self.__class__.__name__,
+                self.text,
+                self.start_position,
+            )
+        else:
+            return "%s(text=%r, start_position=%r, display=%r)" % (
+                self.__class__.__name__,
+                self.text,
+                self.start_position,
+                self.display,
+            )
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Completion):
+            return False
+        return (
+            self.text == other.text
+            and self.start_position == other.start_position
+            and self.display == other.display
+            and self._display_meta == other._display_meta
+        )
+
+    def __hash__(self) -> int:
+        return hash((self.text, self.start_position, self.display, self._display_meta))
+
+    @property
+    def display_text(self) -> str:
+        " The 'display' field as plain text. "
+        from prompt_toolkit.formatted_text import fragment_list_to_text
+
+        return fragment_list_to_text(self.display)
+
+    @property
+    def display_meta(self) -> StyleAndTextTuples:
+        " Return meta-text. (This is lazy when using a callable). "
+        from prompt_toolkit.formatted_text import to_formatted_text
+
+        return to_formatted_text(self._display_meta or "")
+
+    @property
+    def display_meta_text(self) -> str:
+        " The 'meta' field as plain text. "
+        from prompt_toolkit.formatted_text import fragment_list_to_text
+
+        return fragment_list_to_text(self.display_meta)
+
+    def new_completion_from_position(self, position: int) -> "Completion":
+        """
+        (Only for internal use!)
+        Get a new completion by splitting this one. Used by `Application` when
+        it needs to have a list of new completions after inserting the common
+        prefix.
+        """
+        assert position - self.start_position >= 0
+
+        return Completion(
+            text=self.text[position - self.start_position :],
+            display=self.display,
+            display_meta=self._display_meta,
+        )
+
+
+class CompleteEvent:
+    """
+    Event that called the completer.
+
+    :param text_inserted: When True, it means that completions are requested
+        because of a text insert. (`Buffer.complete_while_typing`.)
+    :param completion_requested: When True, it means that the user explicitly
+        pressed the `Tab` key in order to view the completions.
+
+    These two flags can be used for instance to implement a completer that
+    shows some completions when ``Tab`` has been pressed, but not
+    automatically when the user presses a space. (Because of
+    `complete_while_typing`.)
+    """
+
+    def __init__(
+        self, text_inserted: bool = False, completion_requested: bool = False
+    ) -> None:
+        assert not (text_inserted and completion_requested)
+
+        #: Automatic completion while typing.
+        self.text_inserted = text_inserted
+
+        #: Used explicitly requested completion by pressing 'tab'.
+        self.completion_requested = completion_requested
+
+    def __repr__(self) -> str:
+        return "%s(text_inserted=%r, completion_requested=%r)" % (
+            self.__class__.__name__,
+            self.text_inserted,
+            self.completion_requested,
+        )
+
+
+class Completer(metaclass=ABCMeta):
+    """
+    Base class for completer implementations.
+    """
+
+    @abstractmethod
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        """
+        This should be a generator that yields :class:`.Completion` instances.
+
+        If the generation of completions is something expensive (that takes a
+        lot of time), consider wrapping this `Completer` class in a
+        `ThreadedCompleter`. In that case, the completer algorithm runs in a
+        background thread and completions will be displayed as soon as they
+        arrive.
+
+        :param document: :class:`~prompt_toolkit.document.Document` instance.
+        :param complete_event: :class:`.CompleteEvent` instance.
+        """
+        while False:
+            yield
+
+    async def get_completions_async(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> AsyncGenerator[Completion, None]:
+        """
+        Asynchronous generator for completions. (Probably, you won't have to
+        override this.)
+
+        Asynchronous generator of :class:`.Completion` objects.
+        """
+        for item in self.get_completions(document, complete_event):
+            yield item
+
+
+class ThreadedCompleter(Completer):
+    """
+    Wrapper that runs the `get_completions` generator in a thread.
+
+    (Use this to prevent the user interface from becoming unresponsive if the
+    generation of completions takes too much time.)
+
+    The completions will be displayed as soon as they are produced. The user
+    can already select a completion, even if not all completions are displayed.
+    """
+
+    def __init__(self, completer: Completer) -> None:
+        self.completer = completer
+
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        return self.completer.get_completions(document, complete_event)
+
+    async def get_completions_async(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> AsyncGenerator[Completion, None]:
+        """
+        Asynchronous generator of completions.
+        """
+        async for completion in generator_to_async_generator(
+            lambda: self.completer.get_completions(document, complete_event)
+        ):
+            yield completion
+
+    def __repr__(self) -> str:
+        return "ThreadedCompleter(%r)" % (self.completer,)
+
+
+class DummyCompleter(Completer):
+    """
+    A completer that doesn't return any completion.
+    """
+
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        return []
+
+    def __repr__(self) -> str:
+        return "DummyCompleter()"
+
+
+class DynamicCompleter(Completer):
+    """
+    Completer class that can dynamically returns any Completer.
+
+    :param get_completer: Callable that returns a :class:`.Completer` instance.
+    """
+
+    def __init__(self, get_completer: Callable[[], Optional[Completer]]) -> None:
+        self.get_completer = get_completer
+
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        completer = self.get_completer() or DummyCompleter()
+        return completer.get_completions(document, complete_event)
+
+    async def get_completions_async(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> AsyncGenerator[Completion, None]:
+        completer = self.get_completer() or DummyCompleter()
+
+        async for completion in completer.get_completions_async(
+            document, complete_event
+        ):
+            yield completion
+
+    def __repr__(self) -> str:
+        return "DynamicCompleter(%r -> %r)" % (self.get_completer, self.get_completer())
+
+
+class _MergedCompleter(Completer):
+    """
+    Combine several completers into one.
+    """
+
+    def __init__(self, completers: Sequence[Completer]) -> None:
+        self.completers = completers
+
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        # Get all completions from the other completers in a blocking way.
+        for completer in self.completers:
+            for c in completer.get_completions(document, complete_event):
+                yield c
+
+    async def get_completions_async(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> AsyncGenerator[Completion, None]:
+
+        # Get all completions from the other completers in a blocking way.
+        for completer in self.completers:
+            async for item in completer.get_completions_async(document, complete_event):
+                yield item
+
+
+def merge_completers(completers: Sequence[Completer]) -> _MergedCompleter:
+    """
+    Combine several completers into one.
+    """
+    return _MergedCompleter(completers)
+
+
+def get_common_complete_suffix(
+    document: Document, completions: Sequence[Completion]
+) -> str:
+    """
+    Return the common prefix for all completions.
+    """
+    # Take only completions that don't change the text before the cursor.
+    def doesnt_change_before_cursor(completion: Completion) -> bool:
+        end = completion.text[: -completion.start_position]
+        return document.text_before_cursor.endswith(end)
+
+    completions2 = [c for c in completions if doesnt_change_before_cursor(c)]
+
+    # When there is at least one completion that changes the text before the
+    # cursor, don't return any common part.
+    if len(completions2) != len(completions):
+        return ""
+
+    # Return the common prefix.
+    def get_suffix(completion: Completion) -> str:
+        return completion.text[-completion.start_position :]
+
+    return _commonprefix([get_suffix(c) for c in completions2])
+
+
+def _commonprefix(strings: Iterable[str]) -> str:
+    # Similar to os.path.commonprefix
+    if not strings:
+        return ""
+
+    else:
+        s1 = min(strings)
+        s2 = max(strings)
+
+        for i, c in enumerate(s1):
+            if c != s2[i]:
+                return s1[:i]
+
+        return s1

xonsh/vended_ptk/prompt_toolkit/completion/filesystem.py

@@ -0,0 +1,113 @@
+import os
+from typing import Callable, Iterable, List, Optional
+
+from prompt_toolkit.completion import CompleteEvent, Completer, Completion
+from prompt_toolkit.document import Document
+
+__all__ = [
+    "PathCompleter",
+    "ExecutableCompleter",
+]
+
+
+class PathCompleter(Completer):
+    """
+    Complete for Path variables.
+
+    :param get_paths: Callable which returns a list of directories to look into
+                      when the user enters a relative path.
+    :param file_filter: Callable which takes a filename and returns whether
+                        this file should show up in the completion. ``None``
+                        when no filtering has to be done.
+    :param min_input_len: Don't do autocompletion when the input string is shorter.
+    """
+
+    def __init__(
+        self,
+        only_directories: bool = False,
+        get_paths: Optional[Callable[[], List[str]]] = None,
+        file_filter: Optional[Callable[[str], bool]] = None,
+        min_input_len: int = 0,
+        expanduser: bool = False,
+    ) -> None:
+
+        self.only_directories = only_directories
+        self.get_paths = get_paths or (lambda: ["."])
+        self.file_filter = file_filter or (lambda _: True)
+        self.min_input_len = min_input_len
+        self.expanduser = expanduser
+
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        text = document.text_before_cursor
+
+        # Complete only when we have at least the minimal input length,
+        # otherwise, we can too many results and autocompletion will become too
+        # heavy.
+        if len(text) < self.min_input_len:
+            return
+
+        try:
+            # Do tilde expansion.
+            if self.expanduser:
+                text = os.path.expanduser(text)
+
+            # Directories where to look.
+            dirname = os.path.dirname(text)
+            if dirname:
+                directories = [
+                    os.path.dirname(os.path.join(p, text)) for p in self.get_paths()
+                ]
+            else:
+                directories = self.get_paths()
+
+            # Start of current file.
+            prefix = os.path.basename(text)
+
+            # Get all filenames.
+            filenames = []
+            for directory in directories:
+                # Look for matches in this directory.
+                if os.path.isdir(directory):
+                    for filename in os.listdir(directory):
+                        if filename.startswith(prefix):
+                            filenames.append((directory, filename))
+
+            # Sort
+            filenames = sorted(filenames, key=lambda k: k[1])
+
+            # Yield them.
+            for directory, filename in filenames:
+                completion = filename[len(prefix) :]
+                full_name = os.path.join(directory, filename)
+
+                if os.path.isdir(full_name):
+                    # For directories, add a slash to the filename.
+                    # (We don't add them to the `completion`. Users can type it
+                    # to trigger the autocompletion themselves.)
+                    filename += "/"
+                elif self.only_directories:
+                    continue
+
+                if not self.file_filter(full_name):
+                    continue
+
+                yield Completion(completion, 0, display=filename)
+        except OSError:
+            pass
+
+
+class ExecutableCompleter(PathCompleter):
+    """
+    Complete only executable files in the current path.
+    """
+
+    def __init__(self) -> None:
+        super().__init__(
+            only_directories=False,
+            min_input_len=1,
+            get_paths=lambda: os.environ.get("PATH", "").split(os.pathsep),
+            file_filter=lambda name: os.access(name, os.X_OK),
+            expanduser=True,
+        ),

xonsh/vended_ptk/prompt_toolkit/completion/fuzzy_completer.py

@@ -0,0 +1,199 @@
+import re
+from typing import Callable, Dict, Iterable, List, NamedTuple, Optional, Tuple, Union
+
+from prompt_toolkit.document import Document
+from prompt_toolkit.filters import FilterOrBool, to_filter
+from prompt_toolkit.formatted_text import AnyFormattedText, StyleAndTextTuples
+
+from .base import CompleteEvent, Completer, Completion
+from .word_completer import WordCompleter
+
+__all__ = [
+    "FuzzyCompleter",
+    "FuzzyWordCompleter",
+]
+
+
+class FuzzyCompleter(Completer):
+    """
+    Fuzzy completion.
+    This wraps any other completer and turns it into a fuzzy completer.
+
+    If the list of words is: ["leopard" , "gorilla", "dinosaur", "cat", "bee"]
+    Then trying to complete "oar" would yield "leopard" and "dinosaur", but not
+    the others, because they match the regular expression 'o.*a.*r'.
+    Similar, in another application "djm" could expand to "django_migrations".
+
+    The results are sorted by relevance, which is defined as the start position
+    and the length of the match.
+
+    Notice that this is not really a tool to work around spelling mistakes,
+    like what would be possible with difflib. The purpose is rather to have a
+    quicker or more intuitive way to filter the given completions, especially
+    when many completions have a common prefix.
+
+    Fuzzy algorithm is based on this post:
+    https://blog.amjith.com/fuzzyfinder-in-10-lines-of-python
+
+    :param completer: A :class:`~.Completer` instance.
+    :param WORD: When True, use WORD characters.
+    :param pattern: Regex pattern which selects the characters before the
+        cursor that are considered for the fuzzy matching.
+    :param enable_fuzzy: (bool or `Filter`) Enabled the fuzzy behavior. For
+        easily turning fuzzyness on or off according to a certain condition.
+    """
+
+    def __init__(
+        self,
+        completer: Completer,
+        WORD: bool = False,
+        pattern: Optional[str] = None,
+        enable_fuzzy: FilterOrBool = True,
+    ):
+
+        assert pattern is None or pattern.startswith("^")
+
+        self.completer = completer
+        self.pattern = pattern
+        self.WORD = WORD
+        self.pattern = pattern
+        self.enable_fuzzy = to_filter(enable_fuzzy)
+
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        if self.enable_fuzzy():
+            return self._get_fuzzy_completions(document, complete_event)
+        else:
+            return self.completer.get_completions(document, complete_event)
+
+    def _get_pattern(self) -> str:
+        if self.pattern:
+            return self.pattern
+        if self.WORD:
+            return r"[^\s]+"
+        return "^[a-zA-Z0-9_]*"
+
+    def _get_fuzzy_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+
+        word_before_cursor = document.get_word_before_cursor(
+            pattern=re.compile(self._get_pattern())
+        )
+
+        # Get completions
+        document2 = Document(
+            text=document.text[: document.cursor_position - len(word_before_cursor)],
+            cursor_position=document.cursor_position - len(word_before_cursor),
+        )
+
+        completions = list(self.completer.get_completions(document2, complete_event))
+
+        fuzzy_matches: List[_FuzzyMatch] = []
+
+        pat = ".*?".join(map(re.escape, word_before_cursor))
+        pat = "(?=({0}))".format(pat)  # lookahead regex to manage overlapping matches
+        regex = re.compile(pat, re.IGNORECASE)
+        for compl in completions:
+            matches = list(regex.finditer(compl.text))
+            if matches:
+                # Prefer the match, closest to the left, then shortest.
+                best = min(matches, key=lambda m: (m.start(), len(m.group(1))))
+                fuzzy_matches.append(
+                    _FuzzyMatch(len(best.group(1)), best.start(), compl)
+                )
+
+        def sort_key(fuzzy_match: "_FuzzyMatch") -> Tuple[int, int]:
+            " Sort by start position, then by the length of the match. "
+            return fuzzy_match.start_pos, fuzzy_match.match_length
+
+        fuzzy_matches = sorted(fuzzy_matches, key=sort_key)
+
+        for match in fuzzy_matches:
+            # Include these completions, but set the correct `display`
+            # attribute and `start_position`.
+            yield Completion(
+                match.completion.text,
+                start_position=match.completion.start_position
+                - len(word_before_cursor),
+                display_meta=match.completion.display_meta,
+                display=self._get_display(match, word_before_cursor),
+                style=match.completion.style,
+            )
+
+    def _get_display(
+        self, fuzzy_match: "_FuzzyMatch", word_before_cursor: str
+    ) -> AnyFormattedText:
+        """
+        Generate formatted text for the display label.
+        """
+        m = fuzzy_match
+        word = m.completion.text
+
+        if m.match_length == 0:
+            # No highlighting when we have zero length matches (no input text).
+            return word
+
+        result: StyleAndTextTuples = []
+
+        # Text before match.
+        result.append(("class:fuzzymatch.outside", word[: m.start_pos]))
+
+        # The match itself.
+        characters = list(word_before_cursor)
+
+        for c in word[m.start_pos : m.start_pos + m.match_length]:
+            classname = "class:fuzzymatch.inside"
+            if characters and c.lower() == characters[0].lower():
+                classname += ".character"
+                del characters[0]
+
+            result.append((classname, c))
+
+        # Text after match.
+        result.append(
+            ("class:fuzzymatch.outside", word[m.start_pos + m.match_length :])
+        )
+
+        return result
+
+
+class FuzzyWordCompleter(Completer):
+    """
+    Fuzzy completion on a list of words.
+
+    (This is basically a `WordCompleter` wrapped in a `FuzzyCompleter`.)
+
+    :param words: List of words or callable that returns a list of words.
+    :param meta_dict: Optional dict mapping words to their meta-information.
+    :param WORD: When True, use WORD characters.
+    """
+
+    def __init__(
+        self,
+        words: Union[List[str], Callable[[], List[str]]],
+        meta_dict: Optional[Dict[str, str]] = None,
+        WORD: bool = False,
+    ) -> None:
+
+        self.words = words
+        self.meta_dict = meta_dict or {}
+        self.WORD = WORD
+
+        self.word_completer = WordCompleter(
+            words=self.words, WORD=self.WORD, meta_dict=self.meta_dict
+        )
+
+        self.fuzzy_completer = FuzzyCompleter(self.word_completer, WORD=self.WORD)
+
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        return self.fuzzy_completer.get_completions(document, complete_event)
+
+
+_FuzzyMatch = NamedTuple(
+    "_FuzzyMatch",
+    [("match_length", int), ("start_pos", int), ("completion", Completion)],
+)

xonsh/vended_ptk/prompt_toolkit/completion/nested.py

@@ -0,0 +1,109 @@
+"""
+Nestedcompleter for completion of hierarchical data structures.
+"""
+from typing import Any, Dict, Iterable, Mapping, Optional, Set, Union
+
+from prompt_toolkit.completion import CompleteEvent, Completer, Completion
+from prompt_toolkit.completion.word_completer import WordCompleter
+from prompt_toolkit.document import Document
+
+__all__ = ["NestedCompleter"]
+
+# NestedDict = Mapping[str, Union['NestedDict', Set[str], None, Completer]]
+NestedDict = Mapping[str, Union[Any, Set[str], None, Completer]]
+
+
+class NestedCompleter(Completer):
+    """
+    Completer which wraps around several other completers, and calls any the
+    one that corresponds with the first word of the input.
+
+    By combining multiple `NestedCompleter` instances, we can achieve multiple
+    hierarchical levels of autocompletion. This is useful when `WordCompleter`
+    is not sufficient.
+
+    If you need multiple levels, check out the `from_nested_dict` classmethod.
+    """
+
+    def __init__(
+        self, options: Dict[str, Optional[Completer]], ignore_case: bool = True
+    ) -> None:
+
+        self.options = options
+        self.ignore_case = ignore_case
+
+    def __repr__(self) -> str:
+        return "NestedCompleter(%r, ignore_case=%r)" % (self.options, self.ignore_case)
+
+    @classmethod
+    def from_nested_dict(cls, data: NestedDict) -> "NestedCompleter":
+        """
+        Create a `NestedCompleter`, starting from a nested dictionary data
+        structure, like this:
+
+        .. code::
+
+            data = {
+                'show': {
+                    'version': None,
+                    'interfaces': None,
+                    'clock': None,
+                    'ip': {'interface': {'brief'}}
+                },
+                'exit': None
+                'enable': None
+            }
+
+        The value should be `None` if there is no further completion at some
+        point. If all values in the dictionary are None, it is also possible to
+        use a set instead.
+
+        Values in this data structure can be a completers as well.
+        """
+        options: Dict[str, Optional[Completer]] = {}
+        for key, value in data.items():
+            if isinstance(value, Completer):
+                options[key] = value
+            elif isinstance(value, dict):
+                options[key] = cls.from_nested_dict(value)
+            elif isinstance(value, set):
+                options[key] = cls.from_nested_dict({item: None for item in value})
+            else:
+                assert value is None
+                options[key] = None
+
+        return cls(options)
+
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        # Split document.
+        text = document.text_before_cursor.lstrip()
+        stripped_len = len(document.text_before_cursor) - len(text)
+
+        # If there is a space, check for the first term, and use a
+        # subcompleter.
+        if " " in text:
+            first_term = text.split()[0]
+            completer = self.options.get(first_term)
+
+            # If we have a sub completer, use this for the completions.
+            if completer is not None:
+                remaining_text = text[len(first_term) :].lstrip()
+                move_cursor = len(text) - len(remaining_text) + stripped_len
+
+                new_document = Document(
+                    remaining_text,
+                    cursor_position=document.cursor_position - move_cursor,
+                )
+
+                for c in completer.get_completions(new_document, complete_event):
+                    yield c
+
+        # No space in the input: behave exactly like `WordCompleter`.
+        else:
+            completer = WordCompleter(
+                list(self.options.keys()), ignore_case=self.ignore_case
+            )
+            for c in completer.get_completions(document, complete_event):
+                yield c

xonsh/vended_ptk/prompt_toolkit/completion/word_completer.py

@@ -0,0 +1,83 @@
+from typing import Callable, Dict, Iterable, List, Optional, Pattern, Union
+
+from prompt_toolkit.completion import CompleteEvent, Completer, Completion
+from prompt_toolkit.document import Document
+
+__all__ = [
+    "WordCompleter",
+]
+
+
+class WordCompleter(Completer):
+    """
+    Simple autocompletion on a list of words.
+
+    :param words: List of words or callable that returns a list of words.
+    :param ignore_case: If True, case-insensitive completion.
+    :param meta_dict: Optional dict mapping words to their meta-text. (This
+        should map strings to strings or formatted text.)
+    :param WORD: When True, use WORD characters.
+    :param sentence: When True, don't complete by comparing the word before the
+        cursor, but by comparing all the text before the cursor. In this case,
+        the list of words is just a list of strings, where each string can
+        contain spaces. (Can not be used together with the WORD option.)
+    :param match_middle: When True, match not only the start, but also in the
+                         middle of the word.
+    :param pattern: Optional regex. When given, use this regex
+        pattern instead of default one.
+    """
+
+    def __init__(
+        self,
+        words: Union[List[str], Callable[[], List[str]]],
+        ignore_case: bool = False,
+        meta_dict: Optional[Dict[str, str]] = None,
+        WORD: bool = False,
+        sentence: bool = False,
+        match_middle: bool = False,
+        pattern: Optional[Pattern[str]] = None,
+    ) -> None:
+
+        assert not (WORD and sentence)
+
+        self.words = words
+        self.ignore_case = ignore_case
+        self.meta_dict = meta_dict or {}
+        self.WORD = WORD
+        self.sentence = sentence
+        self.match_middle = match_middle
+        self.pattern = pattern
+
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        # Get list of words.
+        words = self.words
+        if callable(words):
+            words = words()
+
+        # Get word/text before cursor.
+        if self.sentence:
+            word_before_cursor = document.text_before_cursor
+        else:
+            word_before_cursor = document.get_word_before_cursor(
+                WORD=self.WORD, pattern=self.pattern
+            )
+
+        if self.ignore_case:
+            word_before_cursor = word_before_cursor.lower()
+
+        def word_matches(word: str) -> bool:
+            """ True when the word before the cursor matches. """
+            if self.ignore_case:
+                word = word.lower()
+
+            if self.match_middle:
+                return word_before_cursor in word
+            else:
+                return word.startswith(word_before_cursor)
+
+        for a in words:
+            if word_matches(a):
+                display_meta = self.meta_dict.get(a, "")
+                yield Completion(a, -len(word_before_cursor), display_meta=display_meta)

xonsh/vended_ptk/prompt_toolkit/contrib/completers/__init__.py

@@ -0,0 +1 @@
+from .system import SystemCompleter

xonsh/vended_ptk/prompt_toolkit/contrib/completers/system.py

@@ -0,0 +1,62 @@
+from prompt_toolkit.completion.filesystem import ExecutableCompleter, PathCompleter
+from prompt_toolkit.contrib.regular_languages.compiler import compile
+from prompt_toolkit.contrib.regular_languages.completion import GrammarCompleter
+
+__all__ = [
+    "SystemCompleter",
+]
+
+
+class SystemCompleter(GrammarCompleter):
+    """
+    Completer for system commands.
+    """
+
+    def __init__(self) -> None:
+        # Compile grammar.
+        g = compile(
+            r"""
+                # First we have an executable.
+                (?P<executable>[^\s]+)
+
+                # Ignore literals in between.
+                (
+                    \s+
+                    ("[^"]*" | '[^']*' | [^'"]+ )
+                )*
+
+                \s+
+
+                # Filename as parameters.
+                (
+                    (?P<filename>[^\s]+) |
+                    "(?P<double_quoted_filename>[^\s]+)" |
+                    '(?P<single_quoted_filename>[^\s]+)'
+                )
+            """,
+            escape_funcs={
+                "double_quoted_filename": (lambda string: string.replace('"', '\\"')),
+                "single_quoted_filename": (lambda string: string.replace("'", "\\'")),
+            },
+            unescape_funcs={
+                "double_quoted_filename": (
+                    lambda string: string.replace('\\"', '"')
+                ),  # XXX: not entirely correct.
+                "single_quoted_filename": (lambda string: string.replace("\\'", "'")),
+            },
+        )
+
+        # Create GrammarCompleter
+        super().__init__(
+            g,
+            {
+                "executable": ExecutableCompleter(),
+                "filename": PathCompleter(only_directories=False, expanduser=True),
+                "double_quoted_filename": PathCompleter(
+                    only_directories=False, expanduser=True
+                ),
+                "single_quoted_filename": PathCompleter(
+                    only_directories=False, expanduser=True
+                ),
+            },
+        )

xonsh/vended_ptk/prompt_toolkit/contrib/regular_languages/__init__.py

@@ -0,0 +1,75 @@
+r"""
+Tool for expressing the grammar of an input as a regular language.
+==================================================================
+
+The grammar for the input of many simple command line interfaces can be
+expressed by a regular language. Examples are PDB (the Python debugger); a
+simple (bash-like) shell with "pwd", "cd", "cat" and "ls" commands; arguments
+that you can pass to an executable; etc. It is possible to use regular
+expressions for validation and parsing of such a grammar. (More about regular
+languages: http://en.wikipedia.org/wiki/Regular_language)
+
+Example
+-------
+
+Let's take the pwd/cd/cat/ls example. We want to have a shell that accepts
+these three commands. "cd" is followed by a quoted directory name and "cat" is
+followed by a quoted file name. (We allow quotes inside the filename when
+they're escaped with a backslash.) We could define the grammar using the
+following regular expression::
+
+    grammar = \s* (
+        pwd |
+        ls |
+        (cd  \s+ " ([^"]|\.)+ ") |
+        (cat \s+ " ([^"]|\.)+ ")
+    ) \s*
+
+
+What can we do with this grammar?
+---------------------------------
+
+- Syntax highlighting: We could use this for instance to give file names
+                       different colour.
+- Parse the result: .. We can extract the file names and commands by using a
+                       regular expression with named groups.
+- Input validation: .. Don't accept anything that does not match this grammar.
+                       When combined with a parser, we can also recursively do
+                       filename validation (and accept only existing files.)
+- Autocompletion: .... Each part of the grammar can have its own autocompleter.
+                       "cat" has to be completed using file names, while "cd"
+                       has to be completed using directory names.
+
+How does it work?
+-----------------
+
+As a user of this library, you have to define the grammar of the input as a
+regular expression. The parts of this grammar where autocompletion, validation
+or any other processing is required need to be marked using a regex named
+group. Like ``(?P<varname>...)`` for instance.
+
+When the input is processed for validation (for instance), the regex will
+execute, the named group is captured, and the validator associated with this
+named group will test the captured string.
+
+There is one tricky bit:
+
+    Often we operate on incomplete input (this is by definition the case for
+    autocompletion) and we have to decide for the cursor position in which
+    possible state the grammar it could be and in which way variables could be
+    matched up to that point.
+
+To solve this problem, the compiler takes the original regular expression and
+translates it into a set of other regular expressions which each match certain
+prefixes of the original regular expression. We generate one prefix regular
+expression for every named variable (with this variable being the end of that
+expression).
+
+
+TODO: some examples of:
+    - How to create a highlighter from this grammar.
+    - How to create a validator from this grammar.
+    - How to create an autocompleter from this grammar.
+    - How to create a parser from this grammar.
+"""
+from .compiler import compile

xonsh/vended_ptk/prompt_toolkit/contrib/regular_languages/compiler.py

@@ -0,0 +1,573 @@
+r"""
+Compiler for a regular grammar.
+
+Example usage::
+
+    # Create and compile grammar.
+    p = compile('add \s+ (?P<var1>[^\s]+)  \s+  (?P<var2>[^\s]+)')
+
+    # Match input string.
+    m = p.match('add 23 432')
+
+    # Get variables.
+    m.variables().get('var1')  # Returns "23"
+    m.variables().get('var2')  # Returns "432"
+
+
+Partial matches are possible::
+
+    # Create and compile grammar.
+    p = compile('''
+        # Operators with two arguments.
+        ((?P<operator1>[^\s]+)  \s+ (?P<var1>[^\s]+)  \s+  (?P<var2>[^\s]+)) |
+
+        # Operators with only one arguments.
+        ((?P<operator2>[^\s]+)  \s+ (?P<var1>[^\s]+))
+    ''')
+
+    # Match partial input string.
+    m = p.match_prefix('add 23')
+
+    # Get variables. (Notice that both operator1 and operator2 contain the
+    # value "add".) This is because our input is incomplete, and we don't know
+    # yet in which rule of the regex we we'll end up. It could also be that
+    # `operator1` and `operator2` have a different autocompleter and we want to
+    # call all possible autocompleters that would result in valid input.)
+    m.variables().get('var1')  # Returns "23"
+    m.variables().get('operator1')  # Returns "add"
+    m.variables().get('operator2')  # Returns "add"
+
+"""
+import re
+from typing import Callable, Dict, Iterable, Iterator, List
+from typing import Match as RegexMatch
+from typing import Optional, Pattern, Tuple, cast
+
+from .regex_parser import (
+    AnyNode,
+    Lookahead,
+    Node,
+    NodeSequence,
+    Regex,
+    Repeat,
+    Variable,
+    parse_regex,
+    tokenize_regex,
+)
+
+__all__ = [
+    "compile",
+]
+
+
+# Name of the named group in the regex, matching trailing input.
+# (Trailing input is when the input contains characters after the end of the
+# expression has been matched.)
+_INVALID_TRAILING_INPUT = "invalid_trailing"
+
+EscapeFuncDict = Dict[str, Callable[[str], str]]
+
+
+class _CompiledGrammar:
+    """
+    Compiles a grammar. This will take the parse tree of a regular expression
+    and compile the grammar.
+
+    :param root_node: :class~`.regex_parser.Node` instance.
+    :param escape_funcs: `dict` mapping variable names to escape callables.
+    :param unescape_funcs: `dict` mapping variable names to unescape callables.
+    """
+
+    def __init__(
+        self,
+        root_node: Node,
+        escape_funcs: Optional[EscapeFuncDict] = None,
+        unescape_funcs: Optional[EscapeFuncDict] = None,
+    ) -> None:
+
+        self.root_node = root_node
+        self.escape_funcs = escape_funcs or {}
+        self.unescape_funcs = unescape_funcs or {}
+
+        #: Dictionary that will map the regex names to Node instances.
+        self._group_names_to_nodes: Dict[
+            str, str
+        ] = {}  # Maps regex group names to varnames.
+        counter = [0]
+
+        def create_group_func(node: Variable) -> str:
+            name = "n%s" % counter[0]
+            self._group_names_to_nodes[name] = node.varname
+            counter[0] += 1
+            return name
+
+        # Compile regex strings.
+        self._re_pattern = "^%s$" % self._transform(root_node, create_group_func)
+        self._re_prefix_patterns = list(
+            self._transform_prefix(root_node, create_group_func)
+        )
+
+        # Compile the regex itself.
+        flags = re.DOTALL  # Note that we don't need re.MULTILINE! (^ and $
+        # still represent the start and end of input text.)
+        self._re = re.compile(self._re_pattern, flags)
+        self._re_prefix = [re.compile(t, flags) for t in self._re_prefix_patterns]
+
+        # We compile one more set of regexes, similar to `_re_prefix`, but accept any trailing
+        # input. This will ensure that we can still highlight the input correctly, even when the
+        # input contains some additional characters at the end that don't match the grammar.)
+        self._re_prefix_with_trailing_input = [
+            re.compile(
+                r"(?:%s)(?P<%s>.*?)$" % (t.rstrip("$"), _INVALID_TRAILING_INPUT), flags
+            )
+            for t in self._re_prefix_patterns
+        ]
+
+    def escape(self, varname: str, value: str) -> str:
+        """
+        Escape `value` to fit in the place of this variable into the grammar.
+        """
+        f = self.escape_funcs.get(varname)
+        return f(value) if f else value
+
+    def unescape(self, varname: str, value: str) -> str:
+        """
+        Unescape `value`.
+        """
+        f = self.unescape_funcs.get(varname)
+        return f(value) if f else value
+
+    @classmethod
+    def _transform(
+        cls, root_node: Node, create_group_func: Callable[[Variable], str]
+    ) -> str:
+        """
+        Turn a :class:`Node` object into a regular expression.
+
+        :param root_node: The :class:`Node` instance for which we generate the grammar.
+        :param create_group_func: A callable which takes a `Node` and returns the next
+            free name for this node.
+        """
+
+        def transform(node: Node) -> str:
+            # Turn `AnyNode` into an OR.
+            if isinstance(node, AnyNode):
+                return "(?:%s)" % "|".join(transform(c) for c in node.children)
+
+            # Concatenate a `NodeSequence`
+            elif isinstance(node, NodeSequence):
+                return "".join(transform(c) for c in node.children)
+
+            # For Regex and Lookahead nodes, just insert them literally.
+            elif isinstance(node, Regex):
+                return node.regex
+
+            elif isinstance(node, Lookahead):
+                before = "(?!" if node.negative else "(="
+                return before + transform(node.childnode) + ")"
+
+            # A `Variable` wraps the children into a named group.
+            elif isinstance(node, Variable):
+                return "(?P<%s>%s)" % (
+                    create_group_func(node),
+                    transform(node.childnode),
+                )
+
+            # `Repeat`.
+            elif isinstance(node, Repeat):
+                if node.max_repeat is None:
+                    if node.min_repeat == 0:
+                        repeat_sign = "*"
+                    elif node.min_repeat == 1:
+                        repeat_sign = "+"
+                else:
+                    repeat_sign = "{%i,%s}" % (
+                        node.min_repeat,
+                        ("" if node.max_repeat is None else str(node.max_repeat)),
+                    )
+
+                return "(?:%s)%s%s" % (
+                    transform(node.childnode),
+                    repeat_sign,
+                    ("" if node.greedy else "?"),
+                )
+            else:
+                raise TypeError("Got %r" % (node,))
+
+        return transform(root_node)
+
+    @classmethod
+    def _transform_prefix(
+        cls, root_node: Node, create_group_func: Callable[[Variable], str]
+    ) -> Iterable[str]:
+        """
+        Yield all the regular expressions matching a prefix of the grammar
+        defined by the `Node` instance.
+
+        For each `Variable`, one regex pattern will be generated, with this
+        named group at the end. This is required because a regex engine will
+        terminate once a match is found. For autocompletion however, we need
+        the matches for all possible paths, so that we can provide completions
+        for each `Variable`.
+
+        - So, in the case of an `Any` (`A|B|C)', we generate a pattern for each
+          clause. This is one for `A`, one for `B` and one for `C`. Unless some
+          groups don't contain a `Variable`, then these can be merged together.
+        - In the case of a `NodeSequence` (`ABC`), we generate a pattern for
+          each prefix that ends with a variable, and one pattern for the whole
+          sequence. So, that's one for `A`, one for `AB` and one for `ABC`.
+
+        :param root_node: The :class:`Node` instance for which we generate the grammar.
+        :param create_group_func: A callable which takes a `Node` and returns the next
+            free name for this node.
+        """
+
+        def contains_variable(node: Node) -> bool:
+            if isinstance(node, Regex):
+                return False
+            elif isinstance(node, Variable):
+                return True
+            elif isinstance(node, (Lookahead, Repeat)):
+                return contains_variable(node.childnode)
+            elif isinstance(node, (NodeSequence, AnyNode)):
+                return any(contains_variable(child) for child in node.children)
+
+            return False
+
+        def transform(node: Node) -> Iterable[str]:
+            # Generate separate pattern for all terms that contain variables
+            # within this OR. Terms that don't contain a variable can be merged
+            # together in one pattern.
+            if isinstance(node, AnyNode):
+                # If we have a definition like:
+                #           (?P<name> .*)  | (?P<city> .*)
+                # Then we want to be able to generate completions for both the
+                # name as well as the city. We do this by yielding two
+                # different regular expressions, because the engine won't
+                # follow multiple paths, if multiple are possible.
+                children_with_variable = []
+                children_without_variable = []
+                for c in node.children:
+                    if contains_variable(c):
+                        children_with_variable.append(c)
+                    else:
+                        children_without_variable.append(c)
+
+                for c in children_with_variable:
+                    yield from transform(c)
+
+                # Merge options without variable together.
+                if children_without_variable:
+                    yield "|".join(
+                        r for c in children_without_variable for r in transform(c)
+                    )
+
+            # For a sequence, generate a pattern for each prefix that ends with
+            # a variable + one pattern of the complete sequence.
+            # (This is because, for autocompletion, we match the text before
+            # the cursor, and completions are given for the variable that we
+            # match right before the cursor.)
+            elif isinstance(node, NodeSequence):
+                # For all components in the sequence, compute prefix patterns,
+                # as well as full patterns.
+                complete = [cls._transform(c, create_group_func) for c in node.children]
+                prefixes = [list(transform(c)) for c in node.children]
+                variable_nodes = [contains_variable(c) for c in node.children]
+
+                # If any child is contains a variable, we should yield a
+                # pattern up to that point, so that we are sure this will be
+                # matched.
+                for i in range(len(node.children)):
+                    if variable_nodes[i]:
+                        for c_str in prefixes[i]:
+                            yield "".join(complete[:i]) + c_str
+
+                # If there are non-variable nodes, merge all the prefixes into
+                # one pattern. If the input is: "[part1] [part2] [part3]", then
+                # this gets compiled into:
+                #  (complete1 + (complete2 + (complete3  | partial3) | partial2) | partial1 )
+                # For nodes that contain a variable, we skip the "|partial"
+                # part here, because thees are matched with the previous
+                # patterns.
+                if not all(variable_nodes):
+                    result = []
+
+                    # Start with complete patterns.
+                    for i in range(len(node.children)):
+                        result.append("(?:")
+                        result.append(complete[i])
+
+                    # Add prefix patterns.
+                    for i in range(len(node.children) - 1, -1, -1):
+                        if variable_nodes[i]:
+                            # No need to yield a prefix for this one, we did
+                            # the variable prefixes earlier.
+                            result.append(")")
+                        else:
+                            result.append("|(?:")
+                            # If this yields multiple, we should yield all combinations.
+                            assert len(prefixes[i]) == 1
+                            result.append(prefixes[i][0])
+                            result.append("))")
+
+                    yield "".join(result)
+
+            elif isinstance(node, Regex):
+                yield "(?:%s)?" % node.regex
+
+            elif isinstance(node, Lookahead):
+                if node.negative:
+                    yield "(?!%s)" % cls._transform(node.childnode, create_group_func)
+                else:
+                    # Not sure what the correct semantics are in this case.
+                    # (Probably it's not worth implementing this.)
+                    raise Exception("Positive lookahead not yet supported.")
+
+            elif isinstance(node, Variable):
+                # (Note that we should not append a '?' here. the 'transform'
+                # method will already recursively do that.)
+                for c_str in transform(node.childnode):
+                    yield "(?P<%s>%s)" % (create_group_func(node), c_str)
+
+            elif isinstance(node, Repeat):
+                # If we have a repetition of 8 times. That would mean that the
+                # current input could have for instance 7 times a complete
+                # match, followed by a partial match.
+                prefix = cls._transform(node.childnode, create_group_func)
+
+                if node.max_repeat == 1:
+                    yield from transform(node.childnode)
+                else:
+                    for c_str in transform(node.childnode):
+                        if node.max_repeat:
+                            repeat_sign = "{,%i}" % (node.max_repeat - 1)
+                        else:
+                            repeat_sign = "*"
+                        yield "(?:%s)%s%s%s" % (
+                            prefix,
+                            repeat_sign,
+                            ("" if node.greedy else "?"),
+                            c_str,
+                        )
+
+            else:
+                raise TypeError("Got %r" % node)
+
+        for r in transform(root_node):
+            yield "^(?:%s)$" % r
+
+    def match(self, string: str) -> Optional["Match"]:
+        """
+        Match the string with the grammar.
+        Returns a :class:`Match` instance or `None` when the input doesn't match the grammar.
+
+        :param string: The input string.
+        """
+        m = self._re.match(string)
+
+        if m:
+            return Match(
+                string, [(self._re, m)], self._group_names_to_nodes, self.unescape_funcs
+            )
+        return None
+
+    def match_prefix(self, string: str) -> Optional["Match"]:
+        """
+        Do a partial match of the string with the grammar. The returned
+        :class:`Match` instance can contain multiple representations of the
+        match. This will never return `None`. If it doesn't match at all, the "trailing input"
+        part will capture all of the input.
+
+        :param string: The input string.
+        """
+        # First try to match using `_re_prefix`. If nothing is found, use the patterns that
+        # also accept trailing characters.
+        for patterns in [self._re_prefix, self._re_prefix_with_trailing_input]:
+            matches = [(r, r.match(string)) for r in patterns]
+            matches2 = [(r, m) for r, m in matches if m]
+
+            if matches2 != []:
+                return Match(
+                    string, matches2, self._group_names_to_nodes, self.unescape_funcs
+                )
+
+        return None
+
+
+class Match:
+    """
+    :param string: The input string.
+    :param re_matches: List of (compiled_re_pattern, re_match) tuples.
+    :param group_names_to_nodes: Dictionary mapping all the re group names to the matching Node instances.
+    """
+
+    def __init__(
+        self,
+        string: str,
+        re_matches: List[Tuple[Pattern[str], RegexMatch[str]]],
+        group_names_to_nodes: Dict[str, str],
+        unescape_funcs: Dict[str, Callable[[str], str]],
+    ):
+        self.string = string
+        self._re_matches = re_matches
+        self._group_names_to_nodes = group_names_to_nodes
+        self._unescape_funcs = unescape_funcs
+
+    def _nodes_to_regs(self) -> List[Tuple[str, Tuple[int, int]]]:
+        """
+        Return a list of (varname, reg) tuples.
+        """
+
+        def get_tuples() -> Iterable[Tuple[str, Tuple[int, int]]]:
+            for r, re_match in self._re_matches:
+                for group_name, group_index in r.groupindex.items():
+                    if group_name != _INVALID_TRAILING_INPUT:
+                        regs = cast(Tuple[Tuple[int, int], ...], re_match.regs)
+                        reg = regs[group_index]
+                        node = self._group_names_to_nodes[group_name]
+                        yield (node, reg)
+
+        return list(get_tuples())
+
+    def _nodes_to_values(self) -> List[Tuple[str, str, Tuple[int, int]]]:
+        """
+        Returns list of (Node, string_value) tuples.
+        """
+
+        def is_none(sl: Tuple[int, int]) -> bool:
+            return sl[0] == -1 and sl[1] == -1
+
+        def get(sl: Tuple[int, int]) -> str:
+            return self.string[sl[0] : sl[1]]
+
+        return [
+            (varname, get(slice), slice)
+            for varname, slice in self._nodes_to_regs()
+            if not is_none(slice)
+        ]
+
+    def _unescape(self, varname: str, value: str) -> str:
+        unwrapper = self._unescape_funcs.get(varname)
+        return unwrapper(value) if unwrapper else value
+
+    def variables(self) -> "Variables":
+        """
+        Returns :class:`Variables` instance.
+        """
+        return Variables(
+            [(k, self._unescape(k, v), sl) for k, v, sl in self._nodes_to_values()]
+        )
+
+    def trailing_input(self) -> Optional["MatchVariable"]:
+        """
+        Get the `MatchVariable` instance, representing trailing input, if there is any.
+        "Trailing input" is input at the end that does not match the grammar anymore, but
+        when this is removed from the end of the input, the input would be a valid string.
+        """
+        slices: List[Tuple[int, int]] = []
+
+        # Find all regex group for the name _INVALID_TRAILING_INPUT.
+        for r, re_match in self._re_matches:
+            for group_name, group_index in r.groupindex.items():
+                if group_name == _INVALID_TRAILING_INPUT:
+                    slices.append(re_match.regs[group_index])
+
+        # Take the smallest part. (Smaller trailing text means that a larger input has
+        # been matched, so that is better.)
+        if slices:
+            slice = (max(i[0] for i in slices), max(i[1] for i in slices))
+            value = self.string[slice[0] : slice[1]]
+            return MatchVariable("<trailing_input>", value, slice)
+        return None
+
+    def end_nodes(self) -> Iterable["MatchVariable"]:
+        """
+        Yields `MatchVariable` instances for all the nodes having their end
+        position at the end of the input string.
+        """
+        for varname, reg in self._nodes_to_regs():
+            # If this part goes until the end of the input string.
+            if reg[1] == len(self.string):
+                value = self._unescape(varname, self.string[reg[0] : reg[1]])
+                yield MatchVariable(varname, value, (reg[0], reg[1]))
+
+
+class Variables:
+    def __init__(self, tuples: List[Tuple[str, str, Tuple[int, int]]]) -> None:
+        #: List of (varname, value, slice) tuples.
+        self._tuples = tuples
+
+    def __repr__(self) -> str:
+        return "%s(%s)" % (
+            self.__class__.__name__,
+            ", ".join("%s=%r" % (k, v) for k, v, _ in self._tuples),
+        )
+
+    def get(self, key: str, default: Optional[str] = None) -> Optional[str]:
+        items = self.getall(key)
+        return items[0] if items else default
+
+    def getall(self, key: str) -> List[str]:
+        return [v for k, v, _ in self._tuples if k == key]
+
+    def __getitem__(self, key: str) -> Optional[str]:
+        return self.get(key)
+
+    def __iter__(self) -> Iterator["MatchVariable"]:
+        """
+        Yield `MatchVariable` instances.
+        """
+        for varname, value, slice in self._tuples:
+            yield MatchVariable(varname, value, slice)
+
+
+class MatchVariable:
+    """
+    Represents a match of a variable in the grammar.
+
+    :param varname: (string) Name of the variable.
+    :param value: (string) Value of this variable.
+    :param slice: (start, stop) tuple, indicating the position of this variable
+                  in the input string.
+    """
+
+    def __init__(self, varname: str, value: str, slice: Tuple[int, int]) -> None:
+        self.varname = varname
+        self.value = value
+        self.slice = slice
+
+        self.start = self.slice[0]
+        self.stop = self.slice[1]
+
+    def __repr__(self) -> str:
+        return "%s(%r, %r)" % (self.__class__.__name__, self.varname, self.value)
+
+
+def compile(
+    expression: str,
+    escape_funcs: Optional[EscapeFuncDict] = None,
+    unescape_funcs: Optional[EscapeFuncDict] = None,
+) -> _CompiledGrammar:
+    """
+    Compile grammar (given as regex string), returning a `CompiledGrammar`
+    instance.
+    """
+    return _compile_from_parse_tree(
+        parse_regex(tokenize_regex(expression)),
+        escape_funcs=escape_funcs,
+        unescape_funcs=unescape_funcs,
+    )
+
+
+def _compile_from_parse_tree(
+    root_node: Node,
+    escape_funcs: Optional[EscapeFuncDict] = None,
+    unescape_funcs: Optional[EscapeFuncDict] = None,
+) -> _CompiledGrammar:
+    """
+    Compile grammar (given as parse tree), returning a `CompiledGrammar`
+    instance.
+    """
+    return _CompiledGrammar(
+        root_node, escape_funcs=escape_funcs, unescape_funcs=unescape_funcs
+    )

xonsh/vended_ptk/prompt_toolkit/contrib/regular_languages/completion.py

@@ -0,0 +1,94 @@
+"""
+Completer for a regular grammar.
+"""
+from typing import Dict, Iterable, List
+
+from prompt_toolkit.completion import CompleteEvent, Completer, Completion
+from prompt_toolkit.document import Document
+
+from .compiler import Match, _CompiledGrammar
+
+__all__ = [
+    "GrammarCompleter",
+]
+
+
+class GrammarCompleter(Completer):
+    """
+    Completer which can be used for autocompletion according to variables in
+    the grammar. Each variable can have a different autocompleter.
+
+    :param compiled_grammar: `GrammarCompleter` instance.
+    :param completers: `dict` mapping variable names of the grammar to the
+                       `Completer` instances to be used for each variable.
+    """
+
+    def __init__(
+        self, compiled_grammar: _CompiledGrammar, completers: Dict[str, Completer]
+    ) -> None:
+
+        self.compiled_grammar = compiled_grammar
+        self.completers = completers
+
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        m = self.compiled_grammar.match_prefix(document.text_before_cursor)
+
+        if m:
+            completions = self._remove_duplicates(
+                self._get_completions_for_match(m, complete_event)
+            )
+
+            for c in completions:
+                yield c
+
+    def _get_completions_for_match(
+        self, match: Match, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        """
+        Yield all the possible completions for this input string.
+        (The completer assumes that the cursor position was at the end of the
+        input string.)
+        """
+        for match_variable in match.end_nodes():
+            varname = match_variable.varname
+            start = match_variable.start
+
+            completer = self.completers.get(varname)
+
+            if completer:
+                text = match_variable.value
+
+                # Unwrap text.
+                unwrapped_text = self.compiled_grammar.unescape(varname, text)
+
+                # Create a document, for the completions API (text/cursor_position)
+                document = Document(unwrapped_text, len(unwrapped_text))
+
+                # Call completer
+                for completion in completer.get_completions(document, complete_event):
+                    new_text = (
+                        unwrapped_text[: len(text) + completion.start_position]
+                        + completion.text
+                    )
+
+                    # Wrap again.
+                    yield Completion(
+                        text=self.compiled_grammar.escape(varname, new_text),
+                        start_position=start - len(match.string),
+                        display=completion.display,
+                        display_meta=completion.display_meta,
+                    )
+
+    def _remove_duplicates(self, items: Iterable[Completion]) -> List[Completion]:
+        """
+        Remove duplicates, while keeping the order.
+        (Sometimes we have duplicates, because the there several matches of the
+        same grammar, each yielding similar completions.)
+        """
+        result: List[Completion] = []
+        for i in items:
+            if i not in result:
+                result.append(i)
+        return result

xonsh/vended_ptk/prompt_toolkit/contrib/regular_languages/lexer.py

@@ -0,0 +1,92 @@
+"""
+`GrammarLexer` is compatible with other lexers and can be used to highlight
+the input using a regular grammar with annotations.
+"""
+from typing import Callable, Dict, Optional
+
+from prompt_toolkit.document import Document
+from prompt_toolkit.formatted_text.base import StyleAndTextTuples
+from prompt_toolkit.formatted_text.utils import split_lines
+from prompt_toolkit.lexers import Lexer
+
+from .compiler import _CompiledGrammar
+
+__all__ = [
+    "GrammarLexer",
+]
+
+
+class GrammarLexer(Lexer):
+    """
+    Lexer which can be used for highlighting of fragments according to variables in the grammar.
+
+    (It does not actual lexing of the string, but it exposes an API, compatible
+    with the Pygments lexer class.)
+
+    :param compiled_grammar: Grammar as returned by the `compile()` function.
+    :param lexers: Dictionary mapping variable names of the regular grammar to
+                   the lexers that should be used for this part. (This can
+                   call other lexers recursively.) If you wish a part of the
+                   grammar to just get one fragment, use a
+                   `prompt_toolkit.lexers.SimpleLexer`.
+    """
+
+    def __init__(
+        self,
+        compiled_grammar: _CompiledGrammar,
+        default_style: str = "",
+        lexers: Optional[Dict[str, Lexer]] = None,
+    ) -> None:
+
+        self.compiled_grammar = compiled_grammar
+        self.default_style = default_style
+        self.lexers = lexers or {}
+
+    def _get_text_fragments(self, text: str) -> StyleAndTextTuples:
+        m = self.compiled_grammar.match_prefix(text)
+
+        if m:
+            characters: StyleAndTextTuples = [(self.default_style, c) for c in text]
+
+            for v in m.variables():
+                # If we have a `Lexer` instance for this part of the input.
+                # Tokenize recursively and apply tokens.
+                lexer = self.lexers.get(v.varname)
+
+                if lexer:
+                    document = Document(text[v.start : v.stop])
+                    lexer_tokens_for_line = lexer.lex_document(document)
+                    text_fragments: StyleAndTextTuples = []
+                    for i in range(len(document.lines)):
+                        text_fragments.extend(lexer_tokens_for_line(i))
+                        text_fragments.append(("", "\n"))
+                    if text_fragments:
+                        text_fragments.pop()
+
+                    i = v.start
+                    for t, s, *_ in text_fragments:
+                        for c in s:
+                            if characters[i][0] == self.default_style:
+                                characters[i] = (t, characters[i][1])
+                            i += 1
+
+            # Highlight trailing input.
+            trailing_input = m.trailing_input()
+            if trailing_input:
+                for i in range(trailing_input.start, trailing_input.stop):
+                    characters[i] = ("class:trailing-input", characters[i][1])
+
+            return characters
+        else:
+            return [("", text)]
+
+    def lex_document(self, document: Document) -> Callable[[int], StyleAndTextTuples]:
+        lines = list(split_lines(self._get_text_fragments(document.text)))
+
+        def get_line(lineno: int) -> StyleAndTextTuples:
+            try:
+                return lines[lineno]
+            except IndexError:
+                return []
+
+        return get_line

xonsh/vended_ptk/prompt_toolkit/contrib/regular_languages/regex_parser.py

@@ -0,0 +1,281 @@
+"""
+Parser for parsing a regular expression.
+Take a string representing a regular expression and return the root node of its
+parse tree.
+
+usage::
+
+    root_node = parse_regex('(hello|world)')
+
+Remarks:
+- The regex parser processes multiline, it ignores all whitespace and supports
+  multiple named groups with the same name and #-style comments.
+
+Limitations:
+- Lookahead is not supported.
+"""
+import re
+from typing import List, Optional
+
+__all__ = [
+    "Repeat",
+    "Variable",
+    "Regex",
+    "Lookahead",
+    "tokenize_regex",
+    "parse_regex",
+]
+
+
+class Node:
+    """
+    Base class for all the grammar nodes.
+    (You don't initialize this one.)
+    """
+
+    def __add__(self, other_node: "Node") -> "NodeSequence":
+        return NodeSequence([self, other_node])
+
+    def __or__(self, other_node: "Node") -> "AnyNode":
+        return AnyNode([self, other_node])
+
+
+class AnyNode(Node):
+    """
+    Union operation (OR operation) between several grammars. You don't
+    initialize this yourself, but it's a result of a "Grammar1 | Grammar2"
+    operation.
+    """
+
+    def __init__(self, children: List[Node]) -> None:
+        self.children = children
+
+    def __or__(self, other_node: Node) -> "AnyNode":
+        return AnyNode(self.children + [other_node])
+
+    def __repr__(self) -> str:
+        return "%s(%r)" % (self.__class__.__name__, self.children)
+
+
+class NodeSequence(Node):
+    """
+    Concatenation operation of several grammars. You don't initialize this
+    yourself, but it's a result of a "Grammar1 + Grammar2" operation.
+    """
+
+    def __init__(self, children: List[Node]) -> None:
+        self.children = children
+
+    def __add__(self, other_node: Node) -> "NodeSequence":
+        return NodeSequence(self.children + [other_node])
+
+    def __repr__(self) -> str:
+        return "%s(%r)" % (self.__class__.__name__, self.children)
+
+
+class Regex(Node):
+    """
+    Regular expression.
+    """
+
+    def __init__(self, regex: str) -> None:
+        re.compile(regex)  # Validate
+
+        self.regex = regex
+
+    def __repr__(self) -> str:
+        return "%s(/%s/)" % (self.__class__.__name__, self.regex)
+
+
+class Lookahead(Node):
+    """
+    Lookahead expression.
+    """
+
+    def __init__(self, childnode: Node, negative: bool = False) -> None:
+        self.childnode = childnode
+        self.negative = negative
+
+    def __repr__(self) -> str:
+        return "%s(%r)" % (self.__class__.__name__, self.childnode)
+
+
+class Variable(Node):
+    """
+    Mark a variable in the regular grammar. This will be translated into a
+    named group. Each variable can have his own completer, validator, etc..
+
+    :param childnode: The grammar which is wrapped inside this variable.
+    :param varname: String.
+    """
+
+    def __init__(self, childnode: Node, varname: str = "") -> None:
+        self.childnode = childnode
+        self.varname = varname
+
+    def __repr__(self) -> str:
+        return "%s(childnode=%r, varname=%r)" % (
+            self.__class__.__name__,
+            self.childnode,
+            self.varname,
+        )
+
+
+class Repeat(Node):
+    def __init__(
+        self,
+        childnode: Node,
+        min_repeat: int = 0,
+        max_repeat: Optional[int] = None,
+        greedy: bool = True,
+    ) -> None:
+        self.childnode = childnode
+        self.min_repeat = min_repeat
+        self.max_repeat = max_repeat
+        self.greedy = greedy
+
+    def __repr__(self) -> str:
+        return "%s(childnode=%r)" % (self.__class__.__name__, self.childnode)
+
+
+def tokenize_regex(input: str) -> List[str]:
+    """
+    Takes a string, representing a regular expression as input, and tokenizes
+    it.
+
+    :param input: string, representing a regular expression.
+    :returns: List of tokens.
+    """
+    # Regular expression for tokenizing other regular expressions.
+    p = re.compile(
+        r"""^(
+        \(\?P\<[a-zA-Z0-9_-]+\>  | # Start of named group.
+        \(\?#[^)]*\)             | # Comment
+        \(\?=                    | # Start of lookahead assertion
+        \(\?!                    | # Start of negative lookahead assertion
+        \(\?<=                   | # If preceded by.
+        \(\?<                    | # If not preceded by.
+        \(?:                     | # Start of group. (non capturing.)
+        \(                       | # Start of group.
+        \(?[iLmsux]              | # Flags.
+        \(?P=[a-zA-Z]+\)         | # Back reference to named group
+        \)                       | # End of group.
+        \{[^{}]*\}               | # Repetition
+        \*\? | \+\? | \?\?\      | # Non greedy repetition.
+        \* | \+ | \?             | # Repetition
+        \#.*\n                   | # Comment
+        \\. |
+
+        # Character group.
+        \[
+            ( [^\]\\]  |  \\.)*
+        \]                  |
+
+        [^(){}]             |
+        .
+    )""",
+        re.VERBOSE,
+    )
+
+    tokens = []
+
+    while input:
+        m = p.match(input)
+        if m:
+            token, input = input[: m.end()], input[m.end() :]
+            if not token.isspace():
+                tokens.append(token)
+        else:
+            raise Exception("Could not tokenize input regex.")
+
+    return tokens
+
+
+def parse_regex(regex_tokens: List[str]) -> Node:
+    """
+    Takes a list of tokens from the tokenizer, and returns a parse tree.
+    """
+    # We add a closing brace because that represents the final pop of the stack.
+    tokens: List[str] = [")"] + regex_tokens[::-1]
+
+    def wrap(lst: List[Node]) -> Node:
+        """ Turn list into sequence when it contains several items. """
+        if len(lst) == 1:
+            return lst[0]
+        else:
+            return NodeSequence(lst)
+
+    def _parse() -> Node:
+        or_list: List[List[Node]] = []
+        result: List[Node] = []
+
+        def wrapped_result() -> Node:
+            if or_list == []:
+                return wrap(result)
+            else:
+                or_list.append(result)
+                return AnyNode([wrap(i) for i in or_list])
+
+        while tokens:
+            t = tokens.pop()
+
+            if t.startswith("(?P<"):
+                variable = Variable(_parse(), varname=t[4:-1])
+                result.append(variable)
+
+            elif t in ("*", "*?"):
+                greedy = t == "*"
+                result[-1] = Repeat(result[-1], greedy=greedy)
+
+            elif t in ("+", "+?"):
+                greedy = t == "+"
+                result[-1] = Repeat(result[-1], min_repeat=1, greedy=greedy)
+
+            elif t in ("?", "??"):
+                if result == []:
+                    raise Exception("Nothing to repeat." + repr(tokens))
+                else:
+                    greedy = t == "?"
+                    result[-1] = Repeat(
+                        result[-1], min_repeat=0, max_repeat=1, greedy=greedy
+                    )
+
+            elif t == "|":
+                or_list.append(result)
+                result = []
+
+            elif t in ("(", "(?:"):
+                result.append(_parse())
+
+            elif t == "(?!":
+                result.append(Lookahead(_parse(), negative=True))
+
+            elif t == "(?=":
+                result.append(Lookahead(_parse(), negative=False))
+
+            elif t == ")":
+                return wrapped_result()
+
+            elif t.startswith("#"):
+                pass
+
+            elif t.startswith("{"):
+                # TODO: implement!
+                raise Exception("{}-style repetition not yet supported".format(t))
+
+            elif t.startswith("(?"):
+                raise Exception("%r not supported" % t)
+
+            elif t.isspace():
+                pass
+            else:
+                result.append(Regex(t))
+
+        raise Exception("Expecting ')' token")
+
+    result = _parse()
+
+    if len(tokens) != 0:
+        raise Exception("Unmatched parentheses.")
+    else:
+        return result

xonsh/vended_ptk/prompt_toolkit/contrib/regular_languages/validation.py

@@ -0,0 +1,60 @@
+"""
+Validator for a regular language.
+"""
+from typing import Dict
+
+from prompt_toolkit.document import Document
+from prompt_toolkit.validation import ValidationError, Validator
+
+from .compiler import _CompiledGrammar
+
+__all__ = [
+    "GrammarValidator",
+]
+
+
+class GrammarValidator(Validator):
+    """
+    Validator which can be used for validation according to variables in
+    the grammar. Each variable can have its own validator.
+
+    :param compiled_grammar: `GrammarCompleter` instance.
+    :param validators: `dict` mapping variable names of the grammar to the
+                       `Validator` instances to be used for each variable.
+    """
+
+    def __init__(
+        self, compiled_grammar: _CompiledGrammar, validators: Dict[str, Validator]
+    ) -> None:
+
+        self.compiled_grammar = compiled_grammar
+        self.validators = validators
+
+    def validate(self, document: Document) -> None:
+        # Parse input document.
+        # We use `match`, not `match_prefix`, because for validation, we want
+        # the actual, unambiguous interpretation of the input.
+        m = self.compiled_grammar.match(document.text)
+
+        if m:
+            for v in m.variables():
+                validator = self.validators.get(v.varname)
+
+                if validator:
+                    # Unescape text.
+                    unwrapped_text = self.compiled_grammar.unescape(v.varname, v.value)
+
+                    # Create a document, for the completions API (text/cursor_position)
+                    inner_document = Document(unwrapped_text, len(unwrapped_text))
+
+                    try:
+                        validator.validate(inner_document)
+                    except ValidationError as e:
+                        raise ValidationError(
+                            cursor_position=v.start + e.cursor_position,
+                            message=e.message,
+                        )
+        else:
+            raise ValidationError(
+                cursor_position=len(document.text), message="Invalid command"
+            )

xonsh/vended_ptk/prompt_toolkit/contrib/ssh/__init__.py

@@ -0,0 +1,6 @@
+from .server import PromptToolkitSession, PromptToolkitSSHServer
+
+__all__ = [
+    "PromptToolkitSession",
+    "PromptToolkitSSHServer",
+]

xonsh/vended_ptk/prompt_toolkit/contrib/ssh/server.py

@@ -0,0 +1,134 @@
+"""
+Utility for running a prompt_toolkit application in an asyncssh server.
+"""
+import asyncio
+import traceback
+from typing import Awaitable, Callable, Optional, TextIO, cast
+
+import asyncssh
+
+from prompt_toolkit.application.current import AppSession, create_app_session
+from prompt_toolkit.data_structures import Size
+from prompt_toolkit.input.posix_pipe import PosixPipeInput
+from prompt_toolkit.output.vt100 import Vt100_Output
+
+__all__ = [
+    "PromptToolkitSession",
+    "PromptToolkitSSHServer",
+]
+
+
+class PromptToolkitSession(asyncssh.SSHServerSession):
+    def __init__(self, interact: Callable[[], Awaitable[None]]) -> None:
+        self.interact = interact
+        self._chan = None
+        self.app_session: Optional[AppSession] = None
+
+        # PipInput object, for sending input in the CLI.
+        # (This is something that we can use in the prompt_toolkit event loop,
+        # but still write date in manually.)
+        self._input = PosixPipeInput()
+
+        # Output object. Don't render to the real stdout, but write everything
+        # in the SSH channel.
+        class Stdout:
+            def write(s, data):
+                if self._chan is not None:
+                    self._chan.write(data.replace("\n", "\r\n"))
+
+            def flush(s):
+                pass
+
+        self._output = Vt100_Output(
+            cast(TextIO, Stdout()), self._get_size, write_binary=False
+        )
+
+    def _get_size(self) -> Size:
+        """
+        Callable that returns the current `Size`, required by Vt100_Output.
+        """
+        if self._chan is None:
+            return Size(rows=20, columns=79)
+        else:
+            width, height, pixwidth, pixheight = self._chan.get_terminal_size()
+            return Size(rows=height, columns=width)
+
+    def connection_made(self, chan):
+        self._chan = chan
+
+    def shell_requested(self) -> bool:
+        return True
+
+    def session_started(self) -> None:
+        asyncio.get_event_loop().create_task(self._interact())
+
+    async def _interact(self) -> None:
+        if self._chan is None:
+            # Should not happen.
+            raise Exception("`_interact` called before `connection_made`.")
+
+        # Disable the line editing provided by asyncssh. Prompt_toolkit
+        # provides the line editing.
+        self._chan.set_line_mode(False)
+
+        with create_app_session(input=self._input, output=self._output) as session:
+            self.app_session = session
+            try:
+                await self.interact()
+            except BaseException:
+                traceback.print_exc()
+            finally:
+                # Close the connection.
+                self._chan.close()
+
+    def terminal_size_changed(self, width, height, pixwidth, pixheight):
+        # Send resize event to the current application.
+        if self.app_session and self.app_session.app:
+            self.app_session.app._on_resize()
+
+    def data_received(self, data, datatype):
+        self._input.send_text(data)
+
+
+class PromptToolkitSSHServer(asyncssh.SSHServer):
+    """
+    Run a prompt_toolkit application over an asyncssh server.
+
+    This takes one argument, an `interact` function, which is called for each
+    connection. This should be an asynchronous function that runs the
+    prompt_toolkit applications. This function runs in an `AppSession`, which
+    means that we can have multiple UI interactions concurrently.
+
+    Example usage:
+
+    .. code:: python
+
+        async def interact() -> None:
+            await yes_no_dialog("my title", "my text").run_async()
+
+            prompt_session = PromptSession()
+            text = await prompt_session.prompt_async("Type something: ")
+            print_formatted_text('You said: ', text)
+
+        server = PromptToolkitSSHServer(interact=interact)
+        loop = get_event_loop()
+        loop.run_until_complete(
+            asyncssh.create_server(
+                lambda: MySSHServer(interact),
+                "",
+                port,
+                server_host_keys=["/etc/ssh/..."],
+            )
+        )
+        loop.run_forever()
+    """
+
+    def __init__(self, interact: Callable[[], Awaitable[None]]) -> None:
+        self.interact = interact
+
+    def begin_auth(self, username):
+        # No authentication.
+        return False
+
+    def session_requested(self) -> PromptToolkitSession:
+        return PromptToolkitSession(self.interact)

xonsh/vended_ptk/prompt_toolkit/contrib/telnet/__init__.py

@@ -0,0 +1,5 @@
+from .server import TelnetServer
+
+__all__ = [
+    "TelnetServer",
+]

xonsh/vended_ptk/prompt_toolkit/contrib/telnet/log.py

@@ -0,0 +1,10 @@
+"""
+Python logger for the telnet server.
+"""
+import logging
+
+logger = logging.getLogger(__package__)
+
+__all__ = [
+    "logger",
+]

xonsh/vended_ptk/prompt_toolkit/contrib/telnet/protocol.py

@@ -0,0 +1,188 @@
+"""
+Parser for the Telnet protocol. (Not a complete implementation of the telnet
+specification, but sufficient for a command line interface.)
+
+Inspired by `Twisted.conch.telnet`.
+"""
+import struct
+from typing import Callable, Generator
+
+from .log import logger
+
+__all__ = [
+    "TelnetProtocolParser",
+]
+
+
+def int2byte(number: int) -> bytes:
+    return bytes((number,))
+
+
+# Telnet constants.
+NOP = int2byte(0)
+SGA = int2byte(3)
+
+IAC = int2byte(255)
+DO = int2byte(253)
+DONT = int2byte(254)
+LINEMODE = int2byte(34)
+SB = int2byte(250)
+WILL = int2byte(251)
+WONT = int2byte(252)
+MODE = int2byte(1)
+SE = int2byte(240)
+ECHO = int2byte(1)
+NAWS = int2byte(31)
+LINEMODE = int2byte(34)
+SUPPRESS_GO_AHEAD = int2byte(3)
+
+DM = int2byte(242)
+BRK = int2byte(243)
+IP = int2byte(244)
+AO = int2byte(245)
+AYT = int2byte(246)
+EC = int2byte(247)
+EL = int2byte(248)
+GA = int2byte(249)
+
+
+class TelnetProtocolParser:
+    """
+    Parser for the Telnet protocol.
+    Usage::
+
+        def data_received(data):
+            print(data)
+
+        def size_received(rows, columns):
+            print(rows, columns)
+
+        p = TelnetProtocolParser(data_received, size_received)
+        p.feed(binary_data)
+    """
+
+    def __init__(
+        self,
+        data_received_callback: Callable[[bytes], None],
+        size_received_callback: Callable[[int, int], None],
+    ) -> None:
+
+        self.data_received_callback = data_received_callback
+        self.size_received_callback = size_received_callback
+
+        self._parser = self._parse_coroutine()
+        self._parser.send(None)  # type: ignore
+
+    def received_data(self, data: bytes) -> None:
+        self.data_received_callback(data)
+
+    def do_received(self, data: bytes) -> None:
+        """ Received telnet DO command. """
+        logger.info("DO %r", data)
+
+    def dont_received(self, data: bytes) -> None:
+        """ Received telnet DONT command. """
+        logger.info("DONT %r", data)
+
+    def will_received(self, data: bytes) -> None:
+        """ Received telnet WILL command. """
+        logger.info("WILL %r", data)
+
+    def wont_received(self, data: bytes) -> None:
+        """ Received telnet WONT command. """
+        logger.info("WONT %r", data)
+
+    def command_received(self, command: bytes, data: bytes) -> None:
+        if command == DO:
+            self.do_received(data)
+
+        elif command == DONT:
+            self.dont_received(data)
+
+        elif command == WILL:
+            self.will_received(data)
+
+        elif command == WONT:
+            self.wont_received(data)
+
+        else:
+            logger.info("command received %r %r", command, data)
+
+    def naws(self, data: bytes) -> None:
+        """
+        Received NAWS. (Window dimensions.)
+        """
+        if len(data) == 4:
+            # NOTE: the first parameter of struct.unpack should be
+            # a 'str' object. Both on Py2/py3. This crashes on OSX
+            # otherwise.
+            columns, rows = struct.unpack(str("!HH"), data)
+            self.size_received_callback(rows, columns)
+        else:
+            logger.warning("Wrong number of NAWS bytes")
+
+    def negotiate(self, data: bytes) -> None:
+        """
+        Got negotiate data.
+        """
+        command, payload = data[0:1], data[1:]
+
+        if command == NAWS:
+            self.naws(payload)
+        else:
+            logger.info("Negotiate (%r got bytes)", len(data))
+
+    def _parse_coroutine(self) -> Generator[None, bytes, None]:
+        """
+        Parser state machine.
+        Every 'yield' expression returns the next byte.
+        """
+        while True:
+            d = yield
+
+            if d == int2byte(0):
+                pass  # NOP
+
+            # Go to state escaped.
+            elif d == IAC:
+                d2 = yield
+
+                if d2 == IAC:
+                    self.received_data(d2)
+
+                # Handle simple commands.
+                elif d2 in (NOP, DM, BRK, IP, AO, AYT, EC, EL, GA):
+                    self.command_received(d2, b"")
+
+                # Handle IAC-[DO/DONT/WILL/WONT] commands.
+                elif d2 in (DO, DONT, WILL, WONT):
+                    d3 = yield
+                    self.command_received(d2, d3)
+
+                # Subnegotiation
+                elif d2 == SB:
+                    # Consume everything until next IAC-SE
+                    data = []
+
+                    while True:
+                        d3 = yield
+
+                        if d3 == IAC:
+                            d4 = yield
+                            if d4 == SE:
+                                break
+                            else:
+                                data.append(d4)
+                        else:
+                            data.append(d3)
+
+                    self.negotiate(b"".join(data))
+            else:
+                self.received_data(d)
+
+    def feed(self, data: bytes) -> None:
+        """
+        Feed data to the parser.
+        """
+        for b in data:
+            self._parser.send(int2byte(b))

xonsh/vended_ptk/prompt_toolkit/contrib/telnet/server.py

@@ -0,0 +1,322 @@
+"""
+Telnet server.
+"""
+import asyncio
+import contextvars  # Requires Python3.7!
+import socket
+from asyncio import get_event_loop
+from typing import Awaitable, Callable, List, Optional, Set, TextIO, Tuple, cast
+
+from prompt_toolkit.application.current import create_app_session, get_app
+from prompt_toolkit.application.run_in_terminal import run_in_terminal
+from prompt_toolkit.data_structures import Size
+from prompt_toolkit.formatted_text import AnyFormattedText, to_formatted_text
+from prompt_toolkit.input.posix_pipe import PosixPipeInput
+from prompt_toolkit.output.vt100 import Vt100_Output
+from prompt_toolkit.renderer import print_formatted_text as print_formatted_text
+from prompt_toolkit.styles import BaseStyle, DummyStyle
+
+from .log import logger
+from .protocol import (
+    DO,
+    ECHO,
+    IAC,
+    LINEMODE,
+    MODE,
+    NAWS,
+    SB,
+    SE,
+    SUPPRESS_GO_AHEAD,
+    WILL,
+    TelnetProtocolParser,
+)
+
+__all__ = [
+    "TelnetServer",
+]
+
+
+def int2byte(number: int) -> bytes:
+    return bytes((number,))
+
+
+def _initialize_telnet(connection: socket.socket) -> None:
+    logger.info("Initializing telnet connection")
+
+    # Iac Do Linemode
+    connection.send(IAC + DO + LINEMODE)
+
+    # Suppress Go Ahead. (This seems important for Putty to do correct echoing.)
+    # This will allow bi-directional operation.
+    connection.send(IAC + WILL + SUPPRESS_GO_AHEAD)
+
+    # Iac sb
+    connection.send(IAC + SB + LINEMODE + MODE + int2byte(0) + IAC + SE)
+
+    # IAC Will Echo
+    connection.send(IAC + WILL + ECHO)
+
+    # Negotiate window size
+    connection.send(IAC + DO + NAWS)
+
+
+class _ConnectionStdout:
+    """
+    Wrapper around socket which provides `write` and `flush` methods for the
+    Vt100_Output output.
+    """
+
+    def __init__(self, connection: socket.socket, encoding: str) -> None:
+        self._encoding = encoding
+        self._connection = connection
+        self._errors = "strict"
+        self._buffer: List[bytes] = []
+
+    def write(self, data: str) -> None:
+        self._buffer.append(data.encode(self._encoding, errors=self._errors))
+        self.flush()
+
+    def flush(self) -> None:
+        try:
+            self._connection.send(b"".join(self._buffer))
+        except socket.error as e:
+            logger.warning("Couldn't send data over socket: %s" % e)
+
+        self._buffer = []
+
+    @property
+    def encoding(self) -> str:
+        return self._encoding
+
+    @property
+    def errors(self) -> str:
+        return self._errors
+
+
+class TelnetConnection:
+    """
+    Class that represents one Telnet connection.
+    """
+
+    def __init__(
+        self,
+        conn: socket.socket,
+        addr: Tuple[str, int],
+        interact: Callable[["TelnetConnection"], Awaitable[None]],
+        server: "TelnetServer",
+        encoding: str,
+        style: Optional[BaseStyle],
+    ) -> None:
+
+        self.conn = conn
+        self.addr = addr
+        self.interact = interact
+        self.server = server
+        self.encoding = encoding
+        self.style = style
+        self._closed = False
+
+        # Create "Output" object.
+        self.size = Size(rows=40, columns=79)
+
+        # Initialize.
+        _initialize_telnet(conn)
+
+        # Create input.
+        self.vt100_input = PosixPipeInput()
+
+        # Create output.
+        def get_size() -> Size:
+            return self.size
+
+        self.stdout = cast(TextIO, _ConnectionStdout(conn, encoding=encoding))
+        self.vt100_output = Vt100_Output(self.stdout, get_size, write_binary=False)
+
+        def data_received(data: bytes) -> None:
+            """ TelnetProtocolParser 'data_received' callback """
+            self.vt100_input.send_bytes(data)
+
+        def size_received(rows: int, columns: int) -> None:
+            """ TelnetProtocolParser 'size_received' callback """
+            self.size = Size(rows=rows, columns=columns)
+            get_app()._on_resize()
+
+        self.parser = TelnetProtocolParser(data_received, size_received)
+        self.context: Optional[contextvars.Context] = None
+
+    async def run_application(self) -> None:
+        """
+        Run application.
+        """
+
+        def handle_incoming_data() -> None:
+            data = self.conn.recv(1024)
+            if data:
+                self.feed(data)
+            else:
+                # Connection closed by client.
+                logger.info("Connection closed by client. %r %r" % self.addr)
+                self.close()
+
+        async def run() -> None:
+            # Add reader.
+            loop = get_event_loop()
+            loop.add_reader(self.conn, handle_incoming_data)
+
+            try:
+                await self.interact(self)
+            except Exception as e:
+                print("Got %s" % type(e).__name__, e)
+                import traceback
+
+                traceback.print_exc()
+                raise
+            finally:
+                self.close()
+
+        with create_app_session(input=self.vt100_input, output=self.vt100_output):
+            self.context = contextvars.copy_context()
+            await run()
+
+    def feed(self, data: bytes) -> None:
+        """
+        Handler for incoming data. (Called by TelnetServer.)
+        """
+        self.parser.feed(data)
+
+    def close(self) -> None:
+        """
+        Closed by client.
+        """
+        if not self._closed:
+            self._closed = True
+
+            self.vt100_input.close()
+            get_event_loop().remove_reader(self.conn)
+            self.conn.close()
+
+    def send(self, formatted_text: AnyFormattedText) -> None:
+        """
+        Send text to the client.
+        """
+        formatted_text = to_formatted_text(formatted_text)
+        print_formatted_text(
+            self.vt100_output, formatted_text, self.style or DummyStyle()
+        )
+
+    def send_above_prompt(self, formatted_text: AnyFormattedText) -> None:
+        """
+        Send text to the client.
+        This is asynchronous, returns a `Future`.
+        """
+        formatted_text = to_formatted_text(formatted_text)
+        return self._run_in_terminal(lambda: self.send(formatted_text))
+
+    def _run_in_terminal(self, func: Callable[[], None]) -> None:
+        # Make sure that when an application was active for this connection,
+        # that we print the text above the application.
+        if self.context:
+            self.context.run(run_in_terminal, func)
+        else:
+            raise RuntimeError("Called _run_in_terminal outside `run_application`.")
+
+    def erase_screen(self) -> None:
+        """
+        Erase the screen and move the cursor to the top.
+        """
+        self.vt100_output.erase_screen()
+        self.vt100_output.cursor_goto(0, 0)
+        self.vt100_output.flush()
+
+
+async def _dummy_interact(connection: TelnetConnection) -> None:
+    pass
+
+
+class TelnetServer:
+    """
+    Telnet server implementation.
+    """
+
+    def __init__(
+        self,
+        host: str = "127.0.0.1",
+        port: int = 23,
+        interact: Callable[[TelnetConnection], Awaitable[None]] = _dummy_interact,
+        encoding: str = "utf-8",
+        style: Optional[BaseStyle] = None,
+    ) -> None:
+
+        self.host = host
+        self.port = port
+        self.interact = interact
+        self.encoding = encoding
+        self.style = style
+        self._application_tasks: List[asyncio.Task] = []
+
+        self.connections: Set[TelnetConnection] = set()
+        self._listen_socket: Optional[socket.socket] = None
+
+    @classmethod
+    def _create_socket(cls, host: str, port: int) -> socket.socket:
+        # Create and bind socket
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        s.bind((host, port))
+
+        s.listen(4)
+        return s
+
+    def start(self) -> None:
+        """
+        Start the telnet server.
+        Don't forget to call `loop.run_forever()` after doing this.
+        """
+        self._listen_socket = self._create_socket(self.host, self.port)
+        logger.info(
+            "Listening for telnet connections on %s port %r", self.host, self.port
+        )
+
+        get_event_loop().add_reader(self._listen_socket, self._accept)
+
+    async def stop(self) -> None:
+        if self._listen_socket:
+            get_event_loop().remove_reader(self._listen_socket)
+            self._listen_socket.close()
+
+        # Wait for all applications to finish.
+        for t in self._application_tasks:
+            t.cancel()
+
+        for t in self._application_tasks:
+            await t
+
+    def _accept(self) -> None:
+        """
+        Accept new incoming connection.
+        """
+        if self._listen_socket is None:
+            return  # Should not happen. `_accept` is called after `start`.
+
+        conn, addr = self._listen_socket.accept()
+        logger.info("New connection %r %r", *addr)
+
+        connection = TelnetConnection(
+            conn, addr, self.interact, self, encoding=self.encoding, style=self.style
+        )
+        self.connections.add(connection)
+
+        # Run application for this connection.
+        async def run() -> None:
+            logger.info("Starting interaction %r %r", *addr)
+            try:
+                await connection.run_application()
+            except Exception as e:
+                print(e)
+            finally:
+                self.connections.remove(connection)
+                self._application_tasks.remove(task)
+                logger.info("Stopping interaction %r %r", *addr)
+
+        task = get_event_loop().create_task(run())
+        self._application_tasks.append(task)

xonsh/vended_ptk/prompt_toolkit/data_structures.py

@@ -0,0 +1,10 @@
+from typing import NamedTuple
+
+__all__ = [
+    "Point",
+    "Size",
+]
+
+
+Point = NamedTuple("Point", [("x", int), ("y", int)])
+Size = NamedTuple("Size", [("rows", int), ("columns", int)])

xonsh/vended_ptk/prompt_toolkit/enums.py

@@ -0,0 +1,17 @@
+from enum import Enum
+
+
+class EditingMode(Enum):
+    # The set of key bindings that is active.
+    VI = "VI"
+    EMACS = "EMACS"
+
+
+#: Name of the search buffer.
+SEARCH_BUFFER = "SEARCH_BUFFER"
+
+#: Name of the default buffer.
+DEFAULT_BUFFER = "DEFAULT_BUFFER"
+
+#: Name of the system buffer.
+SYSTEM_BUFFER = "SYSTEM_BUFFER"

xonsh/vended_ptk/prompt_toolkit/eventloop/__init__.py

@@ -0,0 +1,26 @@
+from .async_generator import generator_to_async_generator
+from .inputhook import (
+    InputHookContext,
+    InputHookSelector,
+    new_eventloop_with_inputhook,
+    set_eventloop_with_inputhook,
+)
+from .utils import (
+    call_soon_threadsafe,
+    get_traceback_from_context,
+    run_in_executor_with_context,
+)
+
+__all__ = [
+    # Async generator
+    "generator_to_async_generator",
+    # Utils.
+    "run_in_executor_with_context",
+    "call_soon_threadsafe",
+    "get_traceback_from_context",
+    # Inputhooks.
+    "new_eventloop_with_inputhook",
+    "set_eventloop_with_inputhook",
+    "InputHookSelector",
+    "InputHookContext",
+]

xonsh/vended_ptk/prompt_toolkit/eventloop/async_context_manager.py

@@ -0,0 +1,123 @@
+"""
+@asynccontextmanager code, copied from Python 3.7's contextlib.
+For usage in Python 3.6.
+"""
+import _collections_abc
+import abc
+from functools import wraps
+
+__all__ = ["asynccontextmanager"]
+
+
+class AbstractAsyncContextManager(abc.ABC):
+
+    """An abstract base class for asynchronous context managers."""
+
+    async def __aenter__(self):
+        """Return `self` upon entering the runtime context."""
+        return self
+
+    @abc.abstractmethod
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        """Raise any exception triggered within the runtime context."""
+        return None
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is AbstractAsyncContextManager:
+            return _collections_abc._check_methods(C, "__aenter__", "__aexit__")
+        return NotImplemented
+
+
+class _GeneratorContextManagerBase:
+    """Shared functionality for @contextmanager and @asynccontextmanager."""
+
+    def __init__(self, func, args, kwds):
+        self.gen = func(*args, **kwds)
+        self.func, self.args, self.kwds = func, args, kwds
+        # Issue 19330: ensure context manager instances have good docstrings
+        doc = getattr(func, "__doc__", None)
+        if doc is None:
+            doc = type(self).__doc__
+        self.__doc__ = doc
+        # Unfortunately, this still doesn't provide good help output when
+        # inspecting the created context manager instances, since pydoc
+        # currently bypasses the instance docstring and shows the docstring
+        # for the class instead.
+        # See http://bugs.python.org/issue19404 for more details.
+
+
+class _AsyncGeneratorContextManager(
+    _GeneratorContextManagerBase, AbstractAsyncContextManager
+):
+    """Helper for @asynccontextmanager."""
+
+    async def __aenter__(self):
+        try:
+            return await self.gen.__anext__()
+        except StopAsyncIteration:
+            raise RuntimeError("generator didn't yield") from None
+
+    async def __aexit__(self, typ, value, traceback):
+        if typ is None:
+            try:
+                await self.gen.__anext__()
+            except StopAsyncIteration:
+                return
+            else:
+                raise RuntimeError("generator didn't stop")
+        else:
+            if value is None:
+                value = typ()
+            # See _GeneratorContextManager.__exit__ for comments on subtleties
+            # in this implementation
+            try:
+                await self.gen.athrow(typ, value, traceback)
+                raise RuntimeError("generator didn't stop after athrow()")
+            except StopAsyncIteration as exc:
+                return exc is not value
+            except RuntimeError as exc:
+                if exc is value:
+                    return False
+                # Avoid suppressing if a StopIteration exception
+                # was passed to throw() and later wrapped into a RuntimeError
+                # (see PEP 479 for sync generators; async generators also
+                # have this behavior). But do this only if the exception wrapped
+                # by the RuntimeError is actully Stop(Async)Iteration (see
+                # issue29692).
+                if isinstance(value, (StopIteration, StopAsyncIteration)):
+                    if exc.__cause__ is value:
+                        return False
+                raise
+            except BaseException as exc:
+                if exc is not value:
+                    raise
+
+
+def asynccontextmanager(func):
+    """@asynccontextmanager decorator.
+    Typical usage:
+        @asynccontextmanager
+        async def some_async_generator(<arguments>):
+            <setup>
+            try:
+                yield <value>
+            finally:
+                <cleanup>
+    This makes this:
+        async with some_async_generator(<arguments>) as <variable>:
+            <body>
+    equivalent to this:
+        <setup>
+        try:
+            <variable> = <value>
+            <body>
+        finally:
+            <cleanup>
+    """
+
+    @wraps(func)
+    def helper(*args, **kwds):
+        return _AsyncGeneratorContextManager(func, args, kwds)
+
+    return helper

xonsh/vended_ptk/prompt_toolkit/eventloop/async_generator.py

@@ -0,0 +1,67 @@
+"""
+Implementation for async generators.
+"""
+from asyncio import Queue, get_event_loop
+from typing import AsyncGenerator, Callable, Iterable, TypeVar, Union
+
+from .utils import run_in_executor_with_context
+
+__all__ = [
+    "generator_to_async_generator",
+]
+
+
+_T = TypeVar("_T")
+
+
+class _Done:
+    pass
+
+
+async def generator_to_async_generator(
+    get_iterable: Callable[[], Iterable[_T]]
+) -> AsyncGenerator[_T, None]:
+    """
+    Turn a generator or iterable into an async generator.
+
+    This works by running the generator in a background thread.
+
+    :param get_iterable: Function that returns a generator or iterable when
+        called.
+    """
+    quitting = False
+    _done = _Done()
+    q: Queue[Union[_T, _Done]] = Queue()
+    loop = get_event_loop()
+
+    def runner() -> None:
+        """
+        Consume the generator in background thread.
+        When items are received, they'll be pushed to the queue.
+        """
+        try:
+            for item in get_iterable():
+                loop.call_soon_threadsafe(q.put_nowait, item)
+
+                # When this async generator was cancelled (closed), stop this
+                # thread.
+                if quitting:
+                    break
+
+        finally:
+            loop.call_soon_threadsafe(q.put_nowait, _done)
+
+    # Start background thread.
+    run_in_executor_with_context(runner)
+
+    try:
+        while True:
+            item = await q.get()
+            if isinstance(item, _Done):
+                break
+            else:
+                yield item
+    finally:
+        # When this async generator is closed (GeneratorExit exception, stop
+        # the background thread as well. - we don't need that anymore.)
+        quitting = True

xonsh/vended_ptk/prompt_toolkit/eventloop/dummy_contextvars.py

@@ -0,0 +1,46 @@
+"""
+Dummy contextvars implementation, to make prompt_toolkit work on Python 3.6.
+
+As long as there is only one application running at a time, we don't need the
+real contextvars. So, stuff like the telnet-server and so on requires 3.7.
+"""
+from typing import Any, Callable, Generic, Optional, TypeVar
+
+
+def copy_context() -> "Context":
+    return Context()
+
+
+_T = TypeVar("_T")
+
+
+class Context:
+    def run(self, callable: Callable[..., _T], *args: Any, **kwargs: Any) -> _T:
+        return callable(*args, **kwargs)
+
+
+class Token(Generic[_T]):
+    pass
+
+
+class ContextVar(Generic[_T]):
+    def __init__(self, name: str, *, default: Optional[_T] = None) -> None:
+        self._name = name
+        self._value = default
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    def get(self, default: Optional[_T] = None) -> _T:
+        result = self._value or default
+        if result is None:
+            raise LookupError
+        return result
+
+    def set(self, value: _T) -> Token[_T]:
+        self._value = value
+        return Token()
+
+    def reset(self, token: Token[_T]) -> None:
+        pass

xonsh/vended_ptk/prompt_toolkit/eventloop/inputhook.py

@@ -0,0 +1,170 @@
+"""
+Similar to `PyOS_InputHook` of the Python API, we can plug in an input hook in
+the asyncio event loop.
+
+The way this works is by using a custom 'selector' that runs the other event
+loop until the real selector is ready.
+
+It's the responsibility of this event hook to return when there is input ready.
+There are two ways to detect when input is ready:
+
+The inputhook itself is a callable that receives an `InputHookContext`. This
+callable should run the other event loop, and return when the main loop has
+stuff to do. There are two ways to detect when to return:
+
+- Call the `input_is_ready` method periodically. Quit when this returns `True`.
+
+- Add the `fileno` as a watch to the external eventloop. Quit when file descriptor
+  becomes readable. (But don't read from it.)
+
+  Note that this is not the same as checking for `sys.stdin.fileno()`. The
+  eventloop of prompt-toolkit allows thread-based executors, for example for
+  asynchronous autocompletion. When the completion for instance is ready, we
+  also want prompt-toolkit to gain control again in order to display that.
+"""
+import asyncio
+import os
+import select
+import selectors
+import threading
+from asyncio import AbstractEventLoop, get_event_loop
+from selectors import BaseSelector
+from typing import Callable
+
+from prompt_toolkit.utils import is_windows
+
+__all__ = [
+    "new_eventloop_with_inputhook",
+    "set_eventloop_with_inputhook",
+    "InputHookSelector",
+    "InputHookContext",
+]
+
+
+def new_eventloop_with_inputhook(
+    inputhook: Callable[["InputHookContext"], None]
+) -> AbstractEventLoop:
+    """
+    Create a new event loop with the given inputhook.
+    """
+    selector = InputHookSelector(selectors.DefaultSelector(), inputhook)
+    loop = asyncio.SelectorEventLoop(selector)
+    return loop
+
+
+def set_eventloop_with_inputhook(
+    inputhook: Callable[["InputHookContext"], None]
+) -> AbstractEventLoop:
+    """
+    Create a new event loop with the given inputhook, and activate it.
+    """
+    loop = new_eventloop_with_inputhook(inputhook)
+    asyncio.set_event_loop(loop)
+    return loop
+
+
+class InputHookSelector(BaseSelector):
+    """
+    Usage:
+
+        selector = selectors.SelectSelector()
+        loop = asyncio.SelectorEventLoop(InputHookSelector(selector, inputhook))
+        asyncio.set_event_loop(loop)
+    """
+
+    def __init__(
+        self, selector: BaseSelector, inputhook: Callable[["InputHookContext"], None]
+    ) -> None:
+        self.selector = selector
+        self.inputhook = inputhook
+        self._r, self._w = os.pipe()
+
+    def register(self, fileobj, events, data=None):
+        return self.selector.register(fileobj, events, data=data)
+
+    def unregister(self, fileobj):
+        return self.selector.unregister(fileobj)
+
+    def modify(self, fileobj, events, data=None):
+        return self.selector.modify(fileobj, events, data=None)
+
+    def select(self, timeout=None):
+        # If there are tasks in the current event loop,
+        # don't run the input hook.
+        if len(get_event_loop()._ready) > 0:
+            return self.selector.select(timeout=timeout)
+
+        ready = False
+        result = None
+
+        # Run selector in other thread.
+        def run_selector() -> None:
+            nonlocal ready, result
+            result = self.selector.select(timeout=timeout)
+            os.write(self._w, b"x")
+            ready = True
+
+        th = threading.Thread(target=run_selector)
+        th.start()
+
+        def input_is_ready() -> bool:
+            return ready
+
+        # Call inputhook.
+        # The inputhook function is supposed to return when our selector
+        # becomes ready. The inputhook can do that by registering the fd in its
+        # own loop, or by checking the `input_is_ready` function regularly.
+        self.inputhook(InputHookContext(self._r, input_is_ready))
+
+        # Flush the read end of the pipe.
+        try:
+            # Before calling 'os.read', call select.select. This is required
+            # when the gevent monkey patch has been applied. 'os.read' is never
+            # monkey patched and won't be cooperative, so that would block all
+            # other select() calls otherwise.
+            # See: http://www.gevent.org/gevent.os.html
+
+            # Note: On Windows, this is apparently not an issue.
+            #       However, if we would ever want to add a select call, it
+            #       should use `windll.kernel32.WaitForMultipleObjects`,
+            #       because `select.select` can't wait for a pipe on Windows.
+            if not is_windows():
+                select.select([self._r], [], [], None)
+
+            os.read(self._r, 1024)
+        except OSError:
+            # This happens when the window resizes and a SIGWINCH was received.
+            # We get 'Error: [Errno 4] Interrupted system call'
+            # Just ignore.
+            pass
+
+        # Wait for the real selector to be done.
+        th.join()
+        return result
+
+    def close(self) -> None:
+        """
+        Clean up resources.
+        """
+        if self._r:
+            os.close(self._r)
+            os.close(self._w)
+
+        self._r = self._w = -1
+        self.selector.close()
+
+    def get_map(self):
+        return self.selector.get_map()
+
+
+class InputHookContext:
+    """
+    Given as a parameter to the inputhook.
+    """
+
+    def __init__(self, fileno: int, input_is_ready: Callable[[], bool]) -> None:
+        self._fileno = fileno
+        self.input_is_ready = input_is_ready
+
+    def fileno(self) -> int:
+        return self._fileno

xonsh/vended_ptk/prompt_toolkit/eventloop/utils.py

@@ -0,0 +1,100 @@
+import sys
+import time
+from asyncio import AbstractEventLoop, get_event_loop
+from types import TracebackType
+from typing import Any, Awaitable, Callable, Dict, Optional, TypeVar
+
+try:
+    import contextvars
+except ImportError:
+    from . import dummy_contextvars as contextvars  # type: ignore
+
+__all__ = [
+    "run_in_executor_with_context",
+    "call_soon_threadsafe",
+    "get_traceback_from_context",
+]
+
+_T = TypeVar("_T")
+
+
+def run_in_executor_with_context(
+    func: Callable[..., _T], *args: Any, loop: Optional[AbstractEventLoop] = None
+) -> Awaitable[_T]:
+    """
+    Run a function in an executor, but make sure it uses the same contextvars.
+    This is required so that the function will see the right application.
+
+    See also: https://bugs.python.org/issue34014
+    """
+    loop = loop or get_event_loop()
+    ctx: contextvars.Context = contextvars.copy_context()
+
+    return loop.run_in_executor(None, ctx.run, func, *args)
+
+
+def call_soon_threadsafe(
+    func: Callable[[], None],
+    max_postpone_time: Optional[float] = None,
+    loop: Optional[AbstractEventLoop] = None,
+) -> None:
+    """
+    Wrapper around asyncio's `call_soon_threadsafe`.
+
+    This takes a `max_postpone_time` which can be used to tune the urgency of
+    the method.
+
+    Asyncio runs tasks in first-in-first-out. However, this is not what we
+    want for the render function of the prompt_toolkit UI. Rendering is
+    expensive, but since the UI is invalidated very often, in some situations
+    we render the UI too often, so much that the rendering CPU usage slows down
+    the rest of the processing of the application.  (Pymux is an example where
+    we have to balance the CPU time spend on rendering the UI, and parsing
+    process output.)
+    However, we want to set a deadline value, for when the rendering should
+    happen. (The UI should stay responsive).
+    """
+    loop2 = loop or get_event_loop()
+
+    # If no `max_postpone_time` has been given, schedule right now.
+    if max_postpone_time is None:
+        loop2.call_soon_threadsafe(func)
+        return
+
+    max_postpone_until = time.time() + max_postpone_time
+
+    def schedule() -> None:
+        # When there are no other tasks scheduled in the event loop. Run it
+        # now.
+        # Notice: uvloop doesn't have this _ready attribute. In that case,
+        #         always call immediately.
+        if not getattr(loop2, "_ready", []):  # type: ignore
+            func()
+            return
+
+        # If the timeout expired, run this now.
+        if time.time() > max_postpone_until:
+            func()
+            return
+
+        # Schedule again for later.
+        loop2.call_soon_threadsafe(schedule)
+
+    loop2.call_soon_threadsafe(schedule)
+
+
+def get_traceback_from_context(context: Dict[str, Any]) -> Optional[TracebackType]:
+    """
+    Get the traceback object from the context.
+    """
+    exception = context.get("exception")
+    if exception:
+        if hasattr(exception, "__traceback__"):
+            return exception.__traceback__
+        else:
+            # call_exception_handler() is usually called indirectly
+            # from an except block. If it's not the case, the traceback
+            # is undefined...
+            return sys.exc_info()[2]
+
+    return None

xonsh/vended_ptk/prompt_toolkit/eventloop/win32.py

@@ -0,0 +1,61 @@
+from ctypes import pointer, windll
+from ctypes.wintypes import BOOL, DWORD, HANDLE
+from typing import List, Optional
+
+from prompt_toolkit.win32_types import SECURITY_ATTRIBUTES
+
+__all__ = ["wait_for_handles", "create_win32_event"]
+
+
+WAIT_TIMEOUT = 0x00000102
+INFINITE = -1
+
+
+def wait_for_handles(
+    handles: List[HANDLE], timeout: int = INFINITE
+) -> Optional[HANDLE]:
+    """
+    Waits for multiple handles. (Similar to 'select') Returns the handle which is ready.
+    Returns `None` on timeout.
+    http://msdn.microsoft.com/en-us/library/windows/desktop/ms687025(v=vs.85).aspx
+
+    Note that handles should be a list of `HANDLE` objects, not integers. See
+    this comment in the patch by @quark-zju for the reason why:
+
+        ''' Make sure HANDLE on Windows has a correct size
+
+        Previously, the type of various HANDLEs are native Python integer
+        types. The ctypes library will treat them as 4-byte integer when used
+        in function arguments. On 64-bit Windows, HANDLE is 8-byte and usually
+        a small integer. Depending on whether the extra 4 bytes are zero-ed out
+        or not, things can happen to work, or break. '''
+
+    This function returns either `None` or one of the given `HANDLE` objects.
+    (The return value can be tested with the `is` operator.)
+    """
+    arrtype = HANDLE * len(handles)
+    handle_array = arrtype(*handles)
+
+    ret = windll.kernel32.WaitForMultipleObjects(
+        len(handle_array), handle_array, BOOL(False), DWORD(timeout)
+    )
+
+    if ret == WAIT_TIMEOUT:
+        return None
+    else:
+        return handles[ret]
+
+
+def create_win32_event() -> HANDLE:
+    """
+    Creates a Win32 unnamed Event .
+    http://msdn.microsoft.com/en-us/library/windows/desktop/ms682396(v=vs.85).aspx
+    """
+    return HANDLE(
+        windll.kernel32.CreateEventA(
+            pointer(SECURITY_ATTRIBUTES()),
+            BOOL(True),  # Manual reset event.
+            BOOL(False),  # Initial state.
+            None,  # Unnamed event object.
+        )
+    )

xonsh/vended_ptk/prompt_toolkit/filters/__init__.py

@@ -0,0 +1,22 @@
+"""
+Filters decide whether something is active or not (they decide about a boolean
+state). This is used to enable/disable features, like key bindings, parts of
+the layout and other stuff. For instance, we could have a `HasSearch` filter
+attached to some part of the layout, in order to show that part of the user
+interface only while the user is searching.
+
+Filters are made to avoid having to attach callbacks to all event in order to
+propagate state. However, they are lazy, they don't automatically propagate the
+state of what they are observing. Only when a filter is called (it's actually a
+callable), it will calculate its value. So, its not really reactive
+programming, but it's made to fit for this framework.
+
+Filters can be chained using ``&`` and ``|`` operations, and inverted using the
+``~`` operator, for instance::
+
+    filter = has_focus('default') & ~ has_selection
+"""
+from .app import *
+from .base import Always, Condition, Filter, FilterOrBool, Never
+from .cli import *
+from .utils import is_true, to_filter

xonsh/vended_ptk/prompt_toolkit/filters/app.py

@@ -0,0 +1,383 @@
+"""
+Filters that accept a `Application` as argument.
+"""
+from typing import TYPE_CHECKING, cast
+
+from prompt_toolkit.application.current import get_app
+from prompt_toolkit.cache import memoized
+from prompt_toolkit.enums import EditingMode
+
+from .base import Condition
+
+if TYPE_CHECKING:
+    from prompt_toolkit.layout.layout import FocusableElement
+
+
+__all__ = [
+    "has_arg",
+    "has_completions",
+    "completion_is_selected",
+    "has_focus",
+    "buffer_has_focus",
+    "has_selection",
+    "has_validation_error",
+    "is_done",
+    "is_read_only",
+    "is_multiline",
+    "renderer_height_is_known",
+    "in_editing_mode",
+    "in_paste_mode",
+    "vi_mode",
+    "vi_navigation_mode",
+    "vi_insert_mode",
+    "vi_insert_multiple_mode",
+    "vi_replace_mode",
+    "vi_selection_mode",
+    "vi_waiting_for_text_object_mode",
+    "vi_digraph_mode",
+    "vi_recording_macro",
+    "emacs_mode",
+    "emacs_insert_mode",
+    "emacs_selection_mode",
+    "shift_selection_mode",
+    "is_searching",
+    "control_is_searchable",
+    "vi_search_direction_reversed",
+]
+
+
+@memoized()
+def has_focus(value: "FocusableElement") -> Condition:
+    """
+    Enable when this buffer has the focus.
+    """
+    from prompt_toolkit.buffer import Buffer
+    from prompt_toolkit.layout.controls import UIControl
+    from prompt_toolkit.layout.containers import to_container, Window, Container
+    from prompt_toolkit.layout import walk
+
+    if isinstance(value, str):
+
+        def test() -> bool:
+            return get_app().current_buffer.name == value
+
+    elif isinstance(value, Buffer):
+
+        def test() -> bool:
+            return get_app().current_buffer == value
+
+    elif isinstance(value, UIControl):
+
+        def test() -> bool:
+            return get_app().layout.current_control == value
+
+    else:
+        value = to_container(value)
+
+        if isinstance(value, Window):
+
+            def test() -> bool:
+                return get_app().layout.current_window == value
+
+        else:
+
+            def test() -> bool:
+                # Consider focused when any window inside this container is
+                # focused.
+                current_window = get_app().layout.current_window
+
+                for c in walk(cast(Container, value)):
+                    if isinstance(c, Window) and c == current_window:
+                        return True
+                return False
+
+    @Condition
+    def has_focus_filter() -> bool:
+        return test()
+
+    return has_focus_filter
+
+
+@Condition
+def buffer_has_focus() -> bool:
+    """
+    Enabled when the currently focused control is a `BufferControl`.
+    """
+    return get_app().layout.buffer_has_focus
+
+
+@Condition
+def has_selection() -> bool:
+    """
+    Enable when the current buffer has a selection.
+    """
+    return bool(get_app().current_buffer.selection_state)
+
+
+@Condition
+def has_completions() -> bool:
+    """
+    Enable when the current buffer has completions.
+    """
+    state = get_app().current_buffer.complete_state
+    return state is not None and len(state.completions) > 0
+
+
+@Condition
+def completion_is_selected() -> bool:
+    """
+    True when the user selected a completion.
+    """
+    complete_state = get_app().current_buffer.complete_state
+    return complete_state is not None and complete_state.current_completion is not None
+
+
+@Condition
+def is_read_only() -> bool:
+    """
+    True when the current buffer is read only.
+    """
+    return get_app().current_buffer.read_only()
+
+
+@Condition
+def is_multiline() -> bool:
+    """
+    True when the current buffer has been marked as multiline.
+    """
+    return get_app().current_buffer.multiline()
+
+
+@Condition
+def has_validation_error() -> bool:
+    " Current buffer has validation error.  "
+    return get_app().current_buffer.validation_error is not None
+
+
+@Condition
+def has_arg() -> bool:
+    " Enable when the input processor has an 'arg'. "
+    return get_app().key_processor.arg is not None
+
+
+@Condition
+def is_done() -> bool:
+    """
+    True when the CLI is returning, aborting or exiting.
+    """
+    return get_app().is_done
+
+
+@Condition
+def renderer_height_is_known() -> bool:
+    """
+    Only True when the renderer knows it's real height.
+
+    (On VT100 terminals, we have to wait for a CPR response, before we can be
+    sure of the available height between the cursor position and the bottom of
+    the terminal. And usually it's nicer to wait with drawing bottom toolbars
+    until we receive the height, in order to avoid flickering -- first drawing
+    somewhere in the middle, and then again at the bottom.)
+    """
+    return get_app().renderer.height_is_known
+
+
+@memoized()
+def in_editing_mode(editing_mode: EditingMode) -> Condition:
+    """
+    Check whether a given editing mode is active. (Vi or Emacs.)
+    """
+
+    @Condition
+    def in_editing_mode_filter() -> bool:
+        return get_app().editing_mode == editing_mode
+
+    return in_editing_mode_filter
+
+
+@Condition
+def in_paste_mode() -> bool:
+    return get_app().paste_mode()
+
+
+@Condition
+def vi_mode() -> bool:
+    return get_app().editing_mode == EditingMode.VI
+
+
+@Condition
+def vi_navigation_mode() -> bool:
+    """
+    Active when the set for Vi navigation key bindings are active.
+    """
+    from prompt_toolkit.key_binding.vi_state import InputMode
+
+    app = get_app()
+
+    if (
+        app.editing_mode != EditingMode.VI
+        or app.vi_state.operator_func
+        or app.vi_state.waiting_for_digraph
+        or app.current_buffer.selection_state
+    ):
+        return False
+
+    return (
+        app.vi_state.input_mode == InputMode.NAVIGATION
+        or app.vi_state.temporary_navigation_mode
+        or app.current_buffer.read_only()
+    )
+
+
+@Condition
+def vi_insert_mode() -> bool:
+    from prompt_toolkit.key_binding.vi_state import InputMode
+
+    app = get_app()
+
+    if (
+        app.editing_mode != EditingMode.VI
+        or app.vi_state.operator_func
+        or app.vi_state.waiting_for_digraph
+        or app.current_buffer.selection_state
+        or app.vi_state.temporary_navigation_mode
+        or app.current_buffer.read_only()
+    ):
+        return False
+
+    return app.vi_state.input_mode == InputMode.INSERT
+
+
+@Condition
+def vi_insert_multiple_mode() -> bool:
+    from prompt_toolkit.key_binding.vi_state import InputMode
+
+    app = get_app()
+
+    if (
+        app.editing_mode != EditingMode.VI
+        or app.vi_state.operator_func
+        or app.vi_state.waiting_for_digraph
+        or app.current_buffer.selection_state
+        or app.vi_state.temporary_navigation_mode
+        or app.current_buffer.read_only()
+    ):
+        return False
+
+    return app.vi_state.input_mode == InputMode.INSERT_MULTIPLE
+
+
+@Condition
+def vi_replace_mode() -> bool:
+    from prompt_toolkit.key_binding.vi_state import InputMode
+
+    app = get_app()
+
+    if (
+        app.editing_mode != EditingMode.VI
+        or app.vi_state.operator_func
+        or app.vi_state.waiting_for_digraph
+        or app.current_buffer.selection_state
+        or app.vi_state.temporary_navigation_mode
+        or app.current_buffer.read_only()
+    ):
+        return False
+
+    return app.vi_state.input_mode == InputMode.REPLACE
+
+
+@Condition
+def vi_selection_mode() -> bool:
+    app = get_app()
+    if app.editing_mode != EditingMode.VI:
+        return False
+
+    return bool(app.current_buffer.selection_state)
+
+
+@Condition
+def vi_waiting_for_text_object_mode() -> bool:
+    app = get_app()
+    if app.editing_mode != EditingMode.VI:
+        return False
+
+    return app.vi_state.operator_func is not None
+
+
+@Condition
+def vi_digraph_mode() -> bool:
+    app = get_app()
+    if app.editing_mode != EditingMode.VI:
+        return False
+
+    return app.vi_state.waiting_for_digraph
+
+
+@Condition
+def vi_recording_macro() -> bool:
+    " When recording a Vi macro. "
+    app = get_app()
+    if app.editing_mode != EditingMode.VI:
+        return False
+
+    return app.vi_state.recording_register is not None
+
+
+@Condition
+def emacs_mode() -> bool:
+    " When the Emacs bindings are active. "
+    return get_app().editing_mode == EditingMode.EMACS
+
+
+@Condition
+def emacs_insert_mode() -> bool:
+    app = get_app()
+    if (
+        app.editing_mode != EditingMode.EMACS
+        or app.current_buffer.selection_state
+        or app.current_buffer.read_only()
+    ):
+        return False
+    return True
+
+
+@Condition
+def emacs_selection_mode() -> bool:
+    app = get_app()
+    return bool(
+        app.editing_mode == EditingMode.EMACS and app.current_buffer.selection_state
+    )
+
+
+@Condition
+def shift_selection_mode() -> bool:
+    app = get_app()
+    return bool(
+        app.current_buffer.selection_state
+        and app.current_buffer.selection_state.shift_mode
+    )
+
+
+@Condition
+def is_searching() -> bool:
+    " When we are searching. "
+    app = get_app()
+    return app.layout.is_searching
+
+
+@Condition
+def control_is_searchable() -> bool:
+    " When the current UIControl is searchable. "
+    from prompt_toolkit.layout.controls import BufferControl
+
+    control = get_app().layout.current_control
+
+    return (
+        isinstance(control, BufferControl) and control.search_buffer_control is not None
+    )
+
+
+@Condition
+def vi_search_direction_reversed() -> bool:
+    " When the '/' and '?' key bindings for Vi-style searching have been reversed. "
+    return get_app().reverse_vi_search_direction()

xonsh/vended_ptk/prompt_toolkit/filters/base.py

@@ -0,0 +1,217 @@
+from abc import ABCMeta, abstractmethod
+from typing import Callable, Dict, Iterable, List, Tuple, Union, cast
+
+__all__ = ["Filter", "Never", "Always", "Condition", "FilterOrBool"]
+
+
+class Filter(metaclass=ABCMeta):
+    """
+    Base class for any filter to activate/deactivate a feature, depending on a
+    condition.
+
+    The return value of ``__call__`` will tell if the feature should be active.
+    """
+
+    @abstractmethod
+    def __call__(self) -> bool:
+        """
+        The actual call to evaluate the filter.
+        """
+        return True
+
+    def __and__(self, other: "Filter") -> "Filter":
+        """
+        Chaining of filters using the & operator.
+        """
+        return _and_cache[self, other]
+
+    def __or__(self, other: "Filter") -> "Filter":
+        """
+        Chaining of filters using the | operator.
+        """
+        return _or_cache[self, other]
+
+    def __invert__(self) -> "Filter":
+        """
+        Inverting of filters using the ~ operator.
+        """
+        return _invert_cache[self]
+
+    def __bool__(self) -> None:
+        """
+        By purpose, we don't allow bool(...) operations directly on a filter,
+        because the meaning is ambiguous.
+
+        Executing a filter has to be done always by calling it. Providing
+        defaults for `None` values should be done through an `is None` check
+        instead of for instance ``filter1 or Always()``.
+        """
+        raise ValueError(
+            "The truth value of a Filter is ambiguous. "
+            "Instead, call it as a function."
+        )
+
+
+class _AndCache(Dict[Tuple[Filter, Filter], "_AndList"]):
+    """
+    Cache for And operation between filters.
+    (Filter classes are stateless, so we can reuse them.)
+
+    Note: This could be a memory leak if we keep creating filters at runtime.
+          If that is True, the filters should be weakreffed (not the tuple of
+          filters), and tuples should be removed when one of these filters is
+          removed. In practise however, there is a finite amount of filters.
+    """
+
+    def __missing__(self, filters: Tuple[Filter, Filter]) -> Filter:
+        a, b = filters
+        assert isinstance(b, Filter), "Expecting filter, got %r" % b
+
+        if isinstance(b, Always) or isinstance(a, Never):
+            return a
+        elif isinstance(b, Never) or isinstance(a, Always):
+            return b
+
+        result = _AndList(filters)
+        self[filters] = result
+        return result
+
+
+class _OrCache(Dict[Tuple[Filter, Filter], "_OrList"]):
+    """ Cache for Or operation between filters. """
+
+    def __missing__(self, filters: Tuple[Filter, Filter]) -> Filter:
+        a, b = filters
+        assert isinstance(b, Filter), "Expecting filter, got %r" % b
+
+        if isinstance(b, Always) or isinstance(a, Never):
+            return b
+        elif isinstance(b, Never) or isinstance(a, Always):
+            return a
+
+        result = _OrList(filters)
+        self[filters] = result
+        return result
+
+
+class _InvertCache(Dict[Filter, "_Invert"]):
+    """ Cache for inversion operator. """
+
+    def __missing__(self, filter: Filter) -> Filter:
+        result = _Invert(filter)
+        self[filter] = result
+        return result
+
+
+_and_cache = _AndCache()
+_or_cache = _OrCache()
+_invert_cache = _InvertCache()
+
+
+class _AndList(Filter):
+    """
+    Result of &-operation between several filters.
+    """
+
+    def __init__(self, filters: Iterable[Filter]) -> None:
+        self.filters: List[Filter] = []
+
+        for f in filters:
+            if isinstance(f, _AndList):  # Turn nested _AndLists into one.
+                self.filters.extend(cast(_AndList, f).filters)
+            else:
+                self.filters.append(f)
+
+    def __call__(self) -> bool:
+        return all(f() for f in self.filters)
+
+    def __repr__(self) -> str:
+        return "&".join(repr(f) for f in self.filters)
+
+
+class _OrList(Filter):
+    """
+    Result of |-operation between several filters.
+    """
+
+    def __init__(self, filters: Iterable[Filter]) -> None:
+        self.filters: List[Filter] = []
+
+        for f in filters:
+            if isinstance(f, _OrList):  # Turn nested _OrLists into one.
+                self.filters.extend(cast(_OrList, f).filters)
+            else:
+                self.filters.append(f)
+
+    def __call__(self) -> bool:
+        return any(f() for f in self.filters)
+
+    def __repr__(self) -> str:
+        return "|".join(repr(f) for f in self.filters)
+
+
+class _Invert(Filter):
+    """
+    Negation of another filter.
+    """
+
+    def __init__(self, filter: Filter) -> None:
+        self.filter = filter
+
+    def __call__(self) -> bool:
+        return not self.filter()
+
+    def __repr__(self) -> str:
+        return "~%r" % self.filter
+
+
+class Always(Filter):
+    """
+    Always enable feature.
+    """
+
+    def __call__(self) -> bool:
+        return True
+
+    def __invert__(self) -> "Never":
+        return Never()
+
+
+class Never(Filter):
+    """
+    Never enable feature.
+    """
+
+    def __call__(self) -> bool:
+        return False
+
+    def __invert__(self) -> Always:
+        return Always()
+
+
+class Condition(Filter):
+    """
+    Turn any callable into a Filter. The callable is supposed to not take any
+    arguments.
+
+    This can be used as a decorator::
+
+        @Condition
+        def feature_is_active():  # `feature_is_active` becomes a Filter.
+            return True
+
+    :param func: Callable which takes no inputs and returns a boolean.
+    """
+
+    def __init__(self, func: Callable[[], bool]):
+        self.func = func
+
+    def __call__(self) -> bool:
+        return self.func()
+
+    def __repr__(self) -> str:
+        return "Condition(%r)" % self.func
+
+
+# Often used as type annotation.
+FilterOrBool = Union[Filter, bool]

xonsh/vended_ptk/prompt_toolkit/filters/cli.py

@@ -0,0 +1,62 @@
+"""
+For backwards-compatibility. keep this file.
+(Many people are going to have key bindings that rely on this file.)
+"""
+from .app import *
+
+__all__ = [
+    # Old names.
+    "HasArg",
+    "HasCompletions",
+    "HasFocus",
+    "HasSelection",
+    "HasValidationError",
+    "IsDone",
+    "IsReadOnly",
+    "IsMultiline",
+    "RendererHeightIsKnown",
+    "InEditingMode",
+    "InPasteMode",
+    "ViMode",
+    "ViNavigationMode",
+    "ViInsertMode",
+    "ViInsertMultipleMode",
+    "ViReplaceMode",
+    "ViSelectionMode",
+    "ViWaitingForTextObjectMode",
+    "ViDigraphMode",
+    "EmacsMode",
+    "EmacsInsertMode",
+    "EmacsSelectionMode",
+    "IsSearching",
+    "HasSearch",
+    "ControlIsSearchable",
+]
+
+# Keep the original classnames for backwards compatibility.
+HasValidationError = lambda: has_validation_error
+HasArg = lambda: has_arg
+IsDone = lambda: is_done
+RendererHeightIsKnown = lambda: renderer_height_is_known
+ViNavigationMode = lambda: vi_navigation_mode
+InPasteMode = lambda: in_paste_mode
+EmacsMode = lambda: emacs_mode
+EmacsInsertMode = lambda: emacs_insert_mode
+ViMode = lambda: vi_mode
+IsSearching = lambda: is_searching
+HasSearch = lambda: is_searching
+ControlIsSearchable = lambda: control_is_searchable
+EmacsSelectionMode = lambda: emacs_selection_mode
+ViDigraphMode = lambda: vi_digraph_mode
+ViWaitingForTextObjectMode = lambda: vi_waiting_for_text_object_mode
+ViSelectionMode = lambda: vi_selection_mode
+ViReplaceMode = lambda: vi_replace_mode
+ViInsertMultipleMode = lambda: vi_insert_multiple_mode
+ViInsertMode = lambda: vi_insert_mode
+HasSelection = lambda: has_selection
+HasCompletions = lambda: has_completions
+IsReadOnly = lambda: is_read_only
+IsMultiline = lambda: is_multiline
+
+HasFocus = has_focus  # No lambda here! (Has_focus is callable that returns a callable.)
+InEditingMode = in_editing_mode

xonsh/vended_ptk/prompt_toolkit/filters/utils.py

@@ -0,0 +1,41 @@
+from typing import Dict
+
+from .base import Always, Filter, FilterOrBool, Never
+
+__all__ = [
+    "to_filter",
+    "is_true",
+]
+
+
+_always = Always()
+_never = Never()
+
+
+_bool_to_filter: Dict[bool, Filter] = {
+    True: _always,
+    False: _never,
+}
+
+
+def to_filter(bool_or_filter: FilterOrBool) -> Filter:
+    """
+    Accept both booleans and Filters as input and
+    turn it into a Filter.
+    """
+    if isinstance(bool_or_filter, bool):
+        return _bool_to_filter[bool_or_filter]
+
+    if isinstance(bool_or_filter, Filter):
+        return bool_or_filter
+
+    raise TypeError("Expecting a bool or a Filter instance. Got %r" % bool_or_filter)
+
+
+def is_true(value: FilterOrBool) -> bool:
+    """
+    Test whether `value` is True. In case of a Filter, call it.
+
+    :param value: Boolean or `Filter` instance.
+    """
+    return to_filter(value)()

xonsh/vended_ptk/prompt_toolkit/formatted_text/__init__.py

@@ -0,0 +1,52 @@
+"""
+Many places in prompt_toolkit can take either plain text, or formatted text.
+For instance the :func:`~prompt_toolkit.shortcuts.prompt` function takes either
+plain text or formatted text for the prompt. The
+:class:`~prompt_toolkit.layout.FormattedTextControl` can also take either plain
+text or formatted text.
+
+In any case, there is an input that can either be just plain text (a string),
+an :class:`.HTML` object, an :class:`.ANSI` object or a sequence of
+`(style_string, text)` tuples. The :func:`.to_formatted_text` conversion
+function takes any of these and turns all of them into such a tuple sequence.
+"""
+from .ansi import ANSI
+from .base import (
+    AnyFormattedText,
+    FormattedText,
+    StyleAndTextTuples,
+    Template,
+    is_formatted_text,
+    merge_formatted_text,
+    to_formatted_text,
+)
+from .html import HTML
+from .pygments import PygmentsTokens
+from .utils import (
+    fragment_list_len,
+    fragment_list_to_text,
+    fragment_list_width,
+    split_lines,
+)
+
+__all__ = [
+    # Base.
+    "AnyFormattedText",
+    "to_formatted_text",
+    "is_formatted_text",
+    "Template",
+    "merge_formatted_text",
+    "FormattedText",
+    "StyleAndTextTuples",
+    # HTML.
+    "HTML",
+    # ANSI.
+    "ANSI",
+    # Pygments.
+    "PygmentsTokens",
+    # Utils.
+    "fragment_list_len",
+    "fragment_list_width",
+    "fragment_list_to_text",
+    "split_lines",
+]

xonsh/vended_ptk/prompt_toolkit/formatted_text/ansi.py

@@ -0,0 +1,246 @@
+from typing import Generator, List, Optional
+
+from prompt_toolkit.output.vt100 import BG_ANSI_COLORS, FG_ANSI_COLORS
+from prompt_toolkit.output.vt100 import _256_colors as _256_colors_table
+
+from .base import StyleAndTextTuples
+
+__all__ = [
+    "ANSI",
+    "ansi_escape",
+]
+
+
+class ANSI:
+    """
+    ANSI formatted text.
+    Take something ANSI escaped text, for use as a formatted string. E.g.
+
+    ::
+
+        ANSI('\\x1b[31mhello \\x1b[32mworld')
+
+    Characters between ``\\001`` and ``\\002`` are supposed to have a zero width
+    when printed, but these are literally sent to the terminal output. This can
+    be used for instance, for inserting Final Term prompt commands.  They will
+    be translated into a prompt_toolkit '[ZeroWidthEscape]' fragment.
+    """
+
+    def __init__(self, value: str) -> None:
+        self.value = value
+        self._formatted_text: StyleAndTextTuples = []
+
+        # Default style attributes.
+        self._color: Optional[str] = None
+        self._bgcolor: Optional[str] = None
+        self._bold = False
+        self._underline = False
+        self._italic = False
+        self._blink = False
+        self._reverse = False
+        self._hidden = False
+
+        # Process received text.
+        parser = self._parse_corot()
+        parser.send(None)  # type: ignore
+        for c in value:
+            parser.send(c)
+
+    def _parse_corot(self) -> Generator[None, str, None]:
+        """
+        Coroutine that parses the ANSI escape sequences.
+        """
+        style = ""
+        formatted_text = self._formatted_text
+
+        while True:
+            csi = False
+            c = yield
+
+            # Everything between \001 and \002 should become a ZeroWidthEscape.
+            if c == "\001":
+                escaped_text = ""
+                while c != "\002":
+                    c = yield
+                    if c == "\002":
+                        formatted_text.append(("[ZeroWidthEscape]", escaped_text))
+                        c = yield
+                        break
+                    else:
+                        escaped_text += c
+
+            if c == "\x1b":
+                # Start of color escape sequence.
+                square_bracket = yield
+                if square_bracket == "[":
+                    csi = True
+                else:
+                    continue
+            elif c == "\x9b":
+                csi = True
+
+            if csi:
+                # Got a CSI sequence. Color codes are following.
+                current = ""
+                params = []
+                while True:
+                    char = yield
+                    if char.isdigit():
+                        current += char
+                    else:
+                        params.append(min(int(current or 0), 9999))
+                        if char == ";":
+                            current = ""
+                        elif char == "m":
+                            # Set attributes and token.
+                            self._select_graphic_rendition(params)
+                            style = self._create_style_string()
+                            break
+                        else:
+                            # Ignore unsupported sequence.
+                            break
+            else:
+                # Add current character.
+                # NOTE: At this point, we could merge the current character
+                #       into the previous tuple if the style did not change,
+                #       however, it's not worth the effort given that it will
+                #       be "Exploded" once again when it's rendered to the
+                #       output.
+                formatted_text.append((style, c))
+
+    def _select_graphic_rendition(self, attrs: List[int]) -> None:
+        """
+        Taken a list of graphics attributes and apply changes.
+        """
+        if not attrs:
+            attrs = [0]
+        else:
+            attrs = list(attrs[::-1])
+
+        while attrs:
+            attr = attrs.pop()
+
+            if attr in _fg_colors:
+                self._color = _fg_colors[attr]
+            elif attr in _bg_colors:
+                self._bgcolor = _bg_colors[attr]
+            elif attr == 1:
+                self._bold = True
+            elif attr == 3:
+                self._italic = True
+            elif attr == 4:
+                self._underline = True
+            elif attr == 5:
+                self._blink = True
+            elif attr == 6:
+                self._blink = True  # Fast blink.
+            elif attr == 7:
+                self._reverse = True
+            elif attr == 8:
+                self._hidden = True
+            elif attr == 22:
+                self._bold = False
+            elif attr == 23:
+                self._italic = False
+            elif attr == 24:
+                self._underline = False
+            elif attr == 25:
+                self._blink = False
+            elif attr == 27:
+                self._reverse = False
+            elif not attr:
+                self._color = None
+                self._bgcolor = None
+                self._bold = False
+                self._underline = False
+                self._italic = False
+                self._blink = False
+                self._reverse = False
+                self._hidden = False
+
+            elif attr in (38, 48) and len(attrs) > 1:
+                n = attrs.pop()
+
+                # 256 colors.
+                if n == 5 and len(attrs) >= 1:
+                    if attr == 38:
+                        m = attrs.pop()
+                        self._color = _256_colors.get(m)
+                    elif attr == 48:
+                        m = attrs.pop()
+                        self._bgcolor = _256_colors.get(m)
+
+                # True colors.
+                if n == 2 and len(attrs) >= 3:
+                    try:
+                        color_str = "#%02x%02x%02x" % (
+                            attrs.pop(),
+                            attrs.pop(),
+                            attrs.pop(),
+                        )
+                    except IndexError:
+                        pass
+                    else:
+                        if attr == 38:
+                            self._color = color_str
+                        elif attr == 48:
+                            self._bgcolor = color_str
+
+    def _create_style_string(self) -> str:
+        """
+        Turn current style flags into a string for usage in a formatted text.
+        """
+        result = []
+        if self._color:
+            result.append(self._color)
+        if self._bgcolor:
+            result.append("bg:" + self._bgcolor)
+        if self._bold:
+            result.append("bold")
+        if self._underline:
+            result.append("underline")
+        if self._italic:
+            result.append("italic")
+        if self._blink:
+            result.append("blink")
+        if self._reverse:
+            result.append("reverse")
+        if self._hidden:
+            result.append("hidden")
+
+        return " ".join(result)
+
+    def __repr__(self) -> str:
+        return "ANSI(%r)" % (self.value,)
+
+    def __pt_formatted_text__(self) -> StyleAndTextTuples:
+        return self._formatted_text
+
+    def format(self, *args: str, **kwargs: str) -> "ANSI":
+        """
+        Like `str.format`, but make sure that the arguments are properly
+        escaped. (No ANSI escapes can be injected.)
+        """
+        # Escape all the arguments.
+        args = tuple(ansi_escape(a) for a in args)
+        kwargs = {k: ansi_escape(v) for k, v in kwargs.items()}
+
+        return ANSI(self.value.format(*args, **kwargs))
+
+
+# Mapping of the ANSI color codes to their names.
+_fg_colors = {v: k for k, v in FG_ANSI_COLORS.items()}
+_bg_colors = {v: k for k, v in BG_ANSI_COLORS.items()}
+
+# Mapping of the escape codes for 256colors to their 'ffffff' value.
+_256_colors = {}
+
+for i, (r, g, b) in enumerate(_256_colors_table.colors):
+    _256_colors[i] = "#%02x%02x%02x" % (r, g, b)
+
+
+def ansi_escape(text: str) -> str:
+    """
+    Replace characters with a special meaning.
+    """
+    return text.replace("\x1b", "?").replace("\b", "?")

xonsh/vended_ptk/prompt_toolkit/formatted_text/base.py

@@ -0,0 +1,174 @@
+from typing import TYPE_CHECKING, Any, Callable, Iterable, List, Tuple, Union, cast
+
+from prompt_toolkit.mouse_events import MouseEvent
+
+if TYPE_CHECKING:
+    from typing_extensions import Protocol
+
+__all__ = [
+    "OneStyleAndTextTuple",
+    "StyleAndTextTuples",
+    "MagicFormattedText",
+    "AnyFormattedText",
+    "to_formatted_text",
+    "is_formatted_text",
+    "Template",
+    "merge_formatted_text",
+    "FormattedText",
+]
+
+OneStyleAndTextTuple = Union[
+    Tuple[str, str], Tuple[str, str, Callable[[MouseEvent], None]],
+]
+
+# List of (style, text) tuples.
+StyleAndTextTuples = List[OneStyleAndTextTuple]
+
+
+if TYPE_CHECKING:
+
+    class MagicFormattedText(Protocol):
+        """
+        Any object that implements ``__pt_formatted_text__`` represents formatted
+        text.
+        """
+
+        def __pt_formatted_text__(self) -> StyleAndTextTuples:
+            ...
+
+
+AnyFormattedText = Union[
+    str,
+    "MagicFormattedText",
+    StyleAndTextTuples,
+    # Callable[[], 'AnyFormattedText']  # Recursive definition not supported by mypy.
+    Callable[[], Any],
+    None,
+]
+
+
+def to_formatted_text(
+    value: AnyFormattedText, style: str = "", auto_convert: bool = False
+) -> "FormattedText":
+    """
+    Convert the given value (which can be formatted text) into a list of text
+    fragments. (Which is the canonical form of formatted text.) The outcome is
+    always a `FormattedText` instance, which is a list of (style, text) tuples.
+
+    It can take an `HTML` object, a plain text string, or anything that
+    implements `__pt_formatted_text__`.
+
+    :param style: An additional style string which is applied to all text
+        fragments.
+    :param auto_convert: If `True`, also accept other types, and convert them
+        to a string first.
+    """
+    result: Union[FormattedText, StyleAndTextTuples]
+
+    if value is None:
+        result = []
+    elif isinstance(value, str):
+        result = [("", value)]
+    elif isinstance(value, list):
+        result = cast(StyleAndTextTuples, value)
+    elif hasattr(value, "__pt_formatted_text__"):
+        result = cast("MagicFormattedText", value).__pt_formatted_text__()
+    elif callable(value):
+        return to_formatted_text(value(), style=style)
+    elif auto_convert:
+        result = [("", "{}".format(value))]
+    else:
+        raise ValueError(
+            "No formatted text. Expecting a unicode object, "
+            "HTML, ANSI or a FormattedText instance. Got %r" % value
+        )
+
+    # Apply extra style.
+    if style:
+        result = cast(
+            StyleAndTextTuples,
+            [(style + " " + item_style, *rest) for item_style, *rest in result],
+        )
+
+    # Make sure the result is wrapped in a `FormattedText`. Among other
+    # reasons, this is important for `print_formatted_text` to work correctly
+    # and distinguish between lists and formatted text.
+    if isinstance(result, FormattedText):
+        return result
+    else:
+        return FormattedText(result)
+
+
+def is_formatted_text(value: object) -> bool:
+    """
+    Check whether the input is valid formatted text (for use in assert
+    statements).
+    In case of a callable, it doesn't check the return type.
+    """
+    if callable(value):
+        return True
+    if isinstance(value, (str, list)):
+        return True
+    if hasattr(value, "__pt_formatted_text__"):
+        return True
+    return False
+
+
+class FormattedText(StyleAndTextTuples):
+    """
+    A list of ``(style, text)`` tuples.
+
+    (In some situations, this can also be ``(style, text, mouse_handler)``
+    tuples.)
+    """
+
+    def __pt_formatted_text__(self) -> StyleAndTextTuples:
+        return self
+
+    def __repr__(self) -> str:
+        return "FormattedText(%s)" % super().__repr__()
+
+
+class Template:
+    """
+    Template for string interpolation with formatted text.
+
+    Example::
+
+        Template(' ... {} ... ').format(HTML(...))
+
+    :param text: Plain text.
+    """
+
+    def __init__(self, text: str) -> None:
+        assert "{0}" not in text
+        self.text = text
+
+    def format(self, *values: AnyFormattedText) -> AnyFormattedText:
+        def get_result():
+            # Split the template in parts.
+            parts = self.text.split("{}")
+            assert len(parts) - 1 == len(values)
+
+            result = FormattedText()
+            for part, val in zip(parts, values):
+                result.append(("", part))
+                result.extend(to_formatted_text(val))
+            result.append(("", parts[-1]))
+            return result
+
+        return get_result
+
+
+def merge_formatted_text(items: Iterable[AnyFormattedText]) -> AnyFormattedText:
+    """
+    Merge (Concatenate) several pieces of formatted text together.
+    """
+
+    def _merge_formatted_text():
+        result = FormattedText()
+        for i in items:
+            result.extend(to_formatted_text(i))
+        return result
+
+    return _merge_formatted_text

xonsh/vended_ptk/prompt_toolkit/formatted_text/html.py

@@ -0,0 +1,137 @@
+import xml.dom.minidom as minidom
+from typing import Any, List, Tuple, Union
+
+from .base import FormattedText, StyleAndTextTuples
+
+__all__ = ["HTML"]
+
+
+class HTML:
+    """
+    HTML formatted text.
+    Take something HTML-like, for use as a formatted string.
+
+    ::
+
+        # Turn something into red.
+        HTML('<style fg="ansired" bg="#00ff44">...</style>')
+
+        # Italic, bold and underline.
+        HTML('<i>...</i>')
+        HTML('<b>...</b>')
+        HTML('<u>...</u>')
+
+    All HTML elements become available as a "class" in the style sheet.
+    E.g. ``<username>...</username>`` can be styled, by setting a style for
+    ``username``.
+    """
+
+    def __init__(self, value: str) -> None:
+        self.value = value
+        document = minidom.parseString("<html-root>%s</html-root>" % (value,))
+
+        result: StyleAndTextTuples = []
+        name_stack: List[str] = []
+        fg_stack: List[str] = []
+        bg_stack: List[str] = []
+
+        def get_current_style() -> str:
+            " Build style string for current node. "
+            parts = []
+            if name_stack:
+                parts.append("class:" + ",".join(name_stack))
+
+            if fg_stack:
+                parts.append("fg:" + fg_stack[-1])
+            if bg_stack:
+                parts.append("bg:" + bg_stack[-1])
+            return " ".join(parts)
+
+        def process_node(node: Any) -> None:
+            " Process node recursively. "
+            for child in node.childNodes:
+                if child.nodeType == child.TEXT_NODE:
+                    result.append((get_current_style(), child.data))
+                else:
+                    add_to_name_stack = child.nodeName not in (
+                        "#document",
+                        "html-root",
+                        "style",
+                    )
+                    fg = bg = ""
+
+                    for k, v in child.attributes.items():
+                        if k == "fg":
+                            fg = v
+                        if k == "bg":
+                            bg = v
+                        if k == "color":
+                            fg = v  # Alias for 'fg'.
+
+                    # Check for spaces in attributes. This would result in
+                    # invalid style strings otherwise.
+                    if " " in fg:
+                        raise ValueError('"fg" attribute contains a space.')
+                    if " " in bg:
+                        raise ValueError('"bg" attribute contains a space.')
+
+                    if add_to_name_stack:
+                        name_stack.append(child.nodeName)
+                    if fg:
+                        fg_stack.append(fg)
+                    if bg:
+                        bg_stack.append(bg)
+
+                    process_node(child)
+
+                    if add_to_name_stack:
+                        name_stack.pop()
+                    if fg:
+                        fg_stack.pop()
+                    if bg:
+                        bg_stack.pop()
+
+        process_node(document)
+
+        self.formatted_text = FormattedText(result)
+
+    def __repr__(self) -> str:
+        return "HTML(%r)" % (self.value,)
+
+    def __pt_formatted_text__(self) -> StyleAndTextTuples:
+        return self.formatted_text
+
+    def format(self, *args: object, **kwargs: object) -> "HTML":
+        """
+        Like `str.format`, but make sure that the arguments are properly
+        escaped.
+        """
+        # Escape all the arguments.
+        escaped_args = [html_escape(a) for a in args]
+        escaped_kwargs = {k: html_escape(v) for k, v in kwargs.items()}
+
+        return HTML(self.value.format(*escaped_args, **escaped_kwargs))
+
+    def __mod__(self, value: Union[object, Tuple[object, ...]]) -> "HTML":
+        """
+        HTML('<b>%s</b>') % value
+        """
+        if not isinstance(value, tuple):
+            value = (value,)
+
+        value = tuple(html_escape(i) for i in value)
+        return HTML(self.value % value)
+
+
+def html_escape(text: object) -> str:
+    # The string interpolation functions also take integers and other types.
+    # Convert to string first.
+    if not isinstance(text, str):
+        text = "{}".format(text)
+
+    return (
+        text.replace("&", "&amp;")
+        .replace("<", "&lt;")
+        .replace(">", "&gt;")
+        .replace('"', "&quot;")
+    )

xonsh/vended_ptk/prompt_toolkit/formatted_text/pygments.py

@@ -0,0 +1,30 @@
+from typing import TYPE_CHECKING, List, Tuple
+
+from prompt_toolkit.styles.pygments import pygments_token_to_classname
+
+from .base import StyleAndTextTuples
+
+if TYPE_CHECKING:
+    from pygments.token import Token
+
+__all__ = [
+    "PygmentsTokens",
+]
+
+
+class PygmentsTokens:
+    """
+    Turn a pygments token list into a list of prompt_toolkit text fragments
+    (``(style_str, text)`` tuples).
+    """
+
+    def __init__(self, token_list: List[Tuple["Token", str]]) -> None:
+        self.token_list = token_list
+
+    def __pt_formatted_text__(self) -> StyleAndTextTuples:
+        result: StyleAndTextTuples = []
+
+        for token, text in self.token_list:
+            result.append(("class:" + pygments_token_to_classname(token), text))
+
+        return result

xonsh/vended_ptk/prompt_toolkit/formatted_text/utils.py

@@ -0,0 +1,85 @@
+"""
+Utilities for manipulating formatted text.
+
+When ``to_formatted_text`` has been called, we get a list of ``(style, text)``
+tuples. This file contains functions for manipulating such a list.
+"""
+from typing import Iterable, cast
+
+from prompt_toolkit.utils import get_cwidth
+
+from .base import OneStyleAndTextTuple, StyleAndTextTuples
+
+__all__ = [
+    "fragment_list_len",
+    "fragment_list_width",
+    "fragment_list_to_text",
+    "split_lines",
+]
+
+
+def fragment_list_len(fragments: StyleAndTextTuples) -> int:
+    """
+    Return the amount of characters in this text fragment list.
+
+    :param fragments: List of ``(style_str, text)`` or
+        ``(style_str, text, mouse_handler)`` tuples.
+    """
+    ZeroWidthEscape = "[ZeroWidthEscape]"
+    return sum(len(item[1]) for item in fragments if ZeroWidthEscape not in item[0])
+
+
+def fragment_list_width(fragments: StyleAndTextTuples) -> int:
+    """
+    Return the character width of this text fragment list.
+    (Take double width characters into account.)
+
+    :param fragments: List of ``(style_str, text)`` or
+        ``(style_str, text, mouse_handler)`` tuples.
+    """
+    ZeroWidthEscape = "[ZeroWidthEscape]"
+    return sum(
+        get_cwidth(c)
+        for item in fragments
+        for c in item[1]
+        if ZeroWidthEscape not in item[0]
+    )
+
+
+def fragment_list_to_text(fragments: StyleAndTextTuples) -> str:
+    """
+    Concatenate all the text parts again.
+
+    :param fragments: List of ``(style_str, text)`` or
+        ``(style_str, text, mouse_handler)`` tuples.
+    """
+    ZeroWidthEscape = "[ZeroWidthEscape]"
+    return "".join(item[1] for item in fragments if ZeroWidthEscape not in item[0])
+
+
+def split_lines(fragments: StyleAndTextTuples) -> Iterable[StyleAndTextTuples]:
+    """
+    Take a single list of (style_str, text) tuples and yield one such list for each
+    line. Just like str.split, this will yield at least one item.
+
+    :param fragments: List of (style_str, text) or (style_str, text, mouse_handler)
+                      tuples.
+    """
+    line: StyleAndTextTuples = []
+
+    for style, string, *mouse_handler in fragments:
+        parts = string.split("\n")
+
+        for part in parts[:-1]:
+            if part:
+                line.append(cast(OneStyleAndTextTuple, (style, part, *mouse_handler)))
+            yield line
+            line = []
+
+        line.append(cast(OneStyleAndTextTuple, (style, parts[-1], *mouse_handler)))
+
+    # Always yield the last line, even when this is an empty line. This ensures
+    # that when `fragments` ends with a newline character, an additional empty
+    # line is yielded. (Otherwise, there's no way to differentiate between the
+    # cases where `fragments` does and doesn't end with a newline.)
+    yield line

xonsh/vended_ptk/prompt_toolkit/history.py

@@ -0,0 +1,232 @@
+"""
+Implementations for the history of a `Buffer`.
+
+NOTE: Notice that there is no `DynamicHistory`. This doesn't work well, because
+      the `Buffer` needs to be able to attach an event handler to the event
+      when a history entry is loaded. This loading can be done asynchronously
+      and making the history swappable would probably break this.
+"""
+import datetime
+import os
+from abc import ABCMeta, abstractmethod
+from threading import Thread
+from typing import Callable, Iterable, List, Optional
+
+__all__ = [
+    "History",
+    "ThreadedHistory",
+    "DummyHistory",
+    "FileHistory",
+    "InMemoryHistory",
+]
+
+
+class History(metaclass=ABCMeta):
+    """
+    Base ``History`` class.
+
+    This also includes abstract methods for loading/storing history.
+    """
+
+    def __init__(self) -> None:
+        # In memory storage for strings.
+        self._loaded = False
+        self._loaded_strings: List[str] = []
+
+    #
+    # Methods expected by `Buffer`.
+    #
+
+    def load(self, item_loaded_callback: Callable[[str], None]) -> None:
+        """
+        Load the history and call the callback for every entry in the history.
+
+        XXX: The callback can be called from another thread, which happens in
+             case of `ThreadedHistory`.
+
+             We can't assume that an asyncio event loop is running, and
+             schedule the insertion into the `Buffer` using the event loop.
+
+             The reason is that the creation of the :class:`.History` object as
+             well as the start of the loading happens *before*
+             `Application.run()` is called, and it can continue even after
+             `Application.run()` terminates. (Which is useful to have a
+             complete history during the next prompt.)
+
+             Calling `get_event_loop()` right here is also not guaranteed to
+             return the same event loop which is used in `Application.run`,
+             because a new event loop can be created during the `run`. This is
+             useful in Python REPLs, where we want to use one event loop for
+             the prompt, and have another one active during the `eval` of the
+             commands. (Otherwise, the user can schedule a while/true loop and
+             freeze the UI.)
+        """
+        if self._loaded:
+            for item in self._loaded_strings[::-1]:
+                item_loaded_callback(item)
+            return
+
+        try:
+            for item in self.load_history_strings():
+                self._loaded_strings.insert(0, item)
+                item_loaded_callback(item)
+        finally:
+            self._loaded = True
+
+    def get_strings(self) -> List[str]:
+        """
+        Get the strings from the history that are loaded so far.
+        """
+        return self._loaded_strings
+
+    def append_string(self, string: str) -> None:
+        " Add string to the history. "
+        self._loaded_strings.append(string)
+        self.store_string(string)
+
+    #
+    # Implementation for specific backends.
+    #
+
+    @abstractmethod
+    def load_history_strings(self) -> Iterable[str]:
+        """
+        This should be a generator that yields `str` instances.
+
+        It should yield the most recent items first, because they are the most
+        important. (The history can already be used, even when it's only
+        partially loaded.)
+        """
+        while False:
+            yield
+
+    @abstractmethod
+    def store_string(self, string: str) -> None:
+        """
+        Store the string in persistent storage.
+        """
+
+
+class ThreadedHistory(History):
+    """
+    Wrapper that runs the `load_history_strings` generator in a thread.
+
+    Use this to increase the start-up time of prompt_toolkit applications.
+    History entries are available as soon as they are loaded. We don't have to
+    wait for everything to be loaded.
+    """
+
+    def __init__(self, history: History) -> None:
+        self.history = history
+        self._load_thread: Optional[Thread] = None
+        self._item_loaded_callbacks: List[Callable[[str], None]] = []
+        super().__init__()
+
+    def load(self, item_loaded_callback: Callable[[str], None]) -> None:
+        self._item_loaded_callbacks.append(item_loaded_callback)
+
+        # Start the load thread, if we don't have a thread yet.
+        if not self._load_thread:
+
+            def call_all_callbacks(item: str) -> None:
+                for cb in self._item_loaded_callbacks:
+                    cb(item)
+
+            self._load_thread = Thread(
+                target=self.history.load, args=(call_all_callbacks,)
+            )
+            self._load_thread.daemon = True
+            self._load_thread.start()
+
+    def get_strings(self) -> List[str]:
+        return self.history.get_strings()
+
+    def append_string(self, string: str) -> None:
+        self.history.append_string(string)
+
+    # All of the following are proxied to `self.history`.
+
+    def load_history_strings(self) -> Iterable[str]:
+        return self.history.load_history_strings()
+
+    def store_string(self, string: str) -> None:
+        self.history.store_string(string)
+
+    def __repr__(self) -> str:
+        return "ThreadedHistory(%r)" % (self.history,)
+
+
+class InMemoryHistory(History):
+    """
+    :class:`.History` class that keeps a list of all strings in memory.
+    """
+
+    def load_history_strings(self) -> Iterable[str]:
+        return []
+
+    def store_string(self, string: str) -> None:
+        pass
+
+
+class DummyHistory(History):
+    """
+    :class:`.History` object that doesn't remember anything.
+    """
+
+    def load_history_strings(self) -> Iterable[str]:
+        return []
+
+    def store_string(self, string: str) -> None:
+        pass
+
+    def append_string(self, string: str) -> None:
+        # Don't remember this.
+        pass
+
+
+class FileHistory(History):
+    """
+    :class:`.History` class that stores all strings in a file.
+    """
+
+    def __init__(self, filename: str) -> None:
+        self.filename = filename
+        super(FileHistory, self).__init__()
+
+    def load_history_strings(self) -> Iterable[str]:
+        strings: List[str] = []
+        lines: List[str] = []
+
+        def add() -> None:
+            if lines:
+                # Join and drop trailing newline.
+                string = "".join(lines)[:-1]
+
+                strings.append(string)
+
+        if os.path.exists(self.filename):
+            with open(self.filename, "rb") as f:
+                for line_bytes in f:
+                    line = line_bytes.decode("utf-8")
+
+                    if line.startswith("+"):
+                        lines.append(line[1:])
+                    else:
+                        add()
+                        lines = []
+
+                add()
+
+        # Reverse the order, because newest items have to go first.
+        return reversed(strings)
+
+    def store_string(self, string: str) -> None:
+        # Save to file.
+        with open(self.filename, "ab") as f:
+
+            def write(t: str) -> None:
+                f.write(t.encode("utf-8"))
+
+            write("\n# %s\n" % datetime.datetime.now())
+            for line in string.split("\n"):
+                write("+%s\n" % line)

xonsh/vended_ptk/prompt_toolkit/input/__init__.py

@@ -0,0 +1,11 @@
+from .base import DummyInput, Input
+from .defaults import create_input, create_pipe_input
+
+__all__ = [
+    # Base.
+    "Input",
+    "DummyInput",
+    # Defaults.
+    "create_input",
+    "create_pipe_input",
+]

xonsh/vended_ptk/prompt_toolkit/input/ansi_escape_sequences.py

@@ -0,0 +1,331 @@
+"""
+Mappings from VT100 (ANSI) escape sequences to the corresponding prompt_toolkit
+keys.
+
+We are not using the terminfo/termcap databases to detect the ANSI escape
+sequences for the input. Instead, we recognize 99% of the most common
+sequences. This works well, because in practice, every modern terminal is
+mostly Xterm compatible.
+
+Some useful docs:
+- Mintty: https://github.com/mintty/mintty/blob/master/wiki/Keycodes.md
+"""
+from typing import Dict, Tuple, Union
+
+from ..keys import Keys
+
+__all__ = [
+    "ANSI_SEQUENCES",
+    "REVERSE_ANSI_SEQUENCES",
+]
+
+# Mapping of vt100 escape codes to Keys.
+ANSI_SEQUENCES: Dict[str, Union[Keys, Tuple[Keys, ...]]] = {
+    # Control keys.
+    "\x00": Keys.ControlAt,  # Control-At (Also for Ctrl-Space)
+    "\x01": Keys.ControlA,  # Control-A (home)
+    "\x02": Keys.ControlB,  # Control-B (emacs cursor left)
+    "\x03": Keys.ControlC,  # Control-C (interrupt)
+    "\x04": Keys.ControlD,  # Control-D (exit)
+    "\x05": Keys.ControlE,  # Control-E (end)
+    "\x06": Keys.ControlF,  # Control-F (cursor forward)
+    "\x07": Keys.ControlG,  # Control-G
+    "\x08": Keys.ControlH,  # Control-H (8) (Identical to '\b')
+    "\x09": Keys.ControlI,  # Control-I (9) (Identical to '\t')
+    "\x0a": Keys.ControlJ,  # Control-J (10) (Identical to '\n')
+    "\x0b": Keys.ControlK,  # Control-K (delete until end of line; vertical tab)
+    "\x0c": Keys.ControlL,  # Control-L (clear; form feed)
+    "\x0d": Keys.ControlM,  # Control-M (13) (Identical to '\r')
+    "\x0e": Keys.ControlN,  # Control-N (14) (history forward)
+    "\x0f": Keys.ControlO,  # Control-O (15)
+    "\x10": Keys.ControlP,  # Control-P (16) (history back)
+    "\x11": Keys.ControlQ,  # Control-Q
+    "\x12": Keys.ControlR,  # Control-R (18) (reverse search)
+    "\x13": Keys.ControlS,  # Control-S (19) (forward search)
+    "\x14": Keys.ControlT,  # Control-T
+    "\x15": Keys.ControlU,  # Control-U
+    "\x16": Keys.ControlV,  # Control-V
+    "\x17": Keys.ControlW,  # Control-W
+    "\x18": Keys.ControlX,  # Control-X
+    "\x19": Keys.ControlY,  # Control-Y (25)
+    "\x1a": Keys.ControlZ,  # Control-Z
+    "\x1b": Keys.Escape,  # Also Control-[
+    "\x9b": Keys.ShiftEscape,
+    "\x1c": Keys.ControlBackslash,  # Both Control-\ (also Ctrl-| )
+    "\x1d": Keys.ControlSquareClose,  # Control-]
+    "\x1e": Keys.ControlCircumflex,  # Control-^
+    "\x1f": Keys.ControlUnderscore,  # Control-underscore (Also for Ctrl-hyphen.)
+    # ASCII Delete (0x7f)
+    # Vt220 (and Linux terminal) send this when pressing backspace. We map this
+    # to ControlH, because that will make it easier to create key bindings that
+    # work everywhere, with the trade-off that it's no longer possible to
+    # handle backspace and control-h individually for the few terminals that
+    # support it. (Most terminals send ControlH when backspace is pressed.)
+    # See: http://www.ibb.net/~anne/keyboard.html
+    "\x7f": Keys.ControlH,
+    # --
+    # Various
+    "\x1b[1~": Keys.Home,  # tmux
+    "\x1b[2~": Keys.Insert,
+    "\x1b[3~": Keys.Delete,
+    "\x1b[4~": Keys.End,  # tmux
+    "\x1b[5~": Keys.PageUp,
+    "\x1b[6~": Keys.PageDown,
+    "\x1b[7~": Keys.Home,  # xrvt
+    "\x1b[8~": Keys.End,  # xrvt
+    "\x1b[Z": Keys.BackTab,  # shift + tab
+    # --
+    # Function keys.
+    "\x1bOP": Keys.F1,
+    "\x1bOQ": Keys.F2,
+    "\x1bOR": Keys.F3,
+    "\x1bOS": Keys.F4,
+    "\x1b[[A": Keys.F1,  # Linux console.
+    "\x1b[[B": Keys.F2,  # Linux console.
+    "\x1b[[C": Keys.F3,  # Linux console.
+    "\x1b[[D": Keys.F4,  # Linux console.
+    "\x1b[[E": Keys.F5,  # Linux console.
+    "\x1b[11~": Keys.F1,  # rxvt-unicode
+    "\x1b[12~": Keys.F2,  # rxvt-unicode
+    "\x1b[13~": Keys.F3,  # rxvt-unicode
+    "\x1b[14~": Keys.F4,  # rxvt-unicode
+    "\x1b[15~": Keys.F5,
+    "\x1b[17~": Keys.F6,
+    "\x1b[18~": Keys.F7,
+    "\x1b[19~": Keys.F8,
+    "\x1b[20~": Keys.F9,
+    "\x1b[21~": Keys.F10,
+    "\x1b[23~": Keys.F11,
+    "\x1b[24~": Keys.F12,
+    "\x1b[25~": Keys.F13,
+    "\x1b[26~": Keys.F14,
+    "\x1b[28~": Keys.F15,
+    "\x1b[29~": Keys.F16,
+    "\x1b[31~": Keys.F17,
+    "\x1b[32~": Keys.F18,
+    "\x1b[33~": Keys.F19,
+    "\x1b[34~": Keys.F20,
+    # Xterm
+    "\x1b[1;2P": Keys.F13,
+    "\x1b[1;2Q": Keys.F14,
+    # '\x1b[1;2R': Keys.F15,  # Conflicts with CPR response.
+    "\x1b[1;2S": Keys.F16,
+    "\x1b[15;2~": Keys.F17,
+    "\x1b[17;2~": Keys.F18,
+    "\x1b[18;2~": Keys.F19,
+    "\x1b[19;2~": Keys.F20,
+    "\x1b[20;2~": Keys.F21,
+    "\x1b[21;2~": Keys.F22,
+    "\x1b[23;2~": Keys.F23,
+    "\x1b[24;2~": Keys.F24,
+    # --
+    # Control + function keys.
+    "\x1b[1;5P": Keys.ControlF1,
+    "\x1b[1;5Q": Keys.ControlF2,
+    # "\x1b[1;5R": Keys.ControlF3,  # Conflicts with CPR response.
+    "\x1b[1;5S": Keys.ControlF4,
+    "\x1b[15;5~": Keys.ControlF5,
+    "\x1b[17;5~": Keys.ControlF6,
+    "\x1b[18;5~": Keys.ControlF7,
+    "\x1b[19;5~": Keys.ControlF8,
+    "\x1b[20;5~": Keys.ControlF9,
+    "\x1b[21;5~": Keys.ControlF10,
+    "\x1b[23;5~": Keys.ControlF11,
+    "\x1b[24;5~": Keys.ControlF12,
+    "\x1b[1;6P": Keys.ControlF13,
+    "\x1b[1;6Q": Keys.ControlF14,
+    # "\x1b[1;6R": Keys.ControlF15,  # Conflicts with CPR response.
+    "\x1b[1;6S": Keys.ControlF16,
+    "\x1b[15;6~": Keys.ControlF17,
+    "\x1b[17;6~": Keys.ControlF18,
+    "\x1b[18;6~": Keys.ControlF19,
+    "\x1b[19;6~": Keys.ControlF20,
+    "\x1b[20;6~": Keys.ControlF21,
+    "\x1b[21;6~": Keys.ControlF22,
+    "\x1b[23;6~": Keys.ControlF23,
+    "\x1b[24;6~": Keys.ControlF24,
+    # --
+    # Tmux (Win32 subsystem) sends the following scroll events.
+    "\x1b[62~": Keys.ScrollUp,
+    "\x1b[63~": Keys.ScrollDown,
+    "\x1b[200~": Keys.BracketedPaste,  # Start of bracketed paste.
+    # --
+    # Sequences generated by numpad 5. Not sure what it means. (It doesn't
+    # appear in 'infocmp'. Just ignore.
+    "\x1b[E": Keys.Ignore,  # Xterm.
+    "\x1b[G": Keys.Ignore,  # Linux console.
+    # --
+    # Meta/control/escape + pageup/pagedown/insert/delete.
+    "\x1b[3;2~": Keys.ShiftDelete,  # xterm, gnome-terminal.
+    "\x1b[5;2~": Keys.ShiftPageUp,
+    "\x1b[6;2~": Keys.ShiftPageDown,
+    "\x1b[2;3~": (Keys.Escape, Keys.Insert),
+    "\x1b[3;3~": (Keys.Escape, Keys.Delete),
+    "\x1b[5;3~": (Keys.Escape, Keys.PageUp),
+    "\x1b[6;3~": (Keys.Escape, Keys.PageDown),
+    "\x1b[2;4~": (Keys.Escape, Keys.ShiftInsert),
+    "\x1b[3;4~": (Keys.Escape, Keys.ShiftDelete),
+    "\x1b[5;4~": (Keys.Escape, Keys.ShiftPageUp),
+    "\x1b[6;4~": (Keys.Escape, Keys.ShiftPageDown),
+    "\x1b[3;5~": Keys.ControlDelete,  # xterm, gnome-terminal.
+    "\x1b[5;5~": Keys.ControlPageUp,
+    "\x1b[6;5~": Keys.ControlPageDown,
+    "\x1b[3;6~": Keys.ControlShiftDelete,
+    "\x1b[5;6~": Keys.ControlShiftPageUp,
+    "\x1b[6;6~": Keys.ControlShiftPageDown,
+    "\x1b[2;7~": (Keys.Escape, Keys.ControlInsert),
+    "\x1b[5;7~": (Keys.Escape, Keys.ControlPageDown),
+    "\x1b[6;7~": (Keys.Escape, Keys.ControlPageDown),
+    "\x1b[2;8~": (Keys.Escape, Keys.ControlShiftInsert),
+    "\x1b[5;8~": (Keys.Escape, Keys.ControlShiftPageDown),
+    "\x1b[6;8~": (Keys.Escape, Keys.ControlShiftPageDown),
+    # --
+    # Arrows.
+    "\x1b[A": Keys.Up,
+    "\x1b[B": Keys.Down,
+    "\x1b[C": Keys.Right,
+    "\x1b[D": Keys.Left,
+    "\x1b[H": Keys.Home,
+    "\x1b[F": Keys.End,
+    # Tmux sends following keystrokes when control+arrow is pressed, but for
+    # Emacs ansi-term sends the same sequences for normal arrow keys. Consider
+    # it a normal arrow press, because that's more important.
+    "\x1bOA": Keys.Up,
+    "\x1bOB": Keys.Down,
+    "\x1bOC": Keys.Right,
+    "\x1bOD": Keys.Left,
+    "\x1bOF": Keys.End,
+    "\x1bOH": Keys.Home,
+    # Shift + arrows.
+    "\x1b[1;2A": Keys.ShiftUp,
+    "\x1b[1;2B": Keys.ShiftDown,
+    "\x1b[1;2C": Keys.ShiftRight,
+    "\x1b[1;2D": Keys.ShiftLeft,
+    "\x1b[1;2F": Keys.ShiftEnd,
+    "\x1b[1;2H": Keys.ShiftHome,
+    # Meta + arrow keys. Several terminals handle this differently.
+    # The following sequences are for xterm and gnome-terminal.
+    #     (Iterm sends ESC followed by the normal arrow_up/down/left/right
+    #     sequences, and the OSX Terminal sends ESCb and ESCf for "alt
+    #     arrow_left" and "alt arrow_right." We don't handle these
+    #     explicitly, in here, because would could not distinguish between
+    #     pressing ESC (to go to Vi navigation mode), followed by just the
+    #     'b' or 'f' key. These combinations are handled in
+    #     the input processor.)
+    "\x1b[1;3A": (Keys.Escape, Keys.Up),
+    "\x1b[1;3B": (Keys.Escape, Keys.Down),
+    "\x1b[1;3C": (Keys.Escape, Keys.Right),
+    "\x1b[1;3D": (Keys.Escape, Keys.Left),
+    "\x1b[1;3F": (Keys.Escape, Keys.End),
+    "\x1b[1;3H": (Keys.Escape, Keys.Home),
+    # Alt+shift+number.
+    "\x1b[1;4A": (Keys.Escape, Keys.ShiftDown),
+    "\x1b[1;4B": (Keys.Escape, Keys.ShiftUp),
+    "\x1b[1;4C": (Keys.Escape, Keys.ShiftRight),
+    "\x1b[1;4D": (Keys.Escape, Keys.ShiftLeft),
+    "\x1b[1;4F": (Keys.Escape, Keys.ShiftEnd),
+    "\x1b[1;4H": (Keys.Escape, Keys.ShiftHome),
+    # Control + arrows.
+    "\x1b[1;5A": Keys.ControlUp,  # Cursor Mode
+    "\x1b[1;5B": Keys.ControlDown,  # Cursor Mode
+    "\x1b[1;5C": Keys.ControlRight,  # Cursor Mode
+    "\x1b[1;5D": Keys.ControlLeft,  # Cursor Mode
+    "\x1b[1;5F": Keys.ControlEnd,
+    "\x1b[1;5H": Keys.ControlHome,
+    # Tmux sends following keystrokes when control+arrow is pressed, but for
+    # Emacs ansi-term sends the same sequences for normal arrow keys. Consider
+    # it a normal arrow press, because that's more important.
+    "\x1b[5A": Keys.ControlUp,
+    "\x1b[5B": Keys.ControlDown,
+    "\x1b[5C": Keys.ControlRight,
+    "\x1b[5D": Keys.ControlLeft,
+    "\x1bOc": Keys.ControlRight,  # rxvt
+    "\x1bOd": Keys.ControlLeft,  # rxvt
+    # Control + shift + arrows.
+    "\x1b[1;6A": Keys.ControlShiftDown,
+    "\x1b[1;6B": Keys.ControlShiftUp,
+    "\x1b[1;6C": Keys.ControlShiftRight,
+    "\x1b[1;6D": Keys.ControlShiftLeft,
+    "\x1b[1;6F": Keys.ControlShiftEnd,
+    "\x1b[1;6H": Keys.ControlShiftHome,
+    # Control + Meta + arrows.
+    "\x1b[1;7A": (Keys.Escape, Keys.ControlDown),
+    "\x1b[1;7B": (Keys.Escape, Keys.ControlUp),
+    "\x1b[1;7C": (Keys.Escape, Keys.ControlRight),
+    "\x1b[1;7D": (Keys.Escape, Keys.ControlLeft),
+    "\x1b[1;7F": (Keys.Escape, Keys.ControlEnd),
+    "\x1b[1;7H": (Keys.Escape, Keys.ControlHome),
+    # Meta + Shift + arrows.
+    "\x1b[1;8A": (Keys.Escape, Keys.ControlShiftDown),
+    "\x1b[1;8B": (Keys.Escape, Keys.ControlShiftUp),
+    "\x1b[1;8C": (Keys.Escape, Keys.ControlShiftRight),
+    "\x1b[1;8D": (Keys.Escape, Keys.ControlShiftLeft),
+    "\x1b[1;8F": (Keys.Escape, Keys.ControlShiftEnd),
+    "\x1b[1;8H": (Keys.Escape, Keys.ControlShiftHome),
+    # Meta + arrow on (some?) Macs when using iTerm defaults (see issue #483).
+    "\x1b[1;9A": (Keys.Escape, Keys.Up),
+    "\x1b[1;9B": (Keys.Escape, Keys.Down),
+    "\x1b[1;9C": (Keys.Escape, Keys.Right),
+    "\x1b[1;9D": (Keys.Escape, Keys.Left),
+    # --
+    # Control/shift/meta + number in mintty.
+    # (c-2 will actually send c-@ and c-6 will send c-^.)
+    "\x1b[1;5p": Keys.Control0,
+    "\x1b[1;5q": Keys.Control1,
+    "\x1b[1;5r": Keys.Control2,
+    "\x1b[1;5s": Keys.Control3,
+    "\x1b[1;5t": Keys.Control4,
+    "\x1b[1;5u": Keys.Control5,
+    "\x1b[1;5v": Keys.Control6,
+    "\x1b[1;5w": Keys.Control7,
+    "\x1b[1;5x": Keys.Control8,
+    "\x1b[1;5y": Keys.Control9,
+    "\x1b[1;6p": Keys.ControlShift0,
+    "\x1b[1;6q": Keys.ControlShift1,
+    "\x1b[1;6r": Keys.ControlShift2,
+    "\x1b[1;6s": Keys.ControlShift3,
+    "\x1b[1;6t": Keys.ControlShift4,
+    "\x1b[1;6u": Keys.ControlShift5,
+    "\x1b[1;6v": Keys.ControlShift6,
+    "\x1b[1;6w": Keys.ControlShift7,
+    "\x1b[1;6x": Keys.ControlShift8,
+    "\x1b[1;6y": Keys.ControlShift9,
+    "\x1b[1;7p": (Keys.Escape, Keys.Control0),
+    "\x1b[1;7q": (Keys.Escape, Keys.Control1),
+    "\x1b[1;7r": (Keys.Escape, Keys.Control2),
+    "\x1b[1;7s": (Keys.Escape, Keys.Control3),
+    "\x1b[1;7t": (Keys.Escape, Keys.Control4),
+    "\x1b[1;7u": (Keys.Escape, Keys.Control5),
+    "\x1b[1;7v": (Keys.Escape, Keys.Control6),
+    "\x1b[1;7w": (Keys.Escape, Keys.Control7),
+    "\x1b[1;7x": (Keys.Escape, Keys.Control8),
+    "\x1b[1;7y": (Keys.Escape, Keys.Control9),
+    "\x1b[1;8p": (Keys.Escape, Keys.ControlShift0),
+    "\x1b[1;8q": (Keys.Escape, Keys.ControlShift1),
+    "\x1b[1;8r": (Keys.Escape, Keys.ControlShift2),
+    "\x1b[1;8s": (Keys.Escape, Keys.ControlShift3),
+    "\x1b[1;8t": (Keys.Escape, Keys.ControlShift4),
+    "\x1b[1;8u": (Keys.Escape, Keys.ControlShift5),
+    "\x1b[1;8v": (Keys.Escape, Keys.ControlShift6),
+    "\x1b[1;8w": (Keys.Escape, Keys.ControlShift7),
+    "\x1b[1;8x": (Keys.Escape, Keys.ControlShift8),
+    "\x1b[1;8y": (Keys.Escape, Keys.ControlShift9),
+}
+
+
+def _get_reverse_ansi_sequences() -> Dict[Keys, str]:
+    """
+    Create a dictionary that maps prompt_toolkit keys back to the VT100 escape
+    sequences.
+    """
+    result: Dict[Keys, str] = {}
+
+    for sequence, key in ANSI_SEQUENCES.items():
+        if not isinstance(key, tuple):
+            if key not in result:
+                result[key] = sequence
+
+    return result
+
+
+REVERSE_ANSI_SEQUENCES = _get_reverse_ansi_sequences()

xonsh/vended_ptk/prompt_toolkit/input/base.py

@@ -0,0 +1,131 @@
+"""
+Abstraction of CLI Input.
+"""
+from abc import ABCMeta, abstractmethod, abstractproperty
+from contextlib import contextmanager
+from typing import Callable, ContextManager, Generator, List
+
+from prompt_toolkit.key_binding import KeyPress
+
+__all__ = [
+    "Input",
+    "DummyInput",
+]
+
+
+class Input(metaclass=ABCMeta):
+    """
+    Abstraction for any input.
+
+    An instance of this class can be given to the constructor of a
+    :class:`~prompt_toolkit.application.Application` and will also be
+    passed to the :class:`~prompt_toolkit.eventloop.base.EventLoop`.
+    """
+
+    @abstractmethod
+    def fileno(self) -> int:
+        """
+        Fileno for putting this in an event loop.
+        """
+
+    @abstractmethod
+    def typeahead_hash(self) -> str:
+        """
+        Identifier for storing type ahead key presses.
+        """
+
+    @abstractmethod
+    def read_keys(self) -> List[KeyPress]:
+        """
+        Return a list of Key objects which are read/parsed from the input.
+        """
+
+    def flush_keys(self) -> List[KeyPress]:
+        """
+        Flush the underlying parser. and return the pending keys.
+        (Used for vt100 input.)
+        """
+        return []
+
+    def flush(self) -> None:
+        " The event loop can call this when the input has to be flushed. "
+        pass
+
+    @property
+    def responds_to_cpr(self) -> bool:
+        """
+        `True` if the `Application` can expect to receive a CPR response from
+        here.
+        """
+        return False
+
+    @abstractproperty
+    def closed(self) -> bool:
+        " Should be true when the input stream is closed. "
+        return False
+
+    @abstractmethod
+    def raw_mode(self) -> ContextManager[None]:
+        """
+        Context manager that turns the input into raw mode.
+        """
+
+    @abstractmethod
+    def cooked_mode(self) -> ContextManager[None]:
+        """
+        Context manager that turns the input into cooked mode.
+        """
+
+    @abstractmethod
+    def attach(self, input_ready_callback: Callable[[], None]) -> ContextManager[None]:
+        """
+        Return a context manager that makes this input active in the current
+        event loop.
+        """
+
+    @abstractmethod
+    def detach(self) -> ContextManager[None]:
+        """
+        Return a context manager that makes sure that this input is not active
+        in the current event loop.
+        """
+
+    def close(self) -> None:
+        " Close input. "
+        pass
+
+
+class DummyInput(Input):
+    """
+    Input for use in a `DummyApplication`
+    """
+
+    def fileno(self) -> int:
+        raise NotImplementedError
+
+    def typeahead_hash(self) -> str:
+        return "dummy-%s" % id(self)
+
+    def read_keys(self) -> List[KeyPress]:
+        return []
+
+    @property
+    def closed(self) -> bool:
+        return True
+
+    def raw_mode(self) -> ContextManager[None]:
+        return _dummy_context_manager()
+
+    def cooked_mode(self) -> ContextManager[None]:
+        return _dummy_context_manager()
+
+    def attach(self, input_ready_callback: Callable[[], None]) -> ContextManager[None]:
+        return _dummy_context_manager()
+
+    def detach(self) -> ContextManager[None]:
+        return _dummy_context_manager()
+
+
+@contextmanager
+def _dummy_context_manager() -> Generator[None, None, None]:
+    yield

xonsh/vended_ptk/prompt_toolkit/input/defaults.py

@@ -0,0 +1,58 @@
+import sys
+from typing import Optional, TextIO
+
+from prompt_toolkit.utils import is_windows
+
+from .base import Input
+
+__all__ = [
+    "create_input",
+    "create_pipe_input",
+]
+
+
+def create_input(
+    stdin: Optional[TextIO] = None, always_prefer_tty: bool = False
+) -> Input:
+    """
+    Create the appropriate `Input` object for the current os/environment.
+
+    :param always_prefer_tty: When set, if `sys.stdin` is connected to a Unix
+        `pipe`, check whether `sys.stdout` or `sys.stderr` are connected to a
+        pseudo terminal. If so, open the tty for reading instead of reading for
+        `sys.stdin`. (We can open `stdout` or `stderr` for reading, this is how
+        a `$PAGER` works.)
+    """
+    if is_windows():
+        from .win32 import Win32Input
+
+        return Win32Input(stdin or sys.stdin)
+    else:
+        from .vt100 import Vt100Input
+
+        # If no input TextIO is given, use stdin/stdout.
+        if stdin is None:
+            stdin = sys.stdin
+
+            if always_prefer_tty:
+                for io in [sys.stdin, sys.stdout, sys.stderr]:
+                    if io.isatty():
+                        stdin = io
+                        break
+
+        return Vt100Input(stdin)
+
+
+def create_pipe_input() -> Input:
+    """
+    Create an input pipe.
+    This is mostly useful for unit testing.
+    """
+    if is_windows():
+        from .win32_pipe import Win32PipeInput
+
+        return Win32PipeInput()
+    else:
+        from .posix_pipe import PosixPipeInput
+
+        return PosixPipeInput()

xonsh/vended_ptk/prompt_toolkit/input/posix_pipe.py

@@ -0,0 +1,73 @@
+import os
+from typing import ContextManager, TextIO, cast
+
+from ..utils import DummyContext
+from .vt100 import Vt100Input
+
+__all__ = [
+    "PosixPipeInput",
+]
+
+
+class PosixPipeInput(Vt100Input):
+    """
+    Input that is send through a pipe.
+    This is useful if we want to send the input programmatically into the
+    application. Mostly useful for unit testing.
+
+    Usage::
+
+        input = PosixPipeInput()
+        input.send_text('inputdata')
+    """
+
+    _id = 0
+
+    def __init__(self, text: str = "") -> None:
+        self._r, self._w = os.pipe()
+
+        class Stdin:
+            def isatty(stdin) -> bool:
+                return True
+
+            def fileno(stdin) -> int:
+                return self._r
+
+        super().__init__(cast(TextIO, Stdin()))
+        self.send_text(text)
+
+        # Identifier for every PipeInput for the hash.
+        self.__class__._id += 1
+        self._id = self.__class__._id
+
+    @property
+    def responds_to_cpr(self) -> bool:
+        return False
+
+    def send_bytes(self, data: bytes) -> None:
+        os.write(self._w, data)
+
+    def send_text(self, data: str) -> None:
+        " Send text to the input. "
+        os.write(self._w, data.encode("utf-8"))
+
+    def raw_mode(self) -> ContextManager[None]:
+        return DummyContext()
+
+    def cooked_mode(self) -> ContextManager[None]:
+        return DummyContext()
+
+    def close(self) -> None:
+        " Close pipe fds. "
+        os.close(self._r)
+        os.close(self._w)
+
+        # We should assign `None` to 'self._r` and 'self._w',
+        # The event loop still needs to know the the fileno for this input in order
+        # to properly remove it from the selectors.
+
+    def typeahead_hash(self) -> str:
+        """
+        This needs to be unique for every `PipeInput`.
+        """
+        return "pipe-input-%s" % (self._id,)

xonsh/vended_ptk/prompt_toolkit/input/posix_utils.py

@@ -0,0 +1,93 @@
+import os
+import select
+from codecs import getincrementaldecoder
+
+__all__ = [
+    "PosixStdinReader",
+]
+
+
+class PosixStdinReader:
+    """
+    Wrapper around stdin which reads (nonblocking) the next available 1024
+    bytes and decodes it.
+
+    Note that you can't be sure that the input file is closed if the ``read``
+    function returns an empty string. When ``errors=ignore`` is passed,
+    ``read`` can return an empty string if all malformed input was replaced by
+    an empty string. (We can't block here and wait for more input.) So, because
+    of that, check the ``closed`` attribute, to be sure that the file has been
+    closed.
+
+    :param stdin_fd: File descriptor from which we read.
+    :param errors:  Can be 'ignore', 'strict' or 'replace'.
+        On Python3, this can be 'surrogateescape', which is the default.
+
+        'surrogateescape' is preferred, because this allows us to transfer
+        unrecognised bytes to the key bindings. Some terminals, like lxterminal
+        and Guake, use the 'Mxx' notation to send mouse events, where each 'x'
+        can be any possible byte.
+    """
+
+    # By default, we want to 'ignore' errors here. The input stream can be full
+    # of junk.  One occurrence of this that I had was when using iTerm2 on OS X,
+    # with "Option as Meta" checked (You should choose "Option as +Esc".)
+
+    def __init__(self, stdin_fd: int, errors: str = "surrogateescape") -> None:
+        self.stdin_fd = stdin_fd
+        self.errors = errors
+
+        # Create incremental decoder for decoding stdin.
+        # We can not just do `os.read(stdin.fileno(), 1024).decode('utf-8')`, because
+        # it could be that we are in the middle of a utf-8 byte sequence.
+        self._stdin_decoder_cls = getincrementaldecoder("utf-8")
+        self._stdin_decoder = self._stdin_decoder_cls(errors=errors)
+
+        #: True when there is nothing anymore to read.
+        self.closed = False
+
+    def read(self, count: int = 1024) -> str:
+        # By default we choose a rather small chunk size, because reading
+        # big amounts of input at once, causes the event loop to process
+        # all these key bindings also at once without going back to the
+        # loop. This will make the application feel unresponsive.
+        """
+        Read the input and return it as a string.
+
+        Return the text. Note that this can return an empty string, even when
+        the input stream was not yet closed. This means that something went
+        wrong during the decoding.
+        """
+        if self.closed:
+            return ""
+
+        # Check whether there is some input to read. `os.read` would block
+        # otherwise.
+        # (Actually, the event loop is responsible to make sure that this
+        # function is only called when there is something to read, but for some
+        # reason this happens in certain situations.)
+        try:
+            if not select.select([self.stdin_fd], [], [], 0)[0]:
+                return ""
+        except IOError:
+            # Happens for instance when the file descriptor was closed.
+            # (We had this in ptterm, where the FD became ready, a callback was
+            # scheduled, but in the meantime another callback closed it already.)
+            self.closed = True
+
+        # Note: the following works better than wrapping `self.stdin` like
+        #       `codecs.getreader('utf-8')(stdin)` and doing `read(1)`.
+        #       Somehow that causes some latency when the escape
+        #       character is pressed. (Especially on combination with the `select`.)
+        try:
+            data = os.read(self.stdin_fd, count)
+
+            # Nothing more to read, stream is closed.
+            if data == b"":
+                self.closed = True
+                return ""
+        except OSError:
+            # In case of SIGWINCH
+            data = b""
+
+        return self._stdin_decoder.decode(data)

xonsh/vended_ptk/prompt_toolkit/input/typeahead.py

@@ -0,0 +1,76 @@
+r"""
+Store input key strokes if we did read more than was required.
+
+The input classes `Vt100Input` and `Win32Input` read the input text in chunks
+of a few kilobytes. This means that if we read input from stdin, it could be
+that we read a couple of lines (with newlines in between) at once.
+
+This creates a problem: potentially, we read too much from stdin. Sometimes
+people paste several lines at once because they paste input in a REPL and
+expect each input() call to process one line. Or they rely on type ahead
+because the application can't keep up with the processing.
+
+However, we need to read input in bigger chunks. We need this mostly to support
+pasting of larger chunks of text. We don't want everything to become
+unresponsive because we:
+  - read one character;
+  - parse one character;
+  - call the key binding, which does a string operation with one character;
+  - and render the user interface.
+Doing text operations on single characters is very inefficient in Python, so we
+prefer to work on bigger chunks of text. This is why we have to read the input
+in bigger chunks.
+
+Further, line buffering is also not an option, because it doesn't work well in
+the architecture. We use lower level Posix APIs, that work better with the
+event loop and so on. In fact, there is also nothing that defines that only \n
+can accept the input, you could create a key binding for any key to accept the
+input.
+
+To support type ahead, this module will store all the key strokes that were
+read too early, so that they can be feed into to the next `prompt()` call or to
+the next prompt_toolkit `Application`.
+"""
+from collections import defaultdict
+from typing import Dict, List
+
+from ..key_binding import KeyPress
+from .base import Input
+
+__all__ = [
+    "store_typeahead",
+    "get_typeahead",
+    "clear_typeahead",
+]
+
+_buffer: Dict[str, List[KeyPress]] = defaultdict(list)
+
+
+def store_typeahead(input_obj: Input, key_presses: List[KeyPress]) -> None:
+    """
+    Insert typeahead key presses for the given input.
+    """
+    global _buffer
+    key = input_obj.typeahead_hash()
+    _buffer[key].extend(key_presses)
+
+
+def get_typeahead(input_obj: Input) -> List[KeyPress]:
+    """
+    Retrieve typeahead and reset the buffer for this input.
+    """
+    global _buffer
+
+    key = input_obj.typeahead_hash()
+    result = _buffer[key]
+    _buffer[key] = []
+    return result
+
+
+def clear_typeahead(input_obj: Input) -> None:
+    """
+    Clear typeahead buffer.
+    """
+    global _buffer
+    key = input_obj.typeahead_hash()
+    _buffer[key] = []

xonsh/vended_ptk/prompt_toolkit/input/vt100.py

@@ -0,0 +1,313 @@
+import contextlib
+import io
+import os
+import sys
+import termios
+import tty
+from asyncio import AbstractEventLoop, get_event_loop
+from typing import (
+    Callable,
+    ContextManager,
+    Dict,
+    Generator,
+    List,
+    Optional,
+    Set,
+    TextIO,
+    Tuple,
+    Union,
+)
+
+from prompt_toolkit.utils import is_dumb_terminal
+
+from ..key_binding import KeyPress
+from .base import Input
+from .posix_utils import PosixStdinReader
+from .vt100_parser import Vt100Parser
+
+__all__ = [
+    "Vt100Input",
+    "raw_mode",
+    "cooked_mode",
+]
+
+
+class Vt100Input(Input):
+    """
+    Vt100 input for Posix systems.
+    (This uses a posix file descriptor that can be registered in the event loop.)
+    """
+
+    # For the error messages. Only display "Input is not a terminal" once per
+    # file descriptor.
+    _fds_not_a_terminal: Set[int] = set()
+
+    def __init__(self, stdin: TextIO) -> None:
+        # Test whether the given input object has a file descriptor.
+        # (Idle reports stdin to be a TTY, but fileno() is not implemented.)
+        try:
+            # This should not raise, but can return 0.
+            stdin.fileno()
+        except io.UnsupportedOperation:
+            if "idlelib.run" in sys.modules:
+                raise io.UnsupportedOperation(
+                    "Stdin is not a terminal. Running from Idle is not supported."
+                )
+            else:
+                raise io.UnsupportedOperation("Stdin is not a terminal.")
+
+        # Even when we have a file descriptor, it doesn't mean it's a TTY.
+        # Normally, this requires a real TTY device, but people instantiate
+        # this class often during unit tests as well. They use for instance
+        # pexpect to pipe data into an application. For convenience, we print
+        # an error message and go on.
+        isatty = stdin.isatty()
+        fd = stdin.fileno()
+
+        if not isatty and fd not in Vt100Input._fds_not_a_terminal:
+            msg = "Warning: Input is not a terminal (fd=%r).\n"
+            sys.stderr.write(msg % fd)
+            sys.stderr.flush()
+            Vt100Input._fds_not_a_terminal.add(fd)
+
+        #
+        self.stdin = stdin
+
+        # Create a backup of the fileno(). We want this to work even if the
+        # underlying file is closed, so that `typeahead_hash()` keeps working.
+        self._fileno = stdin.fileno()
+
+        self._buffer: List[KeyPress] = []  # Buffer to collect the Key objects.
+        self.stdin_reader = PosixStdinReader(self._fileno)
+        self.vt100_parser = Vt100Parser(
+            lambda key_press: self._buffer.append(key_press)
+        )
+
+    @property
+    def responds_to_cpr(self) -> bool:
+        # When the input is a tty, we assume that CPR is supported.
+        # It's not when the input is piped from Pexpect.
+        if os.environ.get("PROMPT_TOOLKIT_NO_CPR", "") == "1":
+            return False
+        if is_dumb_terminal():
+            return False
+        try:
+            return self.stdin.isatty()
+        except ValueError:
+            return False  # ValueError: I/O operation on closed file
+
+    def attach(self, input_ready_callback: Callable[[], None]) -> ContextManager[None]:
+        """
+        Return a context manager that makes this input active in the current
+        event loop.
+        """
+        return _attached_input(self, input_ready_callback)
+
+    def detach(self) -> ContextManager[None]:
+        """
+        Return a context manager that makes sure that this input is not active
+        in the current event loop.
+        """
+        return _detached_input(self)
+
+    def read_keys(self) -> List[KeyPress]:
+        " Read list of KeyPress. "
+        # Read text from stdin.
+        data = self.stdin_reader.read()
+
+        # Pass it through our vt100 parser.
+        self.vt100_parser.feed(data)
+
+        # Return result.
+        result = self._buffer
+        self._buffer = []
+        return result
+
+    def flush_keys(self) -> List[KeyPress]:
+        """
+        Flush pending keys and return them.
+        (Used for flushing the 'escape' key.)
+        """
+        # Flush all pending keys. (This is most important to flush the vt100
+        # 'Escape' key early when nothing else follows.)
+        self.vt100_parser.flush()
+
+        # Return result.
+        result = self._buffer
+        self._buffer = []
+        return result
+
+    @property
+    def closed(self) -> bool:
+        return self.stdin_reader.closed
+
+    def raw_mode(self) -> ContextManager[None]:
+        return raw_mode(self.stdin.fileno())
+
+    def cooked_mode(self) -> ContextManager[None]:
+        return cooked_mode(self.stdin.fileno())
+
+    def fileno(self) -> int:
+        return self.stdin.fileno()
+
+    def typeahead_hash(self) -> str:
+        return "fd-%s" % (self._fileno,)
+
+
+_current_callbacks: Dict[
+    Tuple[AbstractEventLoop, int], Optional[Callable[[], None]]
+] = {}  # (loop, fd) -> current callback
+
+
+@contextlib.contextmanager
+def _attached_input(
+    input: Vt100Input, callback: Callable[[], None]
+) -> Generator[None, None, None]:
+    """
+    Context manager that makes this input active in the current event loop.
+
+    :param input: :class:`~prompt_toolkit.input.Input` object.
+    :param callback: Called when the input is ready to read.
+    """
+    loop = get_event_loop()
+    fd = input.fileno()
+    previous = _current_callbacks.get((loop, fd))
+
+    loop.add_reader(fd, callback)
+    _current_callbacks[loop, fd] = callback
+
+    try:
+        yield
+    finally:
+        loop.remove_reader(fd)
+
+        if previous:
+            loop.add_reader(fd, previous)
+            _current_callbacks[loop, fd] = previous
+        else:
+            del _current_callbacks[loop, fd]
+
+
+@contextlib.contextmanager
+def _detached_input(input: Vt100Input) -> Generator[None, None, None]:
+    loop = get_event_loop()
+    fd = input.fileno()
+    previous = _current_callbacks.get((loop, fd))
+
+    if previous:
+        loop.remove_reader(fd)
+        _current_callbacks[loop, fd] = None
+
+    try:
+        yield
+    finally:
+        if previous:
+            loop.add_reader(fd, previous)
+            _current_callbacks[loop, fd] = previous
+
+
+class raw_mode:
+    """
+    ::
+
+        with raw_mode(stdin):
+            ''' the pseudo-terminal stdin is now used in raw mode '''
+
+    We ignore errors when executing `tcgetattr` fails.
+    """
+
+    # There are several reasons for ignoring errors:
+    # 1. To avoid the "Inappropriate ioctl for device" crash if somebody would
+    #    execute this code (In a Python REPL, for instance):
+    #
+    #         import os; f = open(os.devnull); os.dup2(f.fileno(), 0)
+    #
+    #    The result is that the eventloop will stop correctly, because it has
+    #    to logic to quit when stdin is closed. However, we should not fail at
+    #    this point. See:
+    #      https://github.com/jonathanslenders/python-prompt-toolkit/pull/393
+    #      https://github.com/jonathanslenders/python-prompt-toolkit/issues/392
+
+    # 2. Related, when stdin is an SSH pipe, and no full terminal was allocated.
+    #    See: https://github.com/jonathanslenders/python-prompt-toolkit/pull/165
+    def __init__(self, fileno: int) -> None:
+        self.fileno = fileno
+        self.attrs_before: Optional[List[Union[int, List[bytes]]]]
+        try:
+            self.attrs_before = termios.tcgetattr(fileno)
+        except termios.error:
+            # Ignore attribute errors.
+            self.attrs_before = None
+
+    def __enter__(self) -> None:
+        # NOTE: On os X systems, using pty.setraw() fails. Therefor we are using this:
+        try:
+            newattr = termios.tcgetattr(self.fileno)
+        except termios.error:
+            pass
+        else:
+            newattr[tty.LFLAG] = self._patch_lflag(newattr[tty.LFLAG])
+            newattr[tty.IFLAG] = self._patch_iflag(newattr[tty.IFLAG])
+
+            # VMIN defines the number of characters read at a time in
+            # non-canonical mode. It seems to default to 1 on Linux, but on
+            # Solaris and derived operating systems it defaults to 4. (This is
+            # because the VMIN slot is the same as the VEOF slot, which
+            # defaults to ASCII EOT = Ctrl-D = 4.)
+            newattr[tty.CC][termios.VMIN] = 1  # type: ignore
+
+            termios.tcsetattr(self.fileno, termios.TCSANOW, newattr)
+
+            # Put the terminal in cursor mode. (Instead of application mode.)
+            os.write(self.fileno, b"\x1b[?1l")
+
+    @classmethod
+    def _patch_lflag(cls, attrs):
+        return attrs & ~(termios.ECHO | termios.ICANON | termios.IEXTEN | termios.ISIG)
+
+    @classmethod
+    def _patch_iflag(cls, attrs):
+        return attrs & ~(
+            # Disable XON/XOFF flow control on output and input.
+            # (Don't capture Ctrl-S and Ctrl-Q.)
+            # Like executing: "stty -ixon."
+            termios.IXON
+            | termios.IXOFF
+            |
+            # Don't translate carriage return into newline on input.
+            termios.ICRNL
+            | termios.INLCR
+            | termios.IGNCR
+        )
+
+    def __exit__(self, *a: object) -> None:
+        if self.attrs_before is not None:
+            try:
+                termios.tcsetattr(self.fileno, termios.TCSANOW, self.attrs_before)
+            except termios.error:
+                pass
+
+            # # Put the terminal in application mode.
+            # self._stdout.write('\x1b[?1h')
+
+
+class cooked_mode(raw_mode):
+    """
+    The opposite of ``raw_mode``, used when we need cooked mode inside a
+    `raw_mode` block.  Used in `Application.run_in_terminal`.::
+
+        with cooked_mode(stdin):
+            ''' the pseudo-terminal stdin is now used in cooked mode. '''
+    """
+
+    @classmethod
+    def _patch_lflag(cls, attrs):
+        return attrs | (termios.ECHO | termios.ICANON | termios.IEXTEN | termios.ISIG)
+
+    @classmethod
+    def _patch_iflag(cls, attrs):
+        # Turn the ICRNL flag back on. (Without this, calling `input()` in
+        # run_in_terminal doesn't work and displays ^M instead. Ptpython
+        # evaluates commands using `run_in_terminal`, so it's important that
+        # they translate ^M back into ^J.)
+        return attrs | termios.ICRNL

xonsh/vended_ptk/prompt_toolkit/input/vt100_parser.py

@@ -0,0 +1,247 @@
+"""
+Parser for VT100 input stream.
+"""
+import re
+from typing import Callable, Dict, Generator, Tuple, Union
+
+from ..key_binding.key_processor import KeyPress
+from ..keys import Keys
+from .ansi_escape_sequences import ANSI_SEQUENCES
+
+__all__ = [
+    "Vt100Parser",
+]
+
+
+# Regex matching any CPR response
+# (Note that we use '\Z' instead of '$', because '$' could include a trailing
+# newline.)
+_cpr_response_re = re.compile("^" + re.escape("\x1b[") + r"\d+;\d+R\Z")
+
+# Mouse events:
+# Typical: "Esc[MaB*"  Urxvt: "Esc[96;14;13M" and for Xterm SGR: "Esc[<64;85;12M"
+_mouse_event_re = re.compile("^" + re.escape("\x1b[") + r"(<?[\d;]+[mM]|M...)\Z")
+
+# Regex matching any valid prefix of a CPR response.
+# (Note that it doesn't contain the last character, the 'R'. The prefix has to
+# be shorter.)
+_cpr_response_prefix_re = re.compile("^" + re.escape("\x1b[") + r"[\d;]*\Z")
+
+_mouse_event_prefix_re = re.compile("^" + re.escape("\x1b[") + r"(<?[\d;]*|M.{0,2})\Z")
+
+
+class _Flush:
+    """ Helper object to indicate flush operation to the parser. """
+
+    pass
+
+
+class _IsPrefixOfLongerMatchCache(Dict[str, bool]):
+    """
+    Dictionary that maps input sequences to a boolean indicating whether there is
+    any key that start with this characters.
+    """
+
+    def __missing__(self, prefix: str) -> bool:
+        # (hard coded) If this could be a prefix of a CPR response, return
+        # True.
+        if _cpr_response_prefix_re.match(prefix) or _mouse_event_prefix_re.match(
+            prefix
+        ):
+            result = True
+        else:
+            # If this could be a prefix of anything else, also return True.
+            result = any(
+                v
+                for k, v in ANSI_SEQUENCES.items()
+                if k.startswith(prefix) and k != prefix
+            )
+
+        self[prefix] = result
+        return result
+
+
+_IS_PREFIX_OF_LONGER_MATCH_CACHE = _IsPrefixOfLongerMatchCache()
+
+
+class Vt100Parser:
+    """
+    Parser for VT100 input stream.
+    Data can be fed through the `feed` method and the given callback will be
+    called with KeyPress objects.
+
+    ::
+
+        def callback(key):
+            pass
+        i = Vt100Parser(callback)
+        i.feed('data\x01...')
+
+    :attr feed_key_callback: Function that will be called when a key is parsed.
+    """
+
+    # Lookup table of ANSI escape sequences for a VT100 terminal
+    # Hint: in order to know what sequences your terminal writes to stdin, run
+    #       "od -c" and start typing.
+    def __init__(self, feed_key_callback: Callable[[KeyPress], None]) -> None:
+        self.feed_key_callback = feed_key_callback
+        self.reset()
+
+    def reset(self, request: bool = False) -> None:
+        self._in_bracketed_paste = False
+        self._start_parser()
+
+    def _start_parser(self) -> None:
+        """
+        Start the parser coroutine.
+        """
+        self._input_parser = self._input_parser_generator()
+        self._input_parser.send(None)  # type: ignore
+
+    def _get_match(self, prefix: str) -> Union[None, Keys, Tuple[Keys, ...]]:
+        """
+        Return the key (or keys) that maps to this prefix.
+        """
+        # (hard coded) If we match a CPR response, return Keys.CPRResponse.
+        # (This one doesn't fit in the ANSI_SEQUENCES, because it contains
+        # integer variables.)
+        if _cpr_response_re.match(prefix):
+            return Keys.CPRResponse
+
+        elif _mouse_event_re.match(prefix):
+            return Keys.Vt100MouseEvent
+
+        # Otherwise, use the mappings.
+        try:
+            return ANSI_SEQUENCES[prefix]
+        except KeyError:
+            return None
+
+    def _input_parser_generator(self) -> Generator[None, Union[str, _Flush], None]:
+        """
+        Coroutine (state machine) for the input parser.
+        """
+        prefix = ""
+        retry = False
+        flush = False
+
+        while True:
+            flush = False
+
+            if retry:
+                retry = False
+            else:
+                # Get next character.
+                c = yield
+
+                if isinstance(c, _Flush):
+                    flush = True
+                else:
+                    prefix += c
+
+            # If we have some data, check for matches.
+            if prefix:
+                is_prefix_of_longer_match = _IS_PREFIX_OF_LONGER_MATCH_CACHE[prefix]
+                match = self._get_match(prefix)
+
+                # Exact matches found, call handlers..
+                if (flush or not is_prefix_of_longer_match) and match:
+                    self._call_handler(match, prefix)
+                    prefix = ""
+
+                # No exact match found.
+                elif (flush or not is_prefix_of_longer_match) and not match:
+                    found = False
+                    retry = True
+
+                    # Loop over the input, try the longest match first and
+                    # shift.
+                    for i in range(len(prefix), 0, -1):
+                        match = self._get_match(prefix[:i])
+                        if match:
+                            self._call_handler(match, prefix[:i])
+                            prefix = prefix[i:]
+                            found = True
+
+                    if not found:
+                        self._call_handler(prefix[0], prefix[0])
+                        prefix = prefix[1:]
+
+    def _call_handler(
+        self, key: Union[str, Keys, Tuple[Keys, ...]], insert_text: str
+    ) -> None:
+        """
+        Callback to handler.
+        """
+        if isinstance(key, tuple):
+            # Received ANSI sequence that corresponds with multiple keys
+            # (probably alt+something). Handle keys individually, but only pass
+            # data payload to first KeyPress (so that we won't insert it
+            # multiple times).
+            for i, k in enumerate(key):
+                self._call_handler(k, insert_text if i == 0 else "")
+        else:
+            if key == Keys.BracketedPaste:
+                self._in_bracketed_paste = True
+                self._paste_buffer = ""
+            else:
+                self.feed_key_callback(KeyPress(key, insert_text))
+
+    def feed(self, data: str) -> None:
+        """
+        Feed the input stream.
+
+        :param data: Input string (unicode).
+        """
+        # Handle bracketed paste. (We bypass the parser that matches all other
+        # key presses and keep reading input until we see the end mark.)
+        # This is much faster then parsing character by character.
+        if self._in_bracketed_paste:
+            self._paste_buffer += data
+            end_mark = "\x1b[201~"
+
+            if end_mark in self._paste_buffer:
+                end_index = self._paste_buffer.index(end_mark)
+
+                # Feed content to key bindings.
+                paste_content = self._paste_buffer[:end_index]
+                self.feed_key_callback(KeyPress(Keys.BracketedPaste, paste_content))
+
+                # Quit bracketed paste mode and handle remaining input.
+                self._in_bracketed_paste = False
+                remaining = self._paste_buffer[end_index + len(end_mark) :]
+                self._paste_buffer = ""
+
+                self.feed(remaining)
+
+        # Handle normal input character by character.
+        else:
+            for i, c in enumerate(data):
+                if self._in_bracketed_paste:
+                    # Quit loop and process from this position when the parser
+                    # entered bracketed paste.
+                    self.feed(data[i:])
+                    break
+                else:
+                    self._input_parser.send(c)
+
+    def flush(self) -> None:
+        """
+        Flush the buffer of the input stream.
+
+        This will allow us to handle the escape key (or maybe meta) sooner.
+        The input received by the escape key is actually the same as the first
+        characters of e.g. Arrow-Up, so without knowing what follows the escape
+        sequence, we don't know whether escape has been pressed, or whether
+        it's something else. This flush function should be called after a
+        timeout, and processes everything that's still in the buffer as-is, so
+        without assuming any characters will follow.
+        """
+        self._input_parser.send(_Flush())
+
+    def feed_and_flush(self, data: str) -> None:
+        """
+        Wrapper around ``feed`` and ``flush``.
+        """
+        self.feed(data)
+        self.flush()

xonsh/vended_ptk/prompt_toolkit/input/win32.py

@@ -0,0 +1,655 @@
+import msvcrt
+import os
+import sys
+from abc import abstractmethod
+from asyncio import get_event_loop
+from contextlib import contextmanager
+from ctypes import pointer, windll
+from ctypes.wintypes import DWORD, HANDLE
+from typing import Callable, ContextManager, Dict, Iterable, Optional, TextIO
+
+from prompt_toolkit.eventloop import run_in_executor_with_context
+from prompt_toolkit.eventloop.win32 import create_win32_event, wait_for_handles
+from prompt_toolkit.key_binding.key_processor import KeyPress
+from prompt_toolkit.keys import Keys
+from prompt_toolkit.mouse_events import MouseEventType
+from prompt_toolkit.win32_types import (
+    INPUT_RECORD,
+    KEY_EVENT_RECORD,
+    MOUSE_EVENT_RECORD,
+    STD_INPUT_HANDLE,
+    EventTypes,
+)
+
+from .ansi_escape_sequences import REVERSE_ANSI_SEQUENCES
+from .base import Input
+
+__all__ = [
+    "Win32Input",
+    "ConsoleInputReader",
+    "raw_mode",
+    "cooked_mode",
+    "attach_win32_input",
+    "detach_win32_input",
+]
+
+
+class _Win32InputBase(Input):
+    """
+    Base class for `Win32Input` and `Win32PipeInput`.
+    """
+
+    def __init__(self) -> None:
+        self.win32_handles = _Win32Handles()
+
+    @property
+    @abstractmethod
+    def handle(self) -> HANDLE:
+        pass
+
+
+class Win32Input(_Win32InputBase):
+    """
+    `Input` class that reads from the Windows console.
+    """
+
+    def __init__(self, stdin: Optional[TextIO] = None) -> None:
+        super().__init__()
+        self.console_input_reader = ConsoleInputReader()
+
+    def attach(self, input_ready_callback: Callable[[], None]) -> ContextManager[None]:
+        """
+        Return a context manager that makes this input active in the current
+        event loop.
+        """
+        return attach_win32_input(self, input_ready_callback)
+
+    def detach(self) -> ContextManager[None]:
+        """
+        Return a context manager that makes sure that this input is not active
+        in the current event loop.
+        """
+        return detach_win32_input(self)
+
+    def read_keys(self):
+        return list(self.console_input_reader.read())
+
+    def flush(self) -> None:
+        pass
+
+    @property
+    def closed(self) -> bool:
+        return False
+
+    def raw_mode(self) -> ContextManager[None]:
+        return raw_mode()
+
+    def cooked_mode(self) -> ContextManager[None]:
+        return cooked_mode()
+
+    def fileno(self) -> int:
+        # The windows console doesn't depend on the file handle, so
+        # this is not used for the event loop (which uses the
+        # handle instead). But it's used in `Application.run_system_command`
+        # which opens a subprocess with a given stdin/stdout.
+        return sys.stdin.fileno()
+
+    def typeahead_hash(self) -> str:
+        return "win32-input"
+
+    def close(self) -> None:
+        self.console_input_reader.close()
+
+    @property
+    def handle(self) -> HANDLE:
+        return self.console_input_reader.handle
+
+
+class ConsoleInputReader:
+    """
+    :param recognize_paste: When True, try to discover paste actions and turn
+        the event into a BracketedPaste.
+    """
+
+    # Keys with character data.
+    mappings = {
+        b"\x1b": Keys.Escape,
+        b"\x00": Keys.ControlSpace,  # Control-Space (Also for Ctrl-@)
+        b"\x01": Keys.ControlA,  # Control-A (home)
+        b"\x02": Keys.ControlB,  # Control-B (emacs cursor left)
+        b"\x03": Keys.ControlC,  # Control-C (interrupt)
+        b"\x04": Keys.ControlD,  # Control-D (exit)
+        b"\x05": Keys.ControlE,  # Control-E (end)
+        b"\x06": Keys.ControlF,  # Control-F (cursor forward)
+        b"\x07": Keys.ControlG,  # Control-G
+        b"\x08": Keys.ControlH,  # Control-H (8) (Identical to '\b')
+        b"\x09": Keys.ControlI,  # Control-I (9) (Identical to '\t')
+        b"\x0a": Keys.ControlJ,  # Control-J (10) (Identical to '\n')
+        b"\x0b": Keys.ControlK,  # Control-K (delete until end of line; vertical tab)
+        b"\x0c": Keys.ControlL,  # Control-L (clear; form feed)
+        b"\x0d": Keys.ControlM,  # Control-M (enter)
+        b"\x0e": Keys.ControlN,  # Control-N (14) (history forward)
+        b"\x0f": Keys.ControlO,  # Control-O (15)
+        b"\x10": Keys.ControlP,  # Control-P (16) (history back)
+        b"\x11": Keys.ControlQ,  # Control-Q
+        b"\x12": Keys.ControlR,  # Control-R (18) (reverse search)
+        b"\x13": Keys.ControlS,  # Control-S (19) (forward search)
+        b"\x14": Keys.ControlT,  # Control-T
+        b"\x15": Keys.ControlU,  # Control-U
+        b"\x16": Keys.ControlV,  # Control-V
+        b"\x17": Keys.ControlW,  # Control-W
+        b"\x18": Keys.ControlX,  # Control-X
+        b"\x19": Keys.ControlY,  # Control-Y (25)
+        b"\x1a": Keys.ControlZ,  # Control-Z
+        b"\x1c": Keys.ControlBackslash,  # Both Control-\ and Ctrl-|
+        b"\x1d": Keys.ControlSquareClose,  # Control-]
+        b"\x1e": Keys.ControlCircumflex,  # Control-^
+        b"\x1f": Keys.ControlUnderscore,  # Control-underscore (Also for Ctrl-hyphen.)
+        b"\x7f": Keys.Backspace,  # (127) Backspace   (ASCII Delete.)
+    }
+
+    # Keys that don't carry character data.
+    keycodes = {
+        # Home/End
+        33: Keys.PageUp,
+        34: Keys.PageDown,
+        35: Keys.End,
+        36: Keys.Home,
+        # Arrows
+        37: Keys.Left,
+        38: Keys.Up,
+        39: Keys.Right,
+        40: Keys.Down,
+        45: Keys.Insert,
+        46: Keys.Delete,
+        # F-keys.
+        112: Keys.F1,
+        113: Keys.F2,
+        114: Keys.F3,
+        115: Keys.F4,
+        116: Keys.F5,
+        117: Keys.F6,
+        118: Keys.F7,
+        119: Keys.F8,
+        120: Keys.F9,
+        121: Keys.F10,
+        122: Keys.F11,
+        123: Keys.F12,
+    }
+
+    LEFT_ALT_PRESSED = 0x0002
+    RIGHT_ALT_PRESSED = 0x0001
+    SHIFT_PRESSED = 0x0010
+    LEFT_CTRL_PRESSED = 0x0008
+    RIGHT_CTRL_PRESSED = 0x0004
+
+    def __init__(self, recognize_paste: bool = True) -> None:
+        self._fdcon = None
+        self.recognize_paste = recognize_paste
+
+        # When stdin is a tty, use that handle, otherwise, create a handle from
+        # CONIN$.
+        self.handle: HANDLE
+        if sys.stdin.isatty():
+            self.handle = HANDLE(windll.kernel32.GetStdHandle(STD_INPUT_HANDLE))
+        else:
+            self._fdcon = os.open("CONIN$", os.O_RDWR | os.O_BINARY)
+            self.handle = HANDLE(msvcrt.get_osfhandle(self._fdcon))
+
+    def close(self) -> None:
+        " Close fdcon. "
+        if self._fdcon is not None:
+            os.close(self._fdcon)
+
+    def read(self) -> Iterable[KeyPress]:
+        """
+        Return a list of `KeyPress` instances. It won't return anything when
+        there was nothing to read.  (This function doesn't block.)
+
+        http://msdn.microsoft.com/en-us/library/windows/desktop/ms684961(v=vs.85).aspx
+        """
+        max_count = 2048  # Max events to read at the same time.
+
+        read = DWORD(0)
+        arrtype = INPUT_RECORD * max_count
+        input_records = arrtype()
+
+        # Check whether there is some input to read. `ReadConsoleInputW` would
+        # block otherwise.
+        # (Actually, the event loop is responsible to make sure that this
+        # function is only called when there is something to read, but for some
+        # reason this happened in the asyncio_win32 loop, and it's better to be
+        # safe anyway.)
+        if not wait_for_handles([self.handle], timeout=0):
+            return
+
+        # Get next batch of input event.
+        windll.kernel32.ReadConsoleInputW(
+            self.handle, pointer(input_records), max_count, pointer(read)
+        )
+
+        # First, get all the keys from the input buffer, in order to determine
+        # whether we should consider this a paste event or not.
+        all_keys = list(self._get_keys(read, input_records))
+
+        # Fill in 'data' for key presses.
+        all_keys = [self._insert_key_data(key) for key in all_keys]
+
+        if self.recognize_paste and self._is_paste(all_keys):
+            gen = iter(all_keys)
+            k: Optional[KeyPress]
+
+            for k in gen:
+                # Pasting: if the current key consists of text or \n, turn it
+                # into a BracketedPaste.
+                data = []
+                while k and (isinstance(k.key, str) or k.key == Keys.ControlJ):
+                    data.append(k.data)
+                    try:
+                        k = next(gen)
+                    except StopIteration:
+                        k = None
+
+                if data:
+                    yield KeyPress(Keys.BracketedPaste, "".join(data))
+                if k is not None:
+                    yield k
+        else:
+            for k2 in all_keys:
+                yield k2
+
+    def _insert_key_data(self, key_press: KeyPress) -> KeyPress:
+        """
+        Insert KeyPress data, for vt100 compatibility.
+        """
+        if key_press.data:
+            return key_press
+
+        if isinstance(key_press.key, Keys):
+            data = REVERSE_ANSI_SEQUENCES.get(key_press.key, "")
+        else:
+            data = ""
+
+        return KeyPress(key_press.key, data)
+
+    def _get_keys(self, read, input_records):
+        """
+        Generator that yields `KeyPress` objects from the input records.
+        """
+        for i in range(read.value):
+            ir = input_records[i]
+
+            # Get the right EventType from the EVENT_RECORD.
+            # (For some reason the Windows console application 'cmder'
+            # [http://gooseberrycreative.com/cmder/] can return '0' for
+            # ir.EventType. -- Just ignore that.)
+            if ir.EventType in EventTypes:
+                ev = getattr(ir.Event, EventTypes[ir.EventType])
+
+                # Process if this is a key event. (We also have mouse, menu and
+                # focus events.)
+                if type(ev) == KEY_EVENT_RECORD and ev.KeyDown:
+                    for key_press in self._event_to_key_presses(ev):
+                        yield key_press
+
+                elif type(ev) == MOUSE_EVENT_RECORD:
+                    for key_press in self._handle_mouse(ev):
+                        yield key_press
+
+    @staticmethod
+    def _is_paste(keys) -> bool:
+        """
+        Return `True` when we should consider this list of keys as a paste
+        event. Pasted text on windows will be turned into a
+        `Keys.BracketedPaste` event. (It's not 100% correct, but it is probably
+        the best possible way to detect pasting of text and handle that
+        correctly.)
+        """
+        # Consider paste when it contains at least one newline and at least one
+        # other character.
+        text_count = 0
+        newline_count = 0
+
+        for k in keys:
+            if isinstance(k.key, str):
+                text_count += 1
+            if k.key == Keys.ControlM:
+                newline_count += 1
+
+        return newline_count >= 1 and text_count > 1
+
+    def _event_to_key_presses(self, ev):
+        """
+        For this `KEY_EVENT_RECORD`, return a list of `KeyPress` instances.
+        """
+        assert type(ev) == KEY_EVENT_RECORD and ev.KeyDown
+
+        result = None
+
+        u_char = ev.uChar.UnicodeChar
+        ascii_char = u_char.encode("utf-8")
+
+        # NOTE: We don't use `ev.uChar.AsciiChar`. That appears to be latin-1
+        #       encoded. See also:
+        # https://github.com/ipython/ipython/issues/10004
+        # https://github.com/jonathanslenders/python-prompt-toolkit/issues/389
+
+        if u_char == "\x00":
+            if ev.VirtualKeyCode in self.keycodes:
+                result = KeyPress(self.keycodes[ev.VirtualKeyCode], "")
+        else:
+            if ascii_char in self.mappings:
+                if self.mappings[ascii_char] == Keys.ControlJ:
+                    u_char = (
+                        "\n"  # Windows sends \n, turn into \r for unix compatibility.
+                    )
+                result = KeyPress(self.mappings[ascii_char], u_char)
+            else:
+                result = KeyPress(u_char, u_char)
+
+        # First we handle Shift-Control-Arrow/Home/End (need to do this first)
+        if (
+            (
+                ev.ControlKeyState & self.LEFT_CTRL_PRESSED
+                or ev.ControlKeyState & self.RIGHT_CTRL_PRESSED
+            )
+            and ev.ControlKeyState & self.SHIFT_PRESSED
+            and result
+        ):
+            result.key = {
+                Keys.Left: Keys.ControlShiftLeft,
+                Keys.Right: Keys.ControlShiftRight,
+                Keys.Up: Keys.ControlShiftUp,
+                Keys.Down: Keys.ControlShiftDown,
+                Keys.Home: Keys.ControlShiftHome,
+                Keys.End: Keys.ControlShiftEnd,
+                Keys.Insert: Keys.ControlShiftInsert,
+                Keys.PageUp: Keys.ControlShiftPageUp,
+                Keys.PageDown: Keys.ControlShiftPageDown,
+            }.get(result.key, result.key)
+
+        # Correctly handle Control-Arrow/Home/End and Control-Insert keys.
+        if (
+            ev.ControlKeyState & self.LEFT_CTRL_PRESSED
+            or ev.ControlKeyState & self.RIGHT_CTRL_PRESSED
+        ) and result:
+            result.key = {
+                Keys.Left: Keys.ControlLeft,
+                Keys.Right: Keys.ControlRight,
+                Keys.Up: Keys.ControlUp,
+                Keys.Down: Keys.ControlDown,
+                Keys.Home: Keys.ControlHome,
+                Keys.End: Keys.ControlEnd,
+                Keys.Insert: Keys.ControlInsert,
+                Keys.PageUp: Keys.ControlPageUp,
+                Keys.PageDown: Keys.ControlPageDown,
+            }.get(result.key, result.key)
+
+        # Turn 'Tab' into 'BackTab' when shift was pressed.
+        # Also handle other shift-key combination
+        if ev.ControlKeyState & self.SHIFT_PRESSED and result:
+            result.key = {
+                Keys.Tab: Keys.BackTab,
+                Keys.Left: Keys.ShiftLeft,
+                Keys.Right: Keys.ShiftRight,
+                Keys.Up: Keys.ShiftUp,
+                Keys.Down: Keys.ShiftDown,
+                Keys.Home: Keys.ShiftHome,
+                Keys.End: Keys.ShiftEnd,
+                Keys.Insert: Keys.ShiftInsert,
+                Keys.PageUp: Keys.ShiftPageUp,
+                Keys.PageDown: Keys.ShiftPageDown,
+            }.get(result.key, result.key)
+
+        # Turn 'Space' into 'ControlSpace' when control was pressed.
+        if (
+            (
+                ev.ControlKeyState & self.LEFT_CTRL_PRESSED
+                or ev.ControlKeyState & self.RIGHT_CTRL_PRESSED
+            )
+            and result
+            and result.data == " "
+        ):
+            result = KeyPress(Keys.ControlSpace, " ")
+
+        # Turn Control-Enter into META-Enter. (On a vt100 terminal, we cannot
+        # detect this combination. But it's really practical on Windows.)
+        if (
+            (
+                ev.ControlKeyState & self.LEFT_CTRL_PRESSED
+                or ev.ControlKeyState & self.RIGHT_CTRL_PRESSED
+            )
+            and result
+            and result.key == Keys.ControlJ
+        ):
+            return [KeyPress(Keys.Escape, ""), result]
+
+        # Return result. If alt was pressed, prefix the result with an
+        # 'Escape' key, just like unix VT100 terminals do.
+
+        # NOTE: Only replace the left alt with escape. The right alt key often
+        #       acts as altgr and is used in many non US keyboard layouts for
+        #       typing some special characters, like a backslash. We don't want
+        #       all backslashes to be prefixed with escape. (Esc-\ has a
+        #       meaning in E-macs, for instance.)
+        if result:
+            meta_pressed = ev.ControlKeyState & self.LEFT_ALT_PRESSED
+
+            if meta_pressed:
+                return [KeyPress(Keys.Escape, ""), result]
+            else:
+                return [result]
+
+        else:
+            return []
+
+    def _handle_mouse(self, ev):
+        """
+        Handle mouse events. Return a list of KeyPress instances.
+        """
+        FROM_LEFT_1ST_BUTTON_PRESSED = 0x1
+
+        result = []
+
+        # Check event type.
+        if ev.ButtonState == FROM_LEFT_1ST_BUTTON_PRESSED:
+            # On a key press, generate both the mouse down and up event.
+            for event_type in [MouseEventType.MOUSE_DOWN, MouseEventType.MOUSE_UP]:
+                data = ";".join(
+                    [event_type.value, str(ev.MousePosition.X), str(ev.MousePosition.Y)]
+                )
+                result.append(KeyPress(Keys.WindowsMouseEvent, data))
+
+        return result
+
+
+class _Win32Handles:
+    """
+    Utility to keep track of which handles are connectod to which callbacks.
+
+    `add_win32_handle` starts a tiny event loop in another thread which waits
+    for the Win32 handle to become ready. When this happens, the callback will
+    be called in the current asyncio event loop using `call_soon_threadsafe`.
+
+    `remove_win32_handle` will stop this tiny event loop.
+
+    NOTE: We use this technique, so that we don't have to use the
+          `ProactorEventLoop` on Windows and we can wait for things like stdin
+          in a `SelectorEventLoop`. This is important, because our inputhook
+          mechanism (used by IPython), only works with the `SelectorEventLoop`.
+    """
+
+    def __init__(self) -> None:
+        self._handle_callbacks: Dict[int, Callable[[], None]] = {}
+
+        # Windows Events that are triggered when we have to stop watching this
+        # handle.
+        self._remove_events: Dict[int, HANDLE] = {}
+
+    def add_win32_handle(self, handle: HANDLE, callback: Callable[[], None]) -> None:
+        """
+        Add a Win32 handle to the event loop.
+        """
+        handle_value = handle.value
+
+        if handle_value is None:
+            raise ValueError("Invalid handle.")
+
+        # Make sure to remove a previous registered handler first.
+        self.remove_win32_handle(handle)
+
+        loop = get_event_loop()
+        self._handle_callbacks[handle_value] = callback
+
+        # Create remove event.
+        remove_event = create_win32_event()
+        self._remove_events[handle_value] = remove_event
+
+        # Add reader.
+        def ready() -> None:
+            # Tell the callback that input's ready.
+            try:
+                callback()
+            finally:
+                run_in_executor_with_context(wait, loop=loop)
+
+        # Wait for the input to become ready.
+        # (Use an executor for this, the Windows asyncio event loop doesn't
+        # allow us to wait for handles like stdin.)
+        def wait() -> None:
+            # Wait until either the handle becomes ready, or the remove event
+            # has been set.
+            result = wait_for_handles([remove_event, handle])
+
+            if result is remove_event:
+                windll.kernel32.CloseHandle(remove_event)
+                return
+            else:
+                loop.call_soon_threadsafe(ready)
+
+        run_in_executor_with_context(wait, loop=loop)
+
+    def remove_win32_handle(self, handle: HANDLE) -> Optional[Callable[[], None]]:
+        """
+        Remove a Win32 handle from the event loop.
+        Return either the registered handler or `None`.
+        """
+        if handle.value is None:
+            return None  # Ignore.
+
+        # Trigger remove events, so that the reader knows to stop.
+        try:
+            event = self._remove_events.pop(handle.value)
+        except KeyError:
+            pass
+        else:
+            windll.kernel32.SetEvent(event)
+
+        try:
+            return self._handle_callbacks.pop(handle.value)
+        except KeyError:
+            return None
+
+
+@contextmanager
+def attach_win32_input(input: _Win32InputBase, callback: Callable[[], None]):
+    """
+    Context manager that makes this input active in the current event loop.
+
+    :param input: :class:`~prompt_toolkit.input.Input` object.
+    :param input_ready_callback: Called when the input is ready to read.
+    """
+    win32_handles = input.win32_handles
+    handle = input.handle
+
+    if handle.value is None:
+        raise ValueError("Invalid handle.")
+
+    # Add reader.
+    previous_callback = win32_handles.remove_win32_handle(handle)
+    win32_handles.add_win32_handle(handle, callback)
+
+    try:
+        yield
+    finally:
+        win32_handles.remove_win32_handle(handle)
+
+        if previous_callback:
+            win32_handles.add_win32_handle(handle, previous_callback)
+
+
+@contextmanager
+def detach_win32_input(input: _Win32InputBase):
+    win32_handles = input.win32_handles
+    handle = input.handle
+
+    if handle.value is None:
+        raise ValueError("Invalid handle.")
+
+    previous_callback = win32_handles.remove_win32_handle(handle)
+
+    try:
+        yield
+    finally:
+        if previous_callback:
+            win32_handles.add_win32_handle(handle, previous_callback)
+
+
+class raw_mode:
+    """
+    ::
+
+        with raw_mode(stdin):
+            ''' the windows terminal is now in 'raw' mode. '''
+
+    The ``fileno`` attribute is ignored. This is to be compatible with the
+    `raw_input` method of `.vt100_input`.
+    """
+
+    def __init__(self, fileno=None):
+        self.handle = HANDLE(windll.kernel32.GetStdHandle(STD_INPUT_HANDLE))
+
+    def __enter__(self):
+        # Remember original mode.
+        original_mode = DWORD()
+        windll.kernel32.GetConsoleMode(self.handle, pointer(original_mode))
+        self.original_mode = original_mode
+
+        self._patch()
+
+    def _patch(self) -> None:
+        # Set raw
+        ENABLE_ECHO_INPUT = 0x0004
+        ENABLE_LINE_INPUT = 0x0002
+        ENABLE_PROCESSED_INPUT = 0x0001
+
+        windll.kernel32.SetConsoleMode(
+            self.handle,
+            self.original_mode.value
+            & ~(ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT),
+        )
+
+    def __exit__(self, *a: object) -> None:
+        # Restore original mode
+        windll.kernel32.SetConsoleMode(self.handle, self.original_mode)
+
+
+class cooked_mode(raw_mode):
+    """
+    ::
+
+        with cooked_mode(stdin):
+            ''' The pseudo-terminal stdin is now used in cooked mode. '''
+    """
+
+    def _patch(self) -> None:
+        # Set cooked.
+        ENABLE_ECHO_INPUT = 0x0004
+        ENABLE_LINE_INPUT = 0x0002
+        ENABLE_PROCESSED_INPUT = 0x0001
+
+        windll.kernel32.SetConsoleMode(
+            self.handle,
+            self.original_mode.value
+            | (ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT),
+        )

xonsh/vended_ptk/prompt_toolkit/input/win32_pipe.py

@@ -0,0 +1,137 @@
+from ctypes import windll
+from typing import Callable, ContextManager, List
+
+from prompt_toolkit.eventloop.win32 import create_win32_event
+
+from ..key_binding import KeyPress
+from ..utils import DummyContext
+from .vt100_parser import Vt100Parser
+from .win32 import _Win32InputBase, attach_win32_input, detach_win32_input
+
+__all__ = ["Win32PipeInput"]
+
+
+class Win32PipeInput(_Win32InputBase):
+    """
+    This is an input pipe that works on Windows.
+    Text or bytes can be feed into the pipe, and key strokes can be read from
+    the pipe. This is useful if we want to send the input programmatically into
+    the application. Mostly useful for unit testing.
+
+    Notice that even though it's Windows, we use vt100 escape sequences over
+    the pipe.
+
+    Usage::
+
+        input = Win32PipeInput()
+        input.send_text('inputdata')
+    """
+
+    _id = 0
+
+    def __init__(self) -> None:
+        super().__init__()
+        # Event (handle) for registering this input in the event loop.
+        # This event is set when there is data available to read from the pipe.
+        # Note: We use this approach instead of using a regular pipe, like
+        #       returned from `os.pipe()`, because making such a regular pipe
+        #       non-blocking is tricky and this works really well.
+        self._event = create_win32_event()
+
+        self._closed = False
+
+        # Parser for incoming keys.
+        self._buffer: List[KeyPress] = []  # Buffer to collect the Key objects.
+        self.vt100_parser = Vt100Parser(lambda key: self._buffer.append(key))
+
+        # Identifier for every PipeInput for the hash.
+        self.__class__._id += 1
+        self._id = self.__class__._id
+
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
+    def fileno(self):
+        """
+        The windows pipe doesn't depend on the file handle.
+        """
+        raise NotImplementedError
+
+    @property
+    def handle(self):
+        " The handle used for registering this pipe in the event loop. "
+        return self._event
+
+    def attach(self, input_ready_callback: Callable) -> ContextManager[None]:
+        """
+        Return a context manager that makes this input active in the current
+        event loop.
+        """
+        return attach_win32_input(self, input_ready_callback)
+
+    def detach(self) -> ContextManager[None]:
+        """
+        Return a context manager that makes sure that this input is not active
+        in the current event loop.
+        """
+        return detach_win32_input(self)
+
+    def read_keys(self) -> List[KeyPress]:
+        " Read list of KeyPress. "
+
+        # Return result.
+        result = self._buffer
+        self._buffer = []
+
+        # Reset event.
+        windll.kernel32.ResetEvent(self._event)
+
+        return result
+
+    def flush_keys(self) -> List[KeyPress]:
+        """
+        Flush pending keys and return them.
+        (Used for flushing the 'escape' key.)
+        """
+        # Flush all pending keys. (This is most important to flush the vt100
+        # 'Escape' key early when nothing else follows.)
+        self.vt100_parser.flush()
+
+        # Return result.
+        result = self._buffer
+        self._buffer = []
+        return result
+
+    @property
+    def responds_to_cpr(self) -> bool:
+        return False
+
+    def send_bytes(self, data: bytes) -> None:
+        " Send bytes to the input. "
+        self.send_text(data.decode("utf-8", "ignore"))
+
+    def send_text(self, text: str) -> None:
+        " Send text to the input. "
+        # Pass it through our vt100 parser.
+        self.vt100_parser.feed(text)
+
+        # Set event.
+        windll.kernel32.SetEvent(self._event)
+
+    def raw_mode(self) -> ContextManager[None]:
+        return DummyContext()
+
+    def cooked_mode(self) -> ContextManager[None]:
+        return DummyContext()
+
+    def close(self) -> None:
+        " Close pipe handles. "
+        windll.kernel32.CloseHandle(self._event)
+        self._closed = True
+
+    def typeahead_hash(self) -> str:
+        """
+        This needs to be unique for every `PipeInput`.
+        """
+        return "pipe-input-%s" % (self._id,)

xonsh/vended_ptk/prompt_toolkit/key_binding/__init__.py

@@ -0,0 +1,17 @@
+from .key_bindings import (
+    ConditionalKeyBindings,
+    DynamicKeyBindings,
+    KeyBindings,
+    KeyBindingsBase,
+    merge_key_bindings,
+)
+from .key_processor import KeyPress
+
+__all__ = [
+    "ConditionalKeyBindings",
+    "DynamicKeyBindings",
+    "KeyBindings",
+    "KeyBindingsBase",
+    "merge_key_bindings",
+    "KeyPress",
+]

xonsh/vended_ptk/prompt_toolkit/key_binding/bindings/auto_suggest.py

@@ -0,0 +1,62 @@
+"""
+Key bindings for auto suggestion (for fish-style auto suggestion).
+"""
+import re
+
+from prompt_toolkit.application.current import get_app
+from prompt_toolkit.filters import Condition, emacs_mode
+from prompt_toolkit.key_binding.key_bindings import KeyBindings
+from prompt_toolkit.key_binding.key_processor import KeyPressEvent
+
+__all__ = [
+    "load_auto_suggest_bindings",
+]
+
+E = KeyPressEvent
+
+
+def load_auto_suggest_bindings() -> KeyBindings:
+    """
+    Key bindings for accepting auto suggestion text.
+
+    (This has to come after the Vi bindings, because they also have an
+    implementation for the "right arrow", but we really want the suggestion
+    binding when a suggestion is available.)
+    """
+    key_bindings = KeyBindings()
+    handle = key_bindings.add
+
+    @Condition
+    def suggestion_available() -> bool:
+        app = get_app()
+        return (
+            app.current_buffer.suggestion is not None
+            and app.current_buffer.document.is_cursor_at_the_end
+        )
+
+    @handle("c-f", filter=suggestion_available)
+    @handle("c-e", filter=suggestion_available)
+    @handle("right", filter=suggestion_available)
+    def _accept(event: E) -> None:
+        """
+        Accept suggestion.
+        """
+        b = event.current_buffer
+        suggestion = b.suggestion
+
+        if suggestion:
+            b.insert_text(suggestion.text)
+
+    @handle("escape", "f", filter=suggestion_available & emacs_mode)
+    def _fill(event: E) -> None:
+        """
+        Fill partial suggestion.
+        """
+        b = event.current_buffer
+        suggestion = b.suggestion
+
+        if suggestion:
+            t = re.split(r"(\S+\s+)", suggestion.text)
+            b.insert_text(next(x for x in t if x))
+
+    return key_bindings

xonsh/vended_ptk/prompt_toolkit/key_binding/bindings/basic.py

@@ -0,0 +1,248 @@
+# pylint: disable=function-redefined
+from prompt_toolkit.application.current import get_app
+from prompt_toolkit.filters import (
+    Condition,
+    emacs_insert_mode,
+    has_selection,
+    in_paste_mode,
+    is_multiline,
+    vi_insert_mode,
+)
+from prompt_toolkit.key_binding.key_processor import KeyPress, KeyPressEvent
+from prompt_toolkit.keys import Keys
+
+from ..key_bindings import KeyBindings
+from .named_commands import get_by_name
+
+__all__ = [
+    "load_basic_bindings",
+]
+
+E = KeyPressEvent
+
+
+def if_no_repeat(event: E) -> bool:
+    """ Callable that returns True when the previous event was delivered to
+    another handler. """
+    return not event.is_repeat
+
+
+def load_basic_bindings() -> KeyBindings:
+    key_bindings = KeyBindings()
+    insert_mode = vi_insert_mode | emacs_insert_mode
+    handle = key_bindings.add
+
+    @handle("c-a")
+    @handle("c-b")
+    @handle("c-c")
+    @handle("c-d")
+    @handle("c-e")
+    @handle("c-f")
+    @handle("c-g")
+    @handle("c-h")
+    @handle("c-i")
+    @handle("c-j")
+    @handle("c-k")
+    @handle("c-l")
+    @handle("c-m")
+    @handle("c-n")
+    @handle("c-o")
+    @handle("c-p")
+    @handle("c-q")
+    @handle("c-r")
+    @handle("c-s")
+    @handle("c-t")
+    @handle("c-u")
+    @handle("c-v")
+    @handle("c-w")
+    @handle("c-x")
+    @handle("c-y")
+    @handle("c-z")
+    @handle("f1")
+    @handle("f2")
+    @handle("f3")
+    @handle("f4")
+    @handle("f5")
+    @handle("f6")
+    @handle("f7")
+    @handle("f8")
+    @handle("f9")
+    @handle("f10")
+    @handle("f11")
+    @handle("f12")
+    @handle("f13")
+    @handle("f14")
+    @handle("f15")
+    @handle("f16")
+    @handle("f17")
+    @handle("f18")
+    @handle("f19")
+    @handle("f20")
+    @handle("c-@")  # Also c-space.
+    @handle("c-\\")
+    @handle("c-]")
+    @handle("c-^")
+    @handle("c-_")
+    @handle("backspace")
+    @handle("up")
+    @handle("down")
+    @handle("right")
+    @handle("left")
+    @handle("s-up")
+    @handle("s-down")
+    @handle("s-right")
+    @handle("s-left")
+    @handle("home")
+    @handle("end")
+    @handle("s-home")
+    @handle("s-end")
+    @handle("delete")
+    @handle("s-delete")
+    @handle("c-delete")
+    @handle("pageup")
+    @handle("pagedown")
+    @handle("s-tab")
+    @handle("tab")
+    @handle("c-s-left")
+    @handle("c-s-right")
+    @handle("c-s-home")
+    @handle("c-s-end")
+    @handle("c-left")
+    @handle("c-right")
+    @handle("c-up")
+    @handle("c-down")
+    @handle("c-home")
+    @handle("c-end")
+    @handle("insert")
+    @handle("s-insert")
+    @handle("c-insert")
+    @handle(Keys.Ignore)
+    def _ignore(event: E) -> None:
+        """
+        First, for any of these keys, Don't do anything by default. Also don't
+        catch them in the 'Any' handler which will insert them as data.
+
+        If people want to insert these characters as a literal, they can always
+        do by doing a quoted insert. (ControlQ in emacs mode, ControlV in Vi
+        mode.)
+        """
+        pass
+
+    # Readline-style bindings.
+    handle("home")(get_by_name("beginning-of-line"))
+    handle("end")(get_by_name("end-of-line"))
+    handle("left")(get_by_name("backward-char"))
+    handle("right")(get_by_name("forward-char"))
+    handle("c-up")(get_by_name("previous-history"))
+    handle("c-down")(get_by_name("next-history"))
+    handle("c-l")(get_by_name("clear-screen"))
+
+    handle("c-k", filter=insert_mode)(get_by_name("kill-line"))
+    handle("c-u", filter=insert_mode)(get_by_name("unix-line-discard"))
+    handle("backspace", filter=insert_mode, save_before=if_no_repeat)(
+        get_by_name("backward-delete-char")
+    )
+    handle("delete", filter=insert_mode, save_before=if_no_repeat)(
+        get_by_name("delete-char")
+    )
+    handle("c-delete", filter=insert_mode, save_before=if_no_repeat)(
+        get_by_name("delete-char")
+    )
+    handle(Keys.Any, filter=insert_mode, save_before=if_no_repeat)(
+        get_by_name("self-insert")
+    )
+    handle("c-t", filter=insert_mode)(get_by_name("transpose-chars"))
+    handle("c-i", filter=insert_mode)(get_by_name("menu-complete"))
+    handle("s-tab", filter=insert_mode)(get_by_name("menu-complete-backward"))
+
+    # Control-W should delete, using whitespace as separator, while M-Del
+    # should delete using [^a-zA-Z0-9] as a boundary.
+    handle("c-w", filter=insert_mode)(get_by_name("unix-word-rubout"))
+
+    handle("pageup", filter=~has_selection)(get_by_name("previous-history"))
+    handle("pagedown", filter=~has_selection)(get_by_name("next-history"))
+
+    # CTRL keys.
+
+    @Condition
+    def has_text_before_cursor() -> bool:
+        return bool(get_app().current_buffer.text)
+
+    handle("c-d", filter=has_text_before_cursor & insert_mode)(
+        get_by_name("delete-char")
+    )
+
+    @handle("enter", filter=insert_mode & is_multiline)
+    def _newline(event: E) -> None:
+        """
+        Newline (in case of multiline input.
+        """
+        event.current_buffer.newline(copy_margin=not in_paste_mode())
+
+    @handle("c-j")
+    def _newline2(event: E) -> None:
+        r"""
+        By default, handle \n as if it were a \r (enter).
+        (It appears that some terminals send \n instead of \r when pressing
+        enter. - at least the Linux subsystem for Windows.)
+        """
+        event.key_processor.feed(KeyPress(Keys.ControlM, "\r"), first=True)
+
+    # Delete the word before the cursor.
+
+    @handle("up")
+    def _go_up(event: E) -> None:
+        event.current_buffer.auto_up(count=event.arg)
+
+    @handle("down")
+    def _go_down(event: E) -> None:
+        event.current_buffer.auto_down(count=event.arg)
+
+    @handle("delete", filter=has_selection)
+    def _cut(event: E) -> None:
+        data = event.current_buffer.cut_selection()
+        event.app.clipboard.set_data(data)
+
+    # Global bindings.
+
+    @handle("c-z")
+    def _insert_ctrl_z(event: E) -> None:
+        """
+        By default, control-Z should literally insert Ctrl-Z.
+        (Ansi Ctrl-Z, code 26 in MSDOS means End-Of-File.
+        In a Python REPL for instance, it's possible to type
+        Control-Z followed by enter to quit.)
+
+        When the system bindings are loaded and suspend-to-background is
+        supported, that will override this binding.
+        """
+        event.current_buffer.insert_text(event.data)
+
+    @handle(Keys.BracketedPaste)
+    def _paste(event: E) -> None:
+        """
+        Pasting from clipboard.
+        """
+        data = event.data
+
+        # Be sure to use \n as line ending.
+        # Some terminals (Like iTerm2) seem to paste \r\n line endings in a
+        # bracketed paste. See: https://github.com/ipython/ipython/issues/9737
+        data = data.replace("\r\n", "\n")
+        data = data.replace("\r", "\n")
+
+        event.current_buffer.insert_text(data)
+
+    @Condition
+    def in_quoted_insert() -> bool:
+        return get_app().quoted_insert
+
+    @handle(Keys.Any, filter=in_quoted_insert, eager=True)
+    def _insert_text(event: E) -> None:
+        """
+        Handle quoted insert.
+        """
+        event.current_buffer.insert_text(event.data, overwrite=False)
+        event.app.quoted_insert = False
+
+    return key_bindings

xonsh/vended_ptk/prompt_toolkit/key_binding/bindings/completion.py

@@ -0,0 +1,203 @@
+"""
+Key binding handlers for displaying completions.
+"""
+import asyncio
+import math
+from typing import TYPE_CHECKING, List
+
+from prompt_toolkit.application.run_in_terminal import in_terminal
+from prompt_toolkit.completion import (
+    CompleteEvent,
+    Completion,
+    get_common_complete_suffix,
+)
+from prompt_toolkit.formatted_text import StyleAndTextTuples
+from prompt_toolkit.key_binding.key_bindings import KeyBindings
+from prompt_toolkit.key_binding.key_processor import KeyPressEvent
+from prompt_toolkit.keys import Keys
+from prompt_toolkit.utils import get_cwidth
+
+if TYPE_CHECKING:
+    from prompt_toolkit.application import Application
+    from prompt_toolkit.shortcuts import PromptSession
+
+__all__ = [
+    "generate_completions",
+    "display_completions_like_readline",
+]
+
+E = KeyPressEvent
+
+
+def generate_completions(event: E) -> None:
+    r"""
+    Tab-completion: where the first tab completes the common suffix and the
+    second tab lists all the completions.
+    """
+    b = event.current_buffer
+
+    # When already navigating through completions, select the next one.
+    if b.complete_state:
+        b.complete_next()
+    else:
+        b.start_completion(insert_common_part=True)
+
+
+def display_completions_like_readline(event: E) -> None:
+    """
+    Key binding handler for readline-style tab completion.
+    This is meant to be as similar as possible to the way how readline displays
+    completions.
+
+    Generate the completions immediately (blocking) and display them above the
+    prompt in columns.
+
+    Usage::
+
+        # Call this handler when 'Tab' has been pressed.
+        key_bindings.add(Keys.ControlI)(display_completions_like_readline)
+    """
+    # Request completions.
+    b = event.current_buffer
+    if b.completer is None:
+        return
+    complete_event = CompleteEvent(completion_requested=True)
+    completions = list(b.completer.get_completions(b.document, complete_event))
+
+    # Calculate the common suffix.
+    common_suffix = get_common_complete_suffix(b.document, completions)
+
+    # One completion: insert it.
+    if len(completions) == 1:
+        b.delete_before_cursor(-completions[0].start_position)
+        b.insert_text(completions[0].text)
+    # Multiple completions with common part.
+    elif common_suffix:
+        b.insert_text(common_suffix)
+    # Otherwise: display all completions.
+    elif completions:
+        _display_completions_like_readline(event.app, completions)
+
+
+def _display_completions_like_readline(
+    app: "Application", completions: List[Completion]
+) -> "asyncio.Task[None]":
+    """
+    Display the list of completions in columns above the prompt.
+    This will ask for a confirmation if there are too many completions to fit
+    on a single page and provide a paginator to walk through them.
+    """
+    from prompt_toolkit.shortcuts.prompt import create_confirm_session
+    from prompt_toolkit.formatted_text import to_formatted_text
+
+    # Get terminal dimensions.
+    term_size = app.output.get_size()
+    term_width = term_size.columns
+    term_height = term_size.rows
+
+    # Calculate amount of required columns/rows for displaying the
+    # completions. (Keep in mind that completions are displayed
+    # alphabetically column-wise.)
+    max_compl_width = min(
+        term_width, max(get_cwidth(c.display_text) for c in completions) + 1
+    )
+    column_count = max(1, term_width // max_compl_width)
+    completions_per_page = column_count * (term_height - 1)
+    page_count = int(math.ceil(len(completions) / float(completions_per_page)))
+    # Note: math.ceil can return float on Python2.
+
+    def display(page: int) -> None:
+        # Display completions.
+        page_completions = completions[
+            page * completions_per_page : (page + 1) * completions_per_page
+        ]
+
+        page_row_count = int(math.ceil(len(page_completions) / float(column_count)))
+        page_columns = [
+            page_completions[i * page_row_count : (i + 1) * page_row_count]
+            for i in range(column_count)
+        ]
+
+        result: StyleAndTextTuples = []
+
+        for r in range(page_row_count):
+            for c in range(column_count):
+                try:
+                    completion = page_columns[c][r]
+                    style = "class:readline-like-completions.completion " + (
+                        completion.style or ""
+                    )
+
+                    result.extend(to_formatted_text(completion.display, style=style))
+
+                    # Add padding.
+                    padding = max_compl_width - get_cwidth(completion.display_text)
+                    result.append((completion.style, " " * padding,))
+                except IndexError:
+                    pass
+            result.append(("", "\n"))
+
+        app.print_text(to_formatted_text(result, "class:readline-like-completions"))
+
+    # User interaction through an application generator function.
+    async def run_compl() -> None:
+        " Coroutine. "
+        async with in_terminal(render_cli_done=True):
+            if len(completions) > completions_per_page:
+                # Ask confirmation if it doesn't fit on the screen.
+                confirm = await create_confirm_session(
+                    "Display all {} possibilities?".format(len(completions)),
+                ).prompt_async()
+
+                if confirm:
+                    # Display pages.
+                    for page in range(page_count):
+                        display(page)
+
+                        if page != page_count - 1:
+                            # Display --MORE-- and go to the next page.
+                            show_more = await _create_more_session(
+                                "--MORE--"
+                            ).prompt_async()
+
+                            if not show_more:
+                                return
+                else:
+                    app.output.flush()
+            else:
+                # Display all completions.
+                display(0)
+
+    return app.create_background_task(run_compl())
+
+
+def _create_more_session(message: str = "--MORE--") -> "PromptSession":
+    """
+    Create a `PromptSession` object for displaying the "--MORE--".
+    """
+    from prompt_toolkit.shortcuts import PromptSession
+
+    bindings = KeyBindings()
+
+    @bindings.add(" ")
+    @bindings.add("y")
+    @bindings.add("Y")
+    @bindings.add(Keys.ControlJ)
+    @bindings.add(Keys.ControlM)
+    @bindings.add(Keys.ControlI)  # Tab.
+    def _yes(event: E) -> None:
+        event.app.exit(result=True)
+
+    @bindings.add("n")
+    @bindings.add("N")
+    @bindings.add("q")
+    @bindings.add("Q")
+    @bindings.add(Keys.ControlC)
+    def _no(event: E) -> None:
+        event.app.exit(result=False)
+
+    @bindings.add(Keys.Any)
+    def _ignore(event: E) -> None:
+        " Disable inserting of text. "
+
+    return PromptSession(message, key_bindings=bindings, erase_when_done=True)

xonsh/vended_ptk/prompt_toolkit/key_binding/bindings/cpr.py

@@ -0,0 +1,28 @@
+from prompt_toolkit.key_binding.key_processor import KeyPressEvent
+from prompt_toolkit.keys import Keys
+
+from ..key_bindings import KeyBindings
+
+__all__ = [
+    "load_cpr_bindings",
+]
+
+E = KeyPressEvent
+
+
+def load_cpr_bindings() -> KeyBindings:
+    key_bindings = KeyBindings()
+
+    @key_bindings.add(Keys.CPRResponse, save_before=lambda e: False)
+    def _(event: E) -> None:
+        """
+        Handle incoming Cursor-Position-Request response.
+        """
+        # The incoming data looks like u'\x1b[35;1R'
+        # Parse row/col information.
+        row, col = map(int, event.data[2:-1].split(";"))
+
+        # Report absolute cursor position to the renderer.
+        event.app.renderer.report_absolute_cursor_row(row)
+
+    return key_bindings

xonsh/vended_ptk/prompt_toolkit/key_binding/bindings/emacs.py

@@ -0,0 +1,558 @@
+# pylint: disable=function-redefined
+from typing import Dict, Union
+
+from prompt_toolkit.application.current import get_app
+from prompt_toolkit.buffer import Buffer, SelectionType, indent, unindent
+from prompt_toolkit.completion import CompleteEvent
+from prompt_toolkit.filters import (
+    Condition,
+    emacs_insert_mode,
+    emacs_mode,
+    has_arg,
+    has_selection,
+    in_paste_mode,
+    is_multiline,
+    is_read_only,
+    shift_selection_mode,
+    vi_search_direction_reversed,
+)
+from prompt_toolkit.key_binding.key_bindings import Binding
+from prompt_toolkit.key_binding.key_processor import KeyPressEvent
+from prompt_toolkit.keys import Keys
+
+from ..key_bindings import ConditionalKeyBindings, KeyBindings, KeyBindingsBase
+from .named_commands import get_by_name
+
+__all__ = [
+    "load_emacs_bindings",
+    "load_emacs_search_bindings",
+    "load_emacs_shift_selection_bindings",
+]
+
+E = KeyPressEvent
+
+
+def load_emacs_bindings() -> KeyBindingsBase:
+    """
+    Some e-macs extensions.
+    """
+    # Overview of Readline emacs commands:
+    # http://www.catonmat.net/download/readline-emacs-editing-mode-cheat-sheet.pdf
+    key_bindings = KeyBindings()
+    handle = key_bindings.add
+
+    insert_mode = emacs_insert_mode
+
+    @handle("escape")
+    def _esc(event: E) -> None:
+        """
+        By default, ignore escape key.
+
+        (If we don't put this here, and Esc is followed by a key which sequence
+        is not handled, we'll insert an Escape character in the input stream.
+        Something we don't want and happens to easily in emacs mode.
+        Further, people can always use ControlQ to do a quoted insert.)
+        """
+        pass
+
+    handle("c-a")(get_by_name("beginning-of-line"))
+    handle("c-b")(get_by_name("backward-char"))
+    handle("c-delete", filter=insert_mode)(get_by_name("kill-word"))
+    handle("c-e")(get_by_name("end-of-line"))
+    handle("c-f")(get_by_name("forward-char"))
+    handle("c-left")(get_by_name("backward-word"))
+    handle("c-right")(get_by_name("forward-word"))
+    handle("c-x", "r", "y", filter=insert_mode)(get_by_name("yank"))
+    handle("c-y", filter=insert_mode)(get_by_name("yank"))
+    handle("escape", "b")(get_by_name("backward-word"))
+    handle("escape", "c", filter=insert_mode)(get_by_name("capitalize-word"))
+    handle("escape", "d", filter=insert_mode)(get_by_name("kill-word"))
+    handle("escape", "f")(get_by_name("forward-word"))
+    handle("escape", "l", filter=insert_mode)(get_by_name("downcase-word"))
+    handle("escape", "u", filter=insert_mode)(get_by_name("uppercase-word"))
+    handle("escape", "y", filter=insert_mode)(get_by_name("yank-pop"))
+    handle("escape", "backspace", filter=insert_mode)(get_by_name("backward-kill-word"))
+    handle("escape", "\\", filter=insert_mode)(get_by_name("delete-horizontal-space"))
+
+    handle("c-home")(get_by_name("beginning-of-buffer"))
+    handle("c-end")(get_by_name("end-of-buffer"))
+
+    handle("c-_", save_before=(lambda e: False), filter=insert_mode)(
+        get_by_name("undo")
+    )
+
+    handle("c-x", "c-u", save_before=(lambda e: False), filter=insert_mode)(
+        get_by_name("undo")
+    )
+
+    handle("escape", "<", filter=~has_selection)(get_by_name("beginning-of-history"))
+    handle("escape", ">", filter=~has_selection)(get_by_name("end-of-history"))
+
+    handle("escape", ".", filter=insert_mode)(get_by_name("yank-last-arg"))
+    handle("escape", "_", filter=insert_mode)(get_by_name("yank-last-arg"))
+    handle("escape", "c-y", filter=insert_mode)(get_by_name("yank-nth-arg"))
+    handle("escape", "#", filter=insert_mode)(get_by_name("insert-comment"))
+    handle("c-o")(get_by_name("operate-and-get-next"))
+
+    # ControlQ does a quoted insert. Not that for vt100 terminals, you have to
+    # disable flow control by running ``stty -ixon``, otherwise Ctrl-Q and
+    # Ctrl-S are captured by the terminal.
+    handle("c-q", filter=~has_selection)(get_by_name("quoted-insert"))
+
+    handle("c-x", "(")(get_by_name("start-kbd-macro"))
+    handle("c-x", ")")(get_by_name("end-kbd-macro"))
+    handle("c-x", "e")(get_by_name("call-last-kbd-macro"))
+
+    @handle("c-n")
+    def _next(event: E) -> None:
+        " Next line. "
+        event.current_buffer.auto_down()
+
+    @handle("c-p")
+    def _prev(event: E) -> None:
+        " Previous line. "
+        event.current_buffer.auto_up(count=event.arg)
+
+    def handle_digit(c: str) -> None:
+        """
+        Handle input of arguments.
+        The first number needs to be preceded by escape.
+        """
+
+        @handle(c, filter=has_arg)
+        @handle("escape", c)
+        def _(event: E) -> None:
+            event.append_to_arg_count(c)
+
+    for c in "0123456789":
+        handle_digit(c)
+
+    @handle("escape", "-", filter=~has_arg)
+    def _meta_dash(event: E) -> None:
+        """
+        """
+        if event._arg is None:
+            event.append_to_arg_count("-")
+
+    @handle("-", filter=Condition(lambda: get_app().key_processor.arg == "-"))
+    def _dash(event: E) -> None:
+        """
+        When '-' is typed again, after exactly '-' has been given as an
+        argument, ignore this.
+        """
+        event.app.key_processor.arg = "-"
+
+    @Condition
+    def is_returnable() -> bool:
+        return get_app().current_buffer.is_returnable
+
+    # Meta + Enter: always accept input.
+    handle("escape", "enter", filter=insert_mode & is_returnable)(
+        get_by_name("accept-line")
+    )
+
+    # Enter: accept input in single line mode.
+    handle("enter", filter=insert_mode & is_returnable & ~is_multiline)(
+        get_by_name("accept-line")
+    )
+
+    def character_search(buff: Buffer, char: str, count: int) -> None:
+        if count < 0:
+            match = buff.document.find_backwards(
+                char, in_current_line=True, count=-count
+            )
+        else:
+            match = buff.document.find(char, in_current_line=True, count=count)
+
+        if match is not None:
+            buff.cursor_position += match
+
+    @handle("c-]", Keys.Any)
+    def _goto_char(event: E) -> None:
+        " When Ctl-] + a character is pressed. go to that character. "
+        # Also named 'character-search'
+        character_search(event.current_buffer, event.data, event.arg)
+
+    @handle("escape", "c-]", Keys.Any)
+    def _goto_char_backwards(event: E) -> None:
+        " Like Ctl-], but backwards. "
+        # Also named 'character-search-backward'
+        character_search(event.current_buffer, event.data, -event.arg)
+
+    @handle("escape", "a")
+    def _prev_sentence(event: E) -> None:
+        " Previous sentence. "
+        # TODO:
+
+    @handle("escape", "e")
+    def _end_of_sentence(event: E) -> None:
+        " Move to end of sentence. "
+        # TODO:
+
+    @handle("escape", "t", filter=insert_mode)
+    def _swap_characters(event: E) -> None:
+        """
+        Swap the last two words before the cursor.
+        """
+        # TODO
+
+    @handle("escape", "*", filter=insert_mode)
+    def _insert_all_completions(event: E) -> None:
+        """
+        `meta-*`: Insert all possible completions of the preceding text.
+        """
+        buff = event.current_buffer
+
+        # List all completions.
+        complete_event = CompleteEvent(text_inserted=False, completion_requested=True)
+        completions = list(
+            buff.completer.get_completions(buff.document, complete_event)
+        )
+
+        # Insert them.
+        text_to_insert = " ".join(c.text for c in completions)
+        buff.insert_text(text_to_insert)
+
+    @handle("c-x", "c-x")
+    def _toggle_start_end(event: E) -> None:
+        """
+        Move cursor back and forth between the start and end of the current
+        line.
+        """
+        buffer = event.current_buffer
+
+        if buffer.document.is_cursor_at_the_end_of_line:
+            buffer.cursor_position += buffer.document.get_start_of_line_position(
+                after_whitespace=False
+            )
+        else:
+            buffer.cursor_position += buffer.document.get_end_of_line_position()
+
+    @handle("c-@")  # Control-space or Control-@
+    def _start_selection(event: E) -> None:
+        """
+        Start of the selection (if the current buffer is not empty).
+        """
+        # Take the current cursor position as the start of this selection.
+        buff = event.current_buffer
+        if buff.text:
+            buff.start_selection(selection_type=SelectionType.CHARACTERS)
+
+    @handle("c-g", filter=~has_selection)
+    def _cancel(event: E) -> None:
+        """
+        Control + G: Cancel completion menu and validation state.
+        """
+        event.current_buffer.complete_state = None
+        event.current_buffer.validation_error = None
+
+    @handle("c-g", filter=has_selection)
+    def _cancel_selection(event: E) -> None:
+        """
+        Cancel selection.
+        """
+        event.current_buffer.exit_selection()
+
+    @handle("c-w", filter=has_selection)
+    @handle("c-x", "r", "k", filter=has_selection)
+    def _cut(event: E) -> None:
+        """
+        Cut selected text.
+        """
+        data = event.current_buffer.cut_selection()
+        event.app.clipboard.set_data(data)
+
+    @handle("escape", "w", filter=has_selection)
+    def _copy(event: E) -> None:
+        """
+        Copy selected text.
+        """
+        data = event.current_buffer.copy_selection()
+        event.app.clipboard.set_data(data)
+
+    @handle("escape", "left")
+    def _start_of_word(event: E) -> None:
+        """
+        Cursor to start of previous word.
+        """
+        buffer = event.current_buffer
+        buffer.cursor_position += (
+            buffer.document.find_previous_word_beginning(count=event.arg) or 0
+        )
+
+    @handle("escape", "right")
+    def _start_next_word(event: E) -> None:
+        """
+        Cursor to start of next word.
+        """
+        buffer = event.current_buffer
+        buffer.cursor_position += (
+            buffer.document.find_next_word_beginning(count=event.arg)
+            or buffer.document.get_end_of_document_position()
+        )
+
+    @handle("escape", "/", filter=insert_mode)
+    def _complete(event: E) -> None:
+        """
+        M-/: Complete.
+        """
+        b = event.current_buffer
+        if b.complete_state:
+            b.complete_next()
+        else:
+            b.start_completion(select_first=True)
+
+    @handle("c-c", ">", filter=has_selection)
+    def _indent(event: E) -> None:
+        """
+        Indent selected text.
+        """
+        buffer = event.current_buffer
+
+        buffer.cursor_position += buffer.document.get_start_of_line_position(
+            after_whitespace=True
+        )
+
+        from_, to = buffer.document.selection_range()
+        from_, _ = buffer.document.translate_index_to_position(from_)
+        to, _ = buffer.document.translate_index_to_position(to)
+
+        indent(buffer, from_, to + 1, count=event.arg)
+
+    @handle("c-c", "<", filter=has_selection)
+    def _unindent(event: E) -> None:
+        """
+        Unindent selected text.
+        """
+        buffer = event.current_buffer
+
+        from_, to = buffer.document.selection_range()
+        from_, _ = buffer.document.translate_index_to_position(from_)
+        to, _ = buffer.document.translate_index_to_position(to)
+
+        unindent(buffer, from_, to + 1, count=event.arg)
+
+    return ConditionalKeyBindings(key_bindings, emacs_mode)
+
+
+def load_emacs_search_bindings() -> KeyBindingsBase:
+    key_bindings = KeyBindings()
+    handle = key_bindings.add
+    from . import search
+
+    # NOTE: We don't bind 'Escape' to 'abort_search'. The reason is that we
+    #       want Alt+Enter to accept input directly in incremental search mode.
+    #       Instead, we have double escape.
+
+    handle("c-r")(search.start_reverse_incremental_search)
+    handle("c-s")(search.start_forward_incremental_search)
+
+    handle("c-c")(search.abort_search)
+    handle("c-g")(search.abort_search)
+    handle("c-r")(search.reverse_incremental_search)
+    handle("c-s")(search.forward_incremental_search)
+    handle("up")(search.reverse_incremental_search)
+    handle("down")(search.forward_incremental_search)
+    handle("enter")(search.accept_search)
+
+    # Handling of escape.
+    handle("escape", eager=True)(search.accept_search)
+
+    # Like Readline, it's more natural to accept the search when escape has
+    # been pressed, however instead the following two bindings could be used
+    # instead.
+    # #handle('escape', 'escape', eager=True)(search.abort_search)
+    # #handle('escape', 'enter', eager=True)(search.accept_search_and_accept_input)
+
+    # If Read-only: also include the following key bindings:
+
+    # '/' and '?' key bindings for searching, just like Vi mode.
+    handle("?", filter=is_read_only & ~vi_search_direction_reversed)(
+        search.start_reverse_incremental_search
+    )
+    handle("/", filter=is_read_only & ~vi_search_direction_reversed)(
+        search.start_forward_incremental_search
+    )
+    handle("?", filter=is_read_only & vi_search_direction_reversed)(
+        search.start_forward_incremental_search
+    )
+    handle("/", filter=is_read_only & vi_search_direction_reversed)(
+        search.start_reverse_incremental_search
+    )
+
+    @handle("n", filter=is_read_only)
+    def _jump_next(event: E) -> None:
+        " Jump to next match. "
+        event.current_buffer.apply_search(
+            event.app.current_search_state,
+            include_current_position=False,
+            count=event.arg,
+        )
+
+    @handle("N", filter=is_read_only)
+    def _jump_prev(event: E) -> None:
+        " Jump to previous match. "
+        event.current_buffer.apply_search(
+            ~event.app.current_search_state,
+            include_current_position=False,
+            count=event.arg,
+        )
+
+    return ConditionalKeyBindings(key_bindings, emacs_mode)
+
+
+def load_emacs_shift_selection_bindings() -> KeyBindingsBase:
+    """
+    Bindings to select text with shift + cursor movements
+    """
+
+    key_bindings = KeyBindings()
+    handle = key_bindings.add
+
+    def unshift_move(event: E) -> None:
+        """
+        Used for the shift selection mode. When called with
+        a shift + movement key press event, moves the cursor
+        as if shift is not pressed.
+        """
+        key = event.key_sequence[0].key
+
+        if key == Keys.ShiftUp:
+            event.current_buffer.auto_up(count=event.arg)
+            return
+        if key == Keys.ShiftDown:
+            event.current_buffer.auto_down(count=event.arg)
+            return
+
+        # the other keys are handled through their readline command
+        key_to_command: Dict[Union[Keys, str], str] = {
+            Keys.ShiftLeft: "backward-char",
+            Keys.ShiftRight: "forward-char",
+            Keys.ShiftHome: "beginning-of-line",
+            Keys.ShiftEnd: "end-of-line",
+            Keys.ControlShiftLeft: "backward-word",
+            Keys.ControlShiftRight: "forward-word",
+            Keys.ControlShiftHome: "beginning-of-buffer",
+            Keys.ControlShiftEnd: "end-of-buffer",
+        }
+
+        try:
+            # Both the dict lookup and `get_by_name` can raise KeyError.
+            handler = get_by_name(key_to_command[key])
+        except KeyError:
+            pass
+        else:  # (`else` is not really needed here.)
+            if not isinstance(handler, Binding):
+                # (It should always be a normal callable here, for these
+                # commands.)
+                handler(event)
+
+    @handle("s-left", filter=~has_selection)
+    @handle("s-right", filter=~has_selection)
+    @handle("s-up", filter=~has_selection)
+    @handle("s-down", filter=~has_selection)
+    @handle("s-home", filter=~has_selection)
+    @handle("s-end", filter=~has_selection)
+    @handle("c-s-left", filter=~has_selection)
+    @handle("c-s-right", filter=~has_selection)
+    @handle("c-s-home", filter=~has_selection)
+    @handle("c-s-end", filter=~has_selection)
+    def _start_selection(event: E) -> None:
+        """
+        Start selection with shift + movement.
+        """
+        # Take the current cursor position as the start of this selection.
+        buff = event.current_buffer
+        if buff.text:
+            buff.start_selection(selection_type=SelectionType.CHARACTERS)
+
+            if buff.selection_state is not None:
+                # (`selection_state` should never be `None`, it is created by
+                # `start_selection`.)
+                buff.selection_state.enter_shift_mode()
+
+            # Then move the cursor
+            original_position = buff.cursor_position
+            unshift_move(event)
+            if buff.cursor_position == original_position:
+                # Cursor didn't actually move - so cancel selection
+                # to avoid having an empty selection
+                buff.exit_selection()
+
+    @handle("s-left", filter=shift_selection_mode)
+    @handle("s-right", filter=shift_selection_mode)
+    @handle("s-up", filter=shift_selection_mode)
+    @handle("s-down", filter=shift_selection_mode)
+    @handle("s-home", filter=shift_selection_mode)
+    @handle("s-end", filter=shift_selection_mode)
+    @handle("c-s-left", filter=shift_selection_mode)
+    @handle("c-s-right", filter=shift_selection_mode)
+    @handle("c-s-home", filter=shift_selection_mode)
+    @handle("c-s-end", filter=shift_selection_mode)
+    def _extend_selection(event: E) -> None:
+        """
+        Extend the selection
+        """
+        # Just move the cursor, like shift was not pressed
+        unshift_move(event)
+        buff = event.current_buffer
+
+        if buff.selection_state is not None:
+            if buff.cursor_position == buff.selection_state.original_cursor_position:
+                # selection is now empty, so cancel selection
+                buff.exit_selection()
+
+    @handle(Keys.Any, filter=shift_selection_mode)
+    def _replace_selection(event: E) -> None:
+        """
+        Replace selection by what is typed
+        """
+        event.current_buffer.cut_selection()
+        get_by_name("self-insert").call(event)
+
+    @handle("enter", filter=shift_selection_mode & is_multiline)
+    def _newline(event: E) -> None:
+        """
+        A newline replaces the selection
+        """
+        event.current_buffer.cut_selection()
+        event.current_buffer.newline(copy_margin=not in_paste_mode())
+
+    @handle("backspace", filter=shift_selection_mode)
+    def _delete(event: E) -> None:
+        """
+        Delete selection.
+        """
+        event.current_buffer.cut_selection()
+
+    @handle("c-y", filter=shift_selection_mode)
+    def _yank(event: E) -> None:
+        """
+        In shift selection mode, yanking (pasting) replace the selection.
+        """
+        buff = event.current_buffer
+        if buff.selection_state:
+            buff.cut_selection()
+        get_by_name("yank").call(event)
+
+    # moving the cursor in shift selection mode cancels the selection
+    @handle("left", filter=shift_selection_mode)
+    @handle("right", filter=shift_selection_mode)
+    @handle("up", filter=shift_selection_mode)
+    @handle("down", filter=shift_selection_mode)
+    @handle("home", filter=shift_selection_mode)
+    @handle("end", filter=shift_selection_mode)
+    @handle("c-left", filter=shift_selection_mode)
+    @handle("c-right", filter=shift_selection_mode)
+    @handle("c-home", filter=shift_selection_mode)
+    @handle("c-end", filter=shift_selection_mode)
+    def _cancel(event: E) -> None:
+        """
+        Cancel selection.
+        """
+        event.current_buffer.exit_selection()
+        # we then process the cursor movement
+        key_press = event.key_sequence[0]
+        event.key_processor.feed(key_press, first=True)
+
+    return ConditionalKeyBindings(key_bindings, emacs_mode)

xonsh/vended_ptk/prompt_toolkit/key_binding/bindings/focus.py

@@ -0,0 +1,24 @@
+from prompt_toolkit.key_binding.key_processor import KeyPressEvent
+
+__all__ = [
+    "focus_next",
+    "focus_previous",
+]
+
+E = KeyPressEvent
+
+
+def focus_next(event: E) -> None:
+    """
+    Focus the next visible Window.
+    (Often bound to the `Tab` key.)
+    """
+    event.app.layout.focus_next()
+
+
+def focus_previous(event: E) -> None:
+    """
+    Focus the previous visible Window.
+    (Often bound to the `BackTab` key.)
+    """
+    event.app.layout.focus_previous()