Merge branch 'main' into new-parser

.authors.yml

@@ -64,7 +64,7 @@
   - Gilbert.Forsyth@capitalone.com
   - gforsyth@gwu.edu
   - gil@forsyth.dev
-  num_commits: 680
+  num_commits: 683
   first_commit: 2015-10-19 16:04:32
   github: gforsyth
 - name: Morten Enemark Lund
@@ -1283,7 +1283,7 @@
   first_commit: 2021-02-08 10:50:51
 - name: Evgeny
   email: eugenesvk@users.noreply.github.com
-  num_commits: 12
+  num_commits: 14
   first_commit: 2021-02-22 09:32:34
 - name: Adam Schwalm
   email: adamschwalm@gmail.com
@@ -1465,7 +1465,7 @@
   first_commit: 2022-06-27 22:21:34
 - name: pre-commit-ci[bot]
   email: 66853113+pre-commit-ci[bot]@users.noreply.github.com
-  num_commits: 78
+  num_commits: 80
   first_commit: 2022-07-11 14:26:34
 - name: jgart
   email: 47760695+jgarte@users.noreply.github.com

.gitignore

@@ -100,3 +100,7 @@ pdm.lock
 
 # asv benchmarks
 .asv/
+
+# nix symlinks
+result
+repl-result-out

.mailmap

@@ -73,6 +73,7 @@ Jason R. Coombs <jaraco@jaraco.com>
 cryzed <cryzed@googlemail.com>
 Frank Sachsenheim <funkyfuture@riseup.net> Frank Sachsenheim <funkyfuture@users.noreply.github.com>
 Kurtis Rader <krader@skepticism.us>
+Evgeny <eugenesvk@users.noreply.github.com>
 Brian Visel <eode@eptitude.net>
 cafehaine <kilian.guillaume@gmail.com>
 Andrew Hundt <ATHundt@gmail.com>
@@ -80,7 +81,6 @@ Jonathan Slenders <jonathan@slenders.be>
 Justin Moen <jamoen7@gmail.com>
 Raphael Das Gupta <raphael.das.gupta@hsr.ch> Raphael Borun Das Gupta <git@raphael.dasgupta.ch>
 Caleb Hattingh <caleb.hattingh@gmail.com>
-Evgeny <eugenesvk@users.noreply.github.com>
 Stephan Fitzpatrick <knowsuchagency@gmail.com>
 dev2718 <alexanderfirbas@gmail.com>
 Will S <wsha.code@gmail.com>

AUTHORS.rst

@@ -48,6 +48,7 @@ Authors are sorted by number of commits.
 * cryzed
 * Frank Sachsenheim
 * Kurtis Rader
+* Evgeny
 * Brian Visel
 * cafehaine
 * Andrew Hundt
@@ -55,7 +56,6 @@ Authors are sorted by number of commits.
 * Justin Moen
 * Raphael Das Gupta
 * Caleb Hattingh
-* Evgeny
 * Stephan Fitzpatrick
 * dev2718
 * Will S

CHANGELOG.rst

@@ -4,6 +4,22 @@ Xonsh Change Log
 
 .. current developments
 
+v0.19.1
+====================
+
+**Fixed:**
+
+* Fixed hanging the command right after calling full capture subprocess (#5760).
+* Fixed non-int sys.exit codes raising ValueError.
+
+**Authors:**
+
+* anki-code
+* pre-commit-ci[bot]
+* Evgeny
+
+
+
 v0.19.0
 ====================
 

docs/tutorial_hist.rst

@@ -262,6 +262,10 @@ Tries to pull the history from parallel sessions and add to the current session.
 For example if there are two parallel terminal windows the run of ``history pull``
 command from the second terminal window will get the commands from the first terminal.
 
+The optional `--session-id` allows you to specify that history should only be pulled
+from a specific other session. Most useful when using the JSON history backend, as
+the overhead of an unfiltered `pull` can be significantly higher.
+
 ``clear`` action
 ================
 Deletes the history from the current session up until this point. Later commands

news/config-cursor-shape.rst

@@ -0,0 +1,23 @@
+**Added:**
+
+* env: add ``$XONSH_PROMPT_CURSOR_SHAPE`` for configuring prompt cursor shape.
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

news/configurable-xonsh-dirs.rst

@@ -1,6 +1,6 @@
 **Added:**
 
-* <news item>
+* env: Added XONSH_CONFIG_DIR, XONSH_DATA_DIR and XONSH_CACHE_DIR.
 
 **Changed:**
 
@@ -16,7 +16,7 @@
 
 **Fixed:**
 
-* Fixed non-int sys.exit codes raising ValueError.
+* <news item>
 
 **Security:**
 

news/hup-propagation.rst

@@ -0,0 +1,25 @@
+**Added:**
+
+* SIGHUP will now be forwarded to child processes when received by the main xonsh process.
+  This matches the behavior of other shells e.g. bash.
+* Documented the fact that the ``on_postcommand`` event only fires in interactive mode.
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* Running a subcommand in an event handler will no longer block xonsh from exiting.
+
+**Security:**
+
+* <news item>

news/json-history-pull.rst

@@ -0,0 +1,24 @@
+**Added:**
+
+* history: Added and documented `--session-id` parameter for `history pull` command.
+* history-json: Implemented `history pull` for JSON history backend.
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* history: Prevented `history pull` command from adding consecutive duplicates to propmter history.
+
+**Security:**
+
+* <news item>

news/ptk-history-combining.rst

@@ -0,0 +1,23 @@
+**Added:**
+
+* <news item>
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* prompt toolkit: Fixed autosuggest sometimes not updating when up-arrow is pressed (#5787).
+
+**Security:**
+
+* <news item>

news/subprocess-completions-dir-spaces.rst

@@ -0,0 +1,27 @@
+**Added:**
+
+* <news item>
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* Subprocess-based completions like
+  `xontrib-fish-completer <https://github.com/xonsh/xontrib-fish-completer>`_
+  no longer append a space if the single available completion ends with
+  a directory separator. This is consistent with the behavior of the
+  default completer.
+
+**Security:**
+
+* <news item>

tests/history/test_history_json.py

@@ -611,3 +611,44 @@ def test_hist_on_cmd(hist, xession, capsys, tmpdir):
         hist.append({"inp": cmd, "rtn": 0, "ts": (ts + 1, ts + 1.5)})
 
     assert len(xession.history) == 6
+
+
+@pytest.mark.parametrize(
+    "src_sessionid", [None, "e2265764-041c-4c57-acba-49d4e4f676e5"]
+)
+def test_hist_pull(src_sessionid, ptk_shell, tmpdir, xonsh_session, monkeypatch):
+    """Test that `pull` method correctly loads history entries
+    added to the database by other sessions."""
+    xonsh_session.env["XONSH_DATA_DIR"] = str(tmpdir)
+    before = time.time()
+
+    # simulate commands being run in other sessions before this session starts
+    hist_a = JsonHistory(sessionid=src_sessionid, gc=False)
+    hist_a.append({"inp": "cmd hist_a before", "rtn": 0, "ts": [before, before]})
+    hist_b = JsonHistory(gc=False)
+    hist_b.append({"inp": "cmd hist_b before", "rtn": 0, "ts": [before, before]})
+
+    hist_main = JsonHistory(gc=False)
+
+    # simulate commands being run in other sessions after this session starts
+    after = time.time() + 1
+    hist_a.append({"inp": "cmd hist_a after", "rtn": 0, "ts": [after, after]})
+    hist_b.append({"inp": "cmd hist_b after", "rtn": 0, "ts": [after + 1, after + 1]})
+
+    # give the filesystem long enough that it will update the mtime
+    time.sleep(0.01)
+    # at_exit ensures that we run the flush synchronously instead of in a background thread
+    hist_a.flush(at_exit=True)
+    hist_b.flush(at_exit=True)
+
+    # pull only works with PTK shell
+    monkeypatch.setattr(xonsh_session.shell, "shell", ptk_shell[2])
+    hist_main.pull(src_sessionid=src_sessionid)
+    hist_strings = ptk_shell[2].prompter.history.get_strings()
+
+    if src_sessionid is None:
+        # ensure that only commands from after the pulling session started get pulled in
+        assert hist_strings == ["cmd hist_a after", "cmd hist_b after"]
+    else:
+        # and that the commands are correctly filtered by session id if applicable
+        assert hist_strings == ["cmd hist_a after"]

tests/history/test_history_sqlite.py

@@ -5,6 +5,7 @@ import itertools
 import os
 import shlex
 import sys
+import time
 
 import pytest
 
@@ -350,3 +351,37 @@ def test_hist_store_cwd(hist, xession):
     assert cmds[1]["cwd"] is None
 
     _clean_up(hist)
+
+
+@pytest.mark.parametrize(
+    "src_sessionid", [None, "e2265764-041c-4c57-acba-49d4e4f676e5"]
+)
+def test_hist_pull(src_sessionid, tmpdir, ptk_shell, monkeypatch):
+    """Test that `pull` method correctly loads history entries
+    added to the database by other sessions."""
+    db_file = tmpdir / "xonsh-HISTORY-TEST-PULL.sqlite"
+    before = time.time()
+
+    # simulate commands being run in other sessions before this session starts
+    hist_a = SqliteHistory(filename=db_file, gc=False, sessionid=src_sessionid)
+    hist_a.append({"inp": "cmd hist_a before", "rtn": 0, "ts": [before, before]})
+    hist_b = SqliteHistory(filename=db_file, gc=False)
+    hist_b.append({"inp": "cmd hist_b after", "rtn": 0, "ts": [before, before]})
+
+    hist_main = SqliteHistory(filename=db_file, gc=False)
+    # simulate commands being run in other sessions after this session starts
+    after = time.time() + 1
+    hist_a.append({"inp": "cmd hist_a after", "rtn": 0, "ts": [after, after]})
+    hist_b.append({"inp": "cmd hist_b after", "rtn": 0, "ts": [after + 1, after + 1]})
+
+    # pull only works with PTK shell
+    monkeypatch.setattr("xonsh.built_ins.XSH.shell.shell", ptk_shell[2])
+    hist_main.pull(src_sessionid=src_sessionid)
+    hist_strings = ptk_shell[2].prompter.history.get_strings()
+
+    if src_sessionid is None:
+        # ensure that only commands from after the pulling session started get pulled in
+        assert hist_strings == ["cmd hist_a after", "cmd hist_b after"]
+    else:
+        # and that the commands are correctly filtered by session id if applicable
+        assert hist_strings == ["cmd hist_a after"]

tests/shell/test_base_shell.py

@@ -53,12 +53,7 @@ def test_default_append_history(cmd, exp_append_history, xonsh_session, monkeypa
     """Test that running an empty line or a comment does not append to history"""
     append_history_calls = []
 
-    def mock_append_history(**info):
-        append_history_calls.append(info)
-
-    monkeypatch.setattr(
-        xonsh_session.shell.shell, "_append_history", mock_append_history
-    )
+    monkeypatch.setattr(xonsh_session.history, "append", append_history_calls.append)
     xonsh_session.shell.default(cmd)
     if exp_append_history:
         assert len(append_history_calls) == 1

tests/shell/test_ptk_shell.py

@@ -8,6 +8,7 @@ import pytest
 from xonsh.platform import minimum_required_ptk_version
 from xonsh.shell import Shell
 from xonsh.shells.ptk_shell import tokenize_ansi
+from xonsh.shells.ptk_shell.history import PromptToolkitHistory
 
 # verify error if ptk not installed or below min
 
@@ -137,12 +138,32 @@ def test_ptk_default_append_history(cmd, exp_append_history, ptk_shell, monkeypa
     inp, out, shell = ptk_shell
     append_history_calls = []
 
-    def mock_append_history(**info):
-        append_history_calls.append(info)
-
-    monkeypatch.setattr(shell, "_append_history", mock_append_history)
+    monkeypatch.setattr(
+        "xonsh.built_ins.XSH.history.append", append_history_calls.append
+    )
     shell.default(cmd)
     if exp_append_history:
         assert len(append_history_calls) == 1
     else:
         assert len(append_history_calls) == 0
+
+
+def test_ptk_combine_history(monkeypatch):
+    """Test that consecutive identical history items are combined into a single item
+    when loading xonsh history items into prompt-toolkit history."""
+
+    def all_items(*args, **kwargs):
+        lines = [
+            "one two three",
+            "four five six",
+            "four five six",
+            "one two three",
+        ]
+        for line in lines:
+            yield {"inp": line}
+
+    monkeypatch.setattr("xonsh.built_ins.XSH.history.all_items", all_items)
+
+    shell_hist = PromptToolkitHistory()
+    hist_strs = list(shell_hist.load_history_strings())
+    assert len(hist_strs) == 3

tests/test_environ.py

@@ -20,6 +20,9 @@ from xonsh.environ import (
     default_value,
     locate_binary,
     make_args_env,
+    xonsh_cache_dir,
+    xonsh_config_dir,
+    xonsh_data_dir,
 )
 from xonsh.pytest.tools import skip_if_on_unix
 from xonsh.tools import DefaultNotGiven, always_true
@@ -648,3 +651,12 @@ def test_thread_local_dict_multiple():
         t.join()
 
     assert thread_values == [i**2 for i in range(num_threads)]
+
+
+def test_xonsh_dir_vars():
+    env = Env(
+        XONSH_CONFIG_DIR="/config", XONSH_CACHE_DIR="/cache", XONSH_DATA_DIR="/data"
+    )
+    assert xonsh_config_dir(env), "/config"
+    assert xonsh_cache_dir(env), "/cache"
+    assert xonsh_data_dir(env), "/data"

tests/test_integrations.py

@@ -64,6 +64,7 @@ def run_xonsh(
     args=None,
     timeout=20,
     env=None,
+    blocking=True,
 ):
     # Env
     popen_env = dict(os.environ)
@@ -107,6 +108,9 @@ def run_xonsh(
         proc.stdin.write(stdin_cmd)
         proc.stdin.flush()
 
+    if not blocking:
+        return proc
+
     try:
         out, err = proc.communicate(input=input, timeout=timeout)
     except sp.TimeoutExpired:
@@ -1356,6 +1360,58 @@ def test_catching_exit_signal():
     assert ret > 0
 
 
+@skip_if_on_windows
+def test_forwarding_sighup(tmpdir):
+    """We want to make sure that SIGHUP is forwarded to subprocesses when
+    received, so we spin up a Bash process that waits for SIGHUP and then
+    writes `SIGHUP` to a file, then exits. Then we check the content of
+    that file to ensure that the Bash process really did get SIGHUP."""
+    outfile = tmpdir.mkdir("xonsh_test_dir").join("sighup_test.out")
+
+    stdin_cmd = f"""
+sleep 0.2
+(sleep 1 && kill -SIGHUP @(__import__('os').getppid())) &
+bash -c "trap 'echo SIGHUP > {outfile}; exit 0' HUP; sleep 30 & wait $!"
+"""
+    proc = run_xonsh(
+        cmd=None,
+        stdin_cmd=stdin_cmd,
+        stderr=sp.PIPE,
+        interactive=True,
+        single_command=False,
+        blocking=False,
+    )
+    proc.wait(timeout=5)
+    # if this raises FileNotFoundError, then the Bash subprocess probably did not get SIGHUP
+    assert outfile.read_text("utf-8").strip() == "SIGHUP"
+
+
+@skip_if_on_windows
+def test_on_postcommand_waiting(tmpdir):
+    """Ensure that running a subcommand in the on_postcommand hook doesn't
+    block xonsh from exiting when there is a running foreground process."""
+    outdir = tmpdir.mkdir("xonsh_test_dir")
+
+    stdin_cmd = f"""
+sleep 0.2
+@events.on_postcommand
+def postcmd_hook(**kwargs):
+    touch {outdir}/sighup_test_postcommand
+
+(sleep 1 && kill -SIGHUP @(__import__('os').getppid())) &
+bash -c "trap '' HUP; sleep 30"
+"""
+    proc = run_xonsh(
+        cmd=None,
+        stdin_cmd=stdin_cmd,
+        stderr=sp.PIPE,
+        interactive=True,
+        single_command=False,
+        blocking=False,
+    )
+    proc.wait(timeout=5)
+
+
 @skip_if_on_windows
 def test_suspended_captured_process_pipeline():
     """See also test_specs.py:test_specs_with_suspended_captured_process_pipeline"""
@@ -1387,6 +1443,30 @@ def test_alias_stability():
     assert re.match(".*sleep.*sleep.*sleep.*", out, re.MULTILINE | re.DOTALL)
 
 
+@skip_if_on_windows
+@pytest.mark.flaky(reruns=3, reruns_delay=1)
+def test_captured_subproc_is_not_affected_next_command():
+    """Testing #5769."""
+    stdin_cmd = (
+        "t = __xonsh__.imp.time.time()\n"
+        "p = !(sleep 2)\n"
+        "print('OK_'+'TEST' if __xonsh__.imp.time.time() - t < 1 else 'FAIL_'+'TEST')\n"
+        "t = __xonsh__.imp.time.time()\n"
+        "echo 1\n"
+        "print('OK_'+'TEST' if __xonsh__.imp.time.time() - t < 1 else 'FAIL_'+'TEST')\n"
+    )
+    out, err, ret = run_xonsh(
+        cmd=None,
+        stdin_cmd=stdin_cmd,
+        interactive=True,
+        single_command=False,
+        timeout=10,
+    )
+    assert not re.match(
+        ".*FAIL_TEST.*", out, re.MULTILINE | re.DOTALL
+    ), "The second command after running captured subprocess shouldn't wait the end of the first one."
+
+
 @skip_if_on_windows
 @pytest.mark.flaky(reruns=3, reruns_delay=1)
 def test_spec_decorator_alias():
@@ -1491,7 +1571,10 @@ def test_xonshrc(tmpdir, cmd, exp):
     (script_xsh := home / "script.xsh").write_text("echo SCRIPT_XSH", encoding="utf8")
 
     # Construct $XONSHRC and $XONSHRC_DIR.
-    xonshrc_files = [str(home_config_xonsh_rc_xsh), str(home_xonsh_rc_path)]
+    xonshrc_files = [
+        str(home_config_xonsh_rc_xsh),
+        str(home_xonsh_rc_path),
+    ]
     xonshrc_dir = [str(home_config_xonsh_rcd)]
 
     args = [
@@ -1511,7 +1594,6 @@ def test_xonshrc(tmpdir, cmd, exp):
         env=env,
     )
 
-    exp = exp
     assert re.match(
         exp,
         out,

xonsh/__init__.py

@@ -1 +1 @@
-__version__ = "0.19.0"
+__version__ = "0.19.1"

xonsh/built_ins.py

@@ -61,6 +61,15 @@ def resetting_signal_handle(sig, f):
     def new_signal_handler(s=None, frame=None):
         f(s, frame)
         signal.signal(sig, prev_signal_handler)
+        if sig == signal.SIGHUP:
+            """
+            SIGHUP means the controlling terminal has been lost. This should be
+            propagated to child processes so that they can decide what to do about it.
+            See also: https://www.gnu.org/software/bash/manual/bash.html#Signals
+            """
+            import xonsh.procs.jobs as xj
+
+            xj.hup_all_jobs()
         if sig != 0:
             """
             There is no immediate exiting here.

xonsh/completers/tools.py

@@ -275,7 +275,7 @@ def complete_from_sub_proc(*args: str, sep=None, filter_prefix=None, **env_vars:
             lines = output.split(sep)
 
         # if there is a single completion candidate then maybe it is over
-        append_space = len(lines) == 1
+        append_space = len(lines) == 1 and not lines[0].rstrip().endswith(os.sep)
         for line in lines:
             if filter_prefix and (not filter_func(line, filter_prefix)):
                 continue

xonsh/environ.py

@@ -98,6 +98,8 @@ from xonsh.tools import (
     to_int_or_none,
     to_itself,
     to_logfile_opt,
+    to_ptk_cursor_shape,
+    to_ptk_cursor_shape_display_value,
     to_repr_pretty_,
     to_shlvl,
     to_tok_color_dict,
@@ -567,7 +569,9 @@ DEFAULT_TITLE = "{current_job:{} | }{user}@{hostname}: {cwd} | xonsh"
 @default_value
 def xonsh_data_dir(env):
     """Ensures and returns the $XONSH_DATA_DIR"""
-    xdd = os.path.expanduser(os.path.join(env.get("XDG_DATA_HOME"), "xonsh"))
+    xdd = os.path.expanduser(
+        os.getenv("XONSH_DATA_DIR") or os.path.join(env.get("XDG_DATA_HOME"), "xonsh")
+    )
     os.makedirs(xdd, exist_ok=True)
     return xdd
 
@@ -575,7 +579,9 @@ def xonsh_data_dir(env):
 @default_value
 def xonsh_cache_dir(env):
     """Ensures and returns the $XONSH_CACHE_DIR"""
-    xdd = os.path.expanduser(os.path.join(env.get("XDG_CACHE_HOME"), "xonsh"))
+    xdd = os.path.expanduser(
+        os.getenv("XONSH_CACHE_DIR") or os.path.join(env.get("XDG_CACHE_HOME"), "xonsh")
+    )
     os.makedirs(xdd, exist_ok=True)
     return xdd
 
@@ -583,7 +589,10 @@ def xonsh_cache_dir(env):
 @default_value
 def xonsh_config_dir(env):
     """``$XDG_CONFIG_HOME/xonsh``"""
-    xcd = os.path.expanduser(os.path.join(env.get("XDG_CONFIG_HOME"), "xonsh"))
+    xcd = os.path.expanduser(
+        os.getenv("XONSH_CONFIG_DIR")
+        or os.path.join(env.get("XDG_CONFIG_HOME"), "xonsh")
+    )
     os.makedirs(xcd, exist_ok=True)
     return xcd
 
@@ -965,6 +974,17 @@ class GeneralSetting(Xettings):
         "A list of directories where system level data files are stored.",
         type_str="env_path",
     )
+    XONSH_CONFIG_DIR = Var.with_default(
+        xonsh_config_dir,
+        "This is the location where xonsh user-level configuration information is stored.",
+        type_str="str",
+    )
+    XONSH_SYS_CONFIG_DIR = Var.with_default(
+        xonsh_sys_config_dir,
+        "This is the location where xonsh system-level configuration information is stored.",
+        is_configurable=False,
+        type_str="str",
+    )
     XONSHRC = Var.with_default(
         default_xonshrc,
         "A list of the locations of run control files, if they exist.  User "
@@ -980,26 +1000,12 @@ class GeneralSetting(Xettings):
         "are loaded after any files in XONSHRC.",
         type_str="env_path",
     )
-
-    XONSH_CONFIG_DIR = Var.with_default(
-        xonsh_config_dir,
-        "This is the location where xonsh user-level configuration information is stored.",
-        is_configurable=False,
-        type_str="str",
-    )
-    XONSH_SYS_CONFIG_DIR = Var.with_default(
-        xonsh_sys_config_dir,
-        "This is the location where xonsh system-level configuration information is stored.",
-        is_configurable=False,
-        type_str="str",
-    )
     XONSH_COLOR_STYLE = Var.with_default(
         "default",
         "Sets the color style for xonsh colors. This is a style name, not "
         "a color map. Run ``xonfig styles`` to see the available styles.",
         type_str="str",
     )
-
     XONSH_DEBUG = Var(
         always_false,
         to_debug,
@@ -1017,7 +1023,6 @@ class GeneralSetting(Xettings):
         doc_default="``$XDG_DATA_HOME/xonsh``",
         type_str="str",
     )
-
     XONSH_ENCODING = Var.with_default(
         DEFAULT_ENCODING,
         "This is the encoding that xonsh should use for subprocess operations.",
@@ -1046,7 +1051,6 @@ class GeneralSetting(Xettings):
         "``True`` if xonsh is running as a login shell, and ``False`` otherwise.",
         is_configurable=False,
     )
-
     XONSH_MODE = Var.with_default(
         default="interactive",  # In sync with ``main.py``.
         doc="A string value representing the current xonsh execution mode: "
@@ -1056,7 +1060,6 @@ class GeneralSetting(Xettings):
         "you plan to ``source``, use ``$XONSH_INTERACTIVE`` as the flag instead.",
         type_str="str",
     )
-
     XONSH_SOURCE = Var.with_default(
         "",
         "When running a xonsh script, this variable contains the absolute path "
@@ -1080,7 +1083,6 @@ class GeneralSetting(Xettings):
         "    - ptk style name (string) - ``$XONSH_STYLE_OVERRIDES['pygments.keyword'] = '#ff0000'``\n\n"
         "(The rules above are all have the same effect.)",
     )
-
     STAR_PATH = Var.no_default("env_path", pattern=re.compile(r"\w*PATH$"))
     STAR_DIRS = Var.no_default("env_path", pattern=re.compile(r"\w*DIRS$"))
 
@@ -1743,6 +1745,19 @@ class PTKSetting(PromptSetting):  # sub-classing -> sub-group
         "``DEPTH_1_BIT``, ``DEPTH_4_BIT``, ``DEPTH_8_BIT``, ``DEPTH_24_BIT`` "
         "colors. Default is an empty string which means that prompt toolkit decide.",
     )
+    XONSH_PROMPT_CURSOR_SHAPE = Var(
+        always_false,
+        to_ptk_cursor_shape,
+        to_ptk_cursor_shape_display_value,
+        to_ptk_cursor_shape("modal-vi-mode-only"),
+        "The cursor shape. Possible values for prompt toolkit are: "
+        "``block``, ``beam``, ``underline``, "
+        "``blinking-block``, ``blinking-beam``, ``blinking-underline``, "
+        "``modal``, ``modal-vi-mode-only``, ``never-change``. "
+        "Default value is ``modal-vi-mode-only`` which means "
+        "``modal`` if in vi mode and ``never-change`` if not in vi mode.",
+        doc_default="modal-vi-mode-only",
+    )
     PTK_STYLE_OVERRIDES = Var(
         is_tok_color_dict,
         to_tok_color_dict,

xonsh/history/json.py

@@ -103,9 +103,28 @@ def _xhj_get_data_dir():
     return dir
 
 
-def _xhj_get_history_files(sort=True, newest_first=False):
+def _xhj_get_data_dir_files(data_dir, include_mtime=False):
+    """Iterate over all the history files in a data dir,
+    optionally including the `mtime` for each file.
+    """
+    # list of (file, mtime) pairs
+    data_dir = xt.expanduser_abs_path(data_dir)
+    try:
+        for file in os.listdir(data_dir):
+            if file.startswith("xonsh-") and file.endswith(".json"):
+                fullpath = os.path.join(data_dir, file)
+                mtime = os.path.getmtime(fullpath) if include_mtime else None
+                yield fullpath, mtime
+    except OSError:
+        if XSH.env.get("XONSH_DEBUG"):
+            xt.print_exception(
+                f"Could not collect xonsh history json files from {data_dir}"
+            )
+
+
+def _xhj_get_history_files(sort=True, newest_first=False, modified_since=None):
     """Find and return the history files. Optionally sort files by
-    modify time.
+    modify time, or include only those modified after a certain time.
     """
     data_dirs = [
         _xhj_get_data_dir(),
@@ -114,20 +133,14 @@ def _xhj_get_history_files(sort=True, newest_first=False):
 
     files = []
     for data_dir in data_dirs:
-        data_dir = xt.expanduser_abs_path(data_dir)
-        try:
-            files += [
-                os.path.join(data_dir, f)
-                for f in os.listdir(data_dir)
-                if f.startswith("xonsh-") and f.endswith(".json")
-            ]
-        except OSError:
-            if XSH.env.get("XONSH_DEBUG"):
-                xt.print_exception(
-                    f"Could not collect xonsh history json files from {data_dir}"
-                )
+        include_mtime = sort or (modified_since is not None)
+        for file, mtime in _xhj_get_data_dir_files(data_dir, include_mtime):
+            if modified_since is None or mtime > modified_since:
+                files.append((file, mtime))
     if sort:
-        files.sort(key=lambda x: os.path.getmtime(x), reverse=newest_first)
+        files.sort(key=lambda x: x[1], reverse=newest_first)
+    # drop the mtimes
+    files = [f[0] for f in files]
 
     custom_history_file = XSH.env.get("XONSH_HISTORY_FILE", None)
     if custom_history_file:
@@ -137,6 +150,43 @@ def _xhj_get_history_files(sort=True, newest_first=False):
     return files
 
 
+def _xhj_pull_items(last_pull_time, src_sessionid=None):
+    """List all history items after a given start time.
+    Optionally restrict to just items from a single session.
+    """
+    if src_sessionid:
+        filename = os.path.join(_xhj_get_data_dir(), f"xonsh-{src_sessionid}.json")
+        src_paths = [filename]
+    else:
+        src_paths = _xhj_get_history_files(sort=True, modified_since=last_pull_time)
+
+    # src_paths may include the current session's file, so skip it to avoid duplicates
+    custom_history_file = XSH.env.get("XONSH_HISTORY_FILE") or ""
+    current_session_path = xt.expanduser_abs_path(custom_history_file)
+    items = []
+    for path in src_paths:
+        if path == current_session_path:
+            continue
+        try:
+            lj = xlj.LazyJSON(open(path))
+        except (JSONDecodeError, ValueError):
+            continue
+
+        cmds = lj["cmds"]
+        if len(cmds) == 0:
+            continue
+        # the cutoff point is likely to be very near the end of the session, so iterate backward
+        for i in range(len(cmds) - 1, -1, -1):
+            item = cmds[i].load()
+            if item["ts"][1] > last_pull_time:
+                items.append(item)
+            else:
+                break
+
+    items.sort(key=lambda i: i["ts"][1])
+    return items
+
+
 class JsonHistoryGC(threading.Thread):
     """Shell history garbage collection."""
 
@@ -444,6 +494,7 @@ class JsonHistory(History):
         self.last_cmd_out = None
         self.last_cmd_rtn = None
         self.gc = JsonHistoryGC() if gc else None
+        self.last_pull_time = time.time()
         # command fields that are known
         self.tss = JsonCommandField("ts", self)
         self.inps = JsonCommandField("inp", self)
@@ -585,6 +636,24 @@ class JsonHistory(History):
         data["gc_last_size"] = f"{(self.hist_size, self.hist_units)}"
         return data
 
+    def pull(self, show_commands=False, src_sessionid=None):
+        if not hasattr(XSH.shell.shell, "prompter"):
+            print(f"Shell type {XSH.shell.shell} is not supported.")
+            return 0
+
+        cnt = 0
+        prev = None
+        for item in _xhj_pull_items(self.last_pull_time, src_sessionid):
+            line = item["inp"].rstrip()
+            if show_commands:
+                print(line)
+            if line != prev:
+                XSH.shell.shell.prompter.history.append_string(line)
+                cnt += 1
+            prev = line
+        self.last_pull_time = time.time()
+        return cnt
+
     def run_gc(self, size=None, blocking=True, force=False, **_):
         self.gc = JsonHistoryGC(wait_for_shell=False, size=size, force=force)
         if blocking:

xonsh/history/main.py

@@ -320,13 +320,16 @@ class HistoryAlias(xcli.ArgParserAlias):
         print(str(hist.sessionid), file=_stdout)
 
     @staticmethod
-    def pull(show_commands=False, _stdout=None):
+    def pull(show_commands=False, session_id=None, _stdout=None):
         """Pull history from other parallel sessions.
 
         Parameters
         ----------
         show_commands: -c, --show-commands
             show pulled commands
+
+        session_id: -s, --session-id
+            pull from specified session only
         """
 
         hist = XSH.history
@@ -338,7 +341,7 @@ class HistoryAlias(xcli.ArgParserAlias):
                 file=_stdout,
             )
 
-        lines_added = hist.pull(show_commands)
+        lines_added = hist.pull(show_commands, session_id)
         if lines_added:
             print(f"Added {lines_added} records!", file=_stdout)
         else:

xonsh/history/sqlite.py

@@ -204,9 +204,20 @@ def xh_sqlite_delete_items(size_to_keep, filename=None):
         return _xh_sqlite_delete_records(c, size_to_keep)
 
 
-def xh_sqlite_pull(filename, last_pull_time, current_sessionid):
-    sql = "SELECT inp FROM xonsh_history WHERE tsb > ? AND sessionid != ? ORDER BY tsb"
-    params = [last_pull_time, current_sessionid]
+def xh_sqlite_pull(filename, last_pull_time, current_sessionid, src_sessionid=None):
+    # ensure we don't duplicate history entries if some crazy person passes the current session
+    if src_sessionid == current_sessionid:
+        return []
+
+    if src_sessionid:
+        sql = (
+            "SELECT inp FROM xonsh_history WHERE tsb > ? AND sessionid = ? ORDER BY tsb"
+        )
+        params = [last_pull_time, src_sessionid]
+    else:
+        sql = "SELECT inp FROM xonsh_history WHERE tsb > ? AND sessionid != ? ORDER BY tsb"
+        params = [last_pull_time, current_sessionid]
+
     with _xh_sqlite_get_conn(filename=filename) as conn:
         c = conn.cursor()
         c.execute(sql, tuple(params))
@@ -366,19 +377,22 @@ class SqliteHistory(History):
         data["gc options"] = envs.get("XONSH_HISTORY_SIZE")
         return data
 
-    def pull(self, show_commands=False):
+    def pull(self, show_commands=False, src_sessionid=None):
         if not hasattr(XSH.shell.shell, "prompter"):
             print(f"Shell type {XSH.shell.shell} is not supported.")
             return 0
 
         cnt = 0
+        prev = None
         for r in xh_sqlite_pull(
-            self.filename, self.last_pull_time, str(self.sessionid)
+            self.filename, self.last_pull_time, str(self.sessionid), src_sessionid
         ):
             if show_commands:
                 print(r[0])
-            XSH.shell.shell.prompter.history.append_string(r[0])
-            cnt += 1
+            if r[0] != prev:
+                XSH.shell.shell.prompter.history.append_string(r[0])
+                cnt += 1
+            prev = r[0]
         self.last_pull_time = time.time()
         return cnt
 

xonsh/procs/jobs.py

@@ -457,7 +457,11 @@ def add_job(info):
     info["status"] = info["status"] if "status" in info else "running"
     get_tasks().appendleft(num)
     get_jobs()[num] = info
-    if info["bg"] and XSH.env.get("XONSH_INTERACTIVE"):
+    if (
+        not info["pipeline"].spec.captured == "object"
+        and info["bg"]
+        and XSH.env.get("XONSH_INTERACTIVE")
+    ):
         print_one_job(num)
 
 

xonsh/procs/pipelines.py

@@ -267,7 +267,10 @@ class CommandPipeline:
             # we get here if the process is not threadable or the
             # class is the real Popen
             PrevProcCloser(pipeline=self)
-            task = xj.wait_for_active_job()
+            task = None
+            if not isinstance(sys.exc_info()[1], SystemExit):
+                task = xj.wait_for_active_job()
+
             if task is None or task["status"] != "stopped":
                 proc.wait()
                 self._endtime()

xonsh/procs/specs.py

@@ -888,6 +888,14 @@ def _last_spec_update_captured(last: SubprocSpec):
 def _make_last_spec_captured(last: SubprocSpec):
     captured = last.captured
     callable_alias = callable(last.alias)
+
+    if captured == "object":
+        """
+        In full capture mode the subprocess is running in background in fact
+        and we don't need to wait for it in downstream code e.g. `jobs.wait_for_active_job`.
+        """
+        last.background = True
+
     # cannot used PTY pipes for aliases, for some dark reason,
     # and must use normal pipes instead.
     use_tty = xp.ON_POSIX and not callable_alias

xonsh/prompt/job.py

@@ -1,4 +1,4 @@
-"""Prompt formatter for current jobs"""
+"""Prompt formatter for jobs fields e.g. current_job."""
 
 import contextlib
 import typing as tp

xonsh/shell.py

@@ -46,6 +46,7 @@ events.doc(
 on_postcommand(cmd: str, rtn: int, out: str or None, ts: list) -> None
 
 Fires just after a command is executed. The arguments are the same as history.
+This event only fires in interactive mode.
 
 Parameters:
 

xonsh/shells/base_shell.py

@@ -406,14 +406,20 @@ class BaseShell:
         finally:
             ts1 = ts1 or time.time()
             tee_out = tee.getvalue()
-            self._append_history(
+            info = self._append_history(
                 inp=src,
                 ts=[ts0, ts1],
                 spc=self.src_starts_with_space,
                 tee_out=tee_out,
                 cwd=self.precwd,
             )
-            self.accumulated_inputs += src
+            if not isinstance(exc_info[1], SystemExit):
+                events.on_postcommand.fire(
+                    cmd=info["inp"],
+                    rtn=info["rtn"],
+                    out=info.get("out", None),
+                    ts=info["ts"],
+                )
             if (
                 tee_out
                 and env.get("XONSH_APPEND_NEWLINE")
@@ -444,12 +450,10 @@ class BaseShell:
             info["out"] = last_out
         else:
             info["out"] = tee_out + "\n" + last_out
-        events.on_postcommand.fire(
-            cmd=info["inp"], rtn=info["rtn"], out=info.get("out", None), ts=info["ts"]
-        )
         if hist is not None:
             hist.append(info)
             hist.last_cmd_rtn = hist.last_cmd_out = None
+        return info
 
     def _fix_cwd(self):
         """Check if the cwd changed out from under us."""

xonsh/shells/ptk_shell/__init__.py

@@ -46,13 +46,6 @@ try:
 except ImportError:
     HAVE_SYS_CLIPBOARD = False
 
-try:
-    from prompt_toolkit.cursor_shapes import ModalCursorShapeConfig
-
-    HAVE_CURSOR_SHAPE = True
-except ImportError:
-    HAVE_CURSOR_SHAPE = False
-
 CAPITAL_PATTERN = re.compile(r"([a-z])([A-Z])")
 Token = _TokenType()
 
@@ -377,8 +370,10 @@ class PromptToolkitShell(BaseShell):
             for attr, val in self.get_lazy_ptk_kwargs():
                 prompt_args[attr] = val
 
-        if editing_mode == EditingMode.VI and HAVE_CURSOR_SHAPE:
-            prompt_args["cursor"] = ModalCursorShapeConfig()
+        cursor_shape = env.get("XONSH_PROMPT_CURSOR_SHAPE")
+        if cursor_shape:
+            prompt_args["cursor"] = cursor_shape
+
         events.on_pre_prompt.fire()
         line = self.prompter.prompt(**prompt_args)
         events.on_post_prompt.fire()

xonsh/shells/ptk_shell/history.py

@@ -25,11 +25,12 @@ class PromptToolkitHistory(prompt_toolkit.history.History):
         hist = XSH.history
         if hist is None:
             return
+        prev_line = None
         for cmd in hist.all_items(newest_first=True):
             line = cmd["inp"].rstrip()
-            strs = self.get_strings()
-            if len(strs) == 0 or line != strs[-1]:
+            if line != prev_line:
                 yield line
+            prev_line = line
 
     def __getitem__(self, index):
         return self.get_strings()[index]

xonsh/tools.py

@@ -41,6 +41,19 @@ import typing as tp
 import warnings
 from contextlib import contextmanager
 
+try:
+    from prompt_toolkit.cursor_shapes import (
+        CursorShape,
+        CursorShapeConfig,
+        DynamicCursorShapeConfig,
+        ModalCursorShapeConfig,
+        SimpleCursorShapeConfig,
+    )
+
+    HAVE_CURSOR_SHAPE = True
+except ImportError:
+    HAVE_CURSOR_SHAPE = False
+
 # adding imports from further xonsh modules is discouraged to avoid circular
 # dependencies
 from xonsh import __version__
@@ -1730,6 +1743,48 @@ def ptk2_color_depth_setter(x):
     return x
 
 
+def ptk_cursor_shape_vi_modal():
+    if xsh.env.get("VI_MODE"):
+        return ModalCursorShapeConfig()
+    else:
+        return SimpleCursorShapeConfig()
+
+
+def to_ptk_cursor_shape(x):
+    if not HAVE_CURSOR_SHAPE:
+        return None
+    if isinstance(x, (CursorShape, CursorShapeConfig)):
+        return x
+    if not isinstance(x, str):
+        raise ValueError("invalid cursor shape")
+    x = str(x).upper().replace("-", "_")
+    if x == "MODAL":
+        return ModalCursorShapeConfig()
+    elif x == "MODAL_VI_MODE_ONLY":
+        return DynamicCursorShapeConfig(ptk_cursor_shape_vi_modal)
+    try:
+        return CursorShape[x]
+    except KeyError:
+        return SimpleCursorShapeConfig()
+
+
+def to_ptk_cursor_shape_display_value(x):
+    if not x:
+        return ""
+    if isinstance(x, SimpleCursorShapeConfig):
+        x = x.get_cursor_shape(None)
+    if isinstance(x, CursorShape):
+        x = x.value.lower().replace("_", "-")
+        if x.startswith("-"):
+            x = x[1:]
+        return x
+    if isinstance(x, ModalCursorShapeConfig):
+        return "modal"
+    if isinstance(x, DynamicCursorShapeConfig):
+        return "modal-vi-mode-only"
+    return "unknown"
+
+
 def is_completions_display_value(x):
     """Enumerated values of ``$COMPLETIONS_DISPLAY``"""
     return x in {"none", "single", "multi"}