feat: add subproc output format, autostrip singleline output (#5377)

### Motivation

* To have an ability to manage the output format added ``$XONSH_SUBPROC_OUTPUT_FORMAT`` to switch the way to return the output lines. Default ``stream_lines`` to return text. Alternative ``list_lines`` to return the list of lines. Also supported custom lambda function.
* Additionally the [proposal to change default behavior](https://github.com/xonsh/xonsh/pull/5377#discussion_r1587627131) for a single line case.
* Closes #3924 as soft solution.

### Before

```xsh
mkdir -p /tmp/tst && cd /tmp/tst && touch 1 2 3

$(ls)
# '1\n2\n3\n'

id $(whoami)
# id: ‘pc\n’: no such user: Invalid argument

du $(ls)
# du: cannot access '1'$'\n''2'$'\n''3'$'\n': No such file or directory

ls $(fzf)
# ls: cannot access 'FUNDING.yml'$'\n': No such file or directory
```

### After

```xsh
mkdir -p /tmp/tst && cd /tmp/tst && touch 1 2 3

$XONSH_SUBPROC_OUTPUT_FORMAT = 'list_lines'

$(ls)
# ['1', '2', '3']

[f for f in $(ls)]
# ['1', '2', '3']

id $(whoami)
# uid=501(user) gid=20(staff)

du $(ls)
# 0 1
# 0 2
# 0 3

ls $(fzf)
# FUNDING.yml

# etc
mkdir -p /tmp/@($(whoami))/dir
cat /etc/passwd | grep $(whoami)
```

### Notes

* It will be good to improve parser for cases like `mkdir -p /tmp/$(whoami)/dir`. PR is welcome!
* I named the default mode as `stream_lines` (instead of just `stream` or `raw`) because in fact we transform raw output into stream of lines and possibly reduce the length of output ([replacing `\r\n` to `\n`](c3a12b2a9c/xonsh/procs/pipelines.py (L380-L383))). May be some day we need to add raw "stream" output format.
* Now anybody can implement bash `IFS` behavior in [bashisms](https://github.com/xonsh/xontrib-bashisms).

---------

Co-authored-by: a <1@1.1>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

docs/bash_to_xsh.rst

@@ -31,6 +31,9 @@ line is ``#!/usr/bin/env xonsh``.
         There is no notion of an escaping character in xonsh like the backslash (``\``) in bash.
         Single or double quotes can be used to remove the special meaning of certain
         characters, words or brackets.
+    * - ``IFS``
+      - ``$XONSH_SUBPROC_OUTPUT_FORMAT``
+      - Changing the output representation and splitting.
     * - ``$NAME`` or ``${NAME}``
       - ``$NAME``
       - Look up an environment variable by name.

docs/tutorial.rst

@@ -356,7 +356,7 @@ The ``$(<expr>)`` operator in xonsh executes a subprocess command and
 *captures* some information about that command.
 
 The ``$()`` syntax captures and returns the standard output stream of the
-command as a Python string.  This is similar to how ``$()`` performs in Bash.
+command as a Python string. This is similar to how ``$()`` performs in Bash.
 For example,
 
 .. code-block:: xonshcon
@@ -364,6 +364,13 @@ For example,
     >>> $(ls -l)
     'total 0\n-rw-rw-r-- 1 snail snail 0 Mar  8 15:46 xonsh\n'
 
+
+.. note::
+
+    By default the output is represented as one single block of output with new
+    line characters. You can set ``$XONSH_SUBPROC_OUTPUT_FORMAT`` to ``list_lines``
+    to have a list of distinct lines in the commands like ``du -h $(ls)``.
+
 The ``!()`` syntax captured more information about the command, as an instance
 of a class called ``CommandPipeline``.  This object contains more information
 about the result of the given command, including the return code, the process

news/subproc_output_format.rst

@@ -0,0 +1,28 @@
+**Added:**
+
+* Added ``$XONSH_SUBPROC_OUTPUT_FORMAT`` to switch the way to return the output lines.
+  Default ``stream_lines`` to return text. Alternative ``list_lines`` to return
+  the list of lines. Now you can run ``du $(ls)`` without additional stripping.
+  Also supported custom lambda function to process lines (if you're looking for
+  alternative to bash IFS).
+
+**Changed:**
+
+* Now the ending new line symbol ``\n`` will be stripped from the single line output.
+  For ``$(whoami)`` you will get ``'user'`` instead of ``'user\n'``.
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

tests/procs/test_specs.py

@@ -80,7 +80,7 @@ def test_capture_always(
     exp = "HELLO\nBYE\n"
     cmds = [["echo", "-n", exp]]
     if pipe:
-        exp = exp.splitlines()[1] + "\n"  # second line
+        exp = exp.splitlines()[1]  # second line
         cmds += ["|", ["grep", "--color=never", exp.strip()]]
 
     if alias_type:
@@ -134,6 +134,28 @@ def test_capture_always(
     assert exp in capfd.readouterr().out
 
 
+@skip_if_on_windows
+@pytest.mark.parametrize(
+    "cmds, exp_stream_lines, exp_list_lines",
+    [
+        ([["echo", "-n", "1"]], "1", "1"),
+        ([["echo", "-n", "1\n"]], "1", "1"),
+        ([["echo", "-n", "1\n2\n3\n"]], "1\n2\n3\n", ["1", "2", "3"]),
+        ([["echo", "-n", "1\r\n2\r3\r\n"]], "1\n2\n3\n", ["1", "2", "3"]),
+        ([["echo", "-n", "1\n2\n3"]], "1\n2\n3", ["1", "2", "3"]),
+        ([["echo", "-n", "1\n2 3"]], "1\n2 3", ["1", "2 3"]),
+    ],
+)
+def test_subproc_output_format(cmds, exp_stream_lines, exp_list_lines, xonsh_session):
+    xonsh_session.env["XONSH_SUBPROC_OUTPUT_FORMAT"] = "stream_lines"
+    output = run_subproc(cmds, "stdout")
+    assert output == exp_stream_lines
+
+    xonsh_session.env["XONSH_SUBPROC_OUTPUT_FORMAT"] = "list_lines"
+    output = run_subproc(cmds, "stdout")
+    assert output == exp_list_lines
+
+
 @skip_if_on_windows
 @pytest.mark.parametrize(
     "captured, exp_is_none",
@@ -169,6 +191,11 @@ def test_callable_alias_cls(thread_subprocs, xession):
     assert proc.f == obj
 
 
+def test_specs_resolve_args_list():
+    spec = cmds_to_specs([["echo", ["1", "2", "3"]]], captured="stdout")[0]
+    assert spec.cmd[-3:] == ["1", "2", "3"]
+
+
 @pytest.mark.parametrize("captured", ["hiddenobject", False])
 def test_procproxy_not_captured(xession, captured):
     xession.aliases["tst"] = lambda: 0

tests/test_integrations.py

@@ -410,7 +410,7 @@ from xonsh.lib.subprocess import check_output
 
 print(check_output(["echo", "hello"]).decode("utf8"))
 """,
-        "hello\n\n",
+        "hello\n",
         0,
     ),
     #

tests/test_pipelines.py

@@ -36,14 +36,15 @@ def patched_events(monkeypatch, xonsh_events, xonsh_session):
 
 
 @pytest.mark.parametrize(
-    "cmdline, stdout, stderr",
+    "cmdline, stdout, stderr, raw_stdout",
     (
-        ("!(echo hi)", "hi\n", ""),
+        ("!(echo hi)", "hi", "", "hi\n"),
-        ("!(echo hi o>e)", "", "hi\n"),
+        ("!(echo hi o>e)", "", "hi\n", ""),
         pytest.param(
             "![echo hi]",
-            "hi\n",
+            "hi",
             "",
+            "hi\n",
             marks=pytest.mark.xfail(
                 ON_WINDOWS,
                 reason="ConsoleParallelReader doesn't work without a real console",
@@ -53,34 +54,39 @@ def patched_events(monkeypatch, xonsh_events, xonsh_session):
             "![echo hi o>e]",
             "",
             "hi\n",
+            "",
             marks=pytest.mark.xfail(
                 ON_WINDOWS, reason="stderr isn't captured in ![] on windows"
             ),
         ),
         pytest.param(
-            r"!(echo 'hi\nho')", "hi\nho\n", "", marks=skip_if_on_windows
+            r"!(echo 'hi\nho')", "hi\nho\n", "", "hi\nho\n", marks=skip_if_on_windows
         ),  # won't work with cmd
         # for some reason cmd's echo adds an extra space:
         pytest.param(
-            r"!(cmd /c 'echo hi && echo ho')", "hi \nho\n", "", marks=skip_if_on_unix
+            r"!(cmd /c 'echo hi && echo ho')",
+            "hi \nho\n",
+            "",
+            "hi \nho\n",
+            marks=skip_if_on_unix,
         ),
-        ("!(echo hi | grep h)", "hi\n", ""),
+        ("!(echo hi | grep h)", "hi", "", "hi\n"),
-        ("!(echo hi | grep x)", "", ""),
+        ("!(echo hi | grep x)", "", "", ""),
     ),
 )
-def test_command_pipeline_capture(cmdline, stdout, stderr, xonsh_execer):
+def test_command_pipeline_capture(cmdline, stdout, stderr, raw_stdout, xonsh_execer):
     pipeline: CommandPipeline = xonsh_execer.eval(cmdline)
     assert pipeline.out == stdout
     assert pipeline.err == (stderr or None)
-    assert pipeline.raw_out == stdout.replace("\n", os.linesep).encode()
+    assert pipeline.raw_out == raw_stdout.replace("\n", os.linesep).encode()
     assert pipeline.raw_err == stderr.replace("\n", os.linesep).encode()
 
 
 @pytest.mark.parametrize(
     "cmdline, output",
     (
-        ("echo hi", "hi\n"),
+        ("echo hi", "hi"),
-        ("echo hi | grep h", "hi\n"),
+        ("echo hi | grep h", "hi"),
         ("echo hi | grep x", ""),
         pytest.param("echo -n hi", "hi", marks=skip_if_on_windows),
     ),
@@ -90,7 +96,7 @@ def test_simple_capture(cmdline, output, xonsh_execer):
 
 
 def test_raw_substitution(xonsh_execer):
-    assert xonsh_execer.eval("$(echo @(b'bytes!'))") == "bytes!\n"
+    assert xonsh_execer.eval("$(echo @(b'bytes!'))") == "bytes!"
 
 
 @pytest.mark.parametrize(
@@ -102,7 +108,7 @@ def test_raw_substitution(xonsh_execer):
         ("int(!(nocommand))", 1),
         ("hash(!(echo 1))", 0),
         ("hash(!(nocommand))", 1),
-        ("str(!(echo 1))", "1\n"),
+        ("str(!(echo 1))", "1"),
         ("str(!(nocommand))", ""),
         ("!(echo 1) == 0", True),
         ("!(nocommand) == 1", True),

xonsh/environ.py

@@ -975,6 +975,12 @@ class GeneralSetting(Xettings):
         "not always happen.",
         is_configurable=False,
     )
+    XONSH_SUBPROC_OUTPUT_FORMAT = Var.with_default(
+        "stream_lines",
+        "Set output format for subprocess e.g. ``du $(ls)``. "
+        "By default (``stream_lines``) subprocess operator returns text output. "
+        "Set ``list_lines`` to have list of lines.",
+    )
     XONSH_CAPTURE_ALWAYS = Var.with_default(
         False,
         "Try to capture output of commands run without explicit capturing.\n"

xonsh/procs/pipelines.py

@@ -629,15 +629,33 @@ class CommandPipeline:
         """Creates normalized input string from args."""
         return " ".join(self.args)
 
+    def get_formatted_lines(self, lines):
+        """Format output lines."""
+        format = XSH.env.get("XONSH_SUBPROC_OUTPUT_FORMAT", "stream_lines")
+        if format == "stream_lines":
+            if len(lines) == 1:
+                return lines[0].rstrip("\n")
+            else:
+                return "".join(lines)
+        elif format == "list_lines":
+            if not lines:
+                return lines
+            elif len(lines) == 1:
+                return lines[0].rstrip("\n")
+            else:
+                return [line.rstrip("\n") for line in lines]
+        elif callable(format):
+            return format(lines)
+
     @property
     def output(self):
         """Non-blocking, lazy access to output"""
         if self.ended:
             if self._output is None:
-                self._output = "".join(self.lines)
+                self._output = self.get_formatted_lines(self.lines)
             return self._output
         else:
-            return "".join(self.lines)
+            return self.get_formatted_lines(self.lines)
 
     @property
     def out(self):

xonsh/procs/specs.py

@@ -588,6 +588,7 @@ class SubprocSpec:
         # modifications that do not alter cmds may come before creating instance
         spec = kls(cmd, cls=cls, **kwargs)
         # modifications that alter cmds must come after creating instance
+        spec.resolve_args_list()
         # perform initial redirects
         spec.resolve_redirects()
         # apply aliases
@@ -599,6 +600,13 @@ class SubprocSpec:
         spec.resolve_stack()
         return spec
 
+    def resolve_args_list(self):
+        """Weave a list of arguments into a command."""
+        resolved_cmd = []
+        for c in self.cmd:
+            resolved_cmd += c if isinstance(c, list) else [c]
+        self.cmd = resolved_cmd
+
     def resolve_redirects(self):
         """Manages redirects."""
         new_cmd = []