diff --git a/docs/bash_to_xsh.rst b/docs/bash_to_xsh.rst index 919ef266e..f6aae62d6 100644 --- a/docs/bash_to_xsh.rst +++ b/docs/bash_to_xsh.rst @@ -31,6 +31,9 @@ line is ``#!/usr/bin/env xonsh``. There is no notion of an escaping character in xonsh like the backslash (``\``) in bash. Single or double quotes can be used to remove the special meaning of certain characters, words or brackets. + * - ``IFS`` + - ``$XONSH_SUBPROC_OUTPUT_FORMAT`` + - Changing the output representation and splitting. * - ``$NAME`` or ``${NAME}`` - ``$NAME`` - Look up an environment variable by name. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 9030caa2e..4243b3b31 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -356,7 +356,7 @@ The ``$()`` operator in xonsh executes a subprocess command and *captures* some information about that command. The ``$()`` syntax captures and returns the standard output stream of the -command as a Python string. This is similar to how ``$()`` performs in Bash. +command as a Python string. This is similar to how ``$()`` performs in Bash. For example, .. code-block:: xonshcon @@ -364,6 +364,13 @@ For example, >>> $(ls -l) 'total 0\n-rw-rw-r-- 1 snail snail 0 Mar 8 15:46 xonsh\n' + +.. note:: + + By default the output is represented as one single block of output with new + line characters. You can set ``$XONSH_SUBPROC_OUTPUT_FORMAT`` to ``list_lines`` + to have a list of distinct lines in the commands like ``du -h $(ls)``. + The ``!()`` syntax captured more information about the command, as an instance of a class called ``CommandPipeline``. This object contains more information about the result of the given command, including the return code, the process diff --git a/news/subproc_output_format.rst b/news/subproc_output_format.rst new file mode 100644 index 000000000..9836db7f0 --- /dev/null +++ b/news/subproc_output_format.rst @@ -0,0 +1,28 @@ +**Added:** + +* Added ``$XONSH_SUBPROC_OUTPUT_FORMAT`` to switch the way to return the output lines. + Default ``stream_lines`` to return text. Alternative ``list_lines`` to return + the list of lines. Now you can run ``du $(ls)`` without additional stripping. + Also supported custom lambda function to process lines (if you're looking for + alternative to bash IFS). + +**Changed:** + +* Now the ending new line symbol ``\n`` will be stripped from the single line output. + For ``$(whoami)`` you will get ``'user'`` instead of ``'user\n'``. + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* + +**Security:** + +* diff --git a/tests/procs/test_specs.py b/tests/procs/test_specs.py index dedff4ac3..f2591e74b 100644 --- a/tests/procs/test_specs.py +++ b/tests/procs/test_specs.py @@ -80,7 +80,7 @@ def test_capture_always( exp = "HELLO\nBYE\n" cmds = [["echo", "-n", exp]] if pipe: - exp = exp.splitlines()[1] + "\n" # second line + exp = exp.splitlines()[1] # second line cmds += ["|", ["grep", "--color=never", exp.strip()]] if alias_type: @@ -134,6 +134,28 @@ def test_capture_always( assert exp in capfd.readouterr().out +@skip_if_on_windows +@pytest.mark.parametrize( + "cmds, exp_stream_lines, exp_list_lines", + [ + ([["echo", "-n", "1"]], "1", "1"), + ([["echo", "-n", "1\n"]], "1", "1"), + ([["echo", "-n", "1\n2\n3\n"]], "1\n2\n3\n", ["1", "2", "3"]), + ([["echo", "-n", "1\r\n2\r3\r\n"]], "1\n2\n3\n", ["1", "2", "3"]), + ([["echo", "-n", "1\n2\n3"]], "1\n2\n3", ["1", "2", "3"]), + ([["echo", "-n", "1\n2 3"]], "1\n2 3", ["1", "2 3"]), + ], +) +def test_subproc_output_format(cmds, exp_stream_lines, exp_list_lines, xonsh_session): + xonsh_session.env["XONSH_SUBPROC_OUTPUT_FORMAT"] = "stream_lines" + output = run_subproc(cmds, "stdout") + assert output == exp_stream_lines + + xonsh_session.env["XONSH_SUBPROC_OUTPUT_FORMAT"] = "list_lines" + output = run_subproc(cmds, "stdout") + assert output == exp_list_lines + + @skip_if_on_windows @pytest.mark.parametrize( "captured, exp_is_none", @@ -169,6 +191,11 @@ def test_callable_alias_cls(thread_subprocs, xession): assert proc.f == obj +def test_specs_resolve_args_list(): + spec = cmds_to_specs([["echo", ["1", "2", "3"]]], captured="stdout")[0] + assert spec.cmd[-3:] == ["1", "2", "3"] + + @pytest.mark.parametrize("captured", ["hiddenobject", False]) def test_procproxy_not_captured(xession, captured): xession.aliases["tst"] = lambda: 0 diff --git a/tests/test_integrations.py b/tests/test_integrations.py index d689a2916..25b6aad66 100644 --- a/tests/test_integrations.py +++ b/tests/test_integrations.py @@ -410,7 +410,7 @@ from xonsh.lib.subprocess import check_output print(check_output(["echo", "hello"]).decode("utf8")) """, - "hello\n\n", + "hello\n", 0, ), # diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 6bc7cb069..24f646f02 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -36,14 +36,15 @@ def patched_events(monkeypatch, xonsh_events, xonsh_session): @pytest.mark.parametrize( - "cmdline, stdout, stderr", + "cmdline, stdout, stderr, raw_stdout", ( - ("!(echo hi)", "hi\n", ""), - ("!(echo hi o>e)", "", "hi\n"), + ("!(echo hi)", "hi", "", "hi\n"), + ("!(echo hi o>e)", "", "hi\n", ""), pytest.param( "![echo hi]", - "hi\n", + "hi", "", + "hi\n", marks=pytest.mark.xfail( ON_WINDOWS, reason="ConsoleParallelReader doesn't work without a real console", @@ -53,34 +54,39 @@ def patched_events(monkeypatch, xonsh_events, xonsh_session): "![echo hi o>e]", "", "hi\n", + "", marks=pytest.mark.xfail( ON_WINDOWS, reason="stderr isn't captured in ![] on windows" ), ), pytest.param( - r"!(echo 'hi\nho')", "hi\nho\n", "", marks=skip_if_on_windows + r"!(echo 'hi\nho')", "hi\nho\n", "", "hi\nho\n", marks=skip_if_on_windows ), # won't work with cmd # for some reason cmd's echo adds an extra space: pytest.param( - r"!(cmd /c 'echo hi && echo ho')", "hi \nho\n", "", marks=skip_if_on_unix + r"!(cmd /c 'echo hi && echo ho')", + "hi \nho\n", + "", + "hi \nho\n", + marks=skip_if_on_unix, ), - ("!(echo hi | grep h)", "hi\n", ""), - ("!(echo hi | grep x)", "", ""), + ("!(echo hi | grep h)", "hi", "", "hi\n"), + ("!(echo hi | grep x)", "", "", ""), ), ) -def test_command_pipeline_capture(cmdline, stdout, stderr, xonsh_execer): +def test_command_pipeline_capture(cmdline, stdout, stderr, raw_stdout, xonsh_execer): pipeline: CommandPipeline = xonsh_execer.eval(cmdline) assert pipeline.out == stdout assert pipeline.err == (stderr or None) - assert pipeline.raw_out == stdout.replace("\n", os.linesep).encode() + assert pipeline.raw_out == raw_stdout.replace("\n", os.linesep).encode() assert pipeline.raw_err == stderr.replace("\n", os.linesep).encode() @pytest.mark.parametrize( "cmdline, output", ( - ("echo hi", "hi\n"), - ("echo hi | grep h", "hi\n"), + ("echo hi", "hi"), + ("echo hi | grep h", "hi"), ("echo hi | grep x", ""), pytest.param("echo -n hi", "hi", marks=skip_if_on_windows), ), @@ -90,7 +96,7 @@ def test_simple_capture(cmdline, output, xonsh_execer): def test_raw_substitution(xonsh_execer): - assert xonsh_execer.eval("$(echo @(b'bytes!'))") == "bytes!\n" + assert xonsh_execer.eval("$(echo @(b'bytes!'))") == "bytes!" @pytest.mark.parametrize( @@ -102,7 +108,7 @@ def test_raw_substitution(xonsh_execer): ("int(!(nocommand))", 1), ("hash(!(echo 1))", 0), ("hash(!(nocommand))", 1), - ("str(!(echo 1))", "1\n"), + ("str(!(echo 1))", "1"), ("str(!(nocommand))", ""), ("!(echo 1) == 0", True), ("!(nocommand) == 1", True), diff --git a/xonsh/environ.py b/xonsh/environ.py index 11bf412ba..bf697850a 100644 --- a/xonsh/environ.py +++ b/xonsh/environ.py @@ -975,6 +975,12 @@ class GeneralSetting(Xettings): "not always happen.", is_configurable=False, ) + XONSH_SUBPROC_OUTPUT_FORMAT = Var.with_default( + "stream_lines", + "Set output format for subprocess e.g. ``du $(ls)``. " + "By default (``stream_lines``) subprocess operator returns text output. " + "Set ``list_lines`` to have list of lines.", + ) XONSH_CAPTURE_ALWAYS = Var.with_default( False, "Try to capture output of commands run without explicit capturing.\n" diff --git a/xonsh/procs/pipelines.py b/xonsh/procs/pipelines.py index ba49f7149..5bab58ded 100644 --- a/xonsh/procs/pipelines.py +++ b/xonsh/procs/pipelines.py @@ -629,15 +629,33 @@ class CommandPipeline: """Creates normalized input string from args.""" return " ".join(self.args) + def get_formatted_lines(self, lines): + """Format output lines.""" + format = XSH.env.get("XONSH_SUBPROC_OUTPUT_FORMAT", "stream_lines") + if format == "stream_lines": + if len(lines) == 1: + return lines[0].rstrip("\n") + else: + return "".join(lines) + elif format == "list_lines": + if not lines: + return lines + elif len(lines) == 1: + return lines[0].rstrip("\n") + else: + return [line.rstrip("\n") for line in lines] + elif callable(format): + return format(lines) + @property def output(self): """Non-blocking, lazy access to output""" if self.ended: if self._output is None: - self._output = "".join(self.lines) + self._output = self.get_formatted_lines(self.lines) return self._output else: - return "".join(self.lines) + return self.get_formatted_lines(self.lines) @property def out(self): diff --git a/xonsh/procs/specs.py b/xonsh/procs/specs.py index 64705a464..9d11c90fa 100644 --- a/xonsh/procs/specs.py +++ b/xonsh/procs/specs.py @@ -588,6 +588,7 @@ class SubprocSpec: # modifications that do not alter cmds may come before creating instance spec = kls(cmd, cls=cls, **kwargs) # modifications that alter cmds must come after creating instance + spec.resolve_args_list() # perform initial redirects spec.resolve_redirects() # apply aliases @@ -599,6 +600,13 @@ class SubprocSpec: spec.resolve_stack() return spec + def resolve_args_list(self): + """Weave a list of arguments into a command.""" + resolved_cmd = [] + for c in self.cmd: + resolved_cmd += c if isinstance(c, list) else [c] + self.cmd = resolved_cmd + def resolve_redirects(self): """Manages redirects.""" new_cmd = []