From d202664651995048619dc763d79e7fe4a4596030 Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Fri, 8 May 2015 08:14:39 -0400 Subject: [PATCH 01/25] allow parens, etc to be part of subprocess expressions --- xonsh/lexer.py | 20 +++++++++++++++----- xonsh/parser.py | 15 +++++++++++---- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/xonsh/lexer.py b/xonsh/lexer.py index b2ea3c43b..6328deffa 100644 --- a/xonsh/lexer.py +++ b/xonsh/lexer.py @@ -162,7 +162,7 @@ def handle_lparen(state, token, stream): """ Function for handling ``(`` """ - state['pymode'].append((True, '(', ')', token.start)) + state['pymode'].append((state['pymode'][-1][0], '(', ')', token.start)) state['last'] = token yield _new_token('LPAREN', '(', token.start) @@ -171,7 +171,7 @@ def handle_lbrace(state, token, stream): """ Function for handling ``{`` """ - state['pymode'].append((True, '{', '}', token.start)) + state['pymode'].append((state['pymode'][-1][0], '{', '}', token.start)) state['last'] = token yield _new_token('LBRACE', '{', token.start) @@ -180,7 +180,7 @@ def handle_lbracket(state, token, stream): """ Function for handling ``[`` """ - state['pymode'].append((True, '[', ']', token.start)) + state['pymode'].append((state['pymode'][-1][0], '[', ']', token.start)) state['last'] = token yield _new_token('LBRACKET', '[', token.start) @@ -202,10 +202,14 @@ def handle_rparen(state, token, stream): """ Function for handling ``)`` """ + m = state['pymode'][-1][1] e = _end_delimiter(state, token) if e is None: state['last'] = token - yield _new_token('RPAREN', ')', token.start) + typ = 'RPAREN' + if m.startswith('$'): + typ = 'SUBPROC_END_RPAREN' + yield _new_token(typ, ')', token.start) else: yield _new_token('ERRORTOKEN', e, token.start) @@ -226,10 +230,14 @@ def handle_rbracket(state, token, stream): """ Function for handling ``]`` """ + m = state['pymode'][-1][1] e = _end_delimiter(state, token) if e is None: state['last'] = token - yield _new_token('RBRACKET', ']', token.start) + typ = 'RBRACKET' + if m.startswith('$'): + typ = 'SUBPROC_END_RBRACKET' + yield _new_token(typ, ']', token.start) else: yield _new_token('ERRORTOKEN', e, token.start) @@ -418,6 +426,8 @@ class Lexer(object): 'REGEXPATH', # regex escaped with backticks 'LPAREN', 'RPAREN', # ( ) 'LBRACKET', 'RBRACKET', # [ ] + 'SUBPROC_END_RBRACKET', + 'SUBPROC_END_RPAREN', 'LBRACE', 'RBRACE', # { } 'AT', # @ 'QUESTION', # ? diff --git a/xonsh/parser.py b/xonsh/parser.py index a8cd323d8..94d815414 100644 --- a/xonsh/parser.py +++ b/xonsh/parser.py @@ -1629,8 +1629,8 @@ class Parser(object): | REGEXPATH | DOLLAR_NAME | DOLLAR_LBRACE test RBRACE - | DOLLAR_LPAREN subproc RPAREN - | DOLLAR_LBRACKET subproc RBRACKET + | DOLLAR_LPAREN subproc SUBPROC_END_RPAREN + | DOLLAR_LBRACKET subproc SUBPROC_END_RBRACKET """ p1 = p[1] if len(p) == 2: @@ -2235,8 +2235,8 @@ class Parser(object): | DOLLAR_NAME | AT_LPAREN test RPAREN | DOLLAR_LBRACE test RBRACE - | DOLLAR_LPAREN subproc RPAREN - | DOLLAR_LBRACKET subproc RBRACKET + | DOLLAR_LPAREN subproc SUBPROC_END_RPAREN + | DOLLAR_LBRACKET subproc SUBPROC_END_RBRACKET """ lenp = len(p) p1 = p[1] @@ -2332,6 +2332,13 @@ class Parser(object): | FALSE | NUMBER | STRING + | LPAREN + | RPAREN + | LBRACE + | RBRACE + | LBRACKET + | RBRACKET + | COMMA """ # Many tokens cannot be part of this list, such as $, ', ", () # Use a string atom instead. From c4948905c7e9495411706ea07d76b29592280e54 Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Fri, 8 May 2015 15:58:23 -0400 Subject: [PATCH 02/25] rewrite of ProcProxy (and variant) --- xonsh/built_ins.py | 120 +++++++++++++-------------------------------- xonsh/jobs.py | 20 ++++---- xonsh/proc.py | 62 +++++++++++++++++++++++ 3 files changed, 107 insertions(+), 95 deletions(-) create mode 100644 xonsh/proc.py diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py index ddc5521f6..d0444f25c 100644 --- a/xonsh/built_ins.py +++ b/xonsh/built_ins.py @@ -7,6 +7,7 @@ import sys import shlex import signal import locale +import inspect import builtins import subprocess from io import TextIOWrapper, StringIO @@ -22,7 +23,7 @@ from xonsh.inspectors import Inspector from xonsh.environ import default_env from xonsh.aliases import DEFAULT_ALIASES, bash_aliases from xonsh.jobs import add_job, wait_for_active_job -from xonsh.jobs import ProcProxy +from xonsh.proc import ProcProxy, SimpleProcProxy ENV = None BUILTINS_LOADED = False @@ -310,54 +311,6 @@ def iglobpath(s): WRITER_MODES = {'>': 'w', '>>': 'a'} -def _run_callable_subproc(alias, args, - captured=True, - prev_proc=None, - stdout=None): - """Helper for running callables as a subprocess.""" - # compute stdin for callable - if prev_proc is None: - stdin = None - elif isinstance(prev_proc, ProcProxy): - stdin = prev_proc.stdout - else: - stdin = StringIO(prev_proc.communicate()[0].decode(), None) - stdin.seek(0) - stdin, _ = stdin.read(), stdin.close() - # Redirect the output streams temporarily. merge with possible - # return values from alias function. - if stdout is PIPE: - # handles captured mode - new_stdout, new_stderr = StringIO(), StringIO() - with redirect_stdout(new_stdout), redirect_stderr(new_stderr): - rtn = alias(args, stdin=stdin) - proxy_stdout = new_stdout.getvalue() - proxy_stderr = new_stderr.getvalue() - if isinstance(rtn, str): - proxy_stdout += rtn - elif isinstance(rtn, Sequence): - if rtn[0]: # not None nor '' - proxy_stdout += rtn[0] - if rtn[1]: - proxy_stderr += rtn[1] - return ProcProxy(proxy_stdout, proxy_stderr) - else: - # handles uncaptured mode - rtn = alias(args, stdin=stdin) - rtnout, rtnerr = None, None - if isinstance(rtn, str): - rtnout = rtn - sys.stdout.write(rtn) - elif isinstance(rtn, Sequence): - if rtn[0]: - rtnout = rtn[0] - sys.stdout.write(rtn[0]) - if rtn[1]: - rtnerr = rtn[1] - sys.stderr.write(rtn[1]) - return ProcProxy(rtnout, rtnerr) - - RE_SHEBANG = re.compile(r'#![ \t]*(.+?)$') @@ -467,61 +420,56 @@ def run_subproc(cmds, captured=True): elif alias is None: aliased_cmd = cmd elif callable(alias): - prev_proc = _run_callable_subproc(alias, cmd[1:], - captured=captured, - prev_proc=prev_proc, - stdout=stdout) - continue + aliased_cmd = alias else: aliased_cmd = alias + cmd[1:] # compute stdin for subprocess prev_is_proxy = isinstance(prev_proc, ProcProxy) if prev_proc is None: stdin = None - elif prev_is_proxy: - stdin = PIPE else: stdin = prev_proc.stdout - subproc_kwargs = {} - if os.name == 'posix': - subproc_kwargs['preexec_fn'] = _subproc_pre - try: - proc = Popen(aliased_cmd, - universal_newlines=uninew, - env=ENV.detype(), - stdin=stdin, - stdout=stdout, **subproc_kwargs) - except PermissionError: - e = 'xonsh: subprocess mode: permission denied: {0}' - raise XonshError(e.format(aliased_cmd[0])) - except FileNotFoundError: - cmd = aliased_cmd[0] - e = 'xonsh: subprocess mode: command not found: {0}'.format(cmd) - e += '\n' + suggest_commands(cmd, ENV, builtins.aliases) - raise XonshError(e) + if callable(aliased_cmd): + if len(inspect.signature(aliased_cmd).parameters) == 2: + proc = SimpleProcProxy(aliased_cmd, cmd[1:], stdin, stdout, None) + else: + proc = ProcProxy(aliased_cmd, cmd[1:], stdin, stdout, None) + else: + subproc_kwargs = {} + if os.name == 'posix': + subproc_kwargs['preexec_fn'] = _subproc_pre + try: + proc = Popen(aliased_cmd, + universal_newlines=uninew, + env=ENV.detype(), + stdin=stdin, + stdout=stdout, **subproc_kwargs) + except PermissionError: + e = 'xonsh: subprocess mode: permission denied: {0}' + raise XonshError(e.format(aliased_cmd[0])) + except FileNotFoundError: + cmd = aliased_cmd[0] + e = 'xonsh: subprocess mode: command not found: {0}'.format(cmd) + e += '\n' + suggest_commands(cmd, ENV, builtins.aliases) + raise XonshError(e) procs.append(proc) prev = None - if prev_is_proxy: - proc.stdin.write(prev_proc.stdout) - proc.stdin.close() prev_proc = proc for proc in procs[:-1]: proc.stdout.close() - if not isinstance(prev_proc, ProcProxy): - add_job({ - 'cmds': cmds, - 'pids': [i.pid for i in procs], - 'obj': prev_proc, - 'bg': background - }) + add_job({ + 'cmds': cmds, + 'pids': [i.pid for i in procs], + 'obj': prev_proc, + 'bg': background + }) if background: return wait_for_active_job() if write_target is None: # get output - if isinstance(prev_proc, ProcProxy): - output = prev_proc.stdout - elif prev_proc.stdout is not None: + output = '' + if prev_proc.stdout is not None: output = prev_proc.stdout.read() if captured: return output diff --git a/xonsh/jobs.py b/xonsh/jobs.py index d9a62466a..64114dbe0 100644 --- a/xonsh/jobs.py +++ b/xonsh/jobs.py @@ -8,10 +8,7 @@ import signal import builtins from collections import namedtuple -ProcProxy = namedtuple('ProcProxy', ['stdout', 'stderr']) -""" -A class representing a Python function to be run as a subprocess command. -""" +from xonsh.proc import ProcProxy try: _shell_tty = sys.stderr.fileno() @@ -25,7 +22,7 @@ def _clear_dead_jobs(): to_remove = set() for num, job in builtins.__xonsh_all_jobs__.items(): obj = job['obj'] - if isinstance(obj, ProcProxy) or obj.poll() is not None: + if obj.poll() is not None: to_remove.add(num) for i in to_remove: del builtins.__xonsh_all_jobs__[i] @@ -86,10 +83,13 @@ def add_job(info): """ info['started'] = time.time() info['status'] = 'running' - try: - info['pgrp'] = os.getpgid(info['obj'].pid) - except ProcessLookupError: - return + if isinstance(info['obj'], ProcProxy): + info['pgrp'] = None + else: + try: + info['pgrp'] = os.getpgid(info['obj'].pid) + except ProcessLookupError: + return num = get_next_job_number() builtins.__xonsh_all_jobs__[num] = info builtins.__xonsh_active_job__ = num @@ -113,6 +113,8 @@ def wait_for_active_job(): job = builtins.__xonsh_all_jobs__[act] obj = job['obj'] if isinstance(obj, ProcProxy): + while obj.poll() is None: + time.sleep(0.01) return if job['bg']: return diff --git a/xonsh/proc.py b/xonsh/proc.py new file mode 100644 index 000000000..51a9c4cc5 --- /dev/null +++ b/xonsh/proc.py @@ -0,0 +1,62 @@ +import io + +from threading import Thread +from subprocess import Popen + + +class ProcProxy(Thread, Popen): + def __init__(self, f, args, stdin, stdout, stderr): + self.f = f + self.args = args + self.pid = None + self.retcode = None + + self.stdin = None + self.stdout = None + self.stderr = None + + (p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) = self._get_handles(stdin, stdout, stderr) + + if p2cwrite != -1: + self.stdin = io.open(p2cwrite, 'wb', -1) + self.stdin = io.TextIOWrapper(self.stdin, write_through=True, + line_buffering=(bufsize==1)) + + if c2pread != -1: + self.stdout = io.open(c2pread, 'rb', -1) + self.stdout = io.TextIOWrapper(self.stdout) + if errread != -1: + self.stderr = io.open(errread, 'rb', -1) + self.stderr = io.TextIOWrapper(self.stderr) + + Thread.__init__(self) + self.start() # start executing the function + + def run(self): + if self.f is not None: + r = self.f(self.args, self.stdin, self.stdout, self.stderr) + self.retcode = r if r is not None else True + + def poll(self): + return self.retcode + +class SimpleProcProxy(ProcProxy): + def __init__(self, f, args, stdin, stdout, stderr): + ProcProxy.__init__(self, f, args, stdin, stdout, stderr) + + def run(self): + if self.f is not None: + try: + r = self.f(self.args, self.stdin.read() if self.stdin is not None else "") + if isinstance(r, Sequence): + if self.stdout is not None: + self.stdout.write(r[0]) + if self.stderr is not None: + self.stderr.write(r[1]) + else: + self.stdout.write(r) + self.retcode = True + except: + self.retcode = False From 61235e91d57ad6e07efbf88cb4eb3894bd76c861 Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Fri, 8 May 2015 23:59:01 -0400 Subject: [PATCH 03/25] default output/error should be stdout and stderr --- xonsh/built_ins.py | 8 ++++++-- xonsh/proc.py | 25 ++++++++++++++++--------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py index d0444f25c..5bdeefb63 100644 --- a/xonsh/built_ins.py +++ b/xonsh/built_ins.py @@ -469,12 +469,16 @@ def run_subproc(cmds, captured=True): if write_target is None: # get output output = '' - if prev_proc.stdout is not None: + if prev_proc.stdout not in (None, sys.stdout): output = prev_proc.stdout.read() if captured: return output - elif last_stdout not in (PIPE, None): + elif last_stdout not in (PIPE, None, sys.stdout): last_stdout.close() + o = prev_proc.returncode + if isinstance(o, int): + return o == 0 + return o def subproc_captured(*cmds): diff --git a/xonsh/proc.py b/xonsh/proc.py index 51a9c4cc5..7796eb2c9 100644 --- a/xonsh/proc.py +++ b/xonsh/proc.py @@ -1,15 +1,16 @@ import io +import sys from threading import Thread from subprocess import Popen - +from collections import Sequence class ProcProxy(Thread, Popen): def __init__(self, f, args, stdin, stdout, stderr): self.f = f self.args = args self.pid = None - self.retcode = None + self.returncode = None self.stdin = None self.stdout = None @@ -31,16 +32,21 @@ class ProcProxy(Thread, Popen): self.stderr = io.open(errread, 'rb', -1) self.stderr = io.TextIOWrapper(self.stderr) + if self.stdout is None: + self.stdout = sys.stdout + if self.stderr is None: + self.stderr = sys.stderr + Thread.__init__(self) self.start() # start executing the function def run(self): if self.f is not None: r = self.f(self.args, self.stdin, self.stdout, self.stderr) - self.retcode = r if r is not None else True + self.returncode = r if r is not None else True def poll(self): - return self.retcode + return self.returncode class SimpleProcProxy(ProcProxy): def __init__(self, f, args, stdin, stdout, stderr): @@ -52,11 +58,12 @@ class SimpleProcProxy(ProcProxy): r = self.f(self.args, self.stdin.read() if self.stdin is not None else "") if isinstance(r, Sequence): if self.stdout is not None: - self.stdout.write(r[0]) + self.stdout.write(r[0] or '') if self.stderr is not None: - self.stderr.write(r[1]) + self.stderr.write(r[1] or '') else: - self.stdout.write(r) - self.retcode = True + if self.stdout is not None: + self.stdout.write(r or '') + self.returncode = True except: - self.retcode = False + self.returncode = False From 7c00e8d1417097092c4b1e13f4afd556164c2a2e Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Sat, 9 May 2015 12:39:37 -0400 Subject: [PATCH 04/25] bools are ints --- xonsh/built_ins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py index 5bdeefb63..a45ccade0 100644 --- a/xonsh/built_ins.py +++ b/xonsh/built_ins.py @@ -476,7 +476,7 @@ def run_subproc(cmds, captured=True): elif last_stdout not in (PIPE, None, sys.stdout): last_stdout.close() o = prev_proc.returncode - if isinstance(o, int): + if not isinstance(o, bool): return o == 0 return o From 737ef0fa9fcd9df5914914356d451e2cf4299c25 Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Sat, 9 May 2015 13:05:23 -0400 Subject: [PATCH 05/25] procproxy --- xonsh/proc.py | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/xonsh/proc.py b/xonsh/proc.py index 7796eb2c9..db0007452 100644 --- a/xonsh/proc.py +++ b/xonsh/proc.py @@ -1,4 +1,5 @@ import io +import os import sys from threading import Thread @@ -12,24 +13,24 @@ class ProcProxy(Thread, Popen): self.pid = None self.returncode = None - self.stdin = None - self.stdout = None - self.stderr = None + self.stdin = stdin + self.stdout = stdout + self.stderr = stderr - (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) = self._get_handles(stdin, stdout, stderr) + (self.p2cread, self.p2cwrite, + self.c2pread, self.c2pwrite, + self.errread, self.errwrite) = self._get_handles(stdin, stdout, stderr) - if p2cwrite != -1: - self.stdin = io.open(p2cwrite, 'wb', -1) + if self.p2cwrite != -1: + self.stdin = io.open(self.p2cwrite, 'wb', -1) self.stdin = io.TextIOWrapper(self.stdin, write_through=True, line_buffering=(bufsize==1)) - if c2pread != -1: - self.stdout = io.open(c2pread, 'rb', -1) + if self.c2pread != -1: + self.stdout = io.open(self.c2pread, 'rb', -1) self.stdout = io.TextIOWrapper(self.stdout) - if errread != -1: - self.stderr = io.open(errread, 'rb', -1) + if self.errread != -1: + self.stderr = io.open(self.errread, 'rb', -1) self.stderr = io.TextIOWrapper(self.stderr) if self.stdout is None: @@ -38,16 +39,25 @@ class ProcProxy(Thread, Popen): self.stderr = sys.stderr Thread.__init__(self) - self.start() # start executing the function + self.start() def run(self): if self.f is not None: r = self.f(self.args, self.stdin, self.stdout, self.stderr) self.returncode = r if r is not None else True + self._cleanup() def poll(self): return self.returncode + def _cleanup(self): + if self.p2cread != -1 and self.p2cwrite != -1: + os.close(self.p2cread) + if self.c2pwrite != -1 and self.c2pread != -1: + os.close(self.c2pwrite) + if self.errwrite != -1 and self.errread != -1: + os.close(self.errwrite) + class SimpleProcProxy(ProcProxy): def __init__(self, f, args, stdin, stdout, stderr): ProcProxy.__init__(self, f, args, stdin, stdout, stderr) @@ -56,7 +66,7 @@ class SimpleProcProxy(ProcProxy): if self.f is not None: try: r = self.f(self.args, self.stdin.read() if self.stdin is not None else "") - if isinstance(r, Sequence): + if isinstance(r, tuple): if self.stdout is not None: self.stdout.write(r[0] or '') if self.stderr is not None: @@ -67,3 +77,4 @@ class SimpleProcProxy(ProcProxy): self.returncode = True except: self.returncode = False + self._cleanup() From 5b5b97327f8c8e38846ac05827cb5f818f4783f9 Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Sat, 9 May 2015 14:22:46 -0400 Subject: [PATCH 06/25] fix some issues with piping --- xonsh/built_ins.py | 12 ++++++--- xonsh/proc.py | 62 +++++++++++++++++++++++++++++----------------- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py index a45ccade0..0ed22ae80 100644 --- a/xonsh/built_ins.py +++ b/xonsh/built_ins.py @@ -431,9 +431,12 @@ def run_subproc(cmds, captured=True): stdin = prev_proc.stdout if callable(aliased_cmd): if len(inspect.signature(aliased_cmd).parameters) == 2: - proc = SimpleProcProxy(aliased_cmd, cmd[1:], stdin, stdout, None) + cls = SimpleProcProxy else: - proc = ProcProxy(aliased_cmd, cmd[1:], stdin, stdout, None) + cls = ProcProxy + proc = cls(aliased_cmd, cmd[1:], + stdin, stdout, None, + universal_newlines=uninew) else: subproc_kwargs = {} if os.name == 'posix': @@ -456,7 +459,10 @@ def run_subproc(cmds, captured=True): prev = None prev_proc = proc for proc in procs[:-1]: - proc.stdout.close() + try: + proc.stdout.close() + except OSError: + pass add_job({ 'cmds': cmds, 'pids': [i.pid for i in procs], diff --git a/xonsh/proc.py b/xonsh/proc.py index db0007452..c16b2634b 100644 --- a/xonsh/proc.py +++ b/xonsh/proc.py @@ -6,37 +6,37 @@ from threading import Thread from subprocess import Popen from collections import Sequence + class ProcProxy(Thread, Popen): - def __init__(self, f, args, stdin, stdout, stderr): + def __init__(self, f, args, stdin, stdout, stderr, universal_newlines): self.f = f self.args = args self.pid = None self.returncode = None - self.stdin = stdin - self.stdout = stdout - self.stderr = stderr - + handles = self._get_handles(stdin, stdout, stderr) (self.p2cread, self.p2cwrite, self.c2pread, self.c2pwrite, - self.errread, self.errwrite) = self._get_handles(stdin, stdout, stderr) - + self.errread, self.errwrite) = handles + + # default values + self.stdin = None + self.stdout = None + self.stderr = None + if self.p2cwrite != -1: self.stdin = io.open(self.p2cwrite, 'wb', -1) - self.stdin = io.TextIOWrapper(self.stdin, write_through=True, - line_buffering=(bufsize==1)) - + if universal_newlines: + self.stdin = io.TextIOWrapper(self.stdin, write_through=True, + line_buffering=False) if self.c2pread != -1: self.stdout = io.open(self.c2pread, 'rb', -1) - self.stdout = io.TextIOWrapper(self.stdout) + if universal_newlines: + self.stdout = io.TextIOWrapper(self.stdout) if self.errread != -1: self.stderr = io.open(self.errread, 'rb', -1) - self.stderr = io.TextIOWrapper(self.stderr) - - if self.stdout is None: - self.stdout = sys.stdout - if self.stderr is None: - self.stderr = sys.stderr + if universal_newlines: + self.stderr = io.TextIOWrapper(self.stderr) Thread.__init__(self) self.start() @@ -58,23 +58,39 @@ class ProcProxy(Thread, Popen): if self.errwrite != -1 and self.errread != -1: os.close(self.errwrite) + class SimpleProcProxy(ProcProxy): - def __init__(self, f, args, stdin, stdout, stderr): - ProcProxy.__init__(self, f, args, stdin, stdout, stderr) + def __init__(self, f, args, stdin, stdout, stderr, universal_newlines): + ProcProxy.__init__(self, f, args, + stdin, stdout, stderr, + universal_newlines) def run(self): if self.f is not None: try: - r = self.f(self.args, self.stdin.read() if self.stdin is not None else "") + if self.p2cread != -1: + inp = io.open(self.p2cread, 'rb', -1).read() + else: + inp = b"" + r = self.f(self.args, inp.decode()) if isinstance(r, tuple): if self.stdout is not None: - self.stdout.write(r[0] or '') + os.write(self.c2pwrite, _prep(r[0])) + else: + print(r[0]) if self.stderr is not None: - self.stderr.write(r[1] or '') + os.write(self.errwrite, _prep(r[1])) + else: + print(r[1], file=sys.stderr) else: if self.stdout is not None: - self.stdout.write(r or '') + os.write(self.c2pwrite, _prep(r)) + else: + print(r) self.returncode = True except: self.returncode = False self._cleanup() + +def _prep(x): + return (x or '').encode('utf-8') From 75c9553d803872e5d797a67f488056e0e5410f83 Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Sat, 9 May 2015 14:41:00 -0400 Subject: [PATCH 07/25] fix bug with exit --- xonsh/jobs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xonsh/jobs.py b/xonsh/jobs.py index 64114dbe0..d3153ea66 100644 --- a/xonsh/jobs.py +++ b/xonsh/jobs.py @@ -142,7 +142,9 @@ def kill_all_jobs(): """ _clear_dead_jobs() for job in builtins.__xonsh_all_jobs__.values(): - os.kill(job['obj'].pid, signal.SIGKILL) + p = job['obj'].pid + if p is not None: + os.kill(job['obj'].pid, signal.SIGKILL) def jobs(args, stdin=None): From e8ecb0f65d33c04293002a4c9bf54c16f1c84e3d Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Sat, 9 May 2015 14:43:07 -0400 Subject: [PATCH 08/25] refactor, and hopefully proper handling of IO streams --- xonsh/proc.py | 75 +++++++++++++++++++++++---------------------------- 1 file changed, 33 insertions(+), 42 deletions(-) diff --git a/xonsh/proc.py b/xonsh/proc.py index c16b2634b..c4f05969c 100644 --- a/xonsh/proc.py +++ b/xonsh/proc.py @@ -20,7 +20,7 @@ class ProcProxy(Thread, Popen): self.errread, self.errwrite) = handles # default values - self.stdin = None + self.stdin = stdin self.stdout = None self.stderr = None @@ -43,54 +43,45 @@ class ProcProxy(Thread, Popen): def run(self): if self.f is not None: - r = self.f(self.args, self.stdin, self.stdout, self.stderr) + # need to make file-likes here that work in the "opposite" direction + if self.stdin is not None: + sp_stdin = io.TextIOWrapper(self.stdin) + else: + sp_stdin = io.StringIO("") + if self.c2pwrite != -1: + sp_stdout = io.TextIOWrapper(io.open(self.c2pwrite, 'wb', -1)) + else: + sp_stdout = sys.stdout + if self.errwrite != -1: + sp_stderr = io.TextIOWrapper(io.open(self.errwrite, 'wb', -1)) + else: + sp_stderr = sys.stderr + r = self.f(self.args, sp_stdin, sp_stdout, sp_stderr) self.returncode = r if r is not None else True - self._cleanup() def poll(self): return self.returncode - def _cleanup(self): - if self.p2cread != -1 and self.p2cwrite != -1: - os.close(self.p2cread) - if self.c2pwrite != -1 and self.c2pread != -1: - os.close(self.c2pwrite) - if self.errwrite != -1 and self.errread != -1: - os.close(self.errwrite) - +def _simple_wrapper(f): + def wrapped_simple_command_proxy(args, stdin, stdout, stderr): + try: + i = stdin.read() + r = f(args, i) + if isinstance(r, tuple): + if r[0] is not None: + stdout.write(r[0]) + if r[1] is not None: + stderr.write(r[1]) + else: + if r is not None: + stdout.write(r) + return True + except: + return False + return wrapped_simple_command_proxy class SimpleProcProxy(ProcProxy): def __init__(self, f, args, stdin, stdout, stderr, universal_newlines): - ProcProxy.__init__(self, f, args, + ProcProxy.__init__(self, _simple_wrapper(f), args, stdin, stdout, stderr, universal_newlines) - - def run(self): - if self.f is not None: - try: - if self.p2cread != -1: - inp = io.open(self.p2cread, 'rb', -1).read() - else: - inp = b"" - r = self.f(self.args, inp.decode()) - if isinstance(r, tuple): - if self.stdout is not None: - os.write(self.c2pwrite, _prep(r[0])) - else: - print(r[0]) - if self.stderr is not None: - os.write(self.errwrite, _prep(r[1])) - else: - print(r[1], file=sys.stderr) - else: - if self.stdout is not None: - os.write(self.c2pwrite, _prep(r)) - else: - print(r) - self.returncode = True - except: - self.returncode = False - self._cleanup() - -def _prep(x): - return (x or '').encode('utf-8') From 7723586ce8c96baf83051f8642871b7446fb8045 Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Sat, 9 May 2015 14:49:57 -0400 Subject: [PATCH 09/25] revert paren changes and uncaptured subproc behavior --- xonsh/built_ins.py | 4 ---- xonsh/lexer.py | 20 +++++--------------- xonsh/parser.py | 15 ++++----------- 3 files changed, 9 insertions(+), 30 deletions(-) diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py index 0ed22ae80..0d7841100 100644 --- a/xonsh/built_ins.py +++ b/xonsh/built_ins.py @@ -481,10 +481,6 @@ def run_subproc(cmds, captured=True): return output elif last_stdout not in (PIPE, None, sys.stdout): last_stdout.close() - o = prev_proc.returncode - if not isinstance(o, bool): - return o == 0 - return o def subproc_captured(*cmds): diff --git a/xonsh/lexer.py b/xonsh/lexer.py index 6328deffa..b2ea3c43b 100644 --- a/xonsh/lexer.py +++ b/xonsh/lexer.py @@ -162,7 +162,7 @@ def handle_lparen(state, token, stream): """ Function for handling ``(`` """ - state['pymode'].append((state['pymode'][-1][0], '(', ')', token.start)) + state['pymode'].append((True, '(', ')', token.start)) state['last'] = token yield _new_token('LPAREN', '(', token.start) @@ -171,7 +171,7 @@ def handle_lbrace(state, token, stream): """ Function for handling ``{`` """ - state['pymode'].append((state['pymode'][-1][0], '{', '}', token.start)) + state['pymode'].append((True, '{', '}', token.start)) state['last'] = token yield _new_token('LBRACE', '{', token.start) @@ -180,7 +180,7 @@ def handle_lbracket(state, token, stream): """ Function for handling ``[`` """ - state['pymode'].append((state['pymode'][-1][0], '[', ']', token.start)) + state['pymode'].append((True, '[', ']', token.start)) state['last'] = token yield _new_token('LBRACKET', '[', token.start) @@ -202,14 +202,10 @@ def handle_rparen(state, token, stream): """ Function for handling ``)`` """ - m = state['pymode'][-1][1] e = _end_delimiter(state, token) if e is None: state['last'] = token - typ = 'RPAREN' - if m.startswith('$'): - typ = 'SUBPROC_END_RPAREN' - yield _new_token(typ, ')', token.start) + yield _new_token('RPAREN', ')', token.start) else: yield _new_token('ERRORTOKEN', e, token.start) @@ -230,14 +226,10 @@ def handle_rbracket(state, token, stream): """ Function for handling ``]`` """ - m = state['pymode'][-1][1] e = _end_delimiter(state, token) if e is None: state['last'] = token - typ = 'RBRACKET' - if m.startswith('$'): - typ = 'SUBPROC_END_RBRACKET' - yield _new_token(typ, ']', token.start) + yield _new_token('RBRACKET', ']', token.start) else: yield _new_token('ERRORTOKEN', e, token.start) @@ -426,8 +418,6 @@ class Lexer(object): 'REGEXPATH', # regex escaped with backticks 'LPAREN', 'RPAREN', # ( ) 'LBRACKET', 'RBRACKET', # [ ] - 'SUBPROC_END_RBRACKET', - 'SUBPROC_END_RPAREN', 'LBRACE', 'RBRACE', # { } 'AT', # @ 'QUESTION', # ? diff --git a/xonsh/parser.py b/xonsh/parser.py index 94d815414..a8cd323d8 100644 --- a/xonsh/parser.py +++ b/xonsh/parser.py @@ -1629,8 +1629,8 @@ class Parser(object): | REGEXPATH | DOLLAR_NAME | DOLLAR_LBRACE test RBRACE - | DOLLAR_LPAREN subproc SUBPROC_END_RPAREN - | DOLLAR_LBRACKET subproc SUBPROC_END_RBRACKET + | DOLLAR_LPAREN subproc RPAREN + | DOLLAR_LBRACKET subproc RBRACKET """ p1 = p[1] if len(p) == 2: @@ -2235,8 +2235,8 @@ class Parser(object): | DOLLAR_NAME | AT_LPAREN test RPAREN | DOLLAR_LBRACE test RBRACE - | DOLLAR_LPAREN subproc SUBPROC_END_RPAREN - | DOLLAR_LBRACKET subproc SUBPROC_END_RBRACKET + | DOLLAR_LPAREN subproc RPAREN + | DOLLAR_LBRACKET subproc RBRACKET """ lenp = len(p) p1 = p[1] @@ -2332,13 +2332,6 @@ class Parser(object): | FALSE | NUMBER | STRING - | LPAREN - | RPAREN - | LBRACE - | RBRACE - | LBRACKET - | RBRACKET - | COMMA """ # Many tokens cannot be part of this list, such as $, ', ", () # Use a string atom instead. From ee6703cba46c719860df1449bc26fb4a13faef04 Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Sat, 9 May 2015 15:37:00 -0400 Subject: [PATCH 10/25] redirection of stdout and stderr in procproxy --- xonsh/built_ins.py | 2 +- xonsh/proc.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py index 0d7841100..4f9a03697 100644 --- a/xonsh/built_ins.py +++ b/xonsh/built_ins.py @@ -17,7 +17,7 @@ from contextlib import contextmanager from collections import Sequence, MutableMapping, Iterable, namedtuple, \ MutableSequence, MutableSet -from xonsh.tools import string_types, redirect_stdout, redirect_stderr +from xonsh.tools import string_types from xonsh.tools import suggest_commands, XonshError from xonsh.inspectors import Inspector from xonsh.environ import default_env diff --git a/xonsh/proc.py b/xonsh/proc.py index c4f05969c..30ef604d6 100644 --- a/xonsh/proc.py +++ b/xonsh/proc.py @@ -6,6 +6,7 @@ from threading import Thread from subprocess import Popen from collections import Sequence +from xonsh.tools import redirect_stdout, redirect_stderr class ProcProxy(Thread, Popen): def __init__(self, f, args, stdin, stdout, stderr, universal_newlines): @@ -66,7 +67,8 @@ def _simple_wrapper(f): def wrapped_simple_command_proxy(args, stdin, stdout, stderr): try: i = stdin.read() - r = f(args, i) + with redirect_stdout(stdout), redirect_stderr(stderr): + r = f(args, i) if isinstance(r, tuple): if r[0] is not None: stdout.write(r[0]) From 64ad02431c9f3cb3f9c94bc3f3ee155b6bb54e7c Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Sat, 9 May 2015 15:41:27 -0400 Subject: [PATCH 11/25] better check of return value for procproxy --- xonsh/proc.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/xonsh/proc.py b/xonsh/proc.py index 30ef604d6..31b849cbf 100644 --- a/xonsh/proc.py +++ b/xonsh/proc.py @@ -69,14 +69,15 @@ def _simple_wrapper(f): i = stdin.read() with redirect_stdout(stdout), redirect_stderr(stderr): r = f(args, i) - if isinstance(r, tuple): + if isinstance(r, str): + stdout.write(r) + if isinstance(r, Sequence): if r[0] is not None: stdout.write(r[0]) if r[1] is not None: stderr.write(r[1]) - else: - if r is not None: - stdout.write(r) + elif r is not None: + stdout.write(str(r)) return True except: return False From 3357d584cb40ed0789e9fec2d132b7f907495821 Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Sat, 9 May 2015 20:20:25 -0400 Subject: [PATCH 12/25] procproxy should not be considered a job --- xonsh/built_ins.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py index 4f9a03697..b845f51e7 100644 --- a/xonsh/built_ins.py +++ b/xonsh/built_ins.py @@ -463,12 +463,13 @@ def run_subproc(cmds, captured=True): proc.stdout.close() except OSError: pass - add_job({ - 'cmds': cmds, - 'pids': [i.pid for i in procs], - 'obj': prev_proc, - 'bg': background - }) + if not isinstance(prev_proc, ProcProxy): + add_job({ + 'cmds': cmds, + 'pids': [i.pid for i in procs], + 'obj': prev_proc, + 'bg': background + }) if background: return wait_for_active_job() From aec5a0be4e52bb0bcac31f213d7f1ae5a7481c6a Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Sun, 10 May 2015 19:46:19 -0400 Subject: [PATCH 13/25] default args for ProcProxy --- xonsh/proc.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/xonsh/proc.py b/xonsh/proc.py index 31b849cbf..a070339d2 100644 --- a/xonsh/proc.py +++ b/xonsh/proc.py @@ -9,7 +9,11 @@ from collections import Sequence from xonsh.tools import redirect_stdout, redirect_stderr class ProcProxy(Thread, Popen): - def __init__(self, f, args, stdin, stdout, stderr, universal_newlines): + def __init__(self, f, args, + stdin=None, + stdout=None, + stderr=None, + universal_newlines=False): self.f = f self.args = args self.pid = None @@ -84,7 +88,11 @@ def _simple_wrapper(f): return wrapped_simple_command_proxy class SimpleProcProxy(ProcProxy): - def __init__(self, f, args, stdin, stdout, stderr, universal_newlines): - ProcProxy.__init__(self, _simple_wrapper(f), args, - stdin, stdout, stderr, - universal_newlines) + def __init__(self, f, args, + stdin=None, + stdout=None, + stderr=None, + universal_newlines=False): + super().__init__(_simple_wrapper(f), args, + stdin, stdout, stderr, + universal_newlines) From 253f78c3236fa2ee58b4226170174aa65787e52e Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Tue, 12 May 2015 13:28:06 -0400 Subject: [PATCH 14/25] copy code from subprocess rather than inheriting from Popen --- xonsh/proc.py | 151 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 149 insertions(+), 2 deletions(-) diff --git a/xonsh/proc.py b/xonsh/proc.py index a070339d2..bf2ac20b6 100644 --- a/xonsh/proc.py +++ b/xonsh/proc.py @@ -1,6 +1,7 @@ import io import os import sys +import platform from threading import Thread from subprocess import Popen @@ -8,7 +9,13 @@ from collections import Sequence from xonsh.tools import redirect_stdout, redirect_stderr -class ProcProxy(Thread, Popen): +iswindows = platform.system() == 'Windows' +if iswindows: + import _winapi + import msvcrt + + +class ProcProxy(Thread): def __init__(self, f, args, stdin=None, stdout=None, @@ -18,7 +25,8 @@ class ProcProxy(Thread, Popen): self.args = args self.pid = None self.returncode = None - + self.wait = self.join + handles = self._get_handles(stdin, stdout, stderr) (self.p2cread, self.p2cwrite, self.c2pread, self.c2pwrite, @@ -67,6 +75,145 @@ class ProcProxy(Thread, Popen): def poll(self): return self.returncode + # The code below (_get_devnull, _get_handles, and _make_inheritable) comes + # from subprocess.py in the Python 3.4.2 Standard Library + def _get_devnull(self): + if not hasattr(self, '_devnull'): + self._devnull = os.open(os.devnull, os.O_RDWR) + return self._devnull + + if iswindows: + def _make_inheritable(self, handle): + """Return a duplicate of handle, which is inheritable""" + h = _winapi.DuplicateHandle( + _winapi.GetCurrentProcess(), handle, + _winapi.GetCurrentProcess(), 0, 1, + _winapi.DUPLICATE_SAME_ACCESS) + return Handle(h) + + def _get_handles(self, stdin, stdout, stderr): + """Construct and return tuple with IO objects: + p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite + """ + if stdin is None and stdout is None and stderr is None: + return (-1, -1, -1, -1, -1, -1) + + p2cread, p2cwrite = -1, -1 + c2pread, c2pwrite = -1, -1 + errread, errwrite = -1, -1 + + if stdin is None: + p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE) + if p2cread is None: + p2cread, _ = _winapi.CreatePipe(None, 0) + p2cread = Handle(p2cread) + _winapi.CloseHandle(_) + elif stdin == PIPE: + p2cread, p2cwrite = _winapi.CreatePipe(None, 0) + p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite) + elif stdin == DEVNULL: + p2cread = msvcrt.get_osfhandle(self._get_devnull()) + elif isinstance(stdin, int): + p2cread = msvcrt.get_osfhandle(stdin) + else: + # Assuming file-like object + p2cread = msvcrt.get_osfhandle(stdin.fileno()) + p2cread = self._make_inheritable(p2cread) + + if stdout is None: + c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE) + if c2pwrite is None: + _, c2pwrite = _winapi.CreatePipe(None, 0) + c2pwrite = Handle(c2pwrite) + _winapi.CloseHandle(_) + elif stdout == PIPE: + c2pread, c2pwrite = _winapi.CreatePipe(None, 0) + c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite) + elif stdout == DEVNULL: + c2pwrite = msvcrt.get_osfhandle(self._get_devnull()) + elif isinstance(stdout, int): + c2pwrite = msvcrt.get_osfhandle(stdout) + else: + # Assuming file-like object + c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) + c2pwrite = self._make_inheritable(c2pwrite) + + if stderr is None: + errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE) + if errwrite is None: + _, errwrite = _winapi.CreatePipe(None, 0) + errwrite = Handle(errwrite) + _winapi.CloseHandle(_) + elif stderr == PIPE: + errread, errwrite = _winapi.CreatePipe(None, 0) + errread, errwrite = Handle(errread), Handle(errwrite) + elif stderr == STDOUT: + errwrite = c2pwrite + elif stderr == DEVNULL: + errwrite = msvcrt.get_osfhandle(self._get_devnull()) + elif isinstance(stderr, int): + errwrite = msvcrt.get_osfhandle(stderr) + else: + # Assuming file-like object + errwrite = msvcrt.get_osfhandle(stderr.fileno()) + errwrite = self._make_inheritable(errwrite) + + return (p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) + + else: + # POSIX versions + def _get_handles(self, stdin, stdout, stderr): + """Construct and return tuple with IO objects: + p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite + """ + p2cread, p2cwrite = -1, -1 + c2pread, c2pwrite = -1, -1 + errread, errwrite = -1, -1 + + if stdin is None: + pass + elif stdin == PIPE: + p2cread, p2cwrite = os.pipe() + elif stdin == DEVNULL: + p2cread = self._get_devnull() + elif isinstance(stdin, int): + p2cread = stdin + else: + # Assuming file-like object + p2cread = stdin.fileno() + + if stdout is None: + pass + elif stdout == PIPE: + c2pread, c2pwrite = os.pipe() + elif stdout == DEVNULL: + c2pwrite = self._get_devnull() + elif isinstance(stdout, int): + c2pwrite = stdout + else: + # Assuming file-like object + c2pwrite = stdout.fileno() + + if stderr is None: + pass + elif stderr == PIPE: + errread, errwrite = os.pipe() + elif stderr == STDOUT: + errwrite = c2pwrite + elif stderr == DEVNULL: + errwrite = self._get_devnull() + elif isinstance(stderr, int): + errwrite = stderr + else: + # Assuming file-like object + errwrite = stderr.fileno() + + return (p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) + def _simple_wrapper(f): def wrapped_simple_command_proxy(args, stdin, stdout, stderr): try: From 8292f7557c1a5e60450a3d0035ff6e9ed647660a Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Tue, 12 May 2015 13:29:34 -0400 Subject: [PATCH 15/25] small refactor --- xonsh/proc.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/xonsh/proc.py b/xonsh/proc.py index bf2ac20b6..d45a1ed86 100644 --- a/xonsh/proc.py +++ b/xonsh/proc.py @@ -55,22 +55,22 @@ class ProcProxy(Thread): self.start() def run(self): - if self.f is not None: - # need to make file-likes here that work in the "opposite" direction - if self.stdin is not None: - sp_stdin = io.TextIOWrapper(self.stdin) - else: - sp_stdin = io.StringIO("") - if self.c2pwrite != -1: - sp_stdout = io.TextIOWrapper(io.open(self.c2pwrite, 'wb', -1)) - else: - sp_stdout = sys.stdout - if self.errwrite != -1: - sp_stderr = io.TextIOWrapper(io.open(self.errwrite, 'wb', -1)) - else: - sp_stderr = sys.stderr - r = self.f(self.args, sp_stdin, sp_stdout, sp_stderr) - self.returncode = r if r is not None else True + if self.f is None: + return + if self.stdin is not None: + sp_stdin = io.TextIOWrapper(self.stdin) + else: + sp_stdin = io.StringIO("") + if self.c2pwrite != -1: + sp_stdout = io.TextIOWrapper(io.open(self.c2pwrite, 'wb', -1)) + else: + sp_stdout = sys.stdout + if self.errwrite != -1: + sp_stderr = io.TextIOWrapper(io.open(self.errwrite, 'wb', -1)) + else: + sp_stderr = sys.stderr + r = self.f(self.args, sp_stdin, sp_stdout, sp_stderr) + self.returncode = r if r is not None else True def poll(self): return self.returncode From 076fc3adb00b8124059a6bfd7e331e54c518ab85 Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Tue, 12 May 2015 13:30:02 -0400 Subject: [PATCH 16/25] also import PIPE --- xonsh/proc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xonsh/proc.py b/xonsh/proc.py index d45a1ed86..4c6d02f4a 100644 --- a/xonsh/proc.py +++ b/xonsh/proc.py @@ -4,7 +4,7 @@ import sys import platform from threading import Thread -from subprocess import Popen +from subprocess import Popen, PIPE from collections import Sequence from xonsh.tools import redirect_stdout, redirect_stderr From 576f1c5916ad3cbfd198125b2725513069da625f Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Tue, 12 May 2015 13:35:13 -0400 Subject: [PATCH 17/25] additional imports from subprocess --- xonsh/proc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xonsh/proc.py b/xonsh/proc.py index 4c6d02f4a..437a6d0f3 100644 --- a/xonsh/proc.py +++ b/xonsh/proc.py @@ -4,7 +4,7 @@ import sys import platform from threading import Thread -from subprocess import Popen, PIPE +from subprocess import Popen, PIPE, DEVNULL, STDOUT from collections import Sequence from xonsh.tools import redirect_stdout, redirect_stderr From ccb566b9a31ef69c819bdef6469b63b9a8bdd2d2 Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Tue, 12 May 2015 13:35:27 -0400 Subject: [PATCH 18/25] if->elif duh --- xonsh/proc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xonsh/proc.py b/xonsh/proc.py index 437a6d0f3..49b9f23a1 100644 --- a/xonsh/proc.py +++ b/xonsh/proc.py @@ -222,7 +222,7 @@ def _simple_wrapper(f): r = f(args, i) if isinstance(r, str): stdout.write(r) - if isinstance(r, Sequence): + elif isinstance(r, Sequence): if r[0] is not None: stdout.write(r[0]) if r[1] is not None: From 0c690d507af9ce6608121d1e953e23fa3e2d5546 Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Tue, 12 May 2015 13:41:51 -0400 Subject: [PATCH 19/25] small refactor --- xonsh/proc.py | 45 ++++++++++++++++++++------------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/xonsh/proc.py b/xonsh/proc.py index 49b9f23a1..d30784569 100644 --- a/xonsh/proc.py +++ b/xonsh/proc.py @@ -214,32 +214,27 @@ class ProcProxy(Thread): c2pread, c2pwrite, errread, errwrite) -def _simple_wrapper(f): - def wrapped_simple_command_proxy(args, stdin, stdout, stderr): - try: - i = stdin.read() - with redirect_stdout(stdout), redirect_stderr(stderr): - r = f(args, i) - if isinstance(r, str): - stdout.write(r) - elif isinstance(r, Sequence): - if r[0] is not None: - stdout.write(r[0]) - if r[1] is not None: - stderr.write(r[1]) - elif r is not None: - stdout.write(str(r)) - return True - except: - return False - return wrapped_simple_command_proxy class SimpleProcProxy(ProcProxy): - def __init__(self, f, args, - stdin=None, - stdout=None, - stderr=None, + def __init__(self, f, args, stdin=None, stdout=None, stderr=None, universal_newlines=False): - super().__init__(_simple_wrapper(f), args, - stdin, stdout, stderr, + def wrapped_simple_command(args, stdin, stdout, stderr): + try: + i = stdin.read() + with redirect_stdout(stdout), redirect_stderr(stderr): + r = f(args, i) + if isinstance(r, str): + stdout.write(r) + elif isinstance(r, Sequence): + if r[0] is not None: + stdout.write(r[0]) + if r[1] is not None: + stderr.write(r[1]) + elif r is not None: + stdout.write(str(r)) + return True + except: + return False + super().__init__(wrapped_simple_command, + args, stdin, stdout, stderr, universal_newlines) From 333f553d7a3e03a1bd266910340875aa53b2047f Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Tue, 12 May 2015 13:45:23 -0400 Subject: [PATCH 20/25] wait for procproxy to finish --- xonsh/built_ins.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py index b845f51e7..fb897f4e4 100644 --- a/xonsh/built_ins.py +++ b/xonsh/built_ins.py @@ -463,7 +463,7 @@ def run_subproc(cmds, captured=True): proc.stdout.close() except OSError: pass - if not isinstance(prev_proc, ProcProxy): + if not prev_is_proxy: add_job({ 'cmds': cmds, 'pids': [i.pid for i in procs], @@ -472,7 +472,10 @@ def run_subproc(cmds, captured=True): }) if background: return - wait_for_active_job() + if prev_is_proxy: + prev_proc.wait() + else: + wait_for_active_job() if write_target is None: # get output output = '' From 915ba1b90eeb2611e49e9fdb8f0ea1089d08cb31 Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Tue, 12 May 2015 13:48:18 -0400 Subject: [PATCH 21/25] revert changes to jobs.py --- xonsh/jobs.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/xonsh/jobs.py b/xonsh/jobs.py index d3153ea66..b10c57240 100644 --- a/xonsh/jobs.py +++ b/xonsh/jobs.py @@ -8,7 +8,6 @@ import signal import builtins from collections import namedtuple -from xonsh.proc import ProcProxy try: _shell_tty = sys.stderr.fileno() @@ -83,13 +82,10 @@ def add_job(info): """ info['started'] = time.time() info['status'] = 'running' - if isinstance(info['obj'], ProcProxy): - info['pgrp'] = None - else: - try: - info['pgrp'] = os.getpgid(info['obj'].pid) - except ProcessLookupError: - return + try: + info['pgrp'] = os.getpgid(info['obj'].pid) + except ProcessLookupError: + return num = get_next_job_number() builtins.__xonsh_all_jobs__[num] = info builtins.__xonsh_active_job__ = num @@ -112,10 +108,6 @@ def wait_for_active_job(): return job = builtins.__xonsh_all_jobs__[act] obj = job['obj'] - if isinstance(obj, ProcProxy): - while obj.poll() is None: - time.sleep(0.01) - return if job['bg']: return pgrp = job['pgrp'] @@ -142,9 +134,7 @@ def kill_all_jobs(): """ _clear_dead_jobs() for job in builtins.__xonsh_all_jobs__.values(): - p = job['obj'].pid - if p is not None: - os.kill(job['obj'].pid, signal.SIGKILL) + os.kill(job['obj'].pid, signal.SIGKILL) def jobs(args, stdin=None): From 9323d2446afebb1dd6ca19df2813560259ec0594 Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Tue, 12 May 2015 13:54:22 -0400 Subject: [PATCH 22/25] fix prev_is_proxy --- xonsh/built_ins.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/xonsh/built_ins.py b/xonsh/built_ins.py index fb897f4e4..fa6334098 100644 --- a/xonsh/built_ins.py +++ b/xonsh/built_ins.py @@ -424,20 +424,25 @@ def run_subproc(cmds, captured=True): else: aliased_cmd = alias + cmd[1:] # compute stdin for subprocess - prev_is_proxy = isinstance(prev_proc, ProcProxy) if prev_proc is None: stdin = None else: stdin = prev_proc.stdout if callable(aliased_cmd): - if len(inspect.signature(aliased_cmd).parameters) == 2: + prev_is_proxy = True + numargs = len(inspect.signature(aliased_cmd).parameters) + if numargs == 2: cls = SimpleProcProxy - else: + elif numargs == 4: cls = ProcProxy + else: + e = 'Expected callable with 2 or 4 arguments, not {}' + raise XonshError(e.format(numargs)) proc = cls(aliased_cmd, cmd[1:], stdin, stdout, None, universal_newlines=uninew) else: + prev_is_proxy = False subproc_kwargs = {} if os.name == 'posix': subproc_kwargs['preexec_fn'] = _subproc_pre From f768664d10dae165cb0cfa4afadb62e61cd7b630 Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Tue, 12 May 2015 19:26:27 -0400 Subject: [PATCH 23/25] documentation for ProcProxy --- docs/api/index.rst | 1 + docs/api/proc.rst | 9 +++++++ xonsh/proc.py | 66 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 docs/api/proc.rst diff --git a/docs/api/index.rst b/docs/api/index.rst index 9a72f4788..b74f75028 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -27,6 +27,7 @@ For those of you who want the gritty details. aliases dirstack jobs + proc inspectors completer shell diff --git a/docs/api/proc.rst b/docs/api/proc.rst new file mode 100644 index 000000000..16917526a --- /dev/null +++ b/docs/api/proc.rst @@ -0,0 +1,9 @@ +.. _xonsh_proc: + +****************************************************** +Python Procedures as Subprocess Commands (``xonsh.proc``) +****************************************************** + +.. automodule:: xonsh.proc + :members: + :undoc-members: diff --git a/xonsh/proc.py b/xonsh/proc.py index d30784569..a28b63be3 100644 --- a/xonsh/proc.py +++ b/xonsh/proc.py @@ -1,3 +1,11 @@ +"""Interface for running Python functions as subprocess-mode commands. + +Code for several helper methods in the `ProcProxy` class have been reproduced +without modification from `subprocess.py` in the Python 3.4.2 standard library. +The contents of that file (and, thus, the reproduced methods) is +Copyright (c) 2003-2005 by Peter Astrand , and was +licensed to the Python Software foundation under a Contributor Agreement. +""" import io import os import sys @@ -16,12 +24,54 @@ if iswindows: class ProcProxy(Thread): + """ + Class representing a Python function to be run as a subprocess-mode command. + """ def __init__(self, f, args, stdin=None, stdout=None, stderr=None, universal_newlines=False): + """Parameters + ---------- + f : function + The function to be executed. + args : list + A (possibly empty) list containing the arguments that were given on + the command line + stdin : file-like, optional + A file-like object representing stdin (input can be read from + here). If `stdin` is not provided or if it is explicitly set to + `None`, then an instance of `io.StringIO` representing an empty + file is used. + stdout : file-like, optional + A file-like object representing stdout (normal output can be + written here). If `stdout` is not provided or if it is explicitly + set to `None`, then `sys.stdout` is used. + stderr : file-like, optional + A file-like object representing stderr (error output can be + written here). If `stderr` is not provided or if it is explicitly + set to `None`, then `sys.stderr` is used. + """ self.f = f + """ + The function to be executed. It should be a function of four arguments, described below. + + Parameters + ---------- + args : list + A (possibly empty) list containing the arguments that were given on + the command line + stdin : file-like + A file-like object representing stdin (input can be read from + here). + stdout : file-like + A file-like object representing stdout (normal output can be + written here). + stderr : file-like + A file-like object representing stderr (error output can be + written here). + """ self.args = args self.pid = None self.returncode = None @@ -55,6 +105,9 @@ class ProcProxy(Thread): self.start() def run(self): + """Set up input/output streams and execute the child function in a new + thread. This is part of the `threading.Thread` interface and should + not be called directly.""" if self.f is None: return if self.stdin is not None: @@ -73,6 +126,12 @@ class ProcProxy(Thread): self.returncode = r if r is not None else True def poll(self): + """Check if the function has completed. + + :return: `None` if the function is still executing, `True` if the + function finished successfully, and `False` if there was an + error + """ return self.returncode # The code below (_get_devnull, _get_handles, and _make_inheritable) comes @@ -216,6 +275,13 @@ class ProcProxy(Thread): class SimpleProcProxy(ProcProxy): + """ + Variant of `ProcProxy` for simpler functions. + + The function passed into the initializer for `SimpleProcProxy` should have + the form described in the xonsh tutorial. This function is then wrapped to + make a new function of the form expected by `ProcProxy`. + """ def __init__(self, f, args, stdin=None, stdout=None, stderr=None, universal_newlines=False): def wrapped_simple_command(args, stdin, stdout, stderr): From 04ed8eedb1e925e530d3da7b5affd78b5bd892de Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Tue, 12 May 2015 19:29:17 -0400 Subject: [PATCH 24/25] grammar --- xonsh/proc.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/xonsh/proc.py b/xonsh/proc.py index a28b63be3..241801699 100644 --- a/xonsh/proc.py +++ b/xonsh/proc.py @@ -2,8 +2,8 @@ Code for several helper methods in the `ProcProxy` class have been reproduced without modification from `subprocess.py` in the Python 3.4.2 standard library. -The contents of that file (and, thus, the reproduced methods) is -Copyright (c) 2003-2005 by Peter Astrand , and was +The contents of `subprocess.py` (and, thus, the reproduced methods) are +Copyright (c) 2003-2005 by Peter Astrand and were licensed to the Python Software foundation under a Contributor Agreement. """ import io @@ -25,7 +25,7 @@ if iswindows: class ProcProxy(Thread): """ - Class representing a Python function to be run as a subprocess-mode command. + Class representing a function to be run as a subprocess-mode command. """ def __init__(self, f, args, stdin=None, @@ -55,7 +55,8 @@ class ProcProxy(Thread): """ self.f = f """ - The function to be executed. It should be a function of four arguments, described below. + The function to be executed. It should be a function of four + arguments, described below. Parameters ---------- From c5180cf6bd9bbaf01db7e78024f4caf60993f7b9 Mon Sep 17 00:00:00 2001 From: adam j hartz Date: Tue, 12 May 2015 21:04:19 -0400 Subject: [PATCH 25/25] refactor of check for windows --- xonsh/proc.py | 7 +++---- xonsh/tools.py | 5 +++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/xonsh/proc.py b/xonsh/proc.py index 241801699..480dc3da6 100644 --- a/xonsh/proc.py +++ b/xonsh/proc.py @@ -15,10 +15,9 @@ from threading import Thread from subprocess import Popen, PIPE, DEVNULL, STDOUT from collections import Sequence -from xonsh.tools import redirect_stdout, redirect_stderr +from xonsh.tools import redirect_stdout, redirect_stderr, ON_WINDOWS -iswindows = platform.system() == 'Windows' -if iswindows: +if ON_WINDOWS: import _winapi import msvcrt @@ -142,7 +141,7 @@ class ProcProxy(Thread): self._devnull = os.open(os.devnull, os.O_RDWR) return self._devnull - if iswindows: + if ON_WINDOWS: def _make_inheritable(self, handle): """Return a duplicate of handle, which is inheritable""" h = _winapi.DuplicateHandle( diff --git a/xonsh/tools.py b/xonsh/tools.py index 07f8aa8d4..21ab1c7c3 100644 --- a/xonsh/tools.py +++ b/xonsh/tools.py @@ -20,6 +20,7 @@ import os import re import sys import builtins +import platform from collections import OrderedDict if sys.version_info[0] >= 3: @@ -31,6 +32,10 @@ else: DEFAULT_ENCODING = sys.getdefaultencoding() +ON_WINDOWS = (platform.system() == 'Windows') +ON_MAC = (platform.system() == 'Darwin') +ON_POSIX = (os.name == 'posix') + class XonshError(Exception): pass