Merge remote-tracking branch 'upstream/master' into vox-fs

.travis.yml

@@ -43,8 +43,11 @@ install:
       pip install -r requirements-tests.txt;
     fi
 
+before_script:
+  - rvm get head || true
 
 script:
+  - set -e
   - if [[ $BUILD_DOCS = true ]]; then
       cd docs;
       make html;

docs/index.rst

@@ -40,6 +40,7 @@ the xonsh shell
         "People xonshtantly mispronounce these things",
         "WHAT...is your favorite shell?",
         "Conches for the xonsh god!",
+        "Python-powered, cross-platform, Unix-gazing shell",
         "Exploiting the workers and hanging on to outdated imperialist dogma since 2015."
         ];
     document.write(taglines[Math.floor(Math.random() * taglines.length)]);
@@ -50,11 +51,11 @@ the xonsh shell
     </span>
     </p>
 
-Xonsh is a Python-ish, BASHwards-looking shell language and command prompt.
-The language is a superset of Python 3.4+ with additional shell primitives
-that you are used to from Bash and IPython. It works on all major systems including
-Linux, Mac OSX, and Windows. Xonsh is meant for the daily use of experts and novices
-alike.
+Xonsh is a Python-powered, cross-platform, Unix-gazing shell language and
+command prompt. The language is a superset of Python 3.4+ with additional
+shell primitives that you are used to from Bash and IPython. It works on
+all major systems including Linux, Mac OSX, and Windows. Xonsh is meant
+for the daily use of experts and novices alike.
 
 **Try it out!**
 

news/not_backgroundable.rst

@@ -11,6 +11,7 @@
     * clear.exe
     * cls
     * cmd
+    * ex
     * fish
     * htop
     * ksh
@@ -21,6 +22,8 @@
     * nano
     * psql
     * ranger
+    * rview
+    * rvim
     * scp
     * sh
     * ssh
@@ -29,6 +32,7 @@
     * tcsh
     * top
     * vi
+    * view
     * vim
     * vimpager
     * xo

news/stdout_storage.rst

@@ -0,0 +1,13 @@
+**Added:** None
+
+**Changed:** None
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:**
+
+* STDOUT is only stored when ``$XONSH_STORE_STDOUT=True``
+
+**Security:** None

news/xonsh-debug-level.rst

@@ -0,0 +1,13 @@
+**Added:** None
+
+**Changed:**
+
+* Updated the effectivity of ``$XONSH_DEBUG`` on debug messages.
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None

setup.py

@@ -295,7 +295,7 @@ def main():
         scripts.append('scripts/xonsh')
     skw = dict(
         name='xonsh',
-        description='A general purpose, Python-ish shell',
+        description='Python-powered, cross-platform, Unix-gazing shell',
         long_description=readme,
         license='BSD',
         version=XONSH_VERSION,

tests/test_history.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-"""Tests the xonsh history."""
+"""Tests the json history backend."""
 # pylint: disable=protected-access
 import io
 import os
@@ -31,9 +31,19 @@ def test_hist_init(hist):
 def test_hist_append(hist, xonsh_builtins):
     """Verify appending to the history works."""
     xonsh_builtins.__xonsh_env__['HISTCONTROL'] = set()
-    hf = hist.append({'joco': 'still alive'})
+    hf = hist.append({'inp': 'still alive', 'rtn': 0})
     assert hf is None
-    assert 'still alive' == hist.buffer[0]['joco']
+    assert 'still alive' == hist.buffer[0]['inp']
+    assert 0 == hist.buffer[0]['rtn']
+    assert 0 == hist.rtns[-1]
+    hf = hist.append({'inp': 'dead now', 'rtn': 1})
+    assert 'dead now' == hist.buffer[1]['inp']
+    assert 1 == hist.buffer[1]['rtn']
+    assert 1 == hist.rtns[-1]
+    hf = hist.append({'inp': 'reborn', 'rtn': 0})
+    assert 'reborn' == hist.buffer[2]['inp']
+    assert 0 == hist.buffer[2]['rtn']
+    assert 0 == hist.rtns[-1]
 
 
 def test_hist_flush(hist, xonsh_builtins):
@@ -41,20 +51,61 @@ def test_hist_flush(hist, xonsh_builtins):
     hf = hist.flush()
     assert hf is None
     xonsh_builtins.__xonsh_env__['HISTCONTROL'] = set()
-    hist.append({'joco': 'still alive'})
+    hist.append({'inp': 'still alive?', 'rtn': 0, 'out': 'yes'})
     hf = hist.flush()
     assert hf is not None
     while hf.is_alive():
         pass
     with LazyJSON(hist.filename) as lj:
-        obs = lj['cmds'][0]['joco']
-    assert 'still alive' == obs
+        assert len(lj['cmds']) == 1
+        cmd = lj['cmds'][0]
+        assert cmd['inp'] == 'still alive?'
+        assert not cmd.get('out', None)
+
+
+def test_hist_flush_with_store_stdout(hist, xonsh_builtins):
+    """Verify explicit flushing of the history works."""
+    hf = hist.flush()
+    assert hf is None
+    xonsh_builtins.__xonsh_env__['HISTCONTROL'] = set()
+    xonsh_builtins.__xonsh_env__['XONSH_STORE_STDOUT'] = True
+    hist.append({'inp': 'still alive?', 'rtn': 0, 'out': 'yes'})
+    hf = hist.flush()
+    assert hf is not None
+    while hf.is_alive():
+        pass
+    with LazyJSON(hist.filename) as lj:
+        assert len(lj['cmds']) == 1
+        assert lj['cmds'][0]['inp'] == 'still alive?'
+        assert lj['cmds'][0]['out'].strip() == 'yes'
+
+
+def test_hist_flush_with_hist_control(hist, xonsh_builtins):
+    """Verify explicit flushing of the history works."""
+    hf = hist.flush()
+    assert hf is None
+    xonsh_builtins.__xonsh_env__['HISTCONTROL'] = 'ignoredups,ignoreerr'
+    hist.append({'inp': 'ls foo1', 'rtn': 0})
+    hist.append({'inp': 'ls foo1', 'rtn': 1})
+    hist.append({'inp': 'ls foo1', 'rtn': 0})
+    hist.append({'inp': 'ls foo2', 'rtn': 2})
+    hist.append({'inp': 'ls foo3', 'rtn': 0})
+    hf = hist.flush()
+    assert hf is not None
+    while hf.is_alive():
+        pass
+    assert len(hist.buffer) == 0
+    with LazyJSON(hist.filename) as lj:
+        cmds = list(lj['cmds'])
+        assert len(cmds) == 2
+        assert [x['inp'] for x in cmds] == ['ls foo1', 'ls foo3']
+        assert [x['rtn'] for x in cmds] == [0, 0]
 
 
 def test_cmd_field(hist, xonsh_builtins):
     # in-memory
     xonsh_builtins.__xonsh_env__['HISTCONTROL'] = set()
-    hf = hist.append({'rtn': 1})
+    hf = hist.append({'inp': 'ls foo', 'rtn': 1})
     assert hf is None
     assert 1 == hist.rtns[0]
     assert 1 == hist.rtns[-1]
@@ -106,53 +157,71 @@ def test_histcontrol(hist, xonsh_builtins):
 
     # An error, buffer remains empty
     hist.append({'inp': 'ls foo', 'rtn': 2})
-    assert len(hist.buffer) == 0
+    assert len(hist.buffer) == 1
+    assert hist.rtns[-1] == 2
+    assert hist.inps[-1] == 'ls foo'
 
     # Success
     hist.append({'inp': 'ls foobazz', 'rtn': 0})
-    assert len(hist.buffer) == 1
+    assert len(hist.buffer) == 2
     assert 'ls foobazz' == hist.buffer[-1]['inp']
     assert 0 == hist.buffer[-1]['rtn']
+    assert hist.rtns[-1] == 0
+    assert hist.inps[-1] == 'ls foobazz'
 
     # Error
     hist.append({'inp': 'ls foo', 'rtn': 2})
-    assert len(hist.buffer) == 1
-    assert 'ls foobazz' == hist.buffer[-1]['inp']
-    assert 0 == hist.buffer[-1]['rtn']
+    assert len(hist.buffer) == 3
+    assert 'ls foo' == hist.buffer[-1]['inp']
+    assert 2 == hist.buffer[-1]['rtn']
+    assert hist.rtns[-1] == 2
+    assert hist.inps[-1] == 'ls foo'
 
     # File now exists, success
     hist.append({'inp': 'ls foo', 'rtn': 0})
-    assert len(hist.buffer) == 2
+    assert len(hist.buffer) == 4
     assert 'ls foo' == hist.buffer[-1]['inp']
     assert 0 == hist.buffer[-1]['rtn']
+    assert hist.rtns[-1] == 0
+    assert hist.inps[-1] == 'ls foo'
 
     # Success
     hist.append({'inp': 'ls', 'rtn': 0})
-    assert len(hist.buffer) == 3
+    assert len(hist.buffer) == 5
     assert 'ls' == hist.buffer[-1]['inp']
     assert 0 == hist.buffer[-1]['rtn']
+    assert hist.rtns[-1] == 0
+    assert hist.inps[-1] == 'ls'
 
     # Dup
     hist.append({'inp': 'ls', 'rtn': 0})
-    assert len(hist.buffer) == 3
+    assert len(hist.buffer) == 6
+    assert hist.rtns[-1] == 0
+    assert hist.inps[-1] == 'ls'
 
     # Success
     hist.append({'inp': '/bin/ls', 'rtn': 0})
-    assert len(hist.buffer) == 4
+    assert len(hist.buffer) == 7
     assert '/bin/ls' == hist.buffer[-1]['inp']
     assert 0 == hist.buffer[-1]['rtn']
+    assert hist.rtns[-1] == 0
+    assert hist.inps[-1] == '/bin/ls'
 
     # Error
     hist.append({'inp': 'ls bazz', 'rtn': 1})
-    assert len(hist.buffer) == 4
-    assert '/bin/ls' == hist.buffer[-1]['inp']
-    assert 0 == hist.buffer[-1]['rtn']
+    assert len(hist.buffer) == 8
+    assert 'ls bazz' == hist.buffer[-1]['inp']
+    assert 1 == hist.buffer[-1]['rtn']
+    assert hist.rtns[-1] == 1
+    assert hist.inps[-1] == 'ls bazz'
 
     # Error
     hist.append({'inp': 'ls bazz', 'rtn': -1})
-    assert len(hist.buffer) == 4
-    assert '/bin/ls' == hist.buffer[-1]['inp']
-    assert 0 == hist.buffer[-1]['rtn']
+    assert len(hist.buffer) == 9
+    assert 'ls bazz' == hist.buffer[-1]['inp']
+    assert -1 == hist.buffer[-1]['rtn']
+    assert hist.rtns[-1] == -1
+    assert hist.inps[-1] == 'ls bazz'
 
 
 @pytest.mark.parametrize('args', [ '-h', '--help', 'show -h', 'show --help'])

xonsh/commands_cache.py

@@ -256,6 +256,7 @@ def default_threadable_predictors():
         'clear': predict_false,
         'cls': predict_false,
         'cmd': predict_shell,
+        'ex': predict_false,
         'fish': predict_shell,
         'htop': predict_help_ver,
         'ksh': predict_shell,
@@ -267,6 +268,8 @@ def default_threadable_predictors():
         'nano': predict_help_ver,
         'psql': predict_false,
         'ranger': predict_help_ver,
+        'rview': predict_false,
+        'rvim': predict_false,
         'scp': predict_false,
         'sh': predict_shell,
         'ssh': predict_false,
@@ -275,6 +278,7 @@ def default_threadable_predictors():
         'tcsh': predict_shell,
         'top': predict_help_ver,
         'vi': predict_false,
+        'view': predict_false,
         'vim': predict_false,
         'vimpager': predict_help_ver,
         'xo': predict_help_ver,

xonsh/environ.py

@@ -298,7 +298,7 @@ def DEFAULT_VALUES():
         'XONSH_COLOR_STYLE': 'default',
         'XONSH_CONFIG_DIR': xonsh_config_dir,
         'XONSH_DATA_DIR': xonsh_data_dir,
-        'XONSH_DEBUG': False,
+        'XONSH_DEBUG': 0,
         'XONSH_ENCODING': DEFAULT_ENCODING,
         'XONSH_ENCODING_ERRORS': 'surrogateescape',
         'XONSH_HISTORY_BACKEND': 'json',
@@ -615,10 +615,13 @@ def DEFAULT_DOCS():
         'This is the location where xonsh configuration information is stored.',
         configurable=False, default="``$XDG_CONFIG_HOME/xonsh``"),
     'XONSH_DEBUG': VarDocs(
-        'Sets the xonsh debugging level. This may be an integer or a boolean, '
-        'with higher values cooresponding to higher debuging levels and more '
-        'information presented. Setting this variable prior to stating xonsh '
-        'will supress amalgamated imports.', configurable=False),
+        'Sets the xonsh debugging level. This may be an integer or a boolean. '
+        'Setting this variable prior to stating xonsh to ``1`` or ``True`` '
+        'will supress amalgamated imports. Setting it to ``2`` will get some '
+        'basic information like input transformation, command replacement. '
+        'With ``3`` or a higher number will make more debugging information '
+        'presented, like PLY parsing messages.',
+        configurable=False),
     'XONSH_DATA_DIR': VarDocs(
         'This is the location where xonsh data files are stored, such as '
         'history.', default="``$XDG_DATA_HOME/xonsh``"),

xonsh/execer.py

@@ -54,7 +54,7 @@ class Execer(object):
             filename = self.filename
         if not transform:
             return self.parser.parse(input, filename=filename, mode=mode,
-                                     debug_level=(self.debug_level > 1))
+                                     debug_level=(self.debug_level > 2))
 
         # Parsing actually happens in a couple of phases. The first is a
         # shortcut for a context-free parser. Normally, all subprocess
@@ -161,7 +161,7 @@ class Execer(object):
                 tree = self.parser.parse(input,
                                          filename=filename,
                                          mode=mode,
-                                         debug_level=(self.debug_level > 1))
+                                         debug_level=(self.debug_level > 2))
                 parsed = True
             except IndentationError as e:
                 if original_error is None:
@@ -221,7 +221,7 @@ class Execer(object):
                     # anything
                     raise original_error
                 else:
-                    if self.debug_level:
+                    if self.debug_level > 1:
                         msg = ('{0}:{1}:{2}{3} - {4}\n'
                                '{0}:{1}:{2}{3} + {5}')
                         mstr = '' if maxcol is None else ':' + str(maxcol)

xonsh/foreign_shells.py

@@ -584,7 +584,7 @@ def load_foreign_aliases(shells=None, config=None, issue_warning=True):
             shaliases = {} if shaliases is None else shaliases
             for alias in set(shaliases) & set(xonsh_aliases):
                 del shaliases[alias]
-                if builtins.__xonsh_env__.get('XONSH_DEBUG'):
+                if builtins.__xonsh_env__.get('XONSH_DEBUG') > 1:
                     print('aliases: ignoring alias {!r} of shell {!r} '
                           'which tries to override xonsh alias.'
                           ''.format(alias, shell['shell']),

xonsh/history/json.py

@@ -176,12 +176,28 @@ class JsonHistoryFlusher(threading.Thread):
 
     def dump(self):
         """Write the cached history to external storage."""
+        opts = builtins.__xonsh_env__.get('HISTCONTROL')
+        last_inp = None
+        cmds = []
+        for cmd in self.buffer:
+            if 'ignoredups' in opts and cmd['inp'] == last_inp:
+                # Skipping dup cmd
+                continue
+            if 'ignoreerr' in opts and cmd['rtn'] != 0:
+                # Skipping failed cmd
+                continue
+            cmds.append(cmd)
+            last_inp = cmd['inp']
         with open(self.filename, 'r', newline='\n') as f:
             hist = xlj.LazyJSON(f).load()
-        hist['cmds'].extend(self.buffer)
+        load_hist_len = len(hist['cmds'])
+        hist['cmds'].extend(cmds)
         if self.at_exit:
             hist['ts'][1] = time.time()  # apply end time
             hist['locked'] = False
+        if not builtins.__xonsh_env__.get('XONSH_STORE_STDOUT', False):
+            [cmd.pop('out') for cmd in hist['cmds'][load_hist_len:]
+                if 'out' in cmd]
         with open(self.filename, 'w', newline='\n') as f:
             xlj.ljdump(hist, f, sort_keys=True)
 
@@ -314,15 +330,6 @@ class JsonHistory(History):
         hf : JsonHistoryFlusher or None
             The thread that was spawned to flush history
         """
-        opts = builtins.__xonsh_env__.get('HISTCONTROL')
-        if ('ignoredups' in opts and len(self) > 0 and
-                cmd['inp'] == self.inps[-1]):
-            # Skipping dup cmd
-            return None
-        elif 'ignoreerr' in opts and cmd['rtn'] != 0:
-            # Skipping failed cmd
-            return None
-
         self.buffer.append(cmd)
         self._len += 1  # must come before flushing
         if len(self.buffer) >= self.buffersize:
@@ -363,7 +370,7 @@ class JsonHistory(History):
         """
         Returns all history as found in XONSH_DATA_DIR.
 
-        return format: (cmd, start_time, index)
+        yeild format: {'inp': cmd, 'rtn': 0, ...}
         """
         while self.gc and self.gc.is_alive():
             time.sleep(0.011)  # gc sleeps for 0.01 secs, sleep a beat longer

xonsh/ptk/shortcuts.py

@@ -9,6 +9,7 @@ from prompt_toolkit.shortcuts import (create_prompt_application,
                                       create_eventloop, create_asyncio_eventloop, create_output)
 
 from xonsh.platform import ptk_version_info
+import xonsh.tools as xt
 
 
 class Prompter(object):
@@ -106,9 +107,13 @@ class Prompter(object):
             try:
                 with patch_context:
                     document = cli.run(reset_current_buffer=False)
-
                     if document:
                         return document.text
+            except Exception:
+                xt.print_exception()
+                # return something to prevent xonsh crash when any
+                # exceptions raise
+                return ''
             finally:
                 eventloop.close()
 

xonsh/shell.py

@@ -49,7 +49,8 @@ def fire_precommand(src, show_diff=True):
             print_exception('Modifcations to source input took more than '
                             'the recursion limit number of interations to '
                             'converge.')
-    if show_diff and builtins.__xonsh_env__.get('XONSH_DEBUG') and src != raw:
+    debug_level = builtins.__xonsh_env__.get('XONSH_DEBUG')
+    if show_diff and debug_level > 1 and src != raw:
         sys.stderr.writelines(difflib.unified_diff(
             raw.splitlines(keepends=True),
             src.splitlines(keepends=True),

xonsh/xontribs.json

@@ -21,7 +21,7 @@
     "you might find them useful if you have strong muscle memory.\n\n",
     "**Warning:** This xontrib may modify user command line input to implement ",
     "its behavior. To see the modifications as they are applied (in unified diff",
-    "format), please set ``$XONSH_DEBUG = True``."]
+    "format), please set ``$XONSH_DEBUG`` to ``2`` or higher."]
  },
  {"name": "distributed",
   "package": "xonsh",

xontrib/bashisms.py

@@ -3,4 +3,8 @@
 
 @events.on_precommand
 def bash_preproc(cmd):
+    if not __xonsh_history__.inps:
+        if cmd.strip() == '!!':
+            return ''
+        return cmd
     return cmd.replace('!!', __xonsh_history__.inps[-1].strip())