Add support for rc.d drop-in configuration directories (#4256)

* Add support for rc.d drop-in configuration directories

* main: change how --rc, --no-rc are handled

Explicitly pass --no-rc rather than signalling it as an empty --rc, to
indicate that we should suppress both XONSHRC and XONSHRCDIR in that
case.

* Rename XONSHRCDIR -> XONSHRC_DIR

* xonshrc_context: document setting env for XONSHRC_DIR

* main: --rc foo.xsh should suppress XONSHRC_DIR

docs/xonshrc.rst

@@ -13,9 +13,14 @@ The system-wide ``xonshrc`` file controls options that are applied to all users
 You can create this file in ``/etc/xonshrc`` for Linux and OSX and in ``%ALLUSERSPROFILE%\xonsh\xonshrc`` on Windows.
 
 Xonsh also allows a per-user run control file in your home directory, either
-directly in the home directory at ``~/.xonshrc`` or, for XDG compliance, at ``~/.config/rc.xsh``. 
+directly in the home directory at ``~/.xonshrc`` or, for XDG compliance, at ``~/.config/xonsh/rc.xsh``.
 The options set per user override settings in the system-wide control file.
 
+Xonsh also supports configuration directories, from which all ``.xsh`` files will be sourced in order.
+This allows for drop-in configuration where your configuration can be split across scripts and common
+and local configuration more easily separated. By default, if the directory ``~/.config/xonsh/rc.d``
+exists, any ``xsh`` files within will be sourced at startup.
+
 Xonsh provides 2 wizards to create your own "xonshrc".  ``xonfig web`` provides basic settings, and ``xonfig wizard``
 steps you through all the available options.
 

news/rc_dir.rst

@@ -0,0 +1,30 @@
+**Added:**
+
+* In addition to reading single rc files at startup (``/etc/xonshrc``, ``~/.config/xonsh/rc.xsh``),
+  xonsh now also supports rc.d-style config directories, from which all files are sourced. This is
+  designed to support drop-in style configuration where you could, for example, have a common config
+  file shared across multiple machines and a separate machine specific file.
+
+  This is controlled by the environment variable ``XONSHRC_DIR``, which defaults to
+  ``["/etc/xonsh/rc.d", "~/.config/xonsh/rc.d"]``. If those directories exist, then any ``xsh`` files
+  contained within are sorted and then sourced.
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

tests/test_main.py

@@ -100,6 +100,61 @@ def test_rc_with_modules(shell, tmpdir, monkeypatch, capsys):
     assert tmpdir.strpath not in sys.path
 
 
+def test_rcdir(shell, tmpdir, monkeypatch, capsys):
+    """
+    Test that files are loaded from an rcdir, after a normal rc file,
+    and in lexographic order.
+    """
+
+    rcdir = tmpdir.join("rc.d")
+    rcdir.mkdir()
+    monkeypatch.setattr(sys.stdin, "isatty", lambda: True)
+    monkeypatch.setitem(os.environ, "XONSHRC_DIR", str(rcdir))
+    monkeypatch.setitem(os.environ, "XONSHRC", str(tmpdir.join("rc.xsh")))
+    monkeypatch.setitem(os.environ, "XONSH_CACHE_SCRIPTS", "False")
+
+    rcdir.join("2.xsh").write("print('2.xsh')")
+    rcdir.join("0.xsh").write("print('0.xsh')")
+    rcdir.join("1.xsh").write("print('1.xsh')")
+    tmpdir.join("rc.xsh").write("print('rc.xsh')")
+
+    xonsh.main.premain([])
+    stdout, stderr = capsys.readouterr()
+
+    assert "rc.xsh\n0.xsh\n1.xsh\n2.xsh" in stdout
+    assert len(stderr) == 0
+
+
+def test_rcdir_empty(shell, tmpdir, monkeypatch, capsys):
+    """Test that an empty XONSHRC_DIR is not an error"""
+
+    rcdir = tmpdir.join("rc.d")
+    rcdir.mkdir()
+    monkeypatch.setattr(sys.stdin, "isatty", lambda: True)
+    monkeypatch.setitem(os.environ, "XONSHRC_DIR", str(rcdir))
+
+    xonsh.main.premain([])
+    stdout, stderr = capsys.readouterr()
+    assert len(stderr) == 0
+
+
+def test_rcdir_ignored_with_rc(shell, tmpdir, monkeypatch, capsys):
+    """Test that --rc suppresses loading XONSHRC_DIRs"""
+
+    rcdir = tmpdir.join("rc.d")
+    rcdir.mkdir()
+    monkeypatch.setattr(sys.stdin, "isatty", lambda: True)
+    monkeypatch.setitem(os.environ, "XONSHRC_DIR", str(rcdir))
+    rcdir.join("rcd.xsh").write("print('RCDIR')")
+    tmpdir.join("rc.xsh").write("print('RCFILE')")
+
+    xonsh.main.premain(["--rc", str(tmpdir.join("rc.xsh"))])
+    stdout, stderr = capsys.readouterr()
+    assert "RCDIR" not in stdout
+    assert "RCFILE" in stdout
+    assert not builtins.__xonsh__.env.get("XONSHRC_DIR")
+
+
 @pytest.mark.skipif(ON_WINDOWS, reason="See https://github.com/xonsh/xonsh/issues/3936")
 def test_rc_with_modified_path(shell, tmpdir, monkeypatch, capsys):
     """Test that an RC file can edit the sys.path variable without losing those values."""
@@ -189,6 +244,7 @@ def test_custom_rc_with_script(shell, tmpdir):
 def test_premain_no_rc(shell, tmpdir):
     xonsh.main.premain(["--no-rc", "-i"])
     assert not builtins.__xonsh__.env.get("XONSHRC")
+    assert not builtins.__xonsh__.env.get("XONSHRC_DIR")
 
 
 @pytest.mark.parametrize(

xonsh/environ.py

@@ -6,6 +6,7 @@ import sys
 import pprint
 import textwrap
 import locale
+import glob
 import builtins
 import warnings
 import contextlib
@@ -604,6 +605,15 @@ def default_xonshrc(env):
     return dxrc
 
 
+@default_value
+def default_xonshrcdir(env):
+    xdgrcd = os.path.join(xonsh_config_dir(env), "rc.d")
+    if ON_WINDOWS:
+        return (os.path.join(os_environ["ALLUSERSPROFILE"], "xonsh", "rc.d"), xdgrcd)
+    else:
+        return ("/etc/xonsh/rc.d", xdgrcd)
+
+
 @default_value
 def xonsh_append_newline(env):
     """Appends a newline if we are in interactive mode"""
@@ -940,6 +950,18 @@ class GeneralSetting(Xettings):
         ),
         type_str="env_path",
     )
+    XONSHRC_DIR = Var.with_default(
+        default_xonshrcdir,
+        "A list of directories, from which all .xsh files will be loaded "
+        "at startup, sorted in lexographic order. Files in these directories "
+        "are loaded after any files in XONSHRC.",
+        doc_default=(
+            "On Linux & Mac OSX: ``['/etc/xonsh/rc.d', '~/.config/xonsh/rc.d']``\n"
+            "On Windows: "
+            "``['%ALLUSERSPROFILE%\\\\xonsh\\\\rc.d', '~/.config/xonsh/rc.d']``"
+        ),
+        type_str="env_path",
+    )
     XONSH_APPEND_NEWLINE = Var.with_default(
         xonsh_append_newline,
         "Append new line when a partial line is preserved in output.",
@@ -2129,21 +2151,40 @@ def locate_binary(name):
     return builtins.__xonsh__.commands_cache.locate_binary(name)
 
 
-def xonshrc_context(rcfiles=None, execer=None, ctx=None, env=None, login=True):
-    """Attempts to read in all xonshrc files and return the context."""
+def xonshrc_context(
+    rcfiles=None, rcdirs=None, execer=None, ctx=None, env=None, login=True
+):
+    """
+    Attempts to read in all xonshrc files and return the context.
+    The xonsh environment here is updated to reflect which RC files and
+    directory locations will have been loaded (if they existed). The updated
+    environment vars might be different (or empty) depending on CLI options
+    (--rc, --no-rc) or whether the session is interactive.
+    """
     loaded = env["LOADED_RC_FILES"] = []
     ctx = {} if ctx is None else ctx
-    if rcfiles is None:
+    if rcfiles is None and rcdirs is None:
         return env
     orig_thread = env.get("THREAD_SUBPROCS")
     env["THREAD_SUBPROCS"] = None
-    env["XONSHRC"] = tuple(rcfiles)
-    for rcfile in rcfiles:
-        if not os.path.isfile(rcfile):
-            loaded.append(False)
-            continue
-        status = xonsh_script_run_control(rcfile, ctx, env, execer=execer, login=login)
-        loaded.append(status)
+    if rcfiles is not None:
+        env["XONSHRC"] = tuple(rcfiles)
+        for rcfile in rcfiles:
+            if not os.path.isfile(rcfile):
+                loaded.append(False)
+                continue
+            status = xonsh_script_run_control(
+                rcfile, ctx, env, execer=execer, login=login
+            )
+            loaded.append(status)
+    if rcdirs is not None:
+        env["XONSHRC_DIR"] = tuple(rcdirs)
+        for rcdir in rcdirs:
+            if os.path.isdir(rcdir):
+                for rcfile in sorted(glob.glob(os.path.join(rcdir, "*.xsh"))):
+                    status = xonsh_script_run_control(
+                        rcfile, ctx, env, execer=execer, login=login
+                    )
     if env["THREAD_SUBPROCS"] is None:
         env["THREAD_SUBPROCS"] = orig_thread
     return ctx

xonsh/main.py

@@ -297,17 +297,32 @@ def start_services(shell_kwargs, args, pre_env=None):
     env = builtins.__xonsh__.env
     for k, v in pre_env.items():
         env[k] = v
-    rc = shell_kwargs.get("rc", None)
-    rc = env.get("XONSHRC") if rc is None else rc
-    if (
+
+    # determine which RC files to load, including whether any RC directories
+    # should be scanned for such files
+    if shell_kwargs.get("norc") or (
         args.mode != XonshMode.interactive
         and not args.force_interactive
         and not args.login
     ):
-        #  Don't load xonshrc if not interactive shell
-        rc = None
+        # if --no-rc was passed, or we're not in an interactive shell and
+        # interactive mode was not forced, then disable loading RC files and dirs
+        rc = ()
+        rcd = ()
+    elif shell_kwargs.get("rc"):
+        # if an explicit --rc was passed, then we should load only that RC
+        # file, and nothing else (ignore both XONSHRC and XONSHRC_DIR)
+        rc = shell_kwargs.get("rc")
+        rcd = ()
+    else:
+        # otherwise, get the RC files from XONSHRC, and RC dirs from XONSHRC_DIR
+        rc = env.get("XONSHRC")
+        rcd = env.get("XONSHRC_DIR")
+
     events.on_pre_rc.fire()
-    xonshrc_context(rcfiles=rc, execer=execer, ctx=ctx, env=env, login=login)
+    xonshrc_context(
+        rcfiles=rc, rcdirs=rcd, execer=execer, ctx=ctx, env=env, login=login
+    )
     events.on_post_rc.fire()
     # create shell
     builtins.__xonsh__.shell = Shell(execer=execer, **shell_kwargs)
@@ -340,7 +355,7 @@ def premain(argv=None):
         args.login = True
         shell_kwargs["login"] = True
     if args.norc:
-        shell_kwargs["rc"] = ()
+        shell_kwargs["norc"] = True
     elif args.rc:
         shell_kwargs["rc"] = args.rc
     setattr(sys, "displayhook", _pprint_displayhook)