Add groups to env vars (#4026)

* feat: add grouped-settings for env variables

fixes #4014

style: fix mypy errors

chore: update testing requirements versions

fix: update xonsh.tools import error

* chore: add news item

* fix: update Var.with_default handling env defaults

* fix: set env var.doc_default=DefaultNotGiven

there is a custom handler for it

* chore: update travis

speedup docs generation

* chore: add command to serve docs during development

* docs: add jinja2 helpers/renderers extension for sphinx

* docs: update envvars document

* docs: fix docs failing

* Update xonsh/environ.py

commit suggestion

Co-authored-by: Gil Forsyth <gforsyth@users.noreply.github.com>

* Update xonsh/environ.py

Co-authored-by: Gil Forsyth <gforsyth@users.noreply.github.com>

* Update xonsh/environ.py

Co-authored-by: Gil Forsyth <gforsyth@users.noreply.github.com>

* Update xonsh/environ.py

Co-authored-by: Gil Forsyth <gforsyth@users.noreply.github.com>

* Update xonsh/environ.py

Co-authored-by: Gil Forsyth <gforsyth@users.noreply.github.com>

* Update xonsh/environ.py

Co-authored-by: Gil Forsyth <gforsyth@users.noreply.github.com>

* refactor: update rst-extension

Co-authored-by: Gil Forsyth <gforsyth@users.noreply.github.com>

.travis.yml

@@ -5,34 +5,13 @@ env:
 matrix:
     include:
         - os: linux
-          python: 3.6
+          python: 3.8
           env:
-            - MINICONDA_OS="Linux"
             - BUILD_DOCS=true
 
-before_install:
-  - if [[ ! ("$TRAVIS_PYTHON_VERSION" == "nightly" || "$TRAVIS_PYTHON_VERSION" == "3.6-dev") && ! $BUILD_DOCS ]]; then
-      URL="https://repo.continuum.io/miniconda/Miniconda3-latest-${MINICONDA_OS}-x86_64.sh";
-      wget "${URL}" -O miniconda.sh;
-      bash miniconda.sh -b -p $HOME/miniconda;
-      export PATH="$HOME/miniconda/bin:$PATH";
-      hash -r;
-      conda config --set always_yes yes --set changeps1 no;
-      conda update -q conda;
-      conda info -a;
-    fi
-
 install:
-  - if [[ $BUILD_DOCS = true ]]; then
-      pip install --upgrade -r requirements/docs.txt;
-      pip install .;
-    else
-      pip install --upgrade -r requirements/tests.txt;
-      pip install .;
-    fi
-
-before_script:
-  - rvm get head || true
+  - pip install --upgrade -r requirements/docs.txt
+  - pip install .
 
 script:
   - set -ex
@@ -48,6 +27,4 @@ script:
       make clean html;
       cd ..;
       doctr deploy --deploy-repo xonsh/xonsh-docs .;
-    else
-      xonsh run-tests.xsh test;
     fi

docs/Makefile

@@ -128,3 +128,7 @@ push-root:
 	git commit -am "Pushed root-level docs at $(date)" && \
 	git push
 
+serve:
+	# run as `make serve`
+	# to not watch file changes, run as `make serve watch=no`
+	python serve_docs.py $(watch)

docs/conf.py

@@ -9,18 +9,29 @@
 # serve to show the default.
 import os
 import sys
+from collections import OrderedDict
+from pathlib import Path
+
+# make current docs directory modules importable
+sys.path.append(str(Path(__file__).parent.resolve()))
+
 import builtins
 import inspect
 import importlib
+import typing as tp
 
 os.environ["XONSH_DEBUG"] = "1"
 
 from xonsh import __version__ as XONSH_VERSION
-from xonsh.environ import DEFAULT_VARS, Env
+from xonsh.environ import Env, Var, Xettings
+
+if tp.TYPE_CHECKING:
+    from xonsh.environ import VarKeyType
 from xonsh.xontribs_meta import get_xontribs
-from xonsh import main
 from xonsh.commands_cache import CommandsCache
 
+import rst_helpers
+
 if not hasattr(builtins, "__xonsh__"):
     from argparse import Namespace
 
@@ -57,6 +68,7 @@ extensions = [
     "numpydoc",
     "cmdhelp",
     "runthis.sphinxext",
+    "jinja_rst_ext",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -96,7 +108,11 @@ release = XONSH_VERSION
 # today_fmt = '%B %d, %Y'
 
 # List of documents that shouldn't be included in the build.
-exclude_patterns = ["api/blank.rst"]
+exclude_patterns = [
+    "api/blank.rst",
+    "_build",
+    "_static",
+]
 
 # List of directories, relative to source directory, that shouldn't be searched
 # for source files.
@@ -275,48 +291,55 @@ runthis_server = "https://runthis.xonsh.org:80"
 #
 
 
-def make_envvars():
-    env = Env()
-    vars = sorted(DEFAULT_VARS.keys(), key=lambda x: getattr(x, "pattern", x))
-    s = ".. list-table::\n" "    :header-rows: 0\n\n"
-    table = []
-    ncol = 3
-    row = "    {0} - :ref:`${1} <{2}>`"
-    for i, varname in enumerate(vars):
-        var = getattr(varname, "pattern", varname)
-        star = "*" if i % ncol == 0 else " "
-        table.append(row.format(star, var, var.lower()))
-    table.extend(["      -"] * ((ncol - len(vars) % ncol) % ncol))
-    s += "\n".join(table) + "\n\n"
-    s += "Listing\n" "-------\n\n"
-    sec = (
-        ".. _{low}:\n\n"
-        "{title}\n"
-        "{under}\n"
-        "{docstr}\n\n"
-        "**configurable:** {configurable}\n\n"
-        "**default:** {default}\n\n"
-        "**store_as_str:** {store_as_str}\n\n"
-        "-------\n\n"
-    )
-    for varname in vars:
-        var = getattr(varname, "pattern", varname)
+class VarDoc(tp.NamedTuple):
+    var: Var
+    info: tp.Dict[str, tp.Any]  # for using in template additional info
+
+
+class EnvVarGroup(tp.NamedTuple):
+    vars: tp.Dict["VarKeyType", VarDoc]  # sorted variables in the section
+    children: tp.Dict[Xettings, "EnvVarGroup"]  # child sections
+
+
+def _gather_groups(cls, env: Env, _seen=None):
+    if _seen is None:
+        _seen = set()
+
+    env_vars = list(cls.get_settings())
+    env_vars.sort(key=lambda x: getattr(x[0], "pattern", x[0]))
+    ordered_vars = OrderedDict()  # within that section sort keys
+    for key, var in env_vars:
+        var = getattr(key, "pattern", key)
         title = "$" + var
-        under = "." * len(title)
-        vd = env.get_docs(varname)
-        s += sec.format(
-            low=var.lower(),
+        vd = env.get_docs(key)
+        info = dict(
             title=title,
-            under=under,
             docstr=vd.doc,
-            configurable=vd.doc_configurable,
+            configurable=vd.is_configurable,
             default=vd.doc_default,
-            store_as_str=vd.doc_store_as_str,
+            store_as_str=vd.can_store_as_str,
         )
-    s = s[:-9]
-    fname = os.path.join(os.path.dirname(__file__), "envvarsbody")
-    with open(fname, "w", encoding="utf-8") as f:
-        f.write(s)
+        ordered_vars[key] = VarDoc(var, info)
+
+    vargrp = EnvVarGroup(ordered_vars, {})
+    for sub in cls.__subclasses__():
+        if sub not in _seen:
+            _seen.add(sub)
+            vargrp.children[sub] = _gather_groups(sub, env, _seen)
+    return vargrp
+
+
+def make_envvars():
+    return _gather_groups(Xettings, env=Env())
+
+
+jinja_contexts = {
+    # file-name envvars.rst
+    "envvars": {
+        "env_vars": make_envvars(),
+        "rst": rst_helpers,
+    },
+}
 
 
 def make_xontribs():
@@ -424,7 +447,6 @@ def make_events():
         f.write(s)
 
 
-make_envvars()
 make_xontribs()
 make_events()
 
@@ -436,5 +458,5 @@ builtins.__xonsh__.commands_cache = CommandsCache()
 def setup(app):
     from xonsh.pyghooks import XonshConsoleLexer
 
-    app.add_lexer("xonshcon", XonshConsoleLexer())
+    app.add_lexer("xonshcon", XonshConsoleLexer)
     app.add_css_file("custom.css")

docs/envvars.jinja2

@@ -0,0 +1,50 @@
+{% set ns = namespace(root_index='') %}
+
+{% macro list_table(vars) %}
+    {% for vardoc in rst.iterator_for_divmod(vars.values()) %}
+{% if loop.index0 % 3 == 0 %}* {% else %}  {% endif %}- {% if vardoc %}{{ rst.to_ref_string(vardoc.info.title) }}{% endif %}
+
+    {% endfor %}
+{% endmacro %}
+
+{#titular table#}
+{% for cls, envgrp in env_vars.children.items() recursive %}
+    {% if not loop.depth0 %}
+        {% set ns.root_index=loop.index %}
+    {% endif %}
+
+.. list-table:: {{ rst.to_ref_string(cls.get_group_title()) }}
+    :header-rows: 0
+
+    {{ list_table(envgrp.vars)|indent(4) }}
+    {% if envgrp.children %}{{ loop(envgrp.children.items()) }}{% endif %}
+{% endfor %}
+
+{#variables doc#}
+{% for cls, envgrp in env_vars.children.items() recursive %}
+
+.. _{{ rst.to_valid_id(cls.get_group_title()) }}:
+
+{{ cls.get_group_title() }}
+{{ rst.underline_title(cls.get_group_title(), loop.depth0) }}
+{{ cls.get_group_description() }}
+
+{% for key, vardoc in envgrp.vars.items() %}
+
+.. _{{ rst.to_valid_id(vardoc.info.title) }}:
+
+{{ vardoc.info.title }}
+{{ rst.underline_title(vardoc.info.title) }}
+{{ vardoc.info.docstr }}
+
+**configurable:** {{ vardoc.info.configurable }}
+
+**default:** {{ vardoc.info.default }}
+
+**store_as_str:** {{ vardoc.info.store_as_str }}
+
+-------
+{% endfor %}
+
+{% if envgrp.children %}{{ loop(envgrp.children.items()) }}{% endif %}
+{% endfor %}

docs/envvars.rst

@@ -4,4 +4,4 @@ The following displays information about the environment variables that
 affect xonsh performance in some way. It also lists their default values, if
 applicable.
 
-.. include:: envvarsbody
+{{ content }}

docs/jinja_rst_ext.py

@@ -0,0 +1,46 @@
+"""A sphinx extension to process jinja/rst template"""
+
+# https://www.ericholscher.com/blog/2016/jul/25/integrating-jinja-rst-sphinx/
+
+from pathlib import Path
+
+import jinja2
+
+cur_dir = Path(__file__).parent.resolve()
+
+
+def rstjinja(app, docname, source):
+    """
+    Render our pages as a jinja template for fancy templating goodness.
+    """
+    # Make sure we're outputting HTML
+    if app.builder.format != "html":
+        return
+
+    ctx = app.config.jinja_contexts.get(docname)
+    if ctx is not None:
+        environment = jinja2.Environment(
+            trim_blocks=True,
+            lstrip_blocks=True,
+        )
+
+        src = source[0]
+        if "content" in src:
+            file_path = cur_dir / f"{docname}.jinja2"
+            if file_path.exists():
+                ctx["content"] = environment.from_string(file_path.read_text()).render(
+                    **ctx
+                )
+
+        rendered = app.builder.templates.render_string(src, ctx)
+        source[0] = rendered
+
+        # for debugging purpose write output
+        # Path(cur_dir / "_build" / f"{docname}.rst.out").write_text(rendered)
+
+
+def setup(app):
+    app.connect("source-read", rstjinja)
+
+    # rst files can define the context with their names to be pre-processed with jinja
+    app.add_config_value("jinja_contexts", {}, "env")

docs/rst_helpers.py

@@ -0,0 +1,43 @@
+"""ReST-Jinja2 helpers"""
+import re
+
+
+def underline_title(title: str, level: int = -1):
+    symbols = "-.^"
+    if level < len(symbols):
+        under = symbols[level]
+    else:
+        under = symbols[-1]
+    return under * len(title)
+
+
+def to_valid_id(name: str) -> str:
+    return re.sub(r"[^\w]", "_", name.lower()).strip("_")
+
+
+def to_valid_name(name: str) -> str:
+    return name.replace("`", "")
+
+
+def indent_depth(depth: int = None):
+    return " " * ((1 if depth else 0) * 4)
+
+
+def to_ref_string(title: str, underline=False, depth=-1) -> str:
+    title = str(title)
+    ref = f":ref:`{to_valid_name(title)} <{to_valid_id(title)}>`"
+    if underline:
+        return "\n".join([ref, underline_title(ref, depth)])
+    return ref
+
+
+def iterator_for_divmod(iterable, div: int = 3):
+    items = list(iterable)
+    size = len(items)
+    rem = size % div
+    complement = size + div - rem
+    for i in range(complement):
+        if i < size:
+            yield items[i]
+        else:
+            yield None

docs/serve_docs.py

@@ -0,0 +1,15 @@
+from livereload import Server, shell
+from pathlib import Path
+import sys
+
+cur_dir = Path(__file__).parent
+server = Server()
+
+if "no" not in sys.argv:
+    for ext in ("rst", "py", "jinja2"):
+        server.watch(
+            str(cur_dir / f"*.{ext}"),
+            shell("make html", cwd=str(cur_dir)),
+        )
+
+server.serve(root=str(cur_dir / "_build" / "html"))

news/group-env-variables.rst

@@ -0,0 +1,23 @@
+**Added:**
+
+* <news item>
+
+**Changed:**
+
+* group environment variables into categories.
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

requirements/docs.txt

@@ -11,3 +11,4 @@ tornado
 black
 pre-commit
 runthis-sphinxext
+livereload

requirements/tests.txt

@@ -1,5 +1,5 @@
 py
-pytest>=5.4
+pytest>=6
 flake8
 flake8-docstrings
 pytest-cov
@@ -7,7 +7,7 @@ pytest-timeout
 prompt-toolkit>=3.0
 pygments>=2.2
 codecov
-coverage
+coverage>=5.3.1
 black==20.8b1 --pre
 pre-commit
 mypy==0.790

tests/test_environ.py

@@ -19,6 +19,7 @@ from xonsh.environ import (
     make_args_env,
     LsColors,
     default_value,
+    Var,
 )
 
 from tools import skip_if_on_unix
@@ -431,12 +432,12 @@ def test_register_var_path():
     env = Env()
     env.register("MY_PATH_VAR", type="path")
 
-    path = '/tmp'
+    path = "/tmp"
     env["MY_PATH_VAR"] = path
     assert env["MY_PATH_VAR"] == pathlib.Path(path)
 
     # Empty string is None to avoid uncontrolled converting empty string to Path('.')
-    path = ''
+    path = ""
     env["MY_PATH_VAR"] = path
     assert env["MY_PATH_VAR"] == None
 
@@ -518,8 +519,7 @@ def test_env_iterate_rawkeys():
 
 
 def test_env_get_defaults():
-    """Verify the rather complex rules for env.get("<envvar>",default) value when envvar is not defined.
-    """
+    """Verify the rather complex rules for env.get("<envvar>",default) value when envvar is not defined."""
 
     env = Env(TEST1=0)
     env.register("TEST_REG", default="abc")
@@ -536,3 +536,16 @@ def test_env_get_defaults():
     # var not defined, is registered, reg default is sentinel => value is *immediate* default
     assert env.get("TEST_REG_DNG", 22) == 22
     assert "TEST_REG_DNG" not in env
+
+
+@pytest.mark.parametrize(
+    "val,validator",
+    [
+        ("string", "is_string"),
+        (1, "is_int"),
+        (0.5, "is_float"),
+    ],
+)
+def test_var_with_default_initer(val, validator):
+    var = Var.with_default(val)
+    assert var.validate.__name__ == validator

xonsh/xonfig.py

@@ -301,7 +301,7 @@ def make_envvar(name):
     """Makes a StoreNonEmpty node for an environment variable."""
     env = builtins.__xonsh__.env
     vd = env.get_docs(name)
-    if not vd.doc_configurable:
+    if not vd.is_configurable:
         return
     default = vd.doc_default
     if "\n" in default:
@@ -327,7 +327,7 @@ def make_envvar(name):
         show_conversion=True,
         path=path,
         retry=True,
-        store_raw=vd.doc_store_as_str,
+        store_raw=vd.can_store_as_str,
     )
     return mnode, pnode