refactor(amalgamate): remove amalgamation (#4858)

Co-authored-by: Noorhteen Raja NJ <jnoortheen@gmail.com>

.coveragerc

@@ -5,7 +5,6 @@ source =
   xontrib/
 omit =
   .venv/*
-  */__amalgam__.py
   xonsh/lazyasd.py
   xonsh/parser_table.py
   xonsh/completion_parser_table.py
@@ -24,7 +23,6 @@ precision = 2
 exclude_lines =
     pragma: no cover
     raise NotImplementedError
-    = __amalgam__
 skip_covered = true
 skip_empty = true
 show_missing = true

.github/workflows/test.yml

@@ -64,7 +64,7 @@ jobs:
         if: ${{ startsWith(matrix.python-version, '3.10') }}
         run: |
           python -m pip install -e . --no-deps
-          python -m xonsh run-tests.xsh test --report-coverage --no-amalgam -- --timeout=240
+          python -m xonsh run-tests.xsh test --report-coverage -- --timeout=240
       - name: Upload coverage to Codecov
         if: ${{ startsWith(matrix.python-version, '3.10') }}
         uses: codecov/codecov-action@v3

.gitignore

@@ -4,7 +4,6 @@
 *.egg
 .eggs/
 .pytest_cache/
-__amalgam__.py
 lexer_table.py
 parser_table.py
 parser_test_table.py

CONTRIBUTING.rst

@@ -33,17 +33,6 @@ Finally, run the following commands. You should see the effects of your change
         $ $XONSH_DEBUG=1
         $ xonsh
 
-The xonsh build process collapses all Python source files into a single
-``__amalgam__.py`` file. When xonsh is started with a falsy value for
-`$XONSH_DEBUG <envvars.html>`_, it imports Python modules straight from
-``__amalgam__.py``, which decreases startup times by eliminating the cost of
-runtime imports. But setting ``$ $XONSH_DEBUG=1`` will suppress amalgamated
-imports. Reloading the xonsh shell (``$ xonsh``) won't simply import the stale
-``__amalgam__.py`` file that doesn't contain your new change, but will instead
-import the unamalgamated source code which does contain your change. You can now
-load every subsequent change by reloading xonsh, and if your code changes don't
-seem to have any effect, make sure you check ``$XONSH_DEBUG`` first!
-
 
 Changelog
 =========
@@ -137,28 +126,10 @@ To add this as a git pre-commit hook::
 *******
 Imports
 *******
-Xonsh source code may be amalgamated into a single file (``__amalgam__.py``)
-to speed up imports. The way the code amalgamater works is that other modules
-that are in the same package (and amalgamated) should be imported with::
 
-    from pkg.x import a, c, d
+``xonsh`` imports should be sorted alphabetically, and by module location.  You
-
+can (and should) use ``isort`` either from the command line or use the
-This is because the amalgamater puts all such modules in the same globals(),
+``pre-commit`` hook.
-which is effectively what the from-imports do. For example, ``xonsh.ast`` and
-``xonsh.execer`` are both in the same package (``xonsh``). Thus they should use
-the above from from-import syntax.
-
-Alternatively, for modules outside of the current package (or modules that are
-not amalgamated) the import statement should be either ``import pkg.x`` or
-``import pkg.x as name``. This is because these are the only cases where the
-amalgamater is able to automatically insert lazy imports in way that is guaranteed
-to be safe. This is due to the ambiguity that ``from pkg.x import name`` may
-import a variable that cannot be lazily constructed or may import a module.
-So the simple rules to follow are that:
-
-1. Import objects from modules in the same package directly in using from-import,
-2. Import objects from modules outside of the package via a direct import
-   or import-as statement.
 
 How to Test
 ===========

MANIFEST.in

@@ -7,4 +7,3 @@ include run-tests.xsh
 recursive-include tests *
 exclude tests/test_news.py
 global-exclude *.pyc
-include amalgamate.py

Makefile

@@ -1,27 +0,0 @@
-# Make GNU Make xonshy
-SHELL=xonsh
-.SHELLFLAGS=-c
-.ONESHELL:
-.SILENT:
-
-# Unlike normal makefiles: executes the entire body in one go under xonsh, and doesn't echo
-
-.PHONY: help
-help:
-	print("""
-	Utility file for xonsh project. Try these targets:
-	* amalgamate: Generate __amalgam__.py files
-	* clean: Remove generated files (namely, the amalgamations)
-	""")
-
-.PHONY: clean
-clean:
-	find xonsh -name __amalgam__.py -delete -print
-
-.PHONY: amalgamate
-amalgamate:
-	import sys
-	sys.path.insert(0, '.')
-	import setup
-	setup.amalgamate_source()
-	_ = sys.path.pop(0)

amalgamate.py

@@ -1,565 +0,0 @@
-#!/usr/bin/env python3
-"""A package-based, source code amalgamater."""
-import os
-import sys
-import pprint
-from itertools import repeat
-from collections import namedtuple
-from collections.abc import Mapping
-from ast import parse, walk, Import, ImportFrom
-
-__version__ = "0.1.2"
-
-ModNode = namedtuple("ModNode", ["name", "pkgdeps", "extdeps", "futures"])
-ModNode.__doc__ = """Module node for dependency graph.
-
-Attributes
-----------
-name : str
-    Module name.
-pkgdeps : frozenset of str
-    Module dependencies in the same package.
-extdeps : frozenset of str
-    External module dependencies from outside of the package.
-futures : frozenset of str
-    Import directive names antecedent to 'from __future__ import'
-"""
-
-
-class SourceCache(Mapping):
-    """Stores / loads source code for files based on package and module names."""
-
-    def __init__(self, *args, **kwargs):
-        self._d = dict(*args, **kwargs)
-
-    def __getitem__(self, key):
-        d = self._d
-        if key in d:
-            return d[key]
-        pkg, name = key
-        pkgdir = pkg.replace(".", os.sep)
-        fname = pkgdir + os.sep + name + ".py"
-        with open(fname, encoding="utf-8", errors="surrogateescape") as f:
-            raw = f.read()
-        d[key] = raw
-        return raw
-
-    def __iter__(self):
-        yield from self._d
-
-    def __len__(self):
-        return len(self._d)
-
-
-SOURCES = SourceCache()
-
-
-class GlobalNames:
-    """Stores globally defined names that have been seen on ast nodes."""
-
-    impnodes = frozenset(["import", "importfrom"])
-
-    def __init__(self, pkg="<pkg>"):
-        self.cache = {}
-        self.pkg = pkg
-        self.module = "<mod>"
-        self.topnode = None
-
-    def warn_duplicates(self):
-        s = ""
-        for key in sorted(self.cache.keys()):
-            val = self.cache[key]
-            if len(val) < 2:
-                continue
-            val = sorted(val)
-            if all([val[0][0] == x[0] for x in val[1:]]):
-                continue
-            s += f"WARNING: {key!r} defined in multiple locations:\n"
-            for loc in val:
-                s += "  {}:{} ({})\n".format(*loc)
-        if len(s) > 0:
-            print(s, end="", flush=True, file=sys.stderr)
-
-    def entry(self, name, lineno):
-        if name.startswith("__"):
-            return
-        topnode = self.topnode
-        e = (self.pkg + "." + self.module, lineno, topnode)
-        if name in self.cache:
-            if topnode in self.impnodes and all(
-                [topnode == x[2] for x in self.cache[name]]
-            ):
-                return
-            self.cache[name].add(e)
-        else:
-            self.cache[name] = {e}
-
-    def add(self, node, istopnode=False):
-        """Adds the names from the node to the cache."""
-        nodename = node.__class__.__name__.lower()
-        if istopnode:
-            self.topnode = nodename
-        meth = getattr(self, "_add_" + nodename, None)
-        if meth is not None:
-            meth(node)
-
-    def _add_name(self, node):
-        self.entry(node.id, node.lineno)
-
-    def _add_tuple(self, node):
-        for x in node.elts:
-            self.add(x)
-
-    def _add_assign(self, node):
-        for target in node.targets:
-            self.add(target)
-
-    def _add_functiondef(self, node):
-        self.entry(node.name, node.lineno)
-
-    def _add_classdef(self, node):
-        self.entry(node.name, node.lineno)
-
-    def _add_import(self, node):
-        lineno = node.lineno
-        for target in node.names:
-            if target.asname is None:
-                name, _, _ = target.name.partition(".")
-            else:
-                name = target.asname
-            self.entry(name, lineno)
-
-    def _add_importfrom(self, node):
-        pkg, _ = resolve_package_module(node.module, self.pkg, node.level)
-        if pkg == self.pkg:
-            return
-        lineno = node.lineno
-        for target in node.names:
-            if target.asname is None:
-                name = target.name
-            else:
-                name = target.asname
-            self.entry(name, lineno)
-
-    def _add_with(self, node):
-        for item in node.items:
-            if item.optional_vars is None:
-                continue
-            self.add(item.optional_vars)
-        for child in node.body:
-            self.add(child, istopnode=True)
-
-    def _add_for(self, node):
-        self.add(node.target)
-        for child in node.body:
-            self.add(child, istopnode=True)
-
-    def _add_while(self, node):
-        for child in node.body:
-            self.add(child, istopnode=True)
-
-    def _add_if(self, node):
-        for child in node.body:
-            self.add(child, istopnode=True)
-        for child in node.orelse:
-            self.add(child, istopnode=True)
-
-    def _add_try(self, node):
-        for child in node.body:
-            self.add(child, istopnode=True)
-
-
-def module_is_package(module, pkg, level):
-    """Returns whether or not the module name refers to the package."""
-    if level == 0:
-        return module == pkg
-    elif level == 1:
-        return module is None
-    else:
-        return False
-
-
-def module_from_package(module, pkg, level):
-    """Returns whether or not a module is from the package."""
-    if level == 0:
-        return module.startswith(pkg + ".")
-    elif level == 1:
-        return True
-    else:
-        return False
-
-
-def resolve_package_module(module, pkg, level, default=None):
-    """Returns a 2-tuple of package and module name, even for relative
-    imports
-    """
-    if level == 0:
-        p, _, m = module.rpartition(".")
-    elif level == 1:
-        p = pkg
-        m = module or default
-    else:
-        p = m = None
-    return p, m
-
-
-def make_node(name, pkg, allowed, glbnames):
-    """Makes a node by parsing a file and traversing its AST."""
-    raw = SOURCES[pkg, name]
-    tree = parse(raw, filename=name)
-    # we only want to deal with global import statements
-    pkgdeps = set()
-    extdeps = set()
-    futures = set()
-    glbnames.module = name
-    for a in tree.body:
-        glbnames.add(a, istopnode=True)
-        if isinstance(a, Import):
-            for n in a.names:
-                p, dot, m = n.name.rpartition(".")
-                if p == pkg and m in allowed:
-                    pkgdeps.add(m)
-                else:
-                    extdeps.add(n.name)
-        elif isinstance(a, ImportFrom):
-            if module_is_package(a.module, pkg, a.level):
-                pkgdeps.update(n.name for n in a.names if n.name in allowed)
-            elif module_from_package(a.module, pkg, a.level):
-                p, m = resolve_package_module(
-                    a.module, pkg, a.level, default=a.names[0].name
-                )
-                if p == pkg and m in allowed:
-                    pkgdeps.add(m)
-                else:
-                    extdeps.add(a.module)
-            elif a.module == "__future__":
-                futures.update(n.name for n in a.names)
-    return ModNode(name, frozenset(pkgdeps), frozenset(extdeps), frozenset(futures))
-
-
-def make_graph(pkg, exclude=None):
-    """Create a graph (dict) of module dependencies."""
-    graph = {}
-    pkgdir = pkg.replace(".", os.sep)
-    allowed = set()
-    files = os.listdir(pkgdir)
-    for fname in files:
-        base, ext = os.path.splitext(fname)
-        if base.startswith("__") or ext != ".py":
-            continue
-        allowed.add(base)
-    if exclude:
-        allowed -= exclude
-    glbnames = GlobalNames(pkg=pkg)
-    for base in allowed:
-        graph[base] = make_node(base, pkg, allowed, glbnames)
-    glbnames.warn_duplicates()
-    return graph
-
-
-def depsort(graph):
-    """Sort modules by dependency."""
-    remaining = set(graph.keys())
-    seder = []
-    solved = set()
-    while 0 < len(remaining):
-        nodeps = {m for m in remaining if len(graph[m].pkgdeps - solved) == 0}
-        if len(nodeps) == 0:
-            msg = (
-                "\nsolved order = {}\nremaining = {}\nCycle detected in "
-                "module graph!"
-            ).format(pprint.pformat(seder), pprint.pformat(remaining))
-            raise RuntimeError(msg)
-        solved |= nodeps
-        remaining -= nodeps
-        seder += sorted(nodeps)
-    return seder
-
-
-LAZY_IMPORTS = """
-from sys import modules as _modules
-from types import ModuleType as _ModuleType
-from importlib import import_module as _import_module
-
-
-class _LazyModule(_ModuleType):
-
-    def __init__(self, pkg, mod, asname=None):
-        '''Lazy module 'pkg.mod' in package 'pkg'.'''
-        self.__dct__ = {
-            'loaded': False,
-            'pkg': pkg,  # pkg
-            'mod': mod,  # pkg.mod
-            'asname': asname,  # alias
-            }
-
-    @classmethod
-    def load(cls, pkg, mod, asname=None):
-        if mod in _modules:
-            key = pkg if asname is None else mod
-            return _modules[key]
-        else:
-            return cls(pkg, mod, asname)
-
-    def __getattribute__(self, name):
-        if name == '__dct__':
-            return super(_LazyModule, self).__getattribute__(name)
-        dct = self.__dct__
-        mod = dct['mod']
-        if dct['loaded']:
-            m = _modules[mod]
-        else:
-            m = _import_module(mod)
-            glbs = globals()
-            pkg = dct['pkg']
-            asname = dct['asname']
-            if asname is None:
-                glbs[pkg] = m = _modules[pkg]
-            else:
-                glbs[asname] = m
-            dct['loaded'] = True
-        return getattr(m, name)
-
-"""
-
-
-def get_lineno(node, default=0):
-    """Gets the lineno of a node or returns the default."""
-    return getattr(node, "lineno", default)
-
-
-def min_line(node):
-    """Computes the minimum lineno."""
-    node_line = get_lineno(node)
-    return min(map(get_lineno, walk(node), repeat(node_line)))
-
-
-def format_import(names):
-    """Format an import line"""
-    parts = []
-    for _, name, asname in names:
-        if asname is None:
-            parts.append(name)
-        else:
-            parts.append(name + " as " + asname)
-    line = "import " + ", ".join(parts) + "\n"
-    return line
-
-
-def format_lazy_import(names):
-    """Formats lazy import lines"""
-    lines = ""
-    for _, name, asname in names:
-        pkg, _, _ = name.partition(".")
-        if asname is None:
-            line = "{pkg} = _LazyModule.load({pkg!r}, {mod!r})\n"
-        else:
-            line = "{asname} = _LazyModule.load({pkg!r}, {mod!r}, {asname!r})\n"
-        lines += line.format(pkg=pkg, mod=name, asname=asname)
-    return lines
-
-
-def format_from_import(names):
-    """Format a from import line"""
-    parts = []
-    for _, module, name, asname in names:  # noqa
-        if asname is None:
-            parts.append(name)
-        else:
-            parts.append(name + " as " + asname)
-    line = "from " + module
-    line += " import " + ", ".join(parts) + "\n"
-    return line
-
-
-def rewrite_imports(name, pkg, order, imps):
-    """Rewrite the global imports in the file given the amalgamation."""
-    raw = SOURCES[pkg, name]
-    tree = parse(raw, filename=name)
-    replacements = []  # list of (startline, stopline, str) tuples
-    # collect replacements in forward direction
-    for a, b in zip(tree.body, tree.body[1:] + [None]):
-        if not isinstance(a, (Import, ImportFrom)):
-            continue
-        start = min_line(a) - 1
-        stop = len(tree.body) if b is None else min_line(b) - 1
-        if isinstance(a, Import):
-            keep = []
-            for n in a.names:
-                p, dot, m = n.name.rpartition(".")
-                if p == pkg and m in order:
-                    msg = (
-                        "Cannot amalgamate import of amalgamated module:"
-                        "\n\n  import {0}.{1}\n\nin {0}/{2}.py"
-                    ).format(pkg, n.name, name)
-                    raise RuntimeError(msg)
-                imp = (Import, n.name, n.asname)
-                if imp not in imps:
-                    imps.add(imp)
-                    keep.append(imp)
-            if len(keep) == 0:
-                s = ", ".join(n.name for n in a.names)
-                s = "# amalgamated " + s + "\n"
-            else:
-                s = format_lazy_import(keep)
-            replacements.append((start, stop, s))
-        elif isinstance(a, ImportFrom):
-            p, m = resolve_package_module(a.module, pkg, a.level, default="")
-            if module_is_package(a.module, pkg, a.level):
-                for n in a.names:
-                    if n.name in order:
-                        msg = (
-                            "Cannot amalgamate import of "
-                            "amalgamated module:\n\n  from {0} import {1}\n"
-                            "\nin {0}/{2}.py"
-                        ).format(pkg, n.name, name)
-                        raise RuntimeError(msg)
-            elif p == pkg and m in order:
-                replacements.append(
-                    (start, stop, "# amalgamated " + p + "." + m + "\n")
-                )
-            elif a.module == "__future__":
-                replacements.append(
-                    (start, stop, "# amalgamated __future__ directive\n")
-                )
-            else:
-                keep = []
-                for n in a.names:
-                    imp = (ImportFrom, a.module, n.name, n.asname)
-                    if imp not in imps:
-                        imps.add(imp)
-                        keep.append(imp)
-                if len(keep) == len(a.names):
-                    continue  # all new imports
-                elif len(keep) == 0:
-                    s = ", ".join(n.name for n in a.names)
-                    s = "# amalgamated from " + a.module + " import " + s + "\n"
-                else:
-                    s = format_from_import(keep)
-                replacements.append((start, stop, s))
-    # apply replacements in reverse
-    lines = raw.splitlines(keepends=True)
-    for start, stop, s in replacements[::-1]:
-        lines[start] = s
-        for _ in range(stop - start - 1):
-            del lines[start + 1]
-    return "".join(lines)
-
-
-def sorted_futures(graph):
-    """Returns a sorted, unique list of future imports."""
-    f = set()
-    for value in graph.values():
-        f |= value.futures
-    return sorted(f)
-
-
-def amalgamate(order, graph, pkg):
-    """Create amalgamated source."""
-    src = (
-        '"""Amalgamation of {} package, made up of the following '
-        "modules, in order:\n\n* "
-    ).format(pkg)
-    src += "\n* ".join(order)
-    src += '\n\n"""\n'
-    futures = sorted_futures(graph)
-    if len(futures) > 0:
-        src += "from __future__ import " + ", ".join(futures) + "\n"
-    src += LAZY_IMPORTS
-    imps = set()
-    for name in order:
-        lines = rewrite_imports(name, pkg, order, imps)
-        src += "#\n# " + name + "\n#\n" + lines + "\n"
-    return src
-
-
-def write_amalgam(src, pkg):
-    """Write out __amalgam__.py file"""
-    pkgdir = pkg.replace(".", os.sep)
-    fname = os.path.join(pkgdir, "__amalgam__.py")
-    with open(fname, "w", encoding="utf-8", errors="surrogateescape") as f:
-        f.write(src)
-
-
-def _init_name_lines(pkg):
-    pkgdir = pkg.replace(".", os.sep)
-    fname = os.path.join(pkgdir, "__init__.py")
-    with open(fname, encoding="utf-8", errors="surrogateescape") as f:
-        raw = f.read()
-    lines = raw.splitlines()
-    return fname, lines
-
-
-def read_exclude(pkg):
-    """reads in modules to exclude from __init__.py"""
-    _, lines = _init_name_lines(pkg)
-    exclude = set()
-    for line in lines:
-        if line.startswith("# amalgamate exclude"):
-            exclude.update(line.split()[3:])
-    return exclude
-
-
-FAKE_LOAD = """
-import os as _os
-
-if _os.getenv("{debug}", ""):
-    pass
-else:
-    import sys as _sys
-
-    try:
-        from {pkg} import __amalgam__
-
-        {load}
-        del __amalgam__
-    except ImportError:
-        pass
-    del _sys
-del _os
-""".strip()
-
-
-def rewrite_init(pkg, order, debug="DEBUG"):
-    """Rewrites the init file to insert modules."""
-    fname, lines = _init_name_lines(pkg)
-    start, stop = -1, -1
-    for i, line in enumerate(lines):
-        if line.startswith("# amalgamate end"):
-            stop = i
-        elif line.startswith("# amalgamate"):
-            start = i
-    t = "{1} = __amalgam__\n        " '_sys.modules["{0}.{1}"] = __amalgam__'
-    load = "\n        ".join(t.format(pkg, m) for m in order)
-    s = FAKE_LOAD.format(pkg=pkg, load=load, debug=debug)
-    if start + 1 == stop:
-        lines.insert(stop, s)
-    else:
-        lines[start + 1] = s
-        lines = lines[: start + 2] + lines[stop:]
-    init = "\n".join(lines) + "\n"
-    with open(fname, "w", encoding="utf-8", errors="surrogateescape") as f:
-        f.write(init)
-
-
-def main(args=None):
-    if args is None:
-        args = sys.argv
-    debug = "DEBUG"
-    for pkg in args[1:]:
-        if pkg.startswith("--debug="):
-            debug = pkg[8:]
-            continue
-        print("Amalgamating " + pkg)
-        exclude = read_exclude(pkg)
-        print(f"  excluding {pprint.pformat(exclude or None)}")
-        graph = make_graph(pkg, exclude=exclude)
-        order = depsort(graph)
-        src = amalgamate(order, graph, pkg)
-        write_amalgam(src, pkg)
-        rewrite_init(pkg, order, debug=debug)
-        print(f"  collapsed {len(order)} modules")
-
-
-if __name__ == "__main__":
-    main()

news/bye_bye_amalgamate.rst

@@ -0,0 +1,25 @@
+**Added:**
+
+* <news item>
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* The ``xonsh`` code-base is no longer amalgamated, so tracebacks should be
+  human-readable without intervention.  This may have (minor) impacts on startup
+  speed.
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

pyproject.toml

@@ -113,7 +113,6 @@ test = [
     "coverage>=5.3.1",
     "pyte>=0.8.0",
     "virtualenv>=20.7.2",
-    "amalgamate",
 ]
 dev = [
     "xonsh[test,doc]",
@@ -179,11 +178,10 @@ extend_exclude = '''
 ((xonsh/parser_table.py)|(xonsh/completion_parser_table.py))
 '''
 
-force_exclude = '''.*/__amalgam__.py'''
 
 [tool.isort]
 profile = "black"
-extend_skip_glob = ["xonsh/*_table.py", "xonsh/ply/**.py", "*/__amalgam__.py", "**/__amalgam__.py"]
+extend_skip_glob = ["xonsh/*_table.py", "xonsh/ply/**.py"]
 src_paths = ["xonsh", "xontrib", "xompletions", "tests"]
 known_first_party = ["xonsh", "xontrib", "xompletions", "tests"]
 known_third_party = ["ply", "pytest"]

run-tests.xsh

@@ -10,7 +10,6 @@ import itertools
 
 
 $RAISE_SUBPROC_ERROR = True
-# $XONSH_NO_AMALGAMATE = 1
 # $XONSH_TRACE_SUBPROC = True
 
 
@@ -28,7 +27,6 @@ def _replace_args(args: List[str], num: int) -> List[str]:
 
 def test(
         report_cov: xcli.Arg('--report-coverage', '-c', action="store_true") = False,
-        no_amalgam: xcli.Arg('--no-amalgam', '-n', action="store_true") = False,
         pytest_args: xcli.Arg(nargs='*')=(),
 ):
     """Run pytest.
@@ -40,27 +38,19 @@ def test(
     pytest_args
         arbitrary arguments that gets passed to pytest's invocation.
         Use %%d to parameterize and prevent overwrite
-    no_amalgam
-        Disable amalgamation check
 
     Examples
     --------
     `xonsh run-tests.xsh -- --junitxml=junit/test-results.%%d.xml`
     """
 
-    if (not no_amalgam) and not $(xonsh -c "import xonsh.main; print(xonsh.main.__file__, end='')").endswith("__amalgam__.py"):
-        echo "Tests need to run from the amalgamated xonsh! install with `pip install .` (without `-e`)"
-        exit(1)
-
     if report_cov:
-        $XONSH_NO_AMALGAMATE = True
         ![pytest @(_replace_args(pytest_args, 0)) --cov --cov-report=xml --cov-report=term]
     else:
         # during CI run, some tests take longer to complete on windows
         ![pytest @(_replace_args(pytest_args, 0)) --durations=5]
 
 
-
 def validate_news_items(
         pytest_args: xcli.Arg(nargs='*') = (),
 ):

setup.cfg

@@ -5,9 +5,6 @@
 [flake8]
 max-line-length = 180
 exclude =
-    __amalgam__.py,
-    **/__amalgam__.py,
-    **/*/__amalgam__.py,
     docs/,
     */ply/,
     parser*_table.py,
@@ -102,7 +99,7 @@ warn_unused_configs = True
 warn_no_return = False
 
 ; a regex to exclude certain directories
-exclude = ((xonsh/ply)|(__amalgam__.py)|(xontrib/(mpl.*py|distributed.py|jedi.py)))
+exclude = ((xonsh/ply)|(xontrib/(mpl.*py|distributed.py|jedi.py)))
 
 ;match dmypy semantics - https://github.com/python/mypy/issues/8046
 local_partial_types = True
@@ -115,11 +112,11 @@ pretty = True
 
 # the __init__ files have dynamic check - ignoring the attribute error. others are generated files
 # top level package name only ignores the __init__.py file.
-[mypy-xonsh.parser_table,xonsh.completion_parser_table,xonsh.parsers.parser_table.*,xonsh.parsers.completion_parser_table.*,*.__amalgam__.*,xonsh,xonsh.prompt,xonsh.history,xonsh.completers,xonsh.procs]
+[mypy-xonsh.parser_table,xonsh.completion_parser_table,xonsh.parsers.parser_table.*,xonsh.parsers.completion_parser_table.*,xonsh,xonsh.prompt,xonsh.history,xonsh.completers,xonsh.procs]
 ignore_errors = True
 
 # 3rd party libraries that we dont have control over
-[mypy-zmq.*,setproctitle,xonsh.ply.*,winreg.*,pygments.*,importlib_resources.*,nt.*,prompt_toolkit.*,distro.*,conda_suggest.*,_winreg.*,*.__amalgam__.*]
+[mypy-zmq.*,setproctitle,xonsh.ply.*,winreg.*,pygments.*,importlib_resources.*,nt.*,prompt_toolkit.*,distro.*,conda_suggest.*,_winreg.*]
 ignore_missing_imports = True
 ignore_errors = True
 

setup.py

@@ -17,11 +17,6 @@ TABLES = [
     "xonsh/lexer_table.py",
     "xonsh/parser_table.py",
     "xonsh/completion_parser_table.py",
-    "xonsh/__amalgam__.py",
-    "xonsh/completers/__amalgam__.py",
-    "xonsh/history/__amalgam__.py",
-    "xonsh/prompt/__amalgam__.py",
-    "xonsh/procs/__amalgam__.py",
 ]
 
 
@@ -41,28 +36,6 @@ def clean_tables():
 os.environ["XONSH_DEBUG"] = "1"
 
 
-def amalgamate_source():
-    """Amalgamates source files."""
-    sys.path.insert(0, os.path.dirname(__file__))
-    try:
-        import amalgamate
-    except ImportError:
-        print("Could not import amalgamate, skipping.", file=sys.stderr)
-        return
-    amalgamate.main(
-        [
-            "amalgamate",
-            "--debug=XONSH_NO_AMALGAMATE",
-            "xonsh",
-            "xonsh.completers",
-            "xonsh.history",
-            "xonsh.prompt",
-            "xonsh.procs",
-        ]
-    )
-    sys.path.pop(0)
-
-
 def build_tables():
     """Build the lexer/parser modules."""
     print("Building lexer and parser tables.", file=sys.stderr)
@@ -156,7 +129,6 @@ class xbuild_py(build_py):
     def run(self):
         clean_tables()
         build_tables()
-        amalgamate_source()
         # add dirty version number
         dirty = dirty_version()
         super().run()
@@ -180,7 +152,6 @@ class xinstall(install):
     def run(self):
         clean_tables()
         build_tables()
-        amalgamate_source()
         # add dirty version number
         dirty = dirty_version()
 
@@ -195,7 +166,6 @@ class xsdist(sdist):
     def make_release_tree(self, basedir, files):
         clean_tables()
         build_tables()
-        amalgamate_source()
         dirty = dirty_version()
         files.extend(TABLES)
         super().make_release_tree(basedir, files)

xonsh/__init__.py

@@ -1,100 +1 @@
 __version__ = "0.12.6"
-
-
-# amalgamate exclude jupyter_kernel parser_table completion_parser_table parser_test_table pyghooks
-# amalgamate exclude winutils wizard pytest_plugin fs macutils pygments_cache
-# amalgamate exclude jupyter_shell proc built_ins
-import os as _os
-
-if _os.getenv("XONSH_NO_AMALGAMATE", ""):
-    pass
-else:
-    import sys as _sys
-
-    try:
-        from xonsh import __amalgam__
-
-        cli_utils = __amalgam__
-        _sys.modules["xonsh.cli_utils"] = __amalgam__
-        contexts = __amalgam__
-        _sys.modules["xonsh.contexts"] = __amalgam__
-        lazyasd = __amalgam__
-        _sys.modules["xonsh.lazyasd"] = __amalgam__
-        lazyjson = __amalgam__
-        _sys.modules["xonsh.lazyjson"] = __amalgam__
-        platform = __amalgam__
-        _sys.modules["xonsh.platform"] = __amalgam__
-        pretty = __amalgam__
-        _sys.modules["xonsh.pretty"] = __amalgam__
-        codecache = __amalgam__
-        _sys.modules["xonsh.codecache"] = __amalgam__
-        lazyimps = __amalgam__
-        _sys.modules["xonsh.lazyimps"] = __amalgam__
-        parser = __amalgam__
-        _sys.modules["xonsh.parser"] = __amalgam__
-        tokenize = __amalgam__
-        _sys.modules["xonsh.tokenize"] = __amalgam__
-        tools = __amalgam__
-        _sys.modules["xonsh.tools"] = __amalgam__
-        ast = __amalgam__
-        _sys.modules["xonsh.ast"] = __amalgam__
-        color_tools = __amalgam__
-        _sys.modules["xonsh.color_tools"] = __amalgam__
-        commands_cache = __amalgam__
-        _sys.modules["xonsh.commands_cache"] = __amalgam__
-        completer = __amalgam__
-        _sys.modules["xonsh.completer"] = __amalgam__
-        events = __amalgam__
-        _sys.modules["xonsh.events"] = __amalgam__
-        foreign_shells = __amalgam__
-        _sys.modules["xonsh.foreign_shells"] = __amalgam__
-        jobs = __amalgam__
-        _sys.modules["xonsh.jobs"] = __amalgam__
-        jsonutils = __amalgam__
-        _sys.modules["xonsh.jsonutils"] = __amalgam__
-        lexer = __amalgam__
-        _sys.modules["xonsh.lexer"] = __amalgam__
-        openpy = __amalgam__
-        _sys.modules["xonsh.openpy"] = __amalgam__
-        xontribs = __amalgam__
-        _sys.modules["xonsh.xontribs"] = __amalgam__
-        ansi_colors = __amalgam__
-        _sys.modules["xonsh.ansi_colors"] = __amalgam__
-        diff_history = __amalgam__
-        _sys.modules["xonsh.diff_history"] = __amalgam__
-        dirstack = __amalgam__
-        _sys.modules["xonsh.dirstack"] = __amalgam__
-        execer = __amalgam__
-        _sys.modules["xonsh.execer"] = __amalgam__
-        shell = __amalgam__
-        _sys.modules["xonsh.shell"] = __amalgam__
-        style_tools = __amalgam__
-        _sys.modules["xonsh.style_tools"] = __amalgam__
-        timings = __amalgam__
-        _sys.modules["xonsh.timings"] = __amalgam__
-        xonfig = __amalgam__
-        _sys.modules["xonsh.xonfig"] = __amalgam__
-        base_shell = __amalgam__
-        _sys.modules["xonsh.base_shell"] = __amalgam__
-        environ = __amalgam__
-        _sys.modules["xonsh.environ"] = __amalgam__
-        imphooks = __amalgam__
-        _sys.modules["xonsh.imphooks"] = __amalgam__
-        inspectors = __amalgam__
-        _sys.modules["xonsh.inspectors"] = __amalgam__
-        aliases = __amalgam__
-        _sys.modules["xonsh.aliases"] = __amalgam__
-        main = __amalgam__
-        _sys.modules["xonsh.main"] = __amalgam__
-        readline_shell = __amalgam__
-        _sys.modules["xonsh.readline_shell"] = __amalgam__
-        tracer = __amalgam__
-        _sys.modules["xonsh.tracer"] = __amalgam__
-        dumb_shell = __amalgam__
-        _sys.modules["xonsh.dumb_shell"] = __amalgam__
-        del __amalgam__
-    except ImportError:
-        pass
-    del _sys
-del _os
-# amalgamate end

xonsh/completers/__init__.py

@@ -1,45 +0,0 @@
-# amalgamate exclude
-import os as _os
-
-if _os.getenv("XONSH_NO_AMALGAMATE", ""):
-    pass
-else:
-    import sys as _sys
-
-    try:
-        from xonsh.completers import __amalgam__
-
-        bash_completion = __amalgam__
-        _sys.modules["xonsh.completers.bash_completion"] = __amalgam__
-        dirs = __amalgam__
-        _sys.modules["xonsh.completers.dirs"] = __amalgam__
-        tools = __amalgam__
-        _sys.modules["xonsh.completers.tools"] = __amalgam__
-        bash = __amalgam__
-        _sys.modules["xonsh.completers.bash"] = __amalgam__
-        commands = __amalgam__
-        _sys.modules["xonsh.completers.commands"] = __amalgam__
-        completer = __amalgam__
-        _sys.modules["xonsh.completers.completer"] = __amalgam__
-        environment = __amalgam__
-        _sys.modules["xonsh.completers.environment"] = __amalgam__
-        imports = __amalgam__
-        _sys.modules["xonsh.completers.imports"] = __amalgam__
-        man = __amalgam__
-        _sys.modules["xonsh.completers.man"] = __amalgam__
-        path = __amalgam__
-        _sys.modules["xonsh.completers.path"] = __amalgam__
-        python = __amalgam__
-        _sys.modules["xonsh.completers.python"] = __amalgam__
-        _aliases = __amalgam__
-        _sys.modules["xonsh.completers._aliases"] = __amalgam__
-        base = __amalgam__
-        _sys.modules["xonsh.completers.base"] = __amalgam__
-        init = __amalgam__
-        _sys.modules["xonsh.completers.init"] = __amalgam__
-        del __amalgam__
-    except ImportError:
-        pass
-    del _sys
-del _os
-# amalgamate end

xonsh/environ.py

@@ -1093,11 +1093,6 @@ The file should contain a function with the signature
         "presented, like PLY parsing messages.",
         is_configurable=False,
     )
-    XONSH_NO_AMALGAMATE = Var.with_default(
-        False,
-        "Setting this variable prior to starting xonsh to a truthy value will suppress amalgamated imports.",
-        is_configurable=False,
-    )
     XONSH_DATA_DIR = Var.with_default(
         xonsh_data_dir,
         "This is the location where xonsh data files are stored, such as history, generated completers ...",

xonsh/history/__init__.py

@@ -1,27 +0,0 @@
-# amalgamate exclude
-import os as _os
-
-if _os.getenv("XONSH_NO_AMALGAMATE", ""):
-    pass
-else:
-    import sys as _sys
-
-    try:
-        from xonsh.history import __amalgam__
-
-        base = __amalgam__
-        _sys.modules["xonsh.history.base"] = __amalgam__
-        dummy = __amalgam__
-        _sys.modules["xonsh.history.dummy"] = __amalgam__
-        json = __amalgam__
-        _sys.modules["xonsh.history.json"] = __amalgam__
-        sqlite = __amalgam__
-        _sys.modules["xonsh.history.sqlite"] = __amalgam__
-        main = __amalgam__
-        _sys.modules["xonsh.history.main"] = __amalgam__
-        del __amalgam__
-    except ImportError:
-        pass
-    del _sys
-del _os
-# amalgamate end

xonsh/procs/__init__.py

@@ -1,27 +0,0 @@
-# amalgamate exclude
-import os as _os
-
-if _os.getenv("XONSH_NO_AMALGAMATE", ""):
-    pass
-else:
-    import sys as _sys
-
-    try:
-        from xonsh.procs import __amalgam__
-
-        readers = __amalgam__
-        _sys.modules["xonsh.procs.readers"] = __amalgam__
-        pipelines = __amalgam__
-        _sys.modules["xonsh.procs.pipelines"] = __amalgam__
-        posix = __amalgam__
-        _sys.modules["xonsh.procs.posix"] = __amalgam__
-        proxies = __amalgam__
-        _sys.modules["xonsh.procs.proxies"] = __amalgam__
-        specs = __amalgam__
-        _sys.modules["xonsh.procs.specs"] = __amalgam__
-        del __amalgam__
-    except ImportError:
-        pass
-    del _sys
-del _os
-# amalgamate end

xonsh/prompt/__init__.py

@@ -1,29 +0,0 @@
-# amalgamate exclude gitstatus
-import os as _os
-
-if _os.getenv("XONSH_NO_AMALGAMATE", ""):
-    pass
-else:
-    import sys as _sys
-
-    try:
-        from xonsh.prompt import __amalgam__
-
-        base = __amalgam__
-        _sys.modules["xonsh.prompt.base"] = __amalgam__
-        cwd = __amalgam__
-        _sys.modules["xonsh.prompt.cwd"] = __amalgam__
-        env = __amalgam__
-        _sys.modules["xonsh.prompt.env"] = __amalgam__
-        job = __amalgam__
-        _sys.modules["xonsh.prompt.job"] = __amalgam__
-        times = __amalgam__
-        _sys.modules["xonsh.prompt.times"] = __amalgam__
-        vc = __amalgam__
-        _sys.modules["xonsh.prompt.vc"] = __amalgam__
-        del __amalgam__
-    except ImportError:
-        pass
-    del _sys
-del _os
-# amalgamate end

xonsh/winutils.py

@@ -39,8 +39,7 @@ from ctypes.wintypes import (
     WORD,
 )
 
-from xonsh import lazyimps  # we aren't amalgamated in this module.
+from xonsh import lazyimps, platform
-from xonsh import platform
 from xonsh.lazyasd import lazyobject
 
 __all__ = ("sudo",)

xonsh/xoreutils/__init__.py

@@ -1,2 +0,0 @@
-# amalgamate
-# amalgamate end