mirror of
https://github.com/xonsh/xonsh.git
synced 2025-03-04 00:14:41 +01:00
refactor(amalgamate): remove amalgamation (#4858)
Co-authored-by: Noorhteen Raja NJ <jnoortheen@gmail.com>
This commit is contained in:
Gil Forsyth
GitHub
parent
cbf23e60fb
commit
013fa760a0
20 changed files with 34 additions and 914 deletions
|
|
@ -5,7 +5,6 @@ source =
|
|||
xontrib/
|
||||
omit =
|
||||
.venv/*
|
||||
*/__amalgam__.py
|
||||
xonsh/lazyasd.py
|
||||
xonsh/parser_table.py
|
||||
xonsh/completion_parser_table.py
|
||||
|
|
@ -24,7 +23,6 @@ precision = 2
|
|||
exclude_lines =
|
||||
pragma: no cover
|
||||
raise NotImplementedError
|
||||
= __amalgam__
|
||||
skip_covered = true
|
||||
skip_empty = true
|
||||
show_missing = true
|
||||
|
|
|
|||
2
.github/workflows/test.yml
vendored
2
.github/workflows/test.yml
vendored
|
|
@ -64,7 +64,7 @@ jobs:
|
|||
if: ${{ startsWith(matrix.python-version, '3.10') }}
|
||||
run: |
|
||||
python -m pip install -e . --no-deps
|
||||
python -m xonsh run-tests.xsh test --report-coverage --no-amalgam -- --timeout=240
|
||||
python -m xonsh run-tests.xsh test --report-coverage -- --timeout=240
|
||||
- name: Upload coverage to Codecov
|
||||
if: ${{ startsWith(matrix.python-version, '3.10') }}
|
||||
uses: codecov/codecov-action@v3
|
||||
|
|
|
|||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -4,7 +4,6 @@
|
|||
*.egg
|
||||
.eggs/
|
||||
.pytest_cache/
|
||||
__amalgam__.py
|
||||
lexer_table.py
|
||||
parser_table.py
|
||||
parser_test_table.py
|
||||
|
|
|
|||
|
|
@ -33,17 +33,6 @@ Finally, run the following commands. You should see the effects of your change
|
|||
$ $XONSH_DEBUG=1
|
||||
$ xonsh
|
||||
|
||||
The xonsh build process collapses all Python source files into a single
|
||||
``__amalgam__.py`` file. When xonsh is started with a falsy value for
|
||||
`$XONSH_DEBUG <envvars.html>`_, it imports Python modules straight from
|
||||
``__amalgam__.py``, which decreases startup times by eliminating the cost of
|
||||
runtime imports. But setting ``$ $XONSH_DEBUG=1`` will suppress amalgamated
|
||||
imports. Reloading the xonsh shell (``$ xonsh``) won't simply import the stale
|
||||
``__amalgam__.py`` file that doesn't contain your new change, but will instead
|
||||
import the unamalgamated source code which does contain your change. You can now
|
||||
load every subsequent change by reloading xonsh, and if your code changes don't
|
||||
seem to have any effect, make sure you check ``$XONSH_DEBUG`` first!
|
||||
|
||||
|
||||
Changelog
|
||||
=========
|
||||
|
|
@ -137,28 +126,10 @@ To add this as a git pre-commit hook::
|
|||
*******
|
||||
Imports
|
||||
*******
|
||||
Xonsh source code may be amalgamated into a single file (``__amalgam__.py``)
|
||||
to speed up imports. The way the code amalgamater works is that other modules
|
||||
that are in the same package (and amalgamated) should be imported with::
|
||||
|
||||
from pkg.x import a, c, d
|
||||
|
||||
This is because the amalgamater puts all such modules in the same globals(),
|
||||
which is effectively what the from-imports do. For example, ``xonsh.ast`` and
|
||||
``xonsh.execer`` are both in the same package (``xonsh``). Thus they should use
|
||||
the above from from-import syntax.
|
||||
|
||||
Alternatively, for modules outside of the current package (or modules that are
|
||||
not amalgamated) the import statement should be either ``import pkg.x`` or
|
||||
``import pkg.x as name``. This is because these are the only cases where the
|
||||
amalgamater is able to automatically insert lazy imports in way that is guaranteed
|
||||
to be safe. This is due to the ambiguity that ``from pkg.x import name`` may
|
||||
import a variable that cannot be lazily constructed or may import a module.
|
||||
So the simple rules to follow are that:
|
||||
|
||||
1. Import objects from modules in the same package directly in using from-import,
|
||||
2. Import objects from modules outside of the package via a direct import
|
||||
or import-as statement.
|
||||
``xonsh`` imports should be sorted alphabetically, and by module location. You
|
||||
can (and should) use ``isort`` either from the command line or use the
|
||||
``pre-commit`` hook.
|
||||
|
||||
How to Test
|
||||
===========
|
||||
|
|
|
|||
|
|
@ -7,4 +7,3 @@ include run-tests.xsh
|
|||
recursive-include tests *
|
||||
exclude tests/test_news.py
|
||||
global-exclude *.pyc
|
||||
include amalgamate.py
|
||||
|
|
|
|||
27
Makefile
27
Makefile
|
|
@ -1,27 +0,0 @@
|
|||
# Make GNU Make xonshy
|
||||
SHELL=xonsh
|
||||
.SHELLFLAGS=-c
|
||||
.ONESHELL:
|
||||
.SILENT:
|
||||
|
||||
# Unlike normal makefiles: executes the entire body in one go under xonsh, and doesn't echo
|
||||
|
||||
.PHONY: help
|
||||
help:
|
||||
print("""
|
||||
Utility file for xonsh project. Try these targets:
|
||||
* amalgamate: Generate __amalgam__.py files
|
||||
* clean: Remove generated files (namely, the amalgamations)
|
||||
""")
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
find xonsh -name __amalgam__.py -delete -print
|
||||
|
||||
.PHONY: amalgamate
|
||||
amalgamate:
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
import setup
|
||||
setup.amalgamate_source()
|
||||
_ = sys.path.pop(0)
|
||||
565
amalgamate.py
565
amalgamate.py
|
|
@ -1,565 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""A package-based, source code amalgamater."""
|
||||
import os
|
||||
import sys
|
||||
import pprint
|
||||
from itertools import repeat
|
||||
from collections import namedtuple
|
||||
from collections.abc import Mapping
|
||||
from ast import parse, walk, Import, ImportFrom
|
||||
|
||||
__version__ = "0.1.2"
|
||||
|
||||
ModNode = namedtuple("ModNode", ["name", "pkgdeps", "extdeps", "futures"])
|
||||
ModNode.__doc__ = """Module node for dependency graph.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
name : str
|
||||
Module name.
|
||||
pkgdeps : frozenset of str
|
||||
Module dependencies in the same package.
|
||||
extdeps : frozenset of str
|
||||
External module dependencies from outside of the package.
|
||||
futures : frozenset of str
|
||||
Import directive names antecedent to 'from __future__ import'
|
||||
"""
|
||||
|
||||
|
||||
class SourceCache(Mapping):
|
||||
"""Stores / loads source code for files based on package and module names."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._d = dict(*args, **kwargs)
|
||||
|
||||
def __getitem__(self, key):
|
||||
d = self._d
|
||||
if key in d:
|
||||
return d[key]
|
||||
pkg, name = key
|
||||
pkgdir = pkg.replace(".", os.sep)
|
||||
fname = pkgdir + os.sep + name + ".py"
|
||||
with open(fname, encoding="utf-8", errors="surrogateescape") as f:
|
||||
raw = f.read()
|
||||
d[key] = raw
|
||||
return raw
|
||||
|
||||
def __iter__(self):
|
||||
yield from self._d
|
||||
|
||||
def __len__(self):
|
||||
return len(self._d)
|
||||
|
||||
|
||||
SOURCES = SourceCache()
|
||||
|
||||
|
||||
class GlobalNames:
|
||||
"""Stores globally defined names that have been seen on ast nodes."""
|
||||
|
||||
impnodes = frozenset(["import", "importfrom"])
|
||||
|
||||
def __init__(self, pkg="<pkg>"):
|
||||
self.cache = {}
|
||||
self.pkg = pkg
|
||||
self.module = "<mod>"
|
||||
self.topnode = None
|
||||
|
||||
def warn_duplicates(self):
|
||||
s = ""
|
||||
for key in sorted(self.cache.keys()):
|
||||
val = self.cache[key]
|
||||
if len(val) < 2:
|
||||
continue
|
||||
val = sorted(val)
|
||||
if all([val[0][0] == x[0] for x in val[1:]]):
|
||||
continue
|
||||
s += f"WARNING: {key!r} defined in multiple locations:\n"
|
||||
for loc in val:
|
||||
s += " {}:{} ({})\n".format(*loc)
|
||||
if len(s) > 0:
|
||||
print(s, end="", flush=True, file=sys.stderr)
|
||||
|
||||
def entry(self, name, lineno):
|
||||
if name.startswith("__"):
|
||||
return
|
||||
topnode = self.topnode
|
||||
e = (self.pkg + "." + self.module, lineno, topnode)
|
||||
if name in self.cache:
|
||||
if topnode in self.impnodes and all(
|
||||
[topnode == x[2] for x in self.cache[name]]
|
||||
):
|
||||
return
|
||||
self.cache[name].add(e)
|
||||
else:
|
||||
self.cache[name] = {e}
|
||||
|
||||
def add(self, node, istopnode=False):
|
||||
"""Adds the names from the node to the cache."""
|
||||
nodename = node.__class__.__name__.lower()
|
||||
if istopnode:
|
||||
self.topnode = nodename
|
||||
meth = getattr(self, "_add_" + nodename, None)
|
||||
if meth is not None:
|
||||
meth(node)
|
||||
|
||||
def _add_name(self, node):
|
||||
self.entry(node.id, node.lineno)
|
||||
|
||||
def _add_tuple(self, node):
|
||||
for x in node.elts:
|
||||
self.add(x)
|
||||
|
||||
def _add_assign(self, node):
|
||||
for target in node.targets:
|
||||
self.add(target)
|
||||
|
||||
def _add_functiondef(self, node):
|
||||
self.entry(node.name, node.lineno)
|
||||
|
||||
def _add_classdef(self, node):
|
||||
self.entry(node.name, node.lineno)
|
||||
|
||||
def _add_import(self, node):
|
||||
lineno = node.lineno
|
||||
for target in node.names:
|
||||
if target.asname is None:
|
||||
name, _, _ = target.name.partition(".")
|
||||
else:
|
||||
name = target.asname
|
||||
self.entry(name, lineno)
|
||||
|
||||
def _add_importfrom(self, node):
|
||||
pkg, _ = resolve_package_module(node.module, self.pkg, node.level)
|
||||
if pkg == self.pkg:
|
||||
return
|
||||
lineno = node.lineno
|
||||
for target in node.names:
|
||||
if target.asname is None:
|
||||
name = target.name
|
||||
else:
|
||||
name = target.asname
|
||||
self.entry(name, lineno)
|
||||
|
||||
def _add_with(self, node):
|
||||
for item in node.items:
|
||||
if item.optional_vars is None:
|
||||
continue
|
||||
self.add(item.optional_vars)
|
||||
for child in node.body:
|
||||
self.add(child, istopnode=True)
|
||||
|
||||
def _add_for(self, node):
|
||||
self.add(node.target)
|
||||
for child in node.body:
|
||||
self.add(child, istopnode=True)
|
||||
|
||||
def _add_while(self, node):
|
||||
for child in node.body:
|
||||
self.add(child, istopnode=True)
|
||||
|
||||
def _add_if(self, node):
|
||||
for child in node.body:
|
||||
self.add(child, istopnode=True)
|
||||
for child in node.orelse:
|
||||
self.add(child, istopnode=True)
|
||||
|
||||
def _add_try(self, node):
|
||||
for child in node.body:
|
||||
self.add(child, istopnode=True)
|
||||
|
||||
|
||||
def module_is_package(module, pkg, level):
|
||||
"""Returns whether or not the module name refers to the package."""
|
||||
if level == 0:
|
||||
return module == pkg
|
||||
elif level == 1:
|
||||
return module is None
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def module_from_package(module, pkg, level):
|
||||
"""Returns whether or not a module is from the package."""
|
||||
if level == 0:
|
||||
return module.startswith(pkg + ".")
|
||||
elif level == 1:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def resolve_package_module(module, pkg, level, default=None):
|
||||
"""Returns a 2-tuple of package and module name, even for relative
|
||||
imports
|
||||
"""
|
||||
if level == 0:
|
||||
p, _, m = module.rpartition(".")
|
||||
elif level == 1:
|
||||
p = pkg
|
||||
m = module or default
|
||||
else:
|
||||
p = m = None
|
||||
return p, m
|
||||
|
||||
|
||||
def make_node(name, pkg, allowed, glbnames):
|
||||
"""Makes a node by parsing a file and traversing its AST."""
|
||||
raw = SOURCES[pkg, name]
|
||||
tree = parse(raw, filename=name)
|
||||
# we only want to deal with global import statements
|
||||
pkgdeps = set()
|
||||
extdeps = set()
|
||||
futures = set()
|
||||
glbnames.module = name
|
||||
for a in tree.body:
|
||||
glbnames.add(a, istopnode=True)
|
||||
if isinstance(a, Import):
|
||||
for n in a.names:
|
||||
p, dot, m = n.name.rpartition(".")
|
||||
if p == pkg and m in allowed:
|
||||
pkgdeps.add(m)
|
||||
else:
|
||||
extdeps.add(n.name)
|
||||
elif isinstance(a, ImportFrom):
|
||||
if module_is_package(a.module, pkg, a.level):
|
||||
pkgdeps.update(n.name for n in a.names if n.name in allowed)
|
||||
elif module_from_package(a.module, pkg, a.level):
|
||||
p, m = resolve_package_module(
|
||||
a.module, pkg, a.level, default=a.names[0].name
|
||||
)
|
||||
if p == pkg and m in allowed:
|
||||
pkgdeps.add(m)
|
||||
else:
|
||||
extdeps.add(a.module)
|
||||
elif a.module == "__future__":
|
||||
futures.update(n.name for n in a.names)
|
||||
return ModNode(name, frozenset(pkgdeps), frozenset(extdeps), frozenset(futures))
|
||||
|
||||
|
||||
def make_graph(pkg, exclude=None):
|
||||
"""Create a graph (dict) of module dependencies."""
|
||||
graph = {}
|
||||
pkgdir = pkg.replace(".", os.sep)
|
||||
allowed = set()
|
||||
files = os.listdir(pkgdir)
|
||||
for fname in files:
|
||||
base, ext = os.path.splitext(fname)
|
||||
if base.startswith("__") or ext != ".py":
|
||||
continue
|
||||
allowed.add(base)
|
||||
if exclude:
|
||||
allowed -= exclude
|
||||
glbnames = GlobalNames(pkg=pkg)
|
||||
for base in allowed:
|
||||
graph[base] = make_node(base, pkg, allowed, glbnames)
|
||||
glbnames.warn_duplicates()
|
||||
return graph
|
||||
|
||||
|
||||
def depsort(graph):
|
||||
"""Sort modules by dependency."""
|
||||
remaining = set(graph.keys())
|
||||
seder = []
|
||||
solved = set()
|
||||
while 0 < len(remaining):
|
||||
nodeps = {m for m in remaining if len(graph[m].pkgdeps - solved) == 0}
|
||||
if len(nodeps) == 0:
|
||||
msg = (
|
||||
"\nsolved order = {}\nremaining = {}\nCycle detected in "
|
||||
"module graph!"
|
||||
).format(pprint.pformat(seder), pprint.pformat(remaining))
|
||||
raise RuntimeError(msg)
|
||||
solved |= nodeps
|
||||
remaining -= nodeps
|
||||
seder += sorted(nodeps)
|
||||
return seder
|
||||
|
||||
|
||||
LAZY_IMPORTS = """
|
||||
from sys import modules as _modules
|
||||
from types import ModuleType as _ModuleType
|
||||
from importlib import import_module as _import_module
|
||||
|
||||
|
||||
class _LazyModule(_ModuleType):
|
||||
|
||||
def __init__(self, pkg, mod, asname=None):
|
||||
'''Lazy module 'pkg.mod' in package 'pkg'.'''
|
||||
self.__dct__ = {
|
||||
'loaded': False,
|
||||
'pkg': pkg, # pkg
|
||||
'mod': mod, # pkg.mod
|
||||
'asname': asname, # alias
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def load(cls, pkg, mod, asname=None):
|
||||
if mod in _modules:
|
||||
key = pkg if asname is None else mod
|
||||
return _modules[key]
|
||||
else:
|
||||
return cls(pkg, mod, asname)
|
||||
|
||||
def __getattribute__(self, name):
|
||||
if name == '__dct__':
|
||||
return super(_LazyModule, self).__getattribute__(name)
|
||||
dct = self.__dct__
|
||||
mod = dct['mod']
|
||||
if dct['loaded']:
|
||||
m = _modules[mod]
|
||||
else:
|
||||
m = _import_module(mod)
|
||||
glbs = globals()
|
||||
pkg = dct['pkg']
|
||||
asname = dct['asname']
|
||||
if asname is None:
|
||||
glbs[pkg] = m = _modules[pkg]
|
||||
else:
|
||||
glbs[asname] = m
|
||||
dct['loaded'] = True
|
||||
return getattr(m, name)
|
||||
|
||||
"""
|
||||
|
||||
|
||||
def get_lineno(node, default=0):
|
||||
"""Gets the lineno of a node or returns the default."""
|
||||
return getattr(node, "lineno", default)
|
||||
|
||||
|
||||
def min_line(node):
|
||||
"""Computes the minimum lineno."""
|
||||
node_line = get_lineno(node)
|
||||
return min(map(get_lineno, walk(node), repeat(node_line)))
|
||||
|
||||
|
||||
def format_import(names):
|
||||
"""Format an import line"""
|
||||
parts = []
|
||||
for _, name, asname in names:
|
||||
if asname is None:
|
||||
parts.append(name)
|
||||
else:
|
||||
parts.append(name + " as " + asname)
|
||||
line = "import " + ", ".join(parts) + "\n"
|
||||
return line
|
||||
|
||||
|
||||
def format_lazy_import(names):
|
||||
"""Formats lazy import lines"""
|
||||
lines = ""
|
||||
for _, name, asname in names:
|
||||
pkg, _, _ = name.partition(".")
|
||||
if asname is None:
|
||||
line = "{pkg} = _LazyModule.load({pkg!r}, {mod!r})\n"
|
||||
else:
|
||||
line = "{asname} = _LazyModule.load({pkg!r}, {mod!r}, {asname!r})\n"
|
||||
lines += line.format(pkg=pkg, mod=name, asname=asname)
|
||||
return lines
|
||||
|
||||
|
||||
def format_from_import(names):
|
||||
"""Format a from import line"""
|
||||
parts = []
|
||||
for _, module, name, asname in names: # noqa
|
||||
if asname is None:
|
||||
parts.append(name)
|
||||
else:
|
||||
parts.append(name + " as " + asname)
|
||||
line = "from " + module
|
||||
line += " import " + ", ".join(parts) + "\n"
|
||||
return line
|
||||
|
||||
|
||||
def rewrite_imports(name, pkg, order, imps):
|
||||
"""Rewrite the global imports in the file given the amalgamation."""
|
||||
raw = SOURCES[pkg, name]
|
||||
tree = parse(raw, filename=name)
|
||||
replacements = [] # list of (startline, stopline, str) tuples
|
||||
# collect replacements in forward direction
|
||||
for a, b in zip(tree.body, tree.body[1:] + [None]):
|
||||
if not isinstance(a, (Import, ImportFrom)):
|
||||
continue
|
||||
start = min_line(a) - 1
|
||||
stop = len(tree.body) if b is None else min_line(b) - 1
|
||||
if isinstance(a, Import):
|
||||
keep = []
|
||||
for n in a.names:
|
||||
p, dot, m = n.name.rpartition(".")
|
||||
if p == pkg and m in order:
|
||||
msg = (
|
||||
"Cannot amalgamate import of amalgamated module:"
|
||||
"\n\n import {0}.{1}\n\nin {0}/{2}.py"
|
||||
).format(pkg, n.name, name)
|
||||
raise RuntimeError(msg)
|
||||
imp = (Import, n.name, n.asname)
|
||||
if imp not in imps:
|
||||
imps.add(imp)
|
||||
keep.append(imp)
|
||||
if len(keep) == 0:
|
||||
s = ", ".join(n.name for n in a.names)
|
||||
s = "# amalgamated " + s + "\n"
|
||||
else:
|
||||
s = format_lazy_import(keep)
|
||||
replacements.append((start, stop, s))
|
||||
elif isinstance(a, ImportFrom):
|
||||
p, m = resolve_package_module(a.module, pkg, a.level, default="")
|
||||
if module_is_package(a.module, pkg, a.level):
|
||||
for n in a.names:
|
||||
if n.name in order:
|
||||
msg = (
|
||||
"Cannot amalgamate import of "
|
||||
"amalgamated module:\n\n from {0} import {1}\n"
|
||||
"\nin {0}/{2}.py"
|
||||
).format(pkg, n.name, name)
|
||||
raise RuntimeError(msg)
|
||||
elif p == pkg and m in order:
|
||||
replacements.append(
|
||||
(start, stop, "# amalgamated " + p + "." + m + "\n")
|
||||
)
|
||||
elif a.module == "__future__":
|
||||
replacements.append(
|
||||
(start, stop, "# amalgamated __future__ directive\n")
|
||||
)
|
||||
else:
|
||||
keep = []
|
||||
for n in a.names:
|
||||
imp = (ImportFrom, a.module, n.name, n.asname)
|
||||
if imp not in imps:
|
||||
imps.add(imp)
|
||||
keep.append(imp)
|
||||
if len(keep) == len(a.names):
|
||||
continue # all new imports
|
||||
elif len(keep) == 0:
|
||||
s = ", ".join(n.name for n in a.names)
|
||||
s = "# amalgamated from " + a.module + " import " + s + "\n"
|
||||
else:
|
||||
s = format_from_import(keep)
|
||||
replacements.append((start, stop, s))
|
||||
# apply replacements in reverse
|
||||
lines = raw.splitlines(keepends=True)
|
||||
for start, stop, s in replacements[::-1]:
|
||||
lines[start] = s
|
||||
for _ in range(stop - start - 1):
|
||||
del lines[start + 1]
|
||||
return "".join(lines)
|
||||
|
||||
|
||||
def sorted_futures(graph):
|
||||
"""Returns a sorted, unique list of future imports."""
|
||||
f = set()
|
||||
for value in graph.values():
|
||||
f |= value.futures
|
||||
return sorted(f)
|
||||
|
||||
|
||||
def amalgamate(order, graph, pkg):
|
||||
"""Create amalgamated source."""
|
||||
src = (
|
||||
'"""Amalgamation of {} package, made up of the following '
|
||||
"modules, in order:\n\n* "
|
||||
).format(pkg)
|
||||
src += "\n* ".join(order)
|
||||
src += '\n\n"""\n'
|
||||
futures = sorted_futures(graph)
|
||||
if len(futures) > 0:
|
||||
src += "from __future__ import " + ", ".join(futures) + "\n"
|
||||
src += LAZY_IMPORTS
|
||||
imps = set()
|
||||
for name in order:
|
||||
lines = rewrite_imports(name, pkg, order, imps)
|
||||
src += "#\n# " + name + "\n#\n" + lines + "\n"
|
||||
return src
|
||||
|
||||
|
||||
def write_amalgam(src, pkg):
|
||||
"""Write out __amalgam__.py file"""
|
||||
pkgdir = pkg.replace(".", os.sep)
|
||||
fname = os.path.join(pkgdir, "__amalgam__.py")
|
||||
with open(fname, "w", encoding="utf-8", errors="surrogateescape") as f:
|
||||
f.write(src)
|
||||
|
||||
|
||||
def _init_name_lines(pkg):
|
||||
pkgdir = pkg.replace(".", os.sep)
|
||||
fname = os.path.join(pkgdir, "__init__.py")
|
||||
with open(fname, encoding="utf-8", errors="surrogateescape") as f:
|
||||
raw = f.read()
|
||||
lines = raw.splitlines()
|
||||
return fname, lines
|
||||
|
||||
|
||||
def read_exclude(pkg):
|
||||
"""reads in modules to exclude from __init__.py"""
|
||||
_, lines = _init_name_lines(pkg)
|
||||
exclude = set()
|
||||
for line in lines:
|
||||
if line.startswith("# amalgamate exclude"):
|
||||
exclude.update(line.split()[3:])
|
||||
return exclude
|
||||
|
||||
|
||||
FAKE_LOAD = """
|
||||
import os as _os
|
||||
|
||||
if _os.getenv("{debug}", ""):
|
||||
pass
|
||||
else:
|
||||
import sys as _sys
|
||||
|
||||
try:
|
||||
from {pkg} import __amalgam__
|
||||
|
||||
{load}
|
||||
del __amalgam__
|
||||
except ImportError:
|
||||
pass
|
||||
del _sys
|
||||
del _os
|
||||
""".strip()
|
||||
|
||||
|
||||
def rewrite_init(pkg, order, debug="DEBUG"):
|
||||
"""Rewrites the init file to insert modules."""
|
||||
fname, lines = _init_name_lines(pkg)
|
||||
start, stop = -1, -1
|
||||
for i, line in enumerate(lines):
|
||||
if line.startswith("# amalgamate end"):
|
||||
stop = i
|
||||
elif line.startswith("# amalgamate"):
|
||||
start = i
|
||||
t = "{1} = __amalgam__\n " '_sys.modules["{0}.{1}"] = __amalgam__'
|
||||
load = "\n ".join(t.format(pkg, m) for m in order)
|
||||
s = FAKE_LOAD.format(pkg=pkg, load=load, debug=debug)
|
||||
if start + 1 == stop:
|
||||
lines.insert(stop, s)
|
||||
else:
|
||||
lines[start + 1] = s
|
||||
lines = lines[: start + 2] + lines[stop:]
|
||||
init = "\n".join(lines) + "\n"
|
||||
with open(fname, "w", encoding="utf-8", errors="surrogateescape") as f:
|
||||
f.write(init)
|
||||
|
||||
|
||||
def main(args=None):
|
||||
if args is None:
|
||||
args = sys.argv
|
||||
debug = "DEBUG"
|
||||
for pkg in args[1:]:
|
||||
if pkg.startswith("--debug="):
|
||||
debug = pkg[8:]
|
||||
continue
|
||||
print("Amalgamating " + pkg)
|
||||
exclude = read_exclude(pkg)
|
||||
print(f" excluding {pprint.pformat(exclude or None)}")
|
||||
graph = make_graph(pkg, exclude=exclude)
|
||||
order = depsort(graph)
|
||||
src = amalgamate(order, graph, pkg)
|
||||
write_amalgam(src, pkg)
|
||||
rewrite_init(pkg, order, debug=debug)
|
||||
print(f" collapsed {len(order)} modules")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
25
news/bye_bye_amalgamate.rst
Normal file
25
news/bye_bye_amalgamate.rst
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
**Added:**
|
||||
|
||||
* <news item>
|
||||
|
||||
**Changed:**
|
||||
|
||||
* <news item>
|
||||
|
||||
**Deprecated:**
|
||||
|
||||
* <news item>
|
||||
|
||||
**Removed:**
|
||||
|
||||
* The ``xonsh`` code-base is no longer amalgamated, so tracebacks should be
|
||||
human-readable without intervention. This may have (minor) impacts on startup
|
||||
speed.
|
||||
|
||||
**Fixed:**
|
||||
|
||||
* <news item>
|
||||
|
||||
**Security:**
|
||||
|
||||
* <news item>
|
||||
|
|
@ -113,7 +113,6 @@ test = [
|
|||
"coverage>=5.3.1",
|
||||
"pyte>=0.8.0",
|
||||
"virtualenv>=20.7.2",
|
||||
"amalgamate",
|
||||
]
|
||||
dev = [
|
||||
"xonsh[test,doc]",
|
||||
|
|
@ -179,11 +178,10 @@ extend_exclude = '''
|
|||
((xonsh/parser_table.py)|(xonsh/completion_parser_table.py))
|
||||
'''
|
||||
|
||||
force_exclude = '''.*/__amalgam__.py'''
|
||||
|
||||
[tool.isort]
|
||||
profile = "black"
|
||||
extend_skip_glob = ["xonsh/*_table.py", "xonsh/ply/**.py", "*/__amalgam__.py", "**/__amalgam__.py"]
|
||||
extend_skip_glob = ["xonsh/*_table.py", "xonsh/ply/**.py"]
|
||||
src_paths = ["xonsh", "xontrib", "xompletions", "tests"]
|
||||
known_first_party = ["xonsh", "xontrib", "xompletions", "tests"]
|
||||
known_third_party = ["ply", "pytest"]
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ import itertools
|
|||
|
||||
|
||||
$RAISE_SUBPROC_ERROR = True
|
||||
# $XONSH_NO_AMALGAMATE = 1
|
||||
# $XONSH_TRACE_SUBPROC = True
|
||||
|
||||
|
||||
|
|
@ -28,7 +27,6 @@ def _replace_args(args: List[str], num: int) -> List[str]:
|
|||
|
||||
def test(
|
||||
report_cov: xcli.Arg('--report-coverage', '-c', action="store_true") = False,
|
||||
no_amalgam: xcli.Arg('--no-amalgam', '-n', action="store_true") = False,
|
||||
pytest_args: xcli.Arg(nargs='*')=(),
|
||||
):
|
||||
"""Run pytest.
|
||||
|
|
@ -40,27 +38,19 @@ def test(
|
|||
pytest_args
|
||||
arbitrary arguments that gets passed to pytest's invocation.
|
||||
Use %%d to parameterize and prevent overwrite
|
||||
no_amalgam
|
||||
Disable amalgamation check
|
||||
|
||||
Examples
|
||||
--------
|
||||
`xonsh run-tests.xsh -- --junitxml=junit/test-results.%%d.xml`
|
||||
"""
|
||||
|
||||
if (not no_amalgam) and not $(xonsh -c "import xonsh.main; print(xonsh.main.__file__, end='')").endswith("__amalgam__.py"):
|
||||
echo "Tests need to run from the amalgamated xonsh! install with `pip install .` (without `-e`)"
|
||||
exit(1)
|
||||
|
||||
if report_cov:
|
||||
$XONSH_NO_AMALGAMATE = True
|
||||
![pytest @(_replace_args(pytest_args, 0)) --cov --cov-report=xml --cov-report=term]
|
||||
else:
|
||||
# during CI run, some tests take longer to complete on windows
|
||||
![pytest @(_replace_args(pytest_args, 0)) --durations=5]
|
||||
|
||||
|
||||
|
||||
def validate_news_items(
|
||||
pytest_args: xcli.Arg(nargs='*') = (),
|
||||
):
|
||||
|
|
|
|||
|
|
@ -5,9 +5,6 @@
|
|||
[flake8]
|
||||
max-line-length = 180
|
||||
exclude =
|
||||
__amalgam__.py,
|
||||
**/__amalgam__.py,
|
||||
**/*/__amalgam__.py,
|
||||
docs/,
|
||||
*/ply/,
|
||||
parser*_table.py,
|
||||
|
|
@ -102,7 +99,7 @@ warn_unused_configs = True
|
|||
warn_no_return = False
|
||||
|
||||
; a regex to exclude certain directories
|
||||
exclude = ((xonsh/ply)|(__amalgam__.py)|(xontrib/(mpl.*py|distributed.py|jedi.py)))
|
||||
exclude = ((xonsh/ply)|(xontrib/(mpl.*py|distributed.py|jedi.py)))
|
||||
|
||||
;match dmypy semantics - https://github.com/python/mypy/issues/8046
|
||||
local_partial_types = True
|
||||
|
|
@ -115,11 +112,11 @@ pretty = True
|
|||
|
||||
# the __init__ files have dynamic check - ignoring the attribute error. others are generated files
|
||||
# top level package name only ignores the __init__.py file.
|
||||
[mypy-xonsh.parser_table,xonsh.completion_parser_table,xonsh.parsers.parser_table.*,xonsh.parsers.completion_parser_table.*,*.__amalgam__.*,xonsh,xonsh.prompt,xonsh.history,xonsh.completers,xonsh.procs]
|
||||
[mypy-xonsh.parser_table,xonsh.completion_parser_table,xonsh.parsers.parser_table.*,xonsh.parsers.completion_parser_table.*,xonsh,xonsh.prompt,xonsh.history,xonsh.completers,xonsh.procs]
|
||||
ignore_errors = True
|
||||
|
||||
# 3rd party libraries that we dont have control over
|
||||
[mypy-zmq.*,setproctitle,xonsh.ply.*,winreg.*,pygments.*,importlib_resources.*,nt.*,prompt_toolkit.*,distro.*,conda_suggest.*,_winreg.*,*.__amalgam__.*]
|
||||
[mypy-zmq.*,setproctitle,xonsh.ply.*,winreg.*,pygments.*,importlib_resources.*,nt.*,prompt_toolkit.*,distro.*,conda_suggest.*,_winreg.*]
|
||||
ignore_missing_imports = True
|
||||
ignore_errors = True
|
||||
|
||||
|
|
|
|||
30
setup.py
30
setup.py
|
|
@ -17,11 +17,6 @@ TABLES = [
|
|||
"xonsh/lexer_table.py",
|
||||
"xonsh/parser_table.py",
|
||||
"xonsh/completion_parser_table.py",
|
||||
"xonsh/__amalgam__.py",
|
||||
"xonsh/completers/__amalgam__.py",
|
||||
"xonsh/history/__amalgam__.py",
|
||||
"xonsh/prompt/__amalgam__.py",
|
||||
"xonsh/procs/__amalgam__.py",
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -41,28 +36,6 @@ def clean_tables():
|
|||
os.environ["XONSH_DEBUG"] = "1"
|
||||
|
||||
|
||||
def amalgamate_source():
|
||||
"""Amalgamates source files."""
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
try:
|
||||
import amalgamate
|
||||
except ImportError:
|
||||
print("Could not import amalgamate, skipping.", file=sys.stderr)
|
||||
return
|
||||
amalgamate.main(
|
||||
[
|
||||
"amalgamate",
|
||||
"--debug=XONSH_NO_AMALGAMATE",
|
||||
"xonsh",
|
||||
"xonsh.completers",
|
||||
"xonsh.history",
|
||||
"xonsh.prompt",
|
||||
"xonsh.procs",
|
||||
]
|
||||
)
|
||||
sys.path.pop(0)
|
||||
|
||||
|
||||
def build_tables():
|
||||
"""Build the lexer/parser modules."""
|
||||
print("Building lexer and parser tables.", file=sys.stderr)
|
||||
|
|
@ -156,7 +129,6 @@ class xbuild_py(build_py):
|
|||
def run(self):
|
||||
clean_tables()
|
||||
build_tables()
|
||||
amalgamate_source()
|
||||
# add dirty version number
|
||||
dirty = dirty_version()
|
||||
super().run()
|
||||
|
|
@ -180,7 +152,6 @@ class xinstall(install):
|
|||
def run(self):
|
||||
clean_tables()
|
||||
build_tables()
|
||||
amalgamate_source()
|
||||
# add dirty version number
|
||||
dirty = dirty_version()
|
||||
|
||||
|
|
@ -195,7 +166,6 @@ class xsdist(sdist):
|
|||
def make_release_tree(self, basedir, files):
|
||||
clean_tables()
|
||||
build_tables()
|
||||
amalgamate_source()
|
||||
dirty = dirty_version()
|
||||
files.extend(TABLES)
|
||||
super().make_release_tree(basedir, files)
|
||||
|
|
|
|||
|
|
@ -1,100 +1 @@
|
|||
__version__ = "0.12.6"
|
||||
|
||||
|
||||
# amalgamate exclude jupyter_kernel parser_table completion_parser_table parser_test_table pyghooks
|
||||
# amalgamate exclude winutils wizard pytest_plugin fs macutils pygments_cache
|
||||
# amalgamate exclude jupyter_shell proc built_ins
|
||||
import os as _os
|
||||
|
||||
if _os.getenv("XONSH_NO_AMALGAMATE", ""):
|
||||
pass
|
||||
else:
|
||||
import sys as _sys
|
||||
|
||||
try:
|
||||
from xonsh import __amalgam__
|
||||
|
||||
cli_utils = __amalgam__
|
||||
_sys.modules["xonsh.cli_utils"] = __amalgam__
|
||||
contexts = __amalgam__
|
||||
_sys.modules["xonsh.contexts"] = __amalgam__
|
||||
lazyasd = __amalgam__
|
||||
_sys.modules["xonsh.lazyasd"] = __amalgam__
|
||||
lazyjson = __amalgam__
|
||||
_sys.modules["xonsh.lazyjson"] = __amalgam__
|
||||
platform = __amalgam__
|
||||
_sys.modules["xonsh.platform"] = __amalgam__
|
||||
pretty = __amalgam__
|
||||
_sys.modules["xonsh.pretty"] = __amalgam__
|
||||
codecache = __amalgam__
|
||||
_sys.modules["xonsh.codecache"] = __amalgam__
|
||||
lazyimps = __amalgam__
|
||||
_sys.modules["xonsh.lazyimps"] = __amalgam__
|
||||
parser = __amalgam__
|
||||
_sys.modules["xonsh.parser"] = __amalgam__
|
||||
tokenize = __amalgam__
|
||||
_sys.modules["xonsh.tokenize"] = __amalgam__
|
||||
tools = __amalgam__
|
||||
_sys.modules["xonsh.tools"] = __amalgam__
|
||||
ast = __amalgam__
|
||||
_sys.modules["xonsh.ast"] = __amalgam__
|
||||
color_tools = __amalgam__
|
||||
_sys.modules["xonsh.color_tools"] = __amalgam__
|
||||
commands_cache = __amalgam__
|
||||
_sys.modules["xonsh.commands_cache"] = __amalgam__
|
||||
completer = __amalgam__
|
||||
_sys.modules["xonsh.completer"] = __amalgam__
|
||||
events = __amalgam__
|
||||
_sys.modules["xonsh.events"] = __amalgam__
|
||||
foreign_shells = __amalgam__
|
||||
_sys.modules["xonsh.foreign_shells"] = __amalgam__
|
||||
jobs = __amalgam__
|
||||
_sys.modules["xonsh.jobs"] = __amalgam__
|
||||
jsonutils = __amalgam__
|
||||
_sys.modules["xonsh.jsonutils"] = __amalgam__
|
||||
lexer = __amalgam__
|
||||
_sys.modules["xonsh.lexer"] = __amalgam__
|
||||
openpy = __amalgam__
|
||||
_sys.modules["xonsh.openpy"] = __amalgam__
|
||||
xontribs = __amalgam__
|
||||
_sys.modules["xonsh.xontribs"] = __amalgam__
|
||||
ansi_colors = __amalgam__
|
||||
_sys.modules["xonsh.ansi_colors"] = __amalgam__
|
||||
diff_history = __amalgam__
|
||||
_sys.modules["xonsh.diff_history"] = __amalgam__
|
||||
dirstack = __amalgam__
|
||||
_sys.modules["xonsh.dirstack"] = __amalgam__
|
||||
execer = __amalgam__
|
||||
_sys.modules["xonsh.execer"] = __amalgam__
|
||||
shell = __amalgam__
|
||||
_sys.modules["xonsh.shell"] = __amalgam__
|
||||
style_tools = __amalgam__
|
||||
_sys.modules["xonsh.style_tools"] = __amalgam__
|
||||
timings = __amalgam__
|
||||
_sys.modules["xonsh.timings"] = __amalgam__
|
||||
xonfig = __amalgam__
|
||||
_sys.modules["xonsh.xonfig"] = __amalgam__
|
||||
base_shell = __amalgam__
|
||||
_sys.modules["xonsh.base_shell"] = __amalgam__
|
||||
environ = __amalgam__
|
||||
_sys.modules["xonsh.environ"] = __amalgam__
|
||||
imphooks = __amalgam__
|
||||
_sys.modules["xonsh.imphooks"] = __amalgam__
|
||||
inspectors = __amalgam__
|
||||
_sys.modules["xonsh.inspectors"] = __amalgam__
|
||||
aliases = __amalgam__
|
||||
_sys.modules["xonsh.aliases"] = __amalgam__
|
||||
main = __amalgam__
|
||||
_sys.modules["xonsh.main"] = __amalgam__
|
||||
readline_shell = __amalgam__
|
||||
_sys.modules["xonsh.readline_shell"] = __amalgam__
|
||||
tracer = __amalgam__
|
||||
_sys.modules["xonsh.tracer"] = __amalgam__
|
||||
dumb_shell = __amalgam__
|
||||
_sys.modules["xonsh.dumb_shell"] = __amalgam__
|
||||
del __amalgam__
|
||||
except ImportError:
|
||||
pass
|
||||
del _sys
|
||||
del _os
|
||||
# amalgamate end
|
||||
|
|
|
|||
|
|
@ -1,45 +0,0 @@
|
|||
# amalgamate exclude
|
||||
import os as _os
|
||||
|
||||
if _os.getenv("XONSH_NO_AMALGAMATE", ""):
|
||||
pass
|
||||
else:
|
||||
import sys as _sys
|
||||
|
||||
try:
|
||||
from xonsh.completers import __amalgam__
|
||||
|
||||
bash_completion = __amalgam__
|
||||
_sys.modules["xonsh.completers.bash_completion"] = __amalgam__
|
||||
dirs = __amalgam__
|
||||
_sys.modules["xonsh.completers.dirs"] = __amalgam__
|
||||
tools = __amalgam__
|
||||
_sys.modules["xonsh.completers.tools"] = __amalgam__
|
||||
bash = __amalgam__
|
||||
_sys.modules["xonsh.completers.bash"] = __amalgam__
|
||||
commands = __amalgam__
|
||||
_sys.modules["xonsh.completers.commands"] = __amalgam__
|
||||
completer = __amalgam__
|
||||
_sys.modules["xonsh.completers.completer"] = __amalgam__
|
||||
environment = __amalgam__
|
||||
_sys.modules["xonsh.completers.environment"] = __amalgam__
|
||||
imports = __amalgam__
|
||||
_sys.modules["xonsh.completers.imports"] = __amalgam__
|
||||
man = __amalgam__
|
||||
_sys.modules["xonsh.completers.man"] = __amalgam__
|
||||
path = __amalgam__
|
||||
_sys.modules["xonsh.completers.path"] = __amalgam__
|
||||
python = __amalgam__
|
||||
_sys.modules["xonsh.completers.python"] = __amalgam__
|
||||
_aliases = __amalgam__
|
||||
_sys.modules["xonsh.completers._aliases"] = __amalgam__
|
||||
base = __amalgam__
|
||||
_sys.modules["xonsh.completers.base"] = __amalgam__
|
||||
init = __amalgam__
|
||||
_sys.modules["xonsh.completers.init"] = __amalgam__
|
||||
del __amalgam__
|
||||
except ImportError:
|
||||
pass
|
||||
del _sys
|
||||
del _os
|
||||
# amalgamate end
|
||||
|
|
@ -1093,11 +1093,6 @@ The file should contain a function with the signature
|
|||
"presented, like PLY parsing messages.",
|
||||
is_configurable=False,
|
||||
)
|
||||
XONSH_NO_AMALGAMATE = Var.with_default(
|
||||
False,
|
||||
"Setting this variable prior to starting xonsh to a truthy value will suppress amalgamated imports.",
|
||||
is_configurable=False,
|
||||
)
|
||||
XONSH_DATA_DIR = Var.with_default(
|
||||
xonsh_data_dir,
|
||||
"This is the location where xonsh data files are stored, such as history, generated completers ...",
|
||||
|
|
|
|||
|
|
@ -1,27 +0,0 @@
|
|||
# amalgamate exclude
|
||||
import os as _os
|
||||
|
||||
if _os.getenv("XONSH_NO_AMALGAMATE", ""):
|
||||
pass
|
||||
else:
|
||||
import sys as _sys
|
||||
|
||||
try:
|
||||
from xonsh.history import __amalgam__
|
||||
|
||||
base = __amalgam__
|
||||
_sys.modules["xonsh.history.base"] = __amalgam__
|
||||
dummy = __amalgam__
|
||||
_sys.modules["xonsh.history.dummy"] = __amalgam__
|
||||
json = __amalgam__
|
||||
_sys.modules["xonsh.history.json"] = __amalgam__
|
||||
sqlite = __amalgam__
|
||||
_sys.modules["xonsh.history.sqlite"] = __amalgam__
|
||||
main = __amalgam__
|
||||
_sys.modules["xonsh.history.main"] = __amalgam__
|
||||
del __amalgam__
|
||||
except ImportError:
|
||||
pass
|
||||
del _sys
|
||||
del _os
|
||||
# amalgamate end
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
# amalgamate exclude
|
||||
import os as _os
|
||||
|
||||
if _os.getenv("XONSH_NO_AMALGAMATE", ""):
|
||||
pass
|
||||
else:
|
||||
import sys as _sys
|
||||
|
||||
try:
|
||||
from xonsh.procs import __amalgam__
|
||||
|
||||
readers = __amalgam__
|
||||
_sys.modules["xonsh.procs.readers"] = __amalgam__
|
||||
pipelines = __amalgam__
|
||||
_sys.modules["xonsh.procs.pipelines"] = __amalgam__
|
||||
posix = __amalgam__
|
||||
_sys.modules["xonsh.procs.posix"] = __amalgam__
|
||||
proxies = __amalgam__
|
||||
_sys.modules["xonsh.procs.proxies"] = __amalgam__
|
||||
specs = __amalgam__
|
||||
_sys.modules["xonsh.procs.specs"] = __amalgam__
|
||||
del __amalgam__
|
||||
except ImportError:
|
||||
pass
|
||||
del _sys
|
||||
del _os
|
||||
# amalgamate end
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
# amalgamate exclude gitstatus
|
||||
import os as _os
|
||||
|
||||
if _os.getenv("XONSH_NO_AMALGAMATE", ""):
|
||||
pass
|
||||
else:
|
||||
import sys as _sys
|
||||
|
||||
try:
|
||||
from xonsh.prompt import __amalgam__
|
||||
|
||||
base = __amalgam__
|
||||
_sys.modules["xonsh.prompt.base"] = __amalgam__
|
||||
cwd = __amalgam__
|
||||
_sys.modules["xonsh.prompt.cwd"] = __amalgam__
|
||||
env = __amalgam__
|
||||
_sys.modules["xonsh.prompt.env"] = __amalgam__
|
||||
job = __amalgam__
|
||||
_sys.modules["xonsh.prompt.job"] = __amalgam__
|
||||
times = __amalgam__
|
||||
_sys.modules["xonsh.prompt.times"] = __amalgam__
|
||||
vc = __amalgam__
|
||||
_sys.modules["xonsh.prompt.vc"] = __amalgam__
|
||||
del __amalgam__
|
||||
except ImportError:
|
||||
pass
|
||||
del _sys
|
||||
del _os
|
||||
# amalgamate end
|
||||
|
|
@ -39,8 +39,7 @@ from ctypes.wintypes import (
|
|||
WORD,
|
||||
)
|
||||
|
||||
from xonsh import lazyimps # we aren't amalgamated in this module.
|
||||
from xonsh import platform
|
||||
from xonsh import lazyimps, platform
|
||||
from xonsh.lazyasd import lazyobject
|
||||
|
||||
__all__ = ("sudo",)
|
||||
|
|
|
|||
|
|
@ -1,2 +0,0 @@
|
|||
# amalgamate
|
||||
# amalgamate end
|
||||
Loading…
Add table
Reference in a new issue