Merge remote-tracking branch 'origin/main' into fix_callias_capturing

.coveragerc

@@ -8,7 +8,7 @@ omit =
   xonsh/lazyasd.py
   xonsh/parser_table.py
   xonsh/completion_parser_table.py
-  xonsh/ply/*
+  xonsh/parsers/ply/*
 
 # keep all cache files in one place
 data_file = .cache/coverage

.gitattributes

@@ -22,7 +22,7 @@ Makefile text
 README text
 
 # Files in the lazyjson format require LF line endings
-tests/histories/*.json text eol=lf
+tests/history/histories/*.json text eol=lf
 
 # Binary files
 *.ico binary

CHANGELOG.rst

@@ -2488,7 +2488,7 @@ v0.8.10
   terminal to close. This was problematic for certain command
   pipelines. For example, ``pv /dev/urandom | head`` now works.
 * Prevents recursive errors from being raised when there is no child process
-  in ``xonsh.jobs.wait_for_active_job()``.
+  in ``xonsh.procs.jobs.wait_for_active_job()``.
 * Tweaked ``xonsh.completers.commands.complete_skipper()`` to insert a space following
   certain tokens (``&&``, ``||``, ``|``, ``and``, ``or``) to avoid overwriting existing tokens
   with completer output.

docs/api/index.rst

@@ -31,14 +31,13 @@ For those of you who want the gritty details.
     xonsh.environ
     xonsh.aliases
     xonsh.dirstack
-    xonsh.jobs
     xonsh.procs
     xonsh.inspectors
     xonsh.history
     xonsh.completer
     xonsh.completers
     xonsh.prompt
-    xonsh.shell
+    xonsh.shells
     xonsh.base_shell
     xonsh.readline_shell
     xonsh.ptk_shell

docs/conf.py

@@ -35,7 +35,7 @@ xmain.setup()
 spec = importlib.util.find_spec("prompt_toolkit")
 if spec is not None:
     # hacky runaround to import PTK-specific events
-    from xonsh.ptk_shell.shell import events
+    from xonsh.shells.ptk_shell import events
 else:
     from xonsh.events import events
 

news/xonsh_refactoring.rst

@@ -0,0 +1,25 @@
+**Added:**
+
+* Starting form ``xonsh.api`` (#5383 #5538).
+
+**Changed:**
+
+* Big refactoring of internal modules structure to give clear understanding of internal xonsh components (#5538).
+
+**Deprecated:**
+
+* Starting from this release we notify that in the future we will not recommend to use ``xonsh.procs.run_subproc``
+  and ``xonsh.built_ins.subproc_*`` functions for downstream projects because of #5383.
+  We will develop ``xonsh.api`` as alternative.
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

pyproject.toml

@@ -26,8 +26,10 @@ readme = {file = ["README.rst"]}
 [tool.setuptools]
 packages = [
     "xonsh",
-    "xonsh.ply.ply",
-    "xonsh.ptk_shell",
+    "xonsh.api",
+    "xonsh.shells",
+    "xonsh.shells.ptk_shell",
+    "xonsh.parsers.ply",
     "xonsh.procs",
     "xonsh.platform",
     "xonsh.parsers",
@@ -222,7 +224,7 @@ convention = "numpy"
 [tool.ruff.lint.per-file-ignores]
 "xonsh/timings.py" = ["F401"]
 "xonsh/history.py" = ["F821"]
-"xonsh/lexer.py" = ["E741"]
+"xonsh/parsers/lexer.py" = ["E741"]
 "xonsh/parsers/completion_context.py" = ["B018"]
 "xonsh/lib/tokenize.py" = [
     "F821",
@@ -242,20 +244,20 @@ convention = "numpy"
 "xonsh/style_tools.py" = ["F821"]
 "xonsh/xoreutils/*.py" = ["E722"]
 "xonsh/completers/python.py" = ["E722"]
-"xonsh/ptk/shell.py" = ["E731"]
-"xonsh/ast.py" = ["F401"]
-"xonsh/readline_shell.py" = ["F401"]
+"xonsh/parsers/ast.py" = ["F401"]
+"xonsh/shells/ptk_shell/__init__.py" = ["E731"]
+"xonsh/shells/readline_shell.py" = ["F401"]
 "xonsh/commands_cache.py" = ["F841"]
-"xonsh/ptk/key_bindings.py" = ["F841"]
+"xonsh/shells/ptk_shell/key_bindings.py" = ["F841"]
 "xonsh/tools.py" = [
     "E731",
 ]
 "xonsh/xonfig.py" = ["E731"]
 "xontrib/vox.py" = ["F821"]
 "xonsh/inspectors.py" = ["E722"]
-"xonsh/platform/__init__.py" = ["F401"]
+"xonsh/platform.py" = ["F401"]
 "xonsh/parsers/*.py" = [
-    "E741", #  E741 Ambiguous variable name
+    "E741", # E741 Ambiguous variable name
 ]
 "tests/test*.py" = [
     "E741", #  E741 Ambiguous variable name

setup.cfg

@@ -9,7 +9,7 @@ warn_no_return = False
 packages=xonsh,xontrib,xompletions
 
 ; a regex to exclude certain directories
-exclude = ((xonsh/ply)|(xontrib/(mpl.*py|distributed.py|jedi.py)))
+exclude = ((xonsh/parsers/ply)|(xontrib/(mpl.*py|distributed.py|jedi.py)))
 
 ;match dmypy semantics - https://github.com/python/mypy/issues/8046
 local_partial_types = True
@@ -30,7 +30,7 @@ pretty = True
 ignore_errors = True
 
 # 3rd party libraries that we dont have control over
-[mypy-zmq.*,setproctitle,xonsh.ply.*,winreg.*,pygments.*,importlib_resources.*,nt.*,prompt_toolkit.*,distro.*,conda_suggest.*,_winreg.*]
+[mypy-zmq.*,setproctitle,xonsh.parsers.ply.*,winreg.*,pygments.*,importlib_resources.*,nt.*,prompt_toolkit.*,distro.*,conda_suggest.*,_winreg.*]
 ignore_missing_imports = True
 ignore_errors = True
 

setup.py

@@ -41,7 +41,7 @@ def build_tables():
     print("Building lexer and parser tables.", file=sys.stderr)
     root_dir = os.path.abspath(os.path.dirname(__file__))
     sys.path.insert(0, root_dir)
-    from xonsh.parser import Parser
+    from xonsh.parsers import Parser
     from xonsh.parsers.completion_context import CompletionContextParser
 
     Parser(

tests/api/test_os.xsh

@@ -1,7 +1,7 @@
 import os
 import tempfile
 
-from xonsh.lib.os import indir, rmtree
+from xonsh.api.os import indir, rmtree
 
 import pytest
 

tests/api/test_subprocess.xsh

@@ -2,8 +2,8 @@
 import tempfile
 from subprocess import CalledProcessError
 
-from xonsh.lib.os import indir
-from xonsh.lib.subprocess import run, check_call, check_output
+from xonsh.api.os import indir
+from xonsh.api.subprocess import run, check_call, check_output
 
 import pytest
 

tests/history/test_history_json.py

@@ -14,7 +14,7 @@ from xonsh.history.json import (
     _xhj_gc_seconds_to_rmfiles,
 )
 from xonsh.history.main import HistoryAlias, history_main
-from xonsh.lazyjson import LazyJSON
+from xonsh.lib.lazyjson import LazyJSON
 
 CMDS = ["ls", "cat hello kitty", "abc", "def", "touch me", "grep from me"]
 IGNORE_OPTS = ",".join(["ignoredups", "ignoreerr", "ignorespace"])

tests/lib/test_lazyasd.py

@@ -1,6 +1,6 @@
 """Tests lazy and self destruictive objects."""
 
-from xonsh.lazyasd import LazyObject
+from xonsh.lib.lazyasd import LazyObject
 
 #
 # LazyObject Tests

tests/lib/test_lazyjson.py

@@ -2,7 +2,7 @@
 
 from io import StringIO
 
-from xonsh.lazyjson import LazyJSON, LJNode, index, ljdump
+from xonsh.lib.lazyjson import LazyJSON, LJNode, index, ljdump
 
 
 def test_index_int():

tests/parsers/test_ast.py

@@ -4,8 +4,8 @@ import ast as pyast
 
 import pytest
 
-from xonsh import ast
-from xonsh.ast import BinOp, Call, Name, Store, Tuple, isexpression, min_line
+from xonsh.parsers import ast
+from xonsh.parsers.ast import BinOp, Call, Name, Store, Tuple, isexpression, min_line
 from xonsh.pytest.tools import nodes_equal
 
 

tests/parsers/test_lexer.py

@@ -4,13 +4,13 @@ import os
 import sys
 from collections.abc import Sequence
 
-sys.path.insert(0, os.path.abspath(".."))  # FIXME
+sys.path.insert(0, os.path.abspath("../.."))  # FIXME
 from pprint import pformat
 
 import pytest
 
-from xonsh.lexer import Lexer
-from xonsh.ply.ply.lex import LexToken
+from xonsh.parsers.lexer import Lexer
+from xonsh.parsers.ply.lex import LexToken
 
 LEXER_ARGS = {"lextab": "lexer_test_table", "debug": 0}
 

tests/parsers/test_parser.py

@@ -6,8 +6,8 @@ import textwrap
 
 import pytest
 
-from xonsh.ast import AST, Call, Pass, With, is_const_str
-from xonsh.parser import Parser
+from xonsh.parsers import Parser
+from xonsh.parsers.ast import AST, Call, Pass, With, is_const_str
 from xonsh.parsers.fstring_adaptor import FStringAdaptor
 from xonsh.pytest.tools import (
     VER_MAJOR_MINOR,

tests/procs/test_jobs.py

@@ -1,6 +1,6 @@
 import pytest
 
-from xonsh import jobs
+from xonsh.procs import jobs
 
 
 @pytest.mark.parametrize(

tests/procs/test_pipelines.py

@@ -13,7 +13,7 @@ from xonsh.pytest.tools import skip_if_on_unix, skip_if_on_windows
 
 @pytest.fixture(autouse=True)
 def patched_events(monkeypatch, xonsh_events, xonsh_session):
-    from xonsh.jobs import get_tasks
+    from xonsh.procs.jobs import get_tasks
 
     get_tasks().clear()
     # needed for ci tests

tests/shell/test_base_shell.py

@@ -1,11 +1,11 @@
-"""(A down payment on) Testing for ``xonsh.base_shell.BaseShell`` and associated classes"""
+"""(A down payment on) Testing for ``xonsh.shells.base_shell.BaseShell`` and associated classes"""
 
 import os
 
 import pytest
 
-from xonsh.base_shell import BaseShell
-from xonsh.shell import transform_command
+from xonsh.shells.base_shell import BaseShell
+from xonsh.shells.shell import transform_command
 
 
 def test_pwd_tracks_cwd(xession, xonsh_execer, tmpdir_factory, monkeypatch):

tests/shell/test_ptk_completer.py

@@ -8,7 +8,7 @@ from prompt_toolkit.document import Document
 from xonsh.aliases import Aliases
 from xonsh.completer import Completer
 from xonsh.completers.tools import RichCompletion
-from xonsh.ptk_shell.completer import PromptToolkitCompleter
+from xonsh.shells.ptk_shell.completer import PromptToolkitCompleter
 
 
 @pytest.mark.parametrize(

tests/shell/test_ptk_history.py

@@ -9,7 +9,7 @@ except ImportError:
 @pytest.fixture
 def history_obj():
     """Instantiate `PromptToolkitHistory` and append a line string"""
-    from xonsh.ptk_shell.history import PromptToolkitHistory
+    from xonsh.shells.ptk_shell.history import PromptToolkitHistory
 
     hist = PromptToolkitHistory(load_prev=False)
     hist.append_string("line10")

tests/shell/test_ptk_multiline.py

@@ -17,7 +17,7 @@ Context = namedtuple("Context", ["indent", "buffer", "accept", "cli", "cr"])
 def ctx(xession):
     """Context in which the ptk multiline functionality will be tested."""
     xession.env["INDENT"] = "    "
-    from xonsh.ptk_shell.key_bindings import carriage_return
+    from xonsh.shells.ptk_shell.key_bindings import carriage_return
 
     ptk_buffer = Buffer()
     ptk_buffer.accept_action = MagicMock(name="accept")
@@ -53,14 +53,14 @@ def test_dedent(ctx):
 def test_nodedent(ctx):
     """don't dedent if first line of ctx.buffer"""
     mock = MagicMock(return_value=True)
-    with patch("xonsh.ptk_shell.key_bindings.can_compile", mock):
+    with patch("xonsh.shells.ptk_shell.key_bindings.can_compile", mock):
         document = Document("pass")
         ctx.buffer.set_document(document)
         ctx.cr(ctx.buffer, ctx.cli)
         assert ctx.accept.mock_calls is not None
 
     mock = MagicMock(return_value=True)
-    with patch("xonsh.ptk_shell.key_bindings.can_compile", mock):
+    with patch("xonsh.shells.ptk_shell.key_bindings.can_compile", mock):
         document = Document(ctx.indent + "pass")
         ctx.buffer.set_document(document)
         ctx.cr(ctx.buffer, ctx.cli)
@@ -76,7 +76,7 @@ def test_continuation_line(ctx):
 
 def test_trailing_slash(ctx):
     mock = MagicMock(return_value=True)
-    with patch("xonsh.ptk_shell.key_bindings.can_compile", mock):
+    with patch("xonsh.shells.ptk_shell.key_bindings.can_compile", mock):
         document = Document("this line will \\")
         ctx.buffer.set_document(document)
         ctx.cr(ctx.buffer, ctx.cli)
@@ -88,7 +88,7 @@ def test_trailing_slash(ctx):
 
 def test_cant_compile_newline(ctx):
     mock = MagicMock(return_value=False)
-    with patch("xonsh.ptk_shell.key_bindings.can_compile", mock):
+    with patch("xonsh.shells.ptk_shell.key_bindings.can_compile", mock):
         document = Document("for i in (1, 2, ")
         ctx.buffer.set_document(document)
         ctx.cr(ctx.buffer, ctx.cli)
@@ -97,7 +97,7 @@ def test_cant_compile_newline(ctx):
 
 def test_can_compile_and_executes(ctx):
     mock = MagicMock(return_value=True)
-    with patch("xonsh.ptk_shell.key_bindings.can_compile", mock):
+    with patch("xonsh.shells.ptk_shell.key_bindings.can_compile", mock):
         document = Document("ls")
         ctx.buffer.set_document(document)
         ctx.cr(ctx.buffer, ctx.cli)

tests/shell/test_ptk_shell.py

@@ -6,8 +6,8 @@ import pyte
 import pytest
 
 from xonsh.platform import minimum_required_ptk_version
-from xonsh.ptk_shell.shell import tokenize_ansi
-from xonsh.shell import Shell
+from xonsh.shells.ptk_shell import tokenize_ansi
+from xonsh.shells.shell import Shell
 
 # verify error if ptk not installed or below min
 
@@ -51,13 +51,13 @@ def test_prompt_toolkit_version_checks(
         return ptk_ver is not None
 
     monkeypatch.setattr(
-        "xonsh.shell.warnings.warn", mock_warning
+        "xonsh.shells.shell.warnings.warn", mock_warning
     )  # hardwon: patch the caller!
     monkeypatch.setattr(
-        "xonsh.shell.ptk_above_min_supported", mock_ptk_above_min_supported
+        "xonsh.shells.shell.ptk_above_min_supported", mock_ptk_above_min_supported
     )  # have to patch both callers
     monkeypatch.setattr(
-        "xonsh.platform.ptk_above_min_supported", mock_ptk_above_min_supported
+        "xonsh.shells.shell.ptk_above_min_supported", mock_ptk_above_min_supported
     )
     monkeypatch.setattr("xonsh.platform.has_prompt_toolkit", mock_has_prompt_toolkit)
 

tests/shell/test_readline_shell.py

@@ -1,7 +1,7 @@
 import pytest
 
 from xonsh.completers.tools import RichCompletion
-from xonsh.readline_shell import _render_completions
+from xonsh.shells.readline_shell import _render_completions
 
 
 @pytest.mark.parametrize(

tests/shell/test_shell.py

@@ -1,11 +1,11 @@
-"""Testing for ``xonsh.shell.Shell``"""
+"""Testing for ``xonsh.shells.Shell``"""
 
 import os
 
 from xonsh.history.dummy import DummyHistory
 from xonsh.history.json import JsonHistory
 from xonsh.history.sqlite import SqliteHistory
-from xonsh.shell import Shell
+from xonsh.shells.shell import Shell
 
 
 def test_shell_with_json_history(xession, xonsh_execer, tmpdir_factory):

tests/test_integrations.py

@@ -462,7 +462,7 @@ def _echo(args):
     print(' '.join(args))
 aliases['echo'] = _echo
 
-from xonsh.lib.subprocess import check_output
+from xonsh.api.subprocess import check_output
 
 print(check_output(["echo", "hello"]).decode("utf8"))
 """,

tests/test_pyghooks.py

@@ -328,7 +328,7 @@ def test_colorize_file_symlink(key, file_path, colorizable_files, xs_LS_COLORS):
     assert color_key == tar_color_key, "File classified as expected kind, via symlink"
 
 
-import xonsh.lazyimps
+import xonsh.lib.lazyimps
 
 
 def test_colorize_file_ca(xs_LS_COLORS, monkeypatch):

tests/test_tools.py

@@ -12,7 +12,7 @@ from tempfile import TemporaryDirectory
 import pytest
 
 from xonsh import __version__
-from xonsh.lexer import Lexer
+from xonsh.parsers.lexer import Lexer
 from xonsh.platform import HAS_PYGMENTS, ON_WINDOWS, PYTHON_VERSION_INFO
 from xonsh.pytest.tools import skip_if_on_windows
 from xonsh.tools import (

xonsh/aliases.py

@@ -14,14 +14,13 @@ import typing as tp
 import xonsh.completers._aliases as xca
 import xonsh.history.main as xhm
 import xonsh.xoreutils.which as xxw
-from xonsh.ast import isexpression
 from xonsh.built_ins import XSH
 from xonsh.cli_utils import Annotated, Arg, ArgParserAlias
 from xonsh.dirstack import _get_cwd, cd, dirs, popd, pushd
 from xonsh.environ import locate_binary, make_args_env
 from xonsh.foreign_shells import foreign_shell_data
-from xonsh.jobs import bg, clean_jobs, disown, fg, jobs
-from xonsh.lazyasd import lazyobject
+from xonsh.lib.lazyasd import lazyobject
+from xonsh.parsers.ast import isexpression
 from xonsh.platform import (
     IN_APPIMAGE,
     ON_ANACONDA,
@@ -32,6 +31,7 @@ from xonsh.platform import (
     ON_OPENBSD,
     ON_WINDOWS,
 )
+from xonsh.procs.jobs import bg, clean_jobs, disown, fg, jobs
 from xonsh.procs.specs import SpecAttrModifierAlias, SpecModifierAlias
 from xonsh.timings import timeit_alias
 from xonsh.tools import (

xonsh/ansi_colors.py

@@ -16,7 +16,7 @@ from xonsh.color_tools import (
     short_to_ints,
     warn_deprecated_no_color,
 )
-from xonsh.lazyasd import LazyDict, lazyobject
+from xonsh.lib.lazyasd import LazyDict, lazyobject
 from xonsh.platform import HAS_PYGMENTS
 from xonsh.tools import FORMATTER
 

xonsh/api/__init__.py

@@ -0,0 +1,8 @@
+"""
+Originally posted in https://github.com/xonsh/xonsh/issues/2726#issuecomment-406447196 :
+    Recently @CJ-Wright has started up a ``xonsh.lib`` (``xonsh.api``) sub-package, which is usable from pure Python.
+    This is meant as a standard library for xonsh and downstream tools.
+    Currently there are some xonsh-ish wrappers around ``os`` and ``subprocess``.
+    We'd love to see more contributions to this effort! So if there is something
+    like ``sh()`` that you'd like to see, by all means please help us add it!
+"""

xonsh/api/subprocess.py

@@ -1,8 +1,8 @@
 """Xonsh extension of the standard library subprocess module, using xonsh for
 subprocess calls"""
 
+from xonsh.api.os import indir
 from xonsh.built_ins import XSH, subproc_captured_hiddenobject, subproc_captured_stdout
-from xonsh.lib.os import indir
 
 
 def run(cmd, cwd=None, check=False):

xonsh/built_ins.py

@@ -19,8 +19,8 @@ import types
 import warnings
 from ast import AST
 
-from xonsh.lazyasd import lazyobject
 from xonsh.lib.inspectors import Inspector
+from xonsh.lib.lazyasd import lazyobject
 from xonsh.platform import ON_POSIX
 from xonsh.tools import (
     XonshCalledProcessError,

xonsh/codecache.py

@@ -7,7 +7,7 @@ import sys
 
 from xonsh import __version__ as XONSH_VERSION
 from xonsh.built_ins import XSH
-from xonsh.lazyasd import lazyobject
+from xonsh.lib.lazyasd import lazyobject
 from xonsh.platform import PYTHON_VERSION_INFO_BYTES
 from xonsh.tools import is_writable_file, print_warning
 

xonsh/color_tools.py

@@ -9,7 +9,7 @@ WTFPL http://sam.zoy.org/wtfpl/
 import math
 import re
 
-from xonsh.lazyasd import LazyObject, lazyobject
+from xonsh.lib.lazyasd import LazyObject, lazyobject
 from xonsh.tools import print_warning
 
 _NO_COLOR_WARNING_SHOWN = False

xonsh/commands_cache.py

@@ -14,7 +14,7 @@ import time
 import typing as tp
 from pathlib import Path
 
-from xonsh.lazyasd import lazyobject
+from xonsh.lib.lazyasd import lazyobject
 from xonsh.platform import ON_POSIX, ON_WINDOWS, pathbasename
 from xonsh.tools import executables_in
 

xonsh/completers/imports.py

@@ -21,7 +21,7 @@ from xonsh.completers.tools import (
     contextual_completer,
     get_filter_function,
 )
-from xonsh.lazyasd import lazyobject
+from xonsh.lib.lazyasd import lazyobject
 from xonsh.parsers.completion_context import CompletionContext
 
 _suffixes = all_suffixes()

xonsh/completers/path.py

@@ -3,7 +3,7 @@ import glob
 import os
 import re
 
-import xonsh.lazyasd as xl
+import xonsh.lib.lazyasd as xl
 import xonsh.platform as xp
 import xonsh.tools as xt
 from xonsh.built_ins import XSH

xonsh/completers/python.py

@@ -6,7 +6,7 @@ import inspect
 import re
 import warnings
 
-import xonsh.lazyasd as xl
+import xonsh.lib.lazyasd as xl
 import xonsh.tools as xt
 from xonsh.built_ins import XSH
 from xonsh.completers.tools import (

xonsh/completers/tools.py

@@ -10,7 +10,7 @@ from functools import wraps
 
 import xonsh.tools as xt
 from xonsh.built_ins import XSH
-from xonsh.lazyasd import lazyobject
+from xonsh.lib.lazyasd import lazyobject
 from xonsh.parsers.completion_context import CommandContext, CompletionContext
 
 

xonsh/environ.py

@@ -29,7 +29,7 @@ from xonsh.built_ins import XSH
 from xonsh.codecache import run_script_with_cache
 from xonsh.dirstack import _get_cwd
 from xonsh.events import events
-from xonsh.lazyasd import LazyBool, lazyobject
+from xonsh.lib.lazyasd import LazyBool, lazyobject
 from xonsh.platform import (
     BASH_COMPLETIONS_DEFAULT,
     DEFAULT_ENCODING,
@@ -1535,7 +1535,7 @@ class PromptSetting(Xettings):
         "`prompt_toolkit <https://github.com/jonathanslenders/python-prompt-toolkit>`_"
         " library installed. To specify which shell should be used, do so in "
         "the run control file. "
-        "It also accepts a class type that inherits from ``xonsh.base_shell.BaseShell``",
+        "It also accepts a class type that inherits from ``xonsh.shells.base_shell.BaseShell``",
         doc_default="``best``",
     )
     SUGGEST_COMMANDS = Var.with_default(

xonsh/execer.py

@@ -6,8 +6,8 @@ import inspect
 import sys
 import types
 
-from xonsh.ast import CtxAwareTransformer
-from xonsh.parser import Parser
+from xonsh.parsers import Parser
+from xonsh.parsers.ast import CtxAwareTransformer
 from xonsh.tools import (
     balanced_parens,
     ends_with_colon_token,

xonsh/foreign_shells.py

@@ -11,7 +11,7 @@ import tempfile
 import warnings
 
 from xonsh.built_ins import XSH
-from xonsh.lazyasd import lazyobject
+from xonsh.lib.lazyasd import lazyobject
 from xonsh.platform import ON_CYGWIN, ON_MSYS, ON_WINDOWS
 from xonsh.tools import ensure_string, to_bool
 

xonsh/history/diff_history.py

@@ -5,7 +5,7 @@ import difflib
 import itertools
 
 from xonsh.color_tools import COLORS
-from xonsh.lazyjson import LazyJSON
+from xonsh.lib.lazyjson import LazyJSON
 
 # intern some strings
 REPLACE_S = "replace"

xonsh/history/json.py

@@ -19,7 +19,7 @@ except ImportError:
 
     JSONDecodeError = json.decoder.JSONDecodeError  # type: ignore
 
-import xonsh.lazyjson as xlj
+import xonsh.lib.lazyjson as xlj
 import xonsh.tools as xt
 import xonsh.xoreutils.uptime as uptime
 from xonsh.history.base import History

xonsh/imphooks.py

@@ -15,7 +15,7 @@ from importlib.machinery import ModuleSpec
 from xonsh.built_ins import XSH
 from xonsh.events import events
 from xonsh.execer import Execer
-from xonsh.lazyasd import lazyobject
+from xonsh.lib.lazyasd import lazyobject
 from xonsh.platform import ON_WINDOWS
 from xonsh.tools import print_warning
 

xonsh/lib/__init__.py

@@ -1,9 +1 @@
-"""
-Originally posted by @scopatz in https://github.com/xonsh/xonsh/issues/2726#issuecomment-406447196 :
-
-    Recently @CJ-Wright has started up a ``xonsh.lib`` sub-package, which is usable from pure Python.
-    This is meant as a standard library for xonsh and downstream tools.
-    Currently there are some xonsh-ish wrappers around ``os`` and ``subprocess``.
-    We'd love to see more contributions to this effort! So if there is something
-    like ``sh()`` that you'd like to see, by all means please help us add it!
-"""
+"""Libraries of common functions that used in xonsh components as well as modules borrowed from other projects."""

xonsh/lib/inspectors.py

@@ -15,8 +15,8 @@ import os
 import sys
 import types
 
-from xonsh.lazyasd import LazyObject
-from xonsh.lazyimps import pyghooks, pygments
+from xonsh.lib.lazyasd import LazyObject
+from xonsh.lib.lazyimps import pyghooks, pygments
 from xonsh.lib.openpy import read_py_file
 from xonsh.lib.tokenize import detect_encoding
 from xonsh.platform import HAS_PYGMENTS

xonsh/lib/lazyimps.py

@@ -3,7 +3,7 @@
 import importlib
 import os
 
-from xonsh.lazyasd import LazyObject, lazyobject
+from xonsh.lib.lazyasd import LazyObject, lazyobject
 from xonsh.platform import ON_DARWIN, ON_WINDOWS
 
 pygments = LazyObject(

xonsh/lib/openpy.py

@@ -15,7 +15,7 @@ This file was forked from the IPython project:
 import io
 import re
 
-from xonsh.lazyasd import LazyObject
+from xonsh.lib.lazyasd import LazyObject
 from xonsh.lib.tokenize import detect_encoding, tokopen
 
 cookie_comment_re = LazyObject(

xonsh/lib/pretty.py

@@ -86,7 +86,7 @@ import re
 import sys
 import types
 
-from xonsh.lazyasd import LazyObject, lazyobject
+from xonsh.lib.lazyasd import LazyObject, lazyobject
 
 __all__ = [
     "pretty",

xonsh/lib/tokenize.py

@@ -81,7 +81,7 @@ from token import (
     tok_name,
 )
 
-from xonsh.lazyasd import LazyObject
+from xonsh.lib.lazyasd import LazyObject
 from xonsh.platform import PYTHON_VERSION_INFO
 
 cookie_re = LazyObject(

xonsh/main.py

@@ -17,12 +17,12 @@ from xonsh.environ import get_home_xonshrc_path, make_args_env, xonshrc_context
 from xonsh.events import events
 from xonsh.execer import Execer
 from xonsh.imphooks import install_import_hooks
-from xonsh.jobs import ignore_sigtstp
-from xonsh.lazyasd import lazyobject
-from xonsh.lazyimps import pyghooks, pygments
+from xonsh.lib.lazyasd import lazyobject
+from xonsh.lib.lazyimps import pyghooks, pygments
 from xonsh.lib.pretty import pretty
 from xonsh.platform import HAS_PYGMENTS, ON_WINDOWS
-from xonsh.shell import Shell
+from xonsh.procs.jobs import ignore_sigtstp
+from xonsh.shells.shell import Shell
 from xonsh.timings import setup_timings
 from xonsh.tools import (
     display_error_message,

xonsh/parser.py

@@ -1,17 +0,0 @@
-"""Implements the xonsh parser."""
-
-from xonsh.lazyasd import lazyobject
-from xonsh.platform import PYTHON_VERSION_INFO
-
-
-@lazyobject
-def Parser():
-    if PYTHON_VERSION_INFO > (3, 10):
-        from xonsh.parsers.v310 import Parser as p
-    elif PYTHON_VERSION_INFO > (3, 9):
-        from xonsh.parsers.v39 import Parser as p
-    elif PYTHON_VERSION_INFO > (3, 8):
-        from xonsh.parsers.v38 import Parser as p
-    else:
-        from xonsh.parsers.v36 import Parser as p
-    return p

xonsh/parsers/__init__.py

@@ -0,0 +1,17 @@
+"""Implements the xonsh parser."""
+
+from xonsh.lib.lazyasd import lazyobject
+from xonsh.platform import PYTHON_VERSION_INFO
+
+
+@lazyobject
+def Parser():
+    if PYTHON_VERSION_INFO > (3, 10):
+        from xonsh.parsers.v310 import Parser as p
+    elif PYTHON_VERSION_INFO > (3, 9):
+        from xonsh.parsers.v39 import Parser as p
+    elif PYTHON_VERSION_INFO > (3, 8):
+        from xonsh.parsers.v38 import Parser as p
+    else:
+        from xonsh.parsers.v36 import Parser as p
+    return p

xonsh/parsers/ast.py

@@ -4,10 +4,8 @@
 # pylint: disable=unused-import
 import itertools
 import sys
-
-# pylint: enable=unused-import
 import textwrap
-from ast import (
+from ast import (  # noqa # pylint: disable=unused-import
     AST,
     Add,
     And,

xonsh/parsers/base.py

@@ -10,15 +10,15 @@ from ast import parse as pyparse
 from collections.abc import Iterable, Mapping, Sequence
 from threading import Thread
 
-from xonsh import ast
-from xonsh.ast import has_elts, load_attribute_chain, xonsh_call
-from xonsh.lazyasd import LazyObject
-from xonsh.lexer import Lexer, LexToken
+from xonsh.lib.lazyasd import LazyObject
 from xonsh.lib.tokenize import SearchPath, StringPrefix
+from xonsh.parsers import ast
+from xonsh.parsers.ast import has_elts, load_attribute_chain, xonsh_call
 from xonsh.parsers.context_check import check_contexts
 from xonsh.parsers.fstring_adaptor import FStringAdaptor
+from xonsh.parsers.lexer import Lexer, LexToken
+from xonsh.parsers.ply import yacc
 from xonsh.platform import PYTHON_VERSION_INFO
-from xonsh.ply.ply import yacc
 
 RE_SEARCHPATH = LazyObject(lambda: re.compile(SearchPath), globals(), "RE_SEARCHPATH")
 RE_STRINGPREFIX = LazyObject(

xonsh/parsers/completion_context.py

@@ -17,10 +17,10 @@ from typing import (
     overload,
 )
 
-from xonsh.lazyasd import lazyobject
-from xonsh.lexer import Lexer
+from xonsh.lib.lazyasd import lazyobject
 from xonsh.parsers.base import Location, raise_parse_error
-from xonsh.ply.ply import yacc
+from xonsh.parsers.lexer import Lexer
+from xonsh.parsers.ply import yacc
 from xonsh.tools import check_for_partial_string, get_line_continuation
 
 

xonsh/parsers/context_check.py

@@ -2,7 +2,7 @@ import ast
 import collections
 import keyword
 
-from xonsh import ast as xast
+from xonsh.parsers import ast as xast
 
 _all_keywords = frozenset(keyword.kwlist)
 

xonsh/parsers/fstring_adaptor.py

@@ -3,8 +3,8 @@
 import re
 from ast import parse as pyparse
 
-from xonsh import ast
-from xonsh.lazyasd import lazyobject
+from xonsh.lib.lazyasd import lazyobject
+from xonsh.parsers import ast
 from xonsh.platform import PYTHON_VERSION_INFO
 
 

xonsh/parsers/lexer.py

@@ -10,7 +10,7 @@ import keyword as kwmod
 import re
 import typing as tp
 
-from xonsh.lazyasd import lazyobject
+from xonsh.lib.lazyasd import lazyobject
 from xonsh.lib.tokenize import (
     CASE,
     COMMENT,
@@ -36,8 +36,8 @@ from xonsh.lib.tokenize import (
     TokenError,
     tokenize,
 )
+from xonsh.parsers.ply.lex import LexToken
 from xonsh.platform import PYTHON_VERSION_INFO
-from xonsh.ply.ply.lex import LexToken
 
 
 @lazyobject

xonsh/parsers/v310.py

@@ -10,8 +10,8 @@ handle
 
 import ast
 
+from xonsh.parsers.ply import yacc
 from xonsh.parsers.v39 import Parser as ThreeNineParser
-from xonsh.ply.ply import yacc
 
 
 class Parser(ThreeNineParser):

xonsh/parsers/v36.py

@@ -1,6 +1,6 @@
 """Implements the xonsh parser for Python v3.6."""
 
-import xonsh.ast as ast
+import xonsh.parsers.ast as ast
 from xonsh.parsers.base import BaseParser, lopen_loc, store_ctx
 
 

xonsh/parsers/v38.py

@@ -1,6 +1,6 @@
 """Implements the xonsh parser for Python v3.8."""
 
-import xonsh.ast as ast
+import xonsh.parsers.ast as ast
 from xonsh.parsers.base import store_ctx
 from xonsh.parsers.v36 import Parser as ThreeSixParser
 

xonsh/platform/__init__.py

@@ -14,7 +14,7 @@ import signal
 import subprocess
 import sys
 
-from xonsh.lazyasd import LazyBool, lazybool, lazyobject
+from xonsh.lib.lazyasd import LazyBool, lazybool, lazyobject
 
 # do not import any xonsh-modules here to avoid circular dependencies
 

xonsh/platform/winutils.py

@@ -40,8 +40,9 @@ from ctypes.wintypes import (
     WORD,
 )
 
-from xonsh import lazyimps, platform
-from xonsh.lazyasd import lazyobject
+from xonsh import platform
+from xonsh.lib import lazyimps
+from xonsh.lib.lazyasd import lazyobject
 
 __all__ = ("sudo",)
 

xonsh/procs/jobs.py

@@ -14,7 +14,7 @@ import typing as tp
 from xonsh.built_ins import XSH
 from xonsh.cli_utils import Annotated, Arg, ArgParserAlias
 from xonsh.completers.tools import RichCompletion
-from xonsh.lazyasd import LazyObject
+from xonsh.lib.lazyasd import LazyObject
 from xonsh.platform import FD_STDERR, LIBC, ON_CYGWIN, ON_DARWIN, ON_MSYS, ON_WINDOWS
 from xonsh.tools import get_signal_name, on_main_thread, unthreadable
 

xonsh/procs/pipelines.py

@@ -9,9 +9,9 @@ import sys
 import threading
 import time
 
-import xonsh.jobs as xj
-import xonsh.lazyasd as xl
+import xonsh.lib.lazyasd as xl
 import xonsh.platform as xp
+import xonsh.procs.jobs as xj
 import xonsh.tools as xt
 from xonsh.built_ins import XSH
 from xonsh.procs.readers import ConsoleParallelReader, NonBlockingFDReader, safe_fdclose

xonsh/procs/posix.py

@@ -9,12 +9,12 @@ import sys
 import threading
 import time
 
-import xonsh.lazyasd as xl
-import xonsh.lazyimps as xli
+import xonsh.lib.lazyasd as xl
+import xonsh.lib.lazyimps as xli
 import xonsh.platform as xp
 import xonsh.tools as xt
 from xonsh.built_ins import XSH
-from xonsh.jobs import proc_untraced_waitpid
+from xonsh.procs.jobs import proc_untraced_waitpid
 from xonsh.procs.readers import (
     BufferedFDParallelReader,
     NonBlockingFDReader,

xonsh/procs/proxies.py

@@ -16,7 +16,7 @@ import sys
 import threading
 import time
 
-import xonsh.lazyimps as xli
+import xonsh.lib.lazyimps as xli
 import xonsh.platform as xp
 import xonsh.tools as xt
 from xonsh.built_ins import XSH

xonsh/procs/readers.py

@@ -8,7 +8,7 @@ import sys
 import threading
 import time
 
-import xonsh.lazyimps as xli
+import xonsh.lib.lazyimps as xli
 from xonsh.built_ins import XSH
 
 

xonsh/procs/specs.py

@@ -13,10 +13,10 @@ import subprocess
 import sys
 
 import xonsh.environ as xenv
-import xonsh.jobs as xj
-import xonsh.lazyasd as xl
-import xonsh.lazyimps as xli
+import xonsh.lib.lazyasd as xl
+import xonsh.lib.lazyimps as xli
 import xonsh.platform as xp
+import xonsh.procs.jobs as xj
 import xonsh.tools as xt
 from xonsh.built_ins import XSH
 from xonsh.procs.pipelines import (

xonsh/prompt/vc.py

@@ -12,7 +12,7 @@ import threading
 
 import xonsh.tools as xt
 from xonsh.built_ins import XSH
-from xonsh.lazyasd import LazyObject
+from xonsh.lib.lazyasd import LazyObject
 
 RE_REMOVE_ANSI = LazyObject(
     lambda: re.compile(r"(?:\x1B[@-_]|[\x80-\x9F])[0-?]*[ -/]*[@-~]"),

xonsh/pyghooks.py

@@ -40,8 +40,8 @@ from xonsh.color_tools import (
 )
 from xonsh.commands_cache import CommandsCache
 from xonsh.events import events
-from xonsh.lazyasd import LazyDict, LazyObject, lazyobject
-from xonsh.lazyimps import html, os_listxattr, terminal256
+from xonsh.lib.lazyasd import LazyDict, LazyObject, lazyobject
+from xonsh.lib.lazyimps import html, os_listxattr, terminal256
 from xonsh.platform import (
     os_environ,
     ptk_version_info,

xonsh/pytest/plugin.py

@@ -19,9 +19,9 @@ from xonsh.built_ins import XSH, XonshSession
 from xonsh.completer import Completer
 from xonsh.events import events
 from xonsh.execer import Execer
-from xonsh.jobs import get_tasks
 from xonsh.main import setup
 from xonsh.parsers.completion_context import CompletionContextParser
+from xonsh.procs.jobs import get_tasks
 
 from .tools import DummyHistory, DummyShell, copy_env, sp
 
@@ -363,7 +363,7 @@ def ptk_shell(xonsh_execer):
     from prompt_toolkit.input import create_pipe_input
     from prompt_toolkit.output import DummyOutput
 
-    from xonsh.ptk_shell.shell import PromptToolkitShell
+    from xonsh.shells.ptk_shell import PromptToolkitShell
 
     out = DummyOutput()
     with create_pipe_input() as inp:
@@ -375,7 +375,7 @@ def ptk_shell(xonsh_execer):
 
 @pytest.fixture
 def readline_shell(xonsh_execer, tmpdir, mocker):
-    from xonsh.readline_shell import ReadlineShell
+    from xonsh.shells.readline_shell import ReadlineShell
 
     inp_path = tmpdir / "in"
     inp = inp_path.open("w+")

xonsh/pytest/tools.py

@@ -12,7 +12,7 @@ from collections import defaultdict
 
 import pytest
 
-from xonsh.base_shell import BaseShell
+from xonsh.shells.base_shell import BaseShell
 
 VER_MAJOR_MINOR = sys.version_info[:2]
 VER_FULL = sys.version_info[:3]

xonsh/shells/__init__.py

@@ -0,0 +1 @@
+"""The xonsh interactive shells"""

xonsh/shells/base_shell.py

@@ -17,10 +17,10 @@ from xonsh.codecache import (
 )
 from xonsh.completer import Completer
 from xonsh.events import events
-from xonsh.lazyimps import pyghooks, pygments
+from xonsh.lib.lazyimps import pyghooks, pygments
 from xonsh.platform import HAS_PYGMENTS, ON_WINDOWS
 from xonsh.prompt.base import PromptFormatter, multiline_prompt
-from xonsh.shell import transform_command
+from xonsh.shells.shell import transform_command
 from xonsh.tools import (
     DefaultNotGiven,
     XonshError,