refactoring: move parsers (#5552)

#5538


Co-authored-by: a <1@1.1>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Noorhteen Raja NJ <jnoortheen@gmail.com>

.coveragerc

@@ -8,7 +8,7 @@ omit =
   xonsh/lazyasd.py
   xonsh/parser_table.py
   xonsh/completion_parser_table.py
-  xonsh/ply/*
+  xonsh/parsers/ply/*
 
 # keep all cache files in one place
 data_file = .cache/coverage

pyproject.toml

@@ -26,7 +26,7 @@ readme = {file = ["README.rst"]}
 [tool.setuptools]
 packages = [
     "xonsh",
-    "xonsh.ply.ply",
+    "xonsh.parsers.ply",
     "xonsh.shell",
     "xonsh.shell.ptk_shell",
     "xonsh.procs",
@@ -223,7 +223,7 @@ convention = "numpy"
 [tool.ruff.lint.per-file-ignores]
 "xonsh/timings.py" = ["F401"]
 "xonsh/history.py" = ["F821"]
-"xonsh/lexer.py" = ["E741"]
+"xonsh/parsers/lexer.py" = ["E741"]
 "xonsh/parsers/completion_context.py" = ["B018"]
 "xonsh/lib/tokenize.py" = [
     "F821",
@@ -243,8 +243,8 @@ convention = "numpy"
 "xonsh/style_tools.py" = ["F821"]
 "xonsh/xoreutils/*.py" = ["E722"]
 "xonsh/completers/python.py" = ["E722"]
+"xonsh/parsers/ast.py" = ["F401"]
 "xonsh/shell/ptk_shell/__init__.py" = ["E731"]
-"xonsh/ast.py" = ["F401"]
 "xonsh/shell/readline_shell.py" = ["F401"]
 "xonsh/commands_cache.py" = ["F841"]
 "xonsh/shell/ptk_shell/key_bindings.py" = ["F841"]
@@ -254,7 +254,7 @@ convention = "numpy"
 "xonsh/xonfig.py" = ["E731"]
 "xontrib/vox.py" = ["F821"]
 "xonsh/inspectors.py" = ["E722"]
-"xonsh/platform/__init__.py" = ["F401"]
+"xonsh/platform.py" = ["F401"]
 "xonsh/parsers/*.py" = [
     "E741", #  E741 Ambiguous variable name
 ]

setup.cfg

@@ -9,7 +9,7 @@ warn_no_return = False
 packages=xonsh,xontrib,xompletions
 
 ; a regex to exclude certain directories
-exclude = ((xonsh/ply)|(xontrib/(mpl.*py|distributed.py|jedi.py)))
+exclude = ((xonsh/parsers/ply)|(xontrib/(mpl.*py|distributed.py|jedi.py)))
 
 ;match dmypy semantics - https://github.com/python/mypy/issues/8046
 local_partial_types = True
@@ -30,7 +30,7 @@ pretty = True
 ignore_errors = True
 
 # 3rd party libraries that we dont have control over
-[mypy-zmq.*,setproctitle,xonsh.ply.*,winreg.*,pygments.*,importlib_resources.*,nt.*,prompt_toolkit.*,distro.*,conda_suggest.*,_winreg.*]
+[mypy-zmq.*,setproctitle,xonsh.parsers.ply.*,winreg.*,pygments.*,importlib_resources.*,nt.*,prompt_toolkit.*,distro.*,conda_suggest.*,_winreg.*]
 ignore_missing_imports = True
 ignore_errors = True
 

setup.py

@@ -41,7 +41,7 @@ def build_tables():
     print("Building lexer and parser tables.", file=sys.stderr)
     root_dir = os.path.abspath(os.path.dirname(__file__))
     sys.path.insert(0, root_dir)
-    from xonsh.parser import Parser
+    from xonsh.parsers import Parser
     from xonsh.parsers.completion_context import CompletionContextParser
 
     Parser(

tests/parsers/test_ast.py

@@ -4,8 +4,8 @@ import ast as pyast
 
 import pytest
 
-from xonsh import ast
+from xonsh.parsers import ast
-from xonsh.ast import BinOp, Call, Name, Store, Tuple, isexpression, min_line
+from xonsh.parsers.ast import BinOp, Call, Name, Store, Tuple, isexpression, min_line
 from xonsh.pytest.tools import nodes_equal
 
 

tests/parsers/test_lexer.py

@@ -4,13 +4,13 @@ import os
 import sys
 from collections.abc import Sequence
 
-sys.path.insert(0, os.path.abspath(".."))  # FIXME
+sys.path.insert(0, os.path.abspath("../.."))  # FIXME
 from pprint import pformat
 
 import pytest
 
-from xonsh.lexer import Lexer
+from xonsh.parsers.lexer import Lexer
-from xonsh.ply.ply.lex import LexToken
+from xonsh.parsers.ply.lex import LexToken
 
 LEXER_ARGS = {"lextab": "lexer_test_table", "debug": 0}
 

tests/parsers/test_parser.py

@@ -6,8 +6,8 @@ import textwrap
 
 import pytest
 
-from xonsh.ast import AST, Call, Pass, With, is_const_str
+from xonsh.parsers import Parser
-from xonsh.parser import Parser
+from xonsh.parsers.ast import AST, Call, Pass, With, is_const_str
 from xonsh.parsers.fstring_adaptor import FStringAdaptor
 from xonsh.pytest.tools import (
     VER_MAJOR_MINOR,

tests/test_tools.py

@@ -12,7 +12,7 @@ from tempfile import TemporaryDirectory
 import pytest
 
 from xonsh import __version__
-from xonsh.lexer import Lexer
+from xonsh.parsers.lexer import Lexer
 from xonsh.platform import HAS_PYGMENTS, ON_WINDOWS, PYTHON_VERSION_INFO
 from xonsh.pytest.tools import skip_if_on_windows
 from xonsh.tools import (

xonsh/aliases.py

@@ -14,13 +14,13 @@ import typing as tp
 import xonsh.completers._aliases as xca
 import xonsh.history.main as xhm
 import xonsh.xoreutils.which as xxw
-from xonsh.ast import isexpression
 from xonsh.built_ins import XSH
 from xonsh.cli_utils import Annotated, Arg, ArgParserAlias
 from xonsh.dirstack import _get_cwd, cd, dirs, popd, pushd
 from xonsh.environ import locate_binary, make_args_env
 from xonsh.foreign_shells import foreign_shell_data
 from xonsh.lazyasd import lazyobject
+from xonsh.parsers.ast import isexpression
 from xonsh.platform import (
     IN_APPIMAGE,
     ON_ANACONDA,

xonsh/execer.py

@@ -6,8 +6,8 @@ import inspect
 import sys
 import types
 
-from xonsh.ast import CtxAwareTransformer
+from xonsh.parsers import Parser
-from xonsh.parser import Parser
+from xonsh.parsers.ast import CtxAwareTransformer
 from xonsh.tools import (
     balanced_parens,
     ends_with_colon_token,

xonsh/parser.py

@@ -1,17 +0,0 @@
-"""Implements the xonsh parser."""
-
-from xonsh.lazyasd import lazyobject
-from xonsh.platform import PYTHON_VERSION_INFO
-
-
-@lazyobject
-def Parser():
-    if PYTHON_VERSION_INFO > (3, 10):
-        from xonsh.parsers.v310 import Parser as p
-    elif PYTHON_VERSION_INFO > (3, 9):
-        from xonsh.parsers.v39 import Parser as p
-    elif PYTHON_VERSION_INFO > (3, 8):
-        from xonsh.parsers.v38 import Parser as p
-    else:
-        from xonsh.parsers.v36 import Parser as p
-    return p

xonsh/parsers/__init__.py

@@ -0,0 +1,17 @@
+"""Implements the xonsh parser."""
+
+from xonsh.lazyasd import lazyobject
+from xonsh.platform import PYTHON_VERSION_INFO
+
+
+@lazyobject
+def Parser():
+    if PYTHON_VERSION_INFO > (3, 10):
+        from xonsh.parsers.v310 import Parser as p
+    elif PYTHON_VERSION_INFO > (3, 9):
+        from xonsh.parsers.v39 import Parser as p
+    elif PYTHON_VERSION_INFO > (3, 8):
+        from xonsh.parsers.v38 import Parser as p
+    else:
+        from xonsh.parsers.v36 import Parser as p
+    return p

xonsh/parsers/ast.py

@@ -5,7 +5,7 @@
 import itertools
 import sys
 
-# pylint: enable=unused-import
+# pylint: disable=unused-import
 import textwrap
 from ast import (
     AST,

xonsh/parsers/base.py

@@ -10,15 +10,15 @@ from ast import parse as pyparse
 from collections.abc import Iterable, Mapping, Sequence
 from threading import Thread
 
-from xonsh import ast
-from xonsh.ast import has_elts, load_attribute_chain, xonsh_call
 from xonsh.lazyasd import LazyObject
-from xonsh.lexer import Lexer, LexToken
 from xonsh.lib.tokenize import SearchPath, StringPrefix
+from xonsh.parsers import ast
+from xonsh.parsers.ast import has_elts, load_attribute_chain, xonsh_call
 from xonsh.parsers.context_check import check_contexts
 from xonsh.parsers.fstring_adaptor import FStringAdaptor
+from xonsh.parsers.lexer import Lexer, LexToken
+from xonsh.parsers.ply import yacc
 from xonsh.platform import PYTHON_VERSION_INFO
-from xonsh.ply.ply import yacc
 
 RE_SEARCHPATH = LazyObject(lambda: re.compile(SearchPath), globals(), "RE_SEARCHPATH")
 RE_STRINGPREFIX = LazyObject(

xonsh/parsers/completion_context.py

@@ -18,9 +18,9 @@ from typing import (
 )
 
 from xonsh.lazyasd import lazyobject
-from xonsh.lexer import Lexer
 from xonsh.parsers.base import Location, raise_parse_error
-from xonsh.ply.ply import yacc
+from xonsh.parsers.lexer import Lexer
+from xonsh.parsers.ply import yacc
 from xonsh.tools import check_for_partial_string, get_line_continuation
 
 

xonsh/parsers/context_check.py

@@ -2,7 +2,7 @@ import ast
 import collections
 import keyword
 
-from xonsh import ast as xast
+from xonsh.parsers import ast as xast
 
 _all_keywords = frozenset(keyword.kwlist)
 

xonsh/parsers/fstring_adaptor.py

@@ -3,8 +3,8 @@
 import re
 from ast import parse as pyparse
 
-from xonsh import ast
 from xonsh.lazyasd import lazyobject
+from xonsh.parsers import ast
 from xonsh.platform import PYTHON_VERSION_INFO
 
 

xonsh/parsers/lexer.py

@@ -36,8 +36,8 @@ from xonsh.lib.tokenize import (
     TokenError,
     tokenize,
 )
+from xonsh.parsers.ply.lex import LexToken
 from xonsh.platform import PYTHON_VERSION_INFO
-from xonsh.ply.ply.lex import LexToken
 
 
 @lazyobject

xonsh/parsers/v310.py

@@ -10,8 +10,8 @@ handle
 
 import ast
 
+from xonsh.parsers.ply import yacc
 from xonsh.parsers.v39 import Parser as ThreeNineParser
-from xonsh.ply.ply import yacc
 
 
 class Parser(ThreeNineParser):

xonsh/parsers/v36.py

@@ -1,6 +1,6 @@
 """Implements the xonsh parser for Python v3.6."""
 
-import xonsh.ast as ast
+import xonsh.parsers.ast as ast
 from xonsh.parsers.base import BaseParser, lopen_loc, store_ctx
 
 

xonsh/parsers/v38.py

@@ -1,6 +1,6 @@
 """Implements the xonsh parser for Python v3.8."""
 
-import xonsh.ast as ast
+import xonsh.parsers.ast as ast
 from xonsh.parsers.base import store_ctx
 from xonsh.parsers.v36 import Parser as ThreeSixParser
 

xonsh/xonfig.py

@@ -20,6 +20,7 @@ from xonsh.cli_utils import Arg, ArgParserAlias
 from xonsh.events import events
 from xonsh.foreign_shells import CANON_SHELL_NAMES
 from xonsh.lazyasd import lazyobject
+from xonsh.parsers import ply
 from xonsh.platform import (
     DEFAULT_ENCODING,
     ON_CYGWIN,
@@ -37,7 +38,6 @@ from xonsh.platform import (
     ptk_version,
     pygments_version,
 )
-from xonsh.ply import ply
 from xonsh.prompt.base import is_template_string
 from xonsh.tools import (
     color_style,