From 7ffce23d29fd69025f5dc5ff738d51d22b877f3a Mon Sep 17 00:00:00 2001
From: Andy Kipp <anki-code@users.noreply.github.com>
Date: Sat, 29 Jun 2024 10:28:02 +0200
Subject: [PATCH] refactoring: move parsers (#5552)

#5538


Co-authored-by: a <1@1.1>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Noorhteen Raja NJ <jnoortheen@gmail.com>
---
 .coveragerc                            |  2 +-
 pyproject.toml                         |  8 ++++----
 setup.cfg                              |  4 ++--
 setup.py                               |  2 +-
 tests/{ => parsers}/test_ast.py        |  4 ++--
 tests/{ => parsers}/test_lexer.py      |  6 +++---
 tests/{ => parsers}/test_parser.py     |  4 ++--
 tests/test_tools.py                    |  2 +-
 xonsh/aliases.py                       |  2 +-
 xonsh/execer.py                        |  4 ++--
 xonsh/parser.py                        | 17 -----------------
 xonsh/parsers/__init__.py              | 17 +++++++++++++++++
 xonsh/{ => parsers}/ast.py             |  2 +-
 xonsh/parsers/base.py                  |  8 ++++----
 xonsh/parsers/completion_context.py    |  4 ++--
 xonsh/parsers/context_check.py         |  2 +-
 xonsh/parsers/fstring_adaptor.py       |  2 +-
 xonsh/{ => parsers}/lexer.py           |  2 +-
 xonsh/{ply => parsers}/ply/__init__.py |  0
 xonsh/{ply => parsers}/ply/cpp.py      |  0
 xonsh/{ply => parsers}/ply/ctokens.py  |  0
 xonsh/{ply => parsers}/ply/lex.py      |  0
 xonsh/{ply => parsers}/ply/yacc.py     |  0
 xonsh/{ply => parsers}/ply/ygen.py     |  0
 xonsh/parsers/v310.py                  |  2 +-
 xonsh/parsers/v36.py                   |  2 +-
 xonsh/parsers/v38.py                   |  2 +-
 xonsh/xonfig.py                        |  2 +-
 28 files changed, 50 insertions(+), 50 deletions(-)
 rename tests/{ => parsers}/test_ast.py (96%)
 rename tests/{ => parsers}/test_lexer.py (98%)
 rename tests/{ => parsers}/test_parser.py (99%)
 delete mode 100644 xonsh/parser.py
 rename xonsh/{ => parsers}/ast.py (99%)
 rename xonsh/{ => parsers}/lexer.py (99%)
 rename xonsh/{ply => parsers}/ply/__init__.py (100%)
 rename xonsh/{ply => parsers}/ply/cpp.py (100%)
 rename xonsh/{ply => parsers}/ply/ctokens.py (100%)
 rename xonsh/{ply => parsers}/ply/lex.py (100%)
 rename xonsh/{ply => parsers}/ply/yacc.py (100%)
 rename xonsh/{ply => parsers}/ply/ygen.py (100%)

diff --git a/.coveragerc b/.coveragerc
index 7f6cbab4c..41fbb87e6 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -8,7 +8,7 @@ omit =
   xonsh/lazyasd.py
   xonsh/parser_table.py
   xonsh/completion_parser_table.py
-  xonsh/ply/*
+  xonsh/parsers/ply/*
 
 # keep all cache files in one place
 data_file = .cache/coverage
diff --git a/pyproject.toml b/pyproject.toml
index 672234194..e4673d6e7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -26,7 +26,7 @@ readme = {file = ["README.rst"]}
 [tool.setuptools]
 packages = [
     "xonsh",
-    "xonsh.ply.ply",
+    "xonsh.parsers.ply",
     "xonsh.shell",
     "xonsh.shell.ptk_shell",
     "xonsh.procs",
@@ -223,7 +223,7 @@ convention = "numpy"
 [tool.ruff.lint.per-file-ignores]
 "xonsh/timings.py" = ["F401"]
 "xonsh/history.py" = ["F821"]
-"xonsh/lexer.py" = ["E741"]
+"xonsh/parsers/lexer.py" = ["E741"]
 "xonsh/parsers/completion_context.py" = ["B018"]
 "xonsh/lib/tokenize.py" = [
     "F821",
@@ -243,8 +243,8 @@ convention = "numpy"
 "xonsh/style_tools.py" = ["F821"]
 "xonsh/xoreutils/*.py" = ["E722"]
 "xonsh/completers/python.py" = ["E722"]
+"xonsh/parsers/ast.py" = ["F401"]
 "xonsh/shell/ptk_shell/__init__.py" = ["E731"]
-"xonsh/ast.py" = ["F401"]
 "xonsh/shell/readline_shell.py" = ["F401"]
 "xonsh/commands_cache.py" = ["F841"]
 "xonsh/shell/ptk_shell/key_bindings.py" = ["F841"]
@@ -254,7 +254,7 @@ convention = "numpy"
 "xonsh/xonfig.py" = ["E731"]
 "xontrib/vox.py" = ["F821"]
 "xonsh/inspectors.py" = ["E722"]
-"xonsh/platform/__init__.py" = ["F401"]
+"xonsh/platform.py" = ["F401"]
 "xonsh/parsers/*.py" = [
     "E741", #  E741 Ambiguous variable name
 ]
diff --git a/setup.cfg b/setup.cfg
index fb396157b..7d159b84e 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -9,7 +9,7 @@ warn_no_return = False
 packages=xonsh,xontrib,xompletions
 
 ; a regex to exclude certain directories
-exclude = ((xonsh/ply)|(xontrib/(mpl.*py|distributed.py|jedi.py)))
+exclude = ((xonsh/parsers/ply)|(xontrib/(mpl.*py|distributed.py|jedi.py)))
 
 ;match dmypy semantics - https://github.com/python/mypy/issues/8046
 local_partial_types = True
@@ -30,7 +30,7 @@ pretty = True
 ignore_errors = True
 
 # 3rd party libraries that we dont have control over
-[mypy-zmq.*,setproctitle,xonsh.ply.*,winreg.*,pygments.*,importlib_resources.*,nt.*,prompt_toolkit.*,distro.*,conda_suggest.*,_winreg.*]
+[mypy-zmq.*,setproctitle,xonsh.parsers.ply.*,winreg.*,pygments.*,importlib_resources.*,nt.*,prompt_toolkit.*,distro.*,conda_suggest.*,_winreg.*]
 ignore_missing_imports = True
 ignore_errors = True
 
diff --git a/setup.py b/setup.py
index 22b5ab561..f5f0e1797 100755
--- a/setup.py
+++ b/setup.py
@@ -41,7 +41,7 @@ def build_tables():
     print("Building lexer and parser tables.", file=sys.stderr)
     root_dir = os.path.abspath(os.path.dirname(__file__))
     sys.path.insert(0, root_dir)
-    from xonsh.parser import Parser
+    from xonsh.parsers import Parser
     from xonsh.parsers.completion_context import CompletionContextParser
 
     Parser(
diff --git a/tests/test_ast.py b/tests/parsers/test_ast.py
similarity index 96%
rename from tests/test_ast.py
rename to tests/parsers/test_ast.py
index bee999729..a44040bdf 100644
--- a/tests/test_ast.py
+++ b/tests/parsers/test_ast.py
@@ -4,8 +4,8 @@ import ast as pyast
 
 import pytest
 
-from xonsh import ast
-from xonsh.ast import BinOp, Call, Name, Store, Tuple, isexpression, min_line
+from xonsh.parsers import ast
+from xonsh.parsers.ast import BinOp, Call, Name, Store, Tuple, isexpression, min_line
 from xonsh.pytest.tools import nodes_equal
 
 
diff --git a/tests/test_lexer.py b/tests/parsers/test_lexer.py
similarity index 98%
rename from tests/test_lexer.py
rename to tests/parsers/test_lexer.py
index ad2096550..21c3a01c4 100644
--- a/tests/test_lexer.py
+++ b/tests/parsers/test_lexer.py
@@ -4,13 +4,13 @@ import os
 import sys
 from collections.abc import Sequence
 
-sys.path.insert(0, os.path.abspath(".."))  # FIXME
+sys.path.insert(0, os.path.abspath("../.."))  # FIXME
 from pprint import pformat
 
 import pytest
 
-from xonsh.lexer import Lexer
-from xonsh.ply.ply.lex import LexToken
+from xonsh.parsers.lexer import Lexer
+from xonsh.parsers.ply.lex import LexToken
 
 LEXER_ARGS = {"lextab": "lexer_test_table", "debug": 0}
 
diff --git a/tests/test_parser.py b/tests/parsers/test_parser.py
similarity index 99%
rename from tests/test_parser.py
rename to tests/parsers/test_parser.py
index ee131ed8e..7131f9363 100644
--- a/tests/test_parser.py
+++ b/tests/parsers/test_parser.py
@@ -6,8 +6,8 @@ import textwrap
 
 import pytest
 
-from xonsh.ast import AST, Call, Pass, With, is_const_str
-from xonsh.parser import Parser
+from xonsh.parsers import Parser
+from xonsh.parsers.ast import AST, Call, Pass, With, is_const_str
 from xonsh.parsers.fstring_adaptor import FStringAdaptor
 from xonsh.pytest.tools import (
     VER_MAJOR_MINOR,
diff --git a/tests/test_tools.py b/tests/test_tools.py
index eee08cd65..725bbad4c 100644
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -12,7 +12,7 @@ from tempfile import TemporaryDirectory
 import pytest
 
 from xonsh import __version__
-from xonsh.lexer import Lexer
+from xonsh.parsers.lexer import Lexer
 from xonsh.platform import HAS_PYGMENTS, ON_WINDOWS, PYTHON_VERSION_INFO
 from xonsh.pytest.tools import skip_if_on_windows
 from xonsh.tools import (
diff --git a/xonsh/aliases.py b/xonsh/aliases.py
index 036f6d361..fb1df9b24 100644
--- a/xonsh/aliases.py
+++ b/xonsh/aliases.py
@@ -14,13 +14,13 @@ import typing as tp
 import xonsh.completers._aliases as xca
 import xonsh.history.main as xhm
 import xonsh.xoreutils.which as xxw
-from xonsh.ast import isexpression
 from xonsh.built_ins import XSH
 from xonsh.cli_utils import Annotated, Arg, ArgParserAlias
 from xonsh.dirstack import _get_cwd, cd, dirs, popd, pushd
 from xonsh.environ import locate_binary, make_args_env
 from xonsh.foreign_shells import foreign_shell_data
 from xonsh.lazyasd import lazyobject
+from xonsh.parsers.ast import isexpression
 from xonsh.platform import (
     IN_APPIMAGE,
     ON_ANACONDA,
diff --git a/xonsh/execer.py b/xonsh/execer.py
index 741f06212..9ec9608f9 100644
--- a/xonsh/execer.py
+++ b/xonsh/execer.py
@@ -6,8 +6,8 @@ import inspect
 import sys
 import types
 
-from xonsh.ast import CtxAwareTransformer
-from xonsh.parser import Parser
+from xonsh.parsers import Parser
+from xonsh.parsers.ast import CtxAwareTransformer
 from xonsh.tools import (
     balanced_parens,
     ends_with_colon_token,
diff --git a/xonsh/parser.py b/xonsh/parser.py
deleted file mode 100644
index 3916b0d73..000000000
--- a/xonsh/parser.py
+++ /dev/null
@@ -1,17 +0,0 @@
-"""Implements the xonsh parser."""
-
-from xonsh.lazyasd import lazyobject
-from xonsh.platform import PYTHON_VERSION_INFO
-
-
-@lazyobject
-def Parser():
-    if PYTHON_VERSION_INFO > (3, 10):
-        from xonsh.parsers.v310 import Parser as p
-    elif PYTHON_VERSION_INFO > (3, 9):
-        from xonsh.parsers.v39 import Parser as p
-    elif PYTHON_VERSION_INFO > (3, 8):
-        from xonsh.parsers.v38 import Parser as p
-    else:
-        from xonsh.parsers.v36 import Parser as p
-    return p
diff --git a/xonsh/parsers/__init__.py b/xonsh/parsers/__init__.py
index e69de29bb..3916b0d73 100644
--- a/xonsh/parsers/__init__.py
+++ b/xonsh/parsers/__init__.py
@@ -0,0 +1,17 @@
+"""Implements the xonsh parser."""
+
+from xonsh.lazyasd import lazyobject
+from xonsh.platform import PYTHON_VERSION_INFO
+
+
+@lazyobject
+def Parser():
+    if PYTHON_VERSION_INFO > (3, 10):
+        from xonsh.parsers.v310 import Parser as p
+    elif PYTHON_VERSION_INFO > (3, 9):
+        from xonsh.parsers.v39 import Parser as p
+    elif PYTHON_VERSION_INFO > (3, 8):
+        from xonsh.parsers.v38 import Parser as p
+    else:
+        from xonsh.parsers.v36 import Parser as p
+    return p
diff --git a/xonsh/ast.py b/xonsh/parsers/ast.py
similarity index 99%
rename from xonsh/ast.py
rename to xonsh/parsers/ast.py
index 22fefc161..746e93845 100644
--- a/xonsh/ast.py
+++ b/xonsh/parsers/ast.py
@@ -5,7 +5,7 @@
 import itertools
 import sys
 
-# pylint: enable=unused-import
+# pylint: disable=unused-import
 import textwrap
 from ast import (
     AST,
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
index 7a034334d..91be5987f 100644
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -10,15 +10,15 @@ from ast import parse as pyparse
 from collections.abc import Iterable, Mapping, Sequence
 from threading import Thread
 
-from xonsh import ast
-from xonsh.ast import has_elts, load_attribute_chain, xonsh_call
 from xonsh.lazyasd import LazyObject
-from xonsh.lexer import Lexer, LexToken
 from xonsh.lib.tokenize import SearchPath, StringPrefix
+from xonsh.parsers import ast
+from xonsh.parsers.ast import has_elts, load_attribute_chain, xonsh_call
 from xonsh.parsers.context_check import check_contexts
 from xonsh.parsers.fstring_adaptor import FStringAdaptor
+from xonsh.parsers.lexer import Lexer, LexToken
+from xonsh.parsers.ply import yacc
 from xonsh.platform import PYTHON_VERSION_INFO
-from xonsh.ply.ply import yacc
 
 RE_SEARCHPATH = LazyObject(lambda: re.compile(SearchPath), globals(), "RE_SEARCHPATH")
 RE_STRINGPREFIX = LazyObject(
diff --git a/xonsh/parsers/completion_context.py b/xonsh/parsers/completion_context.py
index c31593c80..09d39f882 100644
--- a/xonsh/parsers/completion_context.py
+++ b/xonsh/parsers/completion_context.py
@@ -18,9 +18,9 @@ from typing import (
 )
 
 from xonsh.lazyasd import lazyobject
-from xonsh.lexer import Lexer
 from xonsh.parsers.base import Location, raise_parse_error
-from xonsh.ply.ply import yacc
+from xonsh.parsers.lexer import Lexer
+from xonsh.parsers.ply import yacc
 from xonsh.tools import check_for_partial_string, get_line_continuation
 
 
diff --git a/xonsh/parsers/context_check.py b/xonsh/parsers/context_check.py
index d8b35f12a..b678ecc7f 100644
--- a/xonsh/parsers/context_check.py
+++ b/xonsh/parsers/context_check.py
@@ -2,7 +2,7 @@ import ast
 import collections
 import keyword
 
-from xonsh import ast as xast
+from xonsh.parsers import ast as xast
 
 _all_keywords = frozenset(keyword.kwlist)
 
diff --git a/xonsh/parsers/fstring_adaptor.py b/xonsh/parsers/fstring_adaptor.py
index c4fb1da32..c77ad9611 100644
--- a/xonsh/parsers/fstring_adaptor.py
+++ b/xonsh/parsers/fstring_adaptor.py
@@ -3,8 +3,8 @@
 import re
 from ast import parse as pyparse
 
-from xonsh import ast
 from xonsh.lazyasd import lazyobject
+from xonsh.parsers import ast
 from xonsh.platform import PYTHON_VERSION_INFO
 
 
diff --git a/xonsh/lexer.py b/xonsh/parsers/lexer.py
similarity index 99%
rename from xonsh/lexer.py
rename to xonsh/parsers/lexer.py
index 60fe33c15..b9c4630aa 100644
--- a/xonsh/lexer.py
+++ b/xonsh/parsers/lexer.py
@@ -36,8 +36,8 @@ from xonsh.lib.tokenize import (
     TokenError,
     tokenize,
 )
+from xonsh.parsers.ply.lex import LexToken
 from xonsh.platform import PYTHON_VERSION_INFO
-from xonsh.ply.ply.lex import LexToken
 
 
 @lazyobject
diff --git a/xonsh/ply/ply/__init__.py b/xonsh/parsers/ply/__init__.py
similarity index 100%
rename from xonsh/ply/ply/__init__.py
rename to xonsh/parsers/ply/__init__.py
diff --git a/xonsh/ply/ply/cpp.py b/xonsh/parsers/ply/cpp.py
similarity index 100%
rename from xonsh/ply/ply/cpp.py
rename to xonsh/parsers/ply/cpp.py
diff --git a/xonsh/ply/ply/ctokens.py b/xonsh/parsers/ply/ctokens.py
similarity index 100%
rename from xonsh/ply/ply/ctokens.py
rename to xonsh/parsers/ply/ctokens.py
diff --git a/xonsh/ply/ply/lex.py b/xonsh/parsers/ply/lex.py
similarity index 100%
rename from xonsh/ply/ply/lex.py
rename to xonsh/parsers/ply/lex.py
diff --git a/xonsh/ply/ply/yacc.py b/xonsh/parsers/ply/yacc.py
similarity index 100%
rename from xonsh/ply/ply/yacc.py
rename to xonsh/parsers/ply/yacc.py
diff --git a/xonsh/ply/ply/ygen.py b/xonsh/parsers/ply/ygen.py
similarity index 100%
rename from xonsh/ply/ply/ygen.py
rename to xonsh/parsers/ply/ygen.py
diff --git a/xonsh/parsers/v310.py b/xonsh/parsers/v310.py
index eabdd3e93..b5ac4ce48 100644
--- a/xonsh/parsers/v310.py
+++ b/xonsh/parsers/v310.py
@@ -10,8 +10,8 @@ handle
 
 import ast
 
+from xonsh.parsers.ply import yacc
 from xonsh.parsers.v39 import Parser as ThreeNineParser
-from xonsh.ply.ply import yacc
 
 
 class Parser(ThreeNineParser):
diff --git a/xonsh/parsers/v36.py b/xonsh/parsers/v36.py
index 7319ea3e2..871d2f99e 100644
--- a/xonsh/parsers/v36.py
+++ b/xonsh/parsers/v36.py
@@ -1,6 +1,6 @@
 """Implements the xonsh parser for Python v3.6."""
 
-import xonsh.ast as ast
+import xonsh.parsers.ast as ast
 from xonsh.parsers.base import BaseParser, lopen_loc, store_ctx
 
 
diff --git a/xonsh/parsers/v38.py b/xonsh/parsers/v38.py
index afec88c70..55ab765a6 100644
--- a/xonsh/parsers/v38.py
+++ b/xonsh/parsers/v38.py
@@ -1,6 +1,6 @@
 """Implements the xonsh parser for Python v3.8."""
 
-import xonsh.ast as ast
+import xonsh.parsers.ast as ast
 from xonsh.parsers.base import store_ctx
 from xonsh.parsers.v36 import Parser as ThreeSixParser
 
diff --git a/xonsh/xonfig.py b/xonsh/xonfig.py
index 5e83c5a40..c5ea9e9ac 100644
--- a/xonsh/xonfig.py
+++ b/xonsh/xonfig.py
@@ -20,6 +20,7 @@ from xonsh.cli_utils import Arg, ArgParserAlias
 from xonsh.events import events
 from xonsh.foreign_shells import CANON_SHELL_NAMES
 from xonsh.lazyasd import lazyobject
+from xonsh.parsers import ply
 from xonsh.platform import (
     DEFAULT_ENCODING,
     ON_CYGWIN,
@@ -37,7 +38,6 @@ from xonsh.platform import (
     ptk_version,
     pygments_version,
 )
-from xonsh.ply import ply
 from xonsh.prompt.base import is_template_string
 from xonsh.tools import (
     color_style,