Py39 support (#4101)

* fix: handle ast.Index and ast.ExtSlice change while Parsing

related to #4100, #4099, #4068, #4038, #4028, #4005,

* fix: py39 raises when lineno & col not given

TypeError: required field "col_offset" missing from keyword

* fix: tuple inside scbscriptor support for py39

* chore: enable py39 ci checks

they were silently failing before

* test: a backlog test for attributes inside subscriptors

this would require debugging deep into the ply parser methods.

* docs: add news item

* test: skip test_rc_with_modified_path on windows

* Update tests/test_parser.py

Co-authored-by: Gil Forsyth <gforsyth@users.noreply.github.com>

* test: mark failing tests for py39 parser

Co-authored-by: Gil Forsyth <gforsyth@users.noreply.github.com>

.github/workflows/genbuilds.py

@@ -25,7 +25,7 @@ OS_IMAGES = {
 PY_MAIN_VERSION = "3.8"
 PYTHON_VERSIONS = ["3.6", "3.7", PY_MAIN_VERSION, "3.9"]
 
-ALLOWED_FAILURES = ["3.9"]
+ALLOWED_FAILURES = []
 
 
 def write_to_file(

.github/workflows/pytest-linux-3.6.yml

@@ -38,6 +38,7 @@ jobs:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
+          use-only-tar-bz2: true
       - name: Get pip cache dir
         id: pip-cache
         run: |

.github/workflows/pytest-linux-3.7.yml

@@ -38,6 +38,7 @@ jobs:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
+          use-only-tar-bz2: true
       - name: Get pip cache dir
         id: pip-cache
         run: |

.github/workflows/pytest-linux-3.8.yml

@@ -38,6 +38,7 @@ jobs:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
+          use-only-tar-bz2: true
       - name: Get pip cache dir
         id: pip-cache
         run: |

.github/workflows/pytest-linux-3.9.yml

@@ -38,6 +38,7 @@ jobs:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
+          use-only-tar-bz2: true
       - name: Get pip cache dir
         id: pip-cache
         run: |
@@ -57,4 +58,3 @@ jobs:
           python -m pip install . --no-deps
       - name: Run tests
         run: python -m xonsh run-tests.xsh test -- --timeout=240
-        continue-on-error: true

.github/workflows/pytest-macos-3.6.yml

@@ -38,6 +38,7 @@ jobs:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
+          use-only-tar-bz2: true
       - name: Get pip cache dir
         id: pip-cache
         run: |

.github/workflows/pytest-macos-3.7.yml

@@ -38,6 +38,7 @@ jobs:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
+          use-only-tar-bz2: true
       - name: Get pip cache dir
         id: pip-cache
         run: |

.github/workflows/pytest-macos-3.8.yml

@@ -38,6 +38,7 @@ jobs:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
+          use-only-tar-bz2: true
       - name: Get pip cache dir
         id: pip-cache
         run: |

.github/workflows/pytest-macos-3.9.yml

@@ -38,6 +38,7 @@ jobs:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
+          use-only-tar-bz2: true
       - name: Get pip cache dir
         id: pip-cache
         run: |
@@ -57,4 +58,3 @@ jobs:
           python -m pip install . --no-deps
       - name: Run tests
         run: python -m xonsh run-tests.xsh test -- --timeout=240
-        continue-on-error: true

.github/workflows/pytest-windows-3.6.yml

@@ -38,6 +38,7 @@ jobs:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
+          use-only-tar-bz2: true
       - name: Get pip cache dir
         id: pip-cache
         run: |

.github/workflows/pytest-windows-3.7.yml

@@ -38,6 +38,7 @@ jobs:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
+          use-only-tar-bz2: true
       - name: Get pip cache dir
         id: pip-cache
         run: |

.github/workflows/pytest-windows-3.8.yml

@@ -38,6 +38,7 @@ jobs:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
+          use-only-tar-bz2: true
       - name: Get pip cache dir
         id: pip-cache
         run: |

.github/workflows/pytest-windows-3.9.yml

@@ -38,6 +38,7 @@ jobs:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
+          use-only-tar-bz2: true
       - name: Get pip cache dir
         id: pip-cache
         run: |
@@ -57,4 +58,3 @@ jobs:
           python -m pip install . --no-deps
       - name: Run tests
         run: python -m xonsh run-tests.xsh test -- --timeout=240
-        continue-on-error: true

.github/workflows/qa-linux-3.8.yml

@@ -38,6 +38,7 @@ jobs:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
+          use-only-tar-bz2: true
       - name: Get pip cache dir
         id: pip-cache
         run: |

news/py39-support.rst

@@ -0,0 +1,23 @@
+**Added:**
+
+* Python3.9 issues with subscriptor forms fixed.
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

tests/test_main.py

@@ -98,7 +98,7 @@ def test_rc_with_modules(shell, tmpdir, monkeypatch, capsys):
     assert tmpdir.strpath not in sys.path
 
 
-@pytest.mark.skipif(ON_WINDOWS and VER_FULL < (3, 9), reason="See https://github.com/xonsh/xonsh/issues/3936")
+@pytest.mark.skipif(ON_WINDOWS, reason="See https://github.com/xonsh/xonsh/issues/3936")
 def test_rc_with_modified_path(shell, tmpdir, monkeypatch, capsys):
     """Test that an RC file can edit the sys.path variable without losing those values."""
 

tests/test_parser.py

@@ -2,8 +2,10 @@
 """Tests the xonsh parser."""
 import ast
 import builtins
+import logging
 import textwrap
 import itertools
+import traceback
 
 import pytest
 
@@ -43,7 +45,6 @@ def check_stmts(inp, run=True, mode="exec", debug_level=0):
 
 
 def check_xonsh_ast(xenv, inp, run=True, mode="eval", debug_level=0, return_obs=False):
-    __tracebackhide__ = True
     builtins.__xonsh__.env = xenv
     obs = PARSER.parse(inp, debug_level=debug_level)
     if obs is None:
@@ -61,6 +62,12 @@ def check_xonsh(xenv, inp, run=True, mode="exec"):
     check_xonsh_ast(xenv, inp, run=run, mode=mode)
 
 
+def eval_code(inp, mode="eval", **loc_vars):
+    obs = PARSER.parse(inp, debug_level=1)
+    bytecode = compile(obs, "<test-xonsh-ast>", mode)
+    return eval(bytecode, loc_vars)
+
+
 #
 # Tests
 #
@@ -390,6 +397,39 @@ def test_if_else_expr_expr():
     check_ast("42+5 if 1 == 2 else 65-5")
 
 
+def test_subscription_syntaxes():
+    assert eval_code("[1, 2, 3][-1]") == 3
+    assert eval_code("[1, 2, 3][-1]") == 3
+    assert eval_code("'string'[-1]") == "g"
+
+
+@pytest.fixture
+def arr_container():
+    # like numpy.r_
+    class Arr:
+        def __getitem__(self, item):
+            return item
+
+    return Arr()
+
+
+def test_subscription_special_syntaxes(arr_container):
+    assert eval_code("arr[1, 2, 3]", arr=arr_container) == (1, 2, 3)
+    # dataframe
+    assert eval_code('arr[["a", "b"]]', arr=arr_container) == ["a", "b"]
+
+
+# todo: enable this test
+@pytest.mark.xfail
+def test_subscription_special_syntaxes_2(arr_container):
+    # aliases
+    d = {}
+    eval_code("d[arr.__name__]=True", arr=arr_container, d=d)
+    assert d == {"Arr": True}
+    # extslice
+    assert eval_code('arr[:, "2"]') == 2
+
+
 def test_str_idx():
     check_ast('"hello"[0]')
 

xonsh/parser.py

@@ -6,7 +6,9 @@ from xonsh.platform import PYTHON_VERSION_INFO
 
 @lazyobject
 def Parser():
-    if PYTHON_VERSION_INFO > (3, 8):
+    if PYTHON_VERSION_INFO > (3, 9):
+        from xonsh.parsers.v39 import Parser as p
+    elif PYTHON_VERSION_INFO > (3, 8):
         from xonsh.parsers.v38 import Parser as p
     else:
         from xonsh.parsers.v36 import Parser as p

xonsh/parsers/base.py

@@ -7,6 +7,7 @@ import textwrap
 from threading import Thread
 from ast import parse as pyparse
 from collections.abc import Iterable, Sequence, Mapping
+import typing as tp
 
 from xonsh.ply.ply import yacc
 
@@ -42,6 +43,14 @@ class Location(object):
         return s
 
 
+class Index(tp.NamedTuple):
+    """From PY39 ast.Index returns the value itself."""
+
+    # todo: since ast.Index is going to be removed in future PY releases, it is better to check for Expr
+    #  rather than Index
+    value: tp.Any
+
+
 def ensure_has_elts(x, lineno=None, col_offset=None):
     """Ensures that x is an AST node with elements."""
     if not has_elts(x):
@@ -2202,11 +2211,21 @@ class BaseParser(object):
         p0 = leader
         for trailer in trailers:
             if isinstance(
-                trailer, (ast.Index, ast.Slice, ast.ExtSlice, ast.Constant, ast.Name)
+                trailer,
+                (
+                    ast.Index,
+                    ast.Slice,
+                    ast.ExtSlice,
+                    ast.Constant,
+                    ast.Name,
+                    Index,
+                ),
             ):
+                # unpack types
+                slice = trailer.value if isinstance(trailer, Index) else trailer
                 p0 = ast.Subscript(
                     value=leader,
-                    slice=trailer,
+                    slice=slice,
                     ctx=ast.Load(),
                     lineno=leader.lineno,
                     col_offset=leader.col_offset,
@@ -2989,7 +3008,9 @@ class BaseParser(object):
 
         envs = self._get_envvars(p2, lineno, col)
         if envs is not None:
-            p0.keywords.append(ast.keyword(arg="envs", value=envs))
+            p0.keywords.append(
+                ast.keyword(arg="envs", value=envs, lineno=lineno, col_offset=col)
+            )
 
         return p0
 

xonsh/parsers/v39.py

@@ -0,0 +1,54 @@
+"""Handles changes since PY39
+
+handle
+- removal of ast.Index and ast.ExtSlice -- https://bugs.python.org/issue34822 -- more info
+    `at<https://docs.python.org/3/whatsnew/3.9.html?highlight=simplified%20ast%20subscription#changes-in-the-python-api>`_.
+"""
+
+import ast
+
+from xonsh.parsers.base import Index
+from xonsh.parsers.v38 import Parser as ThreeEightParser
+
+
+class Parser(ThreeEightParser):
+    def p_subscript_test(self, p):
+        """subscript : test"""
+        p1 = p[1]
+        p[0] = Index(value=p1)
+
+    def p_subscriptlist(self, p):
+        """subscriptlist : subscript comma_subscript_list_opt comma_opt"""
+
+        p1, p2 = p[1], p[2]
+        is_subscript = False
+
+        if p2 is not None:
+            if isinstance(p1, Index):
+                p1 = p1.value
+                is_subscript = True
+            if any((isinstance(p, Index) for p in p2)):
+                is_subscript = True
+
+            after_comma = [p.value if isinstance(p, Index) else p for p in p2]
+            if (
+                isinstance(p1, ast.Slice)
+                or any([isinstance(x, ast.Slice) for x in after_comma])
+                or is_subscript
+            ):
+                p1 = Index(
+                    value=ast.Tuple(
+                        [p1] + after_comma,
+                        ctx=ast.Load(),
+                        lineno=p1.lineno,
+                        col_offset=p1.col_offset,
+                    )
+                )
+            else:
+                p1.value = ast.Tuple(
+                    elts=[p1.value] + [x.value for x in after_comma],
+                    ctx=ast.Load(),
+                    lineno=p1.lineno,
+                    col_offset=p1.col_offset,
+                )
+        p[0] = p1