chore (ci): improve ci workflows and deps management (#3873)

* chore (ci): improve elm job speed by caching

* chore (ci): update ci generator script to be run from any dir

* chore: update pre-commit to use local black binary

* chore: mypy config skip amalgam generated modules

* chore: run-tests.xsh parameterisation

use argparser to make this file a cli

* feat: use requirements.txt during CI

* feat: run black/flake8/mypy during ci from pip-env

* chore: mypy version 0.790 fix

since the loose pinning on mypy version causes new PRs to fail

* refactor: update github actions template

remove poetry/environment.yml files

.github/workflows/black.yml

@@ -1,21 +0,0 @@
-name: Black
-
-on:
-  push:
-    branches:
-    - master
-  pull_request:
-    branches:
-    - master
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Black Check
-      uses: "laloch/black-action@master"
-      with:
-        args: "--check xonsh/ xontrib/"

.github/workflows/elm.yml

@@ -14,6 +14,12 @@ jobs:
     name: Python 3.7 Elm Check Ubuntu
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: ~/conda_pkgs_dir
+          key: elm-env-${{ hashFiles('ci/environment-elm.yml') }}
+          restore-keys: |
+            elm-env-
       - name: Setup conda
         uses: conda-incubator/setup-miniconda@v1
         with:

.github/workflows/genbuilds.xsh

@@ -4,6 +4,7 @@ so that we can restart indivual workflow elements without having to restart
 them all. Rerun this script to regenerate.
 """
 from itertools import product
+import os
 
 
 OS_NAMES = ["linux", "macos", "windows"]
@@ -14,10 +15,13 @@ OS_IMAGES = {
 }
 PYTHON_VERSIONS = ["3.6", "3.7", "3.8"]
 
-template = $(cat pytest.tmpl)
+CURR_DIR = os.path.dirname(__file__)
+template_path = os.path.join(CURR_DIR, "pytest.tmpl")
+
+template = $(cat @(template_path))
 for os_name, python_version in product(OS_NAMES, PYTHON_VERSIONS):
     s = template.replace("OS_NAME", os_name)
     s = s.replace("OS_IMAGE", OS_IMAGES[os_name])
     s = s.replace("PYTHON_VERSION", python_version)
-    fname = f"pytest-{os_name}-{python_version}.yml"
+    fname = os.path.join(CURR_DIR, f"pytest-{os_name}-{python_version}.yml")
     ![echo @(s) > @(fname)]

.github/workflows/pytest-linux-3.6.yml

@@ -18,16 +18,25 @@ jobs:
     name: Python ${{ matrix.python-version }} ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ~/miniconda*/envs/
+          key: ${{ runner.os }}-${{ matrix.python-version }}-env-${{ hashFiles('requirements/tests.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-env-
       - name: Setup conda
         uses: conda-incubator/setup-miniconda@v1
         with:
-          activate-environment: ${{matrix.python-version}}-xonsh-test
-          environment-file: ci/environment-${{matrix.python-version}}.yml
+          activate-environment: xonsh-test
           update-conda: true
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
       - shell: bash -l {0}
         run: |
+          python -m pip --version
+          python -m pip install -r requirements/tests.txt
           python -m pip install . --no-deps
-          python -m xonsh run-tests.xsh --timeout=240
+          python -m xonsh run-tests.xsh test -- --timeout=240
 

.github/workflows/pytest-linux-3.7.yml

@@ -18,16 +18,25 @@ jobs:
     name: Python ${{ matrix.python-version }} ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ~/miniconda*/envs/
+          key: ${{ runner.os }}-${{ matrix.python-version }}-env-${{ hashFiles('requirements/tests.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-env-
       - name: Setup conda
         uses: conda-incubator/setup-miniconda@v1
         with:
-          activate-environment: ${{matrix.python-version}}-xonsh-test
-          environment-file: ci/environment-${{matrix.python-version}}.yml
+          activate-environment: xonsh-test
           update-conda: true
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
       - shell: bash -l {0}
         run: |
+          python -m pip --version
+          python -m pip install -r requirements/tests.txt
           python -m pip install . --no-deps
-          python -m xonsh run-tests.xsh --timeout=240
+          python -m xonsh run-tests.xsh test -- --timeout=240
 

.github/workflows/pytest-linux-3.8.yml

@@ -18,16 +18,25 @@ jobs:
     name: Python ${{ matrix.python-version }} ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ~/miniconda*/envs/
+          key: ${{ runner.os }}-${{ matrix.python-version }}-env-${{ hashFiles('requirements/tests.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-env-
       - name: Setup conda
         uses: conda-incubator/setup-miniconda@v1
         with:
-          activate-environment: ${{matrix.python-version}}-xonsh-test
-          environment-file: ci/environment-${{matrix.python-version}}.yml
+          activate-environment: xonsh-test
           update-conda: true
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
       - shell: bash -l {0}
         run: |
+          python -m pip --version
+          python -m pip install -r requirements/tests.txt
           python -m pip install . --no-deps
-          python -m xonsh run-tests.xsh --timeout=240
+          python -m xonsh run-tests.xsh test -- --timeout=240
 

.github/workflows/pytest-macos-3.6.yml

@@ -18,16 +18,25 @@ jobs:
     name: Python ${{ matrix.python-version }} ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ~/miniconda*/envs/
+          key: ${{ runner.os }}-${{ matrix.python-version }}-env-${{ hashFiles('requirements/tests.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-env-
       - name: Setup conda
         uses: conda-incubator/setup-miniconda@v1
         with:
-          activate-environment: ${{matrix.python-version}}-xonsh-test
-          environment-file: ci/environment-${{matrix.python-version}}.yml
+          activate-environment: xonsh-test
           update-conda: true
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
       - shell: bash -l {0}
         run: |
+          python -m pip --version
+          python -m pip install -r requirements/tests.txt
           python -m pip install . --no-deps
-          python -m xonsh run-tests.xsh --timeout=240
+          python -m xonsh run-tests.xsh test -- --timeout=240
 

.github/workflows/pytest-macos-3.7.yml

@@ -18,16 +18,25 @@ jobs:
     name: Python ${{ matrix.python-version }} ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ~/miniconda*/envs/
+          key: ${{ runner.os }}-${{ matrix.python-version }}-env-${{ hashFiles('requirements/tests.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-env-
       - name: Setup conda
         uses: conda-incubator/setup-miniconda@v1
         with:
-          activate-environment: ${{matrix.python-version}}-xonsh-test
-          environment-file: ci/environment-${{matrix.python-version}}.yml
+          activate-environment: xonsh-test
           update-conda: true
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
       - shell: bash -l {0}
         run: |
+          python -m pip --version
+          python -m pip install -r requirements/tests.txt
           python -m pip install . --no-deps
-          python -m xonsh run-tests.xsh --timeout=240
+          python -m xonsh run-tests.xsh test -- --timeout=240
 

.github/workflows/pytest-macos-3.8.yml

@@ -18,16 +18,25 @@ jobs:
     name: Python ${{ matrix.python-version }} ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ~/miniconda*/envs/
+          key: ${{ runner.os }}-${{ matrix.python-version }}-env-${{ hashFiles('requirements/tests.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-env-
       - name: Setup conda
         uses: conda-incubator/setup-miniconda@v1
         with:
-          activate-environment: ${{matrix.python-version}}-xonsh-test
-          environment-file: ci/environment-${{matrix.python-version}}.yml
+          activate-environment: xonsh-test
           update-conda: true
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
       - shell: bash -l {0}
         run: |
+          python -m pip --version
+          python -m pip install -r requirements/tests.txt
           python -m pip install . --no-deps
-          python -m xonsh run-tests.xsh --timeout=240
+          python -m xonsh run-tests.xsh test -- --timeout=240
 

.github/workflows/pytest-windows-3.6.yml

@@ -18,16 +18,25 @@ jobs:
     name: Python ${{ matrix.python-version }} ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ~/miniconda*/envs/
+          key: ${{ runner.os }}-${{ matrix.python-version }}-env-${{ hashFiles('requirements/tests.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-env-
       - name: Setup conda
         uses: conda-incubator/setup-miniconda@v1
         with:
-          activate-environment: ${{matrix.python-version}}-xonsh-test
-          environment-file: ci/environment-${{matrix.python-version}}.yml
+          activate-environment: xonsh-test
           update-conda: true
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
       - shell: bash -l {0}
         run: |
+          python -m pip --version
+          python -m pip install -r requirements/tests.txt
           python -m pip install . --no-deps
-          python -m xonsh run-tests.xsh --timeout=240
+          python -m xonsh run-tests.xsh test -- --timeout=240
 

.github/workflows/pytest-windows-3.7.yml

@@ -18,16 +18,25 @@ jobs:
     name: Python ${{ matrix.python-version }} ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ~/miniconda*/envs/
+          key: ${{ runner.os }}-${{ matrix.python-version }}-env-${{ hashFiles('requirements/tests.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-env-
       - name: Setup conda
         uses: conda-incubator/setup-miniconda@v1
         with:
-          activate-environment: ${{matrix.python-version}}-xonsh-test
-          environment-file: ci/environment-${{matrix.python-version}}.yml
+          activate-environment: xonsh-test
           update-conda: true
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
       - shell: bash -l {0}
         run: |
+          python -m pip --version
+          python -m pip install -r requirements/tests.txt
           python -m pip install . --no-deps
-          python -m xonsh run-tests.xsh --timeout=240
+          python -m xonsh run-tests.xsh test -- --timeout=240
 

.github/workflows/pytest-windows-3.8.yml

@@ -18,16 +18,25 @@ jobs:
     name: Python ${{ matrix.python-version }} ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ~/miniconda*/envs/
+          key: ${{ runner.os }}-${{ matrix.python-version }}-env-${{ hashFiles('requirements/tests.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-env-
       - name: Setup conda
         uses: conda-incubator/setup-miniconda@v1
         with:
-          activate-environment: ${{matrix.python-version}}-xonsh-test
-          environment-file: ci/environment-${{matrix.python-version}}.yml
+          activate-environment: xonsh-test
           update-conda: true
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
       - shell: bash -l {0}
         run: |
+          python -m pip --version
+          python -m pip install -r requirements/tests.txt
           python -m pip install . --no-deps
-          python -m xonsh run-tests.xsh --timeout=240
+          python -m xonsh run-tests.xsh test -- --timeout=240
 

.github/workflows/pytest.tmpl

@@ -18,15 +18,24 @@ jobs:
     name: Python ${{ matrix.python-version }} ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ~/miniconda*/envs/
+          key: ${{ runner.os }}-${{ matrix.python-version }}-env-${{ hashFiles('requirements/tests.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-env-
       - name: Setup conda
         uses: conda-incubator/setup-miniconda@v1
         with:
-          activate-environment: ${{matrix.python-version}}-xonsh-test
-          environment-file: ci/environment-${{matrix.python-version}}.yml
+          activate-environment: xonsh-test
           update-conda: true
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
           condarc-file: ci/condarc.yml
       - shell: bash -l {0}
         run: |
+          python -m pip --version
+          python -m pip install -r requirements/tests.txt
           python -m pip install . --no-deps
-          python -m xonsh run-tests.xsh --timeout=240
+          python -m xonsh run-tests.xsh test -- --timeout=240

.github/workflows/qa-linux-3.8.yml

@@ -0,0 +1,41 @@
+name: qa linux 3.8
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ 3.8 ]
+    name: Python ${{ matrix.python-version }} ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/conda_pkgs_dir
+            ~/miniconda*/envs/
+          key: ${{ runner.os }}-${{ matrix.python-version }}-env-${{ hashFiles('requirements/tests.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-env-
+      - name: Setup conda
+        uses: conda-incubator/setup-miniconda@v1
+        with:
+          activate-environment: xonsh-test
+          auto-update-conda: true
+          python-version: ${{ matrix.python-version }}  # this itself makes sure that Python version is installed
+          condarc-file: ci/condarc.yml
+      - shell: bash -l {0}
+        run: |
+          python -m pip --version
+          python -m pip install -r requirements/tests.txt
+          python -m pip install . --no-deps
+          python -m xonsh run-tests.xsh qa

.pre-commit-config.yaml

@@ -1,6 +1,11 @@
 repos:
-  - repo: https://github.com/ambv/black
-    rev: stable
+  - repo: local
     hooks:
-    - id: black
-      language_version: python3
+      - id: black
+        name: black
+        # this gets run within development environment.
+        # Otherwise will raise command not found or use system level binary
+        entry: black --check xonsh/ xontrib/
+        language: system
+        types:
+          - python

.travis.yml

@@ -49,5 +49,5 @@ script:
       cd ..;
       doctr deploy --deploy-repo xonsh/xonsh-docs .;
     else
-      xonsh run-tests.xsh;
+      xonsh run-tests.xsh test;
     fi

ci/environment-3.6.yml

@@ -1,19 +0,0 @@
-name: 3.6-xonsh-test
-channels:
-  - conda-forge
-  - defaults
-dependencies:
-  - python=3.6
-  - pygments
-  - prompt_toolkit
-  - pytest
-  - pytest-timeout
-  - numpy
-  - psutil
-  - matplotlib
-  - flake8
-  - mypy=0.782
-  - coverage
-  - pyflakes
-  - pytest-cov
-  - codecov

ci/environment-3.7.yml

@@ -1,19 +0,0 @@
-name: 3.7-xonsh-test
-channels:
-  - conda-forge
-  - defaults
-dependencies:
-  - python=3.7
-  - pygments
-  - prompt_toolkit
-  - pytest
-  - pytest-timeout
-  - numpy
-  - psutil
-  - matplotlib
-  - flake8
-  - mypy=0.782
-  - coverage
-  - pyflakes
-  - pytest-cov
-  - codecov

ci/environment-3.8.yml

@@ -1,19 +0,0 @@
-name: 3.8-xonsh-test
-channels:
-  - conda-forge
-  - defaults
-dependencies:
-  - python=3.8
-  - pygments
-  - prompt_toolkit
-  - pytest
-  - pytest-timeout
-  - numpy
-  - psutil
-  - matplotlib
-  - flake8
-  - mypy=0.782
-  - coverage
-  - pyflakes
-  - pytest-cov
-  - codecov

news/improve-ci-jobs-and-env.rst

@@ -0,0 +1,24 @@
+**Added:**
+
+* <news item>
+
+**Changed:**
+
+* use requirements.txt env in both CI/local/pre-commit checks
+* add caching to CI jobs to improve speed
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>

requirements/tests.txt

@@ -10,4 +10,4 @@ codecov
 coverage
 black==19.10b0 --pre
 pre-commit
-mypy>=0.782
+mypy==0.790

rever.xsh

@@ -20,7 +20,7 @@ $VERSION_BUMP_PATTERNS = [
 $CHANGELOG_FILENAME = 'CHANGELOG.rst'
 $CHANGELOG_TEMPLATE = 'TEMPLATE.rst'
 
-$PYTEST_COMMAND = "./run-tests.xsh"
+$PYTEST_COMMAND = "./run-tests.xsh test -- "
 
 $TAG_REMOTE = 'git@github.com:xonsh/xonsh.git'
 $TAG_TARGET = 'master'

run-tests.xsh

@@ -1,35 +1,72 @@
 #!/usr/bin/env xonsh
-import sys
-args = sys.argv[1:]
+import argparse
+from typing import List
 
 
-def replace_args(num):
-    """
-    Replace %d to num for avoid overwrite files
-
-    Example of args: --junitxml=junit/test-results.%d.xml
-    """
-    return [
-        (arg % num) if "%d" in arg else arg
-        for arg in args]
-
 $RAISE_SUBPROC_ERROR = True
 
-run_separately = [
-    'tests/test_main.py',
-    'tests/test_ptk_highlight.py',
-    ]
-ignores = []
-for fname in run_separately:
-    ignores.append('--ignore')
-    ignores.append(fname)
-![pytest @(replace_args(0)) @(ignores)]
-for index, fname in enumerate(run_separately):
-    ![pytest @(replace_args(index+1)) @(fname)]
 
-echo "---------- Running flake8 ----------"
-python -m flake8
-    
-echo "---------- Running mypy ----------"
-mypy --version
-mypy xonsh
+def _replace_args(args:List[str], num:int) -> List[str]:
+    return [
+         (arg % num) if "%d" in arg else arg
+         for arg in args
+    ]
+
+
+def test(ns: argparse.Namespace):
+    """Run pytest.
+
+    Examples
+    --------
+    `xonsh run-tests.xsh -- --junitxml=junit/test-results.%%d.xml`
+    """
+
+    run_separately = [
+        'tests/test_main.py',
+        'tests/test_ptk_highlight.py',
+    ]
+
+    ignores = []
+    for fname in run_separately:
+        ignores.append('--ignore')
+        ignores.append(fname)
+
+    args = ns.args if "arg" in ns else []
+
+    ![pytest @(_replace_args(args, 0)) @(ignores)]
+    for index, fname in enumerate(run_separately):
+        ![pytest @(_replace_args(args, index+1)) @(fname)]
+
+
+def qa(ns: argparse.Namespace):
+    """QA checks"""
+
+    echo "---------- Check Black formatter -----------"
+    black --check xonsh xontrib
+
+    echo "---------- Running flake8 ----------"
+    python -m flake8
+
+    echo "---------- Running mypy ----------"
+    mypy --version
+    mypy xonsh
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    commands = parser.add_subparsers()
+
+    test_parser = commands.add_parser('test', help=test.__doc__)
+    test_parser.add_argument(
+        'pytest_args',
+        nargs='*',
+        help="arbitrary arguments that gets passed to pytest's invocation."
+             " Use %%d to parameterize and prevent overwrite "
+    )
+    test_parser.set_defaults(func=test)
+
+    qa_parser = commands.add_parser('qa', help=qa.__doc__)
+    qa_parser.set_defaults(func=qa)
+
+    args = parser.parse_args()
+    args.func(args)

setup.cfg

@@ -118,7 +118,7 @@ pretty = True
 ignore_errors = True
 
 # 3rd party libraries that we dont have control over
-[mypy-zmq.*,setproctitle,xonsh.ply.*,jupyter_client.*,winreg.*,pygments.*,prompt_toolkit.*,importlib_resources.*,nt.*,prompt_toolkit.*,distro.*,conda_suggest.*,_winreg.*]
+[mypy-zmq.*,setproctitle,xonsh.ply.*,jupyter_client.*,winreg.*,pygments.*,prompt_toolkit.*,importlib_resources.*,nt.*,prompt_toolkit.*,distro.*,conda_suggest.*,_winreg.*,*.__amalgam__.*]
 ignore_missing_imports = True
 
 [tool:pytest]

xonsh/tools.py

@@ -727,7 +727,6 @@ def fallback(cond, backup):
 # See the Python software license: https://docs.python.org/3/license.html
 # Copyright (c) Python Software Foundation. All rights reserved.
 class _RedirectStream:
-
     _stream: tp.Optional[str] = None
 
     def __init__(self, new_target):
@@ -1296,7 +1295,7 @@ def to_logfile_opt(x):
     the filepath if it is a writable file or None if the filepath is not
     valid, informing the user on stderr about the invalid choice.
     """
-    if isinstance(x, os.PathLike):
+    if isinstance(x, os.PathLike):  # type: ignore
         x = str(x)
     if is_logfile_opt(x):
         return x