[WIP] improve/revisit CI

.travis.yml

@@ -12,124 +12,73 @@ matrix:
   include:
     - python: '2.7'
       env:
-        - TOXENV=py27-t35-c44
+        - TOXENV=py27-t310-c45
     - python: '2.7'
       env:
-        - TOXENV=py27-t35-c45
+        - TOXENV=py27-t40-c45
     - python: '2.7'
       env:
-        - TOXENV=py27-t36-c44
-    - python: '2.7'
-      env:
-        - TOXENV=py27-t36-c45
-    - python: '2.7'
-      env:
-        - TOXENV=py27-t37-c44
-    - python: '2.7'
-      env:
-        - TOXENV=py27-t37-c45
-    - python: '3.4'
-      env:
-        - TOXENV=py34-t35-c44
-    - python: '3.4'
-      env:
-        - TOXENV=py34-t35-c45
-    - python: '3.4'
-      env:
-        - TOXENV=py34-t36-c44
+        - TOXENV=py27-t41-c45
     - python: '3.4'
       env:
-        - TOXENV=py34-t36-c45
+        - TOXENV=py34-t310-c45
     - python: '3.4'
       env:
-        - TOXENV=py34-t37-c44
+        - TOXENV=py34-t40-c45
     - python: '3.4'
       env:
-        - TOXENV=py34-t37-c45
-    - python: '3.5'
-      env:
-        - TOXENV=py35-t35-c44
-    - python: '3.5'
-      env:
-        - TOXENV=py35-t35-c45
-    - python: '3.5'
-      env:
-        - TOXENV=py35-t36-c44
+        - TOXENV=py34-t41-c45
     - python: '3.5'
       env:
-        - TOXENV=py35-t36-c45
+        - TOXENV=py35-t310-c45
     - python: '3.5'
       env:
-        - TOXENV=py35-t37-c44
+        - TOXENV=py35-t40-c45
     - python: '3.5'
       env:
-        - TOXENV=py35-t37-c45
+        - TOXENV=py35-t41-c45
     - python: '3.6'
       env:
-        - TOXENV=py36-t35-c44
+        - TOXENV=py36-t310-c45
     - python: '3.6'
       env:
-        - TOXENV=py36-t35-c45
+        - TOXENV=py36-t40-c45
     - python: '3.6'
       env:
-        - TOXENV=py36-t36-c44
-    - python: '3.6'
-      env:
-        - TOXENV=py36-t36-c45
-    - python: '3.6'
-      env:
-        - TOXENV=py36-t37-c44
-    - python: '3.6'
-      env:
-        - TOXENV=py36-t37-c45
+        - TOXENV=py36-t41-c45
     - python: '3.7'
-      env:
-        - TOXENV=py37-t35-c44
       dist: xenial
       sudo: required
-    - python: '3.7'
       env:
-        - TOXENV=py37-t35-c45
-      dist: xenial
-      sudo: required
+        - TOXENV=py37-t310-c45
     - python: '3.7'
-      env:
-        - TOXENV=py37-t36-c44
       dist: xenial
       sudo: required
-    - python: '3.7'
       env:
-        - TOXENV=py37-t36-c45
-      dist: xenial
-      sudo: required
+        - TOXENV=py37-t40-c45
     - python: '3.7'
-      env:
-        - TOXENV=py37-t37-c44
       dist: xenial
       sudo: required
-    - python: '3.7'
       env:
-        - TOXENV=py37-t37-c45
-      dist: xenial
-      sudo: required
-    - python: 'pypy-5.4'
+        - TOXENV=py37-t41-c45
+    - python: 'pypy'
       env:
-        - TOXENV=pypy-t35-c44
-    - python: 'pypy-5.4'
+        - TOXENV=pypy-t310-c45
+    - python: 'pypy'
       env:
-        - TOXENV=pypy-t35-c45
-    - python: 'pypy-5.4'
+        - TOXENV=pypy-t40-c45
+    - python: 'pypy'
       env:
-        - TOXENV=pypy-t36-c44
-    - python: 'pypy-5.4'
+        - TOXENV=pypy-t41-c45
+    - python: 'pypy3'
       env:
-        - TOXENV=pypy-t36-c45
-    - python: 'pypy-5.4'
+        - TOXENV=pypy3-t310-c45
+    - python: 'pypy3'
       env:
-        - TOXENV=pypy-t37-c44
-    - python: 'pypy-5.4'
+        - TOXENV=pypy3-t40-c45
+    - python: 'pypy3'
       env:
-        - TOXENV=pypy-t37-c45
+        - TOXENV=pypy3-t41-c45
 before_install:
   - python --version
   - uname -a
@@ -140,6 +89,27 @@ install:
   - easy_install --version
   - pip --version
   - tox --version
+  - |
+    set -ex
+    if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then
+        (cd $HOME
+         wget https://bitbucket.org/pypy/pypy/downloads/pypy2-v6.0.0-linux64.tar.bz2
+         tar xf pypy2-*.tar.bz2
+         pypy2-*/bin/pypy -m ensurepip
+         pypy2-*/bin/pypy -m pip install -U virtualenv)
+        export PATH=$(echo $HOME/pypy2-*/bin):$PATH
+        export TOXPYTHON=$(echo $HOME/pypy2-*/bin/pypy)
+    fi
+    if [[ $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then
+        (cd $HOME
+         wget https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-linux64.tar.bz2
+         tar xf pypy3-*.tar.bz2
+         pypy3-*/bin/pypy3 -m ensurepip
+         pypy3-*/bin/pypy3 -m pip install -U virtualenv)
+        export PATH=$(echo $HOME/pypy3-*/bin):$PATH
+        export TOXPYTHON=$(echo $HOME/pypy3-*/bin/pypy3)
+    fi
+    set +x
 script:
   - tox -v
 after_failure:

CONTRIBUTING.rst

@@ -83,7 +83,7 @@ Tips
 
 To run a subset of tests::
 
-    tox -e envname -- py.test -k test_myfeature
+    tox -e envname -- pytest -k test_myfeature
 
 To run all the test environments in *parallel* (you need to ``pip install detox``)::
 

ci/bootstrap.py

@@ -4,53 +4,55 @@
 
 import os
 import sys
+from os.path import abspath
+from os.path import dirname
 from os.path import exists
 from os.path import join
-from os.path import dirname
-from os.path import abspath
 
 
 if __name__ == "__main__":
     base_path = dirname(dirname(abspath(__file__)))
-    print("Project path: {}".format(base_path))
+    print("Project path: {0}".format(base_path))
     env_path = join(base_path, ".tox", "bootstrap")
     if sys.platform == "win32":
         bin_path = join(env_path, "Scripts")
     else:
         bin_path = join(env_path, "bin")
     if not exists(env_path):
         import subprocess
-        print("Making bootstrap env in: {} ...".format(env_path))
+
+        print("Making bootstrap env in: {0} ...".format(env_path))
         try:
             subprocess.check_call(["virtualenv", env_path])
-        except Exception:
+        except subprocess.CalledProcessError:
             subprocess.check_call([sys.executable, "-m", "virtualenv", env_path])
-        print("Installing `jinja2` into bootstrap environment ...")
+        print("Installing `jinja2` into bootstrap environment...")
         subprocess.check_call([join(bin_path, "pip"), "install", "jinja2"])
-    activate = join(bin_path, "activate_this.py")
-    exec(compile(open(activate, "rb").read(), activate, "exec"), dict(__file__=activate))
+    python_executable = join(bin_path, "python")
+    if not os.path.samefile(python_executable, sys.executable):
+        print("Re-executing with: {0}".format(python_executable))
+        os.execv(python_executable, [python_executable, __file__])
 
     import jinja2
 
     import subprocess
 
-
     jinja = jinja2.Environment(
         loader=jinja2.FileSystemLoader(join(base_path, "ci", "templates")),
         trim_blocks=True,
         lstrip_blocks=True,
         keep_trailing_newline=True
     )
 
-    tox_environments = [line.strip() for line in subprocess.check_output(['tox', '--listenvs'], universal_newlines=True).splitlines()]
+    tox_environments = [
+        line.strip()
+        # WARNING: 'tox' must be installed globally or in the project's virtualenv
+        for line in subprocess.check_output(['tox', '--listenvs'], universal_newlines=True).splitlines()
+    ]
     tox_environments = [line for line in tox_environments if line not in ['clean', 'report', 'docs', 'check']]
 
-    template_vars = {'tox_environments': tox_environments}
-    for py_ver in '27 34 35 py'.split():
-        template_vars['py%s_environments' % py_ver] = [x for x in tox_environments if x.startswith('py' + py_ver)]
-
     for name in os.listdir(join("ci", "templates")):
         with open(join(base_path, name), "w") as fh:
-            fh.write(jinja.get_template(name).render(**template_vars))
+            fh.write(jinja.get_template(name).render(tox_environments=tox_environments))
         print("Wrote {}".format(name))
     print("DONE.")

ci/templates/.travis.yml

@@ -11,14 +11,13 @@ env:
 matrix:
   include:
 {%- for env in tox_environments %}{{ '' }}
-    - python: '{{ '{0[0]}-5.4'.format(env.split('-')) if env.startswith('pypy') else '{0[2]}.{0[3]}'.format(env) }}'
-      env:
-        - TOXENV={{ env -}}
+    - python: '{{ env.split("-")[0] if env.startswith("pypy") else "{0[2]}.{0[3]}".format(env) }}'
 {% if env.startswith('py37') %}
-
       dist: xenial
       sudo: required
-{%- endif -%}
+{% endif %}
+      env:
+        - TOXENV={{ env }}{% if 'cover' in env %},report,coveralls,codecov{% endif -%}
 {%- endfor %}{{ '' }}
 before_install:
   - python --version
@@ -30,6 +29,27 @@ install:
   - easy_install --version
   - pip --version
   - tox --version
+  - |
+    set -ex
+    if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then
+        (cd $HOME
+         wget https://bitbucket.org/pypy/pypy/downloads/pypy2-v6.0.0-linux64.tar.bz2
+         tar xf pypy2-*.tar.bz2
+         pypy2-*/bin/pypy -m ensurepip
+         pypy2-*/bin/pypy -m pip install -U virtualenv)
+        export PATH=$(echo $HOME/pypy2-*/bin):$PATH
+        export TOXPYTHON=$(echo $HOME/pypy2-*/bin/pypy)
+    fi
+    if [[ $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then
+        (cd $HOME
+         wget https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-linux64.tar.bz2
+         tar xf pypy3-*.tar.bz2
+         pypy3-*/bin/pypy3 -m ensurepip
+         pypy3-*/bin/pypy3 -m pip install -U virtualenv)
+        export PATH=$(echo $HOME/pypy3-*/bin):$PATH
+        export TOXPYTHON=$(echo $HOME/pypy3-*/bin/pypy3)
+    fi
+    set +x
 script:
   - tox -v
 after_failure:

docs/config.rst

@@ -8,7 +8,7 @@ further control of coverage use a coverage config file.
 For example if tests are contained within the directory tree being measured the tests may be
 excluded if desired by using a .coveragerc file with the omit option set::
 
-    py.test --cov-config .coveragerc
+    pytest --cov-config .coveragerc
             --cov=myproj
             myproj/tests/
 

docs/plugins.rst

@@ -8,7 +8,7 @@ See `pytest/issues/935 <https://github.com/pytest-dev/pytest/issues/935#issuecom
 
 The current way of dealing with this problem is using the append feature and manually starting ``pytest-cov``'s engine, eg:
 
-    COV_CORE_SOURCE=src COV_CORE_CONFIG=.coveragerc COV_CORE_DATAFILE=.coverage.eager py.test --cov=src --cov-append
+    COV_CORE_SOURCE=src COV_CORE_CONFIG=.coveragerc COV_CORE_DATAFILE=.coverage.eager pytest --cov=src --cov-append
 
 Alternatively you can have this in ``tox.ini`` (if you're using `Tox <https://tox.readthedocs.io/en/latest/>`_ of course)::
 

docs/reporting.rst

@@ -8,7 +8,7 @@ annotated source code.
 
 The terminal report without line numbers (default)::
 
-    py.test --cov-report term --cov=myproj tests/
+    pytest --cov-report term --cov=myproj tests/
 
     -------------------- coverage: platform linux2, python 2.6.4-final-0 ---------------------
     Name                 Stmts   Miss  Cover
@@ -22,7 +22,7 @@ The terminal report without line numbers (default)::
 
 The terminal report with line numbers::
 
-    py.test --cov-report term-missing --cov=myproj tests/
+    pytest --cov-report term-missing --cov=myproj tests/
 
     -------------------- coverage: platform linux2, python 2.6.4-final-0 ---------------------
     Name                 Stmts   Miss  Cover   Missing
@@ -35,7 +35,7 @@ The terminal report with line numbers::
 
 The terminal report with skip covered::
 
-    py.test --cov-report term:skip-covered --cov=myproj tests/
+    pytest --cov-report term:skip-covered --cov=myproj tests/
 
     -------------------- coverage: platform linux2, python 2.6.4-final-0 ---------------------
     Name                 Stmts   Miss  Cover
@@ -51,7 +51,7 @@ You can use ``skip-covered`` with ``term-missing`` as well. e.g. ``--cov-report
 
 These three report options output to files without showing anything on the terminal::
 
-    py.test --cov-report html
+    pytest --cov-report html
             --cov-report xml
             --cov-report annotate
             --cov=myproj tests/
@@ -60,14 +60,14 @@ The output location for each of these reports can be specified. The output locat
 report is a file. Where as the output location for the HTML and annotated source code reports are
 directories::
 
-    py.test --cov-report html:cov_html
+    pytest --cov-report html:cov_html
             --cov-report xml:cov.xml
             --cov-report annotate:cov_annotate
             --cov=myproj tests/
 
 The final report option can also suppress printing to the terminal::
 
-    py.test --cov-report= --cov=myproj tests/
+    pytest --cov-report= --cov=myproj tests/
 
 This mode can be especially useful on continuous integration servers, where a coverage file
 is needed for subsequent processing, but no local report needs to be viewed. For example,