From 93cc972441b6c482e54e8c9af45a443bcc4c93ed Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Thu, 30 Aug 2018 14:16:05 +0100 Subject: [PATCH 01/14] PEP-517 source distribution support create a ``.package`` virtual environment to perform build operations inside --- changelog/573.feature.rst | 2 + changelog/820.feature.rst | 1 + setup.py | 1 + src/tox/_pytestplugin.py | 17 ++- src/tox/config.py | 7 ++ src/tox/package.py | 131 +++++++++++++++++++++++- src/tox/session.py | 4 +- src/tox/venv.py | 21 ++-- tests/__init__.py | 0 tests/integration/test_package.py | 71 +++++++++++++ tests/lib/__init__.py | 18 ++++ tests/{ => unit}/test_config.py | 0 tests/{ => unit}/test_docs.py | 0 tests/{ => unit}/test_interpreters.py | 0 tests/{ => unit}/test_package.py | 95 ++++++++++++++++- tests/{ => unit}/test_pytest_plugins.py | 0 tests/{ => unit}/test_quickstart.py | 0 tests/{ => unit}/test_result.py | 0 tests/{ => unit}/test_session.py | 0 tests/{ => unit}/test_venv.py | 0 tests/{ => unit}/test_z_cmdline.py | 0 tox.ini | 4 +- 22 files changed, 352 insertions(+), 20 deletions(-) create mode 100644 changelog/573.feature.rst create mode 100644 changelog/820.feature.rst create mode 100644 tests/__init__.py create mode 100644 tests/integration/test_package.py create mode 100644 tests/lib/__init__.py rename tests/{ => unit}/test_config.py (100%) rename tests/{ => unit}/test_docs.py (100%) rename tests/{ => unit}/test_interpreters.py (100%) rename tests/{ => unit}/test_package.py (61%) rename tests/{ => unit}/test_pytest_plugins.py (100%) rename tests/{ => unit}/test_quickstart.py (100%) rename tests/{ => unit}/test_result.py (100%) rename tests/{ => unit}/test_session.py (100%) rename tests/{ => unit}/test_venv.py (100%) rename tests/{ => unit}/test_z_cmdline.py (100%) diff --git a/changelog/573.feature.rst b/changelog/573.feature.rst new file mode 100644 index 000000000..b4cd95cea --- /dev/null +++ b/changelog/573.feature.rst @@ -0,0 +1,2 @@ +- [PEP-517](https://www.python.org/dev/peps/pep-0517/) source distribution support (create a +``.package`` virtual environment to perform build operations inside) by :user:`gaborbernat` diff --git a/changelog/820.feature.rst b/changelog/820.feature.rst new file mode 100644 index 000000000..5f76e2611 --- /dev/null +++ b/changelog/820.feature.rst @@ -0,0 +1 @@ +- flit support via implementing ``PEP-517`` diff --git a/setup.py b/setup.py index 100cc0017..8e59b9c9d 100644 --- a/setup.py +++ b/setup.py @@ -66,6 +66,7 @@ def main(): "py >= 1.4.17, <2", "six >= 1.0.0, <2", "virtualenv >= 1.11.2", + "toml >=0.9.4", ], extras_require={ "testing": [ diff --git a/src/tox/_pytestplugin.py b/src/tox/_pytestplugin.py index 321d511b8..e77f7a5d0 100644 --- a/src/tox/_pytestplugin.py +++ b/src/tox/_pytestplugin.py @@ -277,7 +277,7 @@ def initproj(tmpdir): setup.py """ - def initproj_(nameversion, filedefs=None, src_root="."): + def initproj_(nameversion, filedefs=None, src_root=".", add_missing_setup_py=True): if filedefs is None: filedefs = {} if not src_root: @@ -297,7 +297,7 @@ def initproj_(nameversion, filedefs=None, src_root="."): base.ensure(dir=1) create_files(base, filedefs) - if not _filedefs_contains(base, filedefs, "setup.py"): + if not _filedefs_contains(base, filedefs, "setup.py") and add_missing_setup_py: create_files( base, { @@ -319,7 +319,18 @@ def initproj_(nameversion, filedefs=None, src_root="."): ) if not _filedefs_contains(base, filedefs, src_root_path.join(name)): create_files( - src_root_path, {name: {"__init__.py": "__version__ = {!r}".format(version)}} + src_root_path, + { + name: { + "__init__.py": textwrap.dedent( + """ + \"\"\" module {} \"\"\" + __version__ = {!r}""" + ) + .strip() + .format(name, version) + } + }, ) manifestlines = [ "include {}".format(p.relto(base)) for p in base.visit(lambda x: x.check(file=1)) diff --git a/src/tox/config.py b/src/tox/config.py index 891eb0a9b..87172ccec 100755 --- a/src/tox/config.py +++ b/src/tox/config.py @@ -1005,6 +1005,13 @@ def __init__(self, config, inipath): # noqa ) config.skipsdist = reader.getbool("skipsdist", all_develop) + config.isolated_build = reader.getbool("isolated_build", False) + if config.isolated_build is True: + name = ".package" + if name not in config.envconfigs: + config.envconfigs[name] = self.make_envconfig( + name, testenvprefix + name, reader._subs, config + ) def _make_thread_safe_path(self, config, attr, unique_id): if config.option.parallel_safe_build: diff --git a/src/tox/package.py b/src/tox/package.py index b23cb1a3b..ec83a993a 100644 --- a/src/tox/package.py +++ b/src/tox/package.py @@ -1,8 +1,17 @@ +import json import sys +import textwrap +from collections import namedtuple +import pkg_resources import py +import toml import tox +from tox.config import DepConfig +from tox.venv import CreationConfig + +BuildInfo = namedtuple("BuildInfo", ["requires", "backend_module", "backend_object"]) @tox.hookimpl @@ -27,10 +36,9 @@ def get_package(session): report.info("using package {!r}, skipping 'sdist' activity ".format(str(path))) else: try: - path = make_sdist(report, config, session) - except tox.exception.InvocationError: - v = sys.exc_info()[1] - report.error("FAIL could not package project - v = {!r}".format(v)) + path = build_package(config, report, session) + except tox.exception.InvocationError as exception: + report.error("FAIL could not package project - v = {!r}".format(exception)) return None sdist_file = config.distshare.join(path.basename) if sdist_file != path: @@ -44,7 +52,14 @@ def get_package(session): return path -def make_sdist(report, config, session): +def build_package(config, report, session): + if not config.isolated_build: + return make_sdist_legacy(report, config, session) + else: + build_isolated(config, report, session) + + +def make_sdist_legacy(report, config, session): setup = config.setupdir.join("setup.py") if not setup.check(): report.error( @@ -83,3 +98,109 @@ def make_sdist(report, config, session): " python setup.py sdist" ) raise SystemExit(1) + + +def build_isolated(config, report, session): + build_info = get_build_info(config.setupdir, report) + package_venv = session.getvenv(".package") + package_venv.envconfig.deps_matches_subset = True + + package_venv.envconfig.deps = [DepConfig(r, None) for r in build_info.requires] + toml_require = {pkg_resources.Requirement(r).key for r in build_info.requires} + if not session.setupenv(package_venv): + raise SystemExit(1) + + live_config = package_venv._getliveconfig() + previous_config = CreationConfig.readconfig(package_venv.path_config) + if not previous_config or not previous_config.matches(live_config, True): + session.finishvenv(package_venv) + + build_requires = get_build_requires(build_info, package_venv, session) + for requirement in build_requires: + pkg_requirement = pkg_resources.Requirement(requirement) + if pkg_requirement.key not in toml_require: + package_venv.envconfig.deps.append(DepConfig(requirement, None)) + + if not session.setupenv(package_venv): + raise SystemExit(1) + + session.finishvenv(package_venv) + return perform_isolated_build(build_info, package_venv, session, config) + + +def get_build_info(folder, report): + toml_file = folder.join("pyproject.toml") + + # as per https://www.python.org/dev/peps/pep-0517/ + + def abort(message): + report.error("{} inside {}".format(message, toml_file)) + raise SystemExit(1) + + if not toml_file.exists(): + abort("missing {}".format(toml_file)) + + with open(toml_file) as file_handler: + config_data = toml.load(file_handler) + + if "build-system" not in config_data: + abort("build-system section missing") + + build_system = config_data["build-system"] + + if "requires" not in build_system: + abort("missing requires key at build-system section") + if "build-backend" not in build_system: + abort("missing build-backend key at build-system section") + + requires = build_system["requires"] + if not isinstance(requires, list) or not all(isinstance(i, str) for i in requires): + abort("requires key at build-system section must be a list of string") + + backend = build_system["build-backend"] + if not isinstance(backend, str): + abort("build-backend key at build-system section must be a string") + + args = backend.split(":") + module = args[0] + obj = "" if len(args) == 1 else ".{}".format(args[1]) + + return BuildInfo(requires, module, "{}{}".format(module, obj)) + + +def perform_isolated_build(build_info, package_venv, session, config): + with session.newaction( + package_venv, "perform isolated build", package_venv.envconfig.envdir + ) as action: + script = textwrap.dedent( + """ + import sys + import {} + basename = {}.build_{}("{}", {{ "--global-option": ["--formats=gztar"]}}) + print(basename)""".format( + build_info.backend_module, build_info.backend_object, "sdist", config.distdir + ) + ) + config.distdir.ensure_dir() + result = action.popen([package_venv.envconfig.envpython, "-c", script], returnout=True) + return config.distdir.join(result.split("\n")[-2]) + + +def get_build_requires(build_info, package_venv, session): + with session.newaction( + package_venv, "get build requires", package_venv.envconfig.envdir + ) as action: + script = textwrap.dedent( + """ + import {} + import json + + backend = {} + for_build_requires = backend.get_requires_for_build_{}(None) + print(json.dumps(for_build_requires)) + """.format( + build_info.backend_module, build_info.backend_object, "sdist" + ) + ).strip() + result = action.popen([package_venv.envconfig.envpython, "-c", script], returnout=True) + return json.loads(result.split("\n")[-2]) diff --git a/src/tox/session.py b/src/tox/session.py index 12db45cff..d13e7d79f 100644 --- a/src/tox/session.py +++ b/src/tox/session.py @@ -263,7 +263,7 @@ class Reporter(object): def __init__(self, session): self.tw = py.io.TerminalWriter() self.session = session - self._reportedlines = [] + self.reported_lines = [] @property def verbosity(self): @@ -338,7 +338,7 @@ def skip(self, msg): self.logline("SKIPPED: {}".format(msg), yellow=True) def logline(self, msg, **opts): - self._reportedlines.append(msg) + self.reported_lines.append(msg) self.tw.line("{}".format(msg), **opts) def verbosity0(self, msg, **opts): diff --git a/src/tox/venv.py b/src/tox/venv.py index f93b94453..978ff5df1 100755 --- a/src/tox/venv.py +++ b/src/tox/venv.py @@ -53,7 +53,7 @@ def readconfig(cls, path): except Exception: return None - def matches(self, other): + def matches(self, other, deps_matches_subset=False): return ( other and self.md5 == other.md5 @@ -62,7 +62,11 @@ def matches(self, other): and self.sitepackages == other.sitepackages and self.usedevelop == other.usedevelop and self.alwayscopy == other.alwayscopy - and self.deps == other.deps + and ( + all(d in self.deps for d in other.deps) + if deps_matches_subset is True + else self.deps == other.deps + ) ) @@ -159,7 +163,13 @@ def update(self, action): if status string is empty, all is ok. """ rconfig = CreationConfig.readconfig(self.path_config) - if not self.envconfig.recreate and rconfig and rconfig.matches(self._getliveconfig()): + if ( + not self.envconfig.recreate + and rconfig + and rconfig.matches( + self._getliveconfig(), getattr(self.envconfig, "deps_matches_subset", False) + ) + ): action.info("reusing", self.envconfig.envdir) return if rconfig is None: @@ -173,9 +183,8 @@ def update(self, action): return sys.exc_info()[1] try: self.hook.tox_testenv_install_deps(action=action, venv=self) - except tox.exception.InvocationError: - v = sys.exc_info()[1] - return "could not install deps {}; v = {!r}".format(self.envconfig.deps, v) + except tox.exception.InvocationError as exception: + return "could not install deps {}; v = {!r}".format(self.envconfig.deps, exception) def _getliveconfig(self): python = self.envconfig.python_info.executable diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/test_package.py b/tests/integration/test_package.py new file mode 100644 index 000000000..09c67b237 --- /dev/null +++ b/tests/integration/test_package.py @@ -0,0 +1,71 @@ +"""Tests that require external access (e.g. pip install, virtualenv creation)""" +import subprocess + +import pytest + +from tests.lib import need_git + + +@pytest.mark.network +def test_package_isolated_build_setuptools(initproj, cmd): + initproj( + "package_toml_setuptools-0.1", + filedefs={ + "tox.ini": """ + [tox] + isolated_build = true + [testenv:.package] + basepython = python + """, + "pyproject.toml": """ + [build-system] + requires = ["setuptools >= 35.0.2", "setuptools_scm >= 2.0.0, <3"] + build-backend = 'setuptools.build_meta' + """, + }, + ) + result = cmd("--sdistonly") + assert result.ret == 0, result.out + + result2 = cmd("--sdistonly") + assert result2.ret == 0, result.out + assert ".package recreate" not in result2.out + + +@pytest.mark.network +@need_git +def test_package_isolated_build_flit(initproj, cmd): + initproj( + "package_toml_flit-0.1", + filedefs={ + "tox.ini": """ + [tox] + isolated_build = true + [testenv:.package] + basepython = python + """, + "pyproject.toml": """ + [build-system] + requires = ["flit"] + build-backend = "flit.buildapi" + + [tool.flit.metadata] + module = "package_toml_flit" + author = "Happy Harry" + author-email = "happy@harry.com" + home-page = "https://github.com/happy-harry/is" + """, + ".gitignore": ".tox", + }, + add_missing_setup_py=False, + ) + subprocess.check_call(["git", "init"]) + subprocess.check_call(["git", "add", "-A", "."]) + subprocess.check_call(["git", "commit", "-m", "first commit"]) + result = cmd("--sdistonly") + assert result.ret == 0, result.out + + result2 = cmd("--sdistonly") + + assert result2.ret == 0, result.out + assert ".package recreate" not in result2.out diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py new file mode 100644 index 000000000..1dbe5d05f --- /dev/null +++ b/tests/lib/__init__.py @@ -0,0 +1,18 @@ +import subprocess + +import pytest + + +def need_executable(name, check_cmd): + def wrapper(fn): + try: + subprocess.check_output(check_cmd) + except OSError: + return pytest.mark.skip(reason="%s is not available" % name)(fn) + return fn + + return wrapper + + +def need_git(fn): + return pytest.mark.mercurial(need_executable("git", ("git", "--version"))(fn)) diff --git a/tests/test_config.py b/tests/unit/test_config.py similarity index 100% rename from tests/test_config.py rename to tests/unit/test_config.py diff --git a/tests/test_docs.py b/tests/unit/test_docs.py similarity index 100% rename from tests/test_docs.py rename to tests/unit/test_docs.py diff --git a/tests/test_interpreters.py b/tests/unit/test_interpreters.py similarity index 100% rename from tests/test_interpreters.py rename to tests/unit/test_interpreters.py diff --git a/tests/test_package.py b/tests/unit/test_package.py similarity index 61% rename from tests/test_package.py rename to tests/unit/test_package.py index e8bd09613..0870950fb 100644 --- a/tests/test_package.py +++ b/tests/unit/test_package.py @@ -1,8 +1,11 @@ import re +import py +import pytest + from tox.config import parseconfig -from tox.package import get_package -from tox.session import Session +from tox.package import get_build_info, get_package +from tox.session import Reporter, Session def test_make_sdist(initproj): @@ -138,3 +141,91 @@ def test_installpkg(tmpdir, newconfig): session = Session(config) sdist_path = get_package(session) assert sdist_path == p + + +def test_package_isolated_no_pyproject_toml(initproj, cmd): + initproj( + "package_no_toml-0.1", + filedefs={ + "tox.ini": """ + [tox] + isolated_build = true + """ + }, + ) + result = cmd("--sdistonly") + assert result.ret == 1 + assert result.outlines == ["ERROR: missing {}".format(py.path.local().join("pyproject.toml"))] + + +def toml_file_check(initproj, version, message, toml): + initproj( + "package_toml-{}".format(version), + filedefs={ + "tox.ini": """ + [tox] + isolated_build = true + """, + "pyproject.toml": toml, + }, + ) + reporter = Reporter(None) + + with pytest.raises(SystemExit, message=1): + get_build_info(py.path.local(), reporter) + toml_file = py.path.local().join("pyproject.toml") + msg = "ERROR: {} inside {}".format(message, toml_file) + assert reporter.reported_lines == [msg] + + +def test_package_isolated_toml_no_build_system(initproj, cmd): + toml_file_check(initproj, "", 1, "build-system section missing") + + +def test_package_isolated_toml_no_requires(initproj, cmd): + toml_file_check( + initproj, + 2, + "missing requires key at build-system section", + """ + [build-system] + """, + ) + + +def test_package_isolated_toml_no_backend(initproj, cmd): + toml_file_check( + initproj, + 3, + "missing build-backend key at build-system section", + """ + [build-system] + requires = [] + """, + ) + + +def test_package_isolated_toml_bad_requires(initproj, cmd): + toml_file_check( + initproj, + 4, + "requires key at build-system section must be a list of string", + """ + [build-system] + requires = "" + build-backend = "" + """, + ) + + +def test_package_isolated_toml_bad_backend(initproj, cmd): + toml_file_check( + initproj, + 5, + "build-backend key at build-system section must be a string", + """ + [build-system] + requires = [] + build-backend = [] + """, + ) diff --git a/tests/test_pytest_plugins.py b/tests/unit/test_pytest_plugins.py similarity index 100% rename from tests/test_pytest_plugins.py rename to tests/unit/test_pytest_plugins.py diff --git a/tests/test_quickstart.py b/tests/unit/test_quickstart.py similarity index 100% rename from tests/test_quickstart.py rename to tests/unit/test_quickstart.py diff --git a/tests/test_result.py b/tests/unit/test_result.py similarity index 100% rename from tests/test_result.py rename to tests/unit/test_result.py diff --git a/tests/test_session.py b/tests/unit/test_session.py similarity index 100% rename from tests/test_session.py rename to tests/unit/test_session.py diff --git a/tests/test_venv.py b/tests/unit/test_venv.py similarity index 100% rename from tests/test_venv.py rename to tests/unit/test_venv.py diff --git a/tests/test_z_cmdline.py b/tests/unit/test_z_cmdline.py similarity index 100% rename from tests/test_z_cmdline.py rename to tests/unit/test_z_cmdline.py diff --git a/tox.ini b/tox.ini index 2c4cde356..ef8ee1dc9 100644 --- a/tox.ini +++ b/tox.ini @@ -126,8 +126,8 @@ multi_line_output = 3 include_trailing_comma = True force_grid_wrap = 0 line_length = 99 -known_first_party = tox -known_third_party = apiclient,git,httplib2,oauth2client,packaging,pkg_resources,pluggy,py,pytest,setuptools,six +known_first_party = tox,tests +known_third_party = apiclient,git,httplib2,oauth2client,packaging,pkg_resources,pluggy,py,pytest,setuptools,six,tests,toml [testenv:release] description = do a release, required posarg of the version number From f09302f771456761eb085fadb8c90b2e67ad73c2 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Thu, 30 Aug 2018 19:02:16 +0100 Subject: [PATCH 02/14] try to fix CI --- .vsts-ci.yml | 3 +++ changelog/573.feature.rst | 2 +- changelog/820.feature.rst | 2 +- src/tox/package.py | 5 +++-- .../{test_package.py => test_package_int.py} | 11 ++++++++--- tests/unit/test_docs.py | 2 +- tests/unit/test_package.py | 2 +- tests/unit/test_pytest_plugins.py | 6 +++--- 8 files changed, 21 insertions(+), 12 deletions(-) rename tests/integration/{test_package.py => test_package_int.py} (87%) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 865a092e2..b41343160 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -1,5 +1,8 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) +variables: + "System.PreferGit": true + trigger: branches: include: diff --git a/changelog/573.feature.rst b/changelog/573.feature.rst index b4cd95cea..a982c1ed1 100644 --- a/changelog/573.feature.rst +++ b/changelog/573.feature.rst @@ -1,2 +1,2 @@ -- [PEP-517](https://www.python.org/dev/peps/pep-0517/) source distribution support (create a +`PEP-517 `_ source distribution support (create a ``.package`` virtual environment to perform build operations inside) by :user:`gaborbernat` diff --git a/changelog/820.feature.rst b/changelog/820.feature.rst index 5f76e2611..1a8be1bd3 100644 --- a/changelog/820.feature.rst +++ b/changelog/820.feature.rst @@ -1 +1 @@ -- flit support via implementing ``PEP-517`` +`flit `_ support via implementing ``PEP-517`` by :user:`gaborbernat` diff --git a/src/tox/package.py b/src/tox/package.py index ec83a993a..03b51fdcf 100644 --- a/src/tox/package.py +++ b/src/tox/package.py @@ -56,7 +56,7 @@ def build_package(config, report, session): if not config.isolated_build: return make_sdist_legacy(report, config, session) else: - build_isolated(config, report, session) + return build_isolated(config, report, session) def make_sdist_legacy(report, config, session): @@ -138,7 +138,8 @@ def abort(message): raise SystemExit(1) if not toml_file.exists(): - abort("missing {}".format(toml_file)) + report.error("missing {}".format(toml_file)) + raise SystemExit(1) with open(toml_file) as file_handler: config_data = toml.load(file_handler) diff --git a/tests/integration/test_package.py b/tests/integration/test_package_int.py similarity index 87% rename from tests/integration/test_package.py rename to tests/integration/test_package_int.py index 09c67b237..64dac45c6 100644 --- a/tests/integration/test_package.py +++ b/tests/integration/test_package_int.py @@ -1,4 +1,5 @@ """Tests that require external access (e.g. pip install, virtualenv creation)""" +import os import subprocess import pytest @@ -59,9 +60,13 @@ def test_package_isolated_build_flit(initproj, cmd): }, add_missing_setup_py=False, ) - subprocess.check_call(["git", "init"]) - subprocess.check_call(["git", "add", "-A", "."]) - subprocess.check_call(["git", "commit", "-m", "first commit"]) + env = os.environ.copy() + env["GIT_COMMITTER_NAME"] = "committer joe" + env["GIT_AUTHOR_NAME"] = "author joe" + env["EMAIL"] = "joe@bloomberg.com" + subprocess.check_call(["git", "init"], env=env) + subprocess.check_call(["git", "add", "-A", "."], env=env) + subprocess.check_call(["git", "commit", "-m", "first commit"], env=env) result = cmd("--sdistonly") assert result.ret == 0, result.out diff --git a/tests/unit/test_docs.py b/tests/unit/test_docs.py index 85d7049be..c3e9b7b70 100644 --- a/tests/unit/test_docs.py +++ b/tests/unit/test_docs.py @@ -18,7 +18,7 @@ RST_FILES = [] -TOX_ROOT = os.path.join(os.path.dirname(os.path.dirname(__file__))) +TOX_ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) for root, _, filenames in os.walk(os.path.join(TOX_ROOT, "doc")): for f in filenames: if f.endswith(".rst"): diff --git a/tests/unit/test_package.py b/tests/unit/test_package.py index 0870950fb..ea885544b 100644 --- a/tests/unit/test_package.py +++ b/tests/unit/test_package.py @@ -179,7 +179,7 @@ def toml_file_check(initproj, version, message, toml): def test_package_isolated_toml_no_build_system(initproj, cmd): - toml_file_check(initproj, "", 1, "build-system section missing") + toml_file_check(initproj, 1, "build-system section missing", "") def test_package_isolated_toml_no_requires(initproj, cmd): diff --git a/tests/unit/test_pytest_plugins.py b/tests/unit/test_pytest_plugins.py index 612800008..2f89564a0 100644 --- a/tests/unit/test_pytest_plugins.py +++ b/tests/unit/test_pytest_plugins.py @@ -16,13 +16,13 @@ class TestInitProj: def test_no_src_root(self, kwargs, tmpdir, initproj): initproj("black_knight-42", **kwargs) init_file = tmpdir.join("black_knight", "black_knight", "__init__.py") - assert init_file.read_binary() == b"__version__ = '42'" + assert init_file.read_binary() == b'""" module black_knight """\n__version__ = \'42\'' def test_existing_src_root(self, tmpdir, initproj): initproj("spam-666", src_root="ham") assert not tmpdir.join("spam", "spam").check(exists=1) init_file = tmpdir.join("spam", "ham", "spam", "__init__.py") - assert init_file.read_binary() == b"__version__ = '666'" + assert init_file.read_binary() == b'""" module spam """\n__version__ = \'666\'' def test_prebuilt_src_dir_with_no_src_root(self, tmpdir, initproj): initproj("spam-1.0", filedefs={"spam": {}}) @@ -56,7 +56,7 @@ def test_broken_py_path_local_join_workaround_on_Windows(self, tmpdir, initproj, initproj("spam-666", src_root=src_root) init_file = tmpdir.join("spam", "spam", "__init__.py") - assert init_file.read_binary() == b"__version__ = '666'" + assert init_file.read_binary() == b'""" module spam """\n__version__ = \'666\'' class TestPathParts: From ac2ce55887b21e8f6463150f30fc3c60fae31652 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Fri, 31 Aug 2018 11:47:45 +0100 Subject: [PATCH 03/14] Fix platform bugs --- src/tox/package.py | 7 ++++--- tests/unit/test_pytest_plugins.py | 17 ++++++++++++++--- tox.ini | 2 +- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/tox/package.py b/src/tox/package.py index 03b51fdcf..44711c957 100644 --- a/src/tox/package.py +++ b/src/tox/package.py @@ -5,6 +5,7 @@ import pkg_resources import py +import six import toml import tox @@ -141,7 +142,7 @@ def abort(message): report.error("missing {}".format(toml_file)) raise SystemExit(1) - with open(toml_file) as file_handler: + with open(str(toml_file)) as file_handler: config_data = toml.load(file_handler) if "build-system" not in config_data: @@ -155,11 +156,11 @@ def abort(message): abort("missing build-backend key at build-system section") requires = build_system["requires"] - if not isinstance(requires, list) or not all(isinstance(i, str) for i in requires): + if not isinstance(requires, list) or not all(isinstance(i, six.text_type) for i in requires): abort("requires key at build-system section must be a list of string") backend = build_system["build-backend"] - if not isinstance(backend, str): + if not isinstance(backend, six.text_type): abort("build-backend key at build-system section must be a string") args = backend.split(":") diff --git a/tests/unit/test_pytest_plugins.py b/tests/unit/test_pytest_plugins.py index 2f89564a0..32f6481ce 100644 --- a/tests/unit/test_pytest_plugins.py +++ b/tests/unit/test_pytest_plugins.py @@ -3,6 +3,8 @@ project test suite, e.g. as shown by the code coverage report. """ +import os + import py.path import pytest @@ -16,13 +18,15 @@ class TestInitProj: def test_no_src_root(self, kwargs, tmpdir, initproj): initproj("black_knight-42", **kwargs) init_file = tmpdir.join("black_knight", "black_knight", "__init__.py") - assert init_file.read_binary() == b'""" module black_knight """\n__version__ = \'42\'' + expected = b'""" module black_knight """' + linesep_bytes() + b"__version__ = '42'" + assert init_file.read_binary() == expected def test_existing_src_root(self, tmpdir, initproj): initproj("spam-666", src_root="ham") assert not tmpdir.join("spam", "spam").check(exists=1) init_file = tmpdir.join("spam", "ham", "spam", "__init__.py") - assert init_file.read_binary() == b'""" module spam """\n__version__ = \'666\'' + expected = b'""" module spam """' + linesep_bytes() + b"__version__ = '666'" + assert init_file.read_binary() == expected def test_prebuilt_src_dir_with_no_src_root(self, tmpdir, initproj): initproj("spam-1.0", filedefs={"spam": {}}) @@ -56,7 +60,14 @@ def test_broken_py_path_local_join_workaround_on_Windows(self, tmpdir, initproj, initproj("spam-666", src_root=src_root) init_file = tmpdir.join("spam", "spam", "__init__.py") - assert init_file.read_binary() == b'""" module spam """\n__version__ = \'666\'' + expected = b'""" module spam """' + linesep_bytes() + b"__version__ = '666'" + assert init_file.read_binary() == expected + + +def linesep_bytes(): + if isinstance(os.linesep, bytes): + return os.linesep + return os.linesep.encode("utf-8") class TestPathParts: diff --git a/tox.ini b/tox.ini index ef8ee1dc9..b41a98fb3 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ passenv = http_proxy https_proxy no_proxy SSL_CERT_FILE TOXENV CI TRAVIS TRAVIS_ deps = extras = testing changedir = {toxinidir}/tests -commands = pytest {posargs:--cov="{envsitepackagesdir}/tox" --cov-config="{toxinidir}/tox.ini" --timeout=180 . -n {env:PYTEST_XDIST_PROC_NR:auto} --junitxml={toxworkdir}/test-results.{envname}.xml } +commands = pytest {posargs:--cov="{envsitepackagesdir}/tox" --cov-config="{toxinidir}/tox.ini" --timeout=180 . -n {env:PYTEST_XDIST_PROC_NR:auto} --junitxml={toxworkdir}/test-results.{envname}.xml} [testenv:docs] description = invoke sphinx-build to build the HTML docs and check that all links are valid From b4095799f31994b226763c11828bcaffa29dacc5 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Fri, 31 Aug 2018 14:19:43 +0100 Subject: [PATCH 04/14] flit not supported on Python 2 --- src/tox/package.py | 5 ++++- tests/integration/test_package_int.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/tox/package.py b/src/tox/package.py index 44711c957..b315facc5 100644 --- a/src/tox/package.py +++ b/src/tox/package.py @@ -180,7 +180,10 @@ def perform_isolated_build(build_info, package_venv, session, config): import {} basename = {}.build_{}("{}", {{ "--global-option": ["--formats=gztar"]}}) print(basename)""".format( - build_info.backend_module, build_info.backend_object, "sdist", config.distdir + build_info.backend_module, + build_info.backend_object, + "sdist", + str(config.distdir).replace("\\", "\\\\"), ) ) config.distdir.ensure_dir() diff --git a/tests/integration/test_package_int.py b/tests/integration/test_package_int.py index 64dac45c6..78d070a28 100644 --- a/tests/integration/test_package_int.py +++ b/tests/integration/test_package_int.py @@ -1,6 +1,7 @@ """Tests that require external access (e.g. pip install, virtualenv creation)""" import os import subprocess +import sys import pytest @@ -35,6 +36,7 @@ def test_package_isolated_build_setuptools(initproj, cmd): @pytest.mark.network @need_git +@pytest.mark.skipif(sys.version_info < (3, 0), reason="flit is Python 3 only") def test_package_isolated_build_flit(initproj, cmd): initproj( "package_toml_flit-0.1", From 1fbd65d39b714f9b7bed41055e70b012668abdfe Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sat, 1 Sep 2018 14:25:31 +0100 Subject: [PATCH 05/14] use !r instead of replace --- src/tox/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tox/package.py b/src/tox/package.py index b315facc5..31a09352b 100644 --- a/src/tox/package.py +++ b/src/tox/package.py @@ -178,12 +178,12 @@ def perform_isolated_build(build_info, package_venv, session, config): """ import sys import {} - basename = {}.build_{}("{}", {{ "--global-option": ["--formats=gztar"]}}) + basename = {}.build_{}({!r}, {{ "--global-option": ["--formats=gztar"]}}) print(basename)""".format( build_info.backend_module, build_info.backend_object, "sdist", - str(config.distdir).replace("\\", "\\\\"), + str(config.distdir), ) ) config.distdir.ensure_dir() From 7034f268622676e60d047f7db6b17aa4d01d2430 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sat, 1 Sep 2018 14:41:29 +0100 Subject: [PATCH 06/14] pull request feedback --- .pre-commit-config.yaml | 2 +- src/tox/_pytestplugin.py | 6 +++--- src/tox/config.py | 5 ++++- src/tox/package.py | 7 ++----- tests/integration/test_package_int.py | 4 ++-- tests/lib/__init__.py | 4 ++-- tests/unit/test_pytest_plugins.py | 4 +--- tox.ini | 2 +- 8 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 28c10ffd5..e4054d81b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: rev: v1.0.1 hooks: - id: seed-isort-config - args: [--application-directories, src] + args: [--application-directories, "src:."] - repo: https://github.com/pre-commit/mirrors-isort rev: v4.3.4 hooks: diff --git a/src/tox/_pytestplugin.py b/src/tox/_pytestplugin.py index e77f7a5d0..9e218cde0 100644 --- a/src/tox/_pytestplugin.py +++ b/src/tox/_pytestplugin.py @@ -323,9 +323,9 @@ def initproj_(nameversion, filedefs=None, src_root=".", add_missing_setup_py=Tru { name: { "__init__.py": textwrap.dedent( - """ - \"\"\" module {} \"\"\" - __version__ = {!r}""" + ''' + """ module {} """ + __version__ = {!r}''' ) .strip() .format(name, version) diff --git a/src/tox/config.py b/src/tox/config.py index 87172ccec..091351ad4 100755 --- a/src/tox/config.py +++ b/src/tox/config.py @@ -1006,8 +1006,11 @@ def __init__(self, config, inipath): # noqa config.skipsdist = reader.getbool("skipsdist", all_develop) config.isolated_build = reader.getbool("isolated_build", False) + config.isolated_build_package_env = reader.getstring( + "isolated_build_package_env", ".package" + ) if config.isolated_build is True: - name = ".package" + name = config.isolated_build_package_env if name not in config.envconfigs: config.envconfigs[name] = self.make_envconfig( name, testenvprefix + name, reader._subs, config diff --git a/src/tox/package.py b/src/tox/package.py index 31a09352b..fae1d9dcf 100644 --- a/src/tox/package.py +++ b/src/tox/package.py @@ -103,7 +103,7 @@ def make_sdist_legacy(report, config, session): def build_isolated(config, report, session): build_info = get_build_info(config.setupdir, report) - package_venv = session.getvenv(".package") + package_venv = session.getvenv(config.isolated_build_package_env) package_venv.envconfig.deps_matches_subset = True package_venv.envconfig.deps = [DepConfig(r, None) for r in build_info.requires] @@ -180,10 +180,7 @@ def perform_isolated_build(build_info, package_venv, session, config): import {} basename = {}.build_{}({!r}, {{ "--global-option": ["--formats=gztar"]}}) print(basename)""".format( - build_info.backend_module, - build_info.backend_object, - "sdist", - str(config.distdir), + build_info.backend_module, build_info.backend_object, "sdist", str(config.distdir) ) ) config.distdir.ensure_dir() diff --git a/tests/integration/test_package_int.py b/tests/integration/test_package_int.py index 78d070a28..0558d0ffd 100644 --- a/tests/integration/test_package_int.py +++ b/tests/integration/test_package_int.py @@ -65,10 +65,10 @@ def test_package_isolated_build_flit(initproj, cmd): env = os.environ.copy() env["GIT_COMMITTER_NAME"] = "committer joe" env["GIT_AUTHOR_NAME"] = "author joe" - env["EMAIL"] = "joe@bloomberg.com" + env["EMAIL"] = "joe@example.com" subprocess.check_call(["git", "init"], env=env) subprocess.check_call(["git", "add", "-A", "."], env=env) - subprocess.check_call(["git", "commit", "-m", "first commit"], env=env) + subprocess.check_call(["git", "commit", "-m", "first commit", "--no-gpg-sign"], env=env) result = cmd("--sdistonly") assert result.ret == 0, result.out diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index 1dbe5d05f..0f711b20e 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -8,11 +8,11 @@ def wrapper(fn): try: subprocess.check_output(check_cmd) except OSError: - return pytest.mark.skip(reason="%s is not available" % name)(fn) + return pytest.mark.skip(reason="{} is not available".format(name))(fn) return fn return wrapper def need_git(fn): - return pytest.mark.mercurial(need_executable("git", ("git", "--version"))(fn)) + return pytest.mark.git(need_executable("git", ("git", "--version"))(fn)) diff --git a/tests/unit/test_pytest_plugins.py b/tests/unit/test_pytest_plugins.py index 32f6481ce..873731b02 100644 --- a/tests/unit/test_pytest_plugins.py +++ b/tests/unit/test_pytest_plugins.py @@ -65,9 +65,7 @@ def test_broken_py_path_local_join_workaround_on_Windows(self, tmpdir, initproj, def linesep_bytes(): - if isinstance(os.linesep, bytes): - return os.linesep - return os.linesep.encode("utf-8") + return os.linesep.encode() class TestPathParts: diff --git a/tox.ini b/tox.ini index b41a98fb3..f33d067ca 100644 --- a/tox.ini +++ b/tox.ini @@ -127,7 +127,7 @@ include_trailing_comma = True force_grid_wrap = 0 line_length = 99 known_first_party = tox,tests -known_third_party = apiclient,git,httplib2,oauth2client,packaging,pkg_resources,pluggy,py,pytest,setuptools,six,tests,toml +known_third_party = apiclient,git,httplib2,oauth2client,packaging,pkg_resources,pluggy,py,pytest,setuptools,six,toml [testenv:release] description = do a release, required posarg of the version number From 53dcb84b4d394daba1377e2c166661d385857195 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sat, 1 Sep 2018 15:43:14 +0100 Subject: [PATCH 07/14] add documentation --- doc/config.rst | 21 ++++++++++++++++++ doc/example/package.rst | 49 +++++++++++++++++++++++++++++++++++++++++ doc/examples.rst | 1 + src/tox/config.py | 6 ++--- src/tox/package.py | 2 +- 5 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 doc/example/package.rst diff --git a/doc/config.rst b/doc/config.rst index ed4b9d420..09ef7addc 100644 --- a/doc/config.rst +++ b/doc/config.rst @@ -79,6 +79,8 @@ and will first lookup global tox settings in this section: .. confval:: requires=LIST + .. versionadded:: 3.2.0 + Specify python packages that need to exist alongside the tox installation for the tox build to be able to start. Use this to specify plugin requirements and build dependencies. @@ -88,6 +90,25 @@ and will first lookup global tox settings in this section: requires = setuptools >= 30.0.0 py +.. confval:: isolated_build=True|False(default) + + .. versionadded:: 3.3.0 + + Activate isolated build environment. tox will use a virtual environment to build + a source distribution from the source tree. For build tools and arguments use + the ``pyproject.toml`` file as specified in + `PEP-517 `_ and + `PEP-518 `_. To specify the virtual + environment Python version define use the :confval:`isolated_build_env` config + section. + +.. confval:: isolated_build_env=str + + .. versionadded:: 3.3.0 + + Name of the virtual environment used to create a source distribution from the + source tree. + Virtualenv test environment settings ------------------------------------ diff --git a/doc/example/package.rst b/doc/example/package.rst new file mode 100644 index 000000000..18c920b6e --- /dev/null +++ b/doc/example/package.rst @@ -0,0 +1,49 @@ +packaging +========= + +Although one can use tox to develop and test applications one of its most popular +usage is to help library creators. Libraries need first to be packaged, so then +they can be installed inside a virtual environment for testing. To help with this +tox implements `PEP-517 `_ and +`PEP-518 `_. This means that by default +tox will build source distribution out of source trees. Before running test commands +``pip`` is used to install the source distribution inside the build environment. + +To create a source distribution there are multiple tools out there and with ``PEP-517`` +and ``PEP-518`` you can easily use your favorite one with tox. Historically tox +only supported ``setuptools``, and always used the tox host environment to build +a source distribution from the source tree. This is still the default behavior. +To opt out of this behaviour you need to set isolated builds to true. + +setuptools +---------- +Using the ``pyproject.toml`` file at the root folder (alongside ``setup.py``) one can specify +build requirements. + +.. code-block:: python + + [build-system] + requires = [ + "setuptools >= 35.0.2", + "setuptools_scm >= 2.0.0, <3" + ] + build-backend = "setuptools.build_meta" + + +flit +---- +`flit `_ requires ``Python 3``, however the generated source +distribution can be installed under ``python 2``. Furthermore it does not require a ``setup.py`` +file as that information is also added to the ``pyproject.toml`` file. + +.. code-block:: python + + [build-system] + requires = ["flit >= 1.1"] + build-backend = "flit.buildapi" + + [tool.flit.metadata] + module = "package_toml_flit" + author = "Happy Harry" + author-email = "happy@harry.com" + home-page = "https://github.com/happy-harry/is" diff --git a/doc/examples.rst b/doc/examples.rst index 6ae0f5885..7975343d1 100644 --- a/doc/examples.rst +++ b/doc/examples.rst @@ -6,6 +6,7 @@ tox configuration and usage examples :maxdepth: 2 example/basic.rst + example/package.rst example/pytest.rst example/unittest example/nose.rst diff --git a/src/tox/config.py b/src/tox/config.py index 091351ad4..ed0cf2a4d 100755 --- a/src/tox/config.py +++ b/src/tox/config.py @@ -1006,11 +1006,11 @@ def __init__(self, config, inipath): # noqa config.skipsdist = reader.getbool("skipsdist", all_develop) config.isolated_build = reader.getbool("isolated_build", False) - config.isolated_build_package_env = reader.getstring( - "isolated_build_package_env", ".package" + config.isolated_build_env = reader.getstring( + "isolated_build_env", ".package" ) if config.isolated_build is True: - name = config.isolated_build_package_env + name = config.isolated_build_env if name not in config.envconfigs: config.envconfigs[name] = self.make_envconfig( name, testenvprefix + name, reader._subs, config diff --git a/src/tox/package.py b/src/tox/package.py index fae1d9dcf..72ecc4093 100644 --- a/src/tox/package.py +++ b/src/tox/package.py @@ -103,7 +103,7 @@ def make_sdist_legacy(report, config, session): def build_isolated(config, report, session): build_info = get_build_info(config.setupdir, report) - package_venv = session.getvenv(config.isolated_build_package_env) + package_venv = session.getvenv(config.isolated_build_env) package_venv.envconfig.deps_matches_subset = True package_venv.envconfig.deps = [DepConfig(r, None) for r in build_info.requires] From 21773440a4037850d649c1161b2d7aded8c7b58e Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sat, 1 Sep 2018 15:48:26 +0100 Subject: [PATCH 08/14] document default isolated build env --- doc/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/config.rst b/doc/config.rst index 09ef7addc..4b3679e24 100644 --- a/doc/config.rst +++ b/doc/config.rst @@ -107,7 +107,7 @@ and will first lookup global tox settings in this section: .. versionadded:: 3.3.0 Name of the virtual environment used to create a source distribution from the - source tree. + source tree. By **default ``.package``** is used. Virtualenv test environment settings From e69f5bebb6e9a7b190d5580665433da39ba83ab9 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sun, 2 Sep 2018 14:18:01 +0100 Subject: [PATCH 09/14] fix documentation bugs --- doc/example/package.rst | 15 +++++++++++++-- src/tox/config.py | 4 +--- tox.ini | 4 ++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/doc/example/package.rst b/doc/example/package.rst index 18c920b6e..34117391a 100644 --- a/doc/example/package.rst +++ b/doc/example/package.rst @@ -20,7 +20,7 @@ setuptools Using the ``pyproject.toml`` file at the root folder (alongside ``setup.py``) one can specify build requirements. -.. code-block:: python +.. code-block:: [build-system] requires = [ @@ -29,6 +29,11 @@ build requirements. ] build-backend = "setuptools.build_meta" +.. code-block:: ini + + # tox.ini + [tox] + build_isolated = True flit ---- @@ -36,7 +41,7 @@ flit distribution can be installed under ``python 2``. Furthermore it does not require a ``setup.py`` file as that information is also added to the ``pyproject.toml`` file. -.. code-block:: python +.. code-block:: [build-system] requires = ["flit >= 1.1"] @@ -47,3 +52,9 @@ file as that information is also added to the ``pyproject.toml`` file. author = "Happy Harry" author-email = "happy@harry.com" home-page = "https://github.com/happy-harry/is" + +.. code-block:: ini + + # tox.ini + [tox] + build_isolated = True diff --git a/src/tox/config.py b/src/tox/config.py index ed0cf2a4d..38f1ec34a 100755 --- a/src/tox/config.py +++ b/src/tox/config.py @@ -1006,9 +1006,7 @@ def __init__(self, config, inipath): # noqa config.skipsdist = reader.getbool("skipsdist", all_develop) config.isolated_build = reader.getbool("isolated_build", False) - config.isolated_build_env = reader.getstring( - "isolated_build_env", ".package" - ) + config.isolated_build_env = reader.getstring("isolated_build_env", ".package") if config.isolated_build is True: name = config.isolated_build_env if name not in config.envconfigs: diff --git a/tox.ini b/tox.ini index f33d067ca..0af688412 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,7 @@ basepython = python3.7 extras = docs changedir = {toxinidir} commands = sphinx-build -d "{toxworkdir}/docs_doctree" doc "{toxworkdir}/docs_out" --color -W -bhtml {posargs} - python -c 'print("documentation available under file://{toxworkdir}/docs_out/index.html")' + python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))' [testenv:package-description] description = check that the long description is valid @@ -52,7 +52,7 @@ deps = pre-commit == 1.10.3 skip_install = True changedir = {toxinidir} commands = pre-commit run --all-files --show-diff-on-failure - python -c 'print("hint: run {envdir}/bin/pre-commit install to add checks as pre-commit hook")' + python -c 'import pathlib; print("hint: run \{\} install to add checks as pre-commit hook".format(pathlib.Path(r"{envdir}") / "bin" / "pre-commit"))' [testenv:coverage] From 43011c48d87c31703939a707c56087525a7c4160 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sun, 2 Sep 2018 21:04:43 +0100 Subject: [PATCH 10/14] allow expanding defintion --- src/tox/config.py | 3 +++ src/tox/package.py | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/tox/config.py b/src/tox/config.py index 38f1ec34a..cfc428b3c 100755 --- a/src/tox/config.py +++ b/src/tox/config.py @@ -1005,6 +1005,9 @@ def __init__(self, config, inipath): # noqa ) config.skipsdist = reader.getbool("skipsdist", all_develop) + self.parse_build_isolation(config, reader) + + def parse_build_isolation(self, config, reader): config.isolated_build = reader.getbool("isolated_build", False) config.isolated_build_env = reader.getstring("isolated_build_env", ".package") if config.isolated_build is True: diff --git a/src/tox/package.py b/src/tox/package.py index 72ecc4093..2909c55e3 100644 --- a/src/tox/package.py +++ b/src/tox/package.py @@ -106,7 +106,8 @@ def build_isolated(config, report, session): package_venv = session.getvenv(config.isolated_build_env) package_venv.envconfig.deps_matches_subset = True - package_venv.envconfig.deps = [DepConfig(r, None) for r in build_info.requires] + build_requires_deps = [DepConfig(r, None) for r in build_info.requires] + package_venv.envconfig.deps.extend(build_requires_deps) toml_require = {pkg_resources.Requirement(r).key for r in build_info.requires} if not session.setupenv(package_venv): raise SystemExit(1) From b501922eeff8d6c64e2204084e9673eb3fe8d6f0 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 3 Sep 2018 07:55:42 +0100 Subject: [PATCH 11/14] documentation fix --- doc/example/package.rst | 4 ++-- setup.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/example/package.rst b/doc/example/package.rst index 34117391a..a5d5e741e 100644 --- a/doc/example/package.rst +++ b/doc/example/package.rst @@ -20,7 +20,7 @@ setuptools Using the ``pyproject.toml`` file at the root folder (alongside ``setup.py``) one can specify build requirements. -.. code-block:: +.. code-block:: toml [build-system] requires = [ @@ -41,7 +41,7 @@ flit distribution can be installed under ``python 2``. Furthermore it does not require a ``setup.py`` file as that information is also added to the ``pyproject.toml`` file. -.. code-block:: +.. code-block:: toml [build-system] requires = ["flit >= 1.1"] diff --git a/setup.py b/setup.py index 8e59b9c9d..471ce2657 100644 --- a/setup.py +++ b/setup.py @@ -77,7 +77,11 @@ def main(): "pytest-xdist >= 1.22.2, <2", "pytest-randomly >= 1.2.3, <2", ], - "docs": ["sphinx >= 1.7.5, < 2", "towncrier >= 18.5.0"], + "docs": [ + "sphinx >= 1.7.5, < 2", + "towncrier >= 18.5.0", + "pygments-github-lexers >= 0.0.5", + ], }, classifiers=[ "Development Status :: 5 - Production/Stable", From 738b078ffd12d0401a935ab98f60ea5b1574b703 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 3 Sep 2018 15:29:58 +0100 Subject: [PATCH 12/14] just install the extra dependencies rather than re-create the env --- src/tox/package.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/tox/package.py b/src/tox/package.py index 2909c55e3..54e6558b2 100644 --- a/src/tox/package.py +++ b/src/tox/package.py @@ -10,7 +10,6 @@ import tox from tox.config import DepConfig -from tox.venv import CreationConfig BuildInfo = namedtuple("BuildInfo", ["requires", "backend_module", "backend_object"]) @@ -106,27 +105,29 @@ def build_isolated(config, report, session): package_venv = session.getvenv(config.isolated_build_env) package_venv.envconfig.deps_matches_subset = True - build_requires_deps = [DepConfig(r, None) for r in build_info.requires] - package_venv.envconfig.deps.extend(build_requires_deps) - toml_require = {pkg_resources.Requirement(r).key for r in build_info.requires} - if not session.setupenv(package_venv): - raise SystemExit(1) + # we allow user specified dependencies so the users can write extensions to + # install additional type of dependencies (e.g. binary) + user_specified_deps = package_venv.envconfig.deps + package_venv.envconfig.deps = [DepConfig(r, None) for r in build_info.requires] + package_venv.envconfig.deps.extend(user_specified_deps) - live_config = package_venv._getliveconfig() - previous_config = CreationConfig.readconfig(package_venv.path_config) - if not previous_config or not previous_config.matches(live_config, True): + if not session.setupenv(package_venv): session.finishvenv(package_venv) build_requires = get_build_requires(build_info, package_venv, session) - for requirement in build_requires: - pkg_requirement = pkg_resources.Requirement(requirement) - if pkg_requirement.key not in toml_require: - package_venv.envconfig.deps.append(DepConfig(requirement, None)) - - if not session.setupenv(package_venv): - raise SystemExit(1) - - session.finishvenv(package_venv) + # we need to filter out requirements already specified in pyproject.toml or user deps + base_build_deps = {pkg_resources.Requirement(r.name).key for r in package_venv.envconfig.deps} + build_requires_dep = [ + DepConfig(r, None) + for r in build_requires + if pkg_resources.Requirement(r).key not in base_build_deps + ] + if build_requires_dep: + with session.newaction( + package_venv, "build_requires", package_venv.envconfig.envdir + ) as action: + package_venv.run_install_command(packages=build_requires_dep, action=action) + session.finishvenv(package_venv) return perform_isolated_build(build_info, package_venv, session, config) From db365311e0a215e484408fe667a8d37e2d5bc602 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sun, 9 Sep 2018 11:49:48 +0100 Subject: [PATCH 13/14] ensure isolated build env is not in list configs, and cannot be part of envlist --- src/tox/config.py | 3 + src/tox/session.py | 4 +- tests/unit/session/__init__.py | 0 tests/unit/session/test_list_env.py | 152 +++++++++++++++++++++++ tests/unit/{ => session}/test_session.py | 0 tests/unit/test_config.py | 148 +++------------------- 6 files changed, 175 insertions(+), 132 deletions(-) create mode 100644 tests/unit/session/__init__.py create mode 100644 tests/unit/session/test_list_env.py rename tests/unit/{ => session}/test_session.py (100%) diff --git a/src/tox/config.py b/src/tox/config.py index cfc428b3c..442011aeb 100755 --- a/src/tox/config.py +++ b/src/tox/config.py @@ -1016,6 +1016,9 @@ def parse_build_isolation(self, config, reader): config.envconfigs[name] = self.make_envconfig( name, testenvprefix + name, reader._subs, config ) + if config.isolated_build_env in config.envlist: + msg = "isolated_build_env {} cannot be part of envlist".format(name) + raise tox.exception.ConfigError(msg) def _make_thread_safe_path(self, config, attr, unique_id): if config.option.parallel_safe_build: diff --git a/src/tox/session.py b/src/tox/session.py index d13e7d79f..ae6eb283a 100644 --- a/src/tox/session.py +++ b/src/tox/session.py @@ -619,7 +619,9 @@ def showconfig(self): def showenvs(self, all_envs=False, description=False): env_conf = self.config.envconfigs # this contains all environments default = self.config.envlist # this only the defaults - extra = sorted(e for e in env_conf if e not in default) if all_envs else [] + ignore = {self.config.isolated_build_env}.union(default) + extra = sorted(e for e in env_conf if e not in ignore) if all_envs else [] + if description: self.report.line("default environments:") max_length = max(len(env) for env in (default + extra)) diff --git a/tests/unit/session/__init__.py b/tests/unit/session/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/session/test_list_env.py b/tests/unit/session/test_list_env.py new file mode 100644 index 000000000..8ccee8979 --- /dev/null +++ b/tests/unit/session/test_list_env.py @@ -0,0 +1,152 @@ +def test_listenvs(cmd, initproj): + initproj( + "listenvs", + filedefs={ + "tox.ini": """ + [tox] + envlist=py36,py27,py34,pypi,docs + description= py27: run pytest on Python 2.7 + py34: run pytest on Python 3.6 + pypi: publish to PyPI + docs: document stuff + notincluded: random extra + + [testenv:notincluded] + changedir = whatever + + [testenv:docs] + changedir = docs + """ + }, + ) + result = cmd("-l") + assert result.outlines == ["py36", "py27", "py34", "pypi", "docs"] + + +def test_listenvs_verbose_description(cmd, initproj): + initproj( + "listenvs_verbose_description", + filedefs={ + "tox.ini": """ + [tox] + envlist=py36,py27,py34,pypi,docs + [testenv] + description= py36: run pytest on Python 3.6 + py27: run pytest on Python 2.7 + py34: run pytest on Python 3.4 + pypi: publish to PyPI + docs: document stuff + notincluded: random extra + + [testenv:notincluded] + changedir = whatever + + [testenv:docs] + changedir = docs + description = let me overwrite that + """ + }, + ) + result = cmd("-lv") + expected = [ + "default environments:", + "py36 -> run pytest on Python 3.6", + "py27 -> run pytest on Python 2.7", + "py34 -> run pytest on Python 3.4", + "pypi -> publish to PyPI", + "docs -> let me overwrite that", + ] + assert result.outlines[2:] == expected + + +def test_listenvs_all(cmd, initproj): + initproj( + "listenvs_all", + filedefs={ + "tox.ini": """ + [tox] + envlist=py36,py27,py34,pypi,docs + + [testenv:notincluded] + changedir = whatever + + [testenv:docs] + changedir = docs + """ + }, + ) + result = cmd("-a") + expected = ["py36", "py27", "py34", "pypi", "docs", "notincluded"] + assert result.outlines == expected + + +def test_listenvs_all_verbose_description(cmd, initproj): + initproj( + "listenvs_all_verbose_description", + filedefs={ + "tox.ini": """ + [tox] + envlist={py27,py36}-{windows,linux} # py35 + [testenv] + description= py27: run pytest on Python 2.7 + py36: run pytest on Python 3.6 + windows: on Windows platform + linux: on Linux platform + docs: generate documentation + commands=pytest {posargs} + + [testenv:docs] + changedir = docs + """ + }, + ) + result = cmd("-av") + expected = [ + "default environments:", + "py27-windows -> run pytest on Python 2.7 on Windows platform", + "py27-linux -> run pytest on Python 2.7 on Linux platform", + "py36-windows -> run pytest on Python 3.6 on Windows platform", + "py36-linux -> run pytest on Python 3.6 on Linux platform", + "", + "additional environments:", + "docs -> generate documentation", + ] + assert result.outlines[-len(expected) :] == expected + + +def test_listenvs_all_verbose_description_no_additional_environments(cmd, initproj): + initproj( + "listenvs_all_verbose_description", + filedefs={ + "tox.ini": """ + [tox] + envlist=py27,py36 + """ + }, + ) + result = cmd("-av") + expected = ["default environments:", "py27 -> [no description]", "py36 -> [no description]"] + assert result.out.splitlines()[-3:] == expected + assert "additional environments" not in result.out + + +def test_listenvs_packaging_excluded(cmd, initproj): + initproj( + "listenvs", + filedefs={ + "tox.ini": """ + [tox] + envlist = py36,py27,py34,pypi,docs + isolated_build = True + + [testenv:notincluded] + changedir = whatever + + [testenv:docs] + changedir = docs + """ + }, + ) + result = cmd("-a") + expected = ["py36", "py27", "py34", "pypi", "docs", "notincluded"] + assert result.outlines == expected, result.outlines diff --git a/tests/unit/test_session.py b/tests/unit/session/test_session.py similarity index 100% rename from tests/unit/test_session.py rename to tests/unit/session/test_session.py diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index a21abd85e..cf458e99a 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -2376,137 +2376,6 @@ class MockEggInfo: assert "some-repr" in version_info assert "1.0" in version_info - def test_listenvs(self, cmd, initproj): - initproj( - "listenvs", - filedefs={ - "tox.ini": """ - [tox] - envlist=py36,py27,py34,pypi,docs - description= py27: run pytest on Python 2.7 - py34: run pytest on Python 3.6 - pypi: publish to PyPI - docs: document stuff - notincluded: random extra - - [testenv:notincluded] - changedir = whatever - - [testenv:docs] - changedir = docs - """ - }, - ) - result = cmd("-l") - assert result.outlines == ["py36", "py27", "py34", "pypi", "docs"] - - def test_listenvs_verbose_description(self, cmd, initproj): - initproj( - "listenvs_verbose_description", - filedefs={ - "tox.ini": """ - [tox] - envlist=py36,py27,py34,pypi,docs - [testenv] - description= py36: run pytest on Python 3.6 - py27: run pytest on Python 2.7 - py34: run pytest on Python 3.4 - pypi: publish to PyPI - docs: document stuff - notincluded: random extra - - [testenv:notincluded] - changedir = whatever - - [testenv:docs] - changedir = docs - description = let me overwrite that - """ - }, - ) - result = cmd("-lv") - expected = [ - "default environments:", - "py36 -> run pytest on Python 3.6", - "py27 -> run pytest on Python 2.7", - "py34 -> run pytest on Python 3.4", - "pypi -> publish to PyPI", - "docs -> let me overwrite that", - ] - assert result.outlines[2:] == expected - - def test_listenvs_all(self, cmd, initproj): - initproj( - "listenvs_all", - filedefs={ - "tox.ini": """ - [tox] - envlist=py36,py27,py34,pypi,docs - - [testenv:notincluded] - changedir = whatever - - [testenv:docs] - changedir = docs - """ - }, - ) - result = cmd("-a") - expected = ["py36", "py27", "py34", "pypi", "docs", "notincluded"] - assert result.outlines == expected - - def test_listenvs_all_verbose_description(self, cmd, initproj): - initproj( - "listenvs_all_verbose_description", - filedefs={ - "tox.ini": """ - [tox] - envlist={py27,py36}-{windows,linux} # py35 - [testenv] - description= py27: run pytest on Python 2.7 - py36: run pytest on Python 3.6 - windows: on Windows platform - linux: on Linux platform - docs: generate documentation - commands=pytest {posargs} - - [testenv:docs] - changedir = docs - """ - }, - ) - result = cmd("-av") - expected = [ - "default environments:", - "py27-windows -> run pytest on Python 2.7 on Windows platform", - "py27-linux -> run pytest on Python 2.7 on Linux platform", - "py36-windows -> run pytest on Python 3.6 on Windows platform", - "py36-linux -> run pytest on Python 3.6 on Linux platform", - "", - "additional environments:", - "docs -> generate documentation", - ] - assert result.outlines[-len(expected) :] == expected - - def test_listenvs_all_verbose_description_no_additional_environments(self, cmd, initproj): - initproj( - "listenvs_all_verbose_description", - filedefs={ - "tox.ini": """ - [tox] - envlist=py27,py36 - """ - }, - ) - result = cmd("-av") - expected = [ - "default environments:", - "py27 -> [no description]", - "py36 -> [no description]", - ] - assert result.out.splitlines()[-3:] == expected - assert "additional environments" not in result.out - def test_config_specific_ini(self, tmpdir, cmd): ini = tmpdir.ensure("hello.ini") result = cmd("-c", ini, "--showconfig") @@ -2738,3 +2607,20 @@ def test_plugin_require(newconfig, capsys): ] ) assert not out + + +def test_isolated_build_env_cannot_be_in_envlist(newconfig, capsys): + inisource = """ + [tox] + envlist = py36,package + isolated_build = True + isolated_build_env = package + """ + with pytest.raises( + tox.exception.ConfigError, match="isolated_build_env package cannot be part of envlist" + ): + newconfig([], inisource) + + out, err = capsys.readouterr() + assert not err + assert not out From ba9572f69a0a20ac23a67f720287636ac83a9b2c Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 10 Sep 2018 09:24:20 +0100 Subject: [PATCH 14/14] Ensure isolated env package is not in envlist --- src/tox/config.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/tox/config.py b/src/tox/config.py index 442011aeb..70dcb4253 100755 --- a/src/tox/config.py +++ b/src/tox/config.py @@ -974,7 +974,8 @@ def __init__(self, config, inipath): # noqa config.logdir = config.toxworkdir.join("log") self._make_thread_safe_path(config, "logdir", unique_id) - config.envlist, all_envs = self._getenvdata(reader) + self.parse_build_isolation(config, reader) + config.envlist, all_envs = self._getenvdata(reader, config) # factors used in config or predefined known_factors = self._list_section_factors("testenv") @@ -1005,7 +1006,6 @@ def __init__(self, config, inipath): # noqa ) config.skipsdist = reader.getbool("skipsdist", all_develop) - self.parse_build_isolation(config, reader) def parse_build_isolation(self, config, reader): config.isolated_build = reader.getbool("isolated_build", False) @@ -1016,9 +1016,6 @@ def parse_build_isolation(self, config, reader): config.envconfigs[name] = self.make_envconfig( name, testenvprefix + name, reader._subs, config ) - if config.isolated_build_env in config.envlist: - msg = "isolated_build_env {} cannot be part of envlist".format(name) - raise tox.exception.ConfigError(msg) def _make_thread_safe_path(self, config, attr, unique_id): if config.option.parallel_safe_build: @@ -1083,7 +1080,7 @@ def make_envconfig(self, name, section, subs, config, replace=True): reader.addsubstitutions(**{env_attr.name: res}) return tc - def _getenvdata(self, reader): + def _getenvdata(self, reader, config): candidates = ( self.config.option.env, os.environ.get("TOXENV"), @@ -1100,9 +1097,17 @@ def _getenvdata(self, reader): if not all_envs: all_envs.add("python") + package_env = config.isolated_build_env + if config.isolated_build is True and package_env in all_envs: + all_envs.remove(package_env) + if not env_list or "ALL" in env_list: env_list = sorted(all_envs) + if config.isolated_build is True and package_env in env_list: + msg = "isolated_build_env {} cannot be part of envlist".format(package_env) + raise tox.exception.ConfigError(msg) + return env_list, all_envs