From a94933e7ef36e62c13ee7c1ab440475743363ce0 Mon Sep 17 00:00:00 2001 From: Obeida Shamoun <2135622+oshmoun@users.noreply.github.com> Date: Tue, 10 Jan 2023 03:34:35 +0100 Subject: [PATCH] [WIP] Make plugin compatible with tox v4 --- tests/fixtures.py | 11 +- tests/test_installer.py | 10 +- tox.ini | 7 +- tox_poetry_installer/__init__.py | 5 +- tox_poetry_installer/hooks.py | 240 ++++++++---------------------- tox_poetry_installer/installer.py | 6 +- tox_poetry_installer/logger.py | 18 +-- tox_poetry_installer/utilities.py | 40 ++--- 8 files changed, 101 insertions(+), 236 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index e50633a..8ffa296 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -6,7 +6,7 @@ import poetry.factory import poetry.installation.pip_installer import poetry.utils.env import pytest -import tox +import tox.tox_env.python.virtual_env.runner from poetry.core.packages.package import Package as PoetryPackage from tox_poetry_installer import utilities @@ -20,11 +20,8 @@ FAKE_VENV_PATH = Path("nowhere") class MockVirtualEnv: """Mock class for the :class:`poetry.utils.env.VirtualEnv` and :class:`tox.venv.VirtualEnv`""" - class MockTestenvConfig: # pylint: disable=missing-class-docstring - envdir = FAKE_VENV_PATH - def __init__(self, *args, **kwargs): - self.envconfig = self.MockTestenvConfig() + self.env_dir = FAKE_VENV_PATH self.installed = [] @staticmethod @@ -53,7 +50,9 @@ def mock_venv(monkeypatch): monkeypatch.setattr( poetry.installation.pip_installer, "PipInstaller", MockPipInstaller ) - monkeypatch.setattr(tox.venv, "VirtualEnv", MockVirtualEnv) + monkeypatch.setattr( + tox.tox_env.python.virtual_env.runner, "VirtualEnvRunner", MockVirtualEnv + ) monkeypatch.setattr(poetry.utils.env, "VirtualEnv", MockVirtualEnv) diff --git a/tests/test_installer.py b/tests/test_installer.py index fa6073d..4926893 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -3,7 +3,7 @@ import time from unittest import mock import pytest -import tox.venv +import tox.tox_env.python.virtual_env.runner from poetry.factory import Factory from .fixtures import mock_poetry_factory @@ -19,7 +19,7 @@ def test_deduplication(mock_venv, mock_poetry_factory): item.name: item for item in poetry.locker.locked_repository().packages } - venv = tox.venv.VirtualEnv() + venv = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner() to_install = [packages["toml"], packages["toml"]] installer.install(poetry, venv, to_install) @@ -43,12 +43,12 @@ def test_parallelization(mock_venv, mock_poetry_factory): packages["attrs"], ] - venv_sequential = tox.venv.VirtualEnv() + venv_sequential = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner() start_sequential = time.time() installer.install(poetry, venv_sequential, to_install, 0) sequential = time.time() - start_sequential - venv_parallel = tox.venv.VirtualEnv() + venv_parallel = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner() start_parallel = time.time() installer.install(poetry, venv_parallel, to_install, 5) parallel = time.time() - start_parallel @@ -76,7 +76,7 @@ def test_propagates_exceptions_during_installation( item.name: item for item in poetry.locker.locked_repository().packages } to_install = [packages["toml"]] - venv = tox.venv.VirtualEnv() + venv = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner() fake_exception = ValueError("my testing exception") with mock.patch.object( diff --git a/tox.ini b/tox.ini index 06dce0c..fe639a5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,5 @@ [tox] envlist = py37, py38, py39, py310, py311, static, static-tests, security -isolated_build = true skip_missing_interpreters = true [testenv] @@ -21,7 +20,7 @@ commands = [testenv:static] description = Static formatting and quality enforcement -basepython = python3.10 +basepython = py3.10 platform = linux ignore_errors = true locked_deps = @@ -46,7 +45,7 @@ commands = [testenv:static-tests] description = Static formatting and quality enforcement for the tests -basepython = python3.10 +basepython = py3.10 platform = linux ignore_errors = true locked_deps = @@ -63,7 +62,7 @@ commands = [testenv:security] description = Security checks -basepython = python3.10 +basepython = py3.10 platform = linux ignore_errors = true skip_install = true diff --git a/tox_poetry_installer/__init__.py b/tox_poetry_installer/__init__.py index 7e22471..46e0b0a 100644 --- a/tox_poetry_installer/__init__.py +++ b/tox_poetry_installer/__init__.py @@ -1,3 +1,4 @@ # pylint: disable=missing-docstring -from tox_poetry_installer.hooks import tox_addoption -from tox_poetry_installer.hooks import tox_testenv_install_deps +from tox_poetry_installer.hooks import tox_add_core_config +from tox_poetry_installer.hooks import tox_add_env_config +from tox_poetry_installer.hooks import tox_on_install diff --git a/tox_poetry_installer/hooks.py b/tox_poetry_installer/hooks.py index ef1ed31..f87a9f6 100644 --- a/tox_poetry_installer/hooks.py +++ b/tox_poetry_installer/hooks.py @@ -5,14 +5,13 @@ specifically related to implementing the hooks (to keep the size/readability of themselves manageable). """ from itertools import chain -from typing import Optional +from typing import List -import tox -from tox.action import Action as ToxAction -from tox.config import Parser as ToxParser -from tox.venv import VirtualEnv as ToxVirtualEnv +from tox.config.sets import ConfigSet +from tox.config.sets import EnvConfigSet +from tox.plugin import impl +from tox.tox_env.api import ToxEnv as ToxVirtualEnv -from tox_poetry_installer import __about__ from tox_poetry_installer import constants from tox_poetry_installer import exceptions from tox_poetry_installer import installer @@ -20,142 +19,61 @@ from tox_poetry_installer import logger from tox_poetry_installer import utilities -def _postprocess_install_project_deps( - testenv_config, value: Optional[str] # pylint: disable=unused-argument -) -> Optional[bool]: - """An awful hack to patch on three-state boolean logic to a config parameter +@impl +def tox_add_core_config(core_conf: ConfigSet): + """Add required core configuration options to the tox INI file""" - .. warning: This logic should 100% be removed in the next feature release. It's here to work - around a bad design for now but should not persist. - - The bug filed in `#61`_ is caused by a combination of poor design and attempted cleverness. The - name of the ``install_project_deps`` config option implies that it has ultimate control over - whether the project dependencies are installed to the testenv, but this is not actually correct. - What it actually allows the user to do is force the project dependencies to not be installed to - an environment that would otherwise install them. This was intended behavior, however the - intention was wrong. - - .. _`#61`: https://github.com/enpaul/tox-poetry-installer/issues/61 - - In an effort to be clever the plugin automatically skips installing project dependencies when - the project package is not installed to the testenv (``skip_install = true``) or if packaging - as a whole is disabled (``skipsdist = true``). The intention of this behavior is to install only - the expected dependencies to a testenv and no more. However, this conflicts with the - ``install_project_deps`` config option, which cannot override this behavior because it defaults - to ``True``. In effect, ``install_project_deps = true`` in fact means "automatically - determine whether to install project dependencies" and ``install_project_deps = false`` means - "never install the project dependencies". This is not ideal and unintuitive. - - To avoid having to make a breaking change this workaround has been added to support three-state - logic between ``True``, ``False``, and ``None``. The ``install_project_deps`` option is now - parsed by Tox as a string with a default value of ``None``. If the value is not ``None`` then - this post processing function will try to convert it to a boolean the same way that Tox's - `SectionReader.getbool()`_ method does, raising an error to mimic the default behavior if it - can't. - - .. _`SectionReader.getbool()`: https://github.com/tox-dev/tox/blob/f8459218ee5ab5753321b3eb989b7beee5b391ad/src/tox/config/__init__.py#L1724 - - The three states for the ``install_project_deps`` setting are: - * ``None`` - User did not configure the setting, package dependency installation is - determined automatically - * ``True`` - User configured the setting to ``True``, package dependencies will be installed - * ``False`` - User configured the setting to ``False``, package dependencies will not be - installed - - This config option should be deprecated with the 1.0.0 release and instead an option like - ``always_install_project_deps`` should be added which overrides the default determination and - just installs the project dependencies. The counterpart (``never_install_project_deps``) - shouldn't be needed, since I don't think there's a real use case for that. - """ - if value is None: - return value - - if value.lower() == "true": - return True - if value.lower() == "false": - return False - - raise tox.exception.ConfigError( - f"install_project_deps: boolean value '{value}' needs to be 'True' or 'False'" - ) - - -@tox.hookimpl -def tox_addoption(parser: ToxParser): - """Add required configuration options to the tox INI file - - Adds the ``require_locked_deps`` configuration option to the venv to check whether all - dependencies should be treated as locked or not. - """ - - parser.add_argument( - "--require-poetry", - action="store_true", - dest="require_poetry", - help="(deprecated) Trigger a failure if Poetry is not available to Tox", - ) - - parser.add_argument( - "--parallelize-locked-install", - type=int, - dest="parallelize_locked_install", - default=None, - help="(deprecated) Number of worker threads to use for installing dependencies from the Poetry lockfile in parallel", - ) - - parser.add_argument( - "--parallel-install-threads", - type=int, - dest="parallel_install_threads", + core_conf.add_config( + "parallel_install_threads", + of_type=int, default=constants.DEFAULT_INSTALL_THREADS, - help="Number of locked dependencies to install simultaneously; set to 0 to disable parallel installation", + desc="Number of locked dependencies to install simultaneously; set to 0 to disable parallel installation", ) - parser.add_testenv_attribute( - name="install_dev_deps", - type="bool", - default=False, - help="(deprecated) Automatically install all Poetry development dependencies to the environment", - ) - parser.add_testenv_attribute( - name="poetry_dep_groups", - type="line-list", +@impl +def tox_add_env_config(env_conf: EnvConfigSet): + """Add required env configuration options to the tox INI file""" + env_conf.add_config( + "poetry_dep_groups", + of_type=List[str], default=[], - help="List of Poetry dependency groups to install to the environment", + desc="List of Poetry dependency groups to install to the environment", ) - parser.add_testenv_attribute( - name="install_project_deps", - type="string", - default=None, - help="Automatically install all Poetry primary dependencies to the environment", - postprocess=_postprocess_install_project_deps, + env_conf.add_config( + "install_project_deps", + of_type=bool, + default=True, + desc="Automatically install all Poetry primary dependencies to the environment", ) - parser.add_testenv_attribute( - name="require_locked_deps", - type="bool", + env_conf.add_config( + "require_locked_deps", + of_type=bool, default=False, - help="Require all dependencies in the environment be installed using the Poetry lockfile", + desc="Require all dependencies in the environment be installed using the Poetry lockfile", ) - parser.add_testenv_attribute( - name="require_poetry", - type="bool", + env_conf.add_config( + "require_poetry", + of_type=bool, default=False, - help="Trigger a failure if Poetry is not available to Tox", + desc="Trigger a failure if Poetry is not available to Tox", ) - parser.add_testenv_attribute( - name="locked_deps", - type="line-list", - help="List of locked dependencies to install to the environment using the Poetry lockfile", + env_conf.add_config( + "locked_deps", + of_type=List[str], + default=[], + desc="List of locked dependencies to install to the environment using the Poetry lockfile", ) -@tox.hookimpl -def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional[bool]: +@impl +def tox_on_install( + tox_env: ToxVirtualEnv, section: str # pylint: disable=unused-argument +) -> None: """Install the dependencies for the current environment Loads the local Poetry environment and the corresponding lockfile then pulls the dependencies @@ -165,22 +83,20 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional :param venv: Tox virtual environment object with configuration for the local Tox environment. :param action: Tox action object """ - try: - poetry = utilities.check_preconditions(venv, action) + poetry = utilities.check_preconditions(tox_env) except exceptions.SkipEnvironment as err: if isinstance(err, exceptions.PoetryNotInstalledError) and ( - venv.envconfig.config.option.require_poetry or venv.envconfig.require_poetry + tox_env.core["require_poetry"] or tox_env.conf["require_poetry"] ): - venv.status = err.__class__.__name__ logger.error(str(err)) - return False + raise err logger.info(str(err)) - return None + return logger.info(f"Loaded project pyproject.toml from {poetry.file}") - virtualenv = utilities.convert_virtualenv(venv) + virtualenv = utilities.convert_virtualenv(tox_env) if not poetry.locker.is_fresh(): logger.warning( @@ -188,28 +104,19 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional ) try: - if venv.envconfig.require_locked_deps and venv.envconfig.deps: + if tox_env.conf["require_locked_deps"] and tox_env.conf["deps"].lines(): raise exceptions.LockedDepsRequiredError( - f"Unlocked dependencies '{venv.envconfig.deps}' specified for environment '{venv.name}' which requires locked dependencies" + f"Unlocked dependencies '{tox_env.conf['deps']}' specified for environment '{tox_env.name}' which requires locked dependencies" ) packages = utilities.build_package_map(poetry) - if venv.envconfig.install_dev_deps: - dev_deps = utilities.find_dev_deps(packages, virtualenv, poetry) - logger.info( - f"Identified {len(dev_deps)} development dependencies to install to env" - ) - else: - dev_deps = [] - logger.info("Env does not install development dependencies, skipping") - group_deps = utilities.dedupe_packages( list( chain( *[ utilities.find_group_deps(group, packages, virtualenv, poetry) - for group in venv.envconfig.poetry_dep_groups + for group in tox_env.conf["poetry_dep_groups"] ] ) ) @@ -219,7 +126,7 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional ) env_deps = utilities.find_additional_deps( - packages, virtualenv, poetry, venv.envconfig.locked_deps + packages, virtualenv, poetry, tox_env.conf["locked_deps"] ) logger.info( @@ -227,16 +134,20 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional ) install_project_deps = ( - venv.envconfig.install_project_deps - if venv.envconfig.install_project_deps is not None - else ( - not venv.envconfig.skip_install and not venv.envconfig.config.skipsdist - ) + tox_env.conf["install_project_deps"] + if tox_env.conf["install_project_deps"] is not None + else (not tox_env.conf["skip_install"] and not tox_env.core["no_package"]) ) + # extras are not set in a testenv if skip_install=true + try: + extras = tox_env.conf["extras"] + except KeyError: + extras = [] + if install_project_deps: project_deps = utilities.find_project_deps( - packages, virtualenv, poetry, venv.envconfig.extras + packages, virtualenv, poetry, extras ) logger.info( f"Identified {len(project_deps)} project dependencies to install to env" @@ -245,39 +156,18 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional project_deps = [] logger.info("Env does not install project package dependencies, skipping") except exceptions.ToxPoetryInstallerException as err: - venv.status = err.__class__.__name__ logger.error(str(err)) - return False + raise err except Exception as err: - venv.status = "InternalError" + # tox_env.status = "InternalError" logger.error(f"Internal plugin error: {err}") raise err - dependencies = utilities.dedupe_packages( - dev_deps + group_deps + env_deps + project_deps - ) - if ( - venv.envconfig.config.option.parallel_install_threads - != constants.DEFAULT_INSTALL_THREADS - ): - parallel_threads = venv.envconfig.config.option.parallel_install_threads - else: - parallel_threads = ( - venv.envconfig.config.option.parallelize_locked_install - if venv.envconfig.config.option.parallelize_locked_install is not None - else constants.DEFAULT_INSTALL_THREADS - ) - log_parallel = f" (using {parallel_threads} threads)" if parallel_threads else "" + dependencies = utilities.dedupe_packages(group_deps + env_deps + project_deps) - action.setactivity( - __about__.__title__, - f"Installing {len(dependencies)} dependencies from Poetry lock file{log_parallel}", - ) installer.install( poetry, - venv, + tox_env, dependencies, - parallel_threads, + tox_env.core["parallel_install_threads"], ) - - return venv.envconfig.require_locked_deps or None diff --git a/tox_poetry_installer/installer.py b/tox_poetry_installer/installer.py index a0d1085..ed6f022 100644 --- a/tox_poetry_installer/installer.py +++ b/tox_poetry_installer/installer.py @@ -10,7 +10,7 @@ from typing import Collection from typing import Set from poetry.core.packages.package import Package as PoetryPackage -from tox.venv import VirtualEnv as ToxVirtualEnv +from tox.tox_env.api import ToxEnv as ToxVirtualEnv from tox_poetry_installer import logger from tox_poetry_installer import utilities @@ -35,9 +35,7 @@ def install( """ from tox_poetry_installer import _poetry - logger.info( - f"Installing {len(packages)} packages to environment at {venv.envconfig.envdir}" - ) + logger.info(f"Installing {len(packages)} packages to environment at {venv.env_dir}") pip = _poetry.PipInstaller( env=utilities.convert_virtualenv(venv), diff --git a/tox_poetry_installer/logger.py b/tox_poetry_installer/logger.py index 11bee82..1833307 100644 --- a/tox_poetry_installer/logger.py +++ b/tox_poetry_installer/logger.py @@ -4,26 +4,26 @@ Calling ``tox.reporter.something()`` and having to format a string with the pref gets really old fast, but more importantly it also makes the flow of the main code more difficult to follow because of the added complexity. """ -import tox +import logging from tox_poetry_installer import constants def error(message: str): - """Wrapper around :func:`tox.reporter.error`""" - tox.reporter.error(f"{constants.REPORTER_PREFIX} {message}") + """Wrapper around :func:`logging.error` that prefixes the reporter prefix onto the message""" + logging.error(f"{constants.REPORTER_PREFIX} {message}") def warning(message: str): - """Wrapper around :func:`tox.reporter.warning`""" - tox.reporter.warning(f"{constants.REPORTER_PREFIX} {message}") + """Wrapper around :func:`logging.warning`""" + logging.warning(f"{constants.REPORTER_PREFIX} {message}") def info(message: str): - """Wrapper around :func:`tox.reporter.verbosity1`""" - tox.reporter.verbosity1(f"{constants.REPORTER_PREFIX} {message}") + """Wrapper around :func:`logging.info`""" + logging.info(f"{constants.REPORTER_PREFIX} {message}") def debug(message: str): - """Wrapper around :func:`tox.reporter.verbosity2`""" - tox.reporter.verbosity2(f"{constants.REPORTER_PREFIX} {message}") + """Wrapper around :func:`logging.debug`""" + logging.debug(f"{constants.REPORTER_PREFIX} {message}") diff --git a/tox_poetry_installer/utilities.py b/tox_poetry_installer/utilities.py index 000cd14..e005b4e 100644 --- a/tox_poetry_installer/utilities.py +++ b/tox_poetry_installer/utilities.py @@ -12,8 +12,8 @@ from typing import Set from poetry.core.packages.dependency import Dependency as PoetryDependency from poetry.core.packages.package import Package as PoetryPackage -from tox.action import Action as ToxAction -from tox.venv import VirtualEnv as ToxVirtualEnv +from tox.tox_env.api import ToxEnv as ToxVirtualEnv +from tox.tox_env.package import PackageToxEnv from tox_poetry_installer import constants from tox_poetry_installer import exceptions @@ -26,50 +26,28 @@ if typing.TYPE_CHECKING: PackageMap = Dict[str, List[PoetryPackage]] -def check_preconditions(venv: ToxVirtualEnv, action: ToxAction) -> "_poetry.Poetry": +def check_preconditions(venv: ToxVirtualEnv) -> "_poetry.Poetry": """Check that the local project environment meets expectations""" + # Skip running the plugin for the provisioning environment. The provisioned environment, # for alternative Tox versions and/or the ``requires`` meta dependencies is specially # handled by Tox and is out of scope for this plugin. Since one of the ways to install this # plugin in the first place is via the Tox provisioning environment, it quickly becomes a # chicken-and-egg problem. - if action.name == venv.envconfig.config.provision_tox_env: - raise exceptions.SkipEnvironment( - f"Skipping Tox provisioning env '{action.name}'" - ) + if isinstance(venv, PackageToxEnv): + raise exceptions.SkipEnvironment(f"Skipping Tox provisioning env '{venv.name}'") - # Skip running the plugin for the packaging environment. PEP-517 front ends can handle - # that better than we can, so let them do their thing. More to the point: if you're having - # problems in the packaging env that this plugin would solve, god help you. - if action.name == venv.envconfig.config.isolated_build_env: - raise exceptions.SkipEnvironment( - f"Skipping isolated packaging build env '{action.name}'" - ) - - if venv.envconfig.config.option.require_poetry: + if venv.conf["require_poetry"]: logger.warning( "DEPRECATION: The '--require-poetry' runtime option is deprecated and will be " "removed in version 1.0.0. Please update test environments that require Poetry to " "set the 'require_poetry = true' option in tox.ini" ) - if venv.envconfig.config.option.parallelize_locked_install is not None: - logger.warning( - "DEPRECATION: The '--parallelize-locked-install' option is deprecated and will " - "be removed in version 1.0.0. Please use the '--parallel-install-threads' option." - ) - - if venv.envconfig.install_dev_deps: - logger.warning( - "DEPRECATION: The 'install_dev_deps' option is deprecated and will be removed in " - "version 1.0.0. Please update test environments that install development dependencies " - "to set the 'poetry_dev_groups = [dev]' option in tox.ini" - ) - from tox_poetry_installer import _poetry try: - return _poetry.Factory().create_poetry(venv.envconfig.config.toxinidir) + return _poetry.Factory().create_poetry(venv.core["tox_root"]) # Support running the plugin when the current tox project does not use Poetry for its # environment/dependency management. # @@ -89,7 +67,7 @@ def convert_virtualenv(venv: ToxVirtualEnv) -> "_poetry.VirtualEnv": """ from tox_poetry_installer import _poetry - return _poetry.VirtualEnv(path=Path(venv.envconfig.envdir)) + return _poetry.VirtualEnv(path=Path(venv.env_dir)) def build_package_map(poetry: "_poetry.Poetry") -> PackageMap: