From 198287a633c22ee34cb60379119d4faa3d295cbe Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Tue, 20 Aug 2024 13:56:17 -0400 Subject: [PATCH] Standardize import structure Standardize on "import module" format rather than "from module import foo" format Remove _poetry stub module since we directly depend on the poetry package now Fix conflicts between modules and parameters both named 'poetry' Fixes #92 --- tests/fixtures.py | 10 +- tests/test_installer.py | 27 ++- tests/test_transients.py | 4 +- tox_poetry_installer/_poetry.py | 44 ----- .../hooks/_tox_on_install_helpers.py | 155 +++++++++--------- .../hooks/tox_add_env_config.py | 8 +- tox_poetry_installer/hooks/tox_add_option.py | 8 +- tox_poetry_installer/hooks/tox_on_install.py | 12 +- 8 files changed, 115 insertions(+), 153 deletions(-) delete mode 100644 tox_poetry_installer/_poetry.py diff --git a/tests/fixtures.py b/tests/fixtures.py index f83a983..0ecb43c 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -5,10 +5,10 @@ from typing import List import poetry.factory import poetry.installation.executor +import poetry.installation.operations.operation import poetry.utils.env import pytest import tox.tox_env.python.virtual_env.runner -from poetry.installation.operations.operation import Operation import tox_poetry_installer.hooks._tox_on_install_helpers @@ -40,7 +40,9 @@ class MockExecutor: def __init__(self, env: MockVirtualEnv, **kwargs): self.env = env - def execute(self, operations: List[Operation]): + def execute( + self, operations: List[poetry.installation.operations.operation.Operation] + ): self.env.installed.extend([operation.package for operation in operations]) time.sleep(1) @@ -61,9 +63,9 @@ def mock_venv(monkeypatch): @pytest.fixture(scope="function") def mock_poetry_factory(monkeypatch): - pypoetry = poetry.factory.Factory().create_poetry(cwd=TEST_PROJECT_PATH) + project = poetry.factory.Factory().create_poetry(cwd=TEST_PROJECT_PATH) def mock_factory(*args, **kwargs): - return pypoetry + return project monkeypatch.setattr(poetry.factory.Factory, "create_poetry", mock_factory) diff --git a/tests/test_installer.py b/tests/test_installer.py index e8158cd..e031f57 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -2,9 +2,10 @@ import time from unittest import mock +import poetry.factory +import poetry.installation.executor import pytest import tox.tox_env.python.virtual_env.runner -from poetry.factory import Factory import tox_poetry_installer.hooks._tox_on_install_helpers @@ -14,16 +15,16 @@ from .fixtures import mock_venv def test_deduplication(mock_venv, mock_poetry_factory): """Test that the installer does not install duplicate dependencies""" - poetry = Factory().create_poetry(None) + project = poetry.factory.Factory().create_poetry(None) packages: tox_poetry_installer.hooks._tox_on_install_helpers.PackageMap = { - item.name: item for item in poetry.locker.locked_repository().packages + item.name: item for item in project.locker.locked_repository().packages } venv = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner() to_install = [packages["toml"], packages["toml"]] tox_poetry_installer.hooks._tox_on_install_helpers.install_package( - poetry, venv, to_install + project, venv, to_install ) assert len(set(to_install)) == len(venv.installed) # pylint: disable=no-member @@ -31,9 +32,9 @@ def test_deduplication(mock_venv, mock_poetry_factory): def test_parallelization(mock_venv, mock_poetry_factory): """Test that behavior is consistent between parallel and non-parallel usage""" - poetry = Factory().create_poetry(None) + project = poetry.factory.Factory().create_poetry(None) packages: tox_poetry_installer.hooks._tox_on_install_helpers.PackageMap = { - item.name: item for item in poetry.locker.locked_repository().packages + item.name: item for item in project.locker.locked_repository().packages } to_install = [ @@ -48,14 +49,14 @@ def test_parallelization(mock_venv, mock_poetry_factory): venv_sequential = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner() start_sequential = time.time() tox_poetry_installer.hooks._tox_on_install_helpers.install_package( - poetry, venv_sequential, to_install, 0 + project, venv_sequential, to_install, 0 ) sequential = time.time() - start_sequential venv_parallel = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner() start_parallel = time.time() tox_poetry_installer.hooks._tox_on_install_helpers.install_package( - poetry, venv_parallel, to_install, 5 + project, venv_parallel, to_install, 5 ) parallel = time.time() - start_parallel @@ -75,24 +76,22 @@ def test_propagates_exceptions_during_installation( Regression test for https://github.com/enpaul/tox-poetry-installer/issues/86 """ - from tox_poetry_installer import _poetry # pylint: disable=import-outside-toplevel - - poetry = Factory().create_poetry(None) + project = poetry.factory.Factory().create_poetry(None) packages: tox_poetry_installer.hooks._tox_on_install_helpers.PackageMap = { - item.name: item for item in poetry.locker.locked_repository().packages + item.name: item for item in project.locker.locked_repository().packages } to_install = [packages["toml"]] venv = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner() fake_exception = ValueError("my testing exception") with mock.patch.object( - _poetry, + poetry.installation.executor, "Executor", **{"return_value.execute.side_effect": fake_exception}, ): with pytest.raises(ValueError) as exc_info: tox_poetry_installer.hooks._tox_on_install_helpers.install_package( - poetry, venv, to_install, num_threads + project, venv, to_install, num_threads ) assert exc_info.value is fake_exception diff --git a/tests/test_transients.py b/tests/test_transients.py index dbfa217..6a07343 100644 --- a/tests/test_transients.py +++ b/tests/test_transients.py @@ -48,9 +48,9 @@ def test_functional(mock_poetry_factory, mock_venv): Trivially test that it resolves dependencies properly and that the parent package is always the last in the returned list. """ - pypoetry = poetry.factory.Factory().create_poetry(None) + project = poetry.factory.Factory().create_poetry(None) packages = tox_poetry_installer.hooks._tox_on_install_helpers.build_package_map( - pypoetry + project ) venv = poetry.utils.env.VirtualEnv() # pylint: disable=no-value-for-parameter diff --git a/tox_poetry_installer/_poetry.py b/tox_poetry_installer/_poetry.py deleted file mode 100644 index d433146..0000000 --- a/tox_poetry_installer/_poetry.py +++ /dev/null @@ -1,44 +0,0 @@ -"""You've heard of vendoirization, now get ready for internal namespace shadowing - -Poetry is an optional dependency of this package explicitly to support the use case of having the -plugin and the `poetry` package installed to the same python environment; this is most common in -containers and/or CI. In this case there are two potential problems that can arise in this case: - -* The installation of the plugin overwrites the installed version of Poetry resulting in - compatibility issues. -* Running `poetry install --no-dev`, when this plugin is in the dev-deps, results in poetry being - uninstalled from the environment. - -To support these edge cases, and more broadly to support not messing with a system package manager, -the `poetry` package dependency is listed as optional dependency. This allows the plugin to be -installed to the same environment as Poetry and import that same Poetry installation here. - -However, simply importing Poetry on the assumption that it is installed breaks another valid use -case: having this plugin installed alongside Tox when not using a Poetry-based project. To account -for this the imports in this module are isolated and the resultant import error that would result -is converted to an internal error that can be caught by callers. Rather than importing this module -at the module scope it is imported into function scope wherever Poetry components are needed. This -moves import errors from load time to runtime which allows the plugin to be skipped if Poetry isn't -installed and/or a more helpful error be raised within the Tox framework. -""" - -import sys - -from tox_poetry_installer import exceptions - - -try: - # pylint: disable=import-outside-toplevel,unused-import - from cleo.io.null_io import NullIO - from poetry.config.config import Config - from poetry.core.packages.dependency import Dependency as PoetryDependency - from poetry.core.packages.package import Package as PoetryPackage - from poetry.factory import Factory - from poetry.installation.executor import Executor - from poetry.installation.operations.install import Install - from poetry.poetry import Poetry - from poetry.utils.env import VirtualEnv -except ImportError as err: - raise exceptions.PoetryNotInstalledError( - f"Failed to import a supported version of Poetry under the current environment '{sys.executable}': {err}" - ) from None diff --git a/tox_poetry_installer/hooks/_tox_on_install_helpers.py b/tox_poetry_installer/hooks/_tox_on_install_helpers.py index 23738f7..6531f26 100644 --- a/tox_poetry_installer/hooks/_tox_on_install_helpers.py +++ b/tox_poetry_installer/hooks/_tox_on_install_helpers.py @@ -3,37 +3,36 @@ import collections import concurrent.futures import contextlib -import typing -from datetime import datetime -from pathlib import Path +import datetime +import pathlib from typing import Collection from typing import Dict from typing import List from typing import Sequence from typing import Set -from packaging.utils import NormalizedName -from poetry.core.packages.dependency import Dependency as PoetryDependency -from poetry.core.packages.package import Package as PoetryPackage -from tox.tox_env.api import ToxEnv as ToxVirtualEnv -from tox.tox_env.package import PackageToxEnv +import cleo.io.null_io +import packaging.utils +import poetry.config.config +import poetry.core.packages.dependency +import poetry.core.packages.package +import poetry.factory +import poetry.installation.executor +import poetry.installation.operations.install +import poetry.poetry +import poetry.utils.env +import tox.tox_env.api +import tox.tox_env.package from tox_poetry_installer import constants from tox_poetry_installer import exceptions from tox_poetry_installer import logger -if typing.TYPE_CHECKING: - from tox_poetry_installer import _poetry - -# This is globally disabled to support the usage of the _poetry shadow module -# pylint: disable=import-outside-toplevel +PackageMap = Dict[str, List[poetry.core.packages.package.Package]] -PackageMap = Dict[str, List[PoetryPackage]] - - -def check_preconditions(venv: ToxVirtualEnv) -> "_poetry.Poetry": +def check_preconditions(venv: tox.tox_env.api.ToxEnv) -> poetry.poetry.Poetry: """Check that the local project environment meets expectations""" # Skip running the plugin for the provisioning environment. The provisioned environment, @@ -41,13 +40,11 @@ def check_preconditions(venv: ToxVirtualEnv) -> "_poetry.Poetry": # handled by Tox and is out of scope for this plugin. Since one of the ways to install this # plugin in the first place is via the Tox provisioning environment, it quickly becomes a # chicken-and-egg problem. - if isinstance(venv, PackageToxEnv): + if isinstance(venv, tox.tox_env.package.PackageToxEnv): raise exceptions.SkipEnvironment(f"Skipping Tox provisioning env '{venv.name}'") - from tox_poetry_installer import _poetry - try: - return _poetry.Factory().create_poetry(venv.core["tox_root"]) + return poetry.factory.Factory().create_poetry(venv.core["tox_root"]) # Support running the plugin when the current tox project does not use Poetry for its # environment/dependency management. # @@ -62,9 +59,9 @@ def check_preconditions(venv: ToxVirtualEnv) -> "_poetry.Poetry": def identify_transients( dep_name: str, packages: PackageMap, - venv: "_poetry.VirtualEnv", + venv: poetry.utils.env.VirtualEnv, allow_missing: Sequence[str] = (), -) -> List[PoetryPackage]: +) -> List[poetry.core.packages.package.Package]: """Using a pool of packages, identify all transient dependencies of a given package name :param dep_name: Either the Poetry dependency or the dependency's bare package name to recursively @@ -81,10 +78,12 @@ def identify_transients( """ searched: Set[str] = set() - def _transients(transient: PoetryDependency) -> List[PoetryPackage]: + def _transients( + transient: poetry.core.packages.dependency.Dependency, + ) -> List[poetry.core.packages.package.Package]: searched.add(transient.name) - results: List[PoetryPackage] = [] + results: List[poetry.core.packages.package.Package] = [] for option in packages[transient.name]: if venv.is_valid_for_marker(option.to_dependency().marker): for requirement in option.requires: @@ -133,22 +132,22 @@ def identify_transients( def find_project_deps( packages: PackageMap, - venv: "_poetry.VirtualEnv", - poetry: "_poetry.Poetry", + venv: poetry.utils.env.VirtualEnv, + project: poetry.poetry.Poetry, extras: Sequence[str] = (), -) -> List[PoetryPackage]: +) -> List[poetry.core.packages.package.Package]: """Find the root project dependencies Recursively identify the dependencies of the root project package :param packages: Mapping of all locked package names to their corresponding package object :param venv: Poetry virtual environment to use for package compatibility checks - :param poetry: Poetry object for the current project + :param project: Poetry object for the current project :param extras: Sequence of extra names to include the dependencies of """ required_dep_names = [ - item.name for item in poetry.package.requires if not item.is_optional() + item.name for item in project.package.requires if not item.is_optional() ] extra_dep_names: List[str] = [] @@ -156,17 +155,20 @@ def find_project_deps( logger.info(f"Processing project extra '{extra}'") try: extra_dep_names += [ - item.name for item in poetry.package.extras[NormalizedName(extra)] + item.name + for item in project.package.extras[ + packaging.utils.NormalizedName(extra) + ] ] except KeyError: raise exceptions.ExtraNotFoundError( f"Environment specifies project extra '{extra}' which was not found in the lockfile" ) from None - dependencies: List[PoetryPackage] = [] + dependencies: List[poetry.core.packages.package.Package] = [] for dep_name in required_dep_names + extra_dep_names: dependencies += identify_transients( - dep_name.lower(), packages, venv, allow_missing=[poetry.package.name] + dep_name.lower(), packages, venv, allow_missing=[project.package.name] ) return dedupe_packages(dependencies) @@ -174,24 +176,24 @@ def find_project_deps( def find_additional_deps( packages: PackageMap, - venv: "_poetry.VirtualEnv", - poetry: "_poetry.Poetry", + venv: poetry.utils.env.VirtualEnv, + project: poetry.poetry.Poetry, dep_names: Sequence[str], -) -> List[PoetryPackage]: +) -> List[poetry.core.packages.package.Package]: """Find additional dependencies Recursively identify the dependencies of an arbitrary list of package names :param packages: Mapping of all locked package names to their corresponding package object :param venv: Poetry virtual environment to use for package compatibility checks - :param poetry: Poetry object for the current project + :param project: Poetry object for the current project :param dep_names: Sequence of additional dependency names to recursively find the transient dependencies for """ - dependencies: List[PoetryPackage] = [] + dependencies: List[poetry.core.packages.package.Package] = [] for dep_name in dep_names: dependencies += identify_transients( - dep_name.lower(), packages, venv, allow_missing=[poetry.package.name] + dep_name.lower(), packages, venv, allow_missing=[project.package.name] ) return dedupe_packages(dependencies) @@ -200,9 +202,9 @@ def find_additional_deps( def find_group_deps( group: str, packages: PackageMap, - venv: "_poetry.VirtualEnv", - poetry: "_poetry.Poetry", -) -> List[PoetryPackage]: + venv: poetry.utils.env.VirtualEnv, + project: poetry.poetry.Poetry, +) -> List[poetry.core.packages.package.Package]: """Find the dependencies belonging to a dependency group Recursively identify the Poetry dev dependencies @@ -210,17 +212,17 @@ def find_group_deps( :param group: Name of the dependency group from the project's ``pyproject.toml`` :param packages: Mapping of all locked package names to their corresponding package object :param venv: Poetry virtual environment to use for package compatibility checks - :param poetry: Poetry object for the current project + :param project: Poetry object for the current project """ return find_additional_deps( packages, venv, - poetry, + project, # the type ignore here is due to the difficulties around getting nested data # from the inherrently unstructured toml structure (which necessarily is flexibly # typed) but in a situation where there is a meta-structure applied to it (i.e. a # pyproject.toml structure). - poetry.pyproject.data["tool"]["poetry"] # type: ignore + project.pyproject.data["tool"]["poetry"] # type: ignore .get("group", {}) .get(group, {}) .get("dependencies", {}) @@ -229,28 +231,30 @@ def find_group_deps( def find_dev_deps( - packages: PackageMap, venv: "_poetry.VirtualEnv", poetry: "_poetry.Poetry" -) -> List[PoetryPackage]: + packages: PackageMap, + venv: poetry.utils.env.VirtualEnv, + project: poetry.poetry.Poetry, +) -> List[poetry.core.packages.package.Package]: """Find the dev dependencies Recursively identify the Poetry dev dependencies :param packages: Mapping of all locked package names to their corresponding package object :param venv: Poetry virtual environment to use for package compatibility checks - :param poetry: Poetry object for the current project + :param project: Poetry object for the current project """ - dev_group_deps = find_group_deps("dev", packages, venv, poetry) + dev_group_deps = find_group_deps("dev", packages, venv, project) # Legacy pyproject.toml poetry format: legacy_dev_group_deps = find_additional_deps( packages, venv, - poetry, + project, # the type ignore here is due to the difficulties around getting nested data # from the inherrently unstructured toml structure (which necessarily is flexibly # typed) but in a situation where there is a meta-structure applied to it (i.e. a # pyproject.toml structure). - poetry.pyproject.data["tool"]["poetry"].get("dev-dependencies", {}).keys(), # type: ignore + project.pyproject.data["tool"]["poetry"].get("dev-dependencies", {}).keys(), # type: ignore ) # Poetry 1.2 unions these two toml sections. @@ -273,37 +277,38 @@ def _optional_parallelize(parallels: int): def install_package( - poetry: "_poetry.Poetry", - venv: ToxVirtualEnv, - packages: Collection["_poetry.PoetryPackage"], + project: poetry.poetry.Poetry, + venv: tox.tox_env.api.ToxEnv, + packages: Collection[poetry.core.packages.package.Package], parallels: int = 0, ): """Install a bunch of packages to a virtualenv - :param poetry: Poetry object the packages were sourced from + :param project: Poetry object the packages were sourced from :param venv: Tox virtual environment to install the packages to :param packages: List of packages to install to the virtual environment :param parallels: Number of parallel processes to use for installing dependency packages, or ``None`` to disable parallelization. """ - from tox_poetry_installer import _poetry logger.info(f"Installing {len(packages)} packages to environment at {venv.env_dir}") - install_executor = _poetry.Executor( + install_executor = poetry.installation.executor.Executor( env=convert_virtualenv(venv), - io=_poetry.NullIO(), - pool=poetry.pool, - config=_poetry.Config(), + io=cleo.io.null_io.NullIO(), + pool=project.pool, + config=poetry.config.config.Config(), ) - installed: Set[_poetry.PoetryPackage] = set() + installed: Set[poetry.core.packages.package.Package] = set() - def logged_install(dependency: _poetry.PoetryPackage) -> None: - start = datetime.now() + def logged_install(dependency: poetry.core.packages.package.Package) -> None: + start = datetime.datetime.now() logger.debug(f"Installing {dependency}") - install_executor.execute([_poetry.Install(package=dependency)]) - end = datetime.now() + install_executor.execute( + [poetry.installation.operations.install.Install(package=dependency)] + ) + end = datetime.datetime.now() logger.debug(f"Finished installing {dependency} in {end - start}") with _optional_parallelize(parallels) as executor: @@ -326,36 +331,36 @@ def install_package( future.result() -def dedupe_packages(packages: Sequence[PoetryPackage]) -> List[PoetryPackage]: - """Deduplicates a sequence of PoetryPackages while preserving ordering +def dedupe_packages( + packages: Sequence[poetry.core.packages.package.Package], +) -> List[poetry.core.packages.package.Package]: + """Deduplicates a sequence of Packages while preserving ordering Adapted from StackOverflow: https://stackoverflow.com/a/480227 """ - seen: Set[PoetryPackage] = set() + seen: Set[poetry.core.packages.package.Package] = set() # Make this faster, avoid method lookup below seen_add = seen.add return [p for p in packages if not (p in seen or seen_add(p))] -def convert_virtualenv(venv: ToxVirtualEnv) -> "_poetry.VirtualEnv": +def convert_virtualenv(venv: tox.tox_env.api.ToxEnv) -> poetry.utils.env.VirtualEnv: """Convert a Tox venv to a Poetry venv :param venv: Tox ``VirtualEnv`` object representing a tox virtual environment :returns: Poetry ``VirtualEnv`` object representing a poetry virtual environment """ - from tox_poetry_installer import _poetry - - return _poetry.VirtualEnv(path=Path(venv.env_dir)) + return poetry.utils.env.VirtualEnv(path=pathlib.Path(venv.env_dir)) -def build_package_map(poetry: "_poetry.Poetry") -> PackageMap: +def build_package_map(project: poetry.poetry.Poetry) -> PackageMap: """Build the mapping of package names to objects - :param poetry: Populated poetry object to load locked packages from + :param project: Populated poetry object to load locked packages from :returns: Mapping of package names to Poetry package objects """ packages = collections.defaultdict(list) - for package in poetry.locker.locked_repository().packages: + for package in project.locker.locked_repository().packages: packages[str(package.name)].append(package) return packages diff --git a/tox_poetry_installer/hooks/tox_add_env_config.py b/tox_poetry_installer/hooks/tox_add_env_config.py index fe77723..d29c1e5 100644 --- a/tox_poetry_installer/hooks/tox_add_env_config.py +++ b/tox_poetry_installer/hooks/tox_add_env_config.py @@ -2,14 +2,14 @@ from typing import List -from tox.config.sets import EnvConfigSet -from tox.plugin import impl +import tox.config.sets +import tox.plugin # pylint: disable=missing-function-docstring -@impl +@tox.plugin.impl def tox_add_env_config( - env_conf: EnvConfigSet, + env_conf: tox.config.sets.EnvConfigSet, ): env_conf.add_config( "poetry_dep_groups", diff --git a/tox_poetry_installer/hooks/tox_add_option.py b/tox_poetry_installer/hooks/tox_add_option.py index 8f07751..4ec1432 100644 --- a/tox_poetry_installer/hooks/tox_add_option.py +++ b/tox_poetry_installer/hooks/tox_add_option.py @@ -1,14 +1,14 @@ """Add additional command line arguments to tox to configure plugin behavior""" -from tox.config.cli.parser import ToxParser -from tox.plugin import impl +import tox.config.cli.parser +import tox.plugin from tox_poetry_installer import constants # pylint: disable=missing-function-docstring -@impl -def tox_add_option(parser: ToxParser): +@tox.plugin.impl +def tox_add_option(parser: tox.config.cli.parser.ToxParser): parser.add_argument( "--parallel-install-threads", type=int, diff --git a/tox_poetry_installer/hooks/tox_on_install.py b/tox_poetry_installer/hooks/tox_on_install.py index d9b9e47..ba4c086 100644 --- a/tox_poetry_installer/hooks/tox_on_install.py +++ b/tox_poetry_installer/hooks/tox_on_install.py @@ -5,10 +5,10 @@ specified by the Tox environment. Finally these dependencies are installed into environment using the Poetry ``PipInstaller`` backend. """ -from itertools import chain +import itertools -from tox.plugin import impl -from tox.tox_env.api import ToxEnv as ToxVirtualEnv +import tox.plugin +import tox.tox_env.api from tox_poetry_installer import exceptions from tox_poetry_installer import logger @@ -23,8 +23,8 @@ from tox_poetry_installer.hooks._tox_on_install_helpers import install_package # pylint: disable=missing-function-docstring,unused-argument -@impl -def tox_on_install(tox_env: ToxVirtualEnv, *args) -> None: +@tox.plugin.impl +def tox_on_install(tox_env: tox.tox_env.api.ToxEnv, *args) -> None: try: poetry = check_preconditions(tox_env) except exceptions.SkipEnvironment as err: @@ -62,7 +62,7 @@ def tox_on_install(tox_env: ToxVirtualEnv, *args) -> None: group_deps = dedupe_packages( list( - chain( + itertools.chain( *[ find_group_deps(group, packages, virtualenv, poetry) for group in tox_env.conf["poetry_dep_groups"]