From d910b6ee8d9bf924edf82e5ee59ccf9478c75c2c Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Thu, 12 Nov 2020 22:14:49 -0500 Subject: [PATCH] Refactor dep processing to improve efficiency of installation Assemble single list of dependencies to reduce duplication and reduce installation overhead --- tox_poetry_installer/exceptions.py | 5 + tox_poetry_installer/hooks.py | 153 +++++++++-------------------- tox_poetry_installer/utilities.py | 29 +++++- 3 files changed, 80 insertions(+), 107 deletions(-) diff --git a/tox_poetry_installer/exceptions.py b/tox_poetry_installer/exceptions.py index 86102fd..d11facd 100644 --- a/tox_poetry_installer/exceptions.py +++ b/tox_poetry_installer/exceptions.py @@ -5,6 +5,7 @@ All exceptions should inherit from the common base exception :exc:`ToxPoetryInst :: ToxPoetryInstallerException + +-- SkipEnvironment +-- LockedDepVersionConflictError +-- LockedDepNotFoundError +-- ExtraNotFoundError @@ -17,6 +18,10 @@ class ToxPoetryInstallerException(Exception): """Error while installing locked dependencies to the test environment""" +class SkipEnvironment(ToxPoetryInstallerException): + """Current environment does not meet preconditions and should be skipped by the plugin""" + + class LockedDepVersionConflictError(ToxPoetryInstallerException): """Locked dependencies cannot specify an alternate version for installation""" diff --git a/tox_poetry_installer/hooks.py b/tox_poetry_installer/hooks.py index 1ffb266..34645cc 100644 --- a/tox_poetry_installer/hooks.py +++ b/tox_poetry_installer/hooks.py @@ -8,7 +8,6 @@ from typing import List from typing import Optional from poetry.core.packages import Package as PoetryPackage -from poetry.factory import Factory as PoetryFactory from poetry.poetry import Poetry from tox import hookimpl from tox import reporter @@ -63,116 +62,75 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional :param action: Tox action object """ - if action.name == venv.envconfig.config.isolated_build_env: - # Skip running the plugin for the packaging environment. PEP-517 front ends can handle - # that better than we can, so let them do their thing. More to the point: if you're having - # problems in the packaging env that this plugin would solve, god help you. - reporter.verbosity1( - f"{constants.REPORTER_PREFIX} skipping isolated build env '{action.name}'" - ) - return None - try: - poetry = PoetryFactory().create_poetry(venv.envconfig.config.toxinidir) - except RuntimeError: - # Support running the plugin when the current tox project does not use Poetry for its - # environment/dependency management. - # - # ``RuntimeError`` is dangerous to blindly catch because it can be (and in Poetry's case, - # is) raised in many different places for different purposes. - reporter.verbosity1( - f"{constants.REPORTER_PREFIX} project does not use Poetry for env management, skipping installation of locked dependencies" - ) + poetry = utilities.check_preconditions(venv, action) + except exceptions.SkipEnvironment as err: + reporter.verbosity1(str(err)) return None reporter.verbosity1( - f"{constants.REPORTER_PREFIX} loaded project pyproject.toml from {poetry.file}" + f"{constants.REPORTER_PREFIX} Loaded project pyproject.toml from {poetry.file}" ) - package_map: PackageMap = { - package.name: package - for package in poetry.locker.locked_repository(True).packages - } - if venv.envconfig.require_locked_deps and venv.envconfig.deps: raise exceptions.LockedDepsRequiredError( f"Unlocked dependencies '{venv.envconfig.deps}' specified for environment '{venv.name}' which requires locked dependencies" ) - # Handle the installation of any locked env dependencies from the lockfile - _install_env_dependencies(venv, poetry, package_map) + package_map: PackageMap = { + package.name: package + for package in poetry.locker.locked_repository(True).packages + } - # Handle the installation of the package dependencies from the lockfile if the package is - # being installed to this venv; otherwise skip installing the package dependencies - if venv.envconfig.skip_install: + if venv.envconfig.install_dev_deps: + dev_deps: List[PoetryPackage] = [ + dep + for dep in package_map.values() + if dep not in poetry.locker.locked_repository(False).packages + ] + else: + dev_deps = [] + + reporter.verbosity1( + f"{constants.REPORTER_PREFIX} Identified {len(dev_deps)} development dependencies to install to env" + ) + + try: + env_deps: List[PoetryPackage] = [] + for dep in venv.envconfig.locked_deps: + env_deps += utilities.find_transients(package_map, dep.lower()) reporter.verbosity1( - f"{constants.REPORTER_PREFIX} env specifies 'skip_install = true', skipping installation of project package" + f"{constants.REPORTER_PREFIX} Identified {len(env_deps)} environment dependencies to install to env" ) - return venv.envconfig.require_locked_deps or None - if venv.envconfig.config.skipsdist: + if not venv.envconfig.skip_install and not venv.envconfig.config.skipsdist: + project_deps: List[PoetryPackage] = _find_project_dependencies( + venv, poetry, package_map + ) + else: + reporter.verbosity1( + f"{constants.REPORTER_PREFIX} Skipping installation of project dependencies, env does not install project package" + ) reporter.verbosity1( - f"{constants.REPORTER_PREFIX} config specifies 'skipsdist = true', skipping installation of project package" + f"{constants.REPORTER_PREFIX} Identified {len(project_deps)} project dependencies to install to env" ) - return venv.envconfig.require_locked_deps or None + except exceptions.ToxPoetryInstallerException as err: + venv.status = "lockfile installation failed" + reporter.error(f"{constants.REPORTER_PREFIX} {err}") + raise err - _install_project_dependencies(venv, poetry, package_map) + dependencies = list(set(dev_deps + env_deps + project_deps)) + reporter.verbosity0( + f"{constants.REPORTER_PREFIX} Installing {len(dependencies)} dependencies to env '{action.name}'" + ) + utilities.install_to_venv(poetry, venv, dependencies) return venv.envconfig.require_locked_deps or None -def _install_env_dependencies( +def _find_project_dependencies( venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap -): - """Install the packages for a specified testenv - - Processes the tox environment config, identifies any locked environment dependencies, pulls - them from the lockfile, and installs them to the virtual environment. - - :param venv: Tox virtual environment to install the packages to - :param poetry: Poetry object the packages were sourced from - :param packages: Mapping of package names to the corresponding package object - """ - - dependencies: List[PoetryPackage] = [] - for dep in venv.envconfig.locked_deps: - try: - dependencies += utilities.find_transients(packages, dep.lower()) - except exceptions.ToxPoetryInstallerException as err: - venv.status = "lockfile installation failed" - reporter.error(f"{constants.REPORTER_PREFIX} {err}") - raise err - - if venv.envconfig.install_dev_deps: - reporter.verbosity1( - f"{constants.REPORTER_PREFIX} env specifies 'install_env_deps = true', including Poetry dev dependencies" - ) - - dev_dependencies = [ - dep - for dep in poetry.locker.locked_repository(True).packages - if dep not in poetry.locker.locked_repository(False).packages - ] - - reporter.verbosity1( - f"{constants.REPORTER_PREFIX} identified {len(dev_dependencies)} Poetry dev dependencies" - ) - - dependencies = list(set(dev_dependencies + dependencies)) - - reporter.verbosity1( - f"{constants.REPORTER_PREFIX} identified {len(dependencies)} total dependencies from {len(venv.envconfig.locked_deps)} locked env dependencies" - ) - - reporter.verbosity0( - f"{constants.REPORTER_PREFIX} ({venv.name}) installing {len(dependencies)} env dependencies from lockfile" - ) - utilities.install_to_venv(poetry, venv, dependencies) - - -def _install_project_dependencies( - venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap -): +) -> List[PoetryPackage]: """Install the dependencies of the project package Install all primary dependencies of the project package. @@ -181,9 +139,6 @@ def _install_project_dependencies( :param poetry: Poetry object the packages were sourced from :param packages: Mapping of package names to the corresponding package object """ - reporter.verbosity1( - f"{constants.REPORTER_PREFIX} performing installation of project dependencies" - ) base_dependencies: List[PoetryPackage] = [ packages[item.name] @@ -204,18 +159,6 @@ def _install_project_dependencies( dependencies: List[PoetryPackage] = [] for dep in base_dependencies + extra_dependencies: - try: - dependencies += utilities.find_transients(packages, dep.name.lower()) - except exceptions.ToxPoetryInstallerException as err: - venv.status = "lockfile installation failed" - reporter.error(f"{constants.REPORTER_PREFIX} {err}") - raise err + dependencies += utilities.find_transients(packages, dep.name.lower()) - reporter.verbosity1( - f"{constants.REPORTER_PREFIX} identified {len(dependencies)} total dependencies from {len(poetry.package.requires)} project dependencies" - ) - - reporter.verbosity0( - f"{constants.REPORTER_PREFIX} ({venv.name}) installing {len(dependencies)} project dependencies from lockfile" - ) - utilities.install_to_venv(poetry, venv, dependencies) + return dependencies diff --git a/tox_poetry_installer/utilities.py b/tox_poetry_installer/utilities.py index f4bb650..828d685 100644 --- a/tox_poetry_installer/utilities.py +++ b/tox_poetry_installer/utilities.py @@ -4,12 +4,14 @@ from typing import Sequence from typing import Set from poetry.core.packages import Package as PoetryPackage +from poetry.factory import Factory as PoetryFactory from poetry.installation.pip_installer import PipInstaller as PoetryPipInstaller from poetry.io.null_io import NullIO as PoetryNullIO from poetry.poetry import Poetry from poetry.puzzle.provider import Provider as PoetryProvider from poetry.utils.env import VirtualEnv as PoetryVirtualEnv from tox import reporter +from tox.action import Action as ToxAction from tox.venv import VirtualEnv as ToxVirtualEnv from tox_poetry_installer import constants @@ -38,7 +40,7 @@ def install_to_venv( ) for dependency in packages: - reporter.verbosity1(f"{constants.REPORTER_PREFIX} installing {dependency}") + reporter.verbosity1(f"{constants.REPORTER_PREFIX} Installing {dependency}") installer.install(dependency) @@ -60,7 +62,7 @@ def find_transients(packages: PackageMap, dependency_name: str) -> Set[PoetryPac def find_deps_of_deps(name: str, transients: PackageMap): if name in PoetryProvider.UNSAFE_PACKAGES: reporter.warning( - f"{constants.REPORTER_PREFIX} installing package '{name}' using Poetry is not supported; skipping installation of package '{name}'" + f"{constants.REPORTER_PREFIX} Installing package '{name}' using Poetry is not supported; skipping installation of package '{name}'" ) else: transients[name] = packages[name] @@ -83,3 +85,26 @@ def find_transients(packages: PackageMap, dependency_name: str) -> Set[PoetryPac raise exceptions.LockedDepNotFoundError( f"No version of locked dependency '{dependency_name}' found in the project lockfile" ) from None + + +def check_preconditions(venv: ToxVirtualEnv, action: ToxAction) -> Poetry: + """Check that the local project environment meets expectations""" + # Skip running the plugin for the packaging environment. PEP-517 front ends can handle + # that better than we can, so let them do their thing. More to the point: if you're having + # problems in the packaging env that this plugin would solve, god help you. + if action.name == venv.envconfig.config.isolated_build_env: + raise exceptions.SkipEnvironment( + f"Skipping isolated packaging build env '{action.name}'" + ) + + try: + return PoetryFactory().create_poetry(venv.envconfig.config.toxinidir) + # Support running the plugin when the current tox project does not use Poetry for its + # environment/dependency management. + # + # ``RuntimeError`` is dangerous to blindly catch because it can be (and in Poetry's case, + # is) raised in many different places for different purposes. + except RuntimeError: + raise exceptions.SkipEnvironment( + "Project does not use Poetry for env management, skipping installation of locked dependencies" + ) from None