Refactor dep processing to improve efficiency of installation

Assemble single list of dependencies to reduce duplication and reduce installation overhead

tox_poetry_installer/exceptions.py

@@ -5,6 +5,7 @@ All exceptions should inherit from the common base exception :exc:`ToxPoetryInst
 ::
 
   ToxPoetryInstallerException
+   +-- SkipEnvironment
    +-- LockedDepVersionConflictError
    +-- LockedDepNotFoundError
    +-- ExtraNotFoundError
@@ -17,6 +18,10 @@ class ToxPoetryInstallerException(Exception):
     """Error while installing locked dependencies to the test environment"""
 
 
+class SkipEnvironment(ToxPoetryInstallerException):
+    """Current environment does not meet preconditions and should be skipped by the plugin"""
+
+
 class LockedDepVersionConflictError(ToxPoetryInstallerException):
     """Locked dependencies cannot specify an alternate version for installation"""
 

tox_poetry_installer/hooks.py

@@ -8,7 +8,6 @@ from typing import List
 from typing import Optional
 
 from poetry.core.packages import Package as PoetryPackage
-from poetry.factory import Factory as PoetryFactory
 from poetry.poetry import Poetry
 from tox import hookimpl
 from tox import reporter
@@ -63,30 +62,19 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
     :param action: Tox action object
     """
 
-    if action.name == venv.envconfig.config.isolated_build_env:
-        # Skip running the plugin for the packaging environment. PEP-517 front ends can handle
-        # that better than we can, so let them do their thing. More to the point: if you're having
-        # problems in the packaging env that this plugin would solve, god help you.
-        reporter.verbosity1(
-            f"{constants.REPORTER_PREFIX} skipping isolated build env '{action.name}'"
-        )
-        return None
-
     try:
-        poetry = PoetryFactory().create_poetry(venv.envconfig.config.toxinidir)
+        poetry = utilities.check_preconditions(venv, action)
-    except RuntimeError:
+    except exceptions.SkipEnvironment as err:
-        # Support running the plugin when the current tox project does not use Poetry for its
+        reporter.verbosity1(str(err))
-        # environment/dependency management.
-        #
-        # ``RuntimeError`` is dangerous to blindly catch because it can be (and in Poetry's case,
-        # is) raised in many different places for different purposes.
-        reporter.verbosity1(
-            f"{constants.REPORTER_PREFIX} project does not use Poetry for env management, skipping installation of locked dependencies"
-        )
         return None
 
     reporter.verbosity1(
-        f"{constants.REPORTER_PREFIX} loaded project pyproject.toml from {poetry.file}"
+        f"{constants.REPORTER_PREFIX} Loaded project pyproject.toml from {poetry.file}"
+    )
+
+    if venv.envconfig.require_locked_deps and venv.envconfig.deps:
+        raise exceptions.LockedDepsRequiredError(
+            f"Unlocked dependencies '{venv.envconfig.deps}' specified for environment '{venv.name}' which requires locked dependencies"
         )
 
     package_map: PackageMap = {
@@ -94,85 +82,55 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
         for package in poetry.locker.locked_repository(True).packages
     }
 
-    if venv.envconfig.require_locked_deps and venv.envconfig.deps:
+    if venv.envconfig.install_dev_deps:
-        raise exceptions.LockedDepsRequiredError(
+        dev_deps: List[PoetryPackage] = [
-            f"Unlocked dependencies '{venv.envconfig.deps}' specified for environment '{venv.name}' which requires locked dependencies"
+            dep
-        )
+            for dep in package_map.values()
+            if dep not in poetry.locker.locked_repository(False).packages
+        ]
+    else:
+        dev_deps = []
 
-    # Handle the installation of any locked env dependencies from the lockfile
-    _install_env_dependencies(venv, poetry, package_map)
-
-    # Handle the installation of the package dependencies from the lockfile if the package is
-    # being installed to this venv; otherwise skip installing the package dependencies
-    if venv.envconfig.skip_install:
     reporter.verbosity1(
-            f"{constants.REPORTER_PREFIX} env specifies 'skip_install = true', skipping installation of project package"
+        f"{constants.REPORTER_PREFIX} Identified {len(dev_deps)} development dependencies to install to env"
     )
-        return venv.envconfig.require_locked_deps or None
 
-    if venv.envconfig.config.skipsdist:
-        reporter.verbosity1(
-            f"{constants.REPORTER_PREFIX} config specifies 'skipsdist = true', skipping installation of project package"
-        )
-        return venv.envconfig.require_locked_deps or None
-
-    _install_project_dependencies(venv, poetry, package_map)
-
-    return venv.envconfig.require_locked_deps or None
-
-
-def _install_env_dependencies(
-    venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap
-):
-    """Install the packages for a specified testenv
-
-    Processes the tox environment config, identifies any locked environment dependencies, pulls
-    them from the lockfile, and installs them to the virtual environment.
-
-    :param venv: Tox virtual environment to install the packages to
-    :param poetry: Poetry object the packages were sourced from
-    :param packages: Mapping of package names to the corresponding package object
-    """
-
-    dependencies: List[PoetryPackage] = []
-    for dep in venv.envconfig.locked_deps:
     try:
-            dependencies += utilities.find_transients(packages, dep.lower())
+        env_deps: List[PoetryPackage] = []
+        for dep in venv.envconfig.locked_deps:
+            env_deps += utilities.find_transients(package_map, dep.lower())
+        reporter.verbosity1(
+            f"{constants.REPORTER_PREFIX} Identified {len(env_deps)} environment dependencies to install to env"
+        )
+
+        if not venv.envconfig.skip_install and not venv.envconfig.config.skipsdist:
+            project_deps: List[PoetryPackage] = _find_project_dependencies(
+                venv, poetry, package_map
+            )
+        else:
+            reporter.verbosity1(
+                f"{constants.REPORTER_PREFIX} Skipping installation of project dependencies, env does not install project package"
+            )
+        reporter.verbosity1(
+            f"{constants.REPORTER_PREFIX} Identified {len(project_deps)} project dependencies to install to env"
+        )
     except exceptions.ToxPoetryInstallerException as err:
         venv.status = "lockfile installation failed"
         reporter.error(f"{constants.REPORTER_PREFIX} {err}")
         raise err
 
-    if venv.envconfig.install_dev_deps:
+    dependencies = list(set(dev_deps + env_deps + project_deps))
-        reporter.verbosity1(
-            f"{constants.REPORTER_PREFIX} env specifies 'install_env_deps = true', including Poetry dev dependencies"
-        )
-
-        dev_dependencies = [
-            dep
-            for dep in poetry.locker.locked_repository(True).packages
-            if dep not in poetry.locker.locked_repository(False).packages
-        ]
-
-        reporter.verbosity1(
-            f"{constants.REPORTER_PREFIX} identified {len(dev_dependencies)} Poetry dev dependencies"
-        )
-
-        dependencies = list(set(dev_dependencies + dependencies))
-
-    reporter.verbosity1(
-        f"{constants.REPORTER_PREFIX} identified {len(dependencies)} total dependencies from {len(venv.envconfig.locked_deps)} locked env dependencies"
-    )
-
     reporter.verbosity0(
-        f"{constants.REPORTER_PREFIX} ({venv.name}) installing {len(dependencies)} env dependencies from lockfile"
+        f"{constants.REPORTER_PREFIX} Installing {len(dependencies)} dependencies to env '{action.name}'"
     )
     utilities.install_to_venv(poetry, venv, dependencies)
 
+    return venv.envconfig.require_locked_deps or None
 
-def _install_project_dependencies(
+
+def _find_project_dependencies(
     venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap
-):
+) -> List[PoetryPackage]:
     """Install the dependencies of the project package
 
     Install all primary dependencies of the project package.
@@ -181,9 +139,6 @@ def _install_project_dependencies(
     :param poetry: Poetry object the packages were sourced from
     :param packages: Mapping of package names to the corresponding package object
     """
-    reporter.verbosity1(
-        f"{constants.REPORTER_PREFIX} performing installation of project dependencies"
-    )
 
     base_dependencies: List[PoetryPackage] = [
         packages[item.name]
@@ -204,18 +159,6 @@ def _install_project_dependencies(
 
     dependencies: List[PoetryPackage] = []
     for dep in base_dependencies + extra_dependencies:
-        try:
         dependencies += utilities.find_transients(packages, dep.name.lower())
-        except exceptions.ToxPoetryInstallerException as err:
-            venv.status = "lockfile installation failed"
-            reporter.error(f"{constants.REPORTER_PREFIX} {err}")
-            raise err
 
-    reporter.verbosity1(
+    return dependencies
-        f"{constants.REPORTER_PREFIX} identified {len(dependencies)} total dependencies from {len(poetry.package.requires)} project dependencies"
-    )
-
-    reporter.verbosity0(
-        f"{constants.REPORTER_PREFIX} ({venv.name}) installing {len(dependencies)} project dependencies from lockfile"
-    )
-    utilities.install_to_venv(poetry, venv, dependencies)

tox_poetry_installer/utilities.py

@@ -4,12 +4,14 @@ from typing import Sequence
 from typing import Set
 
 from poetry.core.packages import Package as PoetryPackage
+from poetry.factory import Factory as PoetryFactory
 from poetry.installation.pip_installer import PipInstaller as PoetryPipInstaller
 from poetry.io.null_io import NullIO as PoetryNullIO
 from poetry.poetry import Poetry
 from poetry.puzzle.provider import Provider as PoetryProvider
 from poetry.utils.env import VirtualEnv as PoetryVirtualEnv
 from tox import reporter
+from tox.action import Action as ToxAction
 from tox.venv import VirtualEnv as ToxVirtualEnv
 
 from tox_poetry_installer import constants
@@ -38,7 +40,7 @@ def install_to_venv(
     )
 
     for dependency in packages:
-        reporter.verbosity1(f"{constants.REPORTER_PREFIX} installing {dependency}")
+        reporter.verbosity1(f"{constants.REPORTER_PREFIX} Installing {dependency}")
         installer.install(dependency)
 
 
@@ -60,7 +62,7 @@ def find_transients(packages: PackageMap, dependency_name: str) -> Set[PoetryPac
         def find_deps_of_deps(name: str, transients: PackageMap):
             if name in PoetryProvider.UNSAFE_PACKAGES:
                 reporter.warning(
-                    f"{constants.REPORTER_PREFIX} installing package '{name}' using Poetry is not supported; skipping installation of package '{name}'"
+                    f"{constants.REPORTER_PREFIX} Installing package '{name}' using Poetry is not supported; skipping installation of package '{name}'"
                 )
             else:
                 transients[name] = packages[name]
@@ -83,3 +85,26 @@ def find_transients(packages: PackageMap, dependency_name: str) -> Set[PoetryPac
         raise exceptions.LockedDepNotFoundError(
             f"No version of locked dependency '{dependency_name}' found in the project lockfile"
         ) from None
+
+
+def check_preconditions(venv: ToxVirtualEnv, action: ToxAction) -> Poetry:
+    """Check that the local project environment meets expectations"""
+    # Skip running the plugin for the packaging environment. PEP-517 front ends can handle
+    # that better than we can, so let them do their thing. More to the point: if you're having
+    # problems in the packaging env that this plugin would solve, god help you.
+    if action.name == venv.envconfig.config.isolated_build_env:
+        raise exceptions.SkipEnvironment(
+            f"Skipping isolated packaging build env '{action.name}'"
+        )
+
+    try:
+        return PoetryFactory().create_poetry(venv.envconfig.config.toxinidir)
+    # Support running the plugin when the current tox project does not use Poetry for its
+    # environment/dependency management.
+    #
+    # ``RuntimeError`` is dangerous to blindly catch because it can be (and in Poetry's case,
+    # is) raised in many different places for different purposes.
+    except RuntimeError:
+        raise exceptions.SkipEnvironment(
+            "Project does not use Poetry for env management, skipping installation of locked dependencies"
+        ) from None