mirror of
https://github.com/enpaul/tox-poetry-installer.git
synced 2024-10-29 19:47:00 +00:00
Refactor dep processing to improve efficiency of installation
Assemble single list of dependencies to reduce duplication and reduce installation overhead
This commit is contained in:
parent
e37c166a8b
commit
d910b6ee8d
@ -5,6 +5,7 @@ All exceptions should inherit from the common base exception :exc:`ToxPoetryInst
|
|||||||
::
|
::
|
||||||
|
|
||||||
ToxPoetryInstallerException
|
ToxPoetryInstallerException
|
||||||
|
+-- SkipEnvironment
|
||||||
+-- LockedDepVersionConflictError
|
+-- LockedDepVersionConflictError
|
||||||
+-- LockedDepNotFoundError
|
+-- LockedDepNotFoundError
|
||||||
+-- ExtraNotFoundError
|
+-- ExtraNotFoundError
|
||||||
@ -17,6 +18,10 @@ class ToxPoetryInstallerException(Exception):
|
|||||||
"""Error while installing locked dependencies to the test environment"""
|
"""Error while installing locked dependencies to the test environment"""
|
||||||
|
|
||||||
|
|
||||||
|
class SkipEnvironment(ToxPoetryInstallerException):
|
||||||
|
"""Current environment does not meet preconditions and should be skipped by the plugin"""
|
||||||
|
|
||||||
|
|
||||||
class LockedDepVersionConflictError(ToxPoetryInstallerException):
|
class LockedDepVersionConflictError(ToxPoetryInstallerException):
|
||||||
"""Locked dependencies cannot specify an alternate version for installation"""
|
"""Locked dependencies cannot specify an alternate version for installation"""
|
||||||
|
|
||||||
|
@ -8,7 +8,6 @@ from typing import List
|
|||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from poetry.core.packages import Package as PoetryPackage
|
from poetry.core.packages import Package as PoetryPackage
|
||||||
from poetry.factory import Factory as PoetryFactory
|
|
||||||
from poetry.poetry import Poetry
|
from poetry.poetry import Poetry
|
||||||
from tox import hookimpl
|
from tox import hookimpl
|
||||||
from tox import reporter
|
from tox import reporter
|
||||||
@ -63,116 +62,75 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
|
|||||||
:param action: Tox action object
|
:param action: Tox action object
|
||||||
"""
|
"""
|
||||||
|
|
||||||
if action.name == venv.envconfig.config.isolated_build_env:
|
|
||||||
# Skip running the plugin for the packaging environment. PEP-517 front ends can handle
|
|
||||||
# that better than we can, so let them do their thing. More to the point: if you're having
|
|
||||||
# problems in the packaging env that this plugin would solve, god help you.
|
|
||||||
reporter.verbosity1(
|
|
||||||
f"{constants.REPORTER_PREFIX} skipping isolated build env '{action.name}'"
|
|
||||||
)
|
|
||||||
return None
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
poetry = PoetryFactory().create_poetry(venv.envconfig.config.toxinidir)
|
poetry = utilities.check_preconditions(venv, action)
|
||||||
except RuntimeError:
|
except exceptions.SkipEnvironment as err:
|
||||||
# Support running the plugin when the current tox project does not use Poetry for its
|
reporter.verbosity1(str(err))
|
||||||
# environment/dependency management.
|
|
||||||
#
|
|
||||||
# ``RuntimeError`` is dangerous to blindly catch because it can be (and in Poetry's case,
|
|
||||||
# is) raised in many different places for different purposes.
|
|
||||||
reporter.verbosity1(
|
|
||||||
f"{constants.REPORTER_PREFIX} project does not use Poetry for env management, skipping installation of locked dependencies"
|
|
||||||
)
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
reporter.verbosity1(
|
reporter.verbosity1(
|
||||||
f"{constants.REPORTER_PREFIX} loaded project pyproject.toml from {poetry.file}"
|
f"{constants.REPORTER_PREFIX} Loaded project pyproject.toml from {poetry.file}"
|
||||||
)
|
)
|
||||||
|
|
||||||
package_map: PackageMap = {
|
|
||||||
package.name: package
|
|
||||||
for package in poetry.locker.locked_repository(True).packages
|
|
||||||
}
|
|
||||||
|
|
||||||
if venv.envconfig.require_locked_deps and venv.envconfig.deps:
|
if venv.envconfig.require_locked_deps and venv.envconfig.deps:
|
||||||
raise exceptions.LockedDepsRequiredError(
|
raise exceptions.LockedDepsRequiredError(
|
||||||
f"Unlocked dependencies '{venv.envconfig.deps}' specified for environment '{venv.name}' which requires locked dependencies"
|
f"Unlocked dependencies '{venv.envconfig.deps}' specified for environment '{venv.name}' which requires locked dependencies"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Handle the installation of any locked env dependencies from the lockfile
|
package_map: PackageMap = {
|
||||||
_install_env_dependencies(venv, poetry, package_map)
|
package.name: package
|
||||||
|
for package in poetry.locker.locked_repository(True).packages
|
||||||
|
}
|
||||||
|
|
||||||
# Handle the installation of the package dependencies from the lockfile if the package is
|
if venv.envconfig.install_dev_deps:
|
||||||
# being installed to this venv; otherwise skip installing the package dependencies
|
dev_deps: List[PoetryPackage] = [
|
||||||
if venv.envconfig.skip_install:
|
dep
|
||||||
|
for dep in package_map.values()
|
||||||
|
if dep not in poetry.locker.locked_repository(False).packages
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
dev_deps = []
|
||||||
|
|
||||||
|
reporter.verbosity1(
|
||||||
|
f"{constants.REPORTER_PREFIX} Identified {len(dev_deps)} development dependencies to install to env"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
env_deps: List[PoetryPackage] = []
|
||||||
|
for dep in venv.envconfig.locked_deps:
|
||||||
|
env_deps += utilities.find_transients(package_map, dep.lower())
|
||||||
reporter.verbosity1(
|
reporter.verbosity1(
|
||||||
f"{constants.REPORTER_PREFIX} env specifies 'skip_install = true', skipping installation of project package"
|
f"{constants.REPORTER_PREFIX} Identified {len(env_deps)} environment dependencies to install to env"
|
||||||
)
|
)
|
||||||
return venv.envconfig.require_locked_deps or None
|
|
||||||
|
|
||||||
if venv.envconfig.config.skipsdist:
|
if not venv.envconfig.skip_install and not venv.envconfig.config.skipsdist:
|
||||||
|
project_deps: List[PoetryPackage] = _find_project_dependencies(
|
||||||
|
venv, poetry, package_map
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
reporter.verbosity1(
|
||||||
|
f"{constants.REPORTER_PREFIX} Skipping installation of project dependencies, env does not install project package"
|
||||||
|
)
|
||||||
reporter.verbosity1(
|
reporter.verbosity1(
|
||||||
f"{constants.REPORTER_PREFIX} config specifies 'skipsdist = true', skipping installation of project package"
|
f"{constants.REPORTER_PREFIX} Identified {len(project_deps)} project dependencies to install to env"
|
||||||
)
|
)
|
||||||
return venv.envconfig.require_locked_deps or None
|
except exceptions.ToxPoetryInstallerException as err:
|
||||||
|
venv.status = "lockfile installation failed"
|
||||||
|
reporter.error(f"{constants.REPORTER_PREFIX} {err}")
|
||||||
|
raise err
|
||||||
|
|
||||||
_install_project_dependencies(venv, poetry, package_map)
|
dependencies = list(set(dev_deps + env_deps + project_deps))
|
||||||
|
reporter.verbosity0(
|
||||||
|
f"{constants.REPORTER_PREFIX} Installing {len(dependencies)} dependencies to env '{action.name}'"
|
||||||
|
)
|
||||||
|
utilities.install_to_venv(poetry, venv, dependencies)
|
||||||
|
|
||||||
return venv.envconfig.require_locked_deps or None
|
return venv.envconfig.require_locked_deps or None
|
||||||
|
|
||||||
|
|
||||||
def _install_env_dependencies(
|
def _find_project_dependencies(
|
||||||
venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap
|
venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap
|
||||||
):
|
) -> List[PoetryPackage]:
|
||||||
"""Install the packages for a specified testenv
|
|
||||||
|
|
||||||
Processes the tox environment config, identifies any locked environment dependencies, pulls
|
|
||||||
them from the lockfile, and installs them to the virtual environment.
|
|
||||||
|
|
||||||
:param venv: Tox virtual environment to install the packages to
|
|
||||||
:param poetry: Poetry object the packages were sourced from
|
|
||||||
:param packages: Mapping of package names to the corresponding package object
|
|
||||||
"""
|
|
||||||
|
|
||||||
dependencies: List[PoetryPackage] = []
|
|
||||||
for dep in venv.envconfig.locked_deps:
|
|
||||||
try:
|
|
||||||
dependencies += utilities.find_transients(packages, dep.lower())
|
|
||||||
except exceptions.ToxPoetryInstallerException as err:
|
|
||||||
venv.status = "lockfile installation failed"
|
|
||||||
reporter.error(f"{constants.REPORTER_PREFIX} {err}")
|
|
||||||
raise err
|
|
||||||
|
|
||||||
if venv.envconfig.install_dev_deps:
|
|
||||||
reporter.verbosity1(
|
|
||||||
f"{constants.REPORTER_PREFIX} env specifies 'install_env_deps = true', including Poetry dev dependencies"
|
|
||||||
)
|
|
||||||
|
|
||||||
dev_dependencies = [
|
|
||||||
dep
|
|
||||||
for dep in poetry.locker.locked_repository(True).packages
|
|
||||||
if dep not in poetry.locker.locked_repository(False).packages
|
|
||||||
]
|
|
||||||
|
|
||||||
reporter.verbosity1(
|
|
||||||
f"{constants.REPORTER_PREFIX} identified {len(dev_dependencies)} Poetry dev dependencies"
|
|
||||||
)
|
|
||||||
|
|
||||||
dependencies = list(set(dev_dependencies + dependencies))
|
|
||||||
|
|
||||||
reporter.verbosity1(
|
|
||||||
f"{constants.REPORTER_PREFIX} identified {len(dependencies)} total dependencies from {len(venv.envconfig.locked_deps)} locked env dependencies"
|
|
||||||
)
|
|
||||||
|
|
||||||
reporter.verbosity0(
|
|
||||||
f"{constants.REPORTER_PREFIX} ({venv.name}) installing {len(dependencies)} env dependencies from lockfile"
|
|
||||||
)
|
|
||||||
utilities.install_to_venv(poetry, venv, dependencies)
|
|
||||||
|
|
||||||
|
|
||||||
def _install_project_dependencies(
|
|
||||||
venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap
|
|
||||||
):
|
|
||||||
"""Install the dependencies of the project package
|
"""Install the dependencies of the project package
|
||||||
|
|
||||||
Install all primary dependencies of the project package.
|
Install all primary dependencies of the project package.
|
||||||
@ -181,9 +139,6 @@ def _install_project_dependencies(
|
|||||||
:param poetry: Poetry object the packages were sourced from
|
:param poetry: Poetry object the packages were sourced from
|
||||||
:param packages: Mapping of package names to the corresponding package object
|
:param packages: Mapping of package names to the corresponding package object
|
||||||
"""
|
"""
|
||||||
reporter.verbosity1(
|
|
||||||
f"{constants.REPORTER_PREFIX} performing installation of project dependencies"
|
|
||||||
)
|
|
||||||
|
|
||||||
base_dependencies: List[PoetryPackage] = [
|
base_dependencies: List[PoetryPackage] = [
|
||||||
packages[item.name]
|
packages[item.name]
|
||||||
@ -204,18 +159,6 @@ def _install_project_dependencies(
|
|||||||
|
|
||||||
dependencies: List[PoetryPackage] = []
|
dependencies: List[PoetryPackage] = []
|
||||||
for dep in base_dependencies + extra_dependencies:
|
for dep in base_dependencies + extra_dependencies:
|
||||||
try:
|
dependencies += utilities.find_transients(packages, dep.name.lower())
|
||||||
dependencies += utilities.find_transients(packages, dep.name.lower())
|
|
||||||
except exceptions.ToxPoetryInstallerException as err:
|
|
||||||
venv.status = "lockfile installation failed"
|
|
||||||
reporter.error(f"{constants.REPORTER_PREFIX} {err}")
|
|
||||||
raise err
|
|
||||||
|
|
||||||
reporter.verbosity1(
|
return dependencies
|
||||||
f"{constants.REPORTER_PREFIX} identified {len(dependencies)} total dependencies from {len(poetry.package.requires)} project dependencies"
|
|
||||||
)
|
|
||||||
|
|
||||||
reporter.verbosity0(
|
|
||||||
f"{constants.REPORTER_PREFIX} ({venv.name}) installing {len(dependencies)} project dependencies from lockfile"
|
|
||||||
)
|
|
||||||
utilities.install_to_venv(poetry, venv, dependencies)
|
|
||||||
|
@ -4,12 +4,14 @@ from typing import Sequence
|
|||||||
from typing import Set
|
from typing import Set
|
||||||
|
|
||||||
from poetry.core.packages import Package as PoetryPackage
|
from poetry.core.packages import Package as PoetryPackage
|
||||||
|
from poetry.factory import Factory as PoetryFactory
|
||||||
from poetry.installation.pip_installer import PipInstaller as PoetryPipInstaller
|
from poetry.installation.pip_installer import PipInstaller as PoetryPipInstaller
|
||||||
from poetry.io.null_io import NullIO as PoetryNullIO
|
from poetry.io.null_io import NullIO as PoetryNullIO
|
||||||
from poetry.poetry import Poetry
|
from poetry.poetry import Poetry
|
||||||
from poetry.puzzle.provider import Provider as PoetryProvider
|
from poetry.puzzle.provider import Provider as PoetryProvider
|
||||||
from poetry.utils.env import VirtualEnv as PoetryVirtualEnv
|
from poetry.utils.env import VirtualEnv as PoetryVirtualEnv
|
||||||
from tox import reporter
|
from tox import reporter
|
||||||
|
from tox.action import Action as ToxAction
|
||||||
from tox.venv import VirtualEnv as ToxVirtualEnv
|
from tox.venv import VirtualEnv as ToxVirtualEnv
|
||||||
|
|
||||||
from tox_poetry_installer import constants
|
from tox_poetry_installer import constants
|
||||||
@ -38,7 +40,7 @@ def install_to_venv(
|
|||||||
)
|
)
|
||||||
|
|
||||||
for dependency in packages:
|
for dependency in packages:
|
||||||
reporter.verbosity1(f"{constants.REPORTER_PREFIX} installing {dependency}")
|
reporter.verbosity1(f"{constants.REPORTER_PREFIX} Installing {dependency}")
|
||||||
installer.install(dependency)
|
installer.install(dependency)
|
||||||
|
|
||||||
|
|
||||||
@ -60,7 +62,7 @@ def find_transients(packages: PackageMap, dependency_name: str) -> Set[PoetryPac
|
|||||||
def find_deps_of_deps(name: str, transients: PackageMap):
|
def find_deps_of_deps(name: str, transients: PackageMap):
|
||||||
if name in PoetryProvider.UNSAFE_PACKAGES:
|
if name in PoetryProvider.UNSAFE_PACKAGES:
|
||||||
reporter.warning(
|
reporter.warning(
|
||||||
f"{constants.REPORTER_PREFIX} installing package '{name}' using Poetry is not supported; skipping installation of package '{name}'"
|
f"{constants.REPORTER_PREFIX} Installing package '{name}' using Poetry is not supported; skipping installation of package '{name}'"
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
transients[name] = packages[name]
|
transients[name] = packages[name]
|
||||||
@ -83,3 +85,26 @@ def find_transients(packages: PackageMap, dependency_name: str) -> Set[PoetryPac
|
|||||||
raise exceptions.LockedDepNotFoundError(
|
raise exceptions.LockedDepNotFoundError(
|
||||||
f"No version of locked dependency '{dependency_name}' found in the project lockfile"
|
f"No version of locked dependency '{dependency_name}' found in the project lockfile"
|
||||||
) from None
|
) from None
|
||||||
|
|
||||||
|
|
||||||
|
def check_preconditions(venv: ToxVirtualEnv, action: ToxAction) -> Poetry:
|
||||||
|
"""Check that the local project environment meets expectations"""
|
||||||
|
# Skip running the plugin for the packaging environment. PEP-517 front ends can handle
|
||||||
|
# that better than we can, so let them do their thing. More to the point: if you're having
|
||||||
|
# problems in the packaging env that this plugin would solve, god help you.
|
||||||
|
if action.name == venv.envconfig.config.isolated_build_env:
|
||||||
|
raise exceptions.SkipEnvironment(
|
||||||
|
f"Skipping isolated packaging build env '{action.name}'"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
return PoetryFactory().create_poetry(venv.envconfig.config.toxinidir)
|
||||||
|
# Support running the plugin when the current tox project does not use Poetry for its
|
||||||
|
# environment/dependency management.
|
||||||
|
#
|
||||||
|
# ``RuntimeError`` is dangerous to blindly catch because it can be (and in Poetry's case,
|
||||||
|
# is) raised in many different places for different purposes.
|
||||||
|
except RuntimeError:
|
||||||
|
raise exceptions.SkipEnvironment(
|
||||||
|
"Project does not use Poetry for env management, skipping installation of locked dependencies"
|
||||||
|
) from None
|
||||||
|
Loading…
Reference in New Issue
Block a user