mirror of
https://github.com/enpaul/tox-poetry-installer.git
synced 2024-10-29 19:47:00 +00:00
Compare commits
No commits in common. "536fd6536b63cf3d6367b62e0c486508ac88d44e" and "e875008bff5fe135003492120f465dd06060bd84" have entirely different histories.
536fd6536b
...
e875008bff
1
.github/scripts/setup-env.sh
vendored
1
.github/scripts/setup-env.sh
vendored
@ -26,6 +26,7 @@ poetry --version --no-ansi;
|
||||
poetry run pip --version;
|
||||
|
||||
poetry install \
|
||||
--extras poetry \
|
||||
--quiet \
|
||||
--remove-untracked \
|
||||
--no-ansi;
|
||||
|
||||
4
.github/workflows/ci.yaml
vendored
4
.github/workflows/ci.yaml
vendored
@ -12,6 +12,8 @@ jobs:
|
||||
strategy:
|
||||
matrix:
|
||||
python:
|
||||
- version: "3.7"
|
||||
toxenv: py37
|
||||
- version: "3.8"
|
||||
toxenv: py38
|
||||
- version: "3.9"
|
||||
@ -20,8 +22,6 @@ jobs:
|
||||
toxenv: py310
|
||||
- version: "3.11"
|
||||
toxenv: py311
|
||||
- version: "3.12"
|
||||
toxenv: py312
|
||||
fail-fast: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
||||
@ -47,13 +47,12 @@ repos:
|
||||
|
||||
- id: reorder-python-imports
|
||||
name: reorder-python-imports
|
||||
entry: isort
|
||||
entry: reorder-python-imports
|
||||
language: system
|
||||
require_serial: true
|
||||
args:
|
||||
- "--unclassifiable-application-module=tox_poetry_installer"
|
||||
types:
|
||||
- python
|
||||
args:
|
||||
- "--filter-files"
|
||||
|
||||
- id: black
|
||||
name: black
|
||||
|
||||
@ -11,6 +11,7 @@
|
||||
# --disable=W"
|
||||
disable=logging-fstring-interpolation
|
||||
,logging-format-interpolation
|
||||
,bad-continuation
|
||||
,line-too-long
|
||||
,ungrouped-imports
|
||||
,typecheck
|
||||
|
||||
@ -227,6 +227,7 @@ error will be set to one of the "Status" values below to indicate what the error
|
||||
| `LockedDepNotFoundError` | Indicates that an item specified in the `locked_deps` config option does not match the name of a package in the Poetry lockfile. |
|
||||
| `LockedDepsRequiredError` | Indicates that a test environment with the `require_locked_deps` config option set to `true` also specified unlocked dependencies using the [`deps`](https://tox.readthedocs.io/en/latest/config.html#conf-deps) config option. |
|
||||
| `PoetryNotInstalledError` | Indicates that the `poetry` module could not be imported under the current runtime environment, and `require_poetry = true` was specified. |
|
||||
| `RequiresUnsafeDepError` | Indicates that the package-under-test depends on a package that Poetry has classified as unsafe and cannot be installed. |
|
||||
|
||||
> ℹ️ **Note:** One or more of these errors can be caused by the `pyproject.toml` being out
|
||||
> of sync with the Poetry lockfile. If this is the case, than a warning will be logged
|
||||
|
||||
2935
poetry.lock
generated
2935
poetry.lock
generated
File diff suppressed because it is too large
Load Diff
@ -23,46 +23,45 @@ classifiers = [
|
||||
"Natural Language :: English",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.7",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: Implementation :: CPython",
|
||||
]
|
||||
|
||||
[tool.poetry.plugins.tox]
|
||||
poetry_installer = "tox_poetry_installer"
|
||||
|
||||
[tool.poetry.extras]
|
||||
poetry = ["poetry", "cleo"]
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.8"
|
||||
cleo = ">=1.0,<3.0"
|
||||
poetry = "^1.5.0"
|
||||
python = "^3.7"
|
||||
cleo = {version = ">=1.0,<3.0", optional = true}
|
||||
poetry = {version = "^1.5.0", optional = true}
|
||||
poetry-core = "^1.1.0"
|
||||
tox = "^4.1"
|
||||
tox = "^4"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
bandit = {version = "^1.7.7", python = "^3.10"}
|
||||
black = {version = "^24.3.0", python = "^3.10"}
|
||||
blacken-docs = {version = "^1.18.0", python = "^3.10"}
|
||||
ipython = {version = "^8.10.1", python = "^3.10"}
|
||||
isort = {version = "^5.13.2", python = "^3.10"}
|
||||
mdformat = {version = "^0.7", python = "^3.10"}
|
||||
mdformat-gfm = {version = "^0.3", python = "^3.10"}
|
||||
mypy = {version = "^1.11.1", python = "^3.10"}
|
||||
pre-commit = {version = "^3.8.0", python = "^3.10"}
|
||||
pre-commit-hooks = {version = "^4.6.0", python = "^3.10"}
|
||||
pylint = {version = "^3.2.6", python = "^3.10"}
|
||||
pytest = {version = "^8.3.2", python = "^3.10"}
|
||||
pytest-cov = {version = "^5.0.0", python = "^3.10"}
|
||||
toml = {version = "^0.10.1", python = "^3.10"}
|
||||
tox = "^4.1"
|
||||
types-toml = {version = "^0.10.1", python = "^3.10"}
|
||||
|
||||
[tool.isort]
|
||||
profile = "black"
|
||||
force_single_line = "true"
|
||||
lines_after_imports = 2
|
||||
bandit = "^1.6.2"
|
||||
black = "^22.3.0"
|
||||
blacken-docs = "^1.8.0"
|
||||
ipython = {version = "^8.10.1", python = "^3.8"}
|
||||
mdformat = "^0.7"
|
||||
mdformat-gfm = "^0.3"
|
||||
mypy = "^0.930"
|
||||
pre-commit = "^2.7.1"
|
||||
pre-commit-hooks = "^3.3.0"
|
||||
pylint = "^2.13.0"
|
||||
pytest = "^6.0.2"
|
||||
pytest-cov = "^2.10.1"
|
||||
reorder-python-imports = "^2.3.5"
|
||||
safety = "^2.2.0"
|
||||
toml = "^0.10.1"
|
||||
tox = "^4"
|
||||
types-toml = "^0.10.1"
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core>=1.1.0"]
|
||||
|
||||
@ -1,16 +1,16 @@
|
||||
# pylint: disable=missing-module-docstring,missing-function-docstring,unused-argument,too-few-public-methods,protected-access
|
||||
# pylint: disable=missing-module-docstring, missing-function-docstring, unused-argument, too-few-public-methods
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
import poetry.factory
|
||||
import poetry.installation.executor
|
||||
import poetry.installation.operations.operation
|
||||
import poetry.utils.env
|
||||
import pytest
|
||||
import tox.tox_env.python.virtual_env.runner
|
||||
from poetry.installation.operations.operation import Operation
|
||||
|
||||
import tox_poetry_installer.hooks._tox_on_install_helpers
|
||||
from tox_poetry_installer import utilities
|
||||
|
||||
|
||||
TEST_PROJECT_PATH = Path(__file__).parent.resolve() / "test-project"
|
||||
@ -40,20 +40,14 @@ class MockExecutor:
|
||||
def __init__(self, env: MockVirtualEnv, **kwargs):
|
||||
self.env = env
|
||||
|
||||
def execute(
|
||||
self, operations: List[poetry.installation.operations.operation.Operation]
|
||||
):
|
||||
def execute(self, operations: List[Operation]):
|
||||
self.env.installed.extend([operation.package for operation in operations])
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_venv(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
tox_poetry_installer.hooks._tox_on_install_helpers,
|
||||
"convert_virtualenv",
|
||||
lambda venv: venv,
|
||||
)
|
||||
monkeypatch.setattr(utilities, "convert_virtualenv", lambda venv: venv)
|
||||
monkeypatch.setattr(poetry.installation.executor, "Executor", MockExecutor)
|
||||
monkeypatch.setattr(
|
||||
tox.tox_env.python.virtual_env.runner, "VirtualEnvRunner", MockVirtualEnv
|
||||
@ -63,9 +57,9 @@ def mock_venv(monkeypatch):
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def mock_poetry_factory(monkeypatch):
|
||||
project = poetry.factory.Factory().create_poetry(cwd=TEST_PROJECT_PATH)
|
||||
pypoetry = poetry.factory.Factory().create_poetry(cwd=TEST_PROJECT_PATH)
|
||||
|
||||
def mock_factory(*args, **kwargs):
|
||||
return project
|
||||
return pypoetry
|
||||
|
||||
monkeypatch.setattr(poetry.factory.Factory, "create_poetry", mock_factory)
|
||||
|
||||
@ -1,40 +1,37 @@
|
||||
# pylint: disable=missing-module-docstring,redefined-outer-name,unused-argument,unused-import,protected-access
|
||||
# pylint: disable=missing-module-docstring, redefined-outer-name, unused-argument, wrong-import-order, unused-import
|
||||
import time
|
||||
from unittest import mock
|
||||
|
||||
import poetry.factory
|
||||
import poetry.installation.executor
|
||||
import pytest
|
||||
import tox.tox_env.python.virtual_env.runner
|
||||
|
||||
import tox_poetry_installer.hooks._tox_on_install_helpers
|
||||
from poetry.factory import Factory
|
||||
|
||||
from .fixtures import mock_poetry_factory
|
||||
from .fixtures import mock_venv
|
||||
from tox_poetry_installer import installer
|
||||
from tox_poetry_installer import utilities
|
||||
|
||||
|
||||
def test_deduplication(mock_venv, mock_poetry_factory):
|
||||
"""Test that the installer does not install duplicate dependencies"""
|
||||
project = poetry.factory.Factory().create_poetry(None)
|
||||
packages: tox_poetry_installer.hooks._tox_on_install_helpers.PackageMap = {
|
||||
item.name: item for item in project.locker.locked_repository().packages
|
||||
poetry = Factory().create_poetry(None)
|
||||
packages: utilities.PackageMap = {
|
||||
item.name: item for item in poetry.locker.locked_repository().packages
|
||||
}
|
||||
|
||||
venv = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner()
|
||||
to_install = [packages["toml"], packages["toml"]]
|
||||
|
||||
tox_poetry_installer.hooks._tox_on_install_helpers.install_package(
|
||||
project, venv, to_install
|
||||
)
|
||||
installer.install(poetry, venv, to_install)
|
||||
|
||||
assert len(set(to_install)) == len(venv.installed) # pylint: disable=no-member
|
||||
|
||||
|
||||
def test_parallelization(mock_venv, mock_poetry_factory):
|
||||
"""Test that behavior is consistent between parallel and non-parallel usage"""
|
||||
project = poetry.factory.Factory().create_poetry(None)
|
||||
packages: tox_poetry_installer.hooks._tox_on_install_helpers.PackageMap = {
|
||||
item.name: item for item in project.locker.locked_repository().packages
|
||||
poetry = Factory().create_poetry(None)
|
||||
packages: utilities.PackageMap = {
|
||||
item.name: item for item in poetry.locker.locked_repository().packages
|
||||
}
|
||||
|
||||
to_install = [
|
||||
@ -48,16 +45,12 @@ def test_parallelization(mock_venv, mock_poetry_factory):
|
||||
|
||||
venv_sequential = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner()
|
||||
start_sequential = time.time()
|
||||
tox_poetry_installer.hooks._tox_on_install_helpers.install_package(
|
||||
project, venv_sequential, to_install, 0
|
||||
)
|
||||
installer.install(poetry, venv_sequential, to_install, 0)
|
||||
sequential = time.time() - start_sequential
|
||||
|
||||
venv_parallel = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner()
|
||||
start_parallel = time.time()
|
||||
tox_poetry_installer.hooks._tox_on_install_helpers.install_package(
|
||||
project, venv_parallel, to_install, 5
|
||||
)
|
||||
installer.install(poetry, venv_parallel, to_install, 5)
|
||||
parallel = time.time() - start_parallel
|
||||
|
||||
# The mock delay during package install is static (one second) so these values should all
|
||||
@ -76,22 +69,22 @@ def test_propagates_exceptions_during_installation(
|
||||
|
||||
Regression test for https://github.com/enpaul/tox-poetry-installer/issues/86
|
||||
"""
|
||||
project = poetry.factory.Factory().create_poetry(None)
|
||||
packages: tox_poetry_installer.hooks._tox_on_install_helpers.PackageMap = {
|
||||
item.name: item for item in project.locker.locked_repository().packages
|
||||
from tox_poetry_installer import _poetry # pylint: disable=import-outside-toplevel
|
||||
|
||||
poetry = Factory().create_poetry(None)
|
||||
packages: utilities.PackageMap = {
|
||||
item.name: item for item in poetry.locker.locked_repository().packages
|
||||
}
|
||||
to_install = [packages["toml"]]
|
||||
venv = tox.tox_env.python.virtual_env.runner.VirtualEnvRunner()
|
||||
fake_exception = ValueError("my testing exception")
|
||||
|
||||
with mock.patch.object(
|
||||
poetry.installation.executor,
|
||||
_poetry,
|
||||
"Executor",
|
||||
**{"return_value.execute.side_effect": fake_exception},
|
||||
):
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
tox_poetry_installer.hooks._tox_on_install_helpers.install_package(
|
||||
project, venv, to_install, num_threads
|
||||
)
|
||||
installer.install(poetry, venv, to_install, num_threads)
|
||||
|
||||
assert exc_info.value is fake_exception
|
||||
|
||||
@ -3,7 +3,6 @@
|
||||
The next best thing to having one source of truth is having a way to ensure all of your
|
||||
sources of truth agree with each other.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import toml
|
||||
|
||||
@ -1,23 +1,33 @@
|
||||
# pylint: disable=missing-module-docstring,redefined-outer-name,unused-argument,unused-import,protected-access
|
||||
# pylint: disable=missing-module-docstring, redefined-outer-name, unused-argument, wrong-import-order, unused-import
|
||||
import poetry.factory
|
||||
import poetry.utils.env
|
||||
import pytest
|
||||
|
||||
import tox_poetry_installer.hooks._tox_on_install_helpers
|
||||
from tox_poetry_installer import exceptions
|
||||
from poetry.puzzle.provider import Provider
|
||||
|
||||
from .fixtures import mock_poetry_factory
|
||||
from .fixtures import mock_venv
|
||||
from tox_poetry_installer import constants
|
||||
from tox_poetry_installer import exceptions
|
||||
from tox_poetry_installer import utilities
|
||||
|
||||
|
||||
def test_exclude_unsafe():
|
||||
"""Test that the unsafe packages are properly excluded
|
||||
|
||||
Also ensure that the internal constant matches the value from Poetry
|
||||
"""
|
||||
assert Provider.UNSAFE_PACKAGES == constants.UNSAFE_PACKAGES
|
||||
|
||||
for dep in constants.UNSAFE_PACKAGES:
|
||||
assert not utilities.identify_transients(dep, {}, None)
|
||||
|
||||
|
||||
def test_allow_missing():
|
||||
"""Test that the ``allow_missing`` parameter works as expected"""
|
||||
with pytest.raises(exceptions.LockedDepNotFoundError):
|
||||
tox_poetry_installer.hooks._tox_on_install_helpers.identify_transients(
|
||||
"luke-skywalker", {}, None
|
||||
)
|
||||
utilities.identify_transients("luke-skywalker", {}, None)
|
||||
|
||||
assert not tox_poetry_installer.hooks._tox_on_install_helpers.identify_transients(
|
||||
assert not utilities.identify_transients(
|
||||
"darth-vader", {}, None, allow_missing=["darth-vader"]
|
||||
)
|
||||
|
||||
@ -37,9 +47,7 @@ def test_exclude_pep508():
|
||||
"=>foo",
|
||||
]:
|
||||
with pytest.raises(exceptions.LockedDepVersionConflictError):
|
||||
tox_poetry_installer.hooks._tox_on_install_helpers.identify_transients(
|
||||
version, {}, None
|
||||
)
|
||||
utilities.identify_transients(version, {}, None)
|
||||
|
||||
|
||||
def test_functional(mock_poetry_factory, mock_venv):
|
||||
@ -48,10 +56,8 @@ def test_functional(mock_poetry_factory, mock_venv):
|
||||
Trivially test that it resolves dependencies properly and that the parent package
|
||||
is always the last in the returned list.
|
||||
"""
|
||||
project = poetry.factory.Factory().create_poetry(None)
|
||||
packages = tox_poetry_installer.hooks._tox_on_install_helpers.build_package_map(
|
||||
project
|
||||
)
|
||||
pypoetry = poetry.factory.Factory().create_poetry(None)
|
||||
packages = utilities.build_package_map(pypoetry)
|
||||
venv = poetry.utils.env.VirtualEnv() # pylint: disable=no-value-for-parameter
|
||||
|
||||
requests_requires = [
|
||||
@ -62,18 +68,12 @@ def test_functional(mock_poetry_factory, mock_venv):
|
||||
packages["requests"][0],
|
||||
]
|
||||
|
||||
transients = tox_poetry_installer.hooks._tox_on_install_helpers.identify_transients(
|
||||
"requests", packages, venv
|
||||
)
|
||||
transients = utilities.identify_transients("requests", packages, venv)
|
||||
|
||||
assert all((item in requests_requires) for item in transients)
|
||||
assert all((item in transients) for item in requests_requires)
|
||||
|
||||
for package in [packages["requests"][0], packages["tox"][0], packages["flask"][0]]:
|
||||
transients = (
|
||||
tox_poetry_installer.hooks._tox_on_install_helpers.identify_transients(
|
||||
package.name, packages, venv
|
||||
)
|
||||
)
|
||||
transients = utilities.identify_transients(package.name, packages, venv)
|
||||
assert transients[-1] == package
|
||||
assert len(transients) == len(set(transients))
|
||||
|
||||
18
tox.ini
18
tox.ini
@ -1,11 +1,13 @@
|
||||
[tox]
|
||||
envlist = py3{8,9,10,11,12} static, static-tests, security
|
||||
envlist = py37, py38, py39, py310, py311, static, static-tests, security
|
||||
skip_missing_interpreters = true
|
||||
|
||||
[testenv]
|
||||
description = Run the tests
|
||||
require_locked_deps = true
|
||||
require_poetry = true
|
||||
extras =
|
||||
poetry
|
||||
locked_deps =
|
||||
pytest
|
||||
pytest-cov
|
||||
@ -24,10 +26,10 @@ ignore_errors = true
|
||||
locked_deps =
|
||||
black
|
||||
blacken-docs
|
||||
isort
|
||||
mdformat
|
||||
mdformat-gfm
|
||||
mypy
|
||||
reorder-python-imports
|
||||
pre-commit
|
||||
pre-commit-hooks
|
||||
pylint
|
||||
@ -50,7 +52,6 @@ locked_deps =
|
||||
pylint
|
||||
pytest
|
||||
mypy
|
||||
toml
|
||||
types-toml
|
||||
commands =
|
||||
pylint {toxinidir}/tests/ \
|
||||
@ -77,3 +78,14 @@ commands =
|
||||
--recursive \
|
||||
--quiet \
|
||||
--skip B101
|
||||
poetry export \
|
||||
--format requirements.txt \
|
||||
--output {envtmpdir}/requirements.txt \
|
||||
--without-hashes \
|
||||
--with dev \
|
||||
--extras poetry
|
||||
safety check \
|
||||
--file {envtmpdir}/requirements.txt \
|
||||
--output text \
|
||||
# https://github.com/pytest-dev/py/issues/287
|
||||
--ignore 51457
|
||||
|
||||
43
tox_poetry_installer/_poetry.py
Normal file
43
tox_poetry_installer/_poetry.py
Normal file
@ -0,0 +1,43 @@
|
||||
"""You've heard of vendoirization, now get ready for internal namespace shadowing
|
||||
|
||||
Poetry is an optional dependency of this package explicitly to support the use case of having the
|
||||
plugin and the `poetry` package installed to the same python environment; this is most common in
|
||||
containers and/or CI. In this case there are two potential problems that can arise in this case:
|
||||
|
||||
* The installation of the plugin overwrites the installed version of Poetry resulting in
|
||||
compatibility issues.
|
||||
* Running `poetry install --no-dev`, when this plugin is in the dev-deps, results in poetry being
|
||||
uninstalled from the environment.
|
||||
|
||||
To support these edge cases, and more broadly to support not messing with a system package manager,
|
||||
the `poetry` package dependency is listed as optional dependency. This allows the plugin to be
|
||||
installed to the same environment as Poetry and import that same Poetry installation here.
|
||||
|
||||
However, simply importing Poetry on the assumption that it is installed breaks another valid use
|
||||
case: having this plugin installed alongside Tox when not using a Poetry-based project. To account
|
||||
for this the imports in this module are isolated and the resultant import error that would result
|
||||
is converted to an internal error that can be caught by callers. Rather than importing this module
|
||||
at the module scope it is imported into function scope wherever Poetry components are needed. This
|
||||
moves import errors from load time to runtime which allows the plugin to be skipped if Poetry isn't
|
||||
installed and/or a more helpful error be raised within the Tox framework.
|
||||
"""
|
||||
# pylint: disable=unused-import
|
||||
import sys
|
||||
|
||||
from tox_poetry_installer import exceptions
|
||||
|
||||
|
||||
try:
|
||||
from cleo.io.null_io import NullIO
|
||||
from poetry.config.config import Config
|
||||
from poetry.core.packages.dependency import Dependency as PoetryDependency
|
||||
from poetry.core.packages.package import Package as PoetryPackage
|
||||
from poetry.factory import Factory
|
||||
from poetry.installation.executor import Executor
|
||||
from poetry.installation.operations.install import Install
|
||||
from poetry.poetry import Poetry
|
||||
from poetry.utils.env import VirtualEnv
|
||||
except ImportError:
|
||||
raise exceptions.PoetryNotInstalledError(
|
||||
f"No version of Poetry could be imported under the current environment for '{sys.executable}'"
|
||||
) from None
|
||||
@ -5,7 +5,7 @@ in this module.
|
||||
|
||||
All constants should be type hinted.
|
||||
"""
|
||||
|
||||
from typing import Set
|
||||
from typing import Tuple
|
||||
|
||||
from tox_poetry_installer import __about__
|
||||
@ -19,5 +19,9 @@ PEP508_VERSION_DELIMITERS: Tuple[str, ...] = ("~=", "==", "!=", ">", "<")
|
||||
# console output.
|
||||
REPORTER_PREFIX: str = f"{__about__.__title__}:"
|
||||
|
||||
# Internal list of packages that poetry has deemed unsafe and are excluded from the lockfile
|
||||
# TODO: This functionality is no longer needed, should remove in a future update.
|
||||
UNSAFE_PACKAGES: Set[str] = set()
|
||||
|
||||
# Number of threads to use for installing dependencies by default
|
||||
DEFAULT_INSTALL_THREADS: int = 10
|
||||
|
||||
@ -11,7 +11,7 @@ All exceptions should inherit from the common base exception :exc:`ToxPoetryInst
|
||||
+-- LockedDepNotFoundError
|
||||
+-- ExtraNotFoundError
|
||||
+-- LockedDepsRequiredError
|
||||
+-- LockfileParsingError
|
||||
+-- RequiresUnsafeDepError
|
||||
|
||||
"""
|
||||
|
||||
@ -44,5 +44,5 @@ class LockedDepsRequiredError(ToxPoetryInstallerException):
|
||||
"""Environment cannot specify unlocked dependencies when locked dependencies are required"""
|
||||
|
||||
|
||||
class LockfileParsingError(ToxPoetryInstallerException):
|
||||
"""Failed to load or parse the Poetry lockfile"""
|
||||
class RequiresUnsafeDepError(ToxPoetryInstallerException):
|
||||
"""Package under test depends on an unsafe dependency and cannot be installed"""
|
||||
|
||||
175
tox_poetry_installer/hooks.py
Normal file
175
tox_poetry_installer/hooks.py
Normal file
@ -0,0 +1,175 @@
|
||||
"""Main hook definition module
|
||||
|
||||
All implementations of tox hooks are defined here, as well as any single-use helper functions
|
||||
specifically related to implementing the hooks (to keep the size/readability of the hook functions
|
||||
themselves manageable).
|
||||
"""
|
||||
from itertools import chain
|
||||
from typing import List
|
||||
|
||||
from tox.config.cli.parser import ToxParser
|
||||
from tox.config.sets import EnvConfigSet
|
||||
from tox.plugin import impl
|
||||
from tox.tox_env.api import ToxEnv as ToxVirtualEnv
|
||||
|
||||
from tox_poetry_installer import constants
|
||||
from tox_poetry_installer import exceptions
|
||||
from tox_poetry_installer import installer
|
||||
from tox_poetry_installer import logger
|
||||
from tox_poetry_installer import utilities
|
||||
|
||||
|
||||
@impl
|
||||
def tox_add_option(parser: ToxParser):
|
||||
"""Add additional command line arguments to tox to configure plugin behavior"""
|
||||
parser.add_argument(
|
||||
"--require-poetry",
|
||||
action="store_true",
|
||||
dest="require_poetry",
|
||||
help="(deprecated) Trigger a failure if Poetry is not available to Tox",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--parallel-install-threads",
|
||||
type=int,
|
||||
dest="parallel_install_threads",
|
||||
default=constants.DEFAULT_INSTALL_THREADS,
|
||||
help="Number of locked dependencies to install simultaneously; set to 0 to disable parallel installation",
|
||||
)
|
||||
|
||||
|
||||
@impl
|
||||
def tox_add_env_config(env_conf: EnvConfigSet):
|
||||
"""Add required env configuration options to the tox INI file"""
|
||||
env_conf.add_config(
|
||||
"poetry_dep_groups",
|
||||
of_type=List[str],
|
||||
default=[],
|
||||
desc="List of Poetry dependency groups to install to the environment",
|
||||
)
|
||||
|
||||
env_conf.add_config(
|
||||
"install_project_deps",
|
||||
of_type=bool,
|
||||
default=True,
|
||||
desc="Automatically install all Poetry primary dependencies to the environment",
|
||||
)
|
||||
|
||||
env_conf.add_config(
|
||||
"require_locked_deps",
|
||||
of_type=bool,
|
||||
default=False,
|
||||
desc="Require all dependencies in the environment be installed using the Poetry lockfile",
|
||||
)
|
||||
|
||||
env_conf.add_config(
|
||||
"require_poetry",
|
||||
of_type=bool,
|
||||
default=False,
|
||||
desc="Trigger a failure if Poetry is not available to Tox",
|
||||
)
|
||||
|
||||
env_conf.add_config(
|
||||
"locked_deps",
|
||||
of_type=List[str],
|
||||
default=[],
|
||||
desc="List of locked dependencies to install to the environment using the Poetry lockfile",
|
||||
)
|
||||
|
||||
|
||||
@impl
|
||||
def tox_on_install(
|
||||
tox_env: ToxVirtualEnv, section: str # pylint: disable=unused-argument
|
||||
) -> None:
|
||||
"""Install the dependencies for the current environment
|
||||
|
||||
Loads the local Poetry environment and the corresponding lockfile then pulls the dependencies
|
||||
specified by the Tox environment. Finally these dependencies are installed into the Tox
|
||||
environment using the Poetry ``PipInstaller`` backend.
|
||||
|
||||
:param venv: Tox virtual environment object with configuration for the local Tox environment.
|
||||
:param action: Tox action object
|
||||
"""
|
||||
try:
|
||||
poetry = utilities.check_preconditions(tox_env)
|
||||
except exceptions.SkipEnvironment as err:
|
||||
if (
|
||||
isinstance(err, exceptions.PoetryNotInstalledError)
|
||||
and tox_env.conf["require_poetry"]
|
||||
):
|
||||
logger.error(str(err))
|
||||
raise err
|
||||
logger.info(str(err))
|
||||
return
|
||||
|
||||
logger.info(f"Loaded project pyproject.toml from {poetry.file}")
|
||||
|
||||
virtualenv = utilities.convert_virtualenv(tox_env)
|
||||
|
||||
if not poetry.locker.is_fresh():
|
||||
logger.warning(
|
||||
f"The Poetry lock file is not up to date with the latest changes in {poetry.file}"
|
||||
)
|
||||
|
||||
try:
|
||||
if tox_env.conf["require_locked_deps"] and tox_env.conf["deps"].lines():
|
||||
raise exceptions.LockedDepsRequiredError(
|
||||
f"Unlocked dependencies '{tox_env.conf['deps']}' specified for environment '{tox_env.name}' which requires locked dependencies"
|
||||
)
|
||||
|
||||
packages = utilities.build_package_map(poetry)
|
||||
|
||||
group_deps = utilities.dedupe_packages(
|
||||
list(
|
||||
chain(
|
||||
*[
|
||||
utilities.find_group_deps(group, packages, virtualenv, poetry)
|
||||
for group in tox_env.conf["poetry_dep_groups"]
|
||||
]
|
||||
)
|
||||
)
|
||||
)
|
||||
logger.info(
|
||||
f"Identified {len(group_deps)} group dependencies to install to env"
|
||||
)
|
||||
|
||||
env_deps = utilities.find_additional_deps(
|
||||
packages, virtualenv, poetry, tox_env.conf["locked_deps"]
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Identified {len(env_deps)} environment dependencies to install to env"
|
||||
)
|
||||
|
||||
# extras are not set in a testenv if skip_install=true
|
||||
try:
|
||||
extras = tox_env.conf["extras"]
|
||||
except KeyError:
|
||||
extras = []
|
||||
|
||||
if tox_env.conf["install_project_deps"]:
|
||||
project_deps = utilities.find_project_deps(
|
||||
packages, virtualenv, poetry, extras
|
||||
)
|
||||
logger.info(
|
||||
f"Identified {len(project_deps)} project dependencies to install to env"
|
||||
)
|
||||
else:
|
||||
project_deps = []
|
||||
logger.info("Env does not install project package dependencies, skipping")
|
||||
except exceptions.ToxPoetryInstallerException as err:
|
||||
logger.error(str(err))
|
||||
raise err
|
||||
except Exception as err:
|
||||
logger.error(f"Internal plugin error: {err}")
|
||||
raise err
|
||||
|
||||
dependencies = utilities.dedupe_packages(group_deps + env_deps + project_deps)
|
||||
|
||||
logger.info(f"Installing {len(dependencies)} dependencies from Poetry lock file")
|
||||
installer.install(
|
||||
poetry,
|
||||
tox_env,
|
||||
dependencies,
|
||||
tox_env.options.parallel_install_threads,
|
||||
)
|
||||
@ -1,4 +0,0 @@
|
||||
# pylint: disable=missing-module-docstring
|
||||
from tox_poetry_installer.hooks.tox_add_env_config import tox_add_env_config
|
||||
from tox_poetry_installer.hooks.tox_add_option import tox_add_option
|
||||
from tox_poetry_installer.hooks.tox_on_install import tox_on_install
|
||||
@ -1,47 +0,0 @@
|
||||
"""Add required env configuration options to the tox INI file"""
|
||||
|
||||
from typing import List
|
||||
|
||||
import tox.config.sets
|
||||
import tox.plugin
|
||||
|
||||
|
||||
# pylint: disable=missing-function-docstring
|
||||
@tox.plugin.impl
|
||||
def tox_add_env_config(
|
||||
env_conf: tox.config.sets.EnvConfigSet,
|
||||
):
|
||||
env_conf.add_config(
|
||||
"poetry_dep_groups",
|
||||
of_type=List[str],
|
||||
default=[],
|
||||
desc="List of Poetry dependency groups to install to the environment",
|
||||
)
|
||||
|
||||
env_conf.add_config(
|
||||
"install_project_deps",
|
||||
of_type=bool,
|
||||
default=True,
|
||||
desc="Automatically install all Poetry primary dependencies to the environment",
|
||||
)
|
||||
|
||||
env_conf.add_config(
|
||||
"require_locked_deps",
|
||||
of_type=bool,
|
||||
default=False,
|
||||
desc="Require all dependencies in the environment be installed using the Poetry lockfile",
|
||||
)
|
||||
|
||||
env_conf.add_config(
|
||||
"require_poetry",
|
||||
of_type=bool,
|
||||
default=False,
|
||||
desc="Trigger a failure if Poetry is not available to Tox",
|
||||
)
|
||||
|
||||
env_conf.add_config(
|
||||
"locked_deps",
|
||||
of_type=List[str],
|
||||
default=[],
|
||||
desc="List of locked dependencies to install to the environment using the Poetry lockfile",
|
||||
)
|
||||
@ -1,18 +0,0 @@
|
||||
"""Add additional command line arguments to tox to configure plugin behavior"""
|
||||
|
||||
import tox.config.cli.parser
|
||||
import tox.plugin
|
||||
|
||||
from tox_poetry_installer import constants
|
||||
|
||||
|
||||
# pylint: disable=missing-function-docstring
|
||||
@tox.plugin.impl
|
||||
def tox_add_option(parser: tox.config.cli.parser.ToxParser):
|
||||
parser.add_argument(
|
||||
"--parallel-install-threads",
|
||||
type=int,
|
||||
dest="parallel_install_threads",
|
||||
default=constants.DEFAULT_INSTALL_THREADS,
|
||||
help="Number of locked dependencies to install simultaneously; set to 0 to disable parallel installation",
|
||||
)
|
||||
@ -1,114 +0,0 @@
|
||||
"""Install the dependencies for the current environment
|
||||
|
||||
Loads the local Poetry environment and the corresponding lockfile then pulls the dependencies
|
||||
specified by the Tox environment. Finally these dependencies are installed into the Tox
|
||||
environment using the Poetry ``PipInstaller`` backend.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
|
||||
import tox.plugin
|
||||
import tox.tox_env.api
|
||||
|
||||
from tox_poetry_installer import exceptions
|
||||
from tox_poetry_installer import logger
|
||||
from tox_poetry_installer.hooks._tox_on_install_helpers import build_package_map
|
||||
from tox_poetry_installer.hooks._tox_on_install_helpers import check_preconditions
|
||||
from tox_poetry_installer.hooks._tox_on_install_helpers import convert_virtualenv
|
||||
from tox_poetry_installer.hooks._tox_on_install_helpers import dedupe_packages
|
||||
from tox_poetry_installer.hooks._tox_on_install_helpers import find_additional_deps
|
||||
from tox_poetry_installer.hooks._tox_on_install_helpers import find_group_deps
|
||||
from tox_poetry_installer.hooks._tox_on_install_helpers import find_project_deps
|
||||
from tox_poetry_installer.hooks._tox_on_install_helpers import install_package
|
||||
|
||||
|
||||
# pylint: disable=missing-function-docstring,unused-argument
|
||||
@tox.plugin.impl
|
||||
def tox_on_install(tox_env: tox.tox_env.api.ToxEnv, *args) -> None:
|
||||
try:
|
||||
poetry = check_preconditions(tox_env)
|
||||
except exceptions.SkipEnvironment as err:
|
||||
if (
|
||||
isinstance(err, exceptions.PoetryNotInstalledError)
|
||||
and tox_env.conf["require_poetry"]
|
||||
):
|
||||
logger.error(str(err))
|
||||
raise err
|
||||
logger.info(str(err))
|
||||
return
|
||||
|
||||
logger.info(f"Loaded project pyproject.toml from {poetry.file}")
|
||||
|
||||
virtualenv = convert_virtualenv(tox_env)
|
||||
|
||||
try:
|
||||
if not poetry.locker.is_fresh():
|
||||
logger.warning(
|
||||
f"The Poetry lock file is not up to date with the latest changes in {poetry.file}"
|
||||
)
|
||||
except FileNotFoundError as err:
|
||||
logger.error(f"Could not parse lockfile: {err}")
|
||||
raise exceptions.LockfileParsingError(
|
||||
f"Could not parse lockfile: {err}"
|
||||
) from err
|
||||
|
||||
try:
|
||||
if tox_env.conf["require_locked_deps"] and tox_env.conf["deps"].lines():
|
||||
raise exceptions.LockedDepsRequiredError(
|
||||
f"Unlocked dependencies '{tox_env.conf['deps']}' specified for environment '{tox_env.name}' which requires locked dependencies"
|
||||
)
|
||||
|
||||
packages = build_package_map(poetry)
|
||||
|
||||
group_deps = dedupe_packages(
|
||||
list(
|
||||
itertools.chain(
|
||||
*[
|
||||
find_group_deps(group, packages, virtualenv, poetry)
|
||||
for group in tox_env.conf["poetry_dep_groups"]
|
||||
]
|
||||
)
|
||||
)
|
||||
)
|
||||
logger.info(
|
||||
f"Identified {len(group_deps)} group dependencies to install to env"
|
||||
)
|
||||
|
||||
env_deps = find_additional_deps(
|
||||
packages, virtualenv, poetry, tox_env.conf["locked_deps"]
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Identified {len(env_deps)} environment dependencies to install to env"
|
||||
)
|
||||
|
||||
# extras are not set in a testenv if skip_install=true
|
||||
try:
|
||||
extras = tox_env.conf["extras"]
|
||||
except KeyError:
|
||||
extras = []
|
||||
|
||||
if tox_env.conf["install_project_deps"]:
|
||||
project_deps = find_project_deps(packages, virtualenv, poetry, extras)
|
||||
logger.info(
|
||||
f"Identified {len(project_deps)} project dependencies to install to env"
|
||||
)
|
||||
else:
|
||||
project_deps = []
|
||||
logger.info("Env does not install project package dependencies, skipping")
|
||||
except exceptions.ToxPoetryInstallerException as err:
|
||||
logger.error(str(err))
|
||||
raise err
|
||||
except Exception as err:
|
||||
logger.error(f"Internal plugin error: {err}")
|
||||
raise err
|
||||
|
||||
dependencies = dedupe_packages(group_deps + env_deps + project_deps)
|
||||
|
||||
logger.info(f"Installing {len(dependencies)} dependencies from Poetry lock file")
|
||||
install_package(
|
||||
poetry,
|
||||
tox_env,
|
||||
dependencies,
|
||||
tox_env.options.parallel_install_threads,
|
||||
)
|
||||
88
tox_poetry_installer/installer.py
Normal file
88
tox_poetry_installer/installer.py
Normal file
@ -0,0 +1,88 @@
|
||||
"""Funcationality for performing virtualenv installation"""
|
||||
# Silence this one globally to support the internal function imports for the proxied poetry module.
|
||||
# See the docstring in 'tox_poetry_installer._poetry' for more context.
|
||||
# pylint: disable=import-outside-toplevel
|
||||
import concurrent.futures
|
||||
import contextlib
|
||||
import typing
|
||||
from datetime import datetime
|
||||
from typing import Collection
|
||||
from typing import Set
|
||||
|
||||
from tox.tox_env.api import ToxEnv as ToxVirtualEnv
|
||||
|
||||
from tox_poetry_installer import logger
|
||||
from tox_poetry_installer import utilities
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from tox_poetry_installer import _poetry
|
||||
|
||||
|
||||
def install(
|
||||
poetry: "_poetry.Poetry",
|
||||
venv: ToxVirtualEnv,
|
||||
packages: Collection["_poetry.PoetryPackage"],
|
||||
parallels: int = 0,
|
||||
):
|
||||
"""Install a bunch of packages to a virtualenv
|
||||
|
||||
:param poetry: Poetry object the packages were sourced from
|
||||
:param venv: Tox virtual environment to install the packages to
|
||||
:param packages: List of packages to install to the virtual environment
|
||||
:param parallels: Number of parallel processes to use for installing dependency packages, or
|
||||
``None`` to disable parallelization.
|
||||
"""
|
||||
from tox_poetry_installer import _poetry
|
||||
|
||||
logger.info(f"Installing {len(packages)} packages to environment at {venv.env_dir}")
|
||||
|
||||
install_executor = _poetry.Executor(
|
||||
env=utilities.convert_virtualenv(venv),
|
||||
io=_poetry.NullIO(),
|
||||
pool=poetry.pool,
|
||||
config=_poetry.Config(),
|
||||
)
|
||||
|
||||
installed: Set[_poetry.PoetryPackage] = set()
|
||||
|
||||
def logged_install(dependency: _poetry.PoetryPackage) -> None:
|
||||
start = datetime.now()
|
||||
logger.debug(f"Installing {dependency}")
|
||||
install_executor.execute([_poetry.Install(package=dependency)])
|
||||
end = datetime.now()
|
||||
logger.debug(f"Finished installing {dependency} in {end - start}")
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _optional_parallelize():
|
||||
"""A bit of cheat, really
|
||||
|
||||
A context manager that exposes a common interface for the caller that optionally
|
||||
enables/disables the usage of the parallel thread pooler depending on the value of
|
||||
the ``parallels`` parameter.
|
||||
"""
|
||||
if parallels > 0:
|
||||
with concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=parallels
|
||||
) as executor:
|
||||
yield executor.submit
|
||||
else:
|
||||
yield lambda func, arg: func(arg)
|
||||
|
||||
with _optional_parallelize() as executor:
|
||||
futures = []
|
||||
for dependency in packages:
|
||||
if dependency not in installed:
|
||||
installed.add(dependency)
|
||||
logger.debug(f"Queuing {dependency}")
|
||||
future = executor(logged_install, dependency)
|
||||
if future is not None:
|
||||
futures.append(future)
|
||||
else:
|
||||
logger.debug(f"Skipping {dependency}, already installed")
|
||||
logger.debug("Waiting for installs to finish...")
|
||||
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
# Don't actually care about the return value, just waiting on the
|
||||
# future to ensure any exceptions that were raised in the called
|
||||
# function are propagated.
|
||||
future.result()
|
||||
@ -4,7 +4,6 @@ Calling ``tox.reporter.something()`` and having to format a string with the pref
|
||||
gets really old fast, but more importantly it also makes the flow of the main code
|
||||
more difficult to follow because of the added complexity.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from tox_poetry_installer import constants
|
||||
|
||||
@ -1,38 +1,32 @@
|
||||
"""Helper functions for the :func:`tox_on_install` hook"""
|
||||
|
||||
"""Helper utility functions, usually bridging Tox and Poetry functionality"""
|
||||
# Silence this one globally to support the internal function imports for the proxied poetry module.
|
||||
# See the docstring in 'tox_poetry_installer._poetry' for more context.
|
||||
# pylint: disable=import-outside-toplevel
|
||||
import collections
|
||||
import concurrent.futures
|
||||
import contextlib
|
||||
import datetime
|
||||
import pathlib
|
||||
from typing import Collection
|
||||
import typing
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Sequence
|
||||
from typing import Set
|
||||
|
||||
import cleo.io.null_io
|
||||
import packaging.utils
|
||||
import poetry.config.config
|
||||
import poetry.core.packages.dependency
|
||||
import poetry.core.packages.package
|
||||
import poetry.factory
|
||||
import poetry.installation.executor
|
||||
import poetry.installation.operations.install
|
||||
import poetry.poetry
|
||||
import poetry.utils.env
|
||||
import tox.tox_env.api
|
||||
import tox.tox_env.package
|
||||
from poetry.core.packages.dependency import Dependency as PoetryDependency
|
||||
from poetry.core.packages.package import Package as PoetryPackage
|
||||
from tox.tox_env.api import ToxEnv as ToxVirtualEnv
|
||||
from tox.tox_env.package import PackageToxEnv
|
||||
|
||||
from tox_poetry_installer import constants
|
||||
from tox_poetry_installer import exceptions
|
||||
from tox_poetry_installer import logger
|
||||
|
||||
|
||||
PackageMap = Dict[str, List[poetry.core.packages.package.Package]]
|
||||
if typing.TYPE_CHECKING:
|
||||
from tox_poetry_installer import _poetry
|
||||
|
||||
|
||||
def check_preconditions(venv: tox.tox_env.api.ToxEnv) -> poetry.poetry.Poetry:
|
||||
PackageMap = Dict[str, List[PoetryPackage]]
|
||||
|
||||
|
||||
def check_preconditions(venv: ToxVirtualEnv) -> "_poetry.Poetry":
|
||||
"""Check that the local project environment meets expectations"""
|
||||
|
||||
# Skip running the plugin for the provisioning environment. The provisioned environment,
|
||||
@ -40,28 +34,61 @@ def check_preconditions(venv: tox.tox_env.api.ToxEnv) -> poetry.poetry.Poetry:
|
||||
# handled by Tox and is out of scope for this plugin. Since one of the ways to install this
|
||||
# plugin in the first place is via the Tox provisioning environment, it quickly becomes a
|
||||
# chicken-and-egg problem.
|
||||
if isinstance(venv, tox.tox_env.package.PackageToxEnv):
|
||||
if isinstance(venv, PackageToxEnv):
|
||||
raise exceptions.SkipEnvironment(f"Skipping Tox provisioning env '{venv.name}'")
|
||||
|
||||
if venv.options.require_poetry:
|
||||
logger.warning(
|
||||
"DEPRECATION: The '--require-poetry' runtime option is deprecated and will be "
|
||||
"removed in version 1.0.0. Please update test environments that require Poetry to "
|
||||
"set the 'require_poetry = true' option in tox.ini"
|
||||
)
|
||||
|
||||
from tox_poetry_installer import _poetry
|
||||
|
||||
try:
|
||||
return poetry.factory.Factory().create_poetry(venv.core["tox_root"])
|
||||
return _poetry.Factory().create_poetry(venv.core["tox_root"])
|
||||
# Support running the plugin when the current tox project does not use Poetry for its
|
||||
# environment/dependency management.
|
||||
#
|
||||
# ``RuntimeError`` is dangerous to blindly catch because it can be (and in Poetry's case,
|
||||
# is) raised in many different places for different purposes.
|
||||
except RuntimeError as err:
|
||||
except RuntimeError:
|
||||
raise exceptions.SkipEnvironment(
|
||||
f"Skipping installation of locked dependencies due to a Poetry error: {err}"
|
||||
"Project does not use Poetry for env management, skipping installation of locked dependencies"
|
||||
) from None
|
||||
|
||||
|
||||
def convert_virtualenv(venv: ToxVirtualEnv) -> "_poetry.VirtualEnv":
|
||||
"""Convert a Tox venv to a Poetry venv
|
||||
|
||||
:param venv: Tox ``VirtualEnv`` object representing a tox virtual environment
|
||||
:returns: Poetry ``VirtualEnv`` object representing a poetry virtual environment
|
||||
"""
|
||||
from tox_poetry_installer import _poetry
|
||||
|
||||
return _poetry.VirtualEnv(path=Path(venv.env_dir))
|
||||
|
||||
|
||||
def build_package_map(poetry: "_poetry.Poetry") -> PackageMap:
|
||||
"""Build the mapping of package names to objects
|
||||
|
||||
:param poetry: Populated poetry object to load locked packages from
|
||||
:returns: Mapping of package names to Poetry package objects
|
||||
"""
|
||||
packages = collections.defaultdict(list)
|
||||
for package in poetry.locker.locked_repository().packages:
|
||||
packages[package.name].append(package)
|
||||
|
||||
return packages
|
||||
|
||||
|
||||
def identify_transients(
|
||||
dep_name: str,
|
||||
packages: PackageMap,
|
||||
venv: poetry.utils.env.VirtualEnv,
|
||||
venv: "_poetry.VirtualEnv",
|
||||
allow_missing: Sequence[str] = (),
|
||||
) -> List[poetry.core.packages.package.Package]:
|
||||
) -> List[PoetryPackage]:
|
||||
"""Using a pool of packages, identify all transient dependencies of a given package name
|
||||
|
||||
:param dep_name: Either the Poetry dependency or the dependency's bare package name to recursively
|
||||
@ -78,12 +105,10 @@ def identify_transients(
|
||||
"""
|
||||
searched: Set[str] = set()
|
||||
|
||||
def _transients(
|
||||
transient: poetry.core.packages.dependency.Dependency,
|
||||
) -> List[poetry.core.packages.package.Package]:
|
||||
def _transients(transient: PoetryDependency) -> List[PoetryPackage]:
|
||||
searched.add(transient.name)
|
||||
|
||||
results: List[poetry.core.packages.package.Package] = []
|
||||
results: List[PoetryPackage] = []
|
||||
for option in packages[transient.name]:
|
||||
if venv.is_valid_for_marker(option.to_dependency().marker):
|
||||
for requirement in option.requires:
|
||||
@ -114,6 +139,13 @@ def identify_transients(
|
||||
except KeyError as err:
|
||||
missing = err.args[0]
|
||||
|
||||
if missing in constants.UNSAFE_PACKAGES:
|
||||
logger.warning(
|
||||
f"Installing package '{missing}' using Poetry is not supported and will be skipped"
|
||||
)
|
||||
logger.debug(f"Skipping {missing}: designated unsafe by Poetry")
|
||||
return []
|
||||
|
||||
if missing in allow_missing:
|
||||
logger.debug(f"Skipping {missing}: package is allowed to be unlocked")
|
||||
return []
|
||||
@ -132,43 +164,43 @@ def identify_transients(
|
||||
|
||||
def find_project_deps(
|
||||
packages: PackageMap,
|
||||
venv: poetry.utils.env.VirtualEnv,
|
||||
project: poetry.poetry.Poetry,
|
||||
venv: "_poetry.VirtualEnv",
|
||||
poetry: "_poetry.Poetry",
|
||||
extras: Sequence[str] = (),
|
||||
) -> List[poetry.core.packages.package.Package]:
|
||||
) -> List[PoetryPackage]:
|
||||
"""Find the root project dependencies
|
||||
|
||||
Recursively identify the dependencies of the root project package
|
||||
|
||||
:param packages: Mapping of all locked package names to their corresponding package object
|
||||
:param venv: Poetry virtual environment to use for package compatibility checks
|
||||
:param project: Poetry object for the current project
|
||||
:param poetry: Poetry object for the current project
|
||||
:param extras: Sequence of extra names to include the dependencies of
|
||||
"""
|
||||
|
||||
if any(dep.name in constants.UNSAFE_PACKAGES for dep in poetry.package.requires):
|
||||
raise exceptions.RequiresUnsafeDepError(
|
||||
f"Project package requires one or more unsafe dependencies ({', '.join(constants.UNSAFE_PACKAGES)}) which cannot be installed with Poetry"
|
||||
)
|
||||
|
||||
required_dep_names = [
|
||||
item.name for item in project.package.requires if not item.is_optional()
|
||||
item.name for item in poetry.package.requires if not item.is_optional()
|
||||
]
|
||||
|
||||
extra_dep_names: List[str] = []
|
||||
for extra in extras:
|
||||
logger.info(f"Processing project extra '{extra}'")
|
||||
try:
|
||||
extra_dep_names += [
|
||||
item.name
|
||||
for item in project.package.extras[
|
||||
packaging.utils.NormalizedName(extra)
|
||||
]
|
||||
]
|
||||
extra_dep_names += [item.name for item in poetry.package.extras[extra]]
|
||||
except KeyError:
|
||||
raise exceptions.ExtraNotFoundError(
|
||||
f"Environment specifies project extra '{extra}' which was not found in the lockfile"
|
||||
) from None
|
||||
|
||||
dependencies: List[poetry.core.packages.package.Package] = []
|
||||
dependencies: List[PoetryPackage] = []
|
||||
for dep_name in required_dep_names + extra_dep_names:
|
||||
dependencies += identify_transients(
|
||||
dep_name.lower(), packages, venv, allow_missing=[project.package.name]
|
||||
dep_name.lower(), packages, venv, allow_missing=[poetry.package.name]
|
||||
)
|
||||
|
||||
return dedupe_packages(dependencies)
|
||||
@ -176,24 +208,24 @@ def find_project_deps(
|
||||
|
||||
def find_additional_deps(
|
||||
packages: PackageMap,
|
||||
venv: poetry.utils.env.VirtualEnv,
|
||||
project: poetry.poetry.Poetry,
|
||||
venv: "_poetry.VirtualEnv",
|
||||
poetry: "_poetry.Poetry",
|
||||
dep_names: Sequence[str],
|
||||
) -> List[poetry.core.packages.package.Package]:
|
||||
) -> List[PoetryPackage]:
|
||||
"""Find additional dependencies
|
||||
|
||||
Recursively identify the dependencies of an arbitrary list of package names
|
||||
|
||||
:param packages: Mapping of all locked package names to their corresponding package object
|
||||
:param venv: Poetry virtual environment to use for package compatibility checks
|
||||
:param project: Poetry object for the current project
|
||||
:param poetry: Poetry object for the current project
|
||||
:param dep_names: Sequence of additional dependency names to recursively find the transient
|
||||
dependencies for
|
||||
"""
|
||||
dependencies: List[poetry.core.packages.package.Package] = []
|
||||
dependencies: List[PoetryPackage] = []
|
||||
for dep_name in dep_names:
|
||||
dependencies += identify_transients(
|
||||
dep_name.lower(), packages, venv, allow_missing=[project.package.name]
|
||||
dep_name.lower(), packages, venv, allow_missing=[poetry.package.name]
|
||||
)
|
||||
|
||||
return dedupe_packages(dependencies)
|
||||
@ -202,9 +234,9 @@ def find_additional_deps(
|
||||
def find_group_deps(
|
||||
group: str,
|
||||
packages: PackageMap,
|
||||
venv: poetry.utils.env.VirtualEnv,
|
||||
project: poetry.poetry.Poetry,
|
||||
) -> List[poetry.core.packages.package.Package]:
|
||||
venv: "_poetry.VirtualEnv",
|
||||
poetry: "_poetry.Poetry",
|
||||
) -> List[PoetryPackage]:
|
||||
"""Find the dependencies belonging to a dependency group
|
||||
|
||||
Recursively identify the Poetry dev dependencies
|
||||
@ -212,17 +244,13 @@ def find_group_deps(
|
||||
:param group: Name of the dependency group from the project's ``pyproject.toml``
|
||||
:param packages: Mapping of all locked package names to their corresponding package object
|
||||
:param venv: Poetry virtual environment to use for package compatibility checks
|
||||
:param project: Poetry object for the current project
|
||||
:param poetry: Poetry object for the current project
|
||||
"""
|
||||
return find_additional_deps(
|
||||
packages,
|
||||
venv,
|
||||
project,
|
||||
# the type ignore here is due to the difficulties around getting nested data
|
||||
# from the inherrently unstructured toml structure (which necessarily is flexibly
|
||||
# typed) but in a situation where there is a meta-structure applied to it (i.e. a
|
||||
# pyproject.toml structure).
|
||||
project.pyproject.data["tool"]["poetry"] # type: ignore
|
||||
poetry,
|
||||
poetry.pyproject.data["tool"]["poetry"]
|
||||
.get("group", {})
|
||||
.get(group, {})
|
||||
.get("dependencies", {})
|
||||
@ -231,136 +259,36 @@ def find_group_deps(
|
||||
|
||||
|
||||
def find_dev_deps(
|
||||
packages: PackageMap,
|
||||
venv: poetry.utils.env.VirtualEnv,
|
||||
project: poetry.poetry.Poetry,
|
||||
) -> List[poetry.core.packages.package.Package]:
|
||||
packages: PackageMap, venv: "_poetry.VirtualEnv", poetry: "_poetry.Poetry"
|
||||
) -> List[PoetryPackage]:
|
||||
"""Find the dev dependencies
|
||||
|
||||
Recursively identify the Poetry dev dependencies
|
||||
|
||||
:param packages: Mapping of all locked package names to their corresponding package object
|
||||
:param venv: Poetry virtual environment to use for package compatibility checks
|
||||
:param project: Poetry object for the current project
|
||||
:param poetry: Poetry object for the current project
|
||||
"""
|
||||
dev_group_deps = find_group_deps("dev", packages, venv, project)
|
||||
dev_group_deps = find_group_deps("dev", packages, venv, poetry)
|
||||
|
||||
# Legacy pyproject.toml poetry format:
|
||||
legacy_dev_group_deps = find_additional_deps(
|
||||
packages,
|
||||
venv,
|
||||
project,
|
||||
# the type ignore here is due to the difficulties around getting nested data
|
||||
# from the inherrently unstructured toml structure (which necessarily is flexibly
|
||||
# typed) but in a situation where there is a meta-structure applied to it (i.e. a
|
||||
# pyproject.toml structure).
|
||||
project.pyproject.data["tool"]["poetry"].get("dev-dependencies", {}).keys(), # type: ignore
|
||||
poetry,
|
||||
poetry.pyproject.data["tool"]["poetry"].get("dev-dependencies", {}).keys(),
|
||||
)
|
||||
|
||||
# Poetry 1.2 unions these two toml sections.
|
||||
return dedupe_packages(dev_group_deps + legacy_dev_group_deps)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _optional_parallelize(parallels: int):
|
||||
"""A bit of cheat, really
|
||||
|
||||
A context manager that exposes a common interface for the caller that optionally
|
||||
enables/disables the usage of the parallel thread pooler depending on the value of
|
||||
the ``parallels`` parameter.
|
||||
"""
|
||||
if parallels > 0:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=parallels) as executor:
|
||||
yield executor.submit
|
||||
else:
|
||||
yield lambda func, arg: func(arg)
|
||||
|
||||
|
||||
def install_package(
|
||||
project: poetry.poetry.Poetry,
|
||||
venv: tox.tox_env.api.ToxEnv,
|
||||
packages: Collection[poetry.core.packages.package.Package],
|
||||
parallels: int = 0,
|
||||
):
|
||||
"""Install a bunch of packages to a virtualenv
|
||||
|
||||
:param project: Poetry object the packages were sourced from
|
||||
:param venv: Tox virtual environment to install the packages to
|
||||
:param packages: List of packages to install to the virtual environment
|
||||
:param parallels: Number of parallel processes to use for installing dependency packages, or
|
||||
``None`` to disable parallelization.
|
||||
"""
|
||||
|
||||
logger.info(f"Installing {len(packages)} packages to environment at {venv.env_dir}")
|
||||
|
||||
install_executor = poetry.installation.executor.Executor(
|
||||
env=convert_virtualenv(venv),
|
||||
io=cleo.io.null_io.NullIO(),
|
||||
pool=project.pool,
|
||||
config=poetry.config.config.Config(),
|
||||
)
|
||||
|
||||
installed: Set[poetry.core.packages.package.Package] = set()
|
||||
|
||||
def logged_install(dependency: poetry.core.packages.package.Package) -> None:
|
||||
start = datetime.datetime.now()
|
||||
logger.debug(f"Installing {dependency}")
|
||||
install_executor.execute(
|
||||
[poetry.installation.operations.install.Install(package=dependency)]
|
||||
)
|
||||
end = datetime.datetime.now()
|
||||
logger.debug(f"Finished installing {dependency} in {end - start}")
|
||||
|
||||
with _optional_parallelize(parallels) as executor:
|
||||
futures = []
|
||||
for dependency in packages:
|
||||
if dependency not in installed:
|
||||
installed.add(dependency)
|
||||
logger.debug(f"Queuing {dependency}")
|
||||
future = executor(logged_install, dependency)
|
||||
if future is not None:
|
||||
futures.append(future)
|
||||
else:
|
||||
logger.debug(f"Skipping {dependency}, already installed")
|
||||
logger.debug("Waiting for installs to finish...")
|
||||
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
# Don't actually care about the return value, just waiting on the
|
||||
# future to ensure any exceptions that were raised in the called
|
||||
# function are propagated.
|
||||
future.result()
|
||||
|
||||
|
||||
def dedupe_packages(
|
||||
packages: Sequence[poetry.core.packages.package.Package],
|
||||
) -> List[poetry.core.packages.package.Package]:
|
||||
"""Deduplicates a sequence of Packages while preserving ordering
|
||||
def dedupe_packages(packages: Sequence[PoetryPackage]) -> List[PoetryPackage]:
|
||||
"""Deduplicates a sequence of PoetryPackages while preserving ordering
|
||||
|
||||
Adapted from StackOverflow: https://stackoverflow.com/a/480227
|
||||
"""
|
||||
seen: Set[poetry.core.packages.package.Package] = set()
|
||||
seen: Set[PoetryPackage] = set()
|
||||
# Make this faster, avoid method lookup below
|
||||
seen_add = seen.add
|
||||
return [item for item in packages if not (item in seen or seen_add(item))]
|
||||
|
||||
|
||||
def convert_virtualenv(venv: tox.tox_env.api.ToxEnv) -> poetry.utils.env.VirtualEnv:
|
||||
"""Convert a Tox venv to a Poetry venv
|
||||
|
||||
:param venv: Tox ``VirtualEnv`` object representing a tox virtual environment
|
||||
:returns: Poetry ``VirtualEnv`` object representing a poetry virtual environment
|
||||
"""
|
||||
return poetry.utils.env.VirtualEnv(path=pathlib.Path(venv.env_dir))
|
||||
|
||||
|
||||
def build_package_map(project: poetry.poetry.Poetry) -> PackageMap:
|
||||
"""Build the mapping of package names to objects
|
||||
|
||||
:param project: Populated poetry object to load locked packages from
|
||||
:returns: Mapping of package names to Poetry package objects
|
||||
"""
|
||||
packages = collections.defaultdict(list)
|
||||
for package in project.locker.locked_repository().packages:
|
||||
packages[str(package.name)].append(package)
|
||||
|
||||
return packages
|
||||
return [p for p in packages if not (p in seen or seen_add(p))]
|
||||
Loading…
Reference in New Issue
Block a user