Update default setup to use parallelized dep installation

Move deprecation warnings to the precondition function
Deprecate --parallelize-locked-install option
Add --parallel-install-threads option
This commit is contained in:
Ethan Paul 2021-05-05 16:18:41 -04:00
parent 915233c529
commit dbbbf8186f
No known key found for this signature in database
GPG Key ID: D0E2CBF1245E92BF
4 changed files with 42 additions and 18 deletions

View File

@ -21,3 +21,6 @@ REPORTER_PREFIX: str = f"{__about__.__title__}:"
# Internal list of packages that poetry has deemed unsafe and are excluded from the lockfile # Internal list of packages that poetry has deemed unsafe and are excluded from the lockfile
UNSAFE_PACKAGES: Set[str] = {"distribute", "pip", "setuptools", "wheel"} UNSAFE_PACKAGES: Set[str] = {"distribute", "pip", "setuptools", "wheel"}
# Number of threads to use for installing dependencies by default
DEFAULT_INSTALL_THREADS: int = 10

View File

@ -12,6 +12,7 @@ from tox.config import Parser as ToxParser
from tox.venv import VirtualEnv as ToxVirtualEnv from tox.venv import VirtualEnv as ToxVirtualEnv
from tox_poetry_installer import __about__ from tox_poetry_installer import __about__
from tox_poetry_installer import constants
from tox_poetry_installer import exceptions from tox_poetry_installer import exceptions
from tox_poetry_installer import installer from tox_poetry_installer import installer
from tox_poetry_installer import logger from tox_poetry_installer import logger
@ -31,7 +32,7 @@ def tox_addoption(parser: ToxParser):
"--require-poetry", "--require-poetry",
action="store_true", action="store_true",
dest="require_poetry", dest="require_poetry",
help="Trigger a failure if Poetry is not available to Tox", help="(deprecated) Trigger a failure if Poetry is not available to Tox",
) )
parser.add_argument( parser.add_argument(
@ -39,7 +40,15 @@ def tox_addoption(parser: ToxParser):
type=int, type=int,
dest="parallelize_locked_install", dest="parallelize_locked_install",
default=None, default=None,
help="Number of worker threads to use for installing dependencies from the Poetry lockfile in parallel", help="(deprecated) Number of worker threads to use for installing dependencies from the Poetry lockfile in parallel",
)
parser.add_argument(
"--parallel-install-threads",
type=int,
dest="parallel_install_threads",
default=constants.DEFAULT_INSTALL_THREADS,
help="Number of locked dependencies to install simultaneously; set to 0 to disable parallel installation",
) )
parser.add_testenv_attribute( parser.add_testenv_attribute(
@ -82,13 +91,6 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
:param action: Tox action object :param action: Tox action object
""" """
if venv.envconfig.config.option.require_poetry:
logger.warning(
"DEPRECATION WARNING: The '--require-poetry' runtime option is deprecated and will be "
"removed in version 1.0.0. Please update test environments that require Poetry to "
"set the 'require_poetry = true' option in tox.ini"
)
try: try:
poetry = utilities.check_preconditions(venv, action) poetry = utilities.check_preconditions(venv, action)
except exceptions.SkipEnvironment as err: except exceptions.SkipEnvironment as err:
@ -158,11 +160,18 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
raise err raise err
dependencies = dev_deps + env_deps + project_deps dependencies = dev_deps + env_deps + project_deps
log_parallel = ( if (
f" (using {venv.envconfig.config.option.parallelize_locked_install} threads)" venv.envconfig.config.option.parallel_install_threads
if venv.envconfig.config.option.parallelize_locked_install != constants.DEFAULT_INSTALL_THREADS
else "" ):
) parallel_threads = venv.envconfig.config.option.parallel_install_threads
else:
parallel_threads = (
venv.envconfig.config.option.parallelize_locked_install
if venv.envconfig.config.option.parallelize_locked_install is not None
else constants.DEFAULT_INSTALL_THREADS
)
log_parallel = f" (using {parallel_threads} threads)" if parallel_threads else ""
action.setactivity( action.setactivity(
__about__.__title__, __about__.__title__,
@ -172,7 +181,7 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
poetry, poetry,
venv, venv,
dependencies, dependencies,
venv.envconfig.config.option.parallelize_locked_install, parallel_threads,
) )
return venv.envconfig.require_locked_deps or None return venv.envconfig.require_locked_deps or None

View File

@ -5,7 +5,6 @@
import concurrent.futures import concurrent.futures
import contextlib import contextlib
import typing import typing
from typing import Optional
from typing import Sequence from typing import Sequence
from typing import Set from typing import Set
@ -23,7 +22,7 @@ def install(
poetry: "_poetry.Poetry", poetry: "_poetry.Poetry",
venv: ToxVirtualEnv, venv: ToxVirtualEnv,
packages: Sequence[PoetryPackage], packages: Sequence[PoetryPackage],
parallels: Optional[int] = None, parallels: int = 0,
): ):
"""Install a bunch of packages to a virtualenv """Install a bunch of packages to a virtualenv
@ -55,7 +54,7 @@ def install(
enables/disables the usage of the parallel thread pooler depending on the value of enables/disables the usage of the parallel thread pooler depending on the value of
the ``parallels`` parameter. the ``parallels`` parameter.
""" """
if parallels: if parallels > 0:
with concurrent.futures.ThreadPoolExecutor( with concurrent.futures.ThreadPoolExecutor(
max_workers=parallels max_workers=parallels
) as executor: ) as executor:

View File

@ -43,6 +43,19 @@ def check_preconditions(venv: ToxVirtualEnv, action: ToxAction) -> "_poetry.Poet
f"Skipping isolated packaging build env '{action.name}'" f"Skipping isolated packaging build env '{action.name}'"
) )
if venv.envconfig.config.option.require_poetry:
logger.warning(
"DEPRECATION: The '--require-poetry' runtime option is deprecated and will be "
"removed in version 1.0.0. Please update test environments that require Poetry to "
"set the 'require_poetry = true' option in tox.ini"
)
if venv.envconfig.config.option.parallelize_locked_install is not None:
logger.warning(
"DEPRECATION: The '--parallelize-locked-install' option is deprecated and will "
"be removed in version 1.0.0. Please use the '--parallel-install-threads' option."
)
from tox_poetry_installer import _poetry from tox_poetry_installer import _poetry
try: try: