Merge pull request #47 from enpaul/enp/parallel

Add support for parallelizing installs

tox_poetry_installer/hooks.py

@@ -34,6 +34,14 @@ def tox_addoption(parser: ToxParser):
         help="Trigger a failure if Poetry is not available to Tox",
     )
 
+    parser.add_argument(
+        "--parallelize-locked-install",
+        type=int,
+        dest="parallelize_locked_install",
+        default=None,
+        help="Number of worker threads to use for installing dependencies from the Poetry lockfile in parallel",
+    )
+
     parser.add_testenv_attribute(
         name="install_dev_deps",
         type="bool",
@@ -143,10 +151,21 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
         raise err
 
     dependencies = dev_deps + env_deps + project_deps
+    log_parallel = (
+        f" (using {venv.envconfig.config.option.parallelize_locked_install} threads)"
+        if venv.envconfig.config.option.parallelize_locked_install
+        else ""
+    )
+
     action.setactivity(
         __about__.__title__,
-        f"Installing {len(dependencies)} dependencies from Poetry lock file",
+        f"Installing {len(dependencies)} dependencies from Poetry lock file{log_parallel}",
+    )
+    installer.install(
+        poetry,
+        venv,
+        dependencies,
+        venv.envconfig.config.option.parallelize_locked_install,
     )
-    installer.install(poetry, venv, dependencies)
 
     return venv.envconfig.require_locked_deps or None

tox_poetry_installer/installer.py

@@ -2,7 +2,10 @@
 # Silence this one globally to support the internal function imports for the proxied poetry module.
 # See the docstring in 'tox_poetry_installer._poetry' for more context.
 # pylint: disable=import-outside-toplevel
+import concurrent.futures
+import contextlib
 import typing
+from typing import Optional
 from typing import Sequence
 from typing import Set
 
@@ -18,13 +21,18 @@ if typing.TYPE_CHECKING:
 
 
 def install(
-    poetry: "_poetry.Poetry", venv: ToxVirtualEnv, packages: Sequence[PoetryPackage]
+    poetry: "_poetry.Poetry",
+    venv: ToxVirtualEnv,
+    packages: Sequence[PoetryPackage],
+    parallels: Optional[int] = None,
 ):
     """Install a bunch of packages to a virtualenv
 
     :param poetry: Poetry object the packages were sourced from
     :param venv: Tox virtual environment to install the packages to
     :param packages: List of packages to install to the virtual environment
+    :param parallels: Number of parallel processes to use for installing dependency packages, or
+                      ``None`` to disable parallelization.
     """
     from tox_poetry_installer import _poetry
 
@@ -40,14 +48,34 @@ def install(
 
     installed: Set[PoetryPackage] = set()
 
-    for dependency in packages:
-        if dependency not in installed:
-            tox.reporter.verbosity2(
-                f"{constants.REPORTER_PREFIX} Installing {dependency}"
-            )
-            pip.install(dependency)
-            installed.add(dependency)
+    @contextlib.contextmanager
+    def _optional_parallelize():
+        """A bit of cheat, really
+
+        A context manager that exposes a common interface for the caller that optionally
+        enables/disables the usage of the parallel thread pooler depending on the value of
+        the ``parallels`` parameter.
+        """
+        if parallels:
+            with concurrent.futures.ThreadPoolExecutor(
+                max_workers=parallels
+            ) as executor:
+                yield executor.submit
         else:
-            tox.reporter.verbosity2(
-                f"{constants.REPORTER_PREFIX} Skipping {dependency}, already installed"
-            )
+            yield lambda func, arg: func(arg)
+
+    with _optional_parallelize() as executor:
+        for dependency in packages:
+            if dependency not in installed:
+                installed.add(dependency)
+                tox.reporter.verbosity2(
+                    f"{constants.REPORTER_PREFIX} Installing {dependency}"
+                )
+                executor(pip.install, dependency)
+            else:
+                tox.reporter.verbosity2(
+                    f"{constants.REPORTER_PREFIX} Skipping {dependency}, already installed"
+                )
+        tox.reporter.verbosity2(
+            f"{constants.REPORTER_PREFIX} Waiting for installs to finish..."
+        )