Overhaul CI to improve resilance and efficiency

Add caching for pip and poetry downloads to reduce runtime
Add pinned pip version
Add poetry installation of local project
Remove bare pip install for local project installation

.github/scripts/setup-env.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+#
+# Environment setup script for the local project. Intended to be used with automation
+# to create a repeatable local environment for tests to be run in. The python env
+# this script creates can be accessed at the location defined by the CI_VENV variable
+# below.
+
+set -e;
+
+# ##### Prereqs #####
+#
+# Set global vars for usage in the script, create the cache directory so we can rely
+# on that existing, then dump some diagnostic info for later reference.
+#
+CI_VENV=$HOME/ci;
+CI_CACHE=$HOME/.cache;
+CI_CACHE_GET_POETRY="$CI_CACHE/get-poetry.py";
+CI_POETRY=$HOME/.poetry/bin/poetry;
+CI_VENV_PIP="$CI_VENV/bin/pip";
+CI_VENV_PIP_VERSION=19.3.1;
+CI_VENV_TOX="$CI_VENV/bin/tox";
+
+mkdir --parents "$CI_CACHE";
+
+command -v python;
+python --version;
+
+# ##### Install Poetry #####
+#
+# Download the poetry install script to the cache directory and then install poetry.
+# After dump the poetry version for later reference.
+#
+curl https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py \
+  --output "$CI_CACHE_GET_POETRY" \
+  --silent \
+  --show-error \
+  --location;
+python "$CI_CACHE_GET_POETRY" --yes 1>/dev/null;
+
+python "$CI_POETRY" --version --no-ansi;
+
+# ##### Setup Runtime Venv #####
+#
+# Create a virtual environment for poetry to use, upgrade pip in that venv to a pinned
+# version, then install the current project to the venv.
+#
+# Note 1: Poetry, Tox, and this project plugin all use pip under the hood for package
+#         installation. This means that even though we are creating up to eight venvs
+#         during a given CI run they all share the same download cache.
+# Note 2: The "VIRTUAL_ENV=$CI_VENV" prefix on the poetry commands below sets the venv
+#         that poetry will use for operations. There is no CLI flag for poetry that
+#         directs it to use a given environment, but if it finds itself in an existing
+#         environment it will use it and skip environment creation.
+#
+python -m venv "$CI_VENV";
+
+$CI_VENV_PIP install "pip==$CI_VENV_PIP_VERSION" \
+  --upgrade \
+  --quiet;
+
+VIRTUAL_ENV=$CI_VENV "$CI_POETRY" install \
+  --extras poetry \
+  --quiet \
+  --no-ansi \
+  &>/dev/null;
+
+# ##### Print Debug Info #####
+#
+# Print the pip and tox versions (which will include registered plugins)
+#
+$CI_VENV_PIP --version;
+echo "tox $($CI_VENV_TOX --version)";

.github/workflows/ci.yaml

@@ -20,24 +20,51 @@ jobs:
           - version: 3.9
             toxenv: py39
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
-      - name: Set up Python ${{ matrix.python.version }}
+        uses: actions/checkout@v2
+      - name: Setup:python${{ matrix.python.version }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python.version }}
-      - name: Install project
+      - name: Setup:cache
-        run: pip install .
+        uses: actions/cache@v2
-      - name: Run tests via ${{ matrix.python.toxenv }}
+        with:
-        run: tox -e ${{ matrix.python.toxenv }}
+          path: |
+            ~/.cache/pip
+            ~/.cache/pypoetry/cache
+            ~/.poetry
+          # Including the hashed poetry.lock in the cache slug ensures that the cache
+          # will be invalidated, and thus all packages will be redownloaded, if the
+          # lockfile is updated
+          key: ${{ runner.os }}-${{ matrix.python.toxenv }}-${{ hashFiles('**/poetry.lock') }}
+      - name: Setup:env
+        run: .github/scripts/setup-env.sh
+      - name: Run:${{ matrix.python.toxenv }}
+        run: $HOME/ci/bin/tox -e ${{ matrix.python.toxenv }}
   Check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
-      - name: Set up Python 3.8
+        uses: actions/checkout@v2
+      - name: Setup:python3.8
         uses: actions/setup-python@v1
         with:
           python-version: 3.8
-      - name: Install project
+      - name: Setup:cache
-        run: pip install .
+        uses: actions/cache@v2
-      - name: Run meta checks
+        with:
-        run: tox -e static -e static-tests -e security
+          path: |
+            ~/.cache/pip
+            ~/.cache/pypoetry/cache
+            ~/.poetry
+          # Hardcoded 'py38' slug here lets this cache piggyback on the 'py38' cache
+          # that is generated for the tests above
+          key: ${{ runner.os }}-py38-${{ hashFiles('**/poetry.lock') }}
+      - name: Setup:env
+        run: .github/scripts/setup-env.sh
+      - name: Run:static
+        run: $HOME/ci/bin/tox -e static
+      - name: Run:static-tests
+        run: $HOME/ci/bin/tox -e static-tests
+      - name: Run:security
+        run: $HOME/ci/bin/tox -e security