From 52aaeba93c4ed16dc06a649872db4a459862c1c7 Mon Sep 17 00:00:00 2001
From: Ethan Paul <24588726+enpaul@users.noreply.github.com>
Date: Sat, 5 Dec 2020 12:36:35 -0500
Subject: [PATCH] Overhaul CI to improve resilance and efficiency

Add caching for pip and poetry downloads to reduce runtime
Add pinned pip version
Add poetry installation of local project
Remove bare pip install for local project installation
---
 .github/scripts/setup-env.sh | 72 ++++++++++++++++++++++++++++++++++++
 .github/workflows/ci.yaml    | 51 +++++++++++++++++++------
 2 files changed, 111 insertions(+), 12 deletions(-)
 create mode 100755 .github/scripts/setup-env.sh

diff --git a/.github/scripts/setup-env.sh b/.github/scripts/setup-env.sh
new file mode 100755
index 0000000..1b43d94
--- /dev/null
+++ b/.github/scripts/setup-env.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+#
+# Environment setup script for the local project. Intended to be used with automation
+# to create a repeatable local environment for tests to be run in. The python env
+# this script creates can be accessed at the location defined by the CI_VENV variable
+# below.
+
+set -e;
+
+# ##### Prereqs #####
+#
+# Set global vars for usage in the script, create the cache directory so we can rely
+# on that existing, then dump some diagnostic info for later reference.
+#
+CI_VENV=$HOME/ci;
+CI_CACHE=$HOME/.cache;
+CI_CACHE_GET_POETRY="$CI_CACHE/get-poetry.py";
+CI_POETRY=$HOME/.poetry/bin/poetry;
+CI_VENV_PIP="$CI_VENV/bin/pip";
+CI_VENV_PIP_VERSION=19.3.1;
+CI_VENV_TOX="$CI_VENV/bin/tox";
+
+mkdir --parents "$CI_CACHE";
+
+command -v python;
+python --version;
+
+# ##### Install Poetry #####
+#
+# Download the poetry install script to the cache directory and then install poetry.
+# After dump the poetry version for later reference.
+#
+curl https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py \
+  --output "$CI_CACHE_GET_POETRY" \
+  --silent \
+  --show-error \
+  --location;
+python "$CI_CACHE_GET_POETRY" --yes 1>/dev/null;
+
+python "$CI_POETRY" --version --no-ansi;
+
+# ##### Setup Runtime Venv #####
+#
+# Create a virtual environment for poetry to use, upgrade pip in that venv to a pinned
+# version, then install the current project to the venv.
+#
+# Note 1: Poetry, Tox, and this project plugin all use pip under the hood for package
+#         installation. This means that even though we are creating up to eight venvs
+#         during a given CI run they all share the same download cache.
+# Note 2: The "VIRTUAL_ENV=$CI_VENV" prefix on the poetry commands below sets the venv
+#         that poetry will use for operations. There is no CLI flag for poetry that
+#         directs it to use a given environment, but if it finds itself in an existing
+#         environment it will use it and skip environment creation.
+#
+python -m venv "$CI_VENV";
+
+$CI_VENV_PIP install "pip==$CI_VENV_PIP_VERSION" \
+  --upgrade \
+  --quiet;
+
+VIRTUAL_ENV=$CI_VENV "$CI_POETRY" install \
+  --extras poetry \
+  --quiet \
+  --no-ansi \
+  &>/dev/null;
+
+# ##### Print Debug Info #####
+#
+# Print the pip and tox versions (which will include registered plugins)
+#
+$CI_VENV_PIP --version;
+echo "tox $($CI_VENV_TOX --version)";
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 6fa8c21..30c31b8 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -20,24 +20,51 @@ jobs:
           - version: 3.9
             toxenv: py39
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python ${{ matrix.python.version }}
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup:python${{ matrix.python.version }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python.version }}
-      - name: Install project
-        run: pip install .
-      - name: Run tests via ${{ matrix.python.toxenv }}
-        run: tox -e ${{ matrix.python.toxenv }}
+      - name: Setup:cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.cache/pypoetry/cache
+            ~/.poetry
+          # Including the hashed poetry.lock in the cache slug ensures that the cache
+          # will be invalidated, and thus all packages will be redownloaded, if the
+          # lockfile is updated
+          key: ${{ runner.os }}-${{ matrix.python.toxenv }}-${{ hashFiles('**/poetry.lock') }}
+      - name: Setup:env
+        run: .github/scripts/setup-env.sh
+      - name: Run:${{ matrix.python.toxenv }}
+        run: $HOME/ci/bin/tox -e ${{ matrix.python.toxenv }}
   Check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.8
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup:python3.8
         uses: actions/setup-python@v1
         with:
           python-version: 3.8
-      - name: Install project
-        run: pip install .
-      - name: Run meta checks
-        run: tox -e static -e static-tests -e security
+      - name: Setup:cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.cache/pypoetry/cache
+            ~/.poetry
+          # Hardcoded 'py38' slug here lets this cache piggyback on the 'py38' cache
+          # that is generated for the tests above
+          key: ${{ runner.os }}-py38-${{ hashFiles('**/poetry.lock') }}
+      - name: Setup:env
+        run: .github/scripts/setup-env.sh
+      - name: Run:static
+        run: $HOME/ci/bin/tox -e static
+      - name: Run:static-tests
+        run: $HOME/ci/bin/tox -e static-tests
+      - name: Run:security
+        run: $HOME/ci/bin/tox -e security