Merge pull request #4 from enpaul/enp/ci

Fix CI
2024-11-24 11:06:53 +00:00 · 2023-03-29 20:17:40 -04:00 · 2023-03-29 20:17:40 -04:00 · 90e4a32753
commit 90e4a32753
parent a98dd16358 3dc062c849
10 changed files with 2158 additions and 1447 deletions
--- a/.github/scripts/setup-env.sh
+++ b/.github/scripts/setup-env.sh
@ -4,11 +4,13 @@
 # to create a repeatable local environment for tests to be run in. The python env
 # this script creates can be accessed at the location defined by the CI_VENV variable
 # below.
+#
+# POETRY_VERSION can be set to install a specific version of Poetry

 set -e;

 CI_CACHE=$HOME/.cache;
-POETRY_VERSION=1.1.12;
+INSTALL_POETRY_VERSION="${POETRY_VERSION:-1.3.2}";

 mkdir --parents "$CI_CACHE";

@ -20,15 +22,16 @@ curl --location https://install.python-poetry.org \
  --silent \
  --show-error;
 python "$CI_CACHE/install-poetry.py" \
-  --version "$POETRY_VERSION" \
+  --version "$INSTALL_POETRY_VERSION" \
  --yes;
 poetry --version --no-ansi;
 poetry run pip --version;

 poetry install \
-  --quiet \
-  --remove-untracked \
-  --no-ansi;
+  --sync \
+  --no-ansi \
+  --no-root \
+  --only ci;

 poetry env info;
 poetry run tox --version;
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@ -5,14 +5,16 @@ on:
    types: ["opened", "synchronize"]
  push:
    branches: ["devel"]
+env:
+  POETRY_VERSION: 1.4.1
 jobs:
  Test:
+    name: Python ${{ matrix.python.version }}
    runs-on: ubuntu-latest
    strategy:
+      fail-fast: true
      matrix:
        python:
-          - version: "3.6"
-            toxenv: py36
          - version: "3.7"
            toxenv: py37
          - version: "3.8"
@ -21,15 +23,19 @@ jobs:
            toxenv: py39
          - version: "3.10"
            toxenv: py310
+          - version: "3.11"
+            toxenv: py311
    steps:
      - name: Checkout
        uses: actions/checkout@v2
+
      - name: Install Python ${{ matrix.python.version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python.version }}
+
      - name: Configure Job Cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
        with:
          path: |
            ~/.cache/pip
@ -39,38 +45,49 @@ jobs:
          # will be invalidated, and thus all packages will be redownloaded, if the
          # lockfile is updated
          key: ${{ runner.os }}-${{ matrix.python.toxenv }}-${{ hashFiles('**/poetry.lock') }}
+
      - name: Configure Path
        run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+
      - name: Configure Environment
        run: .github/scripts/setup-env.sh
+
      - name: Run Toxenv ${{ matrix.python.toxenv }}
        run: poetry run tox -e ${{ matrix.python.toxenv }}
+
  Check:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v2
-      - name: Install Python 3.8
-        uses: actions/setup-python@v1
+
+      - name: Install Python 3.10
+        uses: actions/setup-python@v4
        with:
-          python-version: 3.8
+          python-version: "3.10"
+
      - name: Configure Job Cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
        with:
          path: |
            ~/.cache/pip
            ~/.cache/pypoetry/cache
            ~/.poetry
-          # Hardcoded 'py38' slug here lets this cache piggyback on the 'py38' cache
+          # Hardcoded 'py310' slug here lets this cache piggyback on the 'py310' cache
          # that is generated for the tests above
-          key: ${{ runner.os }}-py38-${{ hashFiles('**/poetry.lock') }}
+          key: ${{ runner.os }}-py310-${{ hashFiles('**/poetry.lock') }}
+
      - name: Configure Path
        run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+
      - name: Configure Environment
        run: .github/scripts/setup-env.sh
+
      - name: Run Static Analysis Checks
        run: poetry run tox -e static
+
      - name: Run Static Analysis Checks (Tests)
        run: poetry run tox -e static-tests
+
      - name: Run Security Checks
        run: poetry run tox -e security
--- a/.pylintrc
+++ b/.pylintrc
@ -11,7 +11,6 @@
 # --disable=W"
 disable=logging-fstring-interpolation
        ,logging-format-interpolation
-        ,bad-continuation
        ,line-too-long
        ,ungrouped-imports
        ,typecheck
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -18,8 +18,8 @@ View this release on: [Github](https://github.com/enpaul/vault2vault/releases/ta

 - Fix bug causing stack trace when the same vaulted block appears in a YAML file more than
  once
- Fix bug where the `--ignore-undecryptable` option was not respected for vaulted variables
-  in YAML files
+- Fix bug where the `--ignore-undecryptable` option was not respected for vaulted
+  variables in YAML files
 - Update logging messages and levels to improve verbose output

 ## Version 0.1.0
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@ -27,9 +27,10 @@ Examples of unacceptable behavior include:
 - The use of sexualized language or imagery, and sexual attention or advances of any kind
 - Trolling, insulting or derogatory comments, and personal or political attacks
 - Public or private harassment
- Publishing others' private information, such as a physical or email address, without their
-  explicit permission
- Other conduct which could reasonably be considered inappropriate in a professional setting
+- Publishing others' private information, such as a physical or email address, without
+  their explicit permission
+- Other conduct which could reasonably be considered inappropriate in a professional
+  setting

 ## Enforcement Responsibilities

@ -52,8 +53,8 @@ offline event.
 ## Enforcement

 Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the
-community leaders responsible for enforcement at \[INSERT CONTACT METHOD\]. All
-complaints will be reviewed and investigated promptly and fairly.
+community leaders responsible for enforcement at \[INSERT CONTACT METHOD\]. All complaints
+will be reviewed and investigated promptly and fairly.

 All community leaders are obligated to respect the privacy and security of the reporter of
 any incident.
@ -105,8 +106,8 @@ toward or disparagement of classes of individuals.
 This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0,
 available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.

-Community Impact Guidelines were inspired by [Mozilla's code of conduct
-enforcement ladder](https://github.com/mozilla/diversity).
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity).

 For answers to common questions about this code of conduct, see the FAQ at
 https://www.contributor-covenant.org/faq. Translations are available at
--- a/2
+++ b/2
@ -33,7 +33,7 @@ test: ## Run the project testsuite(s)
 	poetry run tox --recreate --parallel

 dev: ## Create the local dev environment
-	poetry install
+	poetry install --with dev --extras ansible --sync
 	poetry run pre-commit install

 publish: test wheel source ## Build and upload to pypi (requires $PYPI_API_KEY be set)
--- a/README.md
+++ b/README.md
@ -30,10 +30,9 @@ terminal. Whatever, these things happen.
 The built-in tool Ansible provides,
 [`ansible-vault rekey`](https://docs.ansible.com/ansible/latest/cli/ansible-vault.html#rekey),
 suffers from two main drawbacks: first, it only works on vault encrypted files and not on
-vault encrypted YAML data; and second, it only works on a single vault encrypted file at
-a time. To rekey everything in a large project you'd need to write a script that
-recursively goes through every file and rekeys every encrypted file and YAML variable all
-at once.
+vault encrypted YAML data; and second, it only works on a single vault encrypted file at a
+time. To rekey everything in a large project you'd need to write a script that recursively
+goes through every file and rekeys every encrypted file and YAML variable all at once.

 This is that script.

@ -58,8 +57,8 @@ install `vault2vault` using [PipX](https://pypa.github.io/pipx/) and the `ansibl
 pipx install vault2vault[ansible]
 ```

-> Note: vault2vault requires an Ansible installation to function. If you are installing to a
-> standalone virtual environment (like with PipX) then you must install it with the
+> Note: vault2vault requires an Ansible installation to function. If you are installing to
+> a standalone virtual environment (like with PipX) then you must install it with the
 > `ansible` extra to ensure a version of Ansible is available to the application.\*\*

 ## Usage
@ -117,8 +116,8 @@ data successfully rekeyed and the other half not.

 In the spirit of the
 [Unix philosophy](https://hackaday.com/2018/09/10/doing-one-thing-well-the-unix-philosophy/)
-this tool does not include any built-in way to recover from this state. However, it can
-be done very effectively using a version control tool.
+this tool does not include any built-in way to recover from this state. However, it can be
+done very effectively using a version control tool.

 If you are using Git to track your project files then you can use the command
 `git reset --hard` to restore all files to the state of the currently checked out commit.
@ -160,7 +159,8 @@ roadmap focuses on stability and user experience ahead of a 1.0 release.
 ## Developer Documentation

 All project contributors and participants are expected to adhere to the
-[Contributor Covenant Code of Conduct, v2](CODE_OF_CONDUCT.md) ([external link](https://www.contributor-covenant.org/version/2/0/code_of_conduct/)).
+[Contributor Covenant Code of Conduct, v2](CODE_OF_CONDUCT.md)
+([external link](https://www.contributor-covenant.org/version/2/0/code_of_conduct/)).

 The `devel` branch has the latest (and potentially unstable) changes. The stable releases
 are tracked on [Github](https://github.com/enpaul/vault2vault/releases),
--- a/poetry.lock
+++ b/poetry.lock
--- a/pyproject.toml
+++ b/pyproject.toml
@ -22,11 +22,11 @@ classifiers = [
  "Natural Language :: English",
  "Operating System :: OS Independent",
  "Programming Language :: Python :: 3",
-  "Programming Language :: Python :: 3.6",
  "Programming Language :: Python :: 3.7",
  "Programming Language :: Python :: 3.8",
  "Programming Language :: Python :: 3.9",
  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
  "Programming Language :: Python :: Implementation :: CPython"
 ]

@ -37,30 +37,43 @@ vault2vault = "vault2vault:main"
 ansible = ["ansible-core"]

 [tool.poetry.dependencies]
-python = "^3.6.1"
+python = "^3.7"
 "ruamel.yaml" = "^0.17.16"
 ansible-core = {version = "^2.11.5", optional = true}

-[tool.poetry.dev-dependencies]
-bandit = "^1.6.2"
-black = { version = "^21.9b0", allow-prereleases = true, python = "^3.7" }
-blacken-docs = "^1.8.0"
-ipython = { version = "^7.18.1", python = "^3.7" }
-mypy = "^0.800"
-pre-commit = "^2.7.1"
-pre-commit-hooks = "^3.3.0"
-pylint = "^2.4.4"
-pytest = "^6.0.2"
-pytest-cov = "^2.10.1"
-reorder-python-imports = "^2.3.5"
-safety = "^1.9.0"
-toml = "^0.10.1"
-tox = "^3.20.0"
-tox-poetry-installer = { version = "^0.8.1", extras = ["poetry"] }
-types-toml = "^0.10.4"
-mdformat = "^0.6.4"
-mdformat-gfm = "^0.2"
+[tool.poetry.group.dev.dependencies]
+black = {version = "^23.1.0", python = "^3.10"}
+blacken-docs = {version = "^1.13.0", python = "^3.10"}
+ipython = {version = "^8.10.1", python = "^3.10"}
+mdformat = {version = "^0.7.16", python = "^3.10"}
+mdformat-gfm = {version = "^0.3.5", python = "^3.10"}
+mypy = {version = "^1.1.1", python = "^3.10"}
+pre-commit = {version = "^2.7.1", python = "^3.10"}
+pre-commit-hooks = {version = "^3.3.0", python = "^3.10"}
+pylint = {version = "^2.4.4", python = "^3.10"}
+reorder-python-imports = {version = "^2.3.5", python = "^3.10"}
+types-toml = {version = "^0.10.4", python = "^3.10"}
+# Implicit python version check fails for this one
+packaging = {version = "^23.0", python = "^3.10"}
+
+[tool.poetry.group.security.dependencies]
+bandit = {version = "^1.6.2", python = "^3.10"}
+safety = {version = "^2.2.0", python = "^3.10"}
+poetry = {version = "^1.2.0", python = "^3.10"}
+
+[tool.poetry.group.test.dependencies]
+pytest = {version = "^6.0.2"}
+pytest-cov = {version = "^2.10.1"}
+toml = {version = "^0.10.1"}
+typing-extensions = {version = "^4.5.0", python = "^3.8"}
+
+[tool.poetry.group.ci.dependencies]
+tox = {version = "^3.20.0"}
+tox-poetry-installer = {version = "^0.10.1", extras = ["poetry"]}
+# This doesn't get installed under py3.7 for some reason, but it's
+# required for poetry. Will need to debug this more in the future
+backports-cached-property = "^1.0.2"

 [build-system]
-requires = ["poetry-core>=1.0.0"]
+requires = ["poetry-core>=1.1.0"]
 build-backend = "poetry.core.masonry.api"
--- a/tox.ini
+++ b/tox.ini
@ -1,5 +1,5 @@
 [tox]
-envlist = py36, py37, py38, py39, py310, static, static-tests, security
+envlist = py3{7-11}, static, static-tests, security
 isolated_build = true
 skip_missing_interpreters = true

@ -9,10 +9,8 @@ require_locked_deps = true
 require_poetry = true
 extras =
    ansible
-locked_deps =
-    pytest
-    pytest-cov
-    toml
+poetry_dep_groups =
+    test
 commands =
    pytest {toxinidir}/tests/ \
      --cov vault2vault \
@ -21,20 +19,11 @@ commands =

 [testenv:static]
 description = Static formatting and quality enforcement
-basepython = python3.8
+basepython = python3.10
 platform = linux
 ignore_errors = true
-locked_deps =
-    black
-    blacken-docs
-    mdformat
-    mdformat-gfm
-    mypy
-    reorder-python-imports
-    pre-commit
-    pre-commit-hooks
-    pylint
-    types-toml
+poetry_dep_groups =
+    dev
 commands =
    pre-commit run \
      --all-files
@ -46,7 +35,7 @@ commands =

 [testenv:static-tests]
 description = Static formatting and quality enforcement for the tests
-basepython = python3.8
+basepython = python3.10
 platform = linux
 ignore_errors = true
 locked_deps =
@ -63,14 +52,12 @@ commands =

 [testenv:security]
 description = Security checks
-basepython = python3.8
+basepython = python3.10
 platform = linux
 ignore_errors = true
 skip_install = true
-locked_deps =
-    bandit
-    safety
-    poetry
+poetry_dep_groups =
+    security
 commands =
    bandit {toxinidir}/vault2vault.py \
      --recursive \
@ -82,8 +69,14 @@ commands =
    poetry export \
      --format requirements.txt \
      --output {envtmpdir}/requirements.txt \
-      --without-hashes \
-      --dev
+      --without-hashes
+# For now these groups are disabled until this bug is resolved
+# in poetry-plugin-export:
+# https://github.com/python-poetry/poetry-plugin-export/issues/176
+#      --with dev \
+#      --with ci \
+#      --with security \
+#      --with test
    safety check \
      --file {envtmpdir}/requirements.txt \
      --json