Add role for adding docker nodes to swarm

inventory.yaml

@@ -32,8 +32,8 @@ en1:
         skylab_targets: [cluster, datastore]
         skylab_roles:
           - server
-          - docker-swarm
           - datastore
+          - swarm
         skylab_direct_peers:
           - hostname: pegasus.local  jupiter.svr.local
             address: 192.168.42.10
@@ -49,7 +49,9 @@ en1:
           skylab_description: Arbiter Node
           skylab_cluster:
             address: 10.42.101.10/24
-            interface: bond0
+            interface:
+              access: bond0
+              internal: bond0.99
           skylab_networking:
             eno1:
               bond: bond0
@@ -78,7 +80,9 @@ en1:
           skylab_description: Operational Node
           skylab_cluster:
             address: 10.42.101.11/24
-            interface: bond0
+            interface:
+              access: bond0
+              internal: bond0.99
           skylab_networking:
             eno1:
               bond: bond0
@@ -107,7 +111,9 @@ en1:
           skylab_description: Operational Node
           skylab_cluster:
             address: 10.42.101.12/24
-            interface: bond0
+            interface:
+              access: bond0
+              internal: bond0.99
           skylab_datastore_device: sdb
           skylab_networking:
             eno1:

playbooks/configure.yaml

@@ -15,6 +15,7 @@
   tasks:
     - include_tasks: tasks/meta/bootstrap-remote-env.yaml
 
+
 - name: Configure hosts by role
   hosts: linux
   gather_facts: false
@@ -23,3 +24,5 @@
       when: "'server' in skylab_roles | default([])"
     - role: datastore
       when: "'datastore' in skylab_roles | default([])"
+    - role: swarm
+      when: "'swarm' in skylab_roles | default([])"

playbooks/node-down.yaml

@@ -43,10 +43,10 @@
     - name: Set common fact for node addresses
       vars:
         _node_addresses:
-          - "{{ lookup('vars', 'ansible_' + skylab_cluster.interface).ipv4.address }}"
+          - "{{ lookup('vars', 'ansible_' + skylab_cluster.interface.access).ipv4.address }}"
       ansible.builtin.set_fact:
         _node_addresses: "{{ _node_addresses + [item.address] }}"
-      loop: "{{ lookup('vars', 'ansible_' + skylab_cluster.interface).ipv4_secondaries }}"
+      loop: "{{ lookup('vars', 'ansible_' + skylab_cluster.interface.access).ipv4_secondaries }}"
       loop_control:
         label: "{{ item.address }}"
 
@@ -117,7 +117,7 @@
       become: true
       when: skylab_cluster.address | ansible.netcommon.ipaddr('address') in _node_addresses
       ansible.builtin.command:
-        cmd: ip address delete {{ skylab_cluster.address | ansible.netcommon.ipaddr('host/prefix') }} dev {{ skylab_cluster.interface }}
+        cmd: ip address delete {{ skylab_cluster.address | ansible.netcommon.ipaddr('host/prefix') }} dev {{ skylab_cluster.interface.access }}
       changed_when: true
 
     - name: Assign address to alt node
@@ -125,5 +125,5 @@
       become: true
       when: skylab_cluster.address | ansible.netcommon.ipaddr('address') not in hostvars[_target_alt]._node_addresses
       ansible.builtin.command:
-        cmd: ip address add {{ skylab_cluster.address | ansible.netcommon.ipaddr('host/prefix') }} dev {{ hostvars[_target_alt].skylab_cluster.interface }}
+        cmd: ip address add {{ skylab_cluster.address | ansible.netcommon.ipaddr('host/prefix') }} dev {{ hostvars[_target_alt].skylab_cluster.interface.access }}
       changed_when: true

roles/server/tasks/firewalld.yaml

@@ -18,12 +18,3 @@
   loop: "{{ skylab_networking | dict2items }}"
   loop_control:
     label: "{{ item.key }}"
-
-- name: Configure firewall for docker interface
-  become: true
-  when: "'docker0' in ansible_interfaces"
-  ansible.posix.firewalld:
-    interface: docker0
-    zone: dmz
-    permanent: true
-    immediate: true

roles/swarm/tasks/check.yaml

@@ -0,0 +1,69 @@
+---
+- name: Check cluster swarm status
+  run_once: true
+  block:
+    - name: Fetch cluster server swarm info
+      delegate_to: "{{ item }}"
+      ansible.builtin.command:
+        cmd: !unsafe docker info --format '{{json .Swarm}}'
+      changed_when: false
+      register: _docker_cluster_swarm_state_raw
+      loop: "{{ groups.cluster }}"
+
+    - name: Process cluster server swarm info
+      vars:
+        _docker_cluster_swarm_state: {}
+      ansible.builtin.set_fact:
+        _docker_cluster_swarm_state: "{{ _docker_cluster_swarm_state | combine({item.item: (item.stdout | from_json)}) }}"
+      loop: "{{ _docker_cluster_swarm_state_raw.results }}"
+      loop_control:
+        label: "{{ item.item }}"
+
+    - name: Identify swarm managers
+      vars:
+        _docker_cluster_swarm_managers: []
+      when: item.value.LocalNodeState == 'active' and item.value.ControlAvailable
+      ansible.builtin.set_fact:
+        _docker_cluster_swarm_managers: "{{ _docker_cluster_swarm_managers + [item.key] }}"
+      loop: "{{ _docker_cluster_swarm_state | dict2items }}"
+      loop_control:
+        label: "{{ item.key }}"
+
+    - name: Check that swarm managers were discovered
+      ansible.builtin.assert:
+        that:
+          - _docker_cluster_swarm_managers
+        fail_msg: >-
+          ERROR: None of the member cluster servers ({{ groups.cluster | join(', ') }}) are joined to
+          a docker swarm or is a swarm manager. Please join at least one cluster server to a swarm and
+          promote it to swarm manager
+        success_msg: >-
+          Identified {{ _docker_cluster_swarm_managers | count }} swarm managers
+          ({{ _docker_cluster_swarm_managers | join(', ') }})
+
+    - name: Determine swarm manager cluster IDs
+      vars:
+        _docker_cluster_swarm_manager_cluster_ids: []
+      ansible.builtin.set_fact:
+        _docker_cluster_swarm_manager_cluster_ids: "{{ _docker_cluster_swarm_manager_cluster_ids + [_docker_cluster_swarm_state[item].Cluster.ID] }}"
+      loop: "{{ _docker_cluster_swarm_managers }}"
+
+    - name: Check swarm managers are part of the same swarm
+      ansible.builtin.assert:
+        that:
+          - _docker_cluster_swarm_manager_cluster_ids | unique | count == 1
+        fail_msg: >-
+          ERROR: Swarm managers ({{ _docker_cluster_swarm_managers | join(', ') }}) appear to be
+          joined to different swarms
+          (IDs {{ _docker_cluster_swarm_manager_cluster_ids | join(', ') }})
+        success_msg: >-
+          Swarm managers are joined to swarm with ID
+          {{ _docker_cluster_swarm_manager_cluster_ids[0] }}
+
+    - name: Determine swarm manager to use for host configuration
+      ansible.builtin.set_fact:
+        _docker_swarm_manager: "{{ _docker_cluster_swarm_managers[0] }}"
+
+- name: Determine whether host needs to be added to the swarm
+  ansible.builtin.set_fact:
+    _docker_swarm_needs_join: "{{ not _docker_cluster_swarm_state[inventory_hostname].Cluster.ID | default('') == _docker_cluster_swarm_manager_cluster_ids[0] }}"

roles/swarm/tasks/configure.yaml

@@ -0,0 +1,53 @@
+---
+- name: Determine docker daemon DNS servers
+  vars:
+    _docker_daemon_dns: []
+  ansible.builtin.set_fact:
+    _docker_daemon_dns: "{{ _docker_daemon_dns + (item.value.dns | default([])) }}"
+  loop: "{{ skylab_networking | dict2items }}"
+  loop_control:
+    label: "{{ item.key }}"
+
+- name: Create docker config directory
+  become: true
+  ansible.builtin.file:
+    path: /etc/docker
+    state: directory
+    owner: "{{ ansible_user }}"
+    group: docker
+    mode: 0750
+
+- name: Configure docker daemon
+  become: true
+  ansible.builtin.template:
+    src: daemon.json.j2
+    dest: /etc/docker/daemon.json
+    mode: 0640
+    owner: "{{ ansible_user }}"
+    group: docker
+
+- name: Start and enable docker service
+  become: true
+  ansible.builtin.systemd:
+    name: docker
+    state: started
+    enabled: true
+
+- name: Include access variables
+  ansible.builtin.include_vars:
+    file: vars/access.yaml
+
+- name: Add administrators to docker group
+  become: true
+  when: item.admin | default(false) and 'cluster' in item.targets
+  ansible.builtin.user:
+    name: "{{ item.name }}"
+    group: "{{ item.name }}"
+    groups: docker
+    append: true
+  loop: "{{ skylab_accounts }}"
+  loop_control:
+    label: "{{ item.name }},{{ item.uid }}"
+
+- name: Reset connection to get new group membership
+  ansible.builtin.meta: reset_connection

roles/swarm/tasks/install.yaml

@@ -0,0 +1,26 @@
+---
+- name: Install Docker repository
+  become: true
+  ansible.builtin.get_url:
+    url: https://download.docker.com/linux/centos/docker-ce.repo
+    dest: /etc/yum.repos.d/docker-ce.repo
+    owner: root
+    group: "{{ ansible_user }}"
+    mode: 0644
+  register: _docker_repo_status
+
+- name: Install docker repository GPG key
+  become: true
+  ansible.builtin.rpm_key:
+    key: https://download.docker.com/linux/centos/gpg
+    state: present
+
+- name: Install Docker
+  become: true
+  ansible.builtin.dnf:
+    state: present
+    name:
+      - docker-ce
+      - docker-ce-cli
+      - containerd.io
+    update_cache: "{{ _docker_repo_status.changed }}"

roles/swarm/tasks/join.yaml

@@ -0,0 +1,41 @@
+---
+- name: Fetch join token from existing manager
+  delegate_to: "{{ _docker_swarm_manager }}"
+  changed_when: false
+  ansible.builtin.command:
+    cmd: docker swarm join-token manager --quiet
+  register: _docker_swarm_join_token
+
+- name: Fetch manager addresses from existing manager
+  delegate_to: "{{ _docker_swarm_manager }}"
+  changed_when: false
+  ansible.builtin.command:
+    cmd: !unsafe docker info --format '{{json .Swarm.RemoteManagers}}'
+  register: _docker_swarm_manager_info_raw
+
+- name: Process manager addresses
+  vars:
+    _docker_swarm_manager_addresses: []
+  ansible.builtin.set_fact:
+    _docker_swarm_manager_addresses: "{{ _docker_swarm_manager_addresses + [item.Addr] }}"
+  loop: "{{ _docker_swarm_manager_info_raw.stdout | from_json }}"
+
+- name: Join node to swarm
+  vars:
+    ansible_python_interpreter: "{{ skylab_ansible_venv }}/bin/python"
+  community.docker.docker_swarm:
+    state: join
+    advertise_addr: "{{ lookup('vars', 'ansible_' + skylab_cluster.interface.internal).ipv4.address }}"
+    listen_addr: "{{ lookup('vars', 'ansible_' + skylab_cluster.interface.internal).ipv4.address }}"
+    remote_addrs: "{{ _docker_swarm_manager_addresses }}"
+    join_token: "{{ _docker_swarm_join_token.stdout.strip() }}"
+
+# For newly added nodes we don't want to have services be automatically scheduled on them
+# until the configuration is complete. The node-up playbook will be responsible for updating
+# the node to make it available in the cluster again
+- name: Update node to drain
+  vars:
+    ansible_python_interpreter: "{{ skylab_ansible_venv }}/bin/python"
+  community.docker.docker_node:
+    availability: drain
+    hostname: "{{ skylab_hostname }}"

roles/swarm/tasks/main.yaml

@@ -0,0 +1,18 @@
+---
+- name: Install Docker
+  ansible.builtin.import_tasks: install.yaml
+
+- name: Configure Docker
+  ansible.builtin.import_tasks: configure.yaml
+
+# This taskfile will set two facts that will be used in subsequent tasks:
+# * _docker_swarm_needs_join: a boolean indicating whether the host needs to be joined to the swarm
+#   or is already joined
+# * _docker_swarm_manager: the inventory hostname of a swarm manager that can be delegated to to
+#   fetch swarm joining info
+- name: Check swarm state ahead of swarm configuration
+  ansible.builtin.import_tasks: check.yaml
+
+- name: Join server to swarm
+  when: _docker_swarm_needs_join
+  ansible.builtin.include_tasks: join.yaml

roles/swarm/templates/daemon.json.j2

@@ -0,0 +1,7 @@
+{
+    "dns": [
+        {% for dns_server in _docker_daemon_dns %}
+        "{{ dns_server }}"{{ ',' if not loop.last else '' }}
+        {% endfor %}
+    ]
+}