diff --git a/playbooks/configure.yaml b/playbooks/configure.yaml index 13e5097..51e309c 100644 --- a/playbooks/configure.yaml +++ b/playbooks/configure.yaml @@ -15,6 +15,7 @@ tasks: - include_tasks: tasks/meta/bootstrap-remote-env.yaml + - name: Configure hosts by role hosts: linux gather_facts: false @@ -23,3 +24,5 @@ when: "'server' in skylab_roles | default([])" - role: datastore when: "'datastore' in skylab_roles | default([])" + - role: swarm + when: "'swarm' in skylab_roles | default([])" diff --git a/roles/server/tasks/firewalld.yaml b/roles/server/tasks/firewalld.yaml index 132aece..75738e4 100644 --- a/roles/server/tasks/firewalld.yaml +++ b/roles/server/tasks/firewalld.yaml @@ -18,12 +18,3 @@ loop: "{{ skylab_networking | dict2items }}" loop_control: label: "{{ item.key }}" - -- name: Configure firewall for docker interface - become: true - when: "'docker0' in ansible_interfaces" - ansible.posix.firewalld: - interface: docker0 - zone: dmz - permanent: true - immediate: true diff --git a/roles/swarm/tasks/check.yaml b/roles/swarm/tasks/check.yaml new file mode 100644 index 0000000..f0fc9b4 --- /dev/null +++ b/roles/swarm/tasks/check.yaml @@ -0,0 +1,69 @@ +--- +- name: Check cluster swarm status + run_once: true + block: + - name: Fetch cluster server swarm info + delegate_to: "{{ item }}" + ansible.builtin.command: + cmd: !unsafe docker info --format '{{json .Swarm}}' + changed_when: false + register: _docker_cluster_swarm_state_raw + loop: "{{ groups.cluster }}" + + - name: Process cluster server swarm info + vars: + _docker_cluster_swarm_state: {} + ansible.builtin.set_fact: + _docker_cluster_swarm_state: "{{ _docker_cluster_swarm_state | combine({item.item: (item.stdout | from_json)}) }}" + loop: "{{ _docker_cluster_swarm_state_raw.results }}" + loop_control: + label: "{{ item.item }}" + + - name: Identify swarm managers + vars: + _docker_cluster_swarm_managers: [] + when: item.value.LocalNodeState == 'active' and item.value.ControlAvailable + ansible.builtin.set_fact: + _docker_cluster_swarm_managers: "{{ _docker_cluster_swarm_managers + [item.key] }}" + loop: "{{ _docker_cluster_swarm_state | dict2items }}" + loop_control: + label: "{{ item.key }}" + + - name: Check that swarm managers were discovered + ansible.builtin.assert: + that: + - _docker_cluster_swarm_managers + fail_msg: >- + ERROR: None of the member cluster servers ({{ groups.cluster | join(', ') }}) are joined to + a docker swarm or is a swarm manager. Please join at least one cluster server to a swarm and + promote it to swarm manager + success_msg: >- + Identified {{ _docker_cluster_swarm_managers | count }} swarm managers + ({{ _docker_cluster_swarm_managers | join(', ') }}) + + - name: Determine swarm manager cluster IDs + vars: + _docker_cluster_swarm_manager_cluster_ids: [] + ansible.builtin.set_fact: + _docker_cluster_swarm_manager_cluster_ids: "{{ _docker_cluster_swarm_manager_cluster_ids + [_docker_cluster_swarm_state[item].Cluster.ID] }}" + loop: "{{ _docker_cluster_swarm_managers }}" + + - name: Check swarm managers are part of the same swarm + ansible.builtin.assert: + that: + - _docker_cluster_swarm_manager_cluster_ids | unique | count == 1 + fail_msg: >- + ERROR: Swarm managers ({{ _docker_cluster_swarm_managers | join(', ') }}) appear to be + joined to different swarms + (IDs {{ _docker_cluster_swarm_manager_cluster_ids | join(', ') }}) + success_msg: >- + Swarm managers are joined to swarm with ID + {{ _docker_cluster_swarm_manager_cluster_ids[0] }} + + - name: Determine swarm manager to use for host configuration + ansible.builtin.set_fact: + _docker_swarm_manager: "{{ _docker_cluster_swarm_managers[0] }}" + +- name: Determine whether host needs to be added to the swarm + ansible.builtin.set_fact: + _docker_swarm_needs_join: "{{ not _docker_cluster_swarm_state[inventory_hostname].Cluster.ID | default('') == _docker_cluster_swarm_manager_cluster_ids[0] }}" diff --git a/roles/swarm/tasks/configure.yaml b/roles/swarm/tasks/configure.yaml new file mode 100644 index 0000000..3e63ce9 --- /dev/null +++ b/roles/swarm/tasks/configure.yaml @@ -0,0 +1,53 @@ +--- +- name: Determine docker daemon DNS servers + vars: + _docker_daemon_dns: [] + ansible.builtin.set_fact: + _docker_daemon_dns: "{{ _docker_daemon_dns + (item.value.dns | default([])) }}" + loop: "{{ skylab_networking | dict2items }}" + loop_control: + label: "{{ item.key }}" + +- name: Create docker config directory + become: true + ansible.builtin.file: + path: /etc/docker + state: directory + owner: "{{ ansible_user }}" + group: docker + mode: 0750 + +- name: Configure docker daemon + become: true + ansible.builtin.template: + src: daemon.json.j2 + dest: /etc/docker/daemon.json + mode: 0640 + owner: "{{ ansible_user }}" + group: docker + +- name: Start and enable docker service + become: true + ansible.builtin.systemd: + name: docker + state: started + enabled: true + +- name: Include access variables + ansible.builtin.include_vars: + file: vars/access.yaml + +- name: Add administrators to docker group + become: true + when: item.admin | default(false) and 'cluster' in item.targets + ansible.builtin.user: + name: "{{ item.name }}" + group: "{{ item.name }}" + groups: docker + append: true + loop: "{{ skylab_accounts }}" + loop_control: + label: "{{ item.name }},{{ item.uid }}" + +- name: Reset connection to get new group membership + ansible.builtin.meta: reset_connection diff --git a/roles/swarm/tasks/install.yaml b/roles/swarm/tasks/install.yaml new file mode 100644 index 0000000..975c707 --- /dev/null +++ b/roles/swarm/tasks/install.yaml @@ -0,0 +1,26 @@ +--- +- name: Install Docker repository + become: true + ansible.builtin.get_url: + url: https://download.docker.com/linux/centos/docker-ce.repo + dest: /etc/yum.repos.d/docker-ce.repo + owner: root + group: "{{ ansible_user }}" + mode: 0644 + register: _docker_repo_status + +- name: Install docker repository GPG key + become: true + ansible.builtin.rpm_key: + key: https://download.docker.com/linux/centos/gpg + state: present + +- name: Install Docker + become: true + ansible.builtin.dnf: + state: present + name: + - docker-ce + - docker-ce-cli + - containerd.io + update_cache: "{{ _docker_repo_status.changed }}" diff --git a/roles/swarm/tasks/join.yaml b/roles/swarm/tasks/join.yaml new file mode 100644 index 0000000..8a779a1 --- /dev/null +++ b/roles/swarm/tasks/join.yaml @@ -0,0 +1,41 @@ +--- +- name: Fetch join token from existing manager + delegate_to: "{{ _docker_swarm_manager }}" + changed_when: false + ansible.builtin.command: + cmd: docker swarm join-token manager --quiet + register: _docker_swarm_join_token + +- name: Fetch manager addresses from existing manager + delegate_to: "{{ _docker_swarm_manager }}" + changed_when: false + ansible.builtin.command: + cmd: !unsafe docker info --format '{{json .Swarm.RemoteManagers}}' + register: _docker_swarm_manager_info_raw + +- name: Process manager addresses + vars: + _docker_swarm_manager_addresses: [] + ansible.builtin.set_fact: + _docker_swarm_manager_addresses: "{{ _docker_swarm_manager_addresses + [item.Addr] }}" + loop: "{{ _docker_swarm_manager_info_raw.stdout | from_json }}" + +- name: Join node to swarm + vars: + ansible_python_interpreter: "{{ skylab_ansible_venv }}/bin/python" + community.docker.docker_swarm: + state: join + advertise_addr: "{{ lookup('vars', 'ansible_' + skylab_cluster.interface.internal).ipv4.address }}" + listen_addr: "{{ lookup('vars', 'ansible_' + skylab_cluster.interface.internal).ipv4.address }}" + remote_addrs: "{{ _docker_swarm_manager_addresses }}" + join_token: "{{ _docker_swarm_join_token.stdout.strip() }}" + +# For newly added nodes we don't want to have services be automatically scheduled on them +# until the configuration is complete. The node-up playbook will be responsible for updating +# the node to make it available in the cluster again +- name: Update node to drain + vars: + ansible_python_interpreter: "{{ skylab_ansible_venv }}/bin/python" + community.docker.docker_node: + availability: drain + hostname: "{{ skylab_hostname }}" diff --git a/roles/swarm/tasks/main.yaml b/roles/swarm/tasks/main.yaml new file mode 100644 index 0000000..3ff7665 --- /dev/null +++ b/roles/swarm/tasks/main.yaml @@ -0,0 +1,18 @@ +--- +- name: Install Docker + ansible.builtin.import_tasks: install.yaml + +- name: Configure Docker + ansible.builtin.import_tasks: configure.yaml + +# This taskfile will set two facts that will be used in subsequent tasks: +# * _docker_swarm_needs_join: a boolean indicating whether the host needs to be joined to the swarm +# or is already joined +# * _docker_swarm_manager: the inventory hostname of a swarm manager that can be delegated to to +# fetch swarm joining info +- name: Check swarm state ahead of swarm configuration + ansible.builtin.import_tasks: check.yaml + +- name: Join server to swarm + when: _docker_swarm_needs_join + ansible.builtin.include_tasks: join.yaml diff --git a/roles/swarm/templates/daemon.json.j2 b/roles/swarm/templates/daemon.json.j2 new file mode 100644 index 0000000..1020a71 --- /dev/null +++ b/roles/swarm/templates/daemon.json.j2 @@ -0,0 +1,7 @@ +{ + "dns": [ + {% for dns_server in _docker_daemon_dns %} + "{{ dns_server }}"{{ ',' if not loop.last else '' }} + {% endfor %} + ] +}