Add initial config for dedicated monitoring server

Add server role to iridium

inventory.yaml

@@ -10,6 +10,7 @@ all:
 en1:
   vars:
     skylab_location: Newton MA
+    skylab_dashboard: info.en1.local
     # gross hack for now, will be refactored later
     _skylab_adguard_nat_rule: 8
 
@@ -23,7 +24,17 @@ en1:
     iridium:
       ansible_host: 10.42.101.200
       skylab_description: Local Monitor Node
+      skylab_hostname: iridium.skylab.enp.one
       skylab_targets: [network]
+      skylab_networking:
+        enp4s0:
+          firewall: internal
+          dhcp: false
+          gateway: 10.42.101.1/24
+          dns:
+            - 10.42.101.1
+          addresses:
+            - 10.42.101.200/24
 
   children:
 
@@ -31,10 +42,6 @@ en1:
       vars:
         skylab_targets: [cluster, datastore]
         skylab_compose_version: 3.8
-        skylab_roles:
-          - server
-          - datastore
-          - swarm
 
       hosts:
         pegasus:  # jupiter
@@ -156,4 +163,3 @@ en2:
       skylab_hostname: hubble.en2.enp.one
       skylab_description: Cloud Web Server
       skylab_targets: [cloud]
-      skylab_roles: [server]

playbooks/configure.yaml

@@ -16,13 +16,25 @@
     - include_tasks: tasks/meta/bootstrap-remote-env.yaml
 
 
-- name: Configure hosts by role
-  hosts: linux
+# [lemony snicket voice] "server" here being a word used to mean "not a workstation"
+- name: Configure servers
+  hosts: linux:!workstation
   gather_facts: false
   roles:
     - role: server
-      when: "'server' in skylab_roles | default([])"
+
+
+- name: Configure cluster
+  hosts: linux:&cluster
+  gather_facts: false
+  roles:
     - role: datastore
-      when: "'datastore' in skylab_roles | default([])"
     - role: swarm
-      when: "'swarm' in skylab_roles | default([])"
+
+
+- name: Configure dashboard nodes
+  hosts: iridium
+  gather_facts: false
+  roles:
+    - role: dashboard
+      dashboard_hostname: "{{ skylab_dashboard }}"

playbooks/node-up.yaml

@@ -7,7 +7,6 @@
       private: true
   roles:
     - role: datastore
-      when: "'datastore' in skylab_roles | default([])"
   tasks:
     - name: Fetch node swarm ID
       ansible.builtin.command:

playbooks/update.yaml

@@ -47,6 +47,7 @@
         state: present
         update_cache: true
 
+
 - name: Update unix accounts
   hosts: linux
   tags:
@@ -75,38 +76,55 @@
         cmd: 'grep "{{ skylab_group.name }}:" /etc/group | cut --delimiter : --fields 4 | tr "," "\n"'
       register: _existing_skylab_accounts
 
-    - name: Delete removed user accounts
-      become: true
+    - name: Determine deleted skylab users
+      vars:
+        _deleted_accounts: []
       when: item not in (skylab_accounts | items2dict(key_name='name', value_name='uid'))
-      ansible.builtin.user:
-        name: "{{ item }}"
-        state: absent
+      ansible.builtin.set_fact:
+        _deleted_accounts: "{{ _deleted_accounts + [item] }}"
       loop: "{{ _existing_skylab_accounts.stdout_lines }}"
 
-    - name: Delete removed user groups
-      become: true
-      when: item not in (skylab_accounts | items2dict(key_name='name', value_name='uid'))
-      ansible.builtin.group:
-        name: "{{ item }}"
-        state: absent
-      loop: "{{ _existing_skylab_accounts.stdout_lines }}"
+    - name: Delete accounts
+      when: _deleted_accounts | default(false)
+      block:
+        - name: Delete removed user accounts
+          become: true
+          ansible.builtin.user:
+            name: "{{ item }}"
+            state: absent
+          loop: "{{ _deleted_accounts }}"
 
-    - name: Delete removed user home directories
-      become: true
-      when: item not in (skylab_accounts | items2dict(key_name='name', value_name='uid'))
-      ansible.builtin.file:
-        path: "/home/{{ item }}"
-        state: absent
-      loop: "{{ _existing_skylab_accounts.stdout_lines }}"
+        - name: Delete removed user groups
+          become: true
+          ansible.builtin.group:
+            name: "{{ item }}"
+            state: absent
+          loop: "{{ _deleted_accounts }}"
+
+        - name: Delete removed user home directories
+          become: true
+          ansible.builtin.file:
+            path: "/home/{{ item }}"
+            state: absent
+          loop: "{{ _deleted_accounts }}"
+
+    - name: Determine active users
+      when: item.targets | default([]) | intersect(skylab_targets)
+      vars:
+        _active_accounts: []
+      ansible.builtin.set_fact:
+        _active_accounts: "{{ _active_accounts + [item] }}"
+      loop: "{{ skylab_accounts }}"
+      loop_control:
+        label: "{{ item.uid }},{{ item.name }}"
 
     - name: Create account groups
-      when: item.targets | intersect(skylab_targets)
       become: true
       ansible.builtin.group:
         name: "{{ item.name }}"
         gid: "{{ item.uid }}"
         state: present
-      loop: "{{ skylab_accounts }}"
+      loop: "{{ _active_accounts }}"
       loop_control:
         label: "{{ item.uid }},{{ item.name }}"
 
@@ -119,12 +137,11 @@
           skylab_group_admin.name if item.admin | default(false) else '',
           skylab_group_automation.name if item.service | default(false) else '',
         ]}) }}"
-      loop: "{{ skylab_accounts }}"
+      loop: "{{ _active_accounts }}"
       loop_control:
         label: "{{ item.uid }},{{ item.name }}"
 
     - name: Create accounts
-      when: item.targets | intersect(skylab_targets)
       become: true
       ansible.builtin.user:
         name: "{{ item.name }}"
@@ -136,7 +153,7 @@
         system: "{{ item.service | default(false) }}"
         generate_ssh_key: false
         password: "{{ item.password }}"
-      loop: "{{ skylab_accounts }}"
+      loop: "{{ _active_accounts }}"
       loop_control:
         label: "{{ item.uid }},{{ item.name }}"
 
@@ -148,17 +165,10 @@
         group: "{{ item.name }}"
         owner: "{{ item.name }}"
         mode: 0700
-      loop: "{{ skylab_accounts }}"
+      loop: "{{ _active_accounts }}"
       loop_control:
         label: "{{ item.uid }},{{ item.name }}"
 
-    - name: Enforce root password
-      become: true
-      ansible.builtin.user:
-        name: root
-        password: "{{ skylab_root_password }}"
-        state: present
-
     - name: Create SSH directory
       become: true
       ansible.builtin.file:
@@ -167,31 +177,36 @@
         group: "{{ item.name }}"
         state: directory
         mode: 0700
-      loop: "{{ skylab_accounts }}"
+      loop: "{{ _active_accounts }}"
       loop_control:
         label: "{{ item.uid }},{{ item.name }}"
 
     - name: Update authorized keys
       become: true
-      when: item.targets | intersect(skylab_targets)
       ansible.builtin.authorized_key:
         user: "{{ item.name }}"
         key: "{{ skylab_ssh_keys[item.name] | join('\n') }}"
         state: present
         exclusive: true
-      loop: "{{ skylab_accounts }}"
+      loop: "{{ _active_accounts }}"
       loop_control:
         label: "{{ item.uid }},{{ item.name }}"
 
     - name: Enforce ownership of authorized keys
       become: true
-      when: item.targets | intersect(skylab_targets)
       ansible.builtin.file:
         path: /home/{{ item.name }}/.ssh/authorized_keys
         state: file
         owner: "{{ item.name }}"
         group: "{{ item.name }}"
         mode: 0400
-      loop: "{{ skylab_accounts }}"
+      loop: "{{ _active_accounts }}"
       loop_control:
         label: "{{ item.uid }},{{ item.name }}"
+
+    - name: Enforce root password
+      become: true
+      ansible.builtin.user:
+        name: root
+        password: "{{ skylab_root_password }}"
+        state: present

roles/dashboard/files/grafana.repo

@@ -0,0 +1,9 @@
+[grafana]
+name=grafana
+baseurl=https://packages.grafana.com/enterprise/rpm
+repo_gpgcheck=1
+enabled=1
+gpgcheck=1
+gpgkey=https://packages.grafana.com/gpg.key
+sslverify=1
+sslcacert=/etc/pki/tls/certs/ca-bundle.crt

roles/dashboard/files/ssl-options.conf

@@ -0,0 +1,22 @@
+# Ansible managed file - DO NOT EDIT
+#
+# https://www.digitalocean.com/community/tutorials/how-to-create-a-self-signed-ssl-certificate-for-nginx-in-ubuntu-16-04
+#
+
+ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+ssl_prefer_server_ciphers on;
+ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH";
+ssl_ecdh_curve secp384r1;
+ssl_session_cache shared:SSL:10m;
+ssl_session_tickets off;
+ssl_stapling on;
+ssl_stapling_verify on;
+resolver 1.1.1.1 1.0.0.1 valid=300s;
+resolver_timeout 5s;
+add_header Strict-Transport-Security "max-age=63072000; includeSubdomains";
+add_header X-Frame-Options DENY;
+add_header X-Content-Type-Options nosniff;
+
+ssl_dhparam /etc/nginx/ssl-dhparam.pem;
+
+# EOF

roles/dashboard/handlers/main.yaml

@@ -0,0 +1,12 @@
+---
+- name: restart-nginx
+  become: true
+  ansible.builtin.systemd:
+    name: nginx
+    state: restarted
+
+- name: restart-grafana
+  become: true
+  ansible.builtin.systemd:
+    name: grafana-server
+    state: restarted

roles/dashboard/tasks/grafana.yaml

@@ -0,0 +1,48 @@
+---
+- name: Install Grafana Enterprise repository
+  become: true
+  ansible.builtin.copy:
+    src: grafana.repo
+    dest: /etc/yum.repos.d/grafana.repo
+    owner: root
+    group: "{{ ansible_user }}"
+    mode: 0644
+  register: _grafana_repo
+
+- name: Install Grafana repository GPG key
+  become: true
+  ansible.builtin.rpm_key:
+    state: present
+    key: https://packages.grafana.com/gpg.key
+
+- name: Install Grafana
+  become: true
+  ansible.builtin.dnf:
+    name: grafana
+    state: present
+    update_cache: "{{ _grafana_repo.changed }}"
+
+- name: Enable and start Grafana
+  become: true
+  ansible.builtin.systemd:
+    name: grafana-server
+    state: started
+    enabled: true
+
+- name: Fetch installed grafana plugins
+  become: true
+  ansible.builtin.command:
+    cmd: grafana-cli plugins ls
+  changed_when: false
+  register: _grafana_plugins_raw
+
+- name: Install plugins
+  become: true
+  ansible.builtin.command:
+    cmd: grafana-cli plugins install {{ item }}
+  changed_when: item not in _grafana_plugins_raw.stdout
+  notify: [restart-grafana]
+  loop:
+    - marcusolsson-json-datasource
+    - grafana-clock-panel
+    - ayoungprogrammer-finance-datasource

roles/dashboard/tasks/main.yaml

@@ -0,0 +1,6 @@
+---
+- name: Install and configure Grafana
+  ansible.builtin.import_tasks: grafana.yaml
+
+- name: Install and configure Nginx
+  ansible.builtin.import_tasks: nginx.yaml

roles/dashboard/tasks/nginx.yaml

@@ -0,0 +1,107 @@
+---
+- name: Install nginx
+  become: true
+  ansible.builtin.dnf:
+    name: nginx
+    state: present
+
+- name: Enable and start nginx
+  become: true
+  ansible.builtin.systemd:
+    name: nginx
+    state: started
+    enabled: true
+
+- name: Configure firewall for Nginx
+  become: true
+  ansible.posix.firewalld:
+    service: "{{ item }}"
+    state: enabled
+    zone: internal
+    permanent: true
+    immediate: true
+  loop:
+    - http
+    - https
+
+- name: Configure SELinux for Nginx
+  when: ansible_selinux.status | default("") == "enabled"
+  become: true
+  ansible.posix.seboolean:
+    name: httpd_can_network_connect
+    state: true
+    persistent: true
+  notify: [restart-nginx]
+
+- name: Create certificate directory
+  become: true
+  ansible.builtin.file:
+    path: "{{ dashboard_certificate_directory }}"
+    state: directory
+    owner: nginx
+    group: "{{ ansible_user }}"
+    mode: 0570
+
+- name: Generate X509 private key
+  become: true
+  vars:
+    ansible_python_interpreter: "{{ skylab_ansible_venv }}/bin/python"
+  community.crypto.openssl_privatekey:
+    path: "{{ dashboard_certificate_directory }}/{{ dashboard_hostname }}.key"
+    type: RSA
+    size: 8192
+    passphrase: "{{ dashboard_certificate_password }}"
+    cipher: auto
+    owner: nginx
+    group: "{{ ansible_user }}"
+    mode: 0460
+
+- name: Install private key password file
+  become: true
+  ansible.builtin.copy:
+    content: "{{ dashboard_certificate_password }}"
+    dest: "{{ dashboard_certificate_directory }}/{{ dashboard_hostname }}.password"
+    owner: nginx
+    group: "{{ ansible_user }}"
+    mode: 0460
+
+- name: Create self-signed certificate
+  become: true
+  vars:
+    ansible_python_interpreter: "{{ skylab_ansible_venv }}/bin/python"
+  community.crypto.x509_certificate:
+    path: "{{ dashboard_certificate_directory }}/{{ dashboard_hostname }}.pem"
+    privatekey_path: "{{ dashboard_certificate_directory }}/{{ dashboard_hostname }}.key"
+    privatekey_passphrase: "{{ dashboard_certificate_password }}"
+    provider: selfsigned
+    owner: nginx
+    group: "{{ ansible_user }}"
+    mode: 0460
+  notify: [restart-nginx]
+
+- name: Copy nginx SSL parameters
+  become: true
+  ansible.builtin.copy:
+    src: ssl-options.conf
+    dest: /etc/nginx/ssl-options.conf
+    owner: nginx
+    group: "{{ ansible_user }}"
+    mode: 0664
+  notify: [restart-nginx]
+
+- name: Export Diffie-Hellman parameters
+  become: true
+  ansible.builtin.command:
+    cmd: openssl dhparam -out /etc/nginx/ssl-dhparam.pem 2048
+    creates: /etc/nginx/ssl-dhparam.pem
+  notify: [restart-nginx]
+
+- name: Configure nginx server
+  become: true
+  ansible.builtin.template:
+    src: nginx.conf.j2
+    dest: /etc/nginx/conf.d/{{ dashboard_hostname }}.conf
+    owner: nginx
+    group: "{{ ansible_user }}"
+    mode: 0444
+  notify: [restart-nginx]

roles/dashboard/templates/nginx.conf.j2

@@ -0,0 +1,29 @@
+# Ansible managed file - DO NOT MANUALLY EDIT
+#
+server {
+    server_name  {{ dashboard_hostname }};
+    root         /usr/share/nginx/html;
+
+    location / {
+        proxy_pass        http://127.0.0.1:3000/;
+        proxy_set_header  Host $host;
+    }
+
+    listen 443            ssl http2;
+    ssl_certificate       {{ dashboard_certificate_directory }}/{{ dashboard_hostname }}.pem;
+    ssl_certificate_key   {{ dashboard_certificate_directory }}/{{ dashboard_hostname }}.key;
+    ssl_password_file     {{ dashboard_certificate_directory }}/{{ dashboard_hostname }}.password;
+    include               /etc/nginx/ssl-options.conf;
+}
+
+server {
+    if ($host = {{ dashboard_hostname }}) {
+        return 301  https://$host$request_uri;
+    }
+
+    server_name     {{ dashboard_hostname }};
+    listen          80;
+    return          404;
+}
+#
+# EOF

roles/dashboard/vars/main.yaml

@@ -0,0 +1,15 @@
+---
+dashboard_certificate_directory: /etc/nginx/certs
+dashboard_certificate_password: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          62373265623036656632396637363539313437656433656461356561393538333536303961363462
+          3964353831633165363430313533623563343732623930630a393030393336613563313431306233
+          62393235303234336365313138633137663430653061343737616466303136616130643061356566
+          3165313038393163340a396365643335343332333335363539326635633466313264373639353930
+          36646462396139346432353233646635303031613639323266366235373132346363653431323666
+          38336365303431646530613030613437663035613332653865366432636238303437323633666239
+          64366435353762656362666531393865383639343461616365316634326334623733653664666161
+          63366234646466326531363666633966326462373562313839393731633931383762306663396562
+          65663031653661333439373461333234613863623364643464323863656630386561316565353232
+          35313338373631356231376361346662353365373030653965626434336339613936656138656637
+          666430306334623563306236616663623438

roles/server/templates/motd.j2

@@ -1,11 +1,11 @@
 
-    /#######   /##   /##   /##  /##   /##  /#######   /#####
-   /##        /##___/##   /##__/##   /##  /##___/##  /##__/##
-  /##____    /#######    /######    /##  /########  /######
-  ########  /##   |##   /##        /##  /##   /##  /##__/##
-      /##  /##    |##   \###      /##  /##   /##  /######
+    /#######   /##   /##  /##   /##   /##  /#######   /#####
+   /##        /##___/##  /##___/##   /##  /##___/##  /##__/##
+  /##____    /#######   /########   /##  /########  /######
+  ########  /##   |##        /##   /##  /##   /##  /##__/##
+      /##  /##    |##  /#######   /##  /##   /##  /######
  ____/##                         /##_____
-/######  ````````````````````   /########  ``````````````
+/######   *******************   /########   ************
 
  ✨ {{ skylab_description }} @{{ skylab_location }}
 {{ ' ' }}

roles/swarm/tasks/configure.yaml

@@ -39,7 +39,7 @@
 
 - name: Add administrators to docker group
   become: true
-  when: item.admin | default(false) and 'cluster' in item.targets
+  when: item.admin | default(false) and 'cluster' in (item.targets | default([]))
   ansible.builtin.user:
     name: "{{ item.name }}"
     group: "{{ item.name }}"

roles/swarm/tasks/gluster.yaml

@@ -31,7 +31,6 @@
 
 - name: Determine gluster servers
   run_once: true
-  when: "'datastore' in hostvars[item].skylab_roles | default([])"
   vars:
     _docker_glusterfs_hostnames: []
   ansible.builtin.set_fact: