setup.yaml

---
- name: Setup Machine
  hosts: localhost
  connection: local

  handlers:
    - name: Update package information
      become: true
      ansible.builtin.apt:
        update_cache: true
      changed_when: false

    - name: Restart SSH
      become: true
      ansible.builtin.service:
        name: sshd
        state: restarted
      when: dotenv_environment != "development"

    - name: Restart Docker
      become: true
      ansible.builtin.service:
        name: docker
        state: restarted
      when: dotenv_environment != "development"

    - name: Restart containerd
      become: true
      ansible.builtin.service:
        name: containerd
        state: restarted
      when: dotenv_environment != "development"

    - name: Restart SSHGuard
      become: true
      ansible.builtin.service:
        name: sshguard
        state: restarted
      when: dotenv_environment != "development"

    - name: Reload Vector
      become: true
      ansible.builtin.service:
        name: vector
        state: reloaded
      when: dotenv_environment != "development"

    - name: Reload Monit
      become: true
      ansible.builtin.service:
        name: monit
        state: reloaded
      when: dotenv_environment != "development"

    - name: Send test email as root
      become: true
      ansible.builtin.shell:
        cmd: |
          set -o pipefail
          echo "Test email as root user from new/updated MSMTP at" `hostname` \
            | mail -s "Test SMTP `hostname`" root
        executable: /bin/bash
      changed_when: false
      when: dotenv_environment != "development"

    - name: Send test email as cloud
      ansible.builtin.shell:
        cmd: |
          set -o pipefail
          echo "Test email as {{ ansible_facts['user_id'] | ansible.builtin.mandatory }} user from new/updated MSMTP at" `hostname` \
            | mail -s "Test SMTP `hostname`" {{ ansible_facts['user_id'] | ansible.builtin.mandatory }}
        executable: /bin/bash
      changed_when: false
      when: dotenv_environment != "development"

  pre_tasks:
    - name: Always set facts from environment variables
      ansible.builtin.set_fact:
        # The filter `ansible.builtin.default(undef(), true)` yields undefined if the value evaluates to false or an empty string.
        dotenv_environment: "{{ lookup('ansible.builtin.env', 'ENVIRONMENT') | ansible.builtin.default(undef(), true) | ansible.builtin.mandatory }}"
        host: "{{ lookup('ansible.builtin.env', 'HOST') | ansible.builtin.default(undef(), true) | ansible.builtin.mandatory }}"
        production_subdomain: "{{ lookup('ansible.builtin.env', 'PRODUCTION_SUBDOMAIN') | ansible.builtin.default(undef(), true) | ansible.builtin.mandatory }}"
        staging_subdomain: "{{ lookup('ansible.builtin.env', 'STAGING_SUBDOMAIN') | ansible.builtin.default(undef(), true) | ansible.builtin.mandatory }}"
        telemetry_subdomain: "{{ lookup('ansible.builtin.env', 'TELEMETRY_SUBDOMAIN') | ansible.builtin.default(undef(), true) | ansible.builtin.mandatory }}"
        monitor_http_port: "{{ lookup('ansible.builtin.env', 'MONITOR_HTTP_PORT') | ansible.builtin.default(undef(), true) | ansible.builtin.mandatory }}"
        logs_http_port: "{{ lookup('ansible.builtin.env', 'LOGS_HTTP_PORT') | ansible.builtin.default(undef(), true) | ansible.builtin.mandatory }}"
        metrics_http_port: "{{ lookup('ansible.builtin.env', 'METRICS_HTTP_PORT') | ansible.builtin.default(undef(), true) | ansible.builtin.mandatory }}"
        telemetry_grpc_port: "{{ lookup('ansible.builtin.env', 'TELEMETRY_GPRC_PORT') | ansible.builtin.default(undef(), true) | ansible.builtin.mandatory }}"
        telemetry_http_port: "{{ lookup('ansible.builtin.env', 'TELEMETRY_HTTP_PORT') | ansible.builtin.default(undef(), true) | ansible.builtin.mandatory }}"
        email_address: "{{ lookup('ansible.builtin.env', 'EMAIL_ADDRESS') | ansible.builtin.default(undef(), true) | ansible.builtin.mandatory }}"
        smtp_host: "{{ lookup('ansible.builtin.env', 'SMTP_HOST') | ansible.builtin.default(undef(), true) | ansible.builtin.mandatory }}"
        smtp_port: "{{ lookup('ansible.builtin.env', 'SMTP_PORT') | ansible.builtin.default(undef(), true) | ansible.builtin.mandatory }}"
        network_interface: "{{ lookup('ansible.builtin.env', 'NETWORK_INTERFACE') | ansible.builtin.default(undef(), true) | ansible.builtin.mandatory }}"
      tags: always

    - name: Always update package information
      become: true
      ansible.builtin.apt:
        update_cache: true
      changed_when: false
      tags: always

    - name: Always install access control list utilities (needed to escalate privileges, see https://github.com/ansible/ansible/issues/16052)
      become: true
      ansible.builtin.package:
        name: acl
        state: present
      tags: always

  post_tasks:
    - name: Clean-up Advanced Package Tool (APT) cache and remove unused dependencies
      become: true
      block:
        - name: Remove unused dependencies (autoremove)
          ansible.builtin.apt:
            autoremove: true
            update_cache: true

        - name: Clear obsolete package files (autoclean)
          ansible.builtin.apt:
            autoclean: true

  tasks:
    - name: Install curl
      become: true
      ansible.builtin.package:
        name: curl
        state: present

    - name: Grant user `cloud` read-only access to journald
      become: true
      ansible.builtin.user:
        name: "{{ ansible_facts['user_id'] | ansible.builtin.mandatory }}"
        groups: systemd-journal
        append: true

    - name: Install Bash completion
      become: true
      ansible.builtin.package:
        name: bash-completion
        state: present

    - name: Upgrades
      block:
        # Inspired by https://wiki.debian.org/UnattendedUpgrades
        # and https://askubuntu.com/questions/1305955/ansible-dpkg-reconfigure-plow-unattended-upgrades-stopped-while-running/1323117#1323117
        - name: Install `unattended-upgrades`
          become: true
          ansible.builtin.package:
            name: unattended-upgrades
            state: present

        - name: Install `apt-listchanges`
          become: true
          ansible.builtin.package:
            name: apt-listchanges
            state: present

        - name: Configure Debian to auto install security updates
          become: true
          ansible.builtin.debconf:
            name: unattended-upgrades
            question: unattended-upgrades/enable_auto_updates
            vtype: boolean
            value: "true"
        - name: Activate unattended upgrades
          become: true
          ansible.builtin.command:
            cmd: dpkg-reconfigure --force --frontend=noninteractive unattended-upgrades
            creates: /etc/apt/apt.conf.d/20auto-upgrades
          changed_when: false

    - name: Install GNU Make
      become: true
      ansible.builtin.package:
        name: make
        state: present

    - name: Hard disk management
      block:
        - name: Install Small Computer System Interface (SCSI) tools
          # Provides `rescan-scsi-bus.sh` used by `make scan`.
          become: true
          ansible.builtin.package:
            name: scsitools
            state: present

        # See https://opensource.com/article/18/11/partition-format-drive-linux
        - name: Install parted to partition hard disks
          become: true
          ansible.builtin.package:
            name: parted
            state: present

        - name: Install ext2/ext3/ext4 file system utilities to format hard disks
          become: true
          ansible.builtin.package:
            name: e2fsprogs
            state: present

    - name: Install Apache 2 utils to manage passwords for HTTP Basic Authentication using htpasswd
      become: true
      ansible.builtin.package:
        name: apache2-utils
        state: present

    - name: Install `moreutils` as it includes `chronic` that we use to cure Cron's chronic email problem
      # For details on the problem see https://habilis.net/cronic/
      become: true
      ansible.builtin.package:
        name: moreutils
        state: present

    - name: Install and configure `msmtp` to be used by Cron and other services to send emails
      vars:
        from_email_address: "{{ email_address }}"
      block:
        - name: Uninstall `mailutils` which includes GNU `mail`
          # The problem with GNU `mail` combined with `msmtp` is that aliases
          # in `/etc/aliases` are ignored. For example, when sending an email
          # by running `echo 'body' | mail -s 'subject' root`, the recipient
          # address is `root@hostname` instead of whatever email address is
          # given in `/etc/aliases` for `root`.
          become: true
          ansible.builtin.package:
            name: mailutils
            state: absent

        - name: Install `bsd-mailx` which provides the `mail` command
          # An alternative would be `s-nail`
          become: true
          ansible.builtin.package:
            name: bsd-mailx
            state: present

        - name: Install `msmtp` to send emails from mail user agents
          # For details see https://wiki.debian.org/msmtp
          # The manual is published under https://marlam.de/msmtp/msmtp.html
          # We do not use `sSMTP` because it is unmaintained according to https://wiki.debian.org/sSMTP
          become: true
          ansible.builtin.package:
            name: msmtp
            state: present

        - name: Install `msmtp-mta` to symbolically link `/usr/sbin/sendmail` to `msmtp` that other software can use to send mail
          become: true
          ansible.builtin.package:
            name: msmtp-mta
            state: present

        - name: Copy `mail.rc` configuration file
          become: true
          ansible.builtin.template:
            src: mail.rc.j2
            dest: /etc/mail.rc
            owner: root
            group: root
            mode: "0644"
          notify:
            - Send test email as root
            - Send test email as cloud

        - name: Copy system-wide `mstprc` configuration file
          become: true
          ansible.builtin.template:
            src: msmtprc.j2
            dest: /etc/msmtprc
            owner: root
            group: root
            mode: "0644"
          notify: Send test email as root

        - name: Copy user-specific `mstprc` configuration file
          ansible.builtin.template:
            src: msmtprc.j2
            dest: "{{ ansible_facts['env']['HOME'] | ansible.builtin.mandatory }}/.msmtprc"
            mode: "0644"
          notify: Send test email as cloud

        - name: Copy `aliases` configuration file
          become: true
          ansible.builtin.template:
            src: aliases.j2
            dest: /etc/aliases
            owner: root
            group: root
            mode: "0644"
          notify:
            - Send test email as root
            - Send test email as cloud

        - name: Flush handlers to send test emails
          ansible.builtin.meta: flush_handlers

    - name: Configure Cron
      block:
        - name: Add paths to environment variable `PATH` for root's crontab
          become: true
          community.general.cronvar:
            name: PATH
            value: /usr/local/bin/:/usr/bin:/bin
            state: present

        - name: Add paths to environment variable `PATH` for user's crontab
          community.general.cronvar:
            name: PATH
            value: /usr/local/bin/:/usr/bin:/bin
            state: present

        - name: Backup database and prune backups
          ansible.builtin.cron:
            name: database-backup
            minute: 0
            hour: 3
            job: "chronic make --directory=/app/machine --file=/app/machine/maintenance.mk backup prune-backups"

        - name: Renew Transport Layer Security (TLS) certificates needed for the `S` in `HTTPS`
          ansible.builtin.cron:
            name: tls-renewal
            minute: 5
            hour: 3
            job: "chronic make --directory=/app/machine --file=/app/machine/maintenance.mk renew-tls"

        - name: Set periodic docker system prune
          # See https://docs.docker.com/config/pruning/#prune-everything
          ansible.builtin.cron:
            name: docker-prune
            minute: 10
            hour: 3
            job: "chronic make --directory=/app/machine --file=/app/machine/maintenance.mk prune-docker"

        - name: Restart Docker backend service in production to reload rotated JWT certificates
          ansible.builtin.cron:
            name: restart-backend-service-in-production
            minute: 15
            hour: 3
            job: "chronic make --directory=/app/machine --file=/app/machine/maintenance.mk restart SERVICE=backend ENV=production"

        - name: Restart Docker backend service in staging to reload rotated JWT certificates
          ansible.builtin.cron:
            name: restart-backend-service-in-staging
            minute: 15
            hour: 3
            job: "chronic make --directory=/app/machine --file=/app/machine/maintenance.mk restart SERVICE=backend ENV=staging"

        - name: Rotate logs, vacuum `journald`, and clean-up backed up logs (as the root user)
          become: true
          ansible.builtin.cron:
            name: journald-vacuuming
            minute: 20
            hour: 3
            job: "chronic make --directory=/app/machine --file=/app/machine/logs.mk rotate vacuum clean-up"

    - name: Harden SSH
      become: true
      block:
        - name: Install nftables and SSHGuard
          ansible.builtin.package:
            name:
              - nftables
              - sshguard
            state: present

        - name: Configure SSHGuard whitelist
          vars:
            host: "{{ host }}"
          ansible.builtin.template:
            src: sshguard-whitelist.j2
            dest: /etc/sshguard/whitelist
            owner: root
            group: root
            mode: "0644"
          notify:
            - Restart SSHGuard

        - name: Enable SSHGuard service to start on boot
          ansible.builtin.service:
            name: sshguard
            enabled: true
            state: started

        - name: Ensure ~/.ssh directory exists with 0700 permissions
          ansible.builtin.file:
            path: "{{ ansible_facts['env']['HOME'] | ansible.builtin.mandatory }}/.ssh"
            mode: "0700"
            owner: "{{ ansible_facts['user_id'] | ansible.builtin.mandatory }}"
            state: directory

        - name: Ensure ~/.ssh/authorized_keys file exists with 0600 permissions
          ansible.builtin.file:
            path: "{{ ansible_facts['env']['HOME'] | ansible.builtin.mandatory }}/.ssh/authorized_keys"
            mode: "0600"
            owner: "{{ ansible_facts['user_id'] | ansible.builtin.mandatory }}"
            state: touch

        - name: Ensure at least one SSH key is authorized
          block:
            - name: Count authorized SSH keys
              ansible.builtin.command:
                cmd: "grep --count --invert-match '^#|^$' {{ ansible_facts['env']['HOME'] | ansible.builtin.mandatory }}/.ssh/authorized_keys"
              register: authorized_key_count
              # `rc` below means return code, aka, status code.
              failed_when: authorized_key_count.rc == 2
              changed_when: false

            - name: Fail if there are no authorized SSH keys
              ansible.builtin.fail:
                msg: "The ~/.ssh/authorized_keys file does not contain valid keys."
              # Check if grep found 0 lines or if the file was missing entirely
              when: authorized_key_count.stdout == "0"

        - name: Configure SSH
          ansible.builtin.copy:
            src: sshd_config.conf
            dest: /etc/ssh/sshd_config
            owner: root
            group: root
            mode: "0644"
          notify:
            - Restart SSH

        - name: Flush handlers to restart SSHGuard and SSH
          ansible.builtin.meta: flush_handlers

    - name: GnuPG
      become: true
      block:
        - name: Add GnuPG repository
          ansible.builtin.deb822_repository:
            name: gnupg2
            types: deb
            uris: "https://repos.gnupg.org/deb/gnupg/{{ ansible_facts['distribution_release'] | ansible.builtin.mandatory }}/"
            suites: "{{ ansible_facts['distribution_release'] | ansible.builtin.mandatory }}"
            components: main
            architectures: amd64
            signed_by: "https://repos.gnupg.org/deb/gnupg/{{ ansible_facts['distribution_release'] | ansible.builtin.mandatory }}/gnupg-signing-key.gpg"
            state: present
          notify: Update package information

        - name: Flush handlers to update package information
          ansible.builtin.meta: flush_handlers

        - name: Install GnuPG
          ansible.builtin.package:
            name: gnupg2
            state: present

    - name: Docker
      # Inspired by https://ops.tips/blog/docker-ansible-role/
      become: true
      block:
        - name: Add Docker repository
          ansible.builtin.deb822_repository:
            name: docker
            types: deb
            uris: "https://download.docker.com/linux/debian"
            suites: "{{ ansible_facts['distribution_release'] | ansible.builtin.mandatory }}"
            components: stable
            architectures: amd64
            signed_by: "https://download.docker.com/linux/debian/gpg"
            state: present
          notify: Update package information

        - name: Flush handlers to update package information
          ansible.builtin.meta: flush_handlers

        - name: Install Docker Engine, containerd, and Docker Compose
          ansible.builtin.package:
            name:
              - docker-ce
              - docker-ce-cli
              - containerd.io
              - docker-compose-plugin
            state: present

        - name: Prepare default daemon configuration
          # For a list of options see
          # https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-configuration-file
          ansible.builtin.copy:
            src: ./docker-daemon.json
            dest: /etc/docker/daemon.json
            mode: "0644"
          notify:
            - Restart Docker

        - name: Add user `cloud` to group `docker`
          ansible.builtin.user:
            name: "{{ ansible_facts['user_id'] | ansible.builtin.mandatory }}"
            groups: docker
            append: true

        - name: Enable Docker systemd service to start on boot
          ansible.builtin.service:
            name: docker
            enabled: true
            state: started

        - name: Configure containerd
          # For a list of options see
          # https://containerd.io/docs/main/man/containerd-config.toml.5/
          ansible.builtin.copy:
            src: ./containerd.toml
            dest: /etc/containerd/config.toml
            mode: "0644"
          notify:
            - Restart containerd

        - name: Flush handlers to restart Docker and containerd
          ansible.builtin.meta: flush_handlers

    - name: Vector
      # https://vector.dev/
      become: true
      block:
        - name: Add Vector repository
          ansible.builtin.deb822_repository:
            name: vector
            types: deb
            uris: "https://apt.vector.dev"
            suites: stable
            components: vector-0
            architectures: amd64
            signed_by: "https://keys.datadoghq.com/DATADOG_APT_KEY_CURRENT.public"
            state: present
          notify: Update package information

        - name: Flush handlers to update package information
          ansible.builtin.meta: flush_handlers

        - name: Install Vector
          ansible.builtin.package:
            name: vector
            state: present

        - name: Configure Vector
          ansible.builtin.template:
            src: vector.yaml.j2
            dest: /etc/vector/vector.yaml
            owner: root
            group: root
            mode: "0644"
          notify:
            - Reload Vector

        - name: Add user `vector` to group `docker`
          ansible.builtin.user:
            name: vector
            groups: docker
            append: true

        - name: Enable Vector systemd service to start on boot
          ansible.builtin.systemd:
            name: vector.service
            enabled: true
            state: started

        - name: Flush handlers to reload Vector
          ansible.builtin.meta: flush_handlers

    - name: Monitor system with Monit
      # https://mmonit.com/monit/
      become: true
      block:
        - name: Install Monit
          ansible.builtin.package:
            name: monit
            state: present

        - name: Configure Monit
          vars:
            from_email_address: "{{ email_address }}"
            production_host: "{{ production_subdomain }}.{{ host }}"
            staging_host: "{{ staging_subdomain }}.{{ host }}"
            telemetry_host: "{{ telemetry_subdomain }}.{{ host }}"
          ansible.builtin.template:
            src: monitrc.j2
            dest: /etc/monit/monitrc
            owner: root
            group: root
            mode: "0600"
          notify:
            - Reload Monit

        - name: Make Monit a systemd service
          ansible.builtin.copy:
            src: monit.service
            dest: /etc/systemd/system/monit.service
            owner: root
            group: root
            mode: "0644"

        - name: Enable Monit systemd service to start on boot
          ansible.builtin.systemd:
            name: monit.service
            enabled: true
            # reload daemon to make it notice the new monit.service file
            daemon_reload: true
            state: started

        - name: Grant user `cloud` read-only access to Monit without a password
          community.general.sudoers:
            name: monit-nopasswd
            user: "{{ ansible_facts['user_id'] | ansible.builtin.mandatory }}"
            commands: /usr/bin/monit
            nopassword: true
            state: present

        - name: Flush handlers to reload Monit
          ansible.builtin.meta: flush_handlers