Node exporter ecs monitoring (#179)

This PR adds the node exporter service to ECS cluster machines. This is
necessary for #178 (and therefore for #104).

Node exporter is installed with the user-data script that initializes
cluster machines

ansible/deploy-clickhouse-proxy.yml

@@ -11,3 +11,13 @@
       vars: 
         clickhouse_url: "clickhouse3.prod.ooni.io"
         clickhouse_port: 9000
+    - role: dehydrated
+      vars:
+        ssl_domains: "clickhouseproxy.dev.ooni.io"
+        tls_cert_dir: /var/lib/dehydrated/certs
+    - role: prometheus_node_exporter
+      vars:
+        node_exporter_port: 9100
+        prometheus_nginx_proxy_config: 
+          - location: /metrics/node_exporter
+            proxy_pass: http://127.0.0.1:9100/metrics

ansible/requirements/ansible-galaxy.yml

@@ -12,4 +12,4 @@
   name: idealista.clickhouse_role
 - src: https://github.com/ooni/airflow-role.git
   scm: git
-  name: ooni.airflow_role
+  name: ooni.airflow_role

tf/modules/ecs_cluster/main.tf

@@ -145,6 +145,7 @@ resource "aws_launch_template" "container_host" {
   user_data = base64encode(templatefile("${path.module}/templates/ecs-setup.sh", {
     ecs_cluster_name = var.name,
     ecs_cluster_tags = var.tags
+    node_exporter_port = var.node_exporter_port
   }))
 
   update_default_version               = true

tf/modules/ecs_cluster/templates/ecs-setup.sh

@@ -7,3 +7,62 @@ ECS_CONTAINER_INSTANCE_TAGS=${jsonencode(ecs_cluster_tags)}
 ECS_ENABLE_TASK_IAM_ROLE=true
 EOF
 
+# Exit on error and show running commands
+set -ex
+
+# Install node exporter on this machine
+# IN CASE OF UPDATE: You can find this downloads and its checksums here: 
+# https://prometheus.io/download/#node_exporter
+DOWNLOAD_LINK='https://github.com/prometheus/node_exporter/releases/download/v1.8.2/node_exporter-1.8.2.linux-amd64.tar.gz'
+CHECKSUM='6809dd0b3ec45fd6e992c19071d6b5253aed3ead7bf0686885a51d85c6643c66'
+
+# Download node exporter binary
+echo "Downloading node exporter..."
+cd /tmp
+curl -O -L $DOWNLOAD_LINK
+
+# Checksum the file
+ACTUAL_FILE=$(ls | grep node_exporter-*.*-amd64.tar.gz)
+echo "$CHECKSUM $ACTUAL_FILE" | sha256sum -c -
+if [[ $? -eq 0 ]]; then
+    echo "Node exporter checksum validation OK!"
+else
+    echo "[ERROR] Checksum validation for node exporter failed!" >&2
+    exit 1
+fi
+
+# Move it to an executable path
+tar xvfz node_exporter-*.*-amd64.tar.gz
+chmod 555 node_exporter-*.*-amd64/node_exporter
+sudo mv node_exporter-*.*-amd64/node_exporter /usr/local/bin/
+# Delete remaning files
+rm -r node_exporter-*.*-amd64 node_exporter-*.*-amd64.tar.gz
+
+
+# Add node exporter service user
+echo "Creating node exporter user..."
+sudo useradd -rs /bin/false node_exporter
+
+# Create service file for node exporter
+echo "Setting up service file..."
+cat <<'EOF' >> /tmp/node_exporter.service
+[Unit]
+Description=Node Exporter
+After=network.target
+[Service]
+User=node_exporter
+Group=node_exporter
+Type=simple
+ExecStart=/usr/local/bin/node_exporter --web.listen-address=:${node_exporter_port}
+Restart=always
+[Install]
+WantedBy=multi-user.target
+EOF
+
+sudo mv /tmp/node_exporter.service /etc/systemd/system
+
+# update systemd
+echo "Updating systemd..."
+sudo systemctl daemon-reload
+sudo systemctl enable node_exporter
+sudo systemctl start node_exporter

tf/modules/ecs_cluster/variables.tf

@@ -61,3 +61,7 @@ variable "instance_type" {
 variable "instance_volume_size" {
   default = "5"
 }
+
+variable "node_exporter_port" {
+  default = "9100"
+}