From 4a3a43141fe8ad577bafd130b42ce2dac7765f3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Mon, 17 Jul 2017 10:30:18 +0200
Subject: [PATCH 1/8] Add roles for superset and wiki.js

---
 ansible/deploy-superset.yml                   | 31 +++++++
 ansible/inventory                             |  2 +
 .../letsencrypt/tasks/letsencrypt_gen.yml     | 13 +++
 ansible/roles/superset/defaults/main.yml      |  6 ++
 ansible/roles/superset/tasks/main.yml         | 29 +++++++
 ansible/roles/superset/tasks/setup-nginx.yml  | 31 +++++++
 .../superset/templates/superset_config.py.j2  | 28 +++++++
 .../superset/templates/superset_nginx.conf.j2 | 21 +++++
 ansible/roles/wiki/tasks/main.yml             | 83 +++++++++++++++++++
 .../roles/wiki/templates/node-wiki.service.j2 | 11 +++
 .../roles/wiki/templates/wiki_nginx.conf.j2   | 19 +++++
 ansible/roles/wiki/vars/main.yml              |  3 +
 requirements.txt                              |  2 +-
 13 files changed, 278 insertions(+), 1 deletion(-)
 create mode 100644 ansible/deploy-superset.yml
 create mode 100644 ansible/roles/superset/defaults/main.yml
 create mode 100644 ansible/roles/superset/tasks/main.yml
 create mode 100644 ansible/roles/superset/tasks/setup-nginx.yml
 create mode 100644 ansible/roles/superset/templates/superset_config.py.j2
 create mode 100644 ansible/roles/superset/templates/superset_nginx.conf.j2
 create mode 100644 ansible/roles/wiki/tasks/main.yml
 create mode 100644 ansible/roles/wiki/templates/node-wiki.service.j2
 create mode 100644 ansible/roles/wiki/templates/wiki_nginx.conf.j2
 create mode 100644 ansible/roles/wiki/vars/main.yml

diff --git a/ansible/deploy-superset.yml b/ansible/deploy-superset.yml
new file mode 100644
index 00000000..8641f507
--- /dev/null
+++ b/ansible/deploy-superset.yml
@@ -0,0 +1,31 @@
+---
+- include: ansible-version.yml
+
+- hosts: hkgsuperset.ooni.io
+  become: false
+  remote_user: root
+  gather_facts: true
+  pre_tasks:
+    - name: bootstap python
+      raw: if [ ! -x /usr/bin/python ]; then apt-get update && apt-get -y install python-simplejson python-apt; fi
+      register: output
+      changed_when: output.stdout != ""
+  roles:
+    - role: adm
+      adm_passwd:
+        - "{{ passwd.art }}"
+        - "{{ passwd.darkk }}"
+
+- hosts: hkgsuperset.ooni.io
+  roles:
+    - docker_py
+
+- hosts: hkgsuperset.ooni.io
+  gather_facts: false # already gathered
+  vars:
+    ansible_python_interpreter: "/root/venv/bin/python2.7"
+    letsencrypt_nginx: yes
+    letsencrypt_domains: "hkgsuperset.ooni.io"
+  roles:
+    #- letsencrypt
+    - superset
diff --git a/ansible/inventory b/ansible/inventory
index 47e63a72..bdb1fcf2 100644
--- a/ansible/inventory
+++ b/ansible/inventory
@@ -32,6 +32,8 @@ datacollector.infra.ooni.io
 vpn-gate-runner.infra.ooni.io
 stage.ooni.io # NOT-GH, ooni-zoo
 test.ooni.io # NOT-GH, ooni-zoo
+hkgsuperset.ooni.io
+wiki.ooni.io
 
 test-lists.openobservatory.org # This was used for testing out test-lists web interface. Not critical
 
diff --git a/ansible/roles/letsencrypt/tasks/letsencrypt_gen.yml b/ansible/roles/letsencrypt/tasks/letsencrypt_gen.yml
index 91b0a073..d43ea1e2 100644
--- a/ansible/roles/letsencrypt/tasks/letsencrypt_gen.yml
+++ b/ansible/roles/letsencrypt/tasks/letsencrypt_gen.yml
@@ -6,6 +6,19 @@
     repo='deb http://httpredir.debian.org/debian {{ ansible_distribution_release }}-backports main'
   when: ansible_distribution == "Debian"
 
+- name: Add certbot repository on Ubuntu
+  apt_repository: >
+    state=present
+    repo='ppa:certbot/certbot'
+  when: ansible_distribution == "Ubuntu"
+
+- name: Install Letsencrypt certbot
+  apt: >
+    name=certbot
+    state=latest
+    update_cache=yes
+  when: ansible_distribution == "Ubuntu"
+
 - name: Install Letsencrypt certbot
   apt: >
     name=certbot
diff --git a/ansible/roles/superset/defaults/main.yml b/ansible/roles/superset/defaults/main.yml
new file mode 100644
index 00000000..c4d39c64
--- /dev/null
+++ b/ansible/roles/superset/defaults/main.yml
@@ -0,0 +1,6 @@
+superset_path: /srv/superset
+superset_config_path: "{{ superset_path }}/config"
+superset_port: "3001"
+superset_listen_address: "127.0.0.1:{{ superset_port }}"
+superset_secret_key: "XXXCHANGEMEXXX"
+superset_docker_image: "amancevice/superset:0.18.5"
diff --git a/ansible/roles/superset/tasks/main.yml b/ansible/roles/superset/tasks/main.yml
new file mode 100644
index 00000000..2cf68f43
--- /dev/null
+++ b/ansible/roles/superset/tasks/main.yml
@@ -0,0 +1,29 @@
+---
+- name: mkdir for config and data
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: "u=rwx,g=rx,o="
+  with_items:
+    - "{{ superset_path }}"
+    - "{{ superset_config_path }}"
+
+- name: configure superset
+  template:
+    src: superset_config.py.j2
+    dest: "{{ superset_config_path }}/superset_config.py"
+
+- name: superset service
+  docker_container:
+    image: "{{ superset_docker_image }}"
+    name: superset
+    ports:
+      - "{{ superset_port }}:3000"
+    volumes:
+      - "{{ superset_config_path }}:/home/superset/.superset"
+
+# XXX also do:
+# docker run --detach --name superset [options] amancevice/superset
+# docker exec -it superset superset-init
+
+- include: setup-nginx.yml
diff --git a/ansible/roles/superset/tasks/setup-nginx.yml b/ansible/roles/superset/tasks/setup-nginx.yml
new file mode 100644
index 00000000..13b1916e
--- /dev/null
+++ b/ansible/roles/superset/tasks/setup-nginx.yml
@@ -0,0 +1,31 @@
+---
+- name: Install nginx
+  environment:
+    RUNLEVEL: 1 # To avoid nginx being launched right after it's installed
+  apt:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - "nginx"
+    - "python-passlib"
+
+- name: Remove default nginx virtual host
+  file:
+    name: /etc/nginx/sites-enabled/default
+    state: absent
+  register: nginx
+
+- name: Add nginx virtual host config
+  template:
+    src: superset_nginx.conf.j2
+    dest: /etc/nginx/sites-enabled/superset
+    owner: root
+    group: root
+    mode: 0644
+  register: nginx
+
+- name: Restart nginx
+  service:
+    name: nginx
+    state: restarted
+  when: nginx.changed
diff --git a/ansible/roles/superset/templates/superset_config.py.j2 b/ansible/roles/superset/templates/superset_config.py.j2
new file mode 100644
index 00000000..cf10538b
--- /dev/null
+++ b/ansible/roles/superset/templates/superset_config.py.j2
@@ -0,0 +1,28 @@
+#---------------------------------------------------------
+# Superset specific config
+#---------------------------------------------------------
+ROW_LIMIT = 5000
+SUPERSET_WORKERS = 4
+
+SUPERSET_WEBSERVER_PORT = {{ superset_port }}
+#---------------------------------------------------------
+
+#---------------------------------------------------------
+# Flask App Builder configuration
+#---------------------------------------------------------
+# Your App secret key
+SECRET_KEY = '{{ superset_secret_key }}'
+
+# The SQLAlchemy connection string to your database backend
+# This connection defines the path to the database that stores your
+# superset metadata (slices, connections, tables, dashboards, ...).
+# Note that the connection information to connect to the datasources
+# you want to explore are managed directly in the web UI
+SQLALCHEMY_DATABASE_URI = 'sqlite:////path/to/superset.db'
+
+# Flask-WTF flag for CSRF
+WTF_CSRF_ENABLED = True
+
+# Set this API key to enable Mapbox visualizations
+MAPBOX_API_KEY = ''
+
diff --git a/ansible/roles/superset/templates/superset_nginx.conf.j2 b/ansible/roles/superset/templates/superset_nginx.conf.j2
new file mode 100644
index 00000000..a9e12c29
--- /dev/null
+++ b/ansible/roles/superset/templates/superset_nginx.conf.j2
@@ -0,0 +1,21 @@
+# ansible-managed in ooni-sysadmin.git/ansible/roles/superset/templates/superset_nginx.conf.j2
+server {
+    server_name _;
+
+    listen 443 ssl default_server;
+    listen [::]:443 ssl default_server;
+
+    ssl_certificate /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/privkey.pem;
+    ssl_trusted_certificate /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/fullchain.pem;
+
+  location / {
+    proxy_pass http://{{ superset_listen_address }};
+    proxy_http_version 1.1;
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_read_timeout 900;
+  }
+}
diff --git a/ansible/roles/wiki/tasks/main.yml b/ansible/roles/wiki/tasks/main.yml
new file mode 100644
index 00000000..63ddff8d
--- /dev/null
+++ b/ansible/roles/wiki/tasks/main.yml
@@ -0,0 +1,83 @@
+---
+
+- name: Install apt-transport-https
+  apt:
+    name: apt-transport-https
+
+- name: Add NodeSource apt package signing key
+  apt_key:
+    url: "https://deb.nodesource.com/gpgkey/nodesource.gpg.key"
+
+- name: Add Mongodb apt package signing key
+  apt_key:
+    url: "https://www.mongodb.org/static/pgp/server-3.4.asc"
+
+- name: Add MongoDB apt repository
+  apt_repository:
+    repo: 'deb http://repo.mongodb.org/apt/ubuntu {{ ansible_distribution_release }}/mongodb-org/3.4 multiverse'
+
+- name: Add NodeSource apt repository
+  apt_repository:
+    repo: 'deb https://deb.nodesource.com/node_7.x {{ ansible_distribution_release }} main'
+
+- name: Install required apt packages
+  apt:
+    name: "{{ item }}"
+  with_items:
+    - git-core
+    - nodejs
+    - nginx
+    - mongodb-org
+    - bzip2
+
+- name: Create wiki user
+  user:
+    name: "{{ wiki_user }}"
+
+- name: Creating wiki dirs
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0755
+  with_items:
+    - "{{ wiki_user_path }}"
+
+- name: Remove default nginx virtual host config
+  file:
+    name: /etc/nginx/sites-enabled/default
+    state: absent
+  register: nginx
+
+- name: Add wiki nginx virtual host config
+  template:
+    src: wiki_nginx.conf.j2
+    dest: /etc/nginx/sites-enabled/wiki
+    owner: root
+    group: root
+    mode: 0644
+  register: nginx
+
+- name: Restart nginx
+  service:
+    name: nginx
+    state: restarted
+  when: nginx.changed
+
+- name: Install wiki.js package
+  npm:
+    name: "{{ item }}"
+    global: yes
+  with_items:
+    - wiki.js
+
+- name: Install node-wiki.service
+  template:
+    src: node-wiki.service.j2
+    dest: "{{ node_wiki_unit_path }}"
+
+- name: Start node-wiki.service
+  systemd:
+    name: node-wiki.service
+    state: started
+    enabled: yes
+    daemon_reload: yes
diff --git a/ansible/roles/wiki/templates/node-wiki.service.j2 b/ansible/roles/wiki/templates/node-wiki.service.j2
new file mode 100644
index 00000000..dbdbf5b6
--- /dev/null
+++ b/ansible/roles/wiki/templates/node-wiki.service.j2
@@ -0,0 +1,11 @@
+[Service]
+ExecStart=/usr/bin/node wiki start
+Restart=always
+StandardOutput=syslog
+StandardError=syslog
+User={{ wiki_user }}
+Group={{ wiki_user }}
+RestartSec=15s
+
+[Install]
+WantedBy=multi-user.target
diff --git a/ansible/roles/wiki/templates/wiki_nginx.conf.j2 b/ansible/roles/wiki/templates/wiki_nginx.conf.j2
new file mode 100644
index 00000000..eb82e26d
--- /dev/null
+++ b/ansible/roles/wiki/templates/wiki_nginx.conf.j2
@@ -0,0 +1,19 @@
+server {
+    server_name _;
+
+    listen 443 ssl default_server;
+    listen [::]:443 ssl default_server;
+
+    ssl_certificate /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/privkey.pem;
+    ssl_trusted_certificate /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/fullchain.pem;
+
+  location / {
+    proxy_pass http://127.0.0.1:3000;
+    proxy_http_version 1.1;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection 'upgrade';
+    proxy_set_header Host $host;
+    proxy_cache_bypass $http_upgrade;
+  }
+}
diff --git a/ansible/roles/wiki/vars/main.yml b/ansible/roles/wiki/vars/main.yml
new file mode 100644
index 00000000..b2a251d9
--- /dev/null
+++ b/ansible/roles/wiki/vars/main.yml
@@ -0,0 +1,3 @@
+wiki_user: wiki
+wiki_user_path: "/home/{{ wiki_user }}/data"
+node_wiki_unit_path: /etc/systemd/system/node-wiki.service
diff --git a/requirements.txt b/requirements.txt
index ddcefd85..30a4f943 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,2 @@
-ansible
+ansible==2.2.2
 boto

From ee18480a44e6bee1d93b9ecbad16378a52e71c90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Mon, 17 Jul 2017 18:16:53 +0200
Subject: [PATCH 2/8] Add role for deploying superset

---
 ansible/deploy-superset.yml                   |  7 +-
 ansible/group_vars/all/vars.yml               |  2 +
 ansible/host_vars/hkgsuperset.ooni.io/vault   | 11 +++
 ansible/roles/superset/defaults/main.yml      | 16 +++-
 ansible/roles/superset/tasks/main.yml         | 84 ++++++++++++++++---
 .../superset/templates/superset.service.j2    | 12 +++
 .../superset/templates/superset_config.py.j2  |  2 +-
 7 files changed, 112 insertions(+), 22 deletions(-)
 create mode 100644 ansible/host_vars/hkgsuperset.ooni.io/vault
 create mode 100644 ansible/roles/superset/templates/superset.service.j2

diff --git a/ansible/deploy-superset.yml b/ansible/deploy-superset.yml
index 8641f507..976ddef9 100644
--- a/ansible/deploy-superset.yml
+++ b/ansible/deploy-superset.yml
@@ -16,16 +16,11 @@
         - "{{ passwd.art }}"
         - "{{ passwd.darkk }}"
 
-- hosts: hkgsuperset.ooni.io
-  roles:
-    - docker_py
-
 - hosts: hkgsuperset.ooni.io
   gather_facts: false # already gathered
   vars:
-    ansible_python_interpreter: "/root/venv/bin/python2.7"
     letsencrypt_nginx: yes
     letsencrypt_domains: "hkgsuperset.ooni.io"
   roles:
-    #- letsencrypt
+    - letsencrypt
     - superset
diff --git a/ansible/group_vars/all/vars.yml b/ansible/group_vars/all/vars.yml
index 46ecb2c1..28024fb3 100644
--- a/ansible/group_vars/all/vars.yml
+++ b/ansible/group_vars/all/vars.yml
@@ -12,6 +12,8 @@ passwd:
   nodeexp:  {login: nodeexp,  group: nodeexp, id: 2200, comment: Node Exporter}
   prometh:  {login: prometh,  group: prometh, id: 2201, comment: Prometheus}
   alertman: {login: alertman, group: alertman, id: 2202, comment: Alert Manager}
+
+  superset: {login: superset, group: superset, id: 2400, comment: Superset User}
   # people
   agrabeli:
     login: agrabeli
diff --git a/ansible/host_vars/hkgsuperset.ooni.io/vault b/ansible/host_vars/hkgsuperset.ooni.io/vault
new file mode 100644
index 00000000..5f6192bb
--- /dev/null
+++ b/ansible/host_vars/hkgsuperset.ooni.io/vault
@@ -0,0 +1,11 @@
+$ANSIBLE_VAULT;1.1;AES256
+30646230393239663434653430343837336665633132396463326631373436343636353566346363
+3861393335663865623030386366393537333030346136320a303961356563303232313066396666
+31663639333932323764663736383865396339636237353866653036623965316633386530346138
+3436616364386534620a373062353438646561613963316365636364376633353632396566653338
+39343132393364333633316163393836393134616365346662333561373164383037373736643333
+37343862353131643161393932333164616363643330326137383630333938356539383433356661
+34386632636634396564346539306535663565393731616133323166626362616534366564303562
+34313539303632663031373634353930333135383162386235306236643639306639376636346461
+31626436623439323630383330656666313163346364306363613463363135333165366137376437
+3464306138646166376335613465326437376564643139663334
diff --git a/ansible/roles/superset/defaults/main.yml b/ansible/roles/superset/defaults/main.yml
index c4d39c64..87c30a4c 100644
--- a/ansible/roles/superset/defaults/main.yml
+++ b/ansible/roles/superset/defaults/main.yml
@@ -1,6 +1,16 @@
 superset_path: /srv/superset
 superset_config_path: "{{ superset_path }}/config"
-superset_port: "3001"
+superset_venv_path: "{{ superset_path }}/venv"
+superset_port: "8088"
 superset_listen_address: "127.0.0.1:{{ superset_port }}"
-superset_secret_key: "XXXCHANGEMEXXX"
-superset_docker_image: "amancevice/superset:0.18.5"
+superset_secret_key: "{{ vault_superset_secret_key }}"
+superset_version: "0.18.5"
+
+superset_admin_username: "admin"
+superset_admin_password: "{{ vault_superset_admin_password }}"
+
+superset_user: "{{ passwd.superset.login }}"
+superset_group: "{{ passwd.superset.group }}"
+superset_uid: "{{ passwd.superset.id }}"
+superset_user_comment: "{{ passwd.superset.comment }}"
+
diff --git a/ansible/roles/superset/tasks/main.yml b/ansible/roles/superset/tasks/main.yml
index 2cf68f43..6b66a1d0 100644
--- a/ansible/roles/superset/tasks/main.yml
+++ b/ansible/roles/superset/tasks/main.yml
@@ -1,9 +1,39 @@
 ---
+- name: create superset group
+  group:
+    name: "{{ superset_group }}"
+    state: present
+
+- name: create superset user
+  user:
+    name: "{{ superset_user }}"
+    group: "{{ superset_group }}"
+    uid: "{{ superset_uid }}"
+    createhome: yes # Superset needs a home
+    shell: /bin/bash # XXX maybe we can harden this
+    comment: "{{ superset_user_comment }}"
+    state: present
+
+- name: install apt requirements
+  apt:
+    name: "{{ item }}"
+  with_items:
+    - "build-essential"
+    - "libssl-dev"
+    - "libffi-dev"
+    - "python-dev"
+    - "python-pip"
+    - "libsasl2-dev"
+    - "libldap2-dev"
+    - "libpq-dev"
+
 - name: mkdir for config and data
   file:
     path: "{{ item }}"
     state: directory
     mode: "u=rwx,g=rx,o="
+    owner: "{{ superset_user }}"
+    group: "{{ superset_group }}"
   with_items:
     - "{{ superset_path }}"
     - "{{ superset_config_path }}"
@@ -13,17 +43,47 @@
     src: superset_config.py.j2
     dest: "{{ superset_config_path }}/superset_config.py"
 
-- name: superset service
-  docker_container:
-    image: "{{ superset_docker_image }}"
-    name: superset
-    ports:
-      - "{{ superset_port }}:3000"
-    volumes:
-      - "{{ superset_config_path }}:/home/superset/.superset"
-
-# XXX also do:
-# docker run --detach --name superset [options] amancevice/superset
-# docker exec -it superset superset-init
+- name: create superset virtualenv
+  pip:
+    name: "{{ item }}"
+    virtualenv: "{{ superset_venv_path }}"
+  with_items:
+    - "superset=={{ superset_version }}"
+    - "psycopg2"
+  become: true
+  become_user: "{{ superset_user }}"
+
+- name: create admin user
+  command: "{{ superset_venv_path}}/fabmanager create-admin --app superset --app superset --username {{ superset_admin_username}} --password {{ superset_admin_password }} --firstname admin --lastname admin --email admin@openobservatory.org"
+	environment:
+    PYTHONPATH: "{{ superset_config_path }}" # We pass the PYTHONPATH so that superset can get our custom config
+  become: true
+  become_user: "{{ superset_user }}"
+
+- name: upgrade db
+  command: "{{ superset_venv_path}}/superset db upgrade"
+	environment:
+    PYTHONPATH: "{{ superset_config_path }}"
+  become: true
+  become_user: "{{ superset_user }}"
+
+- name: init db
+  command: "{{ superset_venv_path}}/superset init"
+	environment:
+    PYTHONPATH: "{{ superset_config_path }}"
+  become: true
+  become_user: "{{ superset_user }}"
+
+- name: Install superset.service
+  template:
+    src: superset.service.j2
+    dest: "/etc/systemd/system/superset.service"
+
+- name: Start superset.service
+  systemd:
+    name: superset.service
+    state: started
+    enabled: yes
+    daemon_reload: yes
 
 - include: setup-nginx.yml
diff --git a/ansible/roles/superset/templates/superset.service.j2 b/ansible/roles/superset/templates/superset.service.j2
new file mode 100644
index 00000000..5a8fdab1
--- /dev/null
+++ b/ansible/roles/superset/templates/superset.service.j2
@@ -0,0 +1,12 @@
+[Service]
+Environment="PYTHONPATH={{ superset_config_path }}"
+Environment="PATH={{ superset_venv_path }}/bin:/usr/local/bin:/usr/bin:/bin"
+Type=simple
+User={{ superset_user }}
+Group={{ superset_user }}
+ExecStart={{ superset_venv_path }}/bin/superset runserver -p {{ superset_port }}
+StandardOutput=syslog
+StandardError=syslog
+
+[Install]
+WantedBy=default.target
diff --git a/ansible/roles/superset/templates/superset_config.py.j2 b/ansible/roles/superset/templates/superset_config.py.j2
index cf10538b..113f40da 100644
--- a/ansible/roles/superset/templates/superset_config.py.j2
+++ b/ansible/roles/superset/templates/superset_config.py.j2
@@ -18,7 +18,7 @@ SECRET_KEY = '{{ superset_secret_key }}'
 # superset metadata (slices, connections, tables, dashboards, ...).
 # Note that the connection information to connect to the datasources
 # you want to explore are managed directly in the web UI
-SQLALCHEMY_DATABASE_URI = 'sqlite:////path/to/superset.db'
+SQLALCHEMY_DATABASE_URI = 'sqlite:////{{ superset_config_path }}/superset.db'
 
 # Flask-WTF flag for CSRF
 WTF_CSRF_ENABLED = True

From 906627b1f093931d43a21a096c7baf22b39f9be9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Fri, 3 May 2019 18:16:38 +0200
Subject: [PATCH 3/8] Bump superset version

---
 ansible/roles/superset/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/roles/superset/defaults/main.yml b/ansible/roles/superset/defaults/main.yml
index 87c30a4c..5a854b00 100644
--- a/ansible/roles/superset/defaults/main.yml
+++ b/ansible/roles/superset/defaults/main.yml
@@ -4,7 +4,7 @@ superset_venv_path: "{{ superset_path }}/venv"
 superset_port: "8088"
 superset_listen_address: "127.0.0.1:{{ superset_port }}"
 superset_secret_key: "{{ vault_superset_secret_key }}"
-superset_version: "0.18.5"
+superset_version: "0.28.1"
 
 superset_admin_username: "admin"
 superset_admin_password: "{{ vault_superset_admin_password }}"

From 920cb63ee04b1e6721fd8c75182aa8f02d093572 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Mon, 20 May 2019 12:52:15 +0200
Subject: [PATCH 4/8] Add scripts for dw-superset

---
 ansible/deploy-superset-ec2.yml               | 19 ++++++++++++++++++
 ansible/host_vars/dw-superset.ooni.io/vault   | 12 +++++++++++
 ansible/inventory                             |  3 +++
 ansible/roles/superset/tasks/main.yml         | 20 +++++++++++--------
 .../superset/templates/superset_nginx.conf.j2 | 11 +++++++---
 5 files changed, 54 insertions(+), 11 deletions(-)
 create mode 100644 ansible/deploy-superset-ec2.yml
 create mode 100644 ansible/host_vars/dw-superset.ooni.io/vault

diff --git a/ansible/deploy-superset-ec2.yml b/ansible/deploy-superset-ec2.yml
new file mode 100644
index 00000000..d3e73f07
--- /dev/null
+++ b/ansible/deploy-superset-ec2.yml
@@ -0,0 +1,19 @@
+---
+- hosts: dw-superset.ooni.io
+  roles:
+    - role: adm
+      adm_passwd:
+        - "{{ passwd.art }}"
+        - "{{ passwd.sbs }}"
+        - "{{ passwd.sarath }}"
+
+- hosts: dw-superset.ooni.io
+  gather_facts: false # already gathered
+  roles:
+    - role: letsencrypt
+      letsencrypt_nginx: yes
+      letsencrypt_domains: ["dw-superset.ooni.io"]
+      tags: letsencrypt
+    - role: superset
+      letsencrypt_domains: ["dw-superset.ooni.io"]
+      tags: superset
diff --git a/ansible/host_vars/dw-superset.ooni.io/vault b/ansible/host_vars/dw-superset.ooni.io/vault
new file mode 100644
index 00000000..84bb37ee
--- /dev/null
+++ b/ansible/host_vars/dw-superset.ooni.io/vault
@@ -0,0 +1,12 @@
+$ANSIBLE_VAULT;1.1;AES256
+64333737663061316165313263373835633063396562346230383736393034373261363932633463
+3338623665383835313965363061343732383634636432330a353334333537383936373363623562
+32333739613539383632336439356662643264633732646136326633386631636237663536393063
+6461373862656533320a353938623939613131346330313363623933323731393265393233663633
+39386132393631643632356664323839353735353131336436396464393766333238636638373963
+32623730366339633863396163386430613730666632643837613461653365646166663137653634
+65633165656537333466376330333566633334396436313233316430653238626138323330626365
+61653037323262303431323465653236383633356139303339633835633966393233393566353065
+36663031623164343436653737383635316336383265313661333239363861663636313238313235
+35346562663332306331363362616330363633633962333263306534613361623734656136613535
+643664653030623463653436353063366136
diff --git a/ansible/inventory b/ansible/inventory
index 13d8686c..c6e108f9 100644
--- a/ansible/inventory
+++ b/ansible/inventory
@@ -83,6 +83,9 @@ deb.ooni.nu # NOT-GH, moritz?
 [bigv] # Bytemark @ London
 b.echo.th.ooni.io # NOT-GH, down, mirror1.reports.ooni.nu ooni-1.default.orgtech.uk0.bigv.io, OK (u: $login)
 
+[otf-aws]
+dw-superset.ooni.io
+
 ########################################################################
 # PSK (pre-shared key) tags
 
diff --git a/ansible/roles/superset/tasks/main.yml b/ansible/roles/superset/tasks/main.yml
index 6b66a1d0..6dca920a 100644
--- a/ansible/roles/superset/tasks/main.yml
+++ b/ansible/roles/superset/tasks/main.yml
@@ -21,8 +21,9 @@
     - "build-essential"
     - "libssl-dev"
     - "libffi-dev"
-    - "python-dev"
-    - "python-pip"
+    - "python3-dev"
+    - "python3-pip"
+    - "python3-venv"
     - "libsasl2-dev"
     - "libldap2-dev"
     - "libpq-dev"
@@ -46,30 +47,33 @@
 - name: create superset virtualenv
   pip:
     name: "{{ item }}"
+    virtualenv_command: "/usr/bin/python3 -m venv"
     virtualenv: "{{ superset_venv_path }}"
   with_items:
+    - "sqlalchemy==1.2.18" # fixes: https://github.com/apache/incubator-superset/issues/6977
+    - "pandas==0.23.4" # Workaround for: https://github.com/apache/incubator-superset/issues/6770
     - "superset=={{ superset_version }}"
     - "psycopg2"
   become: true
   become_user: "{{ superset_user }}"
 
 - name: create admin user
-  command: "{{ superset_venv_path}}/fabmanager create-admin --app superset --app superset --username {{ superset_admin_username}} --password {{ superset_admin_password }} --firstname admin --lastname admin --email admin@openobservatory.org"
-	environment:
+  command: "{{ superset_venv_path}}/bin/fabmanager create-admin --app superset --username {{ superset_admin_username}} --password {{ superset_admin_password }} --firstname admin --lastname admin --email admin@openobservatory.org"
+  environment:
     PYTHONPATH: "{{ superset_config_path }}" # We pass the PYTHONPATH so that superset can get our custom config
   become: true
   become_user: "{{ superset_user }}"
 
 - name: upgrade db
-  command: "{{ superset_venv_path}}/superset db upgrade"
-	environment:
+  command: "{{ superset_venv_path}}/bin/superset db upgrade"
+  environment:
     PYTHONPATH: "{{ superset_config_path }}"
   become: true
   become_user: "{{ superset_user }}"
 
 - name: init db
-  command: "{{ superset_venv_path}}/superset init"
-	environment:
+  command: "{{ superset_venv_path}}/bin/superset init"
+  environment:
     PYTHONPATH: "{{ superset_config_path }}"
   become: true
   become_user: "{{ superset_user }}"
diff --git a/ansible/roles/superset/templates/superset_nginx.conf.j2 b/ansible/roles/superset/templates/superset_nginx.conf.j2
index a9e12c29..5b43cc64 100644
--- a/ansible/roles/superset/templates/superset_nginx.conf.j2
+++ b/ansible/roles/superset/templates/superset_nginx.conf.j2
@@ -1,13 +1,16 @@
 # ansible-managed in ooni-sysadmin.git/ansible/roles/superset/templates/superset_nginx.conf.j2
+
+{% import 'common.j2' as c %}
 server {
     server_name _;
 
+    listen 80;
     listen 443 ssl default_server;
     listen [::]:443 ssl default_server;
 
-    ssl_certificate /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/privkey.pem;
-    ssl_trusted_certificate /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/fullchain.pem;
+    ssl_certificate /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/privkey.pem;
+    ssl_trusted_certificate /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/fullchain.pem;
 
   location / {
     proxy_pass http://{{ superset_listen_address }};
@@ -18,4 +21,6 @@ server {
     proxy_set_header X-Forwarded-Proto $scheme;
     proxy_read_timeout 900;
   }
+
+  {{ c.location_letsencrypt() }}
 }

From bed1674c51477948a0c6a4bd05bbe16ace251499 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Mon, 20 May 2019 16:33:32 +0200
Subject: [PATCH 5/8] Rename superset-ec to website-monitoring

---
 ansible/deploy-superset-ec2.yml               | 19 --------------
 ansible/deploy-website-monitoring.yml         | 25 +++++++++++++++++++
 ansible/host_vars/dw-superset.ooni.io/vault   | 12 ---------
 ansible/host_vars/dw.wsm.ooni.io/vault        | 15 +++++++++++
 ansible/inventory                             |  2 +-
 ansible/roles/jupyter/defaults/main.yml       |  1 +
 ansible/roles/jupyter/tasks/setup-jupyter.yml |  7 ++++++
 .../jupyter/templates/nginx-site-jupyter.j2   |  8 +++---
 ansible/roles/superset/defaults/main.yml      |  2 ++
 .../superset/templates/superset_nginx.conf.j2 |  2 +-
 10 files changed, 56 insertions(+), 37 deletions(-)
 delete mode 100644 ansible/deploy-superset-ec2.yml
 create mode 100644 ansible/deploy-website-monitoring.yml
 delete mode 100644 ansible/host_vars/dw-superset.ooni.io/vault
 create mode 100644 ansible/host_vars/dw.wsm.ooni.io/vault

diff --git a/ansible/deploy-superset-ec2.yml b/ansible/deploy-superset-ec2.yml
deleted file mode 100644
index d3e73f07..00000000
--- a/ansible/deploy-superset-ec2.yml
+++ /dev/null
@@ -1,19 +0,0 @@
----
-- hosts: dw-superset.ooni.io
-  roles:
-    - role: adm
-      adm_passwd:
-        - "{{ passwd.art }}"
-        - "{{ passwd.sbs }}"
-        - "{{ passwd.sarath }}"
-
-- hosts: dw-superset.ooni.io
-  gather_facts: false # already gathered
-  roles:
-    - role: letsencrypt
-      letsencrypt_nginx: yes
-      letsencrypt_domains: ["dw-superset.ooni.io"]
-      tags: letsencrypt
-    - role: superset
-      letsencrypt_domains: ["dw-superset.ooni.io"]
-      tags: superset
diff --git a/ansible/deploy-website-monitoring.yml b/ansible/deploy-website-monitoring.yml
new file mode 100644
index 00000000..a3fa9f24
--- /dev/null
+++ b/ansible/deploy-website-monitoring.yml
@@ -0,0 +1,25 @@
+---
+- hosts: dw.wsm.ooni.io
+  roles:
+    - role: adm
+      adm_passwd:
+        - "{{ passwd.art }}"
+        - "{{ passwd.sbs }}"
+        - "{{ passwd.sarath }}"
+
+- hosts: dw.wsm.ooni.io
+  gather_facts: false # already gathered
+  roles:
+    - role: letsencrypt
+      letsencrypt_nginx: yes
+      letsencrypt_domains: ["dw.wsm.ooni.io"]
+      tags: letsencrypt
+    - role: superset
+      letsencrypt_domains: ["dw.wsm.ooni.io"]
+      superset_web_path: "/superset"
+      tags: superset
+    - role: jupyter
+      jupyter_web_path: "/jupyter"
+      jupyter_password_hash: "{{ vault_jupyter_password_hash }}"
+      letsencrypt_domains: ["dw.wsm.ooni.io"]
+      tags: jupyter
diff --git a/ansible/host_vars/dw-superset.ooni.io/vault b/ansible/host_vars/dw-superset.ooni.io/vault
deleted file mode 100644
index 84bb37ee..00000000
--- a/ansible/host_vars/dw-superset.ooni.io/vault
+++ /dev/null
@@ -1,12 +0,0 @@
-$ANSIBLE_VAULT;1.1;AES256
-64333737663061316165313263373835633063396562346230383736393034373261363932633463
-3338623665383835313965363061343732383634636432330a353334333537383936373363623562
-32333739613539383632336439356662643264633732646136326633386631636237663536393063
-6461373862656533320a353938623939613131346330313363623933323731393265393233663633
-39386132393631643632356664323839353735353131336436396464393766333238636638373963
-32623730366339633863396163386430613730666632643837613461653365646166663137653634
-65633165656537333466376330333566633334396436313233316430653238626138323330626365
-61653037323262303431323465653236383633356139303339633835633966393233393566353065
-36663031623164343436653737383635316336383265313661333239363861663636313238313235
-35346562663332306331363362616330363633633962333263306534613361623734656136613535
-643664653030623463653436353063366136
diff --git a/ansible/host_vars/dw.wsm.ooni.io/vault b/ansible/host_vars/dw.wsm.ooni.io/vault
new file mode 100644
index 00000000..38775041
--- /dev/null
+++ b/ansible/host_vars/dw.wsm.ooni.io/vault
@@ -0,0 +1,15 @@
+$ANSIBLE_VAULT;1.1;AES256
+36303138663732643638303632386633393661616638616261663430356261363838663632656536
+3738313130383033376132333466316436613839323830660a353464313335306264353561313664
+36333562363362303939393365366436393763383066613062356661316234353231323733393164
+3839636639316230660a343134396162326561643538646432313166373566323566613234316339
+35353063353232336133383436386539323166646235326239363730343036636463626166346637
+34653839343233333938646362643861316363373763363035326337376662626363373062616535
+31336237616335333436633465383033626330313336393564636631306633643864653438626562
+35376363373162353432663230636565333735633762303133636462313539373136653763613263
+66613066303033626664373831636366653965326231306530643733313735383366666536613465
+65336563653932373030626462353062376432353365633236353963396162666163356437303864
+38303730356437663936366233623635383638313362386132383636646566636638666431643134
+39653630383466316231303464656165303538616531353739666662623634653235663034316261
+63663432343164663633313736393865306531343662393465376430663234353133356664393837
+6434323139636361613632653734323632663039653731336235
diff --git a/ansible/inventory b/ansible/inventory
index c6e108f9..ea330405 100644
--- a/ansible/inventory
+++ b/ansible/inventory
@@ -84,7 +84,7 @@ deb.ooni.nu # NOT-GH, moritz?
 b.echo.th.ooni.io # NOT-GH, down, mirror1.reports.ooni.nu ooni-1.default.orgtech.uk0.bigv.io, OK (u: $login)
 
 [otf-aws]
-dw-superset.ooni.io
+dw.wsm.ooni.io
 
 ########################################################################
 # PSK (pre-shared key) tags
diff --git a/ansible/roles/jupyter/defaults/main.yml b/ansible/roles/jupyter/defaults/main.yml
index c0af86fa..c3e92750 100644
--- a/ansible/roles/jupyter/defaults/main.yml
+++ b/ansible/roles/jupyter/defaults/main.yml
@@ -30,4 +30,5 @@ jupyter_user: "{{ passwd.jupyter.login }}"
 jupyter_group: "{{ passwd.jupyter.login }}"
 
 jupyter_port: 8080
+jupyter_web_path: "/"
 jupyter_password_hash: "{{ CHANGE_ME }}"
diff --git a/ansible/roles/jupyter/tasks/setup-jupyter.yml b/ansible/roles/jupyter/tasks/setup-jupyter.yml
index 2f958dd8..65625965 100644
--- a/ansible/roles/jupyter/tasks/setup-jupyter.yml
+++ b/ansible/roles/jupyter/tasks/setup-jupyter.yml
@@ -1,4 +1,11 @@
 ---
+- name: ensure juypter user exists
+  user:
+    name: "{{ jupyter_user }}"
+    shell: /bin/sh
+    groups: ["{{ jupyter_group }}"]
+    state: present
+
 - name: set permissions, owner and group
   file:
     path: "{{ jupyter_path }}"
diff --git a/ansible/roles/jupyter/templates/nginx-site-jupyter.j2 b/ansible/roles/jupyter/templates/nginx-site-jupyter.j2
index b943248e..154b9f9b 100644
--- a/ansible/roles/jupyter/templates/nginx-site-jupyter.j2
+++ b/ansible/roles/jupyter/templates/nginx-site-jupyter.j2
@@ -4,11 +4,11 @@ server {
     listen 443 ssl default_server;
     listen [::]:443 ssl default_server;
 
-    ssl_certificate /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/privkey.pem;
-    ssl_trusted_certificate /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/fullchain.pem;
+    ssl_certificate /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/privkey.pem;
+    ssl_trusted_certificate /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/fullchain.pem;
 
-  location / {
+  location {{ jupyter_web_path }} {
     proxy_pass http://127.0.0.1:{{ jupyter_port }};
     proxy_http_version 1.1;
     proxy_set_header Upgrade $http_upgrade;
diff --git a/ansible/roles/superset/defaults/main.yml b/ansible/roles/superset/defaults/main.yml
index 5a854b00..7eaef56d 100644
--- a/ansible/roles/superset/defaults/main.yml
+++ b/ansible/roles/superset/defaults/main.yml
@@ -9,6 +9,8 @@ superset_version: "0.28.1"
 superset_admin_username: "admin"
 superset_admin_password: "{{ vault_superset_admin_password }}"
 
+superset_web_path: "/"
+
 superset_user: "{{ passwd.superset.login }}"
 superset_group: "{{ passwd.superset.group }}"
 superset_uid: "{{ passwd.superset.id }}"
diff --git a/ansible/roles/superset/templates/superset_nginx.conf.j2 b/ansible/roles/superset/templates/superset_nginx.conf.j2
index 5b43cc64..89b475c0 100644
--- a/ansible/roles/superset/templates/superset_nginx.conf.j2
+++ b/ansible/roles/superset/templates/superset_nginx.conf.j2
@@ -12,7 +12,7 @@ server {
     ssl_certificate_key /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/privkey.pem;
     ssl_trusted_certificate /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/fullchain.pem;
 
-  location / {
+  location {{ superset_web_path }} {
     proxy_pass http://{{ superset_listen_address }};
     proxy_http_version 1.1;
     proxy_set_header Host $http_host;

From 6549dd24ce8c5ab48aee5faba56cd748423626ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Mon, 20 May 2019 17:25:21 +0200
Subject: [PATCH 6/8] Superset cannot go into a subdir

---
 ansible/roles/jupyter/tasks/setup-anaconda.yml          | 6 ------
 ansible/roles/jupyter/tasks/setup-jupyter.yml           | 1 -
 ansible/roles/superset/defaults/main.yml                | 2 --
 ansible/roles/superset/templates/superset_nginx.conf.j2 | 2 +-
 4 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/ansible/roles/jupyter/tasks/setup-anaconda.yml b/ansible/roles/jupyter/tasks/setup-anaconda.yml
index 36ca656d..61a36823 100644
--- a/ansible/roles/jupyter/tasks/setup-anaconda.yml
+++ b/ansible/roles/jupyter/tasks/setup-anaconda.yml
@@ -59,9 +59,3 @@
   command: '{{anaconda_link_dir}}/bin/pip install {{ item }}'
   with_items: "{{ pip_pkgs }}"
 
-- name: remove conda-curl since it conflicts with the system curl
-  become: yes
-  become_user: root
-  command: '{{anaconda_conda_bin}} remove -y curl'
-  args:
-    removes: '{{anaconda_link_dir}}/lib/libcurl.a'
diff --git a/ansible/roles/jupyter/tasks/setup-jupyter.yml b/ansible/roles/jupyter/tasks/setup-jupyter.yml
index 65625965..96e07170 100644
--- a/ansible/roles/jupyter/tasks/setup-jupyter.yml
+++ b/ansible/roles/jupyter/tasks/setup-jupyter.yml
@@ -3,7 +3,6 @@
   user:
     name: "{{ jupyter_user }}"
     shell: /bin/sh
-    groups: ["{{ jupyter_group }}"]
     state: present
 
 - name: set permissions, owner and group
diff --git a/ansible/roles/superset/defaults/main.yml b/ansible/roles/superset/defaults/main.yml
index 7eaef56d..5a854b00 100644
--- a/ansible/roles/superset/defaults/main.yml
+++ b/ansible/roles/superset/defaults/main.yml
@@ -9,8 +9,6 @@ superset_version: "0.28.1"
 superset_admin_username: "admin"
 superset_admin_password: "{{ vault_superset_admin_password }}"
 
-superset_web_path: "/"
-
 superset_user: "{{ passwd.superset.login }}"
 superset_group: "{{ passwd.superset.group }}"
 superset_uid: "{{ passwd.superset.id }}"
diff --git a/ansible/roles/superset/templates/superset_nginx.conf.j2 b/ansible/roles/superset/templates/superset_nginx.conf.j2
index 89b475c0..5b43cc64 100644
--- a/ansible/roles/superset/templates/superset_nginx.conf.j2
+++ b/ansible/roles/superset/templates/superset_nginx.conf.j2
@@ -12,7 +12,7 @@ server {
     ssl_certificate_key /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/privkey.pem;
     ssl_trusted_certificate /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/fullchain.pem;
 
-  location {{ superset_web_path }} {
+  location / {
     proxy_pass http://{{ superset_listen_address }};
     proxy_http_version 1.1;
     proxy_set_header Host $http_host;

From fa8c494a5069e47009b842b7a6cbee79e9a05f0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Mon, 20 May 2019 17:31:09 +0200
Subject: [PATCH 7/8] Drop default_server

---
 ansible/roles/jupyter/templates/nginx-site-jupyter.j2 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ansible/roles/jupyter/templates/nginx-site-jupyter.j2 b/ansible/roles/jupyter/templates/nginx-site-jupyter.j2
index 154b9f9b..289aa6cf 100644
--- a/ansible/roles/jupyter/templates/nginx-site-jupyter.j2
+++ b/ansible/roles/jupyter/templates/nginx-site-jupyter.j2
@@ -1,8 +1,8 @@
 server {
     server_name _;
 
-    listen 443 ssl default_server;
-    listen [::]:443 ssl default_server;
+    listen 443 ssl;
+    listen [::]:443 ssl;
 
     ssl_certificate /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/fullchain.pem;
     ssl_certificate_key /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/privkey.pem;

From e39c54cd2f836805a50bb7716645b16b2c5e565f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Tue, 21 May 2019 12:33:34 +0200
Subject: [PATCH 8/8] Add notes on pg deployment

---
 ansible/deploy-website-monitoring.yml  | 17 +++++++++++++++
 ansible/host_vars/dw.wsm.ooni.io/vault | 30 ++++++++++++++------------
 2 files changed, 33 insertions(+), 14 deletions(-)

diff --git a/ansible/deploy-website-monitoring.yml b/ansible/deploy-website-monitoring.yml
index a3fa9f24..1e945bc9 100644
--- a/ansible/deploy-website-monitoring.yml
+++ b/ansible/deploy-website-monitoring.yml
@@ -7,6 +7,23 @@
         - "{{ passwd.sbs }}"
         - "{{ passwd.sarath }}"
 
+# Manually:
+# apt install postgresql postgresql-contrib
+# sudo -u postgres psql
+# create extension postgres_fdw;
+#
+# CREATE SERVER metadb FOREIGN DATA WRAPPER
+#   postgres_fdw OPTIONS
+#   (dbname 'metadb', host 'ec2-3-92-83-13.compute-1.amazonaws.com', port '5432');
+#
+# CREATE USER MAPPING FOR postgres
+#         SERVER metadb
+#        OPTIONS (user 'postgres');
+#
+# Create the FOREIGN tables using: https://gist.github.com/hellais/6effbf5c728469e67482a60ae90aeb5c
+# Move the data directory of postgres into /srv/postgresql_data_dir:
+# https://www.dbrnd.com/2018/04/postgresql-move-main-data-directory-in-linux-ubuntu-16-4/
+
 - hosts: dw.wsm.ooni.io
   gather_facts: false # already gathered
   roles:
diff --git a/ansible/host_vars/dw.wsm.ooni.io/vault b/ansible/host_vars/dw.wsm.ooni.io/vault
index 38775041..47de6658 100644
--- a/ansible/host_vars/dw.wsm.ooni.io/vault
+++ b/ansible/host_vars/dw.wsm.ooni.io/vault
@@ -1,15 +1,17 @@
 $ANSIBLE_VAULT;1.1;AES256
-36303138663732643638303632386633393661616638616261663430356261363838663632656536
-3738313130383033376132333466316436613839323830660a353464313335306264353561313664
-36333562363362303939393365366436393763383066613062356661316234353231323733393164
-3839636639316230660a343134396162326561643538646432313166373566323566613234316339
-35353063353232336133383436386539323166646235326239363730343036636463626166346637
-34653839343233333938646362643861316363373763363035326337376662626363373062616535
-31336237616335333436633465383033626330313336393564636631306633643864653438626562
-35376363373162353432663230636565333735633762303133636462313539373136653763613263
-66613066303033626664373831636366653965326231306530643733313735383366666536613465
-65336563653932373030626462353062376432353365633236353963396162666163356437303864
-38303730356437663936366233623635383638313362386132383636646566636638666431643134
-39653630383466316231303464656165303538616531353739666662623634653235663034316261
-63663432343164663633313736393865306531343662393465376430663234353133356664393837
-6434323139636361613632653734323632663039653731336235
+39643036643739646337383131666434363236313062353639393230356538643430613464343138
+3238353430666438393430353466656333626139343632380a393966316566316134363934393237
+35303733666233303165646136383866316430313564373033643031363938326366643063313137
+6335376532303838300a373063383334353361356538316434616563303639353137353533666536
+38396166353935663730663736356166366464366462393863343565666165373932663338363865
+66373833316330393530633963336632613564333236343565323037363064666630636534363765
+35656661343537666435373864636665393039636238636432653961343231393134653239346434
+63623466306462626236306532396430333461323261633032333530623335666539373636663161
+32376262326338613031323939326163656564313965663033343934306165336637323065326130
+32383135623938343433613937353330393236323734333939643833313930653662336161623666
+66333765366131356466613437613735366564353633663731383563646334333062373032353039
+34323938373236323862633938396438376130323662333031613137346435646339656336336130
+33303166646138666139356461373139316337303531393666316132623461326234623133623930
+36356464613564616236363062376135346639656134663265356233633339653863333638353066
+34346663336536353132333239393565383537643865383265336232626362333136616437613431
+61336661633165323263