Feature/website monitoring

ansible/deploy-superset.yml

@@ -1,5 +1,5 @@
 ---
-- import_playbook: ansible-version.yml
+- include: ansible-version.yml
 
 - hosts: hkgmetadb.infra.ooni.io
   gather_facts: false # no useful facts there :)
@@ -10,4 +10,27 @@
       db: metadb
       name: oosuperset
       password: '{{ hkgsuperset_postgres_password }}'
-...
+
+- hosts: hkgsuperset.ooni.io
+  become: false
+  remote_user: root
+  gather_facts: true
+  pre_tasks:
+    - name: bootstap python
+      raw: if [ ! -x /usr/bin/python ]; then apt-get update && apt-get -y install python-simplejson python-apt; fi
+      register: output
+      changed_when: output.stdout != ""
+  roles:
+    - role: adm
+      adm_passwd:
+        - "{{ passwd.art }}"
+        - "{{ passwd.darkk }}"
+
+- hosts: hkgsuperset.ooni.io
+  gather_facts: false # already gathered
+  vars:
+    letsencrypt_nginx: yes
+    letsencrypt_domains: "hkgsuperset.ooni.io"
+  roles:
+    - letsencrypt
+    - superset

ansible/deploy-website-monitoring.yml

@@ -0,0 +1,42 @@
+---
+- hosts: dw.wsm.ooni.io
+  roles:
+    - role: adm
+      adm_passwd:
+        - "{{ passwd.art }}"
+        - "{{ passwd.sbs }}"
+        - "{{ passwd.sarath }}"
+
+# Manually:
+# apt install postgresql postgresql-contrib
+# sudo -u postgres psql
+# create extension postgres_fdw;
+#
+# CREATE SERVER metadb FOREIGN DATA WRAPPER
+#   postgres_fdw OPTIONS
+#   (dbname 'metadb', host 'ec2-3-92-83-13.compute-1.amazonaws.com', port '5432');
+#
+# CREATE USER MAPPING FOR postgres
+#         SERVER metadb
+#        OPTIONS (user 'postgres');
+#
+# Create the FOREIGN tables using: https://gist.github.com/hellais/6effbf5c728469e67482a60ae90aeb5c
+# Move the data directory of postgres into /srv/postgresql_data_dir:
+# https://www.dbrnd.com/2018/04/postgresql-move-main-data-directory-in-linux-ubuntu-16-4/
+
+- hosts: dw.wsm.ooni.io
+  gather_facts: false # already gathered
+  roles:
+    - role: letsencrypt
+      letsencrypt_nginx: yes
+      letsencrypt_domains: ["dw.wsm.ooni.io"]
+      tags: letsencrypt
+    - role: superset
+      letsencrypt_domains: ["dw.wsm.ooni.io"]
+      superset_web_path: "/superset"
+      tags: superset
+    - role: jupyter
+      jupyter_web_path: "/jupyter"
+      jupyter_password_hash: "{{ vault_jupyter_password_hash }}"
+      letsencrypt_domains: ["dw.wsm.ooni.io"]
+      tags: jupyter

ansible/group_vars/all/vars.yml

@@ -44,6 +44,7 @@ passwd:
   piwik:    {login: piwik,    id: 2300, comment: Piwik user}
 
   grav:     {login: grav,     id: 2400, comment: Grav user}
+  superset: {login: superset, group: superset, id: 2401, comment: Superset User}
 
   prtreg:   {login: prtreg,   id: 4800, comment: Orchestration Registry}
   prtevent: {login: prtevent, id: 4801, comment: Orchestration Events}

ansible/host_vars/dw.wsm.ooni.io/vault

@@ -0,0 +1,17 @@
+$ANSIBLE_VAULT;1.1;AES256
+39643036643739646337383131666434363236313062353639393230356538643430613464343138
+3238353430666438393430353466656333626139343632380a393966316566316134363934393237
+35303733666233303165646136383866316430313564373033643031363938326366643063313137
+6335376532303838300a373063383334353361356538316434616563303639353137353533666536
+38396166353935663730663736356166366464366462393863343565666165373932663338363865
+66373833316330393530633963336632613564333236343565323037363064666630636534363765
+35656661343537666435373864636665393039636238636432653961343231393134653239346434
+63623466306462626236306532396430333461323261633032333530623335666539373636663161
+32376262326338613031323939326163656564313965663033343934306165336637323065326130
+32383135623938343433613937353330393236323734333939643833313930653662336161623666
+66333765366131356466613437613735366564353633663731383563646334333062373032353039
+34323938373236323862633938396438376130323662333031613137346435646339656336336130
+33303166646138666139356461373139316337303531393666316132623461326234623133623930
+36356464613564616236363062376135346639656134663265356233633339653863333638353066
+34346663336536353132333239393565383537643865383265336232626362333136616437613431
+61336661633165323263

ansible/host_vars/hkgsuperset.ooni.io/vault

@@ -0,0 +1,11 @@
+$ANSIBLE_VAULT;1.1;AES256
+30646230393239663434653430343837336665633132396463326631373436343636353566346363
+3861393335663865623030386366393537333030346136320a303961356563303232313066396666
+31663639333932323764663736383865396339636237353866653036623965316633386530346138
+3436616364386534620a373062353438646561613963316365636364376633353632396566653338
+39343132393364333633316163393836393134616365346662333561373164383037373736643333
+37343862353131643161393932333164616363643330326137383630333938356539383433356661
+34386632636634396564346539306535663565393731616133323166626362616534366564303562
+34313539303632663031373634353930333135383162386235306236643639306639376636346461
+31626436623439323630383330656666313163346364306363613463363135333165366137376437
+3464306138646166376335613465326437376564643139663334

ansible/inventory

@@ -83,6 +83,9 @@ deb.ooni.nu # NOT-GH, moritz?
 [bigv] # Bytemark @ London
 b.echo.th.ooni.io # NOT-GH, down, mirror1.reports.ooni.nu ooni-1.default.orgtech.uk0.bigv.io, OK (u: $login)
 
+[otf-aws]
+dw.wsm.ooni.io
+
 ########################################################################
 # PSK (pre-shared key) tags
 

ansible/roles/jupyter/defaults/main.yml

@@ -30,4 +30,5 @@ jupyter_user: "{{ passwd.jupyter.login }}"
 jupyter_group: "{{ passwd.jupyter.login }}"
 
 jupyter_port: 8080
+jupyter_web_path: "/"
 jupyter_password_hash: "{{ CHANGE_ME }}"

ansible/roles/jupyter/tasks/setup-anaconda.yml

@@ -59,9 +59,3 @@
   command: '{{anaconda_link_dir}}/bin/pip install {{ item }}'
   with_items: "{{ pip_pkgs }}"
 
-- name: remove conda-curl since it conflicts with the system curl
-  become: yes
-  become_user: root
-  command: '{{anaconda_conda_bin}} remove -y curl'
-  args:
-    removes: '{{anaconda_link_dir}}/lib/libcurl.a'

ansible/roles/jupyter/tasks/setup-jupyter.yml

@@ -1,4 +1,10 @@
 ---
+- name: ensure juypter user exists
+  user:
+    name: "{{ jupyter_user }}"
+    shell: /bin/sh
+    state: present
+
 - name: set permissions, owner and group
   file:
     path: "{{ jupyter_path }}"

ansible/roles/jupyter/templates/nginx-site-jupyter.j2

@@ -1,14 +1,14 @@
 server {
     server_name _;
 
-    listen 443 ssl default_server;
-    listen [::]:443 ssl default_server;
+    listen 443 ssl;
+    listen [::]:443 ssl;
 
-    ssl_certificate /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/privkey.pem;
-    ssl_trusted_certificate /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/fullchain.pem;
+    ssl_certificate /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/privkey.pem;
+    ssl_trusted_certificate /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/fullchain.pem;
 
-  location / {
+  location {{ jupyter_web_path }} {
     proxy_pass http://127.0.0.1:{{ jupyter_port }};
     proxy_http_version 1.1;
     proxy_set_header Upgrade $http_upgrade;

ansible/roles/superset/defaults/main.yml

@@ -0,0 +1,16 @@
+superset_path: /srv/superset
+superset_config_path: "{{ superset_path }}/config"
+superset_venv_path: "{{ superset_path }}/venv"
+superset_port: "8088"
+superset_listen_address: "127.0.0.1:{{ superset_port }}"
+superset_secret_key: "{{ vault_superset_secret_key }}"
+superset_version: "0.28.1"
+
+superset_admin_username: "admin"
+superset_admin_password: "{{ vault_superset_admin_password }}"
+
+superset_user: "{{ passwd.superset.login }}"
+superset_group: "{{ passwd.superset.group }}"
+superset_uid: "{{ passwd.superset.id }}"
+superset_user_comment: "{{ passwd.superset.comment }}"
+

ansible/roles/superset/tasks/main.yml

@@ -0,0 +1,93 @@
+---
+- name: create superset group
+  group:
+    name: "{{ superset_group }}"
+    state: present
+
+- name: create superset user
+  user:
+    name: "{{ superset_user }}"
+    group: "{{ superset_group }}"
+    uid: "{{ superset_uid }}"
+    createhome: yes # Superset needs a home
+    shell: /bin/bash # XXX maybe we can harden this
+    comment: "{{ superset_user_comment }}"
+    state: present
+
+- name: install apt requirements
+  apt:
+    name: "{{ item }}"
+  with_items:
+    - "build-essential"
+    - "libssl-dev"
+    - "libffi-dev"
+    - "python3-dev"
+    - "python3-pip"
+    - "python3-venv"
+    - "libsasl2-dev"
+    - "libldap2-dev"
+    - "libpq-dev"
+
+- name: mkdir for config and data
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: "u=rwx,g=rx,o="
+    owner: "{{ superset_user }}"
+    group: "{{ superset_group }}"
+  with_items:
+    - "{{ superset_path }}"
+    - "{{ superset_config_path }}"
+
+- name: configure superset
+  template:
+    src: superset_config.py.j2
+    dest: "{{ superset_config_path }}/superset_config.py"
+
+- name: create superset virtualenv
+  pip:
+    name: "{{ item }}"
+    virtualenv_command: "/usr/bin/python3 -m venv"
+    virtualenv: "{{ superset_venv_path }}"
+  with_items:
+    - "sqlalchemy==1.2.18" # fixes: https://github.com/apache/incubator-superset/issues/6977
+    - "pandas==0.23.4" # Workaround for: https://github.com/apache/incubator-superset/issues/6770
+    - "superset=={{ superset_version }}"
+    - "psycopg2"
+  become: true
+  become_user: "{{ superset_user }}"
+
+- name: create admin user
+  command: "{{ superset_venv_path}}/bin/fabmanager create-admin --app superset --username {{ superset_admin_username}} --password {{ superset_admin_password }} --firstname admin --lastname admin --email [email protected]"
+  environment:
+    PYTHONPATH: "{{ superset_config_path }}" # We pass the PYTHONPATH so that superset can get our custom config
+  become: true
+  become_user: "{{ superset_user }}"
+
+- name: upgrade db
+  command: "{{ superset_venv_path}}/bin/superset db upgrade"
+  environment:
+    PYTHONPATH: "{{ superset_config_path }}"
+  become: true
+  become_user: "{{ superset_user }}"
+
+- name: init db
+  command: "{{ superset_venv_path}}/bin/superset init"
+  environment:
+    PYTHONPATH: "{{ superset_config_path }}"
+  become: true
+  become_user: "{{ superset_user }}"
+
+- name: Install superset.service
+  template:
+    src: superset.service.j2
+    dest: "/etc/systemd/system/superset.service"
+
+- name: Start superset.service
+  systemd:
+    name: superset.service
+    state: started
+    enabled: yes
+    daemon_reload: yes
+
+- include: setup-nginx.yml

ansible/roles/superset/tasks/setup-nginx.yml

@@ -0,0 +1,31 @@
+---
+- name: Install nginx
+  environment:
+    RUNLEVEL: 1 # To avoid nginx being launched right after it's installed
+  apt:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - "nginx"
+    - "python-passlib"
+
+- name: Remove default nginx virtual host
+  file:
+    name: /etc/nginx/sites-enabled/default
+    state: absent
+  register: nginx
+
+- name: Add nginx virtual host config
+  template:
+    src: superset_nginx.conf.j2
+    dest: /etc/nginx/sites-enabled/superset
+    owner: root
+    group: root
+    mode: 0644
+  register: nginx
+
+- name: Restart nginx
+  service:
+    name: nginx
+    state: restarted
+  when: nginx.changed

ansible/roles/superset/templates/superset.service.j2

@@ -0,0 +1,12 @@
+[Service]
+Environment="PYTHONPATH={{ superset_config_path }}"
+Environment="PATH={{ superset_venv_path }}/bin:/usr/local/bin:/usr/bin:/bin"
+Type=simple
+User={{ superset_user }}
+Group={{ superset_user }}
+ExecStart={{ superset_venv_path }}/bin/superset runserver -p {{ superset_port }}
+StandardOutput=syslog
+StandardError=syslog
+
+[Install]
+WantedBy=default.target

ansible/roles/superset/templates/superset_config.py.j2

@@ -0,0 +1,28 @@
+#---------------------------------------------------------
+# Superset specific config
+#---------------------------------------------------------
+ROW_LIMIT = 5000
+SUPERSET_WORKERS = 4
+
+SUPERSET_WEBSERVER_PORT = {{ superset_port }}
+#---------------------------------------------------------
+
+#---------------------------------------------------------
+# Flask App Builder configuration
+#---------------------------------------------------------
+# Your App secret key
+SECRET_KEY = '{{ superset_secret_key }}'
+
+# The SQLAlchemy connection string to your database backend
+# This connection defines the path to the database that stores your
+# superset metadata (slices, connections, tables, dashboards, ...).
+# Note that the connection information to connect to the datasources
+# you want to explore are managed directly in the web UI
+SQLALCHEMY_DATABASE_URI = 'sqlite:////{{ superset_config_path }}/superset.db'
+
+# Flask-WTF flag for CSRF
+WTF_CSRF_ENABLED = True
+
+# Set this API key to enable Mapbox visualizations
+MAPBOX_API_KEY = ''
+

ansible/roles/superset/templates/superset_nginx.conf.j2

@@ -0,0 +1,26 @@
+# ansible-managed in ooni-sysadmin.git/ansible/roles/superset/templates/superset_nginx.conf.j2
+
+{% import 'common.j2' as c %}
+server {
+    server_name _;
+
+    listen 80;
+    listen 443 ssl default_server;
+    listen [::]:443 ssl default_server;
+
+    ssl_certificate /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/privkey.pem;
+    ssl_trusted_certificate /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/fullchain.pem;
+
+  location / {
+    proxy_pass http://{{ superset_listen_address }};
+    proxy_http_version 1.1;
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_read_timeout 900;
+  }
+
+  {{ c.location_letsencrypt() }}
+}