Migrate to AWS

README.md

@@ -13,6 +13,7 @@ To run the containers, generate `fullchain.pem` and `privkey.pem` (certificate
 and the private key) using Let's Encrypt and put these files into the
 `./config/` directory.
 
+### Development
 While in development, you can generate self-signed certificates:
 
 ```zsh
@@ -21,10 +22,35 @@ openssl req \
   -keyout ./config/privkey.pem \
   -out ./config/fullchain.pem
 ```
-
-Note, production deployment expects `privkey.pem` and `fullchain.pem` to be in
+### Production
+In a production environment, the deployment expects `privkey.pem` and `fullchain.pem` to be in
 the `/etc/letsencrypt/live/test.specifysystems.org-0001/privkey.pem` directory
 
+Before the Test Panel is running, you can install `nginx` locally 
+
+```bash
+sudo apt install nginx
+sudo service nginx start
+```
+
+The SSL certs must be for both `test.specifysystems.org` and `*.test.specifysystems.org`.
+We use [DigitalOcean to manage the DNS configuration](https://www.digitalocean.com/community/tutorials/how-to-create-let-s-encrypt-wildcard-certificates-with-certbot) (Dreamhost doesn't support wilcard certificates).
+
+Follow [these instructions](https://www.digitalocean.com/community/tutorials/how-to-create-let-s-encrypt-wildcard-certificates-with-certbot) to create the `~/certbot-creds.ini` file necessary to generate the SSL certificates.
+
+Once it is started, you can use certbot to generate the certificates:
+
+```bash
+sudo certbot certonly --dns-digitalocean --dns-digitalocean-credentials ~/certbot-creds.ini -d test.specifysystems.org,*.test.specifysystems.org
+```
+
+Now that this is done, make sure to remove the local `nginx` installation so it does not conflict with the one in Docker:
+
+```
+sudo systemctl stop nginx
+sudo apt-get purge nginx
+```
+
 ## Create a GitHub OAuth App
 
 In order to enable authentication though GitHub and usage of GitHub APIs, a
@@ -118,8 +144,32 @@ GITHUB_PERSONAL_TOKEN=<github_token>
 MYSQL_USERNAME=root
 MYSQL_PASSWORD=root
 MYSQL_HOST=mariadb
+
+REPORT_RUNNER_HOST=report-runner
+
+SECRET_KEY=<secret_key>
+
+ASSET_SERVER_KEY=<asset_secret_key>
+ASSET_SERVER_URL=asset-server
+```
+
+### Using RDS
+
+Since we do not use MariaDB in Docker when working with Amazon RDS, you need to set up the connection details in this same file:
+
+```ini
+NEXT_PUBLIC_GITHUB_CLIENT_ID=<client_id>
+GITHUB_CLIENT_SECRET=<client_secret>
+
+GITHUB_PERSONAL_TOKEN=<github_token>
+
+MYSQL_USERNAME=<master_username>
+MYSQL_PASSWORD=<master_password>
+MYSQL_HOST=something.something.us-east-1.rds.amazonaws.com
 ```
 
+### Configuration
+
 Replace `<client_id>` and `<client_secret>` with the actual values from the
 OAuth app configuration page on GitHub
 ([see more details](#create-a-github-oauth-app))
@@ -211,6 +261,10 @@ types.
 
 ## Watch for configuration file changes
 
+You can choose either approach. You do not need to use both `systemd` and `fswatch` solutions as they do the same thing.
+
+The `systemd` approach may be preferable as it can be configured to automatically start when the system boots up.
+
 ### Using systemd
 
 After user changes the configuration in the panel, the file
@@ -227,7 +281,7 @@ Description=Run docker-compose up for test panel.
 
 [Service]
 Type=oneshot
-WorkingDirectory=/home/specify/specify7-test-panel
+WorkingDirectory=/home/ubuntu/specify7-test-panel
 ExecStart=docker compose -f docker-compose.yml -f docker-compose.production.yml -f /var/lib/docker/volumes/specify7-test-panel_state/_data/docker-compose.yml up --remove-orphans -d
 ```
 

app/lib/dockerCompose.ts

@@ -14,13 +14,12 @@ export const createDockerConfig = (
   // This is used just to make docker Nginx container if config changed
   nginxConfigHash: number
 ): string => `
-version: '3.9'
 services:
 
 ${
   deployments.length === 0
     ? `
-  mariadb:
+  nginx:
     restart: unless-stopped`
     : ''
 }
@@ -43,8 +42,11 @@ ${deployments
       - DATABASE_HOST=${process.env.MYSQL_HOST}
       - MASTER_NAME=${process.env.MYSQL_USERNAME}
       - MASTER_PASSWORD=${process.env.MYSQL_PASSWORD}
-      - SECRET_KEY="change this to some unique random string"
-      - REPORT_RUNNER_HOST=report-runner
+      - SECRET_KEY=${process.env.SECRET_KEY}
+      - ASSET_SERVER_URL=${process.env.ASSET_SERVER_URL}
+      - ASSET_SERVER_KEY=${process.env.ASSET_SERVER_KEY}
+      - ASSET_SERVER_COLLECTION=${process.env.ASSET_SERVER_COLLECTION}
+      - REPORT_RUNNER_HOST=${process.env.REPORT_RUNNER_HOST}
       - REPORT_RUNNER_PORT=8080
       - CELERY_BROKER_URL=redis://redis/0
       - CELERY_RESULT_BACKEND=redis://redis/1
@@ -71,8 +73,11 @@ ${deployments
       - DATABASE_HOST=${process.env.MYSQL_HOST}
       - MASTER_NAME=${process.env.MYSQL_USERNAME}
       - MASTER_PASSWORD=${process.env.MYSQL_PASSWORD}
-      - SECRET_KEY="change this to some unique random string"
-      - REPORT_RUNNER_HOST=report-runner
+      - SECRET_KEY=${process.env.SECRET_KEY}
+      - ASSET_SERVER_URL=${process.env.ASSET_SERVER_URL}
+      - ASSET_SERVER_KEY=${process.env.ASSET_SERVER_KEY}
+      - ASSET_SERVER_COLLECTION=${process.env.ASSET_SERVER_COLLECTION}
+      - REPORT_RUNNER_HOST=${process.env.REPORT_RUNNER_HOST}
       - REPORT_RUNNER_PORT=8080
       - CELERY_BROKER_URL=redis://redis/0
       - CELERY_RESULT_BACKEND=redis://redis/1

app/lib/nginx.ts

@@ -13,8 +13,8 @@ server {
     server_name ${deployment.hostname}.${host};
     root /usr/share/nginx;
 
-    ssl_certificate /etc/letsencrypt/live/test.specifysystems.org/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/test.specifysystems.org/privkey.pem;
+    ssl_certificate /etc/letsencrypt/live/test.specifysystems.org-0001/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/test.specifysystems.org-0001/privkey.pem;
     ssl_session_cache shared:SSL:10m;
     ssl_session_timeout 5m;
     ssl_prefer_server_ciphers on;

docker-compose.development.yml

@@ -1,6 +1,5 @@
 # Development config
 
-version: '3.9'
 services:
   nginx:
     depends_on:
@@ -12,8 +11,8 @@ services:
     build:
       context: ./.
       target: dev-runner
-    depends_on:
-      - mariadb
+    # depends_on:
+    #   - mariadb
     networks:
       - database
       - nginx

docker-compose.production.yml

@@ -1,6 +1,5 @@
 # Production config
 
-version: '3.9'
 services:
   nginx:
     depends_on:
@@ -13,8 +12,8 @@ services:
     build:
       context: ./.
       target: runner
-    depends_on:
-      - mariadb
+    # depends_on:
+    #   - mariadb
     networks:
       - database
       - nginx

docker-compose.yml

@@ -1,18 +1,18 @@
-version: '3.9'
 services:
-  mariadb:
-    image: mariadb:11.4
-    networks:
-      - database
-    restart: unless-stopped
-    ports:
-      - '127.0.0.1:3306:3306'
-    environment:
-      - MYSQL_ROOT_PASSWORD=root
-    volumes:
-      - './config/mariadb.cnf:/etc/mysql/conf.d/mariadb.cnf'
-      - 'database:/var/lib/mysql'
-      - './seed-database/:/docker-entrypoint-initdb.d/:ro'
+  # If you run `mariadb` in Docker, uncomment these lines:
+  # mariadb:
+  #   image: mariadb:11.4
+  #   networks:
+  #     - database
+  #   restart: unless-stopped
+  #   ports:
+  #     - '127.0.0.1:3306:3306'
+  #   environment:
+  #     - MYSQL_ROOT_PASSWORD=root
+  #   volumes:
+  #     - './config/mariadb.cnf:/etc/mysql/conf.d/mariadb.cnf'
+  #     - 'database:/var/lib/mysql'
+  #     - './seed-database/:/docker-entrypoint-initdb.d/:ro'
 
   redis:
     image: redis
@@ -31,15 +31,13 @@ services:
       - 'nginx-conf:/etc/nginx/conf.d/servers:ro'
       - './config/panel.conf:/etc/nginx/conf.d/default.conf:ro'
 
-  report-runner:
-    image: specifyconsortium/report-runner
-    volumes:
-      - './report-fonts.jar:/var/lib/jetty/webapps/ROOT/WEB-INF/lib/report-fonts.jar:ro'
-    networks:
-      - database
+  # If you run `report-runner` in Docker, uncomment these lines:
+  # report-runner:
+  #   image: specifyconsortium/report-runner
+  #   volumes:
+  #     - './report-fonts.jar:/var/lib/jetty/webapps/ROOT/WEB-INF/lib/report-fonts.jar:ro'
 
 volumes:
-  database:
   state:
   nginx-conf:
 

state/docker-compose.yml

@@ -1,7 +1,6 @@
-# docker-compose doesn't like empty files, so this redundant mariadb extend
+# docker-compose doesn't like empty files, so this redundant nginx extend
 # is provided:
 
-version: '3.9'
 services:
-  mariadb:
+  nginx:
     restart: unless-stopped