Merge pull request #6 from MITLibraries/awsorg

Migrate Matomo to AWS Organization (Stage-Workloads)
* deploy Matomo container to Stage-Workloads ECR

Author: cabutlermit

.github/pull-request-template.md

@@ -0,0 +1,39 @@
+### What does this PR do?
+
+Describe the overall purpose of the PR changes. Doesn't need to be as specific as the
+individual commits.
+
+### Helpful background context
+
+Describe any additional context beyond what the PR accomplishes if it is likely to be
+useful to a reviewer.
+
+Delete this section if it isn't applicable to the PR.
+
+### How can a reviewer manually see the effects of these changes?
+
+Explain how to see the proposed changes in the application if possible.
+
+Delete this section if it isn't applicable to the PR.
+
+### Includes new or updated dependencies?
+
+YES | NO
+
+### What are the relevant tickets?
+
+Include links to Jira Software and/or Jira Service Management tickets here.
+
+### Developer
+
+- [ ] All new ENV is documented in README (or there is none)
+- [ ] Stakeholder approval has been confirmed (or is not needed)
+
+### Code Reviewer
+
+- [ ] The commit message is clear and follows our guidelines
+      (not just this pull request message)
+- [ ] There are appropriate tests covering any new functionality
+- [ ] The documentation has been updated or is unnecessary
+- [ ] The changes have been verified
+- [ ] New dependencies are appropriate or there were no changes

.github/workflows/dev_build.yml

@@ -0,0 +1,19 @@
+### This is the Terraform-generated dev-build.yml workflow for the matomo-dev app repository ###
+name: Dev Build and Deploy Fargate Container
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - '.github/**'
+
+jobs:
+  deploy:
+    name: Dev Deploy Fargate Container
+    uses: mitlibraries/.github/.github/workflows/fargate-shared-deploy-dev.yml@main
+    secrets: inherit
+    with:
+      AWS_REGION: "us-east-1"
+      GHA_ROLE: "matomo-gha-dev"
+      ECR: "matomo-dev"

.github/workflows/prod-promote.yml

@@ -0,0 +1,18 @@
+### This is the Terraform-generated prod-promote.yml workflow for the matomo-prod app repository ###
+name: Prod Promote Fargate Container
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+
+jobs:
+  deploy:
+    name: Prod Promote Fargate Container
+    uses: mitlibraries/.github/.github/workflows/fargate-shared-promote-prod.yml@main
+    secrets: inherit
+    with:
+      AWS_REGION: "us-east-1"
+      GHA_ROLE_STAGE: matomo-gha-stage
+      GHA_ROLE_PROD: matomo-gha-prod
+      ECR_STAGE: "matomo-stage"
+      ECR_PROD: "matomo-prod"

.github/workflows/stage-build.yml

@@ -0,0 +1,19 @@
+### This is the Terraform-generated stage-build.yml workflow for the matomo-stage app repository ###
+name: Stage Build and Deploy Fargate Container
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - '.github/**'
+
+jobs:
+  deploy:
+    name: Stage Deploy Fargate Container
+    uses: mitlibraries/.github/.github/workflows/fargate-shared-deploy-stage.yml@main
+    secrets: inherit
+    with:
+      AWS_REGION: "us-east-1"
+      GHA_ROLE: "matomo-gha-stage"
+      ECR: "matomo-stage"

.gitignore

@@ -1 +1,2 @@
 .env
+**/.DS_Store

Dockerfile

@@ -1,15 +1,12 @@
-FROM matomo:latest
-
-# Install system dependencies
-RUN apt-get update && apt-get upgrade -y 
-
-
-EXPOSE 80 443
-
-COPY config.ini.php /var/www/html/config/config.ini.php
-
-
+FROM matomo:3.13.6-apache
 
+# Add the EnvironmentVariables plugin
+COPY ./files/plugin-EnvironmentVariables-3.0.0/ /var/www/html/plugins/EnvironmentVariables
 
+# Preconfigure settings
+COPY ./files/config.ini.php /var/www/html/config/config.ini.php
 
+# Create mount point for EFS partition
+RUN mkdir -p /mnt/efs
 
+EXPOSE 80

Makefile

@@ -1,27 +1,37 @@
-.PHONY: help dist publish promote
+.PHONY: help dist-dev publish-dev dist-stage publish-stage
 SHELL=/bin/bash
-ECR_REGISTRY=672626379771.dkr.ecr.us-east-1.amazonaws.com
-DATETIME:=$(shell date -u +%Y%m%dT%H%M%SZ)
+### This is the Terraform-generated header for matomo-dev ###
+ECR_NAME_DEV:=matomo-dev
+ECR_URL_DEV:=222053980223.dkr.ecr.us-east-1.amazonaws.com/matomo-dev
+### End of Terraform-generated header ###
 
 help: ## Print this message
 	@awk 'BEGIN { FS = ":.*##"; print "Usage:  make <target>\n\nTargets:" } \
 		/^[-_[:alpha:]]+:.?*##/ { printf "  %-15s%s\n", $$1, $$2 }' $(MAKEFILE_LIST)
 
-build: ## Build docker image
-	docker build -t $(ECR_REGISTRY)/analytics-stage:latest \
-	-t analytics-stage:latest .
-	
-		
-dist: dist ## Build, tag and push
-	$$(aws ecr get-login --no-include-email --region us-east-1)
-	docker push $(ECR_REGISTRY)/analytics-stage:latest
-	aws ecs update-service --cluster analytics-stage-cluster --service analytics-stage --region us-east-1 --force-new-deployment
+### Terraform-generated Developer Deploy Commands for Dev environment ###
+dist-dev: ## Build docker container (intended for developer-based manual build)
+	docker build --platform linux/amd64 \
+	    -t $(ECR_URL_DEV):latest \
+		-t $(ECR_URL_DEV):`git describe --always` \
+		-t $(ECR_NAME_DEV):latest .
 
-publish: ## Promote the current staging build to production
-	$$(aws ecr get-login --no-include-email --region us-east-1)
-	docker pull $(ECR_REGISTRY)/analytics-stage:latest
-	docker tag $(ECR_REGISTRY)/analytics-stage:latest $(ECR_REGISTRY)/analytics-prod:latest
-	docker tag $(ECR_REGISTRY)/analytics-stage:latest $(ECR_REGISTRY)/analytics-prod:$(DATETIME)
-	docker push $(ECR_REGISTRY)/analytics-prod:latest
-	docker push $(ECR_REGISTRY)/analytics-prod:$(DATETIME)
-	aws ecs update-service --cluster analytics-prod-cluster --service analytics-prod --region us-east-1 --force-new-deployment
+publish-dev: dist-dev ## Build, tag and push (intended for developer-based manual publish)
+	docker login -u AWS -p $$(aws ecr get-login-password --region us-east-1) $(ECR_URL_DEV)
+	docker push $(ECR_URL_DEV):latest
+	docker push $(ECR_URL_DEV):`git describe --always`
+
+### Terraform-generated manual shortcuts for deploying to Stage ###
+### This requires that ECR_NAME_STAGE & ECR_URL_STAGE environment variables are set locally
+### by the developer and that the developer has authenticated to the correct AWS Account.
+### The values for the environment variables can be found in the stage_build.yml caller workflow.
+dist-stage: ## Only use in an emergency
+	docker build --platform linux/amd64 \
+	    -t $(ECR_URL_STAGE):latest \
+		-t $(ECR_URL_STAGE):`git describe --always` \
+		-t $(ECR_NAME_STAGE):latest .
+
+publish-stage: ## Only use in an emergency
+	docker login -u AWS -p $$(aws ecr get-login-password --region us-east-1) $(ECR_URL_STAGE)
+	docker push $(ECR_URL_STAGE):latest
+	docker push $(ECR_URL_STAGE):`git describe --always`

README.md

@@ -1,85 +1,60 @@
 # Search Analytics (Matomo)
 
-This repository supports MIT Libraries' implementation of [Matomo](https://matomo.org/),
-which we use to collect discovery analytics. Our implementation uses the Docker
-container provided by Matomo. We host the application in AWS Fargate.
+This repository supports MIT Libraries' implementation of [Matomo](https://matomo.org/), which we use to collect discovery analytics. Our implementation uses the Docker container provided by Matomo. We host the application in AWS Fargate.
 
-## Building
+## Dependencies
 
-Run `make build` to create a new container tagged as `analytics-stage:latest`.
+* Deploying this container to AWS requires that the AWS ECR Repository for the container has been created by the [mitlib-tf-workloads-ecr](https://github.com/MITLibraries/mitlib-tf-workloads-ecr) repository.
+* Building this container requires access to [Official Docker Matomo](https://hub.docker.com/_/matomo/).
 
-## Deploying
+## Build
 
-Run `make dist` to build, tag, and push a container to staging. To promote to
-production, run `make publish`.
+Run `make dist-dev` to create a new container tagged as `matomo:latest`.
 
-Please see the corresponding [Terraform module](https://github.com/MITLibraries/mitlib-terraform/tree/master/apps/analytics) for deployment config.
+## Deploy
 
-## URLs
-
-- Staging: analytics-stage.mitlib.net (MIT VPN access only)
-- Production: analytics-prod.mitlib.net
+Run `make publish-dev` to build, tag, and push a container to Dev1 for testing, or open a PR to `main`. Merge a PR to `main` to build, tag, and push the container to Stage-Workloads. After merging the PR to `main`, tag a release to promote to production.
 
 ## Implementation notes
 
-### PHP configuration file
+For detailed instructions on initial setup, migration, database upgrades, and application upgrades, see [docs/HOWTO.md](./docs/HOWTO.md).
+
+### Configuration Management
+
+"State" for Matomo is managed/stored in at most two locations:
 
-`config.ini.php` file contains some basic Matomo configuration. There are a
-couple of sections worth addressing:
+* the `config.ini.php` file
+* the MySQL database
 
-1. The `Plugins` section tells Matomo which plugins to enable on installation.
-   To disable a plugin, simply remove it from this section. Our preference is to
-   modify this section and not the `PluginsInstalled` section, as removing a
-   plugin from `PluginsInstalled` would remove it from the application completely.
-2. The `mail` section configures the SMTP server. Matomo uses mailers for
-   several core functions, including password resets and reporting. Some related
-   config values (e.g., `login_password_recovery_email_address`) are defined in
-   the `General` section.
+The `config.ini.php` file contains some core Matomo configuration. There are a couple of sections worth addressing:
 
-When deploying from scratch, we noticed that the initial Matomo setup in the GUI
-seems to overwrite our config. As a result, it may be necessary on rare occasion
-to spin up an EC2 instance to touch the config file directly. So far, we've
-only had to do this after the initial deploy, so it's unlikely to come up often.
+1. The `[Plugins]` section tells Matomo which plugins to enable on installation. To disable a plugin, simply remove it from this section. Our preference is to modify this section and not the `PluginsInstalled` section, as removing a plugin from `PluginsInstalled` would remove it from the application completely. *Sadly, with the container-based Matomo, it seems to only way to activate a plugin is to use the Matomo web UI*.
+1. The `[mail]` section configures the SMTP server. Matomo uses mailers for several core functions, including password resets and reporting. Some related config values (e.g., `login_password_recovery_email_address`) are defined in the `[General]` section.
+1. The `[General]` section handles other configurations. A few of these settings are covered by the EnvironmentVariables plugin, but the rest are managed in this code.
+
+By using the [EnvironmentVariables](https://plugins.matomo.org/EnvironmentVariables) plugin for Matomo, we generally don't need to store much in the `config.ini.php` file. Instead, the [mitlib-tf-workloads-matomo](https://github.com/MITLibraries/mitlib-tf-workloads-matomo) code defines the key environment variables for the ECS service, overriding anything in `config.ini.php` and `global.ini.php`.
+
+The database stores the rest of the state of Matomo: all the data itself along with the Superuser credentials and the other user accounts and credentials.
 
 ### Data anonymization
 
-We anonymize tracking data to protect our users' privacy. Specifically, we mask
-two bytes of visitors' IPs (e.g., 192.186.x.x), and we have disabled [Matomo's
-User ID plugin](https://matomo.org/docs/user-id/).
+We anonymize tracking data to protect our users' privacy. Specifically, we mask two bytes of visitors' IPs (e.g., 192.186.x.x), and we have disabled [Matomo's User ID plugin](https://matomo.org/docs/user-id/).
 
-Before promoting a new build to production, ensure that IPs are anonymized by
-visiting Administration -> Privacy -> Anonymize data in the GUI. (You shouldn't
-ever need to modify these settings, but it's important to verify as part of the
-deploy process.)
+Before promoting a new build to production, ensure that IPs are anonymized by visiting Administration -> Privacy -> Anonymize data in the GUI. (You shouldn't ever need to modify these settings, but it's important to verify as part of the deploy process.)
 
 ### Archiving reports
 
-Matomo calls the process by which it compiles raw log data into human-readable
-report 'archiving'. By default, archiving occurs on demand, whenever a Matomo
-user attempts to view a report in the GUI. Based on [Matomo's recommendation](https://matomo.org/docs/setup-auto-archiving/), we have configured a cron job to archive reports every
-hour.
-
-The cron job runs as a scheduled ECS task and is defined in the
-[Terraform config](https://github.com/MITLibraries/mitlib-terraform/tree/master/apps/analytics).
+Matomo calls the process by which it compiles raw log data into human-readable report 'archiving'. By default, archiving occurs on demand, whenever a Matomo user attempts to view a report in the GUI. Following [Matomo's recommendation](https://matomo.org/docs/setup-auto-archiving/) to schedule this, the mitlib-tf-workloads-matomo infrastructure repository creates an EventBridge rule/schedule for running the reporting archiving hourly.
 
 ### Authentication
 
-The superuser account can create and manage user
-accounts in the GUI (Administration -> System -> Users). After creating a new
-account, the superuser should notify the account holder. The account holder
-can then reset their password by clicking the 'Lost your password?' link on the
-Matomo sign-in page.
+The superuser account can create and manage user accounts in the GUI (Administration -> System -> Users). After creating a new account, the superuser should notify the account holder. The account holder can then reset their password by clicking the 'Lost your password?' link on the Matomo sign-in page.
 
-We generally assign the 'View' role to new users, unless they require a higher
-permissions level.
+We generally assign the 'View' role to new users, unless they require a higher permissions level.
 
-Matomo has built-in two-factor authentication, which we enforce for all accounts.
-When a user logs in to Matomo for the first time, they will be prompted to
-configure 2FA. At this time, we are recommending Duo as an authenticator.
+Matomo has built-in two-factor authentication, which we enforce for all accounts. When a user logs in to Matomo for the first time, they will be prompted to configure 2FA. At this time, we are recommending Duo as an authenticator.
 
 ## Additional documentation
 
-* [MIT Libraries dev docs](https://mitlibraries.github.io/guides/misc/matomo.html) - includes 
-information about setting up a website for tracking in Matomo.
-* [Matomo help center](https://matomo.org/help/) - offical Matomo docs. 
-Includes user guide, developer guide, FAQ, and community support forum.
+* [MIT Libraries dev docs](https://mitlibraries.github.io/guides/misc/matomo.html) - includes information about setting up a website for tracking in Matomo.
+* [Matomo help center](https://matomo.org/help/) - offical Matomo docs. Includes user guide, developer guide, FAQ, and community support forum.