Merge pull request #26 from MITLibraries/IN-1301

Updates for New Use of EFS

Author: cabutlermit

.github/CODEOWNERS

@@ -13,7 +13,11 @@
 # Teams can be specified as code owners as well. Teams should be identified in
 # the format @org/team-name. Teams must have explicit write access to the 
 # repository.
-* @mitlibraries/infraeng-terraform-reviewers
+* @mitlibraries/infraeng
+
+# We assign Dhruti as an owner of the Dockerfile, since she should approve any
+# core changes to Matomo (especially version upgrades)
+/Dockerfile @dhrutibc
 
 # We set the senior engineer in the team as the owner of the CODEOWNERS file as
 # a layer of protection for unauthorized changes.

Dockerfile

@@ -14,24 +14,31 @@ COPY ./files/plugin-HeatmapSessionRecording-5.2.3/ /var/www/html/plugins/Heatmap
 # Add the UsersFlow plugin
 COPY ./files/plugin-UsersFlow-5.0.5/ /var/www/html/plugins/UsersFlow
 
-# Our custom configuration settings
-COPY ./files/config.ini.php /var/www/html/config/config.ini.php
+# Our custom configuration settings. We put it in /usr/src because the 
+# entrypoint.sh builds the /var/www/html folder from the /usr/src/matomo 
+# folder. This ensures that the config file from our updated container is the 
+# one that is pushed to the persistent EFS storage.
+COPY ./files/config.ini.php /usr/src/matomo/config/config.ini.php
 
 # The HeatmapSessionRecording and UsersFlow update the matomo.js and piwik.js
 # files when they are activated. Those updates have been captured and we
-# pre-load them here (we put them in /usr/src because the entrypoint.sh is what
-# actually builds the /var/www/html folder as a Docker VOLUME)
+# pre-load them here (we put them in /usr/src because the entrypoint.sh builds 
+# the /var/www/html folder from the /usr/src/matomo folder.
 COPY --chmod=0644 ./files/post_premium_plugins_matomo.js /usr/src/matomo/matomo.js
 COPY --chmod=0644 ./files/post_premium_plugins_piwik.js /usr/src/matomo/piwik.js
 
 # Address "public" directories problem
-COPY --chmod=0644 --chown=root:root ./files/lang-htaccess /var/www/html/lang/.htaccess
-COPY --chmod=0644 --chown=root:root ./files/config-htaccess /var/www/html/config/.htaccess
-COPY --chmod=0644 --chown=root:root ./files/tmp-htaccess /var/www/html/tmp/.htaccess
+COPY --chmod=0644 --chown=root:root ./files/lang-htaccess /usr/src/matomo/lang/.htaccess
+COPY --chmod=0644 --chown=root:root ./files/config-htaccess /usr/src/matomo/config/.htaccess
+COPY --chmod=0644 --chown=root:root ./files/tmp-htaccess /usr/src/matomo/tmp/.htaccess
 
 # Create mount point for EFS partition
 RUN mkdir -p /mnt/efs
 
+# Fix the installation line in the entrypoint script. We have to modify this 
+# because we enforce the uid/gid via the AWS EFS Access Point configuration.
+RUN sed -i 's|tar xf -|tar xf - --no-same-owner|' /entrypoint.sh
+
 # Move in the "backup persistent files" script
 COPY --chmod=0755 ./files/backup-data.sh /usr/local/bin/backup-data.sh
 

Makefile

@@ -14,7 +14,8 @@ help: ## Print this message
 
 ### Terraform-generated Developer Deploy Commands for Dev environment ###
 dist-dev: ## Build docker container (intended for developer-based manual build)
-	docker build --platform linux/amd64 \
+	docker build \
+		--platform linux/amd64 \
 	    -t $(ECR_URL_DEV):latest \
 		-t $(ECR_URL_DEV):`git describe --always` \
 		-t $(ECR_NAME_DEV):latest .
@@ -35,7 +36,8 @@ publish-dev: dist-dev ## Build, tag and push (intended for developer-based manua
 ###   authenticated to the correct AWS Account. The values for the environment  ###
 ###   variables can be found in the stage_build.yml caller workflow.            ###
 dist-stage: ## Only use in an emergency
-	docker build --platform linux/amd64 \
+	docker build \
+		--platform linux/amd64 \
 	    -t $(ECR_URL_STAGE):latest \
 		-t $(ECR_URL_STAGE):`git describe --always` \
 		-t $(ECR_NAME_STAGE):latest .

README.md

@@ -20,7 +20,7 @@ Run `make publish-dev` to build, tag, and push a container to Dev1 for testing,
 
 Merge a PR to `main` to build, tag, and push the container to Stage-Workloads. After merging the PR to `main`, tag a release on the `main` branch to promote to production. GitHub Actions in this repo will take care of the build, tag, push to Stage and the copy from Stage to Production.
 
-**Important Note**: There is no automation in GitHub to automatically deploy the new container after it is push to the ECR repository in AWS. At this time, the only method to deploy the updated container is to force a new deployment of the Matomo service via the AWS Console.
+**Important Note**: There is no automation in GitHub to automatically deploy the new container after it is push to the ECR repository in AWS. At this time, the only method to deploy the updated container is to force a new deployment of the Matomo service via the AWS Console or the CLI.
 
 ## Implementation notes
 
@@ -39,7 +39,7 @@ The `config.ini.php` file contains some core Matomo configuration. There are a c
 
 We use the [EnvironmentVariables](https://plugins.matomo.org/EnvironmentVariables) plugin that allows us to set configuration values for Matomo via environment variables via the [mitlib-tf-workloads-matomo](https://github.com/mitlibraries/mitlib-tf-workloads-matomo) infrastructure repo.
 
-The current practice is to set the following core configuration information in `config.ini.php` via environment variables. 
+The current practice is to set the following core configuration information in `config.ini.php` via environment variables.
 
 * Database connection information
 * SSL/TLS configuration
@@ -75,7 +75,7 @@ Before promoting a new build to production, ensure that IPs are anonymized by vi
 
 ### Archiving reports
 
-Matomo calls the process by which it compiles raw log data into human-readable report 'archiving'. By default, archiving occurs on demand, whenever a Matomo user attempts to view a report in the GUI. Following [Matomo's recommendation](https://matomo.org/docs/setup-auto-archiving/) to schedule this, the mitlib-tf-workloads-matomo infrastructure repository creates an EventBridge rule/schedule for running the reporting archiving hourly.
+Matomo calls the process by which it compiles raw log data into human-readable report 'archiving'. By default, archiving occurs on demand, whenever a Matomo user attempts to view a report in the GUI. Following [Matomo's recommendation](https://matomo.org/docs/setup-auto-archiving/) to schedule this, the [mitlib-tf-workloads-matomo](https://github.com/MITLibraries/mitlib-tf-workloads-matomo) infrastructure repository creates an EventBridge rule/schedule for running the reporting archiving hourly (only daily in Dev1).
 
 ### Authentication
 
@@ -97,7 +97,7 @@ See the official [Recover from lost 2FA](https://matomo.org/faq/how-to/faq_27248
 
 * Owner: See [CODEOWNERS](./.github/CODEOWNERS)
 * Team: See [CODEOWNERS](./.github/CODEOWNERS)
-* Last Maintenance: 2025-02
+* Last Maintenance: 2025-05
 * External Documentation:
   * [MIT Libraries dev docs](https://mitlibraries.github.io/guides/misc/matomo.html) - includes information about setting up a website for tracking in Matomo.
   * [Matomo help center](https://matomo.org/help/) - offical Matomo docs. Includes user guide, developer guide, FAQ, and community support forum.

docs/HowTos/HOWTO-matomo-upgrade.md

@@ -10,9 +10,9 @@ The database engine is managed by [mitlib-tf-workloads-matomo](https://github.co
 
 These instructions assume you are working in the **dev** environment.  Change to the appropriate `<env>` tag if you are working in a different environment.
 
-1. Ensure that a backup of the current `config/config.ini.php` exists in the EFS mount.
+1. Ensure that an out-of-band backup of the current `config/config.ini.php` exists
    * SSH (via AWSCLI + Session Manager) to the container(see the [Troubleshooting](./HOWTO-miscellaneous.md) section for the AWS CLI connection command).
-   * Run the **backup-data.sh** script `/usr/local/bin/backup-data.sh`
+   * Run `cp /var/www/html/config/* /mnt/efs/backup/config`
 1. Make any necessary changes to the repo.
    * For version upgrades, change line 1 in **Dockerfile** to the new version.
    * Verify plugin versions for compatibility with new version of Matomo. See **Update Matomo Plugins** below for more details.
@@ -21,33 +21,34 @@ These instructions assume you are working in the **dev** environment.  Change to
    * Run `make publish-dev` to push the new container to ECR and tag it as 'latest'.
 1. Deploy updated container for ECS service.
    * Via the AWS Console:
-     * Go to ECS
-     * Click on **matomo-ecs-dev-cluster**
-     * Click on the checkbox in **Services** next to `matomo-ecs-dev-service`
-     * Click  `Update`
-     * On the next page, expand **Deployment options** and choose `Force new deployment`
-     * Click `Update`
+   * Go to ECS
+      * Click on **matomo-ecs-dev-cluster**
+      * Click on the checkbox in **Services** next to `matomo-ecs-dev-service`
+      * Click  `Update`
+      * On the next page, expand **Deployment options** and choose `Force new deployment`
+      * Click `Update`
    * Via AWS CLI
-     * `aws ecs update-service --cluster matomo-ecs-dev-cluster --service matomo-ecs-dev-service --force-new-deployment`
-   *Either method takes a few minutes to complete the deployment.  It's easiest to verify completion in the AWS Console.  A green bar will appear at the top of the page stating `Service updated: matomo-ecs-dev-cluster:matomo-ecs-dev-service`
-1. Verify via webUI that the Matomo installation is ready to be upgraded. Ask someone in DEP to assist with this step.
+      * `aws ecs update-service --cluster matomo-ecs-dev-cluster --service matomo-ecs-dev-service --force-new-deployment`
+   * Either method takes a few minutes to complete the deployment.  It's easiest to verify completion in the AWS Console.  A green bar will appear at the top of the page stating `Service updated: matomo-ecs-dev-cluster:matomo-ecs-dev-service`
+1. Verify via webUI that the Matomo installation is ready to be upgraded.
+   * (Infra members should have the "superuser" checkbox checked for their Matomo UI login; if not, please verify with UXWS)
    * This step is only necessary when a database update is required.
 1. SSH (via AWSCLI + Session Manager) to the container and run the upgrade on the CLI (see the [Troubleshooting](./HOWTO-miscellaneous.md) section for the AWS CLI connection command).
    * The database update command is `php /var/www/html/console core:update`.
    * This step is only necessary when there is a required update to the table structure in Matomo listed in the release notes.
-1. Copy the updated `config/config.ini.php` to the EFS mount.
-   * Run the **backup-data.sh** script as in step 1.
+1. Compare the out-of-band `/mnt/efs/backup/config/config.ini/php` to the updated `config/config.ini.php` for changes.
 1. Verify that there were no changes to the `config.ini.php` file that need to be captured back in this repo.
    * See [compare-ini-files](./HOWTO-compare-ini-files.md)
-1. Log back in to the webUI to verify that everything is working.  Ask someone in DEP to assist with this step.
+1. Log back in to the webUI to verify that everything is working.
+   * (Infra members should have the "superuser" checkbox checked for their Matomo UI login; if not, please verify with UXWS)
 
 ## Update Matomo Plugins
 
 Often, an update to the version of Matomo will require an update to a plugin version. See below for an overview of the plugin update process.
 
-1. Ensure that a backup of the current `config/config.ini.php` exists in the EFS mount.
+1. Ensure that an out-of-band backup of the current `config/config.ini.php` exists
    * SSH (via AWSCLI + Session Manager) to the container(see the [Troubleshooting](./HOWTO-miscellaneous.md) section for the AWS CLI connection command).
-   * Run the **backup-data.sh** script `/usr/local/bin/backup-data.sh`
+   * Run `cp /var/www/html/config/* /mnt/efs/backup/config`
 1. Visit [Matomo plugins](https://plugins.matomo.org), select the correct version of Matomo, and then search for the plugins that are currently in use in our instance of Matomo.
    * If there is a newer version, download it from the site, unzip it, and store the unzipped folder in the [files/](../../files/) directory, following the naming convention in place (e.g., `plugin-<plugin_name>-<version>`)
 1. Update the [Dockerfile](../../Dockerfile) to reference the newer version of the plugin stored in the [files/](../../files/) directory.

docs/HowTos/HOWTO-miscellaneous.md

@@ -34,9 +34,8 @@ To retrieve the **task number** value for the command:
 OR
 
 ```bash
-aws ecs list-clusters | grep matomo
-aws ecs list-tasks --cluster matomo-ecs-dev-cluster (or result from previous command)
-# The task number is the 32 character hex string at the end of the line
+aws ecs list-clusters --output text | grep matomo | cut -d'/' -f2
+aws ecs list-tasks --cluster $(aws ecs list-clusters --output text | grep matomo | cut -d'/' -f2) --query "taskArns[*]" --output text | cut -d'/' -f3
 ```
 
 ## Reset 2-Factor auth

files/backup-data.sh

@@ -1,7 +1,15 @@
 #!/bin/bash
 
-/bin/cp -r /var/www/html/config/ /mnt/efs/config/
-/bin/cp -r /var/www/html/misc/user /mnt/efs/misc_user
-/bin/cp -r /var/www/html/js/ /mnt/efs/js/
-/bin/cp -r /var/www/html/matomo.js /mnt/efs/matomo.js
-/bin/cp -r /var/www/html/piwik.js /mnt/efs/piwik.js
+target_dir="/mnt/efs"
+
+mkdir -p "$target_dir/config"
+tar -cf - -C "/var/www/html/config" . | tar -xf - -C "$target_dir/config"
+
+mkdir -p "$target_dir/misc"
+tar -cf - -C "/var/www/html/misc" . | tar -xf - -C "$target_dir/misc"
+
+mkdir -p "$target_dir/js"
+tar -cf - -C "/var/www/html/js" . | tar -xf - -C "$target_dir/js"
+
+cp -a "/var/www/html/matomo.js" "$target_dir/matomo.js"
+cp -a "/var/www/html/piwik.js" "$target_dir/piwik.js"

files/config.ini.php

@@ -6,6 +6,13 @@
 collation = utf8mb4_general_ci
 
 [General]
+; disable browser-triggered archiving and expire archives in 30 minutes
+browser_archiving_disabled_enforce = 1 
+time_before_today_archive_considered_outdated = 1800
+
+; for now, disable host checking since we are behind ALB that does host checking for us
+enable_trusted_host_check=0
+
 proxy_client_headers[] = "HTTP_X_FORWARDED_FOR"
 
 ; maximum number of rows for any of the Referers tables (keywords, search engines, campaigns, etc.), and Custom variables names
@@ -24,7 +31,7 @@
 datatable_archiving_maximum_rows_subtable_actions = 5000
 
 ; maximum number of rows for the Site Search table
- datatable_archiving_maximum_rows_site_search = 5000
+datatable_archiving_maximum_rows_site_search = 5000
 
 ; maximum number of rows for any of the Events tables (Categories, Actions, Names)
 datatable_archiving_maximum_rows_events = 5000