Improvements for infra setup

.github/workflows/deploy.yml

@@ -37,7 +37,7 @@ jobs:
     if: github.event_name == 'workflow_dispatch' || needs.check-skip-deploy.outputs.skip-deploy == 'false'
     strategy:
       matrix:
-        environment: [vsl-test, espoo-test]
+        environment: [vsl-test, espoo-test, arho-test]
     environment: ${{ matrix.environment }}
     steps:
       - uses: actions/checkout@v4

infra/Makefile

@@ -70,8 +70,8 @@ load-mml:
 
 decrypt-workspace-secrets:
 	sops -d public_keys/$(shell terraform workspace show).enc > public_keys/$(shell terraform workspace show)
-	sops -d $(shell terraform workspace show).tfvars.enc.json > $(shell terraform workspace show).tfvars.json
+	sops -d var-files/$(shell terraform workspace show).tfvars.enc.json > var-files/$(shell terraform workspace show).tfvars.json
 
 encrypt-workspace-secrets:
 	sops -e public_keys/$(shell terraform workspace show) > public_keys/$(shell terraform workspace show).enc
-	sops -e $(shell terraform workspace show).tfvars.json > $(shell terraform workspace show).tfvars.enc.json
+	sops -e var-files/$(shell terraform workspace show).tfvars.json > var-files/$(shell terraform workspace show).tfvars.enc.json

infra/README.md

@@ -43,15 +43,15 @@ To make changes to instances, first check that your variables and current infra
 
 ```shell
 terraform init
-terraform plan --var-file hame-dev.tfvars.json
+terraform plan --var-file var-files/hame-dev.tfvars.json
 ```
 
-This should report that terraform state is up to date with infra and configuration. You may make changes to configuration or variables and run `terraform plan --var-file hame-dev.tfvars.json` again to check what your changes would mean to the infrastructure.
+This should report that terraform state is up to date with infra and configuration. You may make changes to configuration or variables and run `terraform plan --var-file var-files/hame-dev.tfvars.json` again to check what your changes would mean to the infrastructure.
 
 When you are sure that you want to change AWS infra, run
 
 ```shell
-terraform apply --var-file hame-dev.tfvars.json
+terraform apply --var-file var-files/hame-dev.tfvars.json
 ```
 
 Please verify that the reported changes are desired, and respond `yes` to apply the changes to infrastructure.
@@ -90,49 +90,64 @@ ansible-playbook ansible/playbook.yml \
 
 ## Deploying instances
 
+Change to the `infra` directory and set your AWS MFA session variables:
+```
+cd infra
+# Set AWS MFA session variables
+. get-mfa-vars.sh
+```
+
+Generate the Host Key for the bastion host and add to AWS SSM Parameter Store. This is required for the bastion host to be able to use the same host key across reboots, and to avoid SSH warnings about changed host key when the bastion host is restarted.
+```
+ssh-keygen -t ed25519 -f bastion_key -N ""
+
+aws ssm put-parameter \
+    --name "/infra/<instance-name>-bastion/host_key_ed25519" \
+    --value "$(cat bastion_key)" \
+    --type "SecureString" \
+    --region eu-central-1
+```
+
 To launch new instances, running the following commands should be sufficient:
 
 ```shell
 terraform init
-terraform apply --var-file your-deployment.tfvars.json
+terraform apply --var-file var-files/your-deployment.tfvars.json
 ```
 
-But in practice it is little bit more complicated, as some manual steps are required in between. `Terraform apply` would encounter some errors that need to be fixed manually before proceeding. Here is an example session for deploying a new `arho-dev` instance:
+**But in practice it is little bit more complicated**, as some manual steps are required in between. `Terraform apply` would encounter some errors that need to be fixed manually before proceeding. Here is the complete list of steps to deploy new instances, including the manual steps to fix the errors:
 
 ```shell
-cd infra
-# Set AWS MFA session variables
-. get-mfa-vars.sh
 
 # Create new terraform workspace
-terraform workspace new arho-dev
+terraform workspace new <instance-name>
 
 # Copy sample variables and edit them
-cp arho.tfvars.sample.json arho-dev.tfvars.json
-# Edit arho-dev.tfvars.json
+cp arho.tfvars.sample.json <instance-name>.tfvars.json
+# Edit <instance-name>.tfvars.json
 
 # Test the plan first
-terraform plan -var-file=arho-dev.tfvars.json
+terraform plan -var-file=var-files/<instance-name>.tfvars.json
 
-terraform apply -var-file=arho-dev.tfvars.json
-# Error is expected: Error: creating RDS DB Subnet Group (arho-dev-db): operation error RDS: CreateDBSubnetGroup, https response error StatusCode: 400, RequestID: f89c1d41-e247-48d2-9003-5a40b0e018aa, InvalidSubnet: Subnet IDs are required.
+terraform apply -var-file=var-files/<instance-name>.tfvars.json
+# Error is expected: Error: creating RDS DB Subnet Group (<instance-name>-db): operation error RDS: CreateDBSubnetGroup, https response error StatusCode: 400, RequestID: f89c1d41-e247-48d2-9003-5a40b0e018aa, InvalidSubnet: Subnet IDs are required.
 
-terraform apply -var-file=arho-dev.tfvars.json
-# Error is expected: Error: creating Lambda Function (arho-dev-db_manager): operation error Lambda: CreateFunction, https response error StatusCode: 400, RequestID: acc82829-26b3-4c88-af07-c9bae6f27b81, InvalidParameterValueException: Source image 631260641272.dkr.ecr.eu-central-1.amazonaws.com/arho-dev-db_manager:latest does not exist. Provide a valid source image.
+terraform apply -var-file=var-files/<instance-name>.tfvars.json
+# Error is expected: Error: creating Lambda Function (<instance-name>-db_manager): operation error Lambda: CreateFunction, https response error StatusCode: 400, RequestID: acc82829-26b3-4c88-af07-c9bae6f27b81, InvalidParameterValueException: Source image 631260641272.dkr.ecr.eu-central-1.amazonaws.com/<instance-name>-db_manager:latest does not exist. Provide a valid source image.
 
 # Set AWS_REGION and AWS_ACCOUNT_ID environment variables for Makefile
 export AWS_REGION=<region>
 export AWS_ACCOUNT_ID=<account id>
-export prefix=arho-dev
+export prefix=<instance-name>
 # Build and push lambda images
 make push-lambdas
 
-terraform apply -var-file=arho-dev.tfvars.json
-# Error is expected: Error: creating Lambda Provisioned Concurrency Config (arho-dev-ryhti_client,live): operation error Lambda: PutProvisionedConcurrencyConfig, https response error StatusCode: 400, RequestID: 284038e3-650d-4ccb-951f-764e6cd9161d, InvalidParameterValueException: Provisioned Concurrency Configs cannot be applied to unpublished function versions.
+terraform apply -var-file=var-files/<instance-name>.tfvars.json
+# Error is expected: Error: creating Lambda Provisioned Concurrency Config (<instance-name>-ryhti_client,live): operation error Lambda: PutProvisionedConcurrencyConfig, https response error StatusCode: 400, RequestID: 284038e3-650d-4ccb-951f-764e6cd9161d, InvalidParameterValueException: Provisioned Concurrency Configs cannot be applied to unpublished function versions.
 
 # Update lambda functions
 make update-lambdas
-terraform apply -var-file=arho-dev.tfvars.json
+terraform apply -var-file=var-files/<instance-name>.tfvars.json
 
 # Now the infra should be deployed, but the database is still empty. Initialize the database with:
 make create-db
@@ -155,7 +170,7 @@ This is because you need to apply for a separate permit for your subsystem to be
 When your application is accepted, you are provided with the configuration anchor file needed later.
 2. Create an SSH key and add the public key to `bastion_ec2_user_public_keys` in `your-deployment.tfvars.json`.
 3. Fill in the desired admin username and password in `x-road_secrets`, your desired  `x-road_db_password` (password for x-road database) and your desired `x-road_token_pin` (for accessing authentication tokens), in `your-deployment.tfvars.json`.
-4. Apply the variables to AWS with `terraform apply --var-file your-deployment.tfvars.json`.
+4. Apply the variables to AWS with `terraform apply --var-file var-files/your-deployment.tfvars.json`.
 5. Check the private IP address of your `your-deployment-x-road_securityserver` service task under your AWS Elastic Container Service `your-deployment-x-road_securityserver` cluster in your AWS web console.
 6. Open an SSH tunnel to the X-Road server admin interface (e.g. `ssh -N -L4001:<private-ip>:4000 -i "~/.ssh/arho-ec2-user.pem" ec2-user@your-deployment.<bastion_subdomain>.<aws_hosted_domain>`, where `arho-ec2-user.pem` contains your SSH key created in step 2, and `bastion_subdomain` and `aws_hosted_domain` are the settings in your `your-deployment.tfvars.json`).
 7. Point your web browser to [https://localhost:4001](https://localhost:4001). The connection
@@ -226,7 +241,7 @@ Congratulations! You now have access to X-Road Ryhti APIs!
 
 ## Teardown of instances
 
-Shut down and destroy the instances with `terraform destroy --var-file your-deployment.tfvars.json`
+Shut down and destroy the instances with `terraform destroy --var-file var-files/your-deployment.tfvars.json`
 
 ## Manual interactions
 

infra/api.tf

@@ -76,7 +76,7 @@ resource "aws_api_gateway_integration" "lambda_integration" {
   type        = "AWS_PROXY"
   # Our lambdas may run long if everything is processed. For a single
   # plan, the request will be much faster.
-  timeout_milliseconds = 120000
+  timeout_milliseconds = 29000
   uri         = aws_lambda_function.ryhti_client.invoke_arn
 }
 

infra/bastion.tf

@@ -1,16 +1,48 @@
 # We don't want to create the key in terraform. Otherwise the private key(s) would be saved in terraform state.
 # Let's save the public key(s) here as ec2 instance user data.
 
+data "aws_ssm_parameter" "amazon_linux" {
+  name = "/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64"
+}
+
+data "cloudinit_config" "bastion_config" {
+  gzip          = true  # Compresses data to fit more in the 16KB limit
+  base64_encode = true  # AWS requires base64
+
+  # Part 1: Cloud-Config (Runs first by default)
+  part {
+    content_type = "text/cloud-config"
+    content      = templatefile(
+      "bastion_config/cloud-config.yaml.tftpl",
+      {
+        ec2_user_public_keys    = var.bastion_ec2_user_public_keys,
+      }
+    )
+  }
+
+  # Part 2: Shell Script (Runs after)
+  part {
+    content_type = "text/x-shellscript"
+    content      = templatefile(
+      "bastion_config/host_key_setup.sh",
+      {
+        ssm_parameter_name = "/infra/${var.prefix}-bastion/host_key_ed25519"
+        aws_region         = data.aws_region.current.name
+      }
+    )
+  }
+}
+
 # Just the smallest arm instance available, for routing traffic to postgres
 resource "aws_instance" "bastion-ec2-instance" {
-  ami = "ami-0bf463e49ccd368ed" # Amazon Linux 2023
+  ami = data.aws_ssm_parameter.amazon_linux.value
   instance_type = "t4g.nano"
   subnet_id     = aws_subnet.public[0].id
   vpc_security_group_ids = [aws_security_group.bastion.id]
   iam_instance_profile = aws_iam_instance_profile.ec2-iam-profile.name
   tenancy              = "default"
   user_data_replace_on_change = true  # This is needed to update user data *and* ip address
-  user_data     = local.bastion_user_data
+  user_data_base64 = data.cloudinit_config.bastion_config.rendered
   tags = merge(local.default_tags, {
     Name = "${var.prefix}-bastion"
   })

infra/bastion_config/cloud-config.yaml.tftpl

@@ -0,0 +1,10 @@
+#cloud-config
+users:
+  - name: ec2-user
+    sudo:  ALL=(ALL) NOPASSWD:ALL
+    ssh-authorized-keys:
+%{ for key in ec2_user_public_keys ~}
+      - ${key}
+%{ endfor ~}
+  - name: ec2-tunnel
+    sudo: False

infra/bastion_config/host_key_setup.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+set -e
+
+# Parameters passed from Terraform
+SSM_PARAM_NAME="${ssm_parameter_name}"
+AWS_REGION="${aws_region}"
+
+echo "Configuring exclusive Ed25519 Host Key..."
+
+# Fetch and setup key
+service sshd stop
+
+echo "Fetching Host Key from SSM..."
+aws ssm get-parameter \
+  --name "$SSM_PARAM_NAME" \
+  --with-decryption \
+  --region "$AWS_REGION" \
+  --query "Parameter.Value" \
+  --output text > /etc/ssh/ssh_host_ed25519_key
+
+chmod 640 /etc/ssh/ssh_host_ed25519_key
+chown root:ssh_keys /etc/ssh/ssh_host_ed25519_key
+ssh-keygen -y -f /etc/ssh/ssh_host_ed25519_key > /etc/ssh/ssh_host_ed25519_key.pub
+chmod 644 /etc/ssh/ssh_host_ed25519_key.pub
+
+
+# Comment out any existing HostKey definitions
+sed -i 's/^HostKey/#HostKey/g' /etc/ssh/sshd_config
+
+# Append our specific Ed25519 key as the only valid host key
+echo "HostKey /etc/ssh/ssh_host_ed25519_key" >> /etc/ssh/sshd_config
+
+# 5. Destroy the default OS-generated keys to ensure they are never used
+rm -f /etc/ssh/ssh_host_rsa_key*
+rm -f /etc/ssh/ssh_host_ecdsa_key*
+
+service sshd start

infra/iam.tf

@@ -192,31 +192,55 @@ resource "aws_iam_role_policy_attachment" "api-gateway-cloudwatch" {
 # Bastion
 #
 
+data "aws_iam_policy_document" "bastion_trust" {
+  statement {
+    actions = ["sts:AssumeRole"]
+    principals {
+      type        = "Service"
+      identifiers = ["ec2.amazonaws.com"]
+    }
+  }
+}
+
+data "aws_caller_identity" "current" {}
+data "aws_region" "current" {}
+data "aws_iam_policy_document" "bastion_ssm_read" {
+  statement {
+    sid     = "AllowReadingBastionHostKey"
+    actions = ["ssm:GetParameter"]
+
+    resources = [
+      "arn:aws:ssm:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:parameter/infra/${var.prefix}-bastion/host_key_rsa"
+    ]
+  }
+}
+
+resource "aws_iam_policy" "bastion_ssm_read_policy" {
+  name        = "${var.prefix}-bastion-ssm-read-policy"
+  description = "Allows bastion to read its own host key from SSM"
+  policy      = data.aws_iam_policy_document.bastion_ssm_read.json
+
+  tags = merge(local.default_tags, {
+    Name = "${var.prefix}-bastion_ssm_read_policy"
+  })
+}
+
 # Adding a role for the EC2 machine allows making AWS service APIs available via IAM policies
 resource "aws_iam_role" "ec2-role" {
   name               = "${var.prefix}-ec2-iam-role"
   path               = "/"
-  assume_role_policy = <<EOF
-{
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Action": "sts:AssumeRole",
-            "Principal": {
-               "Service": "ec2.amazonaws.com"
-            },
-            "Effect": "Allow",
-            "Sid": ""
-        }
-    ]
-}
-EOF
+  assume_role_policy = data.aws_iam_policy_document.bastion_trust.json
 
   tags = merge(local.default_tags, {
     Name = "${var.prefix}-ec2-iam-role"
   })
 }
 
+resource "aws_iam_role_policy_attachment" "bastion_attach" {
+  role       = aws_iam_role.ec2-role.name
+  policy_arn = aws_iam_policy.bastion_ssm_read_policy.arn
+}
+
 resource "aws_iam_instance_profile" "ec2-iam-profile" {
   name = "${var.prefix}-ec2-iam-profile"
   role = aws_iam_role.ec2-role.name