diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 11e7f92..1dbf020 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -37,7 +37,7 @@ jobs: if: github.event_name == 'workflow_dispatch' || needs.check-skip-deploy.outputs.skip-deploy == 'false' strategy: matrix: - environment: [vsl-test, espoo-test] + environment: [vsl-test, espoo-test, arho-test] environment: ${{ matrix.environment }} steps: - uses: actions/checkout@v4 diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl index 6afdcb7..b0a280d 100644 --- a/infra/.terraform.lock.hcl +++ b/infra/.terraform.lock.hcl @@ -23,3 +23,22 @@ provider "registry.terraform.io/hashicorp/aws" { "zh:fdad558b1c41aa68123d0da82cc0d65bc86d09eaa1ab1d3a167ec3bce0fc0c66", ] } + +provider "registry.terraform.io/hashicorp/cloudinit" { + version = "2.3.7" + hashes = [ + "h1:iZ27qylcH/2bs685LJTKOKcQ+g7cF3VwN3kHMrzm4Ow=", + "zh:06f1c54e919425c3139f8aeb8fcf9bceca7e560d48c9f0c1e3bb0a8ad9d9da1e", + "zh:0e1e4cf6fd98b019e764c28586a386dc136129fef50af8c7165a067e7e4a31d5", + "zh:1871f4337c7c57287d4d67396f633d224b8938708b772abfc664d1f80bd67edd", + "zh:2b9269d91b742a71b2248439d5e9824f0447e6d261bfb86a8a88528609b136d1", + "zh:3d8ae039af21426072c66d6a59a467d51f2d9189b8198616888c1b7fc42addc7", + "zh:3ef4e2db5bcf3e2d915921adced43929214e0946a6fb11793085d9a48995ae01", + "zh:42ae54381147437c83cbb8790cc68935d71b6357728a154109d3220b1beb4dc9", + "zh:4496b362605ae4cbc9ef7995d102351e2fe311897586ffc7a4a262ccca0c782a", + "zh:652a2401257a12706d32842f66dac05a735693abcb3e6517d6b5e2573729ba13", + "zh:7406c30806f5979eaed5f50c548eced2ea18ea121e01801d2f0d4d87a04f6a14", + "zh:7848429fd5a5bcf35f6fee8487df0fb64b09ec071330f3ff240c0343fe2a5224", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + ] +} diff --git a/infra/Makefile b/infra/Makefile index 874e935..65f2df9 100644 --- a/infra/Makefile +++ b/infra/Makefile @@ -70,8 +70,8 @@ load-mml: decrypt-workspace-secrets: sops -d public_keys/$(shell terraform workspace show).enc > public_keys/$(shell terraform workspace show) - sops -d $(shell terraform workspace show).tfvars.enc.json > $(shell terraform workspace show).tfvars.json + sops -d var-files/$(shell terraform workspace show).tfvars.enc.json > var-files/$(shell terraform workspace show).tfvars.json encrypt-workspace-secrets: sops -e public_keys/$(shell terraform workspace show) > public_keys/$(shell terraform workspace show).enc - sops -e $(shell terraform workspace show).tfvars.json > $(shell terraform workspace show).tfvars.enc.json + sops -e var-files/$(shell terraform workspace show).tfvars.json > var-files/$(shell terraform workspace show).tfvars.enc.json diff --git a/infra/README.md b/infra/README.md index 929bd56..3d644be 100644 --- a/infra/README.md +++ b/infra/README.md @@ -43,15 +43,15 @@ To make changes to instances, first check that your variables and current infra ```shell terraform init -terraform plan --var-file hame-dev.tfvars.json +terraform plan --var-file var-files/hame-dev.tfvars.json ``` -This should report that terraform state is up to date with infra and configuration. You may make changes to configuration or variables and run `terraform plan --var-file hame-dev.tfvars.json` again to check what your changes would mean to the infrastructure. +This should report that terraform state is up to date with infra and configuration. You may make changes to configuration or variables and run `terraform plan --var-file var-files/hame-dev.tfvars.json` again to check what your changes would mean to the infrastructure. When you are sure that you want to change AWS infra, run ```shell -terraform apply --var-file hame-dev.tfvars.json +terraform apply --var-file var-files/hame-dev.tfvars.json ``` Please verify that the reported changes are desired, and respond `yes` to apply the changes to infrastructure. @@ -90,49 +90,64 @@ ansible-playbook ansible/playbook.yml \ ## Deploying instances +Change to the `infra` directory and set your AWS MFA session variables: +``` +cd infra +# Set AWS MFA session variables +. get-mfa-vars.sh +``` + +Generate the Host Key for the bastion host and add to AWS SSM Parameter Store. This is required for the bastion host to be able to use the same host key across reboots, and to avoid SSH warnings about changed host key when the bastion host is restarted. +``` +ssh-keygen -t ed25519 -f bastion_key -N "" + +aws ssm put-parameter \ + --name "/infra/-bastion/host_key_ed25519" \ + --value "$(cat bastion_key)" \ + --type "SecureString" \ + --region eu-central-1 +``` + To launch new instances, running the following commands should be sufficient: ```shell terraform init -terraform apply --var-file your-deployment.tfvars.json +terraform apply --var-file var-files/your-deployment.tfvars.json ``` -But in practice it is little bit more complicated, as some manual steps are required in between. `Terraform apply` would encounter some errors that need to be fixed manually before proceeding. Here is an example session for deploying a new `arho-dev` instance: +**But in practice it is little bit more complicated**, as some manual steps are required in between. `Terraform apply` would encounter some errors that need to be fixed manually before proceeding. Here is the complete list of steps to deploy new instances, including the manual steps to fix the errors: ```shell -cd infra -# Set AWS MFA session variables -. get-mfa-vars.sh # Create new terraform workspace -terraform workspace new arho-dev +terraform workspace new # Copy sample variables and edit them -cp arho.tfvars.sample.json arho-dev.tfvars.json -# Edit arho-dev.tfvars.json +cp arho.tfvars.sample.json .tfvars.json +# Edit .tfvars.json # Test the plan first -terraform plan -var-file=arho-dev.tfvars.json +terraform plan -var-file=var-files/.tfvars.json -terraform apply -var-file=arho-dev.tfvars.json -# Error is expected: Error: creating RDS DB Subnet Group (arho-dev-db): operation error RDS: CreateDBSubnetGroup, https response error StatusCode: 400, RequestID: f89c1d41-e247-48d2-9003-5a40b0e018aa, InvalidSubnet: Subnet IDs are required. +terraform apply -var-file=var-files/.tfvars.json +# Error is expected: Error: creating RDS DB Subnet Group (-db): operation error RDS: CreateDBSubnetGroup, https response error StatusCode: 400, RequestID: f89c1d41-e247-48d2-9003-5a40b0e018aa, InvalidSubnet: Subnet IDs are required. -terraform apply -var-file=arho-dev.tfvars.json -# Error is expected: Error: creating Lambda Function (arho-dev-db_manager): operation error Lambda: CreateFunction, https response error StatusCode: 400, RequestID: acc82829-26b3-4c88-af07-c9bae6f27b81, InvalidParameterValueException: Source image 631260641272.dkr.ecr.eu-central-1.amazonaws.com/arho-dev-db_manager:latest does not exist. Provide a valid source image. +terraform apply -var-file=var-files/.tfvars.json +# Error is expected: Error: creating Lambda Function (-db_manager): operation error Lambda: CreateFunction, https response error StatusCode: 400, RequestID: acc82829-26b3-4c88-af07-c9bae6f27b81, InvalidParameterValueException: Source image 631260641272.dkr.ecr.eu-central-1.amazonaws.com/-db_manager:latest does not exist. Provide a valid source image. # Set AWS_REGION and AWS_ACCOUNT_ID environment variables for Makefile export AWS_REGION= export AWS_ACCOUNT_ID= -export prefix=arho-dev +export prefix= # Build and push lambda images make push-lambdas -terraform apply -var-file=arho-dev.tfvars.json -# Error is expected: Error: creating Lambda Provisioned Concurrency Config (arho-dev-ryhti_client,live): operation error Lambda: PutProvisionedConcurrencyConfig, https response error StatusCode: 400, RequestID: 284038e3-650d-4ccb-951f-764e6cd9161d, InvalidParameterValueException: Provisioned Concurrency Configs cannot be applied to unpublished function versions. +terraform apply -var-file=var-files/.tfvars.json +# Error is expected: Error: creating Lambda Provisioned Concurrency Config (-ryhti_client,live): operation error Lambda: PutProvisionedConcurrencyConfig, https response error StatusCode: 400, RequestID: 284038e3-650d-4ccb-951f-764e6cd9161d, InvalidParameterValueException: Provisioned Concurrency Configs cannot be applied to unpublished function versions. # Update lambda functions make update-lambdas -terraform apply -var-file=arho-dev.tfvars.json +terraform apply -var-file=var-files/.tfvars.json # Now the infra should be deployed, but the database is still empty. Initialize the database with: make create-db @@ -155,7 +170,7 @@ This is because you need to apply for a separate permit for your subsystem to be When your application is accepted, you are provided with the configuration anchor file needed later. 2. Create an SSH key and add the public key to `bastion_ec2_user_public_keys` in `your-deployment.tfvars.json`. 3. Fill in the desired admin username and password in `x-road_secrets`, your desired `x-road_db_password` (password for x-road database) and your desired `x-road_token_pin` (for accessing authentication tokens), in `your-deployment.tfvars.json`. -4. Apply the variables to AWS with `terraform apply --var-file your-deployment.tfvars.json`. +4. Apply the variables to AWS with `terraform apply --var-file var-files/your-deployment.tfvars.json`. 5. Check the private IP address of your `your-deployment-x-road_securityserver` service task under your AWS Elastic Container Service `your-deployment-x-road_securityserver` cluster in your AWS web console. 6. Open an SSH tunnel to the X-Road server admin interface (e.g. `ssh -N -L4001::4000 -i "~/.ssh/arho-ec2-user.pem" ec2-user@your-deployment..`, where `arho-ec2-user.pem` contains your SSH key created in step 2, and `bastion_subdomain` and `aws_hosted_domain` are the settings in your `your-deployment.tfvars.json`). 7. Point your web browser to [https://localhost:4001](https://localhost:4001). The connection @@ -226,7 +241,7 @@ Congratulations! You now have access to X-Road Ryhti APIs! ## Teardown of instances -Shut down and destroy the instances with `terraform destroy --var-file your-deployment.tfvars.json` +Shut down and destroy the instances with `terraform destroy --var-file var-files/your-deployment.tfvars.json` ## Manual interactions diff --git a/infra/api.tf b/infra/api.tf index a4b0e77..4f150ac 100644 --- a/infra/api.tf +++ b/infra/api.tf @@ -76,7 +76,7 @@ resource "aws_api_gateway_integration" "lambda_integration" { type = "AWS_PROXY" # Our lambdas may run long if everything is processed. For a single # plan, the request will be much faster. - timeout_milliseconds = 120000 + timeout_milliseconds = 29000 uri = aws_lambda_function.ryhti_client.invoke_arn } diff --git a/infra/bastion.tf b/infra/bastion.tf index 69927d9..16dff34 100644 --- a/infra/bastion.tf +++ b/infra/bastion.tf @@ -1,16 +1,48 @@ # We don't want to create the key in terraform. Otherwise the private key(s) would be saved in terraform state. # Let's save the public key(s) here as ec2 instance user data. +data "aws_ssm_parameter" "amazon_linux" { + name = "/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64" +} + +data "cloudinit_config" "bastion_config" { + gzip = true # Compresses data to fit more in the 16KB limit + base64_encode = true # AWS requires base64 + + # Part 1: Cloud-Config (Runs first by default) + part { + content_type = "text/cloud-config" + content = templatefile( + "bastion_config/cloud-config.yaml.tftpl", + { + ec2_user_public_keys = var.bastion_ec2_user_public_keys, + } + ) + } + + # Part 2: Shell Script (Runs after) + part { + content_type = "text/x-shellscript" + content = templatefile( + "bastion_config/host_key_setup.sh", + { + ssm_parameter_name = "/infra/${var.prefix}-bastion/host_key_ed25519" + aws_region = data.aws_region.current.name + } + ) + } +} + # Just the smallest arm instance available, for routing traffic to postgres resource "aws_instance" "bastion-ec2-instance" { - ami = "ami-0bf463e49ccd368ed" # Amazon Linux 2023 + ami = data.aws_ssm_parameter.amazon_linux.value instance_type = "t4g.nano" subnet_id = aws_subnet.public[0].id vpc_security_group_ids = [aws_security_group.bastion.id] iam_instance_profile = aws_iam_instance_profile.ec2-iam-profile.name tenancy = "default" user_data_replace_on_change = true # This is needed to update user data *and* ip address - user_data = local.bastion_user_data + user_data_base64 = data.cloudinit_config.bastion_config.rendered tags = merge(local.default_tags, { Name = "${var.prefix}-bastion" }) diff --git a/infra/bastion_config/cloud-config.yaml.tftpl b/infra/bastion_config/cloud-config.yaml.tftpl new file mode 100644 index 0000000..78a460d --- /dev/null +++ b/infra/bastion_config/cloud-config.yaml.tftpl @@ -0,0 +1,10 @@ +#cloud-config +users: + - name: ec2-user + sudo: ALL=(ALL) NOPASSWD:ALL + ssh-authorized-keys: +%{ for key in ec2_user_public_keys ~} + - ${key} +%{ endfor ~} + - name: ec2-tunnel + sudo: False diff --git a/infra/bastion_config/host_key_setup.sh b/infra/bastion_config/host_key_setup.sh new file mode 100644 index 0000000..6642ff3 --- /dev/null +++ b/infra/bastion_config/host_key_setup.sh @@ -0,0 +1,37 @@ +#!/bin/bash +set -e + +# Parameters passed from Terraform +SSM_PARAM_NAME="${ssm_parameter_name}" +AWS_REGION="${aws_region}" + +echo "Configuring exclusive Ed25519 Host Key..." + +# Fetch and setup key +service sshd stop + +echo "Fetching Host Key from SSM..." +aws ssm get-parameter \ + --name "$SSM_PARAM_NAME" \ + --with-decryption \ + --region "$AWS_REGION" \ + --query "Parameter.Value" \ + --output text > /etc/ssh/ssh_host_ed25519_key + +chmod 640 /etc/ssh/ssh_host_ed25519_key +chown root:ssh_keys /etc/ssh/ssh_host_ed25519_key +ssh-keygen -y -f /etc/ssh/ssh_host_ed25519_key > /etc/ssh/ssh_host_ed25519_key.pub +chmod 644 /etc/ssh/ssh_host_ed25519_key.pub + + +# Comment out any existing HostKey definitions +sed -i 's/^HostKey/#HostKey/g' /etc/ssh/sshd_config + +# Append our specific Ed25519 key as the only valid host key +echo "HostKey /etc/ssh/ssh_host_ed25519_key" >> /etc/ssh/sshd_config + +# 5. Destroy the default OS-generated keys to ensure they are never used +rm -f /etc/ssh/ssh_host_rsa_key* +rm -f /etc/ssh/ssh_host_ecdsa_key* + +service sshd start diff --git a/infra/bastion_user_data.tpl b/infra/bastion_user_data.tpl deleted file mode 100644 index 6c3af9a..0000000 --- a/infra/bastion_user_data.tpl +++ /dev/null @@ -1,32 +0,0 @@ -Content-Type: multipart/mixed; boundary="//" -MIME-Version: 1.0 - ---// -Content-Type: text/cloud-config; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="cloud-config.txt" - -#cloud-config -cloud_final_modules: -- [users-groups, once] -users: - - name: ec2-user - sudo: ALL=(ALL) NOPASSWD:ALL - ssh-authorized-keys: -%{ for key in ec2_user_public_keys ~} - - ${key} -%{ endfor ~} - - name: ec2-tunnel - sudo: False - ---// -Content-Type: text/x-shellscript; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="userdata.sh" - -#!/bin/bash -sudo dnf update -sudo dnf install postgresql15 -y ---//-- diff --git a/infra/iam.tf b/infra/iam.tf index bccfaf7..0a468e8 100644 --- a/infra/iam.tf +++ b/infra/iam.tf @@ -192,31 +192,55 @@ resource "aws_iam_role_policy_attachment" "api-gateway-cloudwatch" { # Bastion # +data "aws_iam_policy_document" "bastion_trust" { + statement { + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +data "aws_caller_identity" "current" {} +data "aws_region" "current" {} +data "aws_iam_policy_document" "bastion_ssm_read" { + statement { + sid = "AllowReadingBastionHostKey" + actions = ["ssm:GetParameter"] + + resources = [ + "arn:aws:ssm:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:parameter/infra/${var.prefix}-bastion/host_key_rsa" + ] + } +} + +resource "aws_iam_policy" "bastion_ssm_read_policy" { + name = "${var.prefix}-bastion-ssm-read-policy" + description = "Allows bastion to read its own host key from SSM" + policy = data.aws_iam_policy_document.bastion_ssm_read.json + + tags = merge(local.default_tags, { + Name = "${var.prefix}-bastion_ssm_read_policy" + }) +} + # Adding a role for the EC2 machine allows making AWS service APIs available via IAM policies resource "aws_iam_role" "ec2-role" { name = "${var.prefix}-ec2-iam-role" path = "/" - assume_role_policy = <