From 7af9e093e7ed87aadc85772eff39476cd22637ea Mon Sep 17 00:00:00 2001 From: Chinmay Baikar Date: Wed, 15 Jan 2025 10:30:25 -0700 Subject: [PATCH] Secure Slurm Solution --- ansible/inventory/group_vars/all.yml | 1 - ansible/roles/nfs_server/templates/exports | 2 +- main.tf | 83 +++++++++++++++++++--- variables.tf | 14 ++-- 4 files changed, 80 insertions(+), 20 deletions(-) diff --git a/ansible/inventory/group_vars/all.yml b/ansible/inventory/group_vars/all.yml index ed7ed60..d7b1d6e 100644 --- a/ansible/inventory/group_vars/all.yml +++ b/ansible/inventory/group_vars/all.yml @@ -1,4 +1,3 @@ -ansible_ssh_common_args: "-o StrictHostKeyChecking=no" ansible_ssh_timeout: 600 ansible_ssh_retries: 10 diff --git a/ansible/roles/nfs_server/templates/exports b/ansible/roles/nfs_server/templates/exports index 20fc23c..0db7a17 100644 --- a/ansible/roles/nfs_server/templates/exports +++ b/ansible/roles/nfs_server/templates/exports @@ -1,5 +1,5 @@ /home *(rw,async,no_root_squash,no_subtree_check) -/var/spool/slurmctld *(rw,sync,no_root_squash,no_subtree_check) +/var/spool/slurmctld {{ (hostvars['slurm-nfs-node-0'].slurm_vpc_subnet_cidr)}}(rw,sync,no_root_squash,no_subtree_check) {% if (disk_count | int) > 0 %} /scratch/shared *(rw,sync,no_root_squash,no_subtree_check) {% endif %} diff --git a/main.tf b/main.tf index 36eada8..94de402 100644 --- a/main.tf +++ b/main.tf @@ -12,6 +12,19 @@ terraform { locals { ssh_public_key = file(var.ssh_public_key_path) + bastion_host = crusoe_compute_instance.slurm_login_node[0].network_interfaces[0].public_ipv4.address +} + +resource "crusoe_vpc_network" "slurm_vpc_network" { + name = "slurm-vpc-network" + cidr = "10.0.0.0/8" +} + +resource "crusoe_vpc_subnet" "slurm_vpc_subnet" { + location = var.location + name = "slurm-vpc-subnet-${var.location}" + cidr = "10.0.0.0/16" + network = crusoe_vpc_network.slurm_vpc_network.id } resource "crusoe_compute_instance" "slurm_head_node" { @@ -27,7 +40,7 @@ resource "crusoe_compute_instance" "slurm_head_node" { ib_partition_id = var.slurm_head_node_ib_partition_id }]: null network_interfaces = [{ - subnet = var.vpc_subnet_id, + subnet = crusoe_vpc_subnet.slurm_vpc_subnet.id, public_ipv4 = { type = "static" } @@ -47,7 +60,7 @@ resource "crusoe_compute_instance" "slurm_login_node" { ib_partition_id = var.slurm_login_node_ib_partition_id }]: null network_interfaces = [{ - subnet = var.vpc_subnet_id, + subnet = crusoe_vpc_subnet.slurm_vpc_subnet.id, public_ipv4 = { type = "static" } @@ -84,7 +97,7 @@ resource "crusoe_compute_instance" "slurm_nfs_node" { ib_partition_id = var.slurm_nfs_node_ib_partition_id }]: null network_interfaces = [{ - subnet = var.vpc_subnet_id, + subnet = crusoe_vpc_subnet.slurm_vpc_subnet.id, public_ipv4 = { type = "static" } @@ -104,7 +117,7 @@ resource "crusoe_compute_instance" "slurm_compute_node" { ib_partition_id = var.slurm_compute_node_ib_partition_id }]: null network_interfaces = [{ - subnet = var.vpc_subnet_id, + subnet = crusoe_vpc_subnet.slurm_vpc_subnet.id, public_ipv4 = { type = "static" } @@ -116,6 +129,55 @@ resource "crusoe_compute_instance" "slurm_compute_node" { }] } +resource "crusoe_vpc_firewall_rule" "allow_public_ssh_to_login" { + count = length(crusoe_compute_instance.slurm_login_node) + network = crusoe_vpc_network.slurm_vpc_network.id + name = "allow-ssh-to-${crusoe_compute_instance.slurm_login_node[count.index].name}" + action = "allow" + direction = "ingress" + protocols = "tcp" + source = "0.0.0.0/0" + source_ports = "1-65535" + destination = crusoe_compute_instance.slurm_login_node[count.index].network_interfaces[0].private_ipv4.address + destination_ports = "22" +} + +resource "crusoe_vpc_firewall_rule" "allow_tcpudp_internal" { + network = crusoe_vpc_network.slurm_vpc_network.id + name = "allow-all-internal-tcpudp-${var.location}" + action = "allow" + direction = "ingress" + protocols = "tcp,udp" + source = crusoe_vpc_subnet.slurm_vpc_subnet.cidr + source_ports = "1-65535" + destination = crusoe_vpc_subnet.slurm_vpc_subnet.cidr + destination_ports = "1-65535" +} + +resource "crusoe_vpc_firewall_rule" "allow_all_outbound" { + network = crusoe_vpc_network.slurm_vpc_network.id + name = "allow-all-outbound-${var.location}" + action = "allow" + direction = "egress" + protocols = "tcp,udp" + source = crusoe_vpc_subnet.slurm_vpc_subnet.cidr + source_ports = "1-65535" + destination = "0.0.0.0/0" + destination_ports = "1-65535" +} + +resource "crusoe_vpc_firewall_rule" "allow_icmp_internal" { + network = crusoe_vpc_network.slurm_vpc_network.id + name = "allow-all-internal-icmp-${var.location}" + action = "allow" + direction = "ingress" + protocols = "icmp" + source = crusoe_vpc_subnet.slurm_vpc_subnet.cidr + source_ports = "" + destination = crusoe_vpc_subnet.slurm_vpc_subnet.cidr + destination_ports = "" +} + resource "ansible_host" "slurm_nfs_node_host" { for_each = { for n in crusoe_compute_instance.slurm_nfs_node : n.name => n @@ -127,9 +189,10 @@ resource "ansible_host" "slurm_nfs_node_host" { replace(split(".", each.value.type)[0], "-", "_"), ] variables = { - ansible_host = each.value.network_interfaces[0].public_ipv4.address + ansible_host = each.value.network_interfaces[0].private_ipv4.address instance_type = each.value.type location = each.value.location + slurm_vpc_subnet_cidr = crusoe_vpc_subnet.slurm_vpc_subnet.cidr } } @@ -144,7 +207,7 @@ resource "ansible_host" "slurm_head_node_host" { replace(split(".", each.value.type)[0], "-", "_"), ] variables = { - ansible_host = each.value.network_interfaces[0].public_ipv4.address + ansible_host = each.value.network_interfaces[0].private_ipv4.address instance_type = each.value.type location = each.value.location } @@ -161,7 +224,7 @@ resource "ansible_host" "slurm_login_node_host" { replace(split(".", each.value.type)[0], "-", "_"), ] variables = { - ansible_host = each.value.network_interfaces[0].public_ipv4.address + ansible_host = each.value.network_interfaces[0].private_ipv4.address slurm_features = jsonencode([ "login" ]) instance_type = each.value.type location = each.value.location @@ -180,7 +243,7 @@ resource "ansible_host" "slurm_compute_node_host" { replace(split(".", each.value.type)[0], "-", "_"), ] variables = { - ansible_host = each.value.network_interfaces[0].public_ipv4.address + ansible_host = each.value.network_interfaces[0].private_ipv4.address slurm_features = jsonencode([ "batch" ]) instance_type = each.value.type location = each.value.location @@ -207,7 +270,7 @@ resource "null_resource" "ansible_playbook" { } provisioner "local-exec" { - command = "ansible-playbook -i ansible/inventory/inventory.yml ansible/slurm.yml -f 128" + command = "ansible-playbook --ssh-common-args=\"-o StrictHostKeyChecking=accept-new -o ProxyCommand='ssh -W %h:%p -q ${local.bastion_host} -o UserKnownHostsFile=/dev/null'\" -i ansible/inventory/inventory.yml ansible/slurm.yml -f 128" } depends_on = [ @@ -215,6 +278,6 @@ resource "null_resource" "ansible_playbook" { ansible_host.slurm_head_node_host, ansible_host.slurm_login_node_host, ansible_host.slurm_compute_node_host, - ansible_group.all + ansible_group.all, ] } diff --git a/variables.tf b/variables.tf index 7be15e8..8d9ce45 100644 --- a/variables.tf +++ b/variables.tf @@ -1,5 +1,10 @@ variable "ssh_public_key_path" { - description = "The ssh public key authorized to login to the cluster." + description = "The ssh public key authorized to login to the login node." + type = string +} + +variable "ssh_private_key_path" { + description = "The ssh private key authorized to login to the login node." type = string } @@ -13,12 +18,6 @@ variable "project_id" { type = string } -variable "vpc_subnet_id" { - description = "The vpc subnet id." - type = string - default = null -} - variable "slurm_head_node_count" { description = "The number of slurm head nodes." type = number @@ -108,7 +107,6 @@ variable "slurm_compute_node_count" { variable "slurm_compute_node_ib_partition_id" { description = "The ib partition in which to create the compute nodes." type = string - default = null } variable "slurm_compute_node_reservation_id" {