diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9727e8d..7be6968 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-terraform - name: Terraform apply - run: terraform apply -auto-approve -var "create_vm=true" -var "create_app_service=true" -var "ssh_key=${{ secrets.SSH_PUBLIC_KEY }}" + run: terraform apply -auto-approve -var "create_vm=true" -var "create_app_service=true" -var "create_aks=true" -var "ssh_key=${{ secrets.SSH_PUBLIC_KEY }}" app_service_test: needs: deploy-infrastructure @@ -97,6 +97,45 @@ jobs: SSH_AUTH_SOCK: /tmp/ssh_agent.sock run: bundle exec rake test_azure_vm + aks_test: + needs: deploy-infrastructure + runs-on: ubuntu-latest + steps: + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y libvips sshuttle sqlite3 libsqlite3-dev + - name: SSH key + env: + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + run: | + mkdir -p /home/runner/.ssh + echo "${{ secrets.SSH_PRIVATE_KEY }}" > /home/runner/.ssh/id_rsa + chmod 600 /home/runner/.ssh/id_rsa + ssh-agent -a $SSH_AUTH_SOCK > /dev/null + ssh-add /home/runner/.ssh/id_rsa + - name: Checkout + uses: actions/checkout@v4 + - uses: ./.github/actions/setup-terraform + - name: Azure login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + - name: Setup kubectl + uses: azure/setup-kubectl@v4 + - name: Setup ruby + uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + - name: Tests + env: + AZURE_ACCOUNT_NAME: ${{secrets.AZURE_ACCOUNT_NAME}} + AZURE_PRIVATE_CONTAINER: ${{secrets.AZURE_PRIVATE_CONTAINER}} + AZURE_PUBLIC_CONTAINER: ${{secrets.AZURE_PUBLIC_CONTAINER}} + USE_MANAGED_IDENTITIES: true + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + run: bundle exec rake test_aks + azurite_test: runs-on: ubuntu-latest steps: diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 79b56d4..c8a1981 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -21,3 +21,43 @@ provider "registry.terraform.io/hashicorp/azurerm" { "zh:fb9d78dfeca7489bffca9b1a1f3abee7f16dbbcba31388aea1102062c1d6dce8", ] } + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.38.0" + constraints = "~> 2.0" + hashes = [ + "h1:soK8Lt0SZ6dB+HsypFRDzuX/npqlMU6M0fvyaR1yW0k=", + "zh:0af928d776eb269b192dc0ea0f8a3f0f5ec117224cd644bdacdc682300f84ba0", + "zh:1be998e67206f7cfc4ffe77c01a09ac91ce725de0abaec9030b22c0a832af44f", + "zh:326803fe5946023687d603f6f1bab24de7af3d426b01d20e51d4e6fbe4e7ec1b", + "zh:4a99ec8d91193af961de1abb1f824be73df07489301d62e6141a656b3ebfff12", + "zh:5136e51765d6a0b9e4dbcc3b38821e9736bd2136cf15e9aac11668f22db117d2", + "zh:63fab47349852d7802fb032e4f2b6a101ee1ce34b62557a9ad0f0f0f5b6ecfdc", + "zh:924fb0257e2d03e03e2bfe9c7b99aa73c195b1f19412ca09960001bee3c50d15", + "zh:b63a0be5e233f8f6727c56bed3b61eb9456ca7a8bb29539fba0837f1badf1396", + "zh:d39861aa21077f1bc899bc53e7233262e530ba8a3a2d737449b100daeb303e4d", + "zh:de0805e10ebe4c83ce3b728a67f6b0f9d18be32b25146aa89116634df5145ad4", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:faf23e45f0090eef8ba28a8aac7ec5d4fdf11a36c40a8d286304567d71c1e7db", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.7.2" + constraints = "~> 3.0" + hashes = [ + "h1:KG4NuIBl1mRWU0KD/BGfCi1YN/j3F7H4YgeeM7iSdNs=", + "zh:14829603a32e4bc4d05062f059e545a91e27ff033756b48afbae6b3c835f508f", + "zh:1527fb07d9fea400d70e9e6eb4a2b918d5060d604749b6f1c361518e7da546dc", + "zh:1e86bcd7ebec85ba336b423ba1db046aeaa3c0e5f921039b3f1a6fc2f978feab", + "zh:24536dec8bde66753f4b4030b8f3ef43c196d69cccbea1c382d01b222478c7a3", + "zh:29f1786486759fad9b0ce4fdfbbfece9343ad47cd50119045075e05afe49d212", + "zh:4d701e978c2dd8604ba1ce962b047607701e65c078cb22e97171513e9e57491f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b8434212eef0f8c83f5a90c6d76feaf850f6502b61b53c329e85b3b281cba34", + "zh:ac8a23c212258b7976e1621275e3af7099e7e4a3d4478cf8d5d2a27f3bc3e967", + "zh:b516ca74431f3df4c6cf90ddcdb4042c626e026317a33c53f0b445a3d93b720d", + "zh:dc76e4326aec2490c1600d6871a95e78f9050f9ce427c71707ea412a2f2f1a62", + "zh:eac7b63e86c749c7d48f527671c7aee5b4e26c10be6ad7232d6860167f99dbb0", + ] +} diff --git a/Gemfile.lock b/Gemfile.lock index 8071e3e..96d232e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -82,6 +82,7 @@ GEM base64 (0.2.0) bcrypt_pbkdf (1.1.1) bcrypt_pbkdf (1.1.1-arm64-darwin) + bcrypt_pbkdf (1.1.1-x86_64-darwin) benchmark (0.4.0) bigdecimal (3.1.9) builder (3.3.0) @@ -97,6 +98,7 @@ GEM erb (5.0.1) erubi (1.13.1) ffi (1.17.2-arm64-darwin) + ffi (1.17.2-x86_64-darwin) ffi (1.17.2-x86_64-linux-gnu) globalid (1.2.1) activesupport (>= 6.1) @@ -140,11 +142,13 @@ GEM timeout net-smtp (0.5.1) net-protocol - net-ssh (7.2.3) + net-ssh (7.3.0) nio4r (2.7.4) - nokogiri (1.18.8-arm64-darwin) + nokogiri (1.18.10-arm64-darwin) racc (~> 1.4) - nokogiri (1.18.8-x86_64-linux-gnu) + nokogiri (1.18.10-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.18.10-x86_64-linux-gnu) racc (~> 1.4) parallel (1.24.0) parser (3.3.1.0) @@ -244,6 +248,7 @@ GEM logger securerandom (0.4.1) sqlite3 (2.6.0-arm64-darwin) + sqlite3 (2.6.0-x86_64-darwin) sqlite3 (2.6.0-x86_64-linux-gnu) stringio (3.1.7) strscan (3.1.0) @@ -262,6 +267,7 @@ GEM PLATFORMS arm64-darwin-24 + x86_64-darwin-23 x86_64-linux DEPENDENCIES diff --git a/README.md b/README.md index 41c79f4..3d83f02 100644 --- a/README.md +++ b/README.md @@ -26,13 +26,13 @@ microsoft: ### Managed Identity (Entra ID) -AzureBlob supports managed identities on : +AzureBlob supports managed identities on: - Azure VM - App Service +- AKS (Azure Kubernetes Service) - supports both node identity and workload identity - Azure Functions (Untested but should work) - Azure Containers (Untested but should work) - -AKS support will likely require more work. Contributions are welcome. +- Azure AD Workload Identity (AKS/K8s) To authenticate through managed identities instead of a shared key, omit `storage_access_key` from your `storage.yml` file and pass in the identity `principal_id`. @@ -46,6 +46,21 @@ prod: principal_id: 71b34410-4c50-451d-b456-95ead1b18cce ``` +#### Azure AD Workload Identity (AKS/K8s) + +ActiveStorage config example: + +``` +prod: + service: AzureBlob + container: container_name + storage_account_name: account_name + use_managed_identities: true +``` + +> uses `AZURE_CLIENT_ID`, `AZURE_TENANT_ID` and `AZURE_FEDERATED_TOKEN_FILE` environment variables, made available by AKS cluster when Azure AD Workload Identity is set up properly. + + ### Azurite To use Azurite, pass the `storage_blob_host` config key with the Azurite URL (`http://127.0.0.1:10000/devstoreaccount1` by default) @@ -126,20 +141,34 @@ A dev environment is supplied through Nix with [devenv](https://devenv.sh/). To test with Entra ID, the `AZURE_ACCESS_KEY` environment variable must be unset and the code must be ran or proxied through a VPS with the proper roles. -For cost saving, the terraform variable `create_vm` and `create_app_service` are false by default. -To create the VPS and App service, Create a var file `var.tfvars` containing: +For cost saving, the terraform variables `create_vm`, `create_app_service`, and `create_aks` are false by default. +To create the VM, App Service, and/or AKS cluster, create a var file `var.tfvars` containing: ``` create_vm = true create_app_service = true +create_aks = true ``` and re-apply terraform: `terraform apply -var-file=var.tfvars`. -This will create the VPS and required managed identities. +This will create the infrastructure and required managed identities. + +**Testing:** +- `bin/rake test_azure_vm` - Establishes a VPN connection to the Azure VM and runs tests using node identity +- `bin/rake test_app_service` - Establishes a VPN connection to the App Service container and runs tests +- `bin/rake test_aks` - Establishes a VPN connection to the AKS cluster and runs tests. This tests both: + - **Node identity**: Storage access via the node pool's managed identity + - **Workload identity**: Storage access via Azure AD federated credentials for the pod's service account + +You might be prompted for a sudo password when the VPN starts (sshuttle). -`bin/rake test_azure_vm` and `bin/rake test_app_service` will establish a VPN connection to the VM or App service container and run the test suite. You might be prompted for a sudo password when the VPN starts (sshuttle). +After you are done, run terraform again without the var file (`terraform apply`) to destroy all resources. -After you are done, run terraform again without the var file (`terraform apply`) to destroy the VPS and App service application. +**Note for AKS:** The AKS setup includes: +- A Kubernetes cluster with OIDC issuer and workload identity enabled +- An SSH-enabled pod running the `linuxserver/openssh-server` image +- A LoadBalancer service exposing SSH access for VPN tunneling +- Both kubelet identity (for node-level access) and federated identity credentials (for workload identity) #### Cleanup diff --git a/Rakefile b/Rakefile index 1720411..da9f910 100644 --- a/Rakefile +++ b/Rakefile @@ -5,6 +5,7 @@ require "minitest/test_task" require "azure_blob" require_relative "test/support/app_service_vpn" require_relative "test/support/azure_vm_vpn" +require_relative "test/support/aks_vpn" require_relative "test/support/azurite" Minitest::TestTask.create(:test_rails) do @@ -49,6 +50,15 @@ ensure vpn.kill end +task :test_aks do |t| + vpn = AksVpn.new + ENV["IDENTITY_ENDPOINT"] = vpn.endpoint + ENV["IDENTITY_HEADER"] = vpn.header + Rake::Task["test_entra_id"].execute +ensure + vpn.kill +end + task :test_azurite do |t| azurite = Azurite.new # Azurite well-known credentials diff --git a/bin/proxy-aks b/bin/proxy-aks new file mode 100755 index 0000000..2db0eac --- /dev/null +++ b/bin/proxy-aks @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Get Terraform outputs +resource_group=$(terraform output --raw "resource_group") +cluster_name=$(terraform output --raw "aks_cluster_name") +ssh_username=$(terraform output --raw "aks_ssh_username") + +# Setup kubectl config +echo "Setting up kubectl config..." +az aks get-credentials --resource-group "$resource_group" --name "$cluster_name" --overwrite-existing --only-show-errors 2>/dev/null + +# Wait for pod to be ready +echo "Waiting for SSH pod to be ready..." +kubectl wait --for=condition=ready pod -l app=azure-blob-test --timeout=300s + +# Start port forward in the background +echo "Setting up port forward..." +kubectl port-forward service/azure-blob-test-ssh 2222:22 & +PORT_FORWARD_PID=$! + +# Cleanup function +cleanup() { + echo "Cleaning up..." + kill $PORT_FORWARD_PID 2>/dev/null || true +} +trap cleanup EXIT + +# Wait for port forward to be established +sleep 3 + +# Establish sshuttle VPN +echo "Establishing VPN connection..." +exec sshuttle -e "ssh -o CheckHostIP=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -r "$ssh_username@127.0.0.1:2222" 0/0 diff --git a/devenv.nix b/devenv.nix index 3b08417..dea4d48 100644 --- a/devenv.nix +++ b/devenv.nix @@ -12,10 +12,10 @@ libyaml terraform azure-cli + kubectl glib vips sshuttle - sshpass rsync azurite sqlite diff --git a/input.tf b/input.tf index 6bd3273..520679e 100644 --- a/input.tf +++ b/input.tf @@ -38,6 +38,21 @@ variable "create_app_service" { default = false } +variable "create_aks" { + type = bool + default = false +} + +variable "aks_node_count" { + type = number + default = 1 +} + +variable "aks_ssh_username" { + type = string + default = "testuser" +} + variable "ssh_key" { type = string default = "" diff --git a/lib/azure_blob/identity_token.rb b/lib/azure_blob/identity_token.rb index 4c91df5..5e62ca4 100644 --- a/lib/azure_blob/identity_token.rb +++ b/lib/azure_blob/identity_token.rb @@ -1,21 +1,14 @@ +require_relative "instance_metadata_service" +require_relative "workload_identity" require "json" module AzureBlob class IdentityToken - RESOURCE_URI = "https://storage.azure.com/" EXPIRATION_BUFFER = 600 # 10 minutes - IDENTITY_ENDPOINT = ENV["IDENTITY_ENDPOINT"] || "http://169.254.169.254/metadata/identity/oauth2/token" - API_VERSION = ENV["IDENTITY_ENDPOINT"] ? "2019-08-01" : "2018-02-01" - def initialize(principal_id: nil) - @identity_uri = URI.parse(IDENTITY_ENDPOINT) - params = { - 'api-version': API_VERSION, - resource: RESOURCE_URI, - } - params[:principal_id] = principal_id if principal_id - @identity_uri.query = URI.encode_www_form(params) + @service = AzureBlob::WorkloadIdentity.federated_token? ? + AzureBlob::WorkloadIdentity.new : AzureBlob::InstanceMetadataService.new(principal_id: principal_id) end def to_s @@ -31,13 +24,11 @@ def expired? def refresh return unless expired? - headers = { "Metadata" => "true" } - headers["X-IDENTITY-HEADER"] = ENV["IDENTITY_HEADER"] if ENV["IDENTITY_HEADER"] attempt = 0 begin attempt += 1 - response = JSON.parse(AzureBlob::Http.new(identity_uri, headers).get) + response = JSON.parse(service.request) rescue AzureBlob::Http::Error => error if should_retry?(error, attempt) attempt = 1 if error.status == 410 @@ -48,7 +39,7 @@ def refresh raise end @token = response["access_token"] - @expiration = Time.at(response["expires_on"].to_i) + @expiration = Time.at((response["expires_on"] || response["expires_in"]).to_i) end def should_retry?(error, attempt) @@ -61,6 +52,6 @@ def exponential_backoff(error, attempt) end EXPONENTIAL_BACKOFF = [ 2, 6, 14, 30 ] - attr_reader :identity_uri, :expiration, :token + attr_reader :service, :expiration, :token end end diff --git a/lib/azure_blob/instance_metadata_service.rb b/lib/azure_blob/instance_metadata_service.rb new file mode 100644 index 0000000..2f80e91 --- /dev/null +++ b/lib/azure_blob/instance_metadata_service.rb @@ -0,0 +1,24 @@ +module AzureBlob + class InstanceMetadataService # Azure Instance Metadata Service (IMDS) + IDENTITY_ENDPOINT = ENV["IDENTITY_ENDPOINT"] || "http://169.254.169.254/metadata/identity/oauth2/token" + API_VERSION = ENV["IDENTITY_ENDPOINT"] ? "2019-08-01" : "2018-02-01" + RESOURCE_URI = "https://storage.azure.com/" + + def initialize(principal_id: nil) + @identity_uri = URI.parse(IDENTITY_ENDPOINT) + params = { + 'api-version': API_VERSION, + resource: RESOURCE_URI, + } + params[:principal_id] = principal_id if principal_id + @identity_uri.query = URI.encode_www_form(params) + end + + def request + headers = { "Metadata" => "true" } + headers["X-IDENTITY-HEADER"] = ENV["IDENTITY_HEADER"] if ENV["IDENTITY_HEADER"] + + AzureBlob::Http.new(@identity_uri, headers).get + end + end +end diff --git a/lib/azure_blob/workload_identity.rb b/lib/azure_blob/workload_identity.rb new file mode 100644 index 0000000..53bedbc --- /dev/null +++ b/lib/azure_blob/workload_identity.rb @@ -0,0 +1,33 @@ +module AzureBlob + class WorkloadIdentity # Azure AD Workload Identity + IDENTITY_ENDPOINT = "https://login.microsoftonline.com/#{ENV['AZURE_TENANT_ID']}/oauth2/v2.0/token" + CLIENT_ID = ENV["AZURE_CLIENT_ID"] + SCOPE = "https://storage.azure.com/.default" + GRANT_TYPE = "client_credentials" + CLIENT_ASSERTION_TYPE = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" + + FEDERATED_TOKEN_FILE = ENV["AZURE_FEDERATED_TOKEN_FILE"].to_s + + def self.federated_token? + !FEDERATED_TOKEN_FILE.empty? + end + + def request + AzureBlob::Http.new(URI.parse(IDENTITY_ENDPOINT)).post( + URI.encode_www_form( + client_id: CLIENT_ID, + scope: SCOPE, + client_assertion_type: CLIENT_ASSERTION_TYPE, + client_assertion: federated_token, + grant_type: GRANT_TYPE + ) + ) + end + + private + + def federated_token + File.read(FEDERATED_TOKEN_FILE).strip + end + end +end diff --git a/main.tf b/main.tf index 16ed2b0..3e7d13d 100644 --- a/main.tf +++ b/main.tf @@ -4,13 +4,34 @@ terraform { source = "hashicorp/azurerm" version = "~>3.0" } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~>2.0" + } + random = { + source = "hashicorp/random" + version = "~>3.0" + } } } +data "azurerm_kubernetes_cluster" "main" { + count = var.create_aks ? 0 : 1 + name = "${var.prefix}-aks" + resource_group_name = var.prefix +} + provider "azurerm" { features {} } +provider "kubernetes" { + host = var.create_aks ? azurerm_kubernetes_cluster.main[0].kube_config[0].host : (length(data.azurerm_kubernetes_cluster.main) > 0 ? data.azurerm_kubernetes_cluster.main[0].kube_config[0].host : "") + client_certificate = var.create_aks ? base64decode(azurerm_kubernetes_cluster.main[0].kube_config[0].client_certificate) : (length(data.azurerm_kubernetes_cluster.main) > 0 ? base64decode(data.azurerm_kubernetes_cluster.main[0].kube_config[0].client_certificate) : "") + client_key = var.create_aks ? base64decode(azurerm_kubernetes_cluster.main[0].kube_config[0].client_key) : (length(data.azurerm_kubernetes_cluster.main) > 0 ? base64decode(data.azurerm_kubernetes_cluster.main[0].kube_config[0].client_key) : "") + cluster_ca_certificate = var.create_aks ? base64decode(azurerm_kubernetes_cluster.main[0].kube_config[0].cluster_ca_certificate) : (length(data.azurerm_kubernetes_cluster.main) > 0 ? base64decode(data.azurerm_kubernetes_cluster.main[0].kube_config[0].cluster_ca_certificate) : "") +} + locals { public_ssh_key = var.create_vm && var.ssh_key == "" ? file("~/.ssh/id_rsa.pub") : var.ssh_key } @@ -197,3 +218,163 @@ resource "azurerm_app_service_source_control" "main" { use_manual_integration = true use_mercurial = false } + +# AKS Resources +resource "azurerm_kubernetes_cluster" "main" { + count = var.create_aks ? 1 : 0 + name = "${var.prefix}-aks" + location = azurerm_resource_group.main.location + resource_group_name = azurerm_resource_group.main.name + dns_prefix = "${var.prefix}-aks" + + default_node_pool { + name = "default" + node_count = var.aks_node_count + vm_size = var.vm_size + } + + identity { + type = "SystemAssigned" + } + + oidc_issuer_enabled = true + workload_identity_enabled = true + + tags = { + source = "Terraform" + } + + lifecycle { + ignore_changes = [ + default_node_pool[0].upgrade_settings + ] + } +} + +resource "azurerm_role_assignment" "aks_kubelet" { + count = var.create_aks ? 1 : 0 + scope = azurerm_storage_account.main.id + role_definition_name = "Storage Blob Data Owner" + principal_id = azurerm_kubernetes_cluster.main[0].kubelet_identity[0].object_id +} + +resource "azurerm_federated_identity_credential" "aks" { + count = var.create_aks ? 1 : 0 + name = "${var.prefix}-aks-federated-identity" + resource_group_name = azurerm_resource_group.main.name + audience = ["api://AzureADTokenExchange"] + issuer = azurerm_kubernetes_cluster.main[0].oidc_issuer_url + parent_id = azurerm_user_assigned_identity.vm.id + subject = "system:serviceaccount:default:azure-blob-test" +} + +# Kubernetes Resources for SSH Pod +resource "kubernetes_service_account" "azure_blob_test" { + count = var.create_aks ? 1 : 0 + metadata { + name = "azure-blob-test" + namespace = "default" + annotations = { + "azure.workload.identity/client-id" = azurerm_user_assigned_identity.vm.client_id + } + labels = { + "azure.workload.identity/use" = "true" + } + } +} + +resource "kubernetes_deployment" "ssh" { + count = var.create_aks ? 1 : 0 + metadata { + name = "azure-blob-test" + namespace = "default" + labels = { + app = "azure-blob-test" + } + } + + spec { + replicas = 1 + + selector { + match_labels = { + app = "azure-blob-test" + } + } + + template { + metadata { + labels = { + app = "azure-blob-test" + "azure.workload.identity/use" = "true" + } + } + + spec { + service_account_name = kubernetes_service_account.azure_blob_test[0].metadata[0].name + + container { + name = "openssh-server" + image = "lscr.io/linuxserver/openssh-server:latest" + + port { + container_port = 2222 + protocol = "TCP" + } + + env { + name = "PUID" + value = "1000" + } + + env { + name = "PGID" + value = "1000" + } + + env { + name = "TZ" + value = "Etc/UTC" + } + + env { + name = "USER_NAME" + value = var.aks_ssh_username + } + + env { + name = "PUBLIC_KEY" + value = local.public_ssh_key + } + + env { + name = "SUDO_ACCESS" + value = "true" + } + } + } + } + } +} + +resource "kubernetes_service" "ssh" { + count = var.create_aks ? 1 : 0 + metadata { + name = "azure-blob-test-ssh" + namespace = "default" + } + + spec { + type = "LoadBalancer" + + selector = { + app = "azure-blob-test" + } + + port { + port = 22 + target_port = 2222 + protocol = "TCP" + } + } +} diff --git a/output.tf b/output.tf index fe570ba..e267b6d 100644 --- a/output.tf +++ b/output.tf @@ -28,3 +28,15 @@ output "app_service_app_name" { output "resource_group" { value = azurerm_resource_group.main.name } + +output "aks_cluster_name" { + value = var.create_aks ? azurerm_kubernetes_cluster.main[0].name : "" +} + +output "aks_ssh_ip" { + value = var.create_aks && length(kubernetes_service.ssh) > 0 && length(kubernetes_service.ssh[0].status) > 0 && length(kubernetes_service.ssh[0].status[0].load_balancer) > 0 && length(kubernetes_service.ssh[0].status[0].load_balancer[0].ingress) > 0 ? kubernetes_service.ssh[0].status[0].load_balancer[0].ingress[0].ip : "" +} + +output "aks_ssh_username" { + value = var.aks_ssh_username +} diff --git a/test/client/test_client.rb b/test/client/test_client.rb index 7ced6f9..45f55a9 100644 --- a/test/client/test_client.rb +++ b/test/client/test_client.rb @@ -14,12 +14,14 @@ def setup @container = ENV["AZURE_PRIVATE_CONTAINER"] @public_container = ENV["AZURE_PUBLIC_CONTAINER"] @principal_id = ENV["AZURE_PRINCIPAL_ID"] + @use_managed_identities = ENV["USE_MANAGED_IDENTITIES"] == "true" @host = ENV["STORAGE_BLOB_HOST"] @client = AzureBlob::Client.new( account_name: @account_name, access_key: @access_key, container: @container, principal_id: @principal_id, + use_managed_identities: @use_managed_identities, host: @host, ) @uid = SecureRandom.uuid @@ -367,6 +369,7 @@ def test_get_container_properties access_key: @access_key, container: "missingcontainer", principal_id: @principal_id, + use_managed_identities: @use_managed_identities, ) container = client.get_container_properties refute container.present? @@ -381,6 +384,7 @@ def test_container_exist? access_key: @access_key, container: "missingcontainer", principal_id: @principal_id, + use_managed_identities: @use_managed_identities, ) refute client.container_exist? @@ -392,6 +396,7 @@ def test_create_container access_key: @access_key, container: Random.alphanumeric(20).tr("0-9", "").downcase, principal_id: @principal_id, + use_managed_identities: @use_managed_identities, host: @host, ) container = client.get_container_properties @@ -420,6 +425,7 @@ def test_copy_between_containers access_key: @access_key, container: @public_container, principal_id: @principal_id, + use_managed_identities: @use_managed_identities, host: @host, ) client.create_block_blob(key, content) diff --git a/test/client/test_user_delegation_key_expiration.rb b/test/client/test_user_delegation_key_expiration.rb index 5f26ba8..8d0e2fd 100644 --- a/test/client/test_user_delegation_key_expiration.rb +++ b/test/client/test_user_delegation_key_expiration.rb @@ -13,11 +13,13 @@ def setup @account_name = ENV["AZURE_ACCOUNT_NAME"] @container = ENV["AZURE_PRIVATE_CONTAINER"] @principal_id = ENV["AZURE_PRINCIPAL_ID"] + @use_managed_identities = ENV["USE_MANAGED_IDENTITIES"] == "true" @host = ENV["STORAGE_BLOB_HOST"] @client = AzureBlob::Client.new( account_name: @account_name, container: @container, principal_id: @principal_id, + use_managed_identities: @use_managed_identities, host: @host, ) @uid = SecureRandom.uuid diff --git a/test/rails/service/configurations.yml b/test/rails/service/configurations.yml index aa332ff..748a9d1 100644 --- a/test/rails/service/configurations.yml +++ b/test/rails/service/configurations.yml @@ -3,6 +3,7 @@ DEFAULT: &default storage_account_name: <%= ENV["AZURE_ACCOUNT_NAME"] %> storage_access_key: <%= ENV["AZURE_ACCESS_KEY"] %> principal_id: <%= ENV["AZURE_PRINCIPAL_ID"]%> + use_managed_identities: <%= ENV["USE_MANAGED_IDENTITIES"] == "true" %> storage_blob_host: <%= ENV["STORAGE_BLOB_HOST"] %> azure: diff --git a/test/support/aks_vpn.rb b/test/support/aks_vpn.rb new file mode 100644 index 0000000..91093bc --- /dev/null +++ b/test/support/aks_vpn.rb @@ -0,0 +1,130 @@ +require "open3" +require "net/ssh" +require "shellwords" + +class AksVpn + HOST = "127.0.0.1" + + attr_reader :header, :endpoint + + def initialize(verbose: true) + @verbose = verbose + establish_vpn_connection + end + + def kill + Process.kill("INT", tunnel_wait_thread.pid) if tunnel_wait_thread + Process.kill("INT", port_forward_wait_thread.pid) if port_forward_wait_thread + end + + private + + def establish_vpn_connection + setup_kubeconfig + setup_port_forward + extract_msi_info + + puts "Establishing VPN connection..." + + tunnel_stdin, tunnel_stdout, @tunnel_wait_thread = Open3.popen2e([ "sshuttle", "-e", "ssh -o CheckHostIP=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null", "--python=python3", "-r", "#{username}@#{HOST}:#{port}", "0/0" ].shelljoin) + + connection_successful = false + tunnel_stdout.each do |line| + puts "(sshuttle) #{line}" if verbose + if line.include?("Connected to server") + connection_successful = true + puts "Connection successful!" + break + end + end + + raise "Could not establish VPN connection to AKS" unless connection_successful + end + + def setup_kubeconfig + puts "Setting up kubectl config..." + resource_group = `terraform output --raw resource_group`.strip + cluster_name = `terraform output --raw aks_cluster_name`.strip + + system("az aks get-credentials --resource-group #{resource_group} --name #{cluster_name} --overwrite-existing --only-show-errors 2>/dev/null") + raise "Failed to setup kubectl config" unless $?.success? + + # Wait for pod to be ready + puts "Waiting for SSH pod to be ready..." + system("kubectl wait --for=condition=ready pod -l app=azure-blob-test --timeout=300s") + raise "Pod did not become ready" unless $?.success? + end + + def setup_port_forward + puts "Setting up port forward to SSH service..." + + @username = `terraform output --raw aks_ssh_username`.strip + + @port = 2222 + + system("lsof -ti:#{@port} | xargs kill -9 2>/dev/null") + + # Use kubectl port-forward to forward to the LoadBalancer service + # This is more reliable than waiting for the external IP to be routable + port_forward_stdin, port_forward_stdout, @port_forward_wait_thread = Open3.popen2e("kubectl port-forward service/azure-blob-test-ssh #{@port}:22") + + # Wait for port forward to be established and read confirmation + port_forward_ready = false + Thread.new do + port_forward_stdout.each do |line| + port_forward_ready = true if line.include?("Forwarding from") + end + end + + # Wait up to 10 seconds for port forward + 10.times do + break if port_forward_ready + sleep 1 + end + + raise "Port forward did not establish" unless port_forward_ready + + puts "Port forward established on port #{@port}" + end + + def extract_msi_info + puts "Extracting MSI endpoint info from pod..." + + max_retries = 5 + retry_count = 0 + + begin + endpoint = nil + header = nil + + Net::SSH.start(HOST, username, port:, auth_methods: [ "publickey" ]) do |ssh| + # Extract the IDENTITY_ENDPOINT and IDENTITY_HEADER from the pod environment + endpoint = ssh.exec! [ "bash", "-l", "-c", %(printenv IDENTITY_ENDPOINT || echo "http://169.254.169.254/metadata/identity/oauth2/token") ].shelljoin + header = ssh.exec! [ "bash", "-l", "-c", %(printenv IDENTITY_HEADER || echo "") ].shelljoin + + endpoint = endpoint&.strip + header = header&.strip + end + + # For AKS, we need to use the Azure Instance Metadata Service endpoint + # The workload identity will inject the token via the service account + @endpoint = endpoint || "http://169.254.169.254/metadata/identity/oauth2/token" + @header = header || "" + + puts "MSI Endpoint: #{@endpoint}" + puts "MSI Header: #{@header.empty? ? '(empty)' : '(set)'}" + + rescue Net::SSH::AuthenticationFailed, Errno::ECONNREFUSED => e + retry_count += 1 + if retry_count < max_retries + puts "SSH connection failed (attempt #{retry_count}/#{max_retries}), retrying in 2 seconds..." + sleep 2 + retry + else + raise "Could not extract MSI endpoint information after #{max_retries} attempts: #{e.message}" + end + end + end + + attr_reader :port, :username, :verbose, :tunnel_wait_thread, :port_forward_wait_thread +end