From d9e2d7fdd8bc83c68e2d4c0b3a39e3ca43bffe92 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Sun, 12 Apr 2026 21:24:17 +1000 Subject: [PATCH 01/78] Add Aspire.Hosting.Azure.Kubernetes package (Phase 1) Create core implementation files for AKS hosting support: - AzureKubernetesEnvironmentResource: resource class with BicepOutputReference properties - AzureKubernetesEnvironmentExtensions: AddAzureKubernetesEnvironment and configuration methods - AzureKubernetesInfrastructure: eventing subscriber for compute resource processing - AksNodePoolConfig, AksSkuTier, AksNetworkProfile: supporting types - Project file with dependencies on Hosting.Azure, Hosting.Kubernetes, etc. - Add project to Aspire.slnx solution - Add InternalsVisibleTo in Aspire.Hosting.Kubernetes for internal API access Note: Azure.Provisioning.ContainerService package is not yet available in internal NuGet feeds. ConfigureAksInfrastructure uses placeholder outputs. When the package becomes available, replace with typed ContainerServiceManagedCluster. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Aspire.slnx | 1 + .../AksNetworkProfile.cs | 30 +++ .../AksNodePoolConfig.cs | 35 +++ .../AksSkuTier.cs | 25 +++ .../Aspire.Hosting.Azure.Kubernetes.csproj | 29 +++ .../AzureKubernetesEnvironmentExtensions.cs | 205 ++++++++++++++++++ .../AzureKubernetesEnvironmentResource.cs | 103 +++++++++ .../AzureKubernetesInfrastructure.cs | 62 ++++++ src/Aspire.Hosting.Azure.Kubernetes/README.md | 35 +++ .../Aspire.Hosting.Kubernetes.csproj | 1 + 10 files changed, 526 insertions(+) create mode 100644 src/Aspire.Hosting.Azure.Kubernetes/AksNetworkProfile.cs create mode 100644 src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolConfig.cs create mode 100644 src/Aspire.Hosting.Azure.Kubernetes/AksSkuTier.cs create mode 100644 src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj create mode 100644 src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs create mode 100644 src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs create mode 100644 src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs create mode 100644 src/Aspire.Hosting.Azure.Kubernetes/README.md diff --git a/Aspire.slnx b/Aspire.slnx index 687b4d4c339..4143c90b474 100644 --- a/Aspire.slnx +++ b/Aspire.slnx @@ -93,6 +93,7 @@ + diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksNetworkProfile.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksNetworkProfile.cs new file mode 100644 index 00000000000..ca707af77ed --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AksNetworkProfile.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Network profile configuration for an AKS cluster. +/// +internal sealed class AksNetworkProfile +{ + /// + /// Gets or sets the network plugin. Defaults to "azure" for Azure CNI. + /// + public string NetworkPlugin { get; set; } = "azure"; + + /// + /// Gets or sets the network policy. Defaults to "calico". + /// + public string? NetworkPolicy { get; set; } = "calico"; + + /// + /// Gets or sets the service CIDR. + /// + public string ServiceCidr { get; set; } = "10.0.4.0/22"; + + /// + /// Gets or sets the DNS service IP address. + /// + public string DnsServiceIP { get; set; } = "10.0.4.10"; +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolConfig.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolConfig.cs new file mode 100644 index 00000000000..5af798f6d6c --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolConfig.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Configuration for an AKS node pool. +/// +/// The name of the node pool. +/// The VM size for nodes in the pool. +/// The minimum number of nodes. +/// The maximum number of nodes. +/// The mode of the node pool. +internal sealed record AksNodePoolConfig( + string Name, + string VmSize, + int MinCount, + int MaxCount, + AksNodePoolMode Mode); + +/// +/// Specifies the mode of an AKS node pool. +/// +internal enum AksNodePoolMode +{ + /// + /// System node pool for hosting system pods. + /// + System, + + /// + /// User node pool for hosting application workloads. + /// + User +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksSkuTier.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksSkuTier.cs new file mode 100644 index 00000000000..f2f10865ac4 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AksSkuTier.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Specifies the SKU tier for an AKS cluster. +/// +public enum AksSkuTier +{ + /// + /// Free tier with no SLA. + /// + Free, + + /// + /// Standard tier with financially backed SLA. + /// + Standard, + + /// + /// Premium tier with mission-critical features. + /// + Premium +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj new file mode 100644 index 00000000000..4c918928549 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj @@ -0,0 +1,29 @@ + + + + $(DefaultTargetFramework) + true + false + aspire integration hosting azure kubernetes aks + Azure Kubernetes Service (AKS) resource types for Aspire. + $(SharedDir)Azure_256x.png + + + + + + + + + + + + + + + + + + + + diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs new file mode 100644 index 00000000000..55681ca1d02 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -0,0 +1,205 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Azure; +using Aspire.Hosting.Azure.Kubernetes; +using Aspire.Hosting.Kubernetes; +using Aspire.Hosting.Kubernetes.Extensions; +using Aspire.Hosting.Lifecycle; +using Azure.Provisioning; +using Microsoft.Extensions.DependencyInjection; + +namespace Aspire.Hosting; + +/// +/// Provides extension methods for adding Azure Kubernetes Service (AKS) environments to the application model. +/// +public static class AzureKubernetesEnvironmentExtensions +{ + /// + /// Adds an Azure Kubernetes Service (AKS) environment to the distributed application. + /// This provisions an AKS cluster and configures it as a Kubernetes compute environment. + /// + /// The . + /// The name of the AKS environment resource. + /// A reference to the . + /// + /// This method internally creates a Kubernetes environment for Helm-based deployment + /// and provisions an AKS cluster via Azure Bicep. It combines the functionality of + /// AddKubernetesEnvironment with Azure-specific provisioning. + /// + /// + /// + /// var aks = builder.AddAzureKubernetesEnvironment("aks") + /// .WithVersion("1.30"); + /// + /// + [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + public static IResourceBuilder AddAzureKubernetesEnvironment( + this IDistributedApplicationBuilder builder, + [ResourceName] string name) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrEmpty(name); + + // Set up Azure provisioning infrastructure + builder.AddAzureProvisioning(); + builder.Services.Configure( + o => o.SupportsTargetedRoleAssignments = true); + + // Register the AKS-specific infrastructure eventing subscriber + builder.Services.TryAddEventingSubscriber(); + + // Also register the generic K8s infrastructure (for Helm chart generation) + builder.AddKubernetesInfrastructureCore(); + + // Create the unified environment resource + var resource = new AzureKubernetesEnvironmentResource(name, ConfigureAksInfrastructure); + + // Create the inner KubernetesEnvironmentResource (for Helm deployment) + resource.KubernetesEnvironment = new KubernetesEnvironmentResource($"{name}-k8s") + { + HelmChartName = builder.Environment.ApplicationName.ToHelmChartName(), + }; + + if (builder.ExecutionContext.IsRunMode) + { + return builder.CreateResourceBuilder(resource); + } + + return builder.AddResource(resource); + } + + /// + /// Configures the Kubernetes version for the AKS cluster. + /// + /// The resource builder. + /// The Kubernetes version (e.g., "1.30"). + /// A reference to the for chaining. + [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + public static IResourceBuilder WithVersion( + this IResourceBuilder builder, + string version) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrEmpty(version); + + builder.Resource.KubernetesVersion = version; + return builder; + } + + /// + /// Configures the SKU tier for the AKS cluster. + /// + /// The resource builder. + /// The SKU tier. + /// A reference to the for chaining. + [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + public static IResourceBuilder WithSkuTier( + this IResourceBuilder builder, + AksSkuTier tier) + { + ArgumentNullException.ThrowIfNull(builder); + + builder.Resource.SkuTier = tier; + return builder; + } + + /// + /// Adds a node pool to the AKS cluster. + /// + /// The resource builder. + /// The name of the node pool. + /// The VM size for nodes. + /// The minimum node count for autoscaling. + /// The maximum node count for autoscaling. + /// A reference to the for chaining. + [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + public static IResourceBuilder WithNodePool( + this IResourceBuilder builder, + string name, + string vmSize, + int minCount, + int maxCount) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrEmpty(name); + ArgumentException.ThrowIfNullOrEmpty(vmSize); + + builder.Resource.NodePools.Add( + new AksNodePoolConfig(name, vmSize, minCount, maxCount, AksNodePoolMode.User)); + return builder; + } + + /// + /// Configures the AKS cluster as a private cluster with a private API server endpoint. + /// + /// The resource builder. + /// A reference to the for chaining. + [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + public static IResourceBuilder AsPrivateCluster( + this IResourceBuilder builder) + { + ArgumentNullException.ThrowIfNull(builder); + + builder.Resource.IsPrivateCluster = true; + return builder; + } + + /// + /// Enables Container Insights monitoring on the AKS cluster. + /// + /// The resource builder. + /// Optional Log Analytics workspace. If not provided, one will be auto-created. + /// A reference to the for chaining. + [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + public static IResourceBuilder WithContainerInsights( + this IResourceBuilder builder, + IResourceBuilder? logAnalytics = null) + { + ArgumentNullException.ThrowIfNull(builder); + + builder.Resource.ContainerInsightsEnabled = true; + + if (logAnalytics is not null) + { + builder.Resource.LogAnalyticsWorkspace = logAnalytics.Resource; + } + + return builder; + } + + /// + /// Configures the AKS environment to use a specific Azure Log Analytics workspace. + /// + /// The resource builder. + /// The Log Analytics workspace resource builder. + /// A reference to the for chaining. + [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + public static IResourceBuilder WithAzureLogAnalyticsWorkspace( + this IResourceBuilder builder, + IResourceBuilder workspaceBuilder) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(workspaceBuilder); + + builder.Resource.LogAnalyticsWorkspace = workspaceBuilder.Resource; + return builder; + } + + // NOTE: Full AKS Bicep provisioning with ContainerServiceManagedCluster types requires + // the Azure.Provisioning.ContainerService package (currently unavailable in internal feeds). + // This implementation uses ProvisioningOutput placeholders. When the package becomes available, + // replace this with typed ContainerServiceManagedCluster resource construction. + private static void ConfigureAksInfrastructure(AzureResourceInfrastructure infrastructure) + { + // Outputs will be populated by the Bicep module generated during publish + infrastructure.Add(new ProvisioningOutput("id", typeof(string))); + infrastructure.Add(new ProvisioningOutput("name", typeof(string))); + infrastructure.Add(new ProvisioningOutput("clusterFqdn", typeof(string))); + infrastructure.Add(new ProvisioningOutput("oidcIssuerUrl", typeof(string))); + infrastructure.Add(new ProvisioningOutput("kubeletIdentityObjectId", typeof(string))); + infrastructure.Add(new ProvisioningOutput("nodeResourceGroup", typeof(string))); + } +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs new file mode 100644 index 00000000000..5d067f3fec2 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs @@ -0,0 +1,103 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Hosting.Azure.Kubernetes; +using Aspire.Hosting.Kubernetes; + +namespace Aspire.Hosting.Azure; + +/// +/// Represents an Azure Kubernetes Service (AKS) environment resource that provisions +/// an AKS cluster and serves as a compute environment for Kubernetes workloads. +/// +/// The name of the resource. +/// Callback to configure the Azure infrastructure. +public class AzureKubernetesEnvironmentResource( + string name, + Action configureInfrastructure) + : AzureProvisioningResource(name, configureInfrastructure), + IAzureComputeEnvironmentResource +{ + /// + /// Gets the underlying Kubernetes environment resource used for Helm-based deployment. + /// + internal KubernetesEnvironmentResource KubernetesEnvironment { get; set; } = default!; + + /// + /// Gets the resource ID of the AKS cluster. + /// + public BicepOutputReference Id => new("id", this); + + /// + /// Gets the fully qualified domain name of the AKS cluster. + /// + public BicepOutputReference ClusterFqdn => new("clusterFqdn", this); + + /// + /// Gets the OIDC issuer URL for the AKS cluster, used for workload identity federation. + /// + public BicepOutputReference OidcIssuerUrl => new("oidcIssuerUrl", this); + + /// + /// Gets the object ID of the kubelet managed identity. + /// + public BicepOutputReference KubeletIdentityObjectId => new("kubeletIdentityObjectId", this); + + /// + /// Gets the name of the node resource group. + /// + public BicepOutputReference NodeResourceGroup => new("nodeResourceGroup", this); + + /// + /// Gets the name output reference for the AKS cluster. + /// + public BicepOutputReference NameOutputReference => new("name", this); + + /// + /// Gets or sets the Kubernetes version for the AKS cluster. + /// + internal string? KubernetesVersion { get; set; } + + /// + /// Gets or sets the SKU tier for the AKS cluster. + /// + internal AksSkuTier SkuTier { get; set; } = AksSkuTier.Free; + + /// + /// Gets or sets whether OIDC issuer is enabled on the cluster. + /// + internal bool OidcIssuerEnabled { get; set; } = true; + + /// + /// Gets or sets whether workload identity is enabled on the cluster. + /// + internal bool WorkloadIdentityEnabled { get; set; } = true; + + /// + /// Gets or sets the Log Analytics workspace resource for monitoring. + /// + internal AzureLogAnalyticsWorkspaceResource? LogAnalyticsWorkspace { get; set; } + + /// + /// Gets or sets whether Container Insights is enabled. + /// + internal bool ContainerInsightsEnabled { get; set; } + + /// + /// Gets the node pool configurations. + /// + internal List NodePools { get; } = + [ + new AksNodePoolConfig("system", "Standard_D4s_v5", 1, 3, AksNodePoolMode.System) + ]; + + /// + /// Gets or sets the network profile for the AKS cluster. + /// + internal AksNetworkProfile? NetworkProfile { get; set; } + + /// + /// Gets or sets whether the cluster should be private. + /// + internal bool IsPrivateCluster { get; set; } +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs new file mode 100644 index 00000000000..a57b86edf5f --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -0,0 +1,62 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Eventing; +using Aspire.Hosting.Lifecycle; +using Microsoft.Extensions.Logging; + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Infrastructure eventing subscriber that processes compute resources +/// targeting an AKS environment. +/// +internal sealed class AzureKubernetesInfrastructure( + ILogger logger) + : IDistributedApplicationEventingSubscriber +{ + /// + public Task SubscribeAsync(IDistributedApplicationEventing eventing, DistributedApplicationExecutionContext executionContext, CancellationToken cancellationToken) + { + if (!executionContext.IsRunMode) + { + eventing.Subscribe(OnBeforeStartAsync); + } + + return Task.CompletedTask; + } + + private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cancellationToken = default) + { + var aksEnvironments = @event.Model.Resources + .OfType() + .ToArray(); + + if (aksEnvironments.Length == 0) + { + return Task.CompletedTask; + } + + foreach (var environment in aksEnvironments) + { + logger.LogInformation("Processing AKS environment '{Name}'", environment.Name); + + foreach (var r in @event.Model.GetComputeResources()) + { + var resourceComputeEnvironment = r.GetComputeEnvironment(); + if (resourceComputeEnvironment is not null && resourceComputeEnvironment != environment) + { + continue; + } + + r.Annotations.Add(new DeploymentTargetAnnotation(environment) + { + ComputeEnvironment = environment + }); + } + } + + return Task.CompletedTask; + } +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/README.md b/src/Aspire.Hosting.Azure.Kubernetes/README.md new file mode 100644 index 00000000000..649ba91e442 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/README.md @@ -0,0 +1,35 @@ +# Aspire.Hosting.Azure.Kubernetes library + +Provides extension methods and resource definitions for an Aspire AppHost to configure an Azure Kubernetes Service (AKS) environment. + +## Getting started + +### Prerequisites + +- An Azure subscription - [create one for free](https://azure.microsoft.com/free/) + +### Install the package + +In your AppHost project, install the Aspire Azure Kubernetes Hosting library with [NuGet](https://www.nuget.org): + +```dotnetcli +dotnet add package Aspire.Hosting.Azure.Kubernetes +``` + +## Usage example + +Then, in the _AppHost.cs_ file of `AppHost`, add an AKS environment and deploy services to it: + +```csharp +var aks = builder.AddAzureKubernetesEnvironment("aks"); + +var myService = builder.AddProject(); +``` + +## Additional documentation + +* https://learn.microsoft.com/azure/aks/ + +## Feedback & contributing + +https://github.com/microsoft/aspire diff --git a/src/Aspire.Hosting.Kubernetes/Aspire.Hosting.Kubernetes.csproj b/src/Aspire.Hosting.Kubernetes/Aspire.Hosting.Kubernetes.csproj index de2d534e056..b92a148789a 100644 --- a/src/Aspire.Hosting.Kubernetes/Aspire.Hosting.Kubernetes.csproj +++ b/src/Aspire.Hosting.Kubernetes/Aspire.Hosting.Kubernetes.csproj @@ -19,6 +19,7 @@ + From 61992fb0421692a46c1516e835443a18b4abeef4 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Sun, 12 Apr 2026 21:25:17 +1000 Subject: [PATCH 02/78] Add AKS support implementation spec Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/specs/aks-support.md | 605 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 605 insertions(+) create mode 100644 docs/specs/aks-support.md diff --git a/docs/specs/aks-support.md b/docs/specs/aks-support.md new file mode 100644 index 00000000000..a13ce6dc082 --- /dev/null +++ b/docs/specs/aks-support.md @@ -0,0 +1,605 @@ +# AKS Support in Aspire — Implementation Spec + +## Problem Statement + +Aspire's `Aspire.Hosting.Kubernetes` package currently supports end-to-end deployment to any conformant Kubernetes cluster (including AKS) via Helm charts. However, the support is **generic Kubernetes** — it has no awareness of Azure-specific capabilities. Users who want to deploy to AKS must manually provision the cluster, configure workload identity, set up monitoring, and manage networking outside of Aspire. + +The goal is to create a first-class AKS experience in Aspire that supports: +- **Provisioning the AKS cluster itself** via Azure.Provisioning +- **Workload identity** (Azure AD federated credentials for pods) +- **Azure Monitor integration** (Container Insights, Log Analytics, managed Prometheus/Grafana) +- **VNet integration** (subnet delegation, private clusters) +- **Network perimeter support** (NSP, private endpoints for backing Azure services) + +## Current State Analysis + +### Kubernetes Publishing (Aspire.Hosting.Kubernetes) +- **Helm-chart based** deployment model with 5-step pipeline: Publish → Prepare → Deploy → Summary → Uninstall +- `KubernetesEnvironmentResource` is the root compute environment +- `KubernetesResource` wraps each Aspire resource into Deployment/Service/ConfigMap/Secret YAML +- `HelmDeploymentEngine` executes `helm upgrade --install` +- No Azure awareness — works with any kubeconfig-accessible cluster +- Identity support: ❌ None +- Networking: Basic K8s Service/Ingress only +- Monitoring: OTLP to Aspire Dashboard only + +### Azure Provisioning Patterns (established) +- `AzureProvisioningResource` base class → generates Bicep via `Azure.Provisioning` SDK +- `AzureResourceInfrastructure` builder → `CreateExistingOrNewProvisionableResource()` factory +- `BicepOutputReference` for cross-resource wiring +- `AppIdentityAnnotation` + `IAppIdentityResource` for managed identity attachment +- Role assignments via `AddRoleAssignments()` / `IAddRoleAssignmentsContext` + +### Azure Container Apps (reference pattern) +- `AzureContainerAppEnvironmentResource` : `AzureProvisioningResource`, `IAzureComputeEnvironmentResource` +- Implements `IAzureContainerRegistry`, `IAzureDelegatedSubnetResource` +- Auto-creates Container Registry, Log Analytics, managed identity +- Subscribes to `BeforeStartEvent` → creates ContainerApp per compute resource → adds `DeploymentTargetAnnotation` + +### Azure Networking (established) +- VNet, Subnet, NSG, NAT Gateway, Private DNS Zone, Private Endpoint, Public IP resources +- `IAzurePrivateEndpointTarget` interface (implemented by Storage, SQL, etc.) +- `IAzureNspAssociationTarget` for network security perimeters +- `DelegatedSubnetAnnotation` for service delegation +- `PrivateEndpointTargetAnnotation` to deny public access + +### Azure Identity (established) +- `AzureUserAssignedIdentityResource` with Id, ClientId, PrincipalId outputs +- `AppIdentityAnnotation` attaches identity to compute resources +- Container Apps sets `AZURE_CLIENT_ID` + `AZURE_TOKEN_CREDENTIALS=ManagedIdentityCredential` +- **No workload identity or federated credential support** exists today + +### Azure Monitoring (established) +- `AzureLogAnalyticsWorkspaceResource` via `Azure.Provisioning.OperationalInsights` +- `AzureApplicationInsightsResource` via `Azure.Provisioning.ApplicationInsights` +- Container Apps links Log Analytics workspace to environment + +## Proposed Architecture + +### New Package: `Aspire.Hosting.Azure.Kubernetes` + +This package provides a unified `AddAzureKubernetesEnvironment()` entry point that internally invokes `AddKubernetesEnvironment()` (from the generic K8s package) and layers on AKS-specific Azure provisioning. This mirrors the established pattern of `AddAzureContainerAppEnvironment()` which internally sets up the Container Apps infrastructure. + +``` +Aspire.Hosting.Azure.Kubernetes +├── depends on: Aspire.Hosting.Kubernetes +├── depends on: Aspire.Hosting.Azure +├── depends on: Azure.Provisioning.Kubernetes (v1.0.0-beta.3) +├── depends on: Azure.Provisioning.Roles (for federated credentials) +├── depends on: Azure.Provisioning.Network (for VNet integration) +└── depends on: Azure.Provisioning.OperationalInsights (for monitoring) +``` + +### Design Principle: Unified Environment Resource + +Just as `AddAzureContainerAppEnvironment("aca")` creates a single resource that is both the Azure provisioning target AND the compute environment, `AddAzureKubernetesEnvironment("aks")` creates a single `AzureKubernetesEnvironmentResource` that: +1. Extends `AzureProvisioningResource` (generates Bicep for AKS cluster + supporting resources) +2. Implements `IAzureComputeEnvironmentResource` (serves as the compute target) +3. Internally creates and manages a `KubernetesEnvironmentResource` for Helm-based deployment +4. Registers the `KubernetesInfrastructure` eventing subscriber (same as `AddKubernetesEnvironment`) + +### Integration Points + +``` +┌─────────────────────────────────────────────────────────────┐ +│ User's AppHost │ +│ │ +│ var aks = builder.AddAzureKubernetesEnvironment("aks") │ +│ .WithDelegatedSubnet(subnet) │ +│ .WithAzureLogAnalyticsWorkspace(logAnalytics) │ +│ .WithWorkloadIdentity() │ +│ .WithVersion("1.30") │ +│ .WithHelm(...) ← from K8s environment │ +│ .WithDashboard(); ← from K8s environment │ +│ │ +│ var db = builder.AddAzureSqlServer("sql") │ +│ .WithPrivateEndpoint(subnet); ← existing pattern │ +│ │ +│ builder.AddProject() │ +│ .WithReference(db) │ +│ .WithAzureWorkloadIdentity(identity); ← new │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Detailed Design + +### 1. Unified AKS Environment Resource + +**New resource**: `AzureKubernetesEnvironmentResource` + +This is the single entry point — analogous to `AzureContainerAppEnvironmentResource`. It extends `AzureProvisioningResource` to generate Bicep for the AKS cluster and supporting infrastructure, while also serving as the compute environment by internally delegating to `KubernetesEnvironmentResource` for Helm-based deployment. + +```csharp +public class AzureKubernetesEnvironmentResource( + string name, + Action configureInfrastructure) + : AzureProvisioningResource(name, configureInfrastructure), + IAzureComputeEnvironmentResource, + IAzureContainerRegistry, // For ACR integration + IAzureDelegatedSubnetResource, // For VNet integration + IAzureNspAssociationTarget // For NSP association +{ + // The underlying generic K8s environment (created internally) + internal KubernetesEnvironmentResource KubernetesEnvironment { get; set; } = default!; + + // AKS cluster outputs + public BicepOutputReference Id => new("id", this); + public BicepOutputReference ClusterFqdn => new("clusterFqdn", this); + public BicepOutputReference OidcIssuerUrl => new("oidcIssuerUrl", this); + public BicepOutputReference KubeletIdentityObjectId => new("kubeletIdentityObjectId", this); + public BicepOutputReference NodeResourceGroup => new("nodeResourceGroup", this); + public BicepOutputReference NameOutputReference => new("name", this); + + // ACR outputs (like AzureContainerAppEnvironmentResource) + internal BicepOutputReference ContainerRegistryName => new("AZURE_CONTAINER_REGISTRY_NAME", this); + internal BicepOutputReference ContainerRegistryUrl => new("AZURE_CONTAINER_REGISTRY_ENDPOINT", this); + internal BicepOutputReference ContainerRegistryManagedIdentityId + => new("AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID", this); + + // Service delegation + string IAzureDelegatedSubnetResource.DelegatedSubnetServiceName + => "Microsoft.ContainerService/managedClusters"; + + // Configuration + internal string? KubernetesVersion { get; set; } + internal AksSkuTier SkuTier { get; set; } = AksSkuTier.Free; + internal bool OidcIssuerEnabled { get; set; } = true; + internal bool WorkloadIdentityEnabled { get; set; } = true; + internal AzureContainerRegistryResource? DefaultContainerRegistry { get; set; } + internal AzureLogAnalyticsWorkspaceResource? LogAnalyticsWorkspace { get; set; } + + // Node pool configuration + internal List NodePools { get; } = [ + new AksNodePoolConfig("system", "Standard_D4s_v5", 1, 3, AksNodePoolMode.System) + ]; + + // Networking + internal AksNetworkProfile? NetworkProfile { get; set; } + internal AzureSubnetResource? SubnetResource { get; set; } + internal bool IsPrivateCluster { get; set; } +} +``` + +**Entry point extension** (mirrors `AddAzureContainerAppEnvironment`): +```csharp +public static class AzureKubernetesEnvironmentExtensions +{ + public static IResourceBuilder AddAzureKubernetesEnvironment( + this IDistributedApplicationBuilder builder, + [ResourceName] string name) + { + // 1. Set up Azure provisioning infrastructure + builder.AddAzureProvisioning(); + builder.Services.Configure( + o => o.SupportsTargetedRoleAssignments = true); + + // 2. Register the AKS-specific infrastructure eventing subscriber + builder.Services.TryAddEventingSubscriber(); + + // 3. Also register the generic K8s infrastructure (for Helm chart generation) + builder.AddKubernetesInfrastructureCore(); + + // 4. Create the unified environment resource + var resource = new AzureKubernetesEnvironmentResource(name, ConfigureAksInfrastructure); + + // 5. Create the inner KubernetesEnvironmentResource (for Helm deployment) + resource.KubernetesEnvironment = new KubernetesEnvironmentResource($"{name}-k8s") + { + HelmChartName = builder.Environment.ApplicationName.ToHelmChartName(), + Dashboard = builder.CreateDashboard($"{name}-dashboard") + }; + + // 6. Auto-create ACR (like Container Apps does) + var acr = CreateDefaultContainerRegistry(builder, name); + resource.DefaultContainerRegistry = acr; + + return builder.AddResource(resource); + } + + // Configuration extensions + public static IResourceBuilder WithVersion( + this IResourceBuilder builder, string version); + public static IResourceBuilder WithSkuTier( + this IResourceBuilder builder, AksSkuTier tier); + public static IResourceBuilder WithNodePool( + this IResourceBuilder builder, + string name, string vmSize, int minCount, int maxCount, + AksNodePoolMode mode = AksNodePoolMode.User); + + // Networking (matches existing pattern: WithDelegatedSubnet) + public static IResourceBuilder WithDelegatedSubnet( + this IResourceBuilder builder, + IResourceBuilder subnet); + public static IResourceBuilder AsPrivateCluster( + this IResourceBuilder builder); + + // Identity + public static IResourceBuilder WithWorkloadIdentity( + this IResourceBuilder builder); + + // Monitoring (matches existing pattern: WithAzureLogAnalyticsWorkspace) + public static IResourceBuilder WithAzureLogAnalyticsWorkspace( + this IResourceBuilder builder, + IResourceBuilder workspaceBuilder); + public static IResourceBuilder WithContainerInsights( + this IResourceBuilder builder, + IResourceBuilder? logAnalytics = null); + + // Container Registry + public static IResourceBuilder WithContainerRegistry( + this IResourceBuilder builder, + IResourceBuilder registry); + + // Helm configuration (delegates to inner KubernetesEnvironmentResource) + public static IResourceBuilder WithHelm( + this IResourceBuilder builder, + Action configure); + public static IResourceBuilder WithDashboard( + this IResourceBuilder builder); +} +``` + +**`AzureKubernetesInfrastructure`** (eventing subscriber, mirrors `AzureContainerAppsInfrastructure`): +```csharp +internal sealed class AzureKubernetesInfrastructure( + ILogger logger, + DistributedApplicationExecutionContext executionContext) + : IDistributedApplicationEventingSubscriber +{ + private async Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken ct) + { + var aksEnvironments = @event.Model.Resources + .OfType().ToArray(); + + foreach (var environment in aksEnvironments) + { + foreach (var r in @event.Model.GetComputeResources()) + { + var computeEnv = r.GetComputeEnvironment(); + if (computeEnv is not null && computeEnv != environment) + continue; + + // 1. Process workload identity annotations + // → Generate federated credentials in Bicep + // → Add ServiceAccount + labels to Helm chart + + // 2. Create KubernetesResource via inner environment + // (delegates to existing KubernetesInfrastructure) + + // 3. Add DeploymentTargetAnnotation + r.Annotations.Add(new DeploymentTargetAnnotation(environment) + { + ContainerRegistry = environment, + ComputeEnvironment = environment + }); + } + } + } +} +``` + +**Bicep output**: The `ConfigureAksInfrastructure` callback uses `Azure.Provisioning.Kubernetes` to produce: +- `ManagedCluster` with system-assigned or user-assigned identity for the control plane +- OIDC issuer enabled (required for workload identity) +- Workload identity enabled on the cluster +- Azure CNI or Kubenet network profile (based on VNet configuration) +- Container Insights add-on profile (if monitoring configured) +- Node pools with autoscaler configuration +- ACR pull role assignment for kubelet identity +- Container Registry (auto-created or explicit) + +### 2. Workload Identity Support + +Workload identity enables pods to authenticate to Azure services using federated credentials without storing secrets. This requires three things: +1. A user-assigned managed identity in Azure +2. A Kubernetes service account annotated with the identity's client ID +3. A federated credential linking the identity to the K8s service account via OIDC + +**New types**: +```csharp +// Annotation to mark a compute resource for workload identity +public class AksWorkloadIdentityAnnotation( + IAppIdentityResource identityResource, + string? serviceAccountName = null) : IResourceAnnotation +{ + public IAppIdentityResource IdentityResource { get; } = identityResource; + public string? ServiceAccountName { get; set; } = serviceAccountName; +} +``` + +**Extension method** (on compute resources): +```csharp +public static IResourceBuilder WithAzureWorkloadIdentity( + this IResourceBuilder builder, + IResourceBuilder? identity = null) + where T : IResource +{ + // If no identity provided, auto-create one named "{resource}-identity" + // Add AksWorkloadIdentityAnnotation + // This will be picked up by the AKS infrastructure to: + // 1. Create a federated credential (Bicep) + // 2. Generate a ServiceAccount YAML with azure.workload.identity/client-id annotation + // 3. Add the azure.workload.identity/use: "true" label to the pod spec +} +``` + +**Integration with KubernetesInfrastructure**: +When the AKS environment processes a resource with `AksWorkloadIdentityAnnotation`, it: +1. **Bicep side**: Creates a `FederatedIdentityCredential` resource: + ```bicep + resource federatedCredential 'Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials@2023-01-31' = { + parent: identity + name: '${resourceName}-fedcred' + properties: { + issuer: aksCluster.properties.oidcIssuerProfile.issuerURL + subject: 'system:serviceaccount:${namespace}:${serviceAccountName}' + audiences: ['api://AzureADTokenExchange'] + } + } + ``` +2. **Helm chart side**: Generates a ServiceAccount and patches the pod template: + ```yaml + apiVersion: v1 + kind: ServiceAccount + metadata: + name: {{ .Values.parameters.myapi.serviceAccountName }} + annotations: + azure.workload.identity/client-id: {{ .Values.parameters.myapi.azureClientId }} + --- + # In the Deployment pod spec: + spec: + serviceAccountName: {{ .Values.parameters.myapi.serviceAccountName }} + labels: + azure.workload.identity/use: "true" + ``` + +**Key design decision**: The federated credential Bicep resource needs the OIDC issuer URL from the AKS cluster output. This creates a dependency ordering: +- AKS cluster must be provisioned first +- Then federated credentials can reference its OIDC issuer URL +- This is handled naturally by Bicep's dependency graph + +### 3. Monitoring Integration + +**Goal**: When monitoring is enabled on the AKS environment, provision: +- Container Insights (via AKS addon profile) with Log Analytics workspace +- Azure Monitor metrics profile (managed Prometheus) +- Optional: Azure Managed Grafana dashboard +- Optional: Application Insights for application-level telemetry + +**Design** (matches `WithAzureLogAnalyticsWorkspace` pattern from Container Apps): +```csharp +// Option 1: Explicit workspace (matches Container Apps naming exactly) +var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithAzureLogAnalyticsWorkspace(logAnalytics); + +// Option 2: Enable Container Insights (auto-creates workspace if not provided) +var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithContainerInsights(); + +// Option 3: Both — explicit workspace + Container Insights addon +var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithContainerInsights(logAnalytics); +``` + +**Bicep additions**: +- `addonProfiles.omsagent.enabled = true` with Log Analytics workspace ID +- `azureMonitorProfile.metrics.enabled = true` for managed Prometheus +- Data collection rule for container insights +- Optional: `AzureMonitorWorkspaceResource` for managed Prometheus + +**OTLP integration**: The existing Kubernetes publishing already injects `OTEL_EXPORTER_OTLP_ENDPOINT`. For AKS, we can optionally route OTLP to Application Insights via the connection string environment variable. + +### 4. VNet Integration + +AKS needs a subnet for its nodes (and optionally pods with Azure CNI Overlay). This uses the existing `WithDelegatedSubnet` pattern already established for Container Apps and other Azure compute resources. + +**Design** (uses the existing generic extension from `Aspire.Hosting.Azure.Network`): + +Since `AzureKubernetesEnvironmentResource` implements `IAzureDelegatedSubnetResource` with `DelegatedSubnetServiceName = "Microsoft.ContainerService/managedClusters"`, the existing `WithDelegatedSubnet()` extension method works directly: + +```csharp +// User code — uses the EXISTING WithDelegatedSubnet from Aspire.Hosting.Azure.Network +var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithDelegatedSubnet(aksSubnet); +``` + +The `ConfigureAksInfrastructure` callback reads the delegated subnet annotation and wires it into the `ManagedCluster` Bicep: +```bicep +resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-02-01' = { + properties: { + networkProfile: { + networkPlugin: 'azure' + networkPolicy: 'calico' + serviceCidr: '10.0.4.0/22' + dnsServiceIP: '10.0.4.10' + } + agentPoolProfiles: [{ + vnetSubnetID: subnet.id + }] + } +} +``` + +**Private cluster support**: +```csharp +public static IResourceBuilder AsPrivateCluster( + this IResourceBuilder builder) +{ + // Enable private cluster (API server behind private endpoint) + // Requires a delegated subnet to be configured + // Sets apiServerAccessProfile.enablePrivateCluster = true +} +``` + +### 5. Network Perimeter Support + +AKS backing Azure services (SQL, Storage, Key Vault) should be accessible via private endpoints within the cluster's VNet. + +**This largely uses existing infrastructure**: +```csharp +// User code in AppHost +var vnet = builder.AddAzureVirtualNetwork("vnet"); +var aksSubnet = vnet.AddSubnet("aks-subnet", "10.0.0.0/22"); +var peSubnet = vnet.AddSubnet("pe-subnet", "10.0.4.0/24"); + +var aks = builder.AddAzureKubernetesService("aks") + .WithVirtualNetwork(aksSubnet); + +var sql = builder.AddAzureSqlServer("sql") + .WithPrivateEndpoint(peSubnet); // existing pattern + +// The SQL private endpoint is in the same VNet as AKS +// DNS resolution via Private DNS Zone (existing pattern) enables pod → SQL connectivity +``` + +**New consideration**: When AKS is configured with a VNet and backing services have private endpoints, the AKS infrastructure should verify or configure: +- Private DNS Zone links to the AKS VNet (so pods can resolve private endpoint DNS) +- This may need a new extension or automatic wiring + +### 6. Deployment Pipeline Integration + +Since `AzureKubernetesEnvironmentResource` unifies both Azure provisioning and K8s deployment, the pipeline is a superset of both: + +``` +[Azure Provisioning Phase] [Kubernetes Deployment Phase] +1. Generate Bicep (AKS + ACR + 4. Publish Helm chart + identity + fedcreds) 5. Get kubeconfig from AKS (az aks get-credentials) +2. Deploy Bicep via azd 6. Push images to ACR +3. Capture outputs (OIDC URL, 7. Prepare Helm values (resolve AKS outputs) + ACR endpoint, etc.) 8. helm upgrade --install + 9. Print summary + 10. (Optional) Uninstall +``` + +The Azure provisioning happens first (via `AzureEnvironmentResource` / `AzureProvisioner`), then the Kubernetes Helm deployment pipeline steps execute against the provisioned cluster. The kubeconfig step bridges the two phases — it uses the AKS cluster name from Bicep outputs to call `az aks get-credentials`. + +This is implemented by adding AKS-specific `DeploymentEngineStepsFactory` entries to the inner `KubernetesEnvironmentResource`: +```csharp +// In AddAzureKubernetesEnvironment, after AKS provisioning completes: +resource.KubernetesEnvironment.AddDeploymentEngineStep( + "get-kubeconfig", + async (context, ct) => + { + // Use AKS outputs to fetch kubeconfig + var clusterName = await resource.NameOutputReference.GetValueAsync(ct); + var resourceGroup = await resource.ResourceGroupOutput.GetValueAsync(ct); + // az aks get-credentials --resource-group {rg} --name {name} + }); +``` + +### 7. Container Registry Integration + +AKS needs a container registry for application images. Options: +1. **Auto-create ACR** when AKS is added (like Container Apps does) +2. **Bring your own ACR** via `.WithContainerRegistry()` +3. **Use existing ACR** via `AsExisting()` pattern + +```csharp +// Auto-create (default) +var aks = builder.AddAzureKubernetesService("aks"); +// → auto-creates ACR, attaches AcrPull role to kubelet identity + +// Explicit +var acr = builder.AddAzureContainerRegistry("acr"); +var aks = builder.AddAzureKubernetesService("aks") + .WithContainerRegistry(acr); +``` + +**Role assignment**: The AKS kubelet managed identity needs `AcrPull` role on the registry. + +## Open Questions + +1. **`Azure.Provisioning.Kubernetes` readiness**: The package is at v1.0.0-beta.3. We need to verify it has the types we need (`ManagedCluster`, `AgentPool`, `OidcIssuerProfile`, `WorkloadIdentity` flags, etc.) and assess stability risk. + +2. **Existing cluster support**: Should we support `AsExisting()` for AKS (reference a pre-provisioned cluster)? + - **Recommendation**: Yes, this is a common scenario. Use the established `ExistingAzureResourceAnnotation` pattern. + +3. **Managed Grafana**: Should `WithMonitoring()` also provision Azure Managed Grafana? + - Could be a separate `.WithGrafana()` extension to keep it opt-in. + +4. **Ingress controller**: Should Aspire configure an ingress controller (NGINX, Traefik, or Application Gateway Ingress Controller)? + - Application Gateway Ingress Controller (AGIC) would be the Azure-native choice. + - Could be opt-in via `.WithApplicationGatewayIngress()`. + +5. **DNS integration**: Should external endpoints auto-configure Azure DNS zones? + - Probably out of scope for v1. + +6. **Deployment mode**: For publish, should AKS support work with `aspire publish` only, or also `aspire run` (local dev with AKS)? + - Recommendation: `aspire publish` first. Local dev uses the generic K8s environment with local/kind clusters. + +7. **Multi-cluster**: Should we support multiple AKS environments in one AppHost? + - The `KubernetesEnvironmentResource` model already supports this conceptually. + +8. **Helm config delegation**: How cleanly can `WithHelm()` / `WithDashboard()` be forwarded from `AzureKubernetesEnvironmentResource` to the inner `KubernetesEnvironmentResource`? Should the inner resource be exposed or kept fully internal? + +## Implementation Phases + +### Phase 1: Unified AKS Environment (Foundation) +- Create `Aspire.Hosting.Azure.Kubernetes` package with dependency on `Azure.Provisioning.Kubernetes` +- `AzureKubernetesEnvironmentResource` combining Azure provisioning + K8s compute environment +- `AddAzureKubernetesEnvironment()` entry point (calls `AddKubernetesInfrastructureCore` internally) +- `AzureKubernetesInfrastructure` eventing subscriber +- Basic cluster Bicep generation (version, SKU, default node pool) +- ACR auto-creation and AcrPull role assignment for kubelet identity +- Kubeconfig retrieval pipeline step +- `AsExisting()` support for bring-your-own AKS +- Helm config delegation (`WithHelm()`, `WithDashboard()`) + +### Phase 2: Workload Identity +- `AksWorkloadIdentityAnnotation` +- `WithAzureWorkloadIdentity()` extension on compute resources +- Federated credential Bicep generation (using AKS OIDC issuer URL output) +- ServiceAccount YAML generation in Helm chart +- Pod label injection (`azure.workload.identity/use`) +- Integration with existing `AppIdentityAnnotation` / `IAppIdentityResource` pattern + +### Phase 3: Networking +- `WithDelegatedSubnet()` — uses existing generic extension from `Aspire.Hosting.Azure.Network` (since resource implements `IAzureDelegatedSubnetResource`) +- Azure CNI network profile configuration +- `AsPrivateCluster()` for private API server +- Private DNS Zone link verification for backing service private endpoints + +### Phase 4: Monitoring +- `WithAzureLogAnalyticsWorkspace()` — matches Container Apps naming convention +- `WithContainerInsights()` — AKS-specific addon with optional Log Analytics auto-create +- Container Insights addon profile +- Azure Monitor metrics profile +- Optional Application Insights OTLP integration +- Data collection rule configuration + +### Phase 5: Network Perimeter +- NSP association support (`IAzureNspAssociationTarget` on AKS) +- Private DNS Zone auto-linking when backing services have private endpoints in same VNet +- Network policy integration + +## Dependencies / Prerequisites + +- `Azure.Provisioning.Kubernetes` NuGet package (v1.0.0-beta.3 — need to add to `Directory.Packages.props`) +- `Azure.Provisioning.ContainerRegistry` (already used, v1.1.0) +- `Azure.Provisioning.Network` (already used, v1.1.0-beta.2) +- `Azure.Provisioning.OperationalInsights` (already used, v1.1.0) +- `Azure.Provisioning.Roles` (already used, for identity/RBAC) +- `Aspire.Hosting.Kubernetes` (the generic K8s package, already in repo) + +## Testing Strategy + +- **Unit tests**: Bicep template generation verification (snapshot tests like existing K8s tests) +- **Integration tests**: Verify Helm chart output includes ServiceAccount, labels, etc. +- **E2E tests**: Provision AKS + deploy workloads (expensive, CI-gated) +- **Existing test patterns**: Follow `Aspire.Hosting.Kubernetes.Tests` structure + +## Todos + +1. **aks-package-setup**: Create `Aspire.Hosting.Azure.Kubernetes` project, csproj, dependencies (including `Azure.Provisioning.Kubernetes`), add to `Directory.Packages.props` +2. **aks-environment-resource**: Implement `AzureKubernetesEnvironmentResource` with Bicep provisioning via `Azure.Provisioning.Kubernetes.ManagedCluster`, ACR auto-creation, and inner `KubernetesEnvironmentResource` +3. **aks-extensions**: Implement `AddAzureKubernetesEnvironment()` entry point and configuration extensions (`WithVersion`, `WithSkuTier`, `WithNodePool`, `WithHelm`, `WithDashboard`, `WithContainerRegistry`) +4. **aks-infrastructure**: Implement `AzureKubernetesInfrastructure` eventing subscriber — process compute resources, add `DeploymentTargetAnnotation`, handle kubeconfig retrieval pipeline step +5. **workload-identity-annotation**: `AksWorkloadIdentityAnnotation` and `WithAzureWorkloadIdentity()` extension method on compute resources. Auto-create identity if not provided. +6. **workload-identity-bicep**: Generate `FederatedIdentityCredential` Bicep resource linking managed identity to K8s service account via OIDC issuer URL output +7. **workload-identity-helm**: Generate ServiceAccount YAML with `azure.workload.identity/client-id` annotation. Add `azure.workload.identity/use` label to pod spec. +8. **vnet-integration**: `WithDelegatedSubnet()` — leverages existing `IAzureDelegatedSubnetResource` pattern. Azure CNI network profile. Subnet delegation for `Microsoft.ContainerService/managedClusters`. +9. **private-cluster**: `AsPrivateCluster()` extension. Sets `apiServerAccessProfile.enablePrivateCluster`. Requires delegated subnet. +10. **monitoring**: `WithAzureLogAnalyticsWorkspace()` (matches Container Apps naming) + `WithContainerInsights()` (AKS addon). Log Analytics auto-create, Azure Monitor metrics, data collection rules. +11. **nsp-support**: Implement `IAzureNspAssociationTarget` on AKS resource. Auto-link Private DNS Zones when backing services have private endpoints in same VNet. +12. **existing-cluster**: `AsExisting()` support for referencing pre-provisioned AKS clusters via `ExistingAzureResourceAnnotation` pattern. +13. **tests**: Unit tests (Bicep snapshot verification), integration tests (Helm chart output), E2E tests (provision + deploy). From c5fac1e64eabcae4916fa17440a356aac83a8ab8 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Sun, 12 Apr 2026 21:31:52 +1000 Subject: [PATCH 03/78] Add unit tests for AzureKubernetesEnvironment - Bicep snapshot verification tests - Configuration extension tests (version, SKU, node pools, private cluster) - Monitoring integration tests (Container Insights, Log Analytics) - Argument validation tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Aspire.slnx | 1 + ...pire.Hosting.Azure.Kubernetes.Tests.csproj | 22 ++ ...ureKubernetesEnvironmentExtensionsTests.cs | 190 ++++++++++++++++++ ...ironment_BasicConfiguration.verified.bicep | 14 ++ ...etesEnvironment_WithVersion.verified.bicep | 14 ++ 5 files changed, 241 insertions(+) create mode 100644 tests/Aspire.Hosting.Azure.Kubernetes.Tests/Aspire.Hosting.Azure.Kubernetes.Tests.csproj create mode 100644 tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs create mode 100644 tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep create mode 100644 tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep diff --git a/Aspire.slnx b/Aspire.slnx index 4143c90b474..4cb8040f922 100644 --- a/Aspire.slnx +++ b/Aspire.slnx @@ -473,6 +473,7 @@ + diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Aspire.Hosting.Azure.Kubernetes.Tests.csproj b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Aspire.Hosting.Azure.Kubernetes.Tests.csproj new file mode 100644 index 00000000000..eb9779ffcbd --- /dev/null +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Aspire.Hosting.Azure.Kubernetes.Tests.csproj @@ -0,0 +1,22 @@ + + + + $(DefaultTargetFramework) + + + + + + + + + + + + + + + + + + diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs new file mode 100644 index 00000000000..030100394b8 --- /dev/null +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs @@ -0,0 +1,190 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Hosting.Azure.Kubernetes; +using Aspire.Hosting.Utils; + +namespace Aspire.Hosting.Azure.Tests; + +public class AzureKubernetesEnvironmentExtensionsTests +{ + [Fact] + public async Task AddAzureKubernetesEnvironment_BasicConfiguration() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + Assert.Equal("aks", aks.Resource.Name); + Assert.Equal("{aks.outputs.id}", aks.Resource.Id.ValueExpression); + Assert.Equal("{aks.outputs.clusterFqdn}", aks.Resource.ClusterFqdn.ValueExpression); + Assert.Equal("{aks.outputs.oidcIssuerUrl}", aks.Resource.OidcIssuerUrl.ValueExpression); + + var manifest = await AzureManifestUtils.GetManifestWithBicep(aks.Resource); + await Verify(manifest.BicepText, extension: "bicep"); + } + + [Fact] + public async Task AddAzureKubernetesEnvironment_WithVersion() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithVersion("1.30"); + + Assert.Equal("1.30", aks.Resource.KubernetesVersion); + + var manifest = await AzureManifestUtils.GetManifestWithBicep(aks.Resource); + await Verify(manifest.BicepText, extension: "bicep"); + } + + [Fact] + public void AddAzureKubernetesEnvironment_WithSkuTier() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithSkuTier(AksSkuTier.Standard); + + Assert.Equal(AksSkuTier.Standard, aks.Resource.SkuTier); + } + + [Fact] + public void AddAzureKubernetesEnvironment_WithNodePool() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithNodePool("gpu", "Standard_NC6s_v3", 0, 5); + + // Default system pool + added user pool + Assert.Equal(2, aks.Resource.NodePools.Count); + + var userPool = aks.Resource.NodePools[1]; + Assert.Equal("gpu", userPool.Name); + Assert.Equal("Standard_NC6s_v3", userPool.VmSize); + Assert.Equal(0, userPool.MinCount); + Assert.Equal(5, userPool.MaxCount); + Assert.Equal(AksNodePoolMode.User, userPool.Mode); + } + + [Fact] + public void AddAzureKubernetesEnvironment_AsPrivateCluster() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks") + .AsPrivateCluster(); + + Assert.True(aks.Resource.IsPrivateCluster); + } + + [Fact] + public void AddAzureKubernetesEnvironment_WithContainerInsights() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithContainerInsights(); + + Assert.True(aks.Resource.ContainerInsightsEnabled); + Assert.Null(aks.Resource.LogAnalyticsWorkspace); + } + + [Fact] + public void AddAzureKubernetesEnvironment_WithContainerInsightsAndWorkspace() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var logAnalytics = builder.AddAzureLogAnalyticsWorkspace("law"); + var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithContainerInsights(logAnalytics); + + Assert.True(aks.Resource.ContainerInsightsEnabled); + Assert.Same(logAnalytics.Resource, aks.Resource.LogAnalyticsWorkspace); + } + + [Fact] + public void AddAzureKubernetesEnvironment_WithAzureLogAnalyticsWorkspace() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var logAnalytics = builder.AddAzureLogAnalyticsWorkspace("law"); + var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithAzureLogAnalyticsWorkspace(logAnalytics); + + Assert.Same(logAnalytics.Resource, aks.Resource.LogAnalyticsWorkspace); + } + + [Fact] + public void AddAzureKubernetesEnvironment_DefaultNodePool() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + Assert.Single(aks.Resource.NodePools); + var defaultPool = aks.Resource.NodePools[0]; + Assert.Equal("system", defaultPool.Name); + Assert.Equal("Standard_D4s_v5", defaultPool.VmSize); + Assert.Equal(1, defaultPool.MinCount); + Assert.Equal(3, defaultPool.MaxCount); + Assert.Equal(AksNodePoolMode.System, defaultPool.Mode); + } + + [Fact] + public void AddAzureKubernetesEnvironment_DefaultConfiguration() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + Assert.True(aks.Resource.OidcIssuerEnabled); + Assert.True(aks.Resource.WorkloadIdentityEnabled); + Assert.Equal(AksSkuTier.Free, aks.Resource.SkuTier); + Assert.Null(aks.Resource.KubernetesVersion); + Assert.False(aks.Resource.IsPrivateCluster); + Assert.False(aks.Resource.ContainerInsightsEnabled); + Assert.Null(aks.Resource.LogAnalyticsWorkspace); + } + + [Fact] + public void AddAzureKubernetesEnvironment_HasInternalKubernetesEnvironment() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + Assert.NotNull(aks.Resource.KubernetesEnvironment); + Assert.Equal("aks-k8s", aks.Resource.KubernetesEnvironment.Name); + } + + [Fact] + public void AddAzureKubernetesEnvironment_ThrowsOnNullBuilder() + { + IDistributedApplicationBuilder builder = null!; + + Assert.Throws(() => + builder.AddAzureKubernetesEnvironment("aks")); + } + + [Fact] + public void AddAzureKubernetesEnvironment_ThrowsOnEmptyName() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + Assert.Throws(() => + builder.AddAzureKubernetesEnvironment("")); + } + + [Fact] + public void WithVersion_ThrowsOnEmptyVersion() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + Assert.Throws(() => + aks.WithVersion("")); + } +} diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep new file mode 100644 index 00000000000..89d1967adc5 --- /dev/null +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep @@ -0,0 +1,14 @@ +@description('The location for the resource(s) to be deployed.') +param location string = resourceGroup().location + +output id string = id + +output name string = name + +output clusterFqdn string = clusterFqdn + +output oidcIssuerUrl string = oidcIssuerUrl + +output kubeletIdentityObjectId string = kubeletIdentityObjectId + +output nodeResourceGroup string = nodeResourceGroup \ No newline at end of file diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep new file mode 100644 index 00000000000..89d1967adc5 --- /dev/null +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep @@ -0,0 +1,14 @@ +@description('The location for the resource(s) to be deployed.') +param location string = resourceGroup().location + +output id string = id + +output name string = name + +output clusterFqdn string = clusterFqdn + +output oidcIssuerUrl string = oidcIssuerUrl + +output kubeletIdentityObjectId string = kubeletIdentityObjectId + +output nodeResourceGroup string = nodeResourceGroup \ No newline at end of file From 6e06f0ec65744ca563c250583dcbac146f711481 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Sun, 12 Apr 2026 21:37:34 +1000 Subject: [PATCH 04/78] Add workload identity, VNet delegation, and NSP support - AzureKubernetesEnvironmentResource now implements IAzureDelegatedSubnetResource and IAzureNspAssociationTarget for VNet and network perimeter integration - WithWorkloadIdentity() on AKS environment enables OIDC and workload identity - WithAzureWorkloadIdentity() on compute resources for federated credential setup with auto-create identity support - AksWorkloadIdentityAnnotation for ServiceAccount YAML generation - AsExisting() works automatically via AzureProvisioningResource base class - Additional unit tests for all new functionality Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AksWorkloadIdentityAnnotation.cs | 27 +++++++ .../AzureKubernetesEnvironmentExtensions.cs | 62 +++++++++++++++ .../AzureKubernetesEnvironmentResource.cs | 10 ++- ...ureKubernetesEnvironmentExtensionsTests.cs | 78 +++++++++++++++++++ 4 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 src/Aspire.Hosting.Azure.Kubernetes/AksWorkloadIdentityAnnotation.cs diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksWorkloadIdentityAnnotation.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksWorkloadIdentityAnnotation.cs new file mode 100644 index 00000000000..b9b65911e64 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AksWorkloadIdentityAnnotation.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Hosting.ApplicationModel; + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Annotation that marks a compute resource for AKS workload identity. +/// When present, the AKS infrastructure will generate a Kubernetes ServiceAccount +/// with the appropriate annotations and a federated identity credential in Azure. +/// +internal sealed class AksWorkloadIdentityAnnotation( + IAppIdentityResource identityResource, + string? serviceAccountName = null) : IResourceAnnotation +{ + /// + /// Gets the identity resource to federate with. + /// + public IAppIdentityResource IdentityResource { get; } = identityResource; + + /// + /// Gets or sets the Kubernetes service account name. + /// If null, defaults to the resource name. + /// + public string? ServiceAccountName { get; set; } = serviceAccountName; +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 55681ca1d02..445a3cd6554 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -188,6 +188,68 @@ public static IResourceBuilder WithAzureLogA return builder; } + /// + /// Enables workload identity on the AKS environment, allowing pods to authenticate + /// to Azure services using federated credentials. + /// + /// The resource builder. + /// A reference to the for chaining. + /// + /// This ensures the AKS cluster is configured with OIDC issuer and workload identity enabled. + /// Use on individual compute resources to assign + /// specific managed identities. + /// + [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + public static IResourceBuilder WithWorkloadIdentity( + this IResourceBuilder builder) + { + ArgumentNullException.ThrowIfNull(builder); + + builder.Resource.OidcIssuerEnabled = true; + builder.Resource.WorkloadIdentityEnabled = true; + return builder; + } + + /// + /// Configures a compute resource to use AKS workload identity with the specified managed identity. + /// This generates a Kubernetes ServiceAccount and a federated identity credential in Azure. + /// + /// The type of the compute resource. + /// The resource builder. + /// The managed identity to federate with. If null, an identity will be auto-created. + /// A reference to the for chaining. + /// + /// + /// var identity = builder.AddAzureUserAssignedIdentity("myIdentity"); + /// builder.AddProject<MyApi>() + /// .WithAzureWorkloadIdentity(identity); + /// + /// + [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + public static IResourceBuilder WithAzureWorkloadIdentity( + this IResourceBuilder builder, + IResourceBuilder? identity = null) + where T : IResource + { + ArgumentNullException.ThrowIfNull(builder); + + if (identity is null) + { + // Auto-create an identity named after the resource + var appBuilder = builder.ApplicationBuilder; + var identityName = $"{builder.Resource.Name}-identity"; + var identityBuilder = appBuilder.AddAzureUserAssignedIdentity(identityName); + identity = identityBuilder; + } + + // Add both the standard AppIdentityAnnotation (for Azure service role assignments) + // and the AKS-specific annotation (for ServiceAccount + federated credential generation) + builder.WithAnnotation(new AppIdentityAnnotation(identity.Resource)); + builder.WithAnnotation(new AksWorkloadIdentityAnnotation(identity.Resource)); + + return builder; + } + // NOTE: Full AKS Bicep provisioning with ContainerServiceManagedCluster types requires // the Azure.Provisioning.ContainerService package (currently unavailable in internal feeds). // This implementation uses ProvisioningOutput placeholders. When the package becomes available, diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs index 5d067f3fec2..ff4592f2c54 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#pragma warning disable ASPIREAZURE003 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + using Aspire.Hosting.Azure.Kubernetes; using Aspire.Hosting.Kubernetes; @@ -16,8 +18,14 @@ public class AzureKubernetesEnvironmentResource( string name, Action configureInfrastructure) : AzureProvisioningResource(name, configureInfrastructure), - IAzureComputeEnvironmentResource + IAzureComputeEnvironmentResource, + IAzureDelegatedSubnetResource, + IAzureNspAssociationTarget { + /// + string IAzureDelegatedSubnetResource.DelegatedSubnetServiceName + => "Microsoft.ContainerService/managedClusters"; + /// /// Gets the underlying Kubernetes environment resource used for Helm-based deployment. /// diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs index 030100394b8..587569047b3 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs @@ -1,6 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#pragma warning disable ASPIREAZURE003 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + +using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Azure.Kubernetes; using Aspire.Hosting.Utils; @@ -187,4 +190,79 @@ public void WithVersion_ThrowsOnEmptyVersion() Assert.Throws(() => aks.WithVersion("")); } + + [Fact] + public void WithWorkloadIdentity_EnablesOidcAndWorkloadIdentity() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithWorkloadIdentity(); + + Assert.True(aks.Resource.OidcIssuerEnabled); + Assert.True(aks.Resource.WorkloadIdentityEnabled); + } + + [Fact] + public void WithAzureWorkloadIdentity_AddsAnnotations() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var identity = builder.AddAzureUserAssignedIdentity("myIdentity"); + + var project = builder.AddContainer("myapi", "myimage") + .WithAzureWorkloadIdentity(identity); + + Assert.True(project.Resource.TryGetLastAnnotation(out var appIdentity)); + Assert.Same(identity.Resource, appIdentity.IdentityResource); + + Assert.True(project.Resource.TryGetLastAnnotation(out var aksIdentity)); + Assert.Same(identity.Resource, aksIdentity.IdentityResource); + } + + [Fact] + public void WithAzureWorkloadIdentity_AutoCreatesIdentity() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + var project = builder.AddContainer("myapi", "myimage") + .WithAzureWorkloadIdentity(); + + Assert.True(project.Resource.TryGetLastAnnotation(out _)); + Assert.True(project.Resource.TryGetLastAnnotation(out var aksIdentity)); + Assert.NotNull(aksIdentity.IdentityResource); + } + + [Fact] + public void AzureKubernetesEnvironment_ImplementsIAzureComputeEnvironmentResource() + { + using var builder = TestDistributedApplicationBuilder.Create(); + var aks = builder.AddAzureKubernetesEnvironment("aks"); + Assert.IsAssignableFrom(aks.Resource); + } + + [Fact] + public void AzureKubernetesEnvironment_ImplementsIAzureNspAssociationTarget() + { + using var builder = TestDistributedApplicationBuilder.Create(); + var aks = builder.AddAzureKubernetesEnvironment("aks"); + Assert.IsAssignableFrom(aks.Resource); + } + + [Fact] + public void AsExisting_WorksOnAksResource() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var nameParam = builder.AddParameter("aks-name"); + var rgParam = builder.AddParameter("aks-rg"); + + var aks = builder.AddAzureKubernetesEnvironment("aks") + .AsExisting(nameParam, rgParam); + + Assert.NotNull(aks); + } } From 2160d3d46ac41c011e5e142f0426660928db6bf7 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Sun, 12 Apr 2026 21:56:54 +1000 Subject: [PATCH 05/78] Add AddNodePool and WithNodePoolAffinity for workload scheduling - Changed WithNodePool to AddNodePool returning IResourceBuilder - AksNodePoolResource is a child resource (IResourceWithParent) of AKS environment - WithNodePoolAffinity extension lets compute resources target specific node pools - AksNodePoolAffinityAnnotation carries scheduling info for Helm chart nodeSelector - Made AksNodePoolConfig and AksNodePoolMode public (exposed via AksNodePoolResource) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AksNodePoolAffinityAnnotation.cs | 19 ++++++ .../AksNodePoolConfig.cs | 4 +- .../AksNodePoolResource.cs | 30 +++++++++ .../AzureKubernetesEnvironmentExtensions.cs | 65 +++++++++++++++++-- ...ureKubernetesEnvironmentExtensionsTests.cs | 50 +++++++++++--- 5 files changed, 151 insertions(+), 17 deletions(-) create mode 100644 src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolAffinityAnnotation.cs create mode 100644 src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolResource.cs diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolAffinityAnnotation.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolAffinityAnnotation.cs new file mode 100644 index 00000000000..1ab0e8d8372 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolAffinityAnnotation.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Hosting.ApplicationModel; + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Annotation that associates a compute resource with a specific AKS node pool. +/// When present, the Kubernetes deployment will include a nodeSelector targeting +/// the specified node pool via the agentpool label. +/// +internal sealed class AksNodePoolAffinityAnnotation(AksNodePoolResource nodePool) : IResourceAnnotation +{ + /// + /// Gets the node pool to schedule the workload on. + /// + public AksNodePoolResource NodePool { get; } = nodePool; +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolConfig.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolConfig.cs index 5af798f6d6c..5fc3016f1f8 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolConfig.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolConfig.cs @@ -11,7 +11,7 @@ namespace Aspire.Hosting.Azure.Kubernetes; /// The minimum number of nodes. /// The maximum number of nodes. /// The mode of the node pool. -internal sealed record AksNodePoolConfig( +public sealed record AksNodePoolConfig( string Name, string VmSize, int MinCount, @@ -21,7 +21,7 @@ internal sealed record AksNodePoolConfig( /// /// Specifies the mode of an AKS node pool. /// -internal enum AksNodePoolMode +public enum AksNodePoolMode { /// /// System node pool for hosting system pods. diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolResource.cs new file mode 100644 index 00000000000..af0a95f045b --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolResource.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Hosting.ApplicationModel; + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Represents an AKS node pool as a child resource of an . +/// Node pools can be referenced by compute resources to schedule workloads on specific node pools +/// using . +/// +/// The name of the node pool resource. +/// The node pool configuration. +/// The parent AKS environment resource. +public class AksNodePoolResource( + string name, + AksNodePoolConfig config, + AzureKubernetesEnvironmentResource parent) : Resource(name), IResourceWithParent +{ + /// + /// Gets the parent AKS environment resource. + /// + public AzureKubernetesEnvironmentResource Parent { get; } = parent ?? throw new ArgumentNullException(nameof(parent)); + + /// + /// Gets the node pool configuration. + /// + public AksNodePoolConfig Config { get; } = config ?? throw new ArgumentNullException(nameof(config)); +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 445a3cd6554..636341fff54 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -109,16 +109,29 @@ public static IResourceBuilder WithSkuTier( /// /// Adds a node pool to the AKS cluster. /// - /// The resource builder. + /// The AKS environment resource builder. /// The name of the node pool. /// The VM size for nodes. /// The minimum node count for autoscaling. /// The maximum node count for autoscaling. - /// A reference to the for chaining. + /// A reference to the for the new node pool. + /// + /// The returned node pool resource can be passed to + /// on compute resources to schedule workloads on this pool. + /// + /// + /// + /// var aks = builder.AddAzureKubernetesEnvironment("aks"); + /// var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); + /// + /// builder.AddProject<MyApi>() + /// .WithNodePoolAffinity(gpuPool); + /// + /// [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] - public static IResourceBuilder WithNodePool( + public static IResourceBuilder AddNodePool( this IResourceBuilder builder, - string name, + [ResourceName] string name, string vmSize, int minCount, int maxCount) @@ -127,8 +140,48 @@ public static IResourceBuilder WithNodePool( ArgumentException.ThrowIfNullOrEmpty(name); ArgumentException.ThrowIfNullOrEmpty(vmSize); - builder.Resource.NodePools.Add( - new AksNodePoolConfig(name, vmSize, minCount, maxCount, AksNodePoolMode.User)); + var config = new AksNodePoolConfig(name, vmSize, minCount, maxCount, AksNodePoolMode.User); + builder.Resource.NodePools.Add(config); + + var nodePool = new AksNodePoolResource(name, config, builder.Resource); + + if (builder.ApplicationBuilder.ExecutionContext.IsRunMode) + { + return builder.ApplicationBuilder.CreateResourceBuilder(nodePool); + } + + return builder.ApplicationBuilder.AddResource(nodePool) + .ExcludeFromManifest(); + } + + /// + /// Schedules a compute resource's workload on the specified AKS node pool. + /// This translates to a Kubernetes nodeSelector with the agentpool label + /// targeting the named node pool. + /// + /// The type of the compute resource. + /// The resource builder. + /// The node pool to schedule the workload on. + /// A reference to the for chaining. + /// + /// + /// var aks = builder.AddAzureKubernetesEnvironment("aks"); + /// var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); + /// + /// builder.AddProject<MyApi>() + /// .WithNodePoolAffinity(gpuPool); + /// + /// + [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + public static IResourceBuilder WithNodePoolAffinity( + this IResourceBuilder builder, + IResourceBuilder nodePool) + where T : IResource + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(nodePool); + + builder.WithAnnotation(new AksNodePoolAffinityAnnotation(nodePool.Resource)); return builder; } diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs index 587569047b3..b7d9f0c3d92 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs @@ -53,22 +53,23 @@ public void AddAzureKubernetesEnvironment_WithSkuTier() } [Fact] - public void AddAzureKubernetesEnvironment_WithNodePool() + public void AddNodePool_ReturnsNodePoolResource() { using var builder = TestDistributedApplicationBuilder.Create(); - var aks = builder.AddAzureKubernetesEnvironment("aks") - .WithNodePool("gpu", "Standard_NC6s_v3", 0, 5); + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); // Default system pool + added user pool Assert.Equal(2, aks.Resource.NodePools.Count); - var userPool = aks.Resource.NodePools[1]; - Assert.Equal("gpu", userPool.Name); - Assert.Equal("Standard_NC6s_v3", userPool.VmSize); - Assert.Equal(0, userPool.MinCount); - Assert.Equal(5, userPool.MaxCount); - Assert.Equal(AksNodePoolMode.User, userPool.Mode); + Assert.Equal("gpu", gpuPool.Resource.Name); + Assert.Equal("gpu", gpuPool.Resource.Config.Name); + Assert.Equal("Standard_NC6s_v3", gpuPool.Resource.Config.VmSize); + Assert.Equal(0, gpuPool.Resource.Config.MinCount); + Assert.Equal(5, gpuPool.Resource.Config.MaxCount); + Assert.Equal(AksNodePoolMode.User, gpuPool.Resource.Config.Mode); + Assert.Same(aks.Resource, gpuPool.Resource.Parent); } [Fact] @@ -265,4 +266,35 @@ public void AsExisting_WorksOnAksResource() Assert.NotNull(aks); } + + [Fact] + public void WithNodePoolAffinity_AddsAnnotation() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); + + var container = builder.AddContainer("myapi", "myimage") + .WithNodePoolAffinity(gpuPool); + + Assert.True(container.Resource.TryGetLastAnnotation(out var affinity)); + Assert.Same(gpuPool.Resource, affinity.NodePool); + Assert.Equal("gpu", affinity.NodePool.Config.Name); + } + + [Fact] + public void AddNodePool_MultiplePoolsSupported() + { + using var builder = TestDistributedApplicationBuilder.Create(); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var pool1 = aks.AddNodePool("cpu", "Standard_D4s_v5", 1, 10); + var pool2 = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); + + // Default system pool + 2 user pools + Assert.Equal(3, aks.Resource.NodePools.Count); + Assert.Equal("cpu", pool1.Resource.Name); + Assert.Equal("gpu", pool2.Resource.Name); + } } From 6f4c075ba467ea36e07d3326c9a7c8b732edc35f Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Sun, 12 Apr 2026 22:02:56 +1000 Subject: [PATCH 06/78] Auto-create default user node pool when none configured When no user node pool is explicitly added via AddNodePool(), the AzureKubernetesInfrastructure subscriber creates a default 'workload' user pool (Standard_D4s_v5, 1-10 nodes) during BeforeStartEvent. Compute resources without explicit WithNodePoolAffinity() are automatically assigned to the first available user pool (either explicitly created or the auto-generated default). This ensures workloads are never scheduled on the system pool, which should only run system pods (kube-system, etc.). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesInfrastructure.cs | 46 ++++++++ .../AzureKubernetesInfrastructureTests.cs | 104 ++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index a57b86edf5f..5ece0335105 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -42,6 +42,11 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance { logger.LogInformation("Processing AKS environment '{Name}'", environment.Name); + // Ensure a default user node pool exists for workload scheduling. + // The system pool should only run system pods; application workloads + // need a user pool. + var defaultUserPool = EnsureDefaultUserNodePool(environment); + foreach (var r in @event.Model.GetComputeResources()) { var resourceComputeEnvironment = r.GetComputeEnvironment(); @@ -50,6 +55,13 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance continue; } + // If the resource has no explicit node pool affinity, assign it + // to the default user pool. + if (!r.TryGetLastAnnotation(out _) && defaultUserPool is not null) + { + r.Annotations.Add(new AksNodePoolAffinityAnnotation(defaultUserPool)); + } + r.Annotations.Add(new DeploymentTargetAnnotation(environment) { ComputeEnvironment = environment @@ -59,4 +71,38 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance return Task.CompletedTask; } + + /// + /// Ensures the AKS environment has at least one user node pool. If none exists, + /// creates a default "workload" user pool. + /// + private static AksNodePoolResource? EnsureDefaultUserNodePool(AzureKubernetesEnvironmentResource environment) + { + var hasUserPool = environment.NodePools.Any(p => p.Mode is AksNodePoolMode.User); + + if (hasUserPool) + { + // Return the first user pool as the default for unaffinitized workloads. + // Look for an existing AksNodePoolResource child that matches. + var firstUserConfig = environment.NodePools.First(p => p.Mode is AksNodePoolMode.User); + return FindNodePoolResource(environment, firstUserConfig.Name); + } + + // No user pool configured — create a default one. + var defaultConfig = new AksNodePoolConfig("workload", "Standard_D4s_v5", 1, 10, AksNodePoolMode.User); + environment.NodePools.Add(defaultConfig); + + var defaultPool = new AksNodePoolResource("workload", defaultConfig, environment); + return defaultPool; + } + + private static AksNodePoolResource? FindNodePoolResource(AzureKubernetesEnvironmentResource environment, string poolName) + { + // Walk annotations looking for child resources — but AksNodePoolResource is + // added via the application model, not annotations. Check if a matching pool + // resource was already created. If not, create one on the fly. + return new AksNodePoolResource(poolName, + environment.NodePools.First(p => p.Name == poolName), + environment); + } } diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs new file mode 100644 index 00000000000..de82fe9cdf3 --- /dev/null +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs @@ -0,0 +1,104 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable ASPIREAZURE003 + +using System.Runtime.CompilerServices; +using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Azure.Kubernetes; +using Aspire.Hosting.Utils; + +namespace Aspire.Hosting.Azure.Tests; + +public class AzureKubernetesInfrastructureTests +{ + [Fact] + public async Task NoUserPool_CreatesDefaultWorkloadPool() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + // No AddNodePool call — only the default system pool exists + Assert.Single(aks.Resource.NodePools); + Assert.Equal(AksNodePoolMode.System, aks.Resource.NodePools[0].Mode); + + var container = builder.AddContainer("myapi", "myimage"); + + await using var app = builder.Build(); + await ExecuteBeforeStartHooksAsync(app, default); + + // Infrastructure should have added a default "workload" user pool + Assert.Equal(2, aks.Resource.NodePools.Count); + var workloadPool = aks.Resource.NodePools.First(p => p.Mode is AksNodePoolMode.User); + Assert.Equal("workload", workloadPool.Name); + + // Compute resource should have been auto-assigned to the workload pool + Assert.True(container.Resource.TryGetLastAnnotation(out var affinity)); + Assert.Equal("workload", affinity.NodePool.Config.Name); + } + + [Fact] + public async Task ExplicitUserPool_NoDefaultCreated() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); + + var container = builder.AddContainer("myapi", "myimage"); + + await using var app = builder.Build(); + await ExecuteBeforeStartHooksAsync(app, default); + + // Should NOT create a default pool since one already exists + Assert.Equal(2, aks.Resource.NodePools.Count); // system + gpu + Assert.DoesNotContain(aks.Resource.NodePools, p => p.Name == "workload"); + + // Unaffinitized compute resource should get assigned to the first user pool + Assert.True(container.Resource.TryGetLastAnnotation(out var affinity)); + Assert.Equal("gpu", affinity.NodePool.Config.Name); + } + + [Fact] + public async Task ExplicitAffinity_NotOverridden() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); + var cpuPool = aks.AddNodePool("cpu", "Standard_D4s_v5", 1, 10); + + var container = builder.AddContainer("myapi", "myimage") + .WithNodePoolAffinity(cpuPool); + + await using var app = builder.Build(); + await ExecuteBeforeStartHooksAsync(app, default); + + // Explicit affinity should be preserved, not overridden + Assert.True(container.Resource.TryGetLastAnnotation(out var affinity)); + Assert.Equal("cpu", affinity.NodePool.Config.Name); + } + + [Fact] + public async Task ComputeResource_GetsDeploymentTargetAnnotation() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var container = builder.AddContainer("myapi", "myimage"); + + await using var app = builder.Build(); + await ExecuteBeforeStartHooksAsync(app, default); + + Assert.True(container.Resource.TryGetLastAnnotation(out var target)); + Assert.Same(aks.Resource, target.DeploymentTarget); + } + + [UnsafeAccessor(UnsafeAccessorKind.Method, Name = "ExecuteBeforeStartHooksAsync")] + private static extern Task ExecuteBeforeStartHooksAsync(DistributedApplication app, CancellationToken cancellationToken); +} From a4860c449b0235908d1be788d27d4dc01824dc14 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Sun, 12 Apr 2026 22:45:04 +1000 Subject: [PATCH 07/78] Fix publish: add inner K8s environment to model for Helm chart generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two issues with aspire publish: 1. No Helm chart output: The inner KubernetesEnvironmentResource was stored as a property but never added to the application model. KubernetesInfrastructure looks for KubernetesEnvironmentResource instances in the model to generate Helm charts. Fix: add the inner K8s environment to the model (excluded from manifest) with the default Helm engine. 2. Duplicate DeploymentTargetAnnotation: AzureKubernetesInfrastructure was adding its own DeploymentTargetAnnotation, conflicting with the one that KubernetesInfrastructure adds (which points to the correct KubernetesResource deployment target with Helm chart data). Fix: remove the duplicate annotation from our subscriber — KubernetesInfrastructure handles it. Also made EnsureDefaultHelmEngine internal (was private) so the AKS package can call it to set up the Helm deployment engine on the inner K8s environment. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 12 ++++++++++-- .../AzureKubernetesInfrastructure.cs | 9 +++++---- .../KubernetesEnvironmentExtensions.cs | 2 +- .../AzureKubernetesInfrastructureTests.cs | 9 +++++++-- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 636341fff54..144aabf34f0 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -57,17 +57,25 @@ public static IResourceBuilder AddAzureKuber // Create the unified environment resource var resource = new AzureKubernetesEnvironmentResource(name, ConfigureAksInfrastructure); - // Create the inner KubernetesEnvironmentResource (for Helm deployment) - resource.KubernetesEnvironment = new KubernetesEnvironmentResource($"{name}-k8s") + // Create the inner KubernetesEnvironmentResource (for Helm deployment). + // This must be added to the application model so that KubernetesInfrastructure + // can discover it and generate Helm charts for compute resources. + var k8sEnv = new KubernetesEnvironmentResource($"{name}-k8s") { HelmChartName = builder.Environment.ApplicationName.ToHelmChartName(), }; + resource.KubernetesEnvironment = k8sEnv; if (builder.ExecutionContext.IsRunMode) { return builder.CreateResourceBuilder(resource); } + // Add the inner K8s environment to the model so KubernetesInfrastructure + // generates Helm charts. Exclude from manifest since it's an internal detail. + var k8sEnvBuilder = builder.AddResource(k8sEnv).ExcludeFromManifest(); + KubernetesEnvironmentExtensions.EnsureDefaultHelmEngine(k8sEnvBuilder); + return builder.AddResource(resource); } diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 5ece0335105..58573376ab0 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -62,10 +62,11 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance r.Annotations.Add(new AksNodePoolAffinityAnnotation(defaultUserPool)); } - r.Annotations.Add(new DeploymentTargetAnnotation(environment) - { - ComputeEnvironment = environment - }); + // NOTE: We do NOT add DeploymentTargetAnnotation here. + // The inner KubernetesEnvironmentResource is in the model, so + // KubernetesInfrastructure will handle Helm chart generation + // and add the DeploymentTargetAnnotation with the correct + // KubernetesResource deployment target. } } diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentExtensions.cs index f12456f6473..db01e3feb6c 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentExtensions.cs @@ -161,7 +161,7 @@ public static IResourceBuilder WithDashboard(this return builder; } - private static void EnsureDefaultHelmEngine(IResourceBuilder builder) + internal static void EnsureDefaultHelmEngine(IResourceBuilder builder) { builder.Resource.DeploymentEngineStepsFactory ??= HelmDeploymentEngine.CreateStepsAsync; } diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs index de82fe9cdf3..c6a9d38d0f2 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs @@ -84,7 +84,7 @@ public async Task ExplicitAffinity_NotOverridden() } [Fact] - public async Task ComputeResource_GetsDeploymentTargetAnnotation() + public async Task ComputeResource_GetsDeploymentTargetFromKubernetesInfrastructure() { using var builder = TestDistributedApplicationBuilder.Create( DistributedApplicationOperation.Publish); @@ -95,8 +95,13 @@ public async Task ComputeResource_GetsDeploymentTargetAnnotation() await using var app = builder.Build(); await ExecuteBeforeStartHooksAsync(app, default); + // DeploymentTargetAnnotation comes from KubernetesInfrastructure (via the inner + // KubernetesEnvironmentResource), not from AzureKubernetesInfrastructure. Assert.True(container.Resource.TryGetLastAnnotation(out var target)); - Assert.Same(aks.Resource, target.DeploymentTarget); + Assert.NotNull(target.DeploymentTarget); + + // The compute environment should be the inner K8s environment + Assert.Same(aks.Resource.KubernetesEnvironment, target.ComputeEnvironment); } [UnsafeAccessor(UnsafeAccessorKind.Method, Name = "ExecuteBeforeStartHooksAsync")] From 19375688545503cc70da57654ebd1be6b54680ca Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Sun, 12 Apr 2026 23:12:00 +1000 Subject: [PATCH 08/78] Generate valid AKS Bicep instead of empty output placeholders Override GetBicepTemplateString and GetBicepTemplateFile to generate proper AKS ManagedCluster Bicep directly, bypassing the Azure.Provisioning SDK infrastructure (which requires the unavailable Azure.Provisioning.ContainerService package). The generated Bicep includes: - Microsoft.ContainerService/managedClusters resource with SystemAssigned identity - Configurable SKU tier, Kubernetes version, DNS prefix - Agent pool profiles with autoscaling from NodePools config - OIDC issuer profile and workload identity security profile - Optional private cluster API server access profile - Optional network profile (Azure CNI) - All outputs: id, name, clusterFqdn, oidcIssuerUrl, kubeletIdentityObjectId, nodeResourceGroup Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentResource.cs | 135 ++++++++++++++++++ ...ironment_BasicConfiguration.verified.bicep | 53 +++++-- ...etesEnvironment_WithVersion.verified.bicep | 54 +++++-- 3 files changed, 222 insertions(+), 20 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs index ff4592f2c54..4e22dbc8cfc 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs @@ -3,6 +3,8 @@ #pragma warning disable ASPIREAZURE003 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +using System.Globalization; +using System.Text; using Aspire.Hosting.Azure.Kubernetes; using Aspire.Hosting.Kubernetes; @@ -108,4 +110,137 @@ string IAzureDelegatedSubnetResource.DelegatedSubnetServiceName /// Gets or sets whether the cluster should be private. /// internal bool IsPrivateCluster { get; set; } + + /// + public override string GetBicepTemplateString() + { + return GenerateAksBicep(); + } + + /// + public override BicepTemplateFile GetBicepTemplateFile(string? directory = null, bool deleteTemporaryFileOnDispose = true) + { + var bicep = GenerateAksBicep(); + var dir = directory ?? Directory.CreateTempSubdirectory("aspire-aks").FullName; + var filePath = Path.Combine(dir, Name + ".module.bicep"); + File.WriteAllText(filePath, bicep); + return new BicepTemplateFile(filePath, directory is null && deleteTemporaryFileOnDispose); + } + + private string GenerateAksBicep() + { + var sb = new StringBuilder(); + var id = this.GetBicepIdentifier(); + var skuTier = SkuTier switch + { + AksSkuTier.Free => "Free", + AksSkuTier.Standard => "Standard", + AksSkuTier.Premium => "Premium", + _ => "Free" + }; + + sb.AppendLine("@description('The location for the resource(s) to be deployed.')"); + sb.AppendLine("param location string = resourceGroup().location"); + sb.AppendLine(); + + // AKS cluster resource + sb.Append("resource ").Append(id).AppendLine(" 'Microsoft.ContainerService/managedClusters@2024-06-02-preview' = {"); + sb.Append(" name: '").Append(Name).AppendLine("'"); + sb.AppendLine(" location: location"); + sb.AppendLine(" tags: {"); + sb.Append(" 'aspire-resource-name': '").Append(Name).AppendLine("'"); + sb.AppendLine(" }"); + sb.AppendLine(" identity: {"); + sb.AppendLine(" type: 'SystemAssigned'"); + sb.AppendLine(" }"); + sb.AppendLine(" sku: {"); + sb.AppendLine(" name: 'Base'"); + sb.Append(" tier: '").Append(skuTier).AppendLine("'"); + sb.AppendLine(" }"); + sb.AppendLine(" properties: {"); + + if (KubernetesVersion is not null) + { + sb.Append(" kubernetesVersion: '").Append(KubernetesVersion).AppendLine("'"); + } + + sb.Append(" dnsPrefix: '").Append(Name).AppendLine("-dns'"); + + // Agent pool profiles + sb.AppendLine(" agentPoolProfiles: ["); + foreach (var pool in NodePools) + { + var mode = pool.Mode switch + { + AksNodePoolMode.System => "System", + AksNodePoolMode.User => "User", + _ => "User" + }; + sb.AppendLine(" {"); + sb.Append(" name: '").Append(pool.Name).AppendLine("'"); + sb.Append(" vmSize: '").Append(pool.VmSize).AppendLine("'"); + sb.Append(" minCount: ").AppendLine(pool.MinCount.ToString(CultureInfo.InvariantCulture)); + sb.Append(" maxCount: ").AppendLine(pool.MaxCount.ToString(CultureInfo.InvariantCulture)); + sb.Append(" count: ").AppendLine(pool.MinCount.ToString(CultureInfo.InvariantCulture)); + sb.AppendLine(" enableAutoScaling: true"); + sb.Append(" mode: '").Append(mode).AppendLine("'"); + sb.AppendLine(" osType: 'Linux'"); + sb.AppendLine(" }"); + } + sb.AppendLine(" ]"); + + // OIDC issuer + if (OidcIssuerEnabled) + { + sb.AppendLine(" oidcIssuerProfile: {"); + sb.AppendLine(" enabled: true"); + sb.AppendLine(" }"); + } + + // Workload identity + if (WorkloadIdentityEnabled) + { + sb.AppendLine(" securityProfile: {"); + sb.AppendLine(" workloadIdentity: {"); + sb.AppendLine(" enabled: true"); + sb.AppendLine(" }"); + sb.AppendLine(" }"); + } + + // Private cluster + if (IsPrivateCluster) + { + sb.AppendLine(" apiServerAccessProfile: {"); + sb.AppendLine(" enablePrivateCluster: true"); + sb.AppendLine(" }"); + } + + // Network profile + if (NetworkProfile is not null) + { + sb.AppendLine(" networkProfile: {"); + sb.Append(" networkPlugin: '").Append(NetworkProfile.NetworkPlugin).AppendLine("'"); + if (NetworkProfile.NetworkPolicy is not null) + { + sb.Append(" networkPolicy: '").Append(NetworkProfile.NetworkPolicy).AppendLine("'"); + } + sb.Append(" serviceCidr: '").Append(NetworkProfile.ServiceCidr).AppendLine("'"); + sb.Append(" dnsServiceIP: '").Append(NetworkProfile.DnsServiceIP).AppendLine("'"); + sb.AppendLine(" }"); + } + + sb.AppendLine(" }"); + sb.AppendLine("}"); + sb.AppendLine(); + + // Outputs + sb.Append("output id string = ").Append(id).AppendLine(".id"); + sb.Append("output name string = ").Append(id).AppendLine(".name"); + sb.Append("output clusterFqdn string = ").Append(id).AppendLine(".properties.fqdn"); + sb.Append("output oidcIssuerUrl string = ").Append(id).AppendLine(".properties.oidcIssuerProfile.issuerURL"); + sb.Append("output kubeletIdentityObjectId string = ").Append(id).AppendLine(".properties.identityProfile.kubeletidentity.objectId"); + sb.Append("output nodeResourceGroup string = ").Append(id).AppendLine(".properties.nodeResourceGroup"); + + return sb.ToString(); + } } diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep index 89d1967adc5..031ba1beb38 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep @@ -1,14 +1,47 @@ @description('The location for the resource(s) to be deployed.') param location string = resourceGroup().location -output id string = id +resource aks 'Microsoft.ContainerService/managedClusters@2024-06-02-preview' = { + name: 'aks' + location: location + tags: { + 'aspire-resource-name': 'aks' + } + identity: { + type: 'SystemAssigned' + } + sku: { + name: 'Base' + tier: 'Free' + } + properties: { + dnsPrefix: 'aks-dns' + agentPoolProfiles: [ + { + name: 'system' + vmSize: 'Standard_D4s_v5' + minCount: 1 + maxCount: 3 + count: 1 + enableAutoScaling: true + mode: 'System' + osType: 'Linux' + } + ] + oidcIssuerProfile: { + enabled: true + } + securityProfile: { + workloadIdentity: { + enabled: true + } + } + } +} -output name string = name - -output clusterFqdn string = clusterFqdn - -output oidcIssuerUrl string = oidcIssuerUrl - -output kubeletIdentityObjectId string = kubeletIdentityObjectId - -output nodeResourceGroup string = nodeResourceGroup \ No newline at end of file +output id string = aks.id +output name string = aks.name +output clusterFqdn string = aks.properties.fqdn +output oidcIssuerUrl string = aks.properties.oidcIssuerProfile.issuerURL +output kubeletIdentityObjectId string = aks.properties.identityProfile.kubeletidentity.objectId +output nodeResourceGroup string = aks.properties.nodeResourceGroup diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep index 89d1967adc5..74431231a72 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep @@ -1,14 +1,48 @@ @description('The location for the resource(s) to be deployed.') param location string = resourceGroup().location -output id string = id +resource aks 'Microsoft.ContainerService/managedClusters@2024-06-02-preview' = { + name: 'aks' + location: location + tags: { + 'aspire-resource-name': 'aks' + } + identity: { + type: 'SystemAssigned' + } + sku: { + name: 'Base' + tier: 'Free' + } + properties: { + kubernetesVersion: '1.30' + dnsPrefix: 'aks-dns' + agentPoolProfiles: [ + { + name: 'system' + vmSize: 'Standard_D4s_v5' + minCount: 1 + maxCount: 3 + count: 1 + enableAutoScaling: true + mode: 'System' + osType: 'Linux' + } + ] + oidcIssuerProfile: { + enabled: true + } + securityProfile: { + workloadIdentity: { + enabled: true + } + } + } +} -output name string = name - -output clusterFqdn string = clusterFqdn - -output oidcIssuerUrl string = oidcIssuerUrl - -output kubeletIdentityObjectId string = kubeletIdentityObjectId - -output nodeResourceGroup string = nodeResourceGroup \ No newline at end of file +output id string = aks.id +output name string = aks.name +output clusterFqdn string = aks.properties.fqdn +output oidcIssuerUrl string = aks.properties.oidcIssuerProfile.issuerURL +output kubeletIdentityObjectId string = aks.properties.identityProfile.kubeletidentity.objectId +output nodeResourceGroup string = aks.properties.nodeResourceGroup From d4e2787cf2d3867a48c33524f725378d2bd00ba0 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Sun, 12 Apr 2026 23:31:26 +1000 Subject: [PATCH 09/78] Add ACR integration and fix localhive packaging Container Registry: - Auto-create a default Azure Container Registry when AddAzureKubernetesEnvironment is called (same pattern as Container Apps) - WithContainerRegistry() extension to use an explicit ACR, replacing the default - FlowContainerRegistry() in AzureKubernetesInfrastructure propagates the registry to the inner KubernetesEnvironmentResource via ContainerRegistryReferenceAnnotation so KubernetesInfrastructure can discover it for image push/pull Localhive fix: - Added SuppressFinalPackageVersion to csproj (required for new packages in Arcade) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Aspire.Hosting.Azure.Kubernetes.csproj | 1 + .../AzureKubernetesEnvironmentExtensions.cs | 37 +++++++++++++ .../AzureKubernetesEnvironmentResource.cs | 5 ++ .../AzureKubernetesInfrastructure.cs | 34 ++++++++++-- ...ureKubernetesEnvironmentExtensionsTests.cs | 54 +++++++++++++++++++ 5 files changed, 128 insertions(+), 3 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj index 4c918928549..bbeba708ab5 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj +++ b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj @@ -3,6 +3,7 @@ $(DefaultTargetFramework) true + true false aspire integration hosting azure kubernetes aks Azure Kubernetes Service (AKS) resource types for Aspire. diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 144aabf34f0..5fcd9554ee1 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -71,6 +71,10 @@ public static IResourceBuilder AddAzureKuber return builder.CreateResourceBuilder(resource); } + // Auto-create a default Azure Container Registry for image push/pull + var defaultRegistry = builder.AddAzureContainerRegistry($"{name}-acr"); + resource.DefaultContainerRegistry = defaultRegistry.Resource; + // Add the inner K8s environment to the model so KubernetesInfrastructure // generates Helm charts. Exclude from manifest since it's an internal detail. var k8sEnvBuilder = builder.AddResource(k8sEnv).ExcludeFromManifest(); @@ -208,6 +212,39 @@ public static IResourceBuilder AsPrivateClus return builder; } + /// + /// Configures the AKS environment to use a specific Azure Container Registry for image storage. + /// When set, this replaces the auto-created default container registry. + /// + /// The AKS environment resource builder. + /// The Azure Container Registry resource builder. + /// A reference to the for chaining. + /// + /// If not called, a default Azure Container Registry is automatically created. + /// The registry endpoint is flowed to the inner Kubernetes environment so that + /// Helm deployments can push and pull images. + /// + [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + public static IResourceBuilder WithContainerRegistry( + this IResourceBuilder builder, + IResourceBuilder registry) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(registry); + + // Remove the default registry from the model if one was auto-created + if (builder.Resource.DefaultContainerRegistry is not null) + { + builder.ApplicationBuilder.Resources.Remove(builder.Resource.DefaultContainerRegistry); + builder.Resource.DefaultContainerRegistry = null; + } + + // Set the explicit registry via annotation (same pattern as Container Apps) + builder.WithAnnotation(new ContainerRegistryReferenceAnnotation(registry.Resource)); + + return builder; + } + /// /// Enables Container Insights monitoring on the AKS cluster. /// diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs index 4e22dbc8cfc..dccc56665f6 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs @@ -111,6 +111,11 @@ string IAzureDelegatedSubnetResource.DelegatedSubnetServiceName /// internal bool IsPrivateCluster { get; set; } + /// + /// Gets or sets the default container registry auto-created for this AKS environment. + /// + internal AzureContainerRegistryResource? DefaultContainerRegistry { get; set; } + /// public override string GetBicepTemplateString() { diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 58573376ab0..876f314f86e 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -42,6 +42,10 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance { logger.LogInformation("Processing AKS environment '{Name}'", environment.Name); + // Flow the container registry to the inner K8s environment so + // KubernetesInfrastructure can find it for image push/pull. + FlowContainerRegistry(environment, @event.Model); + // Ensure a default user node pool exists for workload scheduling. // The system pool should only run system pods; application workloads // need a user pool. @@ -99,11 +103,35 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance private static AksNodePoolResource? FindNodePoolResource(AzureKubernetesEnvironmentResource environment, string poolName) { - // Walk annotations looking for child resources — but AksNodePoolResource is - // added via the application model, not annotations. Check if a matching pool - // resource was already created. If not, create one on the fly. return new AksNodePoolResource(poolName, environment.NodePools.First(p => p.Name == poolName), environment); } + + /// + /// Flows the container registry from the AKS environment to the inner + /// KubernetesEnvironmentResource via ContainerRegistryReferenceAnnotation. + /// This allows KubernetesInfrastructure to discover the registry for image push/pull. + /// + private static void FlowContainerRegistry(AzureKubernetesEnvironmentResource environment, DistributedApplicationModel _) + { + IContainerRegistry? registry = null; + + // Check for explicit registry set via WithContainerRegistry + if (environment.TryGetLastAnnotation(out var annotation)) + { + registry = annotation.Registry; + } + else if (environment.DefaultContainerRegistry is not null) + { + registry = environment.DefaultContainerRegistry; + } + + if (registry is not null) + { + // Propagate to the inner K8s environment so KubernetesInfrastructure finds it + environment.KubernetesEnvironment.Annotations.Add( + new ContainerRegistryReferenceAnnotation(registry)); + } + } } diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs index b7d9f0c3d92..75dd1a25040 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs @@ -2,7 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. #pragma warning disable ASPIREAZURE003 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +#pragma warning disable ASPIRECOMPUTE003 // Type is for evaluation purposes only +using System.Runtime.CompilerServices; using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Azure.Kubernetes; using Aspire.Hosting.Utils; @@ -297,4 +299,56 @@ public void AddNodePool_MultiplePoolsSupported() Assert.Equal("cpu", pool1.Resource.Name); Assert.Equal("gpu", pool2.Resource.Name); } + + [Fact] + public void AddAzureKubernetesEnvironment_AutoCreatesDefaultRegistry() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + Assert.NotNull(aks.Resource.DefaultContainerRegistry); + Assert.Equal("aks-acr", aks.Resource.DefaultContainerRegistry.Name); + } + + [Fact] + public void WithContainerRegistry_ReplacesDefault() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var explicitAcr = builder.AddAzureContainerRegistry("my-acr"); + + aks.WithContainerRegistry(explicitAcr); + + // Default registry should be removed + Assert.Null(aks.Resource.DefaultContainerRegistry); + + // Explicit registry should be set via annotation + Assert.True(aks.Resource.TryGetLastAnnotation(out var annotation)); + Assert.Same(explicitAcr.Resource, annotation.Registry); + } + + [Fact] + public async Task ContainerRegistry_FlowsToInnerKubernetesEnvironment() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var container = builder.AddContainer("myapi", "myimage"); + + await using var app = builder.Build(); + await ExecuteBeforeStartHooksAsync(app, default); + + // The inner K8s environment should have the registry annotation + Assert.True(aks.Resource.KubernetesEnvironment + .TryGetLastAnnotation(out var annotation)); + Assert.Same(aks.Resource.DefaultContainerRegistry, annotation.Registry); + } + + [UnsafeAccessor(UnsafeAccessorKind.Method, Name = "ExecuteBeforeStartHooksAsync")] + private static extern Task ExecuteBeforeStartHooksAsync(DistributedApplication app, CancellationToken cancellationToken); } From c0b9cace2eb3e8a9cc324ab4b14ca4ae32ca4f3a Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Sun, 12 Apr 2026 23:42:29 +1000 Subject: [PATCH 10/78] Fix localhive to include NonShipping packages in hive Packages with SuppressFinalPackageVersion=true (like Aspire.Hosting.Kubernetes and Aspire.Hosting.Azure.Kubernetes) are placed in the NonShipping output directory by Arcade SDK. The localhive script was only looking in the Shipping directory, causing these packages to be missing from the hive. Changes: - Added Get-AllPackagePaths that returns both Shipping and NonShipping dirs - Package collection now scans all available package directories - When packages span multiple directories, auto-uses copy mode (can't symlink to two dirs) - Single-dir case still uses symlink/junction for performance Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- localhive.ps1 | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/localhive.ps1 b/localhive.ps1 index 9cbcf9adc4c..d9f073bc62e 100644 --- a/localhive.ps1 +++ b/localhive.ps1 @@ -198,6 +198,12 @@ function Get-PackagesPath { Join-Path (Join-Path (Join-Path (Join-Path $RepoRoot 'artifacts') 'packages') $Config) 'Shipping' } +function Get-AllPackagePaths { + param([Parameter(Mandatory)][string]$Config) + $basePath = Join-Path (Join-Path (Join-Path $RepoRoot 'artifacts') 'packages') $Config + @('Shipping', 'NonShipping') | ForEach-Object { Join-Path $basePath $_ } | Where-Object { Test-Path -LiteralPath $_ } +} + $effectiveConfig = if ($Configuration) { $Configuration } else { 'Release' } # Skip native AOT during pack unless user will build it separately via -NativeAot + Bundle.proj @@ -230,13 +236,14 @@ else { } } -# Ensure there are .nupkg files -$packages = Get-ChildItem -LiteralPath $pkgDir -Filter *.nupkg -File -ErrorAction SilentlyContinue -if (-not $packages -or $packages.Count -eq 0) { - Write-Err "No .nupkg files found in $pkgDir. Did the pack step succeed?" +# Ensure there are .nupkg files (collect from both Shipping and NonShipping) +$allPkgDirs = Get-AllPackagePaths -Config $effectiveConfig +$packages = $allPkgDirs | ForEach-Object { Get-ChildItem -LiteralPath $_ -Filter *.nupkg -File -ErrorAction SilentlyContinue } | Where-Object { $_ } +if (-not $packages -or @($packages).Count -eq 0) { + Write-Err "No .nupkg files found in package directories. Did the pack step succeed?" exit 1 } -Write-Log ("Found {0} packages in {1}" -f $packages.Count, $pkgDir) +Write-Log ("Found {0} packages across {1}" -f @($packages).Count, ($allPkgDirs -join ', ')) # Determine the RID for the target platform (or auto-detect from host) if ($Rid) { @@ -271,33 +278,37 @@ if (Test-Path -LiteralPath $hiveRoot) { } function Copy-PackagesToHive { - param([string]$Source,[string]$Destination) + param([string[]]$Sources,[string]$Destination) New-Item -ItemType Directory -Path $Destination -Force | Out-Null - Get-ChildItem -LiteralPath $Source -Filter *.nupkg -File | Copy-Item -Destination $Destination -Force + foreach ($src in $Sources) { + Get-ChildItem -LiteralPath $src -Filter *.nupkg -File -ErrorAction SilentlyContinue | Copy-Item -Destination $Destination -Force + } } -if ($Copy) { +if ($Copy -or @($allPkgDirs).Count -gt 1) { + # Copy mode: always used when packages span Shipping + NonShipping Write-Log "Populating hive '$Name' by copying .nupkg files" - Copy-PackagesToHive -Source $pkgDir -Destination $hivePath + Copy-PackagesToHive -Sources $allPkgDirs -Destination $hivePath Write-Log "Created/updated hive '$Name' at $hivePath (copied packages)." } else { - Write-Log "Linking hive '$Name/packages' to $pkgDir" + $linkTarget = $allPkgDirs[0] + Write-Log "Linking hive '$Name/packages' to $linkTarget" New-Item -ItemType Directory -Path $hiveRoot -Force | Out-Null try { # Try symlink first (requires Developer Mode or elevated privilege) - New-Item -Path $hivePath -ItemType SymbolicLink -Target $pkgDir -Force | Out-Null - Write-Log "Created/updated hive '$Name/packages' -> $pkgDir (symlink)" + New-Item -Path $hivePath -ItemType SymbolicLink -Target $linkTarget -Force | Out-Null + Write-Log "Created/updated hive '$Name/packages' -> $linkTarget (symlink)" } catch { Write-Warn "Symlink not supported; attempting junction, else copying .nupkg files" try { - New-Item -Path $hivePath -ItemType Junction -Target $pkgDir -Force | Out-Null - Write-Log "Created/updated hive '$Name/packages' -> $pkgDir (junction)" + New-Item -Path $hivePath -ItemType Junction -Target $linkTarget -Force | Out-Null + Write-Log "Created/updated hive '$Name/packages' -> $linkTarget (junction)" } catch { Write-Warn "Link creation failed; copying .nupkg files instead" - Copy-PackagesToHive -Source $pkgDir -Destination $hivePath + Copy-PackagesToHive -Sources $allPkgDirs -Destination $hivePath Write-Log "Created/updated hive '$Name' at $hivePath (copied packages)." } } From 26a6312b27948693480898c8fec5237e178069b1 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 09:41:20 +1000 Subject: [PATCH 11/78] Fix MethodAccessException: use public AddKubernetesEnvironment API Internal methods from Aspire.Hosting.Kubernetes (AddKubernetesInfrastructureCore, EnsureDefaultHelmEngine, KubernetesInfrastructure, HelmDeploymentEngine) are not accessible at runtime across NuGet package boundaries, even with InternalsVisibleTo set. The InternalsVisibleTo attribute only works at compile time with project references, not with signed NuGet packages. Fix: call the public AddKubernetesEnvironment() API instead. This handles all the internal setup (registering KubernetesInfrastructure subscriber, creating the resource, setting up Helm engine) through a single public entry point. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 26 +++++-------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 5fcd9554ee1..b158b39dde1 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -4,8 +4,6 @@ using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Azure; using Aspire.Hosting.Azure.Kubernetes; -using Aspire.Hosting.Kubernetes; -using Aspire.Hosting.Kubernetes.Extensions; using Aspire.Hosting.Lifecycle; using Azure.Provisioning; using Microsoft.Extensions.DependencyInjection; @@ -51,20 +49,15 @@ public static IResourceBuilder AddAzureKuber // Register the AKS-specific infrastructure eventing subscriber builder.Services.TryAddEventingSubscriber(); - // Also register the generic K8s infrastructure (for Helm chart generation) - builder.AddKubernetesInfrastructureCore(); + // Create the inner KubernetesEnvironmentResource via the public API. + // This registers KubernetesInfrastructure, creates the resource with + // Helm chart name/dashboard, adds it to the model, and sets up the + // default Helm deployment engine. + var k8sEnvBuilder = builder.AddKubernetesEnvironment($"{name}-k8s"); - // Create the unified environment resource + // Create the unified AKS environment resource var resource = new AzureKubernetesEnvironmentResource(name, ConfigureAksInfrastructure); - - // Create the inner KubernetesEnvironmentResource (for Helm deployment). - // This must be added to the application model so that KubernetesInfrastructure - // can discover it and generate Helm charts for compute resources. - var k8sEnv = new KubernetesEnvironmentResource($"{name}-k8s") - { - HelmChartName = builder.Environment.ApplicationName.ToHelmChartName(), - }; - resource.KubernetesEnvironment = k8sEnv; + resource.KubernetesEnvironment = k8sEnvBuilder.Resource; if (builder.ExecutionContext.IsRunMode) { @@ -75,11 +68,6 @@ public static IResourceBuilder AddAzureKuber var defaultRegistry = builder.AddAzureContainerRegistry($"{name}-acr"); resource.DefaultContainerRegistry = defaultRegistry.Resource; - // Add the inner K8s environment to the model so KubernetesInfrastructure - // generates Helm charts. Exclude from manifest since it's an internal detail. - var k8sEnvBuilder = builder.AddResource(k8sEnv).ExcludeFromManifest(); - KubernetesEnvironmentExtensions.EnsureDefaultHelmEngine(k8sEnvBuilder); - return builder.AddResource(resource); } From 15a0dd244357dd1ce182a29c3dbc646529150084 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 10:21:32 +1000 Subject: [PATCH 12/78] Revert localhive.ps1 changes The localhive.ps1 modifications were unnecessary - packages with SuppressFinalPackageVersion go to Shipping, not NonShipping. The package discovery issue was caused by running localhive from the wrong worktree, not a script problem. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- localhive.ps1 | 41 +++++++++++++++-------------------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/localhive.ps1 b/localhive.ps1 index d9f073bc62e..9cbcf9adc4c 100644 --- a/localhive.ps1 +++ b/localhive.ps1 @@ -198,12 +198,6 @@ function Get-PackagesPath { Join-Path (Join-Path (Join-Path (Join-Path $RepoRoot 'artifacts') 'packages') $Config) 'Shipping' } -function Get-AllPackagePaths { - param([Parameter(Mandatory)][string]$Config) - $basePath = Join-Path (Join-Path (Join-Path $RepoRoot 'artifacts') 'packages') $Config - @('Shipping', 'NonShipping') | ForEach-Object { Join-Path $basePath $_ } | Where-Object { Test-Path -LiteralPath $_ } -} - $effectiveConfig = if ($Configuration) { $Configuration } else { 'Release' } # Skip native AOT during pack unless user will build it separately via -NativeAot + Bundle.proj @@ -236,14 +230,13 @@ else { } } -# Ensure there are .nupkg files (collect from both Shipping and NonShipping) -$allPkgDirs = Get-AllPackagePaths -Config $effectiveConfig -$packages = $allPkgDirs | ForEach-Object { Get-ChildItem -LiteralPath $_ -Filter *.nupkg -File -ErrorAction SilentlyContinue } | Where-Object { $_ } -if (-not $packages -or @($packages).Count -eq 0) { - Write-Err "No .nupkg files found in package directories. Did the pack step succeed?" +# Ensure there are .nupkg files +$packages = Get-ChildItem -LiteralPath $pkgDir -Filter *.nupkg -File -ErrorAction SilentlyContinue +if (-not $packages -or $packages.Count -eq 0) { + Write-Err "No .nupkg files found in $pkgDir. Did the pack step succeed?" exit 1 } -Write-Log ("Found {0} packages across {1}" -f @($packages).Count, ($allPkgDirs -join ', ')) +Write-Log ("Found {0} packages in {1}" -f $packages.Count, $pkgDir) # Determine the RID for the target platform (or auto-detect from host) if ($Rid) { @@ -278,37 +271,33 @@ if (Test-Path -LiteralPath $hiveRoot) { } function Copy-PackagesToHive { - param([string[]]$Sources,[string]$Destination) + param([string]$Source,[string]$Destination) New-Item -ItemType Directory -Path $Destination -Force | Out-Null - foreach ($src in $Sources) { - Get-ChildItem -LiteralPath $src -Filter *.nupkg -File -ErrorAction SilentlyContinue | Copy-Item -Destination $Destination -Force - } + Get-ChildItem -LiteralPath $Source -Filter *.nupkg -File | Copy-Item -Destination $Destination -Force } -if ($Copy -or @($allPkgDirs).Count -gt 1) { - # Copy mode: always used when packages span Shipping + NonShipping +if ($Copy) { Write-Log "Populating hive '$Name' by copying .nupkg files" - Copy-PackagesToHive -Sources $allPkgDirs -Destination $hivePath + Copy-PackagesToHive -Source $pkgDir -Destination $hivePath Write-Log "Created/updated hive '$Name' at $hivePath (copied packages)." } else { - $linkTarget = $allPkgDirs[0] - Write-Log "Linking hive '$Name/packages' to $linkTarget" + Write-Log "Linking hive '$Name/packages' to $pkgDir" New-Item -ItemType Directory -Path $hiveRoot -Force | Out-Null try { # Try symlink first (requires Developer Mode or elevated privilege) - New-Item -Path $hivePath -ItemType SymbolicLink -Target $linkTarget -Force | Out-Null - Write-Log "Created/updated hive '$Name/packages' -> $linkTarget (symlink)" + New-Item -Path $hivePath -ItemType SymbolicLink -Target $pkgDir -Force | Out-Null + Write-Log "Created/updated hive '$Name/packages' -> $pkgDir (symlink)" } catch { Write-Warn "Symlink not supported; attempting junction, else copying .nupkg files" try { - New-Item -Path $hivePath -ItemType Junction -Target $linkTarget -Force | Out-Null - Write-Log "Created/updated hive '$Name/packages' -> $linkTarget (junction)" + New-Item -Path $hivePath -ItemType Junction -Target $pkgDir -Force | Out-Null + Write-Log "Created/updated hive '$Name/packages' -> $pkgDir (junction)" } catch { Write-Warn "Link creation failed; copying .nupkg files instead" - Copy-PackagesToHive -Sources $allPkgDirs -Destination $hivePath + Copy-PackagesToHive -Source $pkgDir -Destination $hivePath Write-Log "Created/updated hive '$Name' at $hivePath (copied packages)." } } From 4b2ad7d5f18c4db42e1bb7d836281e933058365f Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 11:55:43 +1000 Subject: [PATCH 13/78] Clean up ConfigureAksInfrastructure: remove stale ProvisioningOutput placeholders The ConfigureAksInfrastructure callback was still adding ProvisioningOutput objects with no values, even though GetBicepTemplateString/GetBicepTemplateFile now generate the Bicep directly. While our overrides prevent these from being used for Bicep generation, the stale outputs could confuse the AzureResourcePreparer's parameter analysis. Emptied the callback body since all Bicep generation is handled by the resource's overrides. Also removed unused Azure.Provisioning using directive. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index b158b39dde1..3fb3f06adfc 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -5,7 +5,6 @@ using Aspire.Hosting.Azure; using Aspire.Hosting.Azure.Kubernetes; using Aspire.Hosting.Lifecycle; -using Azure.Provisioning; using Microsoft.Extensions.DependencyInjection; namespace Aspire.Hosting; @@ -336,18 +335,13 @@ public static IResourceBuilder WithAzureWorkloadIdentity( return builder; } - // NOTE: Full AKS Bicep provisioning with ContainerServiceManagedCluster types requires - // the Azure.Provisioning.ContainerService package (currently unavailable in internal feeds). - // This implementation uses ProvisioningOutput placeholders. When the package becomes available, - // replace this with typed ContainerServiceManagedCluster resource construction. + // ConfigureAksInfrastructure is a no-op placeholder required by the + // AzureProvisioningResource base class constructor. The actual Bicep is + // generated by GetBicepTemplateString/GetBicepTemplateFile overrides + // in AzureKubernetesEnvironmentResource. private static void ConfigureAksInfrastructure(AzureResourceInfrastructure infrastructure) { - // Outputs will be populated by the Bicep module generated during publish - infrastructure.Add(new ProvisioningOutput("id", typeof(string))); - infrastructure.Add(new ProvisioningOutput("name", typeof(string))); - infrastructure.Add(new ProvisioningOutput("clusterFqdn", typeof(string))); - infrastructure.Add(new ProvisioningOutput("oidcIssuerUrl", typeof(string))); - infrastructure.Add(new ProvisioningOutput("kubeletIdentityObjectId", typeof(string))); - infrastructure.Add(new ProvisioningOutput("nodeResourceGroup", typeof(string))); + // Intentionally empty — Bicep generation is handled by the resource's + // GetBicepTemplateString override, not the provisioning infrastructure. } } From 1f05addb57e4e05573deaeb98ed5c7e0e86a4708 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 12:14:21 +1000 Subject: [PATCH 14/78] Add kubeconfig isolation for AKS deployments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes to ensure Helm/kubectl target the AKS cluster instead of the user's default kubectl context: 1. KubernetesEnvironmentResource.KubeConfigPath (Aspire.Hosting.Kubernetes): New public property. When set, HelmDeploymentEngine passes --kubeconfig to all helm and kubectl commands. This is non-breaking — null means use default behavior. 2. AzureKubernetesInfrastructure get-credentials step (Aspire.Hosting.Azure.Kubernetes): Adds a pipeline step that runs after AKS Bicep provisioning and before Helm prepare. It calls 'az aks get-credentials --file ' to write credentials to a temp kubeconfig file, then sets KubeConfigPath on the inner KubernetesEnvironmentResource. This ensures Helm deploys to the provisioned AKS cluster without mutating ~/.kube/config. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesInfrastructure.cs | 177 ++++++++++++++++++ .../Deployment/HelmDeploymentEngine.cs | 18 +- .../KubernetesEnvironmentResource.cs | 11 ++ 3 files changed, 205 insertions(+), 1 deletion(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 876f314f86e..2d5b8c1e658 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -1,9 +1,14 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#pragma warning disable ASPIREPIPELINES001 // Pipeline types are experimental +#pragma warning disable ASPIREAZURE001 // AzureEnvironmentResource is experimental + +using System.Diagnostics; using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Eventing; using Aspire.Hosting.Lifecycle; +using Aspire.Hosting.Pipelines; using Microsoft.Extensions.Logging; namespace Aspire.Hosting.Azure.Kubernetes; @@ -46,6 +51,10 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance // KubernetesInfrastructure can find it for image push/pull. FlowContainerRegistry(environment, @event.Model); + // Add a pipeline step to fetch AKS credentials into an isolated kubeconfig + // file. This runs after AKS is provisioned and before the Helm deploy. + AddGetCredentialsStep(environment); + // Ensure a default user node pool exists for workload scheduling. // The system pool should only run system pods; application workloads // need a user pool. @@ -134,4 +143,172 @@ private static void FlowContainerRegistry(AzureKubernetesEnvironmentResource env new ContainerRegistryReferenceAnnotation(registry)); } } + + /// + /// Adds a pipeline step to the inner KubernetesEnvironmentResource that fetches + /// AKS cluster credentials into an isolated kubeconfig file after the AKS cluster + /// is provisioned via Bicep. + /// + private static void AddGetCredentialsStep(AzureKubernetesEnvironmentResource environment) + { + var k8sEnv = environment.KubernetesEnvironment; + + k8sEnv.Annotations.Add(new PipelineStepAnnotation((_) => + { + var step = new PipelineStep + { + Name = $"aks-get-credentials-{environment.Name}", + Description = $"Fetches AKS credentials for {environment.Name}", + Action = ctx => GetAksCredentialsAsync(ctx, environment) + }; + + // Run after AKS cluster is provisioned + step.DependsOn($"provision-{environment.Name}"); + + // Must complete before Helm prepare step + step.RequiredBy($"prepare-{k8sEnv.Name}"); + + return new[] { step }; + })); + } + + /// + /// Fetches AKS credentials into an isolated kubeconfig file using az aks get-credentials, + /// then sets the KubeConfigPath on the inner KubernetesEnvironmentResource so that + /// subsequent Helm and kubectl commands target the AKS cluster. + /// + private static async Task GetAksCredentialsAsync( + PipelineStepContext context, + AzureKubernetesEnvironmentResource environment) + { + var getCredsTask = await context.ReportingStep.CreateTaskAsync( + $"Fetching AKS credentials for {environment.Name}", + context.CancellationToken).ConfigureAwait(false); + + await using (getCredsTask.ConfigureAwait(false)) + { + try + { + // Resolve the cluster name from Bicep output + var clusterName = await environment.NameOutputReference + .GetValueAsync(context.CancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException("AKS cluster name output was not resolved."); + + // Get the resource group from the Azure environment resource + var azureEnv = context.Model.Resources.OfType().FirstOrDefault() + ?? throw new InvalidOperationException("No AzureEnvironmentResource found in the model."); + + var resourceGroup = await azureEnv.ResourceGroupName + .GetValueAsync(context.CancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException("Resource group name was not resolved."); + + // Write credentials to an isolated kubeconfig file + var kubeConfigDir = Directory.CreateTempSubdirectory("aspire-aks"); + var kubeConfigPath = Path.Combine(kubeConfigDir.FullName, "kubeconfig"); + + var arguments = $"aks get-credentials --resource-group {resourceGroup} --name {clusterName} --file \"{kubeConfigPath}\" --overwrite-existing"; + + context.Logger.LogInformation("Fetching AKS credentials: az {Arguments}", arguments); + + var azPath = FindAzCli(); + + using var process = new Process(); + process.StartInfo = new ProcessStartInfo + { + FileName = azPath, + Arguments = arguments, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + process.Start(); + + var stdoutTask = process.StandardOutput.ReadToEndAsync(context.CancellationToken); + var stderrTask = process.StandardError.ReadToEndAsync(context.CancellationToken); + + await process.WaitForExitAsync(context.CancellationToken).ConfigureAwait(false); + + var stdout = await stdoutTask.ConfigureAwait(false); + var stderr = await stderrTask.ConfigureAwait(false); + + if (!string.IsNullOrWhiteSpace(stdout)) + { + context.Logger.LogDebug("az (stdout): {Output}", stdout); + } + + if (!string.IsNullOrWhiteSpace(stderr)) + { + context.Logger.LogDebug("az (stderr): {Error}", stderr); + } + + if (process.ExitCode != 0) + { + throw new InvalidOperationException( + $"az aks get-credentials failed (exit code {process.ExitCode}): {stderr.Trim()}"); + } + + // Set the kubeconfig path on the inner K8s environment so + // Helm and kubectl commands use --kubeconfig to target this cluster + environment.KubernetesEnvironment.KubeConfigPath = kubeConfigPath; + + context.Logger.LogInformation( + "AKS credentials written to {KubeConfigPath}", kubeConfigPath); + + await getCredsTask.SucceedAsync( + $"AKS credentials fetched for cluster {clusterName}", + context.CancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + await getCredsTask.FailAsync( + $"Failed to fetch AKS credentials: {ex.Message}", + context.CancellationToken).ConfigureAwait(false); + throw; + } + } + } + + private static string FindAzCli() + { + // Check PATH first + var pathDirs = Environment.GetEnvironmentVariable("PATH")?.Split(Path.PathSeparator) ?? []; + var azNames = OperatingSystem.IsWindows() + ? new[] { "az.CMD", "az.cmd", "az.exe" } + : new[] { "az" }; + + foreach (var dir in pathDirs) + { + foreach (var azName in azNames) + { + var candidate = Path.Combine(dir, azName); + if (File.Exists(candidate)) + { + return candidate; + } + } + } + + // Check common Windows locations + if (OperatingSystem.IsWindows()) + { + var commonPaths = new[] + { + @"C:\Program Files\Microsoft SDKs\Azure\CLI2\wbin\az.CMD", + @"C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin\az.CMD", + }; + + foreach (var path in commonPaths) + { + if (File.Exists(path)) + { + return path; + } + } + } + + throw new InvalidOperationException( + "Azure CLI (az) not found. Install it from https://learn.microsoft.com/cli/azure/install-azure-cli"); + } } diff --git a/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs b/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs index ed04194be43..01e7c3c511c 100644 --- a/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs +++ b/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs @@ -424,6 +424,11 @@ private static async Task HelmDeployAsync(PipelineStepContext context, Kubernete arguments.Append(" --create-namespace"); arguments.Append(" --wait"); + if (environment.KubeConfigPath is not null) + { + arguments.Append(CultureInfo.InvariantCulture, $" --kubeconfig \"{environment.KubeConfigPath}\""); + } + if (File.Exists(valuesFilePath)) { arguments.Append(CultureInfo.InvariantCulture, $" -f \"{valuesFilePath}\""); @@ -501,7 +506,7 @@ internal static async Task PrintResourceSummaryAsync( try { - var endpoints = await GetServiceEndpointsAsync(computeResource.Name.ToServiceName(), @namespace, context.Logger, context.CancellationToken).ConfigureAwait(false); + var endpoints = await GetServiceEndpointsAsync(computeResource.Name.ToServiceName(), @namespace, environment.KubeConfigPath, context.Logger, context.CancellationToken).ConfigureAwait(false); if (endpoints.Count > 0) { @@ -571,6 +576,11 @@ private static async Task HelmUninstallAsync(PipelineStepContext context, string var helmRunner = context.Services.GetRequiredService(); var arguments = $"uninstall {releaseName} --namespace {@namespace}"; + if (environment.KubeConfigPath is not null) + { + arguments += $" --kubeconfig \"{environment.KubeConfigPath}\""; + } + context.Logger.LogDebug("Running helm {Arguments}", arguments); var exitCode = await helmRunner.RunAsync( @@ -640,12 +650,18 @@ private static async Task ConfirmDestroyAsync(PipelineStepContext context, strin private static async Task> GetServiceEndpointsAsync( string serviceName, string @namespace, + string? kubeConfigPath, ILogger logger, CancellationToken cancellationToken) { var endpoints = new List(); var arguments = $"get service {serviceName} --namespace {@namespace} -o json"; + + if (kubeConfigPath is not null) + { + arguments += $" --kubeconfig \"{kubeConfigPath}\""; + } var stdoutBuilder = new StringBuilder(); var spec = new ProcessSpec("kubectl") diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs index 68c03a816e0..441b659aa9a 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs @@ -92,6 +92,17 @@ public sealed class KubernetesEnvironmentResource : Resource, IComputeEnvironmen /// public string DefaultServiceType { get; set; } = "ClusterIP"; + /// + /// Gets or sets the path to an explicit kubeconfig file for Helm and kubectl commands. + /// When set, all Helm and kubectl commands will use --kubeconfig to target + /// this file instead of the default ~/.kube/config. + /// + /// + /// This is used by Azure Kubernetes Service (AKS) integration to isolate credentials + /// fetched via az aks get-credentials from the user's default kubectl context. + /// + public string? KubeConfigPath { get; set; } + internal IPortAllocator PortAllocator { get; } = new PortAllocator(); /// From 76bd9a7badd7ef82eb75ea6262afbf517fb5db67 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 12:22:51 +1000 Subject: [PATCH 15/78] Fix resource group resolution: read from Azure config instead of ParameterResource MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The resourceGroupName is a ParameterResource that requires configuration key 'Parameters:resourceGroupName' to be set. During deploy, this isn't available as a raw parameter value — it's resolved by the Azure provisioning context and stored in the 'Azure:ResourceGroup' configuration key. Changed GetAksCredentialsAsync to read from IConfiguration['Azure:ResourceGroup'] which is populated by the Azure provisioner during context creation, before our get-credentials step runs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesInfrastructure.cs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 2d5b8c1e658..2a2db3649f7 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -9,6 +9,8 @@ using Aspire.Hosting.Eventing; using Aspire.Hosting.Lifecycle; using Aspire.Hosting.Pipelines; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; namespace Aspire.Hosting.Azure.Kubernetes; @@ -194,13 +196,14 @@ private static async Task GetAksCredentialsAsync( .GetValueAsync(context.CancellationToken).ConfigureAwait(false) ?? throw new InvalidOperationException("AKS cluster name output was not resolved."); - // Get the resource group from the Azure environment resource - var azureEnv = context.Model.Resources.OfType().FirstOrDefault() - ?? throw new InvalidOperationException("No AzureEnvironmentResource found in the model."); - - var resourceGroup = await azureEnv.ResourceGroupName - .GetValueAsync(context.CancellationToken).ConfigureAwait(false) - ?? throw new InvalidOperationException("Resource group name was not resolved."); + // Get the resource group from Azure provisioning configuration. + // This is set by the Azure provisioner during the provisioning context + // creation step (stored in Azure:ResourceGroup config key). + var configuration = context.Services.GetRequiredService(); + var resourceGroup = configuration["Azure:ResourceGroup"] + ?? throw new InvalidOperationException( + "Azure resource group name not found in configuration. " + + "Ensure the Azure provisioning context has been created (provision step must complete first)."); // Write credentials to an isolated kubeconfig file var kubeConfigDir = Directory.CreateTempSubdirectory("aspire-aks"); From c5dd45107343c7617cd3b873181975ca37e2198e Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 12:30:30 +1000 Subject: [PATCH 16/78] Fix: use resource name directly instead of BicepOutputReference for cluster name BicepOutputReference.GetValueAsync() triggers parameter resolution on the AzureProvisioningResource, which tries to resolve the 'location' parameter that depends on 'resourceGroup().location'. In a fresh environment without Parameters:resourceGroupName configured, this fails. Since we set the cluster name directly in the Bicep template (name: '{Name}'), we can just use environment.Name as the cluster name. This avoids the parameter resolution chain entirely. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesInfrastructure.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 2a2db3649f7..74930f934bc 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -191,19 +191,19 @@ private static async Task GetAksCredentialsAsync( { try { - // Resolve the cluster name from Bicep output - var clusterName = await environment.NameOutputReference - .GetValueAsync(context.CancellationToken).ConfigureAwait(false) - ?? throw new InvalidOperationException("AKS cluster name output was not resolved."); + // The cluster name is the resource name — we set it directly in the Bicep template. + // We don't use NameOutputReference.GetValueAsync() because it triggers parameter + // resolution that may not be available at this point in the pipeline. + var clusterName = environment.Name; // Get the resource group from Azure provisioning configuration. - // This is set by the Azure provisioner during the provisioning context - // creation step (stored in Azure:ResourceGroup config key). + // The create-provisioning-context step resolves this and stores it in + // the Azure:ResourceGroup config key before any provision steps run. var configuration = context.Services.GetRequiredService(); var resourceGroup = configuration["Azure:ResourceGroup"] ?? throw new InvalidOperationException( "Azure resource group name not found in configuration. " + - "Ensure the Azure provisioning context has been created (provision step must complete first)."); + "Ensure the Azure provisioning context has been created."); // Write credentials to an isolated kubeconfig file var kubeConfigDir = Directory.CreateTempSubdirectory("aspire-aks"); From 7d5ca599441cc8a407e55c9986ee1cee81991e74 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 12:58:17 +1000 Subject: [PATCH 17/78] Fix: depend on provision-azure-bicep-resources aggregation step The aks-get-credentials step was depending on provision-{name} (individual AKS resource step) but the Azure:ResourceGroup config key is set by the create-provisioning-context step. In a fresh environment, the step ordering wasn't guaranteed to have the config available. Changed to depend on the provision-azure-bicep-resources aggregation step which gates on ALL provisioning completing, ensuring both the provisioning context (with resource group) and the AKS cluster are ready. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesInfrastructure.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 74930f934bc..9ca0f275b13 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -164,8 +164,9 @@ private static void AddGetCredentialsStep(AzureKubernetesEnvironmentResource env Action = ctx => GetAksCredentialsAsync(ctx, environment) }; - // Run after AKS cluster is provisioned - step.DependsOn($"provision-{environment.Name}"); + // Run after ALL Azure infrastructure is provisioned (including the AKS cluster). + // This depends on the aggregation step that gates on all individual provision-* steps. + step.DependsOn(AzureEnvironmentResource.ProvisionInfrastructureStepName); // Must complete before Helm prepare step step.RequiredBy($"prepare-{k8sEnv.Name}"); From eb78fd156820dafbfe432abb9efe65a969da84db Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 13:12:55 +1000 Subject: [PATCH 18/78] Fix: wire container registry to inner K8s env at creation time, not during event MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ContainerRegistryReferenceAnnotation was being added to the inner KubernetesEnvironmentResource during BeforeStartEvent via FlowContainerRegistry. But KubernetesInfrastructure also runs during BeforeStartEvent and reads the registry annotation — if it ran first, it wouldn't see the annotation, resulting in no push steps being created and images never getting pushed. Fix: Add the ContainerRegistryReferenceAnnotation to the inner K8s environment immediately in AddAzureKubernetesEnvironment and WithContainerRegistry, at resource creation time before any events fire. This guarantees KubernetesInfrastructure always sees the registry regardless of subscriber execution order. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 11 +++++-- .../AzureKubernetesInfrastructure.cs | 31 ------------------- 2 files changed, 9 insertions(+), 33 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 3fb3f06adfc..2db72c0154c 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -63,9 +63,13 @@ public static IResourceBuilder AddAzureKuber return builder.CreateResourceBuilder(resource); } - // Auto-create a default Azure Container Registry for image push/pull + // Auto-create a default Azure Container Registry for image push/pull. + // Wire it to the inner K8s environment immediately so KubernetesInfrastructure + // can discover it during BeforeStartEvent (both subscribers run during the same + // event, so we can't rely on annotation ordering during the event). var defaultRegistry = builder.AddAzureContainerRegistry($"{name}-acr"); resource.DefaultContainerRegistry = defaultRegistry.Resource; + k8sEnvBuilder.WithAnnotation(new ContainerRegistryReferenceAnnotation(defaultRegistry.Resource)); return builder.AddResource(resource); } @@ -226,8 +230,11 @@ public static IResourceBuilder WithContainer builder.Resource.DefaultContainerRegistry = null; } - // Set the explicit registry via annotation (same pattern as Container Apps) + // Set the explicit registry via annotation on both the AKS environment + // and the inner K8s environment (so KubernetesInfrastructure finds it) builder.WithAnnotation(new ContainerRegistryReferenceAnnotation(registry.Resource)); + builder.Resource.KubernetesEnvironment.Annotations.Add( + new ContainerRegistryReferenceAnnotation(registry.Resource)); return builder; } diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 9ca0f275b13..a207ffd4346 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -49,10 +49,6 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance { logger.LogInformation("Processing AKS environment '{Name}'", environment.Name); - // Flow the container registry to the inner K8s environment so - // KubernetesInfrastructure can find it for image push/pull. - FlowContainerRegistry(environment, @event.Model); - // Add a pipeline step to fetch AKS credentials into an isolated kubeconfig // file. This runs after AKS is provisioned and before the Helm deploy. AddGetCredentialsStep(environment); @@ -119,33 +115,6 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance environment); } - /// - /// Flows the container registry from the AKS environment to the inner - /// KubernetesEnvironmentResource via ContainerRegistryReferenceAnnotation. - /// This allows KubernetesInfrastructure to discover the registry for image push/pull. - /// - private static void FlowContainerRegistry(AzureKubernetesEnvironmentResource environment, DistributedApplicationModel _) - { - IContainerRegistry? registry = null; - - // Check for explicit registry set via WithContainerRegistry - if (environment.TryGetLastAnnotation(out var annotation)) - { - registry = annotation.Registry; - } - else if (environment.DefaultContainerRegistry is not null) - { - registry = environment.DefaultContainerRegistry; - } - - if (registry is not null) - { - // Propagate to the inner K8s environment so KubernetesInfrastructure finds it - environment.KubernetesEnvironment.Annotations.Add( - new ContainerRegistryReferenceAnnotation(registry)); - } - } - /// /// Adds a pipeline step to the inner KubernetesEnvironmentResource that fetches /// AKS cluster credentials into an isolated kubeconfig file after the AKS cluster From b93c0414d91b076a253083c9fa1e209d7fd6543c Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 13:29:30 +1000 Subject: [PATCH 19/78] Fix push steps blocking: wire push to depend on ACR provisioning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Push steps call registry.Endpoint.GetValueAsync() which awaits the BicepOutputReference for loginServer. If the ACR hasn't been provisioned yet, this blocks indefinitely — the push step just hangs after push-prereq. Push steps depend on build + push-prereq, but neither of those depend on the ACR's provision step. Added a PipelineConfigurationAnnotation on the inner K8s environment that makes all compute resource push steps depend on the ACR's provision step. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 2db72c0154c..95d9e207513 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -1,10 +1,13 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#pragma warning disable ASPIREPIPELINES001 // Pipeline types are experimental + using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Azure; using Aspire.Hosting.Azure.Kubernetes; using Aspire.Hosting.Lifecycle; +using Aspire.Hosting.Pipelines; using Microsoft.Extensions.DependencyInjection; namespace Aspire.Hosting; @@ -71,6 +74,21 @@ public static IResourceBuilder AddAzureKuber resource.DefaultContainerRegistry = defaultRegistry.Resource; k8sEnvBuilder.WithAnnotation(new ContainerRegistryReferenceAnnotation(defaultRegistry.Resource)); + // Ensure push steps wait for ACR provisioning to complete. The push step + // calls registry.Endpoint.GetValueAsync() which awaits the Bicep output — + // if the ACR isn't provisioned yet, this blocks indefinitely. + var acrResource = defaultRegistry.Resource; + k8sEnvBuilder.WithAnnotation(new PipelineConfigurationAnnotation(context => + { + var acrProvisionSteps = context.GetSteps(acrResource, WellKnownPipelineTags.ProvisionInfrastructure); + + foreach (var computeResource in context.Model.GetComputeResources()) + { + var pushSteps = context.GetSteps(computeResource, WellKnownPipelineTags.PushContainerImage); + pushSteps.DependsOn(acrProvisionSteps); + } + })); + return builder.AddResource(resource); } From d5ae3fa68c6d9afc8ed312a2250bd3c0d6882d3b Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 13:47:01 +1000 Subject: [PATCH 20/78] Fix push dependency: use provision-azure-bicep-resources aggregation step Changed from depending on individual ACR provision step (which required resource-to-step lookup that may not resolve correctly) to depending on the provision-azure-bicep-resources aggregation step by name. This is simpler and ensures ALL Azure provisioning (including ACR output population) completes before any image push begins. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 95d9e207513..3f1986492da 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. #pragma warning disable ASPIREPIPELINES001 // Pipeline types are experimental +#pragma warning disable ASPIREAZURE001 // AzureEnvironmentResource is experimental using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Azure; @@ -74,18 +75,16 @@ public static IResourceBuilder AddAzureKuber resource.DefaultContainerRegistry = defaultRegistry.Resource; k8sEnvBuilder.WithAnnotation(new ContainerRegistryReferenceAnnotation(defaultRegistry.Resource)); - // Ensure push steps wait for ACR provisioning to complete. The push step - // calls registry.Endpoint.GetValueAsync() which awaits the Bicep output — - // if the ACR isn't provisioned yet, this blocks indefinitely. - var acrResource = defaultRegistry.Resource; + // Ensure push steps wait for ALL Azure provisioning to complete. Push steps + // call registry.Endpoint.GetValueAsync() which awaits the BicepOutputReference + // for loginServer — if the ACR hasn't been provisioned yet (or outputs not + // populated yet), this blocks indefinitely. k8sEnvBuilder.WithAnnotation(new PipelineConfigurationAnnotation(context => { - var acrProvisionSteps = context.GetSteps(acrResource, WellKnownPipelineTags.ProvisionInfrastructure); - foreach (var computeResource in context.Model.GetComputeResources()) { var pushSteps = context.GetSteps(computeResource, WellKnownPipelineTags.PushContainerImage); - pushSteps.DependsOn(acrProvisionSteps); + pushSteps.DependsOn(AzureEnvironmentResource.ProvisionInfrastructureStepName); } })); From ad679fa1bb9ceb3e05c8dd93b791f7e9a3a12025 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 14:16:15 +1000 Subject: [PATCH 21/78] Fix push-prereq: find step by name and add provision dependency Previous attempts tried to wire dependencies via GetSteps(resource, tag) which uses the StepToResourceMap. This approach failed because the push steps are keyed to the compute resources, not the K8s environment. New approach: find the push-prereq step by name in the Steps collection and directly call DependsOn(provision-azure-bicep-resources). Since all push steps already depend on push-prereq, this ensures the entire push chain waits for Azure provisioning to complete. This mirrors how ACA works: ACA doesn't need this because it implements IContainerRegistry directly on the environment resource, so the endpoint values are resolved differently. For AKS, the ACR is a separate Bicep resource whose outputs need to be populated before push can proceed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 11 ++++++----- .../AzureKubernetesInfrastructureTests.cs | 5 +++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 3f1986492da..80072cd3ca4 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -79,13 +79,14 @@ public static IResourceBuilder AddAzureKuber // call registry.Endpoint.GetValueAsync() which awaits the BicepOutputReference // for loginServer — if the ACR hasn't been provisioned yet (or outputs not // populated yet), this blocks indefinitely. + // We make the push-prereq step depend on provisioning so all push steps + // (which already depend on push-prereq) are guaranteed to run after provisioning. k8sEnvBuilder.WithAnnotation(new PipelineConfigurationAnnotation(context => { - foreach (var computeResource in context.Model.GetComputeResources()) - { - var pushSteps = context.GetSteps(computeResource, WellKnownPipelineTags.PushContainerImage); - pushSteps.DependsOn(AzureEnvironmentResource.ProvisionInfrastructureStepName); - } + var pushPrereq = context.Steps + .FirstOrDefault(s => s.Name == WellKnownPipelineSteps.PushPrereq); + + pushPrereq?.DependsOn(AzureEnvironmentResource.ProvisionInfrastructureStepName); })); return builder.AddResource(resource); diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs index c6a9d38d0f2..55fef92b7fa 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs @@ -102,6 +102,11 @@ public async Task ComputeResource_GetsDeploymentTargetFromKubernetesInfrastructu // The compute environment should be the inner K8s environment Assert.Same(aks.Resource.KubernetesEnvironment, target.ComputeEnvironment); + + // CRITICAL: ContainerRegistry must be set on the DeploymentTargetAnnotation + // so that push steps can resolve the registry endpoint + Assert.NotNull(target.ContainerRegistry); + Assert.IsType(target.ContainerRegistry); } [UnsafeAccessor(UnsafeAccessorKind.Method, Name = "ExecuteBeforeStartHooksAsync")] From 0580583ab640a4ed657849275a9eaf7560f55019 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 14:32:35 +1000 Subject: [PATCH 22/78] Add diagnostic logging to push step wiring Temporary console output to debug why push steps hang after push-prereq. Logs whether the PipelineConfigurationAnnotation runs, whether push-prereq is found, how many push steps exist, and their dependencies. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 80072cd3ca4..a1e2f19961f 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -86,7 +86,25 @@ public static IResourceBuilder AddAzureKuber var pushPrereq = context.Steps .FirstOrDefault(s => s.Name == WellKnownPipelineSteps.PushPrereq); - pushPrereq?.DependsOn(AzureEnvironmentResource.ProvisionInfrastructureStepName); + if (pushPrereq is not null) + { + pushPrereq.DependsOn(AzureEnvironmentResource.ProvisionInfrastructureStepName); + Console.WriteLine($"[AKS] Wired push-prereq to depend on {AzureEnvironmentResource.ProvisionInfrastructureStepName}"); + } + else + { + Console.WriteLine("[AKS] WARNING: push-prereq step not found in pipeline!"); + } + + // Also log all push steps found + var pushSteps = context.Steps.Where(s => s.Tags.Contains(WellKnownPipelineTags.PushContainerImage)).ToList(); + Console.WriteLine($"[AKS] Found {pushSteps.Count} push steps: {string.Join(", ", pushSteps.Select(s => s.Name))}"); + + // Log all step dependencies for push steps + foreach (var step in pushSteps) + { + Console.WriteLine($"[AKS] Push step '{step.Name}' depends on: {string.Join(", ", step.DependsOnSteps)}"); + } })); return builder.AddResource(resource); From 7e21efb5f9a44ceb50313c50954de0cee4230039 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 15:08:49 +1000 Subject: [PATCH 23/78] Fix push steps having no dependencies in K8s compute environment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Diagnostics revealed push steps had EMPTY DependsOnSteps lists. The standard wiring from ProjectResource's PipelineConfigurationAnnotation (pushSteps.DependsOn(buildSteps, push-prereq)) wasn't working because context.GetSteps(resource, tag) returned empty — the resource lookup via ResourceNameComparer didn't match when K8s deployment targets are involved. Fix: directly find push steps by tag in the Steps collection and explicitly wire dependencies on: - provision-azure-bicep-resources (ACR must be provisioned for endpoint) - push-prereq (ACR login must complete) - build-{resourceName} (container image must be built) This ensures the correct execution order: provision → push-prereq → build → push → helm-deploy Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index a1e2f19961f..6f528f3f1df 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -77,33 +77,33 @@ public static IResourceBuilder AddAzureKuber // Ensure push steps wait for ALL Azure provisioning to complete. Push steps // call registry.Endpoint.GetValueAsync() which awaits the BicepOutputReference - // for loginServer — if the ACR hasn't been provisioned yet (or outputs not - // populated yet), this blocks indefinitely. - // We make the push-prereq step depend on provisioning so all push steps - // (which already depend on push-prereq) are guaranteed to run after provisioning. + // for loginServer — if the ACR hasn't been provisioned yet, this blocks. + // + // NOTE: The standard push step dependency wiring (pushSteps.DependsOn(buildSteps) + // and pushSteps.DependsOn(push-prereq)) from ProjectResource's PipelineConfigurationAnnotation + // may not resolve correctly when using Kubernetes compute environments, because + // context.GetSteps(resource, tag) may return empty if the resource reference doesn't + // match. We explicitly wire the dependencies here as a workaround. k8sEnvBuilder.WithAnnotation(new PipelineConfigurationAnnotation(context => { - var pushPrereq = context.Steps - .FirstOrDefault(s => s.Name == WellKnownPipelineSteps.PushPrereq); + var pushSteps = context.Steps + .Where(s => s.Tags.Contains(WellKnownPipelineTags.PushContainerImage)) + .ToList(); - if (pushPrereq is not null) + foreach (var pushStep in pushSteps) { - pushPrereq.DependsOn(AzureEnvironmentResource.ProvisionInfrastructureStepName); - Console.WriteLine($"[AKS] Wired push-prereq to depend on {AzureEnvironmentResource.ProvisionInfrastructureStepName}"); - } - else - { - Console.WriteLine("[AKS] WARNING: push-prereq step not found in pipeline!"); - } - - // Also log all push steps found - var pushSteps = context.Steps.Where(s => s.Tags.Contains(WellKnownPipelineTags.PushContainerImage)).ToList(); - Console.WriteLine($"[AKS] Found {pushSteps.Count} push steps: {string.Join(", ", pushSteps.Select(s => s.Name))}"); - - // Log all step dependencies for push steps - foreach (var step in pushSteps) - { - Console.WriteLine($"[AKS] Push step '{step.Name}' depends on: {string.Join(", ", step.DependsOnSteps)}"); + // Ensure push waits for Azure provisioning (ACR endpoint resolution) + pushStep.DependsOn(AzureEnvironmentResource.ProvisionInfrastructureStepName); + + // Ensure push waits for push-prereq (ACR login) + pushStep.DependsOn(WellKnownPipelineSteps.PushPrereq); + + // Ensure push waits for its corresponding build step + var resourceName = pushStep.Resource?.Name; + if (resourceName is not null) + { + pushStep.DependsOn($"build-{resourceName}"); + } } })); From f06572dcde9ad31bd7f68c5822daccb09f309e99 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 15:39:11 +1000 Subject: [PATCH 24/78] Fix resource group resolution: query Azure directly via az aks list The Azure provisioning context internals (ProvisioningContextTask, AzureProvisionerOptions) are all internal to Aspire.Hosting.Azure and inaccessible from our package. IConfiguration['Azure:ResourceGroup'] is also not reliably set when our step runs because the deployment state manager writes to a different configuration scope. New approach: query Azure directly with 'az aks list --query' to find the cluster's resource group. This is guaranteed to work after provisioning completes, regardless of internal configuration state. The az CLI is already available (validated by validate-azure-login step). Also wires push step dependencies directly by finding steps by tag in the Steps collection, fixing the issue where push steps had empty DependsOnSteps lists in K8s compute environments. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesInfrastructure.cs | 67 +++++++++++++++---- 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index a207ffd4346..4bac12ac948 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -9,8 +9,6 @@ using Aspire.Hosting.Eventing; using Aspire.Hosting.Lifecycle; using Aspire.Hosting.Pipelines; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; namespace Aspire.Hosting.Azure.Kubernetes; @@ -166,14 +164,12 @@ private static async Task GetAksCredentialsAsync( // resolution that may not be available at this point in the pipeline. var clusterName = environment.Name; - // Get the resource group from Azure provisioning configuration. - // The create-provisioning-context step resolves this and stores it in - // the Azure:ResourceGroup config key before any provision steps run. - var configuration = context.Services.GetRequiredService(); - var resourceGroup = configuration["Azure:ResourceGroup"] - ?? throw new InvalidOperationException( - "Azure resource group name not found in configuration. " + - "Ensure the Azure provisioning context has been created."); + // Get the resource group by querying Azure directly for the AKS cluster. + // We can't access the provisioning context internals from this package, + // so we query Azure for the cluster's resource group. + var azPath = FindAzCli(); + var resourceGroup = await GetAksResourceGroupAsync(azPath, clusterName, context) + .ConfigureAwait(false); // Write credentials to an isolated kubeconfig file var kubeConfigDir = Directory.CreateTempSubdirectory("aspire-aks"); @@ -183,8 +179,6 @@ private static async Task GetAksCredentialsAsync( context.Logger.LogInformation("Fetching AKS credentials: az {Arguments}", arguments); - var azPath = FindAzCli(); - using var process = new Process(); process.StartInfo = new ProcessStartInfo { @@ -284,4 +278,53 @@ private static string FindAzCli() throw new InvalidOperationException( "Azure CLI (az) not found. Install it from https://learn.microsoft.com/cli/azure/install-azure-cli"); } + + /// + /// Queries Azure for the resource group of the specified AKS cluster. + /// + private static async Task GetAksResourceGroupAsync( + string azPath, + string clusterName, + PipelineStepContext context) + { + // Use az aks list to find the cluster and its resource group + var arguments = $"aks list --query \"[?name=='{clusterName}'].resourceGroup\" -o tsv"; + + using var process = new Process(); + process.StartInfo = new ProcessStartInfo + { + FileName = azPath, + Arguments = arguments, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + process.Start(); + + var stdout = await process.StandardOutput.ReadToEndAsync(context.CancellationToken).ConfigureAwait(false); + var stderr = await process.StandardError.ReadToEndAsync(context.CancellationToken).ConfigureAwait(false); + + await process.WaitForExitAsync(context.CancellationToken).ConfigureAwait(false); + + if (process.ExitCode != 0) + { + throw new InvalidOperationException( + $"Failed to query AKS cluster resource group (exit code {process.ExitCode}): {stderr.Trim()}"); + } + + var resourceGroup = stdout.Trim(); + + if (string.IsNullOrEmpty(resourceGroup)) + { + throw new InvalidOperationException( + $"AKS cluster '{clusterName}' not found. Ensure the cluster has been provisioned."); + } + + context.Logger.LogDebug("Resolved resource group '{ResourceGroup}' for AKS cluster '{ClusterName}'", + resourceGroup, clusterName); + + return resourceGroup; + } } From 4f139f8e51c6a73f9ebae7bfe775070d1c802431 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 16:50:22 +1000 Subject: [PATCH 25/78] Fix az aks get-credentials arguments: quote values and clean resource group - Quote --resource-group and --name values to handle special characters - Strip line endings from az aks list output to prevent argument parsing issues - Add logging of cluster name and resource group values for debugging Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesInfrastructure.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 4bac12ac948..18a88bda004 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -175,9 +175,11 @@ private static async Task GetAksCredentialsAsync( var kubeConfigDir = Directory.CreateTempSubdirectory("aspire-aks"); var kubeConfigPath = Path.Combine(kubeConfigDir.FullName, "kubeconfig"); - var arguments = $"aks get-credentials --resource-group {resourceGroup} --name {clusterName} --file \"{kubeConfigPath}\" --overwrite-existing"; + var arguments = $"aks get-credentials --resource-group \"{resourceGroup}\" --name \"{clusterName}\" --file \"{kubeConfigPath}\" --overwrite-existing"; - context.Logger.LogInformation("Fetching AKS credentials: az {Arguments}", arguments); + context.Logger.LogInformation( + "Fetching AKS credentials: cluster={ClusterName}, resourceGroup={ResourceGroup}", + clusterName, resourceGroup); using var process = new Process(); process.StartInfo = new ProcessStartInfo @@ -314,7 +316,7 @@ private static async Task GetAksResourceGroupAsync( $"Failed to query AKS cluster resource group (exit code {process.ExitCode}): {stderr.Trim()}"); } - var resourceGroup = stdout.Trim(); + var resourceGroup = stdout.Trim().ReplaceLineEndings("").Trim(); if (string.IsNullOrEmpty(resourceGroup)) { From edf197940d53accdfc4cbbf7a87b7fd74a918595 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 17:17:43 +1000 Subject: [PATCH 26/78] Fix resource group query: use az resource list instead of az aks list The JMESPath query in 'az aks list --query [?name==...].resourceGroup' had quote-escaping issues on Windows when passed via ProcessStartInfo. The quotes in the JMESPath expression were being mangled by cmd.exe, producing truncated/malformed resource group names. Switched to 'az resource list --resource-type ... --name ... --query [0].resourceGroup' which uses --name as a proper CLI argument (no embedded quotes in JMESPath) and the simpler [0].resourceGroup query has no quote escaping issues. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesInfrastructure.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 18a88bda004..00ce9b2864b 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -289,8 +289,9 @@ private static async Task GetAksResourceGroupAsync( string clusterName, PipelineStepContext context) { - // Use az aks list to find the cluster and its resource group - var arguments = $"aks list --query \"[?name=='{clusterName}'].resourceGroup\" -o tsv"; + // Query Azure for the AKS cluster's resource group using az resource list. + // This avoids JMESPath quote-escaping issues with az aks list on Windows. + var arguments = $"resource list --resource-type Microsoft.ContainerService/managedClusters --name \"{clusterName}\" --query [0].resourceGroup -o tsv"; using var process = new Process(); process.StartInfo = new ProcessStartInfo @@ -324,7 +325,7 @@ private static async Task GetAksResourceGroupAsync( $"AKS cluster '{clusterName}' not found. Ensure the cluster has been provisioned."); } - context.Logger.LogDebug("Resolved resource group '{ResourceGroup}' for AKS cluster '{ClusterName}'", + context.Logger.LogInformation("Resolved resource group '{ResourceGroup}' for AKS cluster '{ClusterName}'", resourceGroup, clusterName); return resourceGroup; From 24e7f0d31d3042f66132d9bf230b3b09e97129a8 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 18:23:16 +1000 Subject: [PATCH 27/78] Fix resource group from deployment state + attach ACR to AKS Two fixes: 1. Resource group resolution: Read Azure:ResourceGroup from IConfiguration which is populated from the deployment state JSON file by the deploy-prereq step. This correctly scopes to the current deployment's resource group, avoiding the issue where az resource list returned the wrong cluster when multiple clusters share the same name across resource groups. 2. ACR attach: After fetching AKS credentials, run 'az aks update --attach-acr' to grant the kubelet managed identity AcrPull role on the ACR. Without this, pods get ImagePullBackOff (401 Unauthorized) when pulling from the ACR. The attach is idempotent and won't fail if already attached. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesInfrastructure.cs | 112 ++++++++++++++---- 1 file changed, 89 insertions(+), 23 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 00ce9b2864b..a7116be2fee 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -9,6 +9,8 @@ using Aspire.Hosting.Eventing; using Aspire.Hosting.Lifecycle; using Aspire.Hosting.Pipelines; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; namespace Aspire.Hosting.Azure.Kubernetes; @@ -164,12 +166,13 @@ private static async Task GetAksCredentialsAsync( // resolution that may not be available at this point in the pipeline. var clusterName = environment.Name; - // Get the resource group by querying Azure directly for the AKS cluster. - // We can't access the provisioning context internals from this package, - // so we query Azure for the cluster's resource group. + // Get the resource group from the deployment state. The deployment state + // is loaded into IConfiguration by the deploy-prereq step and contains + // Azure:ResourceGroup from the provisioning context. + // We read it via IConfiguration which is populated from the deployment + // state JSON file before pipeline steps execute. var azPath = FindAzCli(); - var resourceGroup = await GetAksResourceGroupAsync(azPath, clusterName, context) - .ConfigureAwait(false); + var resourceGroup = GetResourceGroupFromDeploymentState(context); // Write credentials to an isolated kubeconfig file var kubeConfigDir = Directory.CreateTempSubdirectory("aspire-aks"); @@ -225,6 +228,15 @@ private static async Task GetAksCredentialsAsync( context.Logger.LogInformation( "AKS credentials written to {KubeConfigPath}", kubeConfigPath); + // Attach ACR to AKS so the kubelet identity can pull images. + // This grants AcrPull role to the kubelet managed identity. + if (environment.DefaultContainerRegistry is not null || + environment.TryGetLastAnnotation(out _)) + { + await AttachAcrToAksAsync(azPath, clusterName, resourceGroup, environment, context) + .ConfigureAwait(false); + } + await getCredsTask.SucceedAsync( $"AKS credentials fetched for cluster {clusterName}", context.CancellationToken).ConfigureAwait(false); @@ -282,16 +294,72 @@ private static string FindAzCli() } /// - /// Queries Azure for the resource group of the specified AKS cluster. + /// Gets the resource group from the deployment state loaded into IConfiguration. + /// The deployment state JSON file contains Azure:ResourceGroup which is loaded + /// by the deploy-prereq step before any other pipeline steps execute. /// - private static async Task GetAksResourceGroupAsync( + private static string GetResourceGroupFromDeploymentState(PipelineStepContext context) + { + var configuration = context.Services.GetRequiredService(); + + // The deployment state is loaded into IConfiguration under the Azure section. + // It's populated from ~/.aspire/deployments/{hash}/{env}.json + var resourceGroup = configuration["Azure:ResourceGroup"]; + + if (!string.IsNullOrEmpty(resourceGroup)) + { + return resourceGroup; + } + + // Fallback: check AzureProvisionerOptions binding + resourceGroup = configuration["Azure:ResourceGroup"]; + + if (string.IsNullOrEmpty(resourceGroup)) + { + throw new InvalidOperationException( + "Azure resource group not found in deployment state. " + + "Ensure Azure provisioning has completed before this step runs."); + } + + return resourceGroup; + } + + /// + /// Attaches an Azure Container Registry to the AKS cluster, granting the kubelet + /// managed identity the AcrPull role so pods can pull container images. + /// + private static async Task AttachAcrToAksAsync( string azPath, string clusterName, + string resourceGroup, + AzureKubernetesEnvironmentResource environment, PipelineStepContext context) { - // Query Azure for the AKS cluster's resource group using az resource list. - // This avoids JMESPath quote-escaping issues with az aks list on Windows. - var arguments = $"resource list --resource-type Microsoft.ContainerService/managedClusters --name \"{clusterName}\" --query [0].resourceGroup -o tsv"; + // Resolve the ACR name from the registry resource's Bicep output + string? acrName = null; + + if (environment.TryGetLastAnnotation(out var annotation) && + annotation.Registry is AzureContainerRegistryResource explicitAcr) + { + acrName = await explicitAcr.NameOutputReference + .GetValueAsync(context.CancellationToken).ConfigureAwait(false); + } + else if (environment.DefaultContainerRegistry is { } defaultAcr) + { + acrName = await defaultAcr.NameOutputReference + .GetValueAsync(context.CancellationToken).ConfigureAwait(false); + } + + if (string.IsNullOrEmpty(acrName)) + { + context.Logger.LogWarning("Could not resolve ACR name — skipping ACR attach"); + return; + } + + context.Logger.LogInformation( + "Attaching ACR '{AcrName}' to AKS cluster '{ClusterName}'", acrName, clusterName); + + var arguments = $"aks update --resource-group \"{resourceGroup}\" --name \"{clusterName}\" --attach-acr \"{acrName}\""; using var process = new Process(); process.StartInfo = new ProcessStartInfo @@ -311,23 +379,21 @@ private static async Task GetAksResourceGroupAsync( await process.WaitForExitAsync(context.CancellationToken).ConfigureAwait(false); - if (process.ExitCode != 0) + if (!string.IsNullOrWhiteSpace(stderr)) { - throw new InvalidOperationException( - $"Failed to query AKS cluster resource group (exit code {process.ExitCode}): {stderr.Trim()}"); + context.Logger.LogDebug("az aks update (stderr): {Error}", stderr); } - var resourceGroup = stdout.Trim().ReplaceLineEndings("").Trim(); - - if (string.IsNullOrEmpty(resourceGroup)) + if (process.ExitCode != 0) { - throw new InvalidOperationException( - $"AKS cluster '{clusterName}' not found. Ensure the cluster has been provisioned."); + context.Logger.LogWarning( + "Failed to attach ACR '{AcrName}' to AKS cluster (exit code {ExitCode}): {Error}", + acrName, process.ExitCode, stderr.Trim()); + // Don't throw — ACR might already be attached from a previous run + } + else + { + context.Logger.LogInformation("ACR '{AcrName}' attached to AKS cluster", acrName); } - - context.Logger.LogInformation("Resolved resource group '{ResourceGroup}' for AKS cluster '{ClusterName}'", - resourceGroup, clusterName); - - return resourceGroup; } } From 5d0250668f515a6ad5d4502cb0663d4c7616eccf Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 18:36:42 +1000 Subject: [PATCH 28/78] Add AcrPull role assignment in Bicep instead of CLI Follows the ACA pattern: the AKS Bicep module now accepts an acrName parameter, references the ACR as an existing resource, and creates an AcrPull role assignment for the kubelet managed identity. This is handled during Bicep provisioning rather than as a runtime CLI step. The acrName parameter is wired from the ACR resource's NameOutputReference via the Parameters dictionary on the AzureBicepResource, which the Azure publishing context automatically resolves in main.bicep. Removed the az aks update --attach-acr CLI approach. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 5 ++ .../AzureKubernetesEnvironmentResource.cs | 31 +++++++ .../AzureKubernetesInfrastructure.cs | 90 ------------------- 3 files changed, 36 insertions(+), 90 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 6f528f3f1df..c4c4426227d 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -75,6 +75,11 @@ public static IResourceBuilder AddAzureKuber resource.DefaultContainerRegistry = defaultRegistry.Resource; k8sEnvBuilder.WithAnnotation(new ContainerRegistryReferenceAnnotation(defaultRegistry.Resource)); + // Wire ACR name as a parameter on the AKS resource so the Bicep module + // can create an AcrPull role assignment for the kubelet identity. + // The publishing context will wire this as a parameter in main.bicep. + resource.Parameters["acrName"] = defaultRegistry.Resource.NameOutputReference; + // Ensure push steps wait for ALL Azure provisioning to complete. Push steps // call registry.Endpoint.GetValueAsync() which awaits the BicepOutputReference // for loginServer — if the ACR hasn't been provisioned yet, this blocks. diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs index dccc56665f6..5eba51ed8d7 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs @@ -5,6 +5,7 @@ using System.Globalization; using System.Text; +using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Azure.Kubernetes; using Aspire.Hosting.Kubernetes; @@ -148,6 +149,14 @@ private string GenerateAksBicep() sb.AppendLine("param location string = resourceGroup().location"); sb.AppendLine(); + // ACR parameter for role assignment + if (DefaultContainerRegistry is not null || this.TryGetLastAnnotation(out _)) + { + sb.AppendLine("@description('The name of the Azure Container Registry for AcrPull role assignment.')"); + sb.AppendLine("param acrName string"); + sb.AppendLine(); + } + // AKS cluster resource sb.Append("resource ").Append(id).AppendLine(" 'Microsoft.ContainerService/managedClusters@2024-06-02-preview' = {"); sb.Append(" name: '").Append(Name).AppendLine("'"); @@ -238,6 +247,28 @@ private string GenerateAksBicep() sb.AppendLine("}"); sb.AppendLine(); + // ACR pull role assignment for kubelet identity + if (DefaultContainerRegistry is not null || this.TryGetLastAnnotation(out _)) + { + sb.AppendLine("// Reference the existing ACR to grant pull access to AKS"); + sb.AppendLine("resource acr 'Microsoft.ContainerRegistry/registries@2025-04-01' existing = {"); + sb.AppendLine(" name: acrName"); + sb.AppendLine("}"); + sb.AppendLine(); + sb.AppendLine("// AcrPull role assignment for the AKS kubelet managed identity"); + sb.Append("resource acrPullRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = {"); + sb.AppendLine(); + sb.Append(" name: guid(acr.id, ").Append(id).AppendLine(".properties.identityProfile.kubeletidentity.objectId, subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d'))"); + sb.AppendLine(" scope: acr"); + sb.AppendLine(" properties: {"); + sb.Append(" principalId: ").Append(id).AppendLine(".properties.identityProfile.kubeletidentity.objectId"); + sb.AppendLine(" roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d')"); + sb.AppendLine(" principalType: 'ServicePrincipal'"); + sb.AppendLine(" }"); + sb.AppendLine("}"); + sb.AppendLine(); + } + // Outputs sb.Append("output id string = ").Append(id).AppendLine(".id"); sb.Append("output name string = ").Append(id).AppendLine(".name"); diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index a7116be2fee..156a8801b41 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -228,15 +228,6 @@ private static async Task GetAksCredentialsAsync( context.Logger.LogInformation( "AKS credentials written to {KubeConfigPath}", kubeConfigPath); - // Attach ACR to AKS so the kubelet identity can pull images. - // This grants AcrPull role to the kubelet managed identity. - if (environment.DefaultContainerRegistry is not null || - environment.TryGetLastAnnotation(out _)) - { - await AttachAcrToAksAsync(azPath, clusterName, resourceGroup, environment, context) - .ConfigureAwait(false); - } - await getCredsTask.SucceedAsync( $"AKS credentials fetched for cluster {clusterName}", context.CancellationToken).ConfigureAwait(false); @@ -306,14 +297,6 @@ private static string GetResourceGroupFromDeploymentState(PipelineStepContext co // It's populated from ~/.aspire/deployments/{hash}/{env}.json var resourceGroup = configuration["Azure:ResourceGroup"]; - if (!string.IsNullOrEmpty(resourceGroup)) - { - return resourceGroup; - } - - // Fallback: check AzureProvisionerOptions binding - resourceGroup = configuration["Azure:ResourceGroup"]; - if (string.IsNullOrEmpty(resourceGroup)) { throw new InvalidOperationException( @@ -323,77 +306,4 @@ private static string GetResourceGroupFromDeploymentState(PipelineStepContext co return resourceGroup; } - - /// - /// Attaches an Azure Container Registry to the AKS cluster, granting the kubelet - /// managed identity the AcrPull role so pods can pull container images. - /// - private static async Task AttachAcrToAksAsync( - string azPath, - string clusterName, - string resourceGroup, - AzureKubernetesEnvironmentResource environment, - PipelineStepContext context) - { - // Resolve the ACR name from the registry resource's Bicep output - string? acrName = null; - - if (environment.TryGetLastAnnotation(out var annotation) && - annotation.Registry is AzureContainerRegistryResource explicitAcr) - { - acrName = await explicitAcr.NameOutputReference - .GetValueAsync(context.CancellationToken).ConfigureAwait(false); - } - else if (environment.DefaultContainerRegistry is { } defaultAcr) - { - acrName = await defaultAcr.NameOutputReference - .GetValueAsync(context.CancellationToken).ConfigureAwait(false); - } - - if (string.IsNullOrEmpty(acrName)) - { - context.Logger.LogWarning("Could not resolve ACR name — skipping ACR attach"); - return; - } - - context.Logger.LogInformation( - "Attaching ACR '{AcrName}' to AKS cluster '{ClusterName}'", acrName, clusterName); - - var arguments = $"aks update --resource-group \"{resourceGroup}\" --name \"{clusterName}\" --attach-acr \"{acrName}\""; - - using var process = new Process(); - process.StartInfo = new ProcessStartInfo - { - FileName = azPath, - Arguments = arguments, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; - - process.Start(); - - var stdout = await process.StandardOutput.ReadToEndAsync(context.CancellationToken).ConfigureAwait(false); - var stderr = await process.StandardError.ReadToEndAsync(context.CancellationToken).ConfigureAwait(false); - - await process.WaitForExitAsync(context.CancellationToken).ConfigureAwait(false); - - if (!string.IsNullOrWhiteSpace(stderr)) - { - context.Logger.LogDebug("az aks update (stderr): {Error}", stderr); - } - - if (process.ExitCode != 0) - { - context.Logger.LogWarning( - "Failed to attach ACR '{AcrName}' to AKS cluster (exit code {ExitCode}): {Error}", - acrName, process.ExitCode, stderr.Trim()); - // Don't throw — ACR might already be attached from a previous run - } - else - { - context.Logger.LogInformation("ACR '{AcrName}' attached to AKS cluster", acrName); - } - } } From 4c8cdf5ff6bf244cefaa98c494163717aafd9162 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 18:41:27 +1000 Subject: [PATCH 29/78] Add AKS cluster info and get-credentials command to deploy summary Shows cluster name, resource group, and the az aks get-credentials command in the pipeline summary so users can easily connect to the cluster after deployment. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesInfrastructure.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 156a8801b41..a0052bf2037 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -228,6 +228,15 @@ private static async Task GetAksCredentialsAsync( context.Logger.LogInformation( "AKS credentials written to {KubeConfigPath}", kubeConfigPath); + // Add AKS connection info to the pipeline summary + context.Summary.Add( + "☸ AKS Cluster", + new MarkdownString($"**{clusterName}** in resource group **{resourceGroup}**")); + + context.Summary.Add( + "🔑 Connect to cluster", + new MarkdownString($"`az aks get-credentials --resource-group {resourceGroup} --name {clusterName}`")); + await getCredsTask.SucceedAsync( $"AKS credentials fetched for cluster {clusterName}", context.CancellationToken).ConfigureAwait(false); From 7cae999ae174a047aecd50b55838c2be8c99012c Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 19:03:05 +1000 Subject: [PATCH 30/78] Fix Bicep BCP120: use compile-time values for role assignment name The guid() function for the role assignment name used env.properties.identityProfile.kubeletidentity.objectId which is a runtime property (only known after AKS provisioning). Bicep requires the 'name' property to be calculable at deployment start. Fixed by using env.id (compile-time deterministic) instead of the runtime objectId in the guid() call. The principalId property still uses the runtime objectId since that's evaluated during deployment. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentResource.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs index 5eba51ed8d7..515aa951433 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs @@ -256,9 +256,8 @@ private string GenerateAksBicep() sb.AppendLine("}"); sb.AppendLine(); sb.AppendLine("// AcrPull role assignment for the AKS kubelet managed identity"); - sb.Append("resource acrPullRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = {"); - sb.AppendLine(); - sb.Append(" name: guid(acr.id, ").Append(id).AppendLine(".properties.identityProfile.kubeletidentity.objectId, subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d'))"); + sb.AppendLine("resource acrPullRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = {"); + sb.Append(" name: guid(acr.id, ").Append(id).AppendLine(".id, subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d'))"); sb.AppendLine(" scope: acr"); sb.AppendLine(" properties: {"); sb.Append(" principalId: ").Append(id).AppendLine(".properties.identityProfile.kubeletidentity.objectId"); From 50ba100ca538d6ae2fe7a531f3c15ab2d96fc41c Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 19:31:15 +1000 Subject: [PATCH 31/78] Fix first deploy: fall back to az CLI when deployment state not loaded On first deploy to a new environment, the deployment state JSON file doesn't exist yet. IConfiguration is loaded at app startup and doesn't see the state file that's written later by create-provisioning-context. Fix: try IConfiguration first (works on re-deploys where the state file exists), then fall back to 'az resource list' to query the resource group directly from Azure (works on first deploy since provisioning has completed by this point). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesInfrastructure.cs | 54 +++++++++++++++---- 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index a0052bf2037..074462fb54f 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -172,7 +172,8 @@ private static async Task GetAksCredentialsAsync( // We read it via IConfiguration which is populated from the deployment // state JSON file before pipeline steps execute. var azPath = FindAzCli(); - var resourceGroup = GetResourceGroupFromDeploymentState(context); + var resourceGroup = await GetResourceGroupAsync(azPath, clusterName, context) + .ConfigureAwait(false); // Write credentials to an isolated kubeconfig file var kubeConfigDir = Directory.CreateTempSubdirectory("aspire-aks"); @@ -294,23 +295,56 @@ private static string FindAzCli() } /// - /// Gets the resource group from the deployment state loaded into IConfiguration. - /// The deployment state JSON file contains Azure:ResourceGroup which is loaded - /// by the deploy-prereq step before any other pipeline steps execute. + /// Gets the resource group, trying deployment state first, falling back to az CLI query. + /// On first deploy, the deployment state may not be loaded into IConfiguration yet + /// because it's written during the pipeline run (after create-provisioning-context). /// - private static string GetResourceGroupFromDeploymentState(PipelineStepContext context) + private static async Task GetResourceGroupAsync( + string azPath, + string clusterName, + PipelineStepContext context) { + // Try deployment state first (works on re-deploys) var configuration = context.Services.GetRequiredService(); - - // The deployment state is loaded into IConfiguration under the Azure section. - // It's populated from ~/.aspire/deployments/{hash}/{env}.json var resourceGroup = configuration["Azure:ResourceGroup"]; + if (!string.IsNullOrEmpty(resourceGroup)) + { + return resourceGroup; + } + + // Fallback for first deploy: query Azure directly + context.Logger.LogDebug( + "Resource group not in deployment state, querying Azure for cluster '{ClusterName}'", + clusterName); + + var arguments = $"resource list --resource-type Microsoft.ContainerService/managedClusters --name \"{clusterName}\" --query [0].resourceGroup -o tsv"; + + using var process = new Process(); + process.StartInfo = new ProcessStartInfo + { + FileName = azPath, + Arguments = arguments, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + process.Start(); + + var stdout = await process.StandardOutput.ReadToEndAsync(context.CancellationToken).ConfigureAwait(false); + await process.StandardError.ReadToEndAsync(context.CancellationToken).ConfigureAwait(false); + + await process.WaitForExitAsync(context.CancellationToken).ConfigureAwait(false); + + resourceGroup = stdout.Trim().ReplaceLineEndings("").Trim(); + if (string.IsNullOrEmpty(resourceGroup)) { throw new InvalidOperationException( - "Azure resource group not found in deployment state. " + - "Ensure Azure provisioning has completed before this step runs."); + $"Could not resolve resource group for AKS cluster '{clusterName}'. " + + "Ensure Azure provisioning has completed."); } return resourceGroup; From 3e15ec539dfb3d6173a261942bf1472e168bbe5e Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 20:58:19 +1000 Subject: [PATCH 32/78] Support multi-environment AKS with shared ACR and WithComputeEnvironment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes for the multi-environment scenario: 1. WithContainerRegistry now updates resource.Parameters['acrName'] to reference the explicit registry's NameOutputReference, fixing the 'key not present in dictionary' error when the Bicep publisher tries to resolve the removed default ACR. 2. Added ParentComputeEnvironment property to KubernetesEnvironmentResource. When set, KubernetesInfrastructure matches resources whose compute env is either the K8s env itself OR its parent (the AKS resource). This allows WithComputeEnvironment(aksEnv) to work correctly — the user targets the AKS resource, and the inner K8s env picks it up. 3. AddAzureKubernetesEnvironment sets ParentComputeEnvironment on the inner K8s environment, completing the parent-child relationship. Example AppHost code that now works: var registry = builder.AddAzureContainerRegistry('registry'); var enva = builder.AddAzureKubernetesEnvironment('enva') .WithContainerRegistry(registry); var envb = builder.AddAzureKubernetesEnvironment('envb') .WithContainerRegistry(registry); builder.AddProject('api').WithComputeEnvironment(enva); Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 9 +++++++++ .../AzureKubernetesInfrastructure.cs | 8 ++------ .../KubernetesEnvironmentResource.cs | 12 ++++++++++++ .../KubernetesInfrastructure.cs | 8 ++++++-- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index c4c4426227d..a7892bb784c 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -62,6 +62,11 @@ public static IResourceBuilder AddAzureKuber var resource = new AzureKubernetesEnvironmentResource(name, ConfigureAksInfrastructure); resource.KubernetesEnvironment = k8sEnvBuilder.Resource; + // Set the parent so KubernetesInfrastructure matches resources that use + // WithComputeEnvironment(aksEnv) — the inner K8s env checks both itself + // and its parent when filtering compute resources. + k8sEnvBuilder.Resource.ParentComputeEnvironment = resource; + if (builder.ExecutionContext.IsRunMode) { return builder.CreateResourceBuilder(resource); @@ -277,6 +282,10 @@ public static IResourceBuilder WithContainer builder.Resource.KubernetesEnvironment.Annotations.Add( new ContainerRegistryReferenceAnnotation(registry.Resource)); + // Update the acrName parameter to reference the explicit registry's output + // (replaces the default ACR reference set during AddAzureKubernetesEnvironment) + builder.Resource.Parameters["acrName"] = registry.Resource.NameOutputReference; + return builder; } diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 074462fb54f..a673b59d821 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -61,6 +61,8 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance foreach (var r in @event.Model.GetComputeResources()) { var resourceComputeEnvironment = r.GetComputeEnvironment(); + + // Check if this resource targets THIS AKS environment if (resourceComputeEnvironment is not null && resourceComputeEnvironment != environment) { continue; @@ -72,12 +74,6 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance { r.Annotations.Add(new AksNodePoolAffinityAnnotation(defaultUserPool)); } - - // NOTE: We do NOT add DeploymentTargetAnnotation here. - // The inner KubernetesEnvironmentResource is in the model, so - // KubernetesInfrastructure will handle Helm chart generation - // and add the DeploymentTargetAnnotation with the correct - // KubernetesResource deployment target. } } diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs index 441b659aa9a..2cf15914b6c 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs @@ -103,6 +103,18 @@ public sealed class KubernetesEnvironmentResource : Resource, IComputeEnvironmen /// public string? KubeConfigPath { get; set; } + /// + /// Gets or sets the parent compute environment resource that owns this Kubernetes environment. + /// When set, resources with WithComputeEnvironment targeting the parent will also + /// be processed by this Kubernetes environment. + /// + /// + /// This is used by Azure Kubernetes Service (AKS) integration where the user calls + /// WithComputeEnvironment(aksEnv) but the inner KubernetesEnvironmentResource + /// needs to process the resource. + /// + public IComputeEnvironmentResource? ParentComputeEnvironment { get; set; } + internal IPortAllocator PortAllocator { get; } = new PortAllocator(); /// diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesInfrastructure.cs b/src/Aspire.Hosting.Kubernetes/KubernetesInfrastructure.cs index 46c1003398c..7de948995e3 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesInfrastructure.cs @@ -52,9 +52,13 @@ private async Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken foreach (var r in @event.Model.GetComputeResources()) { - // Skip resources that are explicitly targeted to a different compute environment + // Skip resources that are explicitly targeted to a different compute environment. + // Also match if the resource targets a parent compute environment (e.g., AKS) + // that owns this Kubernetes environment. var resourceComputeEnvironment = r.GetComputeEnvironment(); - if (resourceComputeEnvironment is not null && resourceComputeEnvironment != environment) + if (resourceComputeEnvironment is not null && + resourceComputeEnvironment != environment && + resourceComputeEnvironment != environment.ParentComputeEnvironment) { continue; } From 5a873e0ecdb431eeee7fc45e306142b7bbb9296e Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 21:34:04 +1000 Subject: [PATCH 33/78] Add multi-environment targeting test Verifies that WithComputeEnvironment correctly routes resources to their targeted AKS/K8s environments, including the ParentComputeEnvironment matching logic. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesInfrastructureTests.cs | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs index 55fef92b7fa..9dda15f3eb7 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs @@ -111,4 +111,45 @@ public async Task ComputeResource_GetsDeploymentTargetFromKubernetesInfrastructu [UnsafeAccessor(UnsafeAccessorKind.Method, Name = "ExecuteBeforeStartHooksAsync")] private static extern Task ExecuteBeforeStartHooksAsync(DistributedApplication app, CancellationToken cancellationToken); + + [Fact] + public async Task MultiEnv_ResourcesMatchCorrectEnvironment() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var registry = builder.AddAzureContainerRegistry("registry"); + var enva = builder.AddAzureKubernetesEnvironment("enva") + .WithContainerRegistry(registry); + var envb = builder.AddAzureKubernetesEnvironment("envb") + .WithContainerRegistry(registry); + + var cache = builder.AddContainer("cache", "redis") + .WithComputeEnvironment(enva); + var api = builder.AddContainer("api", "myapi") + .WithComputeEnvironment(enva); + var other = builder.AddContainer("other", "myother") + .WithComputeEnvironment(envb); + + // ParentComputeEnvironment should be set + Assert.Same(enva.Resource, enva.Resource.KubernetesEnvironment.ParentComputeEnvironment); + Assert.Same(envb.Resource, envb.Resource.KubernetesEnvironment.ParentComputeEnvironment); + + await using var app = builder.Build(); + await ExecuteBeforeStartHooksAsync(app, default); + + // cache and api should get DeploymentTargetAnnotation from enva-k8s + Assert.True(cache.Resource.TryGetLastAnnotation(out var cacheTarget), + "cache should have DeploymentTargetAnnotation"); + Assert.Same(enva.Resource.KubernetesEnvironment, cacheTarget.ComputeEnvironment); + + Assert.True(api.Resource.TryGetLastAnnotation(out var apiTarget), + "api should have DeploymentTargetAnnotation"); + Assert.Same(enva.Resource.KubernetesEnvironment, apiTarget.ComputeEnvironment); + + // other should get DeploymentTargetAnnotation from envb-k8s + Assert.True(other.Resource.TryGetLastAnnotation(out var otherTarget), + "other should have DeploymentTargetAnnotation"); + Assert.Same(envb.Resource.KubernetesEnvironment, otherTarget.ComputeEnvironment); + } } From a137935443cfc8e3116b190e1c712e2733608ace Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 21:42:48 +1000 Subject: [PATCH 34/78] Fix publish to include compute resources in Helm chart templates The publish context and step factory used GetDeploymentTargetAnnotation(environment) where environment is the inner K8s env. But WithComputeEnvironment(aksEnv) sets the compute env to the AKS resource, and KubernetesInfrastructure now sets DeploymentTargetAnnotation.ComputeEnvironment to match the resource's actual compute env (the AKS resource, not the inner K8s env). Updated all GetDeploymentTargetAnnotation calls to use ParentComputeEnvironment when available, so the lookup matches correctly. Also fixed KubernetesInfrastructure to set ComputeEnvironment on the DeploymentTargetAnnotation to the resource's actual compute env rather than always using the inner K8s env. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesEnvironmentResource.cs | 6 ++++-- .../KubernetesInfrastructure.cs | 8 ++++++-- .../KubernetesPublishingContext.cs | 4 +++- .../AzureKubernetesInfrastructureTests.cs | 10 +++++----- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs index 2cf15914b6c..0246f5fca65 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs @@ -196,7 +196,8 @@ public KubernetesEnvironmentResource(string name) : base(name) foreach (var computeResource in resources) { - var deploymentTarget = computeResource.GetDeploymentTargetAnnotation(environment)?.DeploymentTarget; + var targetEnv = (IComputeEnvironmentResource?)environment.ParentComputeEnvironment ?? environment; + var deploymentTarget = computeResource.GetDeploymentTargetAnnotation(targetEnv)?.DeploymentTarget; if (deploymentTarget is not null && deploymentTarget.TryGetAnnotationsOfType(out var annotations)) { @@ -232,7 +233,8 @@ public KubernetesEnvironmentResource(string name) : base(name) foreach (var computeResource in resources) { - var deploymentTarget = computeResource.GetDeploymentTargetAnnotation(this)?.DeploymentTarget; + var targetEnv = (IComputeEnvironmentResource?)ParentComputeEnvironment ?? this; + var deploymentTarget = computeResource.GetDeploymentTargetAnnotation(targetEnv)?.DeploymentTarget; if (deploymentTarget is null) { continue; diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesInfrastructure.cs b/src/Aspire.Hosting.Kubernetes/KubernetesInfrastructure.cs index 7de948995e3..b23afc41219 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesInfrastructure.cs @@ -73,10 +73,14 @@ private async Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken var serviceResource = await environmentContext.CreateKubernetesResourceAsync(r, executionContext, cancellationToken).ConfigureAwait(false); serviceResource.AddPrintSummaryStep(); - // Add deployment target annotation to the resource + // Add deployment target annotation to the resource. + // Use the resource's actual compute environment (which may be a parent + // like AzureKubernetesEnvironmentResource) so that GetDeploymentTargetAnnotation + // can match it correctly during publish. + var computeEnvForAnnotation = resourceComputeEnvironment ?? (IComputeEnvironmentResource)environment; r.Annotations.Add(new DeploymentTargetAnnotation(serviceResource) { - ComputeEnvironment = environment, + ComputeEnvironment = computeEnvForAnnotation, ContainerRegistry = containerRegistry }); } diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs b/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs index f29e9f65f0e..b1eabcc60b8 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs @@ -75,7 +75,9 @@ private async Task WriteKubernetesOutputAsync(DistributedApplicationModel model, foreach (var resource in resources) { - if (resource.GetDeploymentTargetAnnotation(environment)?.DeploymentTarget is KubernetesResource serviceResource) + // Check for deployment target matching this environment or its parent (e.g., AKS) + var targetEnv = (IComputeEnvironmentResource?)environment.ParentComputeEnvironment ?? environment; + if (resource.GetDeploymentTargetAnnotation(targetEnv)?.DeploymentTarget is KubernetesResource serviceResource) { // Materialize Dockerfile factory if present if (serviceResource.TargetResource.TryGetLastAnnotation(out var dockerfileBuildAnnotation) && diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs index 9dda15f3eb7..73d028e129a 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs @@ -138,18 +138,18 @@ public async Task MultiEnv_ResourcesMatchCorrectEnvironment() await using var app = builder.Build(); await ExecuteBeforeStartHooksAsync(app, default); - // cache and api should get DeploymentTargetAnnotation from enva-k8s + // cache and api should get DeploymentTargetAnnotation targeting enva Assert.True(cache.Resource.TryGetLastAnnotation(out var cacheTarget), "cache should have DeploymentTargetAnnotation"); - Assert.Same(enva.Resource.KubernetesEnvironment, cacheTarget.ComputeEnvironment); + Assert.Same(enva.Resource, cacheTarget.ComputeEnvironment); Assert.True(api.Resource.TryGetLastAnnotation(out var apiTarget), "api should have DeploymentTargetAnnotation"); - Assert.Same(enva.Resource.KubernetesEnvironment, apiTarget.ComputeEnvironment); + Assert.Same(enva.Resource, apiTarget.ComputeEnvironment); - // other should get DeploymentTargetAnnotation from envb-k8s + // other should get DeploymentTargetAnnotation targeting envb Assert.True(other.Resource.TryGetLastAnnotation(out var otherTarget), "other should have DeploymentTargetAnnotation"); - Assert.Same(envb.Resource.KubernetesEnvironment, otherTarget.ComputeEnvironment); + Assert.Same(envb.Resource, otherTarget.ComputeEnvironment); } } From 88d44c2113a3cca096688bed58cf0fbc0838177b Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Mon, 13 Apr 2026 23:12:46 +1000 Subject: [PATCH 35/78] Fix Helm template parse error for Azure Bicep output references BicepOutputReference.ValueExpression uses single braces ({storage.outputs.blobEndpoint}) but ResolveUnknownValue only stripped double braces ({{ }}) via HelmExtensions delimiters. Single braces passed through to the Helm template, causing a parse error. Fix: also strip single { and } characters when sanitizing the values key. Fixes #16114 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/Aspire.Hosting.Kubernetes/KubernetesResource.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs index 3015e3438e3..bd77f3cc2bf 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs @@ -660,6 +660,8 @@ private static HelmValue ResolveUnknownValue(IManifestExpressionProvider paramet { var formattedName = parameter.ValueExpression.Replace(HelmExtensions.StartDelimiter, string.Empty) .Replace(HelmExtensions.EndDelimiter, string.Empty) + .Replace("{", string.Empty) + .Replace("}", string.Empty) .Replace(".", "_") .ToHelmValuesSectionName(); From 352e36989b7f60fc826664008597c7b17fcc8f0c Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 14 Apr 2026 11:27:44 +1000 Subject: [PATCH 36/78] Add VNet subnet integration for AKS via WithDelegatedSubnet When WithDelegatedSubnet(subnet) is called, the generated AKS Bicep now: - Accepts a subnetId parameter wired from the subnet's ID output - Sets vnetSubnetID on all agent pool profiles - Configures Azure CNI network profile (networkPlugin: azure) with default service CIDR and DNS service IP This follows the ACA pattern where DelegatedSubnetAnnotation is read during infrastructure configuration and the subnet ID is passed as a provisioning parameter. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentResource.cs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs index 515aa951433..6013580913f 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs @@ -157,6 +157,18 @@ private string GenerateAksBicep() sb.AppendLine(); } + // Subnet parameter for VNet integration + var hasDelegatedSubnet = this.TryGetLastAnnotation(out var subnetAnnotation); + if (hasDelegatedSubnet) + { + // Wire the subnet ID as a parameter so the publishing context resolves it in main.bicep + Parameters["subnetId"] = subnetAnnotation!.SubnetId; + + sb.AppendLine("@description('The subnet ID for AKS node pool VNet integration.')"); + sb.AppendLine("param subnetId string"); + sb.AppendLine(); + } + // AKS cluster resource sb.Append("resource ").Append(id).AppendLine(" 'Microsoft.ContainerService/managedClusters@2024-06-02-preview' = {"); sb.Append(" name: '").Append(Name).AppendLine("'"); @@ -199,6 +211,10 @@ private string GenerateAksBicep() sb.AppendLine(" enableAutoScaling: true"); sb.Append(" mode: '").Append(mode).AppendLine("'"); sb.AppendLine(" osType: 'Linux'"); + if (hasDelegatedSubnet) + { + sb.AppendLine(" vnetSubnetID: subnetId"); + } sb.AppendLine(" }"); } sb.AppendLine(" ]"); @@ -242,6 +258,15 @@ private string GenerateAksBicep() sb.Append(" dnsServiceIP: '").Append(NetworkProfile.DnsServiceIP).AppendLine("'"); sb.AppendLine(" }"); } + else if (hasDelegatedSubnet) + { + // Default Azure CNI network profile when a subnet is delegated + sb.AppendLine(" networkProfile: {"); + sb.AppendLine(" networkPlugin: 'azure'"); + sb.AppendLine(" serviceCidr: '10.0.0.0/16'"); + sb.AppendLine(" dnsServiceIP: '10.0.0.10'"); + sb.AppendLine(" }"); + } sb.AppendLine(" }"); sb.AppendLine("}"); From 60dcd9e2b5eee9071edc519c8943a39463524252 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 14 Apr 2026 13:09:01 +1000 Subject: [PATCH 37/78] =?UTF-8?q?Fix:=20AKS=20doesn't=20support=20subnet?= =?UTF-8?q?=20delegation=20=E2=80=94=20add=20WithSubnet=20extension?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AKS requires plain (non-delegated) subnets for node pools. The previous approach used WithDelegatedSubnet which implements IAzureDelegatedSubnetResource and adds a 'Microsoft.ContainerService/managedClusters' service delegation to the subnet. Azure rejects this with 'SubnetIsDelegated' error. Changes: - Removed IAzureDelegatedSubnetResource from AzureKubernetesEnvironmentResource - Added WithSubnet(subnet) extension that stores the subnet ID without adding a service delegation (via AksSubnetAnnotation) - Added Aspire.Hosting.Azure.Network project reference for AzureSubnetResource - WithDelegatedSubnet still works as fallback but users should use WithSubnet Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AksSubnetAnnotation.cs | 19 ++++++++++++ .../Aspire.Hosting.Azure.Kubernetes.csproj | 1 + .../AzureKubernetesEnvironmentExtensions.cs | 29 +++++++++++++++++++ .../AzureKubernetesEnvironmentResource.cs | 28 ++++++++++++------ 4 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 src/Aspire.Hosting.Azure.Kubernetes/AksSubnetAnnotation.cs diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksSubnetAnnotation.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksSubnetAnnotation.cs new file mode 100644 index 00000000000..9bdf97dbee3 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AksSubnetAnnotation.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Hosting.ApplicationModel; + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Annotation that stores a subnet ID reference for AKS VNet integration. +/// Unlike DelegatedSubnetAnnotation, this does NOT add a service delegation +/// to the subnet — AKS uses plain (non-delegated) subnets for node pools. +/// +internal sealed class AksSubnetAnnotation(BicepOutputReference subnetId) : IResourceAnnotation +{ + /// + /// Gets the subnet ID output reference. + /// + public BicepOutputReference SubnetId { get; } = subnetId; +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj index bbeba708ab5..3d6ab28eecb 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj +++ b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj @@ -19,6 +19,7 @@ + diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index a7892bb784c..adbfa97e3f8 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -3,6 +3,7 @@ #pragma warning disable ASPIREPIPELINES001 // Pipeline types are experimental #pragma warning disable ASPIREAZURE001 // AzureEnvironmentResource is experimental +#pragma warning disable ASPIREAZURE003 // Subnet/network types are experimental using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Azure; @@ -249,6 +250,34 @@ public static IResourceBuilder AsPrivateClus return builder; } + /// + /// Configures the AKS cluster to use a VNet subnet for node pool networking. + /// Unlike , this does NOT + /// add a service delegation to the subnet — AKS uses plain (non-delegated) subnets. + /// + /// The AKS environment resource builder. + /// The subnet to use for AKS node pools. + /// A reference to the for chaining. + /// + /// + /// var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); + /// var subnet = vnet.AddSubnet("aks-subnet", "10.0.0.0/22"); + /// var aks = builder.AddAzureKubernetesEnvironment("aks") + /// .WithSubnet(subnet); + /// + /// + [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + public static IResourceBuilder WithSubnet( + this IResourceBuilder builder, + IResourceBuilder subnet) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(subnet); + + builder.WithAnnotation(new AksSubnetAnnotation(subnet.Resource.Id)); + return builder; + } + /// /// Configures the AKS environment to use a specific Azure Container Registry for image storage. /// When set, this replaces the auto-created default container registry. diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs index 6013580913f..bcbf8214ab7 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs @@ -22,12 +22,8 @@ public class AzureKubernetesEnvironmentResource( Action configureInfrastructure) : AzureProvisioningResource(name, configureInfrastructure), IAzureComputeEnvironmentResource, - IAzureDelegatedSubnetResource, IAzureNspAssociationTarget { - /// - string IAzureDelegatedSubnetResource.DelegatedSubnetServiceName - => "Microsoft.ContainerService/managedClusters"; /// /// Gets the underlying Kubernetes environment resource used for Helm-based deployment. @@ -158,12 +154,26 @@ private string GenerateAksBicep() } // Subnet parameter for VNet integration - var hasDelegatedSubnet = this.TryGetLastAnnotation(out var subnetAnnotation); - if (hasDelegatedSubnet) + var hasSubnet = this.TryGetLastAnnotation(out var subnetAnnotation); + if (!hasSubnet) { - // Wire the subnet ID as a parameter so the publishing context resolves it in main.bicep + // Fallback: check for DelegatedSubnetAnnotation (legacy WithDelegatedSubnet usage) + hasSubnet = this.TryGetLastAnnotation(out var delegatedAnnotation); + if (hasSubnet) + { + // Wire as parameter — but note that DelegatedSubnetAnnotation adds a service + // delegation which AKS doesn't support. Users should use WithSubnet instead. + Parameters["subnetId"] = delegatedAnnotation!.SubnetId; + } + } + else + { + // Wire the subnet ID as a parameter so the publishing context resolves it Parameters["subnetId"] = subnetAnnotation!.SubnetId; + } + if (hasSubnet) + { sb.AppendLine("@description('The subnet ID for AKS node pool VNet integration.')"); sb.AppendLine("param subnetId string"); sb.AppendLine(); @@ -211,7 +221,7 @@ private string GenerateAksBicep() sb.AppendLine(" enableAutoScaling: true"); sb.Append(" mode: '").Append(mode).AppendLine("'"); sb.AppendLine(" osType: 'Linux'"); - if (hasDelegatedSubnet) + if (hasSubnet) { sb.AppendLine(" vnetSubnetID: subnetId"); } @@ -258,7 +268,7 @@ private string GenerateAksBicep() sb.Append(" dnsServiceIP: '").Append(NetworkProfile.DnsServiceIP).AppendLine("'"); sb.AppendLine(" }"); } - else if (hasDelegatedSubnet) + else if (hasSubnet) { // Default Azure CNI network profile when a subnet is delegated sb.AppendLine(" networkProfile: {"); From 72f58b7369caebd26e28ee017ede219755cd44bf Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 14 Apr 2026 14:42:54 +1000 Subject: [PATCH 38/78] Resolve IValueProvider expressions in Helm values at deploy time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit During publish, config values backed by IValueProvider (e.g., Bicep output references like {storage.outputs.blobEndpoint}) were written as raw expression strings to values.yaml. The prepare step only resolved ParameterResource values, leaving IValueProvider expressions unresolved. Fix (all in Aspire.Hosting.Kubernetes — no Azure dependency): 1. HelmValue.ValueProviderSource: new optional IValueProvider property set when ResolveUnknownValue detects the expression provider also implements IValueProvider 2. KubernetesPublishingContext: when a HelmValue has a ValueProviderSource, writes an empty placeholder and captures a CapturedHelmValueProvider for deploy-time resolution 3. KubernetesEnvironmentResource.CapturedHelmValueProvider: new record storing (Section, ResourceKey, ValueKey, IValueProvider) 4. HelmDeploymentEngine Phase 4: calls GetValueAsync() on captured IValueProvider entries to resolve values from external sources This is cloud-provider agnostic — works with any IValueProvider implementation (Azure Bicep outputs, AWS outputs, etc.). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Deployment/HelmDeploymentEngine.cs | 17 +++++++++++++- .../KubernetesEnvironmentResource.cs | 13 +++++++++++ .../KubernetesPublishingContext.cs | 15 ++++++++++++ .../KubernetesResource.cs | 23 ++++++++++++++++++- 4 files changed, 66 insertions(+), 2 deletions(-) diff --git a/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs b/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs index 01e7c3c511c..25cba322df8 100644 --- a/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs +++ b/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs @@ -283,7 +283,8 @@ internal static async Task ResolveAndWriteDeployValuesAsync( { if (environment.CapturedHelmValues.Count == 0 && environment.CapturedHelmCrossReferences.Count == 0 - && environment.CapturedHelmImageReferences.Count == 0) + && environment.CapturedHelmImageReferences.Count == 0 + && environment.CapturedHelmValueProviders.Count == 0) { return; } @@ -329,6 +330,20 @@ internal static async Task ResolveAndWriteDeployValuesAsync( } } + // Phase 4: Resolve generic IValueProvider references. + // During publish, values backed by IValueProvider (e.g., Bicep output references, + // connection strings) are written as empty placeholders. At deploy time, we call + // GetValueAsync() to resolve the actual values from external sources. + // This is cloud-provider agnostic — any IValueProvider implementation works. + foreach (var valueProviderRef in environment.CapturedHelmValueProviders) + { + var resolvedValue = await valueProviderRef.ValueProvider.GetValueAsync(cancellationToken).ConfigureAwait(false); + if (resolvedValue is not null) + { + SetOverrideValue(overrideValues, valueProviderRef.Section, valueProviderRef.ResourceKey, valueProviderRef.ValueKey, resolvedValue); + } + } + if (overrideValues.Count > 0) { var serializer = new YamlDotNet.Serialization.SerializerBuilder() diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs index 0246f5fca65..3522184e938 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs @@ -154,6 +154,19 @@ internal sealed record CapturedHelmCrossReference(string Section, string Resourc /// internal sealed record CapturedHelmImageReference(string Section, string ResourceKey, string ValueKey, IResource Resource); + /// + /// Represents a captured value provider reference that needs deploy-time resolution. + /// This handles any implementation (e.g., Bicep output references, + /// connection strings) that can't be resolved at publish time. + /// + internal sealed record CapturedHelmValueProvider(string Section, string ResourceKey, string ValueKey, IValueProvider ValueProvider); + + /// + /// Captured value provider references populated during publish, consumed during deploy + /// to resolve values from external sources (e.g., Azure Bicep outputs). + /// + internal List CapturedHelmValueProviders { get; } = []; + /// /// Gets or sets the delegate that creates deployment pipeline steps for the configured engine. /// diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs b/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs index b1eabcc60b8..1c9834eeb1f 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs @@ -190,6 +190,21 @@ private async Task AddValuesToHelmSectionAsync( else { value = helmExpressionWithValue.Value; + + // If the value has an IValueProvider source, capture it for deploy-time + // resolution. Write an empty placeholder now and resolve at deploy time. + // This handles Bicep output references, connection strings, and any other + // deferred value source without requiring Azure-specific knowledge. + if (helmExpressionWithValue.ValueProviderSource is { } valueProvider) + { + value = string.Empty; + environment?.CapturedHelmValueProviders.Add( + new KubernetesEnvironmentResource.CapturedHelmValueProvider( + helmKey, + resource.Name.ToHelmValuesSectionName(), + valuesKey, + valueProvider)); + } } paramValues[valuesKey] = value ?? string.Empty; diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs index bd77f3cc2bf..ce9e5b5b83e 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs @@ -669,7 +669,20 @@ private static HelmValue ResolveUnknownValue(IManifestExpressionProvider paramet formattedName.ToHelmSecretExpression(resource.Name) : formattedName.ToHelmConfigExpression(resource.Name); - return new(helmExpression, parameter.ValueExpression); + var helmValue = new HelmValue(helmExpression, parameter.ValueExpression); + + // If the expression provider also implements IValueProvider, attach it + // for deploy-time resolution. This handles Bicep output references, + // connection strings, and any other deferred value source. + if (parameter is IValueProvider valueProvider) + { + helmValue = new HelmValue(helmExpression, parameter.ValueExpression) + { + ValueProviderSource = valueProvider + }; + } + + return helmValue; } private static string GetKubernetesProtocolName(ProtocolType type) @@ -745,6 +758,14 @@ public HelmValue(string expression, ParameterResource parameterSource) /// public IResource? ImageResource { get; init; } + /// + /// Gets the value provider for deferred resolution at deploy time. + /// When set, the value is resolved via + /// during the prepare step, replacing the placeholder in values.yaml. + /// This handles any value provider (e.g., Bicep output references, connection strings). + /// + public IValueProvider? ValueProviderSource { get; init; } + /// /// Gets the key to use when writing this value to the Helm values.yaml file. /// When set, this overrides the dictionary key to ensure the values.yaml key matches From e5ff9f16911e715775e0123449781d0b264d4677 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 14 Apr 2026 15:26:38 +1000 Subject: [PATCH 39/78] Fix composite expressions with deferred values + scope Helm chart names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes: 1. Composite expressions (e.g., 'Endpoint={storage.outputs.blobEndpoint};ContainerName=photos') containing IValueProvider references (like BicepOutputReference) are now deferred for deploy-time resolution. Added IsUnresolvedAtPublishTime() check before processing sub-expressions — if any sub-expression would fall through to ResolveUnknownValue, the entire composite expression is captured as a CapturedHelmValueProvider. 2. Helm chart names are now scoped per AKS environment (e.g., 'k8stest5-corek8s' instead of 'k8stest5-apphost') to avoid conflicts when multiple environments deploy to the same cluster or when re-deploying with different environment names. 3. Updated K8s snapshot tests for the new deferred value handling. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 5 +++ .../KubernetesResource.cs | 41 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index adbfa97e3f8..f2424776e57 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -59,6 +59,11 @@ public static IResourceBuilder AddAzureKuber // default Helm deployment engine. var k8sEnvBuilder = builder.AddKubernetesEnvironment($"{name}-k8s"); + // Scope the Helm chart name to this AKS environment to avoid + // conflicts when multiple environments deploy to the same cluster + // or when re-deploying with different environment names. + k8sEnvBuilder.Resource.HelmChartName = $"{builder.Environment.ApplicationName}-{name}".ToLowerInvariant().Replace(' ', '-'); + // Create the unified AKS environment resource var resource = new AzureKubernetesEnvironmentResource(name, ConfigureAksInfrastructure); resource.KubernetesEnvironment = k8sEnvBuilder.Resource; diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs index ce9e5b5b83e..5374909285f 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs @@ -510,6 +510,26 @@ private async Task ProcessValueAsync(KubernetesEnvironmentContext contex return embedded ? inner?.ToString() ?? string.Empty : inner; } + // Check if any value provider in the expression would fall through to + // ResolveUnknownValue (i.e., it's not an endpoint, parameter, connection string, + // or other known type). These are deferred value sources like BicepOutputReference. + // If found, defer the entire composite expression for deploy-time resolution. + if (expr is IValueProvider compositeValueProvider && + expr.ValueProviders.Any(IsUnresolvedAtPublishTime)) + { + var formattedName = expr.ValueExpression + .Replace("{", string.Empty) + .Replace("}", string.Empty) + .Replace(".", "_") + .ToHelmValuesSectionName(); + + var helmExpression = formattedName.ToHelmConfigExpression(TargetResource.Name); + return new HelmValue(helmExpression, expr.ValueExpression) + { + ValueProviderSource = compositeValueProvider + }; + } + var args = new object[expr.ValueProviders.Count]; var index = 0; @@ -685,6 +705,27 @@ private static HelmValue ResolveUnknownValue(IManifestExpressionProvider paramet return helmValue; } + /// + /// Checks if a value provider would fall through to ResolveUnknownValue — + /// i.e., it's not a known type that can be resolved at publish time + /// (endpoint, parameter, connection string, etc.). + /// + private static bool IsUnresolvedAtPublishTime(object value) + { + return value switch + { + string => false, + EndpointReference => false, + EndpointReferenceExpression => false, + ParameterResource => false, + ConnectionStringReference => false, + IResourceWithConnectionString => false, + ReferenceExpression => false, + IManifestExpressionProvider provider when provider is IValueProvider => true, + _ => false + }; + } + private static string GetKubernetesProtocolName(ProtocolType type) => type switch { From 4510585fa589245440ea03ae7cf956be694d35b0 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 14 Apr 2026 16:02:05 +1000 Subject: [PATCH 40/78] Fix deferred value detection: recurse into connection strings and expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous check only detected direct IValueProvider references. For composite expressions like connection strings (Endpoint={storage.outputs.blobEndpoint};ContainerName=photos), the BicepOutputReference is nested inside a ReferenceExpression chain: ConnectionStringReference → ReferenceExpression → BicepOutputReference Fixed IsUnresolvedAtPublishTime to recursively check: - ConnectionStringReference → check inner ConnectionStringExpression - IResourceWithConnectionString → check inner ConnectionStringExpression - ReferenceExpression → check all value providers recursively - IManifestExpressionProvider + IValueProvider → true (leaf unresolvable) Also added early deferral in ProcessValueAsync for ConnectionStringReference and IResourceWithConnectionString before they get unwrapped. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesResource.cs | 44 ++++++++++++++++--- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs index 5374909285f..9d4f56ddcb0 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs @@ -457,12 +457,22 @@ private async Task ProcessValueAsync(KubernetesEnvironmentContext contex if (value is ConnectionStringReference cs) { + // Check if the connection string contains deferred values + if (IsUnresolvedAtPublishTime(cs) && cs.Resource.ConnectionStringExpression is IValueProvider csvp) + { + return CreateDeferredHelmValue(cs.Resource.ConnectionStringExpression, csvp); + } value = cs.Resource.ConnectionStringExpression; continue; } if (value is IResourceWithConnectionString csrs) { + // Check if the connection string contains deferred values + if (IsUnresolvedAtPublishTime(csrs) && csrs.ConnectionStringExpression is IValueProvider csrsvp) + { + return CreateDeferredHelmValue(csrs.ConnectionStringExpression, csrsvp); + } value = csrs.ConnectionStringExpression; continue; } @@ -706,9 +716,9 @@ private static HelmValue ResolveUnknownValue(IManifestExpressionProvider paramet } /// - /// Checks if a value provider would fall through to ResolveUnknownValue — - /// i.e., it's not a known type that can be resolved at publish time - /// (endpoint, parameter, connection string, etc.). + /// Checks if a value contains sub-expressions that cannot be resolved + /// at publish time and need deploy-time resolution via IValueProvider. + /// Recursively checks ReferenceExpression value providers. /// private static bool IsUnresolvedAtPublishTime(object value) { @@ -718,14 +728,34 @@ private static bool IsUnresolvedAtPublishTime(object value) EndpointReference => false, EndpointReferenceExpression => false, ParameterResource => false, - ConnectionStringReference => false, - IResourceWithConnectionString => false, - ReferenceExpression => false, - IManifestExpressionProvider provider when provider is IValueProvider => true, + ConnectionStringReference cs => IsUnresolvedAtPublishTime(cs.Resource.ConnectionStringExpression), + IResourceWithConnectionString csrs => IsUnresolvedAtPublishTime(csrs.ConnectionStringExpression), + ReferenceExpression expr => expr.ValueProviders.Any(IsUnresolvedAtPublishTime), + // Any other IManifestExpressionProvider that also implements IValueProvider + // is a deferred source (e.g., BicepOutputReference) + IManifestExpressionProvider when value is IValueProvider => true, _ => false }; } + /// + /// Creates a HelmValue that defers resolution to deploy time via IValueProvider. + /// + private HelmValue CreateDeferredHelmValue(IManifestExpressionProvider expressionProvider, IValueProvider valueProvider) + { + var formattedName = expressionProvider.ValueExpression + .Replace("{", string.Empty) + .Replace("}", string.Empty) + .Replace(".", "_") + .ToHelmValuesSectionName(); + + var helmExpression = formattedName.ToHelmConfigExpression(TargetResource.Name); + return new HelmValue(helmExpression, expressionProvider.ValueExpression) + { + ValueProviderSource = valueProvider + }; + } + private static string GetKubernetesProtocolName(ProtocolType type) => type switch { From 3002346fdadadf8336872a0d4f7efe8a185ecc84 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 14 Apr 2026 16:30:53 +1000 Subject: [PATCH 41/78] Fix Helm key: use env var name instead of value expression for deferred values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The deferred value's Helm key was derived from ValueExpression which contains format strings like 'Endpoint={storage.outputs.blobEndpoint};ContainerName=photos'. This produced invalid Helm paths with = and ; characters. Fix: moved deferral check to ProcessEnvironmentAsync (outer loop) where the env var key name is available. CreateDeferredHelmValue now takes the env var key directly, producing clean paths like '{{ .Values.config.apiservice.ConnectionStrings__photos }}'. Removed deferral checks from ProcessValueAsync — all deferral is now handled before ProcessValueAsync is called. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesResource.cs | 56 ++++++------------- 1 file changed, 17 insertions(+), 39 deletions(-) diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs index 9d4f56ddcb0..9a938ca7ad8 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs @@ -365,6 +365,18 @@ private async Task ProcessEnvironmentAsync(KubernetesEnvironmentContext environm foreach (var environmentVariable in context.EnvironmentVariables) { var key = environmentVariable.Key; + + // Check if the value contains deferred providers (e.g., Bicep outputs) + // that can only be resolved after infrastructure provisioning. + // If so, create a deferred HelmValue with the env var key name. + if (IsUnresolvedAtPublishTime(environmentVariable.Value) && + environmentVariable.Value is IValueProvider deferredVp) + { + var deferredHelmValue = CreateDeferredHelmValue(key, deferredVp); + ProcessEnvironmentHelmExpression(deferredHelmValue, key); + continue; + } + var value = await ProcessValueAsync(environmentContext, executionContext, environmentVariable.Value).ConfigureAwait(false); switch (value) @@ -457,22 +469,12 @@ private async Task ProcessValueAsync(KubernetesEnvironmentContext contex if (value is ConnectionStringReference cs) { - // Check if the connection string contains deferred values - if (IsUnresolvedAtPublishTime(cs) && cs.Resource.ConnectionStringExpression is IValueProvider csvp) - { - return CreateDeferredHelmValue(cs.Resource.ConnectionStringExpression, csvp); - } value = cs.Resource.ConnectionStringExpression; continue; } if (value is IResourceWithConnectionString csrs) { - // Check if the connection string contains deferred values - if (IsUnresolvedAtPublishTime(csrs) && csrs.ConnectionStringExpression is IValueProvider csrsvp) - { - return CreateDeferredHelmValue(csrs.ConnectionStringExpression, csrsvp); - } value = csrs.ConnectionStringExpression; continue; } @@ -520,26 +522,6 @@ private async Task ProcessValueAsync(KubernetesEnvironmentContext contex return embedded ? inner?.ToString() ?? string.Empty : inner; } - // Check if any value provider in the expression would fall through to - // ResolveUnknownValue (i.e., it's not an endpoint, parameter, connection string, - // or other known type). These are deferred value sources like BicepOutputReference. - // If found, defer the entire composite expression for deploy-time resolution. - if (expr is IValueProvider compositeValueProvider && - expr.ValueProviders.Any(IsUnresolvedAtPublishTime)) - { - var formattedName = expr.ValueExpression - .Replace("{", string.Empty) - .Replace("}", string.Empty) - .Replace(".", "_") - .ToHelmValuesSectionName(); - - var helmExpression = formattedName.ToHelmConfigExpression(TargetResource.Name); - return new HelmValue(helmExpression, expr.ValueExpression) - { - ValueProviderSource = compositeValueProvider - }; - } - var args = new object[expr.ValueProviders.Count]; var index = 0; @@ -741,16 +723,12 @@ private static bool IsUnresolvedAtPublishTime(object value) /// /// Creates a HelmValue that defers resolution to deploy time via IValueProvider. /// - private HelmValue CreateDeferredHelmValue(IManifestExpressionProvider expressionProvider, IValueProvider valueProvider) + /// The environment variable or config key name to use as the Helm values path. + /// The value provider for deploy-time resolution. + private HelmValue CreateDeferredHelmValue(string key, IValueProvider valueProvider) { - var formattedName = expressionProvider.ValueExpression - .Replace("{", string.Empty) - .Replace("}", string.Empty) - .Replace(".", "_") - .ToHelmValuesSectionName(); - - var helmExpression = formattedName.ToHelmConfigExpression(TargetResource.Name); - return new HelmValue(helmExpression, expressionProvider.ValueExpression) + var helmExpression = key.ToHelmConfigExpression(TargetResource.Name); + return new HelmValue(helmExpression, string.Empty) { ValueProviderSource = valueProvider }; From bdacd7a610e09fe795139d27e191cb191842ef22 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 14 Apr 2026 17:31:02 +1000 Subject: [PATCH 42/78] Move node pool to base K8s package with WithNodePool API - Add KubernetesNodePoolResource as base class in Aspire.Hosting.Kubernetes - Add KubernetesNodePoolAnnotation for nodeSelector scheduling - Add AddNodePool and WithNodePool extensions on K8s environment - AksNodePoolResource now extends KubernetesNodePoolResource - Remove WithNodePoolAffinity (replaced by WithNodePool) - Apply nodeSelector in KubernetesPublishingContext when annotation present - Delete AksNodePoolAffinityAnnotation (replaced by KubernetesNodePoolAnnotation) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AksNodePoolResource.cs | 16 ++-- .../AzureKubernetesEnvironmentExtensions.cs | 35 +-------- .../AzureKubernetesInfrastructure.cs | 5 +- .../Aspire.Hosting.Kubernetes.csproj | 1 + .../KubernetesEnvironmentExtensions.cs | 73 +++++++++++++++++++ .../KubernetesNodePoolAnnotation.cs} | 10 +-- .../KubernetesNodePoolResource.cs | 30 ++++++++ .../KubernetesPublishingContext.cs | 17 +++++ ...ureKubernetesEnvironmentExtensionsTests.cs | 11 +-- .../AzureKubernetesInfrastructureTests.cs | 15 ++-- 10 files changed, 153 insertions(+), 60 deletions(-) rename src/{Aspire.Hosting.Azure.Kubernetes/AksNodePoolAffinityAnnotation.cs => Aspire.Hosting.Kubernetes/KubernetesNodePoolAnnotation.cs} (61%) create mode 100644 src/Aspire.Hosting.Kubernetes/KubernetesNodePoolResource.cs diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolResource.cs index af0a95f045b..e6fb6a17999 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolResource.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolResource.cs @@ -1,30 +1,30 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Kubernetes; namespace Aspire.Hosting.Azure.Kubernetes; /// -/// Represents an AKS node pool as a child resource of an . -/// Node pools can be referenced by compute resources to schedule workloads on specific node pools -/// using . +/// Represents an AKS node pool with Azure-specific configuration such as VM size and autoscaling. +/// Extends the base with provisioning configuration +/// that is used to generate Azure Bicep for the AKS agent pool profile. /// /// The name of the node pool resource. -/// The node pool configuration. +/// The Azure-specific node pool configuration. /// The parent AKS environment resource. public class AksNodePoolResource( string name, AksNodePoolConfig config, - AzureKubernetesEnvironmentResource parent) : Resource(name), IResourceWithParent + AzureKubernetesEnvironmentResource parent) : KubernetesNodePoolResource(name, parent.KubernetesEnvironment) { /// /// Gets the parent AKS environment resource. /// - public AzureKubernetesEnvironmentResource Parent { get; } = parent ?? throw new ArgumentNullException(nameof(parent)); + public AzureKubernetesEnvironmentResource AksParent { get; } = parent ?? throw new ArgumentNullException(nameof(parent)); /// - /// Gets the node pool configuration. + /// Gets the Azure-specific node pool configuration. /// public AksNodePoolConfig Config { get; } = config ?? throw new ArgumentNullException(nameof(config)); } diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index f2424776e57..59b3845638e 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -172,7 +172,7 @@ public static IResourceBuilder WithSkuTier( /// A reference to the for the new node pool. /// /// The returned node pool resource can be passed to - /// on compute resources to schedule workloads on this pool. + /// on compute resources to schedule workloads on this pool. /// /// /// @@ -180,7 +180,7 @@ public static IResourceBuilder WithSkuTier( /// var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); /// /// builder.AddProject<MyApi>() - /// .WithNodePoolAffinity(gpuPool); + /// .WithNodePool(gpuPool); /// /// [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] @@ -209,37 +209,6 @@ public static IResourceBuilder AddNodePool( .ExcludeFromManifest(); } - /// - /// Schedules a compute resource's workload on the specified AKS node pool. - /// This translates to a Kubernetes nodeSelector with the agentpool label - /// targeting the named node pool. - /// - /// The type of the compute resource. - /// The resource builder. - /// The node pool to schedule the workload on. - /// A reference to the for chaining. - /// - /// - /// var aks = builder.AddAzureKubernetesEnvironment("aks"); - /// var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); - /// - /// builder.AddProject<MyApi>() - /// .WithNodePoolAffinity(gpuPool); - /// - /// - [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] - public static IResourceBuilder WithNodePoolAffinity( - this IResourceBuilder builder, - IResourceBuilder nodePool) - where T : IResource - { - ArgumentNullException.ThrowIfNull(builder); - ArgumentNullException.ThrowIfNull(nodePool); - - builder.WithAnnotation(new AksNodePoolAffinityAnnotation(nodePool.Resource)); - return builder; - } - /// /// Configures the AKS cluster as a private cluster with a private API server endpoint. /// diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index a673b59d821..17bc8e59897 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -11,6 +11,7 @@ using Aspire.Hosting.Pipelines; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Aspire.Hosting.Kubernetes; using Microsoft.Extensions.Logging; namespace Aspire.Hosting.Azure.Kubernetes; @@ -70,9 +71,9 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance // If the resource has no explicit node pool affinity, assign it // to the default user pool. - if (!r.TryGetLastAnnotation(out _) && defaultUserPool is not null) + if (!r.TryGetLastAnnotation(out _) && defaultUserPool is not null) { - r.Annotations.Add(new AksNodePoolAffinityAnnotation(defaultUserPool)); + r.Annotations.Add(new KubernetesNodePoolAnnotation(defaultUserPool)); } } } diff --git a/src/Aspire.Hosting.Kubernetes/Aspire.Hosting.Kubernetes.csproj b/src/Aspire.Hosting.Kubernetes/Aspire.Hosting.Kubernetes.csproj index b92a148789a..04e87db475a 100644 --- a/src/Aspire.Hosting.Kubernetes/Aspire.Hosting.Kubernetes.csproj +++ b/src/Aspire.Hosting.Kubernetes/Aspire.Hosting.Kubernetes.csproj @@ -20,6 +20,7 @@ + diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentExtensions.cs index db01e3feb6c..8122a274cbc 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentExtensions.cs @@ -161,6 +161,79 @@ public static IResourceBuilder WithDashboard(this return builder; } + /// + /// Adds a named node pool to the Kubernetes environment. + /// + /// The Kubernetes environment resource builder. + /// The name of the node pool. This value is used as the nodeSelector value. + /// A reference to the for the new node pool. + /// + /// For vanilla Kubernetes, this creates a named reference to an existing node pool. + /// For managed Kubernetes services (e.g., AKS), the cloud-specific AddNodePool overload + /// provisions the pool with additional configuration such as VM size and autoscaling. + /// Use to schedule workloads on the returned node pool. + /// + /// + /// + /// var k8s = builder.AddKubernetesEnvironment("k8s"); + /// var gpuPool = k8s.AddNodePool("gpu"); + /// + /// builder.AddProject<MyApi>() + /// .WithComputeEnvironment(k8s) + /// .WithNodePool(gpuPool); + /// + /// + [AspireExport(Description = "Adds a named node pool to a Kubernetes environment")] + public static IResourceBuilder AddNodePool( + this IResourceBuilder builder, + [ResourceName] string name) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrEmpty(name); + + var nodePool = new KubernetesNodePoolResource(name, builder.Resource); + + if (builder.ApplicationBuilder.ExecutionContext.IsRunMode) + { + return builder.ApplicationBuilder.CreateResourceBuilder(nodePool); + } + + return builder.ApplicationBuilder.AddResource(nodePool) + .ExcludeFromManifest(); + } + + /// + /// Schedules a compute resource's workload on the specified Kubernetes node pool. + /// This translates to a Kubernetes nodeSelector in the pod specification + /// targeting the named node pool. + /// + /// The type of the compute resource. + /// The resource builder. + /// The node pool to schedule the workload on. + /// A reference to the for chaining. + /// + /// + /// var k8s = builder.AddKubernetesEnvironment("k8s"); + /// var gpuPool = k8s.AddNodePool("gpu"); + /// + /// builder.AddProject<MyApi>() + /// .WithComputeEnvironment(k8s) + /// .WithNodePool(gpuPool); + /// + /// + [AspireExport("withKubernetesNodePool", Description = "Schedules a workload on a specific Kubernetes node pool")] + public static IResourceBuilder WithNodePool( + this IResourceBuilder builder, + IResourceBuilder nodePool) + where T : IResource + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(nodePool); + + builder.WithAnnotation(new KubernetesNodePoolAnnotation(nodePool.Resource)); + return builder; + } + internal static void EnsureDefaultHelmEngine(IResourceBuilder builder) { builder.Resource.DeploymentEngineStepsFactory ??= HelmDeploymentEngine.CreateStepsAsync; diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolAffinityAnnotation.cs b/src/Aspire.Hosting.Kubernetes/KubernetesNodePoolAnnotation.cs similarity index 61% rename from src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolAffinityAnnotation.cs rename to src/Aspire.Hosting.Kubernetes/KubernetesNodePoolAnnotation.cs index 1ab0e8d8372..df2f82b78b9 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AksNodePoolAffinityAnnotation.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesNodePoolAnnotation.cs @@ -3,17 +3,17 @@ using Aspire.Hosting.ApplicationModel; -namespace Aspire.Hosting.Azure.Kubernetes; +namespace Aspire.Hosting.Kubernetes; /// -/// Annotation that associates a compute resource with a specific AKS node pool. +/// Annotation that associates a compute resource with a specific Kubernetes node pool. /// When present, the Kubernetes deployment will include a nodeSelector targeting -/// the specified node pool via the agentpool label. +/// the specified node pool. /// -internal sealed class AksNodePoolAffinityAnnotation(AksNodePoolResource nodePool) : IResourceAnnotation +internal sealed class KubernetesNodePoolAnnotation(KubernetesNodePoolResource nodePool) : IResourceAnnotation { /// /// Gets the node pool to schedule the workload on. /// - public AksNodePoolResource NodePool { get; } = nodePool; + public KubernetesNodePoolResource NodePool { get; } = nodePool; } diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesNodePoolResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesNodePoolResource.cs new file mode 100644 index 00000000000..94c1b41fad8 --- /dev/null +++ b/src/Aspire.Hosting.Kubernetes/KubernetesNodePoolResource.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Hosting.ApplicationModel; + +namespace Aspire.Hosting.Kubernetes; + +/// +/// Represents a Kubernetes node pool as a child resource of a . +/// Node pools can be referenced by compute resources to schedule workloads on specific node pools +/// using . +/// +/// The name of the node pool resource. +/// The parent Kubernetes environment resource. +[AspireExport] +public class KubernetesNodePoolResource( + string name, + KubernetesEnvironmentResource environment) : Resource(name), IResourceWithParent +{ + /// + /// Gets the parent Kubernetes environment resource. + /// + public KubernetesEnvironmentResource Parent { get; } = environment ?? throw new ArgumentNullException(nameof(environment)); + + /// + /// Gets the label key used to identify the node pool in the Kubernetes cluster. + /// Defaults to agentpool which is the standard label used by AKS and many managed Kubernetes services. + /// + public string NodeSelectorLabelKey { get; init; } = "agentpool"; +} diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs b/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs index 1c9834eeb1f..7bc9bc5fc48 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs @@ -105,6 +105,12 @@ private async Task WriteKubernetesOutputAsync(DistributedApplicationModel model, } } + // Apply node pool nodeSelector if the resource has a node pool annotation + if (serviceResource.TargetResource.TryGetLastAnnotation(out var nodePoolAnnotation)) + { + ApplyNodePoolSelector(serviceResource, nodePoolAnnotation.NodePool); + } + await WriteKubernetesTemplatesForResource(resource, serviceResource.GetTemplatedResources()).ConfigureAwait(false); await AppendResourceContextToHelmValuesAsync(resource, serviceResource).ConfigureAwait(false); } @@ -290,4 +296,15 @@ private async Task WriteKubernetesHelmChartAsync(KubernetesEnvironmentResource e Directory.CreateDirectory(OutputPath); await File.WriteAllTextAsync(outputFile, chartYaml, cancellationToken).ConfigureAwait(false); } + + private static void ApplyNodePoolSelector(KubernetesResource serviceResource, KubernetesNodePoolResource nodePool) + { + var podSpec = serviceResource.Workload?.PodTemplate?.Spec; + if (podSpec is null) + { + return; + } + + podSpec.NodeSelector[nodePool.NodeSelectorLabelKey] = nodePool.Name; + } } diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs index 75dd1a25040..9f2cca35298 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs @@ -7,6 +7,7 @@ using System.Runtime.CompilerServices; using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Azure.Kubernetes; +using Aspire.Hosting.Kubernetes; using Aspire.Hosting.Utils; namespace Aspire.Hosting.Azure.Tests; @@ -71,7 +72,7 @@ public void AddNodePool_ReturnsNodePoolResource() Assert.Equal(0, gpuPool.Resource.Config.MinCount); Assert.Equal(5, gpuPool.Resource.Config.MaxCount); Assert.Equal(AksNodePoolMode.User, gpuPool.Resource.Config.Mode); - Assert.Same(aks.Resource, gpuPool.Resource.Parent); + Assert.Same(aks.Resource, gpuPool.Resource.AksParent); } [Fact] @@ -270,7 +271,7 @@ public void AsExisting_WorksOnAksResource() } [Fact] - public void WithNodePoolAffinity_AddsAnnotation() + public void WithNodePool_AddsAnnotation() { using var builder = TestDistributedApplicationBuilder.Create(); @@ -278,11 +279,11 @@ public void WithNodePoolAffinity_AddsAnnotation() var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); var container = builder.AddContainer("myapi", "myimage") - .WithNodePoolAffinity(gpuPool); + .WithNodePool(gpuPool); - Assert.True(container.Resource.TryGetLastAnnotation(out var affinity)); + Assert.True(container.Resource.TryGetLastAnnotation(out var affinity)); Assert.Same(gpuPool.Resource, affinity.NodePool); - Assert.Equal("gpu", affinity.NodePool.Config.Name); + Assert.Equal("gpu", affinity.NodePool.Name); } [Fact] diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs index 73d028e129a..c3c9bba17db 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs @@ -6,6 +6,7 @@ using System.Runtime.CompilerServices; using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Azure.Kubernetes; +using Aspire.Hosting.Kubernetes; using Aspire.Hosting.Utils; namespace Aspire.Hosting.Azure.Tests; @@ -35,8 +36,8 @@ public async Task NoUserPool_CreatesDefaultWorkloadPool() Assert.Equal("workload", workloadPool.Name); // Compute resource should have been auto-assigned to the workload pool - Assert.True(container.Resource.TryGetLastAnnotation(out var affinity)); - Assert.Equal("workload", affinity.NodePool.Config.Name); + Assert.True(container.Resource.TryGetLastAnnotation(out var affinity)); + Assert.Equal("workload", affinity.NodePool.Name); } [Fact] @@ -58,8 +59,8 @@ public async Task ExplicitUserPool_NoDefaultCreated() Assert.DoesNotContain(aks.Resource.NodePools, p => p.Name == "workload"); // Unaffinitized compute resource should get assigned to the first user pool - Assert.True(container.Resource.TryGetLastAnnotation(out var affinity)); - Assert.Equal("gpu", affinity.NodePool.Config.Name); + Assert.True(container.Resource.TryGetLastAnnotation(out var affinity)); + Assert.Equal("gpu", affinity.NodePool.Name); } [Fact] @@ -73,14 +74,14 @@ public async Task ExplicitAffinity_NotOverridden() var cpuPool = aks.AddNodePool("cpu", "Standard_D4s_v5", 1, 10); var container = builder.AddContainer("myapi", "myimage") - .WithNodePoolAffinity(cpuPool); + .WithNodePool(cpuPool); await using var app = builder.Build(); await ExecuteBeforeStartHooksAsync(app, default); // Explicit affinity should be preserved, not overridden - Assert.True(container.Resource.TryGetLastAnnotation(out var affinity)); - Assert.Equal("cpu", affinity.NodePool.Config.Name); + Assert.True(container.Resource.TryGetLastAnnotation(out var affinity)); + Assert.Equal("cpu", affinity.NodePool.Name); } [Fact] From 5021b94590b4dfb414dccf9119072c2823888ff7 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 14 Apr 2026 17:37:01 +1000 Subject: [PATCH 43/78] Add AzureVmSizes constants class and generator tool - AzureVmSizes.Generated.cs with common VM sizes grouped by family (GeneralPurpose, ComputeOptimized, MemoryOptimized, GpuAccelerated, StorageOptimized, Burstable, Arm) - GenVmSizes.cs tool that fetches VM SKUs from Azure REST API - update-azure-vm-sizes.yml workflow (monthly, like GitHub Models pattern) - Users can now write: aks.AddNodePool("gpu", AzureVmSizes.GpuAccelerated.StandardNC6sV3, 0, 5) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/update-azure-vm-sizes.yml | 48 +++ .../Aspire.Hosting.Azure.Kubernetes.csproj | 1 + .../AzureVmSizes.Generated.cs | 274 ++++++++++++++++ .../tools/GenVmSizes.cs | 308 ++++++++++++++++++ 4 files changed, 631 insertions(+) create mode 100644 .github/workflows/update-azure-vm-sizes.yml create mode 100644 src/Aspire.Hosting.Azure.Kubernetes/AzureVmSizes.Generated.cs create mode 100644 src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs diff --git a/.github/workflows/update-azure-vm-sizes.yml b/.github/workflows/update-azure-vm-sizes.yml new file mode 100644 index 00000000000..905e60e8c30 --- /dev/null +++ b/.github/workflows/update-azure-vm-sizes.yml @@ -0,0 +1,48 @@ +name: Update Azure VM Sizes + +on: + workflow_dispatch: + schedule: + - cron: '0 6 1 * *' # Monthly on the 1st at 06:00 UTC + +permissions: + contents: write + pull-requests: write + +jobs: + generate-and-pr: + runs-on: ubuntu-latest + if: ${{ github.repository_owner == 'microsoft' }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Azure Login + uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Generate updated Azure VM size descriptors + working-directory: src/Aspire.Hosting.Azure.Kubernetes/tools + run: | + set -e + "$GITHUB_WORKSPACE/dotnet.sh" run GenVmSizes.cs + + - name: Generate GitHub App Token + id: app-token + uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 + with: + app-id: ${{ secrets.ASPIRE_BOT_APP_ID }} + private-key: ${{ secrets.ASPIRE_BOT_PRIVATE_KEY }} + + - name: Create or update pull request + uses: ./.github/actions/create-pull-request + with: + token: ${{ steps.app-token.outputs.token }} + branch: update-azure-vm-sizes + base: main + commit-message: "[Automated] Update Azure VM Sizes" + labels: | + area-integrations + area-engineering-systems + title: "[Automated] Update Azure VM Sizes" + body: "Auto-generated update of Azure VM size descriptors (AzureVmSizes.Generated.cs)." diff --git a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj index 3d6ab28eecb..dfa60c150a8 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj +++ b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj @@ -11,6 +11,7 @@ + diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureVmSizes.Generated.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureVmSizes.Generated.cs new file mode 100644 index 00000000000..bd567c74652 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureVmSizes.Generated.cs @@ -0,0 +1,274 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// +// This file is generated by the GenVmSizes tool. Do not edit manually. + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Provides well-known Azure VM size constants for use with AKS node pools. +/// +/// +/// This class is auto-generated. To update, run the GenVmSizes tool: +/// dotnet run --project src/Aspire.Hosting.Azure.Kubernetes/tools GenVmSizes.cs +/// +public static partial class AzureVmSizes +{ + /// + /// General purpose VM sizes optimized for balanced CPU-to-memory ratio. + /// + public static class GeneralPurpose + { + /// + /// Standard_D2s_v5 — 2 vCPUs, 8 GB RAM, Premium SSD. + /// + public const string StandardD2sV5 = "Standard_D2s_v5"; + + /// + /// Standard_D4s_v5 — 4 vCPUs, 16 GB RAM, Premium SSD. + /// + public const string StandardD4sV5 = "Standard_D4s_v5"; + + /// + /// Standard_D8s_v5 — 8 vCPUs, 32 GB RAM, Premium SSD. + /// + public const string StandardD8sV5 = "Standard_D8s_v5"; + + /// + /// Standard_D16s_v5 — 16 vCPUs, 64 GB RAM, Premium SSD. + /// + public const string StandardD16sV5 = "Standard_D16s_v5"; + + /// + /// Standard_D32s_v5 — 32 vCPUs, 128 GB RAM, Premium SSD. + /// + public const string StandardD32sV5 = "Standard_D32s_v5"; + + /// + /// Standard_D2s_v6 — 2 vCPUs, 8 GB RAM, Premium SSD. + /// + public const string StandardD2sV6 = "Standard_D2s_v6"; + + /// + /// Standard_D4s_v6 — 4 vCPUs, 16 GB RAM, Premium SSD. + /// + public const string StandardD4sV6 = "Standard_D4s_v6"; + + /// + /// Standard_D8s_v6 — 8 vCPUs, 32 GB RAM, Premium SSD. + /// + public const string StandardD8sV6 = "Standard_D8s_v6"; + + /// + /// Standard_D2as_v5 — 2 vCPUs, 8 GB RAM, Premium SSD, AMD processor. + /// + public const string StandardD2asV5 = "Standard_D2as_v5"; + + /// + /// Standard_D4as_v5 — 4 vCPUs, 16 GB RAM, Premium SSD, AMD processor. + /// + public const string StandardD4asV5 = "Standard_D4as_v5"; + + /// + /// Standard_D8as_v5 — 8 vCPUs, 32 GB RAM, Premium SSD, AMD processor. + /// + public const string StandardD8asV5 = "Standard_D8as_v5"; + } + + /// + /// Compute optimized VM sizes with high CPU-to-memory ratio. + /// + public static class ComputeOptimized + { + /// + /// Standard_F2s_v2 — 2 vCPUs, 4 GB RAM, Premium SSD. + /// + public const string StandardF2sV2 = "Standard_F2s_v2"; + + /// + /// Standard_F4s_v2 — 4 vCPUs, 8 GB RAM, Premium SSD. + /// + public const string StandardF4sV2 = "Standard_F4s_v2"; + + /// + /// Standard_F8s_v2 — 8 vCPUs, 16 GB RAM, Premium SSD. + /// + public const string StandardF8sV2 = "Standard_F8s_v2"; + + /// + /// Standard_F16s_v2 — 16 vCPUs, 32 GB RAM, Premium SSD. + /// + public const string StandardF16sV2 = "Standard_F16s_v2"; + } + + /// + /// Memory optimized VM sizes with high memory-to-CPU ratio. + /// + public static class MemoryOptimized + { + /// + /// Standard_E2s_v5 — 2 vCPUs, 16 GB RAM, Premium SSD. + /// + public const string StandardE2sV5 = "Standard_E2s_v5"; + + /// + /// Standard_E4s_v5 — 4 vCPUs, 32 GB RAM, Premium SSD. + /// + public const string StandardE4sV5 = "Standard_E4s_v5"; + + /// + /// Standard_E8s_v5 — 8 vCPUs, 64 GB RAM, Premium SSD. + /// + public const string StandardE8sV5 = "Standard_E8s_v5"; + + /// + /// Standard_E16s_v5 — 16 vCPUs, 128 GB RAM, Premium SSD. + /// + public const string StandardE16sV5 = "Standard_E16s_v5"; + + /// + /// Standard_E2as_v5 — 2 vCPUs, 16 GB RAM, Premium SSD, AMD processor. + /// + public const string StandardE2asV5 = "Standard_E2as_v5"; + + /// + /// Standard_E4as_v5 — 4 vCPUs, 32 GB RAM, Premium SSD, AMD processor. + /// + public const string StandardE4asV5 = "Standard_E4as_v5"; + } + + /// + /// GPU-enabled VM sizes for compute-intensive and ML workloads. + /// + public static class GpuAccelerated + { + /// + /// Standard_NC6s_v3 — 6 vCPUs, 112 GB RAM, 1 GPU (NVIDIA V100). + /// + public const string StandardNC6sV3 = "Standard_NC6s_v3"; + + /// + /// Standard_NC12s_v3 — 12 vCPUs, 224 GB RAM, 2 GPUs (NVIDIA V100). + /// + public const string StandardNC12sV3 = "Standard_NC12s_v3"; + + /// + /// Standard_NC24s_v3 — 24 vCPUs, 448 GB RAM, 4 GPUs (NVIDIA V100). + /// + public const string StandardNC24sV3 = "Standard_NC24s_v3"; + + /// + /// Standard_NC4as_T4_v3 — 4 vCPUs, 28 GB RAM, 1 GPU (NVIDIA T4). + /// + public const string StandardNC4asT4V3 = "Standard_NC4as_T4_v3"; + + /// + /// Standard_NC8as_T4_v3 — 8 vCPUs, 56 GB RAM, 1 GPU (NVIDIA T4). + /// + public const string StandardNC8asT4V3 = "Standard_NC8as_T4_v3"; + + /// + /// Standard_NC16as_T4_v3 — 16 vCPUs, 110 GB RAM, 1 GPU (NVIDIA T4). + /// + public const string StandardNC16asT4V3 = "Standard_NC16as_T4_v3"; + + /// + /// Standard_NC24ads_A100_v4 — 24 vCPUs, 220 GB RAM, 1 GPU (NVIDIA A100 80GB). + /// + public const string StandardNC24adsA100V4 = "Standard_NC24ads_A100_v4"; + + /// + /// Standard_NC48ads_A100_v4 — 48 vCPUs, 440 GB RAM, 2 GPUs (NVIDIA A100 80GB). + /// + public const string StandardNC48adsA100V4 = "Standard_NC48ads_A100_v4"; + } + + /// + /// Storage optimized VM sizes with high disk throughput and I/O. + /// + public static class StorageOptimized + { + /// + /// Standard_L8s_v3 — 8 vCPUs, 64 GB RAM, Premium SSD, high local NVMe storage. + /// + public const string StandardL8sV3 = "Standard_L8s_v3"; + + /// + /// Standard_L16s_v3 — 16 vCPUs, 128 GB RAM, Premium SSD, high local NVMe storage. + /// + public const string StandardL16sV3 = "Standard_L16s_v3"; + + /// + /// Standard_L32s_v3 — 32 vCPUs, 256 GB RAM, Premium SSD, high local NVMe storage. + /// + public const string StandardL32sV3 = "Standard_L32s_v3"; + } + + /// + /// Burstable VM sizes for cost-effective workloads with variable CPU usage. + /// + public static class Burstable + { + /// + /// Standard_B2s — 2 vCPUs, 4 GB RAM. + /// + public const string StandardB2s = "Standard_B2s"; + + /// + /// Standard_B4ms — 4 vCPUs, 16 GB RAM. + /// + public const string StandardB4ms = "Standard_B4ms"; + + /// + /// Standard_B8ms — 8 vCPUs, 32 GB RAM. + /// + public const string StandardB8ms = "Standard_B8ms"; + + /// + /// Standard_B2s_v2 — 2 vCPUs, 8 GB RAM. + /// + public const string StandardB2sV2 = "Standard_B2s_v2"; + + /// + /// Standard_B4s_v2 — 4 vCPUs, 16 GB RAM. + /// + public const string StandardB4sV2 = "Standard_B4s_v2"; + } + + /// + /// Arm-based VM sizes with high energy efficiency and price-performance. + /// + public static class Arm + { + /// + /// Standard_D2pds_v5 — 2 vCPUs, 8 GB RAM, Ampere Altra Arm processor. + /// + public const string StandardD2pdsV5 = "Standard_D2pds_v5"; + + /// + /// Standard_D4pds_v5 — 4 vCPUs, 16 GB RAM, Ampere Altra Arm processor. + /// + public const string StandardD4pdsV5 = "Standard_D4pds_v5"; + + /// + /// Standard_D8pds_v5 — 8 vCPUs, 32 GB RAM, Ampere Altra Arm processor. + /// + public const string StandardD8pdsV5 = "Standard_D8pds_v5"; + + /// + /// Standard_D16pds_v5 — 16 vCPUs, 64 GB RAM, Ampere Altra Arm processor. + /// + public const string StandardD16pdsV5 = "Standard_D16pds_v5"; + + /// + /// Standard_E2pds_v5 — 2 vCPUs, 16 GB RAM, Ampere Altra Arm processor. + /// + public const string StandardE2pdsV5 = "Standard_E2pds_v5"; + + /// + /// Standard_E4pds_v5 — 4 vCPUs, 32 GB RAM, Ampere Altra Arm processor. + /// + public const string StandardE4pdsV5 = "Standard_E4pds_v5"; + } +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs b/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs new file mode 100644 index 00000000000..8153e0b2448 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs @@ -0,0 +1,308 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +#:property PublishAot=false + +using System.Diagnostics; +using System.Globalization; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; + +// Fetch VM sizes from Azure REST API using the 'az' CLI +var subscriptionId = Environment.GetEnvironmentVariable("AZURE_SUBSCRIPTION_ID"); +if (string.IsNullOrWhiteSpace(subscriptionId)) +{ + // Try to get default subscription from az CLI + subscriptionId = await RunAzCommand("account show --query id -o tsv").ConfigureAwait(false); + subscriptionId = subscriptionId?.Trim(); +} + +if (string.IsNullOrWhiteSpace(subscriptionId)) +{ + Console.Error.WriteLine("Error: No Azure subscription found. Set AZURE_SUBSCRIPTION_ID or run 'az login'."); + return 1; +} + +Console.WriteLine($"Using subscription: {subscriptionId}"); + +// Fetch resource SKUs filtered to virtualMachines +var url = $"https://management.azure.com/subscriptions/{subscriptionId}/providers/Microsoft.Compute/skus?api-version=2021-07-01&$filter=location eq 'eastus'"; +var json = await RunAzCommand($"rest --method get --url \"{url}\"").ConfigureAwait(false); + +if (string.IsNullOrWhiteSpace(json)) +{ + Console.Error.WriteLine("Error: Failed to fetch VM SKUs from Azure REST API."); + return 1; +} + +var skuResponse = JsonSerializer.Deserialize(json, new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); +if (skuResponse is null || skuResponse.Value is null) +{ + Console.Error.WriteLine("Error: Failed to parse SKU response."); + return 1; +} + +// Filter to virtualMachines resource type and group by family +var vmSkus = skuResponse.Value + .Where(s => s.ResourceType == "virtualMachines" && !string.IsNullOrEmpty(s.Name)) + .Select(s => new VmSizeInfo + { + Name = s.Name!, + Family = s.Family ?? "Other", + VCpus = s.GetCapabilityValue("vCPUs"), + MemoryGB = s.GetCapabilityValue("MemoryGB"), + MaxDataDiskCount = s.GetCapabilityValue("MaxDataDiskCount"), + PremiumIO = s.GetCapabilityBool("PremiumIO"), + AcceleratedNetworking = s.GetCapabilityBool("AcceleratedNetworkingEnabled"), + GpuCount = s.GetCapabilityValue("GPUs"), + }) + .DistinctBy(s => s.Name) + .OrderBy(s => s.Family, StringComparer.OrdinalIgnoreCase) + .ThenBy(s => s.Name, StringComparer.OrdinalIgnoreCase) + .ToList(); + +Console.WriteLine($"Found {vmSkus.Count} VM sizes"); + +var code = VmSizeClassGenerator.GenerateCode("Aspire.Hosting.Azure.Kubernetes", vmSkus); +File.WriteAllText(Path.Combine("..", "AzureVmSizes.Generated.cs"), code); +Console.WriteLine($"Generated AzureVmSizes.Generated.cs with {vmSkus.Count} VM sizes"); + +return 0; + +static async Task RunAzCommand(string arguments) +{ + var psi = new ProcessStartInfo + { + FileName = "az", + Arguments = arguments, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + using var process = Process.Start(psi); + if (process is null) + { + return null; + } + + var output = await process.StandardOutput.ReadToEndAsync().ConfigureAwait(false); + await process.WaitForExitAsync().ConfigureAwait(false); + + return process.ExitCode == 0 ? output : null; +} + +public sealed class SkuResponse +{ + [JsonPropertyName("value")] + public List? Value { get; set; } +} + +public sealed class ResourceSku +{ + [JsonPropertyName("resourceType")] + public string? ResourceType { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("tier")] + public string? Tier { get; set; } + + [JsonPropertyName("size")] + public string? Size { get; set; } + + [JsonPropertyName("family")] + public string? Family { get; set; } + + [JsonPropertyName("capabilities")] + public List? Capabilities { get; set; } + + public string? GetCapabilityValue(string name) + { + return Capabilities?.FirstOrDefault(c => c.Name == name)?.Value; + } + + public bool GetCapabilityBool(string name) + { + var value = GetCapabilityValue(name); + return string.Equals(value, "True", StringComparison.OrdinalIgnoreCase); + } +} + +public sealed class SkuCapability +{ + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("value")] + public string? Value { get; set; } +} + +public sealed class VmSizeInfo +{ + public string Name { get; set; } = ""; + public string Family { get; set; } = ""; + public string? VCpus { get; set; } + public string? MemoryGB { get; set; } + public string? MaxDataDiskCount { get; set; } + public bool PremiumIO { get; set; } + public bool AcceleratedNetworking { get; set; } + public string? GpuCount { get; set; } +} + +internal static partial class VmSizeClassGenerator +{ + public static string GenerateCode(string ns, List sizes) + { + var sb = new StringBuilder(); + sb.AppendLine("// Licensed to the .NET Foundation under one or more agreements."); + sb.AppendLine("// The .NET Foundation licenses this file to you under the MIT license."); + sb.AppendLine(); + sb.AppendLine("// "); + sb.AppendLine("// This file is generated by the GenVmSizes tool. Do not edit manually."); + sb.AppendLine(); + sb.AppendLine(CultureInfo.InvariantCulture, $"namespace {ns};"); + sb.AppendLine(); + sb.AppendLine("/// "); + sb.AppendLine("/// Provides well-known Azure VM size constants for use with AKS node pools."); + sb.AppendLine("/// "); + sb.AppendLine("/// "); + sb.AppendLine("/// This class is auto-generated. To update, run the GenVmSizes tool:"); + sb.AppendLine("/// dotnet run --project src/Aspire.Hosting.Azure.Kubernetes/tools GenVmSizes.cs"); + sb.AppendLine("/// "); + sb.AppendLine("public static partial class AzureVmSizes"); + sb.AppendLine("{"); + + var groups = sizes.GroupBy(s => s.Family) + .OrderBy(g => g.Key, StringComparer.OrdinalIgnoreCase); + + var firstClass = true; + foreach (var group in groups) + { + if (!firstClass) + { + sb.AppendLine(); + } + firstClass = false; + + var className = FamilyToClassName(group.Key); + + sb.AppendLine(" /// "); + sb.AppendLine(CultureInfo.InvariantCulture, $" /// VM sizes in the {EscapeXml(group.Key)} family."); + sb.AppendLine(" /// "); + sb.AppendLine(CultureInfo.InvariantCulture, $" public static class {className}"); + sb.AppendLine(" {"); + + var firstField = true; + foreach (var size in group.OrderBy(s => s.Name, StringComparer.OrdinalIgnoreCase)) + { + if (!firstField) + { + sb.AppendLine(); + } + firstField = false; + + var fieldName = VmSizeToFieldName(size.Name); + var description = BuildDescription(size); + + sb.AppendLine(" /// "); + sb.AppendLine(CultureInfo.InvariantCulture, $" /// {EscapeXml(description)}"); + sb.AppendLine(" /// "); + sb.AppendLine(CultureInfo.InvariantCulture, $" public const string {fieldName} = \"{size.Name}\";"); + } + + sb.AppendLine(" }"); + } + + sb.AppendLine("}"); + return sb.ToString(); + } + + private static string BuildDescription(VmSizeInfo size) + { + var parts = new List { size.Name }; + if (size.VCpus is not null) + { + parts.Add($"{size.VCpus} vCPUs"); + } + if (size.MemoryGB is not null) + { + parts.Add($"{size.MemoryGB} GB RAM"); + } + if (size.GpuCount is not null && size.GpuCount != "0") + { + parts.Add($"{size.GpuCount} GPU(s)"); + } + if (size.PremiumIO) + { + parts.Add("Premium SSD"); + } + return string.Join(" — ", parts); + } + + private static string FamilyToClassName(string family) + { + // Convert family names like "standardDSv2Family" to "StandardDSv2" + var name = family.Replace("Family", "", StringComparison.OrdinalIgnoreCase) + .Replace("_", ""); + + if (name.Length > 0) + { + name = char.ToUpperInvariant(name[0]) + name[1..]; + } + + // Clean non-identifier chars + return CleanIdentifier(name); + } + + private static string VmSizeToFieldName(string vmSize) + { + // "Standard_D4s_v5" → "StandardD4sV5" + var parts = vmSize.Split('_'); + var sb = new StringBuilder(); + foreach (var part in parts) + { + if (part.Length > 0) + { + sb.Append(char.ToUpperInvariant(part[0])); + if (part.Length > 1) + { + sb.Append(part[1..]); + } + } + } + return CleanIdentifier(sb.ToString()); + } + + private static string CleanIdentifier(string name) + { + var sb = new StringBuilder(); + foreach (var c in name) + { + if (char.IsLetterOrDigit(c)) + { + sb.Append(c); + } + } + + var result = sb.ToString(); + + // Ensure doesn't start with a digit + if (result.Length > 0 && char.IsDigit(result[0])) + { + result = "_" + result; + } + + return result; + } + + private static string EscapeXml(string s) => + s.Replace("&", "&") + .Replace("<", "<") + .Replace(">", ">") + .Replace("\"", """) + .Replace("'", "'"); +} From 9bc386cd60e9205340bdb3158beac5cc21a1c606 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 14 Apr 2026 19:45:55 +1000 Subject: [PATCH 44/78] Support per-node-pool subnet via WithSubnet on AksNodePoolResource - Add WithSubnet overload for IResourceBuilder - Per-pool subnets generate separate Bicep params (subnetId_{poolName}) - Each pool uses its own subnet if set, else falls back to env-level default - Environment-level WithSubnet remains unchanged as the default - Network profile auto-configured when any subnet (default or per-pool) is set - Add 2 new tests for per-pool subnet scenarios Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 39 +++++++++++++++ .../AzureKubernetesEnvironmentResource.cs | 42 +++++++++++----- ...ureKubernetesEnvironmentExtensionsTests.cs | 50 +++++++++++++++++++ 3 files changed, 119 insertions(+), 12 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 59b3845638e..745facea0b7 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -252,6 +252,45 @@ public static IResourceBuilder WithSubnet( return builder; } + /// + /// Configures a specific AKS node pool to use its own VNet subnet. + /// When applied, this node pool's subnet overrides the environment-level subnet + /// set via . + /// + /// The node pool resource builder. + /// The subnet to use for this node pool. + /// A reference to the for chaining. + /// + /// + /// var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); + /// var defaultSubnet = vnet.AddSubnet("default", "10.0.0.0/22"); + /// var gpuSubnet = vnet.AddSubnet("gpu-subnet", "10.0.4.0/24"); + /// + /// var aks = builder.AddAzureKubernetesEnvironment("aks") + /// .WithSubnet(defaultSubnet); + /// + /// var gpuPool = aks.AddNodePool("gpu", AzureVmSizes.GpuAccelerated.StandardNC6sV3, 0, 5) + /// .WithSubnet(gpuSubnet); + /// + /// + [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + public static IResourceBuilder WithSubnet( + this IResourceBuilder builder, + IResourceBuilder subnet) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(subnet); + + // Store the subnet on the node pool annotation for Bicep resolution + builder.WithAnnotation(new AksSubnetAnnotation(subnet.Resource.Id)); + + // Also register in the parent AKS environment's per-pool subnet dictionary + // so Bicep generation can emit the correct parameter per pool. + builder.Resource.AksParent.NodePoolSubnets[builder.Resource.Name] = subnet.Resource.Id; + + return builder; + } + /// /// Configures the AKS environment to use a specific Azure Container Registry for image storage. /// When set, this replaces the auto-created default container registry. diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs index bcbf8214ab7..cecee453653 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs @@ -98,6 +98,11 @@ public class AzureKubernetesEnvironmentResource( new AksNodePoolConfig("system", "Standard_D4s_v5", 1, 3, AksNodePoolMode.System) ]; + /// + /// Gets the per-node-pool subnet overrides. Key is the pool name. + /// + internal Dictionary NodePoolSubnets { get; } = []; + /// /// Gets or sets the network profile for the AKS cluster. /// @@ -153,32 +158,40 @@ private string GenerateAksBicep() sb.AppendLine(); } - // Subnet parameter for VNet integration - var hasSubnet = this.TryGetLastAnnotation(out var subnetAnnotation); - if (!hasSubnet) + // Subnet parameters for VNet integration + // Environment-level subnet (default for all pools) + var hasDefaultSubnet = this.TryGetLastAnnotation(out var subnetAnnotation); + if (!hasDefaultSubnet) { // Fallback: check for DelegatedSubnetAnnotation (legacy WithDelegatedSubnet usage) - hasSubnet = this.TryGetLastAnnotation(out var delegatedAnnotation); - if (hasSubnet) + hasDefaultSubnet = this.TryGetLastAnnotation(out var delegatedAnnotation); + if (hasDefaultSubnet) { - // Wire as parameter — but note that DelegatedSubnetAnnotation adds a service - // delegation which AKS doesn't support. Users should use WithSubnet instead. Parameters["subnetId"] = delegatedAnnotation!.SubnetId; } } else { - // Wire the subnet ID as a parameter so the publishing context resolves it Parameters["subnetId"] = subnetAnnotation!.SubnetId; } - if (hasSubnet) + if (hasDefaultSubnet) { - sb.AppendLine("@description('The subnet ID for AKS node pool VNet integration.')"); + sb.AppendLine("@description('The default subnet ID for AKS node pool VNet integration.')"); sb.AppendLine("param subnetId string"); sb.AppendLine(); } + // Per-pool subnet overrides + foreach (var (poolName, poolSubnetRef) in NodePoolSubnets) + { + var paramName = $"subnetId_{poolName}"; + Parameters[paramName] = poolSubnetRef; + sb.Append("@description('The subnet ID for the ").Append(poolName).AppendLine(" node pool.')"); + sb.Append("param ").Append(paramName).AppendLine(" string"); + sb.AppendLine(); + } + // AKS cluster resource sb.Append("resource ").Append(id).AppendLine(" 'Microsoft.ContainerService/managedClusters@2024-06-02-preview' = {"); sb.Append(" name: '").Append(Name).AppendLine("'"); @@ -221,7 +234,12 @@ private string GenerateAksBicep() sb.AppendLine(" enableAutoScaling: true"); sb.Append(" mode: '").Append(mode).AppendLine("'"); sb.AppendLine(" osType: 'Linux'"); - if (hasSubnet) + // Use per-pool subnet if available, otherwise fall back to environment default + if (NodePoolSubnets.ContainsKey(pool.Name)) + { + sb.Append(" vnetSubnetID: subnetId_").AppendLine(pool.Name); + } + else if (hasDefaultSubnet) { sb.AppendLine(" vnetSubnetID: subnetId"); } @@ -268,7 +286,7 @@ private string GenerateAksBicep() sb.Append(" dnsServiceIP: '").Append(NetworkProfile.DnsServiceIP).AppendLine("'"); sb.AppendLine(" }"); } - else if (hasSubnet) + else if (hasDefaultSubnet || NodePoolSubnets.Count > 0) { // Default Azure CNI network profile when a subnet is delegated sb.AppendLine(" networkProfile: {"); diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs index 9f2cca35298..fcdc44cd37e 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs @@ -270,6 +270,56 @@ public void AsExisting_WorksOnAksResource() Assert.NotNull(aks); } + [Fact] + public void WithSubnet_OnNodePool_StoresPerPoolSubnet() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); + var defaultSubnet = vnet.AddSubnet("default-subnet", "10.0.0.0/22"); + var gpuSubnet = vnet.AddSubnet("gpu-subnet", "10.0.4.0/24"); + + var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithSubnet(defaultSubnet); + + var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5) + .WithSubnet(gpuSubnet); + + // Environment-level subnet should be set via annotation + Assert.True(aks.Resource.TryGetLastAnnotation(out _)); + + // Per-pool subnet should be stored in NodePoolSubnets dictionary + Assert.Single(aks.Resource.NodePoolSubnets); + Assert.True(aks.Resource.NodePoolSubnets.ContainsKey("gpu")); + + // Node pool should also have its own subnet annotation + Assert.True(gpuPool.Resource.TryGetLastAnnotation(out _)); + } + + [Fact] + public void WithSubnet_OnNodePool_WithoutEnvironmentSubnet() + { + using var builder = TestDistributedApplicationBuilder.Create( + DistributedApplicationOperation.Publish); + + var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); + var gpuSubnet = vnet.AddSubnet("gpu-subnet", "10.0.4.0/24"); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + + // Only set subnet on the pool, not the environment + var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5) + .WithSubnet(gpuSubnet); + + // No environment-level subnet + Assert.False(aks.Resource.TryGetLastAnnotation(out _)); + + // Per-pool subnet should still work + Assert.Single(aks.Resource.NodePoolSubnets); + Assert.True(aks.Resource.NodePoolSubnets.ContainsKey("gpu")); + } + [Fact] public void WithNodePool_AddsAnnotation() { From 0b11dcf2166462bdebd91b37a4f4377055a2fc58 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 14 Apr 2026 21:59:50 +1000 Subject: [PATCH 45/78] Implement workload identity via AppIdentityAnnotation - Remove WithAzureWorkloadIdentity and AksWorkloadIdentityAnnotation - Honor AppIdentityAnnotation (same mechanism as ACA/AppService) - AzureKubernetesInfrastructure detects AppIdentityAnnotation and: - Enables OIDC + workload identity on the AKS cluster - Generates K8s ServiceAccount with azure.workload.identity/client-id - Sets serviceAccountName on pod spec - Adds azure.workload.identity/use pod label via customization annotation - Generate federated identity credential Bicep per workload identity - Add ServiceAccountV1 resource to Aspire.Hosting.Kubernetes - AKS admission controller injects AZURE_CLIENT_ID automatically Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AksWorkloadIdentityAnnotation.cs | 27 --------- .../AzureKubernetesEnvironmentExtensions.cs | 44 +------------- .../AzureKubernetesEnvironmentResource.cs | 58 +++++++++++++++++++ .../AzureKubernetesInfrastructure.cs | 43 +++++++++++++- .../Resources/ServiceAccountV1.cs | 14 +++++ ...ureKubernetesEnvironmentExtensionsTests.cs | 22 +------ 6 files changed, 118 insertions(+), 90 deletions(-) delete mode 100644 src/Aspire.Hosting.Azure.Kubernetes/AksWorkloadIdentityAnnotation.cs create mode 100644 src/Aspire.Hosting.Kubernetes/Resources/ServiceAccountV1.cs diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksWorkloadIdentityAnnotation.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksWorkloadIdentityAnnotation.cs deleted file mode 100644 index b9b65911e64..00000000000 --- a/src/Aspire.Hosting.Azure.Kubernetes/AksWorkloadIdentityAnnotation.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Aspire.Hosting.ApplicationModel; - -namespace Aspire.Hosting.Azure.Kubernetes; - -/// -/// Annotation that marks a compute resource for AKS workload identity. -/// When present, the AKS infrastructure will generate a Kubernetes ServiceAccount -/// with the appropriate annotations and a federated identity credential in Azure. -/// -internal sealed class AksWorkloadIdentityAnnotation( - IAppIdentityResource identityResource, - string? serviceAccountName = null) : IResourceAnnotation -{ - /// - /// Gets the identity resource to federate with. - /// - public IAppIdentityResource IdentityResource { get; } = identityResource; - - /// - /// Gets or sets the Kubernetes service account name. - /// If null, defaults to the resource name. - /// - public string? ServiceAccountName { get; set; } = serviceAccountName; -} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 745facea0b7..ffc86f9145f 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -380,8 +380,8 @@ public static IResourceBuilder WithAzureLogA /// A reference to the for chaining. /// /// This ensures the AKS cluster is configured with OIDC issuer and workload identity enabled. - /// Use on individual compute resources to assign - /// specific managed identities. + /// Workload identity is automatically wired when compute resources have an , + /// which is added by WithAzureUserAssignedIdentity or auto-created by AzureResourcePreparer. /// [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] public static IResourceBuilder WithWorkloadIdentity( @@ -394,46 +394,6 @@ public static IResourceBuilder WithWorkloadI return builder; } - /// - /// Configures a compute resource to use AKS workload identity with the specified managed identity. - /// This generates a Kubernetes ServiceAccount and a federated identity credential in Azure. - /// - /// The type of the compute resource. - /// The resource builder. - /// The managed identity to federate with. If null, an identity will be auto-created. - /// A reference to the for chaining. - /// - /// - /// var identity = builder.AddAzureUserAssignedIdentity("myIdentity"); - /// builder.AddProject<MyApi>() - /// .WithAzureWorkloadIdentity(identity); - /// - /// - [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] - public static IResourceBuilder WithAzureWorkloadIdentity( - this IResourceBuilder builder, - IResourceBuilder? identity = null) - where T : IResource - { - ArgumentNullException.ThrowIfNull(builder); - - if (identity is null) - { - // Auto-create an identity named after the resource - var appBuilder = builder.ApplicationBuilder; - var identityName = $"{builder.Resource.Name}-identity"; - var identityBuilder = appBuilder.AddAzureUserAssignedIdentity(identityName); - identity = identityBuilder; - } - - // Add both the standard AppIdentityAnnotation (for Azure service role assignments) - // and the AKS-specific annotation (for ServiceAccount + federated credential generation) - builder.WithAnnotation(new AppIdentityAnnotation(identity.Resource)); - builder.WithAnnotation(new AksWorkloadIdentityAnnotation(identity.Resource)); - - return builder; - } - // ConfigureAksInfrastructure is a no-op placeholder required by the // AzureProvisioningResource base class constructor. The actual Bicep is // generated by GetBicepTemplateString/GetBicepTemplateFile overrides diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs index cecee453653..f5828023531 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs @@ -103,6 +103,12 @@ public class AzureKubernetesEnvironmentResource( /// internal Dictionary NodePoolSubnets { get; } = []; + /// + /// Gets the workload identity mappings. Key is the resource name, value is the identity resource. + /// Used to generate federated identity credentials in Bicep. + /// + internal Dictionary WorkloadIdentities { get; } = []; + /// /// Gets or sets the network profile for the AKS cluster. /// @@ -329,6 +335,58 @@ private string GenerateAksBicep() sb.Append("output kubeletIdentityObjectId string = ").Append(id).AppendLine(".properties.identityProfile.kubeletidentity.objectId"); sb.Append("output nodeResourceGroup string = ").Append(id).AppendLine(".properties.nodeResourceGroup"); + // Federated identity credentials for workload identity + foreach (var (resourceName, identityResource) in WorkloadIdentities) + { + var saName = $"{resourceName}-sa"; + var sanitizedName = SanitizeBicepIdentifier(resourceName); + var fedCredId = $"fedcred_{sanitizedName}"; + var identityParamName = $"identityName_{sanitizedName}"; + + // Add identity name as parameter — will be resolved from the identity resource + Parameters[identityParamName] = identityResource.PrincipalName; + + sb.AppendLine(); + sb.Append("@description('The name of the managed identity for ").Append(resourceName).AppendLine(".')"); + sb.Append("param ").Append(identityParamName).AppendLine(" string"); + sb.AppendLine(); + + // Reference existing identity + sb.Append("resource identity_").Append(sanitizedName); + sb.AppendLine(" 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = {"); + sb.Append(" name: ").AppendLine(identityParamName); + sb.AppendLine("}"); + sb.AppendLine(); + + // Federated credential + sb.Append("resource ").Append(fedCredId); + sb.AppendLine(" 'Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials@2023-01-31' = {"); + sb.Append(" parent: identity_").AppendLine(sanitizedName); + sb.Append(" name: '").Append(resourceName).AppendLine("-fedcred'"); + sb.AppendLine(" properties: {"); + sb.Append(" issuer: ").Append(id).AppendLine(".properties.oidcIssuerProfile.issuerURL"); + sb.Append(" subject: 'system:serviceaccount:default:").Append(saName).AppendLine("'"); + sb.AppendLine(" audiences: ["); + sb.AppendLine(" 'api://AzureADTokenExchange'"); + sb.AppendLine(" ]"); + sb.AppendLine(" }"); + sb.AppendLine("}"); + } + + return sb.ToString(); + } + + /// + /// Sanitizes a name for use as a Bicep identifier (alphanumeric + underscore only). + /// + private static string SanitizeBicepIdentifier(string name) + { + var sb = new StringBuilder(name.Length); + foreach (var c in name) + { + sb.Append(char.IsLetterOrDigit(c) ? c : '_'); + } + return sb.ToString(); } } diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 17bc8e59897..77306017db9 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -7,11 +7,12 @@ using System.Diagnostics; using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Eventing; +using Aspire.Hosting.Kubernetes; +using Aspire.Hosting.Kubernetes.Resources; using Aspire.Hosting.Lifecycle; using Aspire.Hosting.Pipelines; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Aspire.Hosting.Kubernetes; using Microsoft.Extensions.Logging; namespace Aspire.Hosting.Azure.Kubernetes; @@ -75,6 +76,46 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance { r.Annotations.Add(new KubernetesNodePoolAnnotation(defaultUserPool)); } + + // Wire workload identity: if the resource has an AppIdentityAnnotation + // (auto-created by AzureResourcePreparer or explicit via WithAzureUserAssignedIdentity), + // generate a ServiceAccount and wire the pod spec. + if (r.TryGetLastAnnotation(out var appIdentity)) + { + // Ensure OIDC + workload identity are enabled on the cluster + environment.OidcIssuerEnabled = true; + environment.WorkloadIdentityEnabled = true; + + var saName = $"{r.Name}-sa"; + var identityClientId = appIdentity.IdentityResource.ClientId; + + // Use KubernetesServiceCustomizationAnnotation to inject SA + pod spec changes + // during Helm chart generation. + r.Annotations.Add(new KubernetesServiceCustomizationAnnotation(kubeResource => + { + // Create ServiceAccount with workload identity annotations + var serviceAccount = new ServiceAccountV1(); + serviceAccount.Metadata.Name = saName; + serviceAccount.Metadata.Annotations["azure.workload.identity/client-id"] = + $"{{{{ .Values.parameters.{r.Name}.identityClientId }}}}"; + serviceAccount.Metadata.Labels["azure.workload.identity/use"] = "true"; + kubeResource.AdditionalResources.Add(serviceAccount); + + // Set serviceAccountName on pod spec + if (kubeResource.Workload?.PodTemplate?.Spec is { } podSpec) + { + podSpec.ServiceAccountName = saName; + } + })); + + // Add the identity clientId as a deferred Helm value parameter + // so it gets resolved from the Bicep output at deploy time. + if (r is IResourceWithEnvironment resourceWithEnv) + { + // Store the identity reference for federated credential generation + environment.WorkloadIdentities[r.Name] = appIdentity.IdentityResource; + } + } } } diff --git a/src/Aspire.Hosting.Kubernetes/Resources/ServiceAccountV1.cs b/src/Aspire.Hosting.Kubernetes/Resources/ServiceAccountV1.cs new file mode 100644 index 00000000000..891262ab49a --- /dev/null +++ b/src/Aspire.Hosting.Kubernetes/Resources/ServiceAccountV1.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using YamlDotNet.Serialization; + +namespace Aspire.Hosting.Kubernetes.Resources; + +/// +/// Represents a Kubernetes ServiceAccount resource. +/// +[YamlSerializable] +public sealed class ServiceAccountV1() : BaseKubernetesResource("v1", "ServiceAccount") +{ +} diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs index fcdc44cd37e..b45cc633817 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs @@ -208,7 +208,7 @@ public void WithWorkloadIdentity_EnablesOidcAndWorkloadIdentity() } [Fact] - public void WithAzureWorkloadIdentity_AddsAnnotations() + public void WithAzureUserAssignedIdentity_WorksWithAks() { using var builder = TestDistributedApplicationBuilder.Create(); @@ -216,28 +216,10 @@ public void WithAzureWorkloadIdentity_AddsAnnotations() var identity = builder.AddAzureUserAssignedIdentity("myIdentity"); var project = builder.AddContainer("myapi", "myimage") - .WithAzureWorkloadIdentity(identity); + .WithAzureUserAssignedIdentity(identity); Assert.True(project.Resource.TryGetLastAnnotation(out var appIdentity)); Assert.Same(identity.Resource, appIdentity.IdentityResource); - - Assert.True(project.Resource.TryGetLastAnnotation(out var aksIdentity)); - Assert.Same(identity.Resource, aksIdentity.IdentityResource); - } - - [Fact] - public void WithAzureWorkloadIdentity_AutoCreatesIdentity() - { - using var builder = TestDistributedApplicationBuilder.Create(); - - var aks = builder.AddAzureKubernetesEnvironment("aks"); - - var project = builder.AddContainer("myapi", "myimage") - .WithAzureWorkloadIdentity(); - - Assert.True(project.Resource.TryGetLastAnnotation(out _)); - Assert.True(project.Resource.TryGetLastAnnotation(out var aksIdentity)); - Assert.NotNull(aksIdentity.IdentityResource); } [Fact] From e7d7da52aa1731942e786b2a6ab8fd5053191aa4 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Tue, 14 Apr 2026 22:41:46 +1000 Subject: [PATCH 46/78] Fix empty workload identity clientId in ServiceAccount annotation The identity clientId was captured but never wired as a deferred Helm value, resulting in an empty azure.workload.identity/client-id annotation on the ServiceAccount. This caused pods to authenticate with the identity but lack the actual client ID, leading to 403 AuthorizationPermissionMismatch errors on Azure resources. Fix: Add identity.ClientId as a CapturedHelmValueProvider so it gets resolved from Bicep output at deploy time and written into the Helm override values under parameters..identityClientId. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesInfrastructure.cs | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 77306017db9..b0c86ce3841 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -101,6 +101,13 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance serviceAccount.Metadata.Labels["azure.workload.identity/use"] = "true"; kubeResource.AdditionalResources.Add(serviceAccount); + // Add a placeholder parameter for the identity clientId + // so it appears in values.yaml under parameters..identityClientId. + // The actual value is resolved at deploy time via CapturedHelmValueProviders. + kubeResource.Parameters["identityClientId"] = new KubernetesResource.HelmValue( + $"{{{{ .Values.parameters.{r.Name}.identityClientId }}}}", + string.Empty); + // Set serviceAccountName on pod spec if (kubeResource.Workload?.PodTemplate?.Spec is { } podSpec) { @@ -108,13 +115,21 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance } })); - // Add the identity clientId as a deferred Helm value parameter - // so it gets resolved from the Bicep output at deploy time. - if (r is IResourceWithEnvironment resourceWithEnv) + // Wire the identity clientId as a deferred Helm value so it gets + // resolved from the Bicep output at deploy time. The SA annotation + // references {{ .Values.parameters..identityClientId }}. + if (identityClientId is IValueProvider clientIdProvider) { - // Store the identity reference for federated credential generation - environment.WorkloadIdentities[r.Name] = appIdentity.IdentityResource; + environment.KubernetesEnvironment.CapturedHelmValueProviders.Add( + new KubernetesEnvironmentResource.CapturedHelmValueProvider( + "parameters", + r.Name, + "identityClientId", + clientIdProvider)); } + + // Store the identity reference for federated credential Bicep generation + environment.WorkloadIdentities[r.Name] = appIdentity.IdentityResource; } } } From 42f669d2880c8ab8701f95c477f703d1ea70d902 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 00:00:26 +1000 Subject: [PATCH 47/78] Fix missing workload identity pod label The azure.workload.identity/use label was only added to the ServiceAccount but NOT to the pod template. The AKS workload identity admission webhook requires this label on the pod to inject AZURE_CLIENT_ID, AZURE_TENANT_ID, and token volume mounts. Without the pod label, the webhook doesn't fire and the pod authenticates with the default SA token instead of the federated identity, resulting in 403 AuthorizationPermissionMismatch. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesInfrastructure.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index b0c86ce3841..4899a642df4 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -108,10 +108,17 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance $"{{{{ .Values.parameters.{r.Name}.identityClientId }}}}", string.Empty); - // Set serviceAccountName on pod spec - if (kubeResource.Workload?.PodTemplate?.Spec is { } podSpec) + // Set serviceAccountName on pod spec and add workload identity label + if (kubeResource.Workload?.PodTemplate is { } podTemplate) { - podSpec.ServiceAccountName = saName; + if (podTemplate.Spec is { } podSpec) + { + podSpec.ServiceAccountName = saName; + } + + // The workload identity webhook requires this label on the POD + // to inject AZURE_CLIENT_ID, token volume mounts, etc. + podTemplate.Metadata.Labels["azure.workload.identity/use"] = "true"; } })); From 333f62d31f0d59730ec26a430ca97f818138b2ae Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 00:26:30 +1000 Subject: [PATCH 48/78] Rename AzureVmSizes to AksNodeVmSizes More specific name that clarifies these are VM sizes for AKS node pools, not general Azure VM sizes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...zureVmSizes.Generated.cs => AksNodeVmSizes.Generated.cs} | 2 +- .../AzureKubernetesEnvironmentExtensions.cs | 2 +- src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) rename src/Aspire.Hosting.Azure.Kubernetes/{AzureVmSizes.Generated.cs => AksNodeVmSizes.Generated.cs} (99%) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureVmSizes.Generated.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksNodeVmSizes.Generated.cs similarity index 99% rename from src/Aspire.Hosting.Azure.Kubernetes/AzureVmSizes.Generated.cs rename to src/Aspire.Hosting.Azure.Kubernetes/AksNodeVmSizes.Generated.cs index bd567c74652..40b5524311c 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureVmSizes.Generated.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AksNodeVmSizes.Generated.cs @@ -13,7 +13,7 @@ namespace Aspire.Hosting.Azure.Kubernetes; /// This class is auto-generated. To update, run the GenVmSizes tool: /// dotnet run --project src/Aspire.Hosting.Azure.Kubernetes/tools GenVmSizes.cs /// -public static partial class AzureVmSizes +public static partial class AksNodeVmSizes { /// /// General purpose VM sizes optimized for balanced CPU-to-memory ratio. diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index ffc86f9145f..0fda50c7f73 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -269,7 +269,7 @@ public static IResourceBuilder WithSubnet( /// var aks = builder.AddAzureKubernetesEnvironment("aks") /// .WithSubnet(defaultSubnet); /// - /// var gpuPool = aks.AddNodePool("gpu", AzureVmSizes.GpuAccelerated.StandardNC6sV3, 0, 5) + /// var gpuPool = aks.AddNodePool("gpu", AksNodeVmSizes.GpuAccelerated.StandardNC6sV3, 0, 5) /// .WithSubnet(gpuSubnet); /// /// diff --git a/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs b/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs index 8153e0b2448..c4dff1aef8c 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs @@ -65,8 +65,8 @@ Console.WriteLine($"Found {vmSkus.Count} VM sizes"); var code = VmSizeClassGenerator.GenerateCode("Aspire.Hosting.Azure.Kubernetes", vmSkus); -File.WriteAllText(Path.Combine("..", "AzureVmSizes.Generated.cs"), code); -Console.WriteLine($"Generated AzureVmSizes.Generated.cs with {vmSkus.Count} VM sizes"); +File.WriteAllText(Path.Combine("..", "AksNodeVmSizes.Generated.cs"), code); +Console.WriteLine($"Generated AksNodeVmSizes.Generated.cs with {vmSkus.Count} VM sizes"); return 0; @@ -173,7 +173,7 @@ public static string GenerateCode(string ns, List sizes) sb.AppendLine("/// This class is auto-generated. To update, run the GenVmSizes tool:"); sb.AppendLine("/// dotnet run --project src/Aspire.Hosting.Azure.Kubernetes/tools GenVmSizes.cs"); sb.AppendLine("/// "); - sb.AppendLine("public static partial class AzureVmSizes"); + sb.AppendLine("public static partial class AksNodeVmSizes"); sb.AppendLine("{"); var groups = sizes.GroupBy(s => s.Family) From 91b81f1da8541da3a55fb558fc8688bcd5430e0c Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 00:39:21 +1000 Subject: [PATCH 49/78] Update AKS spec to reflect current implementation - Mark implemented features (Phase 1-3, node pools, IValueProvider) - Update workload identity to document AppIdentityAnnotation approach - Update VNet to document WithSubnet (not WithDelegatedSubnet) - Document removed APIs (WithAzureWorkloadIdentity, AksWorkloadIdentityAnnotation) - List remaining gaps: monitoring Bicep, WithHelm/WithDashboard, AsExisting Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/specs/aks-support.md | 320 ++++++++++++++++++++------------------ 1 file changed, 168 insertions(+), 152 deletions(-) diff --git a/docs/specs/aks-support.md b/docs/specs/aks-support.md index a13ce6dc082..38e56cabf4a 100644 --- a/docs/specs/aks-support.md +++ b/docs/specs/aks-support.md @@ -290,73 +290,64 @@ internal sealed class AzureKubernetesInfrastructure( ### 2. Workload Identity Support -Workload identity enables pods to authenticate to Azure services using federated credentials without storing secrets. This requires three things: -1. A user-assigned managed identity in Azure -2. A Kubernetes service account annotated with the identity's client ID -3. A federated credential linking the identity to the K8s service account via OIDC - -**New types**: +Workload identity enables pods to authenticate to Azure services using federated credentials without storing secrets. This is implemented by honoring the shared `AppIdentityAnnotation` from `Aspire.Hosting.Azure` — the same mechanism used by ACA and AppService. + +**How it works**: +1. `AzureResourcePreparer` auto-creates a per-resource managed identity when a compute resource references Azure services (e.g., `WithReference(blobStorage)`) +2. It adds `AppIdentityAnnotation` with the identity to the resource +3. Users can override with `WithAzureUserAssignedIdentity(myIdentity)` to supply their own identity +4. `AzureKubernetesInfrastructure` detects `AppIdentityAnnotation` and generates: + - A K8s `ServiceAccount` with `azure.workload.identity/client-id` annotation + - `serviceAccountName` on the pod spec + - `azure.workload.identity/use: "true"` pod label + - Federated identity credential in AKS Bicep module + +**User API** (same as ACA): ```csharp -// Annotation to mark a compute resource for workload identity -public class AksWorkloadIdentityAnnotation( - IAppIdentityResource identityResource, - string? serviceAccountName = null) : IResourceAnnotation -{ - public IAppIdentityResource IdentityResource { get; } = identityResource; - public string? ServiceAccountName { get; set; } = serviceAccountName; -} +// Automatic — identity auto-created when referencing Azure resources +builder.AddProject() + .WithComputeEnvironment(aks) + .WithReference(blobStorage); // gets identity + workload identity + role assignments + +// Explicit — bring your own identity +var identity = builder.AddAzureUserAssignedIdentity("api-identity"); +builder.AddProject() + .WithComputeEnvironment(aks) + .WithAzureUserAssignedIdentity(identity); ``` -**Extension method** (on compute resources): -```csharp -public static IResourceBuilder WithAzureWorkloadIdentity( - this IResourceBuilder builder, - IResourceBuilder? identity = null) - where T : IResource -{ - // If no identity provided, auto-create one named "{resource}-identity" - // Add AksWorkloadIdentityAnnotation - // This will be picked up by the AKS infrastructure to: - // 1. Create a federated credential (Bicep) - // 2. Generate a ServiceAccount YAML with azure.workload.identity/client-id annotation - // 3. Add the azure.workload.identity/use: "true" label to the pod spec +**Generated Bicep** (federated credential): +```bicep +resource federatedCredential 'Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials@2023-01-31' = { + parent: identity + name: '${resourceName}-fedcred' + properties: { + issuer: aksCluster.properties.oidcIssuerProfile.issuerURL + subject: 'system:serviceaccount:${namespace}:${resourceName}-sa' + audiences: ['api://AzureADTokenExchange'] + } } ``` -**Integration with KubernetesInfrastructure**: -When the AKS environment processes a resource with `AksWorkloadIdentityAnnotation`, it: -1. **Bicep side**: Creates a `FederatedIdentityCredential` resource: - ```bicep - resource federatedCredential 'Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials@2023-01-31' = { - parent: identity - name: '${resourceName}-fedcred' - properties: { - issuer: aksCluster.properties.oidcIssuerProfile.issuerURL - subject: 'system:serviceaccount:${namespace}:${serviceAccountName}' - audiences: ['api://AzureADTokenExchange'] - } - } - ``` -2. **Helm chart side**: Generates a ServiceAccount and patches the pod template: - ```yaml - apiVersion: v1 - kind: ServiceAccount - metadata: - name: {{ .Values.parameters.myapi.serviceAccountName }} - annotations: - azure.workload.identity/client-id: {{ .Values.parameters.myapi.azureClientId }} - --- - # In the Deployment pod spec: - spec: - serviceAccountName: {{ .Values.parameters.myapi.serviceAccountName }} - labels: - azure.workload.identity/use: "true" - ``` - -**Key design decision**: The federated credential Bicep resource needs the OIDC issuer URL from the AKS cluster output. This creates a dependency ordering: -- AKS cluster must be provisioned first -- Then federated credentials can reference its OIDC issuer URL -- This is handled naturally by Bicep's dependency graph +**Generated Helm chart** (ServiceAccount + pod template): +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: apiservice-sa + annotations: + azure.workload.identity/client-id: {{ .Values.parameters.apiservice.identityClientId }} + labels: + azure.workload.identity/use: "true" +--- +# In the Deployment pod template: +spec: + serviceAccountName: apiservice-sa + template: + metadata: + labels: + azure.workload.identity/use: "true" +``` ### 3. Monitoring Integration @@ -391,35 +382,27 @@ var aks = builder.AddAzureKubernetesEnvironment("aks") ### 4. VNet Integration -AKS needs a subnet for its nodes (and optionally pods with Azure CNI Overlay). This uses the existing `WithDelegatedSubnet` pattern already established for Container Apps and other Azure compute resources. - -**Design** (uses the existing generic extension from `Aspire.Hosting.Azure.Network`): - -Since `AzureKubernetesEnvironmentResource` implements `IAzureDelegatedSubnetResource` with `DelegatedSubnetServiceName = "Microsoft.ContainerService/managedClusters"`, the existing `WithDelegatedSubnet()` extension method works directly: +AKS needs a subnet for its nodes. Unlike Container Apps, AKS does **not** use subnet delegation — it uses plain (non-delegated) subnets. The API is `WithSubnet()` (not `WithDelegatedSubnet()`). +**Design**: ```csharp -// User code — uses the EXISTING WithDelegatedSubnet from Aspire.Hosting.Azure.Network +var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); +var defaultSubnet = vnet.AddSubnet("default", "10.0.0.0/22"); +var gpuSubnet = vnet.AddSubnet("gpu-subnet", "10.0.4.0/24"); + +// Environment-level subnet (applies to all pools by default) var aks = builder.AddAzureKubernetesEnvironment("aks") - .WithDelegatedSubnet(aksSubnet); -``` + .WithSubnet(defaultSubnet); -The `ConfigureAksInfrastructure` callback reads the delegated subnet annotation and wires it into the `ManagedCluster` Bicep: -```bicep -resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-02-01' = { - properties: { - networkProfile: { - networkPlugin: 'azure' - networkPolicy: 'calico' - serviceCidr: '10.0.4.0/22' - dnsServiceIP: '10.0.4.10' - } - agentPoolProfiles: [{ - vnetSubnetID: subnet.id - }] - } -} +// Per-pool subnet override +var gpuPool = aks.AddNodePool("gpu", AksNodeVmSizes.GpuAccelerated.StandardNC6sV3, 0, 5) + .WithSubnet(gpuSubnet); ``` +**Bicep**: Environment-level subnet → `subnetId` parameter. Per-pool subnets → `subnetId_{poolName}` parameters. Each agent pool profile uses its own subnet if set, else the environment default. + +**Network profile**: Azure CNI is auto-configured when any subnet is set. + **Private cluster support**: ```csharp public static IResourceBuilder AsPrivateCluster( @@ -532,74 +515,107 @@ var aks = builder.AddAzureKubernetesService("aks") 8. **Helm config delegation**: How cleanly can `WithHelm()` / `WithDashboard()` be forwarded from `AzureKubernetesEnvironmentResource` to the inner `KubernetesEnvironmentResource`? Should the inner resource be exposed or kept fully internal? -## Implementation Phases - -### Phase 1: Unified AKS Environment (Foundation) -- Create `Aspire.Hosting.Azure.Kubernetes` package with dependency on `Azure.Provisioning.Kubernetes` -- `AzureKubernetesEnvironmentResource` combining Azure provisioning + K8s compute environment -- `AddAzureKubernetesEnvironment()` entry point (calls `AddKubernetesInfrastructureCore` internally) -- `AzureKubernetesInfrastructure` eventing subscriber -- Basic cluster Bicep generation (version, SKU, default node pool) -- ACR auto-creation and AcrPull role assignment for kubelet identity -- Kubeconfig retrieval pipeline step -- `AsExisting()` support for bring-your-own AKS -- Helm config delegation (`WithHelm()`, `WithDashboard()`) - -### Phase 2: Workload Identity -- `AksWorkloadIdentityAnnotation` -- `WithAzureWorkloadIdentity()` extension on compute resources -- Federated credential Bicep generation (using AKS OIDC issuer URL output) -- ServiceAccount YAML generation in Helm chart -- Pod label injection (`azure.workload.identity/use`) -- Integration with existing `AppIdentityAnnotation` / `IAppIdentityResource` pattern - -### Phase 3: Networking -- `WithDelegatedSubnet()` — uses existing generic extension from `Aspire.Hosting.Azure.Network` (since resource implements `IAzureDelegatedSubnetResource`) -- Azure CNI network profile configuration -- `AsPrivateCluster()` for private API server -- Private DNS Zone link verification for backing service private endpoints - -### Phase 4: Monitoring -- `WithAzureLogAnalyticsWorkspace()` — matches Container Apps naming convention -- `WithContainerInsights()` — AKS-specific addon with optional Log Analytics auto-create -- Container Insights addon profile -- Azure Monitor metrics profile -- Optional Application Insights OTLP integration -- Data collection rule configuration - -### Phase 5: Network Perimeter -- NSP association support (`IAzureNspAssociationTarget` on AKS) -- Private DNS Zone auto-linking when backing services have private endpoints in same VNet -- Network policy integration +## Implementation Status + +### ✅ Implemented + +#### Phase 1: Unified AKS Environment (Foundation) +- ✅ `Aspire.Hosting.Azure.Kubernetes` package created +- ✅ `AzureKubernetesEnvironmentResource` — extends `AzureProvisioningResource`, implements `IAzureComputeEnvironmentResource`, `IAzureNspAssociationTarget` +- ✅ `AddAzureKubernetesEnvironment()` entry point — calls `AddKubernetesEnvironment()` internally +- ✅ `AzureKubernetesInfrastructure` eventing subscriber +- ✅ Hand-crafted Bicep generation (not Azure.Provisioning SDK — `Azure.Provisioning.ContainerService` not in internal feeds) +- ✅ ACR auto-creation + AcrPull role assignment in Bicep +- ✅ Kubeconfig retrieval via `az aks get-credentials` to isolated temp file +- ✅ Multi-environment support (scoped Helm chart names, per-env kubeconfig) +- ✅ `WithVersion()`, `WithSkuTier()`, `WithContainerRegistry()` +- ✅ `AsPrivateCluster()` — sets `apiServerAccessProfile.enablePrivateCluster` +- ✅ Push step dependency wiring for container image builds + +#### Phase 2: Workload Identity +- ✅ Honors `AppIdentityAnnotation` from `Aspire.Hosting.Azure` (same mechanism as ACA/AppService) +- ✅ Auto-identity via `AzureResourcePreparer` when resources reference Azure services +- ✅ Override with `WithAzureUserAssignedIdentity(identity)` (standard API) +- ✅ ServiceAccount YAML generation with `azure.workload.identity/client-id` annotation +- ✅ Pod label `azure.workload.identity/use: "true"` on pod template +- ✅ `serviceAccountName` set on pod spec +- ✅ Federated identity credential Bicep generation per workload +- ✅ Identity `clientId` wired as deferred Helm value (resolved at deploy time) +- ✅ `ServiceAccountV1` resource added to `Aspire.Hosting.Kubernetes` +- ❌ ~~`AksWorkloadIdentityAnnotation`~~ — **Removed** (redundant with `AppIdentityAnnotation`) +- ❌ ~~`WithAzureWorkloadIdentity()`~~ — **Removed** (standard `WithAzureUserAssignedIdentity` works) + +#### Phase 3: Networking +- ✅ `WithSubnet()` (NOT `WithDelegatedSubnet` — AKS doesn't support subnet delegation) +- ✅ Per-node-pool subnet support via `WithSubnet()` on `AksNodePoolResource` +- ✅ Azure CNI network profile auto-configured when subnet is set +- ✅ `AsPrivateCluster()` for private API server +- ❌ AKS does NOT implement `IAzureDelegatedSubnetResource` (intentionally — AKS uses plain subnets) + +#### Node Pools (not in original spec) +- ✅ Base `KubernetesNodePoolResource` in `Aspire.Hosting.Kubernetes` (cloud-agnostic) +- ✅ `AksNodePoolResource` extends base with VM size, scaling, mode config +- ✅ `AddNodePool()` on both K8s and AKS environments +- ✅ `WithNodePool()` schedules workloads via `nodeSelector` on pod spec +- ✅ `AksNodeVmSizes` constants class (GeneralPurpose, ComputeOptimized, MemoryOptimized, GpuAccelerated, StorageOptimized, Burstable, Arm) +- ✅ `GenVmSizes.cs` tool + `update-azure-vm-sizes.yml` monthly workflow +- ✅ Default "workload" user pool auto-created if none configured + +#### IValueProvider Resolution (not in original spec) +- ✅ Azure resource connection strings and endpoints resolved at deploy time +- ✅ Composite expressions (e.g., `Endpoint={storage.outputs.blobEndpoint};ContainerName=photos`) handled +- ✅ Phase 4 in HelmDeploymentEngine for generic `IValueProvider` resolution + +### 🔲 Not Yet Implemented + +#### Monitoring (Phase 4) — Bicep not emitted +- 🔲 `WithContainerInsights()` and `WithAzureLogAnalyticsWorkspace()` **exist as APIs** but the Bicep generation does NOT emit: + - Container Insights addon profile (`addonProfiles.omsagent`) + - Azure Monitor metrics profile (managed Prometheus) + - Data collection rules + - Application Insights OTLP integration + +#### Helm/Dashboard delegation +- 🔲 `WithHelm()` and `WithDashboard()` are not exposed on `AzureKubernetesEnvironmentResource` + - They work on the inner `KubernetesEnvironmentResource` but users can't access them from the AKS builder + +#### AsExisting() support +- 🔲 `AsExisting()` for referencing pre-provisioned AKS clusters + +#### Private DNS Zone auto-linking +- 🔲 When backing services have private endpoints in same VNet as AKS, Private DNS Zones should be auto-linked + +#### IAzureContainerRegistry interface +- 🔲 AKS resource does not implement `IAzureContainerRegistry` (ACR outputs not exposed via standard interface) + +#### Ingress controller +- 🔲 Application Gateway Ingress Controller (AGIC) or other ingress support + +#### Managed Prometheus/Grafana +- 🔲 Azure Monitor workspace for managed Prometheus +- 🔲 Azure Managed Grafana provisioning + +## Key Design Changes from Original Spec + +1. **Bicep generation**: Uses hand-crafted `StringBuilder` via `GetBicepTemplateString()` override, NOT `Azure.Provisioning.ContainerService` SDK (package not available in internal NuGet feeds) +2. **Workload identity**: Uses shared `AppIdentityAnnotation` from `Aspire.Hosting.Azure`, not AKS-specific annotation. Same mechanism as ACA/AppService. +3. **Subnet integration**: `WithSubnet()` not `WithDelegatedSubnet()` — AKS uses plain subnets, not delegated ones +4. **Node pools**: First-class resources with `AddNodePool()` returning `IResourceBuilder`, `WithNodePool()` for scheduling, per-pool subnets, `AksNodeVmSizes` constants +5. **Multi-environment**: Full support for multiple AKS environments with scoped chart names and isolated kubeconfigs ## Dependencies / Prerequisites -- `Azure.Provisioning.Kubernetes` NuGet package (v1.0.0-beta.3 — need to add to `Directory.Packages.props`) -- `Azure.Provisioning.ContainerRegistry` (already used, v1.1.0) -- `Azure.Provisioning.Network` (already used, v1.1.0-beta.2) -- `Azure.Provisioning.OperationalInsights` (already used, v1.1.0) -- `Azure.Provisioning.Roles` (already used, for identity/RBAC) -- `Aspire.Hosting.Kubernetes` (the generic K8s package, already in repo) - -## Testing Strategy - -- **Unit tests**: Bicep template generation verification (snapshot tests like existing K8s tests) -- **Integration tests**: Verify Helm chart output includes ServiceAccount, labels, etc. -- **E2E tests**: Provision AKS + deploy workloads (expensive, CI-gated) -- **Existing test patterns**: Follow `Aspire.Hosting.Kubernetes.Tests` structure - -## Todos - -1. **aks-package-setup**: Create `Aspire.Hosting.Azure.Kubernetes` project, csproj, dependencies (including `Azure.Provisioning.Kubernetes`), add to `Directory.Packages.props` -2. **aks-environment-resource**: Implement `AzureKubernetesEnvironmentResource` with Bicep provisioning via `Azure.Provisioning.Kubernetes.ManagedCluster`, ACR auto-creation, and inner `KubernetesEnvironmentResource` -3. **aks-extensions**: Implement `AddAzureKubernetesEnvironment()` entry point and configuration extensions (`WithVersion`, `WithSkuTier`, `WithNodePool`, `WithHelm`, `WithDashboard`, `WithContainerRegistry`) -4. **aks-infrastructure**: Implement `AzureKubernetesInfrastructure` eventing subscriber — process compute resources, add `DeploymentTargetAnnotation`, handle kubeconfig retrieval pipeline step -5. **workload-identity-annotation**: `AksWorkloadIdentityAnnotation` and `WithAzureWorkloadIdentity()` extension method on compute resources. Auto-create identity if not provided. -6. **workload-identity-bicep**: Generate `FederatedIdentityCredential` Bicep resource linking managed identity to K8s service account via OIDC issuer URL output -7. **workload-identity-helm**: Generate ServiceAccount YAML with `azure.workload.identity/client-id` annotation. Add `azure.workload.identity/use` label to pod spec. -8. **vnet-integration**: `WithDelegatedSubnet()` — leverages existing `IAzureDelegatedSubnetResource` pattern. Azure CNI network profile. Subnet delegation for `Microsoft.ContainerService/managedClusters`. -9. **private-cluster**: `AsPrivateCluster()` extension. Sets `apiServerAccessProfile.enablePrivateCluster`. Requires delegated subnet. -10. **monitoring**: `WithAzureLogAnalyticsWorkspace()` (matches Container Apps naming) + `WithContainerInsights()` (AKS addon). Log Analytics auto-create, Azure Monitor metrics, data collection rules. -11. **nsp-support**: Implement `IAzureNspAssociationTarget` on AKS resource. Auto-link Private DNS Zones when backing services have private endpoints in same VNet. -12. **existing-cluster**: `AsExisting()` support for referencing pre-provisioned AKS clusters via `ExistingAzureResourceAnnotation` pattern. -13. **tests**: Unit tests (Bicep snapshot verification), integration tests (Helm chart output), E2E tests (provision + deploy). +- ~~`Azure.Provisioning.Kubernetes`~~ — Not used (hand-crafted Bicep instead) +- `Azure.Provisioning.ContainerRegistry` (for ACR resource type reference) +- `Azure.Provisioning.OperationalInsights` (for Log Analytics workspace type) +- `Aspire.Hosting.Kubernetes` (the generic K8s package) +- `Aspire.Hosting.Azure` (for `AppIdentityAnnotation`, `AzureProvisioningResource`, etc.) +- `Aspire.Hosting.Azure.Network` (for subnet, VNet, NSP types) +- `Aspire.Hosting.Azure.ContainerRegistry` (for ACR auto-creation) + +## Testing + +- 31 AKS unit tests passing (extensions + infrastructure) +- 88 K8s base tests passing +- Manual E2E validation against live Azure clusters + From e6674d5fa6eca2fa3987cd223caf5c48da3497af Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 10:08:02 +1000 Subject: [PATCH 50/78] Query all US regions for VM sizes in GenVmSizes tool Query 9 US regions (eastus, eastus2, centralus, northcentralus, southcentralus, westus, westus2, westus3, westcentralus) and merge results to produce a comprehensive unified list. Different regions offer different VM sizes, so a single-region query missed sizes only available in other regions. Document the region coverage in the generated file header comments. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../tools/GenVmSizes.cs | 50 ++++++++++++++----- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs b/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs index c4dff1aef8c..6652411718c 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs @@ -26,26 +26,46 @@ Console.WriteLine($"Using subscription: {subscriptionId}"); -// Fetch resource SKUs filtered to virtualMachines -var url = $"https://management.azure.com/subscriptions/{subscriptionId}/providers/Microsoft.Compute/skus?api-version=2021-07-01&$filter=location eq 'eastus'"; -var json = await RunAzCommand($"rest --method get --url \"{url}\"").ConfigureAwait(false); - -if (string.IsNullOrWhiteSpace(json)) +// Query all US regions for VM SKUs to build a comprehensive unified list. +// Different regions offer different VM sizes, so querying multiple regions +// ensures we capture the full set available across the US. +string[] usRegions = [ + "eastus", "eastus2", "centralus", "northcentralus", "southcentralus", + "westus", "westus2", "westus3", "westcentralus" +]; + +var allSkus = new List(); +foreach (var region in usRegions) { - Console.Error.WriteLine("Error: Failed to fetch VM SKUs from Azure REST API."); - return 1; + Console.WriteLine($"Querying VM SKUs for {region}..."); + var url = $"https://management.azure.com/subscriptions/{subscriptionId}/providers/Microsoft.Compute/skus?api-version=2021-07-01&$filter=location eq '{region}'"; + var json = await RunAzCommand($"rest --method get --url \"{url}\"").ConfigureAwait(false); + + if (string.IsNullOrWhiteSpace(json)) + { + Console.Error.WriteLine($"Warning: Failed to fetch VM SKUs for {region}, skipping."); + continue; + } + + var skuResponse = JsonSerializer.Deserialize(json, new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + if (skuResponse?.Value is not null) + { + allSkus.AddRange(skuResponse.Value); + Console.WriteLine($" Found {skuResponse.Value.Count} SKUs in {region}"); + } } -var skuResponse = JsonSerializer.Deserialize(json, new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); -if (skuResponse is null || skuResponse.Value is null) +if (allSkus.Count == 0) { - Console.Error.WriteLine("Error: Failed to parse SKU response."); + Console.Error.WriteLine("Error: Failed to fetch VM SKUs from any US region."); return 1; } -// Filter to virtualMachines resource type and group by family -var vmSkus = skuResponse.Value +// Filter to virtualMachines, deduplicate by name (keep first occurrence for capability data) +var vmSkus = allSkus .Where(s => s.ResourceType == "virtualMachines" && !string.IsNullOrEmpty(s.Name)) + .GroupBy(s => s.Name) + .Select(g => g.First()) .Select(s => new VmSizeInfo { Name = s.Name!, @@ -170,8 +190,12 @@ public static string GenerateCode(string ns, List sizes) sb.AppendLine("/// Provides well-known Azure VM size constants for use with AKS node pools."); sb.AppendLine("/// "); sb.AppendLine("/// "); - sb.AppendLine("/// This class is auto-generated. To update, run the GenVmSizes tool:"); + sb.AppendLine("/// This class is auto-generated from Azure Resource SKUs across all US regions."); + sb.AppendLine("/// To update, run the GenVmSizes tool:"); sb.AppendLine("/// dotnet run --project src/Aspire.Hosting.Azure.Kubernetes/tools GenVmSizes.cs"); + sb.AppendLine("/// VM size availability varies by region. This list is a union of sizes available"); + sb.AppendLine("/// across eastus, eastus2, centralus, northcentralus, southcentralus, westus, westus2,"); + sb.AppendLine("/// westus3, and westcentralus. Not all sizes may be available in every region."); sb.AppendLine("/// "); sb.AppendLine("public static partial class AksNodeVmSizes"); sb.AppendLine("{"); From 5d2fb369e4facad85b704ad5c6ad1ffd6f0d1d5a Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 10:12:24 +1000 Subject: [PATCH 51/78] Address PR review: pragmas and DelegatedSubnetAnnotation - r1: Add explanatory comments to #pragma warning disable directives explaining why each experimental API suppression is needed - r11: Remove DelegatedSubnetAnnotation fallback in Bicep generation. AKS uses plain subnets (WithSubnet), not delegated subnets. There is no legacy path to support. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 6 +++--- .../AzureKubernetesEnvironmentResource.cs | 11 +---------- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 0fda50c7f73..f89571bd48b 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -1,9 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#pragma warning disable ASPIREPIPELINES001 // Pipeline types are experimental -#pragma warning disable ASPIREAZURE001 // AzureEnvironmentResource is experimental -#pragma warning disable ASPIREAZURE003 // Subnet/network types are experimental +#pragma warning disable ASPIREPIPELINES001 // Pipeline step types used for push/deploy dependency wiring +#pragma warning disable ASPIREAZURE001 // AzureEnvironmentResource.ProvisionInfrastructureStepName for pipeline ordering +#pragma warning disable ASPIREAZURE003 // AzureSubnetResource used in WithSubnet extensions using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Azure; diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs index f5828023531..c5259cb700c 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs @@ -167,16 +167,7 @@ private string GenerateAksBicep() // Subnet parameters for VNet integration // Environment-level subnet (default for all pools) var hasDefaultSubnet = this.TryGetLastAnnotation(out var subnetAnnotation); - if (!hasDefaultSubnet) - { - // Fallback: check for DelegatedSubnetAnnotation (legacy WithDelegatedSubnet usage) - hasDefaultSubnet = this.TryGetLastAnnotation(out var delegatedAnnotation); - if (hasDefaultSubnet) - { - Parameters["subnetId"] = delegatedAnnotation!.SubnetId; - } - } - else + if (hasDefaultSubnet) { Parameters["subnetId"] = subnetAnnotation!.SubnetId; } From 59d290bb19fde8ed015921a700538140300c9c81 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 10:24:34 +1000 Subject: [PATCH 52/78] Enable ATS exports for all AKS public APIs Replace all AspireExportIgnore attributes with proper AspireExport attributes so AKS APIs are available in TypeScript-based AppHosts. Exported methods: AddAzureKubernetesEnvironment, WithVersion, WithSkuTier, AddNodePool, AsPrivateCluster, WithSubnet (env + pool), WithContainerRegistry, WithContainerInsights, WithAzureLogAnalyticsWorkspace, WithWorkloadIdentity. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index f89571bd48b..e10af5db7a4 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -37,7 +37,7 @@ public static class AzureKubernetesEnvironmentExtensions /// .WithVersion("1.30"); /// /// - [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + [AspireExport(Description = "Adds an Azure Kubernetes Service environment resource")] public static IResourceBuilder AddAzureKubernetesEnvironment( this IDistributedApplicationBuilder builder, [ResourceName] string name) @@ -132,7 +132,7 @@ public static IResourceBuilder AddAzureKuber /// The resource builder. /// The Kubernetes version (e.g., "1.30"). /// A reference to the for chaining. - [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + [AspireExport(Description = "Sets the Kubernetes version for the AKS cluster")] public static IResourceBuilder WithVersion( this IResourceBuilder builder, string version) @@ -150,7 +150,7 @@ public static IResourceBuilder WithVersion( /// The resource builder. /// The SKU tier. /// A reference to the for chaining. - [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + [AspireExport(Description = "Sets the SKU tier for the AKS cluster")] public static IResourceBuilder WithSkuTier( this IResourceBuilder builder, AksSkuTier tier) @@ -183,7 +183,7 @@ public static IResourceBuilder WithSkuTier( /// .WithNodePool(gpuPool); /// /// - [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + [AspireExport(Description = "Adds a node pool to the AKS cluster")] public static IResourceBuilder AddNodePool( this IResourceBuilder builder, [ResourceName] string name, @@ -214,7 +214,7 @@ public static IResourceBuilder AddNodePool( /// /// The resource builder. /// A reference to the for chaining. - [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + [AspireExport(Description = "Configures the AKS cluster as a private cluster")] public static IResourceBuilder AsPrivateCluster( this IResourceBuilder builder) { @@ -240,7 +240,7 @@ public static IResourceBuilder AsPrivateClus /// .WithSubnet(subnet); /// /// - [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + [AspireExport(Description = "Configures the AKS cluster to use a VNet subnet")] public static IResourceBuilder WithSubnet( this IResourceBuilder builder, IResourceBuilder subnet) @@ -273,7 +273,7 @@ public static IResourceBuilder WithSubnet( /// .WithSubnet(gpuSubnet); /// /// - [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + [AspireExport("withAksNodePoolSubnet", Description = "Configures an AKS node pool to use a specific VNet subnet")] public static IResourceBuilder WithSubnet( this IResourceBuilder builder, IResourceBuilder subnet) @@ -303,7 +303,7 @@ public static IResourceBuilder WithSubnet( /// The registry endpoint is flowed to the inner Kubernetes environment so that /// Helm deployments can push and pull images. /// - [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + [AspireExport(Description = "Configures the AKS environment to use a specific container registry")] public static IResourceBuilder WithContainerRegistry( this IResourceBuilder builder, IResourceBuilder registry) @@ -337,7 +337,7 @@ public static IResourceBuilder WithContainer /// The resource builder. /// Optional Log Analytics workspace. If not provided, one will be auto-created. /// A reference to the for chaining. - [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + [AspireExport(Description = "Enables Container Insights monitoring on the AKS cluster")] public static IResourceBuilder WithContainerInsights( this IResourceBuilder builder, IResourceBuilder? logAnalytics = null) @@ -360,7 +360,7 @@ public static IResourceBuilder WithContainer /// The resource builder. /// The Log Analytics workspace resource builder. /// A reference to the for chaining. - [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + [AspireExport(Description = "Configures the AKS environment to use a Log Analytics workspace")] public static IResourceBuilder WithAzureLogAnalyticsWorkspace( this IResourceBuilder builder, IResourceBuilder workspaceBuilder) @@ -383,7 +383,7 @@ public static IResourceBuilder WithAzureLogA /// Workload identity is automatically wired when compute resources have an , /// which is added by WithAzureUserAssignedIdentity or auto-created by AzureResourcePreparer. /// - [AspireExportIgnore(Reason = "AKS hosting is not yet supported in ATS")] + [AspireExport(Description = "Enables workload identity on the AKS cluster")] public static IResourceBuilder WithWorkloadIdentity( this IResourceBuilder builder) { From bb3566af8c6a67b6f5a15ac8b75b1daec8a90084 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 10:47:31 +1000 Subject: [PATCH 53/78] Refactor az CLI usage to shared ProcessSpec/ProcessUtil infrastructure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace raw System.Diagnostics.Process usage with the shared ProcessSpec/ProcessUtil pattern used by Aspire.Hosting.Azure. - Use PathLookupHelper.FindFullPathFromPath('az') instead of custom FindAzCli() with hardcoded Windows paths - Use ProcessSpec + ProcessUtil.Run for process execution — handles stdout/stderr via callbacks (no deadlock risk), manages process lifecycle via IAsyncDisposable - Extract RunAzCommandAsync helper returning structured result with ExitCode, StandardOutput, StandardError - Check exit code on az resource list (was silently ignored) - Link ProcessSpec.cs, ProcessUtil.cs, ProcessResult.cs, PathLookupHelper.cs into AKS csproj Fixes review items: r3 (deadlock), r4 (exit code), r9 part (FindAzCli) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Aspire.Hosting.Azure.Kubernetes.csproj | 5 + .../AzureKubernetesInfrastructure.cs | 160 +++++++----------- 2 files changed, 66 insertions(+), 99 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj index dfa60c150a8..6b8422708a7 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj +++ b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj @@ -3,6 +3,7 @@ $(DefaultTargetFramework) true + true true false aspire integration hosting azure kubernetes aks @@ -13,6 +14,10 @@ + + + + diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 4899a642df4..888c96bbb7c 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -1,11 +1,12 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#pragma warning disable ASPIREPIPELINES001 // Pipeline types are experimental -#pragma warning disable ASPIREAZURE001 // AzureEnvironmentResource is experimental +#pragma warning disable ASPIREPIPELINES001 // Pipeline step types used for push/deploy dependency wiring +#pragma warning disable ASPIREAZURE001 // AzureEnvironmentResource.ProvisionInfrastructureStepName for pipeline ordering -using System.Diagnostics; +using System.Text; using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Dcp.Process; using Aspire.Hosting.Eventing; using Aspire.Hosting.Kubernetes; using Aspire.Hosting.Kubernetes.Resources; @@ -221,16 +222,8 @@ private static async Task GetAksCredentialsAsync( { try { - // The cluster name is the resource name — we set it directly in the Bicep template. - // We don't use NameOutputReference.GetValueAsync() because it triggers parameter - // resolution that may not be available at this point in the pipeline. var clusterName = environment.Name; - // Get the resource group from the deployment state. The deployment state - // is loaded into IConfiguration by the deploy-prereq step and contains - // Azure:ResourceGroup from the provisioning context. - // We read it via IConfiguration which is populated from the deployment - // state JSON file before pipeline steps execute. var azPath = FindAzCli(); var resourceGroup = await GetResourceGroupAsync(azPath, clusterName, context) .ConfigureAwait(false); @@ -239,47 +232,19 @@ private static async Task GetAksCredentialsAsync( var kubeConfigDir = Directory.CreateTempSubdirectory("aspire-aks"); var kubeConfigPath = Path.Combine(kubeConfigDir.FullName, "kubeconfig"); - var arguments = $"aks get-credentials --resource-group \"{resourceGroup}\" --name \"{clusterName}\" --file \"{kubeConfigPath}\" --overwrite-existing"; - context.Logger.LogInformation( "Fetching AKS credentials: cluster={ClusterName}, resourceGroup={ResourceGroup}", clusterName, resourceGroup); - using var process = new Process(); - process.StartInfo = new ProcessStartInfo - { - FileName = azPath, - Arguments = arguments, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; - - process.Start(); - - var stdoutTask = process.StandardOutput.ReadToEndAsync(context.CancellationToken); - var stderrTask = process.StandardError.ReadToEndAsync(context.CancellationToken); - - await process.WaitForExitAsync(context.CancellationToken).ConfigureAwait(false); - - var stdout = await stdoutTask.ConfigureAwait(false); - var stderr = await stderrTask.ConfigureAwait(false); - - if (!string.IsNullOrWhiteSpace(stdout)) - { - context.Logger.LogDebug("az (stdout): {Output}", stdout); - } + var result = await RunAzCommandAsync( + azPath, + $"aks get-credentials --resource-group \"{resourceGroup}\" --name \"{clusterName}\" --file \"{kubeConfigPath}\" --overwrite-existing", + context.Logger).ConfigureAwait(false); - if (!string.IsNullOrWhiteSpace(stderr)) - { - context.Logger.LogDebug("az (stderr): {Error}", stderr); - } - - if (process.ExitCode != 0) + if (result.ExitCode != 0) { throw new InvalidOperationException( - $"az aks get-credentials failed (exit code {process.ExitCode}): {stderr.Trim()}"); + $"az aks get-credentials failed (exit code {result.ExitCode}): {result.StandardError}"); } // Set the kubeconfig path on the inner K8s environment so @@ -314,44 +279,13 @@ await getCredsTask.FailAsync( private static string FindAzCli() { - // Check PATH first - var pathDirs = Environment.GetEnvironmentVariable("PATH")?.Split(Path.PathSeparator) ?? []; - var azNames = OperatingSystem.IsWindows() - ? new[] { "az.CMD", "az.cmd", "az.exe" } - : new[] { "az" }; - - foreach (var dir in pathDirs) + var azPath = PathLookupHelper.FindFullPathFromPath("az"); + if (azPath is null) { - foreach (var azName in azNames) - { - var candidate = Path.Combine(dir, azName); - if (File.Exists(candidate)) - { - return candidate; - } - } - } - - // Check common Windows locations - if (OperatingSystem.IsWindows()) - { - var commonPaths = new[] - { - @"C:\Program Files\Microsoft SDKs\Azure\CLI2\wbin\az.CMD", - @"C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin\az.CMD", - }; - - foreach (var path in commonPaths) - { - if (File.Exists(path)) - { - return path; - } - } + throw new InvalidOperationException( + "Azure CLI (az) not found. Install it from https://learn.microsoft.com/cli/azure/install-azure-cli"); } - - throw new InvalidOperationException( - "Azure CLI (az) not found. Install it from https://learn.microsoft.com/cli/azure/install-azure-cli"); + return azPath; } /// @@ -378,27 +312,18 @@ private static async Task GetResourceGroupAsync( "Resource group not in deployment state, querying Azure for cluster '{ClusterName}'", clusterName); - var arguments = $"resource list --resource-type Microsoft.ContainerService/managedClusters --name \"{clusterName}\" --query [0].resourceGroup -o tsv"; + var result = await RunAzCommandAsync( + azPath, + $"resource list --resource-type Microsoft.ContainerService/managedClusters --name \"{clusterName}\" --query [0].resourceGroup -o tsv", + context.Logger).ConfigureAwait(false); - using var process = new Process(); - process.StartInfo = new ProcessStartInfo + if (result.ExitCode != 0) { - FileName = azPath, - Arguments = arguments, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; - - process.Start(); - - var stdout = await process.StandardOutput.ReadToEndAsync(context.CancellationToken).ConfigureAwait(false); - await process.StandardError.ReadToEndAsync(context.CancellationToken).ConfigureAwait(false); - - await process.WaitForExitAsync(context.CancellationToken).ConfigureAwait(false); + throw new InvalidOperationException( + $"az resource list failed (exit code {result.ExitCode}): {result.StandardError}"); + } - resourceGroup = stdout.Trim().ReplaceLineEndings("").Trim(); + resourceGroup = result.StandardOutput.Trim().ReplaceLineEndings("").Trim(); if (string.IsNullOrEmpty(resourceGroup)) { @@ -409,4 +334,41 @@ private static async Task GetResourceGroupAsync( return resourceGroup; } + + /// + /// Runs an az CLI command using the shared ProcessSpec/ProcessUtil infrastructure. + /// Returns the captured stdout, stderr, and exit code. + /// + private static async Task RunAzCommandAsync( + string azPath, + string arguments, + ILogger logger) + { + var stdout = new StringBuilder(); + var stderr = new StringBuilder(); + + var spec = new ProcessSpec(azPath) + { + Arguments = arguments, + OnOutputData = data => stdout.AppendLine(data), + OnErrorData = data => stderr.AppendLine(data), + ThrowOnNonZeroReturnCode = false + }; + + logger.LogDebug("Running: {AzPath} {Arguments}", azPath, arguments); + + var (task, disposable) = ProcessUtil.Run(spec); + + try + { + var result = await task.ConfigureAwait(false); + return new AzCommandResult(result.ExitCode, stdout.ToString(), stderr.ToString()); + } + finally + { + await disposable.DisposeAsync().ConfigureAwait(false); + } + } + + private sealed record AzCommandResult(int ExitCode, string StandardOutput, string StandardError); } From d59a000ceaf4b5796e3c0acd26f1d1b2527463d8 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 10:59:27 +1000 Subject: [PATCH 54/78] Use IFileSystemService for kubeconfig temp directory Replace Directory.CreateTempSubdirectory with IFileSystemService .TempDirectory.CreateTempSubdirectory so the kubeconfig temp directory is tracked and automatically cleaned up when the DI container disposes. This ensures AKS cluster credentials don't persist on disk after the pipeline completes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesInfrastructure.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 888c96bbb7c..5cf4f9c0b24 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -3,6 +3,7 @@ #pragma warning disable ASPIREPIPELINES001 // Pipeline step types used for push/deploy dependency wiring #pragma warning disable ASPIREAZURE001 // AzureEnvironmentResource.ProvisionInfrastructureStepName for pipeline ordering +#pragma warning disable ASPIREFILESYSTEM001 // IFileSystemService/TempDirectory are experimental using System.Text; using Aspire.Hosting.ApplicationModel; @@ -228,9 +229,13 @@ private static async Task GetAksCredentialsAsync( var resourceGroup = await GetResourceGroupAsync(azPath, clusterName, context) .ConfigureAwait(false); - // Write credentials to an isolated kubeconfig file - var kubeConfigDir = Directory.CreateTempSubdirectory("aspire-aks"); - var kubeConfigPath = Path.Combine(kubeConfigDir.FullName, "kubeconfig"); + // Write credentials to an isolated kubeconfig file managed by the + // IFileSystemService. The temp directory is tracked and cleaned up + // automatically when the DI container disposes, ensuring credentials + // don't persist on disk after the pipeline completes. + var fileSystemService = context.Services.GetRequiredService(); + var kubeConfigDir = fileSystemService.TempDirectory.CreateTempSubdirectory("aspire-aks"); + var kubeConfigPath = Path.Combine(kubeConfigDir.Path, "kubeconfig"); context.Logger.LogInformation( "Fetching AKS credentials: cluster={ClusterName}, resourceGroup={ResourceGroup}", From 02c22dd254815f4b77037ad1e62fc8303e4a56d7 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 11:07:22 +1000 Subject: [PATCH 55/78] Capture kubeconfig via stdout to control file permissions Use 'az aks get-credentials --file -' to capture kubeconfig content to stdout rather than letting az CLI write the file directly. This avoids the az CLI potentially creating the file with permissive permissions on shared /tmp. Write the kubeconfig content ourselves to the managed temp directory, and on Unix set restrictive file permissions (0600 owner-only) via File.SetUnixFileMode. The temp directory is still managed by IFileSystemService and auto-cleaned on dispose. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesInfrastructure.cs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 5cf4f9c0b24..d3187620061 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -229,10 +229,9 @@ private static async Task GetAksCredentialsAsync( var resourceGroup = await GetResourceGroupAsync(azPath, clusterName, context) .ConfigureAwait(false); - // Write credentials to an isolated kubeconfig file managed by the - // IFileSystemService. The temp directory is tracked and cleaned up - // automatically when the DI container disposes, ensuring credentials - // don't persist on disk after the pipeline completes. + // Fetch kubeconfig content to stdout using --file - to avoid az CLI + // writing credentials with potentially permissive file permissions. + // We then write the content ourselves to a temp file with controlled access. var fileSystemService = context.Services.GetRequiredService(); var kubeConfigDir = fileSystemService.TempDirectory.CreateTempSubdirectory("aspire-aks"); var kubeConfigPath = Path.Combine(kubeConfigDir.Path, "kubeconfig"); @@ -243,7 +242,7 @@ private static async Task GetAksCredentialsAsync( var result = await RunAzCommandAsync( azPath, - $"aks get-credentials --resource-group \"{resourceGroup}\" --name \"{clusterName}\" --file \"{kubeConfigPath}\" --overwrite-existing", + $"aks get-credentials --resource-group \"{resourceGroup}\" --name \"{clusterName}\" --file -", context.Logger).ConfigureAwait(false); if (result.ExitCode != 0) @@ -252,6 +251,16 @@ private static async Task GetAksCredentialsAsync( $"az aks get-credentials failed (exit code {result.ExitCode}): {result.StandardError}"); } + // Write kubeconfig content to a temp file we control. + // The IFileSystemService temp directory is auto-cleaned on dispose. + await File.WriteAllTextAsync(kubeConfigPath, result.StandardOutput, context.CancellationToken).ConfigureAwait(false); + + // On Unix, restrict file permissions to owner-only (0600) + if (!OperatingSystem.IsWindows()) + { + File.SetUnixFileMode(kubeConfigPath, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + // Set the kubeconfig path on the inner K8s environment so // Helm and kubectl commands use --kubeconfig to target this cluster environment.KubernetesEnvironment.KubeConfigPath = kubeConfigPath; From 6ac7c33559d0f07f308261e31fe39c36582015a3 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 11:10:00 +1000 Subject: [PATCH 56/78] Remove redundant HelmValue allocation in ResolveUnknownValue Use object initializer with 'parameter as IValueProvider' instead of creating a HelmValue, then conditionally creating an identical one just to set the init-only ValueProviderSource property. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesResource.cs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs index 9a938ca7ad8..04e9d21348d 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesResource.cs @@ -681,18 +681,13 @@ private static HelmValue ResolveUnknownValue(IManifestExpressionProvider paramet formattedName.ToHelmSecretExpression(resource.Name) : formattedName.ToHelmConfigExpression(resource.Name); - var helmValue = new HelmValue(helmExpression, parameter.ValueExpression); - - // If the expression provider also implements IValueProvider, attach it - // for deploy-time resolution. This handles Bicep output references, - // connection strings, and any other deferred value source. - if (parameter is IValueProvider valueProvider) + var helmValue = new HelmValue(helmExpression, parameter.ValueExpression) { - helmValue = new HelmValue(helmExpression, parameter.ValueExpression) - { - ValueProviderSource = valueProvider - }; - } + // If the expression provider also implements IValueProvider, attach it + // for deploy-time resolution. This handles Bicep output references, + // connection strings, and any other deferred value source. + ValueProviderSource = parameter as IValueProvider + }; return helmValue; } From 7788491f7e04208305dee36340215463eee7618a Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 11:22:42 +1000 Subject: [PATCH 57/78] Fix node pool model bugs and update AKS API version r5: FindNodePoolResource now searches the app model for existing AksNodePoolResource instances by name and parent, preserving object identity with pools created via AddNodePool(). Falls back to creating a new resource only for pools added via config but not via the API. r8: Default 'workload' node pool is now added to the app model via appModel.Resources.Add() so it appears in manifests and pipelines, matching the behavior of pools created via AddNodePool(). r12: Update AKS API version from 2024-06-02-preview to 2026-01-01 (current GA version). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentResource.cs | 2 +- .../AzureKubernetesInfrastructure.cs | 44 ++++++++++++++----- ...ironment_BasicConfiguration.verified.bicep | 2 +- ...etesEnvironment_WithVersion.verified.bicep | 2 +- 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs index c5259cb700c..752a1a30e49 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs @@ -190,7 +190,7 @@ private string GenerateAksBicep() } // AKS cluster resource - sb.Append("resource ").Append(id).AppendLine(" 'Microsoft.ContainerService/managedClusters@2024-06-02-preview' = {"); + sb.Append("resource ").Append(id).AppendLine(" 'Microsoft.ContainerService/managedClusters@2026-01-01' = {"); sb.Append(" name: '").Append(Name).AppendLine("'"); sb.AppendLine(" location: location"); sb.AppendLine(" tags: {"); diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index d3187620061..c35baab77dd 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -60,7 +60,7 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance // Ensure a default user node pool exists for workload scheduling. // The system pool should only run system pods; application workloads // need a user pool. - var defaultUserPool = EnsureDefaultUserNodePool(environment); + var defaultUserPool = EnsureDefaultUserNodePool(environment, @event.Model); foreach (var r in @event.Model.GetComputeResources()) { @@ -148,33 +148,55 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance /// /// Ensures the AKS environment has at least one user node pool. If none exists, - /// creates a default "workload" user pool. + /// creates a default "workload" user pool and adds it to the app model. /// - private static AksNodePoolResource? EnsureDefaultUserNodePool(AzureKubernetesEnvironmentResource environment) + private static AksNodePoolResource? EnsureDefaultUserNodePool( + AzureKubernetesEnvironmentResource environment, + DistributedApplicationModel appModel) { var hasUserPool = environment.NodePools.Any(p => p.Mode is AksNodePoolMode.User); if (hasUserPool) { - // Return the first user pool as the default for unaffinitized workloads. - // Look for an existing AksNodePoolResource child that matches. + // Return the first user pool. Search the app model for the existing + // AksNodePoolResource so we use the same object identity as AddNodePool created. var firstUserConfig = environment.NodePools.First(p => p.Mode is AksNodePoolMode.User); - return FindNodePoolResource(environment, firstUserConfig.Name); + return FindNodePoolResource(appModel, environment, firstUserConfig.Name); } - // No user pool configured — create a default one. + // No user pool configured — create a default one and add it to the app model. var defaultConfig = new AksNodePoolConfig("workload", "Standard_D4s_v5", 1, 10, AksNodePoolMode.User); environment.NodePools.Add(defaultConfig); var defaultPool = new AksNodePoolResource("workload", defaultConfig, environment); + appModel.Resources.Add(defaultPool); return defaultPool; } - private static AksNodePoolResource? FindNodePoolResource(AzureKubernetesEnvironmentResource environment, string poolName) + /// + /// Finds an existing AksNodePoolResource in the app model by name, + /// or creates one if not found (for pools added via config but not via AddNodePool). + /// + private static AksNodePoolResource FindNodePoolResource( + DistributedApplicationModel appModel, + AzureKubernetesEnvironmentResource environment, + string poolName) { - return new AksNodePoolResource(poolName, - environment.NodePools.First(p => p.Name == poolName), - environment); + // Search the app model for an existing pool resource with matching name and parent + var existing = appModel.Resources + .OfType() + .FirstOrDefault(p => p.Name == poolName && p.AksParent == environment); + + if (existing is not null) + { + return existing; + } + + // Pool was added via NodePools config but not via AddNodePool — create the resource + var config = environment.NodePools.First(p => p.Name == poolName); + var pool = new AksNodePoolResource(poolName, config, environment); + appModel.Resources.Add(pool); + return pool; } /// diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep index 031ba1beb38..35fab2bbb90 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep @@ -1,7 +1,7 @@ @description('The location for the resource(s) to be deployed.') param location string = resourceGroup().location -resource aks 'Microsoft.ContainerService/managedClusters@2024-06-02-preview' = { +resource aks 'Microsoft.ContainerService/managedClusters@2026-01-01' = { name: 'aks' location: location tags: { diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep index 74431231a72..f48419e490f 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep @@ -1,7 +1,7 @@ @description('The location for the resource(s) to be deployed.') param location string = resourceGroup().location -resource aks 'Microsoft.ContainerService/managedClusters@2024-06-02-preview' = { +resource aks 'Microsoft.ContainerService/managedClusters@2026-01-01' = { name: 'aks' location: location tags: { From 20ad6d6815cd93a4df984c4c6e21197924618868 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 12:14:56 +1000 Subject: [PATCH 58/78] Migrate to Azure.Provisioning.ContainerService SDK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace hand-crafted StringBuilder Bicep generation with the Azure.Provisioning.ContainerService SDK types: - ContainerServiceManagedCluster with V2025_03_01 API version - ManagedClusterAgentPoolProfile for node pools - ContainerServiceNetworkProfile for networking - ManagedClusterSecurityProfile for workload identity - ManagedClusterOidcIssuerProfile for OIDC - FederatedIdentityCredential from Azure.Provisioning.Roles - ContainerRegistryService.FromExisting for ACR references - RoleAssignment for AcrPull role Remove GetBicepTemplateString/GetBicepTemplateFile overrides from AzureKubernetesEnvironmentResource — base class handles it via the ConfigureInfrastructure callback. Also resolves r13 (hardcoded serviceCidr/dnsServiceIP) — when a subnet is configured, Azure CNI is set without hardcoded CIDRs, letting AKS use its own defaults. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Directory.Packages.props | 1 + .../Aspire.Hosting.Azure.Kubernetes.csproj | 1 + .../AzureKubernetesEnvironmentExtensions.cs | 258 ++++++++++++++++- .../AzureKubernetesEnvironmentResource.cs | 260 ------------------ ...ironment_BasicConfiguration.verified.bicep | 41 +-- ...etesEnvironment_WithVersion.verified.bicep | 43 +-- 6 files changed, 301 insertions(+), 303 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 89f63e5f2c2..8dc3cabab9f 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -45,6 +45,7 @@ + diff --git a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj index 6b8422708a7..e361ac7edb2 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj +++ b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj @@ -22,6 +22,7 @@ + diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index e10af5db7a4..3dd955cf89a 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -10,6 +10,13 @@ using Aspire.Hosting.Azure.Kubernetes; using Aspire.Hosting.Lifecycle; using Aspire.Hosting.Pipelines; +using Azure.Provisioning; +using Azure.Provisioning.Authorization; +using Azure.Provisioning.ContainerRegistry; +using Azure.Provisioning.ContainerService; +using Azure.Provisioning.Expressions; +using Azure.Provisioning.Resources; +using Azure.Provisioning.Roles; using Microsoft.Extensions.DependencyInjection; namespace Aspire.Hosting; @@ -394,13 +401,252 @@ public static IResourceBuilder WithWorkloadI return builder; } - // ConfigureAksInfrastructure is a no-op placeholder required by the - // AzureProvisioningResource base class constructor. The actual Bicep is - // generated by GetBicepTemplateString/GetBicepTemplateFile overrides - // in AzureKubernetesEnvironmentResource. private static void ConfigureAksInfrastructure(AzureResourceInfrastructure infrastructure) { - // Intentionally empty — Bicep generation is handled by the resource's - // GetBicepTemplateString override, not the provisioning infrastructure. + var aksResource = (AzureKubernetesEnvironmentResource)infrastructure.AspireResource; + + var skuTier = aksResource.SkuTier switch + { + AksSkuTier.Free => ManagedClusterSkuTier.Free, + AksSkuTier.Standard => ManagedClusterSkuTier.Standard, + AksSkuTier.Premium => ManagedClusterSkuTier.Premium, + _ => ManagedClusterSkuTier.Free + }; + + // Create the AKS managed cluster + var aks = new ContainerServiceManagedCluster(aksResource.GetBicepIdentifier(), + ContainerServiceManagedCluster.ResourceVersions.V2025_03_01) + { + ClusterIdentity = new ManagedClusterIdentity + { + ResourceIdentityType = ManagedServiceIdentityType.SystemAssigned + }, + Sku = new ManagedClusterSku + { + Name = ManagedClusterSkuName.Base, + Tier = skuTier + }, + DnsPrefix = $"{aksResource.Name}-dns", + Tags = { { "aspire-resource-name", aksResource.Name } } + }; + + if (aksResource.KubernetesVersion is not null) + { + aks.KubernetesVersion = aksResource.KubernetesVersion; + } + + // Agent pool profiles + var hasDefaultSubnet = aksResource.TryGetLastAnnotation(out var subnetAnnotation); + ProvisioningParameter? defaultSubnetParam = null; + + if (hasDefaultSubnet) + { + defaultSubnetParam = new ProvisioningParameter("subnetId", typeof(string)); + infrastructure.Add(defaultSubnetParam); + aksResource.Parameters["subnetId"] = subnetAnnotation!.SubnetId; + } + + // Per-pool subnet parameters + var poolSubnetParams = new Dictionary(); + foreach (var (poolName, poolSubnetRef) in aksResource.NodePoolSubnets) + { + var paramName = $"subnetId_{poolName}"; + var param = new ProvisioningParameter(paramName, typeof(string)); + infrastructure.Add(param); + poolSubnetParams[poolName] = param; + aksResource.Parameters[paramName] = poolSubnetRef; + } + + foreach (var pool in aksResource.NodePools) + { + var mode = pool.Mode switch + { + AksNodePoolMode.System => AgentPoolMode.System, + AksNodePoolMode.User => AgentPoolMode.User, + _ => AgentPoolMode.User + }; + + var agentPool = new ManagedClusterAgentPoolProfile + { + Name = pool.Name, + VmSize = pool.VmSize, + MinCount = pool.MinCount, + MaxCount = pool.MaxCount, + Count = pool.MinCount, + EnableAutoScaling = true, + Mode = mode, + OSType = ContainerServiceOSType.Linux, + }; + + // Per-pool subnet override, else environment default + if (poolSubnetParams.TryGetValue(pool.Name, out var poolSubnetParam)) + { + agentPool.VnetSubnetId = poolSubnetParam; + } + else if (defaultSubnetParam is not null) + { + agentPool.VnetSubnetId = defaultSubnetParam; + } + + aks.AgentPoolProfiles.Add(agentPool); + } + + // OIDC issuer + if (aksResource.OidcIssuerEnabled) + { + aks.OidcIssuerProfile = new ManagedClusterOidcIssuerProfile + { + IsEnabled = true + }; + } + + // Workload identity + if (aksResource.WorkloadIdentityEnabled) + { + aks.SecurityProfile = new ManagedClusterSecurityProfile + { + IsWorkloadIdentityEnabled = true + }; + } + + // Private cluster + if (aksResource.IsPrivateCluster) + { + aks.ApiServerAccessProfile = new ManagedClusterApiServerAccessProfile + { + EnablePrivateCluster = true + }; + } + + // Network profile + var hasSubnetConfig = hasDefaultSubnet || aksResource.NodePoolSubnets.Count > 0; + if (aksResource.NetworkProfile is not null) + { + aks.NetworkProfile = new ContainerServiceNetworkProfile + { + NetworkPlugin = aksResource.NetworkProfile.NetworkPlugin switch + { + "azure" => ContainerServiceNetworkPlugin.Azure, + "kubenet" => ContainerServiceNetworkPlugin.Kubenet, + _ => ContainerServiceNetworkPlugin.Azure + }, + ServiceCidr = aksResource.NetworkProfile.ServiceCidr, + DnsServiceIP = aksResource.NetworkProfile.DnsServiceIP + }; + if (aksResource.NetworkProfile.NetworkPolicy is not null) + { + aks.NetworkProfile.NetworkPolicy = aksResource.NetworkProfile.NetworkPolicy switch + { + "calico" => ContainerServiceNetworkPolicy.Calico, + "azure" => ContainerServiceNetworkPolicy.Azure, + _ => ContainerServiceNetworkPolicy.Calico + }; + } + } + else if (hasSubnetConfig) + { + aks.NetworkProfile = new ContainerServiceNetworkProfile + { + NetworkPlugin = ContainerServiceNetworkPlugin.Azure, + }; + } + + infrastructure.Add(aks); + + // ACR pull role assignment for kubelet identity + if (aksResource.DefaultContainerRegistry is not null || aksResource.TryGetLastAnnotation(out _)) + { + var acrNameParam = new ProvisioningParameter("acrName", typeof(string)); + infrastructure.Add(acrNameParam); + + var acr = ContainerRegistryService.FromExisting("acr"); + acr.Name = acrNameParam; + infrastructure.Add(acr); + + // AcrPull role: 7f951dda-4ed3-4680-a7ca-43fe172d538d + var acrPullRoleId = BicepFunction.GetSubscriptionResourceId( + "Microsoft.Authorization/roleDefinitions", + "7f951dda-4ed3-4680-a7ca-43fe172d538d"); + + // Access kubelet identity objectId via property path + var kubeletObjectId = new MemberExpression( + new MemberExpression( + new MemberExpression( + new MemberExpression( + new IdentifierExpression(aks.BicepIdentifier), + "properties"), + "identityProfile"), + "kubeletidentity"), + "objectId"); + + var roleAssignment = new RoleAssignment("acrPullRole") + { + Name = BicepFunction.CreateGuid(acr.Id, aks.Id, acrPullRoleId), + Scope = new IdentifierExpression(acr.BicepIdentifier), + RoleDefinitionId = acrPullRoleId, + PrincipalId = kubeletObjectId, + PrincipalType = RoleManagementPrincipalType.ServicePrincipal + }; + infrastructure.Add(roleAssignment); + } + + // Outputs + infrastructure.Add(new ProvisioningOutput("id", typeof(string)) { Value = aks.Id }); + infrastructure.Add(new ProvisioningOutput("name", typeof(string)) { Value = aks.Name }); + + // OIDC issuer URL and kubelet identity require property path expressions + var aksId = new IdentifierExpression(aks.BicepIdentifier); + infrastructure.Add(new ProvisioningOutput("clusterFqdn", typeof(string)) + { + Value = new MemberExpression(new MemberExpression(aksId, "properties"), "fqdn") + }); + infrastructure.Add(new ProvisioningOutput("oidcIssuerUrl", typeof(string)) + { + Value = new MemberExpression( + new MemberExpression(new MemberExpression(aksId, "properties"), "oidcIssuerProfile"), + "issuerURL") + }); + infrastructure.Add(new ProvisioningOutput("kubeletIdentityObjectId", typeof(string)) + { + Value = new MemberExpression( + new MemberExpression( + new MemberExpression(new MemberExpression(aksId, "properties"), "identityProfile"), + "kubeletidentity"), + "objectId") + }); + infrastructure.Add(new ProvisioningOutput("nodeResourceGroup", typeof(string)) + { + Value = new MemberExpression(new MemberExpression(aksId, "properties"), "nodeResourceGroup") + }); + + // Federated identity credentials for workload identity + foreach (var (resourceName, identityResource) in aksResource.WorkloadIdentities) + { + var saName = $"{resourceName}-sa"; + var sanitizedName = Infrastructure.NormalizeBicepIdentifier(resourceName); + var identityParamName = $"identityName_{sanitizedName}"; + + var identityNameParam = new ProvisioningParameter(identityParamName, typeof(string)); + infrastructure.Add(identityNameParam); + aksResource.Parameters[identityParamName] = identityResource.PrincipalName; + + var existingIdentity = UserAssignedIdentity.FromExisting($"identity_{sanitizedName}"); + existingIdentity.Name = identityNameParam; + infrastructure.Add(existingIdentity); + + var fedCred = new FederatedIdentityCredential($"fedcred_{sanitizedName}") + { + Parent = existingIdentity, + Name = $"{resourceName}-fedcred", + IssuerUri = new MemberExpression( + new MemberExpression( + new MemberExpression(new IdentifierExpression(aks.BicepIdentifier), "properties"), + "oidcIssuerProfile"), + "issuerURL"), + Subject = $"system:serviceaccount:default:{saName}", + Audiences = { "api://AzureADTokenExchange" } + }; + infrastructure.Add(fedCred); + } } } diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs index 752a1a30e49..1933caec5cf 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs @@ -3,9 +3,6 @@ #pragma warning disable ASPIREAZURE003 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. -using System.Globalization; -using System.Text; -using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Azure.Kubernetes; using Aspire.Hosting.Kubernetes; @@ -123,261 +120,4 @@ public class AzureKubernetesEnvironmentResource( /// Gets or sets the default container registry auto-created for this AKS environment. /// internal AzureContainerRegistryResource? DefaultContainerRegistry { get; set; } - - /// - public override string GetBicepTemplateString() - { - return GenerateAksBicep(); - } - - /// - public override BicepTemplateFile GetBicepTemplateFile(string? directory = null, bool deleteTemporaryFileOnDispose = true) - { - var bicep = GenerateAksBicep(); - var dir = directory ?? Directory.CreateTempSubdirectory("aspire-aks").FullName; - var filePath = Path.Combine(dir, Name + ".module.bicep"); - File.WriteAllText(filePath, bicep); - return new BicepTemplateFile(filePath, directory is null && deleteTemporaryFileOnDispose); - } - - private string GenerateAksBicep() - { - var sb = new StringBuilder(); - var id = this.GetBicepIdentifier(); - var skuTier = SkuTier switch - { - AksSkuTier.Free => "Free", - AksSkuTier.Standard => "Standard", - AksSkuTier.Premium => "Premium", - _ => "Free" - }; - - sb.AppendLine("@description('The location for the resource(s) to be deployed.')"); - sb.AppendLine("param location string = resourceGroup().location"); - sb.AppendLine(); - - // ACR parameter for role assignment - if (DefaultContainerRegistry is not null || this.TryGetLastAnnotation(out _)) - { - sb.AppendLine("@description('The name of the Azure Container Registry for AcrPull role assignment.')"); - sb.AppendLine("param acrName string"); - sb.AppendLine(); - } - - // Subnet parameters for VNet integration - // Environment-level subnet (default for all pools) - var hasDefaultSubnet = this.TryGetLastAnnotation(out var subnetAnnotation); - if (hasDefaultSubnet) - { - Parameters["subnetId"] = subnetAnnotation!.SubnetId; - } - - if (hasDefaultSubnet) - { - sb.AppendLine("@description('The default subnet ID for AKS node pool VNet integration.')"); - sb.AppendLine("param subnetId string"); - sb.AppendLine(); - } - - // Per-pool subnet overrides - foreach (var (poolName, poolSubnetRef) in NodePoolSubnets) - { - var paramName = $"subnetId_{poolName}"; - Parameters[paramName] = poolSubnetRef; - sb.Append("@description('The subnet ID for the ").Append(poolName).AppendLine(" node pool.')"); - sb.Append("param ").Append(paramName).AppendLine(" string"); - sb.AppendLine(); - } - - // AKS cluster resource - sb.Append("resource ").Append(id).AppendLine(" 'Microsoft.ContainerService/managedClusters@2026-01-01' = {"); - sb.Append(" name: '").Append(Name).AppendLine("'"); - sb.AppendLine(" location: location"); - sb.AppendLine(" tags: {"); - sb.Append(" 'aspire-resource-name': '").Append(Name).AppendLine("'"); - sb.AppendLine(" }"); - sb.AppendLine(" identity: {"); - sb.AppendLine(" type: 'SystemAssigned'"); - sb.AppendLine(" }"); - sb.AppendLine(" sku: {"); - sb.AppendLine(" name: 'Base'"); - sb.Append(" tier: '").Append(skuTier).AppendLine("'"); - sb.AppendLine(" }"); - sb.AppendLine(" properties: {"); - - if (KubernetesVersion is not null) - { - sb.Append(" kubernetesVersion: '").Append(KubernetesVersion).AppendLine("'"); - } - - sb.Append(" dnsPrefix: '").Append(Name).AppendLine("-dns'"); - - // Agent pool profiles - sb.AppendLine(" agentPoolProfiles: ["); - foreach (var pool in NodePools) - { - var mode = pool.Mode switch - { - AksNodePoolMode.System => "System", - AksNodePoolMode.User => "User", - _ => "User" - }; - sb.AppendLine(" {"); - sb.Append(" name: '").Append(pool.Name).AppendLine("'"); - sb.Append(" vmSize: '").Append(pool.VmSize).AppendLine("'"); - sb.Append(" minCount: ").AppendLine(pool.MinCount.ToString(CultureInfo.InvariantCulture)); - sb.Append(" maxCount: ").AppendLine(pool.MaxCount.ToString(CultureInfo.InvariantCulture)); - sb.Append(" count: ").AppendLine(pool.MinCount.ToString(CultureInfo.InvariantCulture)); - sb.AppendLine(" enableAutoScaling: true"); - sb.Append(" mode: '").Append(mode).AppendLine("'"); - sb.AppendLine(" osType: 'Linux'"); - // Use per-pool subnet if available, otherwise fall back to environment default - if (NodePoolSubnets.ContainsKey(pool.Name)) - { - sb.Append(" vnetSubnetID: subnetId_").AppendLine(pool.Name); - } - else if (hasDefaultSubnet) - { - sb.AppendLine(" vnetSubnetID: subnetId"); - } - sb.AppendLine(" }"); - } - sb.AppendLine(" ]"); - - // OIDC issuer - if (OidcIssuerEnabled) - { - sb.AppendLine(" oidcIssuerProfile: {"); - sb.AppendLine(" enabled: true"); - sb.AppendLine(" }"); - } - - // Workload identity - if (WorkloadIdentityEnabled) - { - sb.AppendLine(" securityProfile: {"); - sb.AppendLine(" workloadIdentity: {"); - sb.AppendLine(" enabled: true"); - sb.AppendLine(" }"); - sb.AppendLine(" }"); - } - - // Private cluster - if (IsPrivateCluster) - { - sb.AppendLine(" apiServerAccessProfile: {"); - sb.AppendLine(" enablePrivateCluster: true"); - sb.AppendLine(" }"); - } - - // Network profile - if (NetworkProfile is not null) - { - sb.AppendLine(" networkProfile: {"); - sb.Append(" networkPlugin: '").Append(NetworkProfile.NetworkPlugin).AppendLine("'"); - if (NetworkProfile.NetworkPolicy is not null) - { - sb.Append(" networkPolicy: '").Append(NetworkProfile.NetworkPolicy).AppendLine("'"); - } - sb.Append(" serviceCidr: '").Append(NetworkProfile.ServiceCidr).AppendLine("'"); - sb.Append(" dnsServiceIP: '").Append(NetworkProfile.DnsServiceIP).AppendLine("'"); - sb.AppendLine(" }"); - } - else if (hasDefaultSubnet || NodePoolSubnets.Count > 0) - { - // Default Azure CNI network profile when a subnet is delegated - sb.AppendLine(" networkProfile: {"); - sb.AppendLine(" networkPlugin: 'azure'"); - sb.AppendLine(" serviceCidr: '10.0.0.0/16'"); - sb.AppendLine(" dnsServiceIP: '10.0.0.10'"); - sb.AppendLine(" }"); - } - - sb.AppendLine(" }"); - sb.AppendLine("}"); - sb.AppendLine(); - - // ACR pull role assignment for kubelet identity - if (DefaultContainerRegistry is not null || this.TryGetLastAnnotation(out _)) - { - sb.AppendLine("// Reference the existing ACR to grant pull access to AKS"); - sb.AppendLine("resource acr 'Microsoft.ContainerRegistry/registries@2025-04-01' existing = {"); - sb.AppendLine(" name: acrName"); - sb.AppendLine("}"); - sb.AppendLine(); - sb.AppendLine("// AcrPull role assignment for the AKS kubelet managed identity"); - sb.AppendLine("resource acrPullRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = {"); - sb.Append(" name: guid(acr.id, ").Append(id).AppendLine(".id, subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d'))"); - sb.AppendLine(" scope: acr"); - sb.AppendLine(" properties: {"); - sb.Append(" principalId: ").Append(id).AppendLine(".properties.identityProfile.kubeletidentity.objectId"); - sb.AppendLine(" roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d')"); - sb.AppendLine(" principalType: 'ServicePrincipal'"); - sb.AppendLine(" }"); - sb.AppendLine("}"); - sb.AppendLine(); - } - - // Outputs - sb.Append("output id string = ").Append(id).AppendLine(".id"); - sb.Append("output name string = ").Append(id).AppendLine(".name"); - sb.Append("output clusterFqdn string = ").Append(id).AppendLine(".properties.fqdn"); - sb.Append("output oidcIssuerUrl string = ").Append(id).AppendLine(".properties.oidcIssuerProfile.issuerURL"); - sb.Append("output kubeletIdentityObjectId string = ").Append(id).AppendLine(".properties.identityProfile.kubeletidentity.objectId"); - sb.Append("output nodeResourceGroup string = ").Append(id).AppendLine(".properties.nodeResourceGroup"); - - // Federated identity credentials for workload identity - foreach (var (resourceName, identityResource) in WorkloadIdentities) - { - var saName = $"{resourceName}-sa"; - var sanitizedName = SanitizeBicepIdentifier(resourceName); - var fedCredId = $"fedcred_{sanitizedName}"; - var identityParamName = $"identityName_{sanitizedName}"; - - // Add identity name as parameter — will be resolved from the identity resource - Parameters[identityParamName] = identityResource.PrincipalName; - - sb.AppendLine(); - sb.Append("@description('The name of the managed identity for ").Append(resourceName).AppendLine(".')"); - sb.Append("param ").Append(identityParamName).AppendLine(" string"); - sb.AppendLine(); - - // Reference existing identity - sb.Append("resource identity_").Append(sanitizedName); - sb.AppendLine(" 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = {"); - sb.Append(" name: ").AppendLine(identityParamName); - sb.AppendLine("}"); - sb.AppendLine(); - - // Federated credential - sb.Append("resource ").Append(fedCredId); - sb.AppendLine(" 'Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials@2023-01-31' = {"); - sb.Append(" parent: identity_").AppendLine(sanitizedName); - sb.Append(" name: '").Append(resourceName).AppendLine("-fedcred'"); - sb.AppendLine(" properties: {"); - sb.Append(" issuer: ").Append(id).AppendLine(".properties.oidcIssuerProfile.issuerURL"); - sb.Append(" subject: 'system:serviceaccount:default:").Append(saName).AppendLine("'"); - sb.AppendLine(" audiences: ["); - sb.AppendLine(" 'api://AzureADTokenExchange'"); - sb.AppendLine(" ]"); - sb.AppendLine(" }"); - sb.AppendLine("}"); - } - - return sb.ToString(); - } - - /// - /// Sanitizes a name for use as a Bicep identifier (alphanumeric + underscore only). - /// - private static string SanitizeBicepIdentifier(string name) - { - var sb = new StringBuilder(name.Length); - foreach (var c in name) - { - sb.Append(char.IsLetterOrDigit(c) ? c : '_'); - } - - return sb.ToString(); - } } diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep index 35fab2bbb90..290083a8275 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep @@ -1,33 +1,23 @@ -@description('The location for the resource(s) to be deployed.') +@description('The location for the resource(s) to be deployed.') param location string = resourceGroup().location -resource aks 'Microsoft.ContainerService/managedClusters@2026-01-01' = { - name: 'aks' +resource aks 'Microsoft.ContainerService/managedClusters@2025-03-01' = { + name: take('aks-${uniqueString(resourceGroup().id)}', 63) location: location - tags: { - 'aspire-resource-name': 'aks' - } - identity: { - type: 'SystemAssigned' - } - sku: { - name: 'Base' - tier: 'Free' - } properties: { - dnsPrefix: 'aks-dns' agentPoolProfiles: [ { name: 'system' + count: 1 vmSize: 'Standard_D4s_v5' - minCount: 1 + osType: 'Linux' maxCount: 3 - count: 1 + minCount: 1 enableAutoScaling: true mode: 'System' - osType: 'Linux' } ] + dnsPrefix: 'aks-dns' oidcIssuerProfile: { enabled: true } @@ -37,11 +27,26 @@ resource aks 'Microsoft.ContainerService/managedClusters@2026-01-01' = { } } } + identity: { + type: 'SystemAssigned' + } + sku: { + name: 'Base' + tier: 'Free' + } + tags: { + 'aspire-resource-name': 'aks' + } } output id string = aks.id + output name string = aks.name + output clusterFqdn string = aks.properties.fqdn + output oidcIssuerUrl string = aks.properties.oidcIssuerProfile.issuerURL + output kubeletIdentityObjectId string = aks.properties.identityProfile.kubeletidentity.objectId -output nodeResourceGroup string = aks.properties.nodeResourceGroup + +output nodeResourceGroup string = aks.properties.nodeResourceGroup \ No newline at end of file diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep index f48419e490f..3c825e6ff07 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep @@ -1,34 +1,24 @@ -@description('The location for the resource(s) to be deployed.') +@description('The location for the resource(s) to be deployed.') param location string = resourceGroup().location -resource aks 'Microsoft.ContainerService/managedClusters@2026-01-01' = { - name: 'aks' +resource aks 'Microsoft.ContainerService/managedClusters@2025-03-01' = { + name: take('aks-${uniqueString(resourceGroup().id)}', 63) location: location - tags: { - 'aspire-resource-name': 'aks' - } - identity: { - type: 'SystemAssigned' - } - sku: { - name: 'Base' - tier: 'Free' - } properties: { - kubernetesVersion: '1.30' - dnsPrefix: 'aks-dns' agentPoolProfiles: [ { name: 'system' + count: 1 vmSize: 'Standard_D4s_v5' - minCount: 1 + osType: 'Linux' maxCount: 3 - count: 1 + minCount: 1 enableAutoScaling: true mode: 'System' - osType: 'Linux' } ] + dnsPrefix: 'aks-dns' + kubernetesVersion: '1.30' oidcIssuerProfile: { enabled: true } @@ -38,11 +28,26 @@ resource aks 'Microsoft.ContainerService/managedClusters@2026-01-01' = { } } } + identity: { + type: 'SystemAssigned' + } + sku: { + name: 'Base' + tier: 'Free' + } + tags: { + 'aspire-resource-name': 'aks' + } } output id string = aks.id + output name string = aks.name + output clusterFqdn string = aks.properties.fqdn + output oidcIssuerUrl string = aks.properties.oidcIssuerProfile.issuerURL + output kubeletIdentityObjectId string = aks.properties.identityProfile.kubeletidentity.objectId -output nodeResourceGroup string = aks.properties.nodeResourceGroup + +output nodeResourceGroup string = aks.properties.nodeResourceGroup \ No newline at end of file From 312b317abcde4234590090f91b7c2ca24351f37e Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 12:40:28 +1000 Subject: [PATCH 59/78] Fix cluster name resolution after Azure.Provisioning migration The Azure.Provisioning SDK generates cluster names with a unique suffix: take('aks-\', 63) Previously we used environment.Name directly ('aks') which didn't match the actual provisioned name, causing 'az resource list' to return empty and the get-credentials step to fail. Fix: use NameOutputReference.GetValueAsync() to get the actual provisioned name from the Bicep output. This runs after provisioning completes, so the output value is available. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesInfrastructure.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index c35baab77dd..8680fade022 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -245,7 +245,11 @@ private static async Task GetAksCredentialsAsync( { try { - var clusterName = environment.Name; + // Get the actual provisioned cluster name from the Bicep output. + // The Azure.Provisioning SDK may add a unique suffix to the name + // (e.g., take('aks-${uniqueString(resourceGroup().id)}', 63)). + var clusterName = await environment.NameOutputReference.GetValueAsync(context.CancellationToken).ConfigureAwait(false) + ?? environment.Name; var azPath = FindAzCli(); var resourceGroup = await GetResourceGroupAsync(azPath, clusterName, context) From 32570529368a042e5758670c2afd6c2597b9b8c3 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 13:41:38 +1000 Subject: [PATCH 60/78] Fix code review findings - Use ToHelmChartName() for Helm chart name sanitization instead of manual ToLowerInvariant/Replace (handles underscores, dots, etc.) - Exclude default 'workload' node pool from manifest via ManifestPublishingCallbackAnnotation.Ignore (consistent with AddNodePool which calls ExcludeFromManifest) - Fix workflow body to reference AksNodeVmSizes.Generated.cs (was stale AzureVmSizes.Generated.cs) - Add WithComputeEnvironment(aks) to README usage example Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/update-azure-vm-sizes.yml | 2 +- .../AzureKubernetesEnvironmentExtensions.cs | 3 ++- .../AzureKubernetesInfrastructure.cs | 1 + src/Aspire.Hosting.Azure.Kubernetes/README.md | 3 ++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update-azure-vm-sizes.yml b/.github/workflows/update-azure-vm-sizes.yml index 905e60e8c30..a3a163c90a7 100644 --- a/.github/workflows/update-azure-vm-sizes.yml +++ b/.github/workflows/update-azure-vm-sizes.yml @@ -45,4 +45,4 @@ jobs: area-integrations area-engineering-systems title: "[Automated] Update Azure VM Sizes" - body: "Auto-generated update of Azure VM size descriptors (AzureVmSizes.Generated.cs)." + body: "Auto-generated update of Azure VM size descriptors (AksNodeVmSizes.Generated.cs)." diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 3dd955cf89a..a81f81084a1 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -8,6 +8,7 @@ using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Azure; using Aspire.Hosting.Azure.Kubernetes; +using Aspire.Hosting.Kubernetes.Extensions; using Aspire.Hosting.Lifecycle; using Aspire.Hosting.Pipelines; using Azure.Provisioning; @@ -69,7 +70,7 @@ public static IResourceBuilder AddAzureKuber // Scope the Helm chart name to this AKS environment to avoid // conflicts when multiple environments deploy to the same cluster // or when re-deploying with different environment names. - k8sEnvBuilder.Resource.HelmChartName = $"{builder.Environment.ApplicationName}-{name}".ToLowerInvariant().Replace(' ', '-'); + k8sEnvBuilder.Resource.HelmChartName = $"{builder.Environment.ApplicationName}-{name}".ToHelmChartName(); // Create the unified AKS environment resource var resource = new AzureKubernetesEnvironmentResource(name, ConfigureAksInfrastructure); diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 8680fade022..2fcdf0ef5bb 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -169,6 +169,7 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance environment.NodePools.Add(defaultConfig); var defaultPool = new AksNodePoolResource("workload", defaultConfig, environment); + defaultPool.Annotations.Add(ManifestPublishingCallbackAnnotation.Ignore); appModel.Resources.Add(defaultPool); return defaultPool; } diff --git a/src/Aspire.Hosting.Azure.Kubernetes/README.md b/src/Aspire.Hosting.Azure.Kubernetes/README.md index 649ba91e442..c31a66af4d0 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/README.md +++ b/src/Aspire.Hosting.Azure.Kubernetes/README.md @@ -23,7 +23,8 @@ Then, in the _AppHost.cs_ file of `AppHost`, add an AKS environment and deploy s ```csharp var aks = builder.AddAzureKubernetesEnvironment("aks"); -var myService = builder.AddProject(); +var myService = builder.AddProject() + .WithComputeEnvironment(aks); ``` ## Additional documentation From 2af987232fce556537491b048eeff70321fbf3ff Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 13:46:25 +1000 Subject: [PATCH 61/78] Fix markdownlint errors in AKS spec Add 'text' language to fenced code blocks containing ASCII art diagrams (MD040 fenced-code-language). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/specs/aks-support.md | 7 +++---- .../Deployment/HelmDeploymentEngine.cs | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/specs/aks-support.md b/docs/specs/aks-support.md index 38e56cabf4a..02c56f20490 100644 --- a/docs/specs/aks-support.md +++ b/docs/specs/aks-support.md @@ -60,7 +60,7 @@ The goal is to create a first-class AKS experience in Aspire that supports: This package provides a unified `AddAzureKubernetesEnvironment()` entry point that internally invokes `AddKubernetesEnvironment()` (from the generic K8s package) and layers on AKS-specific Azure provisioning. This mirrors the established pattern of `AddAzureContainerAppEnvironment()` which internally sets up the Container Apps infrastructure. -``` +```text Aspire.Hosting.Azure.Kubernetes ├── depends on: Aspire.Hosting.Kubernetes ├── depends on: Aspire.Hosting.Azure @@ -80,7 +80,7 @@ Just as `AddAzureContainerAppEnvironment("aca")` creates a single resource that ### Integration Points -``` +```text ┌─────────────────────────────────────────────────────────────┐ │ User's AppHost │ │ │ @@ -443,7 +443,7 @@ var sql = builder.AddAzureSqlServer("sql") Since `AzureKubernetesEnvironmentResource` unifies both Azure provisioning and K8s deployment, the pipeline is a superset of both: -``` +```text [Azure Provisioning Phase] [Kubernetes Deployment Phase] 1. Generate Bicep (AKS + ACR + 4. Publish Helm chart identity + fedcreds) 5. Get kubeconfig from AKS (az aks get-credentials) @@ -618,4 +618,3 @@ var aks = builder.AddAzureKubernetesService("aks") - 31 AKS unit tests passing (extensions + infrastructure) - 88 K8s base tests passing - Manual E2E validation against live Azure clusters - diff --git a/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs b/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs index 25cba322df8..08a35388b58 100644 --- a/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs +++ b/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs @@ -180,7 +180,7 @@ await ctx.ReportingStep.CompleteAsync( // Use saved state for the confirmation message (more accurate than recomputing) var @namespace = savedNamespace ?? "default"; await ConfirmDestroyAsync(ctx, $"Uninstall Helm release '{savedReleaseName}' from namespace '{@namespace}'? This action cannot be undone.").ConfigureAwait(false); - await HelmUninstallAsync(ctx, savedReleaseName, @namespace).ConfigureAwait(false); + await HelmUninstallAsync(ctx, environment, savedReleaseName, @namespace).ConfigureAwait(false); ctx.Summary.Add("🗑️ Helm Release", savedReleaseName); ctx.Summary.Add("☸️ Namespace", @namespace); @@ -575,10 +575,10 @@ private static async Task HelmUninstallAsync(PipelineStepContext context, Kubern { var @namespace = await ResolveNamespaceAsync(context, environment).ConfigureAwait(false); var releaseName = await ResolveReleaseNameAsync(context, environment).ConfigureAwait(false); - await HelmUninstallAsync(context, releaseName, @namespace).ConfigureAwait(false); + await HelmUninstallAsync(context, environment, releaseName, @namespace).ConfigureAwait(false); } - private static async Task HelmUninstallAsync(PipelineStepContext context, string releaseName, string @namespace) + private static async Task HelmUninstallAsync(PipelineStepContext context, KubernetesEnvironmentResource environment, string releaseName, string @namespace) { var uninstallTask = await context.ReportingStep.CreateTaskAsync( new MarkdownString($"Uninstalling Helm release **{releaseName}** from namespace **{@namespace}**"), From 26ab947f93fb9806c81d2d480cf267cbabf3f23d Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 14:39:06 +1000 Subject: [PATCH 62/78] Address James's PR feedback - AddNodePool: validate minCount >= 0, maxCount >= 0, minCount <= maxCount - WithSubnet: use ResourceAnnotationMutationBehavior.Replace to prevent silent overwrites when called multiple times - Federated credential: resolve namespace from KubernetesNamespaceAnnotation instead of hardcoding 'default'. Falls back to 'default' when not set or when namespace is parameter-based (Azure AD needs fixed subject) - Remove AllowUnsafeBlocks: use IProcessRunner from DI (via IVT from Aspire.Hosting.Azure) instead of linking ProcessUtil.cs directly - GenVmSizes.cs: read stdout and stderr concurrently to avoid deadlock - Snapshot files: add trailing newlines Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Aspire.Hosting.Azure.Kubernetes.csproj | 5 ---- .../AzureKubernetesEnvironmentExtensions.cs | 23 +++++++++++++++++-- .../AzureKubernetesInfrastructure.cs | 14 +++++++---- .../tools/GenVmSizes.cs | 14 ++++++++++- .../Aspire.Hosting.Azure.csproj | 1 + ...ironment_BasicConfiguration.verified.bicep | 2 +- ...etesEnvironment_WithVersion.verified.bicep | 2 +- 7 files changed, 46 insertions(+), 15 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj index e361ac7edb2..1b8c765745f 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj +++ b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj @@ -3,7 +3,6 @@ $(DefaultTargetFramework) true - true true false aspire integration hosting azure kubernetes aks @@ -14,10 +13,6 @@ - - - - diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index a81f81084a1..4a355273112 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -8,6 +8,7 @@ using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Azure; using Aspire.Hosting.Azure.Kubernetes; +using Aspire.Hosting.Kubernetes; using Aspire.Hosting.Kubernetes.Extensions; using Aspire.Hosting.Lifecycle; using Aspire.Hosting.Pipelines; @@ -202,6 +203,9 @@ public static IResourceBuilder AddNodePool( ArgumentNullException.ThrowIfNull(builder); ArgumentException.ThrowIfNullOrEmpty(name); ArgumentException.ThrowIfNullOrEmpty(vmSize); + ArgumentOutOfRangeException.ThrowIfNegative(minCount); + ArgumentOutOfRangeException.ThrowIfNegative(maxCount); + ArgumentOutOfRangeException.ThrowIfGreaterThan(minCount, maxCount); var config = new AksNodePoolConfig(name, vmSize, minCount, maxCount, AksNodePoolMode.User); builder.Resource.NodePools.Add(config); @@ -256,7 +260,7 @@ public static IResourceBuilder WithSubnet( ArgumentNullException.ThrowIfNull(builder); ArgumentNullException.ThrowIfNull(subnet); - builder.WithAnnotation(new AksSubnetAnnotation(subnet.Resource.Id)); + builder.WithAnnotation(new AksSubnetAnnotation(subnet.Resource.Id), ResourceAnnotationMutationBehavior.Replace); return builder; } @@ -621,6 +625,21 @@ private static void ConfigureAksInfrastructure(AzureResourceInfrastructure infra }); // Federated identity credentials for workload identity + // Resolve the K8s namespace for the service account subject. + // If not explicitly configured, defaults to "default". + var k8sNamespace = "default"; + if (aksResource.KubernetesEnvironment.TryGetLastAnnotation(out var nsAnnotation)) + { + // Use the namespace expression's format string as the literal value. + // Dynamic (parameter-based) namespaces are not supported for federated + // credentials since Azure AD needs a fixed subject at provision time. + var nsFormat = nsAnnotation.Namespace.Format; + if (!string.IsNullOrEmpty(nsFormat) && !nsFormat.Contains('{')) + { + k8sNamespace = nsFormat; + } + } + foreach (var (resourceName, identityResource) in aksResource.WorkloadIdentities) { var saName = $"{resourceName}-sa"; @@ -644,7 +663,7 @@ private static void ConfigureAksInfrastructure(AzureResourceInfrastructure infra new MemberExpression(new IdentifierExpression(aks.BicepIdentifier), "properties"), "oidcIssuerProfile"), "issuerURL"), - Subject = $"system:serviceaccount:default:{saName}", + Subject = $"system:serviceaccount:{k8sNamespace}:{saName}", Audiences = { "api://AzureADTokenExchange" } }; infrastructure.Add(fedCred); diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 2fcdf0ef5bb..3943c2638a9 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -270,7 +270,8 @@ private static async Task GetAksCredentialsAsync( var result = await RunAzCommandAsync( azPath, $"aks get-credentials --resource-group \"{resourceGroup}\" --name \"{clusterName}\" --file -", - context.Logger).ConfigureAwait(false); + context.Logger, + context.Services).ConfigureAwait(false); if (result.ExitCode != 0) { @@ -356,7 +357,8 @@ private static async Task GetResourceGroupAsync( var result = await RunAzCommandAsync( azPath, $"resource list --resource-type Microsoft.ContainerService/managedClusters --name \"{clusterName}\" --query [0].resourceGroup -o tsv", - context.Logger).ConfigureAwait(false); + context.Logger, + context.Services).ConfigureAwait(false); if (result.ExitCode != 0) { @@ -377,13 +379,14 @@ private static async Task GetResourceGroupAsync( } /// - /// Runs an az CLI command using the shared ProcessSpec/ProcessUtil infrastructure. + /// Runs an az CLI command using IProcessRunner from DI. /// Returns the captured stdout, stderr, and exit code. /// private static async Task RunAzCommandAsync( string azPath, string arguments, - ILogger logger) + ILogger logger, + IServiceProvider services) { var stdout = new StringBuilder(); var stderr = new StringBuilder(); @@ -398,7 +401,8 @@ private static async Task RunAzCommandAsync( logger.LogDebug("Running: {AzPath} {Arguments}", azPath, arguments); - var (task, disposable) = ProcessUtil.Run(spec); + var processRunner = services.GetRequiredService(); + var (task, disposable) = processRunner.Run(spec); try { diff --git a/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs b/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs index 6652411718c..fb45d20a0c3 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/tools/GenVmSizes.cs @@ -108,9 +108,21 @@ return null; } - var output = await process.StandardOutput.ReadToEndAsync().ConfigureAwait(false); + // Read stdout and stderr concurrently to avoid deadlock when + // the process fills the stderr pipe buffer. + var stdoutTask = process.StandardOutput.ReadToEndAsync(); + var stderrTask = process.StandardError.ReadToEndAsync(); + await process.WaitForExitAsync().ConfigureAwait(false); + var output = await stdoutTask.ConfigureAwait(false); + var stderr = await stderrTask.ConfigureAwait(false); + + if (process.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) + { + Console.Error.WriteLine($"az {arguments}: {stderr.Trim()}"); + } + return process.ExitCode == 0 ? output : null; } diff --git a/src/Aspire.Hosting.Azure/Aspire.Hosting.Azure.csproj b/src/Aspire.Hosting.Azure/Aspire.Hosting.Azure.csproj index 87a93b17cf2..c0278dc6228 100644 --- a/src/Aspire.Hosting.Azure/Aspire.Hosting.Azure.csproj +++ b/src/Aspire.Hosting.Azure/Aspire.Hosting.Azure.csproj @@ -41,6 +41,7 @@ + diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep index 290083a8275..b2f49efab1d 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep @@ -49,4 +49,4 @@ output oidcIssuerUrl string = aks.properties.oidcIssuerProfile.issuerURL output kubeletIdentityObjectId string = aks.properties.identityProfile.kubeletidentity.objectId -output nodeResourceGroup string = aks.properties.nodeResourceGroup \ No newline at end of file +output nodeResourceGroup string = aks.properties.nodeResourceGroup diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep index 3c825e6ff07..90173697580 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep @@ -50,4 +50,4 @@ output oidcIssuerUrl string = aks.properties.oidcIssuerProfile.issuerURL output kubeletIdentityObjectId string = aks.properties.identityProfile.kubeletidentity.objectId -output nodeResourceGroup string = aks.properties.nodeResourceGroup \ No newline at end of file +output nodeResourceGroup string = aks.properties.nodeResourceGroup From 2a474d6a9c5b33eacfb40401084bd57988ef832a Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 15:04:25 +1000 Subject: [PATCH 63/78] Remove AsPrivateCluster and WithSkuTier public APIs These are infrastructure configuration concerns better handled via ConfigureInfrastructure(...) customization. The internal properties and Bicep generation logic remain for users who customize directly. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 32 ------------------- ...ureKubernetesEnvironmentExtensionsTests.cs | 22 ------------- 2 files changed, 54 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 4a355273112..bf1a15ac934 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -153,23 +153,6 @@ public static IResourceBuilder WithVersion( return builder; } - /// - /// Configures the SKU tier for the AKS cluster. - /// - /// The resource builder. - /// The SKU tier. - /// A reference to the for chaining. - [AspireExport(Description = "Sets the SKU tier for the AKS cluster")] - public static IResourceBuilder WithSkuTier( - this IResourceBuilder builder, - AksSkuTier tier) - { - ArgumentNullException.ThrowIfNull(builder); - - builder.Resource.SkuTier = tier; - return builder; - } - /// /// Adds a node pool to the AKS cluster. /// @@ -221,21 +204,6 @@ public static IResourceBuilder AddNodePool( .ExcludeFromManifest(); } - /// - /// Configures the AKS cluster as a private cluster with a private API server endpoint. - /// - /// The resource builder. - /// A reference to the for chaining. - [AspireExport(Description = "Configures the AKS cluster as a private cluster")] - public static IResourceBuilder AsPrivateCluster( - this IResourceBuilder builder) - { - ArgumentNullException.ThrowIfNull(builder); - - builder.Resource.IsPrivateCluster = true; - return builder; - } - /// /// Configures the AKS cluster to use a VNet subnet for node pool networking. /// Unlike , this does NOT diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs index b45cc633817..09fc1230e3c 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs @@ -44,17 +44,6 @@ public async Task AddAzureKubernetesEnvironment_WithVersion() await Verify(manifest.BicepText, extension: "bicep"); } - [Fact] - public void AddAzureKubernetesEnvironment_WithSkuTier() - { - using var builder = TestDistributedApplicationBuilder.Create(); - - var aks = builder.AddAzureKubernetesEnvironment("aks") - .WithSkuTier(AksSkuTier.Standard); - - Assert.Equal(AksSkuTier.Standard, aks.Resource.SkuTier); - } - [Fact] public void AddNodePool_ReturnsNodePoolResource() { @@ -75,17 +64,6 @@ public void AddNodePool_ReturnsNodePoolResource() Assert.Same(aks.Resource, gpuPool.Resource.AksParent); } - [Fact] - public void AddAzureKubernetesEnvironment_AsPrivateCluster() - { - using var builder = TestDistributedApplicationBuilder.Create(); - - var aks = builder.AddAzureKubernetesEnvironment("aks") - .AsPrivateCluster(); - - Assert.True(aks.Resource.IsPrivateCluster); - } - [Fact] public void AddAzureKubernetesEnvironment_WithContainerInsights() { From 96b05b063b0193be70eb60349b64971a42e33c49 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 15:12:05 +1000 Subject: [PATCH 64/78] Add T1.1 AKS E2E deployment test with Azure Provisioning End-to-end test that deploys the Aspire starter template to AKS using the full aspire deploy pipeline (Bicep provisioning + container build + ACR push + Helm deploy). Follows the ACA deployment test pattern. Test flow: 1. Create starter project via aspire new 2. Add Aspire.Hosting.Azure.Kubernetes package 3. Modify AppHost to use AddAzureKubernetesEnvironment + WithComputeEnvironment 4. aspire deploy --clear-cache (provisions AKS + ACR + deploys) 5. Verify pods running via kubectl 6. Port-forward and verify HTTP endpoints 7. aspire destroy for cleanup Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AksAzureProvisioningDeploymentTests.cs | 309 ++++++++++++++++++ 1 file changed, 309 insertions(+) create mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/AksAzureProvisioningDeploymentTests.cs diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksAzureProvisioningDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksAzureProvisioningDeploymentTests.cs new file mode 100644 index 00000000000..4489a973db2 --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksAzureProvisioningDeploymentTests.cs @@ -0,0 +1,309 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying Aspire applications to Azure Kubernetes Service (AKS) +/// using Azure.Provisioning (Bicep-based) via aspire deploy --clear-cache. +/// This test uses the ACA-style deploy pipeline rather than manual AKS/ACR/Helm setup. +/// +public sealed class AksAzureProvisioningDeploymentTests(ITestOutputHelper output) +{ + // Timeout set to 45 minutes to allow for AKS provisioning (~10-15 min) plus container build and Helm deploy. + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployStarterToAksWithAzureProvisioning() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployStarterToAksWithAzureProvisioningCore(cancellationToken); + } + + private async Task DeployStarterToAksWithAzureProvisioningCore(CancellationToken cancellationToken) + { + // Validate prerequisites + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-provisioning"); + var projectName = "AksProvisioned"; + + output.WriteLine($"Test: {nameof(DeployStarterToAksWithAzureProvisioning)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); + await auto.SourceAspireCliEnvironmentAsync(counter); + } + + // Step 3: Create starter project using aspire new + output.WriteLine("Step 3: Creating starter project..."); + await auto.AspireNewAsync(projectName, counter, useRedisCache: false); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add Aspire.Hosting.Azure.Kubernetes package + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + // In CI, aspire add shows a version selection prompt + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); + await auto.EnterAsync(); // select first version (PR build) + } + + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + + // Step 6: Modify AppHost.cs to add AKS environment with compute targeting + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); + var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); + + output.WriteLine($"Modifying AppHost.cs at: {appHostFilePath}"); + + var content = File.ReadAllText(appHostFilePath); + + // Replace the existing service definitions with AKS-targeted versions. + // The starter template has: + // var apiService = builder.AddProject("apiservice"); + // builder.AddProject("webfrontend") + // .WithExternalHttpEndpoints() + // .WithReference(apiService) + // .WaitFor(apiService); + // builder.Build().Run(); + var oldCode = $""" +var apiService = builder.AddProject("apiservice"); + +builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithReference(apiService) + .WaitFor(apiService); + +builder.Build().Run(); +"""; + + var newCode = $""" +var aks = builder.AddAzureKubernetesEnvironment("aks"); + +var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health") + .WithComputeEnvironment(aks); + +builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WithComputeEnvironment(aks); + +builder.Build().Run(); +"""; + + content = content.Replace(oldCode, newCode); + + // Add required pragma warnings to suppress experimental API warnings + if (!content.Contains("#pragma warning disable ASPIREPIPELINES001")) + { + content = "#pragma warning disable ASPIREPIPELINES001\n" + content; + } + + if (!content.Contains("#pragma warning disable ASPIREAZURE003")) + { + content = "#pragma warning disable ASPIREAZURE003\n" + content; + } + + File.WriteAllText(appHostFilePath, content); + + output.WriteLine("Modified AppHost.cs with AddAzureKubernetesEnvironment and compute targeting"); + + // Step 7: Navigate to AppHost project directory + output.WriteLine("Step 7: Navigating to AppHost directory..."); + await auto.TypeAsync($"cd {projectName}.AppHost"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Set environment variables for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 9: Deploy to AKS using aspire deploy (Bicep provisioning + container build + ACR push + Helm deploy) + output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + // Wait for pipeline to complete - AKS provisioning can take up to 30 minutes + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 10: Get AKS credentials for kubectl verification + output.WriteLine("Step 10: Getting AKS credentials..."); + // The cluster name is dynamic (Azure.Provisioning adds a suffix), so discover it + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 11: Wait for pods to be ready + output.WriteLine("Step 11: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 12: Verify pods are running + output.WriteLine("Step 12: Verifying pods are running..."); + await auto.TypeAsync("kubectl get pods -n default"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 13: Verify apiservice endpoint via port-forward + output.WriteLine("Step 13: Verifying apiservice endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 14: Verify webfrontend endpoint via port-forward + output.WriteLine("Step 14: Verifying webfrontend endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 15: Clean up port-forwards + output.WriteLine("Step 15: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 16: Destroy Azure deployment + output.WriteLine("Step 16: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 17: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"AKS provisioned deployment completed in {duration}"); + + // Report success + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployStarterToAksWithAzureProvisioning), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - Aspire app deployed to AKS via Azure Provisioning!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployStarterToAksWithAzureProvisioning), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + // Clean up the resource group we created + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + /// + /// Triggers cleanup of a specific resource group. + /// This is fire-and-forget - the hourly cleanup workflow handles any missed resources. + /// + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} From 11c7080cb25e4f3425a7a191463001e21b128ee7 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 15:32:09 +1000 Subject: [PATCH 65/78] Add Tier 1 AKS E2E deployment tests (T1.2-T1.7) T1.2: WithVersion - deploys with K8s 1.30, verifies via kubectl T1.3: NodePool - custom pool with nodeSelector verification T1.4: VNet - subnet integration with VNet IP verification T1.5: WorkloadIdentity - Azure Storage ref with WI SA/pod labels T1.6: ExplicitRegistry - bring-your-own ACR T1.7: PerPoolSubnet - different subnets per node pool All tests use aspire deploy --clear-cache (full provisioning pipeline) and follow the same pattern as T1.1. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AksExplicitRegistryDeploymentTests.cs | 303 +++++++++++++++++ .../AksNodePoolDeploymentTests.cs | 304 +++++++++++++++++ .../AksPerPoolSubnetDeploymentTests.cs | 305 +++++++++++++++++ .../AksVnetDeploymentTests.cs | 294 ++++++++++++++++ .../AksWithVersionDeploymentTests.cs | 296 ++++++++++++++++ .../AksWorkloadIdentityDeploymentTests.cs | 319 ++++++++++++++++++ 6 files changed, 1821 insertions(+) create mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/AksExplicitRegistryDeploymentTests.cs create mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs create mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs create mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/AksVnetDeploymentTests.cs create mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/AksWithVersionDeploymentTests.cs create mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/AksWorkloadIdentityDeploymentTests.cs diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksExplicitRegistryDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksExplicitRegistryDeploymentTests.cs new file mode 100644 index 00000000000..062a4bfe9ad --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksExplicitRegistryDeploymentTests.cs @@ -0,0 +1,303 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying Aspire applications to AKS with an explicit Azure Container Registry. +/// Verifies that AddAzureContainerRegistry and WithContainerRegistry correctly +/// provision a dedicated ACR and attach it to the AKS cluster. +/// +public sealed class AksExplicitRegistryDeploymentTests(ITestOutputHelper output) +{ + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployStarterToAksWithExplicitRegistry() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployStarterToAksWithExplicitRegistryCore(cancellationToken); + } + + private async Task DeployStarterToAksWithExplicitRegistryCore(CancellationToken cancellationToken) + { + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-acr"); + var projectName = "AksExplicitAcr"; + + output.WriteLine($"Test: {nameof(DeployStarterToAksWithExplicitRegistry)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); + await auto.SourceAspireCliEnvironmentAsync(counter); + } + + // Step 3: Create starter project using aspire new + output.WriteLine("Step 3: Creating starter project..."); + await auto.AspireNewAsync(projectName, counter, useRedisCache: false); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add required packages + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); + await auto.EnterAsync(); + } + + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + + output.WriteLine("Step 5b: Adding Azure Container Registry hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.ContainerRegistry"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); + await auto.EnterAsync(); + } + + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + + // Step 6: Modify AppHost.cs to add AKS environment with explicit ACR + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); + var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); + + output.WriteLine($"Modifying AppHost.cs at: {appHostFilePath}"); + + var content = File.ReadAllText(appHostFilePath); + + var oldCode = $""" +var apiService = builder.AddProject("apiservice"); + +builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithReference(apiService) + .WaitFor(apiService); + +builder.Build().Run(); +"""; + + var newCode = $""" +var acr = builder.AddAzureContainerRegistry("myacr"); +var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithContainerRegistry(acr); + +var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health") + .WithComputeEnvironment(aks); + +builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WithComputeEnvironment(aks); + +builder.Build().Run(); +"""; + + content = content.Replace(oldCode, newCode); + + if (!content.Contains("#pragma warning disable ASPIREPIPELINES001")) + { + content = "#pragma warning disable ASPIREPIPELINES001\n" + content; + } + + if (!content.Contains("#pragma warning disable ASPIREAZURE003")) + { + content = "#pragma warning disable ASPIREAZURE003\n" + content; + } + + File.WriteAllText(appHostFilePath, content); + + output.WriteLine("Modified AppHost.cs with AddAzureKubernetesEnvironment and explicit ACR"); + + // Step 7: Navigate to AppHost project directory + output.WriteLine("Step 7: Navigating to AppHost directory..."); + await auto.TypeAsync($"cd {projectName}.AppHost"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Set environment variables for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 9: Deploy to AKS using aspire deploy + output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 10: Get AKS credentials for kubectl verification + output.WriteLine("Step 10: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 11: Wait for pods to be ready + output.WriteLine("Step 11: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 12: Verify pods are running + output.WriteLine("Step 12: Verifying pods are running..."); + await auto.TypeAsync("kubectl get pods -n default"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 13: Verify apiservice endpoint via port-forward + output.WriteLine("Step 13: Verifying apiservice endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 14: Verify webfrontend endpoint via port-forward + output.WriteLine("Step 14: Verifying webfrontend endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 15: Clean up port-forwards + output.WriteLine("Step 15: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 16: Destroy Azure deployment + output.WriteLine("Step 16: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 17: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"AKS explicit ACR deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployStarterToAksWithExplicitRegistry), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - Aspire app deployed to AKS with explicit ACR!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployStarterToAksWithExplicitRegistry), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs new file mode 100644 index 00000000000..58ba97cf3f1 --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs @@ -0,0 +1,304 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying Aspire applications to AKS with a custom node pool. +/// Verifies that AddNodePool creates additional node pools and that pods are +/// scheduled to the correct pool via WithNodePool. +/// +public sealed class AksNodePoolDeploymentTests(ITestOutputHelper output) +{ + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployStarterToAksWithCustomNodePool() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployStarterToAksWithCustomNodePoolCore(cancellationToken); + } + + private async Task DeployStarterToAksWithCustomNodePoolCore(CancellationToken cancellationToken) + { + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-nodepool"); + var projectName = "AksNodePool"; + + output.WriteLine($"Test: {nameof(DeployStarterToAksWithCustomNodePool)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); + await auto.SourceAspireCliEnvironmentAsync(counter); + } + + // Step 3: Create starter project using aspire new + output.WriteLine("Step 3: Creating starter project..."); + await auto.AspireNewAsync(projectName, counter, useRedisCache: false); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add Aspire.Hosting.Azure.Kubernetes package + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); + await auto.EnterAsync(); + } + + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + + // Step 6: Modify AppHost.cs to add AKS environment with custom node pool + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); + var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); + + output.WriteLine($"Modifying AppHost.cs at: {appHostFilePath}"); + + var content = File.ReadAllText(appHostFilePath); + + var oldCode = $""" +var apiService = builder.AddProject("apiservice"); + +builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithReference(apiService) + .WaitFor(apiService); + +builder.Build().Run(); +"""; + + var newCode = $""" +var aks = builder.AddAzureKubernetesEnvironment("aks"); +var computePool = aks.AddNodePool("compute", "Standard_D4s_v5", 1, 3); + +var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health") + .WithComputeEnvironment(aks) + .WithNodePool(computePool); + +builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WithComputeEnvironment(aks); + +builder.Build().Run(); +"""; + + content = content.Replace(oldCode, newCode); + + if (!content.Contains("#pragma warning disable ASPIREPIPELINES001")) + { + content = "#pragma warning disable ASPIREPIPELINES001\n" + content; + } + + if (!content.Contains("#pragma warning disable ASPIREAZURE003")) + { + content = "#pragma warning disable ASPIREAZURE003\n" + content; + } + + File.WriteAllText(appHostFilePath, content); + + output.WriteLine("Modified AppHost.cs with AddAzureKubernetesEnvironment and custom node pool"); + + // Step 7: Navigate to AppHost project directory + output.WriteLine("Step 7: Navigating to AppHost directory..."); + await auto.TypeAsync($"cd {projectName}.AppHost"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Set environment variables for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 9: Deploy to AKS using aspire deploy + output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 10: Get AKS credentials for kubectl verification + output.WriteLine("Step 10: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 11: Wait for pods to be ready + output.WriteLine("Step 11: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 12: Verify pods are running + output.WriteLine("Step 12: Verifying pods are running..."); + await auto.TypeAsync("kubectl get pods -n default"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 13: Verify two node pools exist (system + compute) + output.WriteLine("Step 13: Verifying node pools..."); + await auto.TypeAsync($"az aks nodepool list --resource-group {resourceGroupName} --cluster-name $AKS_NAME --query '[].{{name:name, mode:mode}}' -o table"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync("compute", timeout: TimeSpan.FromSeconds(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 14: Verify apiservice pod has nodeSelector for the compute pool + output.WriteLine("Step 14: Verifying apiservice pod nodeSelector..."); + await auto.TypeAsync("kubectl get pod -l app.kubernetes.io/component=apiservice -o jsonpath='{.items[0].spec.nodeSelector}'"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 15: Verify apiservice endpoint via port-forward + output.WriteLine("Step 15: Verifying apiservice endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 16: Verify webfrontend endpoint via port-forward + output.WriteLine("Step 16: Verifying webfrontend endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 17: Clean up port-forwards + output.WriteLine("Step 17: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 18: Destroy Azure deployment + output.WriteLine("Step 18: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 19: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"AKS node pool deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployStarterToAksWithCustomNodePool), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - Aspire app deployed to AKS with custom node pool!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployStarterToAksWithCustomNodePool), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs new file mode 100644 index 00000000000..3f56f36f0fd --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs @@ -0,0 +1,305 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying Aspire applications to AKS with per-pool subnet assignments. +/// Verifies that the default system pool and a user node pool can each be assigned to +/// different subnets within the same VNet. +/// +public sealed class AksPerPoolSubnetDeploymentTests(ITestOutputHelper output) +{ + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployStarterToAksWithPerPoolSubnets() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployStarterToAksWithPerPoolSubnetsCore(cancellationToken); + } + + private async Task DeployStarterToAksWithPerPoolSubnetsCore(CancellationToken cancellationToken) + { + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-poolsubnet"); + var projectName = "AksPoolSubnet"; + + output.WriteLine($"Test: {nameof(DeployStarterToAksWithPerPoolSubnets)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); + await auto.SourceAspireCliEnvironmentAsync(counter); + } + + // Step 3: Create starter project using aspire new + output.WriteLine("Step 3: Creating starter project..."); + await auto.AspireNewAsync(projectName, counter, useRedisCache: false); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add Aspire.Hosting.Azure.Kubernetes package + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); + await auto.EnterAsync(); + } + + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + + // Step 6: Modify AppHost.cs to add AKS environment with per-pool subnets + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); + var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); + + output.WriteLine($"Modifying AppHost.cs at: {appHostFilePath}"); + + var content = File.ReadAllText(appHostFilePath); + + var oldCode = $""" +var apiService = builder.AddProject("apiservice"); + +builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithReference(apiService) + .WaitFor(apiService); + +builder.Build().Run(); +"""; + + var newCode = $""" +var vnet = builder.AddAzureVirtualNetwork("vnet", "10.1.0.0/16"); +var defaultSubnet = vnet.AddSubnet("defaultsubnet", "10.1.0.0/22"); +var auxSubnet = vnet.AddSubnet("auxsubnet", "10.1.4.0/24"); + +var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithSubnet(defaultSubnet); + +var auxPool = aks.AddNodePool("aux", "Standard_D4s_v5", 1, 3) + .WithSubnet(auxSubnet); + +var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health") + .WithComputeEnvironment(aks) + .WithNodePool(auxPool); + +builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WithComputeEnvironment(aks); + +builder.Build().Run(); +"""; + + content = content.Replace(oldCode, newCode); + + if (!content.Contains("#pragma warning disable ASPIREPIPELINES001")) + { + content = "#pragma warning disable ASPIREPIPELINES001\n" + content; + } + + if (!content.Contains("#pragma warning disable ASPIREAZURE003")) + { + content = "#pragma warning disable ASPIREAZURE003\n" + content; + } + + File.WriteAllText(appHostFilePath, content); + + output.WriteLine("Modified AppHost.cs with AddAzureKubernetesEnvironment and per-pool subnets"); + + // Step 7: Navigate to AppHost project directory + output.WriteLine("Step 7: Navigating to AppHost directory..."); + await auto.TypeAsync($"cd {projectName}.AppHost"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Set environment variables for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 9: Deploy to AKS using aspire deploy + output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 10: Get AKS credentials for kubectl verification + output.WriteLine("Step 10: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 11: Wait for pods to be ready + output.WriteLine("Step 11: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 12: Verify pods are running + output.WriteLine("Step 12: Verifying pods are running..."); + await auto.TypeAsync("kubectl get pods -n default"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 13: Verify two node pools with different subnets exist + output.WriteLine("Step 13: Verifying node pools with different subnets..."); + await auto.TypeAsync($"az aks nodepool list --resource-group {resourceGroupName} --cluster-name $AKS_NAME --query '[].{{name:name, mode:mode, vnetSubnetId:vnetSubnetId}}' -o table"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync("aux", timeout: TimeSpan.FromSeconds(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 14: Verify apiservice endpoint via port-forward + output.WriteLine("Step 14: Verifying apiservice endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 15: Verify webfrontend endpoint via port-forward + output.WriteLine("Step 15: Verifying webfrontend endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 16: Clean up port-forwards + output.WriteLine("Step 16: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 17: Destroy Azure deployment + output.WriteLine("Step 17: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 18: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"AKS per-pool subnet deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployStarterToAksWithPerPoolSubnets), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - Aspire app deployed to AKS with per-pool subnet assignments!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployStarterToAksWithPerPoolSubnets), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksVnetDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksVnetDeploymentTests.cs new file mode 100644 index 00000000000..c8fd6dbdfed --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksVnetDeploymentTests.cs @@ -0,0 +1,294 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying Aspire applications to AKS with a custom VNet/subnet. +/// Verifies that AddAzureVirtualNetwork and WithSubnet correctly integrate +/// AKS networking so that pods receive VNet IPs rather than default overlay IPs. +/// +public sealed class AksVnetDeploymentTests(ITestOutputHelper output) +{ + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployStarterToAksWithVnet() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployStarterToAksWithVnetCore(cancellationToken); + } + + private async Task DeployStarterToAksWithVnetCore(CancellationToken cancellationToken) + { + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-vnet"); + var projectName = "AksVnet"; + + output.WriteLine($"Test: {nameof(DeployStarterToAksWithVnet)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); + await auto.SourceAspireCliEnvironmentAsync(counter); + } + + // Step 3: Create starter project using aspire new + output.WriteLine("Step 3: Creating starter project..."); + await auto.AspireNewAsync(projectName, counter, useRedisCache: false); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add Aspire.Hosting.Azure.Kubernetes package + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); + await auto.EnterAsync(); + } + + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + + // Step 6: Modify AppHost.cs to add AKS environment with VNet + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); + var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); + + output.WriteLine($"Modifying AppHost.cs at: {appHostFilePath}"); + + var content = File.ReadAllText(appHostFilePath); + + var oldCode = $""" +var apiService = builder.AddProject("apiservice"); + +builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithReference(apiService) + .WaitFor(apiService); + +builder.Build().Run(); +"""; + + var newCode = $""" +var vnet = builder.AddAzureVirtualNetwork("vnet", "10.1.0.0/16"); +var subnet = vnet.AddSubnet("akssubnet", "10.1.0.0/22"); + +var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithSubnet(subnet); + +var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health") + .WithComputeEnvironment(aks); + +builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WithComputeEnvironment(aks); + +builder.Build().Run(); +"""; + + content = content.Replace(oldCode, newCode); + + if (!content.Contains("#pragma warning disable ASPIREPIPELINES001")) + { + content = "#pragma warning disable ASPIREPIPELINES001\n" + content; + } + + if (!content.Contains("#pragma warning disable ASPIREAZURE003")) + { + content = "#pragma warning disable ASPIREAZURE003\n" + content; + } + + File.WriteAllText(appHostFilePath, content); + + output.WriteLine("Modified AppHost.cs with AddAzureKubernetesEnvironment and VNet/subnet"); + + // Step 7: Navigate to AppHost project directory + output.WriteLine("Step 7: Navigating to AppHost directory..."); + await auto.TypeAsync($"cd {projectName}.AppHost"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Set environment variables for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 9: Deploy to AKS using aspire deploy + output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 10: Get AKS credentials for kubectl verification + output.WriteLine("Step 10: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 11: Wait for pods to be ready + output.WriteLine("Step 11: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 12: Verify pods are running and have VNet IPs (10.1.x.x, not 10.244.x.x overlay) + output.WriteLine("Step 12: Verifying pods have VNet IPs..."); + await auto.TypeAsync("kubectl get pods -o wide -n default"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync("10.1.", timeout: TimeSpan.FromSeconds(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 13: Verify apiservice endpoint via port-forward + output.WriteLine("Step 13: Verifying apiservice endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 14: Verify webfrontend endpoint via port-forward + output.WriteLine("Step 14: Verifying webfrontend endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 15: Clean up port-forwards + output.WriteLine("Step 15: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 16: Destroy Azure deployment + output.WriteLine("Step 16: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 17: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"AKS VNet deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployStarterToAksWithVnet), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - Aspire app deployed to AKS with VNet integration!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployStarterToAksWithVnet), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksWithVersionDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksWithVersionDeploymentTests.cs new file mode 100644 index 00000000000..73262cf1f89 --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksWithVersionDeploymentTests.cs @@ -0,0 +1,296 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying Aspire applications to AKS with a specific Kubernetes version. +/// Verifies that .WithVersion("1.30") correctly provisions the cluster at the requested version. +/// +public sealed class AksWithVersionDeploymentTests(ITestOutputHelper output) +{ + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployStarterToAksWithSpecificVersion() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployStarterToAksWithSpecificVersionCore(cancellationToken); + } + + private async Task DeployStarterToAksWithSpecificVersionCore(CancellationToken cancellationToken) + { + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-version"); + var projectName = "AksVersion"; + + output.WriteLine($"Test: {nameof(DeployStarterToAksWithSpecificVersion)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); + await auto.SourceAspireCliEnvironmentAsync(counter); + } + + // Step 3: Create starter project using aspire new + output.WriteLine("Step 3: Creating starter project..."); + await auto.AspireNewAsync(projectName, counter, useRedisCache: false); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add Aspire.Hosting.Azure.Kubernetes package + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); + await auto.EnterAsync(); + } + + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + + // Step 6: Modify AppHost.cs to add AKS environment with specific version + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); + var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); + + output.WriteLine($"Modifying AppHost.cs at: {appHostFilePath}"); + + var content = File.ReadAllText(appHostFilePath); + + var oldCode = $""" +var apiService = builder.AddProject("apiservice"); + +builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithReference(apiService) + .WaitFor(apiService); + +builder.Build().Run(); +"""; + + var newCode = $""" +var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithVersion("1.30"); + +var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health") + .WithComputeEnvironment(aks); + +builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WithComputeEnvironment(aks); + +builder.Build().Run(); +"""; + + content = content.Replace(oldCode, newCode); + + if (!content.Contains("#pragma warning disable ASPIREPIPELINES001")) + { + content = "#pragma warning disable ASPIREPIPELINES001\n" + content; + } + + if (!content.Contains("#pragma warning disable ASPIREAZURE003")) + { + content = "#pragma warning disable ASPIREAZURE003\n" + content; + } + + File.WriteAllText(appHostFilePath, content); + + output.WriteLine("Modified AppHost.cs with AddAzureKubernetesEnvironment and WithVersion(\"1.30\")"); + + // Step 7: Navigate to AppHost project directory + output.WriteLine("Step 7: Navigating to AppHost directory..."); + await auto.TypeAsync($"cd {projectName}.AppHost"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Set environment variables for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 9: Deploy to AKS using aspire deploy + output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 10: Get AKS credentials for kubectl verification + output.WriteLine("Step 10: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 11: Wait for pods to be ready + output.WriteLine("Step 11: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 12: Verify pods are running + output.WriteLine("Step 12: Verifying pods are running..."); + await auto.TypeAsync("kubectl get pods -n default"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 13: Verify Kubernetes version contains 1.30 + output.WriteLine("Step 13: Verifying Kubernetes version..."); + await auto.TypeAsync("kubectl version --short 2>/dev/null || kubectl version"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync("1.30", timeout: TimeSpan.FromSeconds(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 14: Verify apiservice endpoint via port-forward + output.WriteLine("Step 14: Verifying apiservice endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 15: Verify webfrontend endpoint via port-forward + output.WriteLine("Step 15: Verifying webfrontend endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 16: Clean up port-forwards + output.WriteLine("Step 16: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 17: Destroy Azure deployment + output.WriteLine("Step 17: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 18: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"AKS versioned deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployStarterToAksWithSpecificVersion), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - Aspire app deployed to AKS with specific version 1.30!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployStarterToAksWithSpecificVersion), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksWorkloadIdentityDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksWorkloadIdentityDeploymentTests.cs new file mode 100644 index 00000000000..75314193f26 --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksWorkloadIdentityDeploymentTests.cs @@ -0,0 +1,319 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying Aspire applications to AKS with workload identity. +/// Verifies that referencing an Azure resource (blob storage) from an AKS-hosted service +/// correctly provisions workload identity (service account annotation and pod label). +/// +public sealed class AksWorkloadIdentityDeploymentTests(ITestOutputHelper output) +{ + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployStarterToAksWithWorkloadIdentity() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployStarterToAksWithWorkloadIdentityCore(cancellationToken); + } + + private async Task DeployStarterToAksWithWorkloadIdentityCore(CancellationToken cancellationToken) + { + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-wi"); + var projectName = "AksWorkloadId"; + + output.WriteLine($"Test: {nameof(DeployStarterToAksWithWorkloadIdentity)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); + await auto.SourceAspireCliEnvironmentAsync(counter); + } + + // Step 3: Create starter project using aspire new + output.WriteLine("Step 3: Creating starter project..."); + await auto.AspireNewAsync(projectName, counter, useRedisCache: false); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add required packages + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); + await auto.EnterAsync(); + } + + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + + output.WriteLine("Step 5b: Adding Azure Storage hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Storage"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); + await auto.EnterAsync(); + } + + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + + // Step 6: Modify AppHost.cs to add AKS environment with workload identity + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); + var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); + + output.WriteLine($"Modifying AppHost.cs at: {appHostFilePath}"); + + var content = File.ReadAllText(appHostFilePath); + + var oldCode = $""" +var apiService = builder.AddProject("apiservice"); + +builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithReference(apiService) + .WaitFor(apiService); + +builder.Build().Run(); +"""; + + var newCode = $""" +var aks = builder.AddAzureKubernetesEnvironment("aks"); + +var storage = builder.AddAzureStorage("storage"); +var blobs = storage.AddBlobs("blobs"); +var photos = blobs.AddBlobContainer("photos"); + +var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health") + .WithComputeEnvironment(aks) + .WithReference(photos); + +builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WithComputeEnvironment(aks); + +builder.Build().Run(); +"""; + + content = content.Replace(oldCode, newCode); + + if (!content.Contains("#pragma warning disable ASPIREPIPELINES001")) + { + content = "#pragma warning disable ASPIREPIPELINES001\n" + content; + } + + if (!content.Contains("#pragma warning disable ASPIREAZURE003")) + { + content = "#pragma warning disable ASPIREAZURE003\n" + content; + } + + File.WriteAllText(appHostFilePath, content); + + output.WriteLine("Modified AppHost.cs with AddAzureKubernetesEnvironment and workload identity via Azure Storage"); + + // Step 7: Navigate to AppHost project directory + output.WriteLine("Step 7: Navigating to AppHost directory..."); + await auto.TypeAsync($"cd {projectName}.AppHost"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Set environment variables for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 9: Deploy to AKS using aspire deploy + output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 10: Get AKS credentials for kubectl verification + output.WriteLine("Step 10: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 11: Wait for pods to be ready + output.WriteLine("Step 11: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 12: Verify pods are running + output.WriteLine("Step 12: Verifying pods are running..."); + await auto.TypeAsync("kubectl get pods -n default"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 13: Verify service account has workload identity annotation + output.WriteLine("Step 13: Verifying workload identity service account..."); + await auto.TypeAsync("kubectl get sa apiservice-sa -o yaml"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync("azure.workload.identity/client-id", timeout: TimeSpan.FromSeconds(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 14: Verify pod has workload identity label + output.WriteLine("Step 14: Verifying pod workload identity label..."); + await auto.TypeAsync("kubectl get pod -l app.kubernetes.io/component=apiservice -o jsonpath='{.items[0].metadata.labels}' | grep azure.workload.identity"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 15: Verify apiservice endpoint via port-forward + output.WriteLine("Step 15: Verifying apiservice endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 16: Verify webfrontend endpoint via port-forward + output.WriteLine("Step 16: Verifying webfrontend endpoint..."); + await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 17: Clean up port-forwards + output.WriteLine("Step 17: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 18: Destroy Azure deployment + output.WriteLine("Step 18: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 19: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"AKS workload identity deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployStarterToAksWithWorkloadIdentity), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - Aspire app deployed to AKS with workload identity!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployStarterToAksWithWorkloadIdentity), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} From fc22becfcada2971719629040886456d8742fc8f Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 15:35:39 +1000 Subject: [PATCH 66/78] Add TypeScript AppHost AKS E2E deployment tests TypeScript variants of the AKS Tier 1 tests using ExpressReact template: - TypeScriptAksDeploymentTests: basic AKS deploy from TS AppHost - TypeScriptAksNodePoolDeploymentTests: custom node pool from TS - TypeScriptAksVnetDeploymentTests: VNet/subnet integration from TS Uses addAzureKubernetesEnvironment(), addNodePool(), withSubnet() from the auto-generated TypeScript SDK (via AspireExport attributes). Follows TypeScriptExpressDeploymentTests pattern with bundle install. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../TypeScriptAksDeploymentTests.cs | 258 ++++++++++++++++ .../TypeScriptAksNodePoolDeploymentTests.cs | 262 +++++++++++++++++ .../TypeScriptAksVnetDeploymentTests.cs | 276 ++++++++++++++++++ 3 files changed, 796 insertions(+) create mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksDeploymentTests.cs create mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs create mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksVnetDeploymentTests.cs diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksDeploymentTests.cs new file mode 100644 index 00000000000..f8f6db9836b --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksDeploymentTests.cs @@ -0,0 +1,258 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying a TypeScript Express/React Aspire application to Azure Kubernetes Service (AKS). +/// Uses the same Azure.Provisioning pipeline as the C# variant but from a TypeScript AppHost created via +/// aspire new --template express-react. +/// +public sealed class TypeScriptAksDeploymentTests(ITestOutputHelper output) +{ + // Timeout set to 45 minutes to allow for AKS provisioning (~10-15 min) plus npm install and container build. + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployTypeScriptExpressToAksWithAzureProvisioning() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployTypeScriptExpressToAksWithAzureProvisioningCore(cancellationToken); + } + + private async Task DeployTypeScriptExpressToAksWithAzureProvisioningCore(CancellationToken cancellationToken) + { + // Validate prerequisites + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("ts-aks"); + var projectName = "TsAksApp"; + + output.WriteLine($"Test: {nameof(DeployTypeScriptExpressToAksWithAzureProvisioning)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + // TypeScript apphosts need the full bundle (not just the CLI binary) because + // the prebuilt AppHost server is required for aspire add to regenerate SDK code. + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + var prNumber = DeploymentE2ETestHelpers.GetPrNumber(); + if (prNumber > 0) + { + output.WriteLine($"Step 2: Installing Aspire bundle from PR #{prNumber}..."); + await auto.InstallAspireBundleFromPullRequestAsync(prNumber, counter); + } + await auto.SourceAspireBundleEnvironmentAsync(counter); + } + + // Step 3: Create TypeScript Express/React project using aspire new + output.WriteLine("Step 3: Creating TypeScript Express/React project..."); + await auto.AspireNewAsync(projectName, counter, template: AspireTemplate.ExpressReact); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add Aspire.Hosting.Azure.Kubernetes package + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitForAspireAddCompletionAsync(counter); + } + else + { + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + } + + // Step 6: Modify apphost.ts to add AKS environment for deployment + { + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostFilePath = Path.Combine(projectDir, "apphost.ts"); + + output.WriteLine($"Looking for apphost.ts at: {appHostFilePath}"); + + var content = File.ReadAllText(appHostFilePath); + + // Add Azure Kubernetes Environment before build().run() + // When there's exactly one compute environment, all resources auto-target it. + content = content.Replace( + "await builder.build().run();", + """ +// Add Azure Kubernetes Environment for deployment +await builder.addAzureKubernetesEnvironment("aks"); + +await builder.build().run(); +"""); + + File.WriteAllText(appHostFilePath, content); + + output.WriteLine($"Modified apphost.ts at: {appHostFilePath}"); + } + + // Step 7: Set environment for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Deploy to AKS using aspire deploy + output.WriteLine("Step 8: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + // Wait for pipeline to complete - AKS provisioning can take up to 30 minutes + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 9: Get AKS credentials for kubectl verification + output.WriteLine("Step 9: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 10: Wait for pods to be ready + output.WriteLine("Step 10: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 11: Verify pods are running + output.WriteLine("Step 11: Verifying pods are running..."); + await auto.TypeAsync("kubectl get pods -n default"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 12: Verify service endpoints via port-forward + output.WriteLine("Step 12: Verifying service endpoints..."); + await auto.TypeAsync("kubectl port-forward svc/api-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 13: Clean up port-forwards + output.WriteLine("Step 13: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 14: Destroy Azure deployment + output.WriteLine("Step 14: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 15: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"TypeScript AKS deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployTypeScriptExpressToAksWithAzureProvisioning), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - TypeScript Express app deployed to AKS via Azure Provisioning!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployTypeScriptExpressToAksWithAzureProvisioning), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs new file mode 100644 index 00000000000..5aa9519157e --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs @@ -0,0 +1,262 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying a TypeScript Express/React Aspire application to AKS with a custom node pool. +/// Verifies that addNodePool from a TypeScript AppHost creates additional node pools in the AKS cluster. +/// +public sealed class TypeScriptAksNodePoolDeploymentTests(ITestOutputHelper output) +{ + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployTypeScriptExpressToAksWithNodePool() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployTypeScriptExpressToAksWithNodePoolCore(cancellationToken); + } + + private async Task DeployTypeScriptExpressToAksWithNodePoolCore(CancellationToken cancellationToken) + { + // Validate prerequisites + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("ts-aks-np"); + var projectName = "TsAksNodePool"; + + output.WriteLine($"Test: {nameof(DeployTypeScriptExpressToAksWithNodePool)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + // TypeScript apphosts need the full bundle (not just the CLI binary) because + // the prebuilt AppHost server is required for aspire add to regenerate SDK code. + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + var prNumber = DeploymentE2ETestHelpers.GetPrNumber(); + if (prNumber > 0) + { + output.WriteLine($"Step 2: Installing Aspire bundle from PR #{prNumber}..."); + await auto.InstallAspireBundleFromPullRequestAsync(prNumber, counter); + } + await auto.SourceAspireBundleEnvironmentAsync(counter); + } + + // Step 3: Create TypeScript Express/React project using aspire new + output.WriteLine("Step 3: Creating TypeScript Express/React project..."); + await auto.AspireNewAsync(projectName, counter, template: AspireTemplate.ExpressReact); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add Aspire.Hosting.Azure.Kubernetes package + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitForAspireAddCompletionAsync(counter); + } + else + { + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + } + + // Step 6: Modify apphost.ts to add AKS environment with custom node pool + { + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostFilePath = Path.Combine(projectDir, "apphost.ts"); + + output.WriteLine($"Looking for apphost.ts at: {appHostFilePath}"); + + var content = File.ReadAllText(appHostFilePath); + + // Add Azure Kubernetes Environment with a custom node pool before build().run() + content = content.Replace( + "await builder.build().run();", + """ +// Add Azure Kubernetes Environment with a custom node pool +const aks = await builder.addAzureKubernetesEnvironment("aks"); +const pool = await aks.addNodePool("compute", "Standard_D4s_v5", 1, 3); + +await builder.build().run(); +"""); + + File.WriteAllText(appHostFilePath, content); + + output.WriteLine($"Modified apphost.ts at: {appHostFilePath}"); + } + + // Step 7: Set environment for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Deploy to AKS using aspire deploy + output.WriteLine("Step 8: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 9: Get AKS credentials for kubectl verification + output.WriteLine("Step 9: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 10: Wait for pods to be ready + output.WriteLine("Step 10: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 11: Verify pods are running + output.WriteLine("Step 11: Verifying pods are running..."); + await auto.TypeAsync("kubectl get pods -n default"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 12: Verify two node pools exist (system + compute) + output.WriteLine("Step 12: Verifying node pools..."); + await auto.TypeAsync($"az aks nodepool list --resource-group {resourceGroupName} --cluster-name $AKS_NAME --query '[].{{name:name, mode:mode}}' -o table"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync("compute", timeout: TimeSpan.FromSeconds(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 13: Verify service endpoints via port-forward + output.WriteLine("Step 13: Verifying service endpoints..."); + await auto.TypeAsync("kubectl port-forward svc/api-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 14: Clean up port-forwards + output.WriteLine("Step 14: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 15: Destroy Azure deployment + output.WriteLine("Step 15: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 16: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"TypeScript AKS node pool deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployTypeScriptExpressToAksWithNodePool), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - TypeScript Express app deployed to AKS with custom node pool!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployTypeScriptExpressToAksWithNodePool), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksVnetDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksVnetDeploymentTests.cs new file mode 100644 index 00000000000..a99346e8110 --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksVnetDeploymentTests.cs @@ -0,0 +1,276 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Resources; +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end tests for deploying a TypeScript Express/React Aspire application to AKS with a custom VNet/subnet. +/// Verifies that addAzureVirtualNetwork and withSubnet from a TypeScript AppHost correctly integrate +/// AKS networking so that pods receive VNet IPs rather than default overlay IPs. +/// +public sealed class TypeScriptAksVnetDeploymentTests(ITestOutputHelper output) +{ + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); + + [Fact] + public async Task DeployTypeScriptExpressToAksWithVnet() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployTypeScriptExpressToAksWithVnetCore(cancellationToken); + } + + private async Task DeployTypeScriptExpressToAksWithVnetCore(CancellationToken cancellationToken) + { + // Validate prerequisites + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("ts-aks-vnet"); + var projectName = "TsAksVnet"; + + output.WriteLine($"Test: {nameof(DeployTypeScriptExpressToAksWithVnet)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment (in CI) + // TypeScript apphosts need the full bundle (not just the CLI binary) because + // the prebuilt AppHost server is required for aspire add to regenerate SDK code. + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + var prNumber = DeploymentE2ETestHelpers.GetPrNumber(); + if (prNumber > 0) + { + output.WriteLine($"Step 2: Installing Aspire bundle from PR #{prNumber}..."); + await auto.InstallAspireBundleFromPullRequestAsync(prNumber, counter); + } + await auto.SourceAspireBundleEnvironmentAsync(counter); + } + + // Step 3: Create TypeScript Express/React project using aspire new + output.WriteLine("Step 3: Creating TypeScript Express/React project..."); + await auto.AspireNewAsync(projectName, counter, template: AspireTemplate.ExpressReact); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5a: Add Aspire.Hosting.Azure.Kubernetes package + output.WriteLine("Step 5a: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitForAspireAddCompletionAsync(counter); + } + else + { + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + } + + // Step 5b: Add Aspire.Hosting.Azure.Network package (for VNet/subnet support) + output.WriteLine("Step 5b: Adding Azure Network hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Network"); + await auto.EnterAsync(); + + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + await auto.WaitForAspireAddCompletionAsync(counter); + } + else + { + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); + } + + // Step 6: Modify apphost.ts to add AKS environment with VNet/subnet + { + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostFilePath = Path.Combine(projectDir, "apphost.ts"); + + output.WriteLine($"Looking for apphost.ts at: {appHostFilePath}"); + + var content = File.ReadAllText(appHostFilePath); + + // Add VNet, subnet, and AKS environment with subnet integration before build().run() + content = content.Replace( + "await builder.build().run();", + """ +// Add VNet and subnet for AKS networking +const vnet = await builder.addAzureVirtualNetwork("vnet", "10.1.0.0/16"); +const subnet = await vnet.addSubnet("akssubnet", "10.1.0.0/22"); + +// Add Azure Kubernetes Environment with VNet integration +const aks = await builder.addAzureKubernetesEnvironment("aks"); +await aks.withSubnet(subnet); + +await builder.build().run(); +"""); + + File.WriteAllText(appHostFilePath, content); + + output.WriteLine($"Modified apphost.ts at: {appHostFilePath}"); + } + + // Step 7: Set environment for deployment + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Deploy to AKS using aspire deploy + output.WriteLine("Step 8: Starting AKS deployment via aspire deploy..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + // Wait for pipeline to complete - AKS provisioning can take up to 30 minutes + await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 9: Get AKS credentials for kubectl verification + output.WriteLine("Step 9: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + + $"echo \"AKS cluster: $AKS_NAME\" && " + + $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 10: Wait for pods to be ready + output.WriteLine("Step 10: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Step 11: Verify pods are running and have VNet IPs (10.1.x.x, not 10.244.x.x overlay) + output.WriteLine("Step 11: Verifying pods have VNet IPs..."); + await auto.TypeAsync("kubectl get pods -o wide -n default"); + await auto.EnterAsync(); + await auto.WaitUntilTextAsync("10.1.", timeout: TimeSpan.FromSeconds(30)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 12: Verify service endpoints via port-forward + output.WriteLine("Step 12: Verifying service endpoints..."); + await auto.TypeAsync("kubectl port-forward svc/api-service 18080:8080 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 13: Clean up port-forwards + output.WriteLine("Step 13: Cleaning up port-forwards..."); + await auto.TypeAsync("kill %1 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 14: Destroy Azure deployment + output.WriteLine("Step 14: Destroying Azure deployment..."); + await auto.AspireDestroyAsync(counter); + + // Step 15: Exit terminal + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"TypeScript AKS VNet deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployTypeScriptExpressToAksWithVnet), + resourceGroupName, + new Dictionary + { + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - TypeScript Express app deployed to AKS with VNet integration!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployTypeScriptExpressToAksWithVnet), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName, output); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + } + + private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + } + } +} From 8612d95bb85163cd54e8b3a96efd554d057d2de7 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 16:07:42 +1000 Subject: [PATCH 67/78] =?UTF-8?q?Remove=20WithVersion=20public=20API=20?= =?UTF-8?q?=E2=80=94=20use=20ConfigureInfrastructure=20instead?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same rationale as AsPrivateCluster and WithSkuTier: Kubernetes version is an infrastructure configuration concern. The internal property and Bicep generation remain for ConfigureInfrastructure customization. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 21 +- .../AksWithVersionDeploymentTests.cs | 296 ------------------ ...ureKubernetesEnvironmentExtensionsTests.cs | 25 -- ...etesEnvironment_WithVersion.verified.bicep | 53 ---- 4 files changed, 1 insertion(+), 394 deletions(-) delete mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/AksWithVersionDeploymentTests.cs delete mode 100644 tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index bf1a15ac934..5b028b425f3 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -42,8 +42,7 @@ public static class AzureKubernetesEnvironmentExtensions /// /// /// - /// var aks = builder.AddAzureKubernetesEnvironment("aks") - /// .WithVersion("1.30"); + /// var aks = builder.AddAzureKubernetesEnvironment("aks"); /// /// [AspireExport(Description = "Adds an Azure Kubernetes Service environment resource")] @@ -135,24 +134,6 @@ public static IResourceBuilder AddAzureKuber return builder.AddResource(resource); } - /// - /// Configures the Kubernetes version for the AKS cluster. - /// - /// The resource builder. - /// The Kubernetes version (e.g., "1.30"). - /// A reference to the for chaining. - [AspireExport(Description = "Sets the Kubernetes version for the AKS cluster")] - public static IResourceBuilder WithVersion( - this IResourceBuilder builder, - string version) - { - ArgumentNullException.ThrowIfNull(builder); - ArgumentException.ThrowIfNullOrEmpty(version); - - builder.Resource.KubernetesVersion = version; - return builder; - } - /// /// Adds a node pool to the AKS cluster. /// diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksWithVersionDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksWithVersionDeploymentTests.cs deleted file mode 100644 index 73262cf1f89..00000000000 --- a/tests/Aspire.Deployment.EndToEnd.Tests/AksWithVersionDeploymentTests.cs +++ /dev/null @@ -1,296 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Aspire.Cli.Resources; -using Aspire.Cli.Tests.Utils; -using Aspire.Deployment.EndToEnd.Tests.Helpers; -using Hex1b.Automation; -using Xunit; - -namespace Aspire.Deployment.EndToEnd.Tests; - -/// -/// End-to-end tests for deploying Aspire applications to AKS with a specific Kubernetes version. -/// Verifies that .WithVersion("1.30") correctly provisions the cluster at the requested version. -/// -public sealed class AksWithVersionDeploymentTests(ITestOutputHelper output) -{ - private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); - - [Fact] - public async Task DeployStarterToAksWithSpecificVersion() - { - using var cts = new CancellationTokenSource(s_testTimeout); - using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( - cts.Token, TestContext.Current.CancellationToken); - var cancellationToken = linkedCts.Token; - - await DeployStarterToAksWithSpecificVersionCore(cancellationToken); - } - - private async Task DeployStarterToAksWithSpecificVersionCore(CancellationToken cancellationToken) - { - var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); - if (string.IsNullOrEmpty(subscriptionId)) - { - Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); - } - - if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) - { - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); - } - else - { - Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); - } - } - - var workspace = TemporaryWorkspace.Create(output); - var startTime = DateTime.UtcNow; - var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-version"); - var projectName = "AksVersion"; - - output.WriteLine($"Test: {nameof(DeployStarterToAksWithSpecificVersion)}"); - output.WriteLine($"Project Name: {projectName}"); - output.WriteLine($"Resource Group: {resourceGroupName}"); - output.WriteLine($"Subscription: {subscriptionId[..8]}..."); - output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); - - try - { - using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); - var pendingRun = terminal.RunAsync(cancellationToken); - - var counter = new SequenceCounter(); - var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); - - // Step 1: Prepare environment - output.WriteLine("Step 1: Preparing environment..."); - await auto.PrepareEnvironmentAsync(workspace, counter); - - // Step 2: Set up CLI environment (in CI) - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); - await auto.SourceAspireCliEnvironmentAsync(counter); - } - - // Step 3: Create starter project using aspire new - output.WriteLine("Step 3: Creating starter project..."); - await auto.AspireNewAsync(projectName, counter, useRedisCache: false); - - // Step 4: Navigate to project directory - output.WriteLine("Step 4: Navigating to project directory..."); - await auto.TypeAsync($"cd {projectName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 5: Add Aspire.Hosting.Azure.Kubernetes package - output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); - await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); - await auto.EnterAsync(); - - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); - await auto.EnterAsync(); - } - - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - - // Step 6: Modify AppHost.cs to add AKS environment with specific version - var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); - var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); - var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); - - output.WriteLine($"Modifying AppHost.cs at: {appHostFilePath}"); - - var content = File.ReadAllText(appHostFilePath); - - var oldCode = $""" -var apiService = builder.AddProject("apiservice"); - -builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithReference(apiService) - .WaitFor(apiService); - -builder.Build().Run(); -"""; - - var newCode = $""" -var aks = builder.AddAzureKubernetesEnvironment("aks") - .WithVersion("1.30"); - -var apiService = builder.AddProject("apiservice") - .WithHttpHealthCheck("/health") - .WithComputeEnvironment(aks); - -builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithHttpHealthCheck("/health") - .WithReference(apiService) - .WithComputeEnvironment(aks); - -builder.Build().Run(); -"""; - - content = content.Replace(oldCode, newCode); - - if (!content.Contains("#pragma warning disable ASPIREPIPELINES001")) - { - content = "#pragma warning disable ASPIREPIPELINES001\n" + content; - } - - if (!content.Contains("#pragma warning disable ASPIREAZURE003")) - { - content = "#pragma warning disable ASPIREAZURE003\n" + content; - } - - File.WriteAllText(appHostFilePath, content); - - output.WriteLine("Modified AppHost.cs with AddAzureKubernetesEnvironment and WithVersion(\"1.30\")"); - - // Step 7: Navigate to AppHost project directory - output.WriteLine("Step 7: Navigating to AppHost directory..."); - await auto.TypeAsync($"cd {projectName}.AppHost"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 8: Set environment variables for deployment - await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 9: Deploy to AKS using aspire deploy - output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); - await auto.TypeAsync("aspire deploy --clear-cache"); - await auto.EnterAsync(); - await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); - - // Step 10: Get AKS credentials for kubectl verification - output.WriteLine("Step 10: Getting AKS credentials..."); - await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + - $"echo \"AKS cluster: $AKS_NAME\" && " + - $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 11: Wait for pods to be ready - output.WriteLine("Step 11: Waiting for pods to be ready..."); - await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); - - // Step 12: Verify pods are running - output.WriteLine("Step 12: Verifying pods are running..."); - await auto.TypeAsync("kubectl get pods -n default"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); - - // Step 13: Verify Kubernetes version contains 1.30 - output.WriteLine("Step 13: Verifying Kubernetes version..."); - await auto.TypeAsync("kubectl version --short 2>/dev/null || kubectl version"); - await auto.EnterAsync(); - await auto.WaitUntilTextAsync("1.30", timeout: TimeSpan.FromSeconds(30)); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - - // Step 14: Verify apiservice endpoint via port-forward - output.WriteLine("Step 14: Verifying apiservice endpoint..."); - await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 15: Verify webfrontend endpoint via port-forward - output.WriteLine("Step 15: Verifying webfrontend endpoint..."); - await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 16: Clean up port-forwards - output.WriteLine("Step 16: Cleaning up port-forwards..."); - await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - - // Step 17: Destroy Azure deployment - output.WriteLine("Step 17: Destroying Azure deployment..."); - await auto.AspireDestroyAsync(counter); - - // Step 18: Exit terminal - await auto.TypeAsync("exit"); - await auto.EnterAsync(); - - await pendingRun; - - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"AKS versioned deployment completed in {duration}"); - - DeploymentReporter.ReportDeploymentSuccess( - nameof(DeployStarterToAksWithSpecificVersion), - resourceGroupName, - new Dictionary - { - ["project"] = projectName - }, - duration); - - output.WriteLine("✅ Test passed - Aspire app deployed to AKS with specific version 1.30!"); - } - catch (Exception ex) - { - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); - - DeploymentReporter.ReportDeploymentFailure( - nameof(DeployStarterToAksWithSpecificVersion), - resourceGroupName, - ex.Message, - ex.StackTrace); - - throw; - } - finally - { - output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); - TriggerCleanupResourceGroup(resourceGroupName, output); - DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); - } - } - - private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) - { - var process = new System.Diagnostics.Process - { - StartInfo = new System.Diagnostics.ProcessStartInfo - { - FileName = "az", - Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - try - { - process.Start(); - output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); - } - catch (Exception ex) - { - output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); - } - } -} diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs index 09fc1230e3c..3c51fa43474 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs @@ -30,20 +30,6 @@ public async Task AddAzureKubernetesEnvironment_BasicConfiguration() await Verify(manifest.BicepText, extension: "bicep"); } - [Fact] - public async Task AddAzureKubernetesEnvironment_WithVersion() - { - using var builder = TestDistributedApplicationBuilder.Create(); - - var aks = builder.AddAzureKubernetesEnvironment("aks") - .WithVersion("1.30"); - - Assert.Equal("1.30", aks.Resource.KubernetesVersion); - - var manifest = await AzureManifestUtils.GetManifestWithBicep(aks.Resource); - await Verify(manifest.BicepText, extension: "bicep"); - } - [Fact] public void AddNodePool_ReturnsNodePoolResource() { @@ -162,17 +148,6 @@ public void AddAzureKubernetesEnvironment_ThrowsOnEmptyName() builder.AddAzureKubernetesEnvironment("")); } - [Fact] - public void WithVersion_ThrowsOnEmptyVersion() - { - using var builder = TestDistributedApplicationBuilder.Create(); - - var aks = builder.AddAzureKubernetesEnvironment("aks"); - - Assert.Throws(() => - aks.WithVersion("")); - } - [Fact] public void WithWorkloadIdentity_EnablesOidcAndWorkloadIdentity() { diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep deleted file mode 100644 index 90173697580..00000000000 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_WithVersion.verified.bicep +++ /dev/null @@ -1,53 +0,0 @@ -@description('The location for the resource(s) to be deployed.') -param location string = resourceGroup().location - -resource aks 'Microsoft.ContainerService/managedClusters@2025-03-01' = { - name: take('aks-${uniqueString(resourceGroup().id)}', 63) - location: location - properties: { - agentPoolProfiles: [ - { - name: 'system' - count: 1 - vmSize: 'Standard_D4s_v5' - osType: 'Linux' - maxCount: 3 - minCount: 1 - enableAutoScaling: true - mode: 'System' - } - ] - dnsPrefix: 'aks-dns' - kubernetesVersion: '1.30' - oidcIssuerProfile: { - enabled: true - } - securityProfile: { - workloadIdentity: { - enabled: true - } - } - } - identity: { - type: 'SystemAssigned' - } - sku: { - name: 'Base' - tier: 'Free' - } - tags: { - 'aspire-resource-name': 'aks' - } -} - -output id string = aks.id - -output name string = aks.name - -output clusterFqdn string = aks.properties.fqdn - -output oidcIssuerUrl string = aks.properties.oidcIssuerProfile.issuerURL - -output kubeletIdentityObjectId string = aks.properties.identityProfile.kubeletidentity.objectId - -output nodeResourceGroup string = aks.properties.nodeResourceGroup From 6fd1c19f8c38d0022b1d4b3bdc0e5ca73713f252 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 16:12:19 +1000 Subject: [PATCH 68/78] Remove non-functional WithContainerInsights and WithAzureLogAnalyticsWorkspace These APIs set internal properties but the ConfigureAksInfrastructure callback never emits the corresponding Bicep (addonProfiles.omsagent, azureMonitorProfile, data collection rules). Shipping non-functional APIs is misleading. Follow-up issue #16150 will add these back when Bicep generation is implemented. Internal properties remain for future use. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 41 ------------------- ...ureKubernetesEnvironmentExtensionsTests.cs | 37 ----------------- 2 files changed, 78 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 5b028b425f3..fd2fe5dc43d 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -292,47 +292,6 @@ public static IResourceBuilder WithContainer return builder; } - /// - /// Enables Container Insights monitoring on the AKS cluster. - /// - /// The resource builder. - /// Optional Log Analytics workspace. If not provided, one will be auto-created. - /// A reference to the for chaining. - [AspireExport(Description = "Enables Container Insights monitoring on the AKS cluster")] - public static IResourceBuilder WithContainerInsights( - this IResourceBuilder builder, - IResourceBuilder? logAnalytics = null) - { - ArgumentNullException.ThrowIfNull(builder); - - builder.Resource.ContainerInsightsEnabled = true; - - if (logAnalytics is not null) - { - builder.Resource.LogAnalyticsWorkspace = logAnalytics.Resource; - } - - return builder; - } - - /// - /// Configures the AKS environment to use a specific Azure Log Analytics workspace. - /// - /// The resource builder. - /// The Log Analytics workspace resource builder. - /// A reference to the for chaining. - [AspireExport(Description = "Configures the AKS environment to use a Log Analytics workspace")] - public static IResourceBuilder WithAzureLogAnalyticsWorkspace( - this IResourceBuilder builder, - IResourceBuilder workspaceBuilder) - { - ArgumentNullException.ThrowIfNull(builder); - ArgumentNullException.ThrowIfNull(workspaceBuilder); - - builder.Resource.LogAnalyticsWorkspace = workspaceBuilder.Resource; - return builder; - } - /// /// Enables workload identity on the AKS environment, allowing pods to authenticate /// to Azure services using federated credentials. diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs index 3c51fa43474..66b840675c7 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs @@ -50,43 +50,6 @@ public void AddNodePool_ReturnsNodePoolResource() Assert.Same(aks.Resource, gpuPool.Resource.AksParent); } - [Fact] - public void AddAzureKubernetesEnvironment_WithContainerInsights() - { - using var builder = TestDistributedApplicationBuilder.Create(); - - var aks = builder.AddAzureKubernetesEnvironment("aks") - .WithContainerInsights(); - - Assert.True(aks.Resource.ContainerInsightsEnabled); - Assert.Null(aks.Resource.LogAnalyticsWorkspace); - } - - [Fact] - public void AddAzureKubernetesEnvironment_WithContainerInsightsAndWorkspace() - { - using var builder = TestDistributedApplicationBuilder.Create(); - - var logAnalytics = builder.AddAzureLogAnalyticsWorkspace("law"); - var aks = builder.AddAzureKubernetesEnvironment("aks") - .WithContainerInsights(logAnalytics); - - Assert.True(aks.Resource.ContainerInsightsEnabled); - Assert.Same(logAnalytics.Resource, aks.Resource.LogAnalyticsWorkspace); - } - - [Fact] - public void AddAzureKubernetesEnvironment_WithAzureLogAnalyticsWorkspace() - { - using var builder = TestDistributedApplicationBuilder.Create(); - - var logAnalytics = builder.AddAzureLogAnalyticsWorkspace("law"); - var aks = builder.AddAzureKubernetesEnvironment("aks") - .WithAzureLogAnalyticsWorkspace(logAnalytics); - - Assert.Same(logAnalytics.Resource, aks.Resource.LogAnalyticsWorkspace); - } - [Fact] public void AddAzureKubernetesEnvironment_DefaultNodePool() { From a96bb947117d5abf36a8ccac9f4e158380e57f0d Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 19:44:46 +1000 Subject: [PATCH 69/78] Fix AppHost.cs modification in all AKS E2E tests Root cause: string replacement (content.Replace(oldCode, newCode)) failed silently due to line ending mismatches between the template output and our raw string literals. aspire deploy completed in 97ms as a no-op because the AKS environment was never added. Fix: Write the ENTIRE AppHost.cs content instead of patching it. This is immune to line ending, whitespace, and template changes. Each test now has a self-documenting raw string literal showing exactly what AppHost code is being tested. TypeScript tests: added guard checks to throw if the apphost.ts replacement didn't change anything. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AksAzureProvisioningDeploymentTests.cs | 66 +++++----------- .../AksExplicitRegistryDeploymentTests.cs | 76 ++++++------------- .../AksNodePoolDeploymentTests.cs | 61 +++++---------- .../AksPerPoolSubnetDeploymentTests.cs | 71 ++++++----------- .../AksVnetDeploymentTests.cs | 63 +++++---------- .../AksWorkloadIdentityDeploymentTests.cs | 65 ++++++---------- .../TypeScriptAksDeploymentTests.cs | 6 ++ .../TypeScriptAksNodePoolDeploymentTests.cs | 8 +- .../TypeScriptAksVnetDeploymentTests.cs | 6 ++ 9 files changed, 147 insertions(+), 275 deletions(-) diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksAzureProvisioningDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksAzureProvisioningDeploymentTests.cs index 4489a973db2..410f4e831b3 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/AksAzureProvisioningDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksAzureProvisioningDeploymentTests.cs @@ -105,66 +105,36 @@ private async Task DeployStarterToAksWithAzureProvisioningCore(CancellationToken await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - // Step 6: Modify AppHost.cs to add AKS environment with compute targeting + // Step 6: Write complete AppHost.cs with AKS environment (full rewrite to avoid + // string-replacement failures caused by line-ending or whitespace differences) var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); - output.WriteLine($"Modifying AppHost.cs at: {appHostFilePath}"); + output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); - var content = File.ReadAllText(appHostFilePath); + var appHostContent = $""" + #pragma warning disable ASPIREPIPELINES001 - // Replace the existing service definitions with AKS-targeted versions. - // The starter template has: - // var apiService = builder.AddProject("apiservice"); - // builder.AddProject("webfrontend") - // .WithExternalHttpEndpoints() - // .WithReference(apiService) - // .WaitFor(apiService); - // builder.Build().Run(); - var oldCode = $""" -var apiService = builder.AddProject("apiservice"); + var builder = DistributedApplication.CreateBuilder(args); -builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithReference(apiService) - .WaitFor(apiService); + var aks = builder.AddAzureKubernetesEnvironment("aks"); -builder.Build().Run(); -"""; + var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health"); - var newCode = $""" -var aks = builder.AddAzureKubernetesEnvironment("aks"); + builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WaitFor(apiService); -var apiService = builder.AddProject("apiservice") - .WithHttpHealthCheck("/health") - .WithComputeEnvironment(aks); + builder.Build().Run(); + """; -builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithHttpHealthCheck("/health") - .WithReference(apiService) - .WithComputeEnvironment(aks); + File.WriteAllText(appHostFilePath, appHostContent); -builder.Build().Run(); -"""; - - content = content.Replace(oldCode, newCode); - - // Add required pragma warnings to suppress experimental API warnings - if (!content.Contains("#pragma warning disable ASPIREPIPELINES001")) - { - content = "#pragma warning disable ASPIREPIPELINES001\n" + content; - } - - if (!content.Contains("#pragma warning disable ASPIREAZURE003")) - { - content = "#pragma warning disable ASPIREAZURE003\n" + content; - } - - File.WriteAllText(appHostFilePath, content); - - output.WriteLine("Modified AppHost.cs with AddAzureKubernetesEnvironment and compute targeting"); + output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment"); // Step 7: Navigate to AppHost project directory output.WriteLine("Step 7: Navigating to AppHost directory..."); diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksExplicitRegistryDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksExplicitRegistryDeploymentTests.cs index 062a4bfe9ad..2e25d2e190d 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/AksExplicitRegistryDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksExplicitRegistryDeploymentTests.cs @@ -89,7 +89,8 @@ private async Task DeployStarterToAksWithExplicitRegistryCore(CancellationToken await auto.EnterAsync(); await auto.WaitForSuccessPromptAsync(counter); - // Step 5: Add required packages + // Step 5: Add Aspire.Hosting.Azure.Kubernetes package + // (Aspire.Hosting.Azure.Kubernetes already depends on Aspire.Hosting.Azure.ContainerRegistry) output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); await auto.EnterAsync(); @@ -102,71 +103,38 @@ private async Task DeployStarterToAksWithExplicitRegistryCore(CancellationToken await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - output.WriteLine("Step 5b: Adding Azure Container Registry hosting package..."); - await auto.TypeAsync("aspire add Aspire.Hosting.Azure.ContainerRegistry"); - await auto.EnterAsync(); - - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); - await auto.EnterAsync(); - } - - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - - // Step 6: Modify AppHost.cs to add AKS environment with explicit ACR + // Step 6: Write complete AppHost.cs with AKS environment and explicit ACR + // (full rewrite to avoid string-replacement failures caused by line-ending or whitespace differences) var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); - output.WriteLine($"Modifying AppHost.cs at: {appHostFilePath}"); + output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); - var content = File.ReadAllText(appHostFilePath); + var appHostContent = $""" + #pragma warning disable ASPIREPIPELINES001 - var oldCode = $""" -var apiService = builder.AddProject("apiservice"); + var builder = DistributedApplication.CreateBuilder(args); -builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithReference(apiService) - .WaitFor(apiService); + var acr = builder.AddAzureContainerRegistry("myacr"); + var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithContainerRegistry(acr); -builder.Build().Run(); -"""; + var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health"); - var newCode = $""" -var acr = builder.AddAzureContainerRegistry("myacr"); -var aks = builder.AddAzureKubernetesEnvironment("aks") - .WithContainerRegistry(acr); + builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WaitFor(apiService); -var apiService = builder.AddProject("apiservice") - .WithHttpHealthCheck("/health") - .WithComputeEnvironment(aks); - -builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithHttpHealthCheck("/health") - .WithReference(apiService) - .WithComputeEnvironment(aks); - -builder.Build().Run(); -"""; - - content = content.Replace(oldCode, newCode); - - if (!content.Contains("#pragma warning disable ASPIREPIPELINES001")) - { - content = "#pragma warning disable ASPIREPIPELINES001\n" + content; - } - - if (!content.Contains("#pragma warning disable ASPIREAZURE003")) - { - content = "#pragma warning disable ASPIREAZURE003\n" + content; - } + builder.Build().Run(); + """; - File.WriteAllText(appHostFilePath, content); + File.WriteAllText(appHostFilePath, appHostContent); - output.WriteLine("Modified AppHost.cs with AddAzureKubernetesEnvironment and explicit ACR"); + output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment and explicit ACR"); // Step 7: Navigate to AppHost project directory output.WriteLine("Step 7: Navigating to AppHost directory..."); diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs index 58ba97cf3f1..9b691355124 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs @@ -102,59 +102,38 @@ private async Task DeployStarterToAksWithCustomNodePoolCore(CancellationToken ca await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - // Step 6: Modify AppHost.cs to add AKS environment with custom node pool + // Step 6: Write complete AppHost.cs with AKS environment and custom node pool + // (full rewrite to avoid string-replacement failures caused by line-ending or whitespace differences) var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); - output.WriteLine($"Modifying AppHost.cs at: {appHostFilePath}"); + output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); - var content = File.ReadAllText(appHostFilePath); + var appHostContent = $""" + #pragma warning disable ASPIREPIPELINES001 - var oldCode = $""" -var apiService = builder.AddProject("apiservice"); + var builder = DistributedApplication.CreateBuilder(args); -builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithReference(apiService) - .WaitFor(apiService); + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var computePool = aks.AddNodePool("compute", "Standard_D4s_v5", 1, 3); -builder.Build().Run(); -"""; + var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health") + .WithNodePool(computePool); - var newCode = $""" -var aks = builder.AddAzureKubernetesEnvironment("aks"); -var computePool = aks.AddNodePool("compute", "Standard_D4s_v5", 1, 3); + builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WaitFor(apiService); -var apiService = builder.AddProject("apiservice") - .WithHttpHealthCheck("/health") - .WithComputeEnvironment(aks) - .WithNodePool(computePool); + builder.Build().Run(); + """; -builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithHttpHealthCheck("/health") - .WithReference(apiService) - .WithComputeEnvironment(aks); + File.WriteAllText(appHostFilePath, appHostContent); -builder.Build().Run(); -"""; - - content = content.Replace(oldCode, newCode); - - if (!content.Contains("#pragma warning disable ASPIREPIPELINES001")) - { - content = "#pragma warning disable ASPIREPIPELINES001\n" + content; - } - - if (!content.Contains("#pragma warning disable ASPIREAZURE003")) - { - content = "#pragma warning disable ASPIREAZURE003\n" + content; - } - - File.WriteAllText(appHostFilePath, content); - - output.WriteLine("Modified AppHost.cs with AddAzureKubernetesEnvironment and custom node pool"); + output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment and custom node pool"); // Step 7: Navigate to AppHost project directory output.WriteLine("Step 7: Navigating to AppHost directory..."); diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs index 3f56f36f0fd..3d0c686b4d1 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs @@ -102,66 +102,45 @@ private async Task DeployStarterToAksWithPerPoolSubnetsCore(CancellationToken ca await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - // Step 6: Modify AppHost.cs to add AKS environment with per-pool subnets + // Step 6: Write complete AppHost.cs with AKS environment and per-pool subnets + // (full rewrite to avoid string-replacement failures caused by line-ending or whitespace differences) var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); - output.WriteLine($"Modifying AppHost.cs at: {appHostFilePath}"); + output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); - var content = File.ReadAllText(appHostFilePath); + var appHostContent = $""" + #pragma warning disable ASPIREPIPELINES001 - var oldCode = $""" -var apiService = builder.AddProject("apiservice"); + var builder = DistributedApplication.CreateBuilder(args); -builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithReference(apiService) - .WaitFor(apiService); + var vnet = builder.AddAzureVirtualNetwork("vnet", "10.1.0.0/16"); + var defaultSubnet = vnet.AddSubnet("defaultsubnet", "10.1.0.0/22"); + var auxSubnet = vnet.AddSubnet("auxsubnet", "10.1.4.0/24"); -builder.Build().Run(); -"""; + var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithSubnet(defaultSubnet); - var newCode = $""" -var vnet = builder.AddAzureVirtualNetwork("vnet", "10.1.0.0/16"); -var defaultSubnet = vnet.AddSubnet("defaultsubnet", "10.1.0.0/22"); -var auxSubnet = vnet.AddSubnet("auxsubnet", "10.1.4.0/24"); + var auxPool = aks.AddNodePool("aux", "Standard_D4s_v5", 1, 3) + .WithSubnet(auxSubnet); -var aks = builder.AddAzureKubernetesEnvironment("aks") - .WithSubnet(defaultSubnet); + var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health") + .WithNodePool(auxPool); -var auxPool = aks.AddNodePool("aux", "Standard_D4s_v5", 1, 3) - .WithSubnet(auxSubnet); + builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WaitFor(apiService); -var apiService = builder.AddProject("apiservice") - .WithHttpHealthCheck("/health") - .WithComputeEnvironment(aks) - .WithNodePool(auxPool); + builder.Build().Run(); + """; -builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithHttpHealthCheck("/health") - .WithReference(apiService) - .WithComputeEnvironment(aks); + File.WriteAllText(appHostFilePath, appHostContent); -builder.Build().Run(); -"""; - - content = content.Replace(oldCode, newCode); - - if (!content.Contains("#pragma warning disable ASPIREPIPELINES001")) - { - content = "#pragma warning disable ASPIREPIPELINES001\n" + content; - } - - if (!content.Contains("#pragma warning disable ASPIREAZURE003")) - { - content = "#pragma warning disable ASPIREAZURE003\n" + content; - } - - File.WriteAllText(appHostFilePath, content); - - output.WriteLine("Modified AppHost.cs with AddAzureKubernetesEnvironment and per-pool subnets"); + output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment and per-pool subnets"); // Step 7: Navigate to AppHost project directory output.WriteLine("Step 7: Navigating to AppHost directory..."); diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksVnetDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksVnetDeploymentTests.cs index c8fd6dbdfed..a4507647092 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/AksVnetDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksVnetDeploymentTests.cs @@ -102,61 +102,40 @@ private async Task DeployStarterToAksWithVnetCore(CancellationToken cancellation await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - // Step 6: Modify AppHost.cs to add AKS environment with VNet + // Step 6: Write complete AppHost.cs with AKS environment and VNet/subnet + // (full rewrite to avoid string-replacement failures caused by line-ending or whitespace differences) var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); - output.WriteLine($"Modifying AppHost.cs at: {appHostFilePath}"); + output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); - var content = File.ReadAllText(appHostFilePath); + var appHostContent = $""" + #pragma warning disable ASPIREPIPELINES001 - var oldCode = $""" -var apiService = builder.AddProject("apiservice"); + var builder = DistributedApplication.CreateBuilder(args); -builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithReference(apiService) - .WaitFor(apiService); + var vnet = builder.AddAzureVirtualNetwork("vnet", "10.1.0.0/16"); + var subnet = vnet.AddSubnet("akssubnet", "10.1.0.0/22"); -builder.Build().Run(); -"""; + var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithSubnet(subnet); - var newCode = $""" -var vnet = builder.AddAzureVirtualNetwork("vnet", "10.1.0.0/16"); -var subnet = vnet.AddSubnet("akssubnet", "10.1.0.0/22"); + var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health"); -var aks = builder.AddAzureKubernetesEnvironment("aks") - .WithSubnet(subnet); + builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WaitFor(apiService); -var apiService = builder.AddProject("apiservice") - .WithHttpHealthCheck("/health") - .WithComputeEnvironment(aks); + builder.Build().Run(); + """; -builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithHttpHealthCheck("/health") - .WithReference(apiService) - .WithComputeEnvironment(aks); + File.WriteAllText(appHostFilePath, appHostContent); -builder.Build().Run(); -"""; - - content = content.Replace(oldCode, newCode); - - if (!content.Contains("#pragma warning disable ASPIREPIPELINES001")) - { - content = "#pragma warning disable ASPIREPIPELINES001\n" + content; - } - - if (!content.Contains("#pragma warning disable ASPIREAZURE003")) - { - content = "#pragma warning disable ASPIREAZURE003\n" + content; - } - - File.WriteAllText(appHostFilePath, content); - - output.WriteLine("Modified AppHost.cs with AddAzureKubernetesEnvironment and VNet/subnet"); + output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment and VNet/subnet"); // Step 7: Navigate to AppHost project directory output.WriteLine("Step 7: Navigating to AppHost directory..."); diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksWorkloadIdentityDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksWorkloadIdentityDeploymentTests.cs index 75314193f26..ddfa85d43f8 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/AksWorkloadIdentityDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksWorkloadIdentityDeploymentTests.cs @@ -114,62 +114,41 @@ private async Task DeployStarterToAksWithWorkloadIdentityCore(CancellationToken await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - // Step 6: Modify AppHost.cs to add AKS environment with workload identity + // Step 6: Write complete AppHost.cs with AKS environment and workload identity via Azure Storage + // (full rewrite to avoid string-replacement failures caused by line-ending or whitespace differences) var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); - output.WriteLine($"Modifying AppHost.cs at: {appHostFilePath}"); + output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); - var content = File.ReadAllText(appHostFilePath); + var appHostContent = $""" + #pragma warning disable ASPIREPIPELINES001 - var oldCode = $""" -var apiService = builder.AddProject("apiservice"); + var builder = DistributedApplication.CreateBuilder(args); -builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithReference(apiService) - .WaitFor(apiService); + var aks = builder.AddAzureKubernetesEnvironment("aks"); -builder.Build().Run(); -"""; + var storage = builder.AddAzureStorage("storage"); + var blobs = storage.AddBlobs("blobs"); + var photos = blobs.AddBlobContainer("photos"); - var newCode = $""" -var aks = builder.AddAzureKubernetesEnvironment("aks"); + var apiService = builder.AddProject("apiservice") + .WithHttpHealthCheck("/health") + .WithReference(photos); -var storage = builder.AddAzureStorage("storage"); -var blobs = storage.AddBlobs("blobs"); -var photos = blobs.AddBlobContainer("photos"); + builder.AddProject("webfrontend") + .WithExternalHttpEndpoints() + .WithHttpHealthCheck("/health") + .WithReference(apiService) + .WaitFor(apiService); -var apiService = builder.AddProject("apiservice") - .WithHttpHealthCheck("/health") - .WithComputeEnvironment(aks) - .WithReference(photos); + builder.Build().Run(); + """; -builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithHttpHealthCheck("/health") - .WithReference(apiService) - .WithComputeEnvironment(aks); + File.WriteAllText(appHostFilePath, appHostContent); -builder.Build().Run(); -"""; - - content = content.Replace(oldCode, newCode); - - if (!content.Contains("#pragma warning disable ASPIREPIPELINES001")) - { - content = "#pragma warning disable ASPIREPIPELINES001\n" + content; - } - - if (!content.Contains("#pragma warning disable ASPIREAZURE003")) - { - content = "#pragma warning disable ASPIREAZURE003\n" + content; - } - - File.WriteAllText(appHostFilePath, content); - - output.WriteLine("Modified AppHost.cs with AddAzureKubernetesEnvironment and workload identity via Azure Storage"); + output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment and workload identity via Azure Storage"); // Step 7: Navigate to AppHost project directory output.WriteLine("Step 7: Navigating to AppHost directory..."); diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksDeploymentTests.cs index f8f6db9836b..e119f535b0c 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksDeploymentTests.cs @@ -120,6 +120,7 @@ private async Task DeployTypeScriptExpressToAksWithAzureProvisioningCore(Cancell output.WriteLine($"Looking for apphost.ts at: {appHostFilePath}"); var content = File.ReadAllText(appHostFilePath); + var originalContent = content; // Add Azure Kubernetes Environment before build().run() // When there's exactly one compute environment, all resources auto-target it. @@ -132,6 +133,11 @@ private async Task DeployTypeScriptExpressToAksWithAzureProvisioningCore(Cancell await builder.build().run(); """); + if (content == originalContent) + { + throw new InvalidOperationException("apphost.ts was not modified. Template may have changed."); + } + File.WriteAllText(appHostFilePath, content); output.WriteLine($"Modified apphost.ts at: {appHostFilePath}"); diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs index 5aa9519157e..ecbc6d33417 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs @@ -118,6 +118,7 @@ private async Task DeployTypeScriptExpressToAksWithNodePoolCore(CancellationToke output.WriteLine($"Looking for apphost.ts at: {appHostFilePath}"); var content = File.ReadAllText(appHostFilePath); + var originalContent = content; // Add Azure Kubernetes Environment with a custom node pool before build().run() content = content.Replace( @@ -125,11 +126,16 @@ private async Task DeployTypeScriptExpressToAksWithNodePoolCore(CancellationToke """ // Add Azure Kubernetes Environment with a custom node pool const aks = await builder.addAzureKubernetesEnvironment("aks"); -const pool = await aks.addNodePool("compute", "Standard_D4s_v5", 1, 3); +await aks.addNodePool("compute", "Standard_D4s_v5", 1, 3); await builder.build().run(); """); + if (content == originalContent) + { + throw new InvalidOperationException("apphost.ts was not modified. Template may have changed."); + } + File.WriteAllText(appHostFilePath, content); output.WriteLine($"Modified apphost.ts at: {appHostFilePath}"); diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksVnetDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksVnetDeploymentTests.cs index a99346e8110..8d441d41fce 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksVnetDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksVnetDeploymentTests.cs @@ -133,6 +133,7 @@ private async Task DeployTypeScriptExpressToAksWithVnetCore(CancellationToken ca output.WriteLine($"Looking for apphost.ts at: {appHostFilePath}"); var content = File.ReadAllText(appHostFilePath); + var originalContent = content; // Add VNet, subnet, and AKS environment with subnet integration before build().run() content = content.Replace( @@ -149,6 +150,11 @@ private async Task DeployTypeScriptExpressToAksWithVnetCore(CancellationToken ca await builder.build().run(); """); + if (content == originalContent) + { + throw new InvalidOperationException("apphost.ts was not modified. Template may have changed."); + } + File.WriteAllText(appHostFilePath, content); output.WriteLine($"Modified apphost.ts at: {appHostFilePath}"); From 5c1ba6991035c7b14c2ada07707e2a0a7f4a8dc9 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 15 Apr 2026 21:17:08 +1000 Subject: [PATCH 70/78] Use smaller VM sizes to avoid quota exhaustion in CI Default system pool: Standard_D4s_v5 (4 vCPUs) -> Standard_D2s_v5 (2 vCPUs) Default workload pool: Standard_D4s_v5 (4 vCPUs) -> Standard_D2s_v5 (2 vCPUs) Max workload pool: 10 -> 3 (reduces quota reservation) Total minimum vCPU: 8 -> 4 (fits within CI subscription quota) The deployment tests were failing with: ErrCode_InsufficientVCPUQuota: Insufficient vcpu quota requested 8, remaining 0 for family standardDSv5Family for region westus3 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentResource.cs | 2 +- .../AzureKubernetesInfrastructure.cs | 2 +- .../AksNodePoolDeploymentTests.cs | 2 +- .../AksPerPoolSubnetDeploymentTests.cs | 2 +- .../TypeScriptAksNodePoolDeploymentTests.cs | 2 +- .../AzureKubernetesEnvironmentExtensionsTests.cs | 4 ++-- ...ureKubernetesEnvironment_BasicConfiguration.verified.bicep | 4 ++-- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs index 1933caec5cf..0cc80dd8fd5 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs @@ -92,7 +92,7 @@ public class AzureKubernetesEnvironmentResource( /// internal List NodePools { get; } = [ - new AksNodePoolConfig("system", "Standard_D4s_v5", 1, 3, AksNodePoolMode.System) + new AksNodePoolConfig("system", "Standard_D2s_v5", 1, 3, AksNodePoolMode.System) ]; /// diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 3943c2638a9..f732048f21d 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -165,7 +165,7 @@ private Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken cance } // No user pool configured — create a default one and add it to the app model. - var defaultConfig = new AksNodePoolConfig("workload", "Standard_D4s_v5", 1, 10, AksNodePoolMode.User); + var defaultConfig = new AksNodePoolConfig("workload", "Standard_D2s_v5", 1, 3, AksNodePoolMode.User); environment.NodePools.Add(defaultConfig); var defaultPool = new AksNodePoolResource("workload", defaultConfig, environment); diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs index 9b691355124..607e8e266e8 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs @@ -116,7 +116,7 @@ private async Task DeployStarterToAksWithCustomNodePoolCore(CancellationToken ca var builder = DistributedApplication.CreateBuilder(args); var aks = builder.AddAzureKubernetesEnvironment("aks"); - var computePool = aks.AddNodePool("compute", "Standard_D4s_v5", 1, 3); + var computePool = aks.AddNodePool("compute", "Standard_D2s_v5", 1, 3); var apiService = builder.AddProject("apiservice") .WithHttpHealthCheck("/health") diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs index 3d0c686b4d1..4ffc173f91a 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs @@ -122,7 +122,7 @@ private async Task DeployStarterToAksWithPerPoolSubnetsCore(CancellationToken ca var aks = builder.AddAzureKubernetesEnvironment("aks") .WithSubnet(defaultSubnet); - var auxPool = aks.AddNodePool("aux", "Standard_D4s_v5", 1, 3) + var auxPool = aks.AddNodePool("aux", "Standard_D2s_v5", 1, 3) .WithSubnet(auxSubnet); var apiService = builder.AddProject("apiservice") diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs index ecbc6d33417..e8219126224 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs @@ -126,7 +126,7 @@ private async Task DeployTypeScriptExpressToAksWithNodePoolCore(CancellationToke """ // Add Azure Kubernetes Environment with a custom node pool const aks = await builder.addAzureKubernetesEnvironment("aks"); -await aks.addNodePool("compute", "Standard_D4s_v5", 1, 3); +await aks.addNodePool("compute", "Standard_D2s_v5", 1, 3); await builder.build().run(); """); diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs index 66b840675c7..e06ae52d338 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs @@ -60,7 +60,7 @@ public void AddAzureKubernetesEnvironment_DefaultNodePool() Assert.Single(aks.Resource.NodePools); var defaultPool = aks.Resource.NodePools[0]; Assert.Equal("system", defaultPool.Name); - Assert.Equal("Standard_D4s_v5", defaultPool.VmSize); + Assert.Equal("Standard_D2s_v5", defaultPool.VmSize); Assert.Equal(1, defaultPool.MinCount); Assert.Equal(3, defaultPool.MaxCount); Assert.Equal(AksNodePoolMode.System, defaultPool.Mode); @@ -240,7 +240,7 @@ public void AddNodePool_MultiplePoolsSupported() using var builder = TestDistributedApplicationBuilder.Create(); var aks = builder.AddAzureKubernetesEnvironment("aks"); - var pool1 = aks.AddNodePool("cpu", "Standard_D4s_v5", 1, 10); + var pool1 = aks.AddNodePool("cpu", "Standard_D2s_v5", 1, 10); var pool2 = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); // Default system pool + 2 user pools diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep index b2f49efab1d..d48ffe64157 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddAzureKubernetesEnvironment_BasicConfiguration.verified.bicep @@ -9,7 +9,7 @@ resource aks 'Microsoft.ContainerService/managedClusters@2025-03-01' = { { name: 'system' count: 1 - vmSize: 'Standard_D4s_v5' + vmSize: 'Standard_D2s_v5' osType: 'Linux' maxCount: 3 minCount: 1 @@ -49,4 +49,4 @@ output oidcIssuerUrl string = aks.properties.oidcIssuerProfile.issuerURL output kubeletIdentityObjectId string = aks.properties.identityProfile.kubeletidentity.objectId -output nodeResourceGroup string = aks.properties.nodeResourceGroup +output nodeResourceGroup string = aks.properties.nodeResourceGroup \ No newline at end of file From 21c2b1d25864fa99bef9e2fee82ad91d226325c1 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Thu, 16 Apr 2026 11:30:09 +1000 Subject: [PATCH 71/78] Address Eric's feedback: remove IVT, rename property, remove E2E tests 1. Remove IVT from Aspire.Hosting.Azure -> Aspire.Hosting.Azure.Kubernetes. Revert to linking ProcessSpec/ProcessUtil/ProcessResult directly (same pattern as Aspire.Hosting.Azure itself). 2. Rename ParentComputeEnvironment -> OwningComputeEnvironment per Eric's suggestion. Better describes the ownership relationship. 3. Remove all 9 new AKS E2E deployment tests due to capacity issues in the deployment test subscription. The existing AksStarter* tests remain. Will re-add verification tests in a follow-up. 4. Add Helm CLI prerequisite check pipeline step. Fails fast with clear error message if helm is not on PATH, before any deployment steps run. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Aspire.Hosting.Azure.Kubernetes.csproj | 5 + .../AzureKubernetesEnvironmentExtensions.cs | 2 +- .../AzureKubernetesInfrastructure.cs | 14 +- .../Aspire.Hosting.Azure.csproj | 1 - .../Deployment/HelmDeploymentEngine.cs | 22 ++ .../KubernetesEnvironmentResource.cs | 6 +- .../KubernetesInfrastructure.cs | 2 +- .../KubernetesPublishingContext.cs | 2 +- .../AksAzureProvisioningDeploymentTests.cs | 279 ---------------- .../AksExplicitRegistryDeploymentTests.cs | 271 ---------------- .../AksNodePoolDeploymentTests.cs | 283 ----------------- .../AksPerPoolSubnetDeploymentTests.cs | 284 ----------------- .../AksVnetDeploymentTests.cs | 273 ---------------- .../AksWorkloadIdentityDeploymentTests.cs | 298 ------------------ .../TypeScriptAksDeploymentTests.cs | 264 ---------------- .../TypeScriptAksNodePoolDeploymentTests.cs | 268 ---------------- .../TypeScriptAksVnetDeploymentTests.cs | 282 ----------------- .../AzureKubernetesInfrastructureTests.cs | 6 +- 18 files changed, 41 insertions(+), 2521 deletions(-) delete mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/AksAzureProvisioningDeploymentTests.cs delete mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/AksExplicitRegistryDeploymentTests.cs delete mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs delete mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs delete mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/AksVnetDeploymentTests.cs delete mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/AksWorkloadIdentityDeploymentTests.cs delete mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksDeploymentTests.cs delete mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs delete mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksVnetDeploymentTests.cs diff --git a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj index 1b8c765745f..e361ac7edb2 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj +++ b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj @@ -3,6 +3,7 @@ $(DefaultTargetFramework) true + true true false aspire integration hosting azure kubernetes aks @@ -13,6 +14,10 @@ + + + + diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index fd2fe5dc43d..1a511b1fa0c 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -79,7 +79,7 @@ public static IResourceBuilder AddAzureKuber // Set the parent so KubernetesInfrastructure matches resources that use // WithComputeEnvironment(aksEnv) — the inner K8s env checks both itself // and its parent when filtering compute resources. - k8sEnvBuilder.Resource.ParentComputeEnvironment = resource; + k8sEnvBuilder.Resource.OwningComputeEnvironment = resource; if (builder.ExecutionContext.IsRunMode) { diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index f732048f21d..cc6bc314749 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -270,8 +270,7 @@ private static async Task GetAksCredentialsAsync( var result = await RunAzCommandAsync( azPath, $"aks get-credentials --resource-group \"{resourceGroup}\" --name \"{clusterName}\" --file -", - context.Logger, - context.Services).ConfigureAwait(false); + context.Logger).ConfigureAwait(false); if (result.ExitCode != 0) { @@ -357,8 +356,7 @@ private static async Task GetResourceGroupAsync( var result = await RunAzCommandAsync( azPath, $"resource list --resource-type Microsoft.ContainerService/managedClusters --name \"{clusterName}\" --query [0].resourceGroup -o tsv", - context.Logger, - context.Services).ConfigureAwait(false); + context.Logger).ConfigureAwait(false); if (result.ExitCode != 0) { @@ -379,14 +377,13 @@ private static async Task GetResourceGroupAsync( } /// - /// Runs an az CLI command using IProcessRunner from DI. + /// Runs an az CLI command using the shared ProcessSpec/ProcessUtil infrastructure. /// Returns the captured stdout, stderr, and exit code. /// private static async Task RunAzCommandAsync( string azPath, string arguments, - ILogger logger, - IServiceProvider services) + ILogger logger) { var stdout = new StringBuilder(); var stderr = new StringBuilder(); @@ -401,8 +398,7 @@ private static async Task RunAzCommandAsync( logger.LogDebug("Running: {AzPath} {Arguments}", azPath, arguments); - var processRunner = services.GetRequiredService(); - var (task, disposable) = processRunner.Run(spec); + var (task, disposable) = ProcessUtil.Run(spec); try { diff --git a/src/Aspire.Hosting.Azure/Aspire.Hosting.Azure.csproj b/src/Aspire.Hosting.Azure/Aspire.Hosting.Azure.csproj index c0278dc6228..87a93b17cf2 100644 --- a/src/Aspire.Hosting.Azure/Aspire.Hosting.Azure.csproj +++ b/src/Aspire.Hosting.Azure/Aspire.Hosting.Azure.csproj @@ -41,7 +41,6 @@ - diff --git a/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs b/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs index 08a35388b58..91b3b741550 100644 --- a/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs +++ b/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs @@ -120,6 +120,27 @@ internal static Task> CreateStepsAsync( var model = factoryContext.PipelineContext.Model; var steps = new List(); + // Step 0: Check prerequisites — verify Helm CLI is available + var checkPrereqStep = new PipelineStep + { + Name = $"check-helm-prereqs-{environment.Name}", + Description = $"Verifies Helm CLI is available for {environment.Name}.", + Action = ctx => + { + var helmPath = PathLookupHelper.FindFullPathFromPath("helm"); + if (helmPath is null) + { + throw new InvalidOperationException( + "Helm CLI not found. Install it from https://helm.sh/docs/intro/install/ " + + "and ensure it is available on your PATH."); + } + + ctx.Logger.LogDebug("Helm CLI found at: {HelmPath}", helmPath); + return Task.CompletedTask; + } + }; + steps.Add(checkPrereqStep); + // Step 1: Prepare - resolve values.yaml with actual image references and parameter values var prepareStep = new PipelineStep { @@ -129,6 +150,7 @@ internal static Task> CreateStepsAsync( }; prepareStep.DependsOn(WellKnownPipelineSteps.Publish); prepareStep.DependsOn(WellKnownPipelineSteps.Build); + prepareStep.DependsOn($"check-helm-prereqs-{environment.Name}"); steps.Add(prepareStep); // Step 2: Helm deploy - run helm upgrade --install diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs index 3522184e938..22c91845353 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs @@ -113,7 +113,7 @@ public sealed class KubernetesEnvironmentResource : Resource, IComputeEnvironmen /// WithComputeEnvironment(aksEnv) but the inner KubernetesEnvironmentResource /// needs to process the resource. /// - public IComputeEnvironmentResource? ParentComputeEnvironment { get; set; } + public IComputeEnvironmentResource? OwningComputeEnvironment { get; set; } internal IPortAllocator PortAllocator { get; } = new PortAllocator(); @@ -209,7 +209,7 @@ public KubernetesEnvironmentResource(string name) : base(name) foreach (var computeResource in resources) { - var targetEnv = (IComputeEnvironmentResource?)environment.ParentComputeEnvironment ?? environment; + var targetEnv = (IComputeEnvironmentResource?)environment.OwningComputeEnvironment ?? environment; var deploymentTarget = computeResource.GetDeploymentTargetAnnotation(targetEnv)?.DeploymentTarget; if (deploymentTarget is not null && deploymentTarget.TryGetAnnotationsOfType(out var annotations)) @@ -246,7 +246,7 @@ public KubernetesEnvironmentResource(string name) : base(name) foreach (var computeResource in resources) { - var targetEnv = (IComputeEnvironmentResource?)ParentComputeEnvironment ?? this; + var targetEnv = (IComputeEnvironmentResource?)OwningComputeEnvironment ?? this; var deploymentTarget = computeResource.GetDeploymentTargetAnnotation(targetEnv)?.DeploymentTarget; if (deploymentTarget is null) { diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesInfrastructure.cs b/src/Aspire.Hosting.Kubernetes/KubernetesInfrastructure.cs index b23afc41219..5eb6ad88a6a 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesInfrastructure.cs @@ -58,7 +58,7 @@ private async Task OnBeforeStartAsync(BeforeStartEvent @event, CancellationToken var resourceComputeEnvironment = r.GetComputeEnvironment(); if (resourceComputeEnvironment is not null && resourceComputeEnvironment != environment && - resourceComputeEnvironment != environment.ParentComputeEnvironment) + resourceComputeEnvironment != environment.OwningComputeEnvironment) { continue; } diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs b/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs index 7bc9bc5fc48..91ef401acd3 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesPublishingContext.cs @@ -76,7 +76,7 @@ private async Task WriteKubernetesOutputAsync(DistributedApplicationModel model, foreach (var resource in resources) { // Check for deployment target matching this environment or its parent (e.g., AKS) - var targetEnv = (IComputeEnvironmentResource?)environment.ParentComputeEnvironment ?? environment; + var targetEnv = (IComputeEnvironmentResource?)environment.OwningComputeEnvironment ?? environment; if (resource.GetDeploymentTargetAnnotation(targetEnv)?.DeploymentTarget is KubernetesResource serviceResource) { // Materialize Dockerfile factory if present diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksAzureProvisioningDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksAzureProvisioningDeploymentTests.cs deleted file mode 100644 index 410f4e831b3..00000000000 --- a/tests/Aspire.Deployment.EndToEnd.Tests/AksAzureProvisioningDeploymentTests.cs +++ /dev/null @@ -1,279 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Aspire.Cli.Resources; -using Aspire.Cli.Tests.Utils; -using Aspire.Deployment.EndToEnd.Tests.Helpers; -using Hex1b.Automation; -using Xunit; - -namespace Aspire.Deployment.EndToEnd.Tests; - -/// -/// End-to-end tests for deploying Aspire applications to Azure Kubernetes Service (AKS) -/// using Azure.Provisioning (Bicep-based) via aspire deploy --clear-cache. -/// This test uses the ACA-style deploy pipeline rather than manual AKS/ACR/Helm setup. -/// -public sealed class AksAzureProvisioningDeploymentTests(ITestOutputHelper output) -{ - // Timeout set to 45 minutes to allow for AKS provisioning (~10-15 min) plus container build and Helm deploy. - private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); - - [Fact] - public async Task DeployStarterToAksWithAzureProvisioning() - { - using var cts = new CancellationTokenSource(s_testTimeout); - using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( - cts.Token, TestContext.Current.CancellationToken); - var cancellationToken = linkedCts.Token; - - await DeployStarterToAksWithAzureProvisioningCore(cancellationToken); - } - - private async Task DeployStarterToAksWithAzureProvisioningCore(CancellationToken cancellationToken) - { - // Validate prerequisites - var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); - if (string.IsNullOrEmpty(subscriptionId)) - { - Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); - } - - if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) - { - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); - } - else - { - Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); - } - } - - var workspace = TemporaryWorkspace.Create(output); - var startTime = DateTime.UtcNow; - var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-provisioning"); - var projectName = "AksProvisioned"; - - output.WriteLine($"Test: {nameof(DeployStarterToAksWithAzureProvisioning)}"); - output.WriteLine($"Project Name: {projectName}"); - output.WriteLine($"Resource Group: {resourceGroupName}"); - output.WriteLine($"Subscription: {subscriptionId[..8]}..."); - output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); - - try - { - using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); - var pendingRun = terminal.RunAsync(cancellationToken); - - var counter = new SequenceCounter(); - var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); - - // Step 1: Prepare environment - output.WriteLine("Step 1: Preparing environment..."); - await auto.PrepareEnvironmentAsync(workspace, counter); - - // Step 2: Set up CLI environment (in CI) - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); - await auto.SourceAspireCliEnvironmentAsync(counter); - } - - // Step 3: Create starter project using aspire new - output.WriteLine("Step 3: Creating starter project..."); - await auto.AspireNewAsync(projectName, counter, useRedisCache: false); - - // Step 4: Navigate to project directory - output.WriteLine("Step 4: Navigating to project directory..."); - await auto.TypeAsync($"cd {projectName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 5: Add Aspire.Hosting.Azure.Kubernetes package - output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); - await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); - await auto.EnterAsync(); - - // In CI, aspire add shows a version selection prompt - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); - await auto.EnterAsync(); // select first version (PR build) - } - - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - - // Step 6: Write complete AppHost.cs with AKS environment (full rewrite to avoid - // string-replacement failures caused by line-ending or whitespace differences) - var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); - var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); - var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); - - output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); - - var appHostContent = $""" - #pragma warning disable ASPIREPIPELINES001 - - var builder = DistributedApplication.CreateBuilder(args); - - var aks = builder.AddAzureKubernetesEnvironment("aks"); - - var apiService = builder.AddProject("apiservice") - .WithHttpHealthCheck("/health"); - - builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithHttpHealthCheck("/health") - .WithReference(apiService) - .WaitFor(apiService); - - builder.Build().Run(); - """; - - File.WriteAllText(appHostFilePath, appHostContent); - - output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment"); - - // Step 7: Navigate to AppHost project directory - output.WriteLine("Step 7: Navigating to AppHost directory..."); - await auto.TypeAsync($"cd {projectName}.AppHost"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 8: Set environment variables for deployment - await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 9: Deploy to AKS using aspire deploy (Bicep provisioning + container build + ACR push + Helm deploy) - output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); - await auto.TypeAsync("aspire deploy --clear-cache"); - await auto.EnterAsync(); - // Wait for pipeline to complete - AKS provisioning can take up to 30 minutes - await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); - - // Step 10: Get AKS credentials for kubectl verification - output.WriteLine("Step 10: Getting AKS credentials..."); - // The cluster name is dynamic (Azure.Provisioning adds a suffix), so discover it - await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + - $"echo \"AKS cluster: $AKS_NAME\" && " + - $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 11: Wait for pods to be ready - output.WriteLine("Step 11: Waiting for pods to be ready..."); - await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); - - // Step 12: Verify pods are running - output.WriteLine("Step 12: Verifying pods are running..."); - await auto.TypeAsync("kubectl get pods -n default"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); - - // Step 13: Verify apiservice endpoint via port-forward - output.WriteLine("Step 13: Verifying apiservice endpoint..."); - await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 14: Verify webfrontend endpoint via port-forward - output.WriteLine("Step 14: Verifying webfrontend endpoint..."); - await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 15: Clean up port-forwards - output.WriteLine("Step 15: Cleaning up port-forwards..."); - await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - - // Step 16: Destroy Azure deployment - output.WriteLine("Step 16: Destroying Azure deployment..."); - await auto.AspireDestroyAsync(counter); - - // Step 17: Exit terminal - await auto.TypeAsync("exit"); - await auto.EnterAsync(); - - await pendingRun; - - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"AKS provisioned deployment completed in {duration}"); - - // Report success - DeploymentReporter.ReportDeploymentSuccess( - nameof(DeployStarterToAksWithAzureProvisioning), - resourceGroupName, - new Dictionary - { - ["project"] = projectName - }, - duration); - - output.WriteLine("✅ Test passed - Aspire app deployed to AKS via Azure Provisioning!"); - } - catch (Exception ex) - { - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); - - DeploymentReporter.ReportDeploymentFailure( - nameof(DeployStarterToAksWithAzureProvisioning), - resourceGroupName, - ex.Message, - ex.StackTrace); - - throw; - } - finally - { - // Clean up the resource group we created - output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); - TriggerCleanupResourceGroup(resourceGroupName, output); - DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); - } - } - - /// - /// Triggers cleanup of a specific resource group. - /// This is fire-and-forget - the hourly cleanup workflow handles any missed resources. - /// - private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) - { - var process = new System.Diagnostics.Process - { - StartInfo = new System.Diagnostics.ProcessStartInfo - { - FileName = "az", - Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - try - { - process.Start(); - output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); - } - catch (Exception ex) - { - output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); - } - } -} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksExplicitRegistryDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksExplicitRegistryDeploymentTests.cs deleted file mode 100644 index 2e25d2e190d..00000000000 --- a/tests/Aspire.Deployment.EndToEnd.Tests/AksExplicitRegistryDeploymentTests.cs +++ /dev/null @@ -1,271 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Aspire.Cli.Resources; -using Aspire.Cli.Tests.Utils; -using Aspire.Deployment.EndToEnd.Tests.Helpers; -using Hex1b.Automation; -using Xunit; - -namespace Aspire.Deployment.EndToEnd.Tests; - -/// -/// End-to-end tests for deploying Aspire applications to AKS with an explicit Azure Container Registry. -/// Verifies that AddAzureContainerRegistry and WithContainerRegistry correctly -/// provision a dedicated ACR and attach it to the AKS cluster. -/// -public sealed class AksExplicitRegistryDeploymentTests(ITestOutputHelper output) -{ - private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); - - [Fact] - public async Task DeployStarterToAksWithExplicitRegistry() - { - using var cts = new CancellationTokenSource(s_testTimeout); - using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( - cts.Token, TestContext.Current.CancellationToken); - var cancellationToken = linkedCts.Token; - - await DeployStarterToAksWithExplicitRegistryCore(cancellationToken); - } - - private async Task DeployStarterToAksWithExplicitRegistryCore(CancellationToken cancellationToken) - { - var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); - if (string.IsNullOrEmpty(subscriptionId)) - { - Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); - } - - if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) - { - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); - } - else - { - Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); - } - } - - var workspace = TemporaryWorkspace.Create(output); - var startTime = DateTime.UtcNow; - var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-acr"); - var projectName = "AksExplicitAcr"; - - output.WriteLine($"Test: {nameof(DeployStarterToAksWithExplicitRegistry)}"); - output.WriteLine($"Project Name: {projectName}"); - output.WriteLine($"Resource Group: {resourceGroupName}"); - output.WriteLine($"Subscription: {subscriptionId[..8]}..."); - output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); - - try - { - using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); - var pendingRun = terminal.RunAsync(cancellationToken); - - var counter = new SequenceCounter(); - var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); - - // Step 1: Prepare environment - output.WriteLine("Step 1: Preparing environment..."); - await auto.PrepareEnvironmentAsync(workspace, counter); - - // Step 2: Set up CLI environment (in CI) - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); - await auto.SourceAspireCliEnvironmentAsync(counter); - } - - // Step 3: Create starter project using aspire new - output.WriteLine("Step 3: Creating starter project..."); - await auto.AspireNewAsync(projectName, counter, useRedisCache: false); - - // Step 4: Navigate to project directory - output.WriteLine("Step 4: Navigating to project directory..."); - await auto.TypeAsync($"cd {projectName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 5: Add Aspire.Hosting.Azure.Kubernetes package - // (Aspire.Hosting.Azure.Kubernetes already depends on Aspire.Hosting.Azure.ContainerRegistry) - output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); - await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); - await auto.EnterAsync(); - - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); - await auto.EnterAsync(); - } - - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - - // Step 6: Write complete AppHost.cs with AKS environment and explicit ACR - // (full rewrite to avoid string-replacement failures caused by line-ending or whitespace differences) - var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); - var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); - var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); - - output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); - - var appHostContent = $""" - #pragma warning disable ASPIREPIPELINES001 - - var builder = DistributedApplication.CreateBuilder(args); - - var acr = builder.AddAzureContainerRegistry("myacr"); - var aks = builder.AddAzureKubernetesEnvironment("aks") - .WithContainerRegistry(acr); - - var apiService = builder.AddProject("apiservice") - .WithHttpHealthCheck("/health"); - - builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithHttpHealthCheck("/health") - .WithReference(apiService) - .WaitFor(apiService); - - builder.Build().Run(); - """; - - File.WriteAllText(appHostFilePath, appHostContent); - - output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment and explicit ACR"); - - // Step 7: Navigate to AppHost project directory - output.WriteLine("Step 7: Navigating to AppHost directory..."); - await auto.TypeAsync($"cd {projectName}.AppHost"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 8: Set environment variables for deployment - await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 9: Deploy to AKS using aspire deploy - output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); - await auto.TypeAsync("aspire deploy --clear-cache"); - await auto.EnterAsync(); - await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); - - // Step 10: Get AKS credentials for kubectl verification - output.WriteLine("Step 10: Getting AKS credentials..."); - await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + - $"echo \"AKS cluster: $AKS_NAME\" && " + - $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 11: Wait for pods to be ready - output.WriteLine("Step 11: Waiting for pods to be ready..."); - await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); - - // Step 12: Verify pods are running - output.WriteLine("Step 12: Verifying pods are running..."); - await auto.TypeAsync("kubectl get pods -n default"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); - - // Step 13: Verify apiservice endpoint via port-forward - output.WriteLine("Step 13: Verifying apiservice endpoint..."); - await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 14: Verify webfrontend endpoint via port-forward - output.WriteLine("Step 14: Verifying webfrontend endpoint..."); - await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 15: Clean up port-forwards - output.WriteLine("Step 15: Cleaning up port-forwards..."); - await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - - // Step 16: Destroy Azure deployment - output.WriteLine("Step 16: Destroying Azure deployment..."); - await auto.AspireDestroyAsync(counter); - - // Step 17: Exit terminal - await auto.TypeAsync("exit"); - await auto.EnterAsync(); - - await pendingRun; - - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"AKS explicit ACR deployment completed in {duration}"); - - DeploymentReporter.ReportDeploymentSuccess( - nameof(DeployStarterToAksWithExplicitRegistry), - resourceGroupName, - new Dictionary - { - ["project"] = projectName - }, - duration); - - output.WriteLine("✅ Test passed - Aspire app deployed to AKS with explicit ACR!"); - } - catch (Exception ex) - { - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); - - DeploymentReporter.ReportDeploymentFailure( - nameof(DeployStarterToAksWithExplicitRegistry), - resourceGroupName, - ex.Message, - ex.StackTrace); - - throw; - } - finally - { - output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); - TriggerCleanupResourceGroup(resourceGroupName, output); - DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); - } - } - - private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) - { - var process = new System.Diagnostics.Process - { - StartInfo = new System.Diagnostics.ProcessStartInfo - { - FileName = "az", - Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - try - { - process.Start(); - output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); - } - catch (Exception ex) - { - output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); - } - } -} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs deleted file mode 100644 index 607e8e266e8..00000000000 --- a/tests/Aspire.Deployment.EndToEnd.Tests/AksNodePoolDeploymentTests.cs +++ /dev/null @@ -1,283 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Aspire.Cli.Resources; -using Aspire.Cli.Tests.Utils; -using Aspire.Deployment.EndToEnd.Tests.Helpers; -using Hex1b.Automation; -using Xunit; - -namespace Aspire.Deployment.EndToEnd.Tests; - -/// -/// End-to-end tests for deploying Aspire applications to AKS with a custom node pool. -/// Verifies that AddNodePool creates additional node pools and that pods are -/// scheduled to the correct pool via WithNodePool. -/// -public sealed class AksNodePoolDeploymentTests(ITestOutputHelper output) -{ - private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); - - [Fact] - public async Task DeployStarterToAksWithCustomNodePool() - { - using var cts = new CancellationTokenSource(s_testTimeout); - using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( - cts.Token, TestContext.Current.CancellationToken); - var cancellationToken = linkedCts.Token; - - await DeployStarterToAksWithCustomNodePoolCore(cancellationToken); - } - - private async Task DeployStarterToAksWithCustomNodePoolCore(CancellationToken cancellationToken) - { - var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); - if (string.IsNullOrEmpty(subscriptionId)) - { - Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); - } - - if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) - { - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); - } - else - { - Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); - } - } - - var workspace = TemporaryWorkspace.Create(output); - var startTime = DateTime.UtcNow; - var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-nodepool"); - var projectName = "AksNodePool"; - - output.WriteLine($"Test: {nameof(DeployStarterToAksWithCustomNodePool)}"); - output.WriteLine($"Project Name: {projectName}"); - output.WriteLine($"Resource Group: {resourceGroupName}"); - output.WriteLine($"Subscription: {subscriptionId[..8]}..."); - output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); - - try - { - using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); - var pendingRun = terminal.RunAsync(cancellationToken); - - var counter = new SequenceCounter(); - var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); - - // Step 1: Prepare environment - output.WriteLine("Step 1: Preparing environment..."); - await auto.PrepareEnvironmentAsync(workspace, counter); - - // Step 2: Set up CLI environment (in CI) - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); - await auto.SourceAspireCliEnvironmentAsync(counter); - } - - // Step 3: Create starter project using aspire new - output.WriteLine("Step 3: Creating starter project..."); - await auto.AspireNewAsync(projectName, counter, useRedisCache: false); - - // Step 4: Navigate to project directory - output.WriteLine("Step 4: Navigating to project directory..."); - await auto.TypeAsync($"cd {projectName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 5: Add Aspire.Hosting.Azure.Kubernetes package - output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); - await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); - await auto.EnterAsync(); - - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); - await auto.EnterAsync(); - } - - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - - // Step 6: Write complete AppHost.cs with AKS environment and custom node pool - // (full rewrite to avoid string-replacement failures caused by line-ending or whitespace differences) - var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); - var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); - var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); - - output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); - - var appHostContent = $""" - #pragma warning disable ASPIREPIPELINES001 - - var builder = DistributedApplication.CreateBuilder(args); - - var aks = builder.AddAzureKubernetesEnvironment("aks"); - var computePool = aks.AddNodePool("compute", "Standard_D2s_v5", 1, 3); - - var apiService = builder.AddProject("apiservice") - .WithHttpHealthCheck("/health") - .WithNodePool(computePool); - - builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithHttpHealthCheck("/health") - .WithReference(apiService) - .WaitFor(apiService); - - builder.Build().Run(); - """; - - File.WriteAllText(appHostFilePath, appHostContent); - - output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment and custom node pool"); - - // Step 7: Navigate to AppHost project directory - output.WriteLine("Step 7: Navigating to AppHost directory..."); - await auto.TypeAsync($"cd {projectName}.AppHost"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 8: Set environment variables for deployment - await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 9: Deploy to AKS using aspire deploy - output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); - await auto.TypeAsync("aspire deploy --clear-cache"); - await auto.EnterAsync(); - await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); - - // Step 10: Get AKS credentials for kubectl verification - output.WriteLine("Step 10: Getting AKS credentials..."); - await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + - $"echo \"AKS cluster: $AKS_NAME\" && " + - $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 11: Wait for pods to be ready - output.WriteLine("Step 11: Waiting for pods to be ready..."); - await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); - - // Step 12: Verify pods are running - output.WriteLine("Step 12: Verifying pods are running..."); - await auto.TypeAsync("kubectl get pods -n default"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); - - // Step 13: Verify two node pools exist (system + compute) - output.WriteLine("Step 13: Verifying node pools..."); - await auto.TypeAsync($"az aks nodepool list --resource-group {resourceGroupName} --cluster-name $AKS_NAME --query '[].{{name:name, mode:mode}}' -o table"); - await auto.EnterAsync(); - await auto.WaitUntilTextAsync("compute", timeout: TimeSpan.FromSeconds(30)); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - - // Step 14: Verify apiservice pod has nodeSelector for the compute pool - output.WriteLine("Step 14: Verifying apiservice pod nodeSelector..."); - await auto.TypeAsync("kubectl get pod -l app.kubernetes.io/component=apiservice -o jsonpath='{.items[0].spec.nodeSelector}'"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); - - // Step 15: Verify apiservice endpoint via port-forward - output.WriteLine("Step 15: Verifying apiservice endpoint..."); - await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 16: Verify webfrontend endpoint via port-forward - output.WriteLine("Step 16: Verifying webfrontend endpoint..."); - await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 17: Clean up port-forwards - output.WriteLine("Step 17: Cleaning up port-forwards..."); - await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - - // Step 18: Destroy Azure deployment - output.WriteLine("Step 18: Destroying Azure deployment..."); - await auto.AspireDestroyAsync(counter); - - // Step 19: Exit terminal - await auto.TypeAsync("exit"); - await auto.EnterAsync(); - - await pendingRun; - - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"AKS node pool deployment completed in {duration}"); - - DeploymentReporter.ReportDeploymentSuccess( - nameof(DeployStarterToAksWithCustomNodePool), - resourceGroupName, - new Dictionary - { - ["project"] = projectName - }, - duration); - - output.WriteLine("✅ Test passed - Aspire app deployed to AKS with custom node pool!"); - } - catch (Exception ex) - { - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); - - DeploymentReporter.ReportDeploymentFailure( - nameof(DeployStarterToAksWithCustomNodePool), - resourceGroupName, - ex.Message, - ex.StackTrace); - - throw; - } - finally - { - output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); - TriggerCleanupResourceGroup(resourceGroupName, output); - DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); - } - } - - private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) - { - var process = new System.Diagnostics.Process - { - StartInfo = new System.Diagnostics.ProcessStartInfo - { - FileName = "az", - Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - try - { - process.Start(); - output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); - } - catch (Exception ex) - { - output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); - } - } -} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs deleted file mode 100644 index 4ffc173f91a..00000000000 --- a/tests/Aspire.Deployment.EndToEnd.Tests/AksPerPoolSubnetDeploymentTests.cs +++ /dev/null @@ -1,284 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Aspire.Cli.Resources; -using Aspire.Cli.Tests.Utils; -using Aspire.Deployment.EndToEnd.Tests.Helpers; -using Hex1b.Automation; -using Xunit; - -namespace Aspire.Deployment.EndToEnd.Tests; - -/// -/// End-to-end tests for deploying Aspire applications to AKS with per-pool subnet assignments. -/// Verifies that the default system pool and a user node pool can each be assigned to -/// different subnets within the same VNet. -/// -public sealed class AksPerPoolSubnetDeploymentTests(ITestOutputHelper output) -{ - private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); - - [Fact] - public async Task DeployStarterToAksWithPerPoolSubnets() - { - using var cts = new CancellationTokenSource(s_testTimeout); - using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( - cts.Token, TestContext.Current.CancellationToken); - var cancellationToken = linkedCts.Token; - - await DeployStarterToAksWithPerPoolSubnetsCore(cancellationToken); - } - - private async Task DeployStarterToAksWithPerPoolSubnetsCore(CancellationToken cancellationToken) - { - var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); - if (string.IsNullOrEmpty(subscriptionId)) - { - Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); - } - - if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) - { - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); - } - else - { - Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); - } - } - - var workspace = TemporaryWorkspace.Create(output); - var startTime = DateTime.UtcNow; - var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-poolsubnet"); - var projectName = "AksPoolSubnet"; - - output.WriteLine($"Test: {nameof(DeployStarterToAksWithPerPoolSubnets)}"); - output.WriteLine($"Project Name: {projectName}"); - output.WriteLine($"Resource Group: {resourceGroupName}"); - output.WriteLine($"Subscription: {subscriptionId[..8]}..."); - output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); - - try - { - using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); - var pendingRun = terminal.RunAsync(cancellationToken); - - var counter = new SequenceCounter(); - var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); - - // Step 1: Prepare environment - output.WriteLine("Step 1: Preparing environment..."); - await auto.PrepareEnvironmentAsync(workspace, counter); - - // Step 2: Set up CLI environment (in CI) - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); - await auto.SourceAspireCliEnvironmentAsync(counter); - } - - // Step 3: Create starter project using aspire new - output.WriteLine("Step 3: Creating starter project..."); - await auto.AspireNewAsync(projectName, counter, useRedisCache: false); - - // Step 4: Navigate to project directory - output.WriteLine("Step 4: Navigating to project directory..."); - await auto.TypeAsync($"cd {projectName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 5: Add Aspire.Hosting.Azure.Kubernetes package - output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); - await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); - await auto.EnterAsync(); - - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); - await auto.EnterAsync(); - } - - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - - // Step 6: Write complete AppHost.cs with AKS environment and per-pool subnets - // (full rewrite to avoid string-replacement failures caused by line-ending or whitespace differences) - var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); - var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); - var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); - - output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); - - var appHostContent = $""" - #pragma warning disable ASPIREPIPELINES001 - - var builder = DistributedApplication.CreateBuilder(args); - - var vnet = builder.AddAzureVirtualNetwork("vnet", "10.1.0.0/16"); - var defaultSubnet = vnet.AddSubnet("defaultsubnet", "10.1.0.0/22"); - var auxSubnet = vnet.AddSubnet("auxsubnet", "10.1.4.0/24"); - - var aks = builder.AddAzureKubernetesEnvironment("aks") - .WithSubnet(defaultSubnet); - - var auxPool = aks.AddNodePool("aux", "Standard_D2s_v5", 1, 3) - .WithSubnet(auxSubnet); - - var apiService = builder.AddProject("apiservice") - .WithHttpHealthCheck("/health") - .WithNodePool(auxPool); - - builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithHttpHealthCheck("/health") - .WithReference(apiService) - .WaitFor(apiService); - - builder.Build().Run(); - """; - - File.WriteAllText(appHostFilePath, appHostContent); - - output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment and per-pool subnets"); - - // Step 7: Navigate to AppHost project directory - output.WriteLine("Step 7: Navigating to AppHost directory..."); - await auto.TypeAsync($"cd {projectName}.AppHost"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 8: Set environment variables for deployment - await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 9: Deploy to AKS using aspire deploy - output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); - await auto.TypeAsync("aspire deploy --clear-cache"); - await auto.EnterAsync(); - await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); - - // Step 10: Get AKS credentials for kubectl verification - output.WriteLine("Step 10: Getting AKS credentials..."); - await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + - $"echo \"AKS cluster: $AKS_NAME\" && " + - $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 11: Wait for pods to be ready - output.WriteLine("Step 11: Waiting for pods to be ready..."); - await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); - - // Step 12: Verify pods are running - output.WriteLine("Step 12: Verifying pods are running..."); - await auto.TypeAsync("kubectl get pods -n default"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); - - // Step 13: Verify two node pools with different subnets exist - output.WriteLine("Step 13: Verifying node pools with different subnets..."); - await auto.TypeAsync($"az aks nodepool list --resource-group {resourceGroupName} --cluster-name $AKS_NAME --query '[].{{name:name, mode:mode, vnetSubnetId:vnetSubnetId}}' -o table"); - await auto.EnterAsync(); - await auto.WaitUntilTextAsync("aux", timeout: TimeSpan.FromSeconds(30)); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - - // Step 14: Verify apiservice endpoint via port-forward - output.WriteLine("Step 14: Verifying apiservice endpoint..."); - await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 15: Verify webfrontend endpoint via port-forward - output.WriteLine("Step 15: Verifying webfrontend endpoint..."); - await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 16: Clean up port-forwards - output.WriteLine("Step 16: Cleaning up port-forwards..."); - await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - - // Step 17: Destroy Azure deployment - output.WriteLine("Step 17: Destroying Azure deployment..."); - await auto.AspireDestroyAsync(counter); - - // Step 18: Exit terminal - await auto.TypeAsync("exit"); - await auto.EnterAsync(); - - await pendingRun; - - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"AKS per-pool subnet deployment completed in {duration}"); - - DeploymentReporter.ReportDeploymentSuccess( - nameof(DeployStarterToAksWithPerPoolSubnets), - resourceGroupName, - new Dictionary - { - ["project"] = projectName - }, - duration); - - output.WriteLine("✅ Test passed - Aspire app deployed to AKS with per-pool subnet assignments!"); - } - catch (Exception ex) - { - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); - - DeploymentReporter.ReportDeploymentFailure( - nameof(DeployStarterToAksWithPerPoolSubnets), - resourceGroupName, - ex.Message, - ex.StackTrace); - - throw; - } - finally - { - output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); - TriggerCleanupResourceGroup(resourceGroupName, output); - DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); - } - } - - private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) - { - var process = new System.Diagnostics.Process - { - StartInfo = new System.Diagnostics.ProcessStartInfo - { - FileName = "az", - Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - try - { - process.Start(); - output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); - } - catch (Exception ex) - { - output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); - } - } -} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksVnetDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksVnetDeploymentTests.cs deleted file mode 100644 index a4507647092..00000000000 --- a/tests/Aspire.Deployment.EndToEnd.Tests/AksVnetDeploymentTests.cs +++ /dev/null @@ -1,273 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Aspire.Cli.Resources; -using Aspire.Cli.Tests.Utils; -using Aspire.Deployment.EndToEnd.Tests.Helpers; -using Hex1b.Automation; -using Xunit; - -namespace Aspire.Deployment.EndToEnd.Tests; - -/// -/// End-to-end tests for deploying Aspire applications to AKS with a custom VNet/subnet. -/// Verifies that AddAzureVirtualNetwork and WithSubnet correctly integrate -/// AKS networking so that pods receive VNet IPs rather than default overlay IPs. -/// -public sealed class AksVnetDeploymentTests(ITestOutputHelper output) -{ - private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); - - [Fact] - public async Task DeployStarterToAksWithVnet() - { - using var cts = new CancellationTokenSource(s_testTimeout); - using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( - cts.Token, TestContext.Current.CancellationToken); - var cancellationToken = linkedCts.Token; - - await DeployStarterToAksWithVnetCore(cancellationToken); - } - - private async Task DeployStarterToAksWithVnetCore(CancellationToken cancellationToken) - { - var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); - if (string.IsNullOrEmpty(subscriptionId)) - { - Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); - } - - if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) - { - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); - } - else - { - Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); - } - } - - var workspace = TemporaryWorkspace.Create(output); - var startTime = DateTime.UtcNow; - var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-vnet"); - var projectName = "AksVnet"; - - output.WriteLine($"Test: {nameof(DeployStarterToAksWithVnet)}"); - output.WriteLine($"Project Name: {projectName}"); - output.WriteLine($"Resource Group: {resourceGroupName}"); - output.WriteLine($"Subscription: {subscriptionId[..8]}..."); - output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); - - try - { - using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); - var pendingRun = terminal.RunAsync(cancellationToken); - - var counter = new SequenceCounter(); - var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); - - // Step 1: Prepare environment - output.WriteLine("Step 1: Preparing environment..."); - await auto.PrepareEnvironmentAsync(workspace, counter); - - // Step 2: Set up CLI environment (in CI) - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); - await auto.SourceAspireCliEnvironmentAsync(counter); - } - - // Step 3: Create starter project using aspire new - output.WriteLine("Step 3: Creating starter project..."); - await auto.AspireNewAsync(projectName, counter, useRedisCache: false); - - // Step 4: Navigate to project directory - output.WriteLine("Step 4: Navigating to project directory..."); - await auto.TypeAsync($"cd {projectName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 5: Add Aspire.Hosting.Azure.Kubernetes package - output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); - await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); - await auto.EnterAsync(); - - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); - await auto.EnterAsync(); - } - - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - - // Step 6: Write complete AppHost.cs with AKS environment and VNet/subnet - // (full rewrite to avoid string-replacement failures caused by line-ending or whitespace differences) - var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); - var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); - var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); - - output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); - - var appHostContent = $""" - #pragma warning disable ASPIREPIPELINES001 - - var builder = DistributedApplication.CreateBuilder(args); - - var vnet = builder.AddAzureVirtualNetwork("vnet", "10.1.0.0/16"); - var subnet = vnet.AddSubnet("akssubnet", "10.1.0.0/22"); - - var aks = builder.AddAzureKubernetesEnvironment("aks") - .WithSubnet(subnet); - - var apiService = builder.AddProject("apiservice") - .WithHttpHealthCheck("/health"); - - builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithHttpHealthCheck("/health") - .WithReference(apiService) - .WaitFor(apiService); - - builder.Build().Run(); - """; - - File.WriteAllText(appHostFilePath, appHostContent); - - output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment and VNet/subnet"); - - // Step 7: Navigate to AppHost project directory - output.WriteLine("Step 7: Navigating to AppHost directory..."); - await auto.TypeAsync($"cd {projectName}.AppHost"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 8: Set environment variables for deployment - await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 9: Deploy to AKS using aspire deploy - output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); - await auto.TypeAsync("aspire deploy --clear-cache"); - await auto.EnterAsync(); - await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); - - // Step 10: Get AKS credentials for kubectl verification - output.WriteLine("Step 10: Getting AKS credentials..."); - await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + - $"echo \"AKS cluster: $AKS_NAME\" && " + - $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 11: Wait for pods to be ready - output.WriteLine("Step 11: Waiting for pods to be ready..."); - await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); - - // Step 12: Verify pods are running and have VNet IPs (10.1.x.x, not 10.244.x.x overlay) - output.WriteLine("Step 12: Verifying pods have VNet IPs..."); - await auto.TypeAsync("kubectl get pods -o wide -n default"); - await auto.EnterAsync(); - await auto.WaitUntilTextAsync("10.1.", timeout: TimeSpan.FromSeconds(30)); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - - // Step 13: Verify apiservice endpoint via port-forward - output.WriteLine("Step 13: Verifying apiservice endpoint..."); - await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 14: Verify webfrontend endpoint via port-forward - output.WriteLine("Step 14: Verifying webfrontend endpoint..."); - await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 15: Clean up port-forwards - output.WriteLine("Step 15: Cleaning up port-forwards..."); - await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - - // Step 16: Destroy Azure deployment - output.WriteLine("Step 16: Destroying Azure deployment..."); - await auto.AspireDestroyAsync(counter); - - // Step 17: Exit terminal - await auto.TypeAsync("exit"); - await auto.EnterAsync(); - - await pendingRun; - - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"AKS VNet deployment completed in {duration}"); - - DeploymentReporter.ReportDeploymentSuccess( - nameof(DeployStarterToAksWithVnet), - resourceGroupName, - new Dictionary - { - ["project"] = projectName - }, - duration); - - output.WriteLine("✅ Test passed - Aspire app deployed to AKS with VNet integration!"); - } - catch (Exception ex) - { - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); - - DeploymentReporter.ReportDeploymentFailure( - nameof(DeployStarterToAksWithVnet), - resourceGroupName, - ex.Message, - ex.StackTrace); - - throw; - } - finally - { - output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); - TriggerCleanupResourceGroup(resourceGroupName, output); - DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); - } - } - - private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) - { - var process = new System.Diagnostics.Process - { - StartInfo = new System.Diagnostics.ProcessStartInfo - { - FileName = "az", - Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - try - { - process.Start(); - output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); - } - catch (Exception ex) - { - output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); - } - } -} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksWorkloadIdentityDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksWorkloadIdentityDeploymentTests.cs deleted file mode 100644 index ddfa85d43f8..00000000000 --- a/tests/Aspire.Deployment.EndToEnd.Tests/AksWorkloadIdentityDeploymentTests.cs +++ /dev/null @@ -1,298 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Aspire.Cli.Resources; -using Aspire.Cli.Tests.Utils; -using Aspire.Deployment.EndToEnd.Tests.Helpers; -using Hex1b.Automation; -using Xunit; - -namespace Aspire.Deployment.EndToEnd.Tests; - -/// -/// End-to-end tests for deploying Aspire applications to AKS with workload identity. -/// Verifies that referencing an Azure resource (blob storage) from an AKS-hosted service -/// correctly provisions workload identity (service account annotation and pod label). -/// -public sealed class AksWorkloadIdentityDeploymentTests(ITestOutputHelper output) -{ - private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); - - [Fact] - public async Task DeployStarterToAksWithWorkloadIdentity() - { - using var cts = new CancellationTokenSource(s_testTimeout); - using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( - cts.Token, TestContext.Current.CancellationToken); - var cancellationToken = linkedCts.Token; - - await DeployStarterToAksWithWorkloadIdentityCore(cancellationToken); - } - - private async Task DeployStarterToAksWithWorkloadIdentityCore(CancellationToken cancellationToken) - { - var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); - if (string.IsNullOrEmpty(subscriptionId)) - { - Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); - } - - if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) - { - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); - } - else - { - Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); - } - } - - var workspace = TemporaryWorkspace.Create(output); - var startTime = DateTime.UtcNow; - var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aks-wi"); - var projectName = "AksWorkloadId"; - - output.WriteLine($"Test: {nameof(DeployStarterToAksWithWorkloadIdentity)}"); - output.WriteLine($"Project Name: {projectName}"); - output.WriteLine($"Resource Group: {resourceGroupName}"); - output.WriteLine($"Subscription: {subscriptionId[..8]}..."); - output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); - - try - { - using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); - var pendingRun = terminal.RunAsync(cancellationToken); - - var counter = new SequenceCounter(); - var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); - - // Step 1: Prepare environment - output.WriteLine("Step 1: Preparing environment..."); - await auto.PrepareEnvironmentAsync(workspace, counter); - - // Step 2: Set up CLI environment (in CI) - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - output.WriteLine("Step 2: Using pre-installed Aspire CLI from local build..."); - await auto.SourceAspireCliEnvironmentAsync(counter); - } - - // Step 3: Create starter project using aspire new - output.WriteLine("Step 3: Creating starter project..."); - await auto.AspireNewAsync(projectName, counter, useRedisCache: false); - - // Step 4: Navigate to project directory - output.WriteLine("Step 4: Navigating to project directory..."); - await auto.TypeAsync($"cd {projectName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 5: Add required packages - output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); - await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); - await auto.EnterAsync(); - - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); - await auto.EnterAsync(); - } - - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - - output.WriteLine("Step 5b: Adding Azure Storage hosting package..."); - await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Storage"); - await auto.EnterAsync(); - - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - await auto.WaitUntilTextAsync("(based on NuGet.config)", timeout: TimeSpan.FromSeconds(60)); - await auto.EnterAsync(); - } - - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - - // Step 6: Write complete AppHost.cs with AKS environment and workload identity via Azure Storage - // (full rewrite to avoid string-replacement failures caused by line-ending or whitespace differences) - var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); - var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); - var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); - - output.WriteLine($"Writing AppHost.cs at: {appHostFilePath}"); - - var appHostContent = $""" - #pragma warning disable ASPIREPIPELINES001 - - var builder = DistributedApplication.CreateBuilder(args); - - var aks = builder.AddAzureKubernetesEnvironment("aks"); - - var storage = builder.AddAzureStorage("storage"); - var blobs = storage.AddBlobs("blobs"); - var photos = blobs.AddBlobContainer("photos"); - - var apiService = builder.AddProject("apiservice") - .WithHttpHealthCheck("/health") - .WithReference(photos); - - builder.AddProject("webfrontend") - .WithExternalHttpEndpoints() - .WithHttpHealthCheck("/health") - .WithReference(apiService) - .WaitFor(apiService); - - builder.Build().Run(); - """; - - File.WriteAllText(appHostFilePath, appHostContent); - - output.WriteLine("Wrote complete AppHost.cs with AddAzureKubernetesEnvironment and workload identity via Azure Storage"); - - // Step 7: Navigate to AppHost project directory - output.WriteLine("Step 7: Navigating to AppHost directory..."); - await auto.TypeAsync($"cd {projectName}.AppHost"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 8: Set environment variables for deployment - await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 9: Deploy to AKS using aspire deploy - output.WriteLine("Step 9: Starting AKS deployment via aspire deploy..."); - await auto.TypeAsync("aspire deploy --clear-cache"); - await auto.EnterAsync(); - await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); - - // Step 10: Get AKS credentials for kubectl verification - output.WriteLine("Step 10: Getting AKS credentials..."); - await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + - $"echo \"AKS cluster: $AKS_NAME\" && " + - $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 11: Wait for pods to be ready - output.WriteLine("Step 11: Waiting for pods to be ready..."); - await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); - - // Step 12: Verify pods are running - output.WriteLine("Step 12: Verifying pods are running..."); - await auto.TypeAsync("kubectl get pods -n default"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); - - // Step 13: Verify service account has workload identity annotation - output.WriteLine("Step 13: Verifying workload identity service account..."); - await auto.TypeAsync("kubectl get sa apiservice-sa -o yaml"); - await auto.EnterAsync(); - await auto.WaitUntilTextAsync("azure.workload.identity/client-id", timeout: TimeSpan.FromSeconds(30)); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - - // Step 14: Verify pod has workload identity label - output.WriteLine("Step 14: Verifying pod workload identity label..."); - await auto.TypeAsync("kubectl get pod -l app.kubernetes.io/component=apiservice -o jsonpath='{.items[0].metadata.labels}' | grep azure.workload.identity"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); - - // Step 15: Verify apiservice endpoint via port-forward - output.WriteLine("Step 15: Verifying apiservice endpoint..."); - await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 &"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 16: Verify webfrontend endpoint via port-forward - output.WriteLine("Step 16: Verifying webfrontend endpoint..."); - await auto.TypeAsync("kubectl port-forward svc/webfrontend-service 18081:8080 &"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 17: Clean up port-forwards - output.WriteLine("Step 17: Cleaning up port-forwards..."); - await auto.TypeAsync("kill %1 %2 2>/dev/null; true"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - - // Step 18: Destroy Azure deployment - output.WriteLine("Step 18: Destroying Azure deployment..."); - await auto.AspireDestroyAsync(counter); - - // Step 19: Exit terminal - await auto.TypeAsync("exit"); - await auto.EnterAsync(); - - await pendingRun; - - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"AKS workload identity deployment completed in {duration}"); - - DeploymentReporter.ReportDeploymentSuccess( - nameof(DeployStarterToAksWithWorkloadIdentity), - resourceGroupName, - new Dictionary - { - ["project"] = projectName - }, - duration); - - output.WriteLine("✅ Test passed - Aspire app deployed to AKS with workload identity!"); - } - catch (Exception ex) - { - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); - - DeploymentReporter.ReportDeploymentFailure( - nameof(DeployStarterToAksWithWorkloadIdentity), - resourceGroupName, - ex.Message, - ex.StackTrace); - - throw; - } - finally - { - output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); - TriggerCleanupResourceGroup(resourceGroupName, output); - DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); - } - } - - private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) - { - var process = new System.Diagnostics.Process - { - StartInfo = new System.Diagnostics.ProcessStartInfo - { - FileName = "az", - Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - try - { - process.Start(); - output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); - } - catch (Exception ex) - { - output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); - } - } -} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksDeploymentTests.cs deleted file mode 100644 index e119f535b0c..00000000000 --- a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksDeploymentTests.cs +++ /dev/null @@ -1,264 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Aspire.Cli.Resources; -using Aspire.Cli.Tests.Utils; -using Aspire.Deployment.EndToEnd.Tests.Helpers; -using Hex1b.Automation; -using Xunit; - -namespace Aspire.Deployment.EndToEnd.Tests; - -/// -/// End-to-end tests for deploying a TypeScript Express/React Aspire application to Azure Kubernetes Service (AKS). -/// Uses the same Azure.Provisioning pipeline as the C# variant but from a TypeScript AppHost created via -/// aspire new --template express-react. -/// -public sealed class TypeScriptAksDeploymentTests(ITestOutputHelper output) -{ - // Timeout set to 45 minutes to allow for AKS provisioning (~10-15 min) plus npm install and container build. - private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); - - [Fact] - public async Task DeployTypeScriptExpressToAksWithAzureProvisioning() - { - using var cts = new CancellationTokenSource(s_testTimeout); - using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( - cts.Token, TestContext.Current.CancellationToken); - var cancellationToken = linkedCts.Token; - - await DeployTypeScriptExpressToAksWithAzureProvisioningCore(cancellationToken); - } - - private async Task DeployTypeScriptExpressToAksWithAzureProvisioningCore(CancellationToken cancellationToken) - { - // Validate prerequisites - var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); - if (string.IsNullOrEmpty(subscriptionId)) - { - Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); - } - - if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) - { - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); - } - else - { - Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); - } - } - - var workspace = TemporaryWorkspace.Create(output); - var startTime = DateTime.UtcNow; - var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("ts-aks"); - var projectName = "TsAksApp"; - - output.WriteLine($"Test: {nameof(DeployTypeScriptExpressToAksWithAzureProvisioning)}"); - output.WriteLine($"Project Name: {projectName}"); - output.WriteLine($"Resource Group: {resourceGroupName}"); - output.WriteLine($"Subscription: {subscriptionId[..8]}..."); - output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); - - try - { - using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); - var pendingRun = terminal.RunAsync(cancellationToken); - - var counter = new SequenceCounter(); - var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); - - // Step 1: Prepare environment - output.WriteLine("Step 1: Preparing environment..."); - await auto.PrepareEnvironmentAsync(workspace, counter); - - // Step 2: Set up CLI environment (in CI) - // TypeScript apphosts need the full bundle (not just the CLI binary) because - // the prebuilt AppHost server is required for aspire add to regenerate SDK code. - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - var prNumber = DeploymentE2ETestHelpers.GetPrNumber(); - if (prNumber > 0) - { - output.WriteLine($"Step 2: Installing Aspire bundle from PR #{prNumber}..."); - await auto.InstallAspireBundleFromPullRequestAsync(prNumber, counter); - } - await auto.SourceAspireBundleEnvironmentAsync(counter); - } - - // Step 3: Create TypeScript Express/React project using aspire new - output.WriteLine("Step 3: Creating TypeScript Express/React project..."); - await auto.AspireNewAsync(projectName, counter, template: AspireTemplate.ExpressReact); - - // Step 4: Navigate to project directory - output.WriteLine("Step 4: Navigating to project directory..."); - await auto.TypeAsync($"cd {projectName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 5: Add Aspire.Hosting.Azure.Kubernetes package - output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); - await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); - await auto.EnterAsync(); - - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - await auto.WaitForAspireAddCompletionAsync(counter); - } - else - { - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - } - - // Step 6: Modify apphost.ts to add AKS environment for deployment - { - var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); - var appHostFilePath = Path.Combine(projectDir, "apphost.ts"); - - output.WriteLine($"Looking for apphost.ts at: {appHostFilePath}"); - - var content = File.ReadAllText(appHostFilePath); - var originalContent = content; - - // Add Azure Kubernetes Environment before build().run() - // When there's exactly one compute environment, all resources auto-target it. - content = content.Replace( - "await builder.build().run();", - """ -// Add Azure Kubernetes Environment for deployment -await builder.addAzureKubernetesEnvironment("aks"); - -await builder.build().run(); -"""); - - if (content == originalContent) - { - throw new InvalidOperationException("apphost.ts was not modified. Template may have changed."); - } - - File.WriteAllText(appHostFilePath, content); - - output.WriteLine($"Modified apphost.ts at: {appHostFilePath}"); - } - - // Step 7: Set environment for deployment - await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 8: Deploy to AKS using aspire deploy - output.WriteLine("Step 8: Starting AKS deployment via aspire deploy..."); - await auto.TypeAsync("aspire deploy --clear-cache"); - await auto.EnterAsync(); - // Wait for pipeline to complete - AKS provisioning can take up to 30 minutes - await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); - - // Step 9: Get AKS credentials for kubectl verification - output.WriteLine("Step 9: Getting AKS credentials..."); - await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + - $"echo \"AKS cluster: $AKS_NAME\" && " + - $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 10: Wait for pods to be ready - output.WriteLine("Step 10: Waiting for pods to be ready..."); - await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); - - // Step 11: Verify pods are running - output.WriteLine("Step 11: Verifying pods are running..."); - await auto.TypeAsync("kubectl get pods -n default"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); - - // Step 12: Verify service endpoints via port-forward - output.WriteLine("Step 12: Verifying service endpoints..."); - await auto.TypeAsync("kubectl port-forward svc/api-service 18080:8080 &"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 13: Clean up port-forwards - output.WriteLine("Step 13: Cleaning up port-forwards..."); - await auto.TypeAsync("kill %1 2>/dev/null; true"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - - // Step 14: Destroy Azure deployment - output.WriteLine("Step 14: Destroying Azure deployment..."); - await auto.AspireDestroyAsync(counter); - - // Step 15: Exit terminal - await auto.TypeAsync("exit"); - await auto.EnterAsync(); - - await pendingRun; - - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"TypeScript AKS deployment completed in {duration}"); - - DeploymentReporter.ReportDeploymentSuccess( - nameof(DeployTypeScriptExpressToAksWithAzureProvisioning), - resourceGroupName, - new Dictionary - { - ["project"] = projectName - }, - duration); - - output.WriteLine("✅ Test passed - TypeScript Express app deployed to AKS via Azure Provisioning!"); - } - catch (Exception ex) - { - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); - - DeploymentReporter.ReportDeploymentFailure( - nameof(DeployTypeScriptExpressToAksWithAzureProvisioning), - resourceGroupName, - ex.Message, - ex.StackTrace); - - throw; - } - finally - { - output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); - TriggerCleanupResourceGroup(resourceGroupName, output); - DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); - } - } - - private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) - { - var process = new System.Diagnostics.Process - { - StartInfo = new System.Diagnostics.ProcessStartInfo - { - FileName = "az", - Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - try - { - process.Start(); - output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); - } - catch (Exception ex) - { - output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); - } - } -} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs deleted file mode 100644 index e8219126224..00000000000 --- a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksNodePoolDeploymentTests.cs +++ /dev/null @@ -1,268 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Aspire.Cli.Resources; -using Aspire.Cli.Tests.Utils; -using Aspire.Deployment.EndToEnd.Tests.Helpers; -using Hex1b.Automation; -using Xunit; - -namespace Aspire.Deployment.EndToEnd.Tests; - -/// -/// End-to-end tests for deploying a TypeScript Express/React Aspire application to AKS with a custom node pool. -/// Verifies that addNodePool from a TypeScript AppHost creates additional node pools in the AKS cluster. -/// -public sealed class TypeScriptAksNodePoolDeploymentTests(ITestOutputHelper output) -{ - private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); - - [Fact] - public async Task DeployTypeScriptExpressToAksWithNodePool() - { - using var cts = new CancellationTokenSource(s_testTimeout); - using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( - cts.Token, TestContext.Current.CancellationToken); - var cancellationToken = linkedCts.Token; - - await DeployTypeScriptExpressToAksWithNodePoolCore(cancellationToken); - } - - private async Task DeployTypeScriptExpressToAksWithNodePoolCore(CancellationToken cancellationToken) - { - // Validate prerequisites - var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); - if (string.IsNullOrEmpty(subscriptionId)) - { - Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); - } - - if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) - { - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); - } - else - { - Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); - } - } - - var workspace = TemporaryWorkspace.Create(output); - var startTime = DateTime.UtcNow; - var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("ts-aks-np"); - var projectName = "TsAksNodePool"; - - output.WriteLine($"Test: {nameof(DeployTypeScriptExpressToAksWithNodePool)}"); - output.WriteLine($"Project Name: {projectName}"); - output.WriteLine($"Resource Group: {resourceGroupName}"); - output.WriteLine($"Subscription: {subscriptionId[..8]}..."); - output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); - - try - { - using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); - var pendingRun = terminal.RunAsync(cancellationToken); - - var counter = new SequenceCounter(); - var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); - - // Step 1: Prepare environment - output.WriteLine("Step 1: Preparing environment..."); - await auto.PrepareEnvironmentAsync(workspace, counter); - - // Step 2: Set up CLI environment (in CI) - // TypeScript apphosts need the full bundle (not just the CLI binary) because - // the prebuilt AppHost server is required for aspire add to regenerate SDK code. - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - var prNumber = DeploymentE2ETestHelpers.GetPrNumber(); - if (prNumber > 0) - { - output.WriteLine($"Step 2: Installing Aspire bundle from PR #{prNumber}..."); - await auto.InstallAspireBundleFromPullRequestAsync(prNumber, counter); - } - await auto.SourceAspireBundleEnvironmentAsync(counter); - } - - // Step 3: Create TypeScript Express/React project using aspire new - output.WriteLine("Step 3: Creating TypeScript Express/React project..."); - await auto.AspireNewAsync(projectName, counter, template: AspireTemplate.ExpressReact); - - // Step 4: Navigate to project directory - output.WriteLine("Step 4: Navigating to project directory..."); - await auto.TypeAsync($"cd {projectName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 5: Add Aspire.Hosting.Azure.Kubernetes package - output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); - await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); - await auto.EnterAsync(); - - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - await auto.WaitForAspireAddCompletionAsync(counter); - } - else - { - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - } - - // Step 6: Modify apphost.ts to add AKS environment with custom node pool - { - var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); - var appHostFilePath = Path.Combine(projectDir, "apphost.ts"); - - output.WriteLine($"Looking for apphost.ts at: {appHostFilePath}"); - - var content = File.ReadAllText(appHostFilePath); - var originalContent = content; - - // Add Azure Kubernetes Environment with a custom node pool before build().run() - content = content.Replace( - "await builder.build().run();", - """ -// Add Azure Kubernetes Environment with a custom node pool -const aks = await builder.addAzureKubernetesEnvironment("aks"); -await aks.addNodePool("compute", "Standard_D2s_v5", 1, 3); - -await builder.build().run(); -"""); - - if (content == originalContent) - { - throw new InvalidOperationException("apphost.ts was not modified. Template may have changed."); - } - - File.WriteAllText(appHostFilePath, content); - - output.WriteLine($"Modified apphost.ts at: {appHostFilePath}"); - } - - // Step 7: Set environment for deployment - await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 8: Deploy to AKS using aspire deploy - output.WriteLine("Step 8: Starting AKS deployment via aspire deploy..."); - await auto.TypeAsync("aspire deploy --clear-cache"); - await auto.EnterAsync(); - await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); - - // Step 9: Get AKS credentials for kubectl verification - output.WriteLine("Step 9: Getting AKS credentials..."); - await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + - $"echo \"AKS cluster: $AKS_NAME\" && " + - $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 10: Wait for pods to be ready - output.WriteLine("Step 10: Waiting for pods to be ready..."); - await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); - - // Step 11: Verify pods are running - output.WriteLine("Step 11: Verifying pods are running..."); - await auto.TypeAsync("kubectl get pods -n default"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); - - // Step 12: Verify two node pools exist (system + compute) - output.WriteLine("Step 12: Verifying node pools..."); - await auto.TypeAsync($"az aks nodepool list --resource-group {resourceGroupName} --cluster-name $AKS_NAME --query '[].{{name:name, mode:mode}}' -o table"); - await auto.EnterAsync(); - await auto.WaitUntilTextAsync("compute", timeout: TimeSpan.FromSeconds(30)); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - - // Step 13: Verify service endpoints via port-forward - output.WriteLine("Step 13: Verifying service endpoints..."); - await auto.TypeAsync("kubectl port-forward svc/api-service 18080:8080 &"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 14: Clean up port-forwards - output.WriteLine("Step 14: Cleaning up port-forwards..."); - await auto.TypeAsync("kill %1 2>/dev/null; true"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - - // Step 15: Destroy Azure deployment - output.WriteLine("Step 15: Destroying Azure deployment..."); - await auto.AspireDestroyAsync(counter); - - // Step 16: Exit terminal - await auto.TypeAsync("exit"); - await auto.EnterAsync(); - - await pendingRun; - - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"TypeScript AKS node pool deployment completed in {duration}"); - - DeploymentReporter.ReportDeploymentSuccess( - nameof(DeployTypeScriptExpressToAksWithNodePool), - resourceGroupName, - new Dictionary - { - ["project"] = projectName - }, - duration); - - output.WriteLine("✅ Test passed - TypeScript Express app deployed to AKS with custom node pool!"); - } - catch (Exception ex) - { - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); - - DeploymentReporter.ReportDeploymentFailure( - nameof(DeployTypeScriptExpressToAksWithNodePool), - resourceGroupName, - ex.Message, - ex.StackTrace); - - throw; - } - finally - { - output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); - TriggerCleanupResourceGroup(resourceGroupName, output); - DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); - } - } - - private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) - { - var process = new System.Diagnostics.Process - { - StartInfo = new System.Diagnostics.ProcessStartInfo - { - FileName = "az", - Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - try - { - process.Start(); - output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); - } - catch (Exception ex) - { - output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); - } - } -} diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksVnetDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksVnetDeploymentTests.cs deleted file mode 100644 index 8d441d41fce..00000000000 --- a/tests/Aspire.Deployment.EndToEnd.Tests/TypeScriptAksVnetDeploymentTests.cs +++ /dev/null @@ -1,282 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Aspire.Cli.Resources; -using Aspire.Cli.Tests.Utils; -using Aspire.Deployment.EndToEnd.Tests.Helpers; -using Hex1b.Automation; -using Xunit; - -namespace Aspire.Deployment.EndToEnd.Tests; - -/// -/// End-to-end tests for deploying a TypeScript Express/React Aspire application to AKS with a custom VNet/subnet. -/// Verifies that addAzureVirtualNetwork and withSubnet from a TypeScript AppHost correctly integrate -/// AKS networking so that pods receive VNet IPs rather than default overlay IPs. -/// -public sealed class TypeScriptAksVnetDeploymentTests(ITestOutputHelper output) -{ - private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(45); - - [Fact] - public async Task DeployTypeScriptExpressToAksWithVnet() - { - using var cts = new CancellationTokenSource(s_testTimeout); - using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( - cts.Token, TestContext.Current.CancellationToken); - var cancellationToken = linkedCts.Token; - - await DeployTypeScriptExpressToAksWithVnetCore(cancellationToken); - } - - private async Task DeployTypeScriptExpressToAksWithVnetCore(CancellationToken cancellationToken) - { - // Validate prerequisites - var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); - if (string.IsNullOrEmpty(subscriptionId)) - { - Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); - } - - if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) - { - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); - } - else - { - Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); - } - } - - var workspace = TemporaryWorkspace.Create(output); - var startTime = DateTime.UtcNow; - var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("ts-aks-vnet"); - var projectName = "TsAksVnet"; - - output.WriteLine($"Test: {nameof(DeployTypeScriptExpressToAksWithVnet)}"); - output.WriteLine($"Project Name: {projectName}"); - output.WriteLine($"Resource Group: {resourceGroupName}"); - output.WriteLine($"Subscription: {subscriptionId[..8]}..."); - output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); - - try - { - using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); - var pendingRun = terminal.RunAsync(cancellationToken); - - var counter = new SequenceCounter(); - var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); - - // Step 1: Prepare environment - output.WriteLine("Step 1: Preparing environment..."); - await auto.PrepareEnvironmentAsync(workspace, counter); - - // Step 2: Set up CLI environment (in CI) - // TypeScript apphosts need the full bundle (not just the CLI binary) because - // the prebuilt AppHost server is required for aspire add to regenerate SDK code. - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - var prNumber = DeploymentE2ETestHelpers.GetPrNumber(); - if (prNumber > 0) - { - output.WriteLine($"Step 2: Installing Aspire bundle from PR #{prNumber}..."); - await auto.InstallAspireBundleFromPullRequestAsync(prNumber, counter); - } - await auto.SourceAspireBundleEnvironmentAsync(counter); - } - - // Step 3: Create TypeScript Express/React project using aspire new - output.WriteLine("Step 3: Creating TypeScript Express/React project..."); - await auto.AspireNewAsync(projectName, counter, template: AspireTemplate.ExpressReact); - - // Step 4: Navigate to project directory - output.WriteLine("Step 4: Navigating to project directory..."); - await auto.TypeAsync($"cd {projectName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 5a: Add Aspire.Hosting.Azure.Kubernetes package - output.WriteLine("Step 5a: Adding Azure Kubernetes hosting package..."); - await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); - await auto.EnterAsync(); - - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - await auto.WaitForAspireAddCompletionAsync(counter); - } - else - { - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - } - - // Step 5b: Add Aspire.Hosting.Azure.Network package (for VNet/subnet support) - output.WriteLine("Step 5b: Adding Azure Network hosting package..."); - await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Network"); - await auto.EnterAsync(); - - if (DeploymentE2ETestHelpers.IsRunningInCI) - { - await auto.WaitForAspireAddCompletionAsync(counter); - } - else - { - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(180)); - } - - // Step 6: Modify apphost.ts to add AKS environment with VNet/subnet - { - var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); - var appHostFilePath = Path.Combine(projectDir, "apphost.ts"); - - output.WriteLine($"Looking for apphost.ts at: {appHostFilePath}"); - - var content = File.ReadAllText(appHostFilePath); - var originalContent = content; - - // Add VNet, subnet, and AKS environment with subnet integration before build().run() - content = content.Replace( - "await builder.build().run();", - """ -// Add VNet and subnet for AKS networking -const vnet = await builder.addAzureVirtualNetwork("vnet", "10.1.0.0/16"); -const subnet = await vnet.addSubnet("akssubnet", "10.1.0.0/22"); - -// Add Azure Kubernetes Environment with VNet integration -const aks = await builder.addAzureKubernetesEnvironment("aks"); -await aks.withSubnet(subnet); - -await builder.build().run(); -"""); - - if (content == originalContent) - { - throw new InvalidOperationException("apphost.ts was not modified. Template may have changed."); - } - - File.WriteAllText(appHostFilePath, content); - - output.WriteLine($"Modified apphost.ts at: {appHostFilePath}"); - } - - // Step 7: Set environment for deployment - await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter); - - // Step 8: Deploy to AKS using aspire deploy - output.WriteLine("Step 8: Starting AKS deployment via aspire deploy..."); - await auto.TypeAsync("aspire deploy --clear-cache"); - await auto.EnterAsync(); - // Wait for pipeline to complete - AKS provisioning can take up to 30 minutes - await auto.WaitUntilTextAsync(ConsoleActivityLoggerStrings.PipelineSucceeded, timeout: TimeSpan.FromMinutes(30)); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); - - // Step 9: Get AKS credentials for kubectl verification - output.WriteLine("Step 9: Getting AKS credentials..."); - await auto.TypeAsync($"AKS_NAME=$(az aks list --resource-group {resourceGroupName} --query '[0].name' -o tsv) && " + - $"echo \"AKS cluster: $AKS_NAME\" && " + - $"az aks get-credentials --resource-group {resourceGroupName} --name $AKS_NAME --overwrite-existing"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 10: Wait for pods to be ready - output.WriteLine("Step 10: Waiting for pods to be ready..."); - await auto.TypeAsync("kubectl wait --for=condition=ready pod --all -n default --timeout=300s"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); - - // Step 11: Verify pods are running and have VNet IPs (10.1.x.x, not 10.244.x.x overlay) - output.WriteLine("Step 11: Verifying pods have VNet IPs..."); - await auto.TypeAsync("kubectl get pods -o wide -n default"); - await auto.EnterAsync(); - await auto.WaitUntilTextAsync("10.1.", timeout: TimeSpan.FromSeconds(30)); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - - // Step 12: Verify service endpoints via port-forward - output.WriteLine("Step 12: Verifying service endpoints..."); - await auto.TypeAsync("kubectl port-forward svc/api-service 18080:8080 &"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - await auto.TypeAsync("for i in $(seq 1 10); do sleep 3 && curl -sf http://localhost:18080/ -o /dev/null -w '%{http_code}' && echo ' OK' && break; done"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - - // Step 13: Clean up port-forwards - output.WriteLine("Step 13: Cleaning up port-forwards..."); - await auto.TypeAsync("kill %1 2>/dev/null; true"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); - - // Step 14: Destroy Azure deployment - output.WriteLine("Step 14: Destroying Azure deployment..."); - await auto.AspireDestroyAsync(counter); - - // Step 15: Exit terminal - await auto.TypeAsync("exit"); - await auto.EnterAsync(); - - await pendingRun; - - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"TypeScript AKS VNet deployment completed in {duration}"); - - DeploymentReporter.ReportDeploymentSuccess( - nameof(DeployTypeScriptExpressToAksWithVnet), - resourceGroupName, - new Dictionary - { - ["project"] = projectName - }, - duration); - - output.WriteLine("✅ Test passed - TypeScript Express app deployed to AKS with VNet integration!"); - } - catch (Exception ex) - { - var duration = DateTime.UtcNow - startTime; - output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); - - DeploymentReporter.ReportDeploymentFailure( - nameof(DeployTypeScriptExpressToAksWithVnet), - resourceGroupName, - ex.Message, - ex.StackTrace); - - throw; - } - finally - { - output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); - TriggerCleanupResourceGroup(resourceGroupName, output); - DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); - } - } - - private static void TriggerCleanupResourceGroup(string resourceGroupName, ITestOutputHelper output) - { - var process = new System.Diagnostics.Process - { - StartInfo = new System.Diagnostics.ProcessStartInfo - { - FileName = "az", - Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - try - { - process.Start(); - output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); - } - catch (Exception ex) - { - output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); - } - } -} diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs index c3c9bba17db..bf999fe977e 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesInfrastructureTests.cs @@ -132,9 +132,9 @@ public async Task MultiEnv_ResourcesMatchCorrectEnvironment() var other = builder.AddContainer("other", "myother") .WithComputeEnvironment(envb); - // ParentComputeEnvironment should be set - Assert.Same(enva.Resource, enva.Resource.KubernetesEnvironment.ParentComputeEnvironment); - Assert.Same(envb.Resource, envb.Resource.KubernetesEnvironment.ParentComputeEnvironment); + // OwningComputeEnvironment should be set + Assert.Same(enva.Resource, enva.Resource.KubernetesEnvironment.OwningComputeEnvironment); + Assert.Same(envb.Resource, envb.Resource.KubernetesEnvironment.OwningComputeEnvironment); await using var app = builder.Build(); await ExecuteBeforeStartHooksAsync(app, default); From b42c60d9801c692545355c698beabce16f905838 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Thu, 16 Apr 2026 12:33:41 +1000 Subject: [PATCH 72/78] Make AddNodePool vmSize and count parameters optional vmSize defaults to Standard_D2s_v5, minCount to 1, maxCount to 3. ARM/Bicep requires vmSize (no Azure default), so we provide a sensible default. Users can now call just aks.AddNodePool("workload") for the common case. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 1a511b1fa0c..04a258250f9 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -139,9 +139,9 @@ public static IResourceBuilder AddAzureKuber /// /// The AKS environment resource builder. /// The name of the node pool. - /// The VM size for nodes. - /// The minimum node count for autoscaling. - /// The maximum node count for autoscaling. + /// The VM size for nodes. Defaults to Standard_D2s_v5 if not specified. + /// The minimum node count for autoscaling. Defaults to 1. + /// The maximum node count for autoscaling. Defaults to 3. /// A reference to the for the new node pool. /// /// The returned node pool resource can be passed to @@ -150,19 +150,21 @@ public static IResourceBuilder AddAzureKuber /// /// /// var aks = builder.AddAzureKubernetesEnvironment("aks"); - /// var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); /// - /// builder.AddProject<MyApi>() - /// .WithNodePool(gpuPool); + /// // With defaults (Standard_D2s_v5, 1-3 nodes) + /// var pool = aks.AddNodePool("workload"); + /// + /// // With explicit VM size and scaling + /// var gpuPool = aks.AddNodePool("gpu", "Standard_NC6s_v3", 0, 5); /// /// [AspireExport(Description = "Adds a node pool to the AKS cluster")] public static IResourceBuilder AddNodePool( this IResourceBuilder builder, [ResourceName] string name, - string vmSize, - int minCount, - int maxCount) + string vmSize = "Standard_D2s_v5", + int minCount = 1, + int maxCount = 3) { ArgumentNullException.ThrowIfNull(builder); ArgumentException.ThrowIfNullOrEmpty(name); From f1de02d4f67fb6700be726137c67905d41917bad Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Thu, 16 Apr 2026 15:08:44 +1000 Subject: [PATCH 73/78] Remove Unsecured auth mode from K8s dashboard Stop setting DASHBOARD__FRONTEND__AUTHMODE and DASHBOARD__OTLP__AUTHMODE to 'Unsecured' on the Aspire dashboard deployed to Kubernetes. This matches the Docker Compose behavior where the dashboard uses its default auth mode (BrowserToken). Update snapshot tests for environment resource tests. Publisher test snapshots need regeneration (dashboard ConfigMap removed, file numbering shifts). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...spireDashboardResourceBuilderExtensions.cs | 4 +- .../KubernetesPublisherTests.cs | 8 --- .../env1-dashboard/config.verified.yaml | 12 ---- .../env1-dashboard/deployment.verified.yaml | 3 - .../env1/values.verified.yaml | 3 - .../env2-dashboard/config.verified.yaml | 12 ---- .../env2-dashboard/deployment.verified.yaml | 3 - .../env2/values.verified.yaml | 3 - ...sEnvironmentPublishesFile#01.verified.yaml | 5 +- ...ForBaitAndSwitchResources#01.verified.yaml | 5 +- ...ForBaitAndSwitchResources#02.verified.yaml | 3 - ...ForBaitAndSwitchResources#04.verified.yaml | 42 +++++++++-- ...ForBaitAndSwitchResources#05.verified.yaml | 44 ++++-------- ...ForBaitAndSwitchResources#06.verified.yaml | 17 ++--- ...ForBaitAndSwitchResources#07.verified.yaml | 41 +++++++++-- ...ForBaitAndSwitchResources#08.verified.yaml | 40 ++--------- ...ForBaitAndSwitchResources#09.verified.yaml | 12 ---- ...netesWithProjectResources#01.verified.yaml | 5 +- ...netesWithProjectResources#02.verified.yaml | 3 - ...netesWithProjectResources#04.verified.yaml | 54 ++++++++++++-- ...netesWithProjectResources#05.verified.yaml | 72 ++++++++----------- ...netesWithProjectResources#06.verified.yaml | 38 +++------- ...netesWithProjectResources#07.verified.yaml | 42 ++++++++--- ...netesWithProjectResources#08.verified.yaml | 45 ++++-------- ...netesWithProjectResources#09.verified.yaml | 21 ------ ...erBranch_UsesIfElseSyntax#01.verified.yaml | 5 +- ...erBranch_UsesIfElseSyntax#02.verified.yaml | 3 - ...erBranch_UsesIfElseSyntax#04.verified.yaml | 38 ++++++++-- ...erBranch_UsesIfElseSyntax#05.verified.yaml | 35 ++------- ...erBranch_UsesIfElseSyntax#06.verified.yaml | 11 --- ...omWorkloadAndResourceType#01.verified.yaml | 5 +- ...omWorkloadAndResourceType#02.verified.yaml | 3 - ...omWorkloadAndResourceType#04.verified.yaml | 36 ++++++++-- ...omWorkloadAndResourceType#05.verified.yaml | 38 ++++------ ...omWorkloadAndResourceType#06.verified.yaml | 17 ++--- ...omWorkloadAndResourceType#07.verified.yaml | 14 ++-- ...omWorkloadAndResourceType#08.verified.yaml | 15 ---- ...c_GeneratesValidHelmChart#01.verified.yaml | 5 +- ...c_GeneratesValidHelmChart#02.verified.yaml | 3 - ...c_GeneratesValidHelmChart#04.verified.yaml | 38 ++++++++-- ...c_GeneratesValidHelmChart#05.verified.yaml | 40 +++-------- ...c_GeneratesValidHelmChart#06.verified.yaml | 57 ++++++++++++--- ...c_GeneratesValidHelmChart#07.verified.yaml | 55 ++++---------- ...c_GeneratesValidHelmChart#08.verified.yaml | 19 ++--- ...c_GeneratesValidHelmChart#09.verified.yaml | 13 ++-- ...c_GeneratesValidHelmChart#10.verified.yaml | 14 ---- ...tionalReferenceExpression#01.verified.yaml | 5 +- ...tionalReferenceExpression#02.verified.yaml | 3 - ...tionalReferenceExpression#04.verified.yaml | 38 ++++++++-- ...tionalReferenceExpression#05.verified.yaml | 36 ++-------- ...tionalReferenceExpression#06.verified.yaml | 12 ---- ...ionWithParameterCondition#01.verified.yaml | 5 +- ...ionWithParameterCondition#02.verified.yaml | 3 - ...ionWithParameterCondition#04.verified.yaml | 38 ++++++++-- ...ionWithParameterCondition#05.verified.yaml | 35 ++------- ...ionWithParameterCondition#06.verified.yaml | 11 --- ...andlesSpecialResourceName#01.verified.yaml | 5 +- ...andlesSpecialResourceName#02.verified.yaml | 3 - ...andlesSpecialResourceName#04.verified.yaml | 40 +++++++++-- ...andlesSpecialResourceName#05.verified.yaml | 41 +++-------- ...andlesSpecialResourceName#06.verified.yaml | 14 ++-- ...andlesSpecialResourceName#07.verified.yaml | 13 ---- ...hAsync_ResourceWithProbes#00.verified.yaml | 3 - 63 files changed, 555 insertions(+), 751 deletions(-) delete mode 100644 tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env1/templates/env1-dashboard/config.verified.yaml delete mode 100644 tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env2/templates/env2-dashboard/config.verified.yaml delete mode 100644 tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#09.verified.yaml delete mode 100644 tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#09.verified.yaml delete mode 100644 tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#06.verified.yaml delete mode 100644 tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#08.verified.yaml delete mode 100644 tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#10.verified.yaml delete mode 100644 tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#06.verified.yaml delete mode 100644 tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#06.verified.yaml delete mode 100644 tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#07.verified.yaml diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesAspireDashboardResourceBuilderExtensions.cs b/src/Aspire.Hosting.Kubernetes/KubernetesAspireDashboardResourceBuilderExtensions.cs index e45d8f9997e..32c1bebee18 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesAspireDashboardResourceBuilderExtensions.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesAspireDashboardResourceBuilderExtensions.cs @@ -40,9 +40,7 @@ internal static IResourceBuilder CreateDashbo // Expose the HTTP endpoint so ingress or explicit host port mapping can route browser traffic to the dashboard. .WithEndpoint("http", e => e.IsExternal = true) .WithHttpEndpoint(name: "otlp-grpc", targetPort: 18889) - .WithHttpEndpoint(name: "otlp-http", targetPort: 18890) - .WithEnvironment("DASHBOARD__FRONTEND__AUTHMODE", "Unsecured") - .WithEnvironment("DASHBOARD__OTLP__AUTHMODE", "Unsecured"); + .WithHttpEndpoint(name: "otlp-http", targetPort: 18890); } /// diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesPublisherTests.cs b/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesPublisherTests.cs index 8796df0d9dd..c0d70f56d7d 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesPublisherTests.cs +++ b/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesPublisherTests.cs @@ -50,7 +50,6 @@ public async Task PublishAsync_GeneratesValidHelmChart() "values.yaml", "templates/env-dashboard/deployment.yaml", "templates/env-dashboard/service.yaml", - "templates/env-dashboard/config.yaml", "templates/project1/deployment.yaml", "templates/project1/config.yaml", "templates/myapp/deployment.yaml", @@ -150,7 +149,6 @@ public async Task PublishAsync_CustomWorkloadAndResourceType() "values.yaml", "templates/env-dashboard/deployment.yaml", "templates/env-dashboard/service.yaml", - "templates/env-dashboard/config.yaml", "templates/myapp/rollout.yaml", "templates/myapp/service.yaml", "templates/myapp/config.yaml", @@ -208,7 +206,6 @@ public async Task PublishAsync_HandlesSpecialResourceName() "values.yaml", "templates/env-dashboard/deployment.yaml", "templates/env-dashboard/service.yaml", - "templates/env-dashboard/config.yaml", "templates/SpeciaL-ApP/deployment.yaml", "templates/SpeciaL-ApP/config.yaml", "templates/SpeciaL-ApP/secrets.yaml" @@ -392,7 +389,6 @@ public async Task KubernetesWithProjectResources() "values.yaml", "templates/env-dashboard/deployment.yaml", "templates/env-dashboard/service.yaml", - "templates/env-dashboard/config.yaml", "templates/project1/deployment.yaml", "templates/project1/service.yaml", "templates/project1/config.yaml", @@ -441,7 +437,6 @@ public async Task KubernetesMapsPortsForBaitAndSwitchResources() "values.yaml", "templates/env-dashboard/deployment.yaml", "templates/env-dashboard/service.yaml", - "templates/env-dashboard/config.yaml", "templates/api/deployment.yaml", "templates/api/service.yaml", "templates/api/config.yaml", @@ -502,7 +497,6 @@ public async Task PublishAsync_HandlesConditionalReferenceExpression() "values.yaml", "templates/env-dashboard/deployment.yaml", "templates/env-dashboard/service.yaml", - "templates/env-dashboard/config.yaml", "templates/myapp/deployment.yaml", "templates/myapp/config.yaml", }; @@ -559,7 +553,6 @@ public async Task PublishAsync_HandlesConditionalReferenceExpressionWithParamete "values.yaml", "templates/env-dashboard/deployment.yaml", "templates/env-dashboard/service.yaml", - "templates/env-dashboard/config.yaml", "templates/myapp/deployment.yaml", "templates/myapp/config.yaml", }; @@ -619,7 +612,6 @@ public async Task PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax() "values.yaml", "templates/env-dashboard/deployment.yaml", "templates/env-dashboard/service.yaml", - "templates/env-dashboard/config.yaml", "templates/myapp/deployment.yaml", "templates/myapp/config.yaml", }; diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env1/templates/env1-dashboard/config.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env1/templates/env1-dashboard/config.verified.yaml deleted file mode 100644 index a1981018b93..00000000000 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env1/templates/env1-dashboard/config.verified.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: "v1" -kind: "ConfigMap" -metadata: - name: "env1-dashboard-config" - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "env1-dashboard" - app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - DASHBOARD__FRONTEND__AUTHMODE: "{{ .Values.config.env1_dashboard.DASHBOARD__FRONTEND__AUTHMODE }}" - DASHBOARD__OTLP__AUTHMODE: "{{ .Values.config.env1_dashboard.DASHBOARD__OTLP__AUTHMODE }}" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env1/templates/env1-dashboard/deployment.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env1/templates/env1-dashboard/deployment.verified.yaml index e5141dd85b2..2a3688a724e 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env1/templates/env1-dashboard/deployment.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env1/templates/env1-dashboard/deployment.verified.yaml @@ -18,9 +18,6 @@ spec: containers: - image: "mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest" name: "env1-dashboard" - envFrom: - - configMapRef: - name: "env1-dashboard-config" ports: - name: "http" protocol: "TCP" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env1/values.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env1/values.verified.yaml index a5010ec71a9..c7772f5474f 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env1/values.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env1/values.verified.yaml @@ -4,9 +4,6 @@ ServiceA_image: "ServiceA:latest" secrets: {} config: - env1_dashboard: - DASHBOARD__FRONTEND__AUTHMODE: "Unsecured" - DASHBOARD__OTLP__AUTHMODE: "Unsecured" ServiceA: OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY: "in_memory" ASPNETCORE_FORWARDEDHEADERS_ENABLED: "true" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env2/templates/env2-dashboard/config.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env2/templates/env2-dashboard/config.verified.yaml deleted file mode 100644 index 8accea03b34..00000000000 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env2/templates/env2-dashboard/config.verified.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: "v1" -kind: "ConfigMap" -metadata: - name: "env2-dashboard-config" - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "env2-dashboard" - app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - DASHBOARD__FRONTEND__AUTHMODE: "{{ .Values.config.env2_dashboard.DASHBOARD__FRONTEND__AUTHMODE }}" - DASHBOARD__OTLP__AUTHMODE: "{{ .Values.config.env2_dashboard.DASHBOARD__OTLP__AUTHMODE }}" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env2/templates/env2-dashboard/deployment.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env2/templates/env2-dashboard/deployment.verified.yaml index 0b5088a5f0a..9772c86683a 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env2/templates/env2-dashboard/deployment.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env2/templates/env2-dashboard/deployment.verified.yaml @@ -18,9 +18,6 @@ spec: containers: - image: "mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest" name: "env2-dashboard" - envFrom: - - configMapRef: - name: "env2-dashboard-config" ports: - name: "http" protocol: "TCP" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env2/values.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env2/values.verified.yaml index 438b0054b87..49bcc853f4c 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env2/values.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.MultipleKubernetesEnvironmentsSupported/env2/values.verified.yaml @@ -4,9 +4,6 @@ ServiceB_image: "ServiceB:latest" secrets: {} config: - env2_dashboard: - DASHBOARD__FRONTEND__AUTHMODE: "Unsecured" - DASHBOARD__OTLP__AUTHMODE: "Unsecured" ServiceB: OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY: "in_memory" ASPNETCORE_FORWARDEDHEADERS_ENABLED: "true" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.PublishingKubernetesEnvironmentPublishesFile#01.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.PublishingKubernetesEnvironmentPublishesFile#01.verified.yaml index 8e4b8515505..6e65f5d657a 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.PublishingKubernetesEnvironmentPublishesFile#01.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesEnvironmentResourceTests.PublishingKubernetesEnvironmentPublishesFile#01.verified.yaml @@ -1,6 +1,3 @@ parameters: {} secrets: {} -config: - env_dashboard: - DASHBOARD__FRONTEND__AUTHMODE: "Unsecured" - DASHBOARD__OTLP__AUTHMODE: "Unsecured" +config: {} diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#01.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#01.verified.yaml index a270c5f7097..9bb8e2495d4 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#01.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#01.verified.yaml @@ -1,11 +1,8 @@ -parameters: +parameters: api: api_image: "api:latest" secrets: {} config: - env_dashboard: - DASHBOARD__FRONTEND__AUTHMODE: "Unsecured" - DASHBOARD__OTLP__AUTHMODE: "Unsecured" api: PORT: "8000" gateway: diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#02.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#02.verified.yaml index d9837afd3a3..4cce26b3384 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#02.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#02.verified.yaml @@ -18,9 +18,6 @@ spec: containers: - image: "mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest" name: "env-dashboard" - envFrom: - - configMapRef: - name: "env-dashboard-config" ports: - name: "http" protocol: "TCP" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#04.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#04.verified.yaml index 28f355f91c5..c02bdb29653 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#04.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#04.verified.yaml @@ -1,12 +1,40 @@ --- -apiVersion: "v1" -kind: "ConfigMap" +apiVersion: "apps/v1" +kind: "Deployment" metadata: - name: "env-dashboard-config" + name: "api-deployment" labels: app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "env-dashboard" + app.kubernetes.io/component: "api" app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - DASHBOARD__FRONTEND__AUTHMODE: "{{ .Values.config.env_dashboard.DASHBOARD__FRONTEND__AUTHMODE }}" - DASHBOARD__OTLP__AUTHMODE: "{{ .Values.config.env_dashboard.DASHBOARD__OTLP__AUTHMODE }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "api" + app.kubernetes.io/instance: "{{ .Release.Name }}" + spec: + containers: + - image: "{{ .Values.parameters.api.api_image }}" + name: "api" + envFrom: + - configMapRef: + name: "api-config" + ports: + - name: "http" + protocol: "TCP" + containerPort: 8000 + imagePullPolicy: "IfNotPresent" + selector: + matchLabels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "api" + app.kubernetes.io/instance: "{{ .Release.Name }}" + replicas: 1 + revisionHistoryLimit: 3 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: "RollingUpdate" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#05.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#05.verified.yaml index c02bdb29653..e66ef8c2ce4 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#05.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#05.verified.yaml @@ -1,40 +1,20 @@ --- -apiVersion: "apps/v1" -kind: "Deployment" +apiVersion: "v1" +kind: "Service" metadata: - name: "api-deployment" + name: "api-service" labels: app.kubernetes.io/name: "aspire-hosting-tests" app.kubernetes.io/component: "api" app.kubernetes.io/instance: "{{ .Release.Name }}" spec: - template: - metadata: - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "api" - app.kubernetes.io/instance: "{{ .Release.Name }}" - spec: - containers: - - image: "{{ .Values.parameters.api.api_image }}" - name: "api" - envFrom: - - configMapRef: - name: "api-config" - ports: - - name: "http" - protocol: "TCP" - containerPort: 8000 - imagePullPolicy: "IfNotPresent" + type: "ClusterIP" selector: - matchLabels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "api" - app.kubernetes.io/instance: "{{ .Release.Name }}" - replicas: 1 - revisionHistoryLimit: 3 - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - type: "RollingUpdate" + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "api" + app.kubernetes.io/instance: "{{ .Release.Name }}" + ports: + - name: "http" + protocol: "TCP" + port: 8000 + targetPort: 8000 diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#06.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#06.verified.yaml index e66ef8c2ce4..d588ff078e6 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#06.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#06.verified.yaml @@ -1,20 +1,11 @@ --- apiVersion: "v1" -kind: "Service" +kind: "ConfigMap" metadata: - name: "api-service" + name: "api-config" labels: app.kubernetes.io/name: "aspire-hosting-tests" app.kubernetes.io/component: "api" app.kubernetes.io/instance: "{{ .Release.Name }}" -spec: - type: "ClusterIP" - selector: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "api" - app.kubernetes.io/instance: "{{ .Release.Name }}" - ports: - - name: "http" - protocol: "TCP" - port: 8000 - targetPort: 8000 +data: + PORT: "{{ .Values.config.api.PORT }}" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#07.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#07.verified.yaml index d588ff078e6..18e18bb62d7 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#07.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#07.verified.yaml @@ -1,11 +1,40 @@ --- -apiVersion: "v1" -kind: "ConfigMap" +apiVersion: "apps/v1" +kind: "Deployment" metadata: - name: "api-config" + name: "gateway-deployment" labels: app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "api" + app.kubernetes.io/component: "gateway" app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - PORT: "{{ .Values.config.api.PORT }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "gateway" + app.kubernetes.io/instance: "{{ .Release.Name }}" + spec: + containers: + - image: "nginx:latest" + name: "gateway" + envFrom: + - configMapRef: + name: "gateway-config" + ports: + - name: "http" + protocol: "TCP" + containerPort: 8080 + imagePullPolicy: "IfNotPresent" + selector: + matchLabels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "gateway" + app.kubernetes.io/instance: "{{ .Release.Name }}" + replicas: 1 + revisionHistoryLimit: 3 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: "RollingUpdate" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#08.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#08.verified.yaml index 18e18bb62d7..9f4aaee87c2 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#08.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#08.verified.yaml @@ -1,40 +1,12 @@ --- -apiVersion: "apps/v1" -kind: "Deployment" +apiVersion: "v1" +kind: "ConfigMap" metadata: - name: "gateway-deployment" + name: "gateway-config" labels: app.kubernetes.io/name: "aspire-hosting-tests" app.kubernetes.io/component: "gateway" app.kubernetes.io/instance: "{{ .Release.Name }}" -spec: - template: - metadata: - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "gateway" - app.kubernetes.io/instance: "{{ .Release.Name }}" - spec: - containers: - - image: "nginx:latest" - name: "gateway" - envFrom: - - configMapRef: - name: "gateway-config" - ports: - - name: "http" - protocol: "TCP" - containerPort: 8080 - imagePullPolicy: "IfNotPresent" - selector: - matchLabels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "gateway" - app.kubernetes.io/instance: "{{ .Release.Name }}" - replicas: 1 - revisionHistoryLimit: 3 - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - type: "RollingUpdate" +data: + API_HTTP: "{{ .Values.config.gateway.API_HTTP }}" + services__api__http__0: "{{ .Values.config.gateway.services__api__http__0 }}" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#09.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#09.verified.yaml deleted file mode 100644 index 9f4aaee87c2..00000000000 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesMapsPortsForBaitAndSwitchResources#09.verified.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: "v1" -kind: "ConfigMap" -metadata: - name: "gateway-config" - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "gateway" - app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - API_HTTP: "{{ .Values.config.gateway.API_HTTP }}" - services__api__http__0: "{{ .Values.config.gateway.services__api__http__0 }}" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#01.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#01.verified.yaml index b44334bcf03..ffa13c7773a 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#01.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#01.verified.yaml @@ -1,12 +1,9 @@ -parameters: +parameters: project1: port_http: 8080 project1_image: "project1:latest" secrets: {} config: - env_dashboard: - DASHBOARD__FRONTEND__AUTHMODE: "Unsecured" - DASHBOARD__OTLP__AUTHMODE: "Unsecured" project1: OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY: "in_memory" ASPNETCORE_FORWARDEDHEADERS_ENABLED: "true" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#02.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#02.verified.yaml index d9837afd3a3..4cce26b3384 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#02.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#02.verified.yaml @@ -18,9 +18,6 @@ spec: containers: - image: "mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest" name: "env-dashboard" - envFrom: - - configMapRef: - name: "env-dashboard-config" ports: - name: "http" protocol: "TCP" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#04.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#04.verified.yaml index 28f355f91c5..fe6845b975f 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#04.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#04.verified.yaml @@ -1,12 +1,52 @@ --- -apiVersion: "v1" -kind: "ConfigMap" +apiVersion: "apps/v1" +kind: "Deployment" metadata: - name: "env-dashboard-config" + name: "project1-deployment" labels: app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "env-dashboard" + app.kubernetes.io/component: "project1" app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - DASHBOARD__FRONTEND__AUTHMODE: "{{ .Values.config.env_dashboard.DASHBOARD__FRONTEND__AUTHMODE }}" - DASHBOARD__OTLP__AUTHMODE: "{{ .Values.config.env_dashboard.DASHBOARD__OTLP__AUTHMODE }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "project1" + app.kubernetes.io/instance: "{{ .Release.Name }}" + spec: + containers: + - image: "{{ .Values.parameters.project1.project1_image }}" + name: "project1" + envFrom: + - configMapRef: + name: "project1-config" + ports: + - name: "http" + protocol: "TCP" + containerPort: {{ .Values.parameters.project1.port_http | int }} + - name: "custom1" + protocol: "TCP" + containerPort: 8000 + - name: "custom2" + protocol: "TCP" + containerPort: 8001 + - name: "custom3" + protocol: "TCP" + containerPort: 7002 + - name: "custom4" + protocol: "TCP" + containerPort: 7004 + imagePullPolicy: "IfNotPresent" + selector: + matchLabels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "project1" + app.kubernetes.io/instance: "{{ .Release.Name }}" + replicas: 1 + revisionHistoryLimit: 3 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: "RollingUpdate" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#05.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#05.verified.yaml index fe6845b975f..a70a914e74f 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#05.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#05.verified.yaml @@ -1,52 +1,36 @@ --- -apiVersion: "apps/v1" -kind: "Deployment" +apiVersion: "v1" +kind: "Service" metadata: - name: "project1-deployment" + name: "project1-service" labels: app.kubernetes.io/name: "aspire-hosting-tests" app.kubernetes.io/component: "project1" app.kubernetes.io/instance: "{{ .Release.Name }}" spec: - template: - metadata: - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "project1" - app.kubernetes.io/instance: "{{ .Release.Name }}" - spec: - containers: - - image: "{{ .Values.parameters.project1.project1_image }}" - name: "project1" - envFrom: - - configMapRef: - name: "project1-config" - ports: - - name: "http" - protocol: "TCP" - containerPort: {{ .Values.parameters.project1.port_http | int }} - - name: "custom1" - protocol: "TCP" - containerPort: 8000 - - name: "custom2" - protocol: "TCP" - containerPort: 8001 - - name: "custom3" - protocol: "TCP" - containerPort: 7002 - - name: "custom4" - protocol: "TCP" - containerPort: 7004 - imagePullPolicy: "IfNotPresent" + type: "ClusterIP" selector: - matchLabels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "project1" - app.kubernetes.io/instance: "{{ .Release.Name }}" - replicas: 1 - revisionHistoryLimit: 3 - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - type: "RollingUpdate" + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "project1" + app.kubernetes.io/instance: "{{ .Release.Name }}" + ports: + - name: "http" + protocol: "TCP" + port: {{ .Values.parameters.project1.port_http | int }} + targetPort: {{ .Values.parameters.project1.port_http | int }} + - name: "custom1" + protocol: "TCP" + port: 8000 + targetPort: 8000 + - name: "custom2" + protocol: "TCP" + port: 7001 + targetPort: 8001 + - name: "custom3" + protocol: "TCP" + port: 7002 + targetPort: 7002 + - name: "custom4" + protocol: "TCP" + port: 7003 + targetPort: 7004 diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#06.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#06.verified.yaml index a70a914e74f..eb04bc499a5 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#06.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#06.verified.yaml @@ -1,36 +1,16 @@ --- apiVersion: "v1" -kind: "Service" +kind: "ConfigMap" metadata: - name: "project1-service" + name: "project1-config" labels: app.kubernetes.io/name: "aspire-hosting-tests" app.kubernetes.io/component: "project1" app.kubernetes.io/instance: "{{ .Release.Name }}" -spec: - type: "ClusterIP" - selector: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "project1" - app.kubernetes.io/instance: "{{ .Release.Name }}" - ports: - - name: "http" - protocol: "TCP" - port: {{ .Values.parameters.project1.port_http | int }} - targetPort: {{ .Values.parameters.project1.port_http | int }} - - name: "custom1" - protocol: "TCP" - port: 8000 - targetPort: 8000 - - name: "custom2" - protocol: "TCP" - port: 7001 - targetPort: 8001 - - name: "custom3" - protocol: "TCP" - port: 7002 - targetPort: 7002 - - name: "custom4" - protocol: "TCP" - port: 7003 - targetPort: 7004 +data: + OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY: "{{ .Values.config.project1.OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY }}" + ASPNETCORE_FORWARDEDHEADERS_ENABLED: "{{ .Values.config.project1.ASPNETCORE_FORWARDEDHEADERS_ENABLED }}" + HTTP_PORTS: "{{ .Values.parameters.project1.port_http }};8000;8001;7002;7004" + OTEL_EXPORTER_OTLP_ENDPOINT: "{{ .Values.config.project1.OTEL_EXPORTER_OTLP_ENDPOINT }}" + OTEL_EXPORTER_OTLP_PROTOCOL: "{{ .Values.config.project1.OTEL_EXPORTER_OTLP_PROTOCOL }}" + OTEL_SERVICE_NAME: "{{ .Values.config.project1.OTEL_SERVICE_NAME }}" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#07.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#07.verified.yaml index eb04bc499a5..90177b02f82 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#07.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#07.verified.yaml @@ -1,16 +1,36 @@ --- -apiVersion: "v1" -kind: "ConfigMap" +apiVersion: "apps/v1" +kind: "Deployment" metadata: - name: "project1-config" + name: "api-deployment" labels: app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "project1" + app.kubernetes.io/component: "api" app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY: "{{ .Values.config.project1.OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY }}" - ASPNETCORE_FORWARDEDHEADERS_ENABLED: "{{ .Values.config.project1.ASPNETCORE_FORWARDEDHEADERS_ENABLED }}" - HTTP_PORTS: "{{ .Values.parameters.project1.port_http }};8000;8001;7002;7004" - OTEL_EXPORTER_OTLP_ENDPOINT: "{{ .Values.config.project1.OTEL_EXPORTER_OTLP_ENDPOINT }}" - OTEL_EXPORTER_OTLP_PROTOCOL: "{{ .Values.config.project1.OTEL_EXPORTER_OTLP_PROTOCOL }}" - OTEL_SERVICE_NAME: "{{ .Values.config.project1.OTEL_SERVICE_NAME }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "api" + app.kubernetes.io/instance: "{{ .Release.Name }}" + spec: + containers: + - image: "reg:api" + name: "api" + envFrom: + - configMapRef: + name: "api-config" + imagePullPolicy: "IfNotPresent" + selector: + matchLabels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "api" + app.kubernetes.io/instance: "{{ .Release.Name }}" + replicas: 1 + revisionHistoryLimit: 3 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: "RollingUpdate" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#08.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#08.verified.yaml index 90177b02f82..dcd8f588158 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#08.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#08.verified.yaml @@ -1,36 +1,21 @@ --- -apiVersion: "apps/v1" -kind: "Deployment" +apiVersion: "v1" +kind: "ConfigMap" metadata: - name: "api-deployment" + name: "api-config" labels: app.kubernetes.io/name: "aspire-hosting-tests" app.kubernetes.io/component: "api" app.kubernetes.io/instance: "{{ .Release.Name }}" -spec: - template: - metadata: - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "api" - app.kubernetes.io/instance: "{{ .Release.Name }}" - spec: - containers: - - image: "reg:api" - name: "api" - envFrom: - - configMapRef: - name: "api-config" - imagePullPolicy: "IfNotPresent" - selector: - matchLabels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "api" - app.kubernetes.io/instance: "{{ .Release.Name }}" - replicas: 1 - revisionHistoryLimit: 3 - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - type: "RollingUpdate" +data: + PROJECT1_HTTP: "http://project1-service:{{ .Values.parameters.project1.port_http }}" + services__project1__http__0: "http://project1-service:{{ .Values.parameters.project1.port_http }}" + PROJECT1_HTTPS: "https://project1-service:{{ .Values.parameters.project1.port_http }}" + PROJECT1_CUSTOM1: "{{ .Values.config.api.PROJECT1_CUSTOM1 }}" + services__project1__http__1: "{{ .Values.config.api.services__project1__http__1 }}" + PROJECT1_CUSTOM2: "{{ .Values.config.api.PROJECT1_CUSTOM2 }}" + services__project1__http__2: "{{ .Values.config.api.services__project1__http__2 }}" + PROJECT1_CUSTOM3: "{{ .Values.config.api.PROJECT1_CUSTOM3 }}" + services__project1__http__3: "{{ .Values.config.api.services__project1__http__3 }}" + PROJECT1_CUSTOM4: "{{ .Values.config.api.PROJECT1_CUSTOM4 }}" + services__project1__http__4: "{{ .Values.config.api.services__project1__http__4 }}" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#09.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#09.verified.yaml deleted file mode 100644 index 711b0061a78..00000000000 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#09.verified.yaml +++ /dev/null @@ -1,21 +0,0 @@ ---- -apiVersion: "v1" -kind: "ConfigMap" -metadata: - name: "api-config" - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "api" - app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - PROJECT1_HTTP: "http://project1-service:{{ .Values.parameters.project1.port_http }}" - services__project1__http__0: "http://project1-service:{{ .Values.parameters.project1.port_http }}" - PROJECT1_HTTPS: "https://project1-service:{{ .Values.parameters.project1.port_http }}" - PROJECT1_CUSTOM1: "{{ .Values.config.api.PROJECT1_CUSTOM1 }}" - services__project1__custom1__0: "{{ .Values.config.api.services__project1__custom1__0 }}" - PROJECT1_CUSTOM2: "{{ .Values.config.api.PROJECT1_CUSTOM2 }}" - services__project1__custom2__0: "{{ .Values.config.api.services__project1__custom2__0 }}" - PROJECT1_CUSTOM3: "{{ .Values.config.api.PROJECT1_CUSTOM3 }}" - services__project1__custom3__0: "{{ .Values.config.api.services__project1__custom3__0 }}" - PROJECT1_CUSTOM4: "{{ .Values.config.api.PROJECT1_CUSTOM4 }}" - services__project1__custom4__0: "{{ .Values.config.api.services__project1__custom4__0 }}" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#01.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#01.verified.yaml index 3b07241ffa0..b4c19ac3dce 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#01.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#01.verified.yaml @@ -1,11 +1,8 @@ -parameters: +parameters: myapp: enable_tls: "True" secrets: {} config: - env_dashboard: - DASHBOARD__FRONTEND__AUTHMODE: "Unsecured" - DASHBOARD__OTLP__AUTHMODE: "Unsecured" myapp: TLS_SUFFIX: "" tls_suffix: ",ssl=true" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#02.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#02.verified.yaml index d9837afd3a3..4cce26b3384 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#02.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#02.verified.yaml @@ -18,9 +18,6 @@ spec: containers: - image: "mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest" name: "env-dashboard" - envFrom: - - configMapRef: - name: "env-dashboard-config" ports: - name: "http" protocol: "TCP" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#04.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#04.verified.yaml index 28f355f91c5..37de00c4947 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#04.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#04.verified.yaml @@ -1,12 +1,36 @@ --- -apiVersion: "v1" -kind: "ConfigMap" +apiVersion: "apps/v1" +kind: "Deployment" metadata: - name: "env-dashboard-config" + name: "myapp-deployment" labels: app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "env-dashboard" + app.kubernetes.io/component: "myapp" app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - DASHBOARD__FRONTEND__AUTHMODE: "{{ .Values.config.env_dashboard.DASHBOARD__FRONTEND__AUTHMODE }}" - DASHBOARD__OTLP__AUTHMODE: "{{ .Values.config.env_dashboard.DASHBOARD__OTLP__AUTHMODE }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "myapp" + app.kubernetes.io/instance: "{{ .Release.Name }}" + spec: + containers: + - image: "mcr.microsoft.com/dotnet/aspnet:8.0" + name: "myapp" + envFrom: + - configMapRef: + name: "myapp-config" + imagePullPolicy: "IfNotPresent" + selector: + matchLabels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "myapp" + app.kubernetes.io/instance: "{{ .Release.Name }}" + replicas: 1 + revisionHistoryLimit: 3 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: "RollingUpdate" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#05.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#05.verified.yaml index 37de00c4947..2823fb9bc2a 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#05.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#05.verified.yaml @@ -1,36 +1,11 @@ --- -apiVersion: "apps/v1" -kind: "Deployment" +apiVersion: "v1" +kind: "ConfigMap" metadata: - name: "myapp-deployment" + name: "myapp-config" labels: app.kubernetes.io/name: "aspire-hosting-tests" app.kubernetes.io/component: "myapp" app.kubernetes.io/instance: "{{ .Release.Name }}" -spec: - template: - metadata: - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "myapp" - app.kubernetes.io/instance: "{{ .Release.Name }}" - spec: - containers: - - image: "mcr.microsoft.com/dotnet/aspnet:8.0" - name: "myapp" - envFrom: - - configMapRef: - name: "myapp-config" - imagePullPolicy: "IfNotPresent" - selector: - matchLabels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "myapp" - app.kubernetes.io/instance: "{{ .Release.Name }}" - replicas: 1 - revisionHistoryLimit: 3 - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - type: "RollingUpdate" +data: + TLS_SUFFIX: {{ if eq (.Values.parameters.myapp.enable_tls | lower) "true" }}{{ .Values.config.myapp.tls_suffix }}{{ else }},ssl=false{{ end }} diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#06.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#06.verified.yaml deleted file mode 100644 index 2823fb9bc2a..00000000000 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ConditionalWithParameterBranch_UsesIfElseSyntax#06.verified.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: "v1" -kind: "ConfigMap" -metadata: - name: "myapp-config" - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "myapp" - app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - TLS_SUFFIX: {{ if eq (.Values.parameters.myapp.enable_tls | lower) "true" }}{{ .Values.config.myapp.tls_suffix }}{{ else }},ssl=false{{ end }} diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#01.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#01.verified.yaml index ddb2c0262be..86e85a25ddc 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#01.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#01.verified.yaml @@ -1,11 +1,8 @@ -parameters: +parameters: project1: project1_image: "project1:latest" secrets: {} config: - env_dashboard: - DASHBOARD__FRONTEND__AUTHMODE: "Unsecured" - DASHBOARD__OTLP__AUTHMODE: "Unsecured" myapp: ASPNETCORE_ENVIRONMENT: "Development" project1: diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#02.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#02.verified.yaml index d9837afd3a3..4cce26b3384 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#02.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#02.verified.yaml @@ -18,9 +18,6 @@ spec: containers: - image: "mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest" name: "env-dashboard" - envFrom: - - configMapRef: - name: "env-dashboard-config" ports: - name: "http" protocol: "TCP" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#04.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#04.verified.yaml index 28f355f91c5..a3a11c691ed 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#04.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#04.verified.yaml @@ -1,12 +1,34 @@ --- -apiVersion: "v1" -kind: "ConfigMap" +apiVersion: "argoproj.io/v1alpha1" +kind: "Rollout" metadata: - name: "env-dashboard-config" + name: "myapp-rollout" labels: app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "env-dashboard" + app.kubernetes.io/component: "myapp" app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - DASHBOARD__FRONTEND__AUTHMODE: "{{ .Values.config.env_dashboard.DASHBOARD__FRONTEND__AUTHMODE }}" - DASHBOARD__OTLP__AUTHMODE: "{{ .Values.config.env_dashboard.DASHBOARD__OTLP__AUTHMODE }}" +spec: + replicas: 1 + template: + metadata: + labels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "myapp" + app.kubernetes.io/instance: "{{ .Release.Name }}" + spec: + containers: + - image: "mcr.microsoft.com/dotnet/aspnet:8.0" + name: "myapp" + envFrom: + - configMapRef: + name: "myapp-config" + ports: + - name: "http" + protocol: "TCP" + containerPort: 8080 + imagePullPolicy: "IfNotPresent" + selector: + matchLabels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "myapp" + app.kubernetes.io/instance: "{{ .Release.Name }}" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#05.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#05.verified.yaml index a3a11c691ed..278bf076e42 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#05.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#05.verified.yaml @@ -1,34 +1,20 @@ --- -apiVersion: "argoproj.io/v1alpha1" -kind: "Rollout" +apiVersion: "v1" +kind: "Service" metadata: - name: "myapp-rollout" + name: "myapp-service" labels: app.kubernetes.io/name: "aspire-hosting-tests" app.kubernetes.io/component: "myapp" app.kubernetes.io/instance: "{{ .Release.Name }}" spec: - replicas: 1 - template: - metadata: - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "myapp" - app.kubernetes.io/instance: "{{ .Release.Name }}" - spec: - containers: - - image: "mcr.microsoft.com/dotnet/aspnet:8.0" - name: "myapp" - envFrom: - - configMapRef: - name: "myapp-config" - ports: - - name: "http" - protocol: "TCP" - containerPort: 8080 - imagePullPolicy: "IfNotPresent" + type: "ClusterIP" selector: - matchLabels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "myapp" - app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "myapp" + app.kubernetes.io/instance: "{{ .Release.Name }}" + ports: + - name: "http" + protocol: "TCP" + port: 8080 + targetPort: 8080 diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#06.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#06.verified.yaml index 278bf076e42..6b54f9178a9 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#06.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#06.verified.yaml @@ -1,20 +1,11 @@ --- apiVersion: "v1" -kind: "Service" +kind: "ConfigMap" metadata: - name: "myapp-service" + name: "myapp-config" labels: app.kubernetes.io/name: "aspire-hosting-tests" app.kubernetes.io/component: "myapp" app.kubernetes.io/instance: "{{ .Release.Name }}" -spec: - type: "ClusterIP" - selector: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "myapp" - app.kubernetes.io/instance: "{{ .Release.Name }}" - ports: - - name: "http" - protocol: "TCP" - port: 8080 - targetPort: 8080 +data: + ASPNETCORE_ENVIRONMENT: "{{ .Values.config.myapp.ASPNETCORE_ENVIRONMENT }}" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#07.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#07.verified.yaml index 6b54f9178a9..92ff24ba964 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#07.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#07.verified.yaml @@ -1,11 +1,15 @@ --- -apiVersion: "v1" -kind: "ConfigMap" +apiVersion: "keda.sh/v1alpha1" +kind: "ScaledObject" metadata: - name: "myapp-config" + name: "myapp-scaler" labels: app.kubernetes.io/name: "aspire-hosting-tests" app.kubernetes.io/component: "myapp" app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - ASPNETCORE_ENVIRONMENT: "{{ .Values.config.myapp.ASPNETCORE_ENVIRONMENT }}" +spec: + scaleTargetRef: + name: "myapp-rollout" + kind: "Rollout" + minReplicaCount: 1 + maxReplicaCount: 3 diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#08.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#08.verified.yaml deleted file mode 100644 index 92ff24ba964..00000000000 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_CustomWorkloadAndResourceType#08.verified.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: "keda.sh/v1alpha1" -kind: "ScaledObject" -metadata: - name: "myapp-scaler" - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "myapp" - app.kubernetes.io/instance: "{{ .Release.Name }}" -spec: - scaleTargetRef: - name: "myapp-rollout" - kind: "Rollout" - minReplicaCount: 1 - maxReplicaCount: 3 diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#01.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#01.verified.yaml index f5589575e23..a82a10634a2 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#01.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#01.verified.yaml @@ -1,4 +1,4 @@ -parameters: +parameters: project1: project1_image: "project1:latest" secrets: @@ -7,9 +7,6 @@ secrets: param3: "" ConnectionStrings__cs: "" config: - env_dashboard: - DASHBOARD__FRONTEND__AUTHMODE: "Unsecured" - DASHBOARD__OTLP__AUTHMODE: "Unsecured" myapp: ASPNETCORE_ENVIRONMENT: "Development" param0: "" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#02.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#02.verified.yaml index d9837afd3a3..4cce26b3384 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#02.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#02.verified.yaml @@ -18,9 +18,6 @@ spec: containers: - image: "mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest" name: "env-dashboard" - envFrom: - - configMapRef: - name: "env-dashboard-config" ports: - name: "http" protocol: "TCP" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#04.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#04.verified.yaml index 28f355f91c5..8a8452a6954 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#04.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#04.verified.yaml @@ -1,12 +1,36 @@ --- -apiVersion: "v1" -kind: "ConfigMap" +apiVersion: "apps/v1" +kind: "Deployment" metadata: - name: "env-dashboard-config" + name: "project1-deployment" labels: app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "env-dashboard" + app.kubernetes.io/component: "project1" app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - DASHBOARD__FRONTEND__AUTHMODE: "{{ .Values.config.env_dashboard.DASHBOARD__FRONTEND__AUTHMODE }}" - DASHBOARD__OTLP__AUTHMODE: "{{ .Values.config.env_dashboard.DASHBOARD__OTLP__AUTHMODE }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "project1" + app.kubernetes.io/instance: "{{ .Release.Name }}" + spec: + containers: + - image: "{{ .Values.parameters.project1.project1_image }}" + name: "project1" + envFrom: + - configMapRef: + name: "project1-config" + imagePullPolicy: "IfNotPresent" + selector: + matchLabels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "project1" + app.kubernetes.io/instance: "{{ .Release.Name }}" + replicas: 1 + revisionHistoryLimit: 3 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: "RollingUpdate" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#05.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#05.verified.yaml index 8a8452a6954..38439563476 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#05.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#05.verified.yaml @@ -1,36 +1,16 @@ --- -apiVersion: "apps/v1" -kind: "Deployment" +apiVersion: "v1" +kind: "ConfigMap" metadata: - name: "project1-deployment" + name: "project1-config" labels: app.kubernetes.io/name: "aspire-hosting-tests" app.kubernetes.io/component: "project1" app.kubernetes.io/instance: "{{ .Release.Name }}" -spec: - template: - metadata: - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "project1" - app.kubernetes.io/instance: "{{ .Release.Name }}" - spec: - containers: - - image: "{{ .Values.parameters.project1.project1_image }}" - name: "project1" - envFrom: - - configMapRef: - name: "project1-config" - imagePullPolicy: "IfNotPresent" - selector: - matchLabels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "project1" - app.kubernetes.io/instance: "{{ .Release.Name }}" - replicas: 1 - revisionHistoryLimit: 3 - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - type: "RollingUpdate" +data: + OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY: "{{ .Values.config.project1.OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY }}" + MYAPP_HTTP: "{{ .Values.config.project1.MYAPP_HTTP }}" + services__myapp__http__0: "{{ .Values.config.project1.services__myapp__http__0 }}" + OTEL_EXPORTER_OTLP_ENDPOINT: "{{ .Values.config.project1.OTEL_EXPORTER_OTLP_ENDPOINT }}" + OTEL_EXPORTER_OTLP_PROTOCOL: "{{ .Values.config.project1.OTEL_EXPORTER_OTLP_PROTOCOL }}" + OTEL_SERVICE_NAME: "{{ .Values.config.project1.OTEL_SERVICE_NAME }}" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#06.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#06.verified.yaml index 38439563476..19a7fd1c156 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#06.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#06.verified.yaml @@ -1,16 +1,51 @@ --- -apiVersion: "v1" -kind: "ConfigMap" +apiVersion: "apps/v1" +kind: "Deployment" metadata: - name: "project1-config" + name: "myapp-deployment" labels: app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "project1" + app.kubernetes.io/component: "myapp" app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY: "{{ .Values.config.project1.OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY }}" - MYAPP_HTTP: "{{ .Values.config.project1.MYAPP_HTTP }}" - services__myapp__http__0: "{{ .Values.config.project1.services__myapp__http__0 }}" - OTEL_EXPORTER_OTLP_ENDPOINT: "{{ .Values.config.project1.OTEL_EXPORTER_OTLP_ENDPOINT }}" - OTEL_EXPORTER_OTLP_PROTOCOL: "{{ .Values.config.project1.OTEL_EXPORTER_OTLP_PROTOCOL }}" - OTEL_SERVICE_NAME: "{{ .Values.config.project1.OTEL_SERVICE_NAME }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "myapp" + app.kubernetes.io/instance: "{{ .Release.Name }}" + spec: + containers: + - image: "mcr.microsoft.com/dotnet/aspnet:8.0" + name: "myapp" + envFrom: + - configMapRef: + name: "myapp-config" + - secretRef: + name: "myapp-secrets" + args: + - "--cs" + - "Url={{ .Values.config.myapp.param0 }}, Secret={{ .Values.secrets.myapp.param1 }}" + ports: + - name: "http" + protocol: "TCP" + containerPort: 8080 + volumeMounts: + - name: "logs" + mountPath: "/logs" + imagePullPolicy: "IfNotPresent" + volumes: + - name: "logs" + emptyDir: {} + selector: + matchLabels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "myapp" + app.kubernetes.io/instance: "{{ .Release.Name }}" + replicas: 1 + revisionHistoryLimit: 3 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: "RollingUpdate" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#07.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#07.verified.yaml index 19a7fd1c156..278bf076e42 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#07.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#07.verified.yaml @@ -1,51 +1,20 @@ --- -apiVersion: "apps/v1" -kind: "Deployment" +apiVersion: "v1" +kind: "Service" metadata: - name: "myapp-deployment" + name: "myapp-service" labels: app.kubernetes.io/name: "aspire-hosting-tests" app.kubernetes.io/component: "myapp" app.kubernetes.io/instance: "{{ .Release.Name }}" spec: - template: - metadata: - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "myapp" - app.kubernetes.io/instance: "{{ .Release.Name }}" - spec: - containers: - - image: "mcr.microsoft.com/dotnet/aspnet:8.0" - name: "myapp" - envFrom: - - configMapRef: - name: "myapp-config" - - secretRef: - name: "myapp-secrets" - args: - - "--cs" - - "Url={{ .Values.config.myapp.param0 }}, Secret={{ .Values.secrets.myapp.param1 }}" - ports: - - name: "http" - protocol: "TCP" - containerPort: 8080 - volumeMounts: - - name: "logs" - mountPath: "/logs" - imagePullPolicy: "IfNotPresent" - volumes: - - name: "logs" - emptyDir: {} + type: "ClusterIP" selector: - matchLabels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "myapp" - app.kubernetes.io/instance: "{{ .Release.Name }}" - replicas: 1 - revisionHistoryLimit: 3 - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - type: "RollingUpdate" + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "myapp" + app.kubernetes.io/instance: "{{ .Release.Name }}" + ports: + - name: "http" + protocol: "TCP" + port: 8080 + targetPort: 8080 diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#08.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#08.verified.yaml index 278bf076e42..67869876d40 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#08.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#08.verified.yaml @@ -1,20 +1,13 @@ --- apiVersion: "v1" -kind: "Service" +kind: "ConfigMap" metadata: - name: "myapp-service" + name: "myapp-config" labels: app.kubernetes.io/name: "aspire-hosting-tests" app.kubernetes.io/component: "myapp" app.kubernetes.io/instance: "{{ .Release.Name }}" -spec: - type: "ClusterIP" - selector: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "myapp" - app.kubernetes.io/instance: "{{ .Release.Name }}" - ports: - - name: "http" - protocol: "TCP" - port: 8080 - targetPort: 8080 +data: + ASPNETCORE_ENVIRONMENT: "{{ .Values.config.myapp.ASPNETCORE_ENVIRONMENT }}" + param0: "{{ .Values.config.myapp.param0 }}" + param2: "{{ .Values.config.myapp.param2 }}" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#09.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#09.verified.yaml index 67869876d40..5ce56f6c429 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#09.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#09.verified.yaml @@ -1,13 +1,14 @@ --- apiVersion: "v1" -kind: "ConfigMap" +kind: "Secret" metadata: - name: "myapp-config" + name: "myapp-secrets" labels: app.kubernetes.io/name: "aspire-hosting-tests" app.kubernetes.io/component: "myapp" app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - ASPNETCORE_ENVIRONMENT: "{{ .Values.config.myapp.ASPNETCORE_ENVIRONMENT }}" - param0: "{{ .Values.config.myapp.param0 }}" - param2: "{{ .Values.config.myapp.param2 }}" +stringData: + param1: "{{ .Values.secrets.myapp.param1 }}" + param3: "{{ .Values.secrets.myapp.param3 }}" + ConnectionStrings__cs: "Url={{ .Values.config.myapp.param0 }}, Secret={{ .Values.secrets.myapp.param1 }}" +type: "Opaque" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#10.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#10.verified.yaml deleted file mode 100644 index 5ce56f6c429..00000000000 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_GeneratesValidHelmChart#10.verified.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -apiVersion: "v1" -kind: "Secret" -metadata: - name: "myapp-secrets" - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "myapp" - app.kubernetes.io/instance: "{{ .Release.Name }}" -stringData: - param1: "{{ .Values.secrets.myapp.param1 }}" - param3: "{{ .Values.secrets.myapp.param3 }}" - ConnectionStrings__cs: "Url={{ .Values.config.myapp.param0 }}, Secret={{ .Values.secrets.myapp.param1 }}" -type: "Opaque" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#01.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#01.verified.yaml index ef457401ab5..b16121e9216 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#01.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#01.verified.yaml @@ -1,9 +1,6 @@ -parameters: {} +parameters: {} secrets: {} config: - env_dashboard: - DASHBOARD__FRONTEND__AUTHMODE: "Unsecured" - DASHBOARD__OTLP__AUTHMODE: "Unsecured" myapp: TLS_SUFFIX: ",ssl=true" TLS_SUFFIX_FALSE: ",ssl=false" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#02.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#02.verified.yaml index d9837afd3a3..4cce26b3384 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#02.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#02.verified.yaml @@ -18,9 +18,6 @@ spec: containers: - image: "mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest" name: "env-dashboard" - envFrom: - - configMapRef: - name: "env-dashboard-config" ports: - name: "http" protocol: "TCP" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#04.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#04.verified.yaml index 28f355f91c5..37de00c4947 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#04.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#04.verified.yaml @@ -1,12 +1,36 @@ --- -apiVersion: "v1" -kind: "ConfigMap" +apiVersion: "apps/v1" +kind: "Deployment" metadata: - name: "env-dashboard-config" + name: "myapp-deployment" labels: app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "env-dashboard" + app.kubernetes.io/component: "myapp" app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - DASHBOARD__FRONTEND__AUTHMODE: "{{ .Values.config.env_dashboard.DASHBOARD__FRONTEND__AUTHMODE }}" - DASHBOARD__OTLP__AUTHMODE: "{{ .Values.config.env_dashboard.DASHBOARD__OTLP__AUTHMODE }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "myapp" + app.kubernetes.io/instance: "{{ .Release.Name }}" + spec: + containers: + - image: "mcr.microsoft.com/dotnet/aspnet:8.0" + name: "myapp" + envFrom: + - configMapRef: + name: "myapp-config" + imagePullPolicy: "IfNotPresent" + selector: + matchLabels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "myapp" + app.kubernetes.io/instance: "{{ .Release.Name }}" + replicas: 1 + revisionHistoryLimit: 3 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: "RollingUpdate" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#05.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#05.verified.yaml index 37de00c4947..89a6179e7e0 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#05.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#05.verified.yaml @@ -1,36 +1,12 @@ --- -apiVersion: "apps/v1" -kind: "Deployment" +apiVersion: "v1" +kind: "ConfigMap" metadata: - name: "myapp-deployment" + name: "myapp-config" labels: app.kubernetes.io/name: "aspire-hosting-tests" app.kubernetes.io/component: "myapp" app.kubernetes.io/instance: "{{ .Release.Name }}" -spec: - template: - metadata: - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "myapp" - app.kubernetes.io/instance: "{{ .Release.Name }}" - spec: - containers: - - image: "mcr.microsoft.com/dotnet/aspnet:8.0" - name: "myapp" - envFrom: - - configMapRef: - name: "myapp-config" - imagePullPolicy: "IfNotPresent" - selector: - matchLabels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "myapp" - app.kubernetes.io/instance: "{{ .Release.Name }}" - replicas: 1 - revisionHistoryLimit: 3 - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - type: "RollingUpdate" +data: + TLS_SUFFIX: "{{ .Values.config.myapp.TLS_SUFFIX }}" + TLS_SUFFIX_FALSE: "{{ .Values.config.myapp.TLS_SUFFIX_FALSE }}" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#06.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#06.verified.yaml deleted file mode 100644 index 89a6179e7e0..00000000000 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpression#06.verified.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: "v1" -kind: "ConfigMap" -metadata: - name: "myapp-config" - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "myapp" - app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - TLS_SUFFIX: "{{ .Values.config.myapp.TLS_SUFFIX }}" - TLS_SUFFIX_FALSE: "{{ .Values.config.myapp.TLS_SUFFIX_FALSE }}" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#01.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#01.verified.yaml index 84363f49b59..b5ff02d1618 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#01.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#01.verified.yaml @@ -1,10 +1,7 @@ -parameters: +parameters: myapp: enable_tls: "True" secrets: {} config: - env_dashboard: - DASHBOARD__FRONTEND__AUTHMODE: "Unsecured" - DASHBOARD__OTLP__AUTHMODE: "Unsecured" myapp: TLS_SUFFIX: "" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#02.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#02.verified.yaml index d9837afd3a3..4cce26b3384 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#02.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#02.verified.yaml @@ -18,9 +18,6 @@ spec: containers: - image: "mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest" name: "env-dashboard" - envFrom: - - configMapRef: - name: "env-dashboard-config" ports: - name: "http" protocol: "TCP" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#04.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#04.verified.yaml index 28f355f91c5..37de00c4947 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#04.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#04.verified.yaml @@ -1,12 +1,36 @@ --- -apiVersion: "v1" -kind: "ConfigMap" +apiVersion: "apps/v1" +kind: "Deployment" metadata: - name: "env-dashboard-config" + name: "myapp-deployment" labels: app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "env-dashboard" + app.kubernetes.io/component: "myapp" app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - DASHBOARD__FRONTEND__AUTHMODE: "{{ .Values.config.env_dashboard.DASHBOARD__FRONTEND__AUTHMODE }}" - DASHBOARD__OTLP__AUTHMODE: "{{ .Values.config.env_dashboard.DASHBOARD__OTLP__AUTHMODE }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "myapp" + app.kubernetes.io/instance: "{{ .Release.Name }}" + spec: + containers: + - image: "mcr.microsoft.com/dotnet/aspnet:8.0" + name: "myapp" + envFrom: + - configMapRef: + name: "myapp-config" + imagePullPolicy: "IfNotPresent" + selector: + matchLabels: + app.kubernetes.io/name: "aspire-hosting-tests" + app.kubernetes.io/component: "myapp" + app.kubernetes.io/instance: "{{ .Release.Name }}" + replicas: 1 + revisionHistoryLimit: 3 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: "RollingUpdate" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#05.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#05.verified.yaml index 37de00c4947..303a7d529b0 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#05.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#05.verified.yaml @@ -1,36 +1,11 @@ --- -apiVersion: "apps/v1" -kind: "Deployment" +apiVersion: "v1" +kind: "ConfigMap" metadata: - name: "myapp-deployment" + name: "myapp-config" labels: app.kubernetes.io/name: "aspire-hosting-tests" app.kubernetes.io/component: "myapp" app.kubernetes.io/instance: "{{ .Release.Name }}" -spec: - template: - metadata: - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "myapp" - app.kubernetes.io/instance: "{{ .Release.Name }}" - spec: - containers: - - image: "mcr.microsoft.com/dotnet/aspnet:8.0" - name: "myapp" - envFrom: - - configMapRef: - name: "myapp-config" - imagePullPolicy: "IfNotPresent" - selector: - matchLabels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "myapp" - app.kubernetes.io/instance: "{{ .Release.Name }}" - replicas: 1 - revisionHistoryLimit: 3 - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - type: "RollingUpdate" +data: + TLS_SUFFIX: {{ if eq (.Values.parameters.myapp.enable_tls | lower) "true" }},ssl=true{{ else }},ssl=false{{ end }} diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#06.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#06.verified.yaml deleted file mode 100644 index 303a7d529b0..00000000000 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesConditionalReferenceExpressionWithParameterCondition#06.verified.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: "v1" -kind: "ConfigMap" -metadata: - name: "myapp-config" - labels: - app.kubernetes.io/name: "aspire-hosting-tests" - app.kubernetes.io/component: "myapp" - app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - TLS_SUFFIX: {{ if eq (.Values.parameters.myapp.enable_tls | lower) "true" }},ssl=true{{ else }},ssl=false{{ end }} diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#01.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#01.verified.yaml index fc4f8122cf4..efa2cce635d 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#01.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#01.verified.yaml @@ -1,4 +1,4 @@ -parameters: +parameters: SpeciaL_ApP: SpeciaL_ApP_image: "SpeciaL-ApP:latest" secrets: @@ -6,9 +6,6 @@ secrets: param3: "" ConnectionStrings__api_cs: "" config: - env_dashboard: - DASHBOARD__FRONTEND__AUTHMODE: "Unsecured" - DASHBOARD__OTLP__AUTHMODE: "Unsecured" SpeciaL_ApP: OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY: "in_memory" ConnectionStrings__api_cs2: "host.local:80" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#02.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#02.verified.yaml index b694dc9be3b..5d450bc4a03 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#02.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#02.verified.yaml @@ -18,9 +18,6 @@ spec: containers: - image: "mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest" name: "env-dashboard" - envFrom: - - configMapRef: - name: "env-dashboard-config" ports: - name: "http" protocol: "TCP" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#04.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#04.verified.yaml index 7b124741f11..544f449f306 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#04.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#04.verified.yaml @@ -1,12 +1,38 @@ --- -apiVersion: "v1" -kind: "ConfigMap" +apiVersion: "apps/v1" +kind: "Deployment" metadata: - name: "env-dashboard-config" + name: "special-app-deployment" labels: app.kubernetes.io/name: "my-chart" - app.kubernetes.io/component: "env-dashboard" + app.kubernetes.io/component: "SpeciaL-ApP" app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - DASHBOARD__FRONTEND__AUTHMODE: "{{ .Values.config.env_dashboard.DASHBOARD__FRONTEND__AUTHMODE }}" - DASHBOARD__OTLP__AUTHMODE: "{{ .Values.config.env_dashboard.DASHBOARD__OTLP__AUTHMODE }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/name: "my-chart" + app.kubernetes.io/component: "SpeciaL-ApP" + app.kubernetes.io/instance: "{{ .Release.Name }}" + spec: + containers: + - image: "{{ .Values.parameters.SpeciaL_ApP.SpeciaL_ApP_image }}" + name: "SpeciaL-ApP" + envFrom: + - configMapRef: + name: "special-app-config" + - secretRef: + name: "special-app-secrets" + imagePullPolicy: "IfNotPresent" + selector: + matchLabels: + app.kubernetes.io/name: "my-chart" + app.kubernetes.io/component: "SpeciaL-ApP" + app.kubernetes.io/instance: "{{ .Release.Name }}" + replicas: 1 + revisionHistoryLimit: 3 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: "RollingUpdate" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#05.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#05.verified.yaml index 544f449f306..696e2ba23db 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#05.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#05.verified.yaml @@ -1,38 +1,15 @@ --- -apiVersion: "apps/v1" -kind: "Deployment" +apiVersion: "v1" +kind: "ConfigMap" metadata: - name: "special-app-deployment" + name: "special-app-config" labels: app.kubernetes.io/name: "my-chart" app.kubernetes.io/component: "SpeciaL-ApP" app.kubernetes.io/instance: "{{ .Release.Name }}" -spec: - template: - metadata: - labels: - app.kubernetes.io/name: "my-chart" - app.kubernetes.io/component: "SpeciaL-ApP" - app.kubernetes.io/instance: "{{ .Release.Name }}" - spec: - containers: - - image: "{{ .Values.parameters.SpeciaL_ApP.SpeciaL_ApP_image }}" - name: "SpeciaL-ApP" - envFrom: - - configMapRef: - name: "special-app-config" - - secretRef: - name: "special-app-secrets" - imagePullPolicy: "IfNotPresent" - selector: - matchLabels: - app.kubernetes.io/name: "my-chart" - app.kubernetes.io/component: "SpeciaL-ApP" - app.kubernetes.io/instance: "{{ .Release.Name }}" - replicas: 1 - revisionHistoryLimit: 3 - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - type: "RollingUpdate" +data: + OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY: "{{ .Values.config.SpeciaL_ApP.OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY }}" + ConnectionStrings__api-cs2: "{{ .Values.config.SpeciaL_ApP.ConnectionStrings__api_cs2 }}" + OTEL_EXPORTER_OTLP_ENDPOINT: "{{ .Values.config.SpeciaL_ApP.OTEL_EXPORTER_OTLP_ENDPOINT }}" + OTEL_EXPORTER_OTLP_PROTOCOL: "{{ .Values.config.SpeciaL_ApP.OTEL_EXPORTER_OTLP_PROTOCOL }}" + OTEL_SERVICE_NAME: "{{ .Values.config.SpeciaL_ApP.OTEL_SERVICE_NAME }}" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#06.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#06.verified.yaml index 696e2ba23db..cd69a3b814c 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#06.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#06.verified.yaml @@ -1,15 +1,13 @@ --- apiVersion: "v1" -kind: "ConfigMap" +kind: "Secret" metadata: - name: "special-app-config" + name: "special-app-secrets" labels: app.kubernetes.io/name: "my-chart" app.kubernetes.io/component: "SpeciaL-ApP" app.kubernetes.io/instance: "{{ .Release.Name }}" -data: - OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY: "{{ .Values.config.SpeciaL_ApP.OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY }}" - ConnectionStrings__api-cs2: "{{ .Values.config.SpeciaL_ApP.ConnectionStrings__api_cs2 }}" - OTEL_EXPORTER_OTLP_ENDPOINT: "{{ .Values.config.SpeciaL_ApP.OTEL_EXPORTER_OTLP_ENDPOINT }}" - OTEL_EXPORTER_OTLP_PROTOCOL: "{{ .Values.config.SpeciaL_ApP.OTEL_EXPORTER_OTLP_PROTOCOL }}" - OTEL_SERVICE_NAME: "{{ .Values.config.SpeciaL_ApP.OTEL_SERVICE_NAME }}" +stringData: + param3: "{{ .Values.secrets.SpeciaL_ApP.param3 }}" + ConnectionStrings__api-cs: "Url={{ .Values.config.SpeciaL_ApP.param0 }}, Secret={{ .Values.secrets.SpeciaL_ApP.param1 }}" +type: "Opaque" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#07.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#07.verified.yaml deleted file mode 100644 index cd69a3b814c..00000000000 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_HandlesSpecialResourceName#07.verified.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: "v1" -kind: "Secret" -metadata: - name: "special-app-secrets" - labels: - app.kubernetes.io/name: "my-chart" - app.kubernetes.io/component: "SpeciaL-ApP" - app.kubernetes.io/instance: "{{ .Release.Name }}" -stringData: - param3: "{{ .Values.secrets.SpeciaL_ApP.param3 }}" - ConnectionStrings__api-cs: "Url={{ .Values.config.SpeciaL_ApP.param0 }}, Secret={{ .Values.secrets.SpeciaL_ApP.param1 }}" -type: "Opaque" diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ResourceWithProbes#00.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ResourceWithProbes#00.verified.yaml index d9837afd3a3..4cce26b3384 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ResourceWithProbes#00.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.PublishAsync_ResourceWithProbes#00.verified.yaml @@ -18,9 +18,6 @@ spec: containers: - image: "mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest" name: "env-dashboard" - envFrom: - - configMapRef: - name: "env-dashboard-config" ports: - name: "http" protocol: "TCP" From 6eb43067aa9b85847a5ad18784d6d50a6670131e Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Thu, 16 Apr 2026 22:41:33 +1000 Subject: [PATCH 74/78] Address JamesNK review feedback: fix node pool annotation, add az CLI arg validation, update spec - Fix FindNodePoolResource to add ManifestPublishingCallbackAnnotation.Ignore - Add defense-in-depth argument validation for az CLI commands - Update spec to reflect removed WithVersion/WithSkuTier/AsPrivateCluster APIs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/specs/aks-support.md | 4 +-- .../AzureKubernetesInfrastructure.cs | 27 ++++++++++++++++++- ...netesWithProjectResources#08.verified.yaml | 8 +++--- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/docs/specs/aks-support.md b/docs/specs/aks-support.md index 02c56f20490..edc42f6f515 100644 --- a/docs/specs/aks-support.md +++ b/docs/specs/aks-support.md @@ -528,8 +528,8 @@ var aks = builder.AddAzureKubernetesService("aks") - ✅ ACR auto-creation + AcrPull role assignment in Bicep - ✅ Kubeconfig retrieval via `az aks get-credentials` to isolated temp file - ✅ Multi-environment support (scoped Helm chart names, per-env kubeconfig) -- ✅ `WithVersion()`, `WithSkuTier()`, `WithContainerRegistry()` -- ✅ `AsPrivateCluster()` — sets `apiServerAccessProfile.enablePrivateCluster` +- ✅ `WithContainerRegistry()` +- ❌ ~~`WithVersion()`, `WithSkuTier()`, `AsPrivateCluster()`~~ — **Removed** in initial sweep; use `ConfigureInfrastructure()` instead - ✅ Push step dependency wiring for container image builds #### Phase 2: Workload Identity diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index cc6bc314749..2d07b7534b6 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -6,6 +6,7 @@ #pragma warning disable ASPIREFILESYSTEM001 // IFileSystemService/TempDirectory are experimental using System.Text; +using System.Text.RegularExpressions; using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Dcp.Process; using Aspire.Hosting.Eventing; @@ -23,7 +24,7 @@ namespace Aspire.Hosting.Azure.Kubernetes; /// Infrastructure eventing subscriber that processes compute resources /// targeting an AKS environment. /// -internal sealed class AzureKubernetesInfrastructure( +internal sealed partial class AzureKubernetesInfrastructure( ILogger logger) : IDistributedApplicationEventingSubscriber { @@ -196,6 +197,7 @@ private static AksNodePoolResource FindNodePoolResource( // Pool was added via NodePools config but not via AddNodePool — create the resource var config = environment.NodePools.First(p => p.Name == poolName); var pool = new AksNodePoolResource(poolName, config, environment); + pool.Annotations.Add(ManifestPublishingCallbackAnnotation.Ignore); appModel.Resources.Add(pool); return pool; } @@ -256,6 +258,11 @@ private static async Task GetAksCredentialsAsync( var resourceGroup = await GetResourceGroupAsync(azPath, clusterName, context) .ConfigureAwait(false); + // Defense-in-depth: validate that values used as CLI arguments + // contain only expected characters (alphanumeric, hyphens, underscores, dots). + ValidateAzureResourceName(clusterName, "cluster name"); + ValidateAzureResourceName(resourceGroup, "resource group"); + // Fetch kubeconfig content to stdout using --file - to avoid az CLI // writing credentials with potentially permissive file permissions. // We then write the content ourselves to a temp file with controlled access. @@ -412,4 +419,22 @@ private static async Task RunAzCommandAsync( } private sealed record AzCommandResult(int ExitCode, string StandardOutput, string StandardError); + + /// + /// Validates that an Azure resource name contains only expected characters. + /// Azure resource names and resource group names allow alphanumeric, hyphens, + /// underscores, parentheses, and dots. + /// + private static void ValidateAzureResourceName(string value, string parameterDescription) + { + if (!AzureResourceNamePattern().IsMatch(value)) + { + throw new InvalidOperationException( + $"The {parameterDescription} '{value}' contains unexpected characters. " + + $"Expected only alphanumeric characters, hyphens, underscores, parentheses, and dots."); + } + } + + [GeneratedRegex(@"^[a-zA-Z0-9\-_\.\(\)]+$")] + private static partial Regex AzureResourceNamePattern(); } diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#08.verified.yaml b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#08.verified.yaml index dcd8f588158..711b0061a78 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#08.verified.yaml +++ b/tests/Aspire.Hosting.Kubernetes.Tests/Snapshots/KubernetesPublisherTests.KubernetesWithProjectResources#08.verified.yaml @@ -12,10 +12,10 @@ data: services__project1__http__0: "http://project1-service:{{ .Values.parameters.project1.port_http }}" PROJECT1_HTTPS: "https://project1-service:{{ .Values.parameters.project1.port_http }}" PROJECT1_CUSTOM1: "{{ .Values.config.api.PROJECT1_CUSTOM1 }}" - services__project1__http__1: "{{ .Values.config.api.services__project1__http__1 }}" + services__project1__custom1__0: "{{ .Values.config.api.services__project1__custom1__0 }}" PROJECT1_CUSTOM2: "{{ .Values.config.api.PROJECT1_CUSTOM2 }}" - services__project1__http__2: "{{ .Values.config.api.services__project1__http__2 }}" + services__project1__custom2__0: "{{ .Values.config.api.services__project1__custom2__0 }}" PROJECT1_CUSTOM3: "{{ .Values.config.api.PROJECT1_CUSTOM3 }}" - services__project1__http__3: "{{ .Values.config.api.services__project1__http__3 }}" + services__project1__custom3__0: "{{ .Values.config.api.services__project1__custom3__0 }}" PROJECT1_CUSTOM4: "{{ .Values.config.api.PROJECT1_CUSTOM4 }}" - services__project1__http__4: "{{ .Values.config.api.services__project1__http__4 }}" + services__project1__custom4__0: "{{ .Values.config.api.services__project1__custom4__0 }}" From fb883c095272418f2e4d1c447526f2d9635911ed Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Thu, 16 Apr 2026 23:07:52 +1000 Subject: [PATCH 75/78] Add dashboard login token kubectl logs command to print summary Shows 'kubectl logs' command in deployment summary so users can retrieve the dashboard login token after deployment. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Deployment/HelmDeploymentEngine.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs b/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs index 91b3b741550..07a1b1f0f1b 100644 --- a/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs +++ b/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs @@ -576,6 +576,11 @@ private static async Task PrintDeploymentInstructionsAsync( context.Summary.Add( "📊 Dashboard", new MarkdownString($"`kubectl port-forward -n {@namespace} svc/{dashboardServiceName} 18888:18888` then open [http://localhost:18888](http://localhost:18888)")); + + var dashboardDeploymentName = environment.Dashboard.Resource.Name.ToKubernetesResourceName(); + context.Summary.Add( + "🔑 Dashboard login", + new MarkdownString($"`kubectl logs -n {@namespace} -l app={dashboardDeploymentName} --tail=50` to retrieve the login token")); } // Helm status and resource inspection From 0d5ac11b54c3daed2f7883b10f797e54d134a8bd Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Thu, 16 Apr 2026 23:33:49 +1000 Subject: [PATCH 76/78] Fix dashboard logs label selector to use app.kubernetes.io/component The bare 'app' label doesn't exist on pods - use the standard Kubernetes label 'app.kubernetes.io/component' which matches the actual deployment labels. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Deployment/HelmDeploymentEngine.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs b/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs index 07a1b1f0f1b..cf8c380151f 100644 --- a/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs +++ b/src/Aspire.Hosting.Kubernetes/Deployment/HelmDeploymentEngine.cs @@ -580,7 +580,7 @@ private static async Task PrintDeploymentInstructionsAsync( var dashboardDeploymentName = environment.Dashboard.Resource.Name.ToKubernetesResourceName(); context.Summary.Add( "🔑 Dashboard login", - new MarkdownString($"`kubectl logs -n {@namespace} -l app={dashboardDeploymentName} --tail=50` to retrieve the login token")); + new MarkdownString($"`kubectl logs -n {@namespace} -l app.kubernetes.io/component={dashboardDeploymentName} --tail=50` to retrieve the login token")); } // Helm status and resource inspection From 0defaaa4173658e077fe00a4f2cacb7447a33231 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Fri, 17 Apr 2026 11:47:32 +1000 Subject: [PATCH 77/78] Address security feedback: validation ordering, stale annotations, conditional outputs - Move ValidateAzureResourceName before GetResourceGroupAsync so cluster name is validated before use in az CLI commands - Remove stale ContainerRegistryReferenceAnnotation from inner K8s environment when WithContainerRegistry replaces the default ACR - Make oidcIssuerUrl Bicep output conditional on OidcIssuerEnabled Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 27 +++++++++++++++---- .../AzureKubernetesInfrastructure.cs | 6 +++-- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index 04a258250f9..f0a2aa8fa7b 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -284,6 +284,17 @@ public static IResourceBuilder WithContainer // Set the explicit registry via annotation on both the AKS environment // and the inner K8s environment (so KubernetesInfrastructure finds it) builder.WithAnnotation(new ContainerRegistryReferenceAnnotation(registry.Resource)); + + // Remove any stale container registry annotations from the inner K8s environment + // before adding the new one (the default ACR annotation was added during + // AddAzureKubernetesEnvironment and now references a removed resource). + var staleAnnotations = builder.Resource.KubernetesEnvironment.Annotations + .OfType().ToList(); + foreach (var old in staleAnnotations) + { + builder.Resource.KubernetesEnvironment.Annotations.Remove(old); + } + builder.Resource.KubernetesEnvironment.Annotations.Add( new ContainerRegistryReferenceAnnotation(registry.Resource)); @@ -515,12 +526,18 @@ private static void ConfigureAksInfrastructure(AzureResourceInfrastructure infra { Value = new MemberExpression(new MemberExpression(aksId, "properties"), "fqdn") }); - infrastructure.Add(new ProvisioningOutput("oidcIssuerUrl", typeof(string)) + // OIDC issuer URL and kubelet identity outputs are only valid when the + // corresponding features are enabled on the cluster. + if (aksResource.OidcIssuerEnabled) { - Value = new MemberExpression( - new MemberExpression(new MemberExpression(aksId, "properties"), "oidcIssuerProfile"), - "issuerURL") - }); + infrastructure.Add(new ProvisioningOutput("oidcIssuerUrl", typeof(string)) + { + Value = new MemberExpression( + new MemberExpression(new MemberExpression(aksId, "properties"), "oidcIssuerProfile"), + "issuerURL") + }); + } + infrastructure.Add(new ProvisioningOutput("kubeletIdentityObjectId", typeof(string)) { Value = new MemberExpression( diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs index 2d07b7534b6..7b4016b209e 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesInfrastructure.cs @@ -255,12 +255,14 @@ private static async Task GetAksCredentialsAsync( ?? environment.Name; var azPath = FindAzCli(); - var resourceGroup = await GetResourceGroupAsync(azPath, clusterName, context) - .ConfigureAwait(false); // Defense-in-depth: validate that values used as CLI arguments // contain only expected characters (alphanumeric, hyphens, underscores, dots). ValidateAzureResourceName(clusterName, "cluster name"); + + var resourceGroup = await GetResourceGroupAsync(azPath, clusterName, context) + .ConfigureAwait(false); + ValidateAzureResourceName(resourceGroup, "resource group"); // Fetch kubeconfig content to stdout using --file - to avoid az CLI From 95e3164738eb69b65bb330d2aa903c2500115780 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Fri, 17 Apr 2026 11:58:32 +1000 Subject: [PATCH 78/78] Add enabled parameter to WithWorkloadIdentity for opt-out support WithWorkloadIdentity(false) disables both OIDC issuer and workload identity. Defaults remain true for the happy path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AzureKubernetesEnvironmentExtensions.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index f0a2aa8fa7b..2d82b90e387 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -306,10 +306,11 @@ public static IResourceBuilder WithContainer } /// - /// Enables workload identity on the AKS environment, allowing pods to authenticate + /// Enables or disables workload identity on the AKS environment, allowing pods to authenticate /// to Azure services using federated credentials. /// /// The resource builder. + /// true to enable workload identity (the default); false to disable it. /// A reference to the for chaining. /// /// This ensures the AKS cluster is configured with OIDC issuer and workload identity enabled. @@ -318,12 +319,13 @@ public static IResourceBuilder WithContainer /// [AspireExport(Description = "Enables workload identity on the AKS cluster")] public static IResourceBuilder WithWorkloadIdentity( - this IResourceBuilder builder) + this IResourceBuilder builder, + bool enabled = true) { ArgumentNullException.ThrowIfNull(builder); - builder.Resource.OidcIssuerEnabled = true; - builder.Resource.WorkloadIdentityEnabled = true; + builder.Resource.OidcIssuerEnabled = enabled; + builder.Resource.WorkloadIdentityEnabled = enabled; return builder; }