[tests] Workload tests (dotnet#4168)

This adds `Aspire.Workload.Tests` which are meant to exercise the `aspire` workload. The individual tests create projects from templates just like a user would, and then run, and validate them.

It uses the existing infrastructure from `Aspire.EndToEnd.Tests` to install sdk+workload for testing.

- Enables tests on Linux, and Windows
- Because we don't have docker support on helix/Windows yet, the docker dependent tests are skipped on windows

## Tests

Two kinds of tests are being added here:

1. Tests that create+build a `aspire-starter` template project, and then some tests interact with that running app to validate it
2. Individual tests exercise the various templates, for example to create `aspire-starter` with [xunit,nunit,mstest] options

- The tests also connect to the dashboard and validate that the expected services show up as expected using `Playwright`.
- Some tests also connect to the webapp in `aspire-starter` template to validate some parts of it.

Fixes dotnet#47 .

Follow up work is tracked in dotnet#4222

Co-authored-by: Eric Erhardt <[email protected]>
Co-authored-by: Jose Perez Rodriguez <[email protected]>

Author: 3 people

Aspire.sln

@@ -449,6 +449,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OrleansClient", "playground
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OrleansContracts", "playground\orleans\OrleansContracts\OrleansContracts.csproj", "{75760E8A-7025-4F6A-B152-6622688D0E7D}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Aspire.Workload.Tests", "tests\Aspire.Workload.Tests\Aspire.Workload.Tests.csproj", "{3ECB176E-3CC8-4BEE-AF48-F439F9742C8B}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Aspire.Seq.Tests", "tests\Aspire.Seq.Tests\Aspire.Seq.Tests.csproj", "{0505F739-6F85-4502-A554-77E0D7325F26}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Extensions.ServiceDiscovery.Yarp.Tests", "tests\Microsoft.Extensions.ServiceDiscovery.Yarp.Tests\Microsoft.Extensions.ServiceDiscovery.Yarp.Tests.csproj", "{B5D637F3-7E44-4C7D-AE4B-D56D400671CD}"
@@ -1183,6 +1185,10 @@ Global
 		{8191109E-130C-47F3-B84E-82070A6CD269}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{8191109E-130C-47F3-B84E-82070A6CD269}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{8191109E-130C-47F3-B84E-82070A6CD269}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3ECB176E-3CC8-4BEE-AF48-F439F9742C8B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3ECB176E-3CC8-4BEE-AF48-F439F9742C8B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3ECB176E-3CC8-4BEE-AF48-F439F9742C8B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3ECB176E-3CC8-4BEE-AF48-F439F9742C8B}.Release|Any CPU.Build.0 = Release|Any CPU
 		{906B5687-31AD-4364-AD9F-B4B26113462D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{906B5687-31AD-4364-AD9F-B4B26113462D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{906B5687-31AD-4364-AD9F-B4B26113462D}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -1423,6 +1429,7 @@ Global
 		{E0E1B557-D3CF-4446-B993-E5CE719234FB} = {27381127-6C45-4B4C-8F18-41FF48DFE4B2}
 		{A9CFA376-0C90-4231-9152-FBF14065195A} = {4981B3A5-4AFD-4191-BF7D-8692D9783D60}
 		{8191109E-130C-47F3-B84E-82070A6CD269} = {4981B3A5-4AFD-4191-BF7D-8692D9783D60}
+		{3ECB176E-3CC8-4BEE-AF48-F439F9742C8B} = {4981B3A5-4AFD-4191-BF7D-8692D9783D60}
 		{906B5687-31AD-4364-AD9F-B4B26113462D} = {8BAF2119-8370-4E9E-A887-D92506F8C727}
 		{75760E8A-7025-4F6A-B152-6622688D0E7D} = {8BAF2119-8370-4E9E-A887-D92506F8C727}
 		{0505F739-6F85-4502-A554-77E0D7325F26} = {4981B3A5-4AFD-4191-BF7D-8692D9783D60}

Directory.Packages.props

@@ -141,6 +141,7 @@
     <PackageVersion Include="Microsoft.DotNet.XUnitExtensions" Version="$(MicrosoftDotNetXUnitExtensionsPackageVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Diagnostics.Testing" Version="$(MicrosoftExtensionsDiagnosticsTestingPackageVersion)" />
     <PackageVersion Include="Microsoft.NET.Runtime.WorkloadTesting.Internal" Version="$(MicrosoftNETRuntimeWorkloadTestingInternalVersion)" />
+    <PackageVersion Include="Microsoft.Playwright" Version="1.42.0" />
     <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.6.1" />
     <PackageVersion Include="Testcontainers.MongoDb" Version="$(TestcontainersPackageVersion)" />
     <PackageVersion Include="Testcontainers.MsSql" Version="$(TestcontainersPackageVersion)" />

NuGet.config

@@ -12,7 +12,6 @@
     <!--  Begin: Package sources from dotnet-runtime -->
     <!--  End: Package sources from dotnet-runtime -->
     <!--End: Package sources managed by Dependency Flow automation. Do not edit the sources above.-->
-    <!-- TEST_RESTORE_SOURCES_INSERTION_LINE -->
     <add key="dotnet-public" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json" />
     <add key="dotnet-eng" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />
     <add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />

dogfood.sh

@@ -0,0 +1,48 @@
+if [ -z "$ZSH_VERSION" ]; then
+  source="${BASH_SOURCE[0]}"
+  # resolve $SOURCE until the file is no longer a symlink
+  while [[ -h $source ]]; do
+    scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+    source="$(readlink "$source")"
+
+    # if $source was a relative symlink, we need to resolve it relative to the path where the
+    # symlink file was located
+    [[ $source != /* ]] && source="$scriptroot/$source"
+  done
+  scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+else
+  # :A will resolve all symlinks, :h will truncate last path component leaving you with a directory name
+  scriptroot=${0:A:h}
+fi
+
+REPO_ROOT=$(cd "${scriptroot}";pwd)
+SDK_PATH=$REPO_ROOT/artifacts/bin/dotnet-latest
+if [ ! -x "$SDK_PATH/dotnet" ]; then
+    echo "Error: Could not find dotnet at $SDK_PATH/dotnet"
+    return
+fi
+
+CONFIG=$1
+if [ -n "$CONFIG" ]; then
+    PKG_DIR=$REPO_ROOT/artifacts/packages/$CONFIG/Shipping
+    if [ ! -d "$PKG_DIR" ]; then
+        echo "Error: Could not find packages path $PKG_DIR for CONFIG=$CONFIG"
+        return
+    fi
+else
+    PKG_DIR=$REPO_ROOT/artifacts/packages/Release/Shipping
+    if [ ! -d "$PKG_DIR" ]; then
+      PKG_DIR=$REPO_ROOT/artifacts/packages/Debug/Shipping
+    fi
+    if [ ! -d "$PKG_DIR" ]; then
+        echo "Error: Could not find packages path in $REPO_ROOT/artifacts/packages for Release, or Debug configurations"
+        return
+    fi
+fi
+
+echo "Adding $SDK_PATH to \$PATH"
+export PATH=$SDK_PATH:$PATH
+echo Setting BUILT_NUGETS_PATH="$PKG_DIR" to resolve locally built packages
+export BUILT_NUGETS_PATH=$PKG_DIR
+echo
+echo "Use $REPO_ROOT/tests/Aspire.Workload.Tests/data/nuget8.config as your local nuget.config to ensure the packages can be restored."

eng/pipelines/templates/BuildAndTest.yml

@@ -21,26 +21,14 @@ parameters:
     default: false
 
 steps:
-  - ${{ if eq(parameters.isWindows, 'true') }}:
-    - script: ${{ parameters.buildScript }}
-              -restore -build
-              -configuration ${{ parameters.buildConfig }}
-              /p:ArchiveTests=true
-              /bl:${{ parameters.repoLogPath }}/build.binlog
-              $(_OfficialBuildIdArgs)
-      displayName: Build
-  - ${{ if ne(parameters.isWindows, 'true') }}:
-    # temporary: Use -pack for linux as it is required for workload testing
-    # Issue: https://github.com/dotnet/aspire/issues/2687
-    - script: ${{ parameters.buildScript }}
-              -restore -build
-              -pack
-              -configuration ${{ parameters.buildConfig }}
-              /p:ArchiveTests=true
-              /bl:${{ parameters.repoLogPath }}/build.binlog
-              $(extraBuildArgs)
-              $(_OfficialBuildIdArgs)
-      displayName: Build
+  - script: ${{ parameters.buildScript }}
+            -restore -build
+            -pack
+            -configuration ${{ parameters.buildConfig }}
+            /p:ArchiveTests=true
+            /bl:${{ parameters.repoLogPath }}/build.binlog
+            $(_OfficialBuildIdArgs)
+    displayName: Build
 
   - ${{ if ne(parameters.skipTests, 'true') }}:
     - script: ${{ parameters.dotnetScript }} dotnet-coverage collect
@@ -49,15 +37,11 @@ steps:
               "${{ parameters.buildScript }} -testnobuild -test -configuration ${{ parameters.buildConfig }} /bl:${{ parameters.repoLogPath }}/tests.binlog $(_OfficialBuildIdArgs)"
       displayName: Run non-helix tests
 
-    - ${{ if ne(parameters.isWindows, 'true') }}:
-      # E2E tests are disabled on windows for now
-      - script: ${{ parameters.buildScript }}
-                -p:ContinuousIntegrationBuild=true
-                -p:Configuration=${{ parameters.buildConfig }}
-                -p:TriggerInstallWorkloadForTesting=true
-                -bl:${{ parameters.repoLogPath }}/WorkloadInstallForTesting.binlog
-                --projects $(Build.SourcesDirectory)/tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj
-        displayName: Install sdk+workload for testing
+    - script: ${{ parameters.buildScript }}
+              /p:Configuration=${{ parameters.buildConfig }}
+              /bl:${{ parameters.repoLogPath }}/WorkloadInstallForTesting.binlog
+              -projects $(Build.SourcesDirectory)/tests/workloads.proj
+      displayName: Install sdk+workload for testing
 
     # Helix captures code coverage information and, once tests are complete, the code coverage information is
     # downloaded to <repo root>/artifacts/helixresults folder.

tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj

@@ -11,8 +11,6 @@
     <TestUsingWorkloads Condition="! ('$(ContinuousIntegrationBuild)' == 'true' and '$(OS)' == 'Windows_NT')">true</TestUsingWorkloads>
     <InstallWorkloadForTesting>$(TestUsingWorkloads)</InstallWorkloadForTesting>
 
-    <BuiltNuGetsDir>$(ArtifactsShippingPackagesDir)</BuiltNuGetsDir>
-    <PackageVersionForWorkloadManifests>$(PackageVersion)</PackageVersionForWorkloadManifests>
     <TestsRunningOutsideOfRepo Condition="'$(TestsRunningOutsideOfRepo)' == '' and '$(ContinuousIntegrationBuild)' == 'true'">true</TestsRunningOutsideOfRepo>
     <DefineConstants Condition="'$(TestsRunningOutsideOfRepo)' == 'true'">TESTS_RUNNING_OUTSIDE_OF_REPO;$(DefineConstants)</DefineConstants>
 
@@ -23,7 +21,6 @@
 
   <ItemGroup>
     <Compile Include="..\testproject\Common\TestResourceNames.cs" />
-    <Compile Include="..\Shared\WorkloadTesting\*.cs" Link="WorkloadTestingCommon" />
 
     <None Include="..\testproject\**\*" Link="testassets\testproject\%(RecursiveDir)%(FileName)%(Extension)" CopyToOutputDirectory="PreserveNewest" />
     <None Include="..\.editorconfig" Link="testassets\%(FileName)%(Extension)" CopyToOutputDirectory="PreserveNewest" />
@@ -33,4 +30,6 @@
     <PackageReference Include="Microsoft.Extensions.Http.Resilience" />
     <PackageReference Include="Microsoft.DotNet.XUnitExtensions" />
   </ItemGroup>
+
+  <Import Project="..\Shared\WorkloadTesting\Aspire.Shared.WorkloadTesting.targets" />
 </Project>

tests/Aspire.EndToEnd.Tests/Directory.Build.props

@@ -1,5 +1,6 @@
 <Project>
   <PropertyGroup>
+    <!-- This triggers the import of targets needed for workload testing -->
     <IsWorkloadTestProject>true</IsWorkloadTestProject>
   </PropertyGroup>
 

tests/Aspire.EndToEnd.Tests/README.md

@@ -0,0 +1,54 @@
+# Aspire.EndToEnd.Tests
+
+## TL;DR or How do I use this?
+
+### Steps to prepare for `outside-of-repo` runs
+
+1. [Install the sdk+workload](../Aspire.Workload.Tests/README.md#install-the-sdkworkload)
+
+### Using it from VS
+
+- For switching to `outside-of-repo` add `<TestsRunningOutsideOfRepo>true</TestsRunningOutsideOfRepo>` to `tests/Aspire.EndToEnd.Tests/Directory.Build.props` *before* any imports.
+    - tests cannot be run at this point as they will fail complaining about `artifacts/bin/dotnet-latest` being missing
+    - Install the sdk+workload following the steps above
+    - Run/debug the tests normally now, and they will be using the sdk
+    - Also note that in this case the testproject is run from the bindir for `Aspire.EndToEnd.Tests`, so a path like `artifacts/bin/Aspire.EndToEnd.Tests/Debug/net8.0/testassets/testproject/`
+
+### Using it from command line
+
+- When running the tests you can either:
+    - set `<TestsRunningOutsideOfRepo>true</TestsRunningOutsideOfRepo>` to `tests/Aspire.EndToEnd.Tests/Directory.props` before any imports
+    - or set the environment variable `TestsRunningOutsideOfRepo=true`
+
+## (details) What is the goal here?
+
+1. We want to run some EndToEnd tests on CI, which can `dotnet run` an aspire project,
+and allow individual tests to interact with the services.
+This requires:
+
+    - Ability to build, and run an aspire project - IOW, a sdk with the `aspire` workload installed.
+    - `docker`
+
+2. Also, allow using `TestProject.*` in `tests/testproject`, in two modes:
+- `in-repo` test run which directly reference aspire projects, and repo targets
+- `outside-of-repo` test runs which uses a SDK+workload based on local build output
+
+## Solution:
+
+### SDK+workload
+
+[Sdk+workload](../Aspire.Workload.Tests/README.md#solution-sdkworkload)
+
+### TestProject
+
+- This can switch between the two test run modes using the msbuild property `$(TestsRunningOutsideOfRepo)`
+    - when running `in-repo` the test project directly references hosting targets, and aspire projects via `ProjectReference`
+    - when running `outside-of-repo` the `ProjectReferences` and imports are replaced with `PackageReferences` to the Aspire nugets
+- Default is to run `in-repo`
+
+### Helix
+
+- The tests are built on the build machine
+- The testproject, and the sdk+workload is sent to helix
+  - where the tests run using `dotnet` from the sdk+workload path
+- Since `docker` is needed to helix, this is enabled only for `Linux` in this PR. Blocked on https://github.com/dotnet/dnceng/issues/2067 for windows support.

tests/Aspire.Workload.Tests/.runsettings

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+  <RunConfiguration>
+    <!-- Timeout in ms, 20 minutes -->
+    <TestSessionTimeout>1200000</TestSessionTimeout>
+    <!-- Filter out failing (wrong framework, platform, runtime or activeissue) tests -->
+    <TestCaseFilter>category!=failing</TestCaseFilter>
+  </RunConfiguration>
+  <LoggerRunSettings>
+    <Loggers>
+      <Logger friendlyName="trx">
+        <Configuration>
+          <LogFileName>TestResults.trx</LogFileName>
+        </Configuration>
+      </Logger>
+      <Logger friendlyName="console">
+        <Configuration>
+          <Verbosity>normal</Verbosity>
+        </Configuration>
+      </Logger>
+    </Loggers>
+  </LoggerRunSettings>
+</RunSettings>

tests/Aspire.Workload.Tests/Aspire.Workload.Tests.csproj

@@ -0,0 +1,56 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>$(NetCurrent)</TargetFramework>
+
+    <RunTestsOnHelix>true</RunTestsOnHelix>
+    <TestUsingWorkloads>true</TestUsingWorkloads>
+    <InstallWorkloadForTesting>true</InstallWorkloadForTesting>
+
+    <XunitRunnerJson>xunit.runner.json</XunitRunnerJson>
+    <RunSettingsFilePath>$(MSBuildThisFileDirectory).runsettings</RunSettingsFilePath>
+    <TestArchiveTestsDir>$(TestArchiveTestsDirForWorkloadTests)</TestArchiveTestsDir>
+
+    <InstallChromeForPlaywright Condition="'$(InstallChromeForPlaywright)' == '' and '$(CODESPACES)' == 'true'">true</InstallChromeForPlaywright>
+    <InstallChromeForPlaywright Condition="'$(InstallChromeForPlaywright)' == '' and '$(ArchiveTests)' == 'true' and '$(ContinuousIntegrationBuild)' == 'true'">true</InstallChromeForPlaywright>
+
+    <ExtractTestClassNamesForHelix Condition="'$(ContinuousIntegrationBuild)' == 'true' or '$(ArchiveTests)' == 'true'">true</ExtractTestClassNamesForHelix>
+  </PropertyGroup>
+
+  <Import Project="..\Shared\provisioning.targets" />
+  <Import Project="..\Shared\WorkloadTesting\Aspire.Shared.WorkloadTesting.targets" />
+
+  <ItemGroup>
+    <Compile Include="$(RepoRoot)src\Aspire.Hosting.Redis\RedisContainerImageTags.cs" />
+
+    <None Include="data\**\*" CopyToOutputDirectory="PreserveNewest" />
+    <None Include="$(RepoRoot)eng\helix\runtests.*" CopyToOutputDirectory="PreserveNewest" />
+
+    <PackageReference Include="Microsoft.DotNet.XUnitExtensions" />
+    <PackageReference Include="Microsoft.Extensions.Http.Resilience" />
+  </ItemGroup>
+
+  <!-- Used for running one helix job per test class -->
+  <Target Name="_ExtractTestClassNames"
+          Condition="'$(ExtractTestClassNamesForHelix)' == 'true'"
+          BeforeTargets="ZipTestArchive">
+
+    <Exec Command="&quot;$(DotNetTool)&quot; test --no-build -c $(Configuration) -s $(RunSettingsFilePath) --list-tests --nologo -v:q -p:VsTestUseMSBuildOutput=false" ConsoleToMSBuild="true">
+      <Output TaskParameter="ConsoleOutput" ItemName="_ListOfTestsLines" />
+    </Exec>
+
+    <PropertyGroup>
+      <_Regex>^\s*(Aspire.Workload.Tests[^\($]+)</_Regex>
+    </PropertyGroup>
+    <ItemGroup>
+      <_TestLines0 Include="$([System.Text.RegularExpressions.Regex]::Match('%(_ListOfTestsLines.Identity)', '$(_Regex)'))" />
+      <TestClassName Include="$([System.IO.Path]::GetFileNameWithoutExtension('%(_TestLines0.Identity)'))" />
+    </ItemGroup>
+
+    <Error Text="No Aspire.Workload.Tests test classes found!" Condition="'@(TestClassName)' == ''" />
+
+    <WriteLinesToFile File="$(TestArchiveTestsDir)$(MSBuildProjectName).tests.list"
+                      Lines="@(TestClassName->Distinct())"
+                      Overwrite="true" />
+  </Target>
+</Project>

tests/Aspire.Workload.Tests/Directory.Build.props

@@ -0,0 +1,8 @@
+<Project>
+  <PropertyGroup>
+    <!-- This triggers the import of targets needed for workload testing -->
+    <IsWorkloadTestProject>true</IsWorkloadTestProject>
+  </PropertyGroup>
+
+  <Import Project="..\Directory.Build.props" />
+</Project>

tests/Aspire.Workload.Tests/EmptyTemplateRunFixture.cs

@@ -0,0 +1,20 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Xunit.Abstractions;
+
+namespace Aspire.Workload.Tests;
+
+/// <summary>
+/// This fixture ensures the TestProject.AppHost application is started before a test is executed.
+///
+/// Represents the the IntegrationServiceA project in the test application used to send HTTP requests
+/// to the project's endpoints.
+/// </summary>
+public sealed class EmptyTemplateRunFixture : TemplateAppFixture
+{
+    public EmptyTemplateRunFixture(IMessageSink diagnosticMessageSink)
+        : base(diagnosticMessageSink, "aspire")
+    {
+    }
+}

tests/Aspire.Workload.Tests/EmptyTemplateRunTests.cs

@@ -0,0 +1,28 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Aspire.Workload.Tests;
+
+public class EmptyTemplateRunTests : WorkloadTestsBase, IClassFixture<EmptyTemplateRunFixture>
+{
+    private readonly EmptyTemplateRunFixture _testFixture;
+
+    public EmptyTemplateRunTests(EmptyTemplateRunFixture fixture, ITestOutputHelper testOutput)
+        : base(testOutput)
+    {
+        _testFixture = fixture;
+    }
+
+    [Fact]
+    public async Task ResourcesShowUpOnDashboad()
+    {
+        await using var context = await CreateNewBrowserContextAsync();
+        await CheckDashboardHasResourcesAsync(
+            await _testFixture.Project!.OpenDashboardPageAsync(context),
+            [],
+            timeoutSecs: 1_000);
+    }
+}