Skip to content

Commit fcfea4f

Browse files
radicaleerhardt
radical
and
eerhardt
authored
[tests] Misc improvements for workload based tests (dotnet#4093)
* [tests] Fix deploying per-test .runsettings * [tests] helix: cleanup, and install devcerts on helix - uses scripts from `https://github.com/BorisWilhelms/create-dotnet-devcert` on linux * [tests] BuildEnvironment: cleanup, and add support for TFMs - so tests have a different setup per tfm (net8.0/net9.0/..) * [tests] Misc improvements, and more error checking * [tests] Add tests/workloads.proj to allow installing sdk+workload * cleanup * Remove old unused property * Rename SolutionRoot to RepoRoot - feedback from @ eerhardt * address review feedback from @ eehardt * Make AspireProject.AppExited nullable * address review feedback from @ eerhardt * Move BuildEnvironment.TestsProjectPath to EndToEnd tests since it is e2e specific * disambiguate logs path * Cleanup setting testasssets path for E2E tests * BuildEnvironment: when using system dotnet, don't override NUGET_PACKAGES path * EndToEnd tests: copy .editorconfig also, to get the analyzer warning ignores * cleanup * EndToEnd tests: always build with TestsRunningOutsideOfRepo=true on CI * address review feedback from @ eerhardt * simple cleanup * Rename AspireProject.{Start,Stop}Async to {Start,Stop}AppHostAsync --------- Co-authored-by: Eric Erhardt <[email protected]>
1 parent 6f5c4a2 commit fcfea4f

18 files changed

+246
-145
lines changed

‎Directory.Build.props

-2
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@
2424
<StrongNameKeyId>Open</StrongNameKeyId>
2525
<Nullable>enable</Nullable>
2626
<ImplicitUsings>enable</ImplicitUsings>
27-
<!-- Set RunSettingsFilePath property which is read by VSTest. -->
28-
<RunSettingsFilePath Condition="'$(RunSettingsFilePath)' == ''">$(RepositoryEngineeringDir).runsettings</RunSettingsFilePath>
2927
<!-- Redirect test logs into a subfolder -->
3028
<TestResultsLogDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsLogDir)', 'TestLogs'))</TestResultsLogDir>
3129
<!-- We don't want to use the workload for AppHost projects in this repo -->

‎tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj

+3
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,11 @@
1313

1414
<BuiltNuGetsDir>$(ArtifactsShippingPackagesDir)</BuiltNuGetsDir>
1515
<PackageVersionForWorkloadManifests>$(PackageVersion)</PackageVersionForWorkloadManifests>
16+
<TestsRunningOutsideOfRepo Condition="'$(TestsRunningOutsideOfRepo)' == '' and '$(ContinuousIntegrationBuild)' == 'true'">true</TestsRunningOutsideOfRepo>
1617
<DefineConstants Condition="'$(TestsRunningOutsideOfRepo)' == 'true'">TESTS_RUNNING_OUTSIDE_OF_REPO;$(DefineConstants)</DefineConstants>
1718

1819
<XunitRunnerJson>xunit.runner.json</XunitRunnerJson>
20+
<RunSettingsFilePath>$(MSBuildThisFileDirectory).runsettings</RunSettingsFilePath>
1921
<TestArchiveTestsDir>$(TestArchiveTestsDirForEndToEndTests)</TestArchiveTestsDir>
2022
</PropertyGroup>
2123

@@ -24,6 +26,7 @@
2426
<Compile Include="..\Shared\WorkloadTesting\*.cs" Link="WorkloadTestingCommon" />
2527

2628
<None Include="..\testproject\**\*" Link="testassets\testproject\%(RecursiveDir)%(FileName)%(Extension)" CopyToOutputDirectory="PreserveNewest" />
29+
<None Include="..\.editorconfig" Link="testassets\%(FileName)%(Extension)" CopyToOutputDirectory="PreserveNewest" />
2730
<None Condition="'$(InstallWorkloadForTesting)' == 'true'" Include="$(PatchedNuGetConfigPath)" Link="testassets\testproject\nuget.config" CopyToOutputDirectory="PreserveNewest" />
2831
<None Include="$(RepoRoot)Directory.Packages.props" Link="testassets\testproject\Directory.Packages.repo.props" CopyToOutputDirectory="PreserveNewest" />
2932

‎tests/Aspire.EndToEnd.Tests/Directory.Build.props

-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
<Project>
22
<PropertyGroup>
3-
<IsHelixOnlyTestProject>true</IsHelixOnlyTestProject>
43
<IsWorkloadTestProject>true</IsWorkloadTestProject>
5-
<RunSettingsFilePath>$(MSBuildThisFileDirectory).runsettings</RunSettingsFilePath>
64
</PropertyGroup>
75

86
<Import Project="..\Directory.Build.props" />

‎tests/Aspire.EndToEnd.Tests/IntegrationServicesFixture.cs

+23-6
Original file line numberDiff line numberDiff line change
@@ -38,18 +38,35 @@ public IntegrationServicesFixture(IMessageSink diagnosticMessageSink)
3838
{
3939
_diagnosticMessageSink = diagnosticMessageSink;
4040
_testOutput = new TestOutputWrapper(messageSink: _diagnosticMessageSink);
41-
BuildEnvironment = new(TestsRunningOutsideOfRepo, (probePath, solutionRoot) =>
42-
$"Running outside-of-repo: Could not find {probePath} computed from solutionRoot={solutionRoot}. ");
43-
if (BuildEnvironment.HasSdkWithWorkload)
41+
BuildEnvironment = new(useSystemDotNet: !TestsRunningOutsideOfRepo);
42+
if (TestsRunningOutsideOfRepo)
4443
{
44+
if (!BuildEnvironment.HasWorkloadFromArtifacts)
45+
{
46+
throw new InvalidOperationException("Expected to have sdk+workload from artifacts when running tests outside of the repo");
47+
}
4548
BuildEnvironment.EnvVars["TestsRunningOutsideOfRepo"] = "true";
4649
}
47-
BuildEnvironment.EnvVars.Add("ASPIRE_ALLOW_UNSECURED_TRANSPORT", "true");
50+
else
51+
{
52+
// inside the repo
53+
if (BuildEnvironment.RepoRoot is null)
54+
{
55+
throw new InvalidOperationException("These tests should be run from inside the repo when using `TestsRunningOutsideOfRepo=false`");
56+
}
57+
58+
BuildEnvironment.TestAssetsPath = Path.Combine(BuildEnvironment.RepoRoot.FullName, "tests");
59+
if (!Directory.Exists(BuildEnvironment.TestAssetsPath))
60+
{
61+
throw new ArgumentException($"Cannot find TestAssetsPath={BuildEnvironment.TestAssetsPath}");
62+
}
63+
}
4864
}
4965

5066
public async Task InitializeAsync()
5167
{
52-
_project = new AspireProject("TestProject", BuildEnvironment.TestProjectPath, _testOutput, BuildEnvironment);
68+
string testProjectPath = Path.Combine(BuildEnvironment.TestAssetsPath, "testproject");
69+
_project = new AspireProject("TestProject", testProjectPath, _testOutput, BuildEnvironment);
5370
if (TestsRunningOutsideOfRepo)
5471
{
5572
_testOutput.WriteLine("");
@@ -68,7 +85,7 @@ public async Task InitializeAsync()
6885
{
6986
extraArgs += $"--skip-resources {skipArg}";
7087
}
71-
await Project.StartAsync([extraArgs]);
88+
await Project.StartAppHostAsync([extraArgs]);
7289

7390
foreach (var project in Projects.Values)
7491
{

‎tests/Directory.Build.props

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
<Project>
2-
32
<Import Project="..\Directory.Build.props" />
43
<PropertyGroup>
4+
<!-- Set RunSettingsFilePath property which is read by VSTest. -->
5+
<RunSettingsFilePath>$(RepositoryEngineeringDir)testing\.runsettings</RunSettingsFilePath>
56
<TestArchiveTestsDir Condition="'$(TestArchiveTestsDir)' == ''">$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix', 'tests'))</TestArchiveTestsDir>
67
<TestArchiveTestsDirForWorkloadTests Condition="'$(TestArchiveTestsDirForWorkloadTests)' == ''">$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix', 'workload-tests'))</TestArchiveTestsDirForWorkloadTests>
78
<TestArchiveTestsDirForEndToEndTests Condition="'$(TestArchiveTestsDirForEndToEndTests)' == ''">$([MSBuild]::NormalizeDirectory($(ArtifactsDir), 'helix', 'e2e-tests'))</TestArchiveTestsDirForEndToEndTests>

‎tests/Directory.Build.targets

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
</PropertyGroup>
1010

1111
<ItemGroup>
12-
<None Include="$(RepositoryEngineeringDir)testing\.runsettings" CopyToOutputDirectory="PreserveNewest" Condition="'$(DeployRunSettingsFile)' == 'true'" />
12+
<None Include="$(RunSettingsFilePath)" CopyToOutputDirectory="PreserveNewest" Condition="'$(DeployRunSettingsFile)' == 'true'" />
1313
<None Include="$(XunitRunnerJson)" CopyToOutputDirectory="PreserveNewest" />
1414
</ItemGroup>
1515

‎tests/README.md

+9-2
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,12 @@
22

33
The helix CI job builds `tests/helix/send-to-helix-ci.proj`, which in turns builds the `Test` target on `tests/helix/send-to-helix-inner.proj`. This inner project uses the Helix SDK to construct `@(HelixWorkItem)`s, and send them to helix to run.
44

5-
- `tests/helix/send-to-helix-basic-tests.targets` - this prepares all the tests that don't need special preparation
6-
- `tests/helix/send-to-helix-workload-tests.targets` - this is for tests that require a sdk+workload installed
5+
- `tests/helix/send-to-helix-basictests.targets` - this prepares all the tests that don't need special preparation
6+
- `tests/helix/send-to-helix-endtoend-tests.targets` - this is for tests that require a sdk+workload installed
7+
8+
## Install sdk+workload from artifacts
9+
10+
1. `.\build.cmd -pack`
11+
2. `dotnet build tests\workloads.proj`
12+
13+
.. which results in `artifacts\bin\dotnet-latest` which has a sdk (version from `global.json`) with the `aspire` workload installed using packs from `artifacts/packages`.

‎tests/Shared/WorkloadTesting/AspireProject.cs

+41-17
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,13 @@
55
using System.Text;
66
using Microsoft.Extensions.DependencyInjection;
77
using Xunit.Abstractions;
8+
using Xunit.Sdk;
89

910
namespace Aspire.Workload.Tests;
1011

1112
public class AspireProject : IAsyncDisposable
1213
{
14+
public const string DefaultTargetFramework = "net8.0";
1315
public static Lazy<HttpClient> Client => new(CreateHttpClient);
1416
public Process? AppHostProcess { get; private set; }
1517
public string Id { get; init; }
@@ -19,7 +21,8 @@ public class AspireProject : IAsyncDisposable
1921
public string ServiceDefaultsProjectPath => Path.Combine(RootDir, $"{Id}.ServiceDefaults");
2022
public string TestsProjectDirectory => Path.Combine(RootDir, $"{Id}.Tests");
2123
public Dictionary<string, ProjectInfo> InfoTable { get; private set; } = new(capacity: 0);
22-
public TaskCompletionSource AppExited { get; } = new();
24+
public TaskCompletionSource? AppExited { get; private set; }
25+
public bool IsRunning => AppHostProcess is not null && !AppHostProcess.HasExited;
2326

2427
private readonly ITestOutputHelper _testOutput;
2528
private readonly BuildEnvironment _buildEnv;
@@ -33,17 +36,23 @@ public AspireProject(string id, string baseDir, ITestOutputHelper testOutput, Bu
3336
LogPath = Path.Combine(_buildEnv.LogRootPath, Id);
3437
}
3538

36-
public async Task StartAsync(string[]? extraArgs = default, CancellationToken token = default, Action<ProcessStartInfo>? configureProcess = null)
39+
public async Task StartAppHostAsync(string[]? extraArgs = default, Action<ProcessStartInfo>? configureProcess = null, bool noBuild = true, CancellationToken token = default)
3740
{
41+
if (IsRunning)
42+
{
43+
throw new InvalidOperationException("Project is already running");
44+
}
45+
3846
object outputLock = new();
3947
var output = new StringBuilder();
4048
var projectsParsed = new TaskCompletionSource();
4149
var appRunning = new TaskCompletionSource();
4250
var stdoutComplete = new TaskCompletionSource();
4351
var stderrComplete = new TaskCompletionSource();
52+
AppExited = new();
4453
AppHostProcess = new Process();
4554

46-
var processArguments = $"run --no-build";
55+
var processArguments = $"run {(noBuild ? "--no-build" : "")}";
4756
processArguments += extraArgs is not null ? " " + string.Join(" ", extraArgs) : "";
4857
AppHostProcess.StartInfo = new ProcessStartInfo(_buildEnv.DotNet, processArguments)
4958
{
@@ -155,13 +164,13 @@ public async Task StartAsync(string[]? extraArgs = default, CancellationToken to
155164
throw new ArgumentException(exceptionMessage);
156165
}
157166

158-
lock(outputLock)
167+
lock (outputLock)
159168
{
160169
outputMessage = output.ToString();
161170
}
162171
if (resultTask != successfulTask)
163172
{
164-
throw new InvalidOperationException($"App run failed: {Environment.NewLine}{outputMessage}");
173+
throw new XunitException($"App run failed: {Environment.NewLine}{outputMessage}");
165174
}
166175

167176
foreach (var project in InfoTable.Values)
@@ -172,19 +181,39 @@ public async Task StartAsync(string[]? extraArgs = default, CancellationToken to
172181
_testOutput.WriteLine($"-- Ready to run tests --");
173182
}
174183

175-
public async Task BuildAsync(CancellationToken token = default)
184+
public async Task BuildAsync(string[]? extraBuildArgs = default, CancellationToken token = default)
176185
{
177-
using var restoreCmd = new DotNetCommand(_buildEnv, _testOutput, label: "restore")
186+
using var restoreCmd = new DotNetCommand(_testOutput, buildEnv: _buildEnv, label: "restore")
178187
.WithWorkingDirectory(Path.Combine(RootDir, $"{Id}.AppHost"));
179-
var res = await restoreCmd.ExecuteAsync($"restore -bl:{Path.Combine(LogPath!, $"{Id}-restore.binlog")} /p:TreatWarningsAsErrors=true");
188+
var res = await restoreCmd.ExecuteAsync($"restore \"-bl:{Path.Combine(LogPath!, $"{Id}-restore.binlog")}\" /p:TreatWarningsAsErrors=true");
180189
res.EnsureSuccessful();
181190

182-
using var buildCmd = new DotNetCommand(_buildEnv, _testOutput, label: "build")
191+
var buildArgs = $"build \"-bl:{Path.Combine(LogPath!, $"{Id}-build.binlog")}\" /p:TreatWarningsAsErrors=true";
192+
if (extraBuildArgs is not null)
193+
{
194+
buildArgs += " " + string.Join(" ", extraBuildArgs);
195+
}
196+
using var buildCmd = new DotNetCommand(_testOutput, buildEnv: _buildEnv, label: "build")
183197
.WithWorkingDirectory(Path.Combine(RootDir, $"{Id}.AppHost"));
184-
res = await buildCmd.ExecuteAsync($"build -bl:{Path.Combine(LogPath!, $"{Id}-build.binlog")} /p:TreatWarningsAsErrors=true");
198+
res = await buildCmd.ExecuteAsync(buildArgs);
185199
res.EnsureSuccessful();
186200
}
187201

202+
public async Task StopAppHostAsync(CancellationToken token = default)
203+
{
204+
if (AppHostProcess is null)
205+
{
206+
throw new InvalidOperationException("Tried to stop the app host process but it is not running.");
207+
}
208+
209+
if (AppExited?.Task.IsCompleted == false)
210+
{
211+
AppHostProcess.StandardInput.WriteLine("Stop");
212+
}
213+
await AppHostProcess.WaitForExitAsync(token);
214+
AppHostProcess = null;
215+
}
216+
188217
public async ValueTask DisposeAsync()
189218
{
190219
// TODO: check that everything shutdown
@@ -194,12 +223,7 @@ public async ValueTask DisposeAsync()
194223
}
195224

196225
await DumpDockerInfoAsync(new TestOutputWrapper(null));
197-
198-
if (!AppHostProcess.HasExited)
199-
{
200-
AppHostProcess.StandardInput.WriteLine("Stop");
201-
}
202-
await AppHostProcess.WaitForExitAsync();
226+
await StopAppHostAsync();
203227
}
204228

205229
public async Task DumpDockerInfoAsync(ITestOutputHelper? testOutputArg = null)
@@ -241,7 +265,7 @@ public async Task DumpComponentLogsAsync(string component, ITestOutputHelper? te
241265

242266
public void EnsureAppHostRunning()
243267
{
244-
if (AppHostProcess is null || AppHostProcess.HasExited || AppExited.Task.IsCompleted)
268+
if (AppHostProcess is null || AppHostProcess.HasExited || AppExited?.Task.IsCompleted == true)
245269
{
246270
throw new InvalidOperationException("The app host process is not running.");
247271
}

0 commit comments

Comments
 (0)