Skip to content

Commit 049c256

Browse files
committed
Explicitly opt-in to parameter randomisation
Resolves dotnet#9193 Also update outerloop definitions to match the innerloop
1 parent b2bbbea commit 049c256

File tree

5 files changed

+70
-19
lines changed

5 files changed

+70
-19
lines changed

.github/workflows/tests-outerloop.yml

+19-3
Original file line numberDiff line numberDiff line change
@@ -55,13 +55,28 @@ jobs:
5555
if: ${{ github.repository_owner == 'dotnet' }}
5656

5757
steps:
58+
- name: Trust HTTPS development certificate
59+
if: runner.os == 'Linux'
60+
run: dotnet dev-certs https --trust
61+
5862
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
5963

60-
- name: Trust HTTPS development certificate (Linux)
61-
if: matrix.tests.os == 'ubuntu-latest'
62-
run: ./dotnet.sh dev-certs https --trust
64+
- name: Verify Docker is running
65+
# nested docker containers not supported on windows
66+
if: runner.os == 'Linux'
67+
run: docker info
68+
69+
- name: Install Azure Functions Core Tools
70+
if: runner.os == 'Linux'
71+
run: |
72+
sudo apt-get update
73+
sudo apt-get install -y azure-functions-core-tools-4
74+
75+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
6376

6477
- name: Test ${{ matrix.tests.label }}
78+
env:
79+
CI: false
6580
run: |
6681
${{ matrix.tests.command }}
6782
@@ -73,6 +88,7 @@ jobs:
7388
path: |
7489
${{ github.workspace }}/artifacts/log/*/TestLogs/**/*.log
7590
${{ github.workspace }}/artifacts/TestResults/*/*.trx
91+
${{ github.workspace }}/artifacts/TestResults/*/*_crash.dmp
7692
# Longer retention time to allow scanning runs for quarantined test results
7793
retention-days: 30
7894

eng/QuarantinedTestRunsheetBuilder/QuarantinedTestRunsheetBuilder.targets

+21-8
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,9 @@
124124
we use the project name (which looks something like "Aspire.Cli.Tests").
125125
-->
126126
<_TestRunsheet>$([System.String]::Copy('$(MSBuildProjectName)').Replace('Aspire.', ''))</_TestRunsheet>
127+
<_TestRunsheetFileNameWindows>$(ArtifactsTmpDir)/$(_TestRunsheet).win.runsheet.json</_TestRunsheetFileNameWindows>
128+
<_TestRunsheetFileNameLinux>$(ArtifactsTmpDir)/$(_TestRunsheet).linux.runsheet.json</_TestRunsheetFileNameLinux>
129+
127130
<_TestBinLog>$([MSBuild]::NormalizePath($(ArtifactsLogDir), '$(_TestRunsheet).binlog'))</_TestBinLog>
128131

129132
<_RelativeTestProjectPath>$([System.String]::Copy('$(MSBuildProjectFullPath)').Replace('$(RepoRoot)', '%24(pwd)/'))</_RelativeTestProjectPath>
@@ -133,29 +136,39 @@
133136
<_TestRunnerLinux>./eng/build.sh</_TestRunnerLinux>
134137
<_TestCommand>-restore -build -test -projects &quot;$(_RelativeTestProjectPath)&quot; /bl:&quot;$(_RelativeTestBinLog)&quot; -c $(Configuration) -ci /p:RunQuarantinedTests=true /p:CI=false</_TestCommand>
135138

139+
<_PreCommand>$(TestRunnerPreCommand)</_PreCommand>
140+
136141
<!--
137142
Some quarantinted test may only be executable on Windows or Linux, however we can't possibly know that at this time.
138143
The MTP runner will return exit code 8 if no tests are found, and we need to ignore it instead of failing the test.
139144
-->
140145
<_TestCommand>$(_TestCommand) /p:IgnoreZeroTestResult=true</_TestCommand>
141146

142147
<!-- Replace \ with /, and then escape " with \", so we have a compliant JSON -->
148+
<_PreCommand>$([System.String]::Copy($(_PreCommand)).Replace("\", "/").Replace('&quot;', '\&quot;'))</_PreCommand>
143149
<_TestCommand>$([System.String]::Copy($(_TestCommand)).Replace("\", "/").Replace('&quot;', '\&quot;'))</_TestCommand>
144150

145151
<_TestRunsheetWindows>{ "label": "w: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "windows-latest", "command": "./eng/build.ps1 $(_TestCommand)" }</_TestRunsheetWindows>
146-
<_TestRunsheetLinux>{ "label": "l: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "ubuntu-latest", "command": "./eng/build.sh $(_TestCommand)" }</_TestRunsheetLinux>
152+
<_TestRunsheetLinux>{ "label": "l: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "ubuntu-latest", "command": "$(_PreCommand)./eng/build.sh $(_TestCommand)" }</_TestRunsheetLinux>
147153
</PropertyGroup>
148154

155+
<ItemGroup>
156+
<_OutputFiles Include="$(_TestRunsheetFileNameWindows)" />
157+
<_OutputFiles Include="$(_TestRunsheetFileNameLinux)" />
158+
</ItemGroup>
159+
160+
<MakeDir Directories="@(_OutputFiles->'%(RootDir)%(Directory)')"/>
161+
<Delete Files="@(_OutputFiles)" />
162+
149163
<WriteLinesToFile
150164
Condition=" '$(RunOnGithubActionsWindows)' == 'true' and '$(_HasQuarantinedTests)' == 'true'"
151-
File="$(ArtifactsTmpDir)/$(_TestRunsheet).win.runsheet.json"
165+
File="$(_TestRunsheetFileNameWindows)"
152166
Lines="$(_TestRunsheetWindows)"
153167
Overwrite="true"
154168
WriteOnlyWhenDifferent="true" />
155-
156169
<WriteLinesToFile
157170
Condition=" '$(RunOnGithubActionsLinux)' == 'true' and '$(_HasQuarantinedTests)' == 'true' "
158-
File="$(ArtifactsTmpDir)/$(_TestRunsheet).linux.runsheet.json"
171+
File="$(_TestRunsheetFileNameLinux)"
159172
Lines="$(_TestRunsheetLinux)"
160173
Overwrite="true"
161174
WriteOnlyWhenDifferent="true" />
@@ -164,10 +177,10 @@
164177
On Linux there's a bug in MSBuild, which "normalises" all slashes (see https://github.com/dotnet/msbuild/issues/3468).
165178
This is a workaround to replace `/"` with the required `\"`.
166179
-->
167-
<Exec Command="pwsh -Command &quot;(Get-Content -Path '$(ArtifactsTmpDir)/$(_TestRunsheet).win.runsheet.json') -replace '/\&quot;', '\\\&quot;' | Set-Content -Path '$(ArtifactsTmpDir)/$(_TestRunsheet).win.runsheet.json'&quot; "
168-
Condition=" '$(RunOnGithubActionsWindows)' == 'true' and '$(_HasQuarantinedTests)' == 'true' and '$(BuildOs)' != 'windows' " />
169-
<Exec Command="pwsh -Command &quot;(Get-Content -Path '$(ArtifactsTmpDir)/$(_TestRunsheet).linux.runsheet.json') -replace '/\&quot;', '\\\&quot;' | Set-Content -Path '$(ArtifactsTmpDir)/$(_TestRunsheet).linux.runsheet.json'&quot; "
170-
Condition=" '$(RunOnGithubActionsLinux)' == 'true' and '$(_HasQuarantinedTests)' == 'true' and '$(BuildOs)' != 'windows' " />
180+
<Exec Command="pwsh -Command &quot;(Get-Content -Path '$(_TestRunsheetFileNameWindows)') -replace '/\&quot;', '\\\&quot;' | Set-Content -Path '$(_TestRunsheetFileNameWindows)'&quot; "
181+
Condition=" Exists('$(_TestRunsheetFileNameWindows)') and '$(BuildOs)' != 'windows' " />
182+
<Exec Command="pwsh -Command &quot;(Get-Content -Path '$(_TestRunsheetFileNameLinux)') -replace '/\&quot;', '\\\&quot;' | Set-Content -Path '$(_TestRunsheetFileNameLinux)'&quot; "
183+
Condition=" Exists('$(_TestRunsheetFileNameLinux)') and '$(BuildOs)' != 'windows' " />
171184

172185
<!--
173186
The final piece of the puzzle is in eng/AfterSolutionBuild.targets, where we combine the runsheets from all the test projects into a single runsheet.

tests/Aspire.Playground.Tests/AppHostTests.cs

+21-3
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ public async Task TestEndpointsReturnOk(TestEndpoints testEndpoints)
3535
{
3636
var appHostType = testEndpoints.AppHostType!;
3737
var resourceEndpoints = testEndpoints.ResourceEndpoints!;
38-
var appHost = await DistributedApplicationTestFactory.CreateAsync(appHostType, _testOutput);
38+
var appHost = await DistributedApplicationTestFactory.CreateAsync(appHostType, _testOutput, testEndpoints.EnableParameterRandomisation);
3939
var projects = appHost.Resources.OfType<ProjectResource>();
4040
await using var app = await appHost.BuildAsync();
4141

@@ -144,6 +144,7 @@ public static IList<TestEndpoints> GetAllTestEndpoints()
144144
// whenReady: TestEventHubsAppHost),
145145
new TestEndpoints(typeof(Projects.Redis_AppHost),
146146
resourceEndpoints: new() { { "apiservice", ["/alive", "/health", "/garnet/ping", "/garnet/get", "/garnet/set", "/redis/ping", "/redis/get", "/redis/set", "/valkey/ping", "/valkey/get", "/valkey/set"] } },
147+
enableParameterRandomisation: true,
147148
waitForTexts: [
148149
new ("redis", "Ready to accept connections tcp"),
149150
new ("valkey", "Ready to accept connections tcp"),
@@ -152,11 +153,13 @@ public static IList<TestEndpoints> GetAllTestEndpoints()
152153
]),
153154
new TestEndpoints(typeof(Projects.AzureStorageEndToEnd_AppHost),
154155
resourceEndpoints: new() { { "api", ["/alive", "/health", "/"] } },
156+
enableParameterRandomisation: true,
155157
waitForTexts: [
156158
new ("storage", "Azurite Table service is successfully listening")
157159
]),
158160
new TestEndpoints(typeof(Projects.MilvusPlayground_AppHost),
159161
resourceEndpoints: new() { { "apiservice", ["/alive", "/health", "/create", "/search"] } },
162+
enableParameterRandomisation: true,
160163
waitForTexts: [
161164
new ("milvus", "Milvus Proxy successfully initialized and ready to serve"),
162165
]),
@@ -166,19 +169,22 @@ public static IList<TestEndpoints> GetAllTestEndpoints()
166169
// // "/ef" - disabled due to https://github.com/dotnet/aspire/issues/5415
167170
// waitForTexts: [
168171
// new ("cosmos", "Started$"),
169-
// new ("api", "Application started")
172+
// new("api", "Application started")
170173
// ]),
171174
new TestEndpoints(typeof(Projects.Keycloak_AppHost),
172-
resourceEndpoints: new() { { "apiservice", ["/alive", "/health"] } }),
175+
resourceEndpoints: new() { { "apiservice", ["/alive", "/health"] } },
176+
enableParameterRandomisation: true),
173177
new TestEndpoints(typeof(Projects.Mongo_AppHost),
174178
resourceEndpoints: new() { { "api", ["/alive", "/health", "/"] } },
179+
enableParameterRandomisation: true,
175180
waitForTexts: [
176181
new ("mongo", "Waiting for connections"),
177182
new ("mongo-mongoexpress", "Mongo Express server listening"),
178183
new("api", "Application started.")
179184
]),
180185
new TestEndpoints(typeof(Projects.MySqlDb_AppHost),
181186
resourceEndpoints: new() { { "apiservice", ["/alive", "/health", "/catalog"] } },
187+
enableParameterRandomisation: true,
182188
waitForTexts: [
183189
new ("mysql", "ready for connections.* port: 33060"),
184190
new ("apiservice", "Application started")
@@ -188,12 +194,14 @@ public static IList<TestEndpoints> GetAllTestEndpoints()
188194
{ "api", ["/alive", "/health"] },
189195
{ "backend", ["/alive", "/health"] }
190196
},
197+
enableParameterRandomisation: true,
191198
waitForTexts: [
192199
new ("nats", "Server is ready"),
193200
new("api", "Application started")
194201
]),
195202
new TestEndpoints(typeof(Projects.ParameterEndToEnd_AppHost),
196203
resourceEndpoints: new() { { "api", ["/", "/alive", "/health"] } },
204+
enableParameterRandomisation: false,
197205
waitForTexts: [
198206
new ("sql", "SQL Server is now ready for client connections."),
199207
]),
@@ -202,6 +210,7 @@ public static IList<TestEndpoints> GetAllTestEndpoints()
202210
// Invoking "/" first as that *creates* the databases
203211
{ "api", ["/", "/alive", "/health"] }
204212
},
213+
enableParameterRandomisation: true,
205214
waitForTexts: [
206215
new ("pg1", "PostgreSQL init process complete; ready for start up"),
207216
new ("pg2", "PostgreSQL init process complete; ready for start up"),
@@ -213,26 +222,30 @@ public static IList<TestEndpoints> GetAllTestEndpoints()
213222
]),
214223
new TestEndpoints(typeof(Projects.ProxylessEndToEnd_AppHost),
215224
resourceEndpoints: new() { { "api", ["/alive", "/health", "/redis"] } },
225+
enableParameterRandomisation: true,
216226
waitForTexts: [
217227
new ("redis", "Ready to accept connections"),
218228
new("api", "Application started"),
219229
new("api2", "Application started")
220230
]),
221231
new TestEndpoints(typeof(Projects.Qdrant_AppHost),
222232
resourceEndpoints: new() { { "apiservice", ["/alive", "/health"] } },
233+
enableParameterRandomisation: true,
223234
waitForTexts: [
224235
new ("qdrant", "Qdrant HTTP listening"),
225236
new ("apiservice", "Application started")
226237
]),
227238
new TestEndpoints(typeof(Projects.Seq_AppHost),
228239
resourceEndpoints: new() { { "api", ["/alive", "/health"] } },
240+
enableParameterRandomisation: true,
229241
waitForTexts: [
230242
new ("seq", "Seq listening"),
231243
new ("api", "Application started")
232244
]),
233245
// Invoke "/" first to create the databases
234246
new TestEndpoints(typeof(Projects.SqlServerEndToEnd_AppHost),
235247
resourceEndpoints: new() { { "api", ["/", "/alive", "/health"] } },
248+
enableParameterRandomisation: true,
236249
waitForTexts: [
237250
new ("sql1", "SQL Server is now ready for client connections"),
238251
new ("sql2", "SQL Server is now ready for client connections"),
@@ -242,6 +255,7 @@ public static IList<TestEndpoints> GetAllTestEndpoints()
242255
{ "catalogdbapp", ["/alive", "/health"] },
243256
{ "frontend", ["/alive", "/health"] }
244257
},
258+
enableParameterRandomisation: true,
245259
waitForTexts: [
246260
new ("messaging", "started TCP listener"),
247261
new ("basketcache", "Ready to accept connections"),
@@ -279,15 +293,19 @@ public class TestEndpoints
279293
{
280294
public TestEndpoints(Type appHostType,
281295
Dictionary<string, List<string>> resourceEndpoints,
296+
bool enableParameterRandomisation,
282297
List<ReadyStateText>? waitForTexts = null)
283298
{
284299
AppHostType = appHostType;
285300
ResourceEndpoints = resourceEndpoints;
286301
WaitForTexts = waitForTexts;
302+
EnableParameterRandomisation = enableParameterRandomisation;
287303
}
288304

289305
public Type AppHostType { get; set; }
290306

307+
public bool EnableParameterRandomisation { get; set; }
308+
291309
public List<ResourceWait>? WaitForResources { get; set; }
292310

293311
public List<ReadyStateText>? WaitForTexts { get; set; }

tests/Aspire.Playground.Tests/Infrastructure/DistributedApplicationTestFactory.cs

+6-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ internal static class DistributedApplicationTestFactory
1515
/// <summary>
1616
/// Creates an <see cref="IDistributedApplicationTestingBuilder"/> for the specified app host assembly.
1717
/// </summary>
18-
public static async Task<IDistributedApplicationTestingBuilder> CreateAsync(Type appHostProgramType, ITestOutputHelper? testOutput)
18+
public static async Task<IDistributedApplicationTestingBuilder> CreateAsync(Type appHostProgramType, ITestOutputHelper? testOutput, bool enableParameterRandomisation)
1919
{
2020
var builder = await DistributedApplicationTestingBuilder.CreateAsync(appHostProgramType);
2121

@@ -24,7 +24,11 @@ public static async Task<IDistributedApplicationTestingBuilder> CreateAsync(Type
2424
// always override.
2525
builder.Services.AddLifecycleHook<ContainerRegistryHook>();
2626

27-
builder.WithRandomParameterValues();
27+
if (enableParameterRandomisation)
28+
{
29+
builder.WithRandomParameterValues();
30+
}
31+
2832
builder.WithRandomVolumeNames();
2933

3034
builder.Services.AddLogging(logging =>

tests/Aspire.Playground.Tests/ProjectSpecificTests.cs

+3-3
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ public class ProjectSpecificTests(ITestOutputHelper _testOutput)
1919
[QuarantinedTest("https://github.com/dotnet/aspire/issues/9171")]
2020
public async Task WithDockerfileTest()
2121
{
22-
var appHost = await DistributedApplicationTestFactory.CreateAsync(typeof(Projects.WithDockerfile_AppHost), _testOutput);
22+
var appHost = await DistributedApplicationTestFactory.CreateAsync(typeof(Projects.WithDockerfile_AppHost), _testOutput, enableParameterRandomisation: false);
2323
await using var app = await appHost.BuildAsync();
2424

2525
await app.StartAsync();
@@ -36,7 +36,7 @@ await app.WaitForTextAsync($"I'm Batman. - Batman")
3636
[QuarantinedTest("https://github.com/dotnet/aspire/issues/6867")]
3737
public async Task KafkaTest()
3838
{
39-
var appHost = await DistributedApplicationTestFactory.CreateAsync(typeof(Projects.KafkaBasic_AppHost), _testOutput);
39+
var appHost = await DistributedApplicationTestFactory.CreateAsync(typeof(Projects.KafkaBasic_AppHost), _testOutput, enableParameterRandomisation: false);
4040
await using var app = await appHost.BuildAsync();
4141

4242
await app.StartAsync();
@@ -62,7 +62,7 @@ await WaitForAllTextAsync(app,
6262
[RequiresTools(["func"])]
6363
public async Task AzureFunctionsTest()
6464
{
65-
var appHost = await DistributedApplicationTestFactory.CreateAsync(typeof(Projects.AzureFunctionsEndToEnd_AppHost), _testOutput);
65+
var appHost = await DistributedApplicationTestFactory.CreateAsync(typeof(Projects.AzureFunctionsEndToEnd_AppHost), _testOutput, enableParameterRandomisation: true);
6666
await using var app = await appHost.BuildAsync();
6767

6868
await app.StartAsync();

0 commit comments

Comments
 (0)