[tests] Use ResiliencePipeline in IntegrationTests (dotnet#3385)

radical · eerhardt · web-flow · commit cdb73cf03607 · 2024-04-11T18:21:50.000-04:00
* [tests] Use ResiliencePipeline in IntegrationTests .. with a policy to retry with 1 sec delays, and a total timeout of 90 secs on the service side. The test(client) side has an overall http request timeout for 120 secs which would allow the real errors from the service to get surfaced to the client. * address review feedback from @ eerhardt * Use the pipeline for MySql also * IntegrationServiceFixture.DumpComponentLogsAsync: error out if more than one resource is given * Rename TestUtils -> ResilienceUtils - feedback from @ eerhardt * [tests] Ignore `CA1305` for tests https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/quality-rules/ca1305 ``` dotnet_diagnostic.CA1305.severity = none ``` Addresses review feedback from @ eerhardt . * Remove Polly reference * Make code consistent. * add comment addressing review feedback from @ eerhardt --------- Co-authored-by: Eric Erhardt <eric.erhardt@microsoft.com>
diff --git a/src/Aspire.Hosting/Aspire.Hosting.csproj b/src/Aspire.Hosting/Aspire.Hosting.csproj
@@ -34,7 +34,7 @@
     <PackageReference Include="Grpc.AspNetCore" />
     <PackageReference Include="KubernetesClient" />
     <PackageReference Include="Microsoft.Extensions.Hosting" />
-    <PackageReference Include="Polly" />
+    <PackageReference Include="Polly.Core" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/tests/.editorconfig b/tests/.editorconfig
@@ -2,3 +2,6 @@
 
 # CA2007: Consider calling ConfigureAwait on the awaited task
 dotnet_diagnostic.CA2007.severity = silent
+
+# CA1305: Specify IFormatProvider
+dotnet_diagnostic.CA1305.severity = none
diff --git a/tests/Aspire.EndToEnd.Tests/IntegrationServicesFixture.cs b/tests/Aspire.EndToEnd.Tests/IntegrationServicesFixture.cs
@@ -78,6 +78,15 @@ public async Task InitializeAsync()
 
     public Task DumpComponentLogsAsync(TestResourceNames resource, ITestOutputHelper? testOutputArg = null)
     {
+        if (resource == TestResourceNames.None)
+        {
+            return;
+        }
+        if (resource == TestResourceNames.All || !Enum.IsDefined<TestResourceNames>(resource))
+        {
+            throw new ArgumentException($"Only one resource is supported at a time. resource: {resource}");
+        }
+
         string component = resource switch
         {
             TestResourceNames.cosmos => "cosmos",
diff --git a/tests/testproject/TestProject.IntegrationServiceA/Cosmos/CosmosExtensions.cs b/tests/testproject/TestProject.IntegrationServiceA/Cosmos/CosmosExtensions.cs
@@ -1,6 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text;
+using Aspire.TestProject;
 using Microsoft.Azure.Cosmos;
 using Polly;
 
@@ -13,15 +15,17 @@ public static void MapCosmosApi(this WebApplication app)
 
     private static async Task<IResult> VerifyCosmosAsync(CosmosClient cosmosClient)
     {
+        StringBuilder errorMessageBuilder = new();
         try
         {
-            var policy = Policy
-                .Handle<HttpRequestException>()
-                // retry 60 times with a 1 second delay between retries
-                .WaitAndRetryAsync(60, retryAttempt => TimeSpan.FromSeconds(1));
+            ResiliencePipeline pipeline = ResilienceUtils.GetDefaultResiliencePipelineBuilder<HttpRequestException>(args =>
+            {
+                errorMessageBuilder.AppendLine($"{Environment.NewLine}Service retry #{args.AttemptNumber} due to {args.Outcome.Exception}");
+                return ValueTask.CompletedTask;
+            }).Build();
 
-            var db = await policy.ExecuteAsync(
-                async () => (await cosmosClient.CreateDatabaseIfNotExistsAsync("db")).Database);
+            var db = await pipeline.ExecuteAsync(
+                async token => (await cosmosClient.CreateDatabaseIfNotExistsAsync("db", cancellationToken: token)).Database);
 
             var container = (await db.CreateContainerIfNotExistsAsync("todos", "/id")).Container;
 
@@ -38,7 +42,7 @@ private static async Task<IResult> VerifyCosmosAsync(CosmosClient cosmosClient)
         }
         catch (Exception e)
         {
-            return Results.Problem(e.ToString());
+            return Results.Problem($"Error: {e}{Environment.NewLine}** Previous retries: {errorMessageBuilder}");
         }
     }
 }
diff --git a/tests/testproject/TestProject.IntegrationServiceA/MySql/MySqlExtensions.cs b/tests/testproject/TestProject.IntegrationServiceA/MySql/MySqlExtensions.cs
@@ -1,6 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text;
+using Aspire.TestProject;
 using MySqlConnector;
 using Polly;
 
@@ -13,14 +15,16 @@ public static void MapMySqlApi(this WebApplication app)
 
     private static async Task<IResult> VerifyMySqlAsync(MySqlConnection connection)
     {
+        StringBuilder errorMessageBuilder = new();
         try
         {
-            var policy = Policy
-                .Handle<MySqlException>()
-                // retry 60 times with a 1 second delay between retries
-                .WaitAndRetryAsync(60, retryAttempt => TimeSpan.FromSeconds(1));
+            ResiliencePipeline pipeline = ResilienceUtils.GetDefaultResiliencePipelineBuilder<MySqlException>(args =>
+            {
+                errorMessageBuilder.AppendLine($"{Environment.NewLine}Service retry #{args.AttemptNumber} due to {args.Outcome.Exception}");
+                return ValueTask.CompletedTask;
+            }).Build();
 
-            await policy.ExecuteAsync(connection.OpenAsync);
+            await pipeline.ExecuteAsync(async token => await connection.OpenAsync(token));
 
             var command = connection.CreateCommand();
             command.CommandText = $"SELECT 1";
@@ -30,7 +34,7 @@ private static async Task<IResult> VerifyMySqlAsync(MySqlConnection connection)
         }
         catch (Exception e)
         {
-            return Results.Problem(e.ToString());
+            return Results.Problem($"Error: {e}{Environment.NewLine}** Previous retries: {errorMessageBuilder}");
         }
     }
 }
diff --git a/tests/testproject/TestProject.IntegrationServiceA/Oracle/OracleDatabaseExtensions.cs b/tests/testproject/TestProject.IntegrationServiceA/Oracle/OracleDatabaseExtensions.cs
@@ -1,6 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text;
+using Aspire.TestProject;
 using Microsoft.EntityFrameworkCore;
 using Oracle.ManagedDataAccess.Client;
 using Polly;
@@ -14,22 +16,24 @@ public static void MapOracleDatabaseApi(this WebApplication app)
 
     private static IResult VerifyOracleDatabase(MyDbContext context)
     {
+        StringBuilder errorMessageBuilder = new();
         try
         {
-            var policy = Policy
-                .Handle<OracleException>()
-                // retry 60 times with a 1 second delay between retries
-                .WaitAndRetry(60, retryAttempt => TimeSpan.FromSeconds(1));
+            ResiliencePipeline pipeline = ResilienceUtils.GetDefaultResiliencePipelineBuilder<OracleException>(args =>
+            {
+                errorMessageBuilder.AppendLine($"{Environment.NewLine}Service retry #{args.AttemptNumber} due to {args.Outcome.Exception}");
+                return ValueTask.CompletedTask;
+            }).Build();
 
-            return policy.Execute(() =>
+            return pipeline.Execute(() =>
             {
                 var results = context.Database.SqlQueryRaw<int>("SELECT 1 FROM DUAL");
                 return results.Any() ? Results.Ok("Success!") : Results.Problem("Failed");
             });
         }
         catch (Exception e)
         {
-            return Results.Problem(e.ToString());
+            return Results.Problem($"Error: {e}{Environment.NewLine}** Previous retries: {errorMessageBuilder}");
         }
     }
 }
diff --git a/tests/testproject/TestProject.IntegrationServiceA/ResilienceUtils.cs b/tests/testproject/TestProject.IntegrationServiceA/ResilienceUtils.cs
@@ -0,0 +1,27 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Polly;
+using Polly.Retry;
+
+namespace Aspire.TestProject;
+
+public static class ResilienceUtils
+{
+    public static ResiliencePipelineBuilder GetDefaultResiliencePipelineBuilder<TException>(Func<OnRetryArguments<object>, ValueTask> onRetry, int overallTimeoutSecs = 90) where TException : Exception
+    {
+        // Retry for upto 20 times with delay of 1 sec between
+        // attempts, and also stop before an overall timeout of
+        // @overallTimeoutSecs
+        var optionsOnRetry = new RetryStrategyOptions
+        {
+            MaxRetryAttempts = 20,
+            ShouldHandle = new PredicateBuilder().Handle<TException>(),
+            Delay = TimeSpan.FromSeconds(1),
+            OnRetry = onRetry
+        };
+        return new ResiliencePipelineBuilder()
+                    .AddTimeout(TimeSpan.FromSeconds(overallTimeoutSecs))
+                    .AddRetry(optionsOnRetry);
+    }
+}
diff --git a/tests/testproject/TestProject.IntegrationServiceA/SqlServer/SqlServerExtensions.cs b/tests/testproject/TestProject.IntegrationServiceA/SqlServer/SqlServerExtensions.cs
@@ -1,6 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text;
+using Aspire.TestProject;
 using Microsoft.Data.SqlClient;
 using Polly;
 
@@ -13,14 +15,16 @@ public static void MapSqlServerApi(this WebApplication app)
 
     private static async Task<IResult> VerifySqlServerAsync(SqlConnection connection)
     {
+        StringBuilder errorMessageBuilder = new();
         try
         {
-            var policy = Policy
-                .Handle<SqlException>()
-                // retry 60 times with a 1 second delay between retries
-                .WaitAndRetryAsync(60, retryAttempt => TimeSpan.FromSeconds(1));
+            ResiliencePipeline pipeline = ResilienceUtils.GetDefaultResiliencePipelineBuilder<SqlException>(args =>
+            {
+                errorMessageBuilder.AppendLine($"{Environment.NewLine}Service retry #{args.AttemptNumber} due to {args.Outcome.Exception}");
+                return ValueTask.CompletedTask;
+            }).Build();
 
-            await policy.ExecuteAsync(connection.OpenAsync);
+            await pipeline.ExecuteAsync(async token => await connection.OpenAsync(token));
 
             var command = connection.CreateCommand();
             command.CommandText = $"SELECT 1";
@@ -30,7 +34,7 @@ private static async Task<IResult> VerifySqlServerAsync(SqlConnection connection
         }
         catch (Exception e)
         {
-            return Results.Problem(e.ToString());
+            return Results.Problem($"Error: {e}{Environment.NewLine}** Previous retries: {errorMessageBuilder}");
         }
     }
 }
diff --git a/tests/testproject/TestProject.IntegrationServiceA/TestProject.IntegrationServiceA.csproj b/tests/testproject/TestProject.IntegrationServiceA/TestProject.IntegrationServiceA.csproj
@@ -31,7 +31,7 @@
   <ItemGroup>
     <ProjectReference Condition="'$(TestsRunningOutsideOfRepo)' != 'true'" Include="@(ComponentReferenceForTests -> '$(RepoRoot)src\Components\%(Identity)\%(Identity).csproj')" />
     <PackageReference Condition="'$(TestsRunningOutsideOfRepo)' == 'true'" Include="@(ComponentReferenceForTests)" />
-    <PackageReference Include="Polly" />
+    <PackageReference Include="Polly.Core" />
   </ItemGroup>
 
 </Project>

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,8 @@`
`1`	`1`	`// Licensed to the .NET Foundation under one or more agreements.`
`2`	`2`	`// The .NET Foundation licenses this file to you under the MIT license.`
`3`	`3`
	`4`	`+using System.Text;`
	`5`	`+using Aspire.TestProject;`
`4`	`6`	`using Microsoft.Azure.Cosmos;`
`5`	`7`	`using Polly;`
`6`	`8`
`@@ -13,15 +15,17 @@ public static void MapCosmosApi(this WebApplication app)`
`13`	`15`
`14`	`16`	`private static async Task<IResult> VerifyCosmosAsync(CosmosClient cosmosClient)`
`15`	`17`	`{`
	`18`	`+ StringBuilder errorMessageBuilder = new();`
`16`	`19`	`try`
`17`	`20`	`{`
`18`		`- var policy = Policy`
`19`		`- .Handle<HttpRequestException>()`
`20`		`- // retry 60 times with a 1 second delay between retries`
`21`		`- .WaitAndRetryAsync(60, retryAttempt => TimeSpan.FromSeconds(1));`
	`21`	`+ ResiliencePipeline pipeline = ResilienceUtils.GetDefaultResiliencePipelineBuilder<HttpRequestException>(args =>`
	`22`	`+ {`
	`23`	`+ errorMessageBuilder.AppendLine($"{Environment.NewLine}Service retry #{args.AttemptNumber} due to {args.Outcome.Exception}");`
	`24`	`+ return ValueTask.CompletedTask;`
	`25`	`+ }).Build();`
`22`	`26`
`23`		`- var db = await policy.ExecuteAsync(`
`24`		`- async () => (await cosmosClient.CreateDatabaseIfNotExistsAsync("db")).Database);`
	`27`	`+ var db = await pipeline.ExecuteAsync(`
	`28`	`+ async token => (await cosmosClient.CreateDatabaseIfNotExistsAsync("db", cancellationToken: token)).Database);`
`25`	`29`
`26`	`30`	`var container = (await db.CreateContainerIfNotExistsAsync("todos", "/id")).Container;`
`27`	`31`
`@@ -38,7 +42,7 @@ private static async Task<IResult> VerifyCosmosAsync(CosmosClient cosmosClient)`
`38`	`42`	`}`
`39`	`43`	`catch (Exception e)`
`40`	`44`	`{`
`41`		`- return Results.Problem(e.ToString());`
	`45`	`+ return Results.Problem($"Error: {e}{Environment.NewLine}** Previous retries: {errorMessageBuilder}");`
`42`	`46`	`}`
`43`	`47`	`}`
`44`	`48`	`}`