Skip to content

Commit 846b55c

Browse files
RubenCerna2079RubenCerna2079
committed
Add logic to send logs through File Sink (#2825)
## Why make this change? - Closes issue #2578 - In order to complete the File Sink feature, the logic that sends the logs created by DAB to the file is required. ## What is this change? - This change implements the logic that sends all the logs from DAB to the file through the path the user requests. This is done with the help of Serilog which is a logging library that simplifies the creation of file sinks. - The `Startup.cs` program creates the Serilog logger pipeline and adds it as part of the services so that it is used later by the `Program.cs` to set the different loggers with the Serilog pipeline and allow the logs to be sent to the file sink. - We also deleted the `RollingIntervalMode.cs` since we discovered that Serilog has its own rolling interval enum class, which makes the one implemented in DAB obsolete. ## How was this tested? - [ ] Integration Tests - [X] Unit Tests Created tests that check if the services needed for the File Sink exist when the File Sink property is enabled. Also, created test to check if the file sink with the appropriate name is created when the property is enabled. ## Sample Request(s) <img width="917" height="136" alt="image" src="https://github.com/user-attachments/assets/a470844a-6ea3-4d05-9dfa-650f878c1c59" /> <img width="1873" height="948" alt="image" src="https://github.com/user-attachments/assets/1895eaae-0223-4d48-924e-c5275a98e3dc" />
1 parent b41cfe5 commit 846b55c

File tree

10 files changed

+275
-68
lines changed

10 files changed

+275
-68
lines changed

‎src/Config/Azure.DataApiBuilder.Config.csproj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,15 @@
1818
<PackageReference Include="Microsoft.AspNetCore.Authorization" />
1919
<PackageReference Include="Microsoft.IdentityModel.Protocols" />
2020
<PackageReference Include="Microsoft.IdentityModel.Protocols.OpenIdConnect" />
21+
<PackageReference Include="Serilog.Sinks.File" />
2122
<PackageReference Include="System.IO.Abstractions" />
2223
<PackageReference Include="System.Drawing.Common" />
2324
<PackageReference Include="Microsoft.Data.SqlClient" />
2425
<PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" />
2526
<PackageReference Include="System.IdentityModel.Tokens.Jwt" />
2627
<PackageReference Include="Humanizer" />
2728
<PackageReference Include="Npgsql" />
28-
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" />
29+
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" />
2930
</ItemGroup>
3031

3132
<ItemGroup>

‎src/Config/Converters/FileSinkConverter.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System.Text.Json;
55
using System.Text.Json.Serialization;
66
using Azure.DataApiBuilder.Config.ObjectModel;
7+
using Serilog;
78

89
namespace Azure.DataApiBuilder.Config.Converters;
910
class FileSinkConverter : JsonConverter<FileSinkOptions>
@@ -31,7 +32,7 @@ public FileSinkConverter(bool replaceEnvVar)
3132
{
3233
bool? enabled = null;
3334
string? path = null;
34-
RollingIntervalMode? rollingInterval = null;
35+
RollingInterval? rollingInterval = null;
3536
int? retainedFileCountLimit = null;
3637
int? fileSizeLimitBytes = null;
3738

@@ -66,7 +67,7 @@ public FileSinkConverter(bool replaceEnvVar)
6667
case "rolling-interval":
6768
if (reader.TokenType is not JsonTokenType.Null)
6869
{
69-
rollingInterval = EnumExtensions.Deserialize<RollingIntervalMode>(reader.DeserializeString(_replaceEnvVar)!);
70+
rollingInterval = EnumExtensions.Deserialize<RollingInterval>(reader.DeserializeString(_replaceEnvVar)!);
7071
}
7172

7273
break;

‎src/Config/ObjectModel/FileSinkOptions.cs‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
using System.Diagnostics.CodeAnalysis;
55
using System.Text.Json.Serialization;
6+
using Serilog;
67

78
namespace Azure.DataApiBuilder.Config.ObjectModel;
89

@@ -19,12 +20,12 @@ public record FileSinkOptions
1920
/// <summary>
2021
/// Default path for File Sink.
2122
/// </summary>
22-
public const string DEFAULT_PATH = "/logs/dab-log.txt";
23+
public const string DEFAULT_PATH = @"logs\dab-log.txt";
2324

2425
/// <summary>
2526
/// Default rolling interval for File Sink.
2627
/// </summary>
27-
public const string DEFAULT_ROLLING_INTERVAL = nameof(RollingIntervalMode.Day);
28+
public const string DEFAULT_ROLLING_INTERVAL = nameof(Serilog.RollingInterval.Day);
2829

2930
/// <summary>
3031
/// Default retained file count limit for File Sink.
@@ -44,25 +45,25 @@ public record FileSinkOptions
4445
/// <summary>
4546
/// Path to the file where logs will be uploaded.
4647
/// </summary>
47-
public string? Path { get; init; }
48+
public string Path { get; init; }
4849

4950
/// <summary>
5051
/// Time it takes for files with logs to be discarded.
5152
/// </summary>
52-
public string? RollingInterval { get; init; }
53+
public string RollingInterval { get; init; }
5354

5455
/// <summary>
5556
/// Amount of files that can exist simultaneously in which logs are saved.
5657
/// </summary>
57-
public int? RetainedFileCountLimit { get; init; }
58+
public int RetainedFileCountLimit { get; init; }
5859

5960
/// <summary>
6061
/// File size limit in bytes before a new file needs to be created.
6162
/// </summary>
62-
public int? FileSizeLimitBytes { get; init; }
63+
public int FileSizeLimitBytes { get; init; }
6364

6465
[JsonConstructor]
65-
public FileSinkOptions(bool? enabled = null, string? path = null, RollingIntervalMode? rollingInterval = null, int? retainedFileCountLimit = null, int? fileSizeLimitBytes = null)
66+
public FileSinkOptions(bool? enabled = null, string? path = null, RollingInterval? rollingInterval = null, int? retainedFileCountLimit = null, int? fileSizeLimitBytes = null)
6667
{
6768
if (enabled is not null)
6869
{
@@ -86,7 +87,7 @@ public FileSinkOptions(bool? enabled = null, string? path = null, RollingInterva
8687

8788
if (rollingInterval is not null)
8889
{
89-
RollingInterval = rollingInterval.ToString();
90+
RollingInterval = ((RollingInterval)rollingInterval).ToString();
9091
UserProvidedRollingInterval = true;
9192
}
9293
else
@@ -96,7 +97,7 @@ public FileSinkOptions(bool? enabled = null, string? path = null, RollingInterva
9697

9798
if (retainedFileCountLimit is not null)
9899
{
99-
RetainedFileCountLimit = retainedFileCountLimit;
100+
RetainedFileCountLimit = (int)retainedFileCountLimit;
100101
UserProvidedRetainedFileCountLimit = true;
101102
}
102103
else
@@ -106,7 +107,7 @@ public FileSinkOptions(bool? enabled = null, string? path = null, RollingInterva
106107

107108
if (fileSizeLimitBytes is not null)
108109
{
109-
FileSizeLimitBytes = fileSizeLimitBytes;
110+
FileSizeLimitBytes = (int)fileSizeLimitBytes;
110111
UserProvidedFileSizeLimitBytes = true;
111112
}
112113
else

‎src/Config/ObjectModel/RollingIntervalMode.cs‎

Lines changed: 0 additions & 44 deletions
This file was deleted.

‎src/Directory.Packages.props‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@
6060
<PackageVersion Include="Newtonsoft.Json.Schema" Version="3.0.14" />
6161
<PackageVersion Include="Npgsql" Version="8.0.3" />
6262
<PackageVersion Include="Polly" Version="7.2.3" />
63+
<PackageVersion Include="Serilog.Extensions.Logging" Version="9.0.2" />
64+
<PackageVersion Include="Serilog.Sinks.File" Version="7.0.0" />
6365
<PackageVersion Include="Swashbuckle.AspNetCore.SwaggerUI" Version="6.5.0" />
6466
<PackageVersion Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
6567
<PackageVersion Include="System.Drawing.Common" Version="8.0.3" />

‎src/Service.Tests/Configuration/ConfigurationTests.cs‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
using Microsoft.VisualStudio.TestTools.UnitTesting;
4848
using Moq;
4949
using Moq.Protected;
50+
using Serilog;
5051
using VerifyMSTest;
5152
using static Azure.DataApiBuilder.Config.FileSystemRuntimeConfigLoader;
5253
using static Azure.DataApiBuilder.Service.Tests.Configuration.ConfigurationEndpoints;
@@ -4170,21 +4171,21 @@ public void AzureLogAnalyticsSerialization(
41704171
/// </summary>
41714172
[DataTestMethod]
41724173
[TestCategory(TestCategory.MSSQL)]
4173-
[DataRow(true, "/file/path/exists.txt", RollingIntervalMode.Minute, 27, 256, true, "/file/path/exists.txt", RollingIntervalMode.Minute, 27, 256)]
4174-
[DataRow(true, "/test/path.csv", RollingIntervalMode.Hour, 10, 3000, true, "/test/path.csv", RollingIntervalMode.Hour, 10, 3000)]
4175-
[DataRow(false, "C://absolute/file/path.log", RollingIntervalMode.Month, 2147483647, 2048, false, "C://absolute/file/path.log", RollingIntervalMode.Month, 2147483647, 2048)]
4176-
[DataRow(false, "D://absolute/test/path.txt", RollingIntervalMode.Year, 10, 2147483647, false, "D://absolute/test/path.txt", RollingIntervalMode.Year, 10, 2147483647)]
4177-
[DataRow(false, "", RollingIntervalMode.Infinite, 5, 512, false, "", RollingIntervalMode.Infinite, 5, 512)]
4178-
[DataRow(null, null, null, null, null, false, "/logs/dab-log.txt", RollingIntervalMode.Day, 1, 1048576)]
4174+
[DataRow(true, "/file/path/exists.txt", RollingInterval.Minute, 27, 256, true, "/file/path/exists.txt", RollingInterval.Minute, 27, 256)]
4175+
[DataRow(true, "/test/path.csv", RollingInterval.Hour, 10, 3000, true, "/test/path.csv", RollingInterval.Hour, 10, 3000)]
4176+
[DataRow(false, "C://absolute/file/path.log", RollingInterval.Month, 2147483647, 2048, false, "C://absolute/file/path.log", RollingInterval.Month, 2147483647, 2048)]
4177+
[DataRow(false, "D://absolute/test/path.txt", RollingInterval.Year, 10, 2147483647, false, "D://absolute/test/path.txt", RollingInterval.Year, 10, 2147483647)]
4178+
[DataRow(false, "", RollingInterval.Infinite, 5, 512, false, "", RollingInterval.Infinite, 5, 512)]
4179+
[DataRow(null, null, null, null, null, false, "/logs/dab-log.txt", RollingInterval.Day, 1, 1048576)]
41794180
public void FileSinkSerialization(
41804181
bool? enabled,
41814182
string? path,
4182-
RollingIntervalMode? rollingInterval,
4183+
RollingInterval? rollingInterval,
41834184
int? retainedFileCountLimit,
41844185
int? fileSizeLimitBytes,
41854186
bool expectedEnabled,
41864187
string expectedPath,
4187-
RollingIntervalMode expectedRollingInterval,
4188+
RollingInterval expectedRollingInterval,
41884189
int expectedRetainedFileCountLimit,
41894190
int expectedFileSizeLimitBytes)
41904191
{

‎src/Service.Tests/Configuration/Telemetry/FileSinkTests.cs‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.IO;
6+
using System.Net.Http;
7+
using System.Threading.Tasks;
8+
using Azure.DataApiBuilder.Config.ObjectModel;
9+
using Microsoft.AspNetCore.TestHost;
10+
using Microsoft.Extensions.DependencyInjection;
11+
using Microsoft.VisualStudio.TestTools.UnitTesting;
12+
using Serilog;
13+
using Serilog.Core;
14+
using static Azure.DataApiBuilder.Service.Tests.Configuration.ConfigurationTests;
15+
16+
namespace Azure.DataApiBuilder.Service.Tests.Configuration.Telemetry;
17+
18+
/// <summary>
19+
/// Contains tests for File Sink functionality.
20+
/// </summary>
21+
[TestClass, TestCategory(TestCategory.MSSQL)]
22+
public class FileSinkTests
23+
{
24+
public TestContext TestContext { get; set; }
25+
26+
private const string CONFIG_WITH_TELEMETRY = "dab-file-sink-test-config.json";
27+
private const string CONFIG_WITHOUT_TELEMETRY = "dab-no-file-sink-test-config.json";
28+
private static RuntimeConfig _configuration;
29+
30+
/// <summary>
31+
/// This is a helper function that creates runtime config file with specified telemetry options.
32+
/// </summary>
33+
/// <param name="configFileName">Name of the config file to be created.</param>
34+
/// <param name="isFileSinkEnabled">Whether File Sink is enabled or not.</param>
35+
/// <param name="fileSinkPath">Path where logs will be sent to.</param>
36+
/// <param name="rollingInterval">Time it takes for logs to roll over to next file.</param>
37+
private static void SetUpTelemetryInConfig(string configFileName, bool isFileSinkEnabled, string fileSinkPath, RollingInterval? rollingInterval = null)
38+
{
39+
DataSource dataSource = new(DatabaseType.MSSQL,
40+
GetConnectionStringFromEnvironmentConfig(environment: TestCategory.MSSQL), Options: null);
41+
42+
_configuration = InitMinimalRuntimeConfig(dataSource, graphqlOptions: new(), restOptions: new());
43+
44+
TelemetryOptions _testTelemetryOptions = new(File: new FileSinkOptions(isFileSinkEnabled, fileSinkPath, rollingInterval));
45+
_configuration = _configuration with { Runtime = _configuration.Runtime with { Telemetry = _testTelemetryOptions } };
46+
47+
File.WriteAllText(configFileName, _configuration.ToJson());
48+
}
49+
50+
/// <summary>
51+
/// Cleans up the test environment by deleting the runtime config with telemetry options.
52+
/// </summary>
53+
[TestCleanup]
54+
public void CleanUpTelemetryConfig()
55+
{
56+
if (File.Exists(CONFIG_WITH_TELEMETRY))
57+
{
58+
File.Delete(CONFIG_WITH_TELEMETRY);
59+
}
60+
61+
if (File.Exists(CONFIG_WITHOUT_TELEMETRY))
62+
{
63+
File.Delete(CONFIG_WITHOUT_TELEMETRY);
64+
}
65+
}
66+
67+
/// <summary>
68+
/// Tests if the services are correctly enabled for File Sink.
69+
/// </summary>
70+
[TestMethod]
71+
public void TestFileSinkServicesEnabled()
72+
{
73+
// Arrange
74+
SetUpTelemetryInConfig(CONFIG_WITH_TELEMETRY, true, "/dab-log-test/file-sink-file.txt");
75+
76+
string[] args = new[]
77+
{
78+
$"--ConfigFileName={CONFIG_WITH_TELEMETRY}"
79+
};
80+
using TestServer server = new(Program.CreateWebHostBuilder(args));
81+
82+
// Additional assertions to check if File Sink is enabled correctly in services
83+
IServiceProvider serviceProvider = server.Services;
84+
LoggerConfiguration serilogLoggerConfiguration = serviceProvider.GetService<LoggerConfiguration>();
85+
Logger serilogLogger = serviceProvider.GetService<Logger>();
86+
87+
// If serilogLoggerConfiguration and serilogLogger are not null, File Sink is enabled
88+
Assert.IsNotNull(serilogLoggerConfiguration, "LoggerConfiguration for Serilog should be registered.");
89+
Assert.IsNotNull(serilogLogger, "Logger for Serilog should be registered.");
90+
}
91+
92+
/// <summary>
93+
/// Tests if the logs are flushed to the proper path when File Sink is enabled.
94+
/// </summary>
95+
/// <summary>
96+
/// Tests if the logs are flushed to the proper path when File Sink is enabled.
97+
/// </summary>
98+
[DataTestMethod]
99+
[DataRow("file-sink-test-file.txt")]
100+
[DataRow("file-sink-test-file.log")]
101+
[DataRow("file-sink-test-file.csv")]
102+
public async Task TestFileSinkSucceed(string fileName)
103+
{
104+
// Arrange
105+
SetUpTelemetryInConfig(CONFIG_WITH_TELEMETRY, true, fileName, RollingInterval.Infinite);
106+
107+
string[] args = new[]
108+
{
109+
$"--ConfigFileName={CONFIG_WITH_TELEMETRY}"
110+
};
111+
using TestServer server = new(Program.CreateWebHostBuilder(args));
112+
113+
// Act
114+
using (HttpClient client = server.CreateClient())
115+
{
116+
HttpRequestMessage restRequest = new(HttpMethod.Get, "/api/Book");
117+
await client.SendAsync(restRequest);
118+
}
119+
120+
server.Dispose();
121+
122+
// Assert
123+
Assert.IsTrue(File.Exists(fileName));
124+
125+
bool containsInfo = false;
126+
string[] allLines = File.ReadAllLines(fileName);
127+
foreach (string line in allLines)
128+
{
129+
containsInfo = line.Contains("INF");
130+
if (containsInfo)
131+
{
132+
break;
133+
}
134+
}
135+
136+
Assert.IsTrue(containsInfo);
137+
}
138+
139+
/// <summary>
140+
/// Tests if the services are correctly disabled for File Sink.
141+
/// </summary>
142+
[TestMethod]
143+
public void TestFileSinkServicesDisabled()
144+
{
145+
// Arrange
146+
SetUpTelemetryInConfig(CONFIG_WITHOUT_TELEMETRY, false, null);
147+
148+
string[] args = new[]
149+
{
150+
$"--ConfigFileName={CONFIG_WITHOUT_TELEMETRY}"
151+
};
152+
using TestServer server = new(Program.CreateWebHostBuilder(args));
153+
154+
// Additional assertions to check if File Sink is enabled correctly in services
155+
IServiceProvider serviceProvider = server.Services;
156+
LoggerConfiguration serilogLoggerConfiguration = serviceProvider.GetService<LoggerConfiguration>();
157+
Logger serilogLogger = serviceProvider.GetService<Logger>();
158+
159+
// If serilogLoggerConfiguration and serilogLogger are null, File Sink is disabled
160+
Assert.IsNull(serilogLoggerConfiguration, "LoggerConfiguration for Serilog should not be registered.");
161+
Assert.IsNull(serilogLogger, "Logger for Serilog should not be registered.");
162+
}
163+
}

‎src/Service/Azure.DataApiBuilder.Service.csproj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@
7575
<PackageReference Include="Npgsql" />
7676
<PackageReference Include="Polly" />
7777
<PackageReference Include="DotNetEnv" />
78+
<PackageReference Include="Serilog.Extensions.Logging" />
79+
<PackageReference Include="Serilog.Sinks.File" />
7880
<PackageReference Include="Swashbuckle.AspNetCore.SwaggerUI" />
7981
<PackageReference Include="System.CommandLine" />
8082
<PackageReference Include="System.IO.Abstractions" />

0 commit comments

Comments
 (0)