diff --git a/services/grid-bot/.component.yaml b/services/grid-bot/.component.yaml index 980ca3d6..c4ca82f5 100755 --- a/services/grid-bot/.component.yaml +++ b/services/grid-bot/.component.yaml @@ -41,14 +41,40 @@ deployment: static: 8101 grpc: static: 5000 + http: + static: 8888 services: - - name: grid-bot-${{ env.NOMAD_SHORT_ENVIRONMENT }} + - name: ${{ env.NOMAD_ENVIRONMENT }}-grid-bot port: metrics tags: - ${{ env.NOMAD_ENVIRONMENT }} checks: - type: http path: /metrics + + - name: ${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web + port: http + tags: + - ${{ env.NOMAD_ENVIRONMENT }} + - "traefik.enable=true" + - "traefik.http.routers.${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web-http.rule=(HostRegexp(`{host:[a-zA-Z]+}.sitetest4.robloxlabs.com`) || Host(`versioncompatibility.api.sitetest4.robloxlabs.com`))" + - "traefik.http.routers.${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web-http.entrypoints=http" + checks: + - type: http + path: /health + + - name: ${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web-https + port: http + tags: + - ${{ env.NOMAD_ENVIRONMENT }} + - "traefik.enable=true" + - "traefik.http.routers.${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web-https.rule=(HostRegexp(`{host:[a-zA-Z]+}.sitetest4.robloxlabs.com`) || Host(`versioncompatibility.api.sitetest4.robloxlabs.com`))" + - "traefik.http.routers.${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web-https.entrypoints=https" + - "traefik.http.routers.${{ env.NOMAD_ENVIRONMENT }}-grid-bot-web-https.tls=true" + checks: + - type: http + path: /health + volumes: - '/var/run/docker.sock:/var/run/docker.sock' - '/tmp/.X11-unix:/tmp/.X11-unix' diff --git a/services/grid-bot/grid-bot-bare.sln b/services/grid-bot/grid-bot-bare.sln index e32d0c10..a8ce97ec 100755 --- a/services/grid-bot/grid-bot-bare.sln +++ b/services/grid-bot/grid-bot-bare.sln @@ -17,6 +17,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared.Settings", "lib\sett EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared.Utility", "lib\utility\Shared.Utility.csproj", "{AB0EF72D-505F-4F0E-94AD-BA236218CEB7}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Grid.Bot.Grpc", "lib\grpc\Grid.Bot.Grpc.csproj", "{BDC40A33-FB7D-4B64-8FD1-662092858DA8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Grid.Bot.Web", "lib\web\Grid.Bot.Web.csproj", "{EDD50BF5-0E1D-48FB-9BBB-61A9B3CE7642}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -43,6 +47,14 @@ Global {AB0EF72D-505F-4F0E-94AD-BA236218CEB7}.Debug|Any CPU.Build.0 = debug|Any CPU {AB0EF72D-505F-4F0E-94AD-BA236218CEB7}.Release|Any CPU.ActiveCfg = release|Any CPU {AB0EF72D-505F-4F0E-94AD-BA236218CEB7}.Release|Any CPU.Build.0 = release|Any CPU + {BDC40A33-FB7D-4B64-8FD1-662092858DA8}.Debug|Any CPU.ActiveCfg = debug|Any CPU + {BDC40A33-FB7D-4B64-8FD1-662092858DA8}.Debug|Any CPU.Build.0 = debug|Any CPU + {BDC40A33-FB7D-4B64-8FD1-662092858DA8}.Release|Any CPU.ActiveCfg = release|Any CPU + {BDC40A33-FB7D-4B64-8FD1-662092858DA8}.Release|Any CPU.Build.0 = release|Any CPU + {EDD50BF5-0E1D-48FB-9BBB-61A9B3CE7642}.Debug|Any CPU.ActiveCfg = debug|Any CPU + {EDD50BF5-0E1D-48FB-9BBB-61A9B3CE7642}.Debug|Any CPU.Build.0 = debug|Any CPU + {EDD50BF5-0E1D-48FB-9BBB-61A9B3CE7642}.Release|Any CPU.ActiveCfg = release|Any CPU + {EDD50BF5-0E1D-48FB-9BBB-61A9B3CE7642}.Release|Any CPU.Build.0 = release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -53,5 +65,7 @@ Global {67005F02-FE69-4DCF-8D29-07989E8B4C18} = {4FA0281C-7E8F-45E6-820E-97554DE934AE} {425ECFC3-749B-4725-AF7B-A2CC56A06E6B} = {4FA0281C-7E8F-45E6-820E-97554DE934AE} {AB0EF72D-505F-4F0E-94AD-BA236218CEB7} = {4FA0281C-7E8F-45E6-820E-97554DE934AE} + {BDC40A33-FB7D-4B64-8FD1-662092858DA8} = {4FA0281C-7E8F-45E6-820E-97554DE934AE} + {EDD50BF5-0E1D-48FB-9BBB-61A9B3CE7642} = {4FA0281C-7E8F-45E6-820E-97554DE934AE} EndGlobalSection EndGlobal diff --git a/services/grid-bot/grid-bot.sln b/services/grid-bot/grid-bot.sln index 66852c96..3a086aa7 100755 --- a/services/grid-bot/grid-bot.sln +++ b/services/grid-bot/grid-bot.sln @@ -57,12 +57,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Configuration", "..\..\lib\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Configuration.Core", "..\..\lib\src\configuration\core\Configuration.Core.csproj", "{0F99E3B7-B8A2-4166-94DD-E14E10B82782}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ClientSettings.Client", "..\..\lib\src\clients\client-settings-client\ClientSettings.Client.csproj", "{FDE94473-0D43-4FBC-9CDD-E29A3938BC1E}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Thumbnails.Client", "..\..\lib\src\clients\thumbnails-client\Thumbnails.Client.csproj", "{067AC4F9-F1E9-435B-B837-9A9B05DDD621}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Users.Client", "..\..\lib\src\clients\users-client\Users.Client.csproj", "{8702A9B1-8180-41FE-A639-AFE807B08512}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Grid.Bot.Grpc", "lib\grpc\Grid.Bot.Grpc.csproj", "{C4BAE13A-0315-4B1F-A40F-10CAAD7F184C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Grid.Bot.Web", "lib\web\Grid.Bot.Web.csproj", "{3AC6957F-78D6-4872-A0F1-C927FD545033}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -169,10 +171,6 @@ Global {0F99E3B7-B8A2-4166-94DD-E14E10B82782}.Debug|Any CPU.Build.0 = debug|Any CPU {0F99E3B7-B8A2-4166-94DD-E14E10B82782}.Release|Any CPU.ActiveCfg = release|Any CPU {0F99E3B7-B8A2-4166-94DD-E14E10B82782}.Release|Any CPU.Build.0 = release|Any CPU - {FDE94473-0D43-4FBC-9CDD-E29A3938BC1E}.Debug|Any CPU.ActiveCfg = debug|Any CPU - {FDE94473-0D43-4FBC-9CDD-E29A3938BC1E}.Debug|Any CPU.Build.0 = debug|Any CPU - {FDE94473-0D43-4FBC-9CDD-E29A3938BC1E}.Release|Any CPU.ActiveCfg = release|Any CPU - {FDE94473-0D43-4FBC-9CDD-E29A3938BC1E}.Release|Any CPU.Build.0 = release|Any CPU {067AC4F9-F1E9-435B-B837-9A9B05DDD621}.Debug|Any CPU.ActiveCfg = debug|Any CPU {067AC4F9-F1E9-435B-B837-9A9B05DDD621}.Debug|Any CPU.Build.0 = debug|Any CPU {067AC4F9-F1E9-435B-B837-9A9B05DDD621}.Release|Any CPU.ActiveCfg = release|Any CPU @@ -181,6 +179,14 @@ Global {8702A9B1-8180-41FE-A639-AFE807B08512}.Debug|Any CPU.Build.0 = debug|Any CPU {8702A9B1-8180-41FE-A639-AFE807B08512}.Release|Any CPU.ActiveCfg = release|Any CPU {8702A9B1-8180-41FE-A639-AFE807B08512}.Release|Any CPU.Build.0 = release|Any CPU + {C4BAE13A-0315-4B1F-A40F-10CAAD7F184C}.Debug|Any CPU.ActiveCfg = debug|Any CPU + {C4BAE13A-0315-4B1F-A40F-10CAAD7F184C}.Debug|Any CPU.Build.0 = debug|Any CPU + {C4BAE13A-0315-4B1F-A40F-10CAAD7F184C}.Release|Any CPU.ActiveCfg = release|Any CPU + {C4BAE13A-0315-4B1F-A40F-10CAAD7F184C}.Release|Any CPU.Build.0 = release|Any CPU + {3AC6957F-78D6-4872-A0F1-C927FD545033}.Debug|Any CPU.ActiveCfg = debug|Any CPU + {3AC6957F-78D6-4872-A0F1-C927FD545033}.Debug|Any CPU.Build.0 = debug|Any CPU + {3AC6957F-78D6-4872-A0F1-C927FD545033}.Release|Any CPU.ActiveCfg = release|Any CPU + {3AC6957F-78D6-4872-A0F1-C927FD545033}.Release|Any CPU.Build.0 = release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -211,8 +217,9 @@ Global {67C7D321-75F2-4ADC-9C24-BC02E345E0E0} = {6A79B4E5-D433-4FCF-9E6E-AED94F97099D} {689B303F-6238-4F1A-8F24-2C8B4FE6C8BD} = {6A79B4E5-D433-4FCF-9E6E-AED94F97099D} {0F99E3B7-B8A2-4166-94DD-E14E10B82782} = {6A79B4E5-D433-4FCF-9E6E-AED94F97099D} - {FDE94473-0D43-4FBC-9CDD-E29A3938BC1E} = {6A79B4E5-D433-4FCF-9E6E-AED94F97099D} {067AC4F9-F1E9-435B-B837-9A9B05DDD621} = {6A79B4E5-D433-4FCF-9E6E-AED94F97099D} {8702A9B1-8180-41FE-A639-AFE807B08512} = {6A79B4E5-D433-4FCF-9E6E-AED94F97099D} + {C4BAE13A-0315-4B1F-A40F-10CAAD7F184C} = {4FA0281C-7E8F-45E6-820E-97554DE934AE} + {3AC6957F-78D6-4872-A0F1-C927FD545033} = {4FA0281C-7E8F-45E6-820E-97554DE934AE} EndGlobalSection EndGlobal diff --git a/services/grid-bot/lib/commands/PrivateModules/Commands/ClientSettings.cs b/services/grid-bot/lib/commands/PrivateModules/Commands/ClientSettings.cs index 119925dd..8bb2b0c6 100755 --- a/services/grid-bot/lib/commands/PrivateModules/Commands/ClientSettings.cs +++ b/services/grid-bot/lib/commands/PrivateModules/Commands/ClientSettings.cs @@ -2,7 +2,6 @@ namespace Grid.Bot.Commands.Private; using System; using System.IO; -using System.Net; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -14,8 +13,6 @@ namespace Grid.Bot.Commands.Private; using Newtonsoft.Json; -using ClientSettings.Client; - using Utility; using Extensions; @@ -25,76 +22,44 @@ namespace Grid.Bot.Commands.Private; /// /// Construct a new instance of . /// -/// The . -/// The . +/// The . +/// The . /// -/// - cannot be null. -/// - cannot be null. +/// - cannot be null. +/// - cannot be null. /// [LockDownCommand(BotRole.Administrator)] [RequireBotRole(BotRole.Administrator)] [Group("clientsettings"), Summary("Commands used for managing client settings."), Alias("cs", "client_settings")] -public class ClientSettingsModule(IClientSettingsClient clientSettingsClient, ClientSettingsClientSettings clientSettingsClientSettings) : ModuleBase +public class ClientSettingsModule(IClientSettingsFactory clientSettingsFactory, ClientSettingsSettings clientSettingsSettings) : ModuleBase { - private readonly IClientSettingsClient _clientSettingsClient = clientSettingsClient ?? throw new ArgumentNullException(nameof(clientSettingsClient)); - private readonly ClientSettingsClientSettings _clientSettingsClientSettings = clientSettingsClientSettings ?? throw new ArgumentNullException(nameof(clientSettingsClientSettings)); - - /// - /// Represents the type of client setting. - /// - public enum ClientSettingType - { - /// - /// Represents a string client setting. - /// - String, - - /// - /// Represents an integer client setting. - /// - Int, - - /// - /// Represents a boolean client setting. - /// - Bool, - } + private readonly IClientSettingsFactory _clientSettingsFactory = clientSettingsFactory ?? throw new ArgumentNullException(nameof(clientSettingsFactory)); + private readonly ClientSettingsSettings _clientSettingsSettings = clientSettingsSettings ?? throw new ArgumentNullException(nameof(clientSettingsSettings)); /// /// Gets the client settings for the specified application. /// /// The name of the application. - /// Should the API key be used? This will allow the application to be returned from the client settings API even if $allowed on the backend is false. [Command("get_all"), Summary("Gets all client settings for the specified application.")] [Alias("getall", "all")] - public async Task GetAllAsync(string applicationName, bool useApiKey = false) + public async Task GetAllAsync(string applicationName) { using var _ = Context.Channel.EnterTypingState(); - try + var clientSettings = _clientSettingsFactory.GetSettingsForApplication(applicationName); + if (clientSettings is null) { - var clientSettings = await _clientSettingsClient.GetApplicationSettingsAsync( - applicationName, - useApiKey ? _clientSettingsClientSettings.ClientSettingsApiKey : null - ).ConfigureAwait(false); - - using var stream = new MemoryStream(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(clientSettings, Formatting.Indented))); + await this.ReplyWithReferenceAsync( + text: "The specified application does not exist." + ); - await this.ReplyWithFileAsync(stream, $"{applicationName}.json", "Here are the client settings for the specified application."); - } - catch (ApiException ex) - { - if (ex.StatusCode == (int)HttpStatusCode.BadRequest) - await this.ReplyWithReferenceAsync( - "The specified application does not exist." - ); - else if (ex.StatusCode == (int)HttpStatusCode.Unauthorized) - await this.ReplyWithReferenceAsync( - "The specified application cannot be returned from the client settings API without an API key. Please set the use_api_key parameter to true." - ); - else - throw; + return; } + + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(clientSettings, Formatting.Indented))); + + await this.ReplyWithFileAsync(stream, $"{applicationName}.json", "Here are the client settings for the specified application."); + } /// @@ -102,10 +67,9 @@ await this.ReplyWithReferenceAsync( /// /// The name of the application. /// The dependencies for the application. - /// The reference for the application. /// Is the application allowed to be written to from the API? [Command("import"), Summary("Imports the client settings for the specified application.")] - public async Task ImportAsync(string applicationName, string dependencies = null, string reference = null, bool isAllowedFromApi = false) + public async Task ImportAsync(string applicationName, string dependencies = null, bool isAllowedFromApi = false) { var applicationSettings = Context.Message.Attachments.FirstOrDefault(); if (applicationSettings is null) @@ -120,26 +84,37 @@ await this.ReplyWithReferenceAsync( using var _ = Context.Channel.EnterTypingState(); var contents = await applicationSettings.GetAttachmentContentsAscii(); - - var request = new ImportClientApplicationSettingsRequest + var contentsParsed = JsonConvert.DeserializeObject>(contents); + if (contentsParsed is null) { - ApplicationName = applicationName, - ApplicationSettings = JsonConvert.DeserializeObject>(contents), - }; + await this.ReplyWithReferenceAsync( + text: "Failed to parse the client settings file." + ); + + return; + } + + + _clientSettingsFactory.WriteSettingsForApplication(applicationName, contentsParsed); - var parsedDependencies = dependencies?.Split(',', StringSplitOptions.RemoveEmptyEntries); - if (parsedDependencies is not null && parsedDependencies.Length > 0) - request.Dependencies = parsedDependencies; + var parsedDependencies = string.Join(",", dependencies?.Split(',', StringSplitOptions.RemoveEmptyEntries)); - if (!string.IsNullOrWhiteSpace(reference)) - request.Reference = reference; + if (!string.IsNullOrWhiteSpace(parsedDependencies)) + { + _clientSettingsSettings.ClientSettingsApplicationDependencies[applicationName] = parsedDependencies; + _clientSettingsSettings.ApplyCurrent(); + } - request.IsAllowedFromClientSettingsService = isAllowedFromApi; + if (isAllowedFromApi) + { + var currentPermissibleReadApplications = _clientSettingsSettings.PermissibleReadApplications.ToList(); - await _clientSettingsClient.ImportApplicationSettingAsync( - _clientSettingsClientSettings.ClientSettingsApiKey, - request - ).ConfigureAwait(false); + if (!currentPermissibleReadApplications.Contains(applicationName)) + { + currentPermissibleReadApplications.Add(applicationName); + _clientSettingsSettings.PermissibleReadApplications = [.. currentPermissibleReadApplications]; + } + } await this.ReplyWithReferenceAsync( text: "Successfully imported the client settings for the specified application." @@ -158,26 +133,39 @@ public async Task GetAsync(string applicationName, string settingName) try { - var applicationSetting = await _clientSettingsClient.GetClientApplicationSettingAsync(applicationName, settingName).ConfigureAwait(false); + if (ClientSettingsNameHelper.IsFilteredSetting(settingName)) + { + var (filterName, filterType) = ClientSettingsNameHelper.ExtractFilteredSettingName(settingName); + var filteredSetting = _clientSettingsFactory.GetFilteredSettingForApplication(applicationName, filterName, filterType); + + await this.ReplyWithReferenceAsync( + embed: new EmbedBuilder() + .WithTitle($"{filterName} ({filterType} Filter)") + .AddField("Value", $"```\n{filteredSetting.Value}\n```") + .AddField("Filtered Ids", string.Join(", ", filteredSetting.FilteredIds)) + .WithColor(0x00, 0xff, 0x00) + .Build() + ); + + return; + } + + var applicationSetting = _clientSettingsFactory.GetSettingForApplication(applicationName, settingName); await this.ReplyWithReferenceAsync( embed: new EmbedBuilder() .WithTitle(settingName) - .WithDescription($"```\n{applicationSetting.Value}\n```") + .WithDescription($"```\n{applicationSetting}\n```") .WithColor(0x00, 0xff, 0x00) .Build() ); } - catch (ApiException ex) + catch (Exception ex) { - if (ex.StatusCode == (int)HttpStatusCode.BadRequest) + if (ex is InvalidOperationException) await this.ReplyWithReferenceAsync( text: "The specified application does not exist." ); - else if (ex.StatusCode == (int)HttpStatusCode.NotFound) - await this.ReplyWithReferenceAsync( - text: "The specified application setting does not exist." - ); else throw; } @@ -191,7 +179,7 @@ await this.ReplyWithReferenceAsync( /// The type of the setting. /// The value of the setting. [Command("set"), Summary("Sets a client setting for the specified application.")] - public async Task SetAsync(string applicationName, string settingName, ClientSettingType settingType = ClientSettingType.String, string settingValue = "") + public async Task SetAsync(string applicationName, string settingName, SettingType settingType = SettingType.String, string settingValue = "") { if (settingValue is null) { @@ -208,9 +196,9 @@ await this.ReplyWithReferenceAsync( { value = settingType switch { - ClientSettingType.String => settingValue, - ClientSettingType.Int => int.Parse(settingValue), - ClientSettingType.Bool => bool.Parse(settingValue), + SettingType.String => settingValue, + SettingType.Int => long.Parse(settingValue), + SettingType.Bool => bool.Parse(settingValue), _ => throw new InvalidOperationException($"Unknown setting type: {settingType}"), }; } @@ -225,37 +213,9 @@ await this.ReplyWithReferenceAsync( using var _ = Context.Channel.EnterTypingState(); - try - { - var diff = await _clientSettingsClient.SetClientApplicationSettingAsync( - _clientSettingsClientSettings.ClientSettingsApiKey, - new SetClientApplicationSettingRequest - { - ApplicationName = applicationName, - SettingName = settingName, - Value = value, - } - ).ConfigureAwait(false); + _clientSettingsFactory.SetSettingForApplication(applicationName, settingName, value, settingType); - await this.ReplyWithReferenceAsync( - embed: new EmbedBuilder() - .WithTitle(settingName) - .AddField("Previous Value", diff.OldValue?.Value ?? "null") - .AddField("New Value", value) - .AddField("Did Update", diff.DidUpdate) - .WithColor(0x00, 0xff, 0x00) - .Build() - ); - } - catch (ApiException ex) - { - if (ex.StatusCode == (int)HttpStatusCode.BadRequest) - await this.ReplyWithReferenceAsync( - text: "The specified application does not exist." - ); - else - throw; - } + await this.ReplyWithReferenceAsync(text: $"Successfully set the setting `{settingName}` to `{value}` for the application `{applicationName}`."); } /// @@ -266,7 +226,7 @@ public async Task RefreshAllAsync() { using var _ = Context.Channel.EnterTypingState(); - await _clientSettingsClient.RefreshAllClientApplicationSettingsAsync(_clientSettingsClientSettings.ClientSettingsApiKey).ConfigureAwait(false); + _clientSettingsFactory.Refresh(); await this.ReplyWithReferenceAsync( text: "Successfully refreshed all applications." diff --git a/services/grid-bot/lib/commands/Shared.Commands.csproj b/services/grid-bot/lib/commands/Shared.Commands.csproj index e9ef2231..f798d907 100755 --- a/services/grid-bot/lib/commands/Shared.Commands.csproj +++ b/services/grid-bot/lib/commands/Shared.Commands.csproj @@ -14,7 +14,6 @@ - @@ -24,7 +23,6 @@ - @@ -33,9 +31,9 @@ - - - + + + diff --git a/services/grid-bot/lib/events/Shared.Events.csproj b/services/grid-bot/lib/events/Shared.Events.csproj index 91ff8051..c1ca45ee 100755 --- a/services/grid-bot/lib/events/Shared.Events.csproj +++ b/services/grid-bot/lib/events/Shared.Events.csproj @@ -10,9 +10,9 @@ - - - + + + diff --git a/services/grid-bot/lib/grpc/Extensions/IServiceProviderExtensions.cs b/services/grid-bot/lib/grpc/Extensions/IServiceProviderExtensions.cs new file mode 100644 index 00000000..6d6d8466 --- /dev/null +++ b/services/grid-bot/lib/grpc/Extensions/IServiceProviderExtensions.cs @@ -0,0 +1,101 @@ +namespace Grid.Bot.Grpc; + +using System; +using System.Threading.Tasks; +using System.Collections.Generic; +using System.Security.Authentication; + +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Server.Kestrel.Core; + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.DependencyInjection; + +using Discord.WebSocket; + +using Logging; +using Utility; + +/// +/// gRPC extensions for the +/// +public static class IServiceProviderExtensions +{ + /// + /// Implement the grid-bot gRPC server into the current service provider. + /// + /// The + /// The application arguments. + public static void UseGrpcServer(this IServiceProvider services, IEnumerable args) + { + var grpcSettings = services.GetRequiredService(); + var logger = new Logger( + name: grpcSettings.GrpcServerLoggerName, + logLevelGetter: () => grpcSettings.GrpcServerLoggerLevel, + logToConsole: true, + logToFileSystem: false + ); + + if (!grpcSettings.GridBotGrpcServerEnabled) + { + logger.Warning("The grid-bot gRPC server is disabled in settings, not starting gRPC server!"); + + return; + } + + var client = services.GetRequiredService(); + + logger.Information("Starting gRPC server on {0}", grpcSettings.GridBotGrpcServerEndpoint); + + var builder = WebApplication.CreateBuilder([.. args]); + + builder.Logging.ClearProviders(); + builder.Logging.AddProvider(new MicrosoftLoggerProvider(logger)); + + builder.Services.AddSingleton(client); + + builder.Services.AddGrpc(); + + if (grpcSettings.GrpcServerUseTls) + { + builder.Services.Configure(options => + { + options.ConfigureEndpointDefaults(listenOptions => + { + listenOptions.Protocols = HttpProtocols.Http2; + + try + { + listenOptions.UseHttps(grpcSettings.GrpcServerCertificatePath, grpcSettings.GrpcServerCertificatePassword, httpsOptions => + { + httpsOptions.SslProtocols = SslProtocols.Tls13 | SslProtocols.Tls12; + }); + } + catch (Exception ex) + { + logger.Warning("Failed to configure gRPC with HTTPS because: {0}. Will resort to insecure host instead!", ex.Message); + } + }); + }); + + // set urls + } + else + { + builder.Services.Configure(options => + { + options.ConfigureEndpointDefaults(listenOptions => + { + listenOptions.Protocols = HttpProtocols.Http2; + }); + }); + } + + var app = builder.Build(); + + app.MapGrpcService(); + + Task.Factory.StartNew(() => app.Run(grpcSettings.GridBotGrpcServerEndpoint), TaskCreationOptions.LongRunning); + } +} diff --git a/services/grid-bot/lib/grpc/Grid.Bot.Grpc.csproj b/services/grid-bot/lib/grpc/Grid.Bot.Grpc.csproj new file mode 100755 index 00000000..c867f6fc --- /dev/null +++ b/services/grid-bot/lib/grpc/Grid.Bot.Grpc.csproj @@ -0,0 +1,27 @@ + + + gRPC server exposed by the mfdlabs grid bot. + true + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + \ No newline at end of file diff --git a/services/grid-bot/lib/utility/Implementation/GridBotGrpcServer.cs b/services/grid-bot/lib/grpc/Implementation/GridBotGrpcServer.cs similarity index 98% rename from services/grid-bot/lib/utility/Implementation/GridBotGrpcServer.cs rename to services/grid-bot/lib/grpc/Implementation/GridBotGrpcServer.cs index 30bb4ed2..8788d7a6 100755 --- a/services/grid-bot/lib/utility/Implementation/GridBotGrpcServer.cs +++ b/services/grid-bot/lib/grpc/Implementation/GridBotGrpcServer.cs @@ -1,4 +1,6 @@ -namespace Grid.Bot.Utility; +using Grpc.Core; + +namespace Grid.Bot.Grpc; using System; using System.Linq; @@ -7,8 +9,6 @@ namespace Grid.Bot.Utility; using Discord; using Discord.WebSocket; -using Grpc.Core; - using V1; /// diff --git a/services/grid-bot/lib/settings/Providers/AvatarSettings.cs b/services/grid-bot/lib/settings/Providers/AvatarSettings.cs index 546bdec3..bc4c54a1 100755 --- a/services/grid-bot/lib/settings/Providers/AvatarSettings.cs +++ b/services/grid-bot/lib/settings/Providers/AvatarSettings.cs @@ -10,6 +10,27 @@ public class AvatarSettings : BaseSettingsProvider /// public override string Path => SettingsProvidersDefaults.AvatarPath; + + /// + /// Gets the URL for the Avatar API. + /// + public string AvatarApiUrl => GetOrDefault(nameof(AvatarApiUrl), "https://avatar.roblox.com"); + + /// + /// Gets the interval on which to traverse the avatar fetch cache to search for stale entries. + /// + public TimeSpan AvatarFetchCacheTraversalInterval => GetOrDefault(nameof(AvatarFetchCacheTraversalInterval), TimeSpan.FromMinutes(5)); + + /// + /// Gets the TTL for each avatar-fetch cache entry. + /// + public TimeSpan AvatarFetchCacheEntryTtl => GetOrDefault(nameof(AvatarFetchCacheEntryTtl), TimeSpan.FromMinutes(5)); + + /// + /// Determines whether or not body colors are downgraded to pre v2 (v1.1). + /// + public bool AvatarFetchShouldDowngradeBodyColorsFormat => GetOrDefault(nameof(AvatarFetchShouldDowngradeBodyColorsFormat), true); + /// /// Gets the url to be used for asset fetch. /// diff --git a/services/grid-bot/lib/settings/Providers/ClientSettingsClientSettings.cs b/services/grid-bot/lib/settings/Providers/ClientSettingsClientSettings.cs deleted file mode 100755 index 220f4bf4..00000000 --- a/services/grid-bot/lib/settings/Providers/ClientSettingsClientSettings.cs +++ /dev/null @@ -1,34 +0,0 @@ -namespace Grid.Bot; - -/// -/// Settings provider for client-settings -/// -public class ClientSettingsClientSettings : BaseSettingsProvider -{ - /// - public override string Path => SettingsProvidersDefaults.ClientSettingsClientPath; - - /// - /// Gets the base url for the client settings API. - /// - public string ClientSettingsApiBaseUrl => GetOrDefault( - nameof(ClientSettingsApiBaseUrl), - "https://clientsettingscdn.sitetest4.robloxlabs.com/" - ); - - /// - /// Should certificate validation be enabled when interacting with the client settings API? - /// - public bool ClientSettingsCertificateValidationEnabled => GetOrDefault( - nameof(ClientSettingsCertificateValidationEnabled), - false - ); - - /// - /// Gets the API key used to interact with the client settings API. - /// - public string ClientSettingsApiKey => GetOrDefault( - nameof(ClientSettingsApiKey), - string.Empty - ); -} diff --git a/services/grid-bot/lib/settings/Providers/ClientSettingsSettings.cs b/services/grid-bot/lib/settings/Providers/ClientSettingsSettings.cs new file mode 100755 index 00000000..a68e69d3 --- /dev/null +++ b/services/grid-bot/lib/settings/Providers/ClientSettingsSettings.cs @@ -0,0 +1,78 @@ +namespace Grid.Bot; + +using System; +using System.Collections.Generic; + +/// +/// Settings provider for client-settings +/// +public class ClientSettingsSettings : BaseSettingsProvider +{ + /// + public override string Path => SettingsProvidersDefaults.ClientSettingsPath; + + + /// + /// Determines if data access should be via Vault or local JSON files. + /// + public bool ClientSettingsViaVault => GetOrDefault(nameof(ClientSettingsViaVault), false); + + /// + /// Gets the Vault mount for the client settings. + /// + public string ClientSettingsVaultMount => GetOrDefault(nameof(ClientSettingsVaultMount), "client-settings"); + + /// + /// Gets the absolute path to the client settings vault path. + /// + public string ClientSettingsVaultPath => GetOrDefault(nameof(ClientSettingsVaultPath), "/"); + + /// + /// Gets the Vault address for the client settings. + /// + public string ClientSettingsVaultAddress => GetOrDefault(nameof(ClientSettingsVaultAddress), Environment.GetEnvironmentVariable("VAULT_ADDR")); + + /// + /// Gets the Vault token for the client settings. + /// + public string ClientSettingsVaultToken => GetOrDefault(nameof(ClientSettingsVaultToken), Environment.GetEnvironmentVariable("VAULT_TOKEN")); + + /// + /// Gets the refresh interval for the client settings factory. + /// + public TimeSpan ClientSettingsRefreshInterval => GetOrDefault(nameof(ClientSettingsRefreshInterval), TimeSpan.FromSeconds(30)); + + /// + /// Gets the absolute path to the client settings configuration file if is false. + /// + public string ClientSettingsFilePath => GetOrDefault(nameof(ClientSettingsFilePath), "/var/cache/mfdlabs/client-settings.json"); + + /// + /// Resolves the dependency maps for specific application settings. + /// + /// + /// The value is formatted like this: + /// + /// Group1=Group2,Group3 + /// Group2=Group4 + /// + /// + public Dictionary ClientSettingsApplicationDependencies => GetOrDefault(nameof(ClientSettingsApplicationDependencies), new Dictionary()); + + /// + /// A list of application names that can be read from the API endpoints. + /// + public string[] PermissibleReadApplications + { + get => GetOrDefault(nameof(PermissibleReadApplications), Array.Empty()); + set => Set(nameof(PermissibleReadApplications), value); + } + + /// + /// Gets a command seperated list of API keys that can be used for reading non-permissable applications, + /// as well as executing privalaged commands. + /// + /// If this is empty, it will leave endpoints open!!! + public string[] ClientSettingsApiKeys => GetOrDefault(nameof(ClientSettingsApiKeys), Array.Empty()); + +} diff --git a/services/grid-bot/lib/settings/Providers/DiscordSettings.cs b/services/grid-bot/lib/settings/Providers/DiscordSettings.cs index 6a8b909b..5e938f03 100755 --- a/services/grid-bot/lib/settings/Providers/DiscordSettings.cs +++ b/services/grid-bot/lib/settings/Providers/DiscordSettings.cs @@ -19,7 +19,11 @@ public class DiscordSettings : BaseSettingsProvider /// The setting is required! public string BotToken => GetOrDefault( nameof(BotToken), +#if DEBUG + string.Empty +#else () => throw new InvalidOperationException($"Environment Variable {nameof(BotToken)} is required!") +#endif ); #if DEBUG || DEBUG_LOGGING_IN_PROD @@ -46,6 +50,18 @@ public class DiscordSettings : BaseSettingsProvider 0UL ); + /// + /// Determines if the bot should not be enabled. + /// + /// + /// This is only valid for debug builds. + /// If this is true, then the bot will not be enabled. + /// + public bool DebugBotDisabled => GetOrDefault( + nameof(DebugBotDisabled), + false + ); + #endif /// diff --git a/services/grid-bot/lib/settings/Providers/GlobalSettings.cs b/services/grid-bot/lib/settings/Providers/GlobalSettings.cs index b0e1f46a..5c277224 100755 --- a/services/grid-bot/lib/settings/Providers/GlobalSettings.cs +++ b/services/grid-bot/lib/settings/Providers/GlobalSettings.cs @@ -35,7 +35,7 @@ public class GlobalSettings : BaseSettingsProvider /// public int MetricsPort => GetOrDefault( nameof(MetricsPort), - 8080 + 8081 ); /// @@ -85,52 +85,4 @@ public class GlobalSettings : BaseSettingsProvider nameof(DiscordWebhookUrl), string.Empty ); - - /// - /// Gets the endpoint for the Grid Bot gRPC server. - /// - public string GridBotGrpcServerEndpoint => GetOrDefault( - nameof(GridBotGrpcServerEndpoint), - "http://+:5000" - ); - - /// - /// ASP.NET Core logger name for the gRPC server. - /// - public string GrpcServerLoggerName => GetOrDefault( - nameof(GrpcServerLoggerName), - "grpc" - ); - - /// - /// ASP.NET Core logger level for the gRPC server. - /// - public LogLevel GrpcServerLoggerLevel => GetOrDefault( - nameof(GrpcServerLoggerLevel), - LogLevel.Information - ); - - /// - /// Determines if the gRPC server should use TLS. - /// - public bool GrpcServerUseTls => GetOrDefault( - nameof(GrpcServerUseTls), - true - ); - - /// - /// Gets the certificate path for the gRPC server. - /// - public string GrpcServerCertificatePath => GetOrDefault( - nameof(GrpcServerCertificatePath), - () => throw new InvalidOperationException($"'{nameof(GrpcServerCertificatePath)}' is required when '{nameof(GrpcServerUseTls)}' is true.") - ); - - /// - /// Gets the certificate password for the gRPC server. - /// - public string GrpcServerCertificatePassword => GetOrDefault( - nameof(GrpcServerCertificatePassword), - () => throw new InvalidOperationException($"'{nameof(GrpcServerCertificatePassword)}' is required when '{nameof(GrpcServerUseTls)}' is true.") - ); } diff --git a/services/grid-bot/lib/settings/Providers/GrpcSettings.cs b/services/grid-bot/lib/settings/Providers/GrpcSettings.cs new file mode 100644 index 00000000..4e0cf395 --- /dev/null +++ b/services/grid-bot/lib/settings/Providers/GrpcSettings.cs @@ -0,0 +1,71 @@ +namespace Grid.Bot; + +using System; + +using Logging; + +/// +/// Settings provider for all gRPC Server related stuff. +/// +public class GrpcSettings : BaseSettingsProvider +{ + /// + public override string Path => SettingsProvidersDefaults.GrpcPath; + + /// + /// Determines if the grid-bot grpc server should be enabled or not. + /// + public bool GridBotGrpcServerEnabled => GetOrDefault( + nameof(GridBotGrpcServerEnabled), + false + ); + + /// + /// Gets the endpoint for the Grid Bot gRPC server. + /// + public string GridBotGrpcServerEndpoint => GetOrDefault( + nameof(GridBotGrpcServerEndpoint), + "http://+:5000" + ); + + /// + /// ASP.NET Core logger name for the gRPC server. + /// + public string GrpcServerLoggerName => GetOrDefault( + nameof(GrpcServerLoggerName), + "grpc" + ); + + /// + /// ASP.NET Core logger level for the gRPC server. + /// + public LogLevel GrpcServerLoggerLevel => GetOrDefault( + nameof(GrpcServerLoggerLevel), + LogLevel.Information + ); + + /// + /// Determines if the gRPC server should use TLS. + /// + public bool GrpcServerUseTls => GetOrDefault( + nameof(GrpcServerUseTls), + true + ); + + /// + /// Gets the certificate path for the gRPC server. + /// + public string GrpcServerCertificatePath => GetOrDefault( + nameof(GrpcServerCertificatePath), + () => throw new InvalidOperationException($"'{nameof(GrpcServerCertificatePath)}' is required when '{nameof(GrpcServerUseTls)}' is true.") + ); + + /// + /// Gets the certificate password for the gRPC server. + /// + public string GrpcServerCertificatePassword => GetOrDefault( + nameof(GrpcServerCertificatePassword), + () => throw new InvalidOperationException($"'{nameof(GrpcServerCertificatePassword)}' is required when '{nameof(GrpcServerUseTls)}' is true.") + ); + +} diff --git a/services/grid-bot/lib/settings/Providers/WebSettings.cs b/services/grid-bot/lib/settings/Providers/WebSettings.cs new file mode 100644 index 00000000..323437b7 --- /dev/null +++ b/services/grid-bot/lib/settings/Providers/WebSettings.cs @@ -0,0 +1,73 @@ +namespace Grid.Bot; + +using System; +using System.Collections.Generic; + +using Logging; + +/// +/// Settings provider for all Web Server related stuff. +/// +public class WebSettings : BaseSettingsProvider +{ + /// + public override string Path => SettingsProvidersDefaults.WebPath; + + /// + /// Determines if the web server should be enabled. + /// + public bool IsWebServerEnabled => GetOrDefault(nameof(IsWebServerEnabled), true); + + /// + /// Gets the bind address for the web server. + /// + public string WebServerBindAddress => GetOrDefault(nameof(WebServerBindAddress), "http://+:8888"); + + /// + /// Determines if the web server is behind a reverse proxy. + /// + /// + /// If true, then x-forwarded-for and x-forwarded-proto headers will be used to determine the client IP address. + /// + public bool IsWebServerBehindProxy => GetOrDefault(nameof(IsWebServerBehindProxy), false); + + /// + /// Gets the list of allowed proxy networks. + /// + public string[] WebServerAllowedProxyRanges => GetOrDefault(nameof(WebServerAllowedProxyRanges), new[] { "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16" }); + + /// + /// Determines if the web server should use TLS. + /// + /// + /// This should be set to true always in post-2017 grid servers + /// if this is the standard web server. + /// + public bool WebServerUseTls => GetOrDefault(nameof(WebServerUseTls), false); + + /// + /// Gets the path to the PFX certificate for the web server. + /// + public string WebServerCertificatePath => GetOrDefault( + nameof(WebServerCertificatePath), + () => throw new InvalidOperationException($"'{nameof(WebServerCertificatePath)}' is required when '{nameof(WebServerUseTls)}' is true.") + ); + + /// + /// Gets the password for the PFX certificate for the web server. + /// + public string WebServerCertificatePassword => GetOrDefault( + nameof(WebServerCertificatePassword), + () => throw new InvalidOperationException($"'{nameof(WebServerCertificatePassword)}' is required when '{nameof(WebServerUseTls)}' is true.") + ); + + /// + /// Gets the ASP.NET Core logger name for the web server. + /// + public string WebServerLoggerName => GetOrDefault(nameof(WebServerLoggerName), "web"); + + /// + /// Gets the ASP.NET Core logger level for the web server. + /// + public LogLevel WebServerLoggerLevel => GetOrDefault(nameof(WebServerLoggerLevel), LogLevel.Information); +} diff --git a/services/grid-bot/lib/settings/SettingsProvidersDefaults.cs b/services/grid-bot/lib/settings/SettingsProvidersDefaults.cs index 7392ccdb..f8cf1100 100755 --- a/services/grid-bot/lib/settings/SettingsProvidersDefaults.cs +++ b/services/grid-bot/lib/settings/SettingsProvidersDefaults.cs @@ -17,8 +17,10 @@ internal static class SettingsProvidersDefaults public static string ConsulPath => $"{EnvironmentProvider.EnvironmentName}/consul"; public static string UsersClientPath => $"{EnvironmentProvider.EnvironmentName}/users-client"; public static string ScriptsPath => $"{EnvironmentProvider.EnvironmentName}/scripts"; - public static string ClientSettingsClientPath => $"{EnvironmentProvider.EnvironmentName}/client-settings-client"; + public static string ClientSettingsPath => $"{EnvironmentProvider.EnvironmentName}/client-settings"; public static string GlobalPath => $"{EnvironmentProvider.EnvironmentName}/global"; + public static string WebPath => $"{EnvironmentProvider.EnvironmentName}/web"; + public static string GrpcPath => $"{EnvironmentProvider.EnvironmentName}/grpc"; public const string DefaultMountPath = "grid-bot-settings"; public static string MountPath = Environment.GetEnvironmentVariable(_vaultMountEnvVar) ?? DefaultMountPath; diff --git a/services/grid-bot/lib/settings/Shared.Settings.csproj b/services/grid-bot/lib/settings/Shared.Settings.csproj index bbbb3727..2ca7a8a1 100755 --- a/services/grid-bot/lib/settings/Shared.Settings.csproj +++ b/services/grid-bot/lib/settings/Shared.Settings.csproj @@ -11,8 +11,8 @@ - - + + diff --git a/services/grid-bot/lib/utility/Enums/FilterType.cs b/services/grid-bot/lib/utility/Enums/FilterType.cs new file mode 100644 index 00000000..b1ec898a --- /dev/null +++ b/services/grid-bot/lib/utility/Enums/FilterType.cs @@ -0,0 +1,17 @@ +namespace Grid.Bot.Utility; + +/// +/// Type of a client setting filter. +/// +public enum FilterType +{ + /// + /// The filter is filtering places. _PlaceFilter + /// + Place, + + /// + /// The filter is filtering datacenters. _DataCenterFilter + /// + DataCenter +} diff --git a/services/grid-bot/lib/utility/Enums/SettingType.cs b/services/grid-bot/lib/utility/Enums/SettingType.cs new file mode 100644 index 00000000..0f262377 --- /dev/null +++ b/services/grid-bot/lib/utility/Enums/SettingType.cs @@ -0,0 +1,22 @@ +namespace Grid.Bot.Utility; + +/// +/// The type of a setting. +/// +public enum SettingType +{ + /// + /// String type setting -- default. + /// + String, + + /// + /// Boolean type setting -- a flag. + /// + Bool, + + /// + /// Integer type setting. + /// + Int +} diff --git a/services/grid-bot/lib/utility/Extensions/DictionaryExtensions.cs b/services/grid-bot/lib/utility/Extensions/DictionaryExtensions.cs new file mode 100644 index 00000000..e3a1af07 --- /dev/null +++ b/services/grid-bot/lib/utility/Extensions/DictionaryExtensions.cs @@ -0,0 +1,37 @@ +namespace Grid.Bot.Extensions; + +using System.Linq; +using System.Collections.Generic; + +/// +/// Extension methods for Dictionary. +/// +public static class DictionaryExtensions +{ + /// + /// Merges the current dictionary with others, left to right. + /// If a key exists in multiple dictionaries, the value from the last one will be used. + /// + /// The type of the dictionary. + /// The type of the key. + /// The type of the value. + /// The current dictionary. + /// The dictionaries to merge with. + /// A new dictionary containing the merged key-value pairs. + public static T MergeLeft(this T me, params IDictionary[] others) + where T : IDictionary, new() + { + var newMap = new T(); + + foreach (IDictionary src in new List> { me }.Concat(others)) + { + // ^-- echk. Not quite there type-system. + foreach (KeyValuePair p in src) + { + newMap[p.Key] = p.Value; + } + } + + return newMap; + } +} \ No newline at end of file diff --git a/services/grid-bot/lib/utility/Implementation/ClientSettingsFactory.cs b/services/grid-bot/lib/utility/Implementation/ClientSettingsFactory.cs new file mode 100644 index 00000000..09acf5bc --- /dev/null +++ b/services/grid-bot/lib/utility/Implementation/ClientSettingsFactory.cs @@ -0,0 +1,593 @@ +namespace Grid.Bot.Utility; + +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Collections.Generic; + +using VaultSharp; +using VaultSharp.Core; + +using Prometheus; + +using Logging; +using Threading.Extensions; + +using Grid.Bot.Extensions; + +// Simplify these long ass types +using Secrets = System.Collections.Generic.IDictionary; +using MetaData = System.Collections.Generic.IDictionary; +using CachedValues = RefreshAhead>>; + +/// +/// Implementation for via Vault. +/// +/// +public class ClientSettingsFactory : IClientSettingsFactory +{ + private const string _metadataJsonKeyPrefix = "$$"; + + private readonly ClientSettingsSettings _settings; + private readonly IVaultClient _client; + private readonly ILogger _logger; + private readonly LazyWithRetry _settingsCacheRefreshAhead; + + private static readonly Type[] _supportedTypes = [typeof(string), typeof(bool), typeof(int)]; + + private readonly string _mount; + private readonly string _path; + + private readonly Counter _settingsRefreshCounter = Metrics.CreateCounter( + "rbx_client_settings_refresh", + "Number of times the client settings have been refreshed.", + "mount", "path" + ); + private readonly Counter _settingsWriteCounter = Metrics.CreateCounter( + "rbx_client_settings_write", + "Number of times the client settings have been written.", + "mount", "path" + ); + private readonly Counter _settingsReadCounter = Metrics.CreateCounter( + "rbx_client_settings_read", + "Number of times the client settings have been read.", + "mount", "path" + ); + + + private readonly ReaderWriterLockSlim _settingsCacheLock = new(); + + /// + /// Initializes a new instance of the + /// class. + /// + /// The . + /// The . + /// The . + /// + /// - is , only when is . + /// - is . + /// - is . + /// + /// is or whitespace. is + /// less than . + public ClientSettingsFactory( + IVaultClient client, + ILogger logger, + ClientSettingsSettings settings + ) + { + _client = client; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _settings = settings ?? throw new ArgumentNullException(nameof(settings)); + + if (_settings.ClientSettingsViaVault && _client == null) + throw new ArgumentNullException(nameof(client), "Client cannot be null when using Vault."); + + if (string.IsNullOrWhiteSpace(settings.ClientSettingsVaultMount)) + throw new ArgumentException("Value cannot be null or whitespace.", nameof(settings.ClientSettingsVaultMount)); + + if (settings.ClientSettingsRefreshInterval < TimeSpan.Zero) + throw new ArgumentOutOfRangeException(nameof(settings.ClientSettingsRefreshInterval), settings.ClientSettingsRefreshInterval, "Value cannot be less than zero."); + + _mount = settings.ClientSettingsVaultMount; + _path = settings.ClientSettingsVaultPath ?? "/"; + + _settingsCacheRefreshAhead = new LazyWithRetry(() => CachedValues.ConstructAndPopulate(settings.ClientSettingsRefreshInterval, DoRefresh)); + } + + private Dictionary ParseSecrets(Secrets secrets, MetaData metadata) + { + // rbx-client-settings are like this: + // FFlag is a bool + // FInt is an int + // FString is a string + // + // Settings prefixed with no prefix (such as FFlagTest) are static settings. + // Settings prefixed with D (such as DFFlagTest) are dynamic settings. + // Settings prefixed with S (such as SFFlagTest) are server synchronized + // settings. + // + // Anything that does not follow these prefixes is parsed as a string, + // unless there is a metadata key with the name of the setting that + // corresponds to a type that is not a string, in which case it is parsed as + // that type. + + var settings = new Dictionary(); + + foreach (var entry in secrets) + { + if (entry.Value is not JsonElement el) + { + _logger?.Verbose("Skipping setting '{0}' because it is not a string!", entry.Key); + + continue; + } + + var str = el.GetString(); + + if (!ClientSettingsNameHelper.PrefixedSettingRegex().IsMatch(entry.Key)) + { + // Setting has no prefix so check metadata for type, + // if not present just return as-is + if (metadata == null || !metadata.TryGetValue(entry.Key, out var type)) + { + _logger?.Verbose("Skipping setting '{0}' because it is not prefixed and has no metadata!", entry.Key); + + settings.Add(entry.Key, str); + + continue; + } + + if (!Enum.TryParse(type, true, out var settingType)) + { + _logger?.Verbose("Failed to parse setting type '{0}' for setting '{1}'! Defaulting to string.", type, entry.Key); + + settingType = SettingType.String; + } + + switch (settingType) + { + case SettingType.String: // bogus, but whatever + default: + settings.Add(entry.Key, str); + + break; + case SettingType.Bool: + if (bool.TryParse(str, out var boolValue)) + settings.Add(entry.Key, boolValue); + else + _logger?.Verbose("Failed to parse setting '{0}' as a bool!", entry.Key); + + break; + case SettingType.Int: + if (int.TryParse(str, out var intValue)) + settings.Add(entry.Key, intValue); + else + _logger?.Verbose("Failed to parse setting '{0}' as an int!", entry.Key); + + break; + } + + continue; + } + + var sType = ClientSettingsNameHelper.GetSettingTypeFromName(entry.Key); + switch (sType) + { + case SettingType.Bool when !ClientSettingsNameHelper.IsFilteredSetting(entry.Key): + if (bool.TryParse(str, out var boolValue)) + settings.Add(entry.Key, boolValue); + else + _logger?.Verbose("Failed to parse setting '{0}' as a bool!", entry.Key); + + break; + case SettingType.Int when !ClientSettingsNameHelper.IsFilteredSetting(entry.Key): + if (int.TryParse(str, out var intValue)) + settings.Add(entry.Key, intValue); + else + _logger?.Verbose("Failed to parse setting '{0}' as an int!", entry.Key); + + break; + case SettingType.String: + default: + settings.Add(entry.Key, str); + + break; + } + } + + return settings; + } + + private List<(string name, Secrets data, MetaData metadata)> FetchNewData() + { + var data = new List<(string name, Secrets data, MetaData metadata)>(); + + if (_settings.ClientSettingsViaVault) + { + _logger?.Debug("Refreshing settings from vault at path '{0}/{1}'", _mount, _path); + + // List all the keys in the path + var keys = _client.V1.Secrets.KeyValue.V2.ReadSecretPathsAsync(mountPoint: _mount, path: _path).Sync(); + if (keys.Data == null || keys.Data.Keys?.Count() == 0) + { + _logger?.Debug("No keys found at path '{0}/{1}'", _mount, _path); + + return data; + } + + // For each key, read the secret + foreach (var applicationName in keys.Data.Keys) + { + var secret = _client.V1.Secrets.KeyValue.V2.ReadSecretAsync(mountPoint: _mount, path: applicationName).Sync(); + var metadata = _client.V1.Secrets.KeyValue.V2.ReadSecretMetadataAsync(mountPoint: _mount, path: applicationName).Sync(); + + data.Add((applicationName, secret.Data.Data, metadata.Data.CustomMetadata)); + } + + return data; + } + + if (!File.Exists(_settings.ClientSettingsFilePath)) + { + _logger?.Debug("No file was found at the path '{0}'", _settings.ClientSettingsFilePath); + + return data; + } + + using var jsonStream = File.OpenRead(_settings.ClientSettingsFilePath); + var document = JsonDocument.Parse(jsonStream); + + foreach (JsonProperty prop in document.RootElement.EnumerateObject()) + { + var applicationName = prop.Name; + var applicationMetatdataName = $"{_metadataJsonKeyPrefix}{applicationName}"; + + MetaData metadata = new Dictionary(); + if (document.RootElement.TryGetProperty(applicationMetatdataName, out var metadataProp)) + metadata = metadataProp.Deserialize(); + + var applicationData = prop.Value.Deserialize(); + + data.Add((applicationName, applicationData, metadata)); + } + + return data; + } + + private void DoCommit(string applicationName, Secrets data, MetaData metadata) + { + var serializedData = data.ToDictionary(k => k.Key, v => v.Value.ToString()); + _settingsWriteCounter.WithLabels(_mount, _path).Inc(); + + if (_settings.ClientSettingsViaVault) + { + _logger?.Debug("Writiting settings to vault at path '{0}/{1}/{2}'", + _mount, _path, applicationName); + + _client.V1.Secrets.KeyValue.V2 .WriteSecretAsync( + mountPoint: _mount, + path: $"{_path}/${applicationName}", + data: serializedData).Wait(); + + _client.V1.Secrets.KeyValue.V2.WriteSecretMetadataAsync( + mountPoint: _mount, path: $"{_path}/${applicationName}", + customMetadataRequest: new() { CustomMetadata = metadata.ToDictionary() }).Wait(); + + return; + } + + using var jsonStream = File.Open(_settings.ClientSettingsFilePath, FileMode.OpenOrCreate); + var document = JsonNode.Parse(jsonStream); + + document[applicationName] = JsonSerializer.Serialize(serializedData); + document[$"{_metadataJsonKeyPrefix}{applicationName}"] = JsonSerializer.Serialize(metadata); + + using var writer = new Utf8JsonWriter(jsonStream); + document.WriteTo(writer); + + // Can be made more efficient? can we skip past remote here + _settingsCacheRefreshAhead.LazyValue.Value[applicationName] = data; + } + + private IDictionary DoRefresh(IDictionary oldSettings) + { + _logger?.Debug("Refreshing settings, FromVault = {0}", _settings.ClientSettingsViaVault); + _settingsRefreshCounter.WithLabels(_mount, _path).Inc(); + + var settings = new Dictionary(); + + try + { + // List all the keys in the path + var data = FetchNewData(); + + // For each key, read the secret + foreach (var (applicationName, applicationData, applicationMetaData) in data) + { + var parsedSecrets = ParseSecrets(applicationData, applicationMetaData); + settings.Add(applicationName, parsedSecrets); + } + + return settings; + } + catch (VaultApiException ex) + { + _logger?.Error(ex); + + return settings; + } + } + + /// + public IDictionary RawSettings + { + get + { + _settingsCacheLock.EnterReadLock(); + + _settingsReadCounter.WithLabels(_mount, _path).Inc(); + + try + { + return _settingsCacheRefreshAhead.LazyValue.Value; + } + finally + { + _settingsCacheLock.ExitReadLock(); + } + } + } + + /// + public void Refresh() + { + _settingsCacheLock.EnterWriteLock(); + + try + { + _settingsCacheRefreshAhead.LazyValue.Refresh(); + } + finally + { + _settingsCacheLock.ExitWriteLock(); + } + } + + /// + public Secrets GetSettingsForApplication(string application, bool withDependencies = true) + { + if (string.IsNullOrWhiteSpace(application)) + throw new ArgumentException(string.Format("'{0}' cannot be null or whitespace!", nameof(application)), nameof(application)); + + _settingsCacheLock.EnterReadLock(); + + _settingsReadCounter.WithLabels(_mount, _path).Inc(); + + try + { + var hasDependencies = _settings.ClientSettingsApplicationDependencies.TryGetValue(application, out var dependencies); + + if (!_settingsCacheRefreshAhead.LazyValue.Value.TryGetValue(application, out var settings) && !hasDependencies) + { + _logger?.Debug("Settings for application '{0}' not found!", application); + + return null; + } + + settings ??= new Dictionary(); + + if (withDependencies && hasDependencies) + { + var dependenciesToMerge = new List(); + var dependencyNames = dependencies.Split(','); + + foreach (var dependency in dependencyNames) + { + if (!_settingsCacheRefreshAhead.LazyValue.Value.TryGetValue(dependency, out var dependencySettings)) + { + _logger?.Debug("Dependency '{0}' for application '{1}' not found!", dependency, application); + + continue; + } + + dependenciesToMerge.Add(dependencySettings); + } + + if (dependenciesToMerge.Count > 0) + { + _logger?.Debug("Merging settings for application '{0}' with dependencies: {1}", application, string.Join(", ", dependencyNames)); + + var mergedSettings = new Dictionary(settings); + + settings = mergedSettings.MergeLeft(dependenciesToMerge.ToArray()); + } + } + + return settings; + } + finally + { + _settingsCacheLock.ExitReadLock(); + } + } + + /// + public T GetSettingForApplication(string application, string setting, bool withDependencies = true) + { + if (string.IsNullOrWhiteSpace(application)) + throw new ArgumentException(string.Format("'{0}' cannot be null or whitespace!", nameof(application)), nameof(application)); + + if (string.IsNullOrWhiteSpace(setting)) + throw new ArgumentException(string.Format("'{0}' cannot be null or whitespace!", nameof(setting)), nameof(setting)); + + if (!_supportedTypes.Contains(typeof(T)) || typeof(T) == typeof(FilteredValue<>)) + throw new ArgumentException(string.Format("'{0}' is not a supported type!", typeof(T).Name), nameof(T)); + + var settings = GetSettingsForApplication(application, withDependencies) + ?? throw new InvalidOperationException(string.Format("Application '{0}' not found!", application)); + + if (!settings.TryGetValue(setting, out var value)) + throw new InvalidOperationException(string.Format("Setting '{0}' for application '{1}' not found!", setting, application)); + + try + { + var str = ((JsonElement)value).GetString(); + + return typeof(T) switch + { + Type t when t == typeof(string) => (T)(object)str.ToString(), + Type t when t == typeof(bool) => (T)(object)bool.Parse(str), + Type t when t == typeof(int) => (T)(object)long.Parse(str), + _ => throw new ArgumentException(string.Format("'{0}' is not a supported type!", typeof(T).Name)), + }; + } + catch (Exception ex) when (ex is FormatException or InvalidCastException or OverflowException) + { + _logger?.Error(ex); + + throw new InvalidCastException(string.Format( + "Failed to cast setting '{0}' for application '{1}' to type '{2}'!", + setting, + application, + typeof(T).Name), + ex + ); + } + } + + /// + public FilteredValue GetFilteredSettingForApplication( + string application, + string setting, + FilterType filterType = FilterType.Place, + bool withDependencies = false + ) + { + if (string.IsNullOrWhiteSpace(application)) + throw new ArgumentException(string.Format("'{0}' cannot be null or whitespace!", nameof(application)), nameof(application)); + + if (string.IsNullOrWhiteSpace(setting)) + throw new ArgumentException(string.Format("'{0}' cannot be null or whitespace!", nameof(setting)), nameof(setting)); + + if (!_supportedTypes.Contains(typeof(T))) + throw new ArgumentException(string.Format("'{0}' is not a supported type!", typeof(T).Name), nameof(T)); + + var settings = GetSettingsForApplication(application, withDependencies) + ?? throw new InvalidOperationException(string.Format("Application '{0}' not found!", application)); + + var settingName = $"{setting}_{filterType}Filter"; + + if (!settings.TryGetValue(settingName, out var value)) + throw new InvalidOperationException(string.Format("Setting '{0}' for application '{1}' not found!", setting, application)); + + try + { + return FilteredValue.FromString(settingName, (string)value); + } + catch (Exception ex) when (ex is FormatException or InvalidCastException or OverflowException) + { + _logger?.Error(ex); + + throw new InvalidCastException(string.Format( + "Failed to cast setting '{0}' for application '{1}' to type '{2}'!", + setting, + application, + typeof(T).Name), + ex + ); + } + } + + /// + public void WriteSettingsForApplication(string application, Secrets settings) + { + if (string.IsNullOrWhiteSpace(application)) + throw new ArgumentException(string.Format("'{0}' cannot be null or whitespace!", nameof(application)), nameof(application)); + + ArgumentNullException.ThrowIfNull(settings, nameof(settings)); + + var metadata = new Dictionary(); + + foreach (var kvp in settings) + { + if (!_supportedTypes.Contains(kvp.Value.GetType())) + { + _logger.Warning("{0}.{1} is not a valid type, skipping!", application, kvp.Key); + + continue; + } + + if (!ClientSettingsNameHelper.PrefixedSettingRegex().IsMatch(kvp.Key)) + metadata[kvp.Key] = (kvp.Value switch + { + bool => SettingType.Bool, + int => SettingType.Int, + _ => SettingType.String + }).ToString(); + } + + _settingsCacheLock.EnterWriteLock(); + + try + { + DoCommit(application, settings, metadata); + } + catch (Exception ex) + { + _logger.Error(ex); + + throw; + } + finally + { + _settingsCacheLock.ExitWriteLock(); + } + } + + /// + public void SetSettingForApplication(string application, string setting, T value) + { + if (string.IsNullOrWhiteSpace(application)) + throw new ArgumentException(string.Format("'{0}' cannot be null or whitespace!", nameof(application)), nameof(application)); + + if (string.IsNullOrWhiteSpace(setting)) + throw new ArgumentException(string.Format("'{0}' cannot be null or whitespace!", nameof(setting)), nameof(setting)); + + if (!_supportedTypes.Contains(typeof(T))) + throw new ArgumentException(string.Format("'{0}' is not a supported type!", typeof(T).Name), nameof(T)); + + if (value is null) + throw new ArgumentNullException(nameof(value)); + + var data = GetSettingsForApplication(application, false) ?? new Dictionary(); + data[setting] = value; + + WriteSettingsForApplication(application, data); + } + + /// + public void SetSettingForApplication(string application, string setting, object value, SettingType settingType = SettingType.String) + { + switch (settingType) + { + case SettingType.String: + SetSettingForApplication(application, setting, value.ToString()); + break; + case SettingType.Int: + SetSettingForApplication(application, setting, Convert.ToInt64(value)); + break; + case SettingType.Bool: + SetSettingForApplication(application, setting, Convert.ToBoolean(value)); + break; + default: + throw new ArgumentException(string.Format("'{0}' is not a supported type!", settingType), nameof(settingType)); + } + } +} diff --git a/services/grid-bot/lib/utility/Implementation/ClientSettingsNameHelper.cs b/services/grid-bot/lib/utility/Implementation/ClientSettingsNameHelper.cs new file mode 100644 index 00000000..53be03e7 --- /dev/null +++ b/services/grid-bot/lib/utility/Implementation/ClientSettingsNameHelper.cs @@ -0,0 +1,77 @@ +namespace Grid.Bot.Utility; + +using System; +using System.Text.RegularExpressions; + +/// +/// Helper for client settings. +/// +public static partial class ClientSettingsNameHelper +{ + [GeneratedRegex(@"^([DS])?F(Flag|Log|Int|String)", RegexOptions.Compiled)] + public static partial Regex PrefixedSettingRegex(); + + /// + /// Determines if the setting name is a filtered value setting. + /// + /// The name of the setting. + /// True if the setting is a filtered setting, otherwise false. + public static bool IsFilteredSetting(string name) + => name.EndsWith(FilteredValue.PlaceFilterSuffix) || name.EndsWith(FilteredValue.DataCenterFilterSuffix); + + /// + /// Extracts the filtered setting name and type from the given name. + /// + /// The name of the setting. + /// A tuple containing the name of the setting and its type. + public static (string name, FilterType type) ExtractFilteredSettingName(string name) + { + var type = FilterType.Place; + + if (name.EndsWith(FilteredValue.PlaceFilterSuffix)) + { + name = name[..^FilteredValue.PlaceFilterSuffix.Length]; + } + else if (name.EndsWith(FilteredValue.DataCenterFilterSuffix)) + { + type = FilterType.DataCenter; + name = name[..^FilteredValue.DataCenterFilterSuffix.Length]; + } + + return (name, type); + } + + /// + /// Gets the type of setting from its name. + /// + /// The name of the setting. + /// The type of the setting. + public static SettingType GetSettingTypeFromName(string name) + { + if (!PrefixedSettingRegex().IsMatch(name)) return SettingType.String; + + // F = flag, I = int, S = string, L = log (int) + // e.g: + // FFlagTest + // 012345678 + // ^ + // + // DFFlagTest + // 0123456789 + // ^ + + var prefix = name[0]; + if (prefix != 'F') + prefix = name[1..][1]; + else + prefix = name[1]; + + return prefix switch + { + 'F' => SettingType.Bool, // FFlag + 'I' or 'L' => SettingType.Int, // FInt, FLog + _ => SettingType.String, // FString + }; + + } +} \ No newline at end of file diff --git a/services/grid-bot/lib/utility/Implementation/FilteredValue.cs b/services/grid-bot/lib/utility/Implementation/FilteredValue.cs new file mode 100644 index 00000000..2ebfc270 --- /dev/null +++ b/services/grid-bot/lib/utility/Implementation/FilteredValue.cs @@ -0,0 +1,124 @@ +namespace Grid.Bot.Utility; + +using System; +using System.Linq; +using System.Collections.Generic; + +/// +/// Represents a filtered value. +/// +public struct FilteredValue +{ + private const char _filterDelimiter = ';'; + + /// + /// Suffix for place filters. + /// + public const string PlaceFilterSuffix = "_PlaceFilter"; + + /// + /// Suffix for datacenter filters. + /// + public const string DataCenterFilterSuffix = "_DataCenterFilter"; + + /// + /// Construct a new instance of + /// + public FilteredValue() + { + } + + /// + /// Get or set the raw value. + /// + public T Value { get; set; } + + /// + /// Gets or sets the name. + /// + public string Name { get; set; } + + /// + /// Gets the type of the setting. + /// + public readonly SettingType Type + { + get + { + return Value switch + { + bool => SettingType.Bool, + long => SettingType.Int, + _ => SettingType.String, + }; + } + } + + /// + /// Gets or sets the type of filter. + /// + public FilterType FilterType { get; set; } + + /// + /// Gets the filtered place IDs or datacenter IDs. + /// + public HashSet FilteredIds { get; private set; } = []; + + /// + /// Implicit conversion of to + /// + /// The current + public static implicit operator T(FilteredValue value) => value.Value; + + /// + /// Converts the string representation of the filtered value to a filtered value. + /// + /// The raw name of the setting, used to determine the type of filter. + /// The string value of the setting. + /// A new filtered value. + public static FilteredValue FromString(string name, string value) + { + if (!ClientSettingsNameHelper.IsFilteredSetting(name)) + throw new ArgumentException($"The setting name does not end with {PlaceFilterSuffix} or {DataCenterFilterSuffix}!", nameof(name)); + + var filterType = name.EndsWith(PlaceFilterSuffix) + ? FilterType.Place + : FilterType.DataCenter; + var settingName = filterType == FilterType.Place + ? name[..^PlaceFilterSuffix.Length] + : name[..^DataCenterFilterSuffix.Length]; + var settingType = ClientSettingsNameHelper.GetSettingTypeFromName(name); + + var entries = value.Split(_filterDelimiter).Where(s => !string.IsNullOrWhiteSpace(s)).Distinct().ToList(); + if (entries.Count == 0) throw new ArgumentException("Value had no entries!", nameof(value)); + + var settingValueRaw = entries.First(); + var filteredIds = entries.Skip(1).Select(long.Parse); + object settingValue = settingType switch + { + SettingType.Bool => bool.Parse(settingValueRaw), + SettingType.Int => long.Parse(settingValueRaw), + _ => settingValueRaw, + }; + + return new FilteredValue + { + Name = settingName, + FilterType = filterType, + FilteredIds = [.. filteredIds], + Value = (T)(object)settingValue.ToString() + }; + } + + /// + /// Convert the filtered value to a string. + /// + /// The new name and the stringified value. + public new readonly (string key, string value) ToString() + { + var name = $"{Name}_{FilterType}Filter"; + ICollection values = [Value.ToString(), ..FilteredIds.Select(x => x.ToString())]; + + return (name, string.Join(_filterDelimiter, values)); + } +} diff --git a/services/grid-bot/lib/utility/Implementation/LazyWithRetry.cs b/services/grid-bot/lib/utility/Implementation/LazyWithRetry.cs new file mode 100644 index 00000000..c5fcc400 --- /dev/null +++ b/services/grid-bot/lib/utility/Implementation/LazyWithRetry.cs @@ -0,0 +1,91 @@ +namespace Grid.Bot.Utility; + +using System; + +/// +/// Lazy with retry implementation +/// +public class LazyWithRetry +{ + private readonly TimeSpan _TimeoutBetweenRetries = TimeSpan.FromSeconds(30); + + private Lazy _Lazy; + + private readonly object _Sync = new(); + private readonly Func _ValueFactory; + private readonly Func _NowGetter; + + private DateTime? _LastExceptionTimeStamp; + + /// + public bool IsValueCreated => _Lazy.IsValueCreated; + + /// + /// Initializes a new instance of the class. + /// + /// The value factory. + /// The now getter. + /// is . + public LazyWithRetry(Func valueFactory, Func nowGetter = null) + { + _ValueFactory = valueFactory ?? throw new ArgumentNullException(nameof(valueFactory)); + + _Lazy = new Lazy(_ValueFactory); + _NowGetter = nowGetter ?? (() => DateTime.UtcNow); + } + + /// + /// Initializes a new instance of the class. + /// + /// The value factory. + /// The timeout between retries. + /// The now getter. + public LazyWithRetry(Func valueFactory, TimeSpan timeoutBetweenRetries, Func nowGetter = null) + : this(valueFactory, nowGetter) + { + _TimeoutBetweenRetries = timeoutBetweenRetries; + } + + /// + public T LazyValue + { + get + { + try + { + return _Lazy.Value; + } + catch (Exception) + { + bool isTimeForReset = false; + lock (_Sync) + { + if (_LastExceptionTimeStamp == null) + _LastExceptionTimeStamp = _NowGetter(); + else + { + DateTime t = _NowGetter(); + if (t > _LastExceptionTimeStamp + _TimeoutBetweenRetries) + { + Reset(); + isTimeForReset = true; + } + } + } + if (!isTimeForReset) + throw; + + return _Lazy.Value; + } + } + } + + /// + /// Reset the current lazy value. + /// + public void Reset() + { + _Lazy = new Lazy(_ValueFactory); + _LastExceptionTimeStamp = null; + } +} \ No newline at end of file diff --git a/services/grid-bot/lib/utility/Implementation/MicrosoftLogger.cs b/services/grid-bot/lib/utility/Implementation/MicrosoftLogger.cs new file mode 100644 index 00000000..5a92c796 --- /dev/null +++ b/services/grid-bot/lib/utility/Implementation/MicrosoftLogger.cs @@ -0,0 +1,65 @@ +namespace Grid.Bot.Utility; + +using System; + +using ILogger = Logging.ILogger; + +using IMLogger = Microsoft.Extensions.Logging.ILogger; +using MEventId = Microsoft.Extensions.Logging.EventId; +using MLogLevel = Microsoft.Extensions.Logging.LogLevel; + +/// +/// Implementation of that forwards to our ! +/// +/// Our +/// cannot be null. +public class MicrosoftLogger(ILogger logger) : IMLogger +{ + private class NoopDisposable : IDisposable + { + public void Dispose() { } + } + + private readonly ILogger _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + /// + public IDisposable BeginScope(TState state) => new NoopDisposable(); + + /// + public bool IsEnabled(MLogLevel logLevel) => true; + + /// + public void Log(MLogLevel logLevel, MEventId eventId, TState state, Exception exception, Func formatter) + { + var message = formatter(state, exception); + + switch (logLevel) + { + case MLogLevel.Trace: + _logger.Verbose(message); + + break; + case MLogLevel.Debug: + _logger.Debug(message); + + break; + case MLogLevel.Information: + _logger.Information(message); + + break; + case MLogLevel.Warning: + _logger.Warning(message); + + break; + case MLogLevel.Error: + case MLogLevel.Critical: + _logger.Error(message); + + break; + default: + _logger.Warning(message); + + break; + } + } +} \ No newline at end of file diff --git a/services/grid-bot/lib/utility/Implementation/MicrosoftLoggerProvider.cs b/services/grid-bot/lib/utility/Implementation/MicrosoftLoggerProvider.cs new file mode 100644 index 00000000..8f91abb8 --- /dev/null +++ b/services/grid-bot/lib/utility/Implementation/MicrosoftLoggerProvider.cs @@ -0,0 +1,24 @@ +namespace Grid.Bot.Utility; + +using System; + +using ILogger = Logging.ILogger; + +using IMLogger = Microsoft.Extensions.Logging.ILogger; +using IMLoggerProvider = Microsoft.Extensions.Logging.ILoggerProvider; + +/// +/// Provider for the class. +/// +/// Our +/// cannot be null. +public class MicrosoftLoggerProvider(ILogger logger) : IMLoggerProvider +{ + private readonly ILogger _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + /// + public IMLogger CreateLogger(string categoryName) => new MicrosoftLogger(_logger); + + /// + public void Dispose() { } +} \ No newline at end of file diff --git a/services/grid-bot/lib/utility/Implementation/RefreshAhead.cs b/services/grid-bot/lib/utility/Implementation/RefreshAhead.cs new file mode 100644 index 00000000..0ce2c323 --- /dev/null +++ b/services/grid-bot/lib/utility/Implementation/RefreshAhead.cs @@ -0,0 +1,147 @@ +namespace Grid.Bot.Utility; + +using System; +using System.Threading; + + +/// +/// Refresh ahead cached value. +/// +public class RefreshAhead : IDisposable +{ + private DateTime _LastRefresh = DateTime.MinValue; + private bool _RunningRefresh; + private TimeSpan _RefreshInterval; + + private readonly Timer _RefreshTimer; + private readonly Func _RefreshDelegate; + + /// + /// Gets the interval since refresh. + /// + public TimeSpan IntervalSinceRefresh => DateTime.Now.Subtract(_LastRefresh); + + /// + /// Gets the currently cached value. + /// + public T Value { get; private set; } + + /// + /// Initializes a new instance of the class. + /// + /// The refresh interval. + /// The refresh delegate. + public RefreshAhead(TimeSpan refreshInterval, Func refreshDelegate) + : this(default(T), refreshInterval, _ => refreshDelegate()) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The initial value. + /// The refresh interval. + /// The refresh delegate. + public RefreshAhead(T initialValue, TimeSpan refreshInterval, Func refreshDelegate) + : this(initialValue, refreshInterval, _ => refreshDelegate()) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The initial value. + /// The refresh interval. + /// The refresh delegate. + /// is . + public RefreshAhead(T initialValue, TimeSpan refreshInterval, Func refreshDelegate) + { + _RefreshInterval = refreshInterval; + _RefreshDelegate = refreshDelegate ?? throw new ArgumentNullException(nameof(refreshDelegate)); + + Value = initialValue; + _LastRefresh = DateTime.UtcNow; + + var refreshIntervalInMilliseconds = (int)refreshInterval.TotalMilliseconds; + _RefreshTimer = new Timer(_ => Refresh(), null, refreshIntervalInMilliseconds, refreshIntervalInMilliseconds); + } + + /// + /// Sets a new refresh interval. + /// + /// The new refresh interval. + public void SetRefreshInterval(TimeSpan newRefreshInterval) + { + var timeSinceLastRefresh = DateTime.UtcNow - _LastRefresh; + var nextRunTime = newRefreshInterval - timeSinceLastRefresh; + if (nextRunTime.TotalMilliseconds < 0.0) nextRunTime = TimeSpan.Zero; + + _RefreshTimer.Change(nextRunTime, newRefreshInterval); + _RefreshInterval = newRefreshInterval; + } + + /// + /// Manually refreshes the current data. + /// + public void Refresh() + { + if (_RunningRefresh) return; + + _RunningRefresh = true; + + bool refreshTimer = true; + try + { + Value = _RefreshDelegate.Invoke(Value); + + _LastRefresh = DateTime.UtcNow; + } + catch (ThreadAbortException) + { + refreshTimer = false; + + throw; + } + catch (Exception ex) + { + Logging.Logger.Singleton.Error(ex); + } + finally + { + _RunningRefresh = false; + + if (refreshTimer) + _RefreshTimer.Change(_RefreshInterval, _RefreshInterval); + } + } + + /// + /// Constructs the and populates a new instance of . + /// + /// The refresh interval. + /// The refresh delegate. + /// A new instance of . + public static RefreshAhead ConstructAndPopulate(TimeSpan refreshInterval, Func refreshDelegate) + => ConstructAndPopulate(refreshInterval, _ => refreshDelegate()); + + /// + /// Constructs the and populates a new instance of . + /// + /// The refresh interval. + /// The refresh delegate. + /// A new instance of . + public static RefreshAhead ConstructAndPopulate(TimeSpan refreshInterval, Func refreshDelegate) + => new(refreshDelegate(default(T)), refreshInterval, refreshDelegate); + + #region IDisposable Members + + /// + public void Dispose() + { + GC.SuppressFinalize(this); + + _RefreshTimer?.Dispose(); + } + + #endregion IDisposable Members +} \ No newline at end of file diff --git a/services/grid-bot/lib/utility/Interfaces/IClientSettingsFactory.cs b/services/grid-bot/lib/utility/Interfaces/IClientSettingsFactory.cs new file mode 100644 index 00000000..53452f81 --- /dev/null +++ b/services/grid-bot/lib/utility/Interfaces/IClientSettingsFactory.cs @@ -0,0 +1,97 @@ +namespace Grid.Bot.Utility; + +using System; +using System.Collections.Generic; + +using Secrets = System.Collections.Generic.IDictionary; + +/// +/// A factory that provides client-usable settings. +/// +public interface IClientSettingsFactory +{ + /// + /// Gets the raw settings from the backed refresh ahead. + /// + IDictionary RawSettings { get; } + + /// + /// Refreshes the settings. + /// + void Refresh(); + + /// + /// Gets the settings for the specified application. + /// + /// The name of the application. + /// if set to true [with dependencies]. + /// The settings for the specified application. + /// is null or whitespace. + Secrets GetSettingsForApplication(string application, bool withDependencies = true); + + /// + /// Gets the specific setting for the specified application. + /// + /// The type of the setting. Can either be a string, int or bool. + /// The name of the application. + /// The name of the setting. + /// if set to true [with dependencies]. + /// The setting for the specified application. + /// + /// - can only be a string, int or bool. + /// - is null or whitespace. + /// - is null or whitespace. + /// + /// The setting was not found for the specified application. + /// The setting could not be cast to the specified type. + T GetSettingForApplication(string application, string setting, bool withDependencies = true); + + /// + /// Gets the specific setting for the specified application. + /// + /// The type of the setting. Can either be a string, int or bool. + /// The name of the application. + /// The name of the setting. + /// The type of filtered value to read. + /// if set to true [with dependencies]. + /// The setting for the specified application. + /// + /// - can only be a string, int or bool. + /// - is null or whitespace. + /// - is null or whitespace. + /// + /// The setting was not found for the specified application. + /// The setting could not be cast to the specified type. + FilteredValue GetFilteredSettingForApplication(string application, string setting, FilterType filterType = FilterType.Place, bool withDependencies = true); + + /// + /// Import settings for the specified application. + /// + /// Any non-prefixed settings will have their types placed into metadata. + /// The name of the application + /// The raw setting. + void WriteSettingsForApplication(string application, Secrets settings); + + /// + /// Sets the value of the specified setting. + /// + /// The type of the setting. Can either be a string, int, bool or + /// The name of the application. + /// The name of the setting. + /// The value of the setting, this can be of type + void SetSettingForApplication(string application, string setting, T value); + + /// + /// Sets the value of the specified setting. + /// + /// The name of the application. + /// The name of the setting. + /// The value of the setting. + /// The type of the setting. + /// + /// - is null or whitespace. + /// - is null or whitespace. + /// - is null or whitespace. + /// + void SetSettingForApplication(string application, string setting, object value, SettingType settingType = SettingType.String); +} \ No newline at end of file diff --git a/services/grid-bot/lib/utility/Shared.Utility.csproj b/services/grid-bot/lib/utility/Shared.Utility.csproj index b324b17f..31ccec18 100755 --- a/services/grid-bot/lib/utility/Shared.Utility.csproj +++ b/services/grid-bot/lib/utility/Shared.Utility.csproj @@ -50,22 +50,8 @@ - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - + + diff --git a/services/grid-bot/lib/web/Extensions/HttpContextExtensions.cs b/services/grid-bot/lib/web/Extensions/HttpContextExtensions.cs new file mode 100644 index 00000000..33758c32 --- /dev/null +++ b/services/grid-bot/lib/web/Extensions/HttpContextExtensions.cs @@ -0,0 +1,57 @@ +namespace Grid.Bot.Web.Extensions; + +using System; +using System.Linq; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.Http; + +/// +/// Extensions for , and +/// +public static class HttpContextExtensions +{ + private const string _apiKeyHeaderName = "x-api-key"; + + /// + /// Writes an error to the response. + /// + /// The + /// The error message + public static async Task WriteRbxError(this HttpResponse response, string error) + { + var errors = new object[1] { new { code = 1, message = error } }; + + await response.WriteAsJsonAsync(new { errors }); + } + + /// + /// Determines if the request has a valid API key in it. + /// + /// The + /// The + /// [true] if the request has a valid API key, otherwise false. + public static bool HasValidApiKey(this HttpRequest request, ClientSettingsSettings settings) + { + if (settings.ClientSettingsApiKeys.Length == 0) return true; + if (!request.Headers.TryGetValue(_apiKeyHeaderName, out var apiKeyHeaderValues)) return false; + + var apiKeyHeader = apiKeyHeaderValues.First(); + return settings.ClientSettingsApiKeys.Contains(apiKeyHeader); + } + + /// + /// Tries to get an int64 from the request query string. + /// + /// This is case-insensitive. + /// The + /// The key to look for + /// The value of the key + /// [true] if the key was found and the value is an integer, otherwise false. + public static bool TryParseInt64FromQuery(this HttpRequest request, string key, out long value) + { + value = 0; + + return request.Query.TryGetValue(key, out var valueString) && long.TryParse(valueString, out value); + } +} diff --git a/services/grid-bot/lib/web/Extensions/HttpServerTelemetryExtensions.cs b/services/grid-bot/lib/web/Extensions/HttpServerTelemetryExtensions.cs new file mode 100644 index 00000000..34db0014 --- /dev/null +++ b/services/grid-bot/lib/web/Extensions/HttpServerTelemetryExtensions.cs @@ -0,0 +1,27 @@ +namespace Grid.Bot.Web.Middleware; + +using System; + +using Microsoft.AspNetCore.Builder; + +/// +/// Extension methods for adding telemetry to the HTTP server. +/// +public static class HttpServerTelemetryExtensions +{ + /// + /// Adds telemetry to the HTTP server. + /// + /// The + /// The + public static IApplicationBuilder UseTelemetry(this IApplicationBuilder app) + { + ArgumentNullException.ThrowIfNull(app); + + app.UseMiddleware(); + app.UseMiddleware(); + app.UseMiddleware(); + + return app; + } +} diff --git a/services/grid-bot/lib/web/Extensions/IServiceProviderExtensions.cs b/services/grid-bot/lib/web/Extensions/IServiceProviderExtensions.cs new file mode 100644 index 00000000..11407071 --- /dev/null +++ b/services/grid-bot/lib/web/Extensions/IServiceProviderExtensions.cs @@ -0,0 +1,159 @@ +namespace Grid.Bot.Web; + +using System; +using System.Threading.Tasks; +using System.Collections.Generic; + +using Microsoft.AspNetCore.Routing; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.HttpOverrides; +using Microsoft.AspNetCore.Server.Kestrel.Core; + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.DependencyInjection; + +using Prometheus; + +using Logging; +using Utility; + +using Routes; +using Middleware; +using Extensions; + +/// +/// gRPC extensions for the +/// +public static class IServiceProviderExtensions +{ + /// + /// Implement the grid-service WebSrv server into the current service provider. + /// + /// The + /// The application arguments. + public static void UseWebServer(this IServiceProvider services, IEnumerable args) + { + var webSettings = services.GetRequiredService(); + var logger = new Logger( + name: webSettings.WebServerLoggerName, + logLevelGetter: () => webSettings.WebServerLoggerLevel, + logToConsole: true, + logToFileSystem: true + ); + + if (!webSettings.IsWebServerEnabled) + { + logger.Warning("The grid-bot web server is disabled in settings, not starting web server!"); + + return; + } + + var clientSettingsFactory = services.GetRequiredService(); + + logger.Information("Starting web server on {0}", webSettings.WebServerBindAddress); + + var builder = WebApplication.CreateBuilder([.. args]); + + builder.Logging.ClearProviders(); + builder.Logging.AddProvider(new MicrosoftLoggerProvider(logger)); + + var avatarSettings = services.GetRequiredService(); + var clientSettingsSettings = services.GetRequiredService(); + + builder.Services.AddSingleton(logger); + builder.Services.AddSingleton(webSettings); + builder.Services.AddSingleton(clientSettingsFactory); + builder.Services.AddSingleton(clientSettingsSettings); + builder.Services.AddSingleton(avatarSettings); + + builder.Services.AddHttpClient(); + + // Routes + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + + builder.Services.AddRouting(); + + // Conmfigure case insensitive routing + builder.Services.Configure(options => + { + options.LowercaseQueryStrings = true; + options.LowercaseUrls = true; + }); + + builder.Services.AddHealthChecks(); + builder.Services.AddMetrics(); + + if (webSettings.WebServerUseTls) + { + builder.Services.Configure(options => + { + options.ConfigureEndpointDefaults(listenOptions => + { + try + { + listenOptions.UseHttps(webSettings.WebServerCertificatePath, webSettings.WebServerCertificatePassword); + listenOptions.Protocols = HttpProtocols.Http1AndHttp2; + } + catch (Exception ex) + { + logger.Warning("Failed to load TLS certificate: {0}. Will fall back to HTTP.", ex.Message); + } + }); + }); + } + + if (webSettings.IsWebServerBehindProxy) + { + builder.Services.Configure(options => + { + options.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto; + options.KnownNetworks.Clear(); + options.KnownProxies.Clear(); + + foreach (var ip in webSettings.WebServerAllowedProxyRanges) + options.KnownNetworks.Add(IPNetwork.Parse(ip)); + }); + } + + var app = builder.Build(); + + app.UseMiddleware(); + app.UseMiddleware(); + + app.UseRouting(); + + app.UseMetricServer(); + app.UseHealthChecks("/health"); + + app.UseTelemetry(); + + if (webSettings.IsWebServerBehindProxy) + app.UseForwardedHeaders(); + + app.MapFallback(async context => + { + context.Response.StatusCode = 404; + + await context.Response.WriteRbxError(string.Empty); + }); + + var avatar = app.Services.GetRequiredService(); + var clientSettings = app.Services.GetRequiredService(); + var versionCompatibility = app.Services.GetRequiredService(); + + // Avatar routes + app.MapGet("/v1/avatar-fetch", avatar.GetAvatarFetch); + + // Client settings routes + app.MapGet("/v1/settings/application", clientSettings.GetApplicationSettings); + + // Version compatibility routes + app.MapGet("/GetAllowedMd5Hashes", versionCompatibility.GetAllowedMd5Hashes); + + // Start the web server + Task.Factory.StartNew(() => app.Run(webSettings.WebServerBindAddress), TaskCreationOptions.LongRunning); + } +} \ No newline at end of file diff --git a/services/grid-bot/lib/web/Grid.Bot.Web.csproj b/services/grid-bot/lib/web/Grid.Bot.Web.csproj new file mode 100644 index 00000000..67e52bad --- /dev/null +++ b/services/grid-bot/lib/web/Grid.Bot.Web.csproj @@ -0,0 +1,34 @@ + + + Library containing implementation for Web based code in grid-bot. Port of @mfdlabs/grid-service-websrv to C# + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/services/grid-bot/lib/web/Middleware/HttpServerConcurrentRequestsMiddleware.cs b/services/grid-bot/lib/web/Middleware/HttpServerConcurrentRequestsMiddleware.cs new file mode 100644 index 00000000..added308 --- /dev/null +++ b/services/grid-bot/lib/web/Middleware/HttpServerConcurrentRequestsMiddleware.cs @@ -0,0 +1,38 @@ +namespace Grid.Bot.Web.Middleware; + +using System; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.Http; + +using Prometheus; + +/// +/// Middleware for counting concurrent requests. +/// +public sealed class HttpServerConcurrentRequestsMiddleware +{ + private readonly RequestDelegate _next; + private readonly IGauge _ConcurentRequestsGauge = Metrics.CreateGauge("http_server_concurrent_requests_total", "The number of concurrent requests being processed by the server."); + + /// + /// Construct a new instance of + /// + /// The + /// cannot be null. + public HttpServerConcurrentRequestsMiddleware(RequestDelegate next) + { + _next = next ?? throw new ArgumentNullException(nameof(next)); + } + + /// + /// Invoke the middleware. + /// + /// The + /// An awaitable + public async Task Invoke(HttpContext context) + { + using (_ConcurentRequestsGauge.TrackInProgress()) + await _next(context); + } +} diff --git a/services/grid-bot/lib/web/Middleware/HttpServerMiddlewareBase.cs b/services/grid-bot/lib/web/Middleware/HttpServerMiddlewareBase.cs new file mode 100644 index 00000000..e8cf2471 --- /dev/null +++ b/services/grid-bot/lib/web/Middleware/HttpServerMiddlewareBase.cs @@ -0,0 +1,32 @@ +namespace Grid.Bot.Web.Middleware; + +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; + +/// +/// Base middleware for http servers. +/// +public abstract class HttpServerMiddlewareBase +{ + /// + /// Unknown route value. + /// + protected const string _UnknownRouteLabelValue = "Unknown"; + + /// + /// Get endpoint label value for metrics. + /// + /// The + /// The endpoint label value or + protected static (string controller, string action) GetControllerAndAction(HttpContext context) + { + var routeData = context.GetRouteData(); + var action = routeData?.Values["action"] as string ?? string.Empty; + var controller = routeData?.Values["controller"] as string ?? string.Empty; + + if (string.IsNullOrWhiteSpace(action) || string.IsNullOrWhiteSpace(controller)) + return (_UnknownRouteLabelValue, _UnknownRouteLabelValue); + + return (controller, action); + } +} diff --git a/services/grid-bot/lib/web/Middleware/HttpServerRequestCountMiddleware.cs b/services/grid-bot/lib/web/Middleware/HttpServerRequestCountMiddleware.cs new file mode 100644 index 00000000..c58b2b22 --- /dev/null +++ b/services/grid-bot/lib/web/Middleware/HttpServerRequestCountMiddleware.cs @@ -0,0 +1,50 @@ +namespace Grid.Bot.Web.Middleware; + +using System; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.Http; + +using Prometheus; + +/// +/// Middleware for counting http server requests. +/// +public sealed class HttpServerRequestCountMiddleware : HttpServerMiddlewareBase +{ + private readonly RequestDelegate _next; + private readonly Counter _HttpRequestCounter = Metrics.CreateCounter( + "http_server_requests_total", + "Total number of http requests", + "Method", + "Endpoint" + ); + + /// + /// Construct a new instance of + /// + /// The + /// cannot be null. + public HttpServerRequestCountMiddleware(RequestDelegate next) + { + _next = next ?? throw new ArgumentNullException(nameof(next)); + } + + /// + /// Invoke the middleware. + /// + /// The + /// An awaitable + public async Task Invoke(HttpContext context) + { + var (controller, action) = GetControllerAndAction(context); + var endpoint = controller != _UnknownRouteLabelValue && action != _UnknownRouteLabelValue ? + string.Format("{0}.{1}", controller, action) + : _UnknownRouteLabelValue; + + _HttpRequestCounter.WithLabels(context.Request.Method, endpoint).Inc(); + + await _next(context); + } + +} diff --git a/services/grid-bot/lib/web/Middleware/HttpServerRequestLoggingMiddleware.cs b/services/grid-bot/lib/web/Middleware/HttpServerRequestLoggingMiddleware.cs new file mode 100644 index 00000000..499b1b38 --- /dev/null +++ b/services/grid-bot/lib/web/Middleware/HttpServerRequestLoggingMiddleware.cs @@ -0,0 +1,65 @@ +namespace Grid.Bot.Web.Middleware; + +using System; +using System.Diagnostics; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.Http; + +using Logging; + +/// +/// Middleware for logging HTTP requests. +/// +public sealed class HttpServerRequestLoggingMiddleware : HttpServerMiddlewareBase +{ + private readonly RequestDelegate _next; + private readonly ILogger _logger; + + /// + /// Construct a new instance of + /// + /// The + /// The + /// + /// - is . + /// - is . + /// + public HttpServerRequestLoggingMiddleware(RequestDelegate next, ILogger logger) + { + _next = next ?? throw new ArgumentNullException(nameof(next)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + private static string GetEndPointLogEntry(HttpContext context) + => string.Format("\"{0} {1}\"", context.Request.Method, context.Request.Path); + + /// + /// Invoke the middleware. + /// + /// The + /// An awaitable + public async Task Invoke(HttpContext context) + { + var endpointLogEntry = GetEndPointLogEntry(context); + if (string.Compare(context.Request.Method, "GET", StringComparison.CurrentCultureIgnoreCase) == 0) + _logger.Debug("{0} request called: {1}", endpointLogEntry, context.Request.QueryString); + else + _logger.Debug("{0} request called", endpointLogEntry); + + var latency = Stopwatch.StartNew(); + + try + { + await _next(context); + + _logger.Debug("{0} responded in {1} ms", endpointLogEntry, latency.ElapsedMilliseconds); + } + catch (Exception ex) + { + _logger.Error("{0} failed in {1} ms: {2}", endpointLogEntry, latency.ElapsedMilliseconds, ex); + + throw; + } + } +} diff --git a/services/grid-bot/lib/web/Middleware/HttpServerResponseMiddleware.cs b/services/grid-bot/lib/web/Middleware/HttpServerResponseMiddleware.cs new file mode 100644 index 00000000..87a064ec --- /dev/null +++ b/services/grid-bot/lib/web/Middleware/HttpServerResponseMiddleware.cs @@ -0,0 +1,90 @@ +namespace Grid.Bot.Web.Middleware; + +using System; +using System.Diagnostics; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.Http; + +using Prometheus; + +/// +/// Middleware to handle the response +/// +/// +/// Middleware for counting and recording http response metrics. +/// +public sealed class HttpServerResponseMiddleware : HttpServerMiddlewareBase +{ + private readonly RequestDelegate _next; + + private readonly Counter _HttpResponseCounter = Metrics.CreateCounter( + "http_server_response_total", + "Total number of http responses", + "Method", + "Endpoint", + "StatusCode" + ); + private readonly Histogram _RequestDurationHistogram= Metrics.CreateHistogram( + "http_server_request_duration_seconds", + "Duration in seconds each request takes", + "Method", + "Endpoint" + ); + + /// + /// Initializes a new instance of the class. + /// + /// The . + /// + /// - cannot be null. + /// + public HttpServerResponseMiddleware(RequestDelegate next) + { + _next = next ?? throw new ArgumentNullException(nameof(next)); + } + + /// + /// Invokes the middleware + /// + /// The . + /// A . + /// cannot be null. + public async Task Invoke(HttpContext context) + { + ArgumentNullException.ThrowIfNull(context); + + var latencyStopwatch = Stopwatch.StartNew(); + var (controller, action) = GetControllerAndAction(context); + var endpoint = controller != _UnknownRouteLabelValue && action != _UnknownRouteLabelValue ? + string.Format("{0}.{1}", controller, action) + : _UnknownRouteLabelValue; + + try + { + await _next(context).ConfigureAwait(false); + + context.Response.OnCompleted(() => + { + latencyStopwatch.Stop(); + + var statusCode = context.Response.StatusCode; + + if (context.Response.StatusCode >= 200 && context.Response.StatusCode < 300) + _RequestDurationHistogram.WithLabels(context.Request.Method, endpoint).Observe(latencyStopwatch.Elapsed.TotalSeconds); + + _HttpResponseCounter.WithLabels(context.Request.Method, endpoint, context.Response.StatusCode.ToString()).Inc(); + + return Task.CompletedTask; + }); + } + catch (Exception) + { + _HttpResponseCounter.WithLabels(context.Request.Method, endpoint, context.Response.StatusCode.ToString()).Inc(); + + latencyStopwatch.Stop(); + + throw; + } + } +} diff --git a/services/grid-bot/lib/web/Middleware/UnhandledExceptionMiddleware.cs b/services/grid-bot/lib/web/Middleware/UnhandledExceptionMiddleware.cs new file mode 100644 index 00000000..313bce0a --- /dev/null +++ b/services/grid-bot/lib/web/Middleware/UnhandledExceptionMiddleware.cs @@ -0,0 +1,54 @@ +namespace Grid.Bot.Web.Middleware; + +using System; +using System.Net; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.Http; + +using Logging; + +/// +/// Middleware for logging unhandled exceptions and responding with . +/// +public class UnhandledExceptionMiddleware +{ + private readonly RequestDelegate _NextHandler; + private readonly ILogger _Logger; + + /// + /// Initializes a new . + /// + /// A delegate for triggering the next handler. + /// An . + /// + /// - + /// - + /// + public UnhandledExceptionMiddleware(RequestDelegate nextHandler, ILogger logger) + { + _NextHandler = nextHandler ?? throw new ArgumentNullException(nameof(nextHandler)); + _Logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + /// The method to invoke the handler. + /// + /// An . + public async Task Invoke(HttpContext context) + { + try + { + await _NextHandler(context); + } + catch (Exception ex) + { + _Logger.Error(ex); + + context.Response.StatusCode = (int)HttpStatusCode.InternalServerError; + + context.Response.ContentType = "text/plain"; + await context.Response.WriteAsync(ex.ToString()); + } + } +} diff --git a/services/grid-bot/lib/web/Routes/Avatar.cs b/services/grid-bot/lib/web/Routes/Avatar.cs new file mode 100644 index 00000000..c29e5d15 --- /dev/null +++ b/services/grid-bot/lib/web/Routes/Avatar.cs @@ -0,0 +1,131 @@ +namespace Grid.Bot.Web.Routes; + +using System; +using System.Net.Http; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.Http; + +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +using Logging; +using Utility; +using Extensions; + +using Threading.Extensions; + +/// +/// Routes for the avatar API. +/// +public class Avatar +{ + private const string _avatarFetchCacheKeyFormat = "avatar_fetch:{0}:{1}"; + + private const string _avatarFetchBodyColorsMapKey = "bodyColors"; + private const string _avatarFetchBodyColorsMapHeadColorKey = "headColorId"; + private const string _avatarFetchBodyColorsMapTorsoColorKey = "torsoColorId"; + private const string _avatarFetchBodyColorsMapRightArmColorKey = "rightArmColorId"; + private const string _avatarFetchBodyColorsMapLeftArmColorKey = "leftArmColorId"; + private const string _avatarFetchBodyColorsMapRightLegColorKey = "rightLegColorId"; + private const string _avatarFetchBodyColorsMapLeftLegColorKey = "leftLegColorId"; + + private const string _getAvatarFetchUserIdKey = "userId"; + private const string _getAvatarFetchPlaceIdKey = "placeId"; + private const string _getAvatarFetchUrlFormat = $"{{0}}/v1/avatar-fetch?{_getAvatarFetchUserIdKey}={{1}}&{_getAvatarFetchPlaceIdKey}={{2}}"; + + private readonly ILogger _logger; + private readonly AvatarSettings _settings; + private readonly IHttpClientFactory _httpClientFactory; + + private readonly ExpirableDictionary _avatarFetchCache; + + /// + /// Construct a new instance of + /// + /// The + /// The + /// The + public Avatar(ILogger logger, AvatarSettings settings, IHttpClientFactory httpClientFactory) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _settings = settings ?? throw new ArgumentNullException(nameof(settings)); + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + + _avatarFetchCache = new(_settings.AvatarFetchCacheEntryTtl, _settings.AvatarFetchCacheTraversalInterval); + } + + private static string ConstructAvatarCacheKey(long userId, long placeId) + => string.Format(_avatarFetchCacheKeyFormat, userId, placeId); + + private static dynamic DowngradeBodyColorsFormat(dynamic data) + { + var bodyColors = data[_avatarFetchBodyColorsMapKey]; + + var newBodyColors = new + { + HeadColor = bodyColors[_avatarFetchBodyColorsMapHeadColorKey], + TorsoColor = bodyColors[_avatarFetchBodyColorsMapTorsoColorKey], + RightArmColor = bodyColors[_avatarFetchBodyColorsMapRightArmColorKey], + LeftArmColor = bodyColors[_avatarFetchBodyColorsMapLeftArmColorKey], + RightLegColor = bodyColors[_avatarFetchBodyColorsMapRightLegColorKey], + LeftLegColor = bodyColors[_avatarFetchBodyColorsMapLeftLegColorKey] + }; + + data[_avatarFetchBodyColorsMapKey] = JObject.FromObject(newBodyColors); + + return data; + } + + + private dynamic GetAvatarFetchForUser(long userId, long placeId) + { + return _avatarFetchCache.GetOrAdd( + ConstructAvatarCacheKey(userId, placeId), + (key) => + { + _logger.Information("Cache miss for user {0} in place {1}", userId, placeId); + + using var httpClient = _httpClientFactory.CreateClient(); + var url = string.Format(_getAvatarFetchUrlFormat, _settings.AvatarApiUrl, userId, placeId); + + var response = httpClient.GetAsync(url).Sync(); + var data = response.Content.ReadAsStringAsync().Sync(); + + var dynData = JsonConvert.DeserializeObject(data); + + if (_settings.AvatarFetchShouldDowngradeBodyColorsFormat) + dynData = DowngradeBodyColorsFormat(dynData); + + return dynData; + }); + } + + /// + /// Fetch the avatar for a user in a place. + /// + /// The + public async Task GetAvatarFetch(HttpContext context) + { + if (!context.Request.TryParseInt64FromQuery(_getAvatarFetchUserIdKey, out var userId) || + !context.Request.TryParseInt64FromQuery(_getAvatarFetchPlaceIdKey, out var placeId)) + { + context.Response.StatusCode = 400; + await context.Response.WriteRbxError("Invalid user or place ID."); + + return; + } + + var avatarFetchData = GetAvatarFetchForUser(userId, placeId); + if (avatarFetchData == null) + { + context.Response.StatusCode = 404; + await context.Response.WriteRbxError("Avatar not found."); + + return; + } + + context.Response.ContentType = "application/json"; + await context.Response.WriteAsync(text: (string)JsonConvert.SerializeObject(avatarFetchData)); + } +} diff --git a/services/grid-bot/lib/web/Routes/ClientSettings.cs b/services/grid-bot/lib/web/Routes/ClientSettings.cs new file mode 100644 index 00000000..75719445 --- /dev/null +++ b/services/grid-bot/lib/web/Routes/ClientSettings.cs @@ -0,0 +1,81 @@ +namespace Grid.Bot.Web.Routes; + +using System; +using System.Linq; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.Http; + +using Logging; + +using Utility; +using Extensions; + +/// +/// Routes for the client settings API. +/// +public class ClientSettings +{ + private const string _applicationNameQueryParameter = "applicationName"; + private const string _invalidAppNameError = "The application name is invalid."; // also used for denied access. + + private readonly ClientSettingsSettings _settings; + private readonly ILogger _logger; + private readonly IClientSettingsFactory _clientSettingsFactory; + + /// + /// Construct a new instance of + /// + /// The + /// The + /// The + /// + /// - cannot be null. + /// - cannot be null. + /// - cannot be null. + /// + public ClientSettings(ClientSettingsSettings settings, ILogger logger, IClientSettingsFactory clientSettingsFactory) + { + _settings = settings ?? throw new ArgumentNullException(nameof(settings)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _clientSettingsFactory = clientSettingsFactory ?? throw new ArgumentNullException(nameof(clientSettingsFactory)); + } + + /// + /// Get application settings for the specified application. + /// + /// The + public async Task GetApplicationSettings(HttpContext context) + { + if (!context.Request.Query.TryGetValue(_applicationNameQueryParameter, out var applicationNameValues)) + { + context.Response.StatusCode = 400; + await context.Response.WriteRbxError(_invalidAppNameError); + + return; + } + + var applicationName = applicationNameValues.First(); + + if (!_settings.PermissibleReadApplications.Contains(applicationName) && !context.Request.HasValidApiKey(_settings)) + { + _logger.Warning("User {0} read attempt on non permissible read application ({1})", context.Connection.RemoteIpAddress, applicationName); + + context.Response.StatusCode = 403; + await context.Response.WriteRbxError(_invalidAppNameError); + + return; + } + + var applicationSettings = _clientSettingsFactory.GetSettingsForApplication(applicationName); + if (applicationSettings is null) + { + context.Response.StatusCode = 400; + await context.Response.WriteRbxError(_invalidAppNameError); + + return; + } + + await context.Response.WriteAsJsonAsync(new { applicationSettings }); + } +} diff --git a/services/grid-bot/lib/web/Routes/VersionCompatibility.cs b/services/grid-bot/lib/web/Routes/VersionCompatibility.cs new file mode 100644 index 00000000..8c149595 --- /dev/null +++ b/services/grid-bot/lib/web/Routes/VersionCompatibility.cs @@ -0,0 +1,19 @@ +namespace Grid.Bot.Web.Routes; + +using System; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.Http; + +/// +/// Routes for the versioncompatibility API. +/// +public class VersionCompatibility +{ + /// + /// Get allowed MD5 hashes, for now this responds with an empty response. + /// + /// The + public async Task GetAllowedMd5Hashes(HttpContext context) + => await context.Response.WriteAsJsonAsync(new { data = Array.Empty() }); +} diff --git a/services/grid-bot/src/Grid.Bot.csproj b/services/grid-bot/src/Grid.Bot.csproj index 372a91cb..3e3d0ee4 100755 --- a/services/grid-bot/src/Grid.Bot.csproj +++ b/services/grid-bot/src/Grid.Bot.csproj @@ -16,6 +16,8 @@ + + @@ -37,11 +39,10 @@ - - - - - + + + + diff --git a/services/grid-bot/src/Runner.cs b/services/grid-bot/src/Runner.cs index 0a90183b..3a4d5f9e 100755 --- a/services/grid-bot/src/Runner.cs +++ b/services/grid-bot/src/Runner.cs @@ -6,23 +6,21 @@ using System.Reflection; using System.Threading.Tasks; using System.Collections.Generic; -using System.Security.Authentication; using System.Runtime.InteropServices; -using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Builder; -using Microsoft.Extensions.Logging; -using Microsoft.AspNetCore.Server.Kestrel.Core; using Microsoft.Extensions.DependencyInjection; using Discord; +using Discord.Rest; using Discord.WebSocket; using Discord.Commands; using Discord.Interactions; +using Vault; using Redis; using Random; +using Logging; using Networking; using Configuration; using Text.Extensions; @@ -30,8 +28,9 @@ using Users.Client; using Thumbnails.Client; -using ClientSettings.Client; +using Web; +using Grpc; using Events; using Utility; using Prometheus; @@ -42,10 +41,6 @@ using LoggerFactory = Utility.LoggerFactory; using ILoggerFactory = Utility.ILoggerFactory; -using LogLevel = Logging.LogLevel; -using MELLogLevel = Microsoft.Extensions.Logging.LogLevel; -using Discord.Rest; - internal static class Runner { #if DEBUG @@ -166,13 +161,17 @@ private static ServiceProvider InitializeServices() var usersClient = new UsersClient(usersClientSettings.UsersApiBaseUrl); services.AddSingleton(usersClient); - var clientSettingsClientSettings = providers.FirstOrDefault(s => s.GetType() == typeof(ClientSettingsClientSettings)) as ClientSettingsClientSettings; - var clientSettingsClient = new ClientSettingsClient( - clientSettingsClientSettings.ClientSettingsApiBaseUrl, - clientSettingsClientSettings.ClientSettingsCertificateValidationEnabled + var webSettings = providers.FirstOrDefault(s => s.GetType() == typeof(WebSettings)) as WebSettings; + var clientSettingsSettings = providers.FirstOrDefault(s => s.GetType() == typeof(ClientSettingsSettings)) as ClientSettingsSettings; + var vaultClient = clientSettingsSettings.ClientSettingsViaVault + ? VaultClientFactory.Singleton.GetClient(clientSettingsSettings.ClientSettingsVaultAddress, clientSettingsSettings.ClientSettingsVaultToken) + : null; + var clientSettingsFactory = new ClientSettingsFactory( + vaultClient, + logger, + clientSettingsSettings ); - - services.AddSingleton(clientSettingsClient); + services.AddSingleton(clientSettingsFactory); var avatarSettings = providers.FirstOrDefault(s => s.GetType() == typeof(AvatarSettings)) as AvatarSettings; var thumbnailsClient = new ThumbnailsClient(avatarSettings.RbxThumbnailsUrl); @@ -217,11 +216,11 @@ private static ServiceProvider InitializeServices() private static IEnumerable GetSettingsProviders() { var assembly = Assembly.GetAssembly(typeof(BaseSettingsProvider)); - var ns = typeof(BaseSettingsProvider).Namespace; + var @namespace = typeof(BaseSettingsProvider).Namespace; var types = assembly .GetTypes() - .Where(t => string.Equals(t.Namespace, ns, StringComparison.Ordinal) && + .Where(t => string.Equals(t.Namespace, @namespace, StringComparison.Ordinal) && t.BaseType.Name == typeof(BaseSettingsProvider).Name) .ToList(); // finicky @@ -241,7 +240,7 @@ private static IEnumerable GetSettingsProviders() } var singleton = constructor.Invoke(null); - if (singleton == null || singleton is not IConfigurationProvider provider) + if (singleton is not IConfigurationProvider provider) { Console.Error.WriteLine("Provider {0} did not construct a singleton!", t.FullName); @@ -336,122 +335,6 @@ private static void SetupFloodCheckersRedis(ServiceCollection services, FloodChe services.AddSingleton(floodCheckerRegistry); } - private class MicrosoftLogger(ILogger logger) : Microsoft.Extensions.Logging.ILogger - { - private class NoopDisposable : IDisposable - { - public void Dispose() { } - } - - private readonly ILogger _logger = logger; - - public IDisposable BeginScope(TState state) => new NoopDisposable(); - - public bool IsEnabled(MELLogLevel logLevel) => true; - - public void Log(MELLogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) - { - var message = formatter(state, exception); - - switch (logLevel) - { - case MELLogLevel.Trace: - _logger.Verbose(message); - break; - case MELLogLevel.Debug: - _logger.Debug(message); - break; - case MELLogLevel.Information: - _logger.Information(message); - break; - case MELLogLevel.Warning: - _logger.Warning(message); - break; - case MELLogLevel.Error: - case MELLogLevel.Critical: - _logger.Error(message); - break; - default: - _logger.Warning(message); - break; - } - } - } - - private class MicrosoftLoggerProvider(ILogger logger) : ILoggerProvider - { - private readonly ILogger _logger = logger; - - public Microsoft.Extensions.Logging.ILogger CreateLogger(string categoryName) => new MicrosoftLogger(_logger); - - public void Dispose() { } - } - - private static void StartGrpcServer(IEnumerable args, IServiceProvider services) - { - var client = services.GetRequiredService(); - var globalSettings = services.GetRequiredService(); - var logger = new Logger( - name: globalSettings.GrpcServerLoggerName, - logLevelGetter: () => globalSettings.GrpcServerLoggerLevel, - logToConsole: true, - logToFileSystem: false - ); - - logger.Information("Starting gRPC server on {0}", globalSettings.GridBotGrpcServerEndpoint); - - var builder = WebApplication.CreateBuilder(args.ToArray()); - - builder.Logging.ClearProviders(); - builder.Logging.AddProvider(new MicrosoftLoggerProvider(logger)); - - builder.Services.AddSingleton(client); - - builder.Services.AddGrpc(); - - if (globalSettings.GrpcServerUseTls) - { - builder.Services.Configure(options => - { - options.ConfigureEndpointDefaults(listenOptions => - { - listenOptions.Protocols = HttpProtocols.Http2; - - try - { - listenOptions.UseHttps(globalSettings.GrpcServerCertificatePath, globalSettings.GrpcServerCertificatePassword, httpsOptions => - { - httpsOptions.SslProtocols = SslProtocols.Tls13 | SslProtocols.Tls12; - }); - } - catch (Exception ex) - { - logger.Warning("Failed to configure gRPC with HTTPS because: {0}. Will resort to insecure host instead!", ex.Message); - } - }); - }); - - // set urls - } - else - { - builder.Services.Configure(options => - { - options.ConfigureEndpointDefaults(listenOptions => - { - listenOptions.Protocols = HttpProtocols.Http2; - }); - }); - } - - var app = builder.Build(); - - app.MapGrpcService(); - - - app.Run(globalSettings.GridBotGrpcServerEndpoint); - } - private static async Task InvokeAsync(IEnumerable args) { @@ -491,7 +374,11 @@ private static async Task InvokeAsync(IEnumerable args) var discordSettings = services.GetRequiredService(); +#if DEBUG + if (discordSettings.BotToken.IsNullOrEmpty() && !discordSettings.DebugBotDisabled) +#else if (discordSettings.BotToken.IsNullOrEmpty()) +#endif { logger.Error(_noBotToken); @@ -500,7 +387,8 @@ private static async Task InvokeAsync(IEnumerable args) throw new InvalidOperationException(_noBotToken); } - Task.Factory.StartNew(() => StartGrpcServer(args, services), TaskCreationOptions.LongRunning); + services.UseGrpcServer(args); + services.UseWebServer(args); var client = services.GetRequiredService(); var interactions = services.GetRequiredService(); @@ -521,8 +409,13 @@ private static async Task InvokeAsync(IEnumerable args) port: globalSettings.MetricsPort ).Start(); - await client.LoginAsync(TokenType.Bot, discordSettings.BotToken).ConfigureAwait(false); - await client.StartAsync().ConfigureAwait(false); +#if DEBUG + if (!discordSettings.DebugBotDisabled) +#endif + { + await client.LoginAsync(TokenType.Bot, discordSettings.BotToken).ConfigureAwait(false); + await client.StartAsync().ConfigureAwait(false); + } await Task.Delay(Timeout.Infinite).ConfigureAwait(false); } diff --git a/services/recovery/src/Grid.Bot.Recovery.csproj b/services/recovery/src/Grid.Bot.Recovery.csproj index c33ace51..ac9b6979 100755 --- a/services/recovery/src/Grid.Bot.Recovery.csproj +++ b/services/recovery/src/Grid.Bot.Recovery.csproj @@ -29,9 +29,9 @@ - - - + + +