From 0ba597dcbc720a2a1ef57ec6cd2362d272f2a0a7 Mon Sep 17 00:00:00 2001 From: moe Date: Thu, 30 Apr 2026 14:13:46 -0300 Subject: [PATCH 1/4] Move microservice log level context to tooling common Move the ambient microservice log-level AsyncLocal into beamable.tooling.common so AdminRoutes and runtime logging share the same context without introducing a runtime dependency. Scope OpenAPI docs generation to temporarily raise the log threshold to Warning and restore the previous level afterward. Preserve the existing MicroserviceBootstrapper.ContextLogLevel API as an obsolete forwarder for compatibility. --- .../Microservice/AdminRoutes.cs | 32 +++++++++++++------ .../MicroserviceLogLevelContext.cs | 10 ++++++ .../Api/RealmConfig/RealmConfigService.cs | 2 +- .../dbmicroservice/BeamableMicroService.cs | 24 +++++++------- .../MicroserviceBootstrapper.cs | 3 +- .../dbmicroservice/MicroserviceStartupUtil.cs | 6 ++-- 6 files changed, 51 insertions(+), 26 deletions(-) create mode 100644 microservice/beamable.tooling.common/Microservice/MicroserviceLogLevelContext.cs diff --git a/microservice/beamable.tooling.common/Microservice/AdminRoutes.cs b/microservice/beamable.tooling.common/Microservice/AdminRoutes.cs index d110b983c9..238cda7033 100644 --- a/microservice/beamable.tooling.common/Microservice/AdminRoutes.cs +++ b/microservice/beamable.tooling.common/Microservice/AdminRoutes.cs @@ -70,7 +70,7 @@ public string HealthCheck() { return "responsive"; } - + /// /// Generates an OpenAPI/Swagger 3.0 document that describes the available service endpoints. /// @@ -84,19 +84,31 @@ public string HealthCheck() [CustomResponseSerializationAttribute] public string Docs() { - var docs = new ServiceDocGenerator(); - var ctx = GlobalProvider.GetService(); - var doc = docs.Generate(ctx, GlobalProvider); - - if (!string.IsNullOrEmpty(PublicHost)) + // Suppress info-level logs during OpenAPI doc generation to avoid noise + //Wrapping with try/finally to ensure log level is restored even if an exception occurs + var previousLogLevel = MicroserviceLogLevelContext.CurrentLogLevel.Value; + MicroserviceLogLevelContext.CurrentLogLevel.Value = LogLevel.Warning; + + try { - doc.Servers.Add(new OpenApiServer { Url = PublicHost }); - } + var docs = new ServiceDocGenerator(); + var ctx = GlobalProvider.GetService(); + var doc = docs.Generate(ctx, GlobalProvider); + + if (!string.IsNullOrEmpty(PublicHost)) + { + doc.Servers.Add(new OpenApiServer { Url = PublicHost }); + } - var outputString = doc.Serialize(OpenApiSpecVersion.OpenApi3_0, OpenApiFormat.Json); + var outputString = doc.Serialize(OpenApiSpecVersion.OpenApi3_0, OpenApiFormat.Json); - return outputString; + return outputString; + } + finally + { + MicroserviceLogLevelContext.CurrentLogLevel.Value = previousLogLevel; + } } /// diff --git a/microservice/beamable.tooling.common/Microservice/MicroserviceLogLevelContext.cs b/microservice/beamable.tooling.common/Microservice/MicroserviceLogLevelContext.cs new file mode 100644 index 0000000000..92b77f6a66 --- /dev/null +++ b/microservice/beamable.tooling.common/Microservice/MicroserviceLogLevelContext.cs @@ -0,0 +1,10 @@ +using Microsoft.Extensions.Logging; + +namespace Beamable.Server +{ + public static class MicroserviceLogLevelContext + { + public static readonly AsyncLocal CurrentLogLevel = new(); + } + +} diff --git a/microservice/microservice/Api/RealmConfig/RealmConfigService.cs b/microservice/microservice/Api/RealmConfig/RealmConfigService.cs index c56f0b7cff..6bd7f6d324 100644 --- a/microservice/microservice/Api/RealmConfig/RealmConfigService.cs +++ b/microservice/microservice/Api/RealmConfig/RealmConfigService.cs @@ -40,7 +40,7 @@ public RealmConfigService(IBeamableRequester requester, SocketRequesterContext c public void UpdateLogLevel() { var level = GetLogLevel(); - MicroserviceBootstrapper.ContextLogLevel.Value = level; + MicroserviceLogLevelContext.CurrentLogLevel.Value = level; } private LogLevel GetLogLevel() diff --git a/microservice/microservice/dbmicroservice/BeamableMicroService.cs b/microservice/microservice/dbmicroservice/BeamableMicroService.cs index 7e9a94ba65..59d6952b0b 100644 --- a/microservice/microservice/dbmicroservice/BeamableMicroService.cs +++ b/microservice/microservice/dbmicroservice/BeamableMicroService.cs @@ -827,16 +827,18 @@ async Task HandleWebsocketMessage(IConnection ws, JsonDocument document, Stopwat // First get the Global Realm Config Log Level and apply it by running UpdateLogLevel var configService = InstanceArgs.ServiceScope.GetService(); - if (ctx.Path?.StartsWith(_adminPrefix) ?? false) - { - // when the path starts with admin, use warning. - MicroserviceBootstrapper.ContextLogLevel.Value = LogLevel.Warning; - } - else - { - // otherwise, allow default behaviour. - configService.UpdateLogLevel(); - } + // if (ctx.Path?.StartsWith(_adminPrefix) ?? false) + // { + // // when the path starts with admin, use warning. + // MicroserviceBootstrapper.ContextLogLevel.Value = LogLevel.Warning; + // } + // else + // { + // // otherwise, allow default behaviour. + // configService.UpdateLogLevel(); + // } + + configService.UpdateLogLevel(); string routingKey = InstanceArgs.GetRoutingKey().GetOrElse(string.Empty); try @@ -901,7 +903,7 @@ async Task HandleWebsocketMessage(IConnection ws, JsonDocument document, Stopwat } - MicroserviceBootstrapper.ContextLogLevel.Value = contextLogLevel; + MicroserviceLogLevelContext.CurrentLogLevel.Value = contextLogLevel; void TryToSetNewLogLevel(LogLevel newLogLevel) { diff --git a/microservice/microservice/dbmicroservice/MicroserviceBootstrapper.cs b/microservice/microservice/dbmicroservice/MicroserviceBootstrapper.cs index c57424af3e..0f4eec4006 100644 --- a/microservice/microservice/dbmicroservice/MicroserviceBootstrapper.cs +++ b/microservice/microservice/dbmicroservice/MicroserviceBootstrapper.cs @@ -17,7 +17,8 @@ public class SingleUseUserContext : IUserContext public static partial class MicroserviceBootstrapper { - public static AsyncLocal ContextLogLevel = new(); + [Obsolete("Use MicroserviceLogLevelContext.CurrentLogLevel instead.")] + public static AsyncLocal ContextLogLevel => MicroserviceLogLevelContext.CurrentLogLevel; private static BeamServiceConfigBuilder _preparedBuilder; diff --git a/microservice/microservice/dbmicroservice/MicroserviceStartupUtil.cs b/microservice/microservice/dbmicroservice/MicroserviceStartupUtil.cs index ff46621505..6f8094405f 100644 --- a/microservice/microservice/dbmicroservice/MicroserviceStartupUtil.cs +++ b/microservice/microservice/dbmicroservice/MicroserviceStartupUtil.cs @@ -102,7 +102,7 @@ public static async Task Begin(IBeamServiceConfig configurat if (startupCtx.IsGeneratingOapi) { LogUtil.TryParseSystemLogLevel(configuredArgs.OapiGenLogLevel, out var defaultLevel, LogLevel.Information); - MicroserviceBootstrapper.ContextLogLevel.Value = defaultLevel; + MicroserviceLogLevelContext.CurrentLogLevel.Value = defaultLevel; await GenerateOpenApiSpecification(startupCtx, configurator); startupCtx.result.GeneratedClient = true; return startupCtx.result; @@ -380,7 +380,7 @@ private static void ConfigureLogging(IBeamServiceConfig configurator, StartupCon defaultLogLevel = LogLevel.Warning; } - MicroserviceBootstrapper.ContextLogLevel.Value = defaultLogLevel; + MicroserviceLogLevelContext.CurrentLogLevel.Value = defaultLogLevel; var debugLogOptions = UseBeamJsonFormatter(new ZLoggerOptions()); ctx.debugLogProcessor = new DebugLogProcessor(debugLogOptions); @@ -392,7 +392,7 @@ private static void ConfigureLogging(IBeamServiceConfig configurator, StartupCon // all logs are valid, but may not pass the filter. builder.SetMinimumLevel(LogLevel.Trace); - builder.AddFilter(level => level >= MicroserviceBootstrapper.ContextLogLevel.Value); + builder.AddFilter(level => level >= MicroserviceLogLevelContext.CurrentLogLevel.Value); if (!ctx.InDocker) { builder.AddZLoggerLogProcessor(ctx.debugLogProcessor); From 5958ad254b5b2f8af7aa12606294696e7a51b932 Mon Sep 17 00:00:00 2001 From: moe Date: Mon, 4 May 2026 14:27:52 -0300 Subject: [PATCH 2/4] - Add configurable admin route log policy --- .../Microservice/AdminRoutes.cs | 32 +-- .../Microservice/IMicroserviceArgs.cs | 7 + .../Microservice/MicroserviceArgs.cs | 15 ++ .../dbmicroservice/BeamableMicroService.cs | 110 +++++---- .../OpenAPITests/DocTests.cs | 107 +++++++-- .../microservice/TestArgs.cs | 30 +-- .../AdminRouteLogPolicyTests.cs | 226 ++++++++++++++++++ 7 files changed, 437 insertions(+), 90 deletions(-) create mode 100644 microservice/microserviceTests/microservice/dbmicroservice/BeamableMicroServiceTests/AdminRouteLogPolicyTests.cs diff --git a/microservice/beamable.tooling.common/Microservice/AdminRoutes.cs b/microservice/beamable.tooling.common/Microservice/AdminRoutes.cs index 238cda7033..73a44d0f31 100644 --- a/microservice/beamable.tooling.common/Microservice/AdminRoutes.cs +++ b/microservice/beamable.tooling.common/Microservice/AdminRoutes.cs @@ -84,16 +84,16 @@ public string HealthCheck() [CustomResponseSerializationAttribute] public string Docs() { - // Suppress info-level logs during OpenAPI doc generation to avoid noise - //Wrapping with try/finally to ensure log level is restored even if an exception occurs - var previousLogLevel = MicroserviceLogLevelContext.CurrentLogLevel.Value; - MicroserviceLogLevelContext.CurrentLogLevel.Value = LogLevel.Warning; - - try - { - var docs = new ServiceDocGenerator(); - var ctx = GlobalProvider.GetService(); - var doc = docs.Generate(ctx, GlobalProvider); + // Suppress info-level logs during OpenAPI doc generation to avoid noise + //Wrapping with try/finally to ensure log level is restored even if an exception occurs + var previousLogLevel = MicroserviceLogLevelContext.CurrentLogLevel.Value; + MicroserviceLogLevelContext.CurrentLogLevel.Value = LogLevel.Warning; + + try + { + var docs = new ServiceDocGenerator(); + var ctx = GlobalProvider.GetService(); + var doc = docs.Generate(ctx, GlobalProvider); if (!string.IsNullOrEmpty(PublicHost)) { @@ -104,12 +104,12 @@ public string Docs() return outputString; - } - finally - { - MicroserviceLogLevelContext.CurrentLogLevel.Value = previousLogLevel; - } - } + } + finally + { + MicroserviceLogLevelContext.CurrentLogLevel.Value = previousLogLevel; + } + } /// /// Fetch various Beamable SDK metadata for the Microservice diff --git a/microservice/beamable.tooling.common/Microservice/IMicroserviceArgs.cs b/microservice/beamable.tooling.common/Microservice/IMicroserviceArgs.cs index 412485fac6..fca1c7a5ec 100644 --- a/microservice/beamable.tooling.common/Microservice/IMicroserviceArgs.cs +++ b/microservice/beamable.tooling.common/Microservice/IMicroserviceArgs.cs @@ -8,6 +8,12 @@ public enum LogOutputType DEFAULT, STRUCTURED, UNSTRUCTURED, FILE, STRUCTURED_AND_FILE } +public enum AdminRouteLogPolicy +{ + Protected = 0, //Default where Admin logs are restricted to warning and above + FollowServiceRules = 1 //Admin logs follow service rules +} + public interface IMicroserviceArgs : IRealmInfo, IActivityProviderArgs { public IDependencyProviderScope ServiceScope { get; } @@ -59,4 +65,5 @@ public interface IMicroserviceArgs : IRealmInfo, IActivityProviderArgs public bool AllowStartupWithoutBeamableSettings { get; } public int MaxUniqueEventBindingCount { get; } void SetResolvedCid(string resolvedCid); + public AdminRouteLogPolicy AdminRouteLogPolicy { get; } } diff --git a/microservice/beamable.tooling.common/Microservice/MicroserviceArgs.cs b/microservice/beamable.tooling.common/Microservice/MicroserviceArgs.cs index d26da88847..48840fb7e3 100644 --- a/microservice/beamable.tooling.common/Microservice/MicroserviceArgs.cs +++ b/microservice/beamable.tooling.common/Microservice/MicroserviceArgs.cs @@ -68,6 +68,8 @@ public void SetResolvedCid(string resolvedCid) { throw new NotImplementedException(); } + + public AdminRouteLogPolicy AdminRouteLogPolicy { get; set; } } public static class MicroserviceArgsExtensions @@ -126,6 +128,7 @@ public static MicroserviceArgs Copy(this IMicroserviceArgs args, Action(rawPolicy, true, out var policy) ? + policy : + AdminRouteLogPolicy.Protected; + } + } + public string Host => Environment.GetEnvironmentVariable("HOST"); public string Secret => Environment.GetEnvironmentVariable("SECRET"); public string NamePrefix => Environment.GetEnvironmentVariable("NAME_PREFIX") ?? ""; diff --git a/microservice/microservice/dbmicroservice/BeamableMicroService.cs b/microservice/microservice/dbmicroservice/BeamableMicroService.cs index 59d6952b0b..a314c34963 100644 --- a/microservice/microservice/dbmicroservice/BeamableMicroService.cs +++ b/microservice/microservice/dbmicroservice/BeamableMicroService.cs @@ -794,52 +794,16 @@ async Task HandleClientMessage(MicroserviceRequestContext ctx, Stopwatch sw, Bea } } - async Task HandleWebsocketMessage(IConnection ws, JsonDocument document, Stopwatch sw) + /// + /// Applies logging context rules based on the provided request context. + /// This method utilizes the microservice's routing key and logging context service + /// to determine the appropriate log level and any additional logging rules to be applied. + /// It ensures that logging behavior is dynamically adjusted to reflect the specified configuration. + /// + /// The context of the current request, providing necessary information + /// for logging configuration, such as routing keys and associated metadata. + private void ApplyLoggingContextRules(RequestContext ctx) { - - MicroserviceRequestContext ctx = null; - using (var requestActivity = _activityProvider.Create(Otel.TRACE_CONSTRUCT_CTX, importance: TelemetryImportance.VERBOSE)) - { - if (!document.TryBuildRequestContext(InstanceArgs, out ctx)) - { - requestActivity.SetStatus(ActivityStatusCode.Error); - Log.Debug("WS Message contains no data. Cannot handle. Skipping message."); - return; - } - requestActivity.SetStatus(ActivityStatusCode.Ok); - } - - BeamActivity parentActivity = BeamActivity.Noop; - if (_socketRequesterContext.TryGetListener(ctx.Id, out var existingListener)) - { - parentActivity = existingListener.Activity; - } - - - using var activity = _activityProvider.Create(Otel.TRACE_WS, parentActivity); - MicroserviceRequester.ContextActivity.Value = activity; - ctx.ActivityContext = activity; - - - var extraOtelTags = _telemetryProviders.CreateRequestAttributes(InstanceArgs, ctx, ConnectionId); - activity.SetTags(extraOtelTags); - - - // First get the Global Realm Config Log Level and apply it by running UpdateLogLevel - var configService = InstanceArgs.ServiceScope.GetService(); - // if (ctx.Path?.StartsWith(_adminPrefix) ?? false) - // { - // // when the path starts with admin, use warning. - // MicroserviceBootstrapper.ContextLogLevel.Value = LogLevel.Warning; - // } - // else - // { - // // otherwise, allow default behaviour. - // configService.UpdateLogLevel(); - // } - - configService.UpdateLogLevel(); - string routingKey = InstanceArgs.GetRoutingKey().GetOrElse(string.Empty); try { @@ -920,7 +884,63 @@ void TryToSetNewLogLevel(LogLevel newLogLevel) { BeamableZLoggerProvider.Instance.Error(ex); } + } + + /// + /// Adjusts the log level for the current request context based on specific policy criteria. + /// Determines whether the request is for a protected admin route and sets the log level accordingly. + /// If the route is not protected, applies custom logging rules and updates the log level + /// using the service configuration. + /// + /// The request context object containing information about the current request, + /// including the request path. + private void ApplyRequestLogLevel(RequestContext ctx) + { + var isProtectedAdminRoute = ctx.Path?.StartsWith(_adminPrefix) == true + && InstanceArgs.AdminRouteLogPolicy == AdminRouteLogPolicy.Protected; + + if (isProtectedAdminRoute) + { + MicroserviceLogLevelContext.CurrentLogLevel.Value = LogLevel.Warning; + } + else + { + var configureService = InstanceArgs.ServiceScope.GetService(); + configureService.UpdateLogLevel(); + ApplyLoggingContextRules(ctx); + } + } + + async Task HandleWebsocketMessage(IConnection ws, JsonDocument document, Stopwatch sw) + { + MicroserviceRequestContext ctx = null; + using (var requestActivity = _activityProvider.Create(Otel.TRACE_CONSTRUCT_CTX, importance: TelemetryImportance.VERBOSE)) + { + if (!document.TryBuildRequestContext(InstanceArgs, out ctx)) + { + requestActivity.SetStatus(ActivityStatusCode.Error); + Log.Debug("WS Message contains no data. Cannot handle. Skipping message."); + return; + } + requestActivity.SetStatus(ActivityStatusCode.Ok); + } + + BeamActivity parentActivity = BeamActivity.Noop; + if (_socketRequesterContext.TryGetListener(ctx.Id, out var existingListener)) + { + parentActivity = existingListener.Activity; + } + + + using var activity = _activityProvider.Create(Otel.TRACE_WS, parentActivity); + MicroserviceRequester.ContextActivity.Value = activity; + ctx.ActivityContext = activity; + + + var extraOtelTags = _telemetryProviders.CreateRequestAttributes(InstanceArgs, ctx, ConnectionId); + activity.SetTags(extraOtelTags); + ApplyRequestLogLevel(ctx); var logger = BeamableZLoggerProvider.LogContext.Value = InstanceArgs.ServiceScope.GetLogger(); using var scope = logger.BeginScope(extraOtelTags.ToDictionary()); diff --git a/microservice/microserviceTests/OpenAPITests/DocTests.cs b/microservice/microserviceTests/OpenAPITests/DocTests.cs index d20259b309..bbbc17c44d 100644 --- a/microservice/microserviceTests/OpenAPITests/DocTests.cs +++ b/microservice/microserviceTests/OpenAPITests/DocTests.cs @@ -11,17 +11,19 @@ using NUnit.Framework; using System; using System.Collections.Generic; -using System.Threading.Tasks; -using Beamable.Common.Dependencies; -using microserviceTests.microservice.Util; +using System.Threading.Tasks; +using Beamable.Common.Dependencies; +using microserviceTests.microservice.Util; +using microservice.Common; +using Microsoft.Extensions.Logging; namespace microserviceTests.OpenAPITests; [Serializable] public class DocTests { - public class ExampleAttrs : ITelemetryAttributeProvider - { + public class ExampleAttrs : ITelemetryAttributeProvider + { public List GetDescriptors() { return new List @@ -47,11 +49,34 @@ public void CreateConnectionAttributes(IConnectionAttributeContext ctx) public void CreateRequestAttributes(IRequestAttributeContext ctx) { - } - } - - [Test] - public void TestMethodScanning() + } + } + + public class RecordingTelemetryAttributeProvider : ITelemetryAttributeProvider + { + public static readonly List SeenLogLevels = new List(); + + public List GetDescriptors() + { + SeenLogLevels.Add(MicroserviceLogLevelContext.CurrentLogLevel.Value); + return new List(); + } + + public void CreateDefaultAttributes(IDefaultAttributeContext ctx) + { + } + + public void CreateConnectionAttributes(IConnectionAttributeContext ctx) + { + } + + public void CreateRequestAttributes(IRequestAttributeContext ctx) + { + } + } + + [Test] + public void TestMethodScanning() { LoggingUtil.InitTestCorrelator(); var gen = new ServiceDocGenerator(); @@ -72,11 +97,63 @@ public void TestMethodScanning() // Assert.AreEqual(1, reqSchema.Required.Count); var outputString = doc.Serialize(OpenApiSpecVersion.OpenApi3_0, OpenApiFormat.Json); - Console.WriteLine(outputString); - } - - [Microservice("docs")] - public class DocService : Microservice + Console.WriteLine(outputString); + } + + [Test] + public void AdminRoutesDocsTemporarilyRaisesLogLevelAndRestoresPreviousValue() + { + LoggingUtil.InitTestCorrelator(); + RecordingTelemetryAttributeProvider.SeenLogLevels.Clear(); + + var originalLogLevel = MicroserviceLogLevelContext.CurrentLogLevel.Value; + var previousLogLevel = LogLevel.Debug; + MicroserviceLogLevelContext.CurrentLogLevel.Value = previousLogLevel; + + try + { + var args = new TestArgs(); + var microserviceAttribute = new MicroserviceAttribute("docs"); + var startupContext = new StartupContext + { + args = args, + attributes = microserviceAttribute, + routeSources = new[] + { + new BeamRouteSource + { + InstanceType = typeof(DocService) + } + } + }; + + var builder = new DependencyBuilder(); + builder.AddSingleton(args); + builder.AddSingleton(startupContext); + builder.AddSingleton>(); + builder.AddSingleton(); + + var routes = new AdminRoutes + { + GlobalProvider = builder.Build(), + MicroserviceAttribute = microserviceAttribute + }; + + var outputString = routes.Docs(); + + Assert.That(outputString, Does.Contain("\"openapi\"")); + Assert.That(RecordingTelemetryAttributeProvider.SeenLogLevels, Is.Not.Empty); + Assert.That(RecordingTelemetryAttributeProvider.SeenLogLevels, Has.All.EqualTo(LogLevel.Warning)); + Assert.AreEqual(previousLogLevel, MicroserviceLogLevelContext.CurrentLogLevel.Value); + } + finally + { + MicroserviceLogLevelContext.CurrentLogLevel.Value = originalLogLevel; + } + } + + [Microservice("docs")] + public class DocService : Microservice { [ServerCallable] diff --git a/microservice/microserviceTests/microservice/TestArgs.cs b/microservice/microserviceTests/microservice/TestArgs.cs index 926a41ae95..05aa74aaee 100644 --- a/microservice/microserviceTests/microservice/TestArgs.cs +++ b/microservice/microserviceTests/microservice/TestArgs.cs @@ -29,9 +29,9 @@ public TestSetup(IConnectionProvider provider, IContentResolver resolver=null) _provider = provider; } - public async Task Start(TestArgs dudArgs=null, Action configurator=null) where T : Microservice - { - var args = new TestArgs(); + public async Task Start(TestArgs dudArgs=null, Action configurator=null) where T : Microservice + { + var args = dudArgs ?? new TestArgs(); var attr = typeof(T).GetCustomAttribute(); @@ -171,14 +171,16 @@ public class TestArgs : IMicroserviceArgs public bool OtelExporterShouldRetry { get; } public bool OtelExporterStandardEnabled => false; public string OtelExporterRetryMaxSize { get; } - public bool AllowStartupWithoutBeamableSettings => false; - public int MaxUniqueEventBindingCount => 100; - public bool SkipLocalEnv => true; - public bool SkipAliasResolve => true; - - public void SetResolvedCid(string resolvedCid) - { - throw new NotImplementedException(); - } - } -} + public bool AllowStartupWithoutBeamableSettings => false; + public int MaxUniqueEventBindingCount => 100; + public bool SkipLocalEnv => true; + public bool SkipAliasResolve => true; + public AdminRouteLogPolicy AdminRouteLogPolicy { get; set; } = AdminRouteLogPolicy.Protected; + + public void SetResolvedCid(string resolvedCid) + { + throw new NotImplementedException(); + } + + } +} diff --git a/microservice/microserviceTests/microservice/dbmicroservice/BeamableMicroServiceTests/AdminRouteLogPolicyTests.cs b/microservice/microserviceTests/microservice/dbmicroservice/BeamableMicroServiceTests/AdminRouteLogPolicyTests.cs new file mode 100644 index 0000000000..9b61fcc58e --- /dev/null +++ b/microservice/microserviceTests/microservice/dbmicroservice/BeamableMicroServiceTests/AdminRouteLogPolicyTests.cs @@ -0,0 +1,226 @@ +using Beamable.Api.Autogenerated.Models; +using Beamable.Common; +using Beamable.Common.Dependencies; +using Beamable.Microservice.Tests.Socket; +using Beamable.Server; +using Beamable.Server.Api.Logs; +using Beamable.Server.Api.RealmConfig; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using ClientRequest = Beamable.Microservice.Tests.Socket.ClientRequest; + +namespace microserviceTests.microservice.dbmicroservice.BeamableMicroServiceTests; + +[TestFixture] +public class AdminRouteLogPolicyTests : CommonTest +{ + private const string AdminRouteLogPolicyEnvVar = "BEAM_ADMIN_ROUTE_LOG_POLICY"; + + [TestCase(null, AdminRouteLogPolicy.Protected)] + [TestCase("", AdminRouteLogPolicy.Protected)] + [TestCase("Protected", AdminRouteLogPolicy.Protected)] + [TestCase("FollowServiceRules", AdminRouteLogPolicy.FollowServiceRules)] + [TestCase("followservicerules", AdminRouteLogPolicy.FollowServiceRules)] + [TestCase("NotARealPolicy", AdminRouteLogPolicy.Protected)] + [NonParallelizable] + public void EnvironmentArgsParsesAdminRouteLogPolicy(string rawPolicy, AdminRouteLogPolicy expectedPolicy) + { + var previousPolicy = Environment.GetEnvironmentVariable(AdminRouteLogPolicyEnvVar); + try + { + Environment.SetEnvironmentVariable(AdminRouteLogPolicyEnvVar, rawPolicy); + var args = new EnvironmentArgs(); + + Assert.AreEqual(expectedPolicy, args.AdminRouteLogPolicy); + } + finally + { + Environment.SetEnvironmentVariable(AdminRouteLogPolicyEnvVar, previousPolicy); + } + } + + [Test] + [NonParallelizable] + public async Task ProtectedAdminRouteDoesNotApplyServiceLogRules() + { + var realmConfig = new RecordingRealmConfigService(LogLevel.Debug); + var loggingContext = new RecordingLoggingContextService(); + TestSocket testSocket = null; + + var ms = new TestSetup(new TestSocketProvider(socket => + { + testSocket = socket; + socket + .AddStandardMessageHandlers() + .AddMessageHandler( + MessageMatcher + .WithReqId(1) + .WithStatus(200) + .WithPayload(payload => payload == "responsive"), + MessageResponder.NoResponse(), + MessageFrequency.OnlyOnce()); + })); + + await ms.Start(new TestArgs + { + AdminRouteLogPolicy = AdminRouteLogPolicy.Protected + }, builder => RegisterLogServices(builder, realmConfig, loggingContext)); + + Assert.IsTrue(ms.HasInitialized); + var updateLogLevelCount = realmConfig.UpdateLogLevelCount; + var getLogLevelContextCount = loggingContext.GetLogLevelContextCount; + + testSocket.SendToClient(ClientRequest.Callable("micro_log_policy", "admin/HealthCheck", 1)); + await Task.Delay(50); + + Assert.AreEqual(updateLogLevelCount, realmConfig.UpdateLogLevelCount); + Assert.AreEqual(getLogLevelContextCount, loggingContext.GetLogLevelContextCount); + + await ms.OnShutdown(this, null); + Assert.IsTrue(testSocket.AllMocksCalled()); + } + + [Test] + [NonParallelizable] + public async Task FollowServiceRulesAdminRouteAppliesServiceLogRules() + { + var realmConfig = new RecordingRealmConfigService(LogLevel.Debug); + var loggingContext = new RecordingLoggingContextService(); + TestSocket testSocket = null; + + var ms = new TestSetup(new TestSocketProvider(socket => + { + testSocket = socket; + socket + .AddStandardMessageHandlers() + .AddMessageHandler( + MessageMatcher + .WithReqId(1) + .WithStatus(200) + .WithPayload(payload => payload == "responsive"), + MessageResponder.NoResponse(), + MessageFrequency.OnlyOnce()); + })); + + await ms.Start(new TestArgs + { + AdminRouteLogPolicy = AdminRouteLogPolicy.FollowServiceRules + }, builder => RegisterLogServices(builder, realmConfig, loggingContext)); + + Assert.IsTrue(ms.HasInitialized); + var updateLogLevelCount = realmConfig.UpdateLogLevelCount; + var getLogLevelContextCount = loggingContext.GetLogLevelContextCount; + + testSocket.SendToClient(ClientRequest.Callable("micro_log_policy", "admin/HealthCheck", 1)); + await Task.Delay(50); + + Assert.AreEqual(updateLogLevelCount + 1, realmConfig.UpdateLogLevelCount); + Assert.AreEqual(getLogLevelContextCount + 1, loggingContext.GetLogLevelContextCount); + + await ms.OnShutdown(this, null); + Assert.IsTrue(testSocket.AllMocksCalled()); + } + + [Test] + [NonParallelizable] + public async Task ProtectedPolicyStillAppliesServiceLogRulesForNonAdminRoute() + { + var realmConfig = new RecordingRealmConfigService(LogLevel.Debug); + var loggingContext = new RecordingLoggingContextService(); + TestSocket testSocket = null; + + var ms = new TestSetup(new TestSocketProvider(socket => + { + testSocket = socket; + socket + .AddStandardMessageHandlers() + .AddMessageHandler( + MessageMatcher + .WithReqId(1) + .WithStatus(200) + .WithPayload(payload => payload == "ok"), + MessageResponder.NoResponse(), + MessageFrequency.OnlyOnce()); + })); + + await ms.Start(new TestArgs + { + AdminRouteLogPolicy = AdminRouteLogPolicy.Protected + }, builder => RegisterLogServices(builder, realmConfig, loggingContext)); + + Assert.IsTrue(ms.HasInitialized); + var updateLogLevelCount = realmConfig.UpdateLogLevelCount; + var getLogLevelContextCount = loggingContext.GetLogLevelContextCount; + + testSocket.SendToClient(ClientRequest.ClientCallable("micro_log_policy", nameof(LogPolicyService.Ping), 1, 1)); + await Task.Delay(50); + + Assert.AreEqual(updateLogLevelCount + 1, realmConfig.UpdateLogLevelCount); + Assert.AreEqual(getLogLevelContextCount + 1, loggingContext.GetLogLevelContextCount); + + await ms.OnShutdown(this, null); + Assert.IsTrue(testSocket.AllMocksCalled()); + } + + private static void RegisterLogServices(IDependencyBuilder builder, RecordingRealmConfigService realmConfig, RecordingLoggingContextService loggingContext) + { + builder.RemoveIfExists(); + builder.RemoveIfExists(); + builder.RemoveIfExists(); + builder.AddSingleton(realmConfig); + builder.AddSingleton(realmConfig); + builder.AddSingleton(loggingContext); + } + + private class RecordingRealmConfigService : IRealmConfigService + { + private readonly LogLevel _level; + + public RecordingRealmConfigService(LogLevel level) + { + _level = level; + } + + public int UpdateLogLevelCount { get; private set; } + + public Promise GetRealmConfigSettings() + { + return Promise.Successful(new RealmConfig(new Dictionary())); + } + + public void UpdateLogLevel() + { + UpdateLogLevelCount++; + MicroserviceLogLevelContext.CurrentLogLevel.Value = _level; + } + } + + private class RecordingLoggingContextService : ILoggingContextService + { + public int GetLogLevelContextCount { get; private set; } + + public Promise> GetAllLoggingContexts() + { + return Promise>.Successful(new Dictionary()); + } + + public BeamoV2ServiceLoggingContext GetLogLevelContext(string serviceName, string routingKey) + { + GetLogLevelContextCount++; + return null; + } + } + + [Microservice("log_policy", EnableEagerContentLoading = false)] + public class LogPolicyService : Microservice + { + [ClientCallable] + public string Ping() + { + return "ok"; + } + } +} From e65fcd039949560479ea7340ac64f0ff4101f0ac Mon Sep 17 00:00:00 2001 From: moe Date: Mon, 18 May 2026 14:38:09 -0300 Subject: [PATCH 3/4] - beam content window progress bar - SSL error fix during content sync --- .../Commands/Content/ContentSyncCommand.cs | 10 +- cli/cli/Services/Content/ContentService.cs | 102 ++++++++++++++++-- .../BeamCli/Commands/BeamContentSync.cs | 7 ++ .../ContentService/CliContentService.cs | 101 +++++++++++++++-- .../Editor/UI/ContentWindow/ContentWindow.cs | 9 +- .../ContentWindow_ContentActions.cs | 55 ++++++++-- .../UI/ContentWindow/ContentWindow_Header.cs | 17 +-- .../Modules/Content/ContentConfiguration.cs | 9 +- 8 files changed, 274 insertions(+), 36 deletions(-) diff --git a/cli/cli/Commands/Content/ContentSyncCommand.cs b/cli/cli/Commands/Content/ContentSyncCommand.cs index 9b1c16879a..6843b6dd67 100644 --- a/cli/cli/Commands/Content/ContentSyncCommand.cs +++ b/cli/cli/Commands/Content/ContentSyncCommand.cs @@ -22,6 +22,11 @@ public class ContentSyncCommand : AtomicCommand DOWNLOAD_MAX_PARALLEL_COUNT_OPTION = new( + "--download-max-parallel-count", + () => 64, + "Maximum number of content files to download in parallel while syncing. Use 0 for unbounded parallelism."); + private ContentService _contentService; public ContentSyncCommand() : base("sync", "Synchronizes the local content matching the filters to the latest content stored in the realm") @@ -38,6 +43,7 @@ public override void Configure() AddOption(SYNC_CONFLICTS_OPTION, (args, b) => args.SyncConflicts = b); AddOption(SYNC_DELETED_OPTION, (args, b) => args.SyncDeleted = b); AddOption(TARGET_MANIFEST_UID_OPTION, (args, b) => args.TargetManifestUid = b); + AddOption(DOWNLOAD_MAX_PARALLEL_COUNT_OPTION, (args, i) => args.DownloadMaxParallelCount = i); } public override async Task GetResult(ContentSyncCommandArgs args) @@ -49,7 +55,8 @@ public override async Task GetResult(ContentSyncCommandArgs a foreach (var manifestId in args.ManifestIdsToReset) { var task = _contentService.SyncLocalContent(args.Lifecycle, manifestId, args.FilterType, args.Filter, args.SyncCreated, - args.SyncModified, args.SyncConflicts, args.SyncDeleted, args.TargetManifestUid, this.SendResults); + args.SyncModified, args.SyncConflicts, args.SyncDeleted, args.TargetManifestUid, this.SendResults, + args.DownloadMaxParallelCount); resetPromises.Add(task); } @@ -71,6 +78,7 @@ public class ContentSyncCommandArgs : ContentCommandArgs public bool SyncConflicts; public bool SyncDeleted; public string TargetManifestUid; + public int DownloadMaxParallelCount; } [CliContractType] diff --git a/cli/cli/Services/Content/ContentService.cs b/cli/cli/Services/Content/ContentService.cs index 116b61655c..8e461af46b 100644 --- a/cli/cli/Services/Content/ContentService.cs +++ b/cli/cli/Services/Content/ContentService.cs @@ -78,6 +78,13 @@ public partial class ContentService /// private const string FAKE_EMPTY_MANIFEST_UID = "EmptyManifest"; + private const int CONTENT_DOWNLOAD_MAX_ATTEMPTS = 4; + private const int DEFAULT_CONTENT_DOWNLOAD_MAX_CONCURRENCY = 64; + private const int CONTENT_DOWNLOAD_RETRY_BASE_DELAY_MS = 250; + private const int CONTENT_DOWNLOAD_RETRY_JITTER_MS = 250; + + private static readonly HttpClient _contentDownloadClient = new(CreateContentDownloadHandler()); + private readonly CliRequester _requester; private readonly ConfigService _config; private readonly IContentApi _contentApi; @@ -942,10 +949,11 @@ public async Task PublishContent(AutoSnapshotType autoSnapshotType, int maxLocal public async Task SyncLocalContent(AppLifecycle lifecycle, string manifestId, ContentFilterType filterType = ContentFilterType.ExactIds, string[] filters = null, bool deleteCreated = true, bool syncModified = true, bool forceSyncConflicts = true, bool syncDeleted = true, - string referenceManifestUid = "", Action onContentSyncProgressUpdate = null) + string referenceManifestUid = "", Action onContentSyncProgressUpdate = null, + int downloadMaxParallelCount = DEFAULT_CONTENT_DOWNLOAD_MAX_CONCURRENCY) { var targetManifest = await GetManifest(manifestId, referenceManifestUid, replaceLatest: string.IsNullOrEmpty(referenceManifestUid)); - return await SyncLocalContent(targetManifest, manifestId, filterType, filters, deleteCreated, syncModified, forceSyncConflicts, syncDeleted, onContentSyncProgressUpdate, lifecycle.CancellationToken); + return await SyncLocalContent(targetManifest, manifestId, filterType, filters, deleteCreated, syncModified, forceSyncConflicts, syncDeleted, onContentSyncProgressUpdate, lifecycle.CancellationToken, downloadMaxParallelCount); } /// @@ -954,7 +962,8 @@ public async Task SyncLocalContent(AppLifecycle lifecycle, st public async Task SyncLocalContent(ClientManifestJsonResponse targetManifest, string manifestId, ContentFilterType filterType = ContentFilterType.ExactIds, string[] filters = null, bool syncCreated = true, bool syncModified = true, bool forceSyncConflicts = true, bool syncDeleted = true, - Action onContentSyncProgressUpdate = null, CancellationToken cancellationToken = default) + Action onContentSyncProgressUpdate = null, CancellationToken cancellationToken = default, + int downloadMaxParallelCount = DEFAULT_CONTENT_DOWNLOAD_MAX_CONCURRENCY) { // Reset processed count when calling SyncLocalContent method _syncProcessedCount = 0; @@ -995,10 +1004,14 @@ public async Task SyncLocalContent(ClientManifestJsonResponse // Download and overwrite the local content for things that have changed based on the hash or don't exist. int totalItems = contentToDownload.Length + contentToDelete.Length; + using var contentDownloadSemaphore = downloadMaxParallelCount > 0 ? new SemaphoreSlim(downloadMaxParallelCount) : null; var downloadPromises = contentToDownload.Select(async c => { - Log.Verbose("Downloading content with id. ID={Id}", c.Id); - + if (contentDownloadSemaphore != null) + { + await contentDownloadSemaphore.WaitAsync(cancellationToken); + } + ContentProgressUpdateData contentProgress = new ContentProgressUpdateData { totalItems = totalItems, @@ -1008,8 +1021,7 @@ public async Task SyncLocalContent(ClientManifestJsonResponse JsonElement customRequest; try { - customRequest = await _requester.CustomRequest(Method.GET, c.ReferenceContent.uri, - parser: s => JsonSerializer.Deserialize(s)); + customRequest = await DownloadContentFile(c, cancellationToken); } catch (HttpRequesterException exception) { @@ -1018,6 +1030,17 @@ public async Task SyncLocalContent(ClientManifestJsonResponse onContentSyncProgressUpdate?.Invoke(contentProgress); throw; } + catch (Exception exception) when (!cancellationToken.IsCancellationRequested) + { + contentProgress.EventType = ContentProgressUpdateData.EVT_TYPE_SyncError; + contentProgress.errorMessage = exception.Message; + onContentSyncProgressUpdate?.Invoke(contentProgress); + throw; + } + finally + { + contentDownloadSemaphore?.Release(); + } contentProgress.EventType = ContentProgressUpdateData.EVT_TYPE_SyncComplete; onContentSyncProgressUpdate?.Invoke(contentProgress); @@ -1163,6 +1186,71 @@ public async Task SyncLocalContent(ClientManifestJsonResponse } } + private async Task DownloadContentFile(ContentFile contentFile, CancellationToken cancellationToken) + { + for (var attempt = 1; attempt <= CONTENT_DOWNLOAD_MAX_ATTEMPTS; attempt++) + { + try + { + using var response = await _contentDownloadClient.GetAsync(contentFile.ReferenceContent.uri, cancellationToken); + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); + using var reader = new StreamReader(stream, Encoding.UTF8); + var rawResponse = await reader.ReadToEndAsync(); + + if (!response.IsSuccessStatusCode) + { + throw new RequesterException("Cli", Method.GET.ToReadableString(), contentFile.ReferenceContent.uri, (int)response.StatusCode, rawResponse); + } + + return JsonSerializer.Deserialize(rawResponse); + } + catch (Exception exception) when (!cancellationToken.IsCancellationRequested && IsTransientContentDownloadException(exception) && attempt < CONTENT_DOWNLOAD_MAX_ATTEMPTS) + { + var delay = GetContentDownloadRetryDelay(attempt); + Log.Warning($"Transient content download failure. Retrying content-id=[{contentFile.Id}] attempt=[{attempt + 1}/{CONTENT_DOWNLOAD_MAX_ATTEMPTS}] delay-ms=[{delay.TotalMilliseconds}] error=[{exception.GetType().Name}] message=[{exception.Message}]"); + await Task.Delay(delay, cancellationToken); + } + } + + throw new InvalidOperationException($"Content download retry loop exited unexpectedly. content-id=[{contentFile.Id}]"); + } + + private static HttpClientHandler CreateContentDownloadHandler() + { + return new HttpClientHandler + { + MaxConnectionsPerServer = int.MaxValue, + UseCookies = false + }; + } + + private static TimeSpan GetContentDownloadRetryDelay(int failedAttempt) + { + var exponentialDelayMs = CONTENT_DOWNLOAD_RETRY_BASE_DELAY_MS * (1 << (failedAttempt - 1)); + var jitterMs = Random.Shared.Next(0, CONTENT_DOWNLOAD_RETRY_JITTER_MS); + return TimeSpan.FromMilliseconds(exponentialDelayMs + jitterMs); + } + + private static bool IsTransientContentDownloadException(Exception exception) + { + if (exception is RequesterException requesterException) + { + return requesterException.Status is 408 or 429 || requesterException.Status >= 500 && requesterException.Status < 600; + } + + if (exception is HttpRequestException || exception is TimeoutException) + { + return true; + } + + if (exception is TaskCanceledException) + { + return true; + } + + return exception.InnerException != null && IsTransientContentDownloadException(exception.InnerException); + } + public string[] GetContentSnapshots(bool local, string pid = "") { diff --git a/client/Packages/com.beamable/Editor/BeamCli/Commands/BeamContentSync.cs b/client/Packages/com.beamable/Editor/BeamCli/Commands/BeamContentSync.cs index 2b0470c9d0..602097e4cc 100644 --- a/client/Packages/com.beamable/Editor/BeamCli/Commands/BeamContentSync.cs +++ b/client/Packages/com.beamable/Editor/BeamCli/Commands/BeamContentSync.cs @@ -26,6 +26,8 @@ public partial class ContentSyncArgs : Beamable.Common.BeamCli.IBeamCommandArgs public bool syncDeleted; /// If you pass in a Manifest's UID, we'll sync with that as the target. If filters are provided, will only do this for content that matches the filter public string target; + /// Maximum number of content files to download in parallel while syncing. Use 0 for unbounded parallelism. + public int downloadMaxParallelCount; /// Serializes the arguments for command line usage. public virtual string Serialize() { @@ -75,6 +77,11 @@ public virtual string Serialize() { genBeamCommandArgs.Add(("--target=" + this.target)); } + // If the downloadMaxParallelCount value was not default, then add it to the list of args. + if ((this.downloadMaxParallelCount != default(int))) + { + genBeamCommandArgs.Add(("--download-max-parallel-count=" + this.downloadMaxParallelCount)); + } string genBeamCommandStr = ""; // Join all the args with spaces genBeamCommandStr = string.Join(" ", genBeamCommandArgs); diff --git a/client/Packages/com.beamable/Editor/ContentService/CliContentService.cs b/client/Packages/com.beamable/Editor/ContentService/CliContentService.cs index 2ddf45bcaa..580f0e39e7 100644 --- a/client/Packages/com.beamable/Editor/ContentService/CliContentService.cs +++ b/client/Packages/com.beamable/Editor/ContentService/CliContentService.cs @@ -20,6 +20,18 @@ namespace Beamable.Editor.ContentService { + public class ContentOperationProgress + { + public bool IsActive; + public bool HasError; + public string Title = string.Empty; + public string Description = string.Empty; + public string CurrentContentName = string.Empty; + public int ProcessedItems; + public int TotalItems; + public float Progress; + } + public class CliContentService : IStorageHandler, ILoadWithContext { private const string SYNC_OPERATION_TITLE = "Sync Contents"; @@ -46,6 +58,7 @@ public class CliContentService : IStorageHandler, ILoadWithCo private int syncedContents; private int publishedContents; private readonly Dictionary _lastSavedPropertiesCache = new(); + private bool _showUnityModalProgress; [NonSerialized] public bool isReloading; @@ -55,6 +68,8 @@ public class CliContentService : IStorageHandler, ILoadWithCo private readonly Dictionary _contentScriptableCache; public BeamContentPsProgressMessage LatestProgressUpdate; + public ContentOperationProgress CurrentProgress { get; } = new(); + public int ProgressUpdateVersion { get; private set; } private readonly ContentConfiguration _contentConfiguration; private ValidationContext ValidationContext => _provider.GetService(); @@ -205,11 +220,12 @@ public async Promise SyncContentsWithProgress(bool syncModified, bool syncConflicted, bool syncDeleted, string filter = null, - ContentFilterType filterType = default) + ContentFilterType filterType = default, + bool showUnityModalProgress = false) { syncedContents = 0; - EditorUtility.DisplayProgressBar(SYNC_OPERATION_TITLE, "Synchronizing contents...", 0); + BeginActionProgress(SYNC_OPERATION_TITLE, "Synchronizing contents...", showUnityModalProgress); try { if (_contentWatcher != null) @@ -226,6 +242,7 @@ public async Promise SyncContentsWithProgress(bool syncModified, syncDeleted = syncDeleted, filter = filter, filterType = filterType, + downloadMaxParallelCount = GetEditorSyncDownloadMaxParallelCountCliOption(), }); @@ -259,6 +276,7 @@ public async Task SyncContents(bool syncModified = true, syncDeleted = syncDeleted, filter = filter, filterType = filterType, + downloadMaxParallelCount = GetEditorSyncDownloadMaxParallelCountCliOption(), }); await contentSyncCommand.Run(); @@ -421,7 +439,7 @@ public void ResolveConflict(string contentId, bool useLocal) public async Promise PublishContentsWithProgress() { - EditorUtility.DisplayProgressBar(PUBLISH_OPERATION_TITLE, "Publishing contents...", 0); + BeginActionProgress(PUBLISH_OPERATION_TITLE, "Publishing contents...", true); publishedContents = 0; try @@ -451,10 +469,46 @@ public async Promise PublishContentsWithProgress() private void FinishActionProgress() { - EditorUtility.ClearProgressBar(); + if (_showUnityModalProgress) + { + EditorUtility.ClearProgressBar(); + } + + CurrentProgress.IsActive = false; + ProgressUpdateVersion++; _ = Reload(); } + private void BeginActionProgress(string title, string description, bool showUnityModalProgress) + { + _showUnityModalProgress = showUnityModalProgress; + CurrentProgress.IsActive = true; + CurrentProgress.Title = title; + CurrentProgress.Description = description; + CurrentProgress.CurrentContentName = string.Empty; + CurrentProgress.ProcessedItems = 0; + CurrentProgress.TotalItems = 0; + CurrentProgress.Progress = 0; + CurrentProgress.HasError = false; + ProgressUpdateVersion++; + + if (_showUnityModalProgress) + { + EditorUtility.DisplayProgressBar(title, description, 0); + } + } + + private int GetEditorSyncDownloadMaxParallelCountCliOption() + { + if (_contentConfiguration.EditorSyncDownloadMaxParallelCount?.HasValue != true) + { + return default; + } + + var value = _contentConfiguration.EditorSyncDownloadMaxParallelCount.Value; + return value == 0 ? -1 : value; + } + public async Task PublishContents() { var contentPublishArgs = new ContentPublishArgs @@ -908,13 +962,23 @@ private void HandleProgressUpdate(BeamContentProgressUpdateData data, string onSuccessBaseMessage, string onErrorBaseMessage, ref int countItem) { - string description = string.Empty; - float progress = (float)countItem / data.totalItems; + int totalItems = Math.Max(data.totalItems, 0); + string description = CurrentProgress.Description; + float progress = totalItems == 0 ? 0 : Mathf.Clamp01((float)countItem / totalItems); + int processedItemsForDisplay = countItem; switch (data.EventType) { case 0: // Error on Content Progress Update - EditorUtility.ClearProgressBar(); + CurrentProgress.HasError = true; + CurrentProgress.CurrentContentName = data.contentName; + CurrentProgress.TotalItems = totalItems; + CurrentProgress.Description = string.Format(onErrorBaseMessage, data.contentName, data.errorMessage); + ProgressUpdateVersion++; + if (_showUnityModalProgress) + { + EditorUtility.ClearProgressBar(); + } EditorUtility.DisplayDialog( operationTitle, string.Format(onErrorBaseMessage, data.contentName, data.errorMessage), @@ -923,14 +987,33 @@ private void HandleProgressUpdate(BeamContentProgressUpdateData data, return; case 1: // Sync Complete case 2: // Publish Complete - description = string.Format(onSuccessBaseMessage, data.contentName, countItem, data.totalItems); + CurrentProgress.CurrentContentName = data.contentName; + int visibleCount = totalItems == 0 ? 0 : Mathf.Min(totalItems, countItem + 1); + processedItemsForDisplay = visibleCount; + description = string.Format(onSuccessBaseMessage, data.contentName, visibleCount, totalItems); + progress = totalItems == 0 ? 1 : Mathf.Clamp01((float)visibleCount / totalItems); break; case 3: // Update Processed item count countItem = data.processedItems; + processedItemsForDisplay = totalItems == 0 ? countItem : Mathf.Min(countItem, totalItems); + progress = totalItems == 0 ? 1 : Mathf.Clamp01((float)processedItemsForDisplay / totalItems); + description = totalItems == 0 + ? $"{operationTitle} complete." + : $"{processedItemsForDisplay}/{totalItems} items processed."; break; } - EditorUtility.DisplayProgressBar(operationTitle, description, progress); + CurrentProgress.Title = operationTitle; + CurrentProgress.Description = description; + CurrentProgress.ProcessedItems = processedItemsForDisplay; + CurrentProgress.TotalItems = totalItems; + CurrentProgress.Progress = progress; + ProgressUpdateVersion++; + + if (_showUnityModalProgress) + { + EditorUtility.DisplayProgressBar(operationTitle, description, progress); + } } public void SetManifestId(string id) diff --git a/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow.cs b/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow.cs index 29f094c84c..c27d86e27b 100644 --- a/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow.cs +++ b/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow.cs @@ -31,6 +31,7 @@ public partial class ContentWindow : BeamEditorWindow private ContentConfiguration _contentConfiguration; private Vector2 _horizontalScrollPosition; private int _lastManifestChangedCount; + private int _lastProgressUpdateVersion; private EditorGUISplitView _mainSplitter; static ContentWindow() @@ -109,6 +110,12 @@ private void OnEditorUpdate() ReloadData(); Repaint(); } + + if (_contentService != null && _contentService.ProgressUpdateVersion != _lastProgressUpdateVersion) + { + _lastProgressUpdateVersion = _contentService.ProgressUpdateVersion; + Repaint(); + } } private void ReloadData() @@ -119,7 +126,7 @@ private void ReloadData() if(!_contentService.HasChangedContents && _windowStatus != ContentWindowStatus.SnapshotManager) { - ChangeWindowStatus(ContentWindowStatus.Normal); + ChangeWindowStatusDelayed(ContentWindowStatus.Normal); } } diff --git a/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow_ContentActions.cs b/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow_ContentActions.cs index b08dafd733..ddb965618a 100644 --- a/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow_ContentActions.cs +++ b/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow_ContentActions.cs @@ -31,7 +31,7 @@ private void DrawPublishPanel() { if (_contentService.HasConflictedContent || _contentService.HasInvalidContent || !_contentService.HasChangedContents) { - ChangeWindowStatus(ContentWindowStatus.Normal); + ChangeWindowStatusDelayed(ContentWindowStatus.Normal); return; } @@ -43,7 +43,7 @@ void PublishContents() { _contentService.PublishContentsWithProgress().Then(_ => { - ChangeWindowStatus(ContentWindowStatus.Normal); + ChangeWindowStatusDelayed(ContentWindowStatus.Normal); }); } } @@ -55,9 +55,9 @@ void PublishContents() private void DrawRevertPanel() { - if (!_contentService.HasChangedContents) + if (!_contentService.HasChangedContents && !_contentService.CurrentProgress.IsActive) { - ChangeWindowStatus(ContentWindowStatus.Normal); + ChangeWindowStatusDelayed(ContentWindowStatus.Normal); return; } @@ -69,7 +69,7 @@ void RevertContent() { _revertAction?.Invoke().Then(_ => { - ChangeWindowStatus(ContentWindowStatus.Normal); + ChangeWindowStatusDelayed(ContentWindowStatus.Normal); }); } } @@ -195,7 +195,7 @@ private void DrawValidatePanel() GUILayout.FlexibleSpace(); if (BeamGUI.CancelButton("Back", GUILayout.Width(60))) { - ChangeWindowStatus(ContentWindowStatus.Normal); + ChangeWindowStatusDelayed(ContentWindowStatus.Normal); } } @@ -240,26 +240,63 @@ private void DrawContentActionPanel(Texture icon, string title, string warningMe EditorGUILayout.EndScrollView(); EditorGUILayout.Space(12); + + if (DrawContentOperationProgress()) + { + EditorGUILayout.Space(12); + } var buttonsRect = EditorGUILayout.GetControlRect(GUILayout.ExpandWidth(true), GUILayout.Height(30f)); var buttonsRectController = new EditorGUIRectController(buttonsRect); var primaryBtnContent = new GUIContent(buttonText); var primaryBtnSize = GUI.skin.button.CalcSize(primaryBtnContent); - if (BeamGUI.PrimaryButton(buttonsRectController.ReserveWidthFromRight(primaryBtnSize.x + BASE_PADDING * 2), primaryBtnContent)) + if (BeamGUI.ShowDisabled(!_contentService.CurrentProgress.IsActive, + () => BeamGUI.PrimaryButton(buttonsRectController.ReserveWidthFromRight(primaryBtnSize.x + BASE_PADDING * 2), primaryBtnContent))) { onButtonClicked?.Invoke(); } var cancelBtnContent = new GUIContent("Cancel"); var cancelBtnSize = GUI.skin.button.CalcSize(cancelBtnContent); - if (BeamGUI.CustomButton(buttonsRectController.ReserveWidthFromRight(cancelBtnSize.x + BASE_PADDING * 2), cancelBtnContent, BeamGUI.ColorizeButton(Color.gray))) + if (BeamGUI.ShowDisabled(!_contentService.CurrentProgress.IsActive, + () => BeamGUI.CustomButton(buttonsRectController.ReserveWidthFromRight(cancelBtnSize.x + BASE_PADDING * 2), cancelBtnContent, BeamGUI.ColorizeButton(Color.gray)))) { - ChangeWindowStatus(ContentWindowStatus.Normal); + ChangeWindowStatusDelayed(ContentWindowStatus.Normal); } EditorGUILayout.EndVertical(); } + private bool DrawContentOperationProgress() + { + var progress = _contentService.CurrentProgress; + if (!progress.IsActive) + { + return false; + } + + var progressAction = progress.Title == "Publish Contents" ? "published" : "synced"; + var progressText = progress.TotalItems > 0 + ? $"{progress.ProcessedItems}/{progress.TotalItems} items {progressAction}" + : $"Preparing {progress.Title.ToLowerInvariant()}..."; + + if (!string.IsNullOrEmpty(progress.CurrentContentName)) + { + progressText = $"{progressText} - {progress.CurrentContentName}"; + } + + EditorGUILayout.LabelField(progress.Title, EditorStyles.boldLabel); + var progressRect = EditorGUILayout.GetControlRect(GUILayout.ExpandWidth(true), GUILayout.Height(20)); + EditorGUI.ProgressBar(progressRect, progress.Progress, progressText); + + if (!string.IsNullOrEmpty(progress.Description)) + { + EditorGUILayout.LabelField(progress.Description, _contentHeaderDescriptionStyle); + } + + return true; + } + private void DrawChangedContents(float width, string headerName, Texture icon, diff --git a/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow_Header.cs b/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow_Header.cs index 2a291f8b54..2439219409 100644 --- a/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow_Header.cs +++ b/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow_Header.cs @@ -119,7 +119,7 @@ private void DrawTopBarHeader() { if (BeamGUI.HeaderButton("Content Editor", BeamGUI.iconContentEditorIcon, width: 90, iconPadding: 2)) { - ChangeWindowStatus(ContentWindowStatus.Normal); + ChangeWindowStatusDelayed(ContentWindowStatus.Normal); } } if (_windowStatus is ContentWindowStatus.Normal || _windowStatus is ContentWindowStatus.Validate) @@ -260,6 +260,11 @@ private void ChangeWindowStatus(ContentWindowStatus windowStatus, bool shouldRep Repaint(); } + private void ChangeWindowStatusDelayed(ContentWindowStatus windowStatus) + { + EditorApplication.delayCall += () => ChangeWindowStatus(windowStatus); + } + private void DrawLowBarHeader(Rect rect) { if (NeedsMigration) return; @@ -420,27 +425,27 @@ private void ShowSyncMenu() private async Promise RevertAllContents() { - await _contentService.SyncContentsWithProgress(true, true, true, true); + await _contentService.SyncContentsWithProgress(true, true, true, true, showUnityModalProgress: false); } private async Promise RevertModifiedContents() { - await _contentService.SyncContentsWithProgress(true, false, false, false); + await _contentService.SyncContentsWithProgress(true, false, false, false, showUnityModalProgress: false); } private async Promise RevertConflictedContents() { - await _contentService.SyncContentsWithProgress(false, false, true, false); + await _contentService.SyncContentsWithProgress(false, false, true, false, showUnityModalProgress: false); } private async Promise RevertDeletedContents() { - await _contentService.SyncContentsWithProgress(false, false, false, true); + await _contentService.SyncContentsWithProgress(false, false, false, true, showUnityModalProgress: false); } private async Promise RevertAllNewContents() { - await _contentService.SyncContentsWithProgress(false, true, false, false); + await _contentService.SyncContentsWithProgress(false, true, false, false, showUnityModalProgress: false); } diff --git a/client/Packages/com.beamable/Runtime/Modules/Content/ContentConfiguration.cs b/client/Packages/com.beamable/Runtime/Modules/Content/ContentConfiguration.cs index ab5072136b..f3781754fd 100644 --- a/client/Packages/com.beamable/Runtime/Modules/Content/ContentConfiguration.cs +++ b/client/Packages/com.beamable/Runtime/Modules/Content/ContentConfiguration.cs @@ -30,9 +30,12 @@ public class ContentConfiguration : ModuleConfigurationObject [HideInInspector] public string EditorManifestID = DEFAULT_MANIFEST_ID; - [Tooltip("In editor, when downloading content, this controls the batch size of the download. By default, it is 100.")] - public OptionalInt EditorDownloadBatchSize; -#endif + [Tooltip("In editor, when downloading content, this controls the batch size of the download. By default, it is 100.")] + public OptionalInt EditorDownloadBatchSize; + + [Tooltip("In editor, when syncing content from the realm, this controls how many content files can be downloaded in parallel. Leave unset to use the CLI default. Use 0 for unbounded parallelism.")] + public OptionalInt EditorSyncDownloadMaxParallelCount; +#endif [Tooltip("When enabled, content checksum will be calculated based on default object property order.")] public bool EnablePropertyOrderDependenceForContentChecksum = true; From 91eca62a299f7042859446e048f01a0dcae20f02 Mon Sep 17 00:00:00 2001 From: moe Date: Mon, 18 May 2026 14:48:51 -0300 Subject: [PATCH 4/4] - added xml summaries - updated changelog --- cli/cli/Services/Content/ContentService.cs | 24 +++++++++++++++++++ client/Packages/com.beamable/CHANGELOG.md | 7 +++++- .../ContentService/CliContentService.cs | 21 ++++++++++++++++ .../ContentWindow_ContentActions.cs | 4 ++++ .../UI/ContentWindow/ContentWindow_Header.cs | 6 +++++ .../Modules/Content/ContentConfiguration.cs | 3 +++ 6 files changed, 64 insertions(+), 1 deletion(-) diff --git a/cli/cli/Services/Content/ContentService.cs b/cli/cli/Services/Content/ContentService.cs index 8e461af46b..88e801eed2 100644 --- a/cli/cli/Services/Content/ContentService.cs +++ b/cli/cli/Services/Content/ContentService.cs @@ -83,6 +83,12 @@ public partial class ContentService private const int CONTENT_DOWNLOAD_RETRY_BASE_DELAY_MS = 250; private const int CONTENT_DOWNLOAD_RETRY_JITTER_MS = 250; + /// + /// Shared client for CDN content-file downloads during sync. + /// + /// + /// This intentionally bypasses the generic CLI requester to avoid per-file verbose request/response logging while preserving connection reuse. + /// private static readonly HttpClient _contentDownloadClient = new(CreateContentDownloadHandler()); private readonly CliRequester _requester; @@ -1186,6 +1192,12 @@ public async Task SyncLocalContent(ClientManifestJsonResponse } } + /// + /// Downloads a single content file from its remote content URI with transient retry handling. + /// + /// + /// Content sync may download hundreds or thousands of small files. This path keeps those downloads off the generic requester so Unity does not receive a verbose log event for every response body. + /// private async Task DownloadContentFile(ContentFile contentFile, CancellationToken cancellationToken) { for (var attempt = 1; attempt <= CONTENT_DOWNLOAD_MAX_ATTEMPTS; attempt++) @@ -1215,6 +1227,12 @@ private async Task DownloadContentFile(ContentFile contentFile, Can throw new InvalidOperationException($"Content download retry loop exited unexpectedly. content-id=[{contentFile.Id}]"); } + /// + /// Creates the HTTP handler used by the shared content download client. + /// + /// + /// The sync command owns concurrency limits separately, so the handler allows a high per-server connection ceiling. + /// private static HttpClientHandler CreateContentDownloadHandler() { return new HttpClientHandler @@ -1224,6 +1242,9 @@ private static HttpClientHandler CreateContentDownloadHandler() }; } + /// + /// Calculates exponential retry delay with jitter for a failed content download attempt. + /// private static TimeSpan GetContentDownloadRetryDelay(int failedAttempt) { var exponentialDelayMs = CONTENT_DOWNLOAD_RETRY_BASE_DELAY_MS * (1 << (failedAttempt - 1)); @@ -1231,6 +1252,9 @@ private static TimeSpan GetContentDownloadRetryDelay(int failedAttempt) return TimeSpan.FromMilliseconds(exponentialDelayMs + jitterMs); } + /// + /// Determines whether a content download failure is likely transient and should be retried. + /// private static bool IsTransientContentDownloadException(Exception exception) { if (exception is RequesterException requesterException) diff --git a/client/Packages/com.beamable/CHANGELOG.md b/client/Packages/com.beamable/CHANGELOG.md index f625635b61..4d5beb311a 100644 --- a/client/Packages/com.beamable/CHANGELOG.md +++ b/client/Packages/com.beamable/CHANGELOG.md @@ -6,7 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Added in-window Content Manager progress for sync and revert operations. +- Added editor content sync download concurrency configuration. + ### Fixed +- Improved content sync resilience for transient SSL/socket reset download failures. - Deserialization issue with `properties` field in Score Items of Events - Fixed an issue where the Unity Editor would not detect changes to Icon subObject (for Sprites in Multiple Mode) and thus not saving it correctly <<<<<<< fix/contentNullFields @@ -1569,4 +1574,4 @@ This is a broken package. It includes changes from the 1.1.0 release. Please do - Added OnUserLoggingOut event available from API. The event fires before a user switches account. - Doc Url to package.json. - Event phase validation. Events can no longer have zero phases. This may lead to disappearing Event Phases if your Beamable version is mismatched. -- Switched MatchmakingService API to point to our new backend matchmaking service. \ No newline at end of file +- Switched MatchmakingService API to point to our new backend matchmaking service. diff --git a/client/Packages/com.beamable/Editor/ContentService/CliContentService.cs b/client/Packages/com.beamable/Editor/ContentService/CliContentService.cs index 580f0e39e7..e9f33db47c 100644 --- a/client/Packages/com.beamable/Editor/ContentService/CliContentService.cs +++ b/client/Packages/com.beamable/Editor/ContentService/CliContentService.cs @@ -20,6 +20,9 @@ namespace Beamable.Editor.ContentService { + /// + /// Tracks the currently running content operation so the Content Manager can render progress inside the window. + /// public class ContentOperationProgress { public bool IsActive; @@ -68,7 +71,13 @@ public class CliContentService : IStorageHandler, ILoadWithCo private readonly Dictionary _contentScriptableCache; public BeamContentPsProgressMessage LatestProgressUpdate; + /// + /// Current publish or sync progress state consumed by the Content Manager IMGUI view. + /// public ContentOperationProgress CurrentProgress { get; } = new(); + /// + /// Monotonic value incremented when changes so the Content Manager can repaint on demand. + /// public int ProgressUpdateVersion { get; private set; } private readonly ContentConfiguration _contentConfiguration; @@ -467,6 +476,9 @@ public async Promise PublishContentsWithProgress() } } + /// + /// Finishes the active content operation, clears any modal progress UI, and reloads local content state. + /// private void FinishActionProgress() { if (_showUnityModalProgress) @@ -479,6 +491,9 @@ private void FinishActionProgress() _ = Reload(); } + /// + /// Initializes progress state for a content operation. Sync can use in-window progress only, while publish can still opt into Unity's modal progress UI. + /// private void BeginActionProgress(string title, string description, bool showUnityModalProgress) { _showUnityModalProgress = showUnityModalProgress; @@ -498,6 +513,12 @@ private void BeginActionProgress(string title, string description, bool showUnit } } + /// + /// Converts the optional editor sync concurrency setting to the CLI argument value. + /// + /// + /// Returning the default int omits the CLI argument, allowing the CLI default to apply. A configured value of 0 maps to -1 so the generated command includes an explicit unbounded setting. + /// private int GetEditorSyncDownloadMaxParallelCountCliOption() { if (_contentConfiguration.EditorSyncDownloadMaxParallelCount?.HasValue != true) diff --git a/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow_ContentActions.cs b/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow_ContentActions.cs index ddb965618a..925fd41e48 100644 --- a/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow_ContentActions.cs +++ b/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow_ContentActions.cs @@ -267,6 +267,10 @@ private void DrawContentActionPanel(Texture icon, string title, string warningMe EditorGUILayout.EndVertical(); } + /// + /// Draws the active content operation progress bar in the Content Manager action panel. + /// + /// True when progress UI was drawn and the caller should reserve spacing for it. private bool DrawContentOperationProgress() { var progress = _contentService.CurrentProgress; diff --git a/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow_Header.cs b/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow_Header.cs index 2439219409..909d23653c 100644 --- a/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow_Header.cs +++ b/client/Packages/com.beamable/Editor/UI/ContentWindow/ContentWindow_Header.cs @@ -260,6 +260,12 @@ private void ChangeWindowStatus(ContentWindowStatus windowStatus, bool shouldRep Repaint(); } + /// + /// Defers Content Manager state changes until the current IMGUI event has finished. + /// + /// + /// Some state transitions remove or add controls. Deferring them avoids mismatched layout groups during repaint. + /// private void ChangeWindowStatusDelayed(ContentWindowStatus windowStatus) { EditorApplication.delayCall += () => ChangeWindowStatus(windowStatus); diff --git a/client/Packages/com.beamable/Runtime/Modules/Content/ContentConfiguration.cs b/client/Packages/com.beamable/Runtime/Modules/Content/ContentConfiguration.cs index f3781754fd..b2919850b9 100644 --- a/client/Packages/com.beamable/Runtime/Modules/Content/ContentConfiguration.cs +++ b/client/Packages/com.beamable/Runtime/Modules/Content/ContentConfiguration.cs @@ -33,6 +33,9 @@ public class ContentConfiguration : ModuleConfigurationObject [Tooltip("In editor, when downloading content, this controls the batch size of the download. By default, it is 100.")] public OptionalInt EditorDownloadBatchSize; + /// + /// Controls the maximum parallel content-file downloads used by editor content sync. Leave unset to use the CLI default; use 0 for unbounded parallelism. + /// [Tooltip("In editor, when syncing content from the realm, this controls how many content files can be downloaded in parallel. Leave unset to use the CLI default. Use 0 for unbounded parallelism.")] public OptionalInt EditorSyncDownloadMaxParallelCount; #endif