Skip to content

Commit b9d6c33

Browse files
cdavernascdavernas
authored
Merge pull request #421 from serverlessworkflow/feat-static-workflow-seeding
Add ability to seed the database using static file-based resources
2 parents 30d06e3 + 084b002 commit b9d6c33

File tree

21 files changed

+314
-36
lines changed

21 files changed

+314
-36
lines changed

‎src/api/Synapse.Api.Application/Configuration/ApiServerOptions.cs

+5
Original file line numberDiff line numberDiff line change
@@ -43,4 +43,9 @@ public ApiServerOptions()
4343
/// </summary>
4444
public virtual ApiServerCloudEventOptions CloudEvents { get; set; } = new();
4545

46+
/// <summary>
47+
/// Gets/sets the options used to configure the seeding, if any, of Synapse's database
48+
/// </summary>
49+
public virtual DatabaseSeedingOptions Seeding { get; set; } = new();
50+
4651
}

‎src/api/Synapse.Api.Application/Configuration/DatabaseSeedingOptions.cs

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
// Copyright © 2024-Present The Synapse Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License"),
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
// http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
14+
namespace Synapse.Api.Application.Configuration;
15+
16+
/// <summary>
17+
/// Represents the options used to configure the seeding, if any, of Synapse's database
18+
/// </summary>
19+
public class DatabaseSeedingOptions
20+
{
21+
22+
/// <summary>
23+
/// Gets the path to the directory from which to load the static resources used to seed the database
24+
/// </summary>
25+
public static readonly string DefaultDirectory = Path.Combine(AppContext.BaseDirectory, "data", "seed");
26+
/// <summary>
27+
/// Gets the default GLOB pattern used to match the static resource files to use to seed the database
28+
/// </summary>
29+
public const string DefaultFilePattern = "*.*";
30+
31+
/// <summary>
32+
/// Initializes a new <see cref="DatabaseSeedingOptions"/>
33+
/// </summary>
34+
public DatabaseSeedingOptions()
35+
{
36+
var env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Database.Seeding.Reset);
37+
if (!string.IsNullOrWhiteSpace(env))
38+
{
39+
if (!bool.TryParse(env, out var reset)) throw new Exception($"Failed to parse the value specified as '{SynapseDefaults.EnvironmentVariables.Database.Seeding.Reset}' environment variable into a boolean");
40+
this.Reset = reset;
41+
}
42+
env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Database.Seeding.Directory);
43+
if (!string.IsNullOrWhiteSpace(env))
44+
{
45+
this.Directory = env;
46+
if (!System.IO.Directory.Exists(this.Directory)) System.IO.Directory.CreateDirectory(this.Directory);
47+
}
48+
env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Database.Seeding.Overwrite);
49+
if (!string.IsNullOrWhiteSpace(env))
50+
{
51+
if (!bool.TryParse(env, out var overwrite)) throw new Exception($"Failed to parse the value specified as '{SynapseDefaults.EnvironmentVariables.Database.Seeding.Overwrite}' environment variable into a boolean");
52+
this.Overwrite = overwrite;
53+
}
54+
env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Database.Seeding.FilePattern);
55+
if (!string.IsNullOrWhiteSpace(env)) this.FilePattern = env;
56+
}
57+
58+
/// <summary>
59+
/// Gets/sets a boolean indicating whether or not to reset the database upon starting up the API server
60+
/// </summary>
61+
public virtual bool Reset { get; set; }
62+
63+
/// <summary>
64+
/// Gets/sets the directory from which to load the static resources used to seed the database
65+
/// </summary>
66+
public virtual string Directory { get; set; } = DefaultDirectory;
67+
68+
/// <summary>
69+
/// Gets/sets a boolean indicating whether or not to overwrite existing resources
70+
/// </summary>
71+
public virtual bool Overwrite { get; set; }
72+
73+
/// <summary>
74+
/// Gets/sets the GLOB pattern used to match the static resource files to use to seed the database
75+
/// </summary>
76+
public virtual string FilePattern { get; set; } = DefaultFilePattern;
77+
78+
}

‎src/api/Synapse.Api.Application/Services/WorkflowDatabaseInitializer.cs

+154
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
// Copyright © 2024-Present The Synapse Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License"),
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
// http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
14+
using Microsoft.Extensions.DependencyInjection;
15+
using Microsoft.Extensions.Hosting;
16+
using Microsoft.Extensions.Logging;
17+
using Microsoft.Extensions.Options;
18+
using ServerlessWorkflow.Sdk.IO;
19+
using ServerlessWorkflow.Sdk.Models;
20+
using Synapse.Api.Application.Configuration;
21+
using Synapse.Resources;
22+
using System.Diagnostics;
23+
24+
namespace Synapse.Api.Application.Services;
25+
26+
/// <summary>
27+
/// Defines the fundamentals of a service used to initialize the Synapse workflow database
28+
/// </summary>
29+
/// <param name="serviceProvider">The current <see cref="IServiceProvider"/></param>
30+
/// <param name="logger">The service used to perform logging</param>
31+
/// <param name="workflowDefinitionReader">The service used to read <see cref="WorkflowDefinition"/>s</param>
32+
/// <param name="options">The service used to access the current <see cref="ApiServerOptions"/></param>
33+
public class WorkflowDatabaseInitializer(IServiceProvider serviceProvider, ILogger<WorkflowDatabaseInitializer> logger, IWorkflowDefinitionReader workflowDefinitionReader, IOptions<ApiServerOptions> options)
34+
: IHostedService
35+
{
36+
37+
/// <summary>
38+
/// Gets the current <see cref="IServiceProvider"/>
39+
/// </summary>
40+
protected IServiceProvider ServiceProvider { get; } = serviceProvider;
41+
42+
/// <summary>
43+
/// Gets the service used to perform logging
44+
/// </summary>
45+
protected ILogger Logger { get; } = logger;
46+
47+
/// <summary>
48+
/// Gets the service used to read <see cref="WorkflowDefinition"/>s
49+
/// </summary>
50+
protected IWorkflowDefinitionReader WorkflowDefinitionReader { get; } = workflowDefinitionReader;
51+
52+
/// <summary>
53+
/// Gets the current <see cref="ApiServerOptions"/>
54+
/// </summary>
55+
protected ApiServerOptions Options { get; } = options.Value;
56+
57+
/// <inheritdoc/>
58+
public virtual async Task StartAsync(CancellationToken cancellationToken)
59+
{
60+
using var scope = this.ServiceProvider.CreateScope();
61+
var resources = scope.ServiceProvider.GetRequiredService<IResourceRepository>();
62+
var stopwatch = new Stopwatch();
63+
if (this.Options.Seeding.Reset)
64+
{
65+
this.Logger.LogInformation("Starting resetting database...");
66+
stopwatch.Start();
67+
await foreach (var correlation in resources.GetAllAsync<Correlation>(cancellationToken: cancellationToken).ConfigureAwait(false)) await resources.RemoveAsync<Correlation>(correlation.GetName(), correlation.GetNamespace(), false, cancellationToken).ConfigureAwait(false);
68+
await foreach (var correlator in resources.GetAllAsync<Correlator>(cancellationToken: cancellationToken).ConfigureAwait(false)) await resources.RemoveAsync<Correlator>(correlator.GetName(), correlator.GetNamespace(), false, cancellationToken).ConfigureAwait(false);
69+
await foreach (var ns in resources.GetAllAsync<Namespace>(cancellationToken: cancellationToken).Where(ns => ns.GetName() != Namespace.DefaultNamespaceName)) await resources.RemoveAsync<Namespace>(ns.GetName(), ns.GetNamespace(), false, cancellationToken).ConfigureAwait(false);
70+
await foreach (var @operator in resources.GetAllAsync<Operator>(cancellationToken: cancellationToken).ConfigureAwait(false)) await resources.RemoveAsync<Operator>(@operator.GetName(), @operator.GetNamespace(), false, cancellationToken).ConfigureAwait(false);
71+
await foreach (var serviceAccount in resources.GetAllAsync<ServiceAccount>(cancellationToken: cancellationToken).ConfigureAwait(false)) await resources.RemoveAsync<ServiceAccount>(serviceAccount.GetName(), serviceAccount.GetNamespace(), false, cancellationToken).ConfigureAwait(false);
72+
await foreach (var workflow in resources.GetAllAsync<Workflow>(cancellationToken: cancellationToken).ConfigureAwait(false)) await resources.RemoveAsync<Workflow>(workflow.GetName(), workflow.GetNamespace(), false, cancellationToken).ConfigureAwait(false);
73+
await foreach (var workflowInstance in resources.GetAllAsync<WorkflowInstance>(cancellationToken: cancellationToken).ConfigureAwait(false)) await resources.RemoveAsync<WorkflowInstance>(workflowInstance.GetName(), workflowInstance.GetNamespace(), false, cancellationToken).ConfigureAwait(false);
74+
stopwatch.Stop();
75+
this.Logger.LogInformation("Database reset completed in {ms} milliseconds", stopwatch.Elapsed.TotalMilliseconds);
76+
}
77+
var directory = new DirectoryInfo(this.Options.Seeding.Directory);
78+
if (!directory.Exists)
79+
{
80+
this.Logger.LogWarning("The directory '{directory}' does not exist or cannot be found. Skipping static resource import", directory.FullName);
81+
return;
82+
}
83+
this.Logger.LogInformation("Starting importing static resources from directory '{directory}'...", directory.FullName);
84+
var files = directory.GetFiles(this.Options.Seeding.FilePattern, SearchOption.AllDirectories).Where(f => f.FullName.EndsWith(".json", StringComparison.OrdinalIgnoreCase) || f.FullName.EndsWith(".yml", StringComparison.OrdinalIgnoreCase) || f.FullName.EndsWith(".yaml", StringComparison.OrdinalIgnoreCase));
85+
if (!files.Any())
86+
{
87+
this.Logger.LogWarning("No static resource files matching search pattern '{pattern}' found in directory '{directory}'. Skipping import.", this.Options.Seeding.FilePattern, directory.FullName);
88+
return;
89+
}
90+
var count = 0;
91+
stopwatch.Restart();
92+
foreach (var file in files)
93+
{
94+
try
95+
{
96+
using var stream = file.OpenRead();
97+
var workflowDefinition = await this.WorkflowDefinitionReader.ReadAsync(stream, new() { BaseDirectory = file.Directory!.FullName }, cancellationToken).ConfigureAwait(false);
98+
var workflow = await resources.GetAsync<Workflow>(workflowDefinition.Document.Name, workflowDefinition.Document.Namespace, cancellationToken).ConfigureAwait(false);
99+
if (workflow == null)
100+
{
101+
workflow = new()
102+
{
103+
Metadata = new()
104+
{
105+
Namespace = workflowDefinition.Document.Namespace,
106+
Name = workflowDefinition.Document.Name
107+
},
108+
Spec = new()
109+
{
110+
Versions = [workflowDefinition]
111+
}
112+
};
113+
if (await resources.GetAsync<Namespace>(workflow.GetNamespace()!, cancellationToken: cancellationToken).ConfigureAwait(false) == null)
114+
{
115+
await resources.AddAsync(new Namespace() { Metadata = new() { Name = workflow.GetNamespace()! } }, false, cancellationToken).ConfigureAwait(false);
116+
this.Logger.LogInformation("Successfully created namespace '{namespace}'", workflow.GetNamespace());
117+
}
118+
await resources.AddAsync(workflow, false, cancellationToken).ConfigureAwait(false);
119+
}
120+
else
121+
{
122+
var version = workflow.Spec.Versions.Get(workflowDefinition.Document.Version);
123+
if (version != null)
124+
{
125+
if (this.Options.Seeding.Overwrite)
126+
{
127+
workflow.Spec.Versions.Remove(version);
128+
workflow.Spec.Versions.Add(workflowDefinition);
129+
await resources.ReplaceAsync(workflow, false, cancellationToken).ConfigureAwait(false);
130+
}
131+
else
132+
{
133+
this.Logger.LogInformation("Skipped the import of workflow '{workflow}' from file '{file}' because it already exists", $"{workflowDefinition.Document.Name}.{workflowDefinition.Document.Namespace}:{workflowDefinition.Document.Version}", file.FullName);
134+
continue;
135+
}
136+
}
137+
}
138+
this.Logger.LogInformation("Successfully imported workflow '{workflow}' from file '{file}'", $"{workflowDefinition.Document.Name}.{workflowDefinition.Document.Namespace}:{workflowDefinition.Document.Version}", file.FullName);
139+
count++;
140+
}
141+
catch(Exception ex)
142+
{
143+
this.Logger.LogError("An error occurred while reading a workflow definition from file '{file}': {ex}", file.FullName, ex);
144+
continue;
145+
}
146+
}
147+
stopwatch.Stop();
148+
this.Logger.LogInformation("Completed importing {count} static resources in {ms} milliseconds", count, stopwatch.Elapsed.TotalMilliseconds);
149+
}
150+
151+
/// <inheritdoc/>
152+
public virtual Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
153+
154+
}

‎src/api/Synapse.Api.Application/Synapse.Api.Application.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
<ItemGroup>
4545
<PackageReference Include="IdentityServer4" Version="4.1.2" NoWarn="NU1902" />
4646
<PackageReference Include="IdentityServer4.Storage" Version="4.1.2" NoWarn="NU1902" />
47-
<PackageReference Include="Polly" Version="8.4.1" />
47+
<PackageReference Include="Polly" Version="8.4.2" />
4848
</ItemGroup>
4949

5050
<ItemGroup>

‎src/api/Synapse.Api.Client.Http/Synapse.Api.Client.Http.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343

4444
<ItemGroup>
4545
<PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.8" />
46-
<PackageReference Include="ServerlessWorkflow.Sdk.IO" Version="1.0.0-alpha2.15" />
46+
<PackageReference Include="ServerlessWorkflow.Sdk.IO" Version="1.0.0-alpha2.16" />
4747
<PackageReference Include="System.Reactive" Version="6.0.1" />
4848
</ItemGroup>
4949

‎src/api/Synapse.Api.Http/Synapse.Api.Http.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
<ItemGroup>
4646
<PackageReference Include="Neuroglia.Mediation.AspNetCore" Version="4.15.6" />
4747
<PackageReference Include="Neuroglia.Security.AspNetCore" Version="4.15.6" />
48-
<PackageReference Include="Swashbuckle.AspNetCore.SwaggerGen" Version="6.7.3" />
48+
<PackageReference Include="Swashbuckle.AspNetCore.SwaggerGen" Version="6.8.1" />
4949
</ItemGroup>
5050

5151
<ItemGroup>

‎src/api/Synapse.Api.Server/Program.cs

+1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
builder.Services.AddSynapse(builder.Configuration);
2626
builder.Services.AddSynapseApi();
2727
builder.Services.AddSynapseHttpApi(authority);
28+
builder.Services.AddHostedService<WorkflowDatabaseInitializer>();
2829

2930
var authentication = builder.Services.AddAuthentication(FallbackPolicySchemeDefaults.AuthenticationScheme);
3031
authentication.AddScheme<StaticBearerAuthenticationOptions, StaticBearerAuthenticationHandler>(StaticBearerDefaults.AuthenticationScheme, options =>

‎src/api/Synapse.Api.Server/Synapse.Api.Server.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
<PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="8.0.8" />
3535
<PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly.Server" Version="8.0.8" />
3636
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.0" />
37-
<PackageReference Include="Swashbuckle.AspNetCore.SwaggerUI" Version="6.7.3" />
37+
<PackageReference Include="Swashbuckle.AspNetCore.SwaggerUI" Version="6.8.1" />
3838
</ItemGroup>
3939

4040
<ItemGroup>

‎src/cli/Synapse.Cli/Synapse.Cli.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
3434
<PackageReference Include="moment.net" Version="1.3.4" />
3535
<PackageReference Include="NetEscapades.Configuration.Yaml" Version="3.1.0" />
36-
<PackageReference Include="ServerlessWorkflow.Sdk.IO" Version="1.0.0-alpha2.15" />
36+
<PackageReference Include="ServerlessWorkflow.Sdk.IO" Version="1.0.0-alpha2.16" />
3737
<PackageReference Include="Spectre.Console" Version="0.49.1" />
3838
<PackageReference Include="System.CommandLine.NamingConventionBinder" Version="2.0.0-beta4.22272.1" />
3939
</ItemGroup>

‎src/core/Synapse.Core.Infrastructure/Extensions/IServiceCollectionExtensions.cs

-2
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,6 @@ public static IServiceCollection AddSynapse(this IServiceCollection services, IC
4545
{
4646
services.AddHttpClient();
4747
services.AddSerialization();
48-
services.AddJsonSerializer();
49-
services.AddYamlDotNetSerializer();
5048
services.AddMediator();
5149
services.AddScoped<IUserInfoProvider, UserInfoProvider>();
5250
services.AddServerlessWorkflowIO();

‎src/core/Synapse.Core.Infrastructure/Synapse.Core.Infrastructure.csproj

+2-2
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,13 @@
4444

4545
<ItemGroup>
4646
<PackageReference Include="IdentityModel" Version="7.0.0" />
47-
<PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.0.2" />
47+
<PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.1.1" />
4848
<PackageReference Include="Neuroglia.Data.Expressions.Abstractions" Version="4.15.6" />
4949
<PackageReference Include="Neuroglia.Data.Infrastructure.Redis" Version="4.15.6" />
5050
<PackageReference Include="Neuroglia.Data.Infrastructure.ResourceOriented.Redis" Version="4.15.6" />
5151
<PackageReference Include="Neuroglia.Mediation" Version="4.15.6" />
5252
<PackageReference Include="Neuroglia.Plugins" Version="4.15.6" />
53-
<PackageReference Include="ServerlessWorkflow.Sdk.IO" Version="1.0.0-alpha2.15" />
53+
<PackageReference Include="ServerlessWorkflow.Sdk.IO" Version="1.0.0-alpha2.16" />
5454
</ItemGroup>
5555

5656
<ItemGroup>

‎src/core/Synapse.Core/Synapse.Core.csproj

+2-2
Original file line numberDiff line numberDiff line change
@@ -63,11 +63,11 @@
6363

6464
<ItemGroup>
6565
<PackageReference Include="Docker.DotNet" Version="3.125.15" />
66-
<PackageReference Include="KubernetesClient" Version="14.0.12" />
66+
<PackageReference Include="KubernetesClient" Version="15.0.1" />
6767
<PackageReference Include="Neuroglia.Data.Infrastructure.ResourceOriented" Version="4.15.6" />
6868
<PackageReference Include="Neuroglia.Eventing.CloudEvents" Version="4.15.6" />
6969
<PackageReference Include="Semver" Version="2.3.0" />
70-
<PackageReference Include="ServerlessWorkflow.Sdk" Version="1.0.0-alpha2.15" />
70+
<PackageReference Include="ServerlessWorkflow.Sdk" Version="1.0.0-alpha2.16" />
7171
</ItemGroup>
7272

7373
</Project>

0 commit comments

Comments
 (0)