diff --git a/backend/FwHeadless/FwHeadless.csproj b/backend/FwHeadless/FwHeadless.csproj index 6cb5018ecf..48635318f1 100644 --- a/backend/FwHeadless/FwHeadless.csproj +++ b/backend/FwHeadless/FwHeadless.csproj @@ -11,6 +11,9 @@ + + @@ -35,6 +38,11 @@ + + diff --git a/backend/FwHeadless/FwHeadlessConfig.cs b/backend/FwHeadless/FwHeadlessConfig.cs index 95eb49f3ae..836d732b1f 100644 --- a/backend/FwHeadless/FwHeadlessConfig.cs +++ b/backend/FwHeadless/FwHeadlessConfig.cs @@ -22,6 +22,22 @@ public class FwHeadlessConfig public long MaxUploadFileSizeBytes => MaxUploadFileSizeKb * 1024; public string FdoDataModelVersion { get; init; } = "7000072"; + /// + /// The FLExBridge data version. FieldWorks/LfMergeBridge form the Mercurial branch a project's + /// data lives on by joining this and the FDO model version with a dot (see ). + /// Mirrors FlexBridgeConstants.FlexBridgeDataVersion in FLExBridge, which is internal so we can't + /// reference it directly; bump this if FLExBridge bumps its data version. + /// + public string FlexBridgeDataVersion { get; init; } = "7500002"; + + /// + /// The Mercurial branch FieldWorks/LfMergeBridge keep a project's data on: FlexBridgeDataVersion, + /// a dot, then the FDO model version (e.g. "7500002.7000072"). This is exactly the branch name + /// LfMergeBridge derives from , so a new project's genesis commit + /// must land on this branch for send/receive to find the data. + /// + public string SendReceiveBranchName => $"{FlexBridgeDataVersion}.{FdoDataModelVersion}"; + /// /// Project directory structure in FwHeadless: (Note that FwDataProject.ProjectsPath is the root of a SINGLE project) /// {ProjectStorageRoot}/ diff --git a/backend/FwHeadless/FwHeadlessKernel.cs b/backend/FwHeadless/FwHeadlessKernel.cs index 51700d8f53..769d00c92d 100644 --- a/backend/FwHeadless/FwHeadlessKernel.cs +++ b/backend/FwHeadless/FwHeadlessKernel.cs @@ -25,6 +25,7 @@ public static IServiceCollection AddFwHeadless(this IServiceCollection services) services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); @@ -32,6 +33,14 @@ public static IServiceCollection AddFwHeadless(this IServiceCollection services) .AddLcmCrdtClientCore() .AddFwDataBridge(ServiceLifetime.Scoped) .AddFwLiteProjectSync(); + // The LCM project templates (NewLangProj.fwdata etc.) ship in our output (see FwHeadless.csproj). + // FwDataBridgeConfig's default TemplatesFolder points at the FieldWorks user-data dir, which + // doesn't exist in the container; point it at the shipped templates unless overridden in config. + services.Configure(config => + { + if (config.TemplatesFolder == new FwDataBridgeConfig().TemplatesFolder) + config.TemplatesFolder = Path.Combine(AppContext.BaseDirectory, "Templates"); + }); services.RemoveAll(typeof(IMediaAdapter)); services.AddScoped(); services.AddScoped(); diff --git a/backend/FwHeadless/Program.cs b/backend/FwHeadless/Program.cs index 79e1cb3ae1..cdc459031b 100644 --- a/backend/FwHeadless/Program.cs +++ b/backend/FwHeadless/Program.cs @@ -12,6 +12,17 @@ using WebServiceDefaults; using AppVersion = LexCore.AppVersion; +// Chorus discovers its file-type handler plugins with `new DirectoryCatalog(".", "*-ChorusPlugin.dll")`, +// which scans the *current working directory* -- not the app base dir -- and WebApplication.CreateBuilder +// does not change CWD. When launched from anywhere other than the folder holding the co-published +// LibFLExBridge-ChorusPlugin.dll (e.g. `dotnet run`, whose CWD is the project dir), that plugin isn't +// found. Without it, FieldWorks extensions like `.list` fall back to Chorus's 1 MB LargeFileFilter +// default, so the >1 MB SemanticDomainList.list is silently filtered out of a new project's first push +// (and later merges lose their FieldWorks handlers too). Pin CWD to the base dir so the plugin is always +// discovered. Safe here: hg runs with explicit working dirs and TemplatesFolder resolves against +// AppContext.BaseDirectory already. +Environment.CurrentDirectory = AppContext.BaseDirectory; + var builder = WebApplication.CreateBuilder(args); // Add services to the container. @@ -83,6 +94,7 @@ app.MapDefaultEndpoints(); app.MapMediaFileRoutes(); app.MapMergeRoutes(); +app.MapProjectRoutes(); // DELETE endpoint to delete the FieldWorks repo/project (and nothing else) app.MapDelete("/api/manage/repo/{projectId}", async (Guid projectId, ProjectDeletionService deletionService) => diff --git a/backend/FwHeadless/Routes/ProjectRoutes.cs b/backend/FwHeadless/Routes/ProjectRoutes.cs new file mode 100644 index 0000000000..3e8e42b52f --- /dev/null +++ b/backend/FwHeadless/Routes/ProjectRoutes.cs @@ -0,0 +1,53 @@ +using FwHeadless.Services; +using LexCore.Entities; +using Microsoft.AspNetCore.Http.HttpResults; +using SIL.WritingSystems; + +namespace FwHeadless.Routes; + +public static class ProjectRoutes +{ + public static IEndpointConventionBuilder MapProjectRoutes(this WebApplication app) + { + var group = app.MapGroup("/api/project"); + group.MapPost("/init-fwdata-project", InitFwDataProject); + return group; + } + + // Internal endpoint: LexBoxApi has already created the project row + empty repo and performed the + // admin-auth check. FwHeadless populates the empty repo with a template .fwdata configured for the + // requested writing systems. Not idempotent — on failure LexBoxApi deletes and recreates. + private static async Task> InitFwDataProject( + Guid projectId, + InitFwDataProjectInput input, + IProjectLookupService projectLookupService, + ProjectCreationService projectCreationService, + ILogger logger) + { + // Null-safe against a malformed body (System.Text.Json leaves an absent list null). + if (input.WsVernacular is not { Count: > 0 }) + return TypedResults.Problem("At least one vernacular writing system is required", + statusCode: StatusCodes.Status400BadRequest); + if (input.WsAnalysis is not { Count: > 0 }) + return TypedResults.Problem("At least one analysis writing system is required", + statusCode: StatusCodes.Status400BadRequest); + var invalidWs = input.WsVernacular.Concat(input.WsAnalysis).FirstOrDefault(ws => !IetfLanguageTag.IsValid(ws)); + if (invalidWs is not null) + return TypedResults.Problem($"Invalid writing system code: {invalidWs}", + statusCode: StatusCodes.Status400BadRequest); + if (string.IsNullOrEmpty(input.WsUi) || !IetfLanguageTag.IsValid(input.WsUi)) + return TypedResults.Problem($"Invalid UI writing system code: {input.WsUi}", + statusCode: StatusCodes.Status400BadRequest); + + var projectCode = await projectLookupService.GetProjectCode(projectId); + if (projectCode is null) + { + logger.LogError("Init-fwdata-project request for non-existent project {ProjectId}", projectId); + return TypedResults.Problem("Project not found", statusCode: StatusCodes.Status404NotFound); + } + + await projectCreationService.InitFwDataProject( + projectId, projectCode, input.WsVernacular, input.WsAnalysis, input.WsUi); + return TypedResults.Ok(); + } +} diff --git a/backend/FwHeadless/Services/ISendReceiveService.cs b/backend/FwHeadless/Services/ISendReceiveService.cs index 4755ff1ac8..98c9a573e9 100644 --- a/backend/FwHeadless/Services/ISendReceiveService.cs +++ b/backend/FwHeadless/Services/ISendReceiveService.cs @@ -10,4 +10,7 @@ public interface ISendReceiveService Task PendingCommitCountOutgoing(FwDataProject project, string? projectCode); Task PendingCommitCountBothWays(FwDataProject project, string? projectCode); Task CommitFile(string filePath, string commitMessage); + Task CommitEmpty(string folder, string commitMessage); + Task InitRepo(string folder); + Task SetBranch(string folder, string branchName); } diff --git a/backend/FwHeadless/Services/ProjectCreationService.cs b/backend/FwHeadless/Services/ProjectCreationService.cs new file mode 100644 index 0000000000..169693468a --- /dev/null +++ b/backend/FwHeadless/Services/ProjectCreationService.cs @@ -0,0 +1,210 @@ +using System.Xml; +using System.Xml.Linq; +using FwDataMiniLcmBridge; +using FwDataMiniLcmBridge.LcmUtils; +using LexCore.Exceptions; +using Microsoft.Extensions.Options; +using SIL.Xml; + +namespace FwHeadless.Services; + +/// +/// Creates a brand-new FieldWorks project from the SIL.LCModel template and pushes it into the +/// (empty) Mercurial repo that LexBox has already initialised for the project. +/// +public class ProjectCreationService( + IOptions config, + ISendReceiveService srService, + IProjectLoader projectLoader, + SyncHostedService syncHostedService, + ILogger logger) +{ + private const string InitialCommitMessage = "Initial project creation"; + + // The one file FieldWorks/FLExBridge put in a brand-new FLEx repo's first commit: a minimal custom + // property definition list. Mirrors FlexBridgeConstants.CustomPropertiesFilename (internal in FLExBridge). + private const string CustomPropertiesFilename = "FLExProject.CustomProperties"; + + public async Task InitFwDataProject( + Guid projectId, + string projectCode, + IReadOnlyList vernacularWritingSystems, + IReadOnlyList analysisWritingSystems, + string uiWritingSystem) + { + if (vernacularWritingSystems.Count == 0) + throw new ArgumentException("At least one vernacular writing system is required", nameof(vernacularWritingSystems)); + if (analysisWritingSystems.Count == 0) + throw new ArgumentException("At least one analysis writing system is required", nameof(analysisWritingSystems)); + if (string.IsNullOrEmpty(uiWritingSystem)) + throw new ArgumentException("A UI writing system is required", nameof(uiWritingSystem)); + + // Reserve the project so a concurrent create or sync can't race on the same fw/ folder and repo. + if (!syncHostedService.TryStartProjectCreation(projectId)) + throw new ProjectSyncInProgressException(projectId); + var fwDataProject = config.Value.GetFwDataProject(projectCode, projectId); + try + { + // Build the first commit of a brand-new FLEx repo from scratch, mirroring how FieldWorks/ + // FLExBridge create a repo, since Clone can't establish an empty remote and + // Language_Forge_Send_Receive refuses to make the *first* commit itself: + // 1. hg init the local fw/ folder (no clone -- the empty remote has nothing to clone). + // 2. Build fw.fwdata into that folder from the SIL.LCModel template, configured with all + // of the requested writing systems. + // 3. Commit a minimal FLExProject.CustomProperties file on hg's *default* branch -- the same + // genesis file, on the same branch, FieldWorks commits first. The fwdata itself is NOT + // committed; Send/Receive splits it into the nested files Mercurial actually tracks + // (fwdata is excluded from tracking). + // 4. Set the branch to the send/receive branch name (FlexBridgeDataVersion.modelVersion, + // e.g. 7500002.7000072) AFTER the genesis commit; FLEx/LfMergeBridge look for the data on + // that branch, so the split data must land there while the genesis stays on 'default'. + // 5. Send/Receive: split fw.fwdata into its nested files, commit those on the branch, push. + await srService.InitRepo(fwDataProject.ProjectFolder); + + BuildFromTemplate(fwDataProject, vernacularWritingSystems, analysisWritingSystems, uiWritingSystem); + FixLinkedFilesRootDirSeparator(fwDataProject); + AssertModelVersionMatches(fwDataProject); + + var customPropertiesFile = WriteInitialCustomPropertiesFile(fwDataProject); + await srService.CommitFile(customPropertiesFile, InitialCommitMessage); + + await srService.SetBranch(fwDataProject.ProjectFolder, config.Value.SendReceiveBranchName); + // BUG WORKAROUND (remove once the LanguageForgeSendReceiveActionHandler bug is fixed): that + // handler refuses to commit when doing so "could possibly create a new branch", so it won't + // establish the model-version branch itself. This empty commit (no file changes -- it only + // records the branch change from SetBranch above) creates a head on the branch so Send/Receive + // will proceed. + await srService.CommitEmpty(fwDataProject.ProjectFolder, InitialCommitMessage); + var pushResult = await srService.SendReceive(fwDataProject, projectCode, InitialCommitMessage); + if (!pushResult.Success) + throw new SendReceiveException("Pushing the new project to the repo failed", pushResult); + + logger.LogInformation("Created project {ProjectCode} ({ProjectId}) from template", projectCode, projectId); + } + catch + { + CleanupLocalProject(fwDataProject); + throw; + } + finally + { + syncHostedService.EndProjectCreation(projectId); + } + } + + private void BuildFromTemplate( + FwDataProject fwDataProject, + IReadOnlyList vernacularWritingSystems, + IReadOnlyList analysisWritingSystems, + string uiWs) + { + // NewProject copies the SIL.LCModel template, configured with all of the requested writing + // systems (the first of each list is the default of that type), and returns a loaded cache; + // dispose it right away so its file locks are released before the following hg add/commit. + using var cache = projectLoader.NewProject(fwDataProject, analysisWritingSystems, vernacularWritingSystems, uiWs); + } + + /// + /// FieldWorks (Windows) compares LangProject.LinkedFilesRootDir with an exact string, expecting the + /// Windows-style default %proj%\LinkedFiles. liblcm builds that path with Path.Combine, which on + /// this Linux host yields %proj%/LinkedFiles (forward slash), so FieldWorks warns on its first + /// Send/Receive. Rewrite just that one value to the backslash form. + /// + /// We deliberately do NOT parse the (potentially multi-MB) fwdata as XML -- that would be a needless + /// cost. The value always sits on the line immediately after the <LinkedFilesRootDir> open tag, + /// so stream the file once, fix the <Uni> on the line following that tag, and copy every other + /// line through untouched. + /// + private static void FixLinkedFilesRootDirSeparator(FwDataProject fwDataProject) + { + const string markerLinePrefix = "%proj%\LinkedFiles"; + + var sourcePath = fwDataProject.FilePath; + var tempPath = sourcePath + ".tmp"; + using (var reader = new StreamReader(sourcePath)) + using (var writer = new StreamWriter(tempPath)) + { + var nextLineIsValue = false; + var done = false; + string? line; + while ((line = reader.ReadLine()) is not null) + { + if (nextLineIsValue) + { + line = line.Replace(forwardSlashValue, backslashValue); + nextLineIsValue = false; + done = true; + } + else if (!done && line.StartsWith(markerLinePrefix, StringComparison.Ordinal)) + { + nextLineIsValue = true; + } + writer.WriteLine(line); + } + } + File.Move(tempPath, sourcePath, overwrite: true); + } + + /// + /// Writes the genesis FLExProject.CustomProperties file into the repo root: an empty custom-property + /// list (just an <AdditionalFields /> root). Uses SIL's canonical XML settings so the bytes match + /// exactly what the first Send/Receive's project splitter regenerates, leaving no spurious diff. + /// + private string WriteInitialCustomPropertiesFile(FwDataProject fwDataProject) + { + var path = Path.Join(fwDataProject.ProjectFolder, CustomPropertiesFilename); + using (var writer = XmlWriter.Create(path, CanonicalXmlSettings.CreateXmlWriterSettings())) + { + writer.WriteStartDocument(); + new XElement("AdditionalFields").WriteTo(writer); + } + return path; + } + + /// + /// The freshly-created project is stamped with the SIL.LCModel package's data-model version. + /// If that ever diverges from the version FwHeadless syncs with, pushing it would corrupt data + /// (see FwHeadless AGENTS.md), so fail before the push rather than after. + /// + private void AssertModelVersionMatches(FwDataProject fwDataProject) + { + var expected = config.Value.FdoDataModelVersion; + var actual = ReadModelVersion(fwDataProject.FilePath); + // Fail closed: an unreadable version ('actual' == null) is treated as a mismatch, since this is + // the last safety net before an irreversible push in a data-loss-risk area. + if (actual != expected) + { + throw new InvalidOperationException( + $"New project's data-model version '{actual ?? "(unreadable)"}' does not match FwHeadless " + + $"FdoDataModelVersion '{expected}'; refusing to push to avoid data corruption."); + } + } + + private static string? ReadModelVersion(string fwDataFilePath) + { + using var reader = XmlReader.Create(fwDataFilePath, new XmlReaderSettings { IgnoreComments = true, IgnoreWhitespace = true }); + // The .fwdata root element carries the FDO model version, e.g. . + return reader.MoveToContent() == XmlNodeType.Element ? reader.GetAttribute("version") : null; + } + + private void CleanupLocalProject(FwDataProject fwDataProject) + { + // Mirror SyncWorker.SetupFwData's failure cleanup so a retry starts clean. The remote repo + // (created by LexBox) is torn down by the caller on the LexBox side. + try + { + if (Directory.Exists(fwDataProject.ProjectFolder)) + Directory.Delete(fwDataProject.ProjectFolder, true); + if (Directory.Exists(fwDataProject.ProjectsPath) && + !Directory.EnumerateFileSystemEntries(fwDataProject.ProjectsPath).Any()) + Directory.Delete(fwDataProject.ProjectsPath); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to clean up local project folder after a failed creation: {ProjectFolder}", + fwDataProject.ProjectFolder); + } + } +} diff --git a/backend/FwHeadless/Services/SendReceiveHelpers.cs b/backend/FwHeadless/Services/SendReceiveHelpers.cs index 4caebce379..04b7c6f0e3 100644 --- a/backend/FwHeadless/Services/SendReceiveHelpers.cs +++ b/backend/FwHeadless/Services/SendReceiveHelpers.cs @@ -160,6 +160,41 @@ public static async Task CommitFile(string filePath, string commitMessage, IProg await ExecuteHgSuccess($"hg commit --config ui.username={EscapeShellArg(HgUsername)} --message {EscapeShellArg(commitMessage)}", fileDir, progress); } + public static async Task CommitEmpty(string folder, string commitMessage, IProgress? progress = null) + { + using var activity = FwHeadlessActivitySource.Value.StartActivity(); + activity?.SetTag("app.folder", folder); + progress ??= new NullProgress(); + // No `hg add` and no file changes: this succeeds only because a pending branch change (from a + // preceding SetBranch) is itself a committable change in Mercurial, so it records a new head on + // that branch. A plain commit with truly nothing changed would exit 1 and throw. + await ExecuteHgSuccess($"hg commit --config ui.username={EscapeShellArg(HgUsername)} --message {EscapeShellArg(commitMessage)}", folder, progress); + } + + public static async Task InitRepo(string folder, IProgress? progress = null) + { + using var activity = FwHeadlessActivitySource.Value.StartActivity(); + activity?.SetTag("app.folder", folder); + progress ??= new NullProgress(); + Directory.CreateDirectory(folder); + // Use Chorus rather than a bare `hg init`: CreateOrUseExisting also wires up the custom hg + // extensions FwHeadless relies on (e.g. fixutf8) so later send/receive works correctly. + await Task.Run(() => HgRepository.CreateOrUseExisting(folder, progress)); + } + + public static async Task SetBranch(string folder, string branchName, IProgress? progress = null) + { + using var activity = FwHeadlessActivitySource.Value.StartActivity(); + activity?.SetTag("app.branch", branchName); + progress ??= new NullProgress(); + // FLEx repos keep their data on a branch named after the FLExBridge data + FDO model version + // (e.g. 7500002.7000072); a clone looking for that branch reports "no such branch" if the initial + // commit landed on 'default' instead. hg records the branch for the *next* commit, so this must + // run before the first commit. Numeric/dotted branch names are accepted thanks to the fixutf8 + // extension wired into Mercurial/mercurial.ini. + await ExecuteHgSuccess($"hg branch --force {EscapeShellArg(branchName)}", folder, progress); + } + private static string EscapeShellArg(string arg) { var quote = """ diff --git a/backend/FwHeadless/Services/SendReceiveService.cs b/backend/FwHeadless/Services/SendReceiveService.cs index 735e90b8ae..3ef8955a36 100644 --- a/backend/FwHeadless/Services/SendReceiveService.cs +++ b/backend/FwHeadless/Services/SendReceiveService.cs @@ -70,4 +70,19 @@ public async Task CommitFile(string filePath, string commitMessage) { await SendReceiveHelpers.CommitFile(filePath, commitMessage, progress); } + + public async Task CommitEmpty(string folder, string commitMessage) + { + await SendReceiveHelpers.CommitEmpty(folder, commitMessage, progress); + } + + public async Task InitRepo(string folder) + { + await SendReceiveHelpers.InitRepo(folder, progress); + } + + public async Task SetBranch(string folder, string branchName) + { + await SendReceiveHelpers.SetBranch(folder, branchName, progress); + } } diff --git a/backend/FwHeadless/Services/SyncHostedService.cs b/backend/FwHeadless/Services/SyncHostedService.cs index 91b77edead..d01b878593 100644 --- a/backend/FwHeadless/Services/SyncHostedService.cs +++ b/backend/FwHeadless/Services/SyncHostedService.cs @@ -20,6 +20,14 @@ public class SyncHostedService(IServiceProvider services, ILogger _projectsToSync = Channel.CreateUnbounded(); private readonly ConcurrentDictionary> _projectsQueuedOrRunning = new(); + // Projects currently being created from a template. Tracked here (not just in ProjectCreationService) + // so a sync job can't be queued for a project mid-creation and race on the same fw/ folder and repo. + private readonly ConcurrentDictionary _projectsBeingCreated = new(); + + // Serialises the check-then-add across the two dictionaries above, so a creation and a sync can't + // both slip through by interleaving their checks. + private readonly Lock _reservationLock = new(); + protected override async Task ExecuteAsync(CancellationToken stoppingToken) { await foreach (var projectId in _projectsToSync.Reader.ReadAllAsync(stoppingToken)) @@ -54,7 +62,26 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) public virtual bool IsJobQueuedOrRunning(Guid projectId) { - return _projectsQueuedOrRunning.ContainsKey(projectId); + return _projectsQueuedOrRunning.ContainsKey(projectId) || _projectsBeingCreated.ContainsKey(projectId); + } + + /// + /// Reserves a project for creation-from-template, blocking a concurrent creation or sync of the + /// same project. Returns false if a sync is already queued/running or another creation is in flight. + /// Pair with in a finally block. + /// + public bool TryStartProjectCreation(Guid projectId) + { + lock (_reservationLock) + { + if (_projectsQueuedOrRunning.ContainsKey(projectId)) return false; + return _projectsBeingCreated.TryAdd(projectId, 0); + } + } + + public void EndProjectCreation(Guid projectId) + { + _projectsBeingCreated.TryRemove(projectId, out _); } public async Task AwaitSyncFinished(Guid projectId, CancellationToken cancellationToken) @@ -66,8 +93,18 @@ public virtual bool IsJobQueuedOrRunning(Guid projectId) public bool QueueJob(Guid projectId) { - //will only queue job if it's not already queued - var addedToQueue = _projectsQueuedOrRunning.TryAdd(projectId, new()); + bool addedToQueue; + lock (_reservationLock) + { + //don't sync a project while it's still being created from a template (would race on the repo) + if (_projectsBeingCreated.ContainsKey(projectId)) + { + logger.LogInformation("Project {ProjectId} is being created, not queueing a sync job", projectId); + return false; + } + //will only queue job if it's not already queued + addedToQueue = _projectsQueuedOrRunning.TryAdd(projectId, new()); + } if (addedToQueue) { if (!_projectsToSync.Writer.TryWrite(projectId)) diff --git a/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/MockFwProjectLoader.cs b/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/MockFwProjectLoader.cs index 890bca1624..4123cdd39f 100644 --- a/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/MockFwProjectLoader.cs +++ b/backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/MockFwProjectLoader.cs @@ -19,7 +19,7 @@ public override LcmCache LoadCache(FwDataProject project) return cache; } - public override LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs) + public override LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string uiWs = "en") { Init(); var lcmDirectories = new LcmDirectories(project.ProjectsPath, TemplatesFolder); @@ -28,7 +28,7 @@ public override LcmCache NewProject(FwDataProject project, string analysisWs, st new SimpleProjectId(BackendProviderType.kMemoryOnly, Path.GetFullPath(project.FilePath)), analysisWs, vernacularWs, - null, + uiWs, new LfLcmUi(progress.SynchronizeInvoke), lcmDirectories, new LcmSettings()); diff --git a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs index 3d64f8016e..e9a2f6f52a 100644 --- a/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs +++ b/backend/FwLite/FwDataMiniLcmBridge/LcmUtils/ProjectLoader.cs @@ -19,7 +19,13 @@ public interface IProjectLoader /// LcmCache LoadCache(FwDataProject project); - LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs); + LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string uiWs = "en"); + + /// + /// Like the single-WS overload, but the new project also gets every analysis/vernacular writing + /// system in the given lists. The first item of each list is the default of that type. + /// + LcmCache NewProject(FwDataProject project, IReadOnlyList analysisWss, IReadOnlyList vernacularWss, string uiWs = "en"); } public class ProjectLoader(IOptions config) : IProjectLoader @@ -75,27 +81,57 @@ public virtual LcmCache LoadCache(FwDataProject project) return cache; } - public virtual LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs) + public virtual LcmCache NewProject(FwDataProject project, string analysisWs, string vernacularWs, string uiWs = "en") + { + return NewProject(project, [analysisWs], [vernacularWs], uiWs); + } + + public virtual LcmCache NewProject(FwDataProject project, + IReadOnlyList analysisWss, + IReadOnlyList vernacularWss, + string uiWs = "en") { Init(); var lcmDirectories = new LcmDirectories(project.ProjectsPath, TemplatesFolder); var progress = new LcmThreadedProgress(); + var analysisDefinitions = analysisWss.Select(CreateWritingSystemDefinition).ToList(); + var vernacularDefinitions = vernacularWss.Select(CreateWritingSystemDefinition).ToList(); NewProject(progress, project.Name, lcmDirectories, progress.SynchronizeInvoke, - new CoreWritingSystemDefinition(analysisWs) { Id = analysisWs }, - new CoreWritingSystemDefinition(vernacularWs) { Id = vernacularWs }); + analysisDefinitions[0], + vernacularDefinitions[0], + uiWs, + AdditionalWritingSystems(analysisDefinitions), + AdditionalWritingSystems(vernacularDefinitions)); return LoadCache(project); } + private static CoreWritingSystemDefinition CreateWritingSystemDefinition(string ws) => new(ws) { Id = ws }; + + // The default writing system (the first of the list) is passed to CreateNewLangProj on its own, so + // the "additional" set is the whole list as a hash set with that default removed. + private static HashSet AdditionalWritingSystems(List definitions) + { + var additional = definitions.ToHashSet(); + additional.Remove(definitions[0]); + return additional; + } + private static void NewProject(IThreadedProgress progress, string projectName, ILcmDirectories lcmDirectories, ISynchronizeInvoke syncInvoke, CoreWritingSystemDefinition analysisWs, - CoreWritingSystemDefinition vernacularWs) + CoreWritingSystemDefinition vernacularWs, + string uiWs, + HashSet additionalAnalysisWss, + HashSet additionalVernacularWss) { - LcmCache.CreateNewLangProj(progress, [projectName, lcmDirectories, syncInvoke, analysisWs, vernacularWs]); + // After the default analysis/vernacular definitions and the string UI writing system, + // CreateNewLangProj takes the additional analysis then additional vernacular writing systems. + LcmCache.CreateNewLangProj(progress, + [projectName, lcmDirectories, syncInvoke, analysisWs, vernacularWs, uiWs, additionalAnalysisWss, additionalVernacularWss]); } } diff --git a/backend/FwLite/FwLiteProjectSync.Tests/ProjectTemplateTests.cs b/backend/FwLite/FwLiteProjectSync.Tests/ProjectTemplateTests.cs index b3e298cefd..6469b12e33 100644 --- a/backend/FwLite/FwLiteProjectSync.Tests/ProjectTemplateTests.cs +++ b/backend/FwLite/FwLiteProjectSync.Tests/ProjectTemplateTests.cs @@ -110,6 +110,26 @@ public async Task CreateFromTemplateMatchesTemplateFilePlusRequestedVernacular() actual.Should().BeEquivalentTo(expected); } + [Fact] + public async Task NewProjectAddsEveryRequestedWritingSystemToTheCorrectList() + { + var fwDataBridgeConfig = Services.GetRequiredService>().Value; + // The SIL.LCModel template ships next to the test assembly; point liblcm at it (this project, + // unlike FwDataMiniLcmBridge.Tests, doesn't otherwise configure TemplatesFolder). + fwDataBridgeConfig.TemplatesFolder = Path.Combine(AppContext.BaseDirectory, "Templates"); + var fwDataProject = new FwDataProject("multi-ws-source", fwDataBridgeConfig.ProjectsFolder); + // First of each list is the default; the rest go through CreateNewLangProj's additional-WS sets. + // Distinct codes per type so a swapped analysis/vernacular argument would fail this test. + string[] analysis = ["en", "de"]; + string[] vernacular = ["fr", "es"]; + Services.GetRequiredService().NewProject(fwDataProject, analysis, vernacular, uiWs: "en").Dispose(); + + using var api = Services.GetRequiredService().GetFwDataMiniLcmApi(fwDataProject, false); + var writingSystems = await api.GetWritingSystems(); + writingSystems.Analysis.Select(ws => ws.WsId).Should().Contain(["en", "de"]); + writingSystems.Vernacular.Select(ws => ws.WsId).Should().Contain(["fr", "es"]); + } + private static ProjectSnapshot DeserializeTemplate() { using var stream = File.OpenRead(TemplatePath); diff --git a/backend/LexBoxApi/Controllers/ProjectController.cs b/backend/LexBoxApi/Controllers/ProjectController.cs index b05e8b058c..5360d60d69 100644 --- a/backend/LexBoxApi/Controllers/ProjectController.cs +++ b/backend/LexBoxApi/Controllers/ProjectController.cs @@ -1,6 +1,9 @@ +using System.Net; +using System.Text.RegularExpressions; using LexBoxApi.Auth.Attributes; using LexBoxApi.Controllers.ActionResults; using LexBoxApi.Jobs; +using LexBoxApi.Models.Project; using LexBoxApi.Services; using LexCore.Entities; using LexCore.Exceptions; @@ -20,9 +23,119 @@ public class ProjectController( IHgService hgService, LexBoxDbContext lexBoxDbContext, IPermissionService permissionService, - ISchedulerFactory scheduler) + ISchedulerFactory scheduler, + FwHeadlessClient fwHeadlessClient, + ILogger logger) : ControllerBase { + /// + /// Admin-only: create a new FLEx project whose repo is populated with a template .fwdata + /// (from the SIL.LCModel package) configured for the requested writing systems. + /// + /// Project code for the new project. + /// Vernacular writing system id(s); at least one is required. Repeat the query param for multiple. + /// Analysis writing system id(s); defaults to ["en"] when none are given. Repeat the query param for multiple. + /// Optional display name; defaults to the code. + /// Optional to record as the project's origin; admin-only. + [HttpPost("initFwDataProject")] + [AdminRequired] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + [ProducesResponseType(StatusCodes.Status409Conflict)] + [ProducesResponseType(StatusCodes.Status500InternalServerError)] + public async Task> InitFwDataProject( + string code, + [FromQuery] string[] wsVernacular, + [FromQuery] string[]? wsAnalysis = null, + string wsUi = "en", + string? name = null, + string? projectOrigin = null, + CancellationToken cancellationToken = default) + { + if (wsVernacular is null || wsVernacular.Length == 0) + return Problem("At least one vernacular writing system is required", statusCode: StatusCodes.Status400BadRequest); + if (code is null || code.Length < 4 || !Regex.IsMatch(code, Project.ProjectCodeRegex)) + return Problem($"Invalid project code '{code}'", statusCode: StatusCodes.Status400BadRequest); + + ProjectMigrationStatus? origin = null; + if (!string.IsNullOrEmpty(projectOrigin)) + { + // Only admins may say where a project came from. Currently redundant with [AdminRequired], + // but this endpoint is expected to open up to non-admins later; this check must stay. + permissionService.AssertIsAdmin(); + if (!TryParseProjectOrigin(projectOrigin, out var parsedOrigin)) + return Problem($"Invalid project origin '{projectOrigin}'", statusCode: StatusCodes.Status400BadRequest); + origin = parsedOrigin; + } + if (await projectService.ProjectExists(code)) + return Problem($"A project with code '{code}' already exists", statusCode: StatusCodes.Status409Conflict); + + var wsAnalysisOrDefault = AnalysisWritingSystemsOrDefault(wsAnalysis); + + var projectId = await projectService.CreateProject(new CreateProjectInput( + Id: null, + Name: string.IsNullOrWhiteSpace(name) ? code : name, + Description: string.Empty, + Code: code, + Type: ProjectType.FLEx, + RetentionPolicy: RetentionPolicy.Verified, + IsConfidential: false, + ProjectManagerId: null, + OrgId: null), + projectOrigin: origin); + + // Saga: the project row + empty repo now exist. Have FwHeadless populate the repo with the + // template .fwdata; if that fails, compensate by tearing the project back down. Cleanup runs + // on a fresh token so a client disconnect can't skip it. + (HttpStatusCode statusCode, string? error) result; + try + { + result = await fwHeadlessClient.InitFwDataProject(projectId, wsVernacular, wsAnalysisOrDefault, wsUi, cancellationToken); + } + catch + { + await CleanupFailedCreation(projectId, code); + throw; + } + if (result.error is not null) + { + await CleanupFailedCreation(projectId, code); + // Surface a bad request from FwHeadless (e.g. an invalid writing-system tag) as 400, not 500. + var statusCode = result.statusCode == HttpStatusCode.BadRequest + ? StatusCodes.Status400BadRequest + : StatusCodes.Status500InternalServerError; + return Problem($"Failed to create the project from the template: {result.error}", statusCode: statusCode); + } + + await projectService.UpdateLastCommit(code); + return projectId; + } + + /// + /// Parses a project origin. Enum.TryParse also accepts raw numbers ("1") and comma-separated lists + /// ("Migrated,Migrating"), so the parsed value must round-trip to the name that was passed in. + /// + public static bool TryParseProjectOrigin(string projectOrigin, out ProjectMigrationStatus origin) => + Enum.TryParse(projectOrigin, ignoreCase: true, out origin) + && Enum.IsDefined(origin) + && string.Equals(origin.ToString(), projectOrigin, StringComparison.OrdinalIgnoreCase); + + /// Analysis writing systems to use, defaulting to English when none are supplied. + public static string[] AnalysisWritingSystemsOrDefault(string[]? wsAnalysis) => + wsAnalysis is { Length: > 0 } ? wsAnalysis : ["en"]; + + private async Task CleanupFailedCreation(Guid projectId, string code) + { + try + { + await projectService.CleanupFailedProjectCreation(projectId, code); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to roll back project {ProjectId} ({Code}) after a failed template creation", projectId, code); + } + } + [HttpPost("refreshProjectLastChanged")] public async Task RefreshProjectLastChanged(string projectCode) { diff --git a/backend/LexBoxApi/LexBoxKernel.cs b/backend/LexBoxApi/LexBoxKernel.cs index 25033c4023..eeda566ed3 100644 --- a/backend/LexBoxApi/LexBoxKernel.cs +++ b/backend/LexBoxApi/LexBoxKernel.cs @@ -60,7 +60,13 @@ public static void AddLexBoxApi(this IServiceCollection services, .ValidateOnStart(); services.AddHttpClient(); services.AddServiceDiscovery(); - services.AddHttpClient(client => client.BaseAddress = new ("http://fwHeadless")) + services.AddHttpClient(client => + { + client.BaseAddress = new ("http://fwHeadless"); + // init-fwdata-project runs inline (clone empty repo + build the template project + push), + // which can exceed the default 100s on a cold LCM load; give FwHeadless calls more headroom. + client.Timeout = TimeSpan.FromMinutes(5); + }) .AddServiceDiscovery();//service discovery means that we lookup the hostname in Services__fwHeadless__http in config services.AddHttpContextAccessor(); services.AddMemoryCache(); diff --git a/backend/LexBoxApi/Services/FwHeadlessClient.cs b/backend/LexBoxApi/Services/FwHeadlessClient.cs index 0ffc587077..30693ef257 100644 --- a/backend/LexBoxApi/Services/FwHeadlessClient.cs +++ b/backend/LexBoxApi/Services/FwHeadlessClient.cs @@ -1,5 +1,7 @@ using System.Net; +using System.Net.Http.Json; using System.Text.Json; +using LexCore.Entities; using LexCore.Exceptions; using LexCore.Sync; using Microsoft.AspNetCore.Mvc; @@ -197,4 +199,30 @@ public async Task UnblockProject(Guid projectId) return null; } + /// + /// Populates a newly-created (empty) project's repo with a template .fwdata configured for the + /// given writing systems. Runs inline (a new empty project is small). Returns a null Error on + /// success; on failure returns the FwHeadless status code and error body so the caller can + /// propagate the right status (e.g. surface a 400 rather than flattening it to 500). + /// + public async Task<(HttpStatusCode StatusCode, string? Error)> InitFwDataProject(Guid projectId, + IReadOnlyList wsVernacular, + IReadOnlyList wsAnalysis, + string wsUi, + CancellationToken cancellationToken = default) + { + var input = new InitFwDataProjectInput(wsVernacular, wsAnalysis, wsUi); + var response = await httpClient.PostAsJsonAsync($"/api/project/init-fwdata-project?projectId={projectId}", + input, + cancellationToken); + if (response.IsSuccessStatusCode) return (response.StatusCode, null); + var responseBody = await response.Content.ReadAsStringAsync(cancellationToken); + logger.LogError("Failed to init fwdata project: {StatusCode} {StatusDescription}, projectId: {ProjectId}, response: {Response}", + response.StatusCode, + response.ReasonPhrase, + projectId, + responseBody); + return (response.StatusCode, responseBody); + } + } diff --git a/backend/LexBoxApi/Services/ProjectService.cs b/backend/LexBoxApi/Services/ProjectService.cs index dcab6b7b72..5aa82c252e 100644 --- a/backend/LexBoxApi/Services/ProjectService.cs +++ b/backend/LexBoxApi/Services/ProjectService.cs @@ -25,7 +25,7 @@ public class ProjectService( IEmailService emailService, FwHeadlessClient fwHeadless) { - public async Task CreateProject(CreateProjectInput input) + public async Task CreateProject(CreateProjectInput input, ProjectMigrationStatus? projectOrigin = null) { await using var transaction = await dbContext.Database.BeginTransactionAsync(); var projectId = input.Id ?? Guid.NewGuid(); @@ -41,7 +41,7 @@ public async Task CreateProject(CreateProjectInput input) Id = projectId, Code = input.Code, Name = input.Name, - ProjectOrigin = ProjectMigrationStatus.Migrated, + ProjectOrigin = projectOrigin ?? ProjectMigrationStatus.Migrated, Description = input.Description, Type = input.Type, LastCommit = null, @@ -232,6 +232,31 @@ public async Task FinishReset(string code, Stream? zipFile = null) return project; } + /// + /// Compensating cleanup for a project whose template population failed after CreateProject: + /// deletes FwHeadless's local project state, then removes the repo and the project row and + /// invalidates the same caches as a permanent delete. + /// Unlike DeleteProjectPermanently this is not gated on retention policy, because it only undoes a + /// project this same request just created. + /// + public async Task CleanupFailedProjectCreation(Guid projectId, string code) + { + // do this first, because it throws if a creation or sync is in progress, which stops us from + // tearing down an active or already-completed creation (mirrors DeleteProjectPermanently) + await fwHeadless.DeleteProject(projectId); + var project = await dbContext.Projects.FindAsync(projectId); + if (project is not null) + { + dbContext.Projects.Remove(project); + await dbContext.SaveChangesAsync(); + } + await hgService.DeleteRepoIfExists(code); + // Don't forget to add more Invalidate calls here if we add new caches + InvalidateProjectCodeCache(code); + InvalidateProjectConfidentialityCache(projectId); + InvalidateProjectOrgIdsCache(projectId); + } + public async ValueTask LookupProjectOrgIds(Guid projectId) { var cacheKey = $"ProjectOrgsForId:{projectId}"; diff --git a/backend/LexCore/Entities/InitFwDataProjectInput.cs b/backend/LexCore/Entities/InitFwDataProjectInput.cs new file mode 100644 index 0000000000..ba2e2b72a9 --- /dev/null +++ b/backend/LexCore/Entities/InitFwDataProjectInput.cs @@ -0,0 +1,11 @@ +namespace LexCore.Entities; + +/// +/// Body of the internal FwHeadless "init fwdata project" call. The vernacular/analysis +/// lists and the UI writing system are already validated and defaulted by LexBoxApi (at least one +/// vernacular; analysis and UI default to "en"). +/// +public record InitFwDataProjectInput( + IReadOnlyList WsVernacular, + IReadOnlyList WsAnalysis, + string WsUi); diff --git a/backend/LexCore/Entities/Project.cs b/backend/LexCore/Entities/Project.cs index 667548365a..1b271d44a0 100644 --- a/backend/LexCore/Entities/Project.cs +++ b/backend/LexCore/Entities/Project.cs @@ -94,6 +94,7 @@ public enum ProjectMigrationStatus Migrating, PrivateRedmine, PublicRedmine, + LanguageForgeNonSR, } public enum ResetStatus diff --git a/backend/Testing/ApiTests/ApiTestBase.cs b/backend/Testing/ApiTests/ApiTestBase.cs index 74d9492b0c..e0ffec9e8e 100644 --- a/backend/Testing/ApiTests/ApiTestBase.cs +++ b/backend/Testing/ApiTests/ApiTestBase.cs @@ -29,17 +29,25 @@ public ApiTestBase() /// /// bas url for the client /// enable or disable cookies for the client - public static (SocketsHttpHandler Handler, HttpClient Client) NewHttpClient(string? baseUrl = null, bool useCookies = true) + /// + /// retry transient failures (5xx/timeouts). Leave on for idempotent calls; turn off for + /// non-idempotent operations (e.g. project creation) where a retried request would collide with + /// the resource the first attempt already created. + /// + public static (SocketsHttpHandler Handler, HttpClient Client) NewHttpClient(string? baseUrl = null, bool useCookies = true, bool retryTransientFailures = true) { - var retryPipeline = new ResiliencePipelineBuilder() - .AddRetry(new HttpRetryStrategyOptions { BackoffType = DelayBackoffType.Linear, MaxRetryAttempts = 3 }) - .Build(); - var socketsHttpHandler = new SocketsHttpHandler { PooledConnectionLifetime = TimeSpan.FromMinutes(15), UseCookies = useCookies }; + HttpMessageHandler handler = socketsHttpHandler; + if (retryTransientFailures) + { + var retryPipeline = new ResiliencePipelineBuilder() + .AddRetry(new HttpRetryStrategyOptions { BackoffType = DelayBackoffType.Linear, MaxRetryAttempts = 3 }) + .Build(); #pragma warning disable EXTEXP0001 - var resilienceHandler = new ResilienceHandler(retryPipeline) { InnerHandler = socketsHttpHandler }; + handler = new ResilienceHandler(retryPipeline) { InnerHandler = socketsHttpHandler }; #pragma warning restore EXTEXP0001 - var httpClient = new HttpClient(resilienceHandler); + } + var httpClient = new HttpClient(handler); if (!string.IsNullOrEmpty(baseUrl)) { httpClient.BaseAddress = new Uri(baseUrl); diff --git a/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs new file mode 100644 index 0000000000..1846cebb08 --- /dev/null +++ b/backend/Testing/FwHeadless/SyncHostedServiceCreationReservationTests.cs @@ -0,0 +1,53 @@ +using FwHeadless.Services; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Testing.FwHeadless; + +/// +/// Unit tests for the per-project creation reservation that stops a sync from racing a project that +/// is still being created from a template (and stops two concurrent creations of the same project). +/// +public class SyncHostedServiceCreationReservationTests +{ + private static SyncHostedService NewService() => + new(services: null!, NullLogger.Instance, new MemoryCache(new MemoryCacheOptions())); + + [Fact] + public void TryStartProjectCreation_blocks_a_second_concurrent_creation_and_is_reusable_after_release() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.TryStartProjectCreation(projectId).Should().BeTrue(); + svc.TryStartProjectCreation(projectId).Should().BeFalse("a creation is already in flight for this project"); + svc.IsJobQueuedOrRunning(projectId).Should().BeTrue(); + + svc.EndProjectCreation(projectId); + svc.IsJobQueuedOrRunning(projectId).Should().BeFalse(); + svc.TryStartProjectCreation(projectId).Should().BeTrue("the reservation is released and reusable"); + } + + [Fact] + public void QueueJob_is_refused_while_a_project_is_being_created() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.TryStartProjectCreation(projectId).Should().BeTrue(); + svc.QueueJob(projectId).Should().BeFalse("a sync must not race a project that's still being created"); + + svc.EndProjectCreation(projectId); + svc.QueueJob(projectId).Should().BeTrue("syncing is allowed once creation has finished"); + } + + [Fact] + public void A_queued_sync_blocks_a_creation() + { + var svc = NewService(); + var projectId = Guid.NewGuid(); + + svc.QueueJob(projectId).Should().BeTrue(); + svc.TryStartProjectCreation(projectId).Should().BeFalse("a sync is already queued for this project"); + } +} diff --git a/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs new file mode 100644 index 0000000000..d1db78622f --- /dev/null +++ b/backend/Testing/LexBoxApi/InitFwDataProjectValidationTests.cs @@ -0,0 +1,129 @@ +using LexBoxApi.Controllers; +using LexCore.Entities; +using LexCore.ServiceInterfaces; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; + +namespace Testing.LexBoxApi; + +/// +/// Unit tests for ProjectController.InitFwDataProject input validation. The 400 branches run before +/// any injected service is touched, so the controller can be built with null dependencies; only a +/// ProblemDetailsFactory (from AddControllers) is needed for Problem() to render. +/// +public class InitFwDataProjectValidationTests +{ + private static ProjectController NewController(IPermissionService? permissionService = null) + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddControllers(); // registers the ProblemDetailsFactory that ControllerBase.Problem() resolves + return new ProjectController(null!, null!, null!, permissionService!, null!, null!, NullLogger.Instance) + { + ControllerContext = new ControllerContext + { + HttpContext = new DefaultHttpContext { RequestServices = services.BuildServiceProvider() } + } + }; + } + + [Fact] + public async Task Rejects_when_no_vernacular_writing_system_is_supplied() + { + var result = await NewController().InitFwDataProject("myproj", wsVernacular: []); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("vernacular"); + } + + [Fact] + public async Task Rejects_an_invalid_project_code() + { + var result = await NewController().InitFwDataProject("Bad Code!", wsVernacular: ["fr"]); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("Invalid project code"); + } + + /// A permission service that either passes or fails the admin assertion. + private static IPermissionService PermissionService(bool isAdmin) + { + var mock = new Mock(); + if (!isAdmin) mock.Setup(p => p.AssertIsAdmin()).Throws(); + return mock.Object; + } + + [Fact] + public async Task Rejects_a_project_origin_that_is_not_a_migration_status() + { + var result = await NewController(PermissionService(isAdmin: true)) + .InitFwDataProject("myproj", wsVernacular: ["fr"], projectOrigin: "NotAnOrigin"); + var problem = result.Result.Should().BeOfType() + .Which.Value.Should().BeOfType().Which; + problem.Status.Should().Be(StatusCodes.Status400BadRequest); + problem.Detail.Should().Contain("Invalid project origin"); + } + + [Fact] + public async Task Rejects_a_project_origin_from_a_non_admin() + { + // The admin check must happen before the value is even parsed, and must outlive [AdminRequired], + // which is expected to be relaxed for this endpoint later. + var act = () => NewController(PermissionService(isAdmin: false)) + .InitFwDataProject("myproj", wsVernacular: ["fr"], projectOrigin: "Migrated"); + await act.Should().ThrowAsync(); + } + + [Fact] + public async Task Ignores_an_empty_project_origin_without_requiring_admin() + { + // An empty value isn't "specified", so it needs no admin check; a null permission service + // proves AssertIsAdmin() was never called. Fails later (null project service) than validation. + var act = () => NewController().InitFwDataProject("myproj", wsVernacular: ["fr"], projectOrigin: ""); + await act.Should().ThrowAsync(); + } + + [Theory] + [InlineData("Migrated", ProjectMigrationStatus.Migrated)] + [InlineData("migrating", ProjectMigrationStatus.Migrating)] + [InlineData("PUBLICREDMINE", ProjectMigrationStatus.PublicRedmine)] + public void Project_origin_parses_case_insensitively(string input, ProjectMigrationStatus expected) + { + ProjectController.TryParseProjectOrigin(input, out var origin).Should().BeTrue(); + origin.Should().Be(expected); + } + + [Theory] + [InlineData("NotAnOrigin")] + [InlineData("1")] // Enum.TryParse accepts raw numbers; the API takes names only + [InlineData("42")] + [InlineData("Migrated,Migrating")] // ...and comma-separated lists, which OR into a defined value here + [InlineData("")] + public void Project_origin_rejects_values_that_are_not_named_members(string input) + { + ProjectController.TryParseProjectOrigin(input, out _).Should().BeFalse(); + } + + [Fact] + public void Analysis_writing_systems_default_to_english_when_null() + { + ProjectController.AnalysisWritingSystemsOrDefault(null).Should().Equal("en"); + } + + [Fact] + public void Analysis_writing_systems_default_to_english_when_empty() + { + ProjectController.AnalysisWritingSystemsOrDefault([]).Should().Equal("en"); + } + + [Fact] + public void Analysis_writing_systems_are_kept_when_supplied() + { + ProjectController.AnalysisWritingSystemsOrDefault(["fr", "es"]).Should().Equal("fr", "es"); + } +} diff --git a/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs new file mode 100644 index 0000000000..df3d7e05a2 --- /dev/null +++ b/backend/Testing/SyncReverseProxy/InitFwDataProjectTests.cs @@ -0,0 +1,136 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json.Nodes; +using System.Xml.Linq; +using FluentAssertions; +using Testing.ApiTests; +using Testing.Fixtures; +using Testing.Services; +using Xunit.Abstractions; +using static Testing.Services.Constants; + +namespace Testing.SyncReverseProxy; + +/// +/// End-to-end coverage for the admin "init fwdata project" API. This is the one path that +/// can't be unit-tested: LexBox creates an empty hg repo and FwHeadless does the first push of a +/// template-built .fwdata into it (via LfMergeBridge/Chorus against the real hgweb). Requires the +/// lexbox stack, so it only runs in CI. +/// +[Trait("Category", "Integration")] +public class InitFwDataProjectTests : IClassFixture +{ + private readonly ITestOutputHelper _output; + private readonly ApiTestBase _adminApiTester; + + public InitFwDataProjectTests(ITestOutputHelper output, IntegrationFixture fixture) + { + _output = output; + _adminApiTester = fixture.AdminApiTester; + } + + [Fact] + public async Task InitFwDataProject_PopulatesTheEmptyRepoWithTheRequestedWritingSystems() + { + // Valid code (lowercase/digits/hyphen, doesn't start with a hyphen), unique per run. + var code = $"tmpl-{Guid.NewGuid():N}"[..12]; + var vernacular = new[] { "fr", "es" }; + var analysis = new[] { "de", "pt" }; + var query = $"?code={code}" + + string.Concat(vernacular.Select(ws => $"&wsVernacular={ws}")) + + string.Concat(analysis.Select(ws => $"&wsAnalysis={ws}")); + + Guid projectId = default; + try + { + // 1. Create the project via the admin endpoint (creates the DB row + empty repo, then has + // FwHeadless build the template .fwdata and push it into the empty repo). + // This is a long-running, non-idempotent operation. The shared tester's HttpClient retries + // transient failures/timeouts, so a slow first attempt (which already created the project + // row) would be retried and the retry would return 409 "already exists". Use a client that + // doesn't retry and allows more time. Log it in itself (cookie auth) rather than reusing the + // shared tester's captured JWT, which may be stale. + using var createClient = ApiTestBase.NewHttpClient(_adminApiTester.BaseUrl, retryTransientFailures: false).Client; + createClient.Timeout = TimeSpan.FromMinutes(5); + await JwtHelper.ExecuteLogin(AdminAuth, includeDefaultScope: true, createClient); + var response = await createClient.PostAsync( + $"{_adminApiTester.BaseUrl}/api/project/initFwDataProject{query}", null); + response.StatusCode.Should().Be(HttpStatusCode.OK, + "creation should succeed; body: {0}", await response.Content.ReadAsStringAsync()); + projectId = await response.Content.ReadFromJsonAsync(); + projectId.Should().NotBe(Guid.Empty); + + // 2. The first push landed a commit server-side (the empty repo is no longer empty). + var lastCommit = await _adminApiTester.GetProjectLastCommit(code); + lastCommit.Should().NotBeNull("the template project should have been pushed to the repo"); + + var tagsResponse = await _adminApiTester.HttpClient.GetAsync($"{_adminApiTester.BaseUrl}/hg/{code}/tags?style=json"); + tagsResponse.EnsureSuccessStatusCode(); + var tip = (await tagsResponse.Content.ReadFromJsonAsync())?["node"]?.ToString(); + tip.Should().NotBeNullOrEmpty(); + tip!.Replace("0", "").Should().NotBeEmpty("the repo tip should not be the all-zero empty-repo hash"); + + // 3. The pushed project carries the requested writing systems. Send/Receive split the + // template .fwdata into the nested files hg actually tracks; the LangProject's current + // analysis/vernacular writing systems live in General/LanguageProject.langproj (XML). + var langprojResponse = await _adminApiTester.HttpClient.GetAsync( + $"{_adminApiTester.BaseUrl}/hg/{code}/raw-file/tip/General/LanguageProject.langproj"); + langprojResponse.EnsureSuccessStatusCode(); + var langprojXml = await langprojResponse.Content.ReadAsStringAsync(); + langprojXml.Should().NotBeEmpty(); + + // de en pt + // fr es ... + var langProject = XDocument.Parse(langprojXml).Root?.Element("LangProject"); + langProject.Should().NotBeNull("LanguageProject.langproj should contain a LangProject element"); + var curAnalysisWss = SpaceSeparatedUni(langProject!, "CurAnalysisWss"); + var curVernWss = SpaceSeparatedUni(langProject!, "CurVernWss"); + + // The requested writing systems should be current. FieldWorks may add its own defaults (e.g. + // "en" as an analysis WS), so assert each requested code is present rather than exact equality. + foreach (var ws in analysis) + curAnalysisWss.Should().Contain(ws, "analysis writing system {0} should be current in the project", ws); + foreach (var ws in vernacular) + curVernWss.Should().Contain(ws, "vernacular writing system {0} should be current in the project", ws); + } + finally + { + if (projectId != default) await SoftDeleteProject(projectId); + } + } + + [Fact] + public async Task InitFwDataProject_RejectsWhenNoVernacularWritingSystem() + { + var code = $"tmpl-{Guid.NewGuid():N}"[..12]; + var response = await _adminApiTester.HttpClient.PostAsync( + $"{_adminApiTester.BaseUrl}/api/project/initFwDataProject?code={code}&wsAnalysis=en", null); + response.StatusCode.Should().Be(HttpStatusCode.BadRequest); + } + + // Reads a LangProject child element's text (a space-separated writing-system list) and splits it. + private static string[] SpaceSeparatedUni(XElement langProject, string elementName) + { + var uni = langProject.Element(elementName)?.Element("Uni")?.Value ?? ""; + return uni.Split(' ', StringSplitOptions.RemoveEmptyEntries); + } + + private async Task SoftDeleteProject(Guid projectId) + { + try + { + await _adminApiTester.ExecuteGql($$""" + mutation { + softDeleteProject(input: { projectId: "{{projectId}}" }) { + project { id } + errors { __typename } + } + } + """); + } + catch (Exception ex) + { + _output.WriteLine($"[InitFwDataProjectTests] Ignored cleanup exception: {ex}"); + } + } +}