Lexbox API to create empty FLEx project

backend/FwHeadless/FwHeadless.csproj

@@ -11,6 +11,9 @@
     <PackageReference Include="SIL.ChorusPlugin.LfMergeBridge" />
     <PackageReference Include="SIL.Chorus.Mercurial" />
     <PackageReference Include="SIL.Chorus.ChorusMerge" GeneratePathProperty="true" />
+    <!-- GeneratePathProperty so we can copy the NewLangProj template out of the package (below).
+         SIL.LCModel is otherwise a transitive dependency via FwDataMiniLcmBridge. -->
+    <PackageReference Include="SIL.LCModel" GeneratePathProperty="true" />
   </ItemGroup>
   <Import Project="$(MSBuildThisFileDirectory)..\Harmony.App.References.props" />
   <ItemGroup>
@@ -35,6 +38,11 @@
     <Content Remove="Mercurial\contrib\vs2022-settings.json" />
     <Content Include="Mercurial\**" CopyToOutputDirectory="Always" Watch="false" />
     <Content Include="MercurialExtensions\**" CopyToOutputDirectory="Always" Watch="false" />
+    <!-- Ship the LCM project templates (NewLangProj.fwdata etc.) so ProjectCreationService can build a
+         new project from them. TemplatesFolder is pointed at this output folder in FwHeadlessKernel. -->
+    <Content Include="$(PkgSIL_LCModel)/contentFiles/Templates/*.*"
+      Link="Templates/%(Filename)%(Extension)"
+      CopyToOutputDirectory="PreserveNewest" />
   </ItemGroup>
   <ItemGroup>
     <None Include="chorusmerge">

backend/FwHeadless/FwHeadlessConfig.cs

@@ -22,6 +22,22 @@ public class FwHeadlessConfig
     public long MaxUploadFileSizeBytes => MaxUploadFileSizeKb * 1024;
     public string FdoDataModelVersion { get; init; } = "7000072";
 
+    /// <summary>
+    /// The FLExBridge data version. FieldWorks/LfMergeBridge form the Mercurial branch a project's
+    /// data lives on by joining this and the FDO model version with a dot (see <see cref="SendReceiveBranchName"/>).
+    /// Mirrors FlexBridgeConstants.FlexBridgeDataVersion in FLExBridge, which is internal so we can't
+    /// reference it directly; bump this if FLExBridge bumps its data version.
+    /// </summary>
+    public string FlexBridgeDataVersion { get; init; } = "7500002";
+
+    /// <summary>
+    /// The Mercurial branch FieldWorks/LfMergeBridge keep a project's data on: FlexBridgeDataVersion,
+    /// a dot, then the FDO model version (e.g. "7500002.7000072"). This is exactly the branch name
+    /// LfMergeBridge derives from <see cref="FdoDataModelVersion"/>, so a new project's genesis commit
+    /// must land on this branch for send/receive to find the data.
+    /// </summary>
+    public string SendReceiveBranchName => $"{FlexBridgeDataVersion}.{FdoDataModelVersion}";
+
     /// <summary>
     /// Project directory structure in FwHeadless: (Note that FwDataProject.ProjectsPath is the root of a SINGLE project)
     /// {ProjectStorageRoot}/

backend/FwHeadless/FwHeadlessKernel.cs

@@ -25,13 +25,22 @@ public static IServiceCollection AddFwHeadless(this IServiceCollection services)
         services.AddScoped<ISendReceiveService, SendReceiveService>();
         services.AddScoped<IProjectLookupService, ProjectLookupService>();
         services.AddScoped<ProjectDeletionService>();
+        services.AddScoped<ProjectCreationService>();
         services.AddScoped<LogSanitizerService>();
         services.AddScoped<SafeLoggingProgress>();
         services.AddScoped<IProjectMetadataService, ProjectMetadataService>();
         services
             .AddLcmCrdtClientCore()
             .AddFwDataBridge(ServiceLifetime.Scoped)
             .AddFwLiteProjectSync();
+        // The LCM project templates (NewLangProj.fwdata etc.) ship in our output (see FwHeadless.csproj).
+        // FwDataBridgeConfig's default TemplatesFolder points at the FieldWorks user-data dir, which
+        // doesn't exist in the container; point it at the shipped templates unless overridden in config.
+        services.Configure<FwDataBridgeConfig>(config =>
+        {
+            if (config.TemplatesFolder == new FwDataBridgeConfig().TemplatesFolder)
+                config.TemplatesFolder = Path.Combine(AppContext.BaseDirectory, "Templates");
+        });
         services.RemoveAll(typeof(IMediaAdapter));
         services.AddScoped<IMediaAdapter, LexboxFwDataMediaAdapter>();
         services.AddScoped<MediaFileService>();

backend/FwHeadless/Program.cs

@@ -12,6 +12,17 @@
 using WebServiceDefaults;
 using AppVersion = LexCore.AppVersion;
 
+// Chorus discovers its file-type handler plugins with `new DirectoryCatalog(".", "*-ChorusPlugin.dll")`,
+// which scans the *current working directory* -- not the app base dir -- and WebApplication.CreateBuilder
+// does not change CWD. When launched from anywhere other than the folder holding the co-published
+// LibFLExBridge-ChorusPlugin.dll (e.g. `dotnet run`, whose CWD is the project dir), that plugin isn't
+// found. Without it, FieldWorks extensions like `.list` fall back to Chorus's 1 MB LargeFileFilter
+// default, so the >1 MB SemanticDomainList.list is silently filtered out of a new project's first push
+// (and later merges lose their FieldWorks handlers too). Pin CWD to the base dir so the plugin is always
+// discovered. Safe here: hg runs with explicit working dirs and TemplatesFolder resolves against
+// AppContext.BaseDirectory already.
+Environment.CurrentDirectory = AppContext.BaseDirectory;
+
 var builder = WebApplication.CreateBuilder(args);
 
 // Add services to the container.
@@ -83,6 +94,7 @@
 app.MapDefaultEndpoints();
 app.MapMediaFileRoutes();
 app.MapMergeRoutes();
+app.MapProjectRoutes();
 
 // DELETE endpoint to delete the FieldWorks repo/project (and nothing else)
 app.MapDelete("/api/manage/repo/{projectId}", async (Guid projectId, ProjectDeletionService deletionService) =>

backend/FwHeadless/Routes/ProjectRoutes.cs

@@ -0,0 +1,53 @@
+using FwHeadless.Services;
+using LexCore.Entities;
+using Microsoft.AspNetCore.Http.HttpResults;
+using SIL.WritingSystems;
+
+namespace FwHeadless.Routes;
+
+public static class ProjectRoutes
+{
+    public static IEndpointConventionBuilder MapProjectRoutes(this WebApplication app)
+    {
+        var group = app.MapGroup("/api/project");
+        group.MapPost("/init-fwdata-project", InitFwDataProject);
+        return group;
+    }
+
+    // Internal endpoint: LexBoxApi has already created the project row + empty repo and performed the
+    // admin-auth check. FwHeadless populates the empty repo with a template .fwdata configured for the
+    // requested writing systems. Not idempotent — on failure LexBoxApi deletes and recreates.
+    private static async Task<Results<Ok, ProblemHttpResult>> InitFwDataProject(
+        Guid projectId,
+        InitFwDataProjectInput input,
+        IProjectLookupService projectLookupService,
+        ProjectCreationService projectCreationService,
+        ILogger<Program> logger)
+    {
+        // Null-safe against a malformed body (System.Text.Json leaves an absent list null).
+        if (input.WsVernacular is not { Count: > 0 })
+            return TypedResults.Problem("At least one vernacular writing system is required",
+                statusCode: StatusCodes.Status400BadRequest);
+        if (input.WsAnalysis is not { Count: > 0 })
+            return TypedResults.Problem("At least one analysis writing system is required",
+                statusCode: StatusCodes.Status400BadRequest);
+        var invalidWs = input.WsVernacular.Concat(input.WsAnalysis).FirstOrDefault(ws => !IetfLanguageTag.IsValid(ws));
+        if (invalidWs is not null)
+            return TypedResults.Problem($"Invalid writing system code: {invalidWs}",
+                statusCode: StatusCodes.Status400BadRequest);
+        if (string.IsNullOrEmpty(input.WsUi) || !IetfLanguageTag.IsValid(input.WsUi))
+            return TypedResults.Problem($"Invalid UI writing system code: {input.WsUi}",
+                statusCode: StatusCodes.Status400BadRequest);
+
+        var projectCode = await projectLookupService.GetProjectCode(projectId);
+        if (projectCode is null)
+        {
+            logger.LogError("Init-fwdata-project request for non-existent project {ProjectId}", projectId);
+            return TypedResults.Problem("Project not found", statusCode: StatusCodes.Status404NotFound);
+        }
+
+        await projectCreationService.InitFwDataProject(
+            projectId, projectCode, input.WsVernacular, input.WsAnalysis, input.WsUi);
+        return TypedResults.Ok();
+    }
+}

backend/FwHeadless/Services/ISendReceiveService.cs

@@ -10,4 +10,7 @@ public interface ISendReceiveService
     Task<int> PendingCommitCountOutgoing(FwDataProject project, string? projectCode);
     Task<int> PendingCommitCountBothWays(FwDataProject project, string? projectCode);
     Task CommitFile(string filePath, string commitMessage);
+    Task CommitEmpty(string folder, string commitMessage);
+    Task InitRepo(string folder);
+    Task SetBranch(string folder, string branchName);
 }

backend/FwHeadless/Services/ProjectCreationService.cs

@@ -0,0 +1,210 @@
+using System.Xml;
+using System.Xml.Linq;
+using FwDataMiniLcmBridge;
+using FwDataMiniLcmBridge.LcmUtils;
+using LexCore.Exceptions;
+using Microsoft.Extensions.Options;
+using SIL.Xml;
+
+namespace FwHeadless.Services;
+
+/// <summary>
+/// Creates a brand-new FieldWorks project from the SIL.LCModel template and pushes it into the
+/// (empty) Mercurial repo that LexBox has already initialised for the project.
+/// </summary>
+public class ProjectCreationService(
+    IOptions<FwHeadlessConfig> config,
+    ISendReceiveService srService,
+    IProjectLoader projectLoader,
+    SyncHostedService syncHostedService,
+    ILogger<ProjectCreationService> logger)
+{
+    private const string InitialCommitMessage = "Initial project creation";
+
+    // The one file FieldWorks/FLExBridge put in a brand-new FLEx repo's first commit: a minimal custom
+    // property definition list. Mirrors FlexBridgeConstants.CustomPropertiesFilename (internal in FLExBridge).
+    private const string CustomPropertiesFilename = "FLExProject.CustomProperties";
+
+    public async Task InitFwDataProject(
+        Guid projectId,
+        string projectCode,
+        IReadOnlyList<string> vernacularWritingSystems,
+        IReadOnlyList<string> analysisWritingSystems,
+        string uiWritingSystem)
+    {
+        if (vernacularWritingSystems.Count == 0)
+            throw new ArgumentException("At least one vernacular writing system is required", nameof(vernacularWritingSystems));
+        if (analysisWritingSystems.Count == 0)
+            throw new ArgumentException("At least one analysis writing system is required", nameof(analysisWritingSystems));
+        if (string.IsNullOrEmpty(uiWritingSystem))
+            throw new ArgumentException("A UI writing system is required", nameof(uiWritingSystem));
+
+        // Reserve the project so a concurrent create or sync can't race on the same fw/ folder and repo.
+        if (!syncHostedService.TryStartProjectCreation(projectId))
+            throw new ProjectSyncInProgressException(projectId);
+        var fwDataProject = config.Value.GetFwDataProject(projectCode, projectId);
+        try
+        {
+            // Build the first commit of a brand-new FLEx repo from scratch, mirroring how FieldWorks/
+            // FLExBridge create a repo, since Clone can't establish an empty remote and
+            // Language_Forge_Send_Receive refuses to make the *first* commit itself:
+            //   1. hg init the local fw/ folder (no clone -- the empty remote has nothing to clone).
+            //   2. Build fw.fwdata into that folder from the SIL.LCModel template, configured with all
+            //      of the requested writing systems.
+            //   3. Commit a minimal FLExProject.CustomProperties file on hg's *default* branch -- the same
+            //      genesis file, on the same branch, FieldWorks commits first. The fwdata itself is NOT
+            //      committed; Send/Receive splits it into the nested files Mercurial actually tracks
+            //      (fwdata is excluded from tracking).
+            //   4. Set the branch to the send/receive branch name (FlexBridgeDataVersion.modelVersion,
+            //      e.g. 7500002.7000072) AFTER the genesis commit; FLEx/LfMergeBridge look for the data on
+            //      that branch, so the split data must land there while the genesis stays on 'default'.
+            //   5. Send/Receive: split fw.fwdata into its nested files, commit those on the branch, push.
+            await srService.InitRepo(fwDataProject.ProjectFolder);
+
+            BuildFromTemplate(fwDataProject, vernacularWritingSystems, analysisWritingSystems, uiWritingSystem);
+            FixLinkedFilesRootDirSeparator(fwDataProject);
+            AssertModelVersionMatches(fwDataProject);
+
+            var customPropertiesFile = WriteInitialCustomPropertiesFile(fwDataProject);
+            await srService.CommitFile(customPropertiesFile, InitialCommitMessage);
+
+            await srService.SetBranch(fwDataProject.ProjectFolder, config.Value.SendReceiveBranchName);
+            // BUG WORKAROUND (remove once the LanguageForgeSendReceiveActionHandler bug is fixed): that
+            // handler refuses to commit when doing so "could possibly create a new branch", so it won't
+            // establish the model-version branch itself. This empty commit (no file changes -- it only
+            // records the branch change from SetBranch above) creates a head on the branch so Send/Receive
+            // will proceed.
+            await srService.CommitEmpty(fwDataProject.ProjectFolder, InitialCommitMessage);
+            var pushResult = await srService.SendReceive(fwDataProject, projectCode, InitialCommitMessage);
+            if (!pushResult.Success)
+                throw new SendReceiveException("Pushing the new project to the repo failed", pushResult);
+
+            logger.LogInformation("Created project {ProjectCode} ({ProjectId}) from template", projectCode, projectId);
+        }
+        catch
+        {
+            CleanupLocalProject(fwDataProject);
+            throw;
+        }
+        finally
+        {
+            syncHostedService.EndProjectCreation(projectId);
+        }
+    }
+
+    private void BuildFromTemplate(
+        FwDataProject fwDataProject,
+        IReadOnlyList<string> vernacularWritingSystems,
+        IReadOnlyList<string> analysisWritingSystems,
+        string uiWs)
+    {
+        // NewProject copies the SIL.LCModel template, configured with all of the requested writing
+        // systems (the first of each list is the default of that type), and returns a loaded cache;
+        // dispose it right away so its file locks are released before the following hg add/commit.
+        using var cache = projectLoader.NewProject(fwDataProject, analysisWritingSystems, vernacularWritingSystems, uiWs);
+    }
+
+    /// <summary>
+    /// FieldWorks (Windows) compares LangProject.LinkedFilesRootDir with an exact string, expecting the
+    /// Windows-style default <c>%proj%\LinkedFiles</c>. liblcm builds that path with Path.Combine, which on
+    /// this Linux host yields <c>%proj%/LinkedFiles</c> (forward slash), so FieldWorks warns on its first
+    /// Send/Receive. Rewrite just that one value to the backslash form.
+    ///
+    /// We deliberately do NOT parse the (potentially multi-MB) fwdata as XML -- that would be a needless
+    /// cost. The value always sits on the line immediately after the &lt;LinkedFilesRootDir&gt; open tag,
+    /// so stream the file once, fix the &lt;Uni&gt; on the line following that tag, and copy every other
+    /// line through untouched.
+    /// </summary>
+    private static void FixLinkedFilesRootDirSeparator(FwDataProject fwDataProject)
+    {
+        const string markerLinePrefix = "<LinkedFilesRootDir";
+        const string forwardSlashValue = "<Uni>%proj%/LinkedFiles</Uni>";
+        const string backslashValue = @"<Uni>%proj%\LinkedFiles</Uni>";
+
+        var sourcePath = fwDataProject.FilePath;
+        var tempPath = sourcePath + ".tmp";
+        using (var reader = new StreamReader(sourcePath))
+        using (var writer = new StreamWriter(tempPath))
+        {
+            var nextLineIsValue = false;
+            var done = false;
+            string? line;
+            while ((line = reader.ReadLine()) is not null)
+            {
+                if (nextLineIsValue)
+                {
+                    line = line.Replace(forwardSlashValue, backslashValue);
+                    nextLineIsValue = false;
+                    done = true;
+                }
+                else if (!done && line.StartsWith(markerLinePrefix, StringComparison.Ordinal))
+                {
+                    nextLineIsValue = true;
+                }
+                writer.WriteLine(line);
+            }
+        }
+        File.Move(tempPath, sourcePath, overwrite: true);
+    }
+
+    /// <summary>
+    /// Writes the genesis FLExProject.CustomProperties file into the repo root: an empty custom-property
+    /// list (just an &lt;AdditionalFields /&gt; root). Uses SIL's canonical XML settings so the bytes match
+    /// exactly what the first Send/Receive's project splitter regenerates, leaving no spurious diff.
+    /// </summary>
+    private string WriteInitialCustomPropertiesFile(FwDataProject fwDataProject)
+    {
+        var path = Path.Join(fwDataProject.ProjectFolder, CustomPropertiesFilename);
+        using (var writer = XmlWriter.Create(path, CanonicalXmlSettings.CreateXmlWriterSettings()))
+        {
+            writer.WriteStartDocument();
+            new XElement("AdditionalFields").WriteTo(writer);
+        }
+        return path;
+    }
+
+    /// <summary>
+    /// The freshly-created project is stamped with the SIL.LCModel package's data-model version.
+    /// If that ever diverges from the version FwHeadless syncs with, pushing it would corrupt data
+    /// (see FwHeadless AGENTS.md), so fail before the push rather than after.
+    /// </summary>
+    private void AssertModelVersionMatches(FwDataProject fwDataProject)
+    {
+        var expected = config.Value.FdoDataModelVersion;
+        var actual = ReadModelVersion(fwDataProject.FilePath);
+        // Fail closed: an unreadable version ('actual' == null) is treated as a mismatch, since this is
+        // the last safety net before an irreversible push in a data-loss-risk area.
+        if (actual != expected)
+        {
+            throw new InvalidOperationException(
+                $"New project's data-model version '{actual ?? "(unreadable)"}' does not match FwHeadless " +
+                $"FdoDataModelVersion '{expected}'; refusing to push to avoid data corruption.");
+        }
+    }
+
+    private static string? ReadModelVersion(string fwDataFilePath)
+    {
+        using var reader = XmlReader.Create(fwDataFilePath, new XmlReaderSettings { IgnoreComments = true, IgnoreWhitespace = true });
+        // The .fwdata root element carries the FDO model version, e.g. <languageproject version="7000072">.
+        return reader.MoveToContent() == XmlNodeType.Element ? reader.GetAttribute("version") : null;
+    }
+
+    private void CleanupLocalProject(FwDataProject fwDataProject)
+    {
+        // Mirror SyncWorker.SetupFwData's failure cleanup so a retry starts clean. The remote repo
+        // (created by LexBox) is torn down by the caller on the LexBox side.
+        try
+        {
+            if (Directory.Exists(fwDataProject.ProjectFolder))
+                Directory.Delete(fwDataProject.ProjectFolder, true);
+            if (Directory.Exists(fwDataProject.ProjectsPath) &&
+                !Directory.EnumerateFileSystemEntries(fwDataProject.ProjectsPath).Any())
+                Directory.Delete(fwDataProject.ProjectsPath);
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Failed to clean up local project folder after a failed creation: {ProjectFolder}",
+                fwDataProject.ProjectFolder);
+        }
+    }
+}