Draft: EditorJs draft

Dockerfile

@@ -1,23 +1,26 @@
-FROM alpine/git as version
-WORKDIR /src
-COPY . /src
-RUN echo $(git describe --tags --always 2>/dev/null |  sed 's/-g[a-z0-9]\{7\}//') > /version ;\
-    echo "Version: "$(cat /version)
-
-FROM mcr.microsoft.com/dotnet/sdk:7.0 AS build-env
-WORKDIR /build
-COPY . ./
-COPY --from=version /version /build/version
-RUN apt-get update -yq ;\
-	apt-get install curl gnupg -yq ;\
-	curl -sL https://deb.nodesource.com/setup_14.x | bash - ;\
-	apt-get install -y nodejs
-
-RUN sed -i -e "s/<Version>0-develop<\/Version>/<Version>$(cat version | cut -c2- )<\/Version>/g" src/Mimisbrunnr.Web.Host/Mimisbrunnr.Web.Host.csproj;\
-	dotnet build -c Release --nologo;\
-    dotnet publish src/Mimisbrunnr.Web.Host/Mimisbrunnr.Web.Host.csproj -c Release --no-build -o /dist/app
-
-FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS app
-WORKDIR /app
-COPY --from=build-env /dist/app/. .
-CMD ["dotnet", "Mimisbrunnr.Web.Host.dll"]
+FROM alpine/git as version
+WORKDIR /src
+COPY . /src
+RUN echo $(git describe --tags --always 2>/dev/null |  sed 's/-g[a-z0-9]\{7\}//') > /version ;\
+    echo "Version: "$(cat /version)
+
+FROM mcr.microsoft.com/dotnet/sdk:7.0 AS build-env
+WORKDIR /build
+COPY . ./
+COPY --from=version /version /build/version
+RUN apt-get update -yq ;\
+	apt-get install curl gnupg ca-certificates -yq ;\
+    mkdir -p /etc/apt/keyrings ;\
+    curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg ;\
+	echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list ;\
+    apt-get update ;\
+	apt-get install -y nodejs
+
+RUN sed -i -e "s/<Version>0-develop<\/Version>/<Version>$(cat version | cut -c2- )<\/Version>/g" src/Mimisbrunnr.Web.Host/Mimisbrunnr.Web.Host.csproj;\
+	dotnet build -c Release --nologo;\
+    dotnet publish src/Mimisbrunnr.Web.Host/Mimisbrunnr.Web.Host.csproj -c Release --no-build -o /dist/app
+
+FROM mcr.microsoft.com/dotnet/aspnet:6.0 AS app
+WORKDIR /app
+COPY --from=build-env /dist/app/. .
+CMD ["dotnet", "Mimisbrunnr.Web.Host.dll"]

src/Integration/Mimisbrunnr.Integration/Wiki/PageCreateModel.cs

@@ -2,7 +2,7 @@
 
 namespace Mimisbrunnr.Integration.Wiki;
 
-public class PageCreateModel
+public class PageCreateModel : IValidatableObject
 {
     [Required]
     public string SpaceKey { get; set; }
@@ -14,4 +14,16 @@ public class PageCreateModel
     public string Name { get; set; }
 
     public string Content { get; set; }
+
+    public string PlainTextContent {get;set;}
+
+    public PageEditorTypeModel EditorType { get; set; }
+
+    public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
+    {
+        if(!string.IsNullOrEmpty(Content) 
+            && Content.Length > 10 // additional check if Content contains only MD/editorjs tokens
+            && string.IsNullOrEmpty(PlainTextContent))
+            yield return new ValidationResult("PlainTextContent can't be empty when Content not empty", new []{nameof(PlainTextContent)});
+    }
 }

src/Integration/Mimisbrunnr.Integration/Wiki/PageEditorTypeModel.cs

@@ -0,0 +1,7 @@
+namespace Mimisbrunnr.Integration.Wiki;
+
+public enum PageEditorTypeModel
+{
+    MarkdownEditor,
+    EditorJs
+}

src/Integration/Mimisbrunnr.Integration/Wiki/PageEditorTypeUpdateModel.cs

@@ -0,0 +1,6 @@
+namespace Mimisbrunnr.Integration.Wiki;
+
+public class PageEditorTypeUpdateModel
+{
+    public PageEditorTypeModel Type { get; set; }
+}

src/Integration/Mimisbrunnr.Integration/Wiki/PageModel.cs

@@ -14,11 +14,15 @@ public class PageModel
 
     public string Content { get; set; }
 
+    public string PlainTextContent { get; set; }
+
     public DateTime Created { get; set; }
 
     public DateTime Updated { get; set; }
 
     public UserModel CreatedBy { get; set; }
 
     public UserModel UpdatedBy { get; set; }
+
+    public PageEditorTypeModel EditorType { get; set; }
 }

src/Integration/Mimisbrunnr.Integration/Wiki/PageUpdateModel.cs

@@ -2,10 +2,20 @@
 
 namespace Mimisbrunnr.Integration.Wiki;
 
-public class PageUpdateModel
+public class PageUpdateModel : IValidatableObject
 {
     [Required]
     public string Name { get; set; }
 
     public string Content { get; set; }
+
+    public string PlainTextContent {get;set;}
+
+    public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
+    {
+        if(!string.IsNullOrEmpty(Content) 
+            && Content.Length > 10 // additional check if Content contains only MD/editorjs tokens
+            && string.IsNullOrEmpty(PlainTextContent))
+            yield return new ValidationResult("PlainTextContent can't be empty when Content not empty", new []{nameof(PlainTextContent)});
+    }
 }

src/Mimisbrunnr.Json/AbstractClassJsonConverter.cs

@@ -11,7 +11,7 @@ internal class AbstractClassConverter<T> : JsonConverter<T>
 
     public override T Read(ref Utf8JsonReader reader, Type typeToConvert,
         JsonSerializerOptions options)
-    {
+    {      
         if (reader.TokenType == JsonTokenType.Null) return default;
 
         if (reader.TokenType != JsonTokenType.StartObject)

src/Mimisbrunnr.Storage.MongoDb/Mappings/PageMap.cs

@@ -1,5 +1,6 @@
 using Mimisbrunnr.Wiki.Contracts;
 using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Serializers;
 using Skidbladnir.Repository.MongoDB;
 
 namespace Mimisbrunnr.Storage.MongoDb.Mappings;
@@ -13,5 +14,10 @@ public PageMap()
         MapProperty(x => x.Version)
             .SetDefaultValue(0L)
             .SetIgnoreIfDefault(false);
+
+        MapProperty( x=> x.EditorType)
+            .SetIgnoreIfDefault(false)
+            .SetDefaultValue(PageEditorType.MarkdownEditor)
+            .SetSerializer(new EnumSerializer<PageEditorType>(BsonType.String));
     }
 }

src/Mimisbrunnr.Storage.MongoDb/Migrations/PlainTextContentMigrationModule.cs

@@ -0,0 +1,63 @@
+using System.Text.RegularExpressions;
+using DnsClient.Internal;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Mimisbrunnr.Wiki.Contracts;
+using Skidbladnir.Modules;
+using Skidbladnir.Repository.Abstractions;
+
+namespace Mimisbrunnr.Storage.MongoDb.Migrations;
+
+public class PlainTextContentMigrationModule : BackgroundModule
+{
+    private static readonly Regex _markdownRemoveRegex =
+    new Regex(@"__|\*|\#|(?:\[([^\]]*)\]\([^)]*\))", RegexOptions.Compiled);
+    private static readonly Regex _htmlTagsRemoveRegex = new Regex(@"<.*?>", RegexOptions.Compiled);
+
+    public override async Task ExecuteAsync(IServiceProvider provider, CancellationToken cancellationToken = default)
+    {
+        var logger = provider.GetService<ILogger<PlainTextContentMigrationModule>>();
+        var pageRepository = provider.GetService<IRepository<Page>>();
+        var historicalPageRepository = provider.GetService<IRepository<HistoricalPage>>();
+        var parallelOptions = new ParallelOptions()
+        {
+            CancellationToken = cancellationToken,
+            MaxDegreeOfParallelism = Math.Max(4, Environment.ProcessorCount)
+        };
+        logger.LogInformation("Start fix page plain text");
+        try
+        {
+            var pagesWithoutPlainText = await pageRepository
+                .GetAll()
+                .Where(x => x.EditorType != PageEditorType.EditorJs
+                    && string.IsNullOrEmpty(x.PlainTextContent)
+                    && !string.IsNullOrEmpty(x.Content))
+                .ToArrayAsync();
+            await Parallel.ForEachAsync(pagesWithoutPlainText, parallelOptions, async (page, token) =>
+            {
+                page.PlainTextContent = Sanitize(page.Content);
+                await pageRepository.Update(page, token);
+            });
+            var historicalPagesWithoutPlainText = await historicalPageRepository.GetAll()
+                .Where(x => x.EditorType != PageEditorType.EditorJs
+                    && string.IsNullOrEmpty(x.PlainTextContent)
+                    && !string.IsNullOrEmpty(x.Content))
+                .ToArrayAsync();
+            await Parallel.ForEachAsync(historicalPagesWithoutPlainText, parallelOptions, async (page, token) =>
+            {
+                page.PlainTextContent = Sanitize(page.Content);
+                await historicalPageRepository.Update(page, token);
+            });
+        }
+        catch (Exception e)
+        {
+            logger.LogError(e, "Start fix page plain texts");
+        }
+    }
+
+    private static string Sanitize(string articleBody)
+    {
+        var mdSanitized = _markdownRemoveRegex.Replace(articleBody, "");
+        return _htmlTagsRemoveRegex.Replace(mdSanitized, "");
+    }
+}

src/Mimisbrunnr.Storage.MongoDb/MongoDbIndexExtensions.cs

@@ -0,0 +1,39 @@
+using System.Linq;
+using System.Text.RegularExpressions;
+using MongoDB.Driver;
+
+namespace Mimisbrunnr.Storage.MongoDb
+{
+    public static class MongoDbIndexExtensions
+    {
+        private static readonly Regex _conflictIndexNameSearch = new Regex("name: \\\"([^\\s\\\"]*)\\\"", RegexOptions.Compiled);
+        private const int IndexOptionsConflict = 85;
+        private const int IndexKeySpecsConflict = 86;
+
+        public static async Task<string> TryCreateOneAsync<TDocument>(this IMongoIndexManager<TDocument> indexManager, CreateIndexModel<TDocument> model, CancellationToken cancellationToken = default(CancellationToken))
+        {
+            try
+            {
+                return await indexManager.CreateOneAsync(model, cancellationToken: cancellationToken).ConfigureAwait(false);
+            }
+            catch (MongoCommandException e)
+            {
+                // If index exists with different options or name
+                if (e.Code == IndexKeySpecsConflict || e.Code == IndexOptionsConflict)
+                {
+                    var conflictedIndexNameMatches = _conflictIndexNameSearch.Matches(e.Message);
+                    var conflictedIndexNameMatch = conflictedIndexNameMatches.LastOrDefault();
+                    var conflictedIndexName = conflictedIndexNameMatch.Groups.Count > 1 
+                        ? conflictedIndexNameMatch.Groups[1].Value
+                        : model.Options.Name;
+                    await indexManager.DropOneAsync(conflictedIndexName, cancellationToken).ConfigureAwait(false);
+                    return await indexManager.CreateOneAsync(model, cancellationToken: cancellationToken).ConfigureAwait(false);
+                }
+                else
+                {
+                    throw;
+                }
+            }
+        }
+    }
+}

src/Mimisbrunnr.Storage.MongoDb/MongoDbStoreModule.cs

@@ -173,15 +173,18 @@ private static async Task CreatePageIndexes(IMongoDbContext mongoContext)
         {
             Background = true
         }));
-
         var nameKeyDefinition = Builders<Page>.IndexKeys.Ascending(x => x.Name);
         await collection.Indexes.CreateOneAsync(new CreateIndexModel<Page>(nameKeyDefinition, new CreateIndexOptions()
         {
             Background = true
         }));
-
-        var contentTitleFullTextSearch = Builders<Page>.IndexKeys.Text(x => x.Name).Text(x => x.Content);
-        await collection.Indexes.CreateOneAsync(new CreateIndexModel<Page>(contentTitleFullTextSearch, new CreateIndexOptions()
+        var plainTextMigrationKeyDefinition = Builders<Page>.IndexKeys.Ascending(x => x.EditorType).Ascending(x=> x.Content).Ascending(x => x.PlainTextContent);
+        await collection.Indexes.CreateOneAsync(new CreateIndexModel<Page>(plainTextMigrationKeyDefinition, new CreateIndexOptions()
+        {
+            Background = true
+        }));
+        var contentTitleFullTextSearch = Builders<Page>.IndexKeys.Text(x => x.Name).Text(x => x.PlainTextContent);
+        await collection.Indexes.TryCreateOneAsync(new CreateIndexModel<Page>(contentTitleFullTextSearch, new CreateIndexOptions()
         {
             Background = true
         }));
@@ -201,6 +204,11 @@ private async Task CreatePageHistoryIndexes(BaseMongoDbContext mongoContext)
         {
             Background = true
         }));
+        var plainTextMigrationKeyDefinition = Builders<HistoricalPage>.IndexKeys.Ascending(x => x.EditorType).Ascending(x=> x.Content).Ascending(x => x.PlainTextContent);
+        await collection.Indexes.CreateOneAsync(new CreateIndexModel<HistoricalPage>(plainTextMigrationKeyDefinition, new CreateIndexOptions()
+        {
+            Background = true
+        }));
     }