[TASK] Include and load CKEditor locales if configured

This patch adds support for loading the base CKEditor 5
locales and importing them in the backend if requested.

Because CKEditor doesn't ship a full set of locales, we
needed to add an additional build step that assembles
the locales of all plugins we have installed.

Resolves: #100873
Releases: main, 12.4
Change-Id: I971bf3e55d006a4a378123e8723c1bb8d23b09d1
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/79026
Reviewed-by: Andreas Nedbal <[email protected]>
Reviewed-by: Torben Hansen <[email protected]>
Tested-by: Andreas Fernandez <[email protected]>
Reviewed-by: Oliver Hader <[email protected]>
Tested-by: core-ci <[email protected]>
Reviewed-by: Andreas Fernandez <[email protected]>
Tested-by: Andreas Nedbal <[email protected]>
Tested-by: Torben Hansen <[email protected]>
Tested-by: Oliver Hader <[email protected]>

.gitignore

@@ -41,6 +41,7 @@ nbproject
 /Build/bower_components/*
 /Build/node_modules/*
 /Build/JavaScript
+/Build/ckeditorLocales
 !/Build/typings/no-def*
 /Build/testing-docker/local/.env
 /Build/testing-docker/local/macos_passwd

Build/Gruntfile.js

@@ -19,6 +19,10 @@ module.exports = function (grunt) {
   const sass = require('sass');
   const esModuleLexer = require('es-module-lexer');
 
+  grunt.registerTask('ckeditor:compile-locales', 'locale compilation for CKEditor', function() {
+    require('./Scripts/node/ckeditor-locale-compiler.js');
+  })
+
   /**
    * Grunt stylefmt task
    */
@@ -831,7 +835,7 @@ module.exports = function (grunt) {
    * this task does the following things:
    * - copy some components to a specific destinations because they need to be included via PHP
    */
-  grunt.registerTask('update', ['rollup', 'exec:ckeditor', 'concurrent:npmcopy']);
+  grunt.registerTask('update', ['ckeditor:compile-locales', 'rollup', 'exec:ckeditor', 'concurrent:npmcopy']);
 
   /**
    * grunt compile-typescript task

Build/Scripts/node/ckeditor-locale-compiler.js

@@ -0,0 +1,146 @@
+const { resolve, join } = require('path')
+const { readdirSync, existsSync, readFileSync, mkdirSync, writeFileSync } = require('fs')
+const PO = require('pofile')
+
+/**
+ * This script assembles full locales from all plugins found in node_modules/@ckeditor/
+ *
+ * Parts of this script are based on @ckeditor/ckeditor5-dev-translations/lib/multiplelanguagetranslationservice.js
+ *
+ * Subject to following license terms:
+ * @license Copyright (c) 2003-2023, CKSource Holding sp. z o.o. All rights reserved.
+ * For licensing, see @ckeditor/ckeditor5-dev-translations/LICENSE.md.
+ */
+
+const _languages = new Set();
+const _translationDictionaries = {};
+const _pluralFormsRules = {};
+
+function loadPackage(packagePath) {
+  if (!existsSync(packagePath)) {
+    return;
+  }
+
+  const translationPath = getTranslationPath(packagePath);
+
+  if (!existsSync(translationPath)) {
+    return;
+  }
+
+  for (const fileName of readdirSync(translationPath)) {
+    if (!fileName.endsWith('.po') ) {
+      continue;
+    }
+
+    const language = fileName.replace( /\.po$/, '' );
+    const pathToPoFile = join(translationPath, fileName);
+
+    _languages.add(language);
+    loadPoFile(language, pathToPoFile);
+  }
+}
+
+function getTranslationPath(packagePath) {
+  return join(packagePath, 'lang', 'translations');
+}
+
+function loadPoFile(language, pathToPoFile) {
+  if (!existsSync(pathToPoFile)) {
+    return;
+  }
+
+  const parsedTranslationFile = PO.parse(readFileSync(pathToPoFile, 'utf-8'));
+
+  _pluralFormsRules[language] = _pluralFormsRules[language] || parsedTranslationFile.headers['Plural-Forms'];
+
+  if (!_translationDictionaries[language]) {
+    _translationDictionaries[language] = {};
+  }
+
+  const dictionary = _translationDictionaries[language];
+
+  for (const item of parsedTranslationFile.items) {
+    dictionary[item.msgid] = item.msgstr;
+  }
+}
+
+function getTranslationAssets(outputDirectory, languages) {
+  return languages.map(language => {
+    const outputPath = join(outputDirectory, `${language}.js`);
+
+    if ( !_translationDictionaries[language]) {
+      return { outputBody: '', outputPath };
+    }
+
+    const translations = getTranslations(language);
+
+    // Examples of plural forms:
+    // pluralForms="nplurals=3; plural=(n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<12 || n%100>14) ? 1 : 2)"
+    // pluralForms="nplurals=3; plural=n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<12 || n%100>14) ? 1 : 2"
+
+    /** @type {String} */
+    const pluralFormsRule = _pluralFormsRules[language];
+
+    let pluralFormFunction;
+
+
+    if (!pluralFormsRule) {
+
+    } else {
+      const pluralFormFunctionBodyMatch = pluralFormsRule.match(/(?:plural=)(.+)/);
+
+      // Add support for ES5 - this function will not be transpiled.
+      pluralFormFunction = `function(n){return ${pluralFormFunctionBodyMatch[1]};}`;
+    }
+
+    // Stringify translations and remove unnecessary `""` around property names.
+    const stringifiedTranslations = JSON.stringify(translations)
+      .replace(/"([\w_]+)":/g, '$1:');
+
+    const outputBody = (
+      '(function(d){' +
+      `	const l = d['${language}'] = d['${language}'] || {};` +
+      '	l.dictionary=Object.assign(' +
+      '		l.dictionary||{},' +
+      `		${stringifiedTranslations}` +
+      '	);' +
+      (pluralFormFunction ? `l.getPluralForm=${pluralFormFunction};` : '' ) +
+      '})(window.CKEDITOR_TRANSLATIONS||(window.CKEDITOR_TRANSLATIONS={}));'
+    );
+
+    return { outputBody, outputPath };
+  });
+}
+
+function getTranslations(language) {
+  const langDictionary = _translationDictionaries[language];
+  const translatedStrings = {};
+
+  for ( const messageId of Object.keys(langDictionary)) {
+    const translatedMessage = langDictionary[messageId];
+
+    // Register first form as a default form if only one form was provided.
+    translatedStrings[messageId] = translatedMessage.length > 1 ?
+      translatedMessage :
+      translatedMessage[0];
+  }
+
+  return translatedStrings;
+}
+
+const ckeditorNamespacePath = resolve('./node_modules/@ckeditor/');
+for (const packagePath of readdirSync(ckeditorNamespacePath)) {
+  loadPackage(`${ckeditorNamespacePath}/${packagePath}/`);
+}
+
+if (!existsSync('./ckeditorLocales/')) {
+  mkdirSync('./ckeditorLocales/');
+}
+
+const assets = getTranslationAssets('./ckeditorLocales/', Array.from(_languages));
+
+for (const asset of assets) {
+  if (asset.outputBody !== undefined) {
+    writeFileSync(asset.outputPath, asset.outputBody);
+  }
+}

Build/ckeditor5.rollup.config.js

@@ -3,6 +3,7 @@ import commonjs from '@rollup/plugin-commonjs';
 import postcss from 'rollup-plugin-postcss';
 import svg from 'rollup-plugin-svg';
 import * as path from 'path';
+import { buildConfigForTranslations } from './ckeditor5.rollup.helpers';
 
 const { styles } = require( '@ckeditor/ckeditor5-dev-utils' );
 const postCssConfig = styles.getPostCssConfig({
@@ -57,6 +58,7 @@ export default [
         extensions: ['.js']
       }),
     ]
-  }
+  },
+  ...buildConfigForTranslations()
 ];
 

Build/ckeditor5.rollup.helpers.js

@@ -0,0 +1,25 @@
+import { resolve, join } from 'path'
+import { readdirSync } from 'fs'
+
+/**
+ * Helper function to build rollup bundling configuration for all existing
+ * CKEditor translations
+ */
+export function buildConfigForTranslations() {
+  const translationPath = resolve('./ckeditorLocales/')
+  const translationFiles = readdirSync(translationPath)
+  const configuration = []
+
+  for (const translationFile of translationFiles) {
+    configuration.push({
+      input: join(translationPath, translationFile),
+      output: {
+        compact: true,
+        file: `../typo3/sysext/rte_ckeditor/Resources/Public/Contrib/translations/${translationFile}`,
+        format: 'es',
+      },
+    })
+  }
+
+  return configuration
+}

Build/package.json

@@ -62,6 +62,7 @@
     "lintspaces-cli": "^0.8.0",
     "mime-db": "^1.46.0",
     "patch-package": "^6.5.1",
+    "pofile": "^1.1.4",
     "postcss-banner": "^4.0.1",
     "postcss-clean": "^1.2.2",
     "rollup": "^2.79.1",

typo3/sysext/rte_ckeditor/Classes/Form/Element/RichTextElement.php

@@ -179,6 +179,16 @@ public function render(): array
             $resultArray['javaScriptModules'][] = JavaScriptModuleInstruction::create('@typo3/ckeditor5-inspector.js');
         }
 
+        $uiLanguage = $ckeditorConfiguration['options']['language']['ui'];
+        if ($this->translationExists($uiLanguage)) {
+            $resultArray['javaScriptModules'][] = JavaScriptModuleInstruction::create('@typo3/ckeditor5/translations/' . $uiLanguage . '.js');
+        }
+
+        $contentLanguage = $ckeditorConfiguration['options']['language']['content'];
+        if ($this->translationExists($contentLanguage)) {
+            $resultArray['javaScriptModules'][] = JavaScriptModuleInstruction::create('@typo3/ckeditor5/translations/' . $contentLanguage . '.js');
+        }
+
         $resultArray['stylesheetFiles'][] = PathUtility::getPublicResourceWebPath('EXT:rte_ckeditor/Resources/Public/Css/editor.css');
 
         return $resultArray;
@@ -405,4 +415,10 @@ protected function sanitizeFieldId(string $itemFormElementName): string
         $fieldId = (string)preg_replace('/[^a-zA-Z0-9_:.-]/', '_', $itemFormElementName);
         return htmlspecialchars((string)preg_replace('/^[^a-zA-Z]/', 'x', $fieldId));
     }
+
+    protected function translationExists(string $language): bool
+    {
+        $fileName = GeneralUtility::getFileAbsFileName('EXT:rte_ckeditor/Resources/Public/Contrib/translations/' . $language . '.js');
+        return file_exists($fileName);
+    }
 }

typo3/sysext/rte_ckeditor/Configuration/JavaScriptModules.php

@@ -11,5 +11,6 @@
         '@typo3/rte-ckeditor/' => 'EXT:rte_ckeditor/Resources/Public/JavaScript/',
         '@typo3/ckeditor5-bundle.js' => 'EXT:rte_ckeditor/Resources/Public/Contrib/ckeditor5-bundle.js',
         '@typo3/ckeditor5-inspector.js' => 'EXT:rte_ckeditor/Resources/Public/Contrib/ckeditor5-inspector.js',
+        '@typo3/ckeditor5/translations/' => 'EXT:rte_ckeditor/Resources/Public/Contrib/translations/',
     ],
 ];