diff --git a/.generate_database_lua/Dockerfile b/.generate_database_lua/Dockerfile new file mode 100644 index 00000000..a20ef736 --- /dev/null +++ b/.generate_database_lua/Dockerfile @@ -0,0 +1,13 @@ +FROM nickblah/lua:5.1-luarocks + +RUN apt-get update && apt-get install -y git gcc +# Install OpenSSL for LuaSec +RUN apt-get install -y libssl-dev + +RUN luarocks install bit32 +RUN luarocks install argparse +RUN luarocks install luafilesystem +RUN luarocks install luasocket +RUN luarocks install luasec + +RUN apt-get update && apt-get install -y wget \ No newline at end of file diff --git a/.generate_database_lua/createStatic.lua b/.generate_database_lua/createStatic.lua new file mode 100644 index 00000000..fc905bc4 --- /dev/null +++ b/.generate_database_lua/createStatic.lua @@ -0,0 +1,170 @@ +-- Allow accessing private fields +---@diagnostic disable: invisible +require("cli.dump") +local argparse = require("argparse") +local helpers = require(".helpers") + +require("cli.Addon_Meta") +require("cli.CLI_Helpers") + +assert(Is_CLI, "This function should only be called from the CLI environment") + +local f = string.format + +Is_Create_Static = true + +function DumpDatabase(version) + local lowerVersion = version:lower() + local capitalizedVersion = lowerVersion:gsub("^%l", string.upper) + print(f("\n\27[36mCompiling %s database...\27[0m", capitalizedVersion)) + + -- Reset data objects, load the files and set wow version + LibQuestieDBTable = AddonInitializeVersion(capitalizedVersion) + + -- Drain all the timers + C_Timer.drainTimerList() + + local itemOverride = {} + local npcOverride = {} + local objectOverride = {} + local questOverride = {} + + local Corrections = LibQuestieDBTable.Corrections + + Corrections.DumpFunctions.testDumpFunctions() + + do + CLI_Helpers.loadFile(f(".generate_database/_data/%sItemDB.lua", lowerVersion)) + itemOverride = loadstring(QuestieDB.itemData)() + LibQuestieDBTable.Item.LoadOverrideData(false, true) + local itemMeta = Corrections.ItemMeta + for itemId, corrections in pairs(LibQuestieDBTable.Item.override) do + if not itemOverride[itemId] then + itemOverride[itemId] = {} + end + for key, correction in pairs(corrections) do + local correctionIndex = itemMeta.itemKeys[key] + itemOverride[itemId][correctionIndex] = correction + end + end + end + + do + CLI_Helpers.loadFile(f(".generate_database/_data/%sNpcDB.lua", lowerVersion)) + npcOverride = loadstring(QuestieDB.npcData)() + LibQuestieDBTable.Npc.LoadOverrideData(false, true) + local npcMeta = Corrections.NpcMeta + for npcId, corrections in pairs(LibQuestieDBTable.Npc.override) do + if not npcOverride[npcId] then + npcOverride[npcId] = {} + end + for key, correction in pairs(corrections) do + local correctionIndex = npcMeta.npcKeys[key] + npcOverride[npcId][correctionIndex] = correction + end + end + end + + do + CLI_Helpers.loadFile(f(".generate_database/_data/%sObjectDB.lua", lowerVersion)) + objectOverride = loadstring(QuestieDB.objectData)() + LibQuestieDBTable.Object.LoadOverrideData(false, true) + local objectMeta = Corrections.ObjectMeta + for objectId, corrections in pairs(LibQuestieDBTable.Object.override) do + if not objectOverride[objectId] then + objectOverride[objectId] = {} + end + for key, correction in pairs(corrections) do + local correctionIndex = objectMeta.objectKeys[key] + objectOverride[objectId][correctionIndex] = correction + end + end + end + + do + CLI_Helpers.loadFile(f(".generate_database/_data/%sQuestDB.lua", lowerVersion)) + questOverride = loadstring(QuestieDB.questData)() + LibQuestieDBTable.Quest.LoadOverrideData(false, true) + local questMeta = Corrections.QuestMeta + for questId, corrections in pairs(LibQuestieDBTable.Quest.override) do + if not questOverride[questId] then + questOverride[questId] = {} + end + for key, correction in pairs(corrections) do + local correctionIndex = questMeta.questKeys[key] + questOverride[questId][correctionIndex] = correction + end + end + end + + -- Write all the overrides to disk + ---@diagnostic disable-next-line: param-type-mismatch + -- local file = io.open(f(".generate_database/_data/output/Item/%s/ItemData.lua-table", capitalizedVersion), "w") + print("Dumping item overrides") + -- assert(file, "Failed to open file for writing") + local itemData = helpers.dumpData(itemOverride, Corrections.ItemMeta.itemKeys, Corrections.ItemMeta.dumpFuncs, Corrections.ItemMeta.combine) + ---@diagnostic disable-next-line: undefined-field + -- file:write(itemData) + ---@diagnostic disable-next-line: undefined-field + -- file:close() + + -- ---@diagnostic disable-next-line: param-type-mismatch + -- file = io.open(f(".generate_database/_data/output/Quest/%s/QuestData.lua-table", capitalizedVersion), "w") + -- print("Dumping quest overrides") + -- assert(file, "Failed to open file for writing") + -- local questData = helpers.dumpData(questOverride, Corrections.QuestMeta.questKeys, Corrections.QuestMeta.dumpFuncs) + -- ---@diagnostic disable-next-line: undefined-field + -- file:write(questData) + -- ---@diagnostic disable-next-line: undefined-field + -- file:close() + + -- ---@diagnostic disable-next-line: param-type-mismatch + -- file = io.open(f(".generate_database/_data/output/Npc/%s/NpcData.lua-table", capitalizedVersion), "w") + -- print("Dumping npc overrides") + -- assert(file, "Failed to open file for writing") + -- local npcData = helpers.dumpData(npcOverride, Corrections.NpcMeta.npcKeys, Corrections.NpcMeta.dumpFuncs, Corrections.NpcMeta.combine) + -- ---@diagnostic disable-next-line: undefined-field + -- file:write(npcData) + -- ---@diagnostic disable-next-line: undefined-field + -- file:close() + + -- ---@diagnostic disable-next-line: param-type-mismatch + -- file = io.open(f(".generate_database/_data/output/Object/%s/ObjectData.lua-table", capitalizedVersion), "w") + -- print("Dumping object overrides") + -- assert(file, "Failed to open file for writing") + -- local objectData = helpers.dumpData(objectOverride, Corrections.ObjectMeta.objectKeys, Corrections.ObjectMeta.dumpFuncs) + -- ---@diagnostic disable-next-line: undefined-field + -- file:write(objectData) + -- ---@diagnostic disable-next-line: undefined-field + -- file:close() + + print(f("\n\27[32m%s corrections dumped successfully\27[0m", capitalizedVersion)) +end + +-- local validVersions = { +-- ["era"] = true, +-- ["tbc"] = true, +-- ["wotlk"] = true, +-- } +-- local versionString = "" +-- for version in pairs(validVersions) do +-- local v = string.gsub(version, "^%l", string.upper) +-- versionString = versionString .. v .. "/" +-- end +-- -- Add all +-- versionString = versionString .. "All" + +-- local parser = argparse("createStatic", "createStatic.lua Era") +-- parser:argument("version", f("Game version, %s.", versionString)) + +-- local args = parser:parse() + +-- if args.version and validVersions[args.version:lower()] then +-- DumpDatabase(args.version) +-- elseif args.version and args.version:lower() == "all" then +-- for version in pairs(validVersions) do +-- DumpDatabase(version) +-- end +-- else +-- print("No version specified") +-- end diff --git a/.generate_database_lua/docker-compose.yml b/.generate_database_lua/docker-compose.yml new file mode 100644 index 00000000..e3bb49d5 --- /dev/null +++ b/.generate_database_lua/docker-compose.yml @@ -0,0 +1,9 @@ +version: '3' + +services: + lua_2: + #image: nickblah/lua:5.1-luarocks + build: . + command: sh /QuestieDB/.generate_database_lua/run.sh + volumes: + - '../:/QuestieDB' diff --git a/.generate_database_lua/generate_translation_trie.lua b/.generate_database_lua/generate_translation_trie.lua new file mode 100644 index 00000000..a852747b --- /dev/null +++ b/.generate_database_lua/generate_translation_trie.lua @@ -0,0 +1,183 @@ +require("cli.dump") + +-- Define the maximum number of translations per file +local MAX_TRANSLATIONS_PER_FILE = 35 +local SEGMENT_SIZE = 4000 +local REDUCE_SEGMENT_SIZE = math.max(math.min(SEGMENT_SIZE * 0.05, 100), 10) +print("Max translations per file: " .. MAX_TRANSLATIONS_PER_FILE) +print("Segment size: " .. SEGMENT_SIZE, "Reduced segment size: " .. SEGMENT_SIZE - REDUCE_SEGMENT_SIZE) + +-- Function to sanitize translation strings by replacing special characters with HTML entities +local function sanitize_translation(str) + str = string.gsub(str, "&", "&") + str = string.gsub(str, "<", "<") + str = string.gsub(str, ">", ">") + return str +end + +-- Function to create directories recursively using os.execute +local function mkdir(path) + os.execute("mkdir -p " .. path) +end + +-- Function to split a string into segments based on maximum characters per segment +local function split_into_segments(str, max_chars) + local segments = {} + local total_segments = math.ceil(#str / max_chars) + + -- Loop through the string and create segments + for i = 1, total_segments do + local start_pos = (i - 1) * max_chars + 1 + local end_pos = math.min(i * max_chars, #str) + local segment = string.sub(str, start_pos, end_pos) + -- Add segment number prefix for all segments except the first one + if i > 1 then + segment = tostring(i) .. segment + end + table.insert(segments, segment) + end + + -- Add total segments count to the first segment + segments[1] = total_segments .. segments[1] + + return segments +end + +-- Function to write translations to an HTML file with segmentation +local function write_html_file(key_path, translations) + -- Define the file path + local filename = key_path .. ".html" + + -- Open the file for writing + local file, err = io.open(filename, "w") + if not file then + print("Error opening file " .. filename .. ": " .. err) + return + end + + -- Write the HTML structure + file:write("
\n") + for _, translation in ipairs(translations) do + -- Check if the translation needs to be segmented + if #translation > SEGMENT_SIZE - REDUCE_SEGMENT_SIZE then + print("Splitting translation into segments", translation) + -- Split the translation into segments + local segments = split_into_segments(translation, SEGMENT_SIZE - REDUCE_SEGMENT_SIZE) + -- Write each segment as a separate paragraph + for _, segment in ipairs(segments) do + file:write("" .. sanitize_translation(segment) .. "
\n") + end + else + -- Write the translation as a single paragraph + file:write("" .. sanitize_translation(translation) .. "
\n") + end + end + file:write("\n") + file:close() +end + +-- Function to create a branch in the trie structure +local function create_branch(strings, stringIndex) + local branch = {} + -- Process each string in the input array + for i = 1, #strings do + local string = strings[i] + local cleanedString = string.gsub(string, "%s", "") + -- Remove all numbers from the string + -- cleanedString = string.gsub(cleanedString, "%d+", "") + cleanedString = string.gsub(cleanedString, "%p", "") + cleanedString = string.gsub(cleanedString, "%c", "") + + -- Get the character at the current index + -- local char = string.sub(string.lower(cleanedString), stringIndex, stringIndex) + local char = string.sub(cleanedString, stringIndex, stringIndex) + + if char == "" then + error(string.format("%s: %d out of range, increase MAX_TRANSLATIONS_PER_FILE", string, stringIndex)) + end + + -- Create a new branch for the character if it doesn't exist + if not branch[char] then + branch[char] = {} + end + table.insert(branch[char], string) + end + + -- Recursively create branches for child nodes if needed + for char, child in pairs(branch) do + if #child > MAX_TRANSLATIONS_PER_FILE then + branch[char] = create_branch(child, stringIndex + 1) + end + end + + return branch +end + +-- Function to create trie folders and write translations to HTML files +local function create_trie_folders(trie, current_path) + print("Current path: " .. current_path) + for key, value in pairs(trie) do + local new_path = current_path .. "/" .. key + if type(value) == "table" then + -- If the number of translations is greater than the maximum or there are no translations + if #value > MAX_TRANSLATIONS_PER_FILE or #value == 0 then + mkdir(new_path) + -- print("Continuing recursion for: " .. new_path) + create_trie_folders(value, new_path) + else + print("Writing HTML file for: " .. new_path) + write_html_file(new_path, value) + end + end + end +end + +-- Main function to compile translations to HTML +function Compile_translations_to_html(strings) + -- Initialize the trie + local success, trie + repeat + success, trie = pcall(create_branch, strings, 1) + if not success then + MAX_TRANSLATIONS_PER_FILE = MAX_TRANSLATIONS_PER_FILE + 1 + print(trie) + print("Shortest word does not fit! - increasing MAX_TRANSLATIONS_PER_FILE to: " .. MAX_TRANSLATIONS_PER_FILE) + end + until success + + -- Create the root folder + local root_folder = "translations" + mkdir(root_folder) + + -- Create trie folders and write translations + create_trie_folders(trie, root_folder) + + -- DevTools_Dump(trie) + -- Dump the trie to a lua file + -- local lines = {} + local function dump_trie(trie, indent) + local lines = {} + local indent_str = string.rep(" ", indent) + if type(trie) == "table" then + for char, value in pairs(trie) do + if type(value) == "table" then + table.insert(lines, indent_str .. "['" .. char .. "'] = {") + table.insert(lines, dump_trie(value, indent + 1)) + table.insert(lines, indent_str .. "},") + else + table.insert(lines, indent_str .. "\"" .. value .. "\",") + end + end + else + table.insert(lines, indent_str .. "[\"" .. trie .. "\"]") + end + return table.concat(lines, "\n") + end + + local lua_file = io.open(root_folder .. "/trie.lua", "w") + if lua_file ~= nil then + local dump_str = dump_trie(trie, 1) + lua_file:write("local trie = {\n" .. dump_str .. "\n}") + lua_file:close() + end +end diff --git a/.generate_database_lua/helpers.lua b/.generate_database_lua/helpers.lua new file mode 100644 index 00000000..7b47659e --- /dev/null +++ b/.generate_database_lua/helpers.lua @@ -0,0 +1,219 @@ +-- helpers.lua +local lfs = require("lfs") + +-- Require the necessary LuaSocket modules +local http = require("socket.http") +local https = require("ssl.https") +local ltn12 = require("ltn12") + +--- Get the script directory. +---@return string The directory containing the script. +local function get_script_dir() + local script_path = debug.getinfo(1, "S").source:sub(2) + return script_path:match("(.*/)") +end + +--- Get the project directory path. +---@return string The project directory path. +local function get_project_dir_path() + return get_script_dir() .. ".." +end + +--- Get the data directory path for a given entity type and expansion. +---@param entity_type string The type of entity (e.g., "Quest", "Item"). +---@param expansion string The expansion name (e.g., "Era", "Tbc", "Wotlk"). +---@return string The data directory path. +local function get_data_dir_path(entity_type, expansion) + local path = get_project_dir_path() .. "/Database/" .. entity_type .. "/" .. expansion + -- Does path not exist, l10n has this issue... + if not lfs.attributes(path, "mode") then + print("Path " .. path .. " does not exist, trying lowercase") + path = get_project_dir_path() .. "/Database/" .. entity_type:lower() .. "/" .. expansion + end + return path +end + +--- Find the addon name. +---@return string The addon name. +local function find_addon_name() + local current_dir = lfs.currentdir() + local previous_dir = nil + local addon_dir = nil + local max_level = 20 + local level = 0 + + while level < max_level do + local dir_name = current_dir:match("([^/]+)$") + local parent_dir = current_dir:match("^(.*)/[^/]+$") + if type(dir_name) == "nil" then + addon_dir = previous_dir + break + elseif dir_name:lower() == "addons" and parent_dir:match("([^/]+)$"):lower() == "interface" then + addon_dir = previous_dir + break + else + previous_dir = current_dir + current_dir = parent_dir + level = level + 1 + end + end + + if not addon_dir then + print("Could not find the Addons folder, defaulting to 'QuestieDB'.") + return "QuestieDB" + else + print("Found Addons folder: " .. addon_dir) + end + + return addon_dir +end + +--- Read expansion data from a Lua file. +---@param expansion string The expansion name (e.g., "Era", "Tbc", "Wotlk"). +---@param entity_type string The type of entity (e.g., "Quest", "Item"). +---@return string|nil The content of the Lua file, or nil if not found. +local function read_expansion_data(expansion, entity_type) + local path = get_data_dir_path(entity_type, expansion) + print("Reading " .. expansion .. " lua " .. entity_type:lower() .. " data from " .. path) + local file_path = path .. "/" .. entity_type:sub(1, 1):upper() .. entity_type:sub(2):lower() .. "Data.lua-table" + + if not lfs.attributes(file_path, "mode") then + file_path = path .. "/" .. entity_type:lower() .. "Data.lua-table" + if not lfs.attributes(file_path, "mode") then + print("File not found: " .. file_path) + return nil + end + end + + local file, err = io.open(file_path, "r") + if not file then + print("File not found: " .. path) + return nil + end + + local data = file:read("*all") + file:close() + + -- Perform any necessary replacements on the data string + data = data:gsub("&", "and"):gsub("<", "|"):gsub(">", "|") + return data +end + +--- Download a raw text file from a URL and return its contents as a string. +-- @param url string The URL of the text file to download. +-- @return string|nil The contents of the text file, or nil if the download fails. +local function download_text_file(url) + local response_body = {} + local res, code, response_headers = https.request { + url = url, + sink = ltn12.sink.table(response_body), + } + + if res == 1 and code == 200 then + -- Concatenate the table into a single string + return table.concat(response_body) + else + print("Failed to download file: HTTP response code " .. tostring(code)) + return nil + end +end + +---Dumps the data for Item, Quest, Npc, Object into a string +---@param tbl table