Merge pull request #251 from WycliffeAssociates/adjust-frontend-tests-to-pass

Update for STET

Author: PurpleGuitar

backend/doc/domain/parsing.py

@@ -1140,9 +1140,10 @@ def lookup_verse_text(usfm_book: USFMBook, chapter_num: int, verse_ref: str) ->
         return ""
     verse = chapter.verses.get(verse_ref, "")
     logger.info(
-        "lang_code: %s, book_code: %s, chapter_num: %s, verse_num: %s, verse: %s",
+        "lang_code: %s, book_code: %s, national_book_name: %s, chapter_num: %s, verse_num: %s, verse: %s",
         usfm_book.lang_code,
         usfm_book.book_code,
+        usfm_book.national_book_name,
         chapter_num,
         verse_ref,
         verse,

backend/doc/domain/resource_lookup.py

@@ -177,6 +177,7 @@
 # use.
 BOOK_NAME_CORRECTION_TABLE: dict[tuple[str, str], str] = {
     ("pt-br", "1 Corintios"): "1 Coríntios",
+    ("es-419", "I juan"): "1 Juan",
 }
 
 # List of languages which do not have USFM available for any books. We use this
@@ -1039,9 +1040,7 @@ def maybe_correct_book_name(
     """
     Translate incorrect or undesirable book names to a preferred form.
     """
-    logger.debug("book_name to lookup: %s", book_name)
     book_name_ = BOOK_NAME_CORRECTION_TABLE.get((lang_code, book_name), "")
-    logger.debug("result from book_name_correction_table: %s", book_name_)
     if not book_name_:
         book_name_ = book_name
     return book_name_

backend/stet/data/stet_es-419.docx

@@ -1,3 +1,4 @@
+from datetime import datetime
 from typing import Mapping, Sequence
 
 import mistune
@@ -28,6 +29,12 @@
 from pydantic import Json
 from stet.domain.model import VerseEntry, WordEntry
 from stet.domain.parser import get_word_entry_dtos
+from stet.domain.strings import (
+    LOCALIZED_DATE_FORMAT_STRINGS,
+    TRANSLATED_FOOTER_PHRASES_TABLE,
+    TRANSLATED_HEADER_PHRASES_TABLE,
+    TRANSLATED_TABLE_COLUMN_HEADERS,
+)
 from stet.utils.docx_utils import (
     add_footer,
     add_header,
@@ -59,11 +66,13 @@ def generate_docx_document(
     >>> generate_docx_document()
     """
     word_entries: list[WordEntry] = []
-    word_entry_dtos, book_codes_and_names = get_word_entry_dtos(lang0_code, lang1_code)
+    word_entry_dtos, lang0_book_codes_and_names = get_word_entry_dtos(
+        lang0_code, lang1_code
+    )
     lang0_resource_types = resource_types(
         lang0_code,
         ",".join(
-            [book_code_and_name[0] for book_code_and_name in book_codes_and_names]
+            [book_code_and_name[0] for book_code_and_name in lang0_book_codes_and_names]
         ),
     )
     lang0_resource_types_ = [
@@ -73,7 +82,7 @@ def generate_docx_document(
     lang1_resource_types = resource_types(
         lang1_code,
         ",".join(
-            [book_code_and_name[0] for book_code_and_name in book_codes_and_names]
+            [book_code_and_name[0] for book_code_and_name in lang0_book_codes_and_names]
         ),
     )
     lang1_resource_types_ = [
@@ -115,7 +124,7 @@ def generate_docx_document(
     if lang0_usfm_resource_type and lang1_usfm_resource_type:
         source_usfm_book = None
         target_usfm_book = None
-        for book_code, book_name in book_codes_and_names:
+        for book_code, book_name in lang0_book_codes_and_names:
             current_task.update_state(state="Locating assets")
             lang0_resource_lookup_dto_ = resource_lookup_dto(
                 lang0_code, lang0_usfm_resource_type, book_code
@@ -196,23 +205,13 @@ def generate_docx_document(
                 )
             if target_selected_usfm_books:
                 target_selected_usfm_book = target_selected_usfm_books[0]
-            for verse_ref in verse_ref_dto.verse_refs:
-                if source_selected_usfm_book:
-                    source_verse_text = lookup_verse_text(
-                        source_selected_usfm_book,
-                        verse_ref_dto.chapter_num,
-                        verse_ref.strip(),
-                    )
-                else:
-                    source_verse_text = ""
-                if target_selected_usfm_book:
-                    target_verse_text = lookup_verse_text(
-                        target_selected_usfm_book,
-                        verse_ref_dto.chapter_num,
-                        verse_ref.strip(),
-                    )
-                else:
-                    target_verse_text = ""
+                target_selected_usfm_book.national_book_name = maybe_correct_book_name(
+                    lang1_code, target_selected_usfm_book.national_book_name
+                )
+                logger.debug(
+                    "target_usfm_book.national_book_name: %s",
+                    target_selected_usfm_book.national_book_name,
+                )
             non_book_name_portion_of_source_reference = extract_chapter_and_beyond(
                 verse_ref_dto.source_reference
             )
@@ -231,6 +230,23 @@ def generate_docx_document(
                 and non_book_name_portion_of_target_reference
                 else verse_ref_dto.target_reference
             )
+            for verse_ref in verse_ref_dto.verse_refs:
+                if source_selected_usfm_book:
+                    source_verse_text = lookup_verse_text(
+                        source_selected_usfm_book,
+                        verse_ref_dto.chapter_num,
+                        verse_ref.strip(),
+                    )
+                else:
+                    source_verse_text = ""
+                if target_selected_usfm_book:
+                    target_verse_text = lookup_verse_text(
+                        target_selected_usfm_book,
+                        verse_ref_dto.chapter_num,
+                        verse_ref.strip(),
+                    )
+                else:
+                    target_verse_text = ""
             word_entry.verses.append(
                 VerseEntry(
                     source_reference=localized_source_reference,
@@ -246,7 +262,16 @@ def generate_docx_document(
 
 
 def generate_docx(
-    word_entries: list[WordEntry], docx_filepath: str, lang0_code: str, lang1_code: str
+    word_entries: list[WordEntry],
+    docx_filepath: str,
+    lang0_code: str,
+    lang1_code: str,
+    translated_table_column_headers: dict[
+        str, tuple[str, str, str, str]
+    ] = TRANSLATED_TABLE_COLUMN_HEADERS,
+    translated_footer_phrases_table: dict[str, str] = TRANSLATED_FOOTER_PHRASES_TABLE,
+    localized_date_format_strings: dict[str, str] = LOCALIZED_DATE_FORMAT_STRINGS,
+    translated_header_phrases_table: dict[str, str] = TRANSLATED_HEADER_PHRASES_TABLE,
 ) -> None:
     """
     Generates a DOCX document from a list of word entries and saves it to the given file path.
@@ -273,9 +298,9 @@ def generate_docx(
         table.style = "Table Grid"
         # Set the header of the table and apply bold formatting
         hdr_cells = table.rows[0].cells
-        hdr_cells[0].text = "Source Reference"
-        hdr_cells[1].text = "Target Reference"
-        hdr_cells[2].text = "Status"
+        hdr_cells[0].text = translated_table_column_headers[lang0_code][0]
+        hdr_cells[1].text = translated_table_column_headers[lang0_code][1]
+        hdr_cells[2].text = translated_table_column_headers[lang0_code][2]
         hdr_cells[2].paragraphs[0].alignment = WD_PARAGRAPH_ALIGNMENT.CENTER
         for hdr_cell in hdr_cells:
             hdr_cell.paragraphs[0].runs[0].bold = True
@@ -287,7 +312,11 @@ def generate_docx(
             source_run.bold = True
             target_run = row_cells[1].paragraphs[0].add_run(verse.target_reference)
             target_run.bold = True
-            status_run = row_cells[2].paragraphs[0].add_run("OK")
+            status_run = (
+                row_cells[2]
+                .paragraphs[0]
+                .add_run(translated_table_column_headers[lang0_code][3])
+            )
             status_run.bold = True
             row_cells[2].paragraphs[0].alignment = WD_PARAGRAPH_ALIGNMENT.CENTER
             # Row for texts
@@ -319,8 +348,14 @@ def generate_docx(
             tcPr.append(vAlign)  # Append the vertical alignment to cell properties
         # Adjust column widths to prioritize the first two columns
         adjust_table_columns(table)
-    doc = add_footer(doc)
-    doc = add_header(doc, lang0_code, lang1_code)
+    footer_phrase = translated_footer_phrases_table[lang0_code]
+    current_datetime = datetime.now().strftime(
+        localized_date_format_strings[lang0_code]
+    )
+    date_text = f"{footer_phrase} {current_datetime}"
+    doc = add_footer(doc, date_text)
+    header_phrase = translated_header_phrases_table[lang0_code]
+    doc = add_header(doc, lang0_code, lang1_code, header_phrase)
     doc = add_lined_page_at_end(doc)
     reduce_spacing_around_tables(doc)
     doc.save(docx_filepath)

backend/stet/data/stet_pt-br.docx

@@ -18,7 +18,9 @@ def get_word_entry_dtos(
 ) -> tuple[list[WordEntryDto], list[tuple[str, str]]]:
     # Build data from source doc
     word_entry_dtos: list[WordEntryDto] = []
-    book_codes_and_names__: list[tuple[str, str]] = []
+    lang0_book_codes_and_names = book_codes_for_lang_from_usfm_only(lang0_code)
+    lang1_book_codes_and_names = book_codes_for_lang_from_usfm_only(lang1_code)
+    lang0_book_codes_and_names__: list[tuple[str, str]] = []
     doc = Document(f"{stet_dir}/stet_{lang0_code}.docx")
     for table in doc.tables:
         for row in table.rows:
@@ -59,17 +61,12 @@ def get_word_entry_dtos(
                     book_name = match.group(1)
                     # Some languages, e.g., bem, have a \n in the book name
                     book_name = book_name.replace("\n", "")
-                    # Get book codes and names for the language that has been
-                    # requested from DOC.
-                    book_codes_and_names = book_codes_for_lang_from_usfm_only(
-                        lang0_code
-                    )
                     # We expect this book name to be in localized form according to the
                     # language of the STET input document (as indicated by the input
                     # document's filename, stet_[ietf_code].docx).
                     book_codes_and_names_ = [
                         (book_code, book_name_)
-                        for book_code, book_name_ in book_codes_and_names
+                        for book_code, book_name_ in lang0_book_codes_and_names
                         if book_name_
                         == book_name  # Check if DOC and STET input doc agree on book name
                     ]
@@ -85,7 +82,7 @@ def get_word_entry_dtos(
                         book_codes_and_names_[0] if book_codes_and_names_ else None
                     )
                     if book_code_and_name_:
-                        book_codes_and_names__.append(book_code_and_name_)
+                        lang0_book_codes_and_names__.append(book_code_and_name_)
                     chapter_num = int(match.group(2))
                     verses = match.group(3)
                     comment = match.group(4)
@@ -95,7 +92,23 @@ def get_word_entry_dtos(
                         )
                     else:
                         source_reference = f"{book_name} {chapter_num}:{verses}"
-                    target_reference = f"{book_name} {chapter_num}:{verses}"
+                    lang0_book_code = (
+                        book_code_and_name_[0] if book_code_and_name_ else ""
+                    )
+                    lang1_book_code_and_name_ = next(
+                        (
+                            lang1_book_code_and_name
+                            for lang1_book_code_and_name in lang1_book_codes_and_names
+                            if lang1_book_code_and_name[0] == lang0_book_code
+                        ),
+                        None,
+                    )
+                    lang1_book_name = (
+                        lang1_book_code_and_name_[1]
+                        if lang1_book_code_and_name_
+                        else ""
+                    )
+                    target_reference = f"{lang1_book_name} {chapter_num}:{verses}"
                     verse_refs: list[str] = verses.split(",")
                     valid_verse_refs: list[str] = []
                     for verse_ref in verse_refs:
@@ -129,4 +142,8 @@ def get_word_entry_dtos(
                     keyword.strip() for keyword in row.cells[3].text.split(",")
                 ]
             word_entry_dtos.append(word_entry_dto)
-    return word_entry_dtos, list(set(book_codes_and_names__))
+    # Sort word entry dtos by first word in word list
+    sorted_word_entry_dtos = sorted(
+        word_entry_dtos, key=lambda word_entry_dto: word_entry_dto.words[0]
+    )
+    return sorted_word_entry_dtos, list(set(lang0_book_codes_and_names__))

backend/stet/domain/document_generator.py

@@ -0,0 +1,23 @@
+TRANSLATED_HEADER_PHRASES_TABLE: dict[str, str] = {
+    "en": "Spiritual Terms Evaluation Tool (STET)",
+    "es-419": "Herramienta de Evaluación de Términos Espirituales (STET)",
+    "pt-br": "Ferramenta de Avaliação de Termos Espirituais (STET)",
+}
+
+TRANSLATED_FOOTER_PHRASES_TABLE: dict[str, str] = {
+    "en": "Generated on",
+    "es-419": "Generado el",
+    "pt-br": "Gerado em",
+}
+
+LOCALIZED_DATE_FORMAT_STRINGS: dict[str, str] = {
+    "en": "%m/%d/%Y %H:%M:%S",
+    "es-419": "%d/%m/%Y %H:%M:%S",
+    "pt-br": "%d/%m/%Y %H:%M:%S",
+}
+
+TRANSLATED_TABLE_COLUMN_HEADERS = {
+    "en": ("Source Reference", "Target Reference", "Status", "OK"),
+    "es-419": ("Referencia de origen", "Referencia de destino", "Estado", "OK"),
+    "pt-br": ("Referência de Origem", "Referência de Destino", "Status", "OK"),
+}

backend/stet/domain/parser.py

@@ -1,5 +1,4 @@
 import re
-from datetime import datetime
 from typing import Optional
 
 from docx import Document  # type: ignore
@@ -114,7 +113,7 @@ def add_header(
     doc: Document,
     source_lang_code: str,
     target_lang_code: str,
-    header_text: str = "Spiritual Terms Evaluation Tool",
+    header_text: str = "Spiritual Terms Evaluation Tool (STET)",
 ) -> Document:
     """
     Add a header with:
@@ -148,8 +147,6 @@ def add_header(
     return doc
 
 
-
-
 def add_highlighted_html_to_docx_for_words(
     html: str, paragraph: Paragraph, keywords: list[str]
 ) -> None:
@@ -288,7 +285,7 @@ def set_spacing(
             previous_element = element
 
 
-def add_footer(doc: Document) -> Document:
+def add_footer(doc: Document, date_text: str) -> Document:
     """
     Programmatically add page numbers and a date timestamp in the footer.
     Page number will be centered, and the date timestamp will be aligned to the right
@@ -338,8 +335,6 @@ def add_footer(doc: Document) -> Document:
     page_run.font.color.rgb = RGBColor(169, 169, 169)  # Grey color for page number
     # Add the "Generated on" text
     footer_paragraph.add_run("\t")  # Tab to right position
-    current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    date_text = f"Generated on {current_datetime}"
     date_run = footer_paragraph.add_run(date_text)
     date_run.font.color.rgb = RGBColor(169, 169, 169)  # Grey color for timestamp
     date_run.font.size = Pt(10)  # Optional: Adjust font size for consistency

backend/stet/domain/strings.py

@@ -11,7 +11,7 @@ def is_valid_int(text: str) -> bool:
 
 
 def extract_chapter_and_beyond(text: str) -> Optional[str]:
-    # Regular expression to match "<chapter_num>:<verse_num> [comment]"
+    # Regular expression to match "<chapter_num>:<verse_num> (comment)"
     match = re.search(r"\d+:\d+(\s*\(\*\*?\))?$", text)
     if match:
         return match.group()

backend/stet/utils/docx_utils.py

@@ -3,7 +3,7 @@ import type { PlaywrightTestConfig } from '@playwright/test'
 const config: PlaywrightTestConfig = {
     testDir: 'tests',
     testMatch: '**/*.ts',
-    timeout: 240000 // Set global timeout to 4 minutes
+    timeout: 640000 // Set global timeout 
 }
 
 export default config